"""
Raster and 3d raster mapcalculation functions
(C) 2012-2013 by the GRASS Development Team
This program is free software under the GNU General Public
License (>=v2). Read the file COPYING that comes with GRASS
for details.
:authors: Soeren Gebbert
"""
import copy
from datetime import datetime
from multiprocessing import Process
import grass.script as gscript
from grass.exceptions import CalledModuleError
from .core import (
SQLDatabaseInterfaceConnection,
get_current_mapset,
get_tgis_message_interface,
)
from .open_stds import open_new_stds, open_old_stds, check_new_stds
from .datetime_math import time_delta_to_relative_time
############################################################################
[docs]def dataset_mapcalculator(
inputs,
output,
type,
expression,
base,
method,
nprocs=1,
register_null=False,
spatial=False,
):
"""Perform map-calculations of maps from different space time
raster/raster3d datasets, using a specific sampling method
to select temporal related maps.
A mapcalc expression must be provided to process the temporal
selected maps. Temporal operators are available in addition to
the r.mapcalc operators:
Supported operators for relative and absolute time are:
- td() - the time delta of the current interval in days
and fractions of days or the unit in case of relative time
- start_time() - The start time of the interval from the begin of
the time series in days and fractions of days or the
unit in case of relative time
- end_time() - The end time of the current interval from the begin of
the time series in days and fractions of days or the
unit in case of relative time
Supported operators for absolute time:
- start_doy() - Day of year (doy) from the start time [1 - 366]
- start_dow() - Day of week (dow) from the start time [1 - 7],
the start of the week is monday == 1
- start_year() - The year of the start time [0 - 9999]
- start_month() - The month of the start time [1 - 12]
- start_week() - Week of year of the start time [1 - 54]
- start_day() - Day of month from the start time [1 - 31]
- start_hour() - The hour of the start time [0 - 23]
- start_minute() - The minute of the start time [0 - 59]
- start_second() - The second of the start time [0 - 59]
- end_doy() - Day of year (doy) from the end time [1 - 366]
- end_dow() - Day of week (dow) from the end time [1 - 7],
the start of the week is monday == 1
- end_year() - The year of the end time [0 - 9999]
- end_month() - The month of the end time [1 - 12]
- end_week() - Week of year of the end time [1 - 54]
- end_day() - Day of month from the end time [1 - 31]
- end_hour() - The hour of the end time [0 - 23]
- end_minute() - The minute of the end time [0 - 59]
- end_second() - The minute of the end time [0 - 59]
:param inputs: The names of the input space time raster/raster3d datasets
:param output: The name of the extracted new space time raster(3d) dataset
:param type: The type of the dataset: "raster" or "raster3d"
:param expression: The r(3).mapcalc expression
:param base: The base name of the new created maps in case a
mapclac expression is provided
:param method: The method to be used for temporal sampling
:param nprocs: The number of parallel processes to be used for
mapcalc processing
:param register_null: Set this number True to register empty maps
:param spatial: Check spatial overlap
"""
# We need a database interface for fast computation
dbif = SQLDatabaseInterfaceConnection()
dbif.connect()
mapset = get_current_mapset()
msgr = get_tgis_message_interface()
input_name_list = inputs.split(",")
first_input = open_old_stds(input_name_list[0], type, dbif)
# skip sampling when only one dataset specified (with different
# band filters)
input_name_list_uniq = []
for input_name in input_name_list:
ds = open_old_stds(input_name, type, dbif)
ds_name = ds.get_name(semantic_label=False)
if ds_name not in input_name_list_uniq:
input_name_list_uniq.append(ds_name)
do_sampling = len(input_name_list_uniq) > 1
# All additional inputs in reverse sorted order to avoid
# wrong name substitution
input_name_list = input_name_list[1:]
input_name_list.sort()
input_name_list.reverse()
input_list = []
for input in input_name_list:
sp = open_old_stds(input, type, dbif)
input_list.append(copy.copy(sp))
new_sp = check_new_stds(output, type, dbif, gscript.overwrite())
# Sample all inputs by the first input and create a sample matrix
if spatial:
msgr.message(_("Starting spatio-temporal sampling..."))
else:
msgr.message(_("Starting temporal sampling..."))
map_matrix = []
id_list = []
sample_map_list = []
if len(input_list) > 0 and do_sampling:
# First entry is the first dataset id
id_list.append(first_input.get_name())
has_samples = False
for dataset in input_list:
list = dataset.sample_by_dataset(
stds=first_input, method=method, spatial=spatial, dbif=dbif
)
# In case samples are not found
if not list or len(list) == 0:
dbif.close()
msgr.message(_("No samples found for map calculation"))
return 0
# The fist entries are the samples
map_name_list = []
if not has_samples:
for entry in list:
granule = entry["granule"]
# Do not consider gaps
if granule.get_id() is None:
continue
sample_map_list.append(granule)
map_name_list.append(granule.get_name())
# Attach the map names
map_matrix.append(copy.copy(map_name_list))
has_samples = True
map_name_list = []
for entry in list:
maplist = entry["samples"]
granule = entry["granule"]
# Do not consider gaps in the sampler
if granule.get_id() is None:
continue
if len(maplist) > 1:
msgr.warning(
_(
"Found more than a single map in a sample "
"granule. Only the first map is used for "
"computation. Use t.rast.aggregate.ds to "
"create synchronous raster datasets."
)
)
# Store all maps! This includes non existent maps,
# identified by id == None
map_name_list.append(maplist[0].get_name())
# Attach the map names
map_matrix.append(copy.copy(map_name_list))
id_list.append(dataset.get_name())
else:
input_list.insert(0, first_input)
for dataset in input_list:
list = dataset.get_registered_maps_as_objects(dbif=dbif)
if list is None or len(list) < 1:
dbif.close()
msgr.message(
_("No maps registered in input dataset <{}>").format(
dataset.get_name()
)
)
return 0
map_name_list = []
for map in list:
map_name_list.append(map.get_name())
sample_map_list.append(map)
# Attach the map names
map_matrix.append(copy.copy(map_name_list))
id_list.append(dataset.get_name())
# Needed for map registration
map_list = []
if len(map_matrix) > 0:
msgr.message(_("Starting mapcalc computation..."))
count = 0
# Get the number of samples
num = len(map_matrix[0])
# Parallel processing
proc_list = []
proc_count = 0
# For all samples
for i in range(num):
count += 1
msgr.percent(count, num, 10)
# Create the r.mapcalc statement for the current time step
map_name = "{base}_{suffix}".format(
base=base, suffix=gscript.get_num_suffix(count, num)
)
# Remove spaces and new lines
expr = expression.replace(" ", "")
# Check that all maps are in the sample
valid_maps = True
# Replace all dataset names with their map names of the
# current time step
for j in range(len(map_matrix)):
if map_matrix[j][i] is None:
valid_maps = False
break
# Substitute the dataset name with the map name
expr = expr.replace(id_list[j], map_matrix[j][i])
# Proceed with the next sample
if not valid_maps:
continue
# Create the new map id and check if the map is already
# in the database
map_id = map_name + "@" + mapset
new_map = first_input.get_new_map_instance(map_id)
# Check if new map is in the temporal database
if new_map.is_in_db(dbif):
if gscript.overwrite():
# Remove the existing temporal database entry
new_map.delete(dbif)
new_map = first_input.get_new_map_instance(map_id)
else:
msgr.error(
_(
"Map <%s> is already in temporal database, "
"use overwrite flag to overwrite"
)
)
continue
# Set the time stamp
if sample_map_list[i].is_time_absolute():
start, end = sample_map_list[i].get_absolute_time()
new_map.set_absolute_time(start, end)
else:
start, end, unit = sample_map_list[i].get_relative_time()
new_map.set_relative_time(start, end, unit)
# Set the semantic label
semantic_label = sample_map_list[i].metadata.get_semantic_label()
if semantic_label is not None:
new_map.set_semantic_label(semantic_label)
# Parse the temporal expressions
expr = _operator_parser(expr, sample_map_list[0], sample_map_list[i])
# Add the output map name
expr = "%s=%s" % (map_name, expr)
map_list.append(new_map)
msgr.verbose(_('Apply mapcalc expression: "%s"') % expr)
# Start the parallel r.mapcalc computation
if type == "raster":
proc_list.append(Process(target=_run_mapcalc2d, args=(expr,)))
else:
proc_list.append(Process(target=_run_mapcalc3d, args=(expr,)))
proc_list[proc_count].start()
proc_count += 1
if proc_count == nprocs or proc_count == num or count == num:
proc_count = 0
exitcodes = 0
for proc in proc_list:
proc.join()
exitcodes += proc.exitcode
if exitcodes != 0:
dbif.close()
msgr.fatal(_("Error while mapcalc computation"))
# Empty process list
proc_list = []
# Register the new maps in the output space time dataset
msgr.message(_("Starting map registration in temporal database..."))
(
temporal_type,
semantic_type,
title,
description,
) = first_input.get_initial_values()
new_sp = open_new_stds(
output,
type,
temporal_type,
title,
description,
semantic_type,
dbif,
gscript.overwrite(),
)
count = 0
# collect empty maps to remove them
empty_maps = []
# Insert maps in the temporal database and in the new space time
# dataset
for new_map in map_list:
count += 1
msgr.percent(count, num, 10)
# Read the map data
new_map.load()
# In case of a null map continue, do not register null maps
if (
new_map.metadata.get_min() is None
and new_map.metadata.get_max() is None
):
if not register_null:
empty_maps.append(new_map)
continue
# Insert map in temporal database
new_map.insert(dbif)
new_sp.register_map(new_map, dbif)
# Update the spatio-temporal extent and the metadata table entries
new_sp.update_from_registered_maps(dbif)
# Remove empty maps
if len(empty_maps) > 0:
n_empty, n_tot = len(empty_maps), len(map_list)
msgr.warning(
_("Removing {}/{} ({}%) maps because empty...").format(
n_empty, n_tot, n_empty * 100.0 / n_tot
)
)
names = ""
count = 0
for map in empty_maps:
if count == 0:
names += "%s" % (map.get_name())
else:
names += ",%s" % (map.get_name())
count += 1
if type == "raster":
gscript.run_command(
"g.remove", flags="f", type="raster", name=names, quiet=True
)
elif type == "raster3d":
gscript.run_command(
"g.remove", flags="f", type="raster_3d", name=names, quiet=True
)
dbif.close()
###############################################################################
def _run_mapcalc2d(expr):
"""Helper function to run r.mapcalc in parallel"""
try:
gscript.run_command(
"r.mapcalc", expression=expr, overwrite=gscript.overwrite(), quiet=True
)
except CalledModuleError:
exit(1)
###############################################################################
def _run_mapcalc3d(expr):
"""Helper function to run r3.mapcalc in parallel"""
try:
gscript.run_command(
"r3.mapcalc", expression=expr, overwrite=gscript.overwrite(), quiet=True
)
except CalledModuleError:
exit(1)
###############################################################################
def _operator_parser(expr, first, current):
"""This method parses the expression string and substitutes
the temporal operators with numerical values.
Supported operators for relative and absolute time are:
- td() - the time delta of the current interval in days
and fractions of days or the unit in case of relative time
- start_time() - The start time of the interval from the begin of the
time series in days and fractions of days or the unit
in case of relative time
- end_time() - The end time of the current interval from the begin of
the time series in days and fractions of days or the
unit in case of relative time
Supported operators for absolute time:
- start_doy() - Day of year (doy) from the start time [1 - 366]
- start_dow() - Day of week (dow) from the start time [1 - 7],
the start of the week is monday == 1
- start_year() - The year of the start time [0 - 9999]
- start_month() - The month of the start time [1 - 12]
- start_week() - Week of year of the start time [1 - 54]
- start_day() - Day of month from the start time [1 - 31]
- start_hour() - The hour of the start time [0 - 23]
- start_minute() - The minute of the start time [0 - 59]
- start_second() - The second of the start time [0 - 59]
- end_doy() - Day of year (doy) from the end time [1 - 366]
- end_dow() - Day of week (dow) from the end time [1 - 7],
the start of the week is monday == 1
- end_year() - The year of the end time [0 - 9999]
- end_month() - The month of the end time [1 - 12]
- end_week() - Week of year of the end time [1 - 54]
- end_day() - Day of month from the end time [1 - 31]
- end_hour() - The hour of the end time [0 - 23]
- end_minute() - The minute of the end time [0 - 59]
- end_second() - The minute of the end time [0 - 59]
The modified expression is returned.
"""
is_time_absolute = first.is_time_absolute()
expr = _parse_td_operator(expr, is_time_absolute, first, current)
expr = _parse_start_time_operator(expr, is_time_absolute, first, current)
expr = _parse_end_time_operator(expr, is_time_absolute, first, current)
expr = _parse_start_operators(expr, is_time_absolute, current)
expr = _parse_end_operators(expr, is_time_absolute, current)
return expr
###############################################################################
def _parse_start_operators(expr, is_time_absolute, current):
"""
Supported operators for absolute time:
- start_doy() - Day of year (doy) from the start time [1 - 366]
- start_dow() - Day of week (dow) from the start time [1 - 7],
the start of the week is monday == 1
- start_year() - The year of the start time [0 - 9999]
- start_month() - The month of the start time [1 - 12]
- start_week() - Week of year of the start time [1 - 54]
- start_day() - Day of month from the start time [1 - 31]
- start_hour() - The hour of the start time [0 - 23]
- start_minute() - The minute of the start time [0 - 59]
- start_second() - The second of the start time [0 - 59]
"""
start, end = current.get_absolute_time()
msgr = get_tgis_message_interface()
if expr.find("start_year()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
expr = expr.replace("start_year()", str(start.year))
if expr.find("start_month()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
expr = expr.replace("start_month()", str(start.month))
if expr.find("start_week()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
expr = expr.replace("start_week()", str(start.isocalendar()[1]))
if expr.find("start_day()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
expr = expr.replace("start_day()", str(start.day))
if expr.find("start_hour()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
expr = expr.replace("start_hour()", str(start.hour))
if expr.find("start_minute()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
expr = expr.replace("start_minute()", str(start.minute))
if expr.find("start_second()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
expr = expr.replace("start_second()", str(start.second))
if expr.find("start_dow()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
expr = expr.replace("start_dow()", str(start.isoweekday()))
if expr.find("start_doy()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("start_*")
)
)
year = datetime(start.year, 1, 1)
delta = start - year
expr = expr.replace("start_doy()", str(delta.days + 1))
return expr
###############################################################################
def _parse_end_operators(expr, is_time_absolute, current):
"""
Supported operators for absolute time:
- end_doy() - Day of year (doy) from the end time [1 - 366]
- end_dow() - Day of week (dow) from the end time [1 - 7],
the start of the week is monday == 1
- end_year() - The year of the end time [0 - 9999]
- end_month() - The month of the end time [1 - 12]
- end_week() - Week of year of the end time [1 - 54]
- end_day() - Day of month from the end time [1 - 31]
- end_hour() - The hour of the end time [0 - 23]
- end_minute() - The minute of the end time [0 - 59]
- end_second() - The minute of the end time [0 - 59]
In case of time instances the end* expression will be replaced by
null()
"""
start, end = current.get_absolute_time()
msgr = get_tgis_message_interface()
if expr.find("end_year()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_year()", "null()")
else:
expr = expr.replace("end_year()", str(end.year))
if expr.find("end_month()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_month()", "null()")
else:
expr = expr.replace("end_month()", str(end.month))
if expr.find("end_week()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_week()", "null()")
else:
expr = expr.replace("end_week()", str(end.isocalendar()[1]))
if expr.find("end_day()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_day()", "null()")
else:
expr = expr.replace("end_day()", str(end.day))
if expr.find("end_hour()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_hour()", "null()")
else:
expr = expr.replace("end_hour()", str(end.hour))
if expr.find("end_minute()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_minute()", "null()")
else:
expr = expr.replace("end_minute()", str(end.minute))
if expr.find("end_second()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_second()", "null()")
else:
expr = expr.replace("end_second()", str(end.second))
if expr.find("end_dow()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_dow()", "null()")
else:
expr = expr.replace("end_dow()", str(end.isoweekday()))
if expr.find("end_doy()") >= 0:
if not is_time_absolute:
msgr.fatal(
_(
"The temporal operators <%s> support only absolute "
"time." % ("end_*")
)
)
if not end:
expr = expr.replace("end_doy()", "null()")
else:
year = datetime(end.year, 1, 1)
delta = end - year
expr = expr.replace("end_doy()", str(delta.days + 1))
return expr
###############################################################################
def _parse_td_operator(expr, is_time_absolute, first, current):
"""Parse the time delta operator td(). This operator
represents the size of the current sample time interval
in days and fraction of days for absolute time,
and in relative units in case of relative time.
In case of time instances, the td() operator will be of type null().
"""
if expr.find("td()") >= 0:
td = "null()"
if is_time_absolute:
start, end = current.get_absolute_time()
if end is not None:
td = time_delta_to_relative_time(end - start)
else:
start, end, unit = current.get_relative_time()
if end is not None:
td = end - start
expr = expr.replace("td()", str(td))
return expr
###############################################################################
def _parse_start_time_operator(expr, is_time_absolute, first, current):
"""Parse the start_time() operator. This operator represent
the time difference between the start time of the sample space time
raster dataset and the start time of the current sample interval or
instance. The time is measured in days and fraction of days for absolute
time, and in relative units in case of relative time."""
if expr.find("start_time()") >= 0:
if is_time_absolute:
start1, end = first.get_absolute_time()
start, end = current.get_absolute_time()
x = time_delta_to_relative_time(start - start1)
else:
start1, end, unit = first.get_relative_time()
start, end, unit = current.get_relative_time()
x = start - start1
expr = expr.replace("start_time()", str(x))
return expr
###############################################################################
def _parse_end_time_operator(expr, is_time_absolute, first, current):
"""Parse the end_time() operator. This operator represent
the time difference between the start time of the sample space time
raster dataset and the end time of the current sample interval. The time
is measured in days and fraction of days for absolute time,
and in relative units in case of relative time.
The end_time() will be represented by null() in case of a time instance.
"""
if expr.find("end_time()") >= 0:
x = "null()"
if is_time_absolute:
start1, end = first.get_absolute_time()
start, end = current.get_absolute_time()
if end:
x = time_delta_to_relative_time(end - start1)
else:
start1, end, unit = first.get_relative_time()
start, end, unit = current.get_relative_time()
if end:
x = end - start1
expr = expr.replace("end_time()", str(x))
return expr
