Source code for temporal.register

"""
Functions to register map layer in space time datasets and the temporal database

Usage:

.. code-block:: python

    import grass.temporal as tgis

    tgis.register_maps_in_space_time_dataset(type, name, maps)

(C) 2012-2013 by the GRASS Development Team
This program is free software under the GNU General Public
License (>=v2). Read the file COPYING that comes with GRASS
for details.

:authors: Soeren Gebbert
"""

from datetime import datetime
import grass.script as gs
from .core import get_tgis_message_interface, init_dbif, get_current_mapset
from .open_stds import open_old_stds
from .abstract_map_dataset import AbstractMapDataset
from .factory import dataset_factory
from .datetime_math import (
    check_datetime_string,
    increment_datetime_by_string,
    string_to_datetime,
)

###############################################################################


[docs]def register_maps_in_space_time_dataset( type, name, maps=None, file=None, start=None, end=None, unit=None, increment=None, dbif=None, interval=False, fs="|", update_cmd_list=True, ): """Use this method to register maps in space time datasets. Additionally a start time string and an increment string can be specified to assign a time interval automatically to the maps. It takes care of the correct update of the space time datasets from all registered maps. :param type: The type of the maps raster, raster_3d or vector :param name: The name of the space time dataset. Maps will be registered in the temporal database if the name was set to None :param maps: A comma separated list of map names :param file: Input file, one map per line map with start and optional end time, or the same as io object (with readline capability) :param start: The start date and time of the first map (format absolute: "yyyy-mm-dd HH:MM:SS" or "yyyy-mm-dd", format relative is integer 5) :param end: The end date and time of the first map (format absolute: "yyyy-mm-dd HH:MM:SS" or "yyyy-mm-dd", format relative is integer 5) :param unit: The unit of the relative time: years, months, days, hours, minutes, seconds :param increment: Time increment between maps for time stamp creation (format absolute: NNN seconds, minutes, hours, days, weeks, months, years; format relative: 1.0) :param dbif: The database interface to be used :param interval: If True, time intervals are created in case the start time and an increment is provided :param fs: Field separator used in input file :param update_cmd_list: If is True, the command that was invoking this process will be written to the process history """ start_time_in_file = False end_time_in_file = False semantic_label_in_file = False overwrite = gs.overwrite() msgr = get_tgis_message_interface() msgr.debug(1, "register_maps_in_space_time_dataset()") # Make sure the arguments are of type string if start != "" and start is not None: start = str(start) if end != "" and end is not None: end = str(end) if increment != "" and increment is not None: increment = str(increment) if maps and file: msgr.fatal(_("maps and file are mutually exclusive")) if end and increment: msgr.fatal(_("end and increment are mutually exclusive")) if end and interval: msgr.fatal(_("end and the interval flag are mutually exclusive")) if increment and not start: msgr.fatal(_("The increment option requires the start option")) if interval and not start: msgr.fatal(_("The interval flag requires the start option")) if end and not start: msgr.fatal(_("Please specify start_time and end_time")) if not maps and not file: msgr.fatal(_("Please specify maps or file")) # We may need the mapset mapset = get_current_mapset() dbif, connection_state_changed = init_dbif(dbif) # create new stds only in the current mapset # remove all connections to any other mapsets # ugly hack ! currcon = {} currcon[mapset] = dbif.connections[mapset] dbif.connections = currcon # The name of the space time dataset is optional if name: sp = open_old_stds(name, type, dbif) if sp.is_time_relative() and (start or end) and not unit: dbif.close() msgr.fatal( _( "Space time {sp} dataset <{name}> with relative" " time found, but no relative unit set for {sp} " "maps" ).format(name=name, sp=sp.get_new_map_instance(None).get_type()) ) maplist = [] # Map names as comma separated string if maps: maplist = maps.split(",") # Build the map list again with the ids for idx, maplist_item in enumerate(maplist): maplist[idx] = { "id": AbstractMapDataset.build_id_from_search_path(maplist_item, type) } # Read the map list from file if file: if hasattr(file, "readline"): fd = file else: fd = open(file, "r") line = True while True: line = fd.readline().strip() if not line: break line_list = line.split(fs) # Detect start and end time (and semantic label) if len(line_list) == 2: start_time_in_file = True end_time_in_file = False semantic_label_in_file = False elif len(line_list) == 3: start_time_in_file = True # Check if last column is an end time or a semantic label time_object = check_datetime_string(line_list[2]) if not sp.is_time_relative() and isinstance(time_object, datetime): end_time_in_file = True semantic_label_in_file = False else: end_time_in_file = False semantic_label_in_file = True elif len(line_list) == 4: start_time_in_file = True end_time_in_file = True semantic_label_in_file = True else: start_time_in_file = False end_time_in_file = False semantic_label_in_file = False mapname = line_list[0].strip() row = {} if start_time_in_file: row["start"] = line_list[1].strip() if end_time_in_file: row["end"] = line_list[2].strip() if semantic_label_in_file: idx = 3 if end_time_in_file else 2 # case-sensitive, the user decides on the band name row["semantic_label"] = line_list[idx].strip() row["id"] = AbstractMapDataset.build_id_from_search_path(mapname, type) maplist.append(row) if start_time_in_file is True and increment: increment = None msgr.warning( _( "The increment option will be ignored because of time stamps in " "input file" ) ) if start_time_in_file is True and interval: increment = None msgr.warning( _( "The interval flag will be ignored because of time stamps in input " "file" ) ) fd.close() num_maps = len(maplist) map_object_list = [] statement = "" # Store the ids of datasets that must be updated datatsets_to_modify = {} msgr.debug(2, "Gathering map information...") for count, row in enumerate(maplist): if count % 50 == 0: msgr.percent(count, num_maps, 1) # Get a new instance of the map type map_object = dataset_factory(type, row["id"]) map_object_id = map_object.get_map_id() map_object_layer = map_object.get_layer() map_object_type = map_object.get_type() if not map_object.map_exists(): msgr.fatal( _("Unable to update {t} map <{mid}>. The map does not exist.").format( t=map_object_type, mid=map_object_id ) ) # Use the time data from file if "start" in row: start = row["start"] if "end" in row: end = row["end"] # Use the semantic label from file if "semantic_label" in row: semantic_label = row["semantic_label"] else: semantic_label = None is_in_db = map_object.is_in_db(dbif, mapset) # Put the map into the database of the current mapset if not is_in_db: # Break in case no valid time is provided if (start == "" or start is None) and not map_object.has_grass_timestamp(): dbif.close() if map_object_layer: msgr.fatal( _( "Unable to register {t} map <{mid}> with " "layer {l}. The map has timestamp and " "the start time is not set." ).format( t=map_object_type, mid=map_object_id, l=map_object_layer, ) ) else: msgr.fatal( _( "Unable to register {t} map <{mid}>. The" " map has no timestamp and the start time " "is not set." ).format(t=map_object_type, mid=map_object_id) ) if start != "" and start is not None: # We need to check if the time is absolute and the unit was specified time_object = check_datetime_string(start) if isinstance(time_object, datetime) and unit: msgr.fatal(_("unit can only be set for relative time")) if not isinstance(time_object, datetime) and not unit: msgr.fatal(_("unit must be set in case of relative time stamps")) if unit: map_object.set_time_to_relative() else: map_object.set_time_to_absolute() else: # Check the overwrite flag if not overwrite: if map_object_layer: msgr.warning( _( "Map is already registered in temporal " "database. Unable to update {t} map " "<{mid}> with layer {l}. Overwrite flag" " is not set." ).format( t=map_object_type, mid=map_object_id, l=str(map_object_layer), ) ) else: msgr.warning( _( "Map is already registered in temporal " "database. Unable to update {t} map " "<{mid}>. Overwrite flag is not set." ).format(t=map_object_type, mid=map_object_id) ) # Simple registration is allowed if name: map_object_list.append(map_object) # Jump to next map continue # Reload properties from database map_object.select(dbif) # Save the datasets that must be updated datasets = map_object.get_registered_stds(dbif) if datasets is not None: for dataset in datasets: if dataset != "": datatsets_to_modify[dataset] = dataset if name and map_object.get_temporal_type() != sp.get_temporal_type(): dbif.close() if map_object_layer: msgr.fatal( _( "Unable to update {t} map <{id}> " "with layer {l}. The temporal types " "are different." ).format( t=map_object_type, mid=map_object_id, l=map_object_layer, ) ) else: msgr.fatal( _( "Unable to update {t} map <{mid}>. " "The temporal types are different." ).format(t=map_object_type, mid=map_object_id) ) # Load the data from the grass file database map_object.load() # Try to read an existing time stamp from the grass spatial database # in case this map wasn't already registered in the temporal database # Read the spatial database time stamp only, if no time stamp was provided for # this map # as method argument or in the input file if not is_in_db and not start: map_object.read_timestamp_from_grass() # Set the valid time if start: # In case the time is in the input file we ignore the increment # counter if start_time_in_file: count = 1 assign_valid_time_to_map( ttype=map_object.get_temporal_type(), map_object=map_object, start=start, end=end, unit=unit, increment=increment, mult=count, interval=interval, ) # Set the semantic label (only raster type supported) if semantic_label: # semantic label defined in input file # -> update raster metadata # -> write band identifier to GRASS data base map_object.set_semantic_label(semantic_label) else: # Try to read semantic label from GRASS data base if defined map_object.read_semantic_label_from_grass() if is_in_db: # Gather the SQL update statement statement += map_object.update_all(dbif=dbif, execute=False) else: # Gather the SQL insert statement statement += map_object.insert(dbif=dbif, execute=False) # Sqlite3 performance is better for huge datasets when committing in # small chunks if dbif.get_dbmi().__name__ == "sqlite3": if count % 100 == 0: if statement is not None and statement != "": dbif.execute_transaction(statement) statement = "" # Store the maps in a list to register in a space time dataset if name: map_object_list.append(map_object) msgr.percent(num_maps, num_maps, 1) if statement is not None and statement != "": dbif.execute_transaction(statement) # Finally Register the maps in the space time dataset if name and map_object_list: num_maps = len(map_object_list) for count, map_object in enumerate(map_object_list): if count % 50 == 0: msgr.percent(count, num_maps, 1) sp.register_map(map=map_object, dbif=dbif) # Update the space time tables if name and map_object_list: sp.update_from_registered_maps(dbif) if update_cmd_list is True: sp.update_command_string(dbif=dbif) # Update affected datasets if datatsets_to_modify: for dataset in datatsets_to_modify: if type in ["rast", "raster"]: ds = dataset_factory("strds", dataset) elif type in ["raster_3d", "rast3d", "raster3d"]: ds = dataset_factory("str3ds", dataset) elif type in ["vect", "vector"]: ds = dataset_factory("stvds", dataset) ds.select(dbif) ds.update_from_registered_maps(dbif) if connection_state_changed is True: dbif.close() msgr.percent(num_maps, num_maps, 1)
###############################################################################
[docs]def assign_valid_time_to_map( ttype, map_object, start, end, unit, increment=None, mult=1, interval=False ): """Assign the valid time to a map dataset :param ttype: The temporal type which should be assigned and which the time format is of :param map: A map dataset object derived from abstract_map_dataset :param start: The start date and time of the first map (format absolute: "yyyy-mm-dd HH:MM:SS" or "yyyy-mm-dd", format relative is integer 5) :param end: The end date and time of the first map (format absolute: "yyyy-mm-dd HH:MM:SS" or "yyyy-mm-dd", format relative is integer 5) :param unit: The unit of the relative time: years, months, days, hours, minutes, seconds :param increment: Time increment between maps for time stamp creation (format absolute: NNN seconds, minutes, hours, days, weeks, months, years; format relative is integer 1) :param mult: A multiplier for the increment :param interval: If True, time intervals are created in case the start time and an increment is provided """ msgr = get_tgis_message_interface() if ttype == "absolute": start_time = string_to_datetime(start) if start_time is None: msgr.fatal( _('Unable to convert string "{}" into a datetime object').format(start) ) end_time = None if end: end_time = string_to_datetime(end) if end_time is None: msgr.fatal( _('Unable to convert string "{}" into a datetime object').format( end ) ) # Add the increment if increment: start_time = increment_datetime_by_string(start_time, increment, mult) if start_time is None: msgr.fatal(_("Error occurred in increment computation")) if interval: end_time = increment_datetime_by_string(start_time, increment, 1) if end_time is None: msgr.fatal(_("Error occurred in increment computation")) if map_object.get_layer(): msgr.debug( 1, _( "Set absolute valid time for map <{id}> with " "layer {layer} to {start} - {end}" ).format( id=map_object.get_map_id(), layer=map_object.get_layer(), start=str(start_time), end=str(end_time), ), ) else: msgr.debug( 1, _( "Set absolute valid time for map <{mid}> to " "{start_time} - {end_time}" ).format( mid=map_object.get_map_id(), start_time=str(start_time), end_time=str(end_time), ), ) map_object.set_absolute_time(start_time, end_time) else: start_time = int(start) end_time = None if end: end_time = int(end) if increment: start_time = start_time + mult * int(increment) if interval: end_time = start_time + int(increment) if map_object.get_layer(): msgr.debug( 1, _( "Set relative valid time for map <{mid}> with layer" " {layer} to {start} - {end} with unit {unit}" ).format( mid=map_object.get_map_id(), layer=map_object.get_layer(), start=start_time, end=str(end_time), unit=unit, ), ) else: msgr.debug( 1, _( "Set relative valid time for map <{mid}> to " "{start} - {end} with unit {unit}" ).format( mid=map_object.get_map_id(), start=start_time, end=str(end_time), unit=unit, ), ) map_object.set_relative_time(start_time, end_time, unit)
##############################################################################
[docs]def register_map_object_list( type, map_list, output_stds, delete_empty=False, unit=None, dbif=None ): """Register a list of AbstractMapDataset objects in the temporal database and optional in a space time dataset. :param type: The type of the map layer (raster, raster_3d, vector) :param map_list: List of AbstractMapDataset objects :param output_stds: The output stds :param delete_empty: Set True to delete empty map layer found in the map_list :param unit: The temporal unit of the space time dataset :param dbif: The database interface to be used """ import grass.pygrass.modules as pymod import copy dbif, connection_state_changed = init_dbif(None) filename = gs.tempfile(True) with open(filename, "w") as register_file: empty_maps = [] for map_layer in map_list: # Read the map data map_layer.load() # In case of a empty map continue, do not register empty maps if delete_empty: if type in ["raster", "raster_3d", "rast", "rast3d"]: if ( map_layer.metadata.get_min() is None and map_layer.metadata.get_max() is None ): empty_maps.append(map_layer) continue if type == "vector": if map_layer.metadata.get_number_of_primitives() == 0: empty_maps.append(map_layer) continue start, end = map_layer.get_temporal_extent_as_tuple() id = map_layer.get_id() if not end: end = start string = f"{id}|{start}|{end}\n" register_file.write(string) if output_stds: output_stds_id = output_stds.get_id() else: output_stds_id = None register_maps_in_space_time_dataset( type, output_stds_id, unit=unit, file=filename, dbif=dbif ) # Remove empty maps and unregister them from the temporal database g_remove = pymod.Module("g.remove", flags="f", quiet=True, run_=False, finish_=True) if len(empty_maps) > 0: for map_object in empty_maps: mod = copy.deepcopy(g_remove) if map_object.get_name(): if map_object.get_type() == "raster": mod(type="raster", name=map_object.get_name()) if map_object.get_type() == "raster3d": mod(type="raster_3d", name=map_object.get_name()) if map_object.get_type() == "vector": mod(type="vector", name=map_object.get_name()) mod.run() if map_object.is_in_db(dbif): map_object.delete(dbif) if connection_state_changed: dbif.close()
if __name__ == "__main__": import doctest doctest.testmod()