From abe8b6b1ebb612666667bf6ffdd63660aeb0987e Mon Sep 17 00:00:00 2001 From: Tito Brasolin Date: Mon, 28 Oct 2024 17:16:16 +0100 Subject: [PATCH] chore!: rename commands --- requirements.txt | 20 +- scp_udg_client_app/cli.py | 15 +- scp_udg_client_app/commands/fetch.py | 94 ++++++ ...retrieve_last_dataset.py => fetch_last.py} | 5 +- scp_udg_client_app/commands/poll.py | 269 ----------------- scp_udg_client_app/commands/run.py | 275 +++++++++++++++++- .../commands/search_datasets.py | 63 ---- scp_udg_client_app/commands/test.py | 12 +- scp_udg_client_app/helpers.py | 22 +- setup.py | 16 +- 10 files changed, 418 insertions(+), 373 deletions(-) create mode 100755 scp_udg_client_app/commands/fetch.py rename scp_udg_client_app/commands/{retrieve_last_dataset.py => fetch_last.py} (91%) delete mode 100644 scp_udg_client_app/commands/poll.py delete mode 100755 scp_udg_client_app/commands/search_datasets.py diff --git a/requirements.txt b/requirements.txt index 48ba77e..79a806c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,14 @@ -click -click-log +click~=8.1.7 +click-config-file +click-log~=0.4.0 +click-params scp-udg-client-rest @ git+https://crossserv.bologna.enea.it/gitlab/cross/scp_udg_client_python_rest.git -setuptools -wheel -jproperties -jpype1 -requests-cache +setuptools~=75.1.0 +wheel~=0.44.0 +jproperties~=2.1.2 +jpype1~=1.5.0 +requests-cache~=1.2.1 simple-file-poller -python-dateutil \ No newline at end of file +python-dateutil~=2.9.0.post0 +javaproperties~=0.8.1 +tocase~=1.0.0 \ No newline at end of file diff --git a/scp_udg_client_app/cli.py b/scp_udg_client_app/cli.py index 461548a..5f139d8 100644 --- a/scp_udg_client_app/cli.py +++ b/scp_udg_client_app/cli.py @@ -16,9 +16,7 @@ from jpype.types import * # Launch the JVM jpype.startJVM(classpath=[str(files(__package__).joinpath("jars/*"))]) -from scp_udg_client_app.commands import ( - retrieve_last_dataset, search_datasets, test, login, logout, run, report, validate, poll -) +from scp_udg_client_app.commands import test, fetch_last, fetch, run from scp_udg_client_app.config import Config config = Config() @@ -38,7 +36,7 @@ def setup_logging(): logger.addHandler(handler) -@click.group() +@click.group(context_settings={"auto_envvar_prefix": "SCP_UDG_CLIENT"}) @click.option("-c", "--config-file", type=click.File('rb'), expose_value=False, callback=config.load, is_eager=True, help=f"Path to config properties file; loads {config.DEFAULT_PATH} by default if it exists") @click.version_option() @@ -50,13 +48,14 @@ def cli(ctx): cli.add_command(test.test) -cli.add_command(retrieve_last_dataset.retrieve_last_dataset) -cli.add_command(search_datasets.search_datasets) +cli.add_command(fetch.fetch) +cli.add_command(fetch_last.fetch_last) #cli.add_command(login.login) #cli.add_command(logout.logout) #cli.add_command(run.run) #cli.add_command(report.report) #cli.add_command(validate.validate) -cli.add_command(poll.poll) +cli.add_command(run.run) -main = cli +if __name__ == "__main__": + cli() diff --git a/scp_udg_client_app/commands/fetch.py b/scp_udg_client_app/commands/fetch.py new file mode 100755 index 0000000..61beb59 --- /dev/null +++ b/scp_udg_client_app/commands/fetch.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import logging +import os + +import click +import click_config_file +from click import FloatRange +from click_params import UrlParamType +from scp_udg_client_rest import SearchingRequest + +from scp_udg_client_app.config import DEFAULT_UDG_ENDPOINT +from scp_udg_client_app.helpers import validate_resource_id, create_api_client_instance, active_login, save_dataset, \ + javaproperties_provider + +# Setting up Logger +logger = logging.getLogger() + +# Introduce constant +RESPONSE_CODE_SUCCESS = "03" + + +def fetch_and_save_datasets(api_client, request, resource_id, inbox_dir): + response = api_client.searching(request) + if response.code == RESPONSE_CODE_SUCCESS: + for dataset in response.dataset: + save_dataset(dataset, resource_id, inbox_dir) + else: + logger.warning("Dataset non trovati!") + + +def get_udg_endpoint(): + return os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT) + + +def get_resource_id(): + return os.environ.get("SCP_UDG_CLIENT_RESOURCE_ID", "") + + +def get_user_name(): + return os.environ.get("SCP_UDG_CLIENT_USER_NAME", "") + + +def get_user_password(): + return os.environ.get("SCP_UDG_CLIENT_USER_PASSWORD", "") + + +def get_inbox_dir(): + return os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox") + + +@click.command() +@click_config_file.configuration_option(provider=javaproperties_provider, show_default=True) +@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=get_udg_endpoint) +@click.option('--username', 'user_name', prompt=True, prompt_required=False, default=get_user_name) +@click.option('--password', 'user_password', prompt=True, prompt_required=False, hide_input=True, + default=get_user_password) +@click.option('--resource-id', prompt=True, callback=validate_resource_id, default=get_resource_id) +@click.option('--access-token', hidden=True, prompt=True, prompt_required=False) +@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), default=get_inbox_dir) +@click.option('--period-start', type=click.DateTime(), + help="Date and time from which you want to specify the start of a time interval. If it is absent, the time window includes all possible values.") +@click.option('--period-end', type=click.DateTime(), + help="Date and time from which you want to specify the end of a time interval. If it is absent, the time window includes all possible values from period_start until the moment the call is made.") +@click.option('--center-latitude', type=FloatRange(max=90, min=-90), + help="Latitude of the center where space research will be carried out.") +@click.option('--center-longitude', type=FloatRange(max=180, min=-180), + help="Longitude of the center on which space research will be carried out.") +@click.option('--distance', type=click.FloatRange(min=0), + help="Radius of the circle, in meters, on which space research will be carried out.") +@click.pass_context +def fetch(ctx, udg_endpoint, resource_id, access_token, user_name, user_password, inbox_dir, period_start, + period_end, center_latitude, center_longitude, distance): + """Fetch and save datasets from an UDG endpoint.""" + api_client = create_api_client_instance(udg_endpoint, access_token) + click.echo(user_name) + if not active_login(api_client, user_name, user_password): + logger.warning("Login fallito!") + return + + try: + request = SearchingRequest( + resource_id=resource_id, + period_start=period_start, + period_end=period_end, + center_latitude=center_latitude, + center_longitude=center_longitude, + distance=distance) + fetch_and_save_datasets(api_client, request, resource_id, inbox_dir) + except Exception as e: + logger.error(f"Errore durante il recupero dei dataset: {str(e)}") + + +if __name__ == '__main__': + fetch() diff --git a/scp_udg_client_app/commands/retrieve_last_dataset.py b/scp_udg_client_app/commands/fetch_last.py similarity index 91% rename from scp_udg_client_app/commands/retrieve_last_dataset.py rename to scp_udg_client_app/commands/fetch_last.py index bd246e1..ec539fa 100755 --- a/scp_udg_client_app/commands/retrieve_last_dataset.py +++ b/scp_udg_client_app/commands/fetch_last.py @@ -24,7 +24,8 @@ logger = logging.getLogger() @click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), default=lambda: os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox")) @click.pass_context -def retrieve_last_dataset(ctx, udg_endpoint, resource_id, access_token, user_name, user_password, inbox_dir): +def fetch_last(ctx, udg_endpoint, resource_id, access_token, user_name, user_password, inbox_dir): + """Fetch and save the last dataset from an UDG endpoint.""" api_client = create_api_client_instance(udg_endpoint, access_token) if not active_login(api_client, user_name, user_password): @@ -46,4 +47,4 @@ def process_response(response, resource_id, inbox_dir): if __name__ == '__main__': - retrieve_last_dataset() + fetch_last() diff --git a/scp_udg_client_app/commands/poll.py b/scp_udg_client_app/commands/poll.py deleted file mode 100644 index 23bc55d..0000000 --- a/scp_udg_client_app/commands/poll.py +++ /dev/null @@ -1,269 +0,0 @@ -import os -import re -import shutil -import time -from pathlib import Path -from time import sleep -import click -import requests_cache -import scp_udg_client_rest -from dateutil.parser import parse, ParserError -# import the Java modules -# noinspection PyUnresolvedReferences -from it.enea.validation.Validators import JSONSchematronValidator -from scp_udg_client_rest import LoginRequest, PushRequest -from scp_udg_client_rest.models.scps_urbandataset_schema20 import ScpsUrbandatasetSchema20 -from sfp import Poller, Parameters - -SUPPORTED_EXTS = [".json"] -""" supported file extensions (lower case). """ - -pushed_files_info = {} - - -class ValidationFailed(Exception): pass - - -class AuthenticationFailed(Exception): pass - - -def check_file(fname, poller): - poller.debug("Checking:", fname) - - try: - try: - parse_dict = parse_filename(fname) - except ValueError as e: - raise ValidationFailed(str(e)) from e - - poller.debug("Parsed filename:", parse_dict) - - # We check the validity of timestamps - - collaboration_start = parse_dict["collaboration_start"] - try: - parse(collaboration_start) - except ParserError as e: - raise ValidationFailed(" ".join([e, "in 'collaboration_start'"])) from e - - timestamp_ud = parse_dict["timestamp_ud"] - try: - parse(timestamp_ud) - except ParserError as e: - raise ValidationFailed(" ".join([e, "in 'timestamp_ud'"])) from e - - # We try to read and interpret the file - - try: - ud_content = Path(fname).read_text() - dataset = ScpsUrbandatasetSchema20.from_json(ud_content) - except Exception as e: - raise ValidationFailed(str(e)) from e - - urbandataset_id = parse_dict["urbandataset_id"] - - if dataset.urban_dataset.specification.id.value != urbandataset_id: - raise ValidationFailed(f"Incorrect ID: {urbandataset_id}") - - except ValidationFailed as e: - poller.error(e) - return False - - poller.debug("UrbanDataset ID:", urbandataset_id) - - # Schematron validation can fail for a variety of reasons, but we do not consider this to be fatal. - - nome_urbandataset = parse_dict["nome_urbandataset"] - session = requests_cache.CachedSession() - - try: - response = session.get( - f'https://smartcityplatform.enea.it/UDWebLibrary/it/template/{nome_urbandataset}?templ=schem') - ud_schema_content = response.text - - validator = JSONSchematronValidator() - validation_report = validator.getValidationReport(ud_content, ud_schema_content) - - is_success = validation_report.isSuccess() - poller.debug("Schematron valid:", is_success) - except Exception as e: - poller.error(e) - - # Let's try to send the file to the platform. - - response_code = "" - - try: - alive = False - if api_instance.api_client.configuration.access_token: - is_alive_response = api_instance.is_alive() - poller.info("Is alive response:", is_alive_response) - alive = (is_alive_response.code == "00") - - if not alive: - login_request = LoginRequest(username=poller.params.user_name, password=poller.params.user_password) - login_response = api_instance.login(login_request) - poller.info("Login response:", login_response) - if login_response.code != "01": - response_code = login_response.code - raise AuthenticationFailed(login_response.message) - api_instance.api_client.configuration.access_token = login_response.token - - resource_id = parse_dict["resource_id"] - push_request = PushRequest(resource_id=resource_id, dataset=dataset) - push_response = api_instance.push(push_request) - poller.info("Push response:", push_response) - - response_code = push_response.code - - except Exception as e: - poller.error(e) - - initial_delay = 60 * poller.params.retry_frequency - backoff_factor = initial_delay - - k = os.path.basename(fname) - if k in pushed_files_info: - try_count = pushed_files_info[k]["try_count"] - delay = initial_delay * backoff_factor ** try_count - pushed_files_info[k]["try_count"] = try_count + 1 - pushed_files_info[k]["next_try_at"] = time.time() + delay - pushed_files_info[k]["response_code"] = response_code - else: - pushed_files_info[k] = {"try_count": 1, "next_try_at": time.time() + initial_delay, - "response_code": response_code} - return True - - -def auto_rename(filename): - # rename filename if already exists - name, ext = os.path.splitext(filename) - i = 0 - while 1: - if not os.path.exists(filename): return filename - i += 1 - filename = name + str(i) + ext - - -def parse_filename(filepath, custom_patterns=None): - """ - Parses the filename based on the specified conventions and extracts information. - - :param filepath: The filename to parse - :param custom_patterns: array of regex patterns that should be included at the end of the patterns - :return: A dictionary with the extracted information - """ - - patterns = [ - r"^(?P(?PSCP-(?:0|[1-9][0-9]*))_(?P(?P(?:[A-Z][a-z0-9]*)+)-(?P(?:0|[1-9][0-9]*)))_(?P(?P(?:[A-Z][a-z0-9]*)+)-(?P(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*))*))_(?P[0-9]{14}))_-_(?P[0-9]{14})$", - r"^(?P[0-9]{8}T[0-9]{6})_(?P(?PSCP-(?:0|[1-9][0-9]*))_(?P(?P(?:[A-Z][a-z0-9]*)+)-(?P(?:0|[1-9][0-9]*)))_(?P(?P(?:[A-Z][a-z0-9]*)+)-(?P(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*))*))_(?P[0-9]{14}))$" - ] - - if custom_patterns is not None: - patterns += list(custom_patterns) - - path = Path(filepath) - filename, basename = path.name, path.stem - - for pattern in patterns: - match = re.match(pattern, basename) - try: - result = {k: v for k, v in match.groupdict().items() if v is not None} - return result - except AttributeError: - continue - - raise ValueError('Failed to parse filename: %s' % filename) - - -@click.command() -@click.option('--outbox-dir', type=click.Path(resolve_path=True, file_okay=False), default='Outbox') -@click.option('--oksent-dir', type=click.Path(resolve_path=True, file_okay=False), default='OKSent') -@click.option('--nosent-dir', type=click.Path(resolve_path=True, file_okay=False), default='NOSent') -@click.option('--retry-dirs', multiple=True, default=[], help="Directory interne a NOSent, soggette a retry") -@click.option('--retry', type=click.INT, default=4, help="Numero di tentativi di re-invio") -@click.option('--retry-frequency', type=click.INT, default=9, - help="Frequenza, espressa in minuti, tra un invio e l'altro in caso di retry") -@click.option('--udg-endpoint') -@click.option('--user-name') -@click.option('--user-password') -@click.pass_context -def poll(ctx, outbox_dir, oksent_dir, nosent_dir, retry_dirs, retry, retry_frequency, udg_endpoint, user_name, - user_password): - global api_instance - verbose = ctx.obj['verbose'] - progress = ctx.obj['progress'] - - output_dir = os.path.join(outbox_dir, "output_dir") - Path(output_dir).mkdir(parents=True, exist_ok=True) - Path(oksent_dir).mkdir(parents=True, exist_ok=True) - Path(nosent_dir).mkdir(parents=True, exist_ok=True) - - configuration = scp_udg_client_rest.Configuration(host=udg_endpoint) - api_client = scp_udg_client_rest.ApiClient(configuration) - api_instance = scp_udg_client_rest.UrbanDatasetGatewayApi(api_client) - - params = Parameters() - params.udg_endpoint = udg_endpoint - params.user_name = user_name - params.user_password = user_password - params.retry_frequency = retry_frequency - - p = Poller( - input_dir=outbox_dir, - output_dir=output_dir, - extensions=SUPPORTED_EXTS, - verbose=verbose, - progress=progress, - check_file=check_file, - blacklist_tries=1, # TODO: Gestire "blacklist" - params=params) - - while not p.is_stopped: - # Prima di ogni esecuzione del "poller" verifichiamo se in - # output_dir esistono file da ricollocare in input_dir. - - for file_name in os.listdir(p.output_dir): - src = os.path.join(p.output_dir, file_name) - if os.path.isdir(src): - continue - if file_name not in pushed_files_info or pushed_files_info[file_name]["next_try_at"] >= time.time(): - # Potrebbe trattarsi di un file rimasto da una precedente esecuzione terminata - # in modo brusco, oppure di un invio da ritentare. Lo riportiamo in input_dir, - # ma solo se non è già presente un eventuale file omonimo. - dst = os.path.join(p.input_dir, file_name) - os.path.exists(dst) or shutil.move(src, dst) - - p.poll() - - # Verifichiamo il contenuto della cartella di output. - - for file_name in os.listdir(p.output_dir): - src = os.path.join(p.output_dir, file_name) - p.debug("Src: ", src) - if os.path.isdir(src): - continue - if file_name not in pushed_files_info: - # Si tratta di un file che ha fallito la convalida - # locale, lo spostiamo immediatamente in nosent_dir - shutil.move(src, auto_rename(os.path.join(nosent_dir, file_name))) - else: - # Potrebbe essere un file che è andato a buon fine oppure che ha esaurito i tentativi. - file_info = pushed_files_info[file_name] - response_code = str(file_info["response_code"]) - if response_code != "02": - dst_dir = os.path.join(nosent_dir, response_code) - Path(dst_dir).mkdir(parents=True, exist_ok=True) - dst = auto_rename(os.path.join(dst_dir, file_name)) - if (not response_code or response_code in retry_dirs) and file_info["try_count"] <= retry: - # TODO: Il file viene comunque archiviato, forse questo comportamento potrebbe essere opzionale. - shutil.copy(src, dst) - continue # N.B. Senza cancellare l'elemento da pushed_files_info - else: - shutil.move(src, dst) - else: - shutil.move(src, auto_rename(os.path.join(oksent_dir, file_name))) - del pushed_files_info[file_name] - - p.debug("Waiting %d seconds before next poll" % p.poll_wait) - sleep(p.poll_wait) diff --git a/scp_udg_client_app/commands/run.py b/scp_udg_client_app/commands/run.py index d17f88b..1d8f6a3 100644 --- a/scp_udg_client_app/commands/run.py +++ b/scp_udg_client_app/commands/run.py @@ -1,5 +1,276 @@ +import os +import re +import shutil +import time +from pathlib import Path +from time import sleep import click +import click_config_file +from click_params import UrlParamType, StringListParamType +import requests_cache +import scp_udg_client_rest + +from dateutil.parser import parse, ParserError +# import the Java modules +# noinspection PyUnresolvedReferences +from it.enea.validation.Validators import JSONSchematronValidator +from scp_udg_client_rest import LoginRequest, PushRequest +from scp_udg_client_rest.models.scps_urbandataset_schema20 import ScpsUrbandatasetSchema20 +from sfp import Poller, Parameters + +from scp_udg_client_app.helpers import javaproperties_provider + +SUPPORTED_EXTS = [".json"] +""" supported file extensions (lower case). """ + +pushed_files_info = {} + + +class ValidationFailed(Exception): pass + + +class AuthenticationFailed(Exception): pass + + +def check_file(fname, poller): + poller.debug("Checking:", fname) + + try: + try: + parse_dict = parse_filename(fname) + except ValueError as e: + raise ValidationFailed(str(e)) from e + + poller.debug("Parsed filename:", parse_dict) + + # We check the validity of timestamps + + collaboration_start = parse_dict["collaboration_start"] + try: + parse(collaboration_start) + except ParserError as e: + raise ValidationFailed(" ".join([e, "in 'collaboration_start'"])) from e + + timestamp_ud = parse_dict["timestamp_ud"] + try: + parse(timestamp_ud) + except ParserError as e: + raise ValidationFailed(" ".join([e, "in 'timestamp_ud'"])) from e + + # We try to read and interpret the file + + try: + ud_content = Path(fname).read_text() + dataset = ScpsUrbandatasetSchema20.from_json(ud_content) + except Exception as e: + raise ValidationFailed(str(e)) from e + + urbandataset_id = parse_dict["urbandataset_id"] + + if dataset.urban_dataset.specification.id.value != urbandataset_id: + raise ValidationFailed(f"Incorrect ID: {urbandataset_id}") + + except ValidationFailed as e: + poller.error(e) + return False + + poller.debug("UrbanDataset ID:", urbandataset_id) + + # Schematron validation can fail for a variety of reasons, but we do not consider this to be fatal. + + nome_urbandataset = parse_dict["nome_urbandataset"] + session = requests_cache.CachedSession() + + try: + response = session.get( + f'https://smartcityplatform.enea.it/UDWebLibrary/it/template/{nome_urbandataset}?templ=schem') + ud_schema_content = response.text + + validator = JSONSchematronValidator() + validation_report = validator.getValidationReport(ud_content, ud_schema_content) + + is_success = validation_report.isSuccess() + poller.debug("Schematron valid:", is_success) + except Exception as e: + poller.error(e) + + # Let's try to send the file to the platform. + + response_code = "" + + try: + alive = False + if api_instance.api_client.configuration.access_token: + is_alive_response = api_instance.is_alive() + poller.info("Is alive response:", is_alive_response) + alive = (is_alive_response.code == "00") + + if not alive: + login_request = LoginRequest(username=poller.params.user_name, password=poller.params.user_password) + login_response = api_instance.login(login_request) + poller.info("Login response:", login_response) + if login_response.code != "01": + response_code = login_response.code + raise AuthenticationFailed(login_response.message) + api_instance.api_client.configuration.access_token = login_response.token + + resource_id = parse_dict["resource_id"] + push_request = PushRequest(resource_id=resource_id, dataset=dataset) + push_response = api_instance.push(push_request) + poller.info("Push response:", push_response) + + response_code = push_response.code + + except Exception as e: + poller.error(e) + + initial_delay = 60 * poller.params.retry_frequency + backoff_factor = initial_delay + + k = os.path.basename(fname) + if k in pushed_files_info: + try_count = pushed_files_info[k]["try_count"] + delay = initial_delay * backoff_factor ** try_count + pushed_files_info[k]["try_count"] = try_count + 1 + pushed_files_info[k]["next_try_at"] = time.time() + delay + pushed_files_info[k]["response_code"] = response_code + else: + pushed_files_info[k] = {"try_count": 1, "next_try_at": time.time() + initial_delay, + "response_code": response_code} + return True + + +def auto_rename(filename): + # rename filename if already exists + name, ext = os.path.splitext(filename) + i = 0 + while 1: + if not os.path.exists(filename): return filename + i += 1 + filename = name + str(i) + ext + + +def parse_filename(filepath, custom_patterns=None): + """ + Parses the filename based on the specified conventions and extracts information. + + :param filepath: The filename to parse + :param custom_patterns: array of regex patterns that should be included at the end of the patterns + :return: A dictionary with the extracted information + """ + + patterns = [ + r"^(?P(?PSCP-(?:0|[1-9][0-9]*))_(?P(?P(?:[A-Z][a-z0-9]*)+)-(?P(?:0|[1-9][0-9]*)))_(?P(?P(?:[A-Z][a-z0-9]*)+)-(?P(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*))*))_(?P[0-9]{14}))_-_(?P[0-9]{14})$", + r"^(?P[0-9]{8}T[0-9]{6})_(?P(?PSCP-(?:0|[1-9][0-9]*))_(?P(?P(?:[A-Z][a-z0-9]*)+)-(?P(?:0|[1-9][0-9]*)))_(?P(?P(?:[A-Z][a-z0-9]*)+)-(?P(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*))*))_(?P[0-9]{14}))$" + ] + + if custom_patterns is not None: + patterns += list(custom_patterns) + + path = Path(filepath) + filename, basename = path.name, path.stem + + for pattern in patterns: + match = re.match(pattern, basename) + try: + result = {k: v for k, v in match.groupdict().items() if v is not None} + return result + except AttributeError: + continue + + raise ValueError('Failed to parse filename: %s' % filename) + @click.command() -def run(): - pass \ No newline at end of file +@click_config_file.configuration_option(provider=javaproperties_provider, show_default=True) +@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True)) +@click.option('--user-name') +@click.option('--user-password') +@click.option('--outbox-dir', type=click.Path(resolve_path=True, file_okay=False), default='Outbox', show_default=True) +@click.option('--oksent-dir', type=click.Path(resolve_path=True, file_okay=False), default='OKSent', show_default=True) +@click.option('--nosent-dir', type=click.Path(resolve_path=True, file_okay=False), default='NOSent', show_default=True) +@click.option('--retry-dirs', type=StringListParamType(ignore_empty=True), + help="Directory interne a NOSent, soggette a retry", show_default=True) +@click.option('--retry', type=click.IntRange(min=0), default=4, help="Numero di tentativi di re-invio") +@click.option('--retry-frequency', type=click.IntRange(min=0), default=9, + help="Frequenza, espressa in minuti, tra un invio e l'altro in caso di retry") +@click.pass_context +def run(ctx, outbox_dir, oksent_dir, nosent_dir, retry_dirs, retry, retry_frequency, udg_endpoint, user_name, + user_password): + global api_instance + verbose = True + progress = True + click.echo(retry_dirs) + output_dir = os.path.join(outbox_dir, "output_dir") + Path(output_dir).mkdir(parents=True, exist_ok=True) + Path(oksent_dir).mkdir(parents=True, exist_ok=True) + Path(nosent_dir).mkdir(parents=True, exist_ok=True) + + configuration = scp_udg_client_rest.Configuration(host=udg_endpoint) + api_client = scp_udg_client_rest.ApiClient(configuration) + api_instance = scp_udg_client_rest.UrbanDatasetGatewayApi(api_client) + + params = Parameters() + params.udg_endpoint = udg_endpoint + params.user_name = user_name + params.user_password = user_password + params.retry_frequency = retry_frequency + + p = Poller( + input_dir=outbox_dir, + output_dir=output_dir, + extensions=SUPPORTED_EXTS, + verbose=verbose, + progress=progress, + check_file=check_file, + blacklist_tries=1, # TODO: Gestire "blacklist" + params=params) + + while not p.is_stopped: + # Prima di ogni esecuzione del "poller" verifichiamo se in + # output_dir esistono file da ricollocare in input_dir. + + for file_name in os.listdir(p.output_dir): + src = os.path.join(p.output_dir, file_name) + if os.path.isdir(src): + continue + if file_name not in pushed_files_info or pushed_files_info[file_name]["next_try_at"] >= time.time(): + # Potrebbe trattarsi di un file rimasto da una precedente esecuzione terminata + # in modo brusco, oppure di un invio da ritentare. Lo riportiamo in input_dir, + # ma solo se non è già presente un eventuale file omonimo. + dst = os.path.join(p.input_dir, file_name) + os.path.exists(dst) or shutil.move(src, dst) + + p.poll() + + # Verifichiamo il contenuto della cartella di output. + + for file_name in os.listdir(p.output_dir): + src = os.path.join(p.output_dir, file_name) + p.debug("Src: ", src) + if os.path.isdir(src): + continue + if file_name not in pushed_files_info: + # Si tratta di un file che ha fallito la convalida + # locale, lo spostiamo immediatamente in nosent_dir + shutil.move(src, auto_rename(os.path.join(nosent_dir, file_name))) + else: + # Potrebbe essere un file che è andato a buon fine oppure che ha esaurito i tentativi. + file_info = pushed_files_info[file_name] + response_code = str(file_info["response_code"]) + if response_code != "02": + dst_dir = os.path.join(nosent_dir, response_code) + Path(dst_dir).mkdir(parents=True, exist_ok=True) + dst = auto_rename(os.path.join(dst_dir, file_name)) + if (not response_code or response_code in retry_dirs) and file_info["try_count"] <= retry: + # TODO: Il file viene comunque archiviato, forse questo comportamento potrebbe essere opzionale. + shutil.copy(src, dst) + continue # N.B. Senza cancellare l'elemento da pushed_files_info + else: + shutil.move(src, dst) + else: + shutil.move(src, auto_rename(os.path.join(oksent_dir, file_name))) + del pushed_files_info[file_name] + + p.debug("Waiting %d seconds before next poll" % p.poll_wait) + sleep(p.poll_wait) diff --git a/scp_udg_client_app/commands/search_datasets.py b/scp_udg_client_app/commands/search_datasets.py deleted file mode 100755 index 2bbba1f..0000000 --- a/scp_udg_client_app/commands/search_datasets.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -import logging -import os - -import click -from scp_udg_client_rest import SearchingRequest - -from scp_udg_client_app.config import DEFAULT_UDG_ENDPOINT -from scp_udg_client_app.helpers import validate_resource_id, create_api_client_instance, active_login, save_dataset - -logger = logging.getLogger() - - -def fetch_and_save_datasets(api_client, request, resource_id, inbox_dir): - response = api_client.searching(request) - if response.code == "03": - for dataset in response.dataset: - save_dataset(dataset, resource_id, inbox_dir) - else: - logger.warning("Dataset non trovati!") - - -@click.command() -@click.option('--udg-endpoint', prompt=True, prompt_required=False, - default=lambda: os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT)) -@click.option('--resource-id', prompt=True, prompt_required=False, callback=validate_resource_id, - default=lambda: os.environ.get("SCP_UDG_CLIENT_RESOURCE_ID", "")) -@click.option('--access-token', prompt=True, prompt_required=False) -@click.option('--user-name', prompt=True, prompt_required=False, - default=lambda: os.environ.get("SCP_UDG_CLIENT_USER_NAME", "")) -@click.option('--user-password', prompt=True, prompt_required=False, hide_input=True, - default=lambda: os.environ.get("SCP_UDG_CLIENT_USER_PASSWORD", "")) -@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), - default=lambda: os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox")) -@click.option('--period-start', prompt=True, prompt_required=False) -@click.option('--period-end', prompt=True, prompt_required=False) -@click.option('--center-latitude', prompt=True, prompt_required=False) -@click.option('--center-longitude', prompt=True, prompt_required=False) -@click.option('--distance', prompt=True, prompt_required=False) -@click.pass_context -def search_datasets(ctx, udg_endpoint, resource_id, access_token, user_name, user_password, inbox_dir, period_start, - period_end, center_latitude, center_longitude, distance): - api_client = create_api_client_instance(udg_endpoint, access_token) - - if not active_login(api_client, user_name, user_password): - logger.warning("Login fallito!") - return - - try: - request = SearchingRequest( - resource_id=resource_id, - period_start=period_start, - period_end=period_end, - center_latitude=center_latitude, - center_longitude=center_longitude, - distance=distance) - fetch_and_save_datasets(api_client, request, resource_id, inbox_dir) - except Exception as e: - logger.error(f"Errore durante il recupero dei dataset: {str(e)}") - - -if __name__ == '__main__': - search_datasets() diff --git a/scp_udg_client_app/commands/test.py b/scp_udg_client_app/commands/test.py index 0e4281e..5a8dcc7 100644 --- a/scp_udg_client_app/commands/test.py +++ b/scp_udg_client_app/commands/test.py @@ -15,17 +15,7 @@ logger = logging.getLogger(__name__) help="URL endpoint for the Urban Dataset Gateway API.") @click.pass_context def test(ctx, udg_endpoint): - """ - This function is a Click command-line interface command to test the connection to the - Urban Dataset Gateway API service. - - The function sets up the command with an option for the UDG endpoint URL, which can be - provided by the user or defaults to an environment variable or a predefined default. - - The command creates an instance of the API client using the provided or default UDG - endpoint and attempts to invoke the `test` method of the UrbanDatasetGatewayApi. It logs - the response if successful or logs an error message if an exception occurs. - """ + """Test the connection to the UrbanDatasetGateway Web Service.""" api_client = create_api_client_instance(udg_endpoint=udg_endpoint) logger.info( f"Testing UrbanDatasetGatewayApi at {udg_endpoint}" diff --git a/scp_udg_client_app/helpers.py b/scp_udg_client_app/helpers.py index f5f41b8..1da3dce 100644 --- a/scp_udg_client_app/helpers.py +++ b/scp_udg_client_app/helpers.py @@ -1,15 +1,18 @@ import logging import os import re +from os import path from pathlib import Path from typing import Optional import click +import javaproperties from dateutil.parser import parse, ParserError from scp_udg_client_rest import LoginRequest, Configuration, ApiClient, UrbanDatasetGatewayApi +from tocase.for_strings import ToCase # Logger -log = logging.getLogger(__name__) +logger = logging.getLogger() # Constants RESOURCE_ID_PATTERN = re.compile( @@ -20,6 +23,21 @@ RESOURCE_ID_ERROR_MSG = "format must be '{scp_id}_{solution_id}_{dataset_id}_{co # Functions +def read_properties_file(file_path): + with open(file_path, 'r') as file: + logger.debug(f'Reading config file {file_path}') + return {ToCase(key).snake(): value for key, value in javaproperties.load(file).items() if key and value} + + +def javaproperties_provider(file_path, cmd_name): + if not path.exists(file_path): + logger.warning(f'No configuration file found in [{file_path}]') + return {} + data = read_properties_file(file_path) + logger.debug(data) + return data + + def validate_resource_id(ctx, param, value: str) -> str: match = RESOURCE_ID_PATTERN.match(value) if match: @@ -27,7 +45,7 @@ def validate_resource_id(ctx, param, value: str) -> str: parse(match.groupdict()["collaboration_start"]) return value except ParserError as e: - log.warning(str(e)) + logger.warning(str(e)) raise click.BadParameter(RESOURCE_ID_ERROR_MSG) diff --git a/setup.py b/setup.py index 461222f..f8ddbb8 100644 --- a/setup.py +++ b/setup.py @@ -52,17 +52,17 @@ setuptools.setup( "Operating System :: OS Independent", ], install_requires=[ - "toml", "scp-udg-client-rest @ git+https://crossserv.bologna.enea.it/gitlab/cross/scp_udg_client_python_rest.git", - "click", - "click-log", - "jproperties", - "jpype1", - "requests-cache", - "python-dateutil" + "click>=8.1.7", + "click-config-file>=0.6.0", + "click-log>=0.4.0", + "jproperties>=2.1.2", + "jpype1>=1.5.0", + "requests-cache>=1.2.1", + "python-dateutil>=2.9.0" ], cmdclass={"sdist": SdistCommand, "bdist_wheel": BdistWheelCommand}, entry_points={ - 'console_scripts': ['scp-udg-client-app=scp_udg_client_app.cli:main'] + 'console_scripts': ['scp-udg-client=scp_udg_client_app.cli:cli'] } ) -- GitLab