diff --git a/scp_udg_client_app/__main__.py b/scp_udg_client_app/__main__.py index e7caab698562f888996792cd02feb17b15eb3a72..f123d318ed95c6e6461ff95508ffc7e9a29aa3b1 100755 --- a/scp_udg_client_app/__main__.py +++ b/scp_udg_client_app/__main__.py @@ -13,4 +13,4 @@ if __name__ == '__main__': import sys import scp_udg_client_app.cli - sys.exit(scp_udg_client_app.cli.main()) \ No newline at end of file + sys.exit(scp_udg_client_app.cli.cli()) \ No newline at end of file diff --git a/scp_udg_client_app/cli.py b/scp_udg_client_app/cli.py index 5f139d88bb31feccda21861d9f17e0ac365bb086..ae4105187c10c83efc1016e945c5ca4c4a0312d2 100644 --- a/scp_udg_client_app/cli.py +++ b/scp_udg_client_app/cli.py @@ -1,60 +1,46 @@ -#! /usr/bin/env python3 +#!/usr/bin/env python3 + import logging -# noinspection PyUnresolvedReferences -from importlib.resources import files from logging.handlers import RotatingFileHandler import click import click_log -# noinspection PyUnresolvedReferences -import jpype -# noinspection PyUnresolvedReferences -import jpype.imports -# noinspection PyUnresolvedReferences -from jpype.types import * - -# Launch the JVM -jpype.startJVM(classpath=[str(files(__package__).joinpath("jars/*"))]) from scp_udg_client_app.commands import test, fetch_last, fetch, run -from scp_udg_client_app.config import Config -config = Config() -logger = logging.getLogger() +# Constants +LOG_FORMAT = "%(asctime)s - %(levelname)s [%(funcName)s] %(message)s" + +# Logger configuration +logger = click_log.basic_config() -def setup_logging(): - click_log.basic_config(logger) - if config.logging_log_file: - handler = RotatingFileHandler( - config.logging_log_file, - mode='a', - maxBytes=int(config.logging_max_bytes), - backupCount=int(config.logging_backup_count) - ) - handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s [%(funcName)s] %(message)s")) - logger.addHandler(handler) +def configure_logger(log_file, log_max_bytes, log_backup_count): + handler = RotatingFileHandler( + log_file, + mode='a', + maxBytes=int(log_max_bytes), + backupCount=int(log_backup_count) + ) + handler.setFormatter(logging.Formatter(LOG_FORMAT)) + logger.addHandler(handler) @click.group(context_settings={"auto_envvar_prefix": "SCP_UDG_CLIENT"}) -@click.option("-c", "--config-file", type=click.File('rb'), expose_value=False, callback=config.load, is_eager=True, - help=f"Path to config properties file; loads {config.DEFAULT_PATH} by default if it exists") +@click_log.simple_verbosity_option(logger, default="warning") +@click.option("--log-file", type=click.Path(dir_okay=False, writable=True), help="File where the logs are written to.") +@click.option("--log-max-bytes", type=click.IntRange(min=0, min_open=True), help="Maximum file size.") +@click.option("--log-backup-count", type=click.IntRange(min=0, min_open=True), help="Number of files to keep.") @click.version_option() @click.pass_context -@click_log.simple_verbosity_option(logger, default="warning") -def cli(ctx): - ctx.default_map = config.build_default_map() - setup_logging() +def cli(ctx, log_file, log_max_bytes, log_backup_count): + if log_file: + configure_logger(log_file, log_max_bytes, log_backup_count) cli.add_command(test.test) cli.add_command(fetch.fetch) cli.add_command(fetch_last.fetch_last) -#cli.add_command(login.login) -#cli.add_command(logout.logout) -#cli.add_command(run.run) -#cli.add_command(report.report) -#cli.add_command(validate.validate) cli.add_command(run.run) if __name__ == "__main__": diff --git a/scp_udg_client_app/commands/fetch.py b/scp_udg_client_app/commands/fetch.py index 52e9c4b7af462c02df8214f7b9a5535d5a842ba8..c1bc89ee7230b4747173c552e0dc629dcb8a5bb3 100755 --- a/scp_udg_client_app/commands/fetch.py +++ b/scp_udg_client_app/commands/fetch.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 import logging -import os import click import click_config_file from click import FloatRange from click_params import UrlParamType -from scp_udg_client_rest import SearchingRequest +from scp_udg_client_rest import SearchingRequest, Configuration -from scp_udg_client_app.config import DEFAULT_UDG_ENDPOINT from scp_udg_client_app.helpers import validate_resource_id, create_api_client_instance, active_login, save_dataset, \ javaproperties_provider @@ -28,35 +26,17 @@ def fetch_and_save_datasets(api_client, request, resource_id, inbox_dir): logger.warning("Dataset non trovati!") -def get_udg_endpoint(): - return os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT) - - -def get_resource_id(): - return os.environ.get("SCP_UDG_CLIENT_RESOURCE_ID", "") - - -def get_user_name(): - return os.environ.get("SCP_UDG_CLIENT_USER_NAME", "") - - -def get_user_password(): - return os.environ.get("SCP_UDG_CLIENT_USER_PASSWORD", "") - - -def get_inbox_dir(): - return os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox") - - @click.command() -@click_config_file.configuration_option(provider=javaproperties_provider, show_default=True) -@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=get_udg_endpoint) -@click.option('--username', 'user_name', prompt=True, prompt_required=False, default=get_user_name) -@click.option('--password', 'user_password', prompt=True, prompt_required=False, hide_input=True, - default=get_user_password) -@click.option('--resource-id', prompt=True, callback=validate_resource_id, default=get_resource_id) +@click_config_file.configuration_option(cmd_name="scp-udg-client", provider=javaproperties_provider, + config_file_name="config.properties", show_default=True, + help="Read configuration from a Java .properties FILE.") +@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=Configuration().host, show_default=True, + help="URL endpoint for the Urban Dataset Gateway API.") +@click.option('--resource-id', prompt=True, prompt_required=False, callback=validate_resource_id) +@click.option('--username', 'user_name') +@click.option('--password', 'user_password') @click.option('--access-token', hidden=True, prompt=True, prompt_required=False) -@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), default=get_inbox_dir) +@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), default="Inbox", show_default=True) @click.option('--period-start', type=click.DateTime(), help="Date and time from which you want to specify the start of a time interval. If it is absent, the time window includes all possible values.") @click.option('--period-end', type=click.DateTime(), diff --git a/scp_udg_client_app/commands/fetch_last.py b/scp_udg_client_app/commands/fetch_last.py index ec539faf2ea515ee580c35c7e64a32bb5b2d74fe..fb19c25943a33c97337d2dac801af78686655187 100755 --- a/scp_udg_client_app/commands/fetch_last.py +++ b/scp_udg_client_app/commands/fetch_last.py @@ -1,29 +1,29 @@ #!/usr/bin/env python3 import logging -import os import click -from scp_udg_client_rest import LastRequest +import click_config_file +from click_params import UrlParamType +from scp_udg_client_rest import LastRequest, Configuration -from scp_udg_client_app.config import DEFAULT_UDG_ENDPOINT -from scp_udg_client_app.helpers import validate_resource_id, create_api_client_instance, active_login, save_dataset +from scp_udg_client_app.helpers import create_api_client_instance, active_login, save_dataset, \ + javaproperties_provider, validate_resource_id +# Setting up Logger logger = logging.getLogger() @click.command() -@click.option('--udg-endpoint', prompt=True, prompt_required=False, - default=lambda: os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT)) -@click.option('--resource-id', prompt=True, prompt_required=False, callback=validate_resource_id, - default=lambda: os.environ.get("SCP_UDG_CLIENT_RESOURCE_ID", "")) -@click.option('--access-token', prompt=True, prompt_required=False) -@click.option('--user-name', prompt=True, prompt_required=False, - default=lambda: os.environ.get("SCP_UDG_CLIENT_USER_NAME", "")) -@click.option('--user-password', prompt=True, prompt_required=False, hide_input=True, - default=lambda: os.environ.get("SCP_UDG_CLIENT_USER_PASSWORD", "")) -@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), - default=lambda: os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox")) -@click.pass_context +@click_config_file.configuration_option(cmd_name="scp-udg-client", provider=javaproperties_provider, + config_file_name="config.properties", show_default=True, + help="Read configuration from a Java .properties FILE.") +@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=Configuration().host, show_default=True, + help="URL endpoint for the Urban Dataset Gateway API.") +@click.option('--resource-id', prompt=True, prompt_required=False, callback=validate_resource_id) +@click.option('--username', 'user_name') +@click.option('--password', 'user_password') +@click.option('--access-token', hidden=True, prompt=True, prompt_required=False) +@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), default="Inbox", show_default=True) def fetch_last(ctx, udg_endpoint, resource_id, access_token, user_name, user_password, inbox_dir): """Fetch and save the last dataset from an UDG endpoint.""" api_client = create_api_client_instance(udg_endpoint, access_token) diff --git a/scp_udg_client_app/commands/run.py b/scp_udg_client_app/commands/run.py index 1d8f6a3f455f2032e74c30774c2e2943e1044d32..0e9f485e084eed3d64dd1c32931254ff0fc5d934 100644 --- a/scp_udg_client_app/commands/run.py +++ b/scp_udg_client_app/commands/run.py @@ -1,28 +1,42 @@ +import logging import os import re import shutil import time +# noinspection PyUnresolvedReferences +from importlib.resources import files from pathlib import Path from time import sleep + import click import click_config_file -from click_params import UrlParamType, StringListParamType -import requests_cache import scp_udg_client_rest - +from click_params import UrlParamType, StringListParamType from dateutil.parser import parse, ParserError -# import the Java modules -# noinspection PyUnresolvedReferences -from it.enea.validation.Validators import JSONSchematronValidator -from scp_udg_client_rest import LoginRequest, PushRequest +from scp_udg_client_rest import LoginRequest, PushRequest, Configuration from scp_udg_client_rest.models.scps_urbandataset_schema20 import ScpsUrbandatasetSchema20 -from sfp import Poller, Parameters +from sfp import Poller, Parameters, LOGGING_TYPE_DEBUG, LOGGING_TYPE_INFO, LOGGING_TYPE_ERROR from scp_udg_client_app.helpers import javaproperties_provider SUPPORTED_EXTS = [".json"] """ supported file extensions (lower case). """ +# Setting up Logger +logger = logging.getLogger() + + +def log_message(logging_type, *args): + logger_methods = { + LOGGING_TYPE_DEBUG: logger.debug, + LOGGING_TYPE_INFO: logger.info, + LOGGING_TYPE_ERROR: logger.error + } + log_method = logger_methods.get(logging_type) + if log_method: + log_method(" ".join(args)) + + pushed_files_info = {} @@ -76,24 +90,6 @@ def check_file(fname, poller): poller.debug("UrbanDataset ID:", urbandataset_id) - # Schematron validation can fail for a variety of reasons, but we do not consider this to be fatal. - - nome_urbandataset = parse_dict["nome_urbandataset"] - session = requests_cache.CachedSession() - - try: - response = session.get( - f'https://smartcityplatform.enea.it/UDWebLibrary/it/template/{nome_urbandataset}?templ=schem') - ud_schema_content = response.text - - validator = JSONSchematronValidator() - validation_report = validator.getValidationReport(ud_content, ud_schema_content) - - is_success = validation_report.isSuccess() - poller.debug("Schematron valid:", is_success) - except Exception as e: - poller.error(e) - # Let's try to send the file to the platform. response_code = "" @@ -182,10 +178,14 @@ def parse_filename(filepath, custom_patterns=None): @click.command() -@click_config_file.configuration_option(provider=javaproperties_provider, show_default=True) -@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True)) -@click.option('--user-name') -@click.option('--user-password') +@click_config_file.configuration_option(cmd_name="scp-udg-client", provider=javaproperties_provider, + config_file_name="config.properties", show_default=True, + help="Read configuration from a Java .properties FILE.") +@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=Configuration().host, show_default=True, + help="URL endpoint for the Urban Dataset Gateway API.") +@click.option('--username', 'user_name') +@click.option('--password', 'user_password') +@click.option('--access-token', hidden=True, prompt=True, prompt_required=False) @click.option('--outbox-dir', type=click.Path(resolve_path=True, file_okay=False), default='Outbox', show_default=True) @click.option('--oksent-dir', type=click.Path(resolve_path=True, file_okay=False), default='OKSent', show_default=True) @click.option('--nosent-dir', type=click.Path(resolve_path=True, file_okay=False), default='NOSent', show_default=True) @@ -196,7 +196,7 @@ def parse_filename(filepath, custom_patterns=None): help="Frequenza, espressa in minuti, tra un invio e l'altro in caso di retry") @click.pass_context def run(ctx, outbox_dir, oksent_dir, nosent_dir, retry_dirs, retry, retry_frequency, udg_endpoint, user_name, - user_password): + user_password, access_token): global api_instance verbose = True progress = True @@ -220,6 +220,7 @@ def run(ctx, outbox_dir, oksent_dir, nosent_dir, retry_dirs, retry, retry_freque input_dir=outbox_dir, output_dir=output_dir, extensions=SUPPORTED_EXTS, + logging=log_message, verbose=verbose, progress=progress, check_file=check_file, diff --git a/scp_udg_client_app/commands/test.py b/scp_udg_client_app/commands/test.py index 5a8dcc7203b6aa8a55fc909ecf06454d779babe9..57ce2710c5d79e5c1d37bd0c7fea422a6f264def 100644 --- a/scp_udg_client_app/commands/test.py +++ b/scp_udg_client_app/commands/test.py @@ -1,17 +1,21 @@ import logging -import os import click +import click_config_file +from click_params import UrlParamType +from scp_udg_client_rest import Configuration -from scp_udg_client_app.config import DEFAULT_UDG_ENDPOINT -from scp_udg_client_app.helpers import create_api_client_instance +from scp_udg_client_app.helpers import create_api_client_instance, javaproperties_provider -logger = logging.getLogger(__name__) +# Setting up Logger +logger = logging.getLogger() @click.command() -@click.option('--udg-endpoint', prompt=True, prompt_required=False, show_default=True, - default=lambda: os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT), +@click_config_file.configuration_option(cmd_name="scp-udg-client", provider=javaproperties_provider, + config_file_name="config.properties", show_default=True, + help="Read configuration from a Java .properties FILE.") +@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=Configuration().host, show_default=True, help="URL endpoint for the Urban Dataset Gateway API.") @click.pass_context def test(ctx, udg_endpoint): diff --git a/scp_udg_client_app/config.py b/scp_udg_client_app/config.py deleted file mode 100644 index 90dcd2023219531d477f5cc5844ae9c300dcc194..0000000000000000000000000000000000000000 --- a/scp_udg_client_app/config.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -from pathlib import Path - -import click -from jproperties import Properties, PropertyTuple - -DEFAULT_UDG_ENDPOINT = "https://scp-casaccia.bologna.enea.it:8443/webservices/rest/UrbanDatasetGateway" - - -class Config: - DEFAULT_DIR = Path(click.get_app_dir("scp-udg-client")) - DEFAULT_PATH = Path(DEFAULT_DIR, "app-config.properties") - - def __init__(self): - self.cfg = Properties() - - def load(self, ctx, param, file): - default_path = self.DEFAULT_PATH - if file: - with open(file, "rb") as props_file: - self.cfg.load(props_file) - elif default_path.exists(): - with open(default_path, "rb") as props_file: - self.cfg.load(props_file) - - @property - def scp_name(self): - return self.cfg.get("scp.name", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_SCP_NAME", ""), meta=None)).data - - @property - def client_name(self): - return self.cfg.get("client.name", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_CLIENT_NAME", ""), meta=None)).data - - @property - def gui_url(self): - return self.cfg.get("gui.url", PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_GUI_URL", ""), meta=None)).data - - @property - def udg_endpoint(self): - return self.cfg.get("udg.endpoint", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT), - meta=None)).data - - @property - def user_name(self): - return self.cfg.get("user.name", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_USER_NAME", ""), meta=None)).data - - @property - def user_password(self): - return self.cfg.get("user.password", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_USER_PASSWORD", ""), meta=None)).data - - @property - def outbox_dir(self): - return self.cfg.get("outbox.dir", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_OUTBOX_DIR", "Outbox"), meta=None)).data - - @property - def inbox_dir(self): - return self.cfg.get("inbox.dir", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox"), meta=None)).data - - @property - def oksent_dir(self): - return self.cfg.get("oksent.dir", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_OKSENT_DIR", "OKSent"), meta=None)).data - - @property - def nosent_dir(self): - return self.cfg.get("nosent.dir", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_NOSENT_DIR", "NOSent"), meta=None)).data - - @property - def retry_dirs(self): - return self.cfg.get("retry.dirs", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_RETRY_DIRS", ""), meta=None)).data - - @property - def retry(self): - return self.cfg.get("retry", PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_RETRY", "3"), meta=None)).data - - @property - def retry_frequency(self): - return self.cfg.get("retry_frequency", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_RETRY_FREQUENCY", "2"), meta=None)).data - - @property - def email_address(self): - return self.cfg.get("email.address", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_EMAIL_ADDRESS", ""), meta=None)).data - - @property - def email_frequency(self): - return self.cfg.get("email.frequency", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_EMAIL_FREQUENCY", ""), meta=None)).data - - @property - def logging_log_file(self): - return self.cfg.get("logging.log_file", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_LOGGING_LOG_FILE", ""), meta=None)).data - - @property - def logging_max_bytes(self): - return self.cfg.get("logging.max_bytes", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_LOGGING_MAX_BYTES", ""), meta=None)).data - - @property - def logging_backup_count(self): - return self.cfg.get("logging.backup_count", - PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_LOGGING_BACKUP_COUNT", ""), - meta=None)).data - - def build_default_map(self): - - default_map = { - "last": { - "udg_endpoint": self.udg_endpoint, - "user_name": self.user_name, - "user_password": self.user_password, - "inbox_dir": self.inbox_dir - }, - "test": { - "udg_endpoint": self.udg_endpoint - }, - "login": { - "udg_endpoint": self.udg_endpoint, - "user_name": self.user_name, - "user_password": self.user_password, - }, - "run": { - "scp_name": self.scp_name, - "client_name": self.client_name, - "gui_url": self.gui_url, - "udg_endpoint": self.udg_endpoint, - "user_name": self.user_name, - "user_password": self.user_password, - "outbox_dir": self.outbox_dir, - "inbox_dir": self.inbox_dir, - "oksent_dir": self.oksent_dir, - "nosent_dir": self.nosent_dir, - "retry_dirs": self.retry_dirs, - "retry": self.retry, - "retry_frequency": self.retry_frequency, - "email_address": self.email_address, - "email_frequency": self.email_frequency - }, - "poll": { - "udg_endpoint": self.udg_endpoint, - "user_name": self.user_name, - "user_password": self.user_password, - "outbox_dir": self.outbox_dir, - "oksent_dir": self.oksent_dir, - "nosent_dir": self.nosent_dir, - "retry_dirs": self.retry_dirs, - "retry": self.retry, - "retry_frequency": self.retry_frequency - } - } - return default_map diff --git a/scp_udg_client_app/helpers.py b/scp_udg_client_app/helpers.py index 1da3dcecc274d82ee5bd29badf3d0e469bdba9c0..853e00b259e3e9262ce7d5f4a7c2e8d480248371 100644 --- a/scp_udg_client_app/helpers.py +++ b/scp_udg_client_app/helpers.py @@ -20,6 +20,7 @@ RESOURCE_ID_PATTERN = re.compile( ) DATE_FORMAT = "%Y%m%d%H%M%S" RESOURCE_ID_ERROR_MSG = "format must be '{scp_id}_{solution_id}_{dataset_id}_{collaboration_start}'" +DEFAULT_UDG_ENDPOINT = "https://scp-casaccia.bologna.enea.it:8443/webservices/rest/UrbanDatasetGateway" # Functions