From fd1d2241ff944805b8b9828117ac0dd5c45c8272 Mon Sep 17 00:00:00 2001
From: Tito Brasolin <tito.brasolin@kerberos.energy>
Date: Tue, 29 Oct 2024 15:31:23 +0100
Subject: [PATCH] chore!: major refactoring and cleanup
---
scp_udg_client_app/__main__.py | 2 +-
scp_udg_client_app/cli.py | 60 +++-----
scp_udg_client_app/commands/fetch.py | 40 ++----
scp_udg_client_app/commands/fetch_last.py | 32 ++---
scp_udg_client_app/commands/run.py | 63 ++++-----
scp_udg_client_app/commands/test.py | 16 ++-
scp_udg_client_app/config.py | 162 ----------------------
scp_udg_client_app/helpers.py | 1 +
8 files changed, 93 insertions(+), 283 deletions(-)
delete mode 100644 scp_udg_client_app/config.py
diff --git a/scp_udg_client_app/__main__.py b/scp_udg_client_app/__main__.py
index e7caab6..f123d31 100755
--- a/scp_udg_client_app/__main__.py
+++ b/scp_udg_client_app/__main__.py
@@ -13,4 +13,4 @@
if __name__ == '__main__':
import sys
import scp_udg_client_app.cli
- sys.exit(scp_udg_client_app.cli.main())
\ No newline at end of file
+ sys.exit(scp_udg_client_app.cli.cli())
\ No newline at end of file
diff --git a/scp_udg_client_app/cli.py b/scp_udg_client_app/cli.py
index 5f139d8..ae41051 100644
--- a/scp_udg_client_app/cli.py
+++ b/scp_udg_client_app/cli.py
@@ -1,60 +1,46 @@
-#! /usr/bin/env python3
+#!/usr/bin/env python3
+
import logging
-# noinspection PyUnresolvedReferences
-from importlib.resources import files
from logging.handlers import RotatingFileHandler
import click
import click_log
-# noinspection PyUnresolvedReferences
-import jpype
-# noinspection PyUnresolvedReferences
-import jpype.imports
-# noinspection PyUnresolvedReferences
-from jpype.types import *
-
-# Launch the JVM
-jpype.startJVM(classpath=[str(files(__package__).joinpath("jars/*"))])
from scp_udg_client_app.commands import test, fetch_last, fetch, run
-from scp_udg_client_app.config import Config
-config = Config()
-logger = logging.getLogger()
+# Constants
+LOG_FORMAT = "%(asctime)s - %(levelname)s [%(funcName)s] %(message)s"
+
+# Logger configuration
+logger = click_log.basic_config()
-def setup_logging():
- click_log.basic_config(logger)
- if config.logging_log_file:
- handler = RotatingFileHandler(
- config.logging_log_file,
- mode='a',
- maxBytes=int(config.logging_max_bytes),
- backupCount=int(config.logging_backup_count)
- )
- handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s [%(funcName)s] %(message)s"))
- logger.addHandler(handler)
+def configure_logger(log_file, log_max_bytes, log_backup_count):
+ handler = RotatingFileHandler(
+ log_file,
+ mode='a',
+ maxBytes=int(log_max_bytes),
+ backupCount=int(log_backup_count)
+ )
+ handler.setFormatter(logging.Formatter(LOG_FORMAT))
+ logger.addHandler(handler)
@click.group(context_settings={"auto_envvar_prefix": "SCP_UDG_CLIENT"})
-@click.option("-c", "--config-file", type=click.File('rb'), expose_value=False, callback=config.load, is_eager=True,
- help=f"Path to config properties file; loads {config.DEFAULT_PATH} by default if it exists")
+@click_log.simple_verbosity_option(logger, default="warning")
+@click.option("--log-file", type=click.Path(dir_okay=False, writable=True), help="File where the logs are written to.")
+@click.option("--log-max-bytes", type=click.IntRange(min=0, min_open=True), help="Maximum file size.")
+@click.option("--log-backup-count", type=click.IntRange(min=0, min_open=True), help="Number of files to keep.")
@click.version_option()
@click.pass_context
-@click_log.simple_verbosity_option(logger, default="warning")
-def cli(ctx):
- ctx.default_map = config.build_default_map()
- setup_logging()
+def cli(ctx, log_file, log_max_bytes, log_backup_count):
+ if log_file:
+ configure_logger(log_file, log_max_bytes, log_backup_count)
cli.add_command(test.test)
cli.add_command(fetch.fetch)
cli.add_command(fetch_last.fetch_last)
-#cli.add_command(login.login)
-#cli.add_command(logout.logout)
-#cli.add_command(run.run)
-#cli.add_command(report.report)
-#cli.add_command(validate.validate)
cli.add_command(run.run)
if __name__ == "__main__":
diff --git a/scp_udg_client_app/commands/fetch.py b/scp_udg_client_app/commands/fetch.py
index 52e9c4b..c1bc89e 100755
--- a/scp_udg_client_app/commands/fetch.py
+++ b/scp_udg_client_app/commands/fetch.py
@@ -1,14 +1,12 @@
#!/usr/bin/env python3
import logging
-import os
import click
import click_config_file
from click import FloatRange
from click_params import UrlParamType
-from scp_udg_client_rest import SearchingRequest
+from scp_udg_client_rest import SearchingRequest, Configuration
-from scp_udg_client_app.config import DEFAULT_UDG_ENDPOINT
from scp_udg_client_app.helpers import validate_resource_id, create_api_client_instance, active_login, save_dataset, \
javaproperties_provider
@@ -28,35 +26,17 @@ def fetch_and_save_datasets(api_client, request, resource_id, inbox_dir):
logger.warning("Dataset non trovati!")
-def get_udg_endpoint():
- return os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT)
-
-
-def get_resource_id():
- return os.environ.get("SCP_UDG_CLIENT_RESOURCE_ID", "")
-
-
-def get_user_name():
- return os.environ.get("SCP_UDG_CLIENT_USER_NAME", "")
-
-
-def get_user_password():
- return os.environ.get("SCP_UDG_CLIENT_USER_PASSWORD", "")
-
-
-def get_inbox_dir():
- return os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox")
-
-
@click.command()
-@click_config_file.configuration_option(provider=javaproperties_provider, show_default=True)
-@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=get_udg_endpoint)
-@click.option('--username', 'user_name', prompt=True, prompt_required=False, default=get_user_name)
-@click.option('--password', 'user_password', prompt=True, prompt_required=False, hide_input=True,
- default=get_user_password)
-@click.option('--resource-id', prompt=True, callback=validate_resource_id, default=get_resource_id)
+@click_config_file.configuration_option(cmd_name="scp-udg-client", provider=javaproperties_provider,
+ config_file_name="config.properties", show_default=True,
+ help="Read configuration from a Java .properties FILE.")
+@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=Configuration().host, show_default=True,
+ help="URL endpoint for the Urban Dataset Gateway API.")
+@click.option('--resource-id', prompt=True, prompt_required=False, callback=validate_resource_id)
+@click.option('--username', 'user_name')
+@click.option('--password', 'user_password')
@click.option('--access-token', hidden=True, prompt=True, prompt_required=False)
-@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), default=get_inbox_dir)
+@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), default="Inbox", show_default=True)
@click.option('--period-start', type=click.DateTime(),
help="Date and time from which you want to specify the start of a time interval. If it is absent, the time window includes all possible values.")
@click.option('--period-end', type=click.DateTime(),
diff --git a/scp_udg_client_app/commands/fetch_last.py b/scp_udg_client_app/commands/fetch_last.py
index ec539fa..fb19c25 100755
--- a/scp_udg_client_app/commands/fetch_last.py
+++ b/scp_udg_client_app/commands/fetch_last.py
@@ -1,29 +1,29 @@
#!/usr/bin/env python3
import logging
-import os
import click
-from scp_udg_client_rest import LastRequest
+import click_config_file
+from click_params import UrlParamType
+from scp_udg_client_rest import LastRequest, Configuration
-from scp_udg_client_app.config import DEFAULT_UDG_ENDPOINT
-from scp_udg_client_app.helpers import validate_resource_id, create_api_client_instance, active_login, save_dataset
+from scp_udg_client_app.helpers import create_api_client_instance, active_login, save_dataset, \
+ javaproperties_provider, validate_resource_id
+# Setting up Logger
logger = logging.getLogger()
@click.command()
-@click.option('--udg-endpoint', prompt=True, prompt_required=False,
- default=lambda: os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT))
-@click.option('--resource-id', prompt=True, prompt_required=False, callback=validate_resource_id,
- default=lambda: os.environ.get("SCP_UDG_CLIENT_RESOURCE_ID", ""))
-@click.option('--access-token', prompt=True, prompt_required=False)
-@click.option('--user-name', prompt=True, prompt_required=False,
- default=lambda: os.environ.get("SCP_UDG_CLIENT_USER_NAME", ""))
-@click.option('--user-password', prompt=True, prompt_required=False, hide_input=True,
- default=lambda: os.environ.get("SCP_UDG_CLIENT_USER_PASSWORD", ""))
-@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False),
- default=lambda: os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox"))
-@click.pass_context
+@click_config_file.configuration_option(cmd_name="scp-udg-client", provider=javaproperties_provider,
+ config_file_name="config.properties", show_default=True,
+ help="Read configuration from a Java .properties FILE.")
+@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=Configuration().host, show_default=True,
+ help="URL endpoint for the Urban Dataset Gateway API.")
+@click.option('--resource-id', prompt=True, prompt_required=False, callback=validate_resource_id)
+@click.option('--username', 'user_name')
+@click.option('--password', 'user_password')
+@click.option('--access-token', hidden=True, prompt=True, prompt_required=False)
+@click.option('--inbox-dir', type=click.Path(resolve_path=True, file_okay=False), default="Inbox", show_default=True)
def fetch_last(ctx, udg_endpoint, resource_id, access_token, user_name, user_password, inbox_dir):
"""Fetch and save the last dataset from an UDG endpoint."""
api_client = create_api_client_instance(udg_endpoint, access_token)
diff --git a/scp_udg_client_app/commands/run.py b/scp_udg_client_app/commands/run.py
index 1d8f6a3..0e9f485 100644
--- a/scp_udg_client_app/commands/run.py
+++ b/scp_udg_client_app/commands/run.py
@@ -1,28 +1,42 @@
+import logging
import os
import re
import shutil
import time
+# noinspection PyUnresolvedReferences
+from importlib.resources import files
from pathlib import Path
from time import sleep
+
import click
import click_config_file
-from click_params import UrlParamType, StringListParamType
-import requests_cache
import scp_udg_client_rest
-
+from click_params import UrlParamType, StringListParamType
from dateutil.parser import parse, ParserError
-# import the Java modules
-# noinspection PyUnresolvedReferences
-from it.enea.validation.Validators import JSONSchematronValidator
-from scp_udg_client_rest import LoginRequest, PushRequest
+from scp_udg_client_rest import LoginRequest, PushRequest, Configuration
from scp_udg_client_rest.models.scps_urbandataset_schema20 import ScpsUrbandatasetSchema20
-from sfp import Poller, Parameters
+from sfp import Poller, Parameters, LOGGING_TYPE_DEBUG, LOGGING_TYPE_INFO, LOGGING_TYPE_ERROR
from scp_udg_client_app.helpers import javaproperties_provider
SUPPORTED_EXTS = [".json"]
""" supported file extensions (lower case). """
+# Setting up Logger
+logger = logging.getLogger()
+
+
+def log_message(logging_type, *args):
+ logger_methods = {
+ LOGGING_TYPE_DEBUG: logger.debug,
+ LOGGING_TYPE_INFO: logger.info,
+ LOGGING_TYPE_ERROR: logger.error
+ }
+ log_method = logger_methods.get(logging_type)
+ if log_method:
+ log_method(" ".join(args))
+
+
pushed_files_info = {}
@@ -76,24 +90,6 @@ def check_file(fname, poller):
poller.debug("UrbanDataset ID:", urbandataset_id)
- # Schematron validation can fail for a variety of reasons, but we do not consider this to be fatal.
-
- nome_urbandataset = parse_dict["nome_urbandataset"]
- session = requests_cache.CachedSession()
-
- try:
- response = session.get(
- f'https://smartcityplatform.enea.it/UDWebLibrary/it/template/{nome_urbandataset}?templ=schem')
- ud_schema_content = response.text
-
- validator = JSONSchematronValidator()
- validation_report = validator.getValidationReport(ud_content, ud_schema_content)
-
- is_success = validation_report.isSuccess()
- poller.debug("Schematron valid:", is_success)
- except Exception as e:
- poller.error(e)
-
# Let's try to send the file to the platform.
response_code = ""
@@ -182,10 +178,14 @@ def parse_filename(filepath, custom_patterns=None):
@click.command()
-@click_config_file.configuration_option(provider=javaproperties_provider, show_default=True)
-@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True))
-@click.option('--user-name')
-@click.option('--user-password')
+@click_config_file.configuration_option(cmd_name="scp-udg-client", provider=javaproperties_provider,
+ config_file_name="config.properties", show_default=True,
+ help="Read configuration from a Java .properties FILE.")
+@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=Configuration().host, show_default=True,
+ help="URL endpoint for the Urban Dataset Gateway API.")
+@click.option('--username', 'user_name')
+@click.option('--password', 'user_password')
+@click.option('--access-token', hidden=True, prompt=True, prompt_required=False)
@click.option('--outbox-dir', type=click.Path(resolve_path=True, file_okay=False), default='Outbox', show_default=True)
@click.option('--oksent-dir', type=click.Path(resolve_path=True, file_okay=False), default='OKSent', show_default=True)
@click.option('--nosent-dir', type=click.Path(resolve_path=True, file_okay=False), default='NOSent', show_default=True)
@@ -196,7 +196,7 @@ def parse_filename(filepath, custom_patterns=None):
help="Frequenza, espressa in minuti, tra un invio e l'altro in caso di retry")
@click.pass_context
def run(ctx, outbox_dir, oksent_dir, nosent_dir, retry_dirs, retry, retry_frequency, udg_endpoint, user_name,
- user_password):
+ user_password, access_token):
global api_instance
verbose = True
progress = True
@@ -220,6 +220,7 @@ def run(ctx, outbox_dir, oksent_dir, nosent_dir, retry_dirs, retry, retry_freque
input_dir=outbox_dir,
output_dir=output_dir,
extensions=SUPPORTED_EXTS,
+ logging=log_message,
verbose=verbose,
progress=progress,
check_file=check_file,
diff --git a/scp_udg_client_app/commands/test.py b/scp_udg_client_app/commands/test.py
index 5a8dcc7..57ce271 100644
--- a/scp_udg_client_app/commands/test.py
+++ b/scp_udg_client_app/commands/test.py
@@ -1,17 +1,21 @@
import logging
-import os
import click
+import click_config_file
+from click_params import UrlParamType
+from scp_udg_client_rest import Configuration
-from scp_udg_client_app.config import DEFAULT_UDG_ENDPOINT
-from scp_udg_client_app.helpers import create_api_client_instance
+from scp_udg_client_app.helpers import create_api_client_instance, javaproperties_provider
-logger = logging.getLogger(__name__)
+# Setting up Logger
+logger = logging.getLogger()
@click.command()
-@click.option('--udg-endpoint', prompt=True, prompt_required=False, show_default=True,
- default=lambda: os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT),
+@click_config_file.configuration_option(cmd_name="scp-udg-client", provider=javaproperties_provider,
+ config_file_name="config.properties", show_default=True,
+ help="Read configuration from a Java .properties FILE.")
+@click.option('--udg-endpoint', type=UrlParamType(may_have_port=True), default=Configuration().host, show_default=True,
help="URL endpoint for the Urban Dataset Gateway API.")
@click.pass_context
def test(ctx, udg_endpoint):
diff --git a/scp_udg_client_app/config.py b/scp_udg_client_app/config.py
deleted file mode 100644
index 90dcd20..0000000
--- a/scp_udg_client_app/config.py
+++ /dev/null
@@ -1,162 +0,0 @@
-import os
-from pathlib import Path
-
-import click
-from jproperties import Properties, PropertyTuple
-
-DEFAULT_UDG_ENDPOINT = "https://scp-casaccia.bologna.enea.it:8443/webservices/rest/UrbanDatasetGateway"
-
-
-class Config:
- DEFAULT_DIR = Path(click.get_app_dir("scp-udg-client"))
- DEFAULT_PATH = Path(DEFAULT_DIR, "app-config.properties")
-
- def __init__(self):
- self.cfg = Properties()
-
- def load(self, ctx, param, file):
- default_path = self.DEFAULT_PATH
- if file:
- with open(file, "rb") as props_file:
- self.cfg.load(props_file)
- elif default_path.exists():
- with open(default_path, "rb") as props_file:
- self.cfg.load(props_file)
-
- @property
- def scp_name(self):
- return self.cfg.get("scp.name",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_SCP_NAME", ""), meta=None)).data
-
- @property
- def client_name(self):
- return self.cfg.get("client.name",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_CLIENT_NAME", ""), meta=None)).data
-
- @property
- def gui_url(self):
- return self.cfg.get("gui.url", PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_GUI_URL", ""), meta=None)).data
-
- @property
- def udg_endpoint(self):
- return self.cfg.get("udg.endpoint",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_UDG_ENDPOINT", DEFAULT_UDG_ENDPOINT),
- meta=None)).data
-
- @property
- def user_name(self):
- return self.cfg.get("user.name",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_USER_NAME", ""), meta=None)).data
-
- @property
- def user_password(self):
- return self.cfg.get("user.password",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_USER_PASSWORD", ""), meta=None)).data
-
- @property
- def outbox_dir(self):
- return self.cfg.get("outbox.dir",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_OUTBOX_DIR", "Outbox"), meta=None)).data
-
- @property
- def inbox_dir(self):
- return self.cfg.get("inbox.dir",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_INBOX_DIR", "Inbox"), meta=None)).data
-
- @property
- def oksent_dir(self):
- return self.cfg.get("oksent.dir",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_OKSENT_DIR", "OKSent"), meta=None)).data
-
- @property
- def nosent_dir(self):
- return self.cfg.get("nosent.dir",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_NOSENT_DIR", "NOSent"), meta=None)).data
-
- @property
- def retry_dirs(self):
- return self.cfg.get("retry.dirs",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_RETRY_DIRS", ""), meta=None)).data
-
- @property
- def retry(self):
- return self.cfg.get("retry", PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_RETRY", "3"), meta=None)).data
-
- @property
- def retry_frequency(self):
- return self.cfg.get("retry_frequency",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_RETRY_FREQUENCY", "2"), meta=None)).data
-
- @property
- def email_address(self):
- return self.cfg.get("email.address",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_EMAIL_ADDRESS", ""), meta=None)).data
-
- @property
- def email_frequency(self):
- return self.cfg.get("email.frequency",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_EMAIL_FREQUENCY", ""), meta=None)).data
-
- @property
- def logging_log_file(self):
- return self.cfg.get("logging.log_file",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_LOGGING_LOG_FILE", ""), meta=None)).data
-
- @property
- def logging_max_bytes(self):
- return self.cfg.get("logging.max_bytes",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_LOGGING_MAX_BYTES", ""), meta=None)).data
-
- @property
- def logging_backup_count(self):
- return self.cfg.get("logging.backup_count",
- PropertyTuple(data=os.environ.get("SCP_UDG_CLIENT_LOGGING_BACKUP_COUNT", ""),
- meta=None)).data
-
- def build_default_map(self):
-
- default_map = {
- "last": {
- "udg_endpoint": self.udg_endpoint,
- "user_name": self.user_name,
- "user_password": self.user_password,
- "inbox_dir": self.inbox_dir
- },
- "test": {
- "udg_endpoint": self.udg_endpoint
- },
- "login": {
- "udg_endpoint": self.udg_endpoint,
- "user_name": self.user_name,
- "user_password": self.user_password,
- },
- "run": {
- "scp_name": self.scp_name,
- "client_name": self.client_name,
- "gui_url": self.gui_url,
- "udg_endpoint": self.udg_endpoint,
- "user_name": self.user_name,
- "user_password": self.user_password,
- "outbox_dir": self.outbox_dir,
- "inbox_dir": self.inbox_dir,
- "oksent_dir": self.oksent_dir,
- "nosent_dir": self.nosent_dir,
- "retry_dirs": self.retry_dirs,
- "retry": self.retry,
- "retry_frequency": self.retry_frequency,
- "email_address": self.email_address,
- "email_frequency": self.email_frequency
- },
- "poll": {
- "udg_endpoint": self.udg_endpoint,
- "user_name": self.user_name,
- "user_password": self.user_password,
- "outbox_dir": self.outbox_dir,
- "oksent_dir": self.oksent_dir,
- "nosent_dir": self.nosent_dir,
- "retry_dirs": self.retry_dirs,
- "retry": self.retry,
- "retry_frequency": self.retry_frequency
- }
- }
- return default_map
diff --git a/scp_udg_client_app/helpers.py b/scp_udg_client_app/helpers.py
index 1da3dce..853e00b 100644
--- a/scp_udg_client_app/helpers.py
+++ b/scp_udg_client_app/helpers.py
@@ -20,6 +20,7 @@ RESOURCE_ID_PATTERN = re.compile(
)
DATE_FORMAT = "%Y%m%d%H%M%S"
RESOURCE_ID_ERROR_MSG = "format must be '{scp_id}_{solution_id}_{dataset_id}_{collaboration_start}'"
+DEFAULT_UDG_ENDPOINT = "https://scp-casaccia.bologna.enea.it:8443/webservices/rest/UrbanDatasetGateway"
# Functions
--
GitLab