Skip to content
Snippets Groups Projects
Commit e8e52e9e authored by Tito Brasolin's avatar Tito Brasolin
Browse files

fix: do not retry existing datasets

parent a9497753
Branches
No related tags found
No related merge requests found
......@@ -274,16 +274,24 @@ def process_pending_files(pending_dir, retry_dirs, retry, nosent_dir, oksent_dir
if response_code != PUSH_SUCCESSFUL_CODE or response_detail == "UrbanDataset already exists":
# noinspection PyUnresolvedReferences
try_count = int(file_info["try_count"])
nosent_response_path = os.path.join(nosent_dir, response_code)
Path(nosent_response_path).mkdir(parents=True, exist_ok=True)
if response_code == PUSH_SUCCESSFUL_CODE:
logger.info(
f"- UrbanDataset already exists. File {file_name} will be moved to {nosent_response_path} and never retried.")
move_file(src, nosent_response_path)
del pushed_files_info[file_name]
continue
# noinspection PyUnresolvedReferences
try_count = int(file_info["try_count"])
# noinspection PyUnresolvedReferences
if (not response_code or response_code in retry_dirs) and try_count <= retry:
logger.info(f"- File {file_name} will be copied to {nosent_response_path} and retried later.")
copy_file(src, nosent_response_path)
logger.info(f"- File {file_name} has been attempted {try_count} time(s) and will be retried later.")
# Decommentare la riga seguente se si desidera che resti traccia di tutti i tentativi.
# copy_file(src, nosent_response_path)
pushed_files_info[file_name]["processed"] = True
else:
logger.info(f"- File {file_name} will be moved to {nosent_response_path} and never retried.")
......@@ -334,7 +342,8 @@ def job(poller):
@click.option('--token-file', type=click.Path(readable=True, dir_okay=False), default='token.json', show_envvar=True,
help='Path to the file containing the login response. [default: token.json]')
@click.option('--is-json', is_flag=True, default=False, hidden=True, help='Flag to handle token as JSON.')
@login_options
@click.option('--username', prompt=True, prompt_required=False, show_envvar=True, help='Your username.')
@click.password_option('--password', prompt_required=False, show_envvar=True, help='Your password.')
@outbox_dir_options
@click.option('--retry-dirs', type=StringListParamType(ignore_empty=True), default="",
help="Directory interne a --nosent-dir soggette a retry.", show_envvar=True)
......@@ -365,6 +374,11 @@ def push(ctx, outbox_dir, oksent_dir, nosent_dir, retry_dirs, retry, retry_frequ
if continuous and (not username or not password):
raise click.UsageError("Username and password are required if continuous mode is enabled!")
# Verify if PUSH_SUCCESSFUL_CODE is present in retry_dirs and log a warning if so
if PUSH_SUCCESSFUL_CODE in retry_dirs:
logger.warning(f"Code '{PUSH_SUCCESSFUL_CODE}' is in retry_dirs. These submissions are considered successful "
"and will therefore not be retried, even if the file is placed in the 'no sent' directory.")
global api_instance
api_instance = get_api_instance(ctx, udg_endpoint)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment