From f2d41cd065952424acb465b1b86be27e84ecdcb8 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Tue, 7 Apr 2026 17:15:47 +0200 Subject: [PATCH 1/4] Increase delay when inquiry fails. --- src/cso/cso_dataspace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cso/cso_dataspace.py b/src/cso/cso_dataspace.py index bd7e723..5209280 100644 --- a/src/cso/cso_dataspace.py +++ b/src/cso/cso_dataspace.py @@ -53,7 +53,7 @@ # # 2026-04, Arjo Segers # Fixed definition of bounding box for global selection. -# Added delays to avoid rate limit errors from STAC catalogue inquiry. +# Added increasing delays to avoid rate limit errors from STAC catalogue inquiry. # @@ -340,6 +340,7 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc): maxtry = 10 ntry = 1 nsec_wait = 5 + nsec_wait_max = 600 # loop until success or too many errors: while True: # until success: @@ -429,6 +430,8 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc): time.sleep(nsec_wait) ntry += 1 logging.warning(f"{indent} attempt {ntry} / {maxtry} ...") + # next time, wait a bit longer, but not too long ... + nsec_wait = min(nsec_wait * 2, nsec_wait_max) # endif # endtry -- GitLab From 8e9a649c8b3022c5c5523c865a41e86071cc423c Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Tue, 7 Apr 2026 17:16:21 +0200 Subject: [PATCH 2/4] Initialize empty listing with proper columns. --- src/cso/cso_colhub.py | 10 ++++++++-- src/cso/cso_file.py | 8 ++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/cso/cso_colhub.py b/src/cso/cso_colhub.py index 1e0a4a4..de30345 100644 --- a/src/cso/cso_colhub.py +++ b/src/cso/cso_colhub.py @@ -27,6 +27,9 @@ # 2025-09, Arjo Segers # Added "CSO_ColHubMirror_Cleanup" class. # +# 2026-04, Arjo Segers +# Initialize columns of listing file. +# ######################################################################## ### @@ -180,8 +183,9 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): # info .. logging.info(f"{indent}create %s ..." % lst_file) - # initiallize for (re)creation: - listing = cso_file.CSO_Listing() + # initiallize for (re)creation, + # define extra columns to ensure correct content even for empty files: + listing = cso_file.CSO_Listing( columns=["orbit","processing","collection","processor_version","href"] ) # archive directories archive_dirs = self.GetSetting("dir").split() @@ -282,6 +286,8 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): # endfor # archive_dir + # info ... + logging.warning(f"{indent}save ...") # sort: listing.Sort(by="orbit") # save: diff --git a/src/cso/cso_file.py b/src/cso/cso_file.py index b638e39..1f121f5 100644 --- a/src/cso/cso_file.py +++ b/src/cso/cso_file.py @@ -65,6 +65,9 @@ # Updated use of Dataset.dims following deprication warning. # Removed whitespace from template replacement. # +# 2026-04, Arjo Segers +# Added optional "columns" argument to initialize empty listing object. +# ######################################################################## ### @@ -1306,10 +1309,11 @@ class CSO_Listing(object): Optional arguments: * ``filename`` : listing file read into table + * ``columns`` : extra columns that will be created if an empty listing is initialized """ - def __init__(self, filename=None, indent=""): + def __init__(self, filename=None, columns=[], indent=""): """ Initialize empty table or read existing. """ @@ -1360,7 +1364,7 @@ class CSO_Listing(object): self.dirname = os.curdir # new empty table: - self.df = pandas.DataFrame(columns=["start_time", "end_time"]) + self.df = pandas.DataFrame(columns=["start_time", "end_time"] + columns) # endif -- GitLab From e233c1a9574736687e7e14852c24aaff3b07a035 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Tue, 7 Apr 2026 17:16:55 +0200 Subject: [PATCH 3/4] Updated User Guide. --- doc/source/history.rst | 5 +++++ doc/source/s5p-so2.rst | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/source/history.rst b/doc/source/history.rst index 6db1398..ccdd38c 100644 --- a/doc/source/history.rst +++ b/doc/source/history.rst @@ -127,6 +127,11 @@ A summary of the versions and changes. * | *v2.15* | Updates for access to Copernicus DataSpace: use STAC API to inquire, and download from S3 buckets. + | *(2026-03)* + +* | *vx.y* + | Removed support for SO\ :sub:`2` COBRA product as this has become operational. + | *(2026-04)* To be done diff --git a/doc/source/s5p-so2.rst b/doc/source/s5p-so2.rst index 90761ec..c519d8d 100644 --- a/doc/source/s5p-so2.rst +++ b/doc/source/s5p-so2.rst @@ -10,7 +10,7 @@ This chapter describes the tasks performed for processing Sentinel-5p SO\ :sub:` from the operational processing. Up to March 2026, the current operational product was available as SO2-COBRA (Theys et al., 2021) -from the `S5P-PAL Data Portal `_ . +from the `S5P-PAL Data Portal `_. Product description -- GitLab From 8f1800881144648b8dd233b8130952582e22ecbd Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 8 Apr 2026 09:24:31 +0200 Subject: [PATCH 4/4] Updated change log. --- CHANGELOG | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 3e87281..ea3df7e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -588,3 +588,8 @@ Added missing dependency for STAC inquiry. Fixed definition of bounding box for global selection. src/cso/cso_dataspace.py + +Option to initialize empty listing with extra columns. + src/cso/cso_file.py + src/cso/cso_colhub.py + -- GitLab