diff --git a/CHANGELOG b/CHANGELOG index 3e872815098fb0353200afb194b6fe23bd91e37d..ea3df7eb64e79b3fcc4e2ff1d23e26511e2e98d8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -588,3 +588,8 @@ Added missing dependency for STAC inquiry. Fixed definition of bounding box for global selection. src/cso/cso_dataspace.py + +Option to initialize empty listing with extra columns. + src/cso/cso_file.py + src/cso/cso_colhub.py + diff --git a/doc/source/history.rst b/doc/source/history.rst index 6db139820c4cfc71359e5460fda66db5d3f9c828..ccdd38cbc56cc0e2a533a48f4ccde102165d8c68 100644 --- a/doc/source/history.rst +++ b/doc/source/history.rst @@ -127,6 +127,11 @@ A summary of the versions and changes. * | *v2.15* | Updates for access to Copernicus DataSpace: use STAC API to inquire, and download from S3 buckets. + | *(2026-03)* + +* | *vx.y* + | Removed support for SO\ :sub:`2` COBRA product as this has become operational. + | *(2026-04)* To be done diff --git a/doc/source/s5p-so2.rst b/doc/source/s5p-so2.rst index 90761ec29f06d49f8a79324bf04effb505f79ce1..c519d8d615ae43700d3d1781d4444528be080051 100644 --- a/doc/source/s5p-so2.rst +++ b/doc/source/s5p-so2.rst @@ -10,7 +10,7 @@ This chapter describes the tasks performed for processing Sentinel-5p SO\ :sub:` from the operational processing. Up to March 2026, the current operational product was available as SO2-COBRA (Theys et al., 2021) -from the `S5P-PAL Data Portal `_ . +from the `S5P-PAL Data Portal `_. Product description diff --git a/src/cso/cso_colhub.py b/src/cso/cso_colhub.py index 1e0a4a43f45da6fdf533f9d293c35ee4debc40f8..de30345a9b41c6f57c4d7aaaa9e300adac64a028 100644 --- a/src/cso/cso_colhub.py +++ b/src/cso/cso_colhub.py @@ -27,6 +27,9 @@ # 2025-09, Arjo Segers # Added "CSO_ColHubMirror_Cleanup" class. # +# 2026-04, Arjo Segers +# Initialize columns of listing file. +# ######################################################################## ### @@ -180,8 +183,9 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): # info .. logging.info(f"{indent}create %s ..." % lst_file) - # initiallize for (re)creation: - listing = cso_file.CSO_Listing() + # initiallize for (re)creation, + # define extra columns to ensure correct content even for empty files: + listing = cso_file.CSO_Listing( columns=["orbit","processing","collection","processor_version","href"] ) # archive directories archive_dirs = self.GetSetting("dir").split() @@ -282,6 +286,8 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): # endfor # archive_dir + # info ... + logging.warning(f"{indent}save ...") # sort: listing.Sort(by="orbit") # save: diff --git a/src/cso/cso_dataspace.py b/src/cso/cso_dataspace.py index bd7e7234c1ecb8d9f99a08815c540db33f43e2f4..5209280b823f4d04729ac660b5d461abacb0f8b6 100644 --- a/src/cso/cso_dataspace.py +++ b/src/cso/cso_dataspace.py @@ -53,7 +53,7 @@ # # 2026-04, Arjo Segers # Fixed definition of bounding box for global selection. -# Added delays to avoid rate limit errors from STAC catalogue inquiry. +# Added increasing delays to avoid rate limit errors from STAC catalogue inquiry. # @@ -340,6 +340,7 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc): maxtry = 10 ntry = 1 nsec_wait = 5 + nsec_wait_max = 600 # loop until success or too many errors: while True: # until success: @@ -429,6 +430,8 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc): time.sleep(nsec_wait) ntry += 1 logging.warning(f"{indent} attempt {ntry} / {maxtry} ...") + # next time, wait a bit longer, but not too long ... + nsec_wait = min(nsec_wait * 2, nsec_wait_max) # endif # endtry diff --git a/src/cso/cso_file.py b/src/cso/cso_file.py index b638e39e3631c8467eb34d8336e49fb7a02fce77..1f121f50ccae4164ab790accc23a838050e27560 100644 --- a/src/cso/cso_file.py +++ b/src/cso/cso_file.py @@ -65,6 +65,9 @@ # Updated use of Dataset.dims following deprication warning. # Removed whitespace from template replacement. # +# 2026-04, Arjo Segers +# Added optional "columns" argument to initialize empty listing object. +# ######################################################################## ### @@ -1306,10 +1309,11 @@ class CSO_Listing(object): Optional arguments: * ``filename`` : listing file read into table + * ``columns`` : extra columns that will be created if an empty listing is initialized """ - def __init__(self, filename=None, indent=""): + def __init__(self, filename=None, columns=[], indent=""): """ Initialize empty table or read existing. """ @@ -1360,7 +1364,7 @@ class CSO_Listing(object): self.dirname = os.curdir # new empty table: - self.df = pandas.DataFrame(columns=["start_time", "end_time"]) + self.df = pandas.DataFrame(columns=["start_time", "end_time"] + columns) # endif