TNO Intern

Commit d43713e7 authored by Arjo Segers's avatar Arjo Segers
Browse files

Merge branch 'sesam' into 'master' 

Updates for S5P/SO2 processing.

See merge request !26
parents d06fd9ff 8f180088

CHANGELOG

+5 −0
Original line number Diff line number Diff line
@@ -588,3 +588,8 @@ Added missing dependency for STAC inquiry. 


Fixed definition of bounding box for global selection.

  src/cso/cso_dataspace.py



Option to initialize empty listing with extra columns.

  src/cso/cso_file.py

  src/cso/cso_colhub.py

doc/source/history.rst

+5 −0
Original line number Diff line number Diff line
@@ -127,6 +127,11 @@ A summary of the versions and changes. 
  

* | *v2.15*

  | Updates for access to Copernicus DataSpace: use STAC API to inquire, and download from S3 buckets.

  | *(2026-03)*



* | *vx.y*

  | Removed support for SO\ :sub:`2` COBRA product as this has become operational.

  | *(2026-04)*





To be done

src/cso/cso_colhub.py

+8 −2
Original line number Diff line number Diff line
@@ -27,6 +27,9 @@ 
# 2025-09, Arjo Segers

#   Added "CSO_ColHubMirror_Cleanup" class.

#

# 2026-04, Arjo Segers

#   Initialize columns of listing file.

#



########################################################################

###
@@ -180,8 +183,9 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): 
            # info ..

            logging.info(f"{indent}create %s ..." % lst_file)



            # initiallize for (re)creation:

            listing = cso_file.CSO_Listing()

            # initiallize for (re)creation,

            # define extra columns to ensure correct content even for empty files:

            listing = cso_file.CSO_Listing( columns=["orbit","processing","collection","processor_version","href"] )



            # archive directories

            archive_dirs = self.GetSetting("dir").split()
@@ -282,6 +286,8 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): 


            # endfor # archive_dir



            # info ...

            logging.warning(f"{indent}save ...")

            # sort:

            listing.Sort(by="orbit")

            # save:

src/cso/cso_dataspace.py

+4 −1
Original line number Diff line number Diff line
@@ -53,7 +53,7 @@ 
#

# 2026-04, Arjo Segers

#   Fixed definition of bounding box for global selection.

#   Added delays to avoid rate limit errors from STAC catalogue inquiry.

#   Added increasing delays to avoid rate limit errors from STAC catalogue inquiry.

#
@@ -340,6 +340,7 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc): 
            maxtry = 10

            ntry = 1

            nsec_wait = 5

            nsec_wait_max = 600

            # loop until success or too many errors:

            while True:

                # until success:
@@ -429,6 +430,8 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc): 
                        time.sleep(nsec_wait)

                        ntry += 1

                        logging.warning(f"{indent}    attempt {ntry} / {maxtry} ...")

                        # next time, wait a bit longer, but not too long ...

                        nsec_wait = min(nsec_wait * 2, nsec_wait_max)

                    # endif

                # endtry

src/cso/cso_file.py

+6 −2
Original line number Diff line number Diff line
@@ -65,6 +65,9 @@ 
#   Updated use of Dataset.dims following deprication warning.

#   Removed whitespace from template replacement.

#

# 2026-04, Arjo Segers

#   Added optional "columns" argument to initialize empty listing object.

#



########################################################################

###
@@ -1306,10 +1309,11 @@ class CSO_Listing(object): 
    Optional arguments:



    * ``filename`` : listing file read into table

    * ``columns`` : extra columns that will be created if an empty listing is initialized



    """



    def __init__(self, filename=None, indent=""):

    def __init__(self, filename=None, columns=[], indent=""):

        """

        Initialize empty table or read existing.

        """
@@ -1360,7 +1364,7 @@ class CSO_Listing(object): 
            self.dirname = os.curdir



            # new empty table:

            self.df = pandas.DataFrame(columns=["start_time", "end_time"])

            self.df = pandas.DataFrame(columns=["start_time", "end_time"] + columns)



        # endif

doc/source/s5p-so2.rst

+1 −1

File changed.

Contains only whitespace changes.