config/Copernicus/cso-user-settings.rc

@@ -83,6 +83,10 @@ my.full-timerange.end           :  2023-12-31 23:59
 my.timerange.start              :  2018-06-01 00:00
 my.timerange.end                :  2018-06-01 23:59
 
+!! SO2-COBRA
+!my.timerange.start              :  2023-06-01 00:00
+!my.timerange.end                :  2023-06-01 23:59
+
 
 !----------------------------------------------------------
 ! user specific settings:

config/Copernicus/cso.rc

@@ -15,17 +15,15 @@
 #include cso-user-settings.rc
 
 ! selected tracers:
-my.tracers                :  no2 so2 hcho co chocho
+!my.tracers                :  no2 so2 so2-cobra hcho co chocho
 !~ one by one ..
-!my.tracers                :  no2
+my.tracers                :  no2
 !my.tracers                :  so2
+!my.tracers                :  so2-cobra
 !my.tracers                :  hcho
 !my.tracers                :  co
 !my.tracers                :  chocho
 
-!! testing ...
-!my.tracers                :  so2-cobra
-
 
 !----------------------------------------------------------
 ! job tree
@@ -50,12 +48,13 @@ cso.s5p.elements                              :  ${my.tracers}
 cso.s5p.TRACER.class                             :  utopya.UtopyaJobTree
 
 ! list of sub-elements:
-cso.s5p.TRACER.elements                          :  inquire \
-                                                    convert listing \
-                                                    catalogue \
-                                                    gridded gridded-catalogue
+!cso.s5p.TRACER.elements                          :  inquire \
+!                                                    convert listing \
+!                                                    catalogue \
+!                                                    gridded gridded-catalogue
 !~ preprocessor steps one by one ...
 !cso.s5p.TRACER.elements                          :  inquire
+cso.s5p.TRACER.elements                          :  convert listing catalogue
 !cso.s5p.TRACER.elements                          :  convert
 !cso.s5p.TRACER.elements                          :  listing
 !cso.s5p.TRACER.elements                          :  catalogue

doc/source/conf.py

@@ -22,7 +22,7 @@ copyright = '2020-2023, Arjo Segers'
 author = 'Arjo Segers'
 
 # The full version, including alpha/beta/rc tags
-release = 'v2.5'
+release = 'v2.6'
 
 
 # -- General configuration ---------------------------------------------------

doc/source/history.rst

@@ -91,6 +91,10 @@ A summary of the versions and changes.
   | Support new Copernicus *DataSpace* portal to download Sentinel data.
   | *(2023-11)*
 
+* | *v2.6*
+  | Support SO\ :sub:`2` COBRA product from PAL website.
+  | *(2024-01)*
+
 
 To be included
 ==============
@@ -115,8 +119,6 @@ A wishlist of developments.
         
   * Also output by observation operator could use the same basename.
   
-* Add all S5p filename parts to listingfile, see *Product User Manual*.
-    
 * Add offline tool to re-create listing files.
 
   * Scan directories based on filename patterns.

doc/source/s5p-chocho.rst

@@ -56,7 +56,7 @@ Features:
 
   * :math:`\mathbf{y}_s` is the simulated retrieval (mol/m2) defined on :math:`n_r=1` layers;
   * :math:`\mathbf{A}` is the *tropospheric* averaging kernel matrix with shape :math:`(n_r,n_a)`;
-    in this product, :math:`n_r=1` the number of *a priori* layers; see also the remarks below;
+    with :math:`n_a` the number of *a priori* layers;
   * :math:`\mathbf{x}` is the atmospheric state, which consists of a 3D array of CHOCHO concentrations;
   * :math:`\mathbf{H}` extracts a simulated profile from the state using horizontal and vertical interpolation;
     the result should be defined on the :math:`n_a` *a priori* layers
@@ -221,6 +221,30 @@ The example is based on the S5p CHOCHO file from which the header is available i
 * `doc/samples/S5P_OFFL_L2__CHOCHO___20200601T001519_20200601T015649_13643_01_010000_20210128.txt <../../samples/S5P_OFFL_L2__CHOCHO___20200601T001519_20200601T015649_13643_01_010000_20210128.txt>`_
 
 
+Orbit file selection
+--------------------
+
+Based on the inquiry the download and conversion could be limitted to files created with the most recent processor versions.
+
+For the S5P files a useful property is also the *collection number*, a 2-digit number that defines a collection of files
+(or actually processor versions) that together form a contineous series. The *collection number* is extracted from the filename,
+and stored as a column of the listing file.
+
+The following setting is used to select specific files from the archive based on the properities stored
+in the listing file::
+
+    ! Provide ';' seperated list of to decide if a particular orbit file should be processed.
+    ! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing),
+    ! the file with the first match will be used.
+    ! The expressions should include templates '%{header}' for the column values.
+    ! Example to select files from collection '03', preferably from processing 'RPRO' but otherwise from 'OFFL':
+    !   (%{collection} == '03') and (%{processing} == 'RPRO') ; \
+    !   (%{collection} == '03') and (%{processing} == 'OFFL')
+    !
+    cso.s5p.chocho.convert.selection                  :  (%{collection} == '03') and (%{processing} == 'RPRO') ; \
+                                                         (%{collection} == '03') and (%{processing} == 'OFFL')
+
+
 Pixel selection
 ---------------
 

doc/source/s5p-co.rst

@@ -44,7 +44,7 @@ Notes:
 
     * :math:`\mathbf{y}_s` is the simulated retrieval (mol/m2) defined on :math:`n_r=1` layers;
     * :math:`\mathbf{A}^{trop}` is the *tropospheric* averaging kernel matrix with shape :math:`(n_r,n_a)`;
-      in this product, :math:`n_r=1` the number of *a priori* layers; see also the remarks below;
+      with :math:`n_a` the number of *a priori* layers;
     * :math:`\mathbf{x}` is the atmospheric state, which consists of a 3D array of CO concentrations;
     * :math:`\mathbf{H}` extracts a simulated profile from the state using horizontal and vertical interpolation;
       the result should be defined on the :math:`n_a` *a priori* layers
@@ -171,7 +171,7 @@ The jobtree configuration to inquire the portals and create the overview figure
 
     ! task initialization:
     cso.s5p.co.convert.class     :  cso.CSO_S5p_Convert
-    cso.s5p.co.convert.args      :  '${PWD}/config/ESA-S5p/cso-s5p-co.rc', rcbase='cso.s5p.co.convert'
+    cso.s5p.co.convert.args      :  '${PWD}/config/Copernicus/cso-s5p-co.rc', rcbase='cso.s5p.co.convert'
 
   See the class documentation for the general configuration,
   below some specific choices are described.
@@ -414,7 +414,7 @@ The jobtree configuration to inquire the portals and create the overview figure
 
       ! catalogue creation task:
       cso.s5p.co.catalogue.task.figs.class  :  cso.CSO_Catalogue
-      cso.s5p.co.catalogue.task.figs.args   :  '${PWD}/config/ESA-S5p/cso-s5p-co.rc', \
+      cso.s5p.co.catalogue.task.figs.args   :  '${PWD}/config/Copernicus/cso-s5p-co.rc', \
                                                   rcbase='cso.s5p.co.catalogue'
 
   The configuration describes where to find a *listing* file with orbits, 
@@ -454,7 +454,7 @@ The jobtree configuration to inquire the portals and create the overview figure
 
       ! index creation task:
       cso.s5p.co.catalogue.task.index.class     :  utopya.Indexer
-      cso.s5p.co.catalogue.task.index.args      :  '${PWD}/config/ESA-S5p/cso-s5p-co.rc', \
+      cso.s5p.co.catalogue.task.index.args      :  '${PWD}/config/Copernicus/cso-s5p-co.rc', \
                                                      rcbase='cso.s5p.co.catalogue-index'
 
   When succesful, the index creator displays an url that could be loaded in a browser::
@@ -603,7 +603,7 @@ The jobtree configuration to inquire the portals and create the overview figure
 
       ! catalogue creation task:
       cso.s5p.TRACER.sim-catalogue.task.class          :  cso.CSO_SimCatalogue
-      cso.s5p.TRACER.sim-catalogue.task.args           :  '${PWD}/config/ESA-S5p/cso-s5p-TRACER.rc', \
+      cso.s5p.TRACER.sim-catalogue.task.args           :  '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
                                                         rcbase='cso.s5p.TRACER.sim-catalogue'
 
   The configuration describes where to find a *listing* file with orbits, 
@@ -647,7 +647,7 @@ The jobtree configuration to inquire the portals and create the overview figure
 
       ! index creation task:
       cso.s5p.co.catalogue.task.index.class     :  utopya.Indexer
-      cso.s5p.co.catalogue.task.index.args      :  '${PWD}/config/ESA-S5p/cso-s5p-co.rc', \
+      cso.s5p.co.catalogue.task.index.args      :  '${PWD}/config/Copernicus/cso-s5p-co.rc', \
                                                      rcbase='cso.s5p.co.catalogue-index'
 
   When succesful, the index creator displays an url that could be loaded in a browser::

doc/source/s5p-no2.rst

@@ -39,7 +39,7 @@ Features:
   * :math:`\mathbf{y}_s` is the simulated retrieval (mol/m2) defined on :math:`n_r=1` layers;
   * :math:`\mathbf{A}^{trop}` is the *tropospheric* averaging kernel matrix with shape :math:`(n_r,n_a)`,
     with :math:`n_a` the number of *a priori* layers;
-  * :math:`\mathbf{x}` is the atmospheric state, which probably consists of a 3D array of NO2 concentrations;
+  * :math:`\mathbf{x}` is the atmospheric state, which probably consists of a 3D array of NO\ :sub:`2` concentrations;
   * operators :math:`\mathbf{G}` and :math:`\mathbf{V}` together compute a simulated profile 
     at the :math:`n_a` *a priori* layers from the state, using horizontal (:math:`\mathbf{G}`)
     and vertical (:math:`\mathbf{V}`) mappings;
@@ -180,14 +180,14 @@ CSO processing
 
 *(See* :ref:`tutorial` *chapter for introduction to CSO scripts and configuration)*
 
-An example configuration of the CSO processing of the S5p/NO2 data is available via
+An example configuration of the CSO processing of the S5p/NO\ :sub:`2` data is available via
 the following settings:
 
 * `config/Copernicus/cso.rc <../../../config/Copernicus/cso.rc>`_
 
   Top-level settings that configure the job-tree with various sub-tasks.
   This is a generic file that could be used for multiple S5 products, 
-  edit it to select the NO2 processing.
+  edit it to select the NO\ :sub:`2` processing.
    
 * `config/Copernicus/cso-user-settings.rc <../../../config/Copernicus/cso-user-settings.rc>`_
 
@@ -195,7 +195,7 @@ the following settings:
   
 * `config/Copernicus/cso-s5p-no2.rc <../../../config/Copernicus/cso-s5p-no2.rc>`_
   
-  Specific settings for NO2 product.
+  Specific settings for NO\ :sub:`2` product.
 
 Start the job-tree using::
 
@@ -259,7 +259,7 @@ To visualize what is available from the various portals, the
 .. figure:: figs/NO2/Copernicus_S5p_NO2.png
    :scale: 50 %
    :align: center
-   :alt: Overview of available NO2 processings.
+   :alt: Overview of available NO\ :sub:`2` processings.
 
 The jobtree configuration to inquire the portals and create the overview figure could look like::
 
@@ -280,7 +280,6 @@ The jobtree configuration to inquire the portals and create the overview figure
 
 
 
-
 .. Label between '.. _' and ':' ; use :ref:`text <label>` for reference
 .. _s5p-no2-convert:
 
@@ -302,7 +301,7 @@ which is initialized using the settings file::
 
   ! task initialization:
   cso.s5p.no2.convert.class     :  cso.CSO_S5p_Convert
-  cso.s5p.no2.convert.args      :  '${PWD}/config/ESA-S5p/cso-s5p-no2.rc', rcbase='cso.s5p.no2.convert'
+  cso.s5p.no2.convert.args      :  '${PWD}/config/Copernicus/cso-s5p-no2.rc', rcbase='cso.s5p.no2.convert'
   
 See the class documentation for the general configuration,
 below some specific choices are described.
@@ -311,6 +310,30 @@ The example is based on the S5p NO\ :sub:`2` file from which the header is avail
 * `doc/samples/S5P_OFFL_L2__NO2____20190601T011830_20190601T030000_08451_01_010301_20190607T025115.txt <../../samples/S5P_OFFL_L2__NO2____20190601T011830_20190601T030000_08451_01_010301_20190607T025115.txt>`_
 
 
+Orbit file selection
+--------------------
+
+Based on the inquiry the download and conversion could be limitted to files created with the most recent processor versions.
+
+For the S5P files a useful property is also the *collection number*, a 2-digit number that defines a collection of files
+(or actually processor versions) that together form a contineous series. The *collection number* is extracted from the filename,
+and stored as a column of the listing file.
+
+The following setting is used to select specific files from the archive based on the properities stored
+in the listing file::
+
+    ! Provide ';' seperated list of to decide if a particular orbit file should be processed.
+    ! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing),
+    ! the file with the first match will be used.
+    ! The expressions should include templates '%{header}' for the column values.
+    ! Example to select files from collection '03', preferably from processing 'RPRO' but otherwise from 'OFFL':
+    !   (%{collection} == '03') and (%{processing} == 'RPRO') ; \
+    !   (%{collection} == '03') and (%{processing} == 'OFFL')
+    !
+    cso.s5p.no2.convert.selection                     :  (%{collection} == '03') and (%{processing} == 'RPRO') ; \
+                                                         (%{collection} == '03') and (%{processing} == 'OFFL')
+
+
 Pixel selection
 ---------------
         
@@ -488,8 +511,7 @@ the later could include a template for the orbit number::
     ! output directory and filename:
     ! - times are taken from mid of selection, rounded to hours
     ! - use '%{orbit}' for orbit number
-    cso.s5p.no2.convert.output.dir          :  /Scratch/CSO/S5p/RPRO/NO2/Europe/%Y/%m
-    cso.s5p.no2.convert.output.filename     :  S5p_RPRO_NO2_%{orbit}.nc
+    cso.s5p.no2.convert.output.filename     :  /Scratch/CSO-data/Europe/S5p/NO2/C03/%Y/%m/S5p_NO2_%{orbit}.nc
 
 A flag is read to decide if existing files should be renewed or kept::
 
@@ -557,8 +579,7 @@ file is found, the first match is used.
 This could be explored to create a listing that combines reprocessed data
 with near-real-time data::
 
-    <rcbase>.patterns            :  RPRO/NO2/Europe/%Y/%m/S5p_*.nc \
-                                    OFFL/NO2/Europe/%Y/%m/S5p_*.nc
+    <rcbase>.patterns            :  CO3/%Y/%m/S5p_*.nc
 
 
 
@@ -574,7 +595,7 @@ Configuration could look like::
 
     ! catalogue creation task:
     cso.s5p.no2.catalogue.task.figs.class  :  cso.CSO_Catalogue
-    cso.s5p.no2.catalogue.task.figs.args   :  '${PWD}/config/ESA-S5p/cso-s5p-no2.rc', \
+    cso.s5p.no2.catalogue.task.figs.args   :  '${PWD}/config/Copernicus/cso-s5p-no2.rc', \
                                                 rcbase='cso.s5p.no2.catalogue'
 
 The configuration describes where to find a *listing* file with orbits, 
@@ -606,7 +627,7 @@ Figures are saved to files with the basename of the original orbit file and the
 .. figure:: figs/NO2/S5p_RPRO_NO2_03278__vcd.png 
    :scale: 50 %
    :align: center
-   :alt: S5p NO2 columns
+   :alt: S5p NO\ :sub:`2` columns
 
 To search for interesting features in the data, 
 the :py:class:`Indexer <utopya_index.Indexer>` class could be used to create index pages.
@@ -614,7 +635,7 @@ Configuration could look like::
 
     ! index creation task:
     cso.s5p.no2.catalogue.task.index.class     :  utopya.Indexer
-    cso.s5p.no2.catalogue.task.index.args      :  '${PWD}/config/ESA-S5p/cso-s5p-no2.rc', \
+    cso.s5p.no2.catalogue.task.index.args      :  '${PWD}/config/Copernicus/cso-s5p-no2.rc', \
                                                    rcbase='cso.s5p.no2.catalogue-index'
 
 When succesful, the index creator displays an url that could be loaded in a browser::
@@ -696,7 +717,7 @@ Example settings::
 For the simulated values, also define a list of variable names that should be created::
 
   ! state varaiables to be put out from model:
-  tutorial.S5p.no2.vars                         :  mod_conc mod_hp mod_tcc mod_cc hx ys shx M_m A_m yr_m ys_m
+  tutorial.S5p.no2.vars                         :  mod_conc mod_hp mod_tcc mod_cc xs ys Sx M_m A_m yr_m ys_m
 
 Example settings::
 
@@ -822,7 +843,7 @@ Configuration could look like::
 
     ! catalogue creation task:
     cso.s5p.no2.sim-catalogue.task.class          :  cso.CSO_SimCatalogue
-    cso.s5p.no2.sim-catalogue.task.args           :  '${PWD}/config/ESA-S5p/cso-s5p-TRACER.rc', \
+    cso.s5p.no2.sim-catalogue.task.args           :  '${PWD}/config/Copernicus/cso-s5p-TRACER.rc', \
                                                       rcbase='cso.s5p.no2.sim-catalogue'
 
 The configuration describes where to find a *listing* file with orbits, 
@@ -871,7 +892,7 @@ Configuration could look like::
 
     ! index creation task:
     cso.s5p.no2.catalogue.task.index.class     :  utopya.Indexer
-    cso.s5p.no2.catalogue.task.index.args      :  '${PWD}/config/ESA-S5p/cso-s5p-no2.rc', \
+    cso.s5p.no2.catalogue.task.index.args      :  '${PWD}/config/Copernicus/cso-s5p-no2.rc', \
                                                    rcbase='cso.s5p.no2.catalogue-index'
 
 When succesful, the index creator displays an url that could be loaded in a browser::
@@ -882,5 +903,6 @@ When succesful, the index creator displays an url that could be loaded in a brow
 .. figure:: figs/NO2/CSO_NO2_sim-catalogue.png
    :scale: 50 %
    :align: center
-   :alt: Index for Simulated and S5p NO2 columns
+   :alt: Index for Simulated and S5p NO\ :sub:`2` columns
+
 

py/cso_dataspace.py

@@ -9,6 +9,9 @@
 #   Store access token in object to avoid server errors.
 #   Retry if downloaded zipfile is corrupted.
 #
+# 2024-01, Arjo Segers
+#   Do not use timeout when downloading.
+#
 
 ########################################################################
 ###
@@ -19,9 +22,9 @@
 """
 .. _cso-dataspace:
 
-*************
-CSO DataSpace
-*************
+************************
+``cso_dataspace`` module
+************************
 
 The ``cso_dataspace`` module provides classes for accessing  data from the 
 `Copernicus DataSpace <https://dataspace.copernicus.eu/>`_.
@@ -667,11 +670,14 @@ class CSO_DataSpace_Downloader(object):
 
     # *
 
-    def DownloadFile(self, href, output_file, maxtry=10, timeout=60, indent=""):
+    def DownloadFile(self, href, output_file, maxtry=10, nsec_wait=60, indent=""):
 
         """
         Download file from DataSpace.
 
+        If a request fails it is tried again up to a maximum of ``maxtry`` times,
+        with a delay of ``nsec_wait`` between requsts.
+
         Arguments:
 
         * ``href`` : download url, for example::
@@ -683,7 +689,7 @@ class CSO_DataSpace_Downloader(object):
         Optional arguments:
     
         * ``maxtry`` : number of times to try again if download fails
-        * ``timeout`` : delay in seconds between requests
+        * ``nsec_wait`` : delay in seconds between requests
 
         """
 
@@ -698,9 +704,6 @@ class CSO_DataSpace_Downloader(object):
         # tools:
         import cso_file
 
-        # number of seconds to wait in retry loop:
-        nsec_wait = 10
-
         # no token yet?
         if self.access_token is None:
             # info ..
@@ -720,7 +723,7 @@ class CSO_DataSpace_Downloader(object):
                 # ensure that "~/.netrc" is ignored by passing null-authorization,
                 # otherwise the token in the header is overwritten by a token formed
                 # from the login/password in the rcfile if that is found:
-                r = requests.get(href, auth=NullAuth(), headers=headers, timeout=timeout)
+                r = requests.get(href, auth=NullAuth(), headers=headers)
                 # check status, raise error if request failed:
                 r.raise_for_status()
 

py/cso_file.py

@@ -17,6 +17,9 @@
 # 2023-11, Arjo Segers
 #   Added "CheckDir" method.
 #
+# 2024-01, Arjo Segers
+#   Updated comment.
+#
 
 ########################################################################
 ###
@@ -32,6 +35,15 @@
 Create and access file with satellite data extract.
 
 
+Methods
+=======
+
+The following methods are defined:
+
+* :py:meth:`CheckDir`
+* :py:meth:`Pack_DataArray`
+
+
 Class hierchy
 =============
 
@@ -41,8 +53,8 @@ The classes are defined according to the following hierchy:
 * :py:class:`.CSO_Listing`
 
 
-Classes
-=======
+Methods and classes
+===================
 """
 
 # modules:

py/cso_pal.py

@@ -4,6 +4,9 @@
 # 2023-08, Arjo Segers
 #   Reformatted using 'black'.
 #
+# 2024-01, Arjo Segers
+#   Added 'CSO_PAL_Downloader' class.
+#
 
 ########################################################################
 ###
@@ -34,7 +37,7 @@ PAL API
 See the PAL `API info <https://data-portal.s5p-pal.com/cat-doc>`_ for latest info.
 
 S5P-PAL product files can be selected and downloaded using the *Spatio Temporal Asset Catalog* (STAC).
-The `PySTAC <https://pystac-client.readthedocs.io/en/latest/` Python interface is used for access.
+The `PySTAC <https://pystac-client.readthedocs.io/en/latest/>`_ Python interface is used for access.
 
 
 Class hierchy
@@ -46,6 +49,7 @@ The classes and are defined according to the following hierchy:
 * :py:class:`.UtopyaRc`
 
   * :py:class:`.CSO_PAL_Inquire`
+  * :py:class:`.CSO_PAL_Downloader`
 
 
 Classes
@@ -295,6 +299,159 @@ class CSO_PAL_Inquire(utopya.UtopyaRc):
 # endclass CSO_PAL_Inquire
 
 
+########################################################################
+###
+### PAL download
+###
+########################################################################
+
+
+class CSO_PAL_Downloader(object):
+
+    """
+    Class to download single file from the `Product Algorithm Laboratory <https://www.s5p-pal.com/>`_.
+
+    The :py:class:`DownloadFile` method should be used to
+    actually download a file.
+
+    Usage::
+
+        # initialize downloader:
+        downloader = CSO_PAL_Downloader()
+        # download file, store in specified file:
+        downloader.DownloadFile( "https://data-portal.s5p-pal.com/cat/sentinel-5p/download/88c15681-db43-4219-b391-c8567e39cccf", "orbit.nc" )
+
+    """
+
+    def __init__(self):
+
+        """
+        Initialize downloader.
+        """
+
+    # enddef __init__
+
+    # *
+
+    def DownloadFile(self, href, output_file, maxtry=10, nsec_wait=60, indent=""):
+
+        """
+        Download file from PAL.
+
+        If a request fails it is tried again up to a maximum of ``maxtry`` times,
+        with a delay of ``nsec_wait`` between requsts.
+
+        Arguments:
+
+        * ``href`` : download url, for example::
+    
+            https://data-portal.s5p-pal.com/cat/sentinel-5p/download/88c15681-db43-4219-b391-c8567e39cccf
+
+        * ``output_file`` : target file
+    
+        Optional arguments:
+    
+        * ``maxtry`` : number of times to try again if download fails
+        * ``nsec_wait`` : delay in seconds between requests
+
+        """
+
+        # modules:
+        import sys
+        import os
+        import time
+        import requests
+
+        # tools:
+        import cso_file
+
+        # retry loop ..
+        ntry = 1
+        while True:
+            # try to download and save:
+            try:
+
+                # get data:
+                r = requests.get(href)
+                # check status, raise error if request failed:
+                r.raise_for_status()
+
+                # product is netcdf file, use base name of target file:
+                product_file = os.path.basename(output_file)
+                # info ..
+                logging.info(f"{indent}write to {product_file} ...")
+                # write to temporary target first ..
+                tmpfile = product_file + ".tmp"
+                # open destination file for binary write:
+                with open(tmpfile, "wb") as fd:
+                    # prefered way to write content following:
+                    #   https://docs.python-requests.org/en/master/user/quickstart/
+                    for chunk in r.iter_content(chunk_size=128):
+                        fd.write(chunk)
+                    # endfor
+                # endwith
+                # rename:
+                os.rename(tmpfile, product_file)
+
+                # create target dir if necessary:
+                cso_file.CheckDir(output_file)
+                # move to destination:
+                os.rename(product_file, output_file)
+
+                # all ok, leave retry loop:
+                break
+
+            except requests.exceptions.HTTPError as err:
+                # info ..
+                msg = str(err)
+                logging.warning(f"{indent}exception from download; message received:")
+                logging.warning(f"{indent}  %s" % msg)
+                # catch known problem ...
+                if msg.startswith("401 Client Error: Unauthorized for url:"):
+                    logging.warning(f"{indent}renew token ...")
+                    self.CreateToken(href, indent=indent)
+                # endif
+
+            except MemoryError as err:
+                logging.error("memory error from download; increase resources?")
+                # quit with error:
+                raise
+
+            except Exception as err:
+                # info ..
+                logging.error("from download; message received:")
+                logging.error("  %s" % str(err))
+                # catch known problem ...
+                if msg.startswith("File is not a zip file"):
+                    logging.warning(f"{indent}maybe download was interrupted, try again  ...")
+                else:
+                    # quit with error:
+                    raise
+                # endif
+
+            # endtry
+
+            # increase counter:
+            ntry += 1
+            # switch:
+            if ntry == maxtry:
+                logging.warning(f"{indent}tried {maxtry} times; exit ...")
+                raise Exception
+            else:
+                logging.warning(f"{indent}wait {nsec_wait} seconds ...")
+                time.sleep(nsec_wait)
+                logging.warning(f"{indent}attempt {ntry} / {maxtry} ...")
+                continue  # while-loop
+            # endif
+
+        # endwhile # retry
+
+    # enddef DownloadFile
+
+
+# endclass CSO_PAL_Downloader
+
+
 ########################################################################
 ###
 ### end

py/cso_s5p.py

@@ -29,6 +29,9 @@
 # 2023-12, Arjo Segers
 #   Fixed bug in orbit selection.
 #
+# 2024-01, Arjo Segers
+#   Switch between DataSpace and PAL downloader based on download link.
+#
 
 ########################################################################
 ###
@@ -2184,6 +2187,7 @@ class CSO_S5p_Convert(utopya.UtopyaRc):
         # tools:
         import cso_file
         import cso_dataspace
+        import cso_pal
         import utopya
 
         # info ...
@@ -2523,13 +2527,26 @@ class CSO_S5p_Convert(utopya.UtopyaRc):
                 if not os.path.isfile(input_file):
                     # info ..
                     logging.info("      not present yet, download ...")
+                    # download url:
+                    href = rec["href"]
                     # initialize download?
                     if downloader is None:
-                        # init download ..
-                        downloader = cso_dataspace.CSO_DataSpace_Downloader()
+                        # init downloader based on url:
+                        if "dataspace.copernicus.eu" in href:
+                            # download from Copernicus DataSpace:
+                            downloader = cso_dataspace.CSO_DataSpace_Downloader()
+                        #
+                        elif "s5p-pal.com" in href:
+                            # download from PAL:
+                            downloader = cso_pal.CSO_PAL_Downloader()
+                        #
+                        else:
+                            logging.error("no downloader class defined for url: {href}")
+                            raise Exception
+                        # endif
                     # endif
                     # download ...
-                    downloader.DownloadFile(rec["href"], input_file, indent="        ")
+                    downloader.DownloadFile(href, input_file, indent="        ")
                     # store name:
                     downloads.append(input_file)
                 # endif