TNO Intern

Commit 63f3487f authored by Arjo Segers's avatar Arjo Segers
Browse files

Updated code and settings for latest PAL downloads.

parent ea7c7d5a

config/Copernicus/cso-s5p-chocho.rc

+14 −10
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ cso.s5p.chocho.inquire-table-pal.timerange.start : ${my.full-timerange.s 
cso.s5p.chocho.inquire-table-pal.timerange.end          :  ${my.full-timerange.end}



! server url:

cso.s5p.chocho.inquire-table-pal.url                    :  https://data-portal.s5p-pal.com/cat/sentinel-5p/catalog.json

cso.s5p.chocho.inquire-table-pal.url                    :  https://data-portal.s5p-pal.com/api/s5p-l2



! product type (always 10 characters!):

cso.s5p.chocho.inquire-table-pal.producttype            :  L2__CHOCHO
@@ -164,7 +164,7 @@ cso.s5p.chocho.convert.input.dir : ${my.observations}/GlyR 
cso.s5p.chocho.convert.downloads.cleanup              :  False



! selection names:

cso.s5p.chocho.convert.filters                        :  lons lats valid quality

cso.s5p.chocho.convert.filters                        :  lons lats quality



! filter settings:

cso.s5p.chocho.convert.filter.lons.var                :  PRODUCT/longitude
@@ -178,14 +178,18 @@ cso.s5p.chocho.convert.filter.lats.type : minmax 
cso.s5p.chocho.convert.filter.lats.minmax             :  ${my.region.south} ${my.region.north}

cso.s5p.chocho.convert.filter.lats.units              :  degrees_north



! skip pixel with "no data";

! - PRODUCT/qa_value seems always defined

! - however, some orbit files have "PRODUCT/delta_time" with no-data while qa_value==1.0,

!   for example:

!     2020/04/03/S5P_OFFL_L2__co___20200403T033959_20200403T052129_12808_01_010000_20210128.nc

!   therefore use the time delta to check:

cso.s5p.chocho.convert.filter.valid.var               :  PRODUCT/delta_time

cso.s5p.chocho.convert.filter.valid.type              :  valid

!! ...

!! This could not be used anymore, 

!! as "delta_time" has now shape "(time,scanline)" rather than "(time,scanline,ground_pixel)"

!! ...

!! Skip pixel with "no data";

!! - PRODUCT/qa_value seems always defined

!! - however, some orbit files have "PRODUCT/delta_time" with no-data while qa_value==1.0,

!!   for example:

!!     2020/04/03/S5P_OFFL_L2__co___20200403T033959_20200403T052129_12808_01_010000_20210128.nc

!!   therefore use the time delta to check:

!cso.s5p.chocho.convert.filter.valid.var               :  PRODUCT/delta_time

!cso.s5p.chocho.convert.filter.valid.type              :  valid



! comment in "PRODUCT/qa_value" variable:

!   "A continuous quality descriptor,

config/Copernicus/cso-s5p-so2-cobra.rc

+2 −2
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ cso.s5p.so2-cobra.inquire-table-pal.timerange.start : ${my.full-timerang 
cso.s5p.so2-cobra.inquire-table-pal.timerange.end          :  ${my.full-timerange.end}



! server url:

cso.s5p.so2-cobra.inquire-table-pal.url                    :  https://data-portal.s5p-pal.com/api/s5p-l2/collection.json

cso.s5p.so2-cobra.inquire-table-pal.url                    :  https://data-portal.s5p-pal.com/api/s5p-l2



! product type (always 10 characters!):

cso.s5p.so2-cobra.inquire-table-pal.producttype            :  L2__SO2CBR
@@ -60,7 +60,7 @@ cso.s5p.so2-cobra.inquire-plot.renew : True 
cso.s5p.so2-cobra.inquire-plot.file                     :  ${cso.s5p.so2-cobra.inquire-table-pal.output.file}

!~ specify dates ("yyyy-mm-dd") to use historic tables,

!  default is table of today:

cso.s5p.so2-cobra.inquire-plot.filedate                 :  2024-01-03

cso.s5p.so2-cobra.inquire-plot.filedate                 :  2024-11-06



! annote:

cso.s5p.so2-cobra.inquire-plot.title                    :  S5p/SO2-COBRA %Y-%m-%d

doc/source/s5p-chocho.rst

+1 −1
Original line number Diff line number Diff line
@@ -223,7 +223,7 @@ See the class documentation for the general configuration, 
below some specific choices are described.

The example is based on the S5p CHOCHO file from which the header is available in:



* `doc/samples/S5P_OFFL_L2__CHOCHO___20200601T001519_20200601T015649_13643_01_010000_20210128.txt <../../samples/S5P_OFFL_L2__CHOCHO___20200601T001519_20200601T015649_13643_01_010000_20210128.txt>`_

* `doc/samples/S5P_PAL__L2__CHOCHO_20180601T002121_20180601T020251_03272_03_010002_20240414T115812.txt <../../samples/S5P_PAL__L2__CHOCHO_20180601T002121_20180601T020251_03272_03_010002_20240414T115812.txt>`_





Orbit file selection

py/cso_pal.py

+35 −16
Original line number Diff line number Diff line
@@ -173,10 +173,7 @@ class CSO_PAL_Inquire(utopya.UtopyaRc): 
            #    west,south:east,north

            west, south, east, north = map(float, area.replace(":", " ").replace(",", " ").split())

            # info ...

            logging.info(

                indent

                + "area         : [%8.2f,%8.2f] x [%8.2f,%8.2f]" % (west, east, south, north)

            )

            logging.info( f"{indent}area         : [{west:8.2f},{east:8.2f}] x [{south:8.2f},{north:8.2f}]")

            # intersection description:

            intersects = {

                "type": "Polygon",
@@ -215,21 +212,43 @@ class CSO_PAL_Inquire(utopya.UtopyaRc): 
            # info ..

            logging.info(f"{indent}  %s .." % xt1.strftime("%Y-%m"))



            # collection:

            collection = pystac.Collection.from_file(url)

            # info ..

            logging.info(f"{indent}    search in catalog ...")

            # get catalog:

            catalog = pystac.Catalog.from_file(url)

            # end point for item search:

            endpoint_url = collection.get_single_link("search").target

            # search:

            item_collection = pystac_client.ItemSearch(

            endpoint_url = catalog.get_single_link("search").target

            # search, return item generator; 

            # do not generate before actual use, otherwise times are lost!

            items_generator = pystac_client.ItemSearch(

                endpoint_url, datetime=(xt1, xt2), intersects=intersects, filter=sfilter

            ).item_collection()

            ).items()

            # generate:

            items = list(items_generator)

            # info ..

            logging.info(f"{indent}    found {len(item_collection)} items")

            logging.info(f"{indent}    found {len(items)} items")



            # generate times and loop over them:

            #for item in item_collection:

            for item in items:

                # item properties:

                #  {'uuid': '7d135456-661b-4056-9e24-599a429756a5',

                #  'datetime': None,

                #  'start_datetime': '2018-06-01T00:21:21+00:00',

                #  'end_datetime': '2018-06-01T02:02:51+00:00',

                #  'metadata_date': '2024-03-18T11:04:38.509388',

                #  'archive_date': '2024-02-15T23:24:04.962709',

                #  'product_type': 'L2__SO2CBR',

                #  'product_name': 'S5P_PAL__L2__SO2CBR_20180601T002121_20180601T020251_03272_03_020000_20240215T231348',

                #  'creation_date': '2024-02-15T23:13:48',

                #  's5p:collection': 3,

                #  's5p:file_class': 'PAL_',

                #  's5p:file_type': 'L2__SO2CBR',

                #  's5p:orbit': 3272,

                #  's5p:processor_version': 20000}



            # testing ..

            for item in item_collection:

                # target file:

                filename = item.properties["physical_name"]

                filename = item.properties["product_name"] + ".nc"

                ## info ...

                #logging.info( indent+'    file : %s' % filename )
@@ -259,7 +278,7 @@ class CSO_PAL_Inquire(utopya.UtopyaRc): 
                te = datetime.datetime.strptime(end_time, tfmt)



                # extract download link:

                href = item.assets["download"].href

                href = item.assets["product"].href



                # row:

                rec = {