TNO Intern

Commit a143a4b8 authored by Arjo Segers's avatar Arjo Segers
Browse files

Merge branch 'viirs-updates' into 'master' 

Viirs updates

See merge request !15
parents aedfb69c 7afe471c

config/VIIRS/cso-user-settings.rc

+1 −2
Original line number Diff line number Diff line
@@ -75,8 +75,7 @@ my.attr.institution : CSO 
my.attr.email                   :  Your.Name@cso.org



! base location for work directories:

!my.work                         :  /work/${USER}/CSO-Tutorial

my.work                         :  /Scratch/${USER}/CSO-VIIRS

my.work                         :  /work/${USER}/CSO-Tutorial-VIIRS





!----------------------------------------------------------

config/VIIRS/cso-viirs.rc

+34 −28
Original line number Diff line number Diff line
@@ -140,6 +140,12 @@ cso.download.processor_version : 020000 
! target directory, includiong time values:

cso.download.dir                    :  ${my.arch.dir}/${my.version}/%Y/%j



! ADHOC: some files are double (from different url's);

!  should be ckecked in inquire?

!  for the moment keep blacklist:

cso.download.blacklist              :  https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/5200/AERDB_L2_VIIRS_SNPP/2024/103/AERDB_L2_VIIRS_SNPP.A2024103.0836.002.2024106154554.nc \

                                       https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/5200/AERDB_L2_VIIRS_SNPP/2024/103/AERDB_L2_VIIRS_SNPP.A2024103.1348.002.2024106155539.nc







!-----------------------------------------------------------
@@ -378,34 +384,34 @@ cso.convert.output.var.aot_WVLnm.attrs : { 'valid_range' : Non 






!!======================================================================

!!===

!!=== listing

!!===

!!======================================================================

!

!! csv file that will hold records per file with:

!! - timerange of pixels in file

!! - orbit number

!cso.listing.file                :  ${my.work}/_PRODUCT_/data/${my.region}/${my.selection}__listing.csv

!

!! renew table if file already exists?

!cso.listing.renew               :  True

!

!! time range:

!cso.listing.timerange.start     :  ${my.timerange.start}

!cso.listing.timerange.end       :  ${my.timerange.end}

!

!! filename filters relative to listing file that should be scanned for orbit files;

!! names could include time templates ;

!! if same orbit is found in multiple directories, the first found is used;

!! remove existing table for safety to ensure that this is done correctly ...

!cso.listing.patterns            :  ${my.selection}/%Y/%m/_PRODUCT__*.nc

!

!! extra columns to be added, read from global attributes:

!cso.listing.xcolumns            :  orbit

!

!

!======================================================================

!===

!=== listing

!===

!======================================================================



! csv file that will hold records per file with:

! - timerange of pixels in file

! - orbit number

cso.listing.file                :   ${my.work}/${MY_PRODUCT}/data/${my.region}/listing.csv



! renew table if file already exists?

cso.listing.renew               :  True



! time range:

cso.listing.timerange.start     :  ${my.timerange.start}

cso.listing.timerange.end       :  ${my.timerange.end}



! filename filters relative to listing file that should be scanned for orbit files;

! names could include time templates ;

! if same orbit is found in multiple directories, the first found is used;

! remove existing table for safety to ensure that this is done correctly ...

cso.listing.patterns            :  %Y/%m/${MY_PRODUCT}_*.nc



! extra columns to be added, read from global attributes:

cso.listing.xcolumns            :  









!======================================================================

config/VIIRS/cso.rc

+15 −12
Original line number Diff line number Diff line
@@ -107,26 +107,26 @@ cso.copy.rcwrite : ${my.work}/cso.rc 


#for _PRODUCT_ in viirs1-aod-db viirs2-aod-db viirs1-aod-dt viirs2-aod-dt





! class to create a job tree:

cso._PRODUCT_.class                             :  utopya.UtopyaJobTree



!~ sub steps:

!cso._PRODUCT_.elements                          :  inquire download download-listing convert

!cso._PRODUCT_.elements                          :  inquire download download-listing convert listing

!~ one by one ...

!cso._PRODUCT_.elements                          :  inquire

!cso._PRODUCT_.elements                          :  download

!cso._PRODUCT_.elements                          :  inquire download

cso._PRODUCT_.elements                          :  download-listing

!cso._PRODUCT_.elements                          :  download-listing

!cso._PRODUCT_.elements                          :  convert

cso._PRODUCT_.elements                          :  listing



! *



! inquire tasks:

!cso._PRODUCT_.inquire.tasks                      :  table-earthaccess plot

cso._PRODUCT_.inquire.tasks                      :  table-earthaccess

!cso._PRODUCT_.inquire.tasks                      :  plot







! single step:

cso._PRODUCT_.inquire.class                      :  utopya.UtopyaJobStep

!~ inquire files downloaded from EarthAccess:
@@ -140,6 +140,8 @@ cso._PRODUCT_.inquire.plot.args : '${my.work}/rc/cso-viirs.rc' 
                                                      rcbase='cso.inquire-plot', \

                                                      env={ 'MY_PRODUCT' : '_PRODUCT_' }



! *



!~ download data:

! single step:

cso._PRODUCT_.download.class                     :  utopya.UtopyaJobStep
@@ -165,13 +167,14 @@ cso._PRODUCT_.convert.task.class : cso.CSO_VIIRS_Convert 
cso._PRODUCT_.convert.task.args                 :  '${my.work}/rc/cso-viirs.rc', \

                                                      rcbase='cso.convert', \

                                                      env={ 'MY_PRODUCT' : '_PRODUCT_' }

!! single step:

!cso._PRODUCT_.listing.class                     :  utopya.UtopyaJobStep

!! listing task:

!cso._PRODUCT_.listing.task.class                :  cso.CSO_S5p_Listing

!cso._PRODUCT_.listing.task.args                 :  '${my.work}/rc/cso-s5p-ALL.rc', \

!                                                      rcbase='cso._PRODUCT_.listing' 

!

! single step:

cso._PRODUCT_.listing.class                     :  utopya.UtopyaJobStep

! listing task:

cso._PRODUCT_.listing.task.class                :  cso.CSO_S5p_Listing

cso._PRODUCT_.listing.task.args                 :  '${my.work}/rc/cso-viirs.rc', \

                                                      rcbase='cso.listing', \

                                                      env={ 'MY_PRODUCT' : '_PRODUCT_' }



#endfor

src/cso/cso_catalogue.py

+10 −9
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ 
# 2025-09, Arjo Segers

#   Updated plotting of gridded fields.

#   Extended unit conversions.

#   Fixed reading of settings for map styles.

#
@@ -362,7 +363,7 @@ class CSO_Catalogue(CSO_CatalogueBase): 
            figsize = eval(self.GetSetting("figsize"))



            # no-data color:

            color_nan = self.GetSetting("color_nan", default="0.80")

            color_nan = self.GetSetting("color_nan", default="0.90")

            # extra map properties:

            bmp_kwargs = self.GetSetting("map", totype="dict", default=dict())
@@ -714,7 +715,7 @@ class CSO_SimCatalogue(CSO_CatalogueBase): 
        figsize = eval(self.GetSetting("figsize"))



        # no-data color:

        color_nan = self.GetSetting("color_nan", default="0.80")

        color_nan = self.GetSetting("color_nan", default="0.90")

        # extra map properties:

        bmp_kwargs = self.GetSetting("map", totype="dict", default=dict())
@@ -788,20 +789,20 @@ class CSO_SimCatalogue(CSO_CatalogueBase): 
                        # settings for this variable:

                        vkey = "var.%s" % varname

                        # originating variable:

                        vsource = self.GetSetting("var.{varname}.source", default="data:" + varname)

                        vsource = self.GetSetting(f"var.{varname}.source", default="data:" + varname)

                        # target units:

                        vunits = self.GetSetting("var.{varname}.units", default="None")

                        vunits = self.GetSetting(f"var.{varname}.units", default="None")

                        # plot type:

                        ptype = self.GetSetting("var.{varname}.type", default="map")

                        ptype = self.GetSetting(f"var.{varname}.type", default="map")

                        # long name used in labels:

                        long_name = self.GetSetting("var.{varname}.long_name", default=varname)

                        long_name = self.GetSetting(f"var.{varname}.long_name", default=varname)



                        # switch:

                        if ptype == "map":

                            # style:

                            vmin = eval(self.GetSetting("var.{varname}.vmin", default="None"))

                            vmax = eval(self.GetSetting("var.{varname}.vmax", default="None"))

                            colors = eval(self.GetSetting("var.{varname}.colors", default="None"))

                            vmin = eval(self.GetSetting(f"var.{varname}.vmin", default="None"))

                            vmax = eval(self.GetSetting(f"var.{varname}.vmax", default="None"))

                            colors = eval(self.GetSetting(f"var.{varname}.colors", default="None"))



                            # variable source:

                            #    [data:]vname

src/cso/cso_earthaccess.py

+34 −7
Original line number Diff line number Diff line
@@ -13,6 +13,9 @@ 
# 2025-04, Arjo Segers

#   Changed imports for python packaging.

#

# 2025-09, Arjo Segers

#   Added 'blacklist' for problematic URL's.

#



########################################################################

###
@@ -383,6 +386,15 @@ class CSO_EarthAccess_Inquire(utopya.UtopyaRc): 
                    # update record:

                    listing.UpdateRecord(filename, data, indent=f"{indent}    ")



                    ## ADHOC check on double records ..

                    #dlst = listing.Select( product=data["product"], start_time=data["start_time"], end_time=data["end_time"], processor_version=data["processor_version"] )

                    #if len(dlst) > 1 :

                    #    logging.error( f"record alredy exist?" )

                    #    logging.error( dlst.df )

                    #    logging.error( data )

                    #    raise Exception

                    ##endif



                    ## testing ...

                    # if len(listing) >= 100 :

                    #    logging.warning( f"BREAK after {len(listing)} files ..." )
@@ -462,6 +474,13 @@ class CSO_EarthAccess_Download(utopya.UtopyaRc): 
        ! processor version "v2.0.0"

         <rcbase>.processor_version   :  020000



    Some url's seem not to exist anymore, or actually, these are double available from 2 different url's of which 1 does not work anymore.

    As temporary solution these could be blacklisted::



        ! skip some problematic url's:

        <rcbase>.blacklist            :  https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/5200/AERDB_L2_VIIRS_SNPP/2024/103/AERDB_L2_VIIRS_SNPP.A2024103.0836.002.2024106154554.nc \

                                         https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/5200/AERDB_L2_VIIRS_SNPP/2024/103/AERDB_L2_VIIRS_SNPP.A2024103.1348.002.2024106155539.nc



    Specify the directory where the input files are to be searched,

    or where to download them to if not present yet::
@@ -566,8 +585,8 @@ class CSO_EarthAccess_Download(utopya.UtopyaRc): 
        logging.info(f"{indent}selection:")

        logging.info(f"{indent}  processor version: {processor_version}")



        ## skip some?

        # blacklist = self.GetSetting("blacklist", default="").split()

        # skip some?

        blacklist = self.GetSetting("blacklist", default="").split()



        # target directory, including time templates:

        arch_dir__template = self.GetSetting("dir")
@@ -594,6 +613,14 @@ class CSO_EarthAccess_Download(utopya.UtopyaRc): 
            # info ...

            logging.info(f"{indent}{basename} ...")



            # check ..

            if rec["href"] in blacklist:

                # info ...

                logging.info(f"{indent}  download url is blacklisted, skip ...")

                # next record:

                continue

            #endif



            # expand time templates

            arch_dir = rec["start_time"].strftime(arch_dir__template)

            # full path:
@@ -700,7 +727,7 @@ class CSO_EarthAccess_Download_Listing(utopya.UtopyaRc): 
    """

    Create *listing* file for files downloaded from VIIRS data portals.



    A *listing* file contains the names of the converted orbit files,

    A *listing* file contains the names of orbit files,

    the time range of pixels in the file, and other information extracted from the filenames or file attributes::



        filename                                                       ;start_time         ;end_time           ;orbit