diff --git a/config/VIIRS/cso-user-settings.rc b/config/VIIRS/cso-user-settings.rc new file mode 100644 index 0000000000000000000000000000000000000000..9616ec9988b2f0f673e14ac1743c91bd729ff47e --- /dev/null +++ b/config/VIIRS/cso-user-settings.rc @@ -0,0 +1,112 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO common configuration +!!! +!!! Base settings that are used by multiple tasks: +!!! - time range(s) +!!! - target domain(s) +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + +!----------------------------------------------------------- +! id's +!----------------------------------------------------------- + +! file format: +my.cso.format : 1.0 + +! file format convention: +my.cso.convention : CF-1.7 + + +!----------------------------------------------------------- +! domain +!----------------------------------------------------------- + +! +! Used for: +! - orbit selection durning download +! - pixel selection +! - map plots +! + +! region name: +my.region : CAMS +! CAMS regional ensemble domain: +my.region.west : -30.0 +my.region.east : 45.0 +my.region.south : 30.0 +my.region.north : 76.0 +! size of map figures for this region: +my.region.figsize : (6,6) + +!! region name: +!my.region : globe +!! global domain: +!my.region.west : -180.0 +!my.region.east : 180.0 +!my.region.south : -90.0 +!my.region.north : 90.0 +!! size of map figures for this region: +!my.region.figsize : (8,6) + + +!---------------------------------------------------------- +! timerange +!---------------------------------------------------------- + +! inquire full timerange: +my.inquire.timerange.start : 2011-10-01 00:00 +my.inquire.timerange.end : 2024-12-31 23:59 + +! testing 3 days +my.timerange.start : 2018-06-01 00:00 +my.timerange.end : 2018-06-03 23:59 + + +!---------------------------------------------------------- +! user specific settings: +!---------------------------------------------------------- + +! Attributes written to output files. +my.attr.author : Your Name +my.attr.institution : CSO +my.attr.email : Your.Name@cso.org + +! base location for work directories: +!my.work : /work/${USER}/CSO-Tutorial +my.work : /Scratch/${USER}/CSO-VIIRS + + +!---------------------------------------------------------- +! job step defaults +!---------------------------------------------------------- + +! run jobs in foreground: +*.script.class : utopya.UtopyaJobScriptForeground + +! dummy value, will be defined when running in virtrual environment: +VIRTUAL_ENV : +! running in virutual envionment? +#if "${VIRTUAL_ENV}" > "" +! interpretor from virtual enviornment, the system path is set automatically: +*.shell : ${VIRTUAL_ENV}/bin/python3 +#else +! search path for python modules: +*.pypath : ${CSO_PREFIX}/src:${CSO_PREFIX}/src/utopya:${CSO_PREFIX}/src/cso +#endif + +! work directory for jobs; +*.workdir : ${my.work}/__NAME2PATH__ + +! for new job files, use jobtree settings from this file: +*.rcfile : ${__filename__} + + + +!---------------------------------------------------------- +! end +!---------------------------------------------------------- + + diff --git a/config/VIIRS/cso-viirs.rc b/config/VIIRS/cso-viirs.rc new file mode 100644 index 0000000000000000000000000000000000000000..52a02fa846095df2ba1ed7b239fb6c757342103c --- /dev/null +++ b/config/VIIRS/cso-viirs.rc @@ -0,0 +1,416 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO - CAMS Satellite Operator +!!! +!!! Settings for VIIRS processing. +!!! +!!! Environment: +!!! +!!! MY_PRODUCT : defines the VIIRS product, one of: +!!! viirs1-aod-db viirs1-aod-dt +!!! viirs2-aod-db viirs2-aod-dt +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + +!----------------------------------------------------------------------- +! user specific settings: +!----------------------------------------------------------------------- + +! include user specfic settings: +#include cso-user-settings.rc + + +!----------------------------------------------------------------------- +! multi-product +!----------------------------------------------------------------------- + +! instrument selection: +#if "${MY_PRODUCT}" in ["viirs1-aod-db","viirs1-aod-dt"] +my.instrument : VIIRS-1 +my.platform : SNPP +#elif "${MY_PRODUCT}" in ["viirs2-aod-db","viirs2-aod-dt"] +my.instrument : VIIRS-2 +my.platform : NOAA20 +#else +#error unsupported product '${MY_PRODUCT}' +#endif + +! product selection: +#if "${MY_PRODUCT}" in ["viirs1-aod-db","viirs2-aod-db"] +!~ "Deep Blue" products: +my.retrieval : AERDB +#elif "${MY_PRODUCT}" in ["viirs1-aod-dt","viirs2-aod-dt"] +!~ "Dark Target" products: +my.retrieval : AERDT +#else +#error unsupported product '${MY_PRODUCT}' +#endif + +! version selection: +my.version : v2.0.0 + +! local archive: +my.arch.dir : ${my.work}/EarthData/${my.instrument}/${my.retrieval}_L2 + + +!====================================================================== +!=== +!=== Inquire +!=== +!====================================================================== + + +!----------------------------------------------------------------------- +! inquire EarthData +!----------------------------------------------------------------------- + +! Obtain names of all available VIIRS files. +! Stored as csv with processing type, processor version, filenames, etc. + +! renew table if file already exists? +cso.inquire-table-earthaccess.renew : True + +! full time range: +cso.inquire-table-earthaccess.timerange.start : ${my.inquire.timerange.start} +cso.inquire-table-earthaccess.timerange.end : ${my.inquire.timerange.end} +!! ... TESTING +!cso.inquire-table-earthaccess.timerange.start : 2022-01-01 12:00:00 +!cso.inquire-table-earthaccess.timerange.end : 2022-01-01 15:00:00 + +!~ dataset name: +cso.inquire-table-earthaccess.dataset : ${my.retrieval}_L2_VIIRS_${my.platform} + +!~ domain specified as: west,south,east,north +cso.inquire-table-earthaccess.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} + +! csv file that will hold records per file with: +! - timerange of pixels in file +! - orbit number +cso.inquire-table-earthaccess.output.file : ${my.arch.dir}/inquire/inquire.csv + + + +!----------------------------------------------------------- +! overview plot of versions +!----------------------------------------------------------- + +! renew existing plots? +cso.inquire-plot.renew : True + +! listing files: +cso.inquire-plot.file : ${cso.inquire-table-earthaccess.output.file} +!cso.inquire-plot.filedate : 2023-08-07 + +! annote: +cso.inquire-plot.title : VIIRS-AOD + +! output figure, eventually use templates for time values: +cso.inquire-plot.output.file : ${my.arch.dir}/inquire/inquire.png + + +!====================================================================== +!=== +!=== download +!=== +!====================================================================== + + +! renew existing files (True|False) ? +cso.download.renew : False +!! .. TESTING ... +!cso.download.renew : True + +! time range: +cso.download.timerange.start : ${my.timerange.start} +cso.download.timerange.end : ${my.timerange.end} + +! listing of available source files crated by "inquire" job: +cso.download.inquire.file : ${my.arch.dir}/inquire/inquire.csv +!!~ historic inquire ... +!cso.download.inquire.filedate : 2025-02-01 + +! processor version: +#if "${my.version}" == "v2.0.0" +cso.download.processor_version : 020000 +#else +#error unsupported my.version "${my.version}" +#endif + +! target directory, includiong time values: +cso.download.dir : ${my.arch.dir}/${my.version}/%Y/%j + + + +!----------------------------------------------------------- +! listing of downloaded files +!----------------------------------------------------------- + + +! renew existing file (True|False) ? +cso.download-listing.renew : True + +! create listing of downloaded files; +! eventully include time templates %Y-%m-%d etc: +cso.download-listing.file : ${my.arch.dir}/${my.version}/listing.csv + +! filename pattern for files in subdirs: +cso.download-listing.pattern : AER*.nc + + + +!====================================================================== +!=== +!=== convert (and download) +!=== +!====================================================================== + + +! renew existing files (True|False) ? +!cso.convert.renew : True +! ... TESTING ... +cso.convert.renew : False + +! time range: +cso.convert.timerange.start : ${my.timerange.start} +cso.convert.timerange.end : ${my.timerange.end} + + +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +! input files +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +! listing file with input data: +cso.convert.input.listing : ${cso.download-listing.file} + +! selection names: +cso.convert.filters : lons lats valid quality + +! filter settings: +cso.convert.filter.lons.type : minmax +cso.convert.filter.lons.minmax : ${my.region.west} ${my.region.east} +cso.convert.filter.lons.var : Longitude +cso.convert.filter.lons.units : degrees_east + +! filter settings: +cso.convert.filter.lats.type : minmax +cso.convert.filter.lats.minmax : ${my.region.south} ${my.region.north} +cso.convert.filter.lats.var : Latitude +cso.convert.filter.lats.units : degrees_north + +! skip pixel with "no data", use the combined land/ocean variable to check: +cso.convert.filter.valid.var : Aerosol_Optical_Thickness_550_Land_Ocean_Best_Estimate +cso.convert.filter.valid.type : valid + +! Descriptions in variables: +! Aerosol_Optical_Thickness_QA_Flag_Land :long_name = "Deep Blue quality assurance flag over land. 0=no retrieval, 1=poor, 2=moderate, 3=good" ; +! Aerosol_Optical_Thickness_QA_Flag_Ocean:long_name = "SOAR quality assurance flag over water. 0=no retrieval, 1=poor, 3=good" ; +! compare for both variables, at least one should have a good quality pixel: +cso.convert.filter.quality.type : min2 +cso.convert.filter.quality.min : 3 +cso.convert.filter.quality.var : Aerosol_Optical_Thickness_QA_Flag_Land +cso.convert.filter.quality.var2 : Aerosol_Optical_Thickness_QA_Flag_Ocean +cso.convert.filter.quality.units : 1 + + + +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +! output files +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +! output directory and filename: +! - time values taken from start time +! - templates: %{orbit} +cso.convert.output.filename : ${my.work}/${MY_PRODUCT}/data/${my.region}/%Y/%m/${MY_PRODUCT}_%{orbit}.nc + +! pack variables on output: +cso.convert.output.packed : True +! zlib compression level, 0 for no compression: +cso.convert.output.complevel : 1 + +! global attributes: +cso.convert.output.attrs : format Conventions \ + author institution email +! +cso.convert.output.attr.format : ${my.cso.format} +cso.convert.output.attr.Conventions : ${my.cso.convention} +cso.convert.output.attr.author : ${my.attr.author} +cso.convert.output.attr.institution : ${my.attr.institution} +cso.convert.output.attr.email : ${my.attr.email} + +!! no need to swap layes: +!cso.convert.swap_layers : False + + +! ~ variables + +! +! Describe per variable: +! * .dims : dimensions list: +! pixel : selected pixels +! corner : number of footprint bounds (probably 4) +! layer : number of layers in atmospheric profile (layers in kernel) +! layeri : number of layer interfaces in atmospheric profile (layer+1) +! retr : number of layers in retrieval product (1 for columns) ; +! for error covariance use (retr,retr0) to avoid repeated dimensions +! track_scan : original scan index in 2D track +! track_pixel : original ground pixel in 2D track +! * .specal : keyword to select special processing +! * None : no special processing (default) +! * track_longitude : longitudes at centers of original 2D track +! * track_latitude : latitudes at centers of original 2D track +! * .units : target units if different from original +! * .oper : special postprocessing, currently supported: +! * square : fill variable with squared valued (used to form variance from standard deviation) +! In case no special processing is needed: +! * .from : original variable (group path and variable name) +! + +! which fields to be put out ? +cso.convert.output.vars : longitude longitude_bounds \ + latitude latitude_bounds \ + track_longitude track_longitude_bounds \ + track_latitude track_latitude_bounds \ + time \ + aot_412nm aot_488nm aot_550nm aot_670nm aot_865nm aot_1240nm aot_1640nm aot_2250nm + +! qa_value \ +! vcd vcd_errvar +! +!! vertical column density: +!cso.convert.output.var.vcd.dims : pixel retr +!cso.convert.output.var.vcd.from : PRODUCT/nitrogendioxide_tropospheric_column +!cso.convert.output.var.vcd.attrs : { 'ancillary_variables' : None, 'multiplication_factor_to_convert_to_molecules_percm2' : None } +! +!! error variance in vertical column density (after application of kernel), +!! fill with single element 'covariance matrix', from square of standard error: +!! use dims with different names to avoid that cf-checker complains: +!cso.convert.output.var.vcd_errvar.dims : pixel retr retr0 +!cso.convert.output.var.vcd_errvar.special : square +!cso.convert.output.var.vcd_errvar.from : PRODUCT/nitrogendioxide_tropospheric_column_precision_kernel +!!~ skip standard name, modifier "standard_error" is not valid anymore: +!cso.convert.output.var.vcd_errvar.attrs : { 'standard_name' : None, 'multiplication_factor_to_convert_to_molecules_percm2' : None } + +! centers of pixels; +! use special processing to ensure correction at swath edges; +! remove range, gives problems when packed ... +!~ longitudes +cso.convert.output.var.longitude.dims : pixel +cso.convert.output.var.longitude.special : longitude +cso.convert.output.var.longitude.from : Longitude +cso.convert.output.var.longitude.units : degrees_east +cso.convert.output.var.longitude.attrs : { 'valid_range' : None } +!~ latitudes +cso.convert.output.var.latitude.dims : pixel +cso.convert.output.var.latitude.special : latitude +cso.convert.output.var.latitude.from : Latitude +cso.convert.output.var.latitude.units : degrees_north +cso.convert.output.var.latitude.attrs : { 'valid_range' : None } + +! corners of pixels, interpolate/extrapolate from centers; +! use special processing to ensure correction at swath edges; +! remove range, gives problems when packed ... +!~ longitudes +cso.convert.output.var.longitude_bounds.dims : pixel corner +cso.convert.output.var.longitude_bounds.special : longitude_bounds +cso.convert.output.var.longitude_bounds.from : Longitude +cso.convert.output.var.longitude_bounds.units : degrees_east +cso.convert.output.var.longitude_bounds.attrs : { 'valid_range' : None } +!~ latitudes +cso.convert.output.var.latitude_bounds.dims : pixel corner +cso.convert.output.var.latitude_bounds.special : latitude_bounds +cso.convert.output.var.latitude_bounds.from : Latitude +cso.convert.output.var.latitude_bounds.units : degrees_north +cso.convert.output.var.latitude_bounds.attrs : { 'valid_range' : None } + +! original track: +! use special processing to ensure correction at swath edges; +! remove range, gives problems when packed ... +!~ longitudes +cso.convert.output.var.track_longitude.dims : track_scan track_pixel +cso.convert.output.var.track_longitude.special : track_longitude +cso.convert.output.var.track_longitude.from : Longitude +cso.convert.output.var.track_longitude.attrs : { 'valid_range' : None } +!~ latitudes +cso.convert.output.var.track_latitude.dims : track_scan track_pixel +cso.convert.output.var.track_latitude.special : track_latitude +cso.convert.output.var.track_latitude.from : Latitude +cso.convert.output.var.track_latitude.attrs : { 'valid_range' : None } +!~ corner lons +cso.convert.output.var.track_longitude_bounds.dims : track_scan track_pixel corner +cso.convert.output.var.track_longitude_bounds.special : track_longitude_bounds +cso.convert.output.var.track_longitude_bounds.from : Longitude +cso.convert.output.var.track_longitude_bounds.units : degrees_east +!~ corner lats +cso.convert.output.var.track_latitude_bounds.dims : track_scan track_pixel corner +cso.convert.output.var.track_latitude_bounds.special : track_latitude_bounds +cso.convert.output.var.track_latitude_bounds.from : Latitude +cso.convert.output.var.track_latitude_bounds.units : degrees_north + +! time based on TAI93 units:! use time-delta array for shape: +cso.convert.output.var.time.dims : pixel +cso.convert.output.var.time.special : time +cso.convert.output.var.time.from : Scan_Start_Time + +! AOT, merged product: +cso.convert.output.var.aot_550nm.dims : pixel +cso.convert.output.var.aot_550nm.from : Aerosol_Optical_Thickness_550_Land_Ocean_Best_Estimate +cso.convert.output.var.aot_550nm.attrs : { 'valid_range' : None } + +! AOT from land band only: +#for WVL in 412 488 670 865 1240 1640 2250 +cso.convert.output.var.aot_WVLnm.dims : pixel +cso.convert.output.var.aot_WVLnm.special : wavelength=WVL +#if WVL in [412] +cso.convert.output.var.aot_WVLnm.from : Spectral_Aerosol_Optical_Thickness_Land +cso.convert.output.var.aot_WVLnm.attrs : { 'valid_range' : None, 'long_name' : 'aerosol optical thickness at WVL nm over land' } +#elif WVL in [488,670] +cso.convert.output.var.aot_WVLnm.from : Spectral_Aerosol_Optical_Thickness_Land \ + Spectral_Aerosol_Optical_Thickness_Ocean +cso.convert.output.var.aot_WVLnm.attrs : { 'valid_range' : None, 'long_name' : 'aerosol optical thickness at WVL nm over land and water' } +#elif WVL in [865,1240,1640,2250] +cso.convert.output.var.aot_WVLnm.from : Spectral_Aerosol_Optical_Thickness_Ocean +cso.convert.output.var.aot_WVLnm.attrs : { 'valid_range' : None, 'long_name' : 'aerosol optical thickness at WVL nm over water' } +#else +#error unsupported wavelength WVL +#endif +#endfor + + + +!!====================================================================== +!!=== +!!=== listing +!!=== +!!====================================================================== +! +!! csv file that will hold records per file with: +!! - timerange of pixels in file +!! - orbit number +!cso.listing.file : ${my.work}/_PRODUCT_/data/${my.region}/${my.selection}__listing.csv +! +!! renew table if file already exists? +!cso.listing.renew : True +! +!! time range: +!cso.listing.timerange.start : ${my.timerange.start} +!cso.listing.timerange.end : ${my.timerange.end} +! +!! filename filters relative to listing file that should be scanned for orbit files; +!! names could include time templates ; +!! if same orbit is found in multiple directories, the first found is used; +!! remove existing table for safety to ensure that this is done correctly ... +!cso.listing.patterns : ${my.selection}/%Y/%m/_PRODUCT__*.nc +! +!! extra columns to be added, read from global attributes: +!cso.listing.xcolumns : orbit +! +! + + +!====================================================================== +!=== +!=== end +!=== +!====================================================================== + diff --git a/config/VIIRS/cso.rc b/config/VIIRS/cso.rc new file mode 100644 index 0000000000000000000000000000000000000000..a633c0fe94734774dda378bb5aa30c49c17426af --- /dev/null +++ b/config/VIIRS/cso.rc @@ -0,0 +1,182 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO - CAMS Satellite Operator +!!! +!!! Settings for project. +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +! dummy values for variables defined in 'cso' script, +! these will be replaced by the actual values from the environment, +! but needed for first evaluation of this file ... +CSO_PREFIX : /not/defined/yet +CSO_RCFILE : /not/defined/yet +CSO_RCDIR : /not/defined/yet + + +!---------------------------------------------------------- +! user specific settings: +!---------------------------------------------------------- + +! include user specfic settings: +#include cso-user-settings.rc + + + +!---------------------------------------------------------- +! job tree +!---------------------------------------------------------- + +! class to create a job tree: +cso.class : utopya.UtopyaJobTree +! top level job is virtual: +cso.virtual : True +! list of sub-elements: +!cso.elements : copy viirs1-aod-db viirs2-aod-db viirs1-aod-dt viirs2-aod-dt +cso.elements : copy viirs1-aod-db viirs2-aod-db + + +! ====================================================================== +! === +! === copy to workdir +! === +! ====================================================================== + +! no sub list: +cso.copy.class : utopya.UtopyaJobStep + +! default class is defined in machine specific settings; +! copy is always done in foreground to avoid that files are +! changed before the job is started: +cso.copy.script.class : utopya.UtopyaJobScriptForeground + +!! search path to utopya modules: +!cso.copy.pypath : ${CSO_PREFIX}/py + +! configure and build based on settings in this rcfile; +! the name of this file has been stored in the +! environment variable 'CSO_RCFILE' by the 'cso' script: +cso.copy.task.class : utopya.UtopyaCopy +cso.copy.task.args : '${CSO_RCFILE}', rcbase='cso' + + +! +! * config of "UtopyaCopy" +! + + +! no need to remove existing build directory if present ... +cso.copy.new : False + +! prefix for destination of source and script files +! (base path for subdirectories src, py, etc): +cso.copy.prefix : ${my.work} + +! no extension to build directory need (no objects created) +cso.copy.prefix.extensions : + +! copy will consist of following subdirs: +cso.copy.subdirs : rc + +! directories to be inlcuded in copy, +! otherwise only files are copied: +cso.copy.incdirs : + +! skip files matching these filename patterns +! (tempoary editor files, compiled modules, etc) +cso.copy.skip : .#* *~ *.pyc .DS* + +! list of source directories to be copied; +! here empty, only settings files will be copied: +cso.copy.dirs : +! copy settings to "rc": +cso.copy.rc.dirs : ${CSO_RCDIR} + +! write evaluated rcfile: +cso.copy.rcwrite : ${my.work}/cso.rc + + + + + +! ====================================================================== +! === +! === viirs products +! === +! ====================================================================== + +#for _PRODUCT_ in viirs1-aod-db viirs2-aod-db viirs1-aod-dt viirs2-aod-dt + + +! class to create a job tree: +cso._PRODUCT_.class : utopya.UtopyaJobTree + +!~ sub steps: +!cso._PRODUCT_.elements : inquire download download-listing convert +!~ one by one ... +!cso._PRODUCT_.elements : inquire +!cso._PRODUCT_.elements : download +!cso._PRODUCT_.elements : inquire download +cso._PRODUCT_.elements : download-listing +!cso._PRODUCT_.elements : convert + +! inquire tasks: +!cso._PRODUCT_.inquire.tasks : table-earthaccess plot +cso._PRODUCT_.inquire.tasks : table-earthaccess +!cso._PRODUCT_.inquire.tasks : plot + + + +! single step: +cso._PRODUCT_.inquire.class : utopya.UtopyaJobStep +!~ inquire files downloaded from EarthAccess: +cso._PRODUCT_.inquire.table-earthaccess.class : cso.CSO_EarthAccess_Inquire +cso._PRODUCT_.inquire.table-earthaccess.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.inquire-table-earthaccess', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } +!~ create plot of available versions: +cso._PRODUCT_.inquire.plot.class : cso.CSO_Inquire_Plot +cso._PRODUCT_.inquire.plot.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.inquire-plot', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } + +!~ download data: +! single step: +cso._PRODUCT_.download.class : utopya.UtopyaJobStep +! download task: +cso._PRODUCT_.download.task.class : cso.CSO_EarthAccess_Download +cso._PRODUCT_.download.task.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.download', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } +! single step: +cso._PRODUCT_.download-listing.class : utopya.UtopyaJobStep +! download task: +cso._PRODUCT_.download-listing.task.class : cso.CSO_EarthAccess_Download_Listing +cso._PRODUCT_.download-listing.task.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.download-listing', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } + +! * + +! single step: +cso._PRODUCT_.convert.class : utopya.UtopyaJobStep +! convert task: +cso._PRODUCT_.convert.task.class : cso.CSO_VIIRS_Convert +cso._PRODUCT_.convert.task.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.convert', \ + env={ 'MY_PRODUCT' : '_PRODUCT_' } +!! single step: +!cso._PRODUCT_.listing.class : utopya.UtopyaJobStep +!! listing task: +!cso._PRODUCT_.listing.task.class : cso.CSO_S5p_Listing +!cso._PRODUCT_.listing.task.args : '${my.work}/rc/cso-s5p-ALL.rc', \ +! rcbase='cso._PRODUCT_.listing' +! +#endfor + + +! ====================================================================== +! === +! === end +! === +! ====================================================================== diff --git a/doc/source/figs/VIIRS/viirs1-aod-db_inquire.png b/doc/source/figs/VIIRS/viirs1-aod-db_inquire.png new file mode 100644 index 0000000000000000000000000000000000000000..6f0b8b815f4bc5d5c944eda37815f53b0eec6341 Binary files /dev/null and b/doc/source/figs/VIIRS/viirs1-aod-db_inquire.png differ diff --git a/doc/source/gridding.rst b/doc/source/gridding.rst index e8c44645d34e851037ca28f18db670b597a86724..61f54831f1ba524f2720b4829cba43fba4baf07e 100644 --- a/doc/source/gridding.rst +++ b/doc/source/gridding.rst @@ -25,7 +25,7 @@ where: * :math:`w_p` is the footprint area [m\ :sup:`2`] of pixel :math:`p` * :math:`w_{p,k}` is the area [m\ :sup:`2`] of pixel :math:`p` that overlaps with the cell :math:`k`. -The overlapping area is computed using the :py:meth:`LonLatPolygonCentroids ` method. +The overlapping area is computed using the :py:meth:`LonLatPolygonCentroids <.LonLatPolygonCentroids>` method. This fractions a footprint area into a large number of triangles, and returns for each triangle the centroid and the area. The centroids are collected per grid cell, and sum over the associated triangle area's is @@ -80,7 +80,7 @@ that is configured as:: rcbase='cso.tutorial.gridded-catalogue-index' The actual work is done by the :py:class:`.CSO_GriddedCatalogue` -and :py:class:`utopya.Indexer ` classes, +and :py:class:`utopya.Indexer <.Indexer>` classes, see their description for details on the configuration. The result could be viewed in a browser: diff --git a/doc/source/obsoper.rst b/doc/source/obsoper.rst index 5ada849df3e78efa8b266c235f57e432bc9956f5..785b45fb7e590c303acc15de7e2326a33d9be088 100644 --- a/doc/source/obsoper.rst +++ b/doc/source/obsoper.rst @@ -1565,7 +1565,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: Simulated S5p NO2 columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/pymod-cso_viirs.rst b/doc/source/pymod-cso_viirs.rst new file mode 100644 index 0000000000000000000000000000000000000000..6afc9360d0d07fec14356971febc1eb02068083d --- /dev/null +++ b/doc/source/pymod-cso_viirs.rst @@ -0,0 +1,5 @@ +.. Documentation for module. + +.. Import documentation from ".py" file: +.. automodule:: cso.cso_viirs + diff --git a/doc/source/pymods.rst b/doc/source/pymods.rst index d73838598816b3d4cfb218696c2032e8ddcf84e9..ce62f4a81566b5c78c3d7f0221f5c19659190271 100644 --- a/doc/source/pymods.rst +++ b/doc/source/pymods.rst @@ -33,6 +33,7 @@ Overview of the Python module(s) and classes. pymod-cso_colhub pymod-cso_earthaccess pymod-cso_s5p + pymod-cso_viirs pymod-cso_file pymod-cso_gridded pymod-cso_superobs diff --git a/doc/source/s5p-chocho.rst b/doc/source/s5p-chocho.rst index c30269d1e1cb26a387874462f2a1c7e1c6f12bac..e520e3a97f1663999d54225f80c6605af85f8734 100644 --- a/doc/source/s5p-chocho.rst +++ b/doc/source/s5p-chocho.rst @@ -153,7 +153,7 @@ The portals provide data files created with the same retrieval algorithm, but mo It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -167,12 +167,12 @@ as well as the filename of the data and the url that should be used to actually See the section on *File name convention* in the *Product User Manual* for the meaning of all parts of the filename. -A similar class :py:class:`CSO_S5p_Download_Listing ` +A similar class :py:class:`CSO_S5p_Download_Listing <.CSO_S5p_Download_Listing>` class is available to list the content of the downloaded GlyRetro files. Also this will produce a table file. To visualize what is available from the portal, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/CHOCHO/Copernicus_S5p_CHOCHO.png :scale: 50 % @@ -514,7 +514,7 @@ The *listing* is a csv file that looks something like:: :alt: S5p glyox columns To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -707,7 +707,7 @@ The *listing* is a csv file that looks something like:: To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-co.rst b/doc/source/s5p-co.rst index 214cb8a4a8d2d3381c301cc25690b54eb85559c3..be5b5d00e8ee7a09ae02b77c2c0633ad82b67bf1 100644 --- a/doc/source/s5p-co.rst +++ b/doc/source/s5p-co.rst @@ -112,7 +112,7 @@ but with different processor versions. It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such orbit number and processor version, @@ -470,7 +470,7 @@ The jobtree configuration to inquire the portals and create the overview figure :alt: S5p co columns To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -663,7 +663,7 @@ The jobtree configuration to inquire the portals and create the overview figure To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-hcho.rst b/doc/source/s5p-hcho.rst index a538e047cd7f7a72a180b14212f9352bf220a461..df4fad162e270aa4ddc9af88febdcfcffd21de7f 100644 --- a/doc/source/s5p-hcho.rst +++ b/doc/source/s5p-hcho.rst @@ -117,7 +117,7 @@ The portal provides data files created with different processor versions. It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -132,7 +132,7 @@ See the section on *File name convention* in the *Product User Manual* for the m parts of the filename. To visualize what is available from the various portals, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/HCHO/Copernicus_S5p_HCHO.png :scale: 50 % @@ -533,7 +533,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p hcho columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -720,7 +720,7 @@ Figures are saved to files with the basename of the original orbit file and the To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-no2.rst b/doc/source/s5p-no2.rst index e8da3dcd11219579a07ebbb1da7b728736addbaf..b876d7e41cc6aaf0f011199ba709b578222bb47a 100644 --- a/doc/source/s5p-no2.rst +++ b/doc/source/s5p-no2.rst @@ -236,7 +236,7 @@ The portals provide data files created with the same retrieval algorithm, but mo It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -250,11 +250,11 @@ as well as the filename of the data and the url that should be used to actually See the section on *File name convention* in the *Product User Manual* for the meaning of all parts of the filename. -A similar class :py:class:`CSO_PAL_Inquire ` class is available to list the content +A similar class :py:class:`CSO_PAL_Inquire <.CSO_PAL_Inquire>` class is available to list the content of the *Product Algorithm Laboratory* portal. Also this will produce a table file. To visualize what is available from the various portals, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/NO2/Copernicus_S5p_NO2.png :scale: 50 % @@ -651,7 +651,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p NO\ :sub:`2` columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -908,7 +908,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p NO2 columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-o3.rst b/doc/source/s5p-o3.rst index c65d30703da4cd918ceed0ea950fb6451fd68e15..fc0a20d9a8c1770bdb9aec49f45ea5d34df7c901 100644 --- a/doc/source/s5p-o3.rst +++ b/doc/source/s5p-o3.rst @@ -162,7 +162,7 @@ Data is available for different processing streams, each identified by a 4-chara * ``OFFL`` : `Offline`, available within weeks after observations; * ``RPRO`` : re-processing of all previously made observations. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -177,7 +177,7 @@ See the section on *File name convention* in the *Product User Manual* for the m parts of the filename. To visualize the available data, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/O3-PR/Copernicus_S5p_O3-PR.png :scale: 50 % @@ -578,7 +578,7 @@ The example is based on the S5p O\ :sub:`3`-profile file from which the header i :alt: S5p O\ :sub:`3` columns To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -835,7 +835,7 @@ The example is based on the S5p O\ :sub:`3`-profile file from which the header i :alt: S5p O3 columns To search for interesting features in the data, - the :py:class:`Indexer ` class could be used to create index pages. + the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-so2-cobra.rst b/doc/source/s5p-so2-cobra.rst index 4a4c570a035c89686cfcb6d7045613c4877906b9..ec32e78c523ba14d80ed7de96b059a01706b55e5 100644 --- a/doc/source/s5p-so2-cobra.rst +++ b/doc/source/s5p-so2-cobra.rst @@ -127,7 +127,7 @@ There might be data available from more than one processor version. It is therefore necessary to inquire the archive first to see which data is available, and what the version numbers are. -The :py:class:`CSO_PAL_Inquire ` class is available to inquire the remote archive. +The :py:class:`CSO_PAL_Inquire <.CSO_PAL_Inquire>` class is available to inquire the remote archive. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, as well as the filename of the data and the url that should be used to actually download the data:: @@ -141,7 +141,7 @@ See the section on *File name convention* in the *Product User Manual* for the m parts of the filename. To visualize what is available from the various portals, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/SO2-COBRA/Copernicus_S5p_SO2-COBRA.png :scale: 50 % @@ -485,7 +485,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p SO\ :sub:`2` columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/s5p-so2.rst b/doc/source/s5p-so2.rst index 8571833fc8bd5f21b2180131c65c0da3dccc64bc..52b2aa3e5e19909876e8f1d6e90a4bc20b26780e 100644 --- a/doc/source/s5p-so2.rst +++ b/doc/source/s5p-so2.rst @@ -128,7 +128,7 @@ The portal provides data files created with different processor versions. It is therefore necessary to first inquire both archives to see which data is available where, and what the version numbers are. -The :py:class:`CSO_DataSpace_Inquire ` class is available to inquire the +The :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class is available to inquire the *Copernicus DataSpace*. The settings used by this class allow selection on for example time range and intersection area. The result is a csv file which with columns for keywords such as orbit number and processor version, @@ -143,7 +143,7 @@ See the section on *File name convention* in the *Product User Manual* for the m parts of the filename. To visualize what is available from the various portals, the -:py:class:`CSO_Inquire_Plot ` could be used to create an overview figure: +:py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` could be used to create an overview figure: .. figure:: figs/SO2/Copernicus_S5p_SO2.png :scale: 50 % @@ -558,7 +558,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p SO\ :sub:`2` columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: @@ -759,7 +759,7 @@ Figures are saved to files with the basename of the original orbit file and the :alt: S5p SO2 columns To search for interesting features in the data, -the :py:class:`Indexer ` class could be used to create index pages. +the :py:class:`Indexer <.Indexer>` class could be used to create index pages. Configuration could look like:: ! index creation task: diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 823c5a6237dbe2d9ec8ad339c2f99a65718e067a..453629aaef9fa8b7b09108981a5ed5a776854807 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -9,6 +9,85 @@ Tutorial This chapter describes step by step how to run the CSO pre-processor and observation operator. +Setup Python environment +======================== + +There are two ways to setup the correct Python environment for CSO. + + +Running CSO within your own Python environment +---------------------------------------------- + +The CSO tools could be started within your own Python environment. +You should then ensure that the required (versions of) packages have been installed correctly. + +A probablly incomplete list of packages is: + +* ``numpy`` +* ``numba`` +* ``netcdf4`` +* ``xarray`` + +If you want CSO to create plots, you probably also need: + +* ``matplotlib`` +* ``cartopy`` + +To download satellite data, you might need: + +* ``reqests`` : to access Sentinel data; +* ``earthaccess`` : to access VIIRS data. + +To generate a local copy of the User Guide, install: + +* ``sphinx`` +* ``toml`` + + + +Running CSO within a virtual environment +---------------------------------------- + +A *virtual environment* could be created that has all required (versions) of Python packages installed. +For this, the ``pyproject.toml`` file is included which contains the dependencies needed by CSO. + +To create a new virtual enviroment run:: + + python3 -m venv --prompt cso .venv + +This will create a new Python evironment folder called ``.venv``. +To activate it run:: + + source .venv/bin/activate + +The terminal prompt will then be preceded by ``(cso)`` which holds the name specified above +with the ``--prompt`` argument. + +You may need to upgrade ``pip`` with the command:: + + pip install --upgrade pip + +We can then install CSO and its dependencies into the virtual environment using:: + + pip install --editable . + +or, if also the dependencies that are required to generate a local copy of the User Guide:: + + pip install --editable .[docs] + +The ``--editable`` flag installs CSO in editable mode, meaning that the user can change the source code +in the ``src/`` directory and run CSO anywhere using these changes. + +As described below, a typical way to run the CSO tools is then:: + + ./bin/cso config/tutorial/tutorial.rc + +When finished, leave from the virtual environment using:: + + deactivate + + + Run script ========== @@ -56,7 +135,7 @@ This list is actually defined as tree, using lists in which the elements could b For each element in the tree, the configuration file should specify the name of a python class that takes care of the job creation. -If the element is a *tree*, use the :py:class:`utopya.UtopyaJobTree ` class, +If the element is a *tree*, use the :py:class:`utopya.UtopyaJobTree <.UtopyaJobTree>` class, and add a line to specify the names of the sub-elements. For the main ``cso`` job, this looks like:: @@ -79,7 +158,7 @@ in this example again a tree is defined:: cso.tutorial.elements : inquire convert listing catalogue A job that is no container for sub-jobs but should actually do some work should -be defined with the :py:class:`utopya.UtopyaJobStep ` class. +be defined with the :py:class:`utopya.UtopyaJobStep <.UtopyaJobStep>` class. This is for example the case for the ``cso.tutorial.inquire`` job:: ! single step: @@ -169,7 +248,7 @@ The first setting defines that this is single job step that should do some work. The ``tasks`` list defines keywords for the two tasks to be performed. -* For the ``table-dataspace`` task, define that the :py:class:`CSO_DataSpace_Inquire ` class +* For the ``table-dataspace`` task, define that the :py:class:`CSO_DataSpace_Inquire <.CSO_DataSpace_Inquire>` class should be used to do the work; the class is accessible from the :py:mod:`cso` module (implemented in``py/cso.py``). The first arguments that initialize the class specifies the name of an rcfile with settings; @@ -179,7 +258,7 @@ The ``tasks`` list defines keywords for the two tasks to be performed. ``'cso.tutorial.inquire-table-dataspace'``. * Similar for the ``plot`` task, the settings define that the - :py:class:`CSO_Inquire_Plot ` class should be used to do the work. + :py:class:`CSO_Inquire_Plot <.CSO_Inquire_Plot>` class should be used to do the work. The tutorial settings will inquire the time range 2018-2023. @@ -281,7 +360,7 @@ The conversion job is configured with:: cso.tutorial.convert.task.args : '${__filename__}', \ rcbase='cso.tutorial.convert' -The conversion is thus done using the :py:class:`CSO_S5p_Convert ` class +The conversion is thus done using the :py:class:`CSO_S5p_Convert <.CSO_S5p_Convert>` class that can be accessed from the :py:mod:`cso` module. The arguments that initialize the class specify the name of a rcfile with settings (in this case the ``tutorial.rc`` that holds the job-tree definition) @@ -333,7 +412,7 @@ The following is a demo Python code that creates an S5p/NO2 map:: title = os.path.basename(filename) label = '%s [%s]' % (vname,attrs['units']) # create map figure (single layer): - fig = cso.QuickMap( values[:,:,0], xx=xx, yy=yy, vmin=0.0, vmax=vmax, + fig = cso.QuickPat( values[:,:,0], x=xx, y=yy, vmin=0.0, vmax=vmax, bmp=dict(resolution='l',countries=True,domain=domain,title=title), cbar=dict(label=label), figsize=(8,6) ) # save: @@ -432,7 +511,7 @@ and the variable that is plotted:: Index pages are created to facilitate browsing through the figures. The index is created with the ``index`` task of the job. -As shown above, the work is done by the :py:class:`Indexer ` class +As shown above, the work is done by the :py:class:`Indexer <.Indexer>` class that can be accessed from the :py:mod:`utopya` module. The arguments that initialize the class specify the name of an rcfile with settings (``tutorial.rc``) and that the settings start with keywords ``'cso.tutorial.catalogue-index'``. @@ -664,7 +743,7 @@ and the variable that is plotted:: Index pages are created to facilitate browsing through the figures. The index is created with the ``index`` task of the job. -As shown above, the work is done by the :py:class:`Indexer ` class +As shown above, the work is done by the :py:class:`Indexer <.Indexer>` class that can be accessed from the :py:mod:`utopya` module. The arguments that initialize the class specify the name of an rcfile with settings (``tutorial.rc``) and that the settings start with keywords ``'cso.tutorial.sim-catalogue-index'``. diff --git a/doc/source/viirs-aod.rst b/doc/source/viirs-aod.rst index 73d042b645a0e644beeec5efb9ed3b438fe1f25e..c0e1342495648209e12d9a78e1590455a8b940fd 100644 --- a/doc/source/viirs-aod.rst +++ b/doc/source/viirs-aod.rst @@ -141,11 +141,14 @@ CSO processing An example configuration of the CSO processing of the VIIRS/AOD data is available via the following settings: -* `config/VIIRS/viirs-aod.rc <../../../config/VIIRS/viirs-aod.rc>`_ +* `config/VIIRS/cso.rc <../../../config/VIIRS/cso.rc>`_ defines a job-tree of sub-steps to perform; +* `config/VIIRS/cso-user-settings.rc <../../../config/VIIRS/cso-user-settings.rc>`_ + defines user-settings such as domain, time range, and paths; +* `config/VIIRS/cso-viirs.rc <../../../config/VIIRS/cso-viirs.rc>`_ configures the operations on VIIRS/AOD data. Start the job-tree using:: - ./bin/cso /config/VIIRS/viirs-aod.rc + ./bin/cso /config/VIIRS/cso.rc Selected sub-steps in the processing are described below. @@ -157,6 +160,47 @@ Selected sub-steps in the processing are described below. Inquire VIIRS archives ====================== +The :py:class:`.CSO_EarthAccess_Inquire` class is available to inquire the +*EarthData* portal. The settings used by this class allow selection on the data product, time range, and intersection area. +The result is a csv file which with records for available files and properties of their conentent, +as well as the url that should be used to actually download the data:: + + filename ;start_time ;end_time ;product;platform;processor_version;href + AERDB_L2_VIIRS_SNPP.A2012064.0354.002.2023077205454.nc;20120304T035400;20120304T040000;AERDB ;SNPP ;020000 ;https://data.laadsdaac.earthdatacloud.nasa.gov/prod-lads/AERDB_L2_VIIRS_SNPP/AERDB_L2_VIIRS_SNPP.A2012064.0354.002.2023077205454.nc + AERDB_L2_VIIRS_SNPP.A2012064.0536.002.2023077205330.nc;20120304T053600;20120304T054200;AERDB ;SNPP ;020000 ;https://data.laadsdaac.earthdatacloud.nasa.gov/prod-lads/AERDB_L2_VIIRS_SNPP/AERDB_L2_VIIRS_SNPP.A2012064.0536.002.2023077205330.nc + : + +To visualize what is available from the portal, the +:py:class:`.CSO_Inquire_Plot` could be used to create an overview figure. +The following example shows that for the VIIRS-1 "Deep Blue" AOD product only data from processor version ``v2.0.0`` is available: + +.. figure:: figs/VIIRS/viirs1-aod-db_inquire.png + :scale: 50 % + :align: center + :alt: Example of available VIIRS-1 "Deep Blue" AOD settings. + +The jobtree configuration to inquire the portals and create the overview figure looks like:: + + ! single step: + cso.viirs1-aod-db.inquire.class : utopya.UtopyaJobStep + + ! inquire tasks: + cso.viirs1-aod-db.inquire.tasks : table-earthaccess plot + + + !~ inquire files downloaded from EarthAccess: + cso.viirs1-aod-db.inquire.table-earthaccess.class : cso.CSO_EarthAccess_Inquire + cso.viirs1-aod-db.inquire.table-earthaccess.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.inquire-table-earthaccess', \ + env={ 'MY_PRODUCT' : 'viirs1-aod-db' } + + !~ create plot of available versions: + cso.viirs1-aod-db.inquire.plot.class : cso.CSO_Inquire_Plot + cso.viirs1-aod-db.inquire.plot.args : '${my.work}/rc/cso-viirs.rc', \ + rcbase='cso.inquire-plot', \ + env={ 'MY_PRODUCT' : 'viirs1-aod-db' } + + .. Label between '.. _' and ':' ; use :ref:`text