From dd495cfa56aef2938ca993a8cc1d0ac3023954bf Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Tue, 28 Jan 2025 16:46:25 +0100 Subject: [PATCH 1/4] Support ColHub mirror. --- CHANGELOG | 8 + config/_test-colhub/cso-s5p-no2.rc | 1232 ++++++++++++++++++ config/_test-colhub/cso-user-settings.rc | 169 +++ config/_test-colhub/cso.rc | 311 +++++ doc/source/history.rst | 3 + py/cso.py | 1 + py/cso_colhub.py | 1472 ++++++++++++++++++++++ py/cso_file.py | 215 +++- py/cso_s5p.py | 245 ++-- 9 files changed, 3475 insertions(+), 181 deletions(-) create mode 100644 config/_test-colhub/cso-s5p-no2.rc create mode 100644 config/_test-colhub/cso-user-settings.rc create mode 100644 config/_test-colhub/cso.rc create mode 100644 py/cso_colhub.py diff --git a/CHANGELOG b/CHANGELOG index 35db4a1..00aa344 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -466,3 +466,11 @@ Increased maximum number of records and introduced sort order to avoid that search request return a different result when repeated. py/cso_dataspace.py +colhub branch +~~~~~~~~~~~~~ + +Support inquiry and use of ColHub mirror archive. + py/cso_colhub.py + py/cso_file.py + py/cso_s5p.py + py/cso.py diff --git a/config/_test-colhub/cso-s5p-no2.rc b/config/_test-colhub/cso-s5p-no2.rc new file mode 100644 index 0000000..e775834 --- /dev/null +++ b/config/_test-colhub/cso-s5p-no2.rc @@ -0,0 +1,1232 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO - CAMS Satellite Operator +!!! +!!! Settings for S5p/NO2 processing. +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + +!----------------------------------------------------------------------- +! user specific settings: +!----------------------------------------------------------------------- + +! include user specfic settings: +#include cso-user-settings.rc + + +!====================================================================== +!=== +!=== Inquire +!=== +!====================================================================== + + +!----------------------------------------------------------------------- +! inquire DataSpace +!----------------------------------------------------------------------- + +! Obtain names of all available S5p files. +! Stored as csv with processing type, processor version, filenames, etc. + +! full time range: +cso.s5p.no2.inquire-table-dataspace.timerange.start : ${my.full-timerange.start} +cso.s5p.no2.inquire-table-dataspace.timerange.end : ${my.full-timerange.end} + +! API url: +cso.s5p.no2.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api + +! collection name: +cso.s5p.no2.inquire-table-dataspace.collection : Sentinel5P + +! product type, always 10 characters! +! L2__NO2___ +! L2__CO____ +! ... +cso.s5p.no2.inquire-table-dataspace.producttype : L2__NO2___ + +! target area; +!!~ empty for no limitation: +!cso.s5p.no2.inquire-table-dataspace.area : +!~ domain specified as: west,south,east,north +cso.s5p.no2.inquire-table-dataspace.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} + +! template for download url given "{product_id}": +cso.s5p.no2.inquire-table-dataspace.download_url : https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value + +! output table, date of today: +cso.s5p.no2.inquire-table-dataspace.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__${my.region}__%Y-%m-%d.csv + + +!----------------------------------------------------------------------- +! inquire ColHub portal +!----------------------------------------------------------------------- + +! Obtain names of all available S5p files. +! Stored as csv with processing type, processor version, filenames, etc. + +!! inquire full time range: +!cso.s5p.no2.inquire-table-colhub.timerange.start : ${my.full-timerange.start} +!cso.s5p.no2.inquire-table-colhub.timerange.end : ${my.full-timerange.end} +! TESTING ... +cso.s5p.no2.inquire-table-colhub.timerange.start : 2024-01-01 00:00:00 +cso.s5p.no2.inquire-table-colhub.timerange.end : 2025-02-01 00:00:00 + +! +! server url ; +! provide login/password in ~/.netrc: +! +! machine s5phub.copernicus.eu login s5pguest password s5pguest +! +!cso.s5p.no2.inquire-table-colhub.url : https://s5phub.copernicus.eu/dhus +cso.s5p.no2.inquire-table-colhub.url : https://colhub.met.no + +! target area; +!!~ empty for no limitation: +!cso.s5p.no2.inquire-table-colhub.area : +!~ domain specified as: west,south:east,north +cso.s5p.no2.inquire-table-colhub.area : ${my.region.west},${my.region.south}:${my.region.east},${my.region.north} + +! search query, obtained from interactive download: +! +! platformname : Sentinel-5 +! producttype : L2__NO2___ | L2__CO____ (always 10 characters!) +! processinglevel : L2 +! +cso.s5p.no2.inquire-table-colhub.query : platformname:Sentinel-5 AND \ + producttype:L2__NO2___ AND \ + processinglevel:L2 + +! output table, date of today: +cso.s5p.no2.inquire-table-colhub.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub__%Y-%m-%d.csv + + +!----------------------------------------------------------------------- +! inquire mirror archive +!----------------------------------------------------------------------- + +! csv file that will hold records per file with: +! - timerange of pixels in file +! - orbit number +cso.s5p.no2.inquire-table-mirror.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-listing__%Y-%m-%d.csv + +! renew table if file already exists? +cso.s5p.no2.inquire-table-mirror.renew : True + +! base path; example file: +! /x0/x0/x1/S5P_OFFL_L2__NO2____20230701T002408_20230701T020537_29604_03_020500_20230702T161349.nc +cso.s5p.no2.inquire-table-mirror.dir : /lustre/storeB/project/ESAcolhub/production-backend-AOI/S5p/all +! filename filter: +cso.s5p.no2.inquire-table-mirror.pattern : S5P_*_L2__NO2_*.nc + + +! ** create list of missing files + +! renew table if file already exists? +cso.s5p.no2.inquire-table-missing.renew : True + +! csv file that will hold records per file with: +! - timerange of pixels in file +! - orbit number +cso.s5p.no2.inquire-table-missing.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-missing__%Y-%m-%d.csv + +! listing from DataSpace with all potentially available files: +cso.s5p.no2.inquire-table-missing.all.file : ${cso.s5p.no2.inquire-table-dataspace.output.file} +cso.s5p.no2.inquire-table-missing.all.filedate : 2025-01-23 + +! listing from current archive: +cso.s5p.no2.inquire-table-missing.curr.file : ${cso.s5p.no2.inquire-table-mirror.output.file} +cso.s5p.no2.inquire-table-missing.curr.filedate : 2025-01-24 + +! selection of orbits, see "convert" below: +cso.s5p.no2.inquire-table-missing.selection : ${cso.s5p.no2.convert.selection} + + +!!----------------------------------------------------------------------- +!! inquire PAL portal +!!----------------------------------------------------------------------- +! +!! Obtain names of all available S5p files. +!! Stored as csv with processing type, processor version, filenames, etc. +! +!! inquire full time range: +!cso.s5p.no2.inquire-table-pal.timerange.start : ${my.full-timerange.start} +!cso.s5p.no2.inquire-table-pal.timerange.end : ${my.full-timerange.end} +! +!! server url: +!cso.s5p.no2.inquire-table-pal.url : https://data-portal.s5p-pal.com/cat/sentinel-5p/catalog.json +! +!! product type (always 10 characters!): +!cso.s5p.no2.inquire-table-pal.producttype : L2__NO2___ +! +!! target area; +!!!~ empty for no limitation: +!!cso.s5p.no2.inquire-table-pal.area : +!!~ domain specified as: west,south:east,north +!cso.s5p.no2.inquire-table-pal.area : ${my.region.west},${my.region.south}:${my.region.east},${my.region.north} +! +!! output table, date of today: +!cso.s5p.no2.inquire-table-pal.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_pal_%Y-%m-%d.csv + + +!----------------------------------------------------------- +! overview plot of versions +!----------------------------------------------------------- + +! renew existing plots? +cso.s5p.no2.inquire-plot.renew : True + +! listing files: +cso.s5p.no2.inquire-plot.file : ${cso.s5p.no2.inquire-table-dataspace.output.file} +!cso.s5p.no2.inquire-plot.file : ${cso.s5p.no2.inquire-table-colhub.output.file} +!cso.s5p.no2.inquire-plot.file : ${cso.s5p.no2.inquire-table-mirror.output.file} +!!~ specify dates ("yyyy-mm-dd") to use historic tables, +!! default is table of today: +cso.s5p.no2.inquire-plot.filedate : 2025-01-23 + +! annote: +cso.s5p.no2.inquire-plot.title : S5p/NO2 %Y-%m-%d + +! output figure, date of today: +cso.s5p.no2.inquire-plot.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__${my.region}__%Y-%m-%d.png +!cso.s5p.no2.inquire-plot.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub__%Y-%m-%d.png +!cso.s5p.no2.inquire-plot.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-archive__%Y-%m-%d.png + + +!====================================================================== +!=== +!=== download (without convert) +!=== +!====================================================================== + + +! time range: +cso.s5p.no2.download.timerange.start : ${my.timerange.start} +cso.s5p.no2.download.timerange.end : ${my.timerange.end} + +! listing of available source files, +! created by 'inquire-dataspace' job: +cso.s5p.no2.download.inquire.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv +!!~ historic inquire ... +!cso.s5p.no2.download.inquire.filedate : 2023-08-07 + +! selection keyword: +my.s5p.no2.download.selection : C03 + +! Provide ';' seperated list of to decide if a particular orbit file should be processed. +! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing), +! the file with the first match will be used. +! The expressions should include templates '%{header}' for the column values. +! Example to select files from collection '03', preferably from processing 'RPRO' but otherwise from 'OFFL': +! (%{collection} == '03') and (%{processing} == 'RPRO') ; \ +! (%{collection} == '03') and (%{processing} == 'OFFL') +! +#if "${my.s5p.no2.download.selection}" == "C03" +cso.s5p.no2.download.selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ + (%{collection} == '03') and (%{processing} == 'OFFL') +#else +#error unsupported my.s5p.no2.download.selection "${my.s5p.no2.download.selection}" +#endif + +! input directory; +! files are searched here or downloaded to if not present yet; +! supported templates: +! %{processing} +cso.s5p.no2.download.input.dir : ${my.work}/Copernicus/S5P/%{processing}/NO2/%Y/%m + + +!====================================================================== +!=== +!=== convert (and download) +!=== +!====================================================================== + + +! renew existing files (True|False) ? +cso.s5p.no2.convert.renew : True +!cso.s5p.no2.convert.renew : False + +! time range: +cso.s5p.no2.convert.timerange.start : ${my.timerange.start} +cso.s5p.no2.convert.timerange.end : ${my.timerange.end} + + +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +! input files +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +! listing of available source files, +! created by 'inquire-dataspace' job: +!cso.s5p.no2.convert.inquire.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv +!cso.s5p.no2.convert.inquire.filedate : ${cso.s5p.no2.inquire-plot.filedate} +! TESTING .. +cso.s5p.no2.convert.inquire.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-listing__%Y-%m-%d.csv +cso.s5p.no2.convert.inquire.filedate : 2025-01-24 + +! selection keyword: +my.s5p.no2.selection : C03 + +! Provide ';' seperated list of to decide if a particular orbit file should be processed. +! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing), +! the file with the first match will be used. +! The expressions should include templates '%{header}' for the column values. +! Example to select files from collection '03', preferably from processing 'RPRO' but otherwise from 'OFFL': +! (%{collection} == '03') and (%{processing} == 'RPRO') ; \ +! (%{collection} == '03') and (%{processing} == 'OFFL') +! +#if "${my.s5p.no2.selection}" == "C03" +cso.s5p.no2.convert.selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ + (%{collection} == '03') and (%{processing} == 'OFFL') +#else +#error unsupported my.s5p.no2.selection "${my.s5p.no2.selection}" +#endif + +! input directory; +! files are searched here or downloaded to if not present yet; +! supported templates: +! %{processing} +cso.s5p.no2.convert.input.dir : ${my.work}/Copernicus/S5P/%{processing}/NO2/%Y/%m + +! remove downloaded input files after convert? +cso.s5p.no2.convert.downloads.cleanup : False + +! selection names: +cso.s5p.no2.convert.filters : lons lats valid quality +!~ optionally also: +! cloud_fraction + +! filter settings: +cso.s5p.no2.convert.filter.lons.var : PRODUCT/longitude +cso.s5p.no2.convert.filter.lons.type : minmax +cso.s5p.no2.convert.filter.lons.minmax : ${my.region.west} ${my.region.east} +cso.s5p.no2.convert.filter.lons.units : degrees_east + +! filter settings: +cso.s5p.no2.convert.filter.lats.var : PRODUCT/latitude +cso.s5p.no2.convert.filter.lats.type : minmax +cso.s5p.no2.convert.filter.lats.minmax : ${my.region.south} ${my.region.north} +cso.s5p.no2.convert.filter.lats.units : degrees_north + +! skip pixel with "no data", use the "qa_value" variable to check: +cso.s5p.no2.convert.filter.valid.var : PRODUCT/qa_value +cso.s5p.no2.convert.filter.valid.type : valid + +! Comment in "PRODUCT/qa_value" variable: +! "A continuous quality descriptor, +! varying between 0 (no data) and 1 (full quality data). +! Recommend to ignore data with qa_value < 0.5" +! Tests suggest that better threshold is 0.54, +! this removes the kernels with very high values. +! 0.75 is the recommended use so what we will use here +cso.s5p.no2.convert.filter.quality.var : PRODUCT/qa_value +cso.s5p.no2.convert.filter.quality.type : min +cso.s5p.no2.convert.filter.quality.min : 0.75 +cso.s5p.no2.convert.filter.quality.units : 1 + +!! recommended 0.3 cloud fraction max (https://agupubs.onlinelibrary.wiley.com/doi/full/10.1029/2018GL081095) +!cso.s5p.no2.convert.filter.cloud_fraction.var : PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/cloud_fraction_crb_nitrogendioxide_window +!cso.s5p.no2.convert.filter.cloud_fraction.type : max +!cso.s5p.no2.convert.filter.cloud_fraction.max : 0.3 +!cso.s5p.no2.convert.filter.cloud_fraction.units : 1 + + + +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +! output files +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +! output directory and filename: +! - times are taken from mid of selection, rounded to hours +! - use '%{processing}' for the processing name +! - use '%{orbit}' for orbit number +cso.s5p.no2.convert.output.filename : ${my.work}/CSO-data/${my.region}/S5p/NO2/${my.s5p.no2.selection}/%Y/%m/S5p_NO2_%{orbit}.nc + +! pack variables on output: +cso.s5p.no2.convert.output.packed : True +! zlib compression level, 0 for no compression: +cso.s5p.no2.convert.output.complevel : 1 + +! global attributes: +cso.s5p.no2.convert.output.attrs : format Conventions \ + author institution email +! +cso.s5p.no2.convert.output.attr.format : ${my.cso.format} +cso.s5p.no2.convert.output.attr.Conventions : ${my.cso.convention} +cso.s5p.no2.convert.output.attr.author : ${my.attr.author} +cso.s5p.no2.convert.output.attr.institution : ${my.attr.institution} +cso.s5p.no2.convert.output.attr.email : ${my.attr.email} + +! no need to swap layes: +cso.s5p.no2.convert.swap_layers : False + + +! ~ variables + +! which fields to be put out ? +cso.s5p.no2.convert.output.vars : longitude longitude_bounds \ + latitude latitude_bounds \ + track_longitude track_longitude_bounds \ + track_latitude track_latitude_bounds \ + time \ + qa_value \ + pressure kernel_trop amf amf_trop nla \ + vcd vcd_errvar \ + cloud_fraction cloud_radiance_fraction + +! ... optional ... +! ground_pixel \ +! cloud_pressure_crb \ +! surface_albedo \ +! viewing_zenith_angle \ +! viewing_azimuth_angle \ +! solar_azimuth_angle \ +! solar_zenith_angle \ +! surface_classification \ +! snow_ice_flag + +! +! Describe per variable: +! * .dims : dimensions list: +! pixel : selected pixels +! corner : number of footprint bounds (probably 4) +! layer : number of layers in atmospheric profile (layers in kernel) +! layeri : number of layer interfaces in atmospheric profile (layer+1) +! retr : number of layers in retrieval product (1 for columns) ; +! for error covariance use (retr,retr0) to avoid repeated dimensions +! track_scan : original scan index in 2D track +! track_pixel : original ground pixel in 2D track +! * .specal : keyword to select special processing +! * None : no special processing (default) +! * track_longitude : longitudes at centers of original 2D track +! * track_latitude : latitudes at centers of original 2D track +! * track_longitude_bounds : longitude bounds at centers of original 2D track +! * track_latitude_bounds : latitude bounds at centers of original 2D track +! * .units : target units if different from original +! * .oper : special postprocessing, currently supported: +! * square : fill variable with squared valued (used to form variance from standard deviation) +! In case no special processing is needed: +! * .from : original variable (group path and variable name) +! + +! center longitudes; remove bounds attribute, no coordinate ... +cso.s5p.no2.convert.output.var.longitude.dims : pixel +cso.s5p.no2.convert.output.var.longitude.from : PRODUCT/longitude +cso.s5p.no2.convert.output.var.longitude.attrs : { 'bounds' : None } +! center latitudes; remove bounds attribute, no coordinate ... +cso.s5p.no2.convert.output.var.latitude.dims : pixel +cso.s5p.no2.convert.output.var.latitude.from : PRODUCT/latitude +cso.s5p.no2.convert.output.var.latitude.attrs : { 'bounds' : None } + +! corner longitudes; no units in file: +cso.s5p.no2.convert.output.var.longitude_bounds.dims : pixel corner +cso.s5p.no2.convert.output.var.longitude_bounds.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds +cso.s5p.no2.convert.output.var.longitude_bounds.units : degrees_east +! corner latitudes, no units in file: +cso.s5p.no2.convert.output.var.latitude_bounds.dims : pixel corner +cso.s5p.no2.convert.output.var.latitude_bounds.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds +cso.s5p.no2.convert.output.var.latitude_bounds.units : degrees_north + +! original track: +!~ center lon; remove bounds attribute, no coordinate ... +cso.s5p.no2.convert.output.var.track_longitude.dims : track_scan track_pixel +cso.s5p.no2.convert.output.var.track_longitude.special : track_longitude +cso.s5p.no2.convert.output.var.track_longitude.from : PRODUCT/longitude +cso.s5p.no2.convert.output.var.track_longitude.attrs : { 'bounds' : None } +!~ center lat; remove bounds attribute, no coordinate ... +cso.s5p.no2.convert.output.var.track_latitude.dims : track_scan track_pixel +cso.s5p.no2.convert.output.var.track_latitude.special : track_latitude +cso.s5p.no2.convert.output.var.track_latitude.from : PRODUCT/latitude +cso.s5p.no2.convert.output.var.track_latitude.attrs : { 'bounds' : None } +!~ corner lons +cso.s5p.no2.convert.output.var.track_longitude_bounds.dims : track_scan track_pixel corner +cso.s5p.no2.convert.output.var.track_longitude_bounds.special : track_longitude_bounds +cso.s5p.no2.convert.output.var.track_longitude_bounds.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds +cso.s5p.no2.convert.output.var.track_longitude_bounds.units : degrees_east +!~ corner lats +cso.s5p.no2.convert.output.var.track_latitude_bounds.dims : track_scan track_pixel corner +cso.s5p.no2.convert.output.var.track_latitude_bounds.special : track_latitude_bounds +cso.s5p.no2.convert.output.var.track_latitude_bounds.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds +cso.s5p.no2.convert.output.var.track_latitude_bounds.units : degrees_north + +! time value per scan line +cso.s5p.no2.convert.output.var.time.dims : pixel +cso.s5p.no2.convert.output.var.time.special : time-delta +cso.s5p.no2.convert.output.var.time.tref : PRODUCT/time +cso.s5p.no2.convert.output.var.time.dt : PRODUCT/delta_time + +! vertical column density: +cso.s5p.no2.convert.output.var.vcd.dims : pixel retr +cso.s5p.no2.convert.output.var.vcd.from : PRODUCT/nitrogendioxide_tropospheric_column +cso.s5p.no2.convert.output.var.vcd.attrs : { 'ancillary_variables' : None } + +! error variance in vertical column density (after application of kernel), +! fill with single element 'covariance matrix', from square of standard error: +! use dims with different names to avoid that cf-checker complains: +cso.s5p.no2.convert.output.var.vcd_errvar.dims : pixel retr retr0 +cso.s5p.no2.convert.output.var.vcd_errvar.special : square +cso.s5p.no2.convert.output.var.vcd_errvar.from : PRODUCT/nitrogendioxide_tropospheric_column_precision_kernel +!~ skip standard name, modifier "standard_error" is not valid anymore: +cso.s5p.no2.convert.output.var.vcd_errvar.attrs : { 'standard_name' : None } + +! Convert from hybride coefficient bounds in (2,nlev) aray to 3D half level pressure: +cso.s5p.no2.convert.output.var.pressure.dims : pixel layeri +cso.s5p.no2.convert.output.var.pressure.special : hybounds_to_pressure +cso.s5p.no2.convert.output.var.pressure.sp : PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure +cso.s5p.no2.convert.output.var.pressure.hyab : PRODUCT/tm5_constant_a +cso.s5p.no2.convert.output.var.pressure.hybb : PRODUCT/tm5_constant_b +cso.s5p.no2.convert.output.var.pressure.units : Pa + +! (total) airmass factor: +cso.s5p.no2.convert.output.var.amf.dims : pixel retr +cso.s5p.no2.convert.output.var.amf.from : PRODUCT/air_mass_factor_total +cso.s5p.no2.convert.output.var.amf.attrs : { 'coordinates' : None, 'ancillary_variables' : None } + +! tropospheric airmass factor: +cso.s5p.no2.convert.output.var.amf_trop.dims : pixel retr +cso.s5p.no2.convert.output.var.amf_trop.from : PRODUCT/air_mass_factor_troposphere +cso.s5p.no2.convert.output.var.amf_trop.attrs : { 'coordinates' : None, 'ancillary_variables' : None } + +! number of apriori layers in retrieval layer, +! enforce that it is stored as a short integer: +cso.s5p.no2.convert.output.var.nla.dims : pixel retr +cso.s5p.no2.convert.output.var.nla.dtype : i2 +cso.s5p.no2.convert.output.var.nla.from : PRODUCT/tm5_tropopause_layer_index +cso.s5p.no2.convert.output.var.nla.attrs : { 'coordinates' : None, 'ancillary_variables' : None } + +! description: +! kernel := averaging_kernel * amf/amf_trop +! for layers up to l_trop, zero above +cso.s5p.no2.convert.output.var.kernel_trop.dims : pixel layer retr +cso.s5p.no2.convert.output.var.kernel_trop.special : kernel_trop +cso.s5p.no2.convert.output.var.kernel_trop.avk : PRODUCT/averaging_kernel +cso.s5p.no2.convert.output.var.kernel_trop.amf : PRODUCT/air_mass_factor_total +cso.s5p.no2.convert.output.var.kernel_trop.amft : PRODUCT/air_mass_factor_troposphere +cso.s5p.no2.convert.output.var.kernel_trop.troplayer : PRODUCT/tm5_tropopause_layer_index +cso.s5p.no2.convert.output.var.kernel_trop.attrs : { 'coordinates' : None, 'ancillary_variables' : None } + +! cloud property: +cso.s5p.no2.convert.output.var.cloud_fraction.dims : pixel +cso.s5p.no2.convert.output.var.cloud_fraction.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/cloud_fraction_crb +cso.s5p.no2.convert.output.var.cloud_fraction.attrs : { 'coordinates' : None, 'source' : None } + +! cloud property: +cso.s5p.no2.convert.output.var.cloud_radiance_fraction.dims : pixel +cso.s5p.no2.convert.output.var.cloud_radiance_fraction.from : PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/cloud_radiance_fraction_nitrogendioxide_window +cso.s5p.no2.convert.output.var.cloud_radiance_fraction.attrs : { 'coordinates' : None, 'ancillary_variables' : None } + +! quality flag: +cso.s5p.no2.convert.output.var.qa_value.dims : pixel +cso.s5p.no2.convert.output.var.qa_value.from : PRODUCT/qa_value +!~ skip some attributes, cf-checker complains ... +cso.s5p.no2.convert.output.var.qa_value.attrs : { 'valid_min' : None, 'valid_max' : None } + +!! description: +!cso.s5p.no2.convert.output.var.cloud_pressure_crb.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/cloud_pressure_crb +!cso.s5p.no2.convert.output.var.cloud_pressure_crb.units : Pa +!cso.s5p.no2.convert.output.var.cloud_pressure_crb.dims : pixel + +!! description: +!cso.s5p.no2.convert.output.var.surface_albedo.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_albedo_nitrogendioxide_window +!cso.s5p.no2.convert.output.var.surface_albedo.units : 1 +!cso.s5p.no2.convert.output.var.surface_albedo.dims : pixel + +!! description: +!cso.s5p.no2.convert.output.var.viewing_zenith_angle.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/viewing_zenith_angle +!cso.s5p.no2.convert.output.var.viewing_zenith_angle.units : degree +!cso.s5p.no2.convert.output.var.viewing_zenith_angle.dims : pixel + +!! description: +!cso.s5p.no2.convert.output.var.viewing_azimuth_angle.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/viewing_azimuth_angle +!cso.s5p.no2.convert.output.var.viewing_azimuth_angle.units : degree +!cso.s5p.no2.convert.output.var.viewing_azimuth_angle.dims : pixel + +!! description: +!cso.s5p.no2.convert.output.var.solar_azimuth_angle.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/solar_azimuth_angle +!cso.s5p.no2.convert.output.var.solar_azimuth_angle.units : degree +!cso.s5p.no2.convert.output.var.solar_azimuth_angle.dims : pixel + +!! description: +!cso.s5p.no2.convert.output.var.solar_zenith_angle.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/solar_zenith_angle +!cso.s5p.no2.convert.output.var.solar_zenith_angle.units : degree +!cso.s5p.no2.convert.output.var.solar_zenith_angle.dims : pixel + +!! description: +!cso.s5p.no2.convert.output.var.snow_ice_flag.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/snow_ice_flag +!cso.s5p.no2.convert.output.var.snow_ice_flag.units : None +!cso.s5p.no2.convert.output.var.snow_ice_flag.dims : pixel + +!! description: +!cso.s5p.no2.convert.output.var.surface_classification.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_classification +!cso.s5p.no2.convert.output.var.surface_classification.units : None +!cso.s5p.no2.convert.output.var.surface_classification.dims : pixel + +!! for traceback: +!cso.s5p.no2.convert.output.var.orbit_number.special : orbit_number +!cso.s5p.no2.convert.output.var.orbit_number.units : 1 +!cso.s5p.no2.convert.output.var.orbit_number.dims : pixel + +!! for traceback: +!cso.s5p.no2.convert.output.var.image_number.special : scan_number +!cso.s5p.no2.convert.output.var.image_number.units : 1 +!cso.s5p.no2.convert.output.var.image_number.dims : pixel + +!! for traceback: +!cso.s5p.no2.convert.output.var.ground_pixel.dims : track_pixel +!cso.s5p.no2.convert.output.var.ground_pixel.special : ground_pixel +!cso.s5p.no2.convert.output.var.ground_pixel.from : PRODUCT/ground_pixel + + + + +!====================================================================== +!=== +!=== listing +!=== +!====================================================================== + +! csv file that will hold records per file with: +! - timerange of pixels in file +! - orbit number +cso.s5p.no2.listing.file : ${my.work}/CSO-data/${my.region}/S5p/NO2/${my.s5p.no2.selection}__listing.csv + +! renew table if file already exists? +cso.s5p.no2.listing.renew : True + +! time range: +cso.s5p.no2.listing.timerange.start : ${my.timerange.start} +cso.s5p.no2.listing.timerange.end : ${my.timerange.end} + +! filename filters relative to listing file that should be scanned for orbit files; +! names could include time templates ; +! if same orbit is found in multiple directories, the first found is used; +! remove existing table for safety to ensure that this is done correctly ... +cso.s5p.no2.listing.patterns : ${my.s5p.no2.selection}/%Y/%m/S5p_*.nc + +! extra columns to be added, read from global attributes: +cso.s5p.no2.listing.xcolumns : orbit + + +!====================================================================== +!=== +!=== catalogue +!=== +!====================================================================== + +! listing file with filenames/timerange. +cso.s5p.no2.catalogue.input.listing : ${cso.s5p.no2.listing.file} + +! time range: +cso.s5p.no2.catalogue.timerange.start : ${my.timerange.start} +cso.s5p.no2.catalogue.timerange.end : ${my.timerange.end} + +! target filenames; templates: +! - time values +! - %{orbit} : from listing +! - %{varname} for variable +cso.s5p.no2.catalogue.output.file : ${my.work}/CSO-data-catalogue/${my.region}/S5p/NO2/${my.s5p.no2.selection}/%Y/%m/%d/S5p_NO2_%{orbit}__%{varname}.png + +! map domain (west east south north) +cso.s5p.no2.catalogue.domain : ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north} + +! figure size (inches), default is (8,6): +cso.s5p.no2.catalogue.figsize : ${my.region.figsize} + +! renew existing files? +!cso.s5p.no2.catalogue.renew : False +cso.s5p.no2.catalogue.renew : True + +! variables to be plotted: +cso.s5p.no2.catalogue.vars : vcd vcd_errvar qa_value \ + amf amf_trop nla \ + cloud_fraction cloud_radiance_fraction + +!! color for no-data values in track, default '0.8' (gray): +!cso.s5p.no2.catalogue.color_nan : white + +!! extra keyword arguments for map: +!cso.s5p.no2.catalogue.map : resolution='h' + +! convert units: +cso.s5p.no2.catalogue.var.vcd.units : umol/m2 +! style: +cso.s5p.no2.catalogue.var.vcd.vmin : 0.0 +cso.s5p.no2.catalogue.var.vcd.vmax : 100.0 + +! show error as std.dev, convert to vcd units: +cso.s5p.no2.catalogue.var.vcd_errvar.units : umol/m2 +! style: +cso.s5p.no2.catalogue.var.vcd_errvar.vmax : 100.0 + +! style: +cso.s5p.no2.catalogue.var.amf.vmin : 0.0 +cso.s5p.no2.catalogue.var.amf.vmax : 4.0 + +! style: +cso.s5p.no2.catalogue.var.amf_trop.vmin : 0.0 +cso.s5p.no2.catalogue.var.amf_trop.vmax : 4.0 + +! style: +cso.s5p.no2.catalogue.var.nla.vmin : 1 +cso.s5p.no2.catalogue.var.nla.vmax : 34 + +! style: +cso.s5p.no2.catalogue.var.qa_value.vmin : 0.5 +cso.s5p.no2.catalogue.var.qa_value.vmax : 1.0 +cso.s5p.no2.catalogue.var.qa_value.colors : ['red','yellow','green'] + +! style: +cso.s5p.no2.catalogue.var.cloud_fraction.vmax : 1.0 +cso.s5p.no2.catalogue.var.cloud_fraction.colors : ['blue','cyan','white'] + +! style: +cso.s5p.no2.catalogue.var.cloud_radiance_fraction.vmax : 1.0 +cso.s5p.no2.catalogue.var.cloud_radiance_fraction.colors : ['blue','cyan','white'] + + +!----------------------------------------------------------- +! catalogue index +!----------------------------------------------------------- + +! target location: +cso.s5p.no2.catalogue-index.outdir : ${my.work}/CSO-data-catalogue/${my.region}/S5p/NO2/${my.s5p.no2.selection} + +! title: +cso.s5p.no2.catalogue-index.header : CSO catalogue +! show info on created page? +cso.s5p.no2.catalogue-index.info : True + +! create new page for each value? +cso.s5p.no2.catalogue-index.newpage : True + +! content type: +cso.s5p.no2.catalogue-index.type : list +! define row values: +cso.s5p.no2.catalogue-index.name : date +cso.s5p.no2.catalogue-index.values : TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' ) + +! create new page for each value: +cso.s5p.no2.catalogue-index.date.newpage : True +! content type: +cso.s5p.no2.catalogue-index.date.type : table-row +! define row values: +cso.s5p.no2.catalogue-index.date.name : orbit +cso.s5p.no2.catalogue-index.date.values : CsvFile( '%{date[0:4]}/%{date[4:6]}/%{date[6:8]}/orbits.csv' ) + +! content type: +cso.s5p.no2.catalogue-index.date.orbit.type : table-col +! define column values: +cso.s5p.no2.catalogue-index.date.orbit.name : var +cso.s5p.no2.catalogue-index.date.orbit.values : ${cso.s5p.no2.catalogue.vars} + +! content type: +cso.s5p.no2.catalogue-index.date.orbit.var.type : img +! define image: +cso.s5p.no2.catalogue-index.date.orbit.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{orbit}__%{var}.png +cso.s5p.no2.catalogue-index.date.orbit.var.kwargs : height=300 + + + +!====================================================================== +!=== +!=== gridded orbits +!=== +!====================================================================== + + +!----------------------------------------------------------- +! gridded orbits +!----------------------------------------------------------- + +! time range: +cso.s5p.no2.gridded.timerange.start : ${my.timerange.start} +cso.s5p.no2.gridded.timerange.end : ${my.timerange.end} +! create one gridded file per hour: +cso.s5p.no2.gridded.timerange.step : hour + +! renew existing files? +cso.s5p.no2.gridded.renew : True + +! target directory, incl. subdir for resolution and filters: +my.gridded.dir : ${my.work}/CSO-gridded/${my.region}__r01x01/S5p/NO2/${my.s5p.no2.selection}__${my.s5p.no2.gridded-selection} + +!~ + +! grid definition: +!~ same as pixel selection on conversion: +cso.s5p.no2.gridded.grid.west : ${my.region.west} +cso.s5p.no2.gridded.grid.east : ${my.region.east} +cso.s5p.no2.gridded.grid.south : ${my.region.south} +cso.s5p.no2.gridded.grid.north : ${my.region.north} +! resolution: +cso.s5p.no2.gridded.grid.dlon : 0.1 +cso.s5p.no2.gridded.grid.dlat : 0.1 + +! level of recursive splitting of footprint into triangles, +! and assignment of centroids to grid cells; +! for 4-corner footprints, number of centroids is: +! 1 (levels=0), 4 (1), 8 (2), 16 (3), 64 (5), 256 (7) +cso.s5p.no2.gridded.mapping.levels : 5 + +!~ + +! keywords for source files; +! the first one should have the footprints; +! here the only source are the converted files: +cso.s5p.no2.gridded.sources : data + +! input files for each source type: +!~ here: specify listing file, +! this is only supported for a single source +cso.s5p.no2.gridded.source.data.listing : ${cso.s5p.no2.listing.file} +!!~ alternative: filename patterns with time templates, +!! but here the source files do not have time stamps +!cso.s5p.no2.gridded.source.data.filenames : ${my.work}/CSO-data/S5p/RPRO/NO2/${my.region}/%Y/%m/S5p_*.nc + +!~ + +! filter description: +my.s5p.no2.gridded-selection : qa08 + +! switch: +#if "${my.s5p.no2.gridded-selection}" == "qa08" + +! keywords for filters: +cso.s5p.no2.gridded.filters : quality + +! minimum quality value required: +cso.s5p.no2.gridded.filter.quality.var : qa_value +cso.s5p.no2.gridded.filter.quality.type : min +cso.s5p.no2.gridded.filter.quality.min : 0.8 +cso.s5p.no2.gridded.filter.quality.units : 1 + +#else +#error unsupported my.s5p.no2.gridded-selection "${my.s5p.no2.gridded-selection}" +#endif + + +!~ + +! target file, might contain templates: +! %Y,%m,etc : time values +cso.s5p.no2.gridded.output.file : ${my.gridded.dir}/%Y/%m/S5p_NO2_%Y%m%d_%H%M_gridded.nc + +! pack floats as short values? +cso.s5p.no2.gridded.output.packed : True +! zlib compression level (default 1, 0 for no compression): +cso.s5p.no2.gridded.output.complevel : 1 + +! data variables to be created: +cso.s5p.no2.gridded.output.vars : yr + +! input variables: +! data:vcd : variable "vcd" from source "data" +cso.s5p.no2.gridded.output.yr.source : data:vcd + + +!----------------------------------------------------------- +! catalogue of gridded fields +!----------------------------------------------------------- + +! time range: +cso.s5p.no2.gridded-catalogue.timerange.start : ${my.timerange.start} +cso.s5p.no2.gridded-catalogue.timerange.end : ${my.timerange.end} +! create one gridded file per hour: +cso.s5p.no2.gridded-catalogue.timerange.step : hour + +! renew existing files? +cso.s5p.no2.gridded-catalogue.renew : True +!cso.s5p.no2.gridded-catalogue.renew : False + +! target directory for catalogue: +my.no2.gridded-catalogue.output.dir : ${my.gridded.dir}/catalogue + +! input files: +cso.s5p.no2.gridded-catalogue.input.file : ${my.gridded.dir}/%Y/%m/S5p_NO2_%Y%m%d_%H%M_gridded.nc +!!~ idem for daily average: +!cso.s5p.no2.gridded-catalogue.input.file : ${my.gridded.dir}/S5p_NO2_%Y%m%d_aver_gridded.nc + +! target files, time tempates and variable name are replaced: +cso.s5p.no2.gridded-catalogue.output.file : ${my.no2.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_gridded_%{var}.png +!!~ idem for daily average: +!cso.s5p.no2.gridded-catalogue.output.file : ${my.no2.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_NO2_%Y%m%d_aver_gridded_%{var}.png + +! figure size (inches), default is (8,6): +cso.s5p.no2.gridded-catalogue.figsize : (6,6) + +! variables to be plotted: +cso.s5p.no2.gridded-catalogue.vars : yr + +! variable: +cso.s5p.no2.gridded-catalogue.var.yr.source : yr +! convert units: +cso.s5p.no2.gridded-catalogue.var.yr.units : umol/m2 +! style: +cso.s5p.no2.gridded-catalogue.var.yr.vmin : 0.0 +cso.s5p.no2.gridded-catalogue.var.yr.vmax : 100.0 + + + + +!----------------------------------------------------------- +! gridded catalogue index +!----------------------------------------------------------- + +! target location: +cso.s5p.no2.gridded-catalogue-index.outdir : ${my.no2.gridded-catalogue.output.dir} + +! title: +cso.s5p.no2.gridded-catalogue-index.header : CSO catalogue +! show info on created page? +cso.s5p.no2.gridded-catalogue-index.info : True + +! create new page for each value? +cso.s5p.no2.gridded-catalogue-index.newpage : True + +! content type: +cso.s5p.no2.gridded-catalogue-index.type : list +! define row values: +cso.s5p.no2.gridded-catalogue-index.name : date +cso.s5p.no2.gridded-catalogue-index.values : TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' ) + +! create new page for each value: +cso.s5p.no2.gridded-catalogue-index.date.newpage : True +! content type: +cso.s5p.no2.gridded-catalogue-index.date.type : table-row +! define row values: +cso.s5p.no2.gridded-catalogue-index.date.name : time +cso.s5p.no2.gridded-catalogue-index.date.values : Range( 0, 23, 1, '%2.2i00' ) + +! content type: +cso.s5p.no2.gridded-catalogue-index.date.time.type : table-col +! define column values: +cso.s5p.no2.gridded-catalogue-index.date.time.name : var +cso.s5p.no2.gridded-catalogue-index.date.time.values : ${cso.s5p.no2.gridded-catalogue.vars} + +! content type: +cso.s5p.no2.gridded-catalogue-index.date.time.var.type : img +! define image: +cso.s5p.no2.gridded-catalogue-index.date.time.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_gridded_%{var}.png +cso.s5p.no2.gridded-catalogue-index.date.time.var.kwargs : height=300 + + + +!====================================================================== +!=== +!=== simulation catalogue +!=== +!====================================================================== + + +!----------------------------------------------------------- +! simulation catalogue +!----------------------------------------------------------- + +! time range: +cso.s5p.no2.sim-catalogue.timerange.start : ${my.timerange.start} +cso.s5p.no2.sim-catalogue.timerange.end : ${my.timerange.end} +cso.s5p.no2.sim-catalogue.timerange.step : hour + +! input files: +cso.s5p.no2.sim-catalogue.input.data.file : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc +cso.s5p.no2.sim-catalogue.input.state.file : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc + +! target files, time tempates and variable name are replaced: +cso.s5p.no2.sim-catalogue.output.file : ${my.work}/CSO-data-sim-catalogue/S5p/NO2/${my.region}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_%{var}.png + +! map domain used for simulations (west east south north): +cso.s5p.no2.sim-catalogue.domain : ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north} + +! figure size (inches), default is (8,6): +cso.s5p.no2.sim-catalogue.figsize : ${my.region.figsize} + +! renew existing files? +cso.s5p.no2.sim-catalogue.renew : False +!cso.s5p.no2.sim-catalogue.renew : True + +! variables to be plotted: +cso.s5p.no2.sim-catalogue.vars : yr ys yr_m ys_m + +! variable: +cso.s5p.no2.sim-catalogue.var.yr.source : data:yr +! convert units: +cso.s5p.no2.sim-catalogue.var.yr.units : umol/m2 +! style: +cso.s5p.no2.sim-catalogue.var.yr.vmin : 0.0 +cso.s5p.no2.sim-catalogue.var.yr.vmax : 100.0 + +! variable: +cso.s5p.no2.sim-catalogue.var.ys.source : state:ys +! convert units: +cso.s5p.no2.sim-catalogue.var.ys.units : umol/m2 +! style: +cso.s5p.no2.sim-catalogue.var.ys.vmin : 0.0 +cso.s5p.no2.sim-catalogue.var.ys.vmax : 100.0 + +! variable: +cso.s5p.no2.sim-catalogue.var.yr_m.source : state:yr_m +! convert units: +cso.s5p.no2.sim-catalogue.var.yr_m.units : umol/m2 +! style: +cso.s5p.no2.sim-catalogue.var.yr_m.vmin : 0.0 +cso.s5p.no2.sim-catalogue.var.yr_m.vmax : 100.0 + +! variable: +cso.s5p.no2.sim-catalogue.var.ys_m.source : state:ys_m +! convert units: +cso.s5p.no2.sim-catalogue.var.ys_m.units : umol/m2 +! style: +cso.s5p.no2.sim-catalogue.var.ys_m.vmin : 0.0 +cso.s5p.no2.sim-catalogue.var.ys_m.vmax : 100.0 + + + +!----------------------------------------------------------- +! sim catalogue index +!----------------------------------------------------------- + +! target location: +cso.s5p.no2.sim-catalogue-index.outdir : ${my.work}/CSO-sim-catalogue/S5p/NO2/${my.region} + +! title: +cso.s5p.no2.sim-catalogue-index.header : CSO catalogue +! show info on created page? +cso.s5p.no2.sim-catalogue-index.info : True + +! create new page for each value? +cso.s5p.no2.sim-catalogue-index.newpage : True + +! content type: +cso.s5p.no2.sim-catalogue-index.type : list +! define row values: +cso.s5p.no2.sim-catalogue-index.name : date +cso.s5p.no2.sim-catalogue-index.values : TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' ) + +! create new page for each value: +cso.s5p.no2.sim-catalogue-index.date.newpage : True +! content type: +cso.s5p.no2.sim-catalogue-index.date.type : table-row +! define row values: +cso.s5p.no2.sim-catalogue-index.date.name : time +cso.s5p.no2.sim-catalogue-index.date.values : Range( 0, 23, 1, '%2.2i00' ) + +! content type: +cso.s5p.no2.sim-catalogue-index.date.time.type : table-col +! define column values: +cso.s5p.no2.sim-catalogue-index.date.time.name : var +cso.s5p.no2.sim-catalogue-index.date.time.values : ${cso.s5p.no2.sim-catalogue.vars} + +! content type: +cso.s5p.no2.sim-catalogue-index.date.time.var.type : img +! define image: +cso.s5p.no2.sim-catalogue-index.date.time.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_%{var}.png +cso.s5p.no2.sim-catalogue-index.date.time.var.kwargs : height=300 + + + +!====================================================================== +!=== +!=== gridded simulated orbits +!=== +!====================================================================== + + +!----------------------------------------------------------- +! gridded orbits +!----------------------------------------------------------- + +! time range: +cso.s5p.no2.sim-gridded.timerange.start : ${my.timerange.start} +cso.s5p.no2.sim-gridded.timerange.end : ${my.timerange.end} +! create one gridded file per hour: +cso.s5p.no2.sim-gridded.timerange.step : hour + +! renew existing files? +cso.s5p.no2.sim-gridded.renew : True +!cso.s5p.no2.sim-gridded.renew : False + +! target directory, incl. subdir for resolution and filters: +my.sim-gridded.dir : ${my.work}/CSO-sim-gridded/${my.region}__r01x01__qa08 + +!~ + +! testing .. +my.gridded.region : ${my.region} +! grid definition: +!!~ same as pixel selection on conversion: +!cso.s5p.no2.sim-gridded.grid.west : ${my.region.west} +!cso.s5p.no2.sim-gridded.grid.east : ${my.region.east} +!cso.s5p.no2.sim-gridded.grid.south : ${my.region.south} +!cso.s5p.no2.sim-gridded.grid.north : ${my.region.north} +!~ observation operator tutorial: +cso.s5p.no2.sim-gridded.grid.west : -10 +cso.s5p.no2.sim-gridded.grid.east : 30 +cso.s5p.no2.sim-gridded.grid.south : 35 +cso.s5p.no2.sim-gridded.grid.north : 65 +! resolution: +cso.s5p.no2.sim-gridded.grid.dlon : 0.1 +cso.s5p.no2.sim-gridded.grid.dlat : 0.1 + +! level of recursive splitting of footprint into triangles, +! and assignment of centroids to grid cells; +! for 4-corner footprints, number of centroids is: +! 1 (levels=0), 4 (1), 8 (2), 16 (3), 64 (5), 256 (7) +cso.s5p.no2.sim-gridded.mapping.levels : 5 + +!~ + +! keywords for source files; +! the first one should have the footprints: +cso.s5p.no2.sim-gridded.sources : data state + +! input files for each source type: +cso.s5p.no2.sim-gridded.source.data.filenames : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc +cso.s5p.no2.sim-gridded.source.state.filenames : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc + +!~ + +! keywords for filters: +cso.s5p.no2.sim-gridded.filters : quality + +! minimum quality value required: +cso.s5p.no2.sim-gridded.filter.quality.var : qa_value +cso.s5p.no2.sim-gridded.filter.quality.type : min +cso.s5p.no2.sim-gridded.filter.quality.min : 0.8 +cso.s5p.no2.sim-gridded.filter.quality.units : 1 + +!~ + +! target file, might contain templates: +! %Y,%m,etc : time values +! %{basename} : basename (without extension) of first source file +cso.s5p.no2.sim-gridded.output.file : ${my.sim-gridded.dir}/%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc + +! pack variables on output: +cso.s5p.no2.sim-gridded.output.packed : True +! zlib compression level, 0 for no compression: +cso.s5p.no2.sim-gridded.output.complevel : 1 + +! data variables to be created: +cso.s5p.no2.sim-gridded.output.vars : yr ys yr_m ys_m + +! input variables: +! data:yr : from data file +! state:ys : from state file +cso.s5p.no2.sim-gridded.output.yr.source : data:yr +cso.s5p.no2.sim-gridded.output.ys.source : state:ys +cso.s5p.no2.sim-gridded.output.yr_m.source : state:yr_m +cso.s5p.no2.sim-gridded.output.ys_m.source : state:ys_m + + +!----------------------------------------------------------- +! catalogue of gridded simulations +!----------------------------------------------------------- + +! time range: +cso.s5p.no2.sim-gridded-catalogue.timerange.start : ${my.timerange.start} +cso.s5p.no2.sim-gridded-catalogue.timerange.end : ${my.timerange.end} +! hourly fields: +cso.s5p.no2.sim-gridded-catalogue.timerange.step : hour + +! renew existing files? +cso.s5p.no2.sim-gridded-catalogue.renew : True +!cso.s5p.no2.sim-gridded-catalogue.renew : False + +! input files: +cso.s5p.no2.sim-gridded-catalogue.input.file : ${my.sim-gridded.dir}/%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc +!!~ idem for daily average: +!cso.s5p.no2.sim-gridded-catalogue.input.file : ${my.sim-gridded.dir}/CSO_output_%Y%m%d_aver_gridded.nc + +! target files, time tempates and variable name are replaced: +cso.s5p.no2.sim-gridded-catalogue.output.file : ${my.sim-gridded.dir}/catalogue/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_gridded_%{var}.png +!!~ idem for daily average: +!cso.s5p.no2.sim-gridded-catalogue.output.file : ${my.sim-gridded.dir}/catalogue/%Y/%m/%d/S5p_NO2_%Y%m%d_aver_gridded_%{var}.png + +! figure size (inches), default is (8,6): +cso.s5p.no2.sim-gridded-catalogue.figsize : (6,6) + +! variables to be plotted: +cso.s5p.no2.sim-gridded-catalogue.vars : yr ys yr_m ys_m + +! variable: +cso.s5p.no2.sim-gridded-catalogue.var.yr.source : yr +! convert units: +cso.s5p.no2.sim-gridded-catalogue.var.yr.units : umol/m2 +! style: +cso.s5p.no2.sim-gridded-catalogue.var.yr.vmin : 0.0 +cso.s5p.no2.sim-gridded-catalogue.var.yr.vmax : 100.0 + +! variable: +cso.s5p.no2.sim-gridded-catalogue.var.ys.source : ys +! convert units: +cso.s5p.no2.sim-gridded-catalogue.var.ys.units : umol/m2 +! style: +cso.s5p.no2.sim-gridded-catalogue.var.ys.vmin : 0.0 +cso.s5p.no2.sim-gridded-catalogue.var.ys.vmax : 100.0 + +! variable: +cso.s5p.no2.sim-gridded-catalogue.var.yr_m.source : yr_m +! convert units: +cso.s5p.no2.sim-gridded-catalogue.var.yr_m.units : umol/m2 +! style: +cso.s5p.no2.sim-gridded-catalogue.var.yr_m.vmin : 0.0 +cso.s5p.no2.sim-gridded-catalogue.var.yr_m.vmax : 100.0 + +! variable: +cso.s5p.no2.sim-gridded-catalogue.var.ys_m.source : ys_m +! convert units: +cso.s5p.no2.sim-gridded-catalogue.var.ys_m.units : umol/m2 +! style: +cso.s5p.no2.sim-gridded-catalogue.var.ys_m.vmin : 0.0 +cso.s5p.no2.sim-gridded-catalogue.var.ys_m.vmax : 100.0 + + + +!----------------------------------------------------------- +! gridded catalogue index +!----------------------------------------------------------- + +! target location: +cso.s5p.no2.sim-gridded-catalogue-index.outdir : ${my.sim-gridded.dir}/catalogue + +! title: +cso.s5p.no2.sim-gridded-catalogue-index.header : CSO catalogue +! show info on created page? +cso.s5p.no2.sim-gridded-catalogue-index.info : True + +! create new page for each value? +cso.s5p.no2.sim-gridded-catalogue-index.newpage : True + +! content type: +cso.s5p.no2.sim-gridded-catalogue-index.type : list +! define row values: +cso.s5p.no2.sim-gridded-catalogue-index.name : date +cso.s5p.no2.sim-gridded-catalogue-index.values : TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' ) + +! create new page for each value: +cso.s5p.no2.sim-gridded-catalogue-index.date.newpage : True +! content type: +cso.s5p.no2.sim-gridded-catalogue-index.date.type : table-row +! define row values: +cso.s5p.no2.sim-gridded-catalogue-index.date.name : time +cso.s5p.no2.sim-gridded-catalogue-index.date.values : Range( 0, 23, 1, '%2.2i00' ) + +! content type: +cso.s5p.no2.sim-gridded-catalogue-index.date.time.type : table-col +! define column values: +cso.s5p.no2.sim-gridded-catalogue-index.date.time.name : var +cso.s5p.no2.sim-gridded-catalogue-index.date.time.values : ${cso.s5p.no2.sim-gridded-catalogue.vars} + +! content type: +cso.s5p.no2.sim-gridded-catalogue-index.date.time.var.type : img +! define image: +cso.s5p.no2.sim-gridded-catalogue-index.date.time.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_gridded_%{var}.png +cso.s5p.no2.sim-gridded-catalogue-index.date.time.var.kwargs : height=300 + + +!====================================================================== +!=== +!=== end +!=== +!====================================================================== + + diff --git a/config/_test-colhub/cso-user-settings.rc b/config/_test-colhub/cso-user-settings.rc new file mode 100644 index 0000000..10bc13f --- /dev/null +++ b/config/_test-colhub/cso-user-settings.rc @@ -0,0 +1,169 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO common configuration +!!! +!!! Base settings that are used by multiple tasks: +!!! - time range(s) +!!! - target domain(s) +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + +!----------------------------------------------------------- +! id's +!----------------------------------------------------------- + +! file format: +my.cso.format : 1.0 + +! file format convention: +my.cso.convention : CF-1.7 + + +!----------------------------------------------------------- +! domain +!----------------------------------------------------------- + +! +! Used for: +! - orbit selection durning download +! - pixel selection +! - map plots +! + +! region name: +!my.region : globe +!my.region : CAMS +my.region : xEMEP + +! switch: +!.................................... +#if "${my.region}" == "globe" +!.................................... + +! global domain: +my.region.west : -180.0 +my.region.east : 180.0 +my.region.south : -90.0 +my.region.north : 90.0 + +! size of map figures for this region, default size is (8,6) +my.region.figsize : (8,6) + +!.................................... +#elif "${my.region}" == "CAMS" +!.................................... + +! CAMS regional ensemble domain: +my.region.west : -30.0 +my.region.east : 45.0 +my.region.south : 30.0 +my.region.north : 76.0 + +! size of map figures for this region, default size is (8,6) +my.region.figsize : (8,6) + +!.................................... +#elif "${my.region}" == "xEMEP" +!.................................... + +! outer bound of all known domains: +my.region.west : -50.0 +my.region.east : 90.0 +my.region.south : 25.0 +my.region.north : 85.0 + +! size of map figures for this region, default size is (8,6) +my.region.figsize : (8,5) + +!.................................... +#else +#error unsupported my.region "${my.region}" +#endif +!.................................... + + + + +!---------------------------------------------------------- +! timerange +!---------------------------------------------------------- + +!! full range, used for inquiry jobs: +!my.full-timerange.start : 2018-01-01 00:00 +!my.full-timerange.end : 2024-12-31 23:59 + +! TESTING: 1 month ... +my.full-timerange.start : 2023-01-01 00:00 +my.full-timerange.end : 2023-01-31 23:59 + +! processing for selected period: +my.timerange.start : 2018-06-01 00:00 +my.timerange.end : 2018-06-01 23:59 + +!! SO2-COBRA +!my.timerange.start : 2023-06-01 00:00 +!my.timerange.end : 2023-06-01 23:59 + + +!---------------------------------------------------------- +! user specific settings: +!---------------------------------------------------------- + +!.............................. +#if "${USER}" == "you" +!.............................. + +! Attributes written to output files. +my.attr.author : Your Name +my.attr.institution : CSO +my.attr.email : Your.Name@cso.org + +! base location for work directories: +my.work : /Scratch/${USER}/CSO-Copernicus + +! storage for downloaded observations: +my.observations : /Scratch/${USER}/observations + +!.............................. +#elif "${USER}" == "arjos" +!.............................. + +! Attributes written to output files. +my.attr.author : Arjo Segers +my.attr.institution : MET Norway +my.attr.email : arjos@met.no + +! base location for work directories: +my.work : ${WORK}/projects/SESAM/test-colhub + +! storage for downloaded observations: +my.observations : ${WORK}/../observations + +!.............................. +#elif "${USER}" == "segersaj" +!.............................. + +! Attributes written to output files. +my.attr.author : Arjo Segers +my.attr.institution : TNO +my.attr.email : Arjo.Segers@tno.nl + +! base location for work directories: +my.work : ${SCRATCH}/CSO-Copernicus + +! storage for downloaded observations: +my.observations : ${SCRATCH}/observations + +!.............................. +#else +#error unsupported USER "${USER}" +#endif +!.............................. + + +!---------------------------------------------------------- +! end +!---------------------------------------------------------- + + diff --git a/config/_test-colhub/cso.rc b/config/_test-colhub/cso.rc new file mode 100644 index 0000000..3de54af --- /dev/null +++ b/config/_test-colhub/cso.rc @@ -0,0 +1,311 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!! +!!! CSO - CAMS Satellite Operator +!!! +!!! Settings for project. +!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +! dummy keys, actual values defined in environment by "cso" script: +CSO_RCFILE : +CSO_RCDIR : + + +!---------------------------------------------------------- +! user specific settings: +!---------------------------------------------------------- + +! include user specfic settings: +#include cso-user-settings.rc + +! selected tracers: +!my.tracers : no2 so2 so2-cobra hcho co chocho +!~ one by one .. +my.tracers : no2 +!my.tracers : so2 +!my.tracers : so2-cobra +!my.tracers : hcho +!my.tracers : co +!my.tracers : o3-col +!my.tracers : o3-pr +!my.tracers : chocho + + +!---------------------------------------------------------- +! job tree +!---------------------------------------------------------- + +! class to create a job tree: +cso.class : utopya.UtopyaJobTree +! list of sub-elements: +cso.elements : copy s5p + +! class to create a job tree: +cso.s5p.class : utopya.UtopyaJobTree +! list of sub-elements: +cso.s5p.elements : ${my.tracers} + + +! * + +#for TRACER in ${my.tracers} + +! class to create a job tree: +cso.s5p.TRACER.class : utopya.UtopyaJobTree + +! list of sub-elements: +!cso.s5p.TRACER.elements : inquire \ +! convert listing \ +! catalogue \ +! gridded gridded-catalogue +!~ preprocessor steps one by one ... +cso.s5p.TRACER.elements : inquire +!cso.s5p.TRACER.elements : convert listing catalogue +!cso.s5p.TRACER.elements : convert +!cso.s5p.TRACER.elements : listing +!cso.s5p.TRACER.elements : catalogue +!cso.s5p.TRACER.elements : gridded +!cso.s5p.TRACER.elements : gridded-catalogue + +!!~ downloading only, no convert: +!cso.s5p.TRACER.elements : download +! +!~ process simulator output: +!cso.s5p.TRACER.elements : sim-catalogue +!cso.s5p.TRACER.elements : sim-gridded +!cso.s5p.TRACER.elements : sim-gridded-catalogue + +! * + +! single step: +cso.s5p.TRACER.inquire.class : utopya.UtopyaJobStep +! two or more tasks: +#if "TRACER" in ["no2","so2","hcho","co","o3-pr","o3-col"] +!cso.s5p.TRACER.inquire.tasks : table-dataspace plot +cso.s5p.TRACER.inquire.tasks : table-dataspace +!cso.s5p.TRACER.inquire.tasks : table-mirror +!cso.s5p.TRACER.inquire.tasks : table-missing +!cso.s5p.TRACER.inquire.tasks : plot +#elif "TRACER" in ["so2-cobra"] +cso.s5p.TRACER.inquire.tasks : table-pal plot +#elif "TRACER" in ["chocho"] +cso.s5p.TRACER.inquire.tasks : table-glyretro table-pal plot +#else +#error unsupported tracer "TRACER" +#endif +!~ inquire files available on DataSpace: +cso.s5p.TRACER.inquire.table-dataspace.class : cso.CSO_DataSpace_Inquire +cso.s5p.TRACER.inquire.table-dataspace.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-table-dataspace' +!~ inquire files available on ColHub: +cso.s5p.TRACER.inquire.table-colhub.class : cso.CSO_ColHub_Inquire +cso.s5p.TRACER.inquire.table-colhub.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-table-colhub' +!~ inquire file archive: +cso.s5p.TRACER.inquire.table-mirror.class : cso.CSO_ColHubMirror_Inquire +cso.s5p.TRACER.inquire.table-mirror.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-table-mirror' +!~ create table with files that are missing: +cso.s5p.TRACER.inquire.table-missing.class : cso.CSO_ColHubMirror_Missing +cso.s5p.TRACER.inquire.table-missing.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-table-missing' +!~ inquire files available on PAL: +cso.s5p.TRACER.inquire.table-pal.class : cso.CSO_PAL_Inquire +cso.s5p.TRACER.inquire.table-pal.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-table-pal' +!~ inquire files downloaded from GlyRetro: +cso.s5p.TRACER.inquire.table-glyretro.class : cso.CSO_S5p_Download_Listing +cso.s5p.TRACER.inquire.table-glyretro.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-table-glyretro' +!~ create plot of available versions: +cso.s5p.TRACER.inquire.plot.class : cso.CSO_Inquire_Plot +cso.s5p.TRACER.inquire.plot.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-plot' + +! * + +! single step: +cso.s5p.TRACER.download.class : utopya.UtopyaJobStep +! convert task: +cso.s5p.TRACER.download.task.class : cso.CSO_S5p_Download +cso.s5p.TRACER.download.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.download' +! single step: +cso.s5p.TRACER.convert.class : utopya.UtopyaJobStep +! convert task: +cso.s5p.TRACER.convert.task.class : cso.CSO_S5p_Convert +cso.s5p.TRACER.convert.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.convert' +! single step: +cso.s5p.TRACER.listing.class : utopya.UtopyaJobStep +! listing task: +cso.s5p.TRACER.listing.task.class : cso.CSO_S5p_Listing +cso.s5p.TRACER.listing.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.listing' + +! single step: +cso.s5p.TRACER.catalogue.class : utopya.UtopyaJobStep +! two tasks: +cso.s5p.TRACER.catalogue.tasks : figs index + +! catalogue creation task: +cso.s5p.TRACER.catalogue.figs.class : cso.CSO_Catalogue +cso.s5p.TRACER.catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.catalogue' +! indexer task: +cso.s5p.TRACER.catalogue.index.class : utopya.Indexer +cso.s5p.TRACER.catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.catalogue-index' + +! * + +! single step: +cso.s5p.TRACER.gridded.class : utopya.UtopyaJobStep +! catalogue creation task: +cso.s5p.TRACER.gridded.task.class : cso.CSO_GriddedAverage +cso.s5p.TRACER.gridded.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.gridded' + +! single step: +cso.s5p.TRACER.gridded-catalogue.class : utopya.UtopyaJobStep +! two tasks: +cso.s5p.TRACER.gridded-catalogue.tasks : figs index +! catalogue creation task: +cso.s5p.TRACER.gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue +cso.s5p.TRACER.gridded-catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.gridded-catalogue' +! indexer task: +cso.s5p.TRACER.gridded-catalogue.index.class : utopya.Indexer +cso.s5p.TRACER.gridded-catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.gridded-catalogue-index' + +! * + +! single step: +cso.s5p.TRACER.sim-catalogue.class : utopya.UtopyaJobStep +! two tasks: +cso.s5p.TRACER.sim-catalogue.tasks : figs index + +! catalogue creation task: +cso.s5p.TRACER.sim-catalogue.figs.class : cso.CSO_SimCatalogue +cso.s5p.TRACER.sim-catalogue.figs.args : , \ + rcbase='cso.s5p.TRACER.sim-catalogue' +! indexer task: +cso.s5p.TRACER.sim-catalogue.index.class : utopya.Indexer +cso.s5p.TRACER.sim-catalogue.index.args : , \ + rcbase='cso.s5p.TRACER.sim-catalogue-index' + +! +! * +! + +! single step: +cso.s5p.TRACER.sim-gridded.class : utopya.UtopyaJobStep +! catalogue creation task: +cso.s5p.TRACER.sim-gridded.task.class : cso.CSO_GriddedAverage +cso.s5p.TRACER.sim-gridded.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.sim-gridded' + +! single step: +cso.s5p.TRACER.sim-gridded-catalogue.class : utopya.UtopyaJobStep +! two tasks: +cso.s5p.TRACER.sim-gridded-catalogue.tasks : figs index +! catalogue creation task: +cso.s5p.TRACER.sim-gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue +cso.s5p.TRACER.sim-gridded-catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.sim-gridded-catalogue' +! indexer task: +cso.s5p.TRACER.sim-gridded-catalogue.index.class : utopya.Indexer +cso.s5p.TRACER.sim-gridded-catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.sim-gridded-catalogue-index' + +#endfor + + +!---------------------------------------------------------- +! job step defaults +!---------------------------------------------------------- + +! run jobs in foreground: +*.script.class : utopya.UtopyaJobScriptForeground + +! search path for python modules: +*.pypath : ${my.work}/py + +! work directory for jobs; +! here path including subdirectories for job name elements: +*.workdir : ${my.work}/__NAME2PATH__ +!! TESTING ... +!*.workdir : ${my.work}/test-difflist-__NAME2PATH__ + +! jobtree settings from this file: +*.rcfile : ${my.work}/rc/cso.rc + +! script interpreter: +*.shell : /usr/bin/env python3 + + +!---------------------------------------------------------- +! copy +! - create run directory +! - copy scripts and settings for jobtree +!---------------------------------------------------------- + +! no sub list: +cso.copy.class : utopya.UtopyaJobStep + +! default class is defined in machine specific settings; +! copy is always done in foreground to avoid that files are +! changed before the job is started: +cso.copy.script.class : utopya.UtopyaJobScriptForeground + +! utopya jobstep settings: this file, +! this task will copy it to the workdir: +cso.copy.rcfile : ${CSO_RCFILE} + +! search path to utopya modules; +! the 'build.utopya' job will create a copy on the workdir +! for which the path is included in the '*.pypath' setting: +cso.copy.pypath : ${CSO_PREFIX}/py + +! configure and build: +cso.copy.task.class : utopya.UtopyaCopy +cso.copy.task.args : '${CSO_RCFILE}', rcbase='main' + +! ~ + +! prefix for destination of source and script files +! (base path for subdirectories src, py, etc) ; +! this should be an absolute path, +! use ${PWD} for the present dir if necessary: +main.copy.prefix : ${my.work} + +! space separated extensions: +main.copy.prefix.extensions : + +! remove existing build directory if present ? +main.copy.new : False + +! sub directories to be included ; +! leave empty if no subdirs are defined: +main.copy.subdirs : py rc + +! directories to be inlcuded in copy, +! otherwise only files are copied: +main.copy.incdirs : + +! list of source directories to be copied; +! files in latest directory overwrite previously copied versions: +main.copy.dirs : ${CSO_PREFIX} + +! extra directories to be copied into "rc": +main.copy.rc.dirs : ${CSO_RCDIR} + +! skip files matching these filename patterns +! (tempoary editor files, compiled modules, etc) +main.copy.skip : .#* *~ .DS* *.pyc + + +!---------------------------------------------------------- +! end +!---------------------------------------------------------- diff --git a/doc/source/history.rst b/doc/source/history.rst index 5b2a9e2..69cc8c6 100644 --- a/doc/source/history.rst +++ b/doc/source/history.rst @@ -109,6 +109,9 @@ A summary of the versions and changes. * | *v2.9* | Initial support for S5P O3-profile product. + +* | *colhub branch* + | Support inquiry of ColHub mirror archive. To be included diff --git a/py/cso.py b/py/cso.py index a6350ea..b7f572b 100644 --- a/py/cso.py +++ b/py/cso.py @@ -109,6 +109,7 @@ and are defined according to the following hierchy: from cso_file import * from cso_inquire import * from cso_dataspace import * +from cso_colhub import * from cso_pal import * from cso_s5p import * from cso_gridded import * diff --git a/py/cso_colhub.py b/py/cso_colhub.py new file mode 100644 index 0000000..5c730d4 --- /dev/null +++ b/py/cso_colhub.py @@ -0,0 +1,1472 @@ +# +# Changes +# +# 2022-09, Arjo Segers +# Updated documentation. +# +# 2023-06, Arjo Segers +# Use "pandas.concat()" instead of "df.append()" to avoid warnings. +# +# 2023-08, Arjo Segers +# Updated logging messages. +# +# 2023-08, Arjo Segers +# Reformatted using 'black'. +# +# 2035-01, Arjo Segers +# Extracted parts from former "cso_scihub.py" module to support "colhub", +# the Norwegian mirror site. +# + +######################################################################## +### +### help +### +######################################################################## + +""" +.. _cso-colhub: + +********************* +``cso_colhub`` module +********************* + +The :py:mod:`cso_colhub` module provides classes for accessing data from the +`Copernicus Open Access Hub `_. +Below first the data hub itself is described, followed by how the CSO +pre-processor could be used for batch download of a selection. + + +Copernicus Open Access Hub +========================== + +The `Copernicus Open Access Hub `_ is the official +portal for Copernicus satellite data. + +.. figure:: figs/sci-hub.png + :scale: 50 % + :align: center + :alt: Sci Hub main page. + + *Home page of Copernicus Open Access Hub* (https://colhub.copernicus.eu/) + +Different hubs are provided for different data sets. +Below we describe: + +* **Open Hub** Provides access to Sentinel-1,2,3 data. +* **S-5P Pre-Ops** Provides access to (pre-operational) Sentinel-5P data. + + +.. _ColHub-OpenHub: + +Open Hub +-------- + +The *Open Hub* provides access to Sentinel-1,2,3 data. +Since this is the oldest data, most examples in the User Guide refer to this hub. + +To download data it is necessary to register. +On the main page, select: + +* `User Guide `_, + follow instructions on `Self Registration `_ . + +The user name and password should be stored in your home directory in the ``~/.netrc`` file +(ensure that it has read/write permissions only for you):: + + # access Copernicus Open Access Hub + machine colhub.copernicus.eu login ********** password ********* + + +S-5P Pre-Ops Hub +---------------- + +On the main page, select *'S-5P Pre-Ops'* for the +`Sentinel-5P Pre-Operations Data Hub `_ . + +Access is open, the login/password are shown in the login pop-up. +The user name and password should be stored in your home directory in the ``~/.netrc`` file +(ensure that it has read/write permissions only for you):: + + # access Copernicus Open Access Hub + machine s5phub.copernicus.eu login s5pguest password s5pguest + +Data can be selected and downloaded interactively. +In the search bar, open the *'Advanced Search'* menu and specify a selection. +The figure below shows an example for Level2 NO2 data. + + +.. figure:: figs/s5p-hub-advanced-search.png + :scale: 75 % + :align: center + :alt: Advanced Search menu at S-5P Hub. + + *Advanced Search menu at S-5P Hub.* + +The result of a search is a list of product files. +Each product file contains data from one orbit. +The name of a product file contains the scan period, the orbit number, +and a processing time:: + + S5P_OFFL_L2__NO2____20190701T102357_20190701T120527_08882_01_010302_20190707T120219 + | | \_prodid_/ \_start-time__/ \__end-time___/ orbit | | \__processed__/ + | stream | processor + mission collection + +The search query is shown above the product list, and contains the time +and product selections; the later could be useful for batch download:: + + ( beginPosition:[2019-07-01T00:00:00.000Z TO 2019-07-01T23:59:59.999Z] AND + endPosition:[2019-07-01T00:00:00.000Z TO 2019-07-01T23:59:59.999Z] ) + AND ( ( platformname:Sentinel-5 AND + producttype:L2__NO2___ AND + processinglevel:L2 AND + processingmode:Offline ) ) + + + +.. _colhub-opensearch-download: + +OpenSearch API +-------------- + +For batch processing one can use the +`OpenSearch API `_ . +First a search query is send to the server, and the result then contains links that can be used to download +selected files. + + +Batch download via CSO +====================== + +The CSO download tools use by default the :ref:`colhub-opensearch-download`, +but could also use the :ref:`colhub-batch-download` option. + +The :py:class:`.CSO_ColHub_Download` class performs the download using the OpenSearch API. +This class is prefered since: + +* existing files could be kept without a new download; +* error messages are cleaner. + +See the documentation of the class for the settings to be used. + +Alternatively, the :py:class:`.CSO_ColHub_Download_DHuS` class could be used +which will call the ``dhusget.sh`` script to perform the download. +This will give a lot of intermediate log files, and error messages that are not easily interpreted. + + +Class hierchy +============= + +The classes and are defined according to the following hierchy: + + +* :py:class:`.CSO_ColHub_DownloadFile` +* :py:class:`.UtopyaRc` + + * :py:class:`.CSO_ColHub_Inquire` + * :py:class:`.CSO_ColHub_DownloadFile` + * :py:class:`.CSO_ColHub_Download` + * :py:class:`.CSO_ColHubMirror_Inquire` + * :py:class:`.CSO_ColHubMirror_Missing` + + + +Classes +======= + + +""" + + +######################################################################## +### +### modules +### +######################################################################## + +# modules: +import logging + +# tools: +import utopya + + +######################################################################## +### +### OpenSearch inquire +### +######################################################################## + + +class CSO_ColHub_Inquire(utopya.UtopyaRc): + + """ + Inquire available Sentinel data from the `Copernicus Open Access Hub `_ + using the OpenSearch API: + + * `Copernicus Open Access Hub `_ + + * `User Guide `_ + + * `6 OpenSearch API `_ + + A query is sent to search for products that are available + for a certain time and overlap with a specified region. + The result is a list with orbit files and instructions on how to download them. + + In the settings, specify the url of the hub, which is either the Open Data hub or the S5-P hub:: + + ! server url; provide login/password in ~/.netrc: + .url : https://s5phub.copernicus.eu/dhus + + Specify the time range over which files should be downloaded:: + + .timerange.start : 2018-07-01 00:00 + .timerange.end : 2018-07-01 23:59 + + Specify a target area, only orbits with some pixels within the defined box will be downloaded:: + + ! target area, leave empty for globe; format: west,south:east,north + .area : + !.area : -30,30:35,76 + + A query is used to select the required data. + The search box on the hub could be used for inspiration on the format. + Note that the '``producttype``' should have exactly 10 characters, + with the first 3 used for the retrieval level, and the last 6 for the product; + empty characters should have an underscore instead:: + + ! search query, obtained from interactive download: + .query : platformname:Sentinel-5 AND \\ + producttype:L2__NO2___ AND \\ + processinglevel:L2 + + Name of ouptut csv file:: + + ! output table, date of today: + .output.file : ${my.work}/Copernicus_S5P_NO2_%Y-%m-%d.csv + + """ + + def __init__(self, rcfile, rcbase="", env={}, indent=""): + + """ + Download orbits using OpenSearch API. + """ + + # modules: + import sys + import os + import datetime + import requests + import xml.etree.ElementTree + import pandas + + # info ... + logging.info( f"{indent}") + logging.info( f"{indent}** Inquire files available on Copernicus Data Hub") + logging.info( f"{indent}") + + # init base object: + utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) + + # domain: + url = self.GetSetting("url") + # info ... + logging.info( f"{indent}url : %s" % url) + + # area of interest: west,south:east,north + area = self.GetSetting("area") + # defined? + if len(area) > 0: + # convert from format for "dhusget.sh": + # west,south:east,north + west, south, east, north = map(float, area.replace(":", " ").replace(",", " ").split()) + else: + # globe: + west, south, east, north = -180, -90, 180, 90 + # endif + # info ... + logging.info( + f"{indent}area : [%8.2f,%8.2f] x [%8.2f,%8.2f]" % (west, east, south, north) + ) + + # query, probably obtained from interactive download page: + product = self.GetSetting("query") + # remove excesive whitespace: + product = " ".join(product.split()) + # info ... + logging.info( f"{indent}product : %s" % product) + + # target file, might include time templates: + output_file__template = self.GetSetting("output.file") + # current time: + output_file = datetime.datetime.now().strftime(output_file__template) + + # new output table: + output_df = pandas.DataFrame() + + # time range: + t1 = self.GetSetting("timerange.start", totype="datetime") + t2 = self.GetSetting("timerange.end", totype="datetime") + # info ... + tfmt = "%Y-%m-%d %H:%M" + logging.info( f"{indent}timerange: [%s,%s]" % (t1.strftime(tfmt), t2.strftime(tfmt))) + + # targefile: + qfile = "query-%i.xml" % os.getpid() + + # timeout of requests in seconds: + timeout = 60 + + # search query could only return a maximum number of records; + # a 'page' of records is requested using a row offet and the number of rows: + row0 = 0 + nrow = 100 # maximum allowed is 100 ... + + # time labels: + tfmt = "%Y-%m-%dT%H:%M:%SZ" + # time of interest: + toi = "beginPosition:[%s TO %s]" % (t1.strftime(tfmt), t2.strftime(tfmt)) + + # geographic area: + # 'POLYGON((P1Lon P1Lat, P2Lon P2Lat, ..., PnLon PnLat))' + area = "POLYGON((%f %f, %f %f, %f %f, %f %f, %f %f))" % ( + west, + south, + east, + south, + east, + north, + west, + north, + west, + south, + ) + # area of interest: + aoi = 'footprint:"Intersects(%s)"' % area + + # info .. + logging.info( f"{indent} time of interest: %s" % toi) + logging.info( + indent + + " area of interest: [%8.2f,%8.2f] x [%8.2f,%8.2f]" % (west, east, south, north) + ) + logging.info( f"{indent} product : %s" % product) + + # combine: + query = "%s AND %s AND (%s)" % (toi, product, aoi) + + # init counter: + ipage = 0 + + # loop over pages of query result: + while True: + + # increase counter: + ipage += 1 + # info ... + logging.info( f"{indent} page %i (entries %i,..,%i)" % (ipage, row0 + 1, row0 + nrow)) + + # send query to search page; + # result is an xml tree with entries for matching orbit files; + # a maximum of 'nrow' entries is returned, from a zero-based 'start' index onwards: + r = requests.get( + os.path.join(url, "search"), + params=dict(q=query, start=row0, rows=nrow), + timeout=timeout, + ) + # check status, raise error if request failed: + try: + r.raise_for_status() + except requests.exceptions.HTTPError as err: + msg = str(err) + logging.error("from query; message received:") + logging.error(" %s" % msg) + if msg.startswith("401 Client Error: Unauthorized for url:"): + logging.error( + 'Interpretation: the (username,password) received from your "~/.netrc" file is incorrect.' + ) + logging.error("For the S5p-hub, the file should contain the following entry:") + logging.error( + " machine s5phub.copernicus.eu login s5pguest password s5pguest" + ) + # get authorization as test: + try: + username, password = requests.utils.get_netrc_auth(url, raise_errors=True) + logging.error('The username extracted from "~/.netrc" for this url is:') + logging.error(" %s" % username) + logging.error("Maybe this is the default entry?") + except Exception as err: + logging.error( + 'The "~/.netrc" file could not be parsed correctly; the error raised from:' + ) + logging.error( + ' username,password = requests.utils.get_netrc_auth( "%s", raise_errors=True )' + % url + ) + logging.error("is:") + logging.error(" %s" % str(err)) + # endtry + # endif + except Exception as err: + msg = str(err) + logging.error("from query; message received:") + logging.error(" %s" % msg) + sys.exit(1) + # endtry + + # save: + with open(qfile, "w") as f: + f.write(r.text) + # endwith + + # + # Example content: + # + # + # + # Data Hub Service search results for: beginPosition:[2018-07-01T00:00:00Z TO 2018-07-02T00:00:00Z] AND platformname:Sentinel-5 AND producttype:L2__NO2___ AND processinglevel:L2 AND processingmode:Reprocessing AND (footprint:"Intersects(POLYGON((-30.000000 30.000000, 45.000000 30.000000, 45.000000 76.000000, -30.000000 76.000000, -30.000000 30.000000)))") + # ... + # + # S5P_RPRO_L2__NO2____20180701T024001_20180701T042329_03699_01_010202_20190211T185158 + # + # S5P_RPRO_L2__NO2____20180701T024001_20180701T042329_03699_01_010202_20190211T185158.nc + # a66d4240-4999-4724-9903-aa2db410bbad + # ... + # + # ... + # + # + + # open query result: + tree = xml.etree.ElementTree.parse(qfile) + # get root element ("feed"): + root = tree.getroot() + + # the tag contains an url prefix: + # root.tag = "{http://www.w3.org/2005/Atom}feed" + # prefixed? + if root.tag.startswith("{"): + # extract prefix: + prefix, tag = root.tag[1:].split("}") + else: + # no prefix: + prefix = "" + # endif + # create namespace to translate the "atom:" to the prefix: + ns = {"atom": prefix} + + # find "entry" elements: + entries = root.findall("atom:entry", namespaces=ns) + + # count: + nrec = len(entries) + # info .. + logging.info( f"{indent} number of records found: %i" % nrec) + # check .. + if (ipage == 1) and (nrec == 0): + logging.warning( + " no records found for this day; something wrong in query? continue with next day ..." + ) + break + # endif + + # loop over entries: + for entry in entries: + + # get target filename, this is the content of the "str" element + # with attribute "name" equal to "filename": + # S5P_RPRO_L2__NO2____20180701T024001_20180701T042329_03699_01_010202_20190211T185158.nc + filename = entry.find('atom:str[@name="filename"]', ns).text + + # info ... + logging.info( f"{indent} file : %s" % filename) + + # + # filenames: + # + # S5P_OFFL_L2__NO2____20180701T005930_20180701T024100_03698_01_010002_20180707T022838.nc + # plt proc [product-] [starttime....] [endtime......] orbit cl prvers [prodtime.....] + # + bname = os.path.basename(filename).replace(".nc", "") + # split: + platform_name, processing, rest = bname.split("_", 2) + product_type = rest[0:10] + start_time, end_time, orbit, collection, processor_version, production_time = rest[ + 11: + ].split("_") + + # convert: + tfmt = "%Y%m%dT%H%M%S" + ts = datetime.datetime.strptime(start_time, tfmt) + te = datetime.datetime.strptime(end_time, tfmt) + + # find first "link" element: + link = entry.find("atom:link", ns) + # extract download link: + href = link.attrib["href"] + + # row: + rec = { + "orbit": [orbit], + "start_time": [ts], + "end_time": [te], + "processing": [processing], + "collection": [collection], + "processor_version": [processor_version], + "filename": [filename], + "href": [href], + } + # add record: + output_df = pandas.concat((output_df, pandas.DataFrame(rec)), ignore_index=True) + + # endfor # entries + + # cleanup: + os.remove(qfile) + + # leave loop over pages? + if nrec < nrow: + break + # increse offset: + row0 += nrow + + # endwhile # pages + + # info .. + logging.info("save to: %s ..." % output_file) + # create directory: + dirname = os.path.dirname(output_file) + if len(dirname) > 0: + if not os.path.isdir(dirname): + os.makedirs(dirname) + # endif + # write: + output_df.to_csv(output_file, sep=";", index=False) + + # info ... + logging.info( f"{indent}") + logging.info( f"{indent}** end ColHub inquire") + logging.info( f"{indent}") + + # enddef __init__ + + +# endclass CSO_ColHub_Inquire + + +######################################################################## +### +### OpenSearch download +### +######################################################################## + + +class CSO_ColHub_DownloadFile(object): + + """ + Download single file from ColHub. + + Arguments: + + * ``href`` : download url: ``https://s5phub.copernicus.eu/dhus/odata/v1/Products('d483baa0-3a61-4985-aa0c-5642a83c9214')/$value`` + * ``output_file`` : target file + + Optional arguments: + + * ``maxtry`` : number of times to try again if download fails + * ``timeout`` : delay in seconds between requests + """ + + def __init__(self, href, output_file, maxtry=10, timeout=60, indent=""): + + """ + Download file. + """ + + # modules: + import os + import requests + + # retry loop .. + ntry = 0 + while True: + # try to download and save: + try: + + # try to download: + try: + + # download: + r = requests.get(href, timeout=timeout) + # check status, raise error if request failed: + r.raise_for_status() + + # info .. + logging.info( f"{indent} write to: %s ..." % os.path.dirname(output_file)) + # target dir: + dname = os.path.dirname(output_file) + if len(dname) > 0: + if not os.path.isdir(dname): + os.makedirs(dname) + # endif + # write to temporary target first .. + tmpfile = output_file + ".tmp" + # open destination file for binary write: + with open(tmpfile, "wb") as fd: + # prefered way to write content following: + # https://docs.python-requests.org/en/master/user/quickstart/ + for chunk in r.iter_content(chunk_size=128): + fd.write(chunk) + # endfor + # endwith + # rename: + os.rename(tmpfile, output_file) + + except requests.exceptions.HTTPError as err: + # info .. + msg = str(err) + logging.error("exception from download; message received:") + logging.error(" %s" % msg) + # catch known problem ... + if msg.startswith("401 Client Error: Unauthorized for url:"): + logging.error( + 'Interpretation: the (username,password) received from your "~/.netrc" file is incorrect.' + ) + logging.error( + "For the S5p-hub, the file should contain the following entry:" + ) + logging.error( + " machine s5phub.copernicus.eu login s5pguest password s5pguest" + ) + # try to get authorization as test: + try: + username, password = requests.utils.get_netrc_auth( + url, raise_errors=True + ) + logging.error( + 'The username extracted from "~/.netrc" for this url is:' + ) + logging.error(" %s" % username) + logging.error("Maybe this is the default entry?") + except Exception as err: + logging.error( + 'The "~/.netrc" file could not be parsed correctly; the error raised from:' + ) + logging.error( + ' username,password = requests.utils.get_netrc_auth( "%s", raise_errors=True )' + % url + ) + logging.error("is:") + logging.error(" %s" % str(err)) + # endtry + # should be solved first, leave retry loop ... + break + # endif + + except MemoryError as err: + logging.error( + "memory error from download; need to increase allocated resources?" + ) + # quit with error: + raise + + except Exception as err: + # info .. + logging.error("from download; message received:") + logging.error(" %s" % str(err)) + # quit with error: + raise + + # endtry + + # error from download or save: + except: + # increase counter: + ntry += 1 + # switch: + if ntry == maxtry: + logging.warning( + f"{indent} exception from download; tried %i times ..." % maxtry + ) + raise Exception + else: + logging.warning( f"{indent} exception from download; try again ...") + continue # while-loop + # endif + # endtry + # leave retry loop, + # either because download was ok, + # or because maximum number of retries was reached: + break + + # endwhile # retry + + # enddef __init__ + + +# endclass CSO_ColHub_DownloadFile + +# * + + +class CSO_ColHub_Download(utopya.UtopyaRc): + + """ + Download Sentinel data from the `Copernicus Open Access Hub `_ + using the OpenSearch API: + + * `Copernicus Open Access Hub `_ + + * `User Guide `_ + + * `6 OpenSearch API `_ + + To download orbit files, first a query is sent to search for products that are available + for a certain time and overlap with a specified region. + The result is a list with orbit files and instructions on how to download them. + + In the settings, specify the url of the hub, which is either the Open Data hub or the S5-P hub:: + + ! server url; provide login/password in ~/.netrc: + .url : https://s5phub.copernicus.eu/dhus + + Specify the time range over which files should be downloaded:: + + .timerange.start : 2018-07-01 00:00 + .timerange.end : 2018-07-01 23:59 + + Specify a target area, only orbits with some pixels within the defined box will be downloaded:: + + ! target area, leave empty for globe; format: west,south:east,north + .area : + !.area : -30,30:35,76 + + A query is used to select the required data. + The search box on the hub could be used for inspiration on the format. + Note that the '``producttype``' should have exactly 10 characters, + with the first 3 used for the retrieval level, and the last 6 for the product; + empty characters should have an underscore instead:: + + ! search query, obtained from interactive download: + .query : platformname:Sentinel-5 AND \\ + producttype:L2__NO2___ AND \\ + processinglevel:L2 AND \\ + processingmode:Offline + + The target directory for downloaded file could include templates for time values:: + + ! output archive, store per month: + .output.dir : /data/Copernicus/S5P/OFFL/NO2/%Y/%m + + Use the following flag to keep files that are already present:: + + ! renew existing files? + + # + # Data Hub Service search results for: beginPosition:[2018-07-01T00:00:00Z TO 2018-07-02T00:00:00Z] AND platformname:Sentinel-5 AND producttype:L2__NO2___ AND processinglevel:L2 AND processingmode:Reprocessing AND (footprint:"Intersects(POLYGON((-30.000000 30.000000, 45.000000 30.000000, 45.000000 76.000000, -30.000000 76.000000, -30.000000 30.000000)))") + # ... + # + # S5P_RPRO_L2__NO2____20180701T024001_20180701T042329_03699_01_010202_20190211T185158 + # + # S5P_RPRO_L2__NO2____20180701T024001_20180701T042329_03699_01_010202_20190211T185158.nc + # a66d4240-4999-4724-9903-aa2db410bbad + # ... + # + # ... + # + # + + # open query result: + tree = xml.etree.ElementTree.parse(qfile) + # get root element ("feed"): + root = tree.getroot() + + # the tag contains an url prefix: + # root.tag = "{http://www.w3.org/2005/Atom}feed" + # prefixed? + if root.tag.startswith("{"): + # extract prefix: + prefix, tag = root.tag[1:].split("}") + else: + # no prefix: + prefix = "" + # endif + # create namespace to translate the "atom:" to the prefix: + ns = {"atom": prefix} + + # find "entry" elements: + entries = root.findall("atom:entry", namespaces=ns) + + # count: + nrec = len(entries) + # info .. + logging.info( f"{indent} number of records found: %i" % nrec) + # check .. + if (ipage == 1) and (nrec == 0): + logging.warning( + " no records found for this day; something wrong in query? continue with next day ..." + ) + break + # endif + + # loop over entries: + for entry in entries: + + # get target filename, this is the content of the "str" element + # with attribute "name" equal to "filename": + # S5P_RPRO_L2__NO2____20180701T024001_20180701T042329_03699_01_010202_20190211T185158.nc + filename = entry.find('atom:str[@name="filename"]', ns).text + + # full path: + output_file = os.path.join(output_dir, filename) + + # info ... + logging.info( f"{indent} file : %s" % filename) + # (re)new? + if (not os.path.isfile(output_file)) or renew: + + # find first "link" element: + link = entry.find("atom:link", ns) + # extract download link: + href = link.attrib["href"] + # info .. + logging.info( f"{indent} download: %s" % href) + + # sinlge file: + CSO_ColHub_DownloadFile( + href, output_file, maxtry=maxtry, timeout=timeout, indent=indent + ) + + else: + # info .. + logging.info( f"{indent} keep in: %s ..." % output_dir) + # endif + + ## testing ... + # break + + # endfor # entries + + # cleanup: + os.remove(qfile) + + # leave loop over pages? + if nrec < nrow: + break + # increse offset: + row0 += nrow + + # endwhile # pages + + # start of next day: + t = te + # endwhile # day loop + + # info ... + logging.info( f"{indent}") + logging.info( f"{indent}** end ColHub Download") + logging.info( f"{indent}") + + # enddef __init__ + + +# endclass CSO_ColHub_Download + + +######################################################################## +### +### create listing file for file archive +### +######################################################################## + + +class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): + + """ + Create *listing* file for files available in file archive. + + The format is similar to the output of *inquiry* classes, + with per line a filename, the time range of pixels in the file, and other information extracted from the filenames: + + filename ;processing;start_time ;end_time ;orbit;collection;processor_version;href + S5P_RPRO_L2__CH4____20180430T001851_20180430T020219_02818_01_010301_20190513T141133.nc;RPRO ;2018-04-30T00:18:51;2018-04-30T02:02:19;02818;01 ;010301 ;/archive/mirror/S5P_RPRO_L2__CH4____20180430T001851_20180430T020219_02818_01_010301_20190513T141133.nc + S5P_RPRO_L2__CH4____20180430T020021_20180430T034349_02819_01_010301_20190513T135953.nc;RPRO ;2018-04-30T02:00:21;2018-04-30T03:43:49;02819;01 ;010301 ;/archive/mirror/S5P_RPRO_L2__CH4____20180430T020021_20180430T034349_02819_01_010301_20190513T135953.nc + : + + This file could be used to scan for available versions and how they were produced. + + In the settings, define the base directory of the archive: + + .dir : /archive/mirror + + This directory is recursively scanned using the :py:class:`os.walk` class on files with a specified form:: + + ! search S5P CH4 files: + .pattern : S5P_*_L2_CH4___*.nc + + Specifiy the output file:: + + ! csv file that will hold records per file with: + ! - timerange of pixels in file + ! - orbit number + ! time templates are replaced with todays date + .file : /Scratch/Copernicus/S5p/listing-CH4__%Y-%m-%d.csv + + An existing listing file is not replaced, + unless the following flag is set:: + + ! renew table? + .renew : True + + """ + + def __init__(self, rcfile, rcbase="", env={}, indent=""): + + """ + Scan file archive. + """ + + # modules: + import os + import datetime + #import glob + import collections + import fnmatch + + # tools: + import cso_file + + # info ... + logging.info( f"{indent}") + logging.info( f"{indent}** create listing file") + logging.info( f"{indent}") + + # init base object: + utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) + + # renew output? + renew = self.GetSetting("renew", totype="bool") + + # table file to be written: + lst_file = self.GetSetting("output.file") + # evaluate current time: + lst_file = datetime.datetime.now().strftime(lst_file) + + # create? + if (not os.path.isfile(lst_file)) or renew: + # info .. + logging.info( f"{indent}create %s ..." % lst_file) + + # initiallize for (re)creation: + listing = cso_file.CSO_Listing() + + # archive directory: + archive_dir = self.GetSetting("dir") + # check .. + if not os.path.isdir(archive_dir): + logging.error(f"archive directory not found: {archive_dir}") + raise Exception + #endif + # pattern for data files: + fpattern = self.GetSetting("pattern") + + # recursively search for files: + for root, dirs, files in os.walk(archive_dir): + + # loop over files: + for fname in files : + # data file? + if fnmatch.fnmatch( fname, fpattern ) : + + # already in table? + if fname in listing: + # info ... + logging.info( f"{indent} keep entry %s ..." % fname) + else: + # info ... + logging.info( f"{indent} add entry %s ..." % fname) + # Example filename: + # S5P_RPRO_L2__CH4____20180430T001851_20180430T020219_02818_01_010301_20190513T141133.nc + # + # Some products have incorrect product id (should be 10 characters): + # S5P_OFFL_L2__CHOCHO___20200101T005246_20200101T023416_11487_01_010000_20210128.nc + # The extracted product id is then truncated to 10 characters. + # + # basename: + bname, ext = os.path.splitext(fname) + # extract: + try: + # split in 3 parts: + mission, processing, rest = bname.split("_", 2) + # extract product id, should be 10 chars: + product_id = rest[0:10] + # adhoc fix: some files have too long product id: + if rest.startswith("L2__CHOCHO__"): + rest = res[13:] + else : + rest = rest[11:] + #endif + # remaining parts: + ( + start_time, + end_time, + orbit, + collection, + processor_version, + prod_time, + ) = rest.split("_") + except: + logging.error(f"could not extract filename parts; expected format:") + logging.error(f" S5P_RPRO_L2__CH4____20180430T001851_" + +"20180430T020219_02818_01_010301_20190513T141133" ) + logging.error(f"found:") + logging.error(f" {bname}") + raise + # endif + # headers in "inquire" tables: + # orbit;start_time;end_time;processing;collection;processor_version;filename;href + # fill data record: + data = collections.OrderedDict() + tfmt = "%Y%m%dT%H%M%S" + data["orbit"] = orbit + data["start_time"] = datetime.datetime.strptime(start_time, tfmt) + data["end_time"] = datetime.datetime.strptime(end_time, tfmt) + data["processing"] = processing + data["collection"] = collection + data["processor_version"] = processor_version + data["href"] = os.path.join(root,fname) + # update record: + listing.UpdateRecord( fname, data, indent= f"{indent} ") + # endif # new record? + + #endif # name of data file + + #endfor # files + + # testing . + if len(listing) >= 100 : break + + #endfor # walk + + # sort: + listing.Sort(by="orbit") + # save: + listing.Save(lst_file,indent= f"{indent} ") + + else: + # info .. + logging.info( f"{indent}keep %s ..." % lst_file) + # endif + + # info ... + logging.info( f"{indent}") + logging.info( f"{indent}** end archive listing") + logging.info( f"{indent}") + + # enddef __init__ + + +# endclass CSO_ColHubMirror_Inquire + + + + +######################################################################## +### +### create listing files missing in archive +### +######################################################################## + + +class CSO_ColHubMirror_Missing(utopya.UtopyaRc): + + """ + Create *listing* file for files that are in one inquiry table but not in another one. + This could be used to complete a mirror archive. + + The format is similar to the output of *inquiry* classes, + with per line a filename, the time range of pixels in the file, and other information extracted from the filenames:: + + filename ;start_time ;end_time ;orbit;processing;collection;processor_version;href + S5P_RPRO_L2__NO2____20180504T073130_20180504T091300_02879_03_020400_20221208T160012.nc;2018-05-04 07:31:30;2018-05-04 09:13:00;02879 ;RPRO ;03 ;020400 ;https://zipper.dataspace.copernicus.eu/odata/v1/Products(ae43b35c-0569-4e1f-b8cb-0afc49790716)/$value + : + + In the settings, define the listing file with all available data, for example the result of an inquiry step; + eventually add a timestamp to replace templates in the filename:: + + .all.file : /work/inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv + !.all.filedate : 2025-01-24 + + Similar specify the name of the file that is listing the current mirror, + probably the output of the :py:class:`CSO_ColHubMirror_Inquire` class:: + + .curr.file : /work/inquire/Copernicus_S5p_NO2_colhub-mirror__%Y-%m-%d.csv + !.curr.filedate : 2025-01-24 + + Specify a selection filter; this defines which of the orbit files are actually needed. + + ! Provide ';' seperated list of to decide if a particular orbit file should be processed. + ! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing), + ! the file with the first match will be used. + ! The expressions should include templates '%{header}' for the column values. + ! + .selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ + (%{collection} == '03') and (%{processing} == 'OFFL') + + Specifiy the output file:: + + ! csv file that will hold records per file with: + ! - timerange of pixels in file + ! - orbit number + ! time templates are replaced with todays date + .file : /work/inquire/Copernicus_S5p_NO2_colhub-mirror-missing__%Y-%m-%d.csv + + An existing listing file is not replaced, + unless the following flag is set:: + + ! renew table? + .renew : True + + """ + + def __init__(self, rcfile, rcbase="", env={}, indent=""): + + """ + Convert data. + """ + + # modules: + import os + import datetime + #import glob + import collections + import fnmatch + + # tools: + import cso_file + + # info ... + logging.info( f"{indent}") + logging.info( f"{indent}** create list of missing files") + logging.info( f"{indent}") + + # init base object: + utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) + + # renew output? + renew = self.GetSetting("renew", totype="bool") + + # table file to be written: + lst_file = self.GetSetting("output.file") + # evaluate current time: + lst_file = datetime.datetime.now().strftime(lst_file) + + # create? + if (not os.path.isfile(lst_file)) or renew: + # info .. + logging.info( f"{indent}create %s ..." % lst_file) + + # initiallize for (re)creation: + listing = cso_file.CSO_Listing() + + # table with all available files: + listfile_all = self.GetSetting("all.file") + # evaluate time? + filedate = self.GetSetting( "all.filedate", totype="datetime", default=datetime.datetime.now() ) + listfile_all = filedate.strftime(listfile_all) + # read: + listing_all = cso_file.CSO_Listing(listfile_all) + + # table with currently already available files: + listfile_curr = self.GetSetting("curr.file") + # evaluate time? + filedate = self.GetSetting( "curr.filedate", totype="datetime", default=datetime.datetime.now() ) + listfile_curr = filedate.strftime(listfile_curr) + # read: + listing_curr = cso_file.CSO_Listing(listfile_curr) + + # extract orbits: + orbits = listing_all.GetValues( "orbit" ).unique() + # info .. + logging.info( f"{indent}found {len(orbits)} unique orbit numbers ..." ) + + # list of ';' seperated selection expression: + # (%{processor_version} == '020400') & (%{processing} == 'RPRO') ; ... + selection_expr = self.GetSetting("selection") + # info .. + logging.info(f"{indent}selection criteria (first with matching orbit is used):") + for expr in selection_expr.split(";") : + logging.info(f"{indent} {expr.strip()}") + # endif + + # info .. + logging.info( f"{indent}loop over available orbits ..." ) + # loop: + for orbit in orbits : + # info .. + logging.info( f"{indent} orbit '{orbit}' ..." ) + + # selection based on orbit and filter expression; + # returns None if no records are found: + xlst = listing_all.Select( orbit=orbit, expr=selection_expr, indent=" " ) + # nothing selected? + if len(xlst) == 0 : continue + + # loop over selected records: + for irec in range(len(xlst)): + # current: + rec = xlst.GetRecord(irec) + ## info .. + #logging.info( f"{indent} {fname}" ) + # check if already available: + if rec["filename"] in listing_curr : + # info ... + logging.info( f"{indent} file already present ..." ) + else : + # info ... + logging.info( f"{indent} file not present yet, add to list ..." ) + # add record to list: + listing.UpdateRecord( rec["filename"], rec ) + #endif + #endfor + + # testing ... + #break + if len(listing) > 0 : + logging.warning(f"BREAK!") + break + #endif + #endfor + + # save: + listing.Save(lst_file, indent= f"{indent} ") + + else: + # info .. + logging.info( f"{indent}keep %s ..." % lst_file) + # endif # renew + + # info ... + logging.info( f"{indent}") + logging.info( f"{indent}** end missing") + logging.info( f"{indent}") + + # enddef __init__ + + +# endclass CSO_ColHubMirror_Missing + +######################################################################## +### +### end +### +######################################################################## diff --git a/py/cso_file.py b/py/cso_file.py index e926dbd..c3bf175 100644 --- a/py/cso_file.py +++ b/py/cso_file.py @@ -21,7 +21,10 @@ # Do not pack coordinates. Updated comment. # # 2024-08, Arjo Segers -# Updated formatting. +# Updated formatting. +# +# 2025-01, Arjo Segers +# Extened CSO_Listing class with selection and other methods. # ######################################################################## @@ -1209,15 +1212,13 @@ class CSO_Listing(object): 2007/06/RAL-IASI-CH4_20070601T022359.nc;2007-06-01 02:23:59.512000000;2007-06-01 04:05:57.328000000 ... - Arguments: + Optional arguments: - * ``filename`` : listing file, content is read if this file is already present unless ``renew==True`` ; - if ``exist==True`` the file should exist - * ``renew`` : (optinal) if ``True``, an existing file is ignored + * ``filename`` : listing file read into table """ - def __init__(self, filename, exist=False, renew=False, indent=""): + def __init__(self, filename=None, indent=""): """ Initialize empty table or read existing. """ @@ -1226,31 +1227,25 @@ class CSO_Listing(object): import os import pandas - # store: - self.filename = filename - # base path: - self.dirname = os.path.dirname(self.filename) - # csv style: self.sep = ";" # head for index column: self.index_label = "filename" + + # read? + if filename is not None : - # check .. - if exist: + # check .. if not os.path.isfile(filename): logging.error("listing file not found: %s" % filename) raise Exception # endif - # endif - - # already present? - if os.path.isfile(filename) and (not renew): + # info ... - logging.info(indent + "read listing %s ..." % self.filename) + logging.info(f"{indent} read listing {filename} ...") # read: self.df = pandas.read_csv( - self.filename, + filename, sep=self.sep, index_col=self.index_label, parse_dates=["start_time", "end_time"], @@ -1267,15 +1262,25 @@ class CSO_Listing(object): # * - def Close(self, indent=""): + def Save(self, filename, indent=""): """ Write table to file. """ - + # info ... - logging.info(indent + "save listing %s ..." % self.filename) - # save: - self.df.to_csv(self.filename, sep=self.sep, index_label=self.index_label) + logging.info(f"{indent}save listing {filename} ...") + + # create directory if needed: + CheckDir( filename ) + + # write all columns: + columns = list(self.df.keys()) + # index only once .. + if self.index_label in columns : + columns.remove(self.index_label) + + # save, also write the index column: + self.df.to_csv(filename, sep=self.sep, columns=columns, index_label=self.index_label) # enddef Close @@ -1309,6 +1314,23 @@ class CSO_Listing(object): # * + def GetValues(self, name): + """ + Return :py:class:`pandas.Series` object with all values for provided column ``name``. + """ + + # check .. + if name not in self.df.keys() : + logging.error( f"column '{name}' not found in: {self.filename}" ) + raise Exception + #endif + # extract: + return self.df[name] + + # enddef GetRecord + + # * + def __contains__(self, key): """ Returns True if key is in the table index. @@ -1320,6 +1342,17 @@ class CSO_Listing(object): # * + def keys(self): + """ + Returns list of column names. + """ + + return self.df.keys() + + # enddef keys + + # * + def Cleanup(self, indent=""): """ Remove records from table if filename is not present anymore. @@ -1423,9 +1456,9 @@ class CSO_Listing(object): # * - def Select(self, tr=None, method="overlap"): + def Select(self, tr=None, method="overlap", expr=None, blacklist=[], indent="", **kwargs ): """ - Return dataframe with selected records. + Return :py:class:`CSO_Listing` objects with selection of records. Optional arguments: @@ -1435,13 +1468,28 @@ class CSO_Listing(object): * ``method='overlap'`` (default) : select records that have pixels overlapping with interval * ``method='middle'`` : select pixels with the middle of ``start_time`` and ``end_time`` within the inerval ``(t1,t2]`` + + * ``name=value`` arguments select records based on column values + + * The exprssion ``expr`` provides a list of ';'-seperated selection expressions + with templates ``%{..}`` for the column names:: + + (%{processor_version} == '020400') & (%{processing} == 'RPRO') ; ... + + This is evaluted after the previously described selections. + The result should be None or exactly one record. + Eventually skip files in ``blacklists``. + """ # modules: + import os import pandas # init result with entire dataset: df = self.df + + # * # time range specified? if tr is not None: @@ -1463,9 +1511,103 @@ class CSO_Listing(object): raise Exception # endif # endif + + # * + + # other selections: + for key,value in kwargs.items(): + # check .. + if key not in df.keys(): + logging.error( f"key '{key}' not defined in listing: {self.filename}" ) + raise Exception + #endif + # select: + df = df[ df[key] == value ] + #endfor + + # * + + # evaluate selection expression? + if expr is not None : + # replace templates: + for key in self.df.keys(): + expr = expr.replace("%{" + key + "}", "xrec['" + key + "']") + # endfor + # split: + selections = expr.split(";") + + # storage for status label: "selected", "blacklisted", ... + filestatus = {} + # no match yet .. + seleted = [] + # loop over selection criteria, + # this should give either none or a single file: + for selection in selections: + # make empty again: + selected = [] + # loop over records: + for indx, xrec in df.iterrows(): + # skip? + if os.path.basename(indx) in blacklist: + filestatus[indx] = "blacklisted" + continue + # endif + # evaluate expression including 'xrec[key]' values: + if eval(selection): + selected.append(indx) + filestatus[indx] = "selected" + rec = xrec + # endif + # endfor # records + # exactly one? then leave: + if len(selected) == 1: + break + elif len(selected) > 1: + logging.error( f"found more than one record matching selection: {selection}" ) + for fname in selected: + logging.error(f" {fname}") + # endfor + raise Exception + # endif # number found + # endfor # selection criteria + + # info ... + logging.info(f"{indent}available records(s):") + # loop: + for fname, row in df.iterrows(): + line = fname + if fname in filestatus.keys(): + line = line + " [" + filestatus[fname] + "]" + logging.info(f"{indent} {line}") + # endfor + # no match? + if len(selected) == 0: + # info ... + logging.warning(" no match with any selection criterium ...") + ## degug: + # for selection in selections : + # logging.warning( ' %s' % selection.strip() ) + # logging.warning( ' record:' ) + # for key in rec.keys() : + # logging.warning( ' "%s" : "%s"' % (key,rec[key]) ) + # create empty dataframe as result: + df = pandas.DataFrame(columns=df.columns) + else : + # extract selected record: + df = df.loc[[selected[0]]] + #endif + + #endif + + # * + + # return as listing: + lst = CSO_Listing() + lst.df = df + # ok - return df + return lst # enddef Select @@ -1492,11 +1634,28 @@ class CSO_Listing(object): self.df = pandas.concat((self.df, xdf.set_index(self.index_label))) else: # append: - self.df = pandas.concat((self.df, lst.df)) + if len(self.df) == 0 : + self.df = lst.df + else : + self.df = pandas.concat((self.df, lst.df)) + #endif # endif # enddef Append + + # * + + def Sort(self,by="filename"): + + """ + Sort listing table by filename or other key. + """ + # sort inplace: + print('xxx',self.df.keys()) + self.df.sort_values(by, inplace=True) + + #endef Sort # endclass CSO_Listing diff --git a/py/cso_s5p.py b/py/cso_s5p.py index 6dd1d32..fbdd8fa 100644 --- a/py/cso_s5p.py +++ b/py/cso_s5p.py @@ -54,6 +54,9 @@ # Fixed replacement of units for special processing 'square_sum'. # Fixed check on inquire filedates in case of multiple inquire files. # +# 2025-01, Arjo Segers +# Use updated CSO_Listing class. +# ######################################################################## @@ -2732,7 +2735,7 @@ class CSO_S5p_Convert(utopya.UtopyaRc): raise Exception # endif # init storage: - dfs = [] + listing = cso_file.CSO_Listing() # loop: for ifile in range(len(filename__templates)): # current: @@ -2746,31 +2749,16 @@ class CSO_S5p_Convert(utopya.UtopyaRc): # endif # expand time templates filename = t0.strftime(filename__template) - # check .. - if not os.path.isfile(filename): - logging.error("inquire table not found: %s" % filename) - raise Exception - # endif - # info .. - logging.info(f"{indent}read inquire table: %s" % filename) # read: - dfs.append( - pandas.read_csv( - filename, - sep=";", - skip_blank_lines=True, - parse_dates=["start_time", "end_time"], - dtype="str", - ) - ) + lst = cso_file.CSO_Listing( filename ) + # add: + listing.Append( lst ) # endfor # filenames - # combine: - df = pandas.concat(dfs) # sort by filename: - df = df.sort_values("filename") + listing.Sort() # info ... - logging.info(f"{indent}number of files : %i" % len(df)) + logging.info(f"{indent}number of files : %i" % len(listing)) ## selected proecessings: # processings = self.GetSetting( 'processings' ).split() @@ -2779,18 +2767,11 @@ class CSO_S5p_Convert(utopya.UtopyaRc): # list of ';' seperated selection expression: # (%{processor_version} == '020400') & (%{processing} == 'RPRO') ; ... - line = self.GetSetting("selection") - # replace templates: - # (xrec['processor_version'] == '020400') & (xrec['processing'] == 'RPRO') ; ... - for key in df.keys(): - line = line.replace("%{" + key + "}", "xrec['" + key + "']") - # endfor - # split: - selections = line.split(";") + selection_expr = self.GetSetting("selection") # info .. logging.info("selection criteria (first with matching orbit is used):") - for selection in selections: - logging.info(" %s" % selection.strip()) + for expr in selection_expr.split(";"): + logging.info(f" {expr.strip()}" ) # endif # skip some? @@ -2822,12 +2803,12 @@ class CSO_S5p_Convert(utopya.UtopyaRc): complevel = self.GetSetting("output.complevel", "int") # select records with start time inside time range: - xdf = df[(df["start_time"] >= t1) & (df["start_time"] <= t2)] + xlst = listing.Select( tr=(t1,t2) ) # info .. - logging.info(f"{indent}found %i orbits with overlap of time range .." % len(xdf)) + logging.info(f"{indent}found {len(xlst)} orbits with overlap of time range .." ) # oribit labels: - orbits = xdf["orbit"].unique() + orbits = xlst.GetValues("orbit").unique() # no download initialized yet: downloader = None @@ -2839,70 +2820,14 @@ class CSO_S5p_Convert(utopya.UtopyaRc): # info ... logging.info(indent + ' orbit "%s" ...' % orbit) - # search for other records for same orbit: - odf = xdf[xdf["orbit"] == orbit] - - # storage for status label: "selected", "blacklisted", ... - filestatus = {} - # no match yet .. - seleted = [] - # loop over selection criteria, - # this should give either none or a single file: - for selection in selections: - # make empty again: - selected = [] - # loop over records: - for indx, xrec in odf.iterrows(): - # skip? - if os.path.basename(xrec["filename"]) in blacklist: - filestatus[xrec["filename"]] = "blacklisted" - continue - # endif - # evaluate expression including 'xrec[key]' values: - if eval(selection): - selected.append(xrec["filename"]) - filestatus[xrec["filename"]] = "selected" - rec = xrec - # endif - # endfor # records - # exactly one? then leave: - if len(selected) == 1: - break - elif len(selected) > 1: - logging.error( - "found more than one orbit file matching selection: %s" % selection - ) - for fname in selected: - logging.error(" %s" % fname) - # endfor - raise Exception - # endif # number found - # endfor # selection criteria - - # info ... - logging.info(f"{indent} available file(s):") - # loop: - for fname in odf["filename"]: - line = fname - if fname in filestatus.keys(): - line = line + " [" + filestatus[fname] + "]" - logging.info(f"{indent} " + line) - # endfor - - # no match? - if len(selected) == 0: - # info ... - logging.warning(" no match with any selection criterium; next orbit ...") - ## degug: - # for selection in selections : - # logging.warning( ' %s' % selection.strip() ) - # logging.warning( ' record:' ) - # for key in rec.keys() : - # logging.warning( ' "%s" : "%s"' % (key,rec[key]) ) - # next orbit: + # select single orbit matching expression: + odf = xlst.Select( orbit=orbit, expr=selection_expr, blacklist=blacklist, indent=f" " ) + # no orbit found? next: + if len(odf) == 0 : continue - # endif - + # selected record: + rec = odf.GetRecord(0) + # start time of orbit: t0 = rec["start_time"] @@ -3018,68 +2943,82 @@ class CSO_S5p_Convert(utopya.UtopyaRc): if create: # keep list of downloaded files: downloads = [] + + # source location, either url or local mirror: + href = rec["href"] + # local file? + if os.path.isfile(href): - # input dir: - input_dir = t0.strftime(input_dir__template) - # replace templates: - for key in rec.keys(): - if type(rec[key]) == str: - input_dir = input_dir.replace("%{" + key + "}", rec[key]) - # endif - # endfor - # full path: - input_file = os.path.join(input_dir, rec["filename"]) + # this is the input file: + input_file = href + # info .. + logging.info(" input file: %s" % input_file) + + else: + + # input dir: + input_dir = t0.strftime(input_dir__template) + # replace templates: + for key in rec.keys(): + if type(rec[key]) == str: + input_dir = input_dir.replace("%{" + key + "}", rec[key]) + # endif + # endfor + # full path: + input_file = os.path.join(input_dir, rec["filename"]) - # info .. - logging.info(" input file: %s" % input_file) - # check .. - if not os.path.isfile(input_file): # info .. - logging.info(" not present yet, download ...") - # download url: - href = rec["href"] - # initialize download? - if downloader is None: - # init downloader based on url: - if "dataspace.copernicus.eu" in href: - # download from Copernicus DataSpace: - downloader = cso_dataspace.CSO_DataSpace_Downloader() - # - elif "s5p-pal.com" in href: - # download from PAL: - downloader = cso_pal.CSO_PAL_Downloader() - # - else: - logging.error("no downloader class defined for url: {href}") - raise Exception + logging.info(" input file: %s" % input_file) + # check .. + if not os.path.isfile(input_file): + # info .. + logging.info(" not present yet, download ...") + # download url: + href = rec["href"] + # initialize download? + if downloader is None: + # init downloader based on url: + if "dataspace.copernicus.eu" in href: + # download from Copernicus DataSpace: + downloader = cso_dataspace.CSO_DataSpace_Downloader() + # + elif "s5p-pal.com" in href: + # download from PAL: + downloader = cso_pal.CSO_PAL_Downloader() + # + else: + logging.error(f"no downloader class defined for url: {href}") + raise Exception + # endif # endif + # download ... + downloader.DownloadFile(href, input_file, indent=" ") + # store name: + downloads.append(input_file) # endif - # download ... - downloader.DownloadFile(href, input_file, indent=" ") - # store name: - downloads.append(input_file) - # endif - # download might have failed .. - if not os.path.isfile(input_file): - # write error file or raise error? - if with_error_files: - # info .. - logging.warning(f"{indent} missing input file, write error file ...") - # write error file: - with open(output_errfile, "w") as f: - f.write("missing file: %s\n" % input_file) - # endwith - # next: - continue - else: - # info .. - logging.error(f"missing input file") - logging.error(f" {input_file}") - logging.error(f"enable creation of *error files* to not break on this") - raise Exception + # download might have failed .. + if not os.path.isfile(input_file): + # write error file or raise error? + if with_error_files: + # info .. + logging.warning(f"{indent} missing input file, write error file ...") + # write error file: + with open(output_errfile, "w") as f: + f.write("missing file: %s\n" % input_file) + # endwith + # next: + continue + else: + # info .. + logging.error(f"missing input file") + logging.error(f" {input_file}") + logging.error(f"enable creation of *error files* to not break on this") + raise Exception + # endif # endif - # endif + + # endif # local mirror or remote source file # info ... logging.info(f"{indent} open ...") @@ -3328,7 +3267,7 @@ class CSO_S5p_Download(utopya.UtopyaRc): # combine: df = pandas.concat(dfs) # sort by filename: - df = df.sort_values("filename") + df.sort_values("filename", inplace=True) # info ... logging.info(f"{indent}number of files : %i" % len(df)) -- GitLab From 581c5146f6541b34c8ef5eb63285d379cee69d82 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Wed, 29 Jan 2025 11:40:50 +0100 Subject: [PATCH 2/4] Removed debug statements. --- py/cso_colhub.py | 15 +++++++++------ py/cso_file.py | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/py/cso_colhub.py b/py/cso_colhub.py index 5c730d4..a237e83 100644 --- a/py/cso_colhub.py +++ b/py/cso_colhub.py @@ -1252,8 +1252,11 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): #endfor # files - # testing . - if len(listing) >= 100 : break + ## testing ... + #if len(listing) >= 100 : + # logging.warning( f"BREAK after {len(listing)} files ..." ) + # break + # endif #endfor # walk @@ -1441,10 +1444,10 @@ class CSO_ColHubMirror_Missing(utopya.UtopyaRc): # testing ... #break - if len(listing) > 0 : - logging.warning(f"BREAK!") - break - #endif + #if len(listing) > 0 : + # logging.warning(f"BREAK!") + # break + ##endif #endfor # save: diff --git a/py/cso_file.py b/py/cso_file.py index c3bf175..ca376d8 100644 --- a/py/cso_file.py +++ b/py/cso_file.py @@ -1652,7 +1652,6 @@ class CSO_Listing(object): """ # sort inplace: - print('xxx',self.df.keys()) self.df.sort_values(by, inplace=True) #endef Sort -- GitLab From effd7ff7513c4e251ba2cede7416bea08f93ad90 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Thu, 30 Jan 2025 10:42:43 +0100 Subject: [PATCH 3/4] Added templates for ColHub archive to settings. --- config/Copernicus/cso-s5p-no2.rc | 41 + config/Copernicus/cso-user-settings.rc | 16 +- config/Copernicus/cso.rc | 9 + config/_test-colhub/cso-s5p-no2.rc | 1232 ---------------------- config/_test-colhub/cso-user-settings.rc | 169 --- config/_test-colhub/cso.rc | 311 ------ 6 files changed, 65 insertions(+), 1713 deletions(-) delete mode 100644 config/_test-colhub/cso-s5p-no2.rc delete mode 100644 config/_test-colhub/cso-user-settings.rc delete mode 100644 config/_test-colhub/cso.rc diff --git a/config/Copernicus/cso-s5p-no2.rc b/config/Copernicus/cso-s5p-no2.rc index d9c3cbe..5ae5685 100644 --- a/config/Copernicus/cso-s5p-no2.rc +++ b/config/Copernicus/cso-s5p-no2.rc @@ -58,6 +58,47 @@ cso.s5p.no2.inquire-table-dataspace.download_url : https://zipper.dat cso.s5p.no2.inquire-table-dataspace.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv +!----------------------------------------------------------------------- +! inquire colhub mirror archive +!----------------------------------------------------------------------- + +! csv file that will hold records per file with: +! - timerange of pixels in file +! - orbit number +cso.s5p.no2.inquire-table-colhub-mirror.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-listing__%Y-%m-%d.csv + +! renew table if file already exists? +cso.s5p.no2.inquire-table-colhub-mirror.renew : True + +! base path; example file: +! /x0/x0/x1/S5P_OFFL_L2__NO2____20230701T002408_20230701T020537_29604_03_020500_20230702T161349.nc +cso.s5p.no2.inquire-table-colhub-mirror.dir : /lustre/storeB/project/ESAcolhub/production-backend-AOI/S5p/all +! filename filter: +cso.s5p.no2.inquire-table-colhub-mirror.pattern : S5P_*_L2__NO2_*.nc + + +! ** create list of missing files + +! renew table if file already exists? +cso.s5p.no2.inquire-table-colhub-missing.renew : True + +! csv file that will hold records per file with: +! - timerange of pixels in file +! - orbit number +cso.s5p.no2.inquire-table-colhub-missing.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-missing__%Y-%m-%d.csv + +! listing from DataSpace with all potentially available files: +cso.s5p.no2.inquire-table-colhub-missing.all.file : ${cso.s5p.no2.inquire-table-dataspace.output.file} +!cso.s5p.no2.inquire-table-colhub-missing.all.filedate : 2025-01-23 + +! listing from current archive: +cso.s5p.no2.inquire-table-colhub-missing.curr.file : ${cso.s5p.no2.inquire-table-colhub-mirror.output.file} +!cso.s5p.no2.inquire-table-colhub-missing.curr.filedate : 2025-01-24 + +! selection of orbits, see "convert" below: +cso.s5p.no2.inquire-table-colhub-missing.selection : ${cso.s5p.no2.convert.selection} + + !!----------------------------------------------------------------------- !! inquire PAL portal !!----------------------------------------------------------------------- diff --git a/config/Copernicus/cso-user-settings.rc b/config/Copernicus/cso-user-settings.rc index b07f313..b7252fe 100644 --- a/config/Copernicus/cso-user-settings.rc +++ b/config/Copernicus/cso-user-settings.rc @@ -34,6 +34,7 @@ my.cso.convention : CF-1.7 ! region name: !my.region : globe my.region : CAMS +!my.region : xEMEP ! switch: !.................................... @@ -62,6 +63,19 @@ my.region.north : 76.0 ! size of map figures for this region, default size is (8,6) my.region.figsize : (8,6) +!.................................... +#elif "${my.region}" == "xEMEP" +!.................................... + +! outer bound of all known domains: +my.region.west : -50.0 +my.region.east : 90.0 +my.region.south : 25.0 +my.region.north : 85.0 + +! size of map figures for this region, default size is (8,6) +my.region.figsize : (8,5) + !.................................... #else #error unsupported my.region "${my.region}" @@ -77,7 +91,7 @@ my.region.figsize : (8,6) ! full range, used for inquiry jobs: my.full-timerange.start : 2018-01-01 00:00 -my.full-timerange.end : 2023-12-31 23:59 +my.full-timerange.end : 2024-12-31 23:59 ! processing for selected period: my.timerange.start : 2018-06-01 00:00 diff --git a/config/Copernicus/cso.rc b/config/Copernicus/cso.rc index b773216..bc97fd1 100644 --- a/config/Copernicus/cso.rc +++ b/config/Copernicus/cso.rc @@ -82,6 +82,7 @@ cso.s5p.TRACER.inquire.class : utopya.UtopyaJobStep ! two or more tasks: #if "TRACER" in ["no2","so2","hcho","co","o3-pr","o3-col"] cso.s5p.TRACER.inquire.tasks : table-dataspace plot +!cso.s5p.TRACER.inquire.tasks : table-colhub-mirror table-colhub-missing #elif "TRACER" in ["so2-cobra"] cso.s5p.TRACER.inquire.tasks : table-pal plot #elif "TRACER" in ["chocho"] @@ -93,6 +94,14 @@ cso.s5p.TRACER.inquire.tasks : table-glyretro table-pal pl cso.s5p.TRACER.inquire.table-dataspace.class : cso.CSO_DataSpace_Inquire cso.s5p.TRACER.inquire.table-dataspace.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ rcbase='cso.s5p.TRACER.inquire-table-dataspace' +!~ inquire file archive: +cso.s5p.TRACER.inquire.table-colhub-mirror.class : cso.CSO_ColHubMirror_Inquire +cso.s5p.TRACER.inquire.table-colhub-mirror.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-table-colhub-mirror' +!~ create table with files that are missing: +cso.s5p.TRACER.inquire.table-colhub-missing.class : cso.CSO_ColHubMirror_Missing +cso.s5p.TRACER.inquire.table-colhub-missing.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ + rcbase='cso.s5p.TRACER.inquire-table-colhub-missing' !~ inquire files available on PAL: cso.s5p.TRACER.inquire.table-pal.class : cso.CSO_PAL_Inquire cso.s5p.TRACER.inquire.table-pal.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ diff --git a/config/_test-colhub/cso-s5p-no2.rc b/config/_test-colhub/cso-s5p-no2.rc deleted file mode 100644 index e775834..0000000 --- a/config/_test-colhub/cso-s5p-no2.rc +++ /dev/null @@ -1,1232 +0,0 @@ -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!!! -!!! CSO - CAMS Satellite Operator -!!! -!!! Settings for S5p/NO2 processing. -!!! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - -!----------------------------------------------------------------------- -! user specific settings: -!----------------------------------------------------------------------- - -! include user specfic settings: -#include cso-user-settings.rc - - -!====================================================================== -!=== -!=== Inquire -!=== -!====================================================================== - - -!----------------------------------------------------------------------- -! inquire DataSpace -!----------------------------------------------------------------------- - -! Obtain names of all available S5p files. -! Stored as csv with processing type, processor version, filenames, etc. - -! full time range: -cso.s5p.no2.inquire-table-dataspace.timerange.start : ${my.full-timerange.start} -cso.s5p.no2.inquire-table-dataspace.timerange.end : ${my.full-timerange.end} - -! API url: -cso.s5p.no2.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api - -! collection name: -cso.s5p.no2.inquire-table-dataspace.collection : Sentinel5P - -! product type, always 10 characters! -! L2__NO2___ -! L2__CO____ -! ... -cso.s5p.no2.inquire-table-dataspace.producttype : L2__NO2___ - -! target area; -!!~ empty for no limitation: -!cso.s5p.no2.inquire-table-dataspace.area : -!~ domain specified as: west,south,east,north -cso.s5p.no2.inquire-table-dataspace.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} - -! template for download url given "{product_id}": -cso.s5p.no2.inquire-table-dataspace.download_url : https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value - -! output table, date of today: -cso.s5p.no2.inquire-table-dataspace.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__${my.region}__%Y-%m-%d.csv - - -!----------------------------------------------------------------------- -! inquire ColHub portal -!----------------------------------------------------------------------- - -! Obtain names of all available S5p files. -! Stored as csv with processing type, processor version, filenames, etc. - -!! inquire full time range: -!cso.s5p.no2.inquire-table-colhub.timerange.start : ${my.full-timerange.start} -!cso.s5p.no2.inquire-table-colhub.timerange.end : ${my.full-timerange.end} -! TESTING ... -cso.s5p.no2.inquire-table-colhub.timerange.start : 2024-01-01 00:00:00 -cso.s5p.no2.inquire-table-colhub.timerange.end : 2025-02-01 00:00:00 - -! -! server url ; -! provide login/password in ~/.netrc: -! -! machine s5phub.copernicus.eu login s5pguest password s5pguest -! -!cso.s5p.no2.inquire-table-colhub.url : https://s5phub.copernicus.eu/dhus -cso.s5p.no2.inquire-table-colhub.url : https://colhub.met.no - -! target area; -!!~ empty for no limitation: -!cso.s5p.no2.inquire-table-colhub.area : -!~ domain specified as: west,south:east,north -cso.s5p.no2.inquire-table-colhub.area : ${my.region.west},${my.region.south}:${my.region.east},${my.region.north} - -! search query, obtained from interactive download: -! -! platformname : Sentinel-5 -! producttype : L2__NO2___ | L2__CO____ (always 10 characters!) -! processinglevel : L2 -! -cso.s5p.no2.inquire-table-colhub.query : platformname:Sentinel-5 AND \ - producttype:L2__NO2___ AND \ - processinglevel:L2 - -! output table, date of today: -cso.s5p.no2.inquire-table-colhub.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub__%Y-%m-%d.csv - - -!----------------------------------------------------------------------- -! inquire mirror archive -!----------------------------------------------------------------------- - -! csv file that will hold records per file with: -! - timerange of pixels in file -! - orbit number -cso.s5p.no2.inquire-table-mirror.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-listing__%Y-%m-%d.csv - -! renew table if file already exists? -cso.s5p.no2.inquire-table-mirror.renew : True - -! base path; example file: -! /x0/x0/x1/S5P_OFFL_L2__NO2____20230701T002408_20230701T020537_29604_03_020500_20230702T161349.nc -cso.s5p.no2.inquire-table-mirror.dir : /lustre/storeB/project/ESAcolhub/production-backend-AOI/S5p/all -! filename filter: -cso.s5p.no2.inquire-table-mirror.pattern : S5P_*_L2__NO2_*.nc - - -! ** create list of missing files - -! renew table if file already exists? -cso.s5p.no2.inquire-table-missing.renew : True - -! csv file that will hold records per file with: -! - timerange of pixels in file -! - orbit number -cso.s5p.no2.inquire-table-missing.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-missing__%Y-%m-%d.csv - -! listing from DataSpace with all potentially available files: -cso.s5p.no2.inquire-table-missing.all.file : ${cso.s5p.no2.inquire-table-dataspace.output.file} -cso.s5p.no2.inquire-table-missing.all.filedate : 2025-01-23 - -! listing from current archive: -cso.s5p.no2.inquire-table-missing.curr.file : ${cso.s5p.no2.inquire-table-mirror.output.file} -cso.s5p.no2.inquire-table-missing.curr.filedate : 2025-01-24 - -! selection of orbits, see "convert" below: -cso.s5p.no2.inquire-table-missing.selection : ${cso.s5p.no2.convert.selection} - - -!!----------------------------------------------------------------------- -!! inquire PAL portal -!!----------------------------------------------------------------------- -! -!! Obtain names of all available S5p files. -!! Stored as csv with processing type, processor version, filenames, etc. -! -!! inquire full time range: -!cso.s5p.no2.inquire-table-pal.timerange.start : ${my.full-timerange.start} -!cso.s5p.no2.inquire-table-pal.timerange.end : ${my.full-timerange.end} -! -!! server url: -!cso.s5p.no2.inquire-table-pal.url : https://data-portal.s5p-pal.com/cat/sentinel-5p/catalog.json -! -!! product type (always 10 characters!): -!cso.s5p.no2.inquire-table-pal.producttype : L2__NO2___ -! -!! target area; -!!!~ empty for no limitation: -!!cso.s5p.no2.inquire-table-pal.area : -!!~ domain specified as: west,south:east,north -!cso.s5p.no2.inquire-table-pal.area : ${my.region.west},${my.region.south}:${my.region.east},${my.region.north} -! -!! output table, date of today: -!cso.s5p.no2.inquire-table-pal.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_pal_%Y-%m-%d.csv - - -!----------------------------------------------------------- -! overview plot of versions -!----------------------------------------------------------- - -! renew existing plots? -cso.s5p.no2.inquire-plot.renew : True - -! listing files: -cso.s5p.no2.inquire-plot.file : ${cso.s5p.no2.inquire-table-dataspace.output.file} -!cso.s5p.no2.inquire-plot.file : ${cso.s5p.no2.inquire-table-colhub.output.file} -!cso.s5p.no2.inquire-plot.file : ${cso.s5p.no2.inquire-table-mirror.output.file} -!!~ specify dates ("yyyy-mm-dd") to use historic tables, -!! default is table of today: -cso.s5p.no2.inquire-plot.filedate : 2025-01-23 - -! annote: -cso.s5p.no2.inquire-plot.title : S5p/NO2 %Y-%m-%d - -! output figure, date of today: -cso.s5p.no2.inquire-plot.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__${my.region}__%Y-%m-%d.png -!cso.s5p.no2.inquire-plot.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub__%Y-%m-%d.png -!cso.s5p.no2.inquire-plot.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-archive__%Y-%m-%d.png - - -!====================================================================== -!=== -!=== download (without convert) -!=== -!====================================================================== - - -! time range: -cso.s5p.no2.download.timerange.start : ${my.timerange.start} -cso.s5p.no2.download.timerange.end : ${my.timerange.end} - -! listing of available source files, -! created by 'inquire-dataspace' job: -cso.s5p.no2.download.inquire.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv -!!~ historic inquire ... -!cso.s5p.no2.download.inquire.filedate : 2023-08-07 - -! selection keyword: -my.s5p.no2.download.selection : C03 - -! Provide ';' seperated list of to decide if a particular orbit file should be processed. -! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing), -! the file with the first match will be used. -! The expressions should include templates '%{header}' for the column values. -! Example to select files from collection '03', preferably from processing 'RPRO' but otherwise from 'OFFL': -! (%{collection} == '03') and (%{processing} == 'RPRO') ; \ -! (%{collection} == '03') and (%{processing} == 'OFFL') -! -#if "${my.s5p.no2.download.selection}" == "C03" -cso.s5p.no2.download.selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ - (%{collection} == '03') and (%{processing} == 'OFFL') -#else -#error unsupported my.s5p.no2.download.selection "${my.s5p.no2.download.selection}" -#endif - -! input directory; -! files are searched here or downloaded to if not present yet; -! supported templates: -! %{processing} -cso.s5p.no2.download.input.dir : ${my.work}/Copernicus/S5P/%{processing}/NO2/%Y/%m - - -!====================================================================== -!=== -!=== convert (and download) -!=== -!====================================================================== - - -! renew existing files (True|False) ? -cso.s5p.no2.convert.renew : True -!cso.s5p.no2.convert.renew : False - -! time range: -cso.s5p.no2.convert.timerange.start : ${my.timerange.start} -cso.s5p.no2.convert.timerange.end : ${my.timerange.end} - - -!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -! input files -!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -! listing of available source files, -! created by 'inquire-dataspace' job: -!cso.s5p.no2.convert.inquire.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv -!cso.s5p.no2.convert.inquire.filedate : ${cso.s5p.no2.inquire-plot.filedate} -! TESTING .. -cso.s5p.no2.convert.inquire.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_colhub-listing__%Y-%m-%d.csv -cso.s5p.no2.convert.inquire.filedate : 2025-01-24 - -! selection keyword: -my.s5p.no2.selection : C03 - -! Provide ';' seperated list of to decide if a particular orbit file should be processed. -! If more than one file is available for a particular orbit (from "OFFL" and "RPRO" processing), -! the file with the first match will be used. -! The expressions should include templates '%{header}' for the column values. -! Example to select files from collection '03', preferably from processing 'RPRO' but otherwise from 'OFFL': -! (%{collection} == '03') and (%{processing} == 'RPRO') ; \ -! (%{collection} == '03') and (%{processing} == 'OFFL') -! -#if "${my.s5p.no2.selection}" == "C03" -cso.s5p.no2.convert.selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ - (%{collection} == '03') and (%{processing} == 'OFFL') -#else -#error unsupported my.s5p.no2.selection "${my.s5p.no2.selection}" -#endif - -! input directory; -! files are searched here or downloaded to if not present yet; -! supported templates: -! %{processing} -cso.s5p.no2.convert.input.dir : ${my.work}/Copernicus/S5P/%{processing}/NO2/%Y/%m - -! remove downloaded input files after convert? -cso.s5p.no2.convert.downloads.cleanup : False - -! selection names: -cso.s5p.no2.convert.filters : lons lats valid quality -!~ optionally also: -! cloud_fraction - -! filter settings: -cso.s5p.no2.convert.filter.lons.var : PRODUCT/longitude -cso.s5p.no2.convert.filter.lons.type : minmax -cso.s5p.no2.convert.filter.lons.minmax : ${my.region.west} ${my.region.east} -cso.s5p.no2.convert.filter.lons.units : degrees_east - -! filter settings: -cso.s5p.no2.convert.filter.lats.var : PRODUCT/latitude -cso.s5p.no2.convert.filter.lats.type : minmax -cso.s5p.no2.convert.filter.lats.minmax : ${my.region.south} ${my.region.north} -cso.s5p.no2.convert.filter.lats.units : degrees_north - -! skip pixel with "no data", use the "qa_value" variable to check: -cso.s5p.no2.convert.filter.valid.var : PRODUCT/qa_value -cso.s5p.no2.convert.filter.valid.type : valid - -! Comment in "PRODUCT/qa_value" variable: -! "A continuous quality descriptor, -! varying between 0 (no data) and 1 (full quality data). -! Recommend to ignore data with qa_value < 0.5" -! Tests suggest that better threshold is 0.54, -! this removes the kernels with very high values. -! 0.75 is the recommended use so what we will use here -cso.s5p.no2.convert.filter.quality.var : PRODUCT/qa_value -cso.s5p.no2.convert.filter.quality.type : min -cso.s5p.no2.convert.filter.quality.min : 0.75 -cso.s5p.no2.convert.filter.quality.units : 1 - -!! recommended 0.3 cloud fraction max (https://agupubs.onlinelibrary.wiley.com/doi/full/10.1029/2018GL081095) -!cso.s5p.no2.convert.filter.cloud_fraction.var : PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/cloud_fraction_crb_nitrogendioxide_window -!cso.s5p.no2.convert.filter.cloud_fraction.type : max -!cso.s5p.no2.convert.filter.cloud_fraction.max : 0.3 -!cso.s5p.no2.convert.filter.cloud_fraction.units : 1 - - - -!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -! output files -!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -! output directory and filename: -! - times are taken from mid of selection, rounded to hours -! - use '%{processing}' for the processing name -! - use '%{orbit}' for orbit number -cso.s5p.no2.convert.output.filename : ${my.work}/CSO-data/${my.region}/S5p/NO2/${my.s5p.no2.selection}/%Y/%m/S5p_NO2_%{orbit}.nc - -! pack variables on output: -cso.s5p.no2.convert.output.packed : True -! zlib compression level, 0 for no compression: -cso.s5p.no2.convert.output.complevel : 1 - -! global attributes: -cso.s5p.no2.convert.output.attrs : format Conventions \ - author institution email -! -cso.s5p.no2.convert.output.attr.format : ${my.cso.format} -cso.s5p.no2.convert.output.attr.Conventions : ${my.cso.convention} -cso.s5p.no2.convert.output.attr.author : ${my.attr.author} -cso.s5p.no2.convert.output.attr.institution : ${my.attr.institution} -cso.s5p.no2.convert.output.attr.email : ${my.attr.email} - -! no need to swap layes: -cso.s5p.no2.convert.swap_layers : False - - -! ~ variables - -! which fields to be put out ? -cso.s5p.no2.convert.output.vars : longitude longitude_bounds \ - latitude latitude_bounds \ - track_longitude track_longitude_bounds \ - track_latitude track_latitude_bounds \ - time \ - qa_value \ - pressure kernel_trop amf amf_trop nla \ - vcd vcd_errvar \ - cloud_fraction cloud_radiance_fraction - -! ... optional ... -! ground_pixel \ -! cloud_pressure_crb \ -! surface_albedo \ -! viewing_zenith_angle \ -! viewing_azimuth_angle \ -! solar_azimuth_angle \ -! solar_zenith_angle \ -! surface_classification \ -! snow_ice_flag - -! -! Describe per variable: -! * .dims : dimensions list: -! pixel : selected pixels -! corner : number of footprint bounds (probably 4) -! layer : number of layers in atmospheric profile (layers in kernel) -! layeri : number of layer interfaces in atmospheric profile (layer+1) -! retr : number of layers in retrieval product (1 for columns) ; -! for error covariance use (retr,retr0) to avoid repeated dimensions -! track_scan : original scan index in 2D track -! track_pixel : original ground pixel in 2D track -! * .specal : keyword to select special processing -! * None : no special processing (default) -! * track_longitude : longitudes at centers of original 2D track -! * track_latitude : latitudes at centers of original 2D track -! * track_longitude_bounds : longitude bounds at centers of original 2D track -! * track_latitude_bounds : latitude bounds at centers of original 2D track -! * .units : target units if different from original -! * .oper : special postprocessing, currently supported: -! * square : fill variable with squared valued (used to form variance from standard deviation) -! In case no special processing is needed: -! * .from : original variable (group path and variable name) -! - -! center longitudes; remove bounds attribute, no coordinate ... -cso.s5p.no2.convert.output.var.longitude.dims : pixel -cso.s5p.no2.convert.output.var.longitude.from : PRODUCT/longitude -cso.s5p.no2.convert.output.var.longitude.attrs : { 'bounds' : None } -! center latitudes; remove bounds attribute, no coordinate ... -cso.s5p.no2.convert.output.var.latitude.dims : pixel -cso.s5p.no2.convert.output.var.latitude.from : PRODUCT/latitude -cso.s5p.no2.convert.output.var.latitude.attrs : { 'bounds' : None } - -! corner longitudes; no units in file: -cso.s5p.no2.convert.output.var.longitude_bounds.dims : pixel corner -cso.s5p.no2.convert.output.var.longitude_bounds.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds -cso.s5p.no2.convert.output.var.longitude_bounds.units : degrees_east -! corner latitudes, no units in file: -cso.s5p.no2.convert.output.var.latitude_bounds.dims : pixel corner -cso.s5p.no2.convert.output.var.latitude_bounds.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds -cso.s5p.no2.convert.output.var.latitude_bounds.units : degrees_north - -! original track: -!~ center lon; remove bounds attribute, no coordinate ... -cso.s5p.no2.convert.output.var.track_longitude.dims : track_scan track_pixel -cso.s5p.no2.convert.output.var.track_longitude.special : track_longitude -cso.s5p.no2.convert.output.var.track_longitude.from : PRODUCT/longitude -cso.s5p.no2.convert.output.var.track_longitude.attrs : { 'bounds' : None } -!~ center lat; remove bounds attribute, no coordinate ... -cso.s5p.no2.convert.output.var.track_latitude.dims : track_scan track_pixel -cso.s5p.no2.convert.output.var.track_latitude.special : track_latitude -cso.s5p.no2.convert.output.var.track_latitude.from : PRODUCT/latitude -cso.s5p.no2.convert.output.var.track_latitude.attrs : { 'bounds' : None } -!~ corner lons -cso.s5p.no2.convert.output.var.track_longitude_bounds.dims : track_scan track_pixel corner -cso.s5p.no2.convert.output.var.track_longitude_bounds.special : track_longitude_bounds -cso.s5p.no2.convert.output.var.track_longitude_bounds.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/longitude_bounds -cso.s5p.no2.convert.output.var.track_longitude_bounds.units : degrees_east -!~ corner lats -cso.s5p.no2.convert.output.var.track_latitude_bounds.dims : track_scan track_pixel corner -cso.s5p.no2.convert.output.var.track_latitude_bounds.special : track_latitude_bounds -cso.s5p.no2.convert.output.var.track_latitude_bounds.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/latitude_bounds -cso.s5p.no2.convert.output.var.track_latitude_bounds.units : degrees_north - -! time value per scan line -cso.s5p.no2.convert.output.var.time.dims : pixel -cso.s5p.no2.convert.output.var.time.special : time-delta -cso.s5p.no2.convert.output.var.time.tref : PRODUCT/time -cso.s5p.no2.convert.output.var.time.dt : PRODUCT/delta_time - -! vertical column density: -cso.s5p.no2.convert.output.var.vcd.dims : pixel retr -cso.s5p.no2.convert.output.var.vcd.from : PRODUCT/nitrogendioxide_tropospheric_column -cso.s5p.no2.convert.output.var.vcd.attrs : { 'ancillary_variables' : None } - -! error variance in vertical column density (after application of kernel), -! fill with single element 'covariance matrix', from square of standard error: -! use dims with different names to avoid that cf-checker complains: -cso.s5p.no2.convert.output.var.vcd_errvar.dims : pixel retr retr0 -cso.s5p.no2.convert.output.var.vcd_errvar.special : square -cso.s5p.no2.convert.output.var.vcd_errvar.from : PRODUCT/nitrogendioxide_tropospheric_column_precision_kernel -!~ skip standard name, modifier "standard_error" is not valid anymore: -cso.s5p.no2.convert.output.var.vcd_errvar.attrs : { 'standard_name' : None } - -! Convert from hybride coefficient bounds in (2,nlev) aray to 3D half level pressure: -cso.s5p.no2.convert.output.var.pressure.dims : pixel layeri -cso.s5p.no2.convert.output.var.pressure.special : hybounds_to_pressure -cso.s5p.no2.convert.output.var.pressure.sp : PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_pressure -cso.s5p.no2.convert.output.var.pressure.hyab : PRODUCT/tm5_constant_a -cso.s5p.no2.convert.output.var.pressure.hybb : PRODUCT/tm5_constant_b -cso.s5p.no2.convert.output.var.pressure.units : Pa - -! (total) airmass factor: -cso.s5p.no2.convert.output.var.amf.dims : pixel retr -cso.s5p.no2.convert.output.var.amf.from : PRODUCT/air_mass_factor_total -cso.s5p.no2.convert.output.var.amf.attrs : { 'coordinates' : None, 'ancillary_variables' : None } - -! tropospheric airmass factor: -cso.s5p.no2.convert.output.var.amf_trop.dims : pixel retr -cso.s5p.no2.convert.output.var.amf_trop.from : PRODUCT/air_mass_factor_troposphere -cso.s5p.no2.convert.output.var.amf_trop.attrs : { 'coordinates' : None, 'ancillary_variables' : None } - -! number of apriori layers in retrieval layer, -! enforce that it is stored as a short integer: -cso.s5p.no2.convert.output.var.nla.dims : pixel retr -cso.s5p.no2.convert.output.var.nla.dtype : i2 -cso.s5p.no2.convert.output.var.nla.from : PRODUCT/tm5_tropopause_layer_index -cso.s5p.no2.convert.output.var.nla.attrs : { 'coordinates' : None, 'ancillary_variables' : None } - -! description: -! kernel := averaging_kernel * amf/amf_trop -! for layers up to l_trop, zero above -cso.s5p.no2.convert.output.var.kernel_trop.dims : pixel layer retr -cso.s5p.no2.convert.output.var.kernel_trop.special : kernel_trop -cso.s5p.no2.convert.output.var.kernel_trop.avk : PRODUCT/averaging_kernel -cso.s5p.no2.convert.output.var.kernel_trop.amf : PRODUCT/air_mass_factor_total -cso.s5p.no2.convert.output.var.kernel_trop.amft : PRODUCT/air_mass_factor_troposphere -cso.s5p.no2.convert.output.var.kernel_trop.troplayer : PRODUCT/tm5_tropopause_layer_index -cso.s5p.no2.convert.output.var.kernel_trop.attrs : { 'coordinates' : None, 'ancillary_variables' : None } - -! cloud property: -cso.s5p.no2.convert.output.var.cloud_fraction.dims : pixel -cso.s5p.no2.convert.output.var.cloud_fraction.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/cloud_fraction_crb -cso.s5p.no2.convert.output.var.cloud_fraction.attrs : { 'coordinates' : None, 'source' : None } - -! cloud property: -cso.s5p.no2.convert.output.var.cloud_radiance_fraction.dims : pixel -cso.s5p.no2.convert.output.var.cloud_radiance_fraction.from : PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/cloud_radiance_fraction_nitrogendioxide_window -cso.s5p.no2.convert.output.var.cloud_radiance_fraction.attrs : { 'coordinates' : None, 'ancillary_variables' : None } - -! quality flag: -cso.s5p.no2.convert.output.var.qa_value.dims : pixel -cso.s5p.no2.convert.output.var.qa_value.from : PRODUCT/qa_value -!~ skip some attributes, cf-checker complains ... -cso.s5p.no2.convert.output.var.qa_value.attrs : { 'valid_min' : None, 'valid_max' : None } - -!! description: -!cso.s5p.no2.convert.output.var.cloud_pressure_crb.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/cloud_pressure_crb -!cso.s5p.no2.convert.output.var.cloud_pressure_crb.units : Pa -!cso.s5p.no2.convert.output.var.cloud_pressure_crb.dims : pixel - -!! description: -!cso.s5p.no2.convert.output.var.surface_albedo.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_albedo_nitrogendioxide_window -!cso.s5p.no2.convert.output.var.surface_albedo.units : 1 -!cso.s5p.no2.convert.output.var.surface_albedo.dims : pixel - -!! description: -!cso.s5p.no2.convert.output.var.viewing_zenith_angle.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/viewing_zenith_angle -!cso.s5p.no2.convert.output.var.viewing_zenith_angle.units : degree -!cso.s5p.no2.convert.output.var.viewing_zenith_angle.dims : pixel - -!! description: -!cso.s5p.no2.convert.output.var.viewing_azimuth_angle.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/viewing_azimuth_angle -!cso.s5p.no2.convert.output.var.viewing_azimuth_angle.units : degree -!cso.s5p.no2.convert.output.var.viewing_azimuth_angle.dims : pixel - -!! description: -!cso.s5p.no2.convert.output.var.solar_azimuth_angle.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/solar_azimuth_angle -!cso.s5p.no2.convert.output.var.solar_azimuth_angle.units : degree -!cso.s5p.no2.convert.output.var.solar_azimuth_angle.dims : pixel - -!! description: -!cso.s5p.no2.convert.output.var.solar_zenith_angle.from : PRODUCT/SUPPORT_DATA/GEOLOCATIONS/solar_zenith_angle -!cso.s5p.no2.convert.output.var.solar_zenith_angle.units : degree -!cso.s5p.no2.convert.output.var.solar_zenith_angle.dims : pixel - -!! description: -!cso.s5p.no2.convert.output.var.snow_ice_flag.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/snow_ice_flag -!cso.s5p.no2.convert.output.var.snow_ice_flag.units : None -!cso.s5p.no2.convert.output.var.snow_ice_flag.dims : pixel - -!! description: -!cso.s5p.no2.convert.output.var.surface_classification.from : PRODUCT/SUPPORT_DATA/INPUT_DATA/surface_classification -!cso.s5p.no2.convert.output.var.surface_classification.units : None -!cso.s5p.no2.convert.output.var.surface_classification.dims : pixel - -!! for traceback: -!cso.s5p.no2.convert.output.var.orbit_number.special : orbit_number -!cso.s5p.no2.convert.output.var.orbit_number.units : 1 -!cso.s5p.no2.convert.output.var.orbit_number.dims : pixel - -!! for traceback: -!cso.s5p.no2.convert.output.var.image_number.special : scan_number -!cso.s5p.no2.convert.output.var.image_number.units : 1 -!cso.s5p.no2.convert.output.var.image_number.dims : pixel - -!! for traceback: -!cso.s5p.no2.convert.output.var.ground_pixel.dims : track_pixel -!cso.s5p.no2.convert.output.var.ground_pixel.special : ground_pixel -!cso.s5p.no2.convert.output.var.ground_pixel.from : PRODUCT/ground_pixel - - - - -!====================================================================== -!=== -!=== listing -!=== -!====================================================================== - -! csv file that will hold records per file with: -! - timerange of pixels in file -! - orbit number -cso.s5p.no2.listing.file : ${my.work}/CSO-data/${my.region}/S5p/NO2/${my.s5p.no2.selection}__listing.csv - -! renew table if file already exists? -cso.s5p.no2.listing.renew : True - -! time range: -cso.s5p.no2.listing.timerange.start : ${my.timerange.start} -cso.s5p.no2.listing.timerange.end : ${my.timerange.end} - -! filename filters relative to listing file that should be scanned for orbit files; -! names could include time templates ; -! if same orbit is found in multiple directories, the first found is used; -! remove existing table for safety to ensure that this is done correctly ... -cso.s5p.no2.listing.patterns : ${my.s5p.no2.selection}/%Y/%m/S5p_*.nc - -! extra columns to be added, read from global attributes: -cso.s5p.no2.listing.xcolumns : orbit - - -!====================================================================== -!=== -!=== catalogue -!=== -!====================================================================== - -! listing file with filenames/timerange. -cso.s5p.no2.catalogue.input.listing : ${cso.s5p.no2.listing.file} - -! time range: -cso.s5p.no2.catalogue.timerange.start : ${my.timerange.start} -cso.s5p.no2.catalogue.timerange.end : ${my.timerange.end} - -! target filenames; templates: -! - time values -! - %{orbit} : from listing -! - %{varname} for variable -cso.s5p.no2.catalogue.output.file : ${my.work}/CSO-data-catalogue/${my.region}/S5p/NO2/${my.s5p.no2.selection}/%Y/%m/%d/S5p_NO2_%{orbit}__%{varname}.png - -! map domain (west east south north) -cso.s5p.no2.catalogue.domain : ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north} - -! figure size (inches), default is (8,6): -cso.s5p.no2.catalogue.figsize : ${my.region.figsize} - -! renew existing files? -!cso.s5p.no2.catalogue.renew : False -cso.s5p.no2.catalogue.renew : True - -! variables to be plotted: -cso.s5p.no2.catalogue.vars : vcd vcd_errvar qa_value \ - amf amf_trop nla \ - cloud_fraction cloud_radiance_fraction - -!! color for no-data values in track, default '0.8' (gray): -!cso.s5p.no2.catalogue.color_nan : white - -!! extra keyword arguments for map: -!cso.s5p.no2.catalogue.map : resolution='h' - -! convert units: -cso.s5p.no2.catalogue.var.vcd.units : umol/m2 -! style: -cso.s5p.no2.catalogue.var.vcd.vmin : 0.0 -cso.s5p.no2.catalogue.var.vcd.vmax : 100.0 - -! show error as std.dev, convert to vcd units: -cso.s5p.no2.catalogue.var.vcd_errvar.units : umol/m2 -! style: -cso.s5p.no2.catalogue.var.vcd_errvar.vmax : 100.0 - -! style: -cso.s5p.no2.catalogue.var.amf.vmin : 0.0 -cso.s5p.no2.catalogue.var.amf.vmax : 4.0 - -! style: -cso.s5p.no2.catalogue.var.amf_trop.vmin : 0.0 -cso.s5p.no2.catalogue.var.amf_trop.vmax : 4.0 - -! style: -cso.s5p.no2.catalogue.var.nla.vmin : 1 -cso.s5p.no2.catalogue.var.nla.vmax : 34 - -! style: -cso.s5p.no2.catalogue.var.qa_value.vmin : 0.5 -cso.s5p.no2.catalogue.var.qa_value.vmax : 1.0 -cso.s5p.no2.catalogue.var.qa_value.colors : ['red','yellow','green'] - -! style: -cso.s5p.no2.catalogue.var.cloud_fraction.vmax : 1.0 -cso.s5p.no2.catalogue.var.cloud_fraction.colors : ['blue','cyan','white'] - -! style: -cso.s5p.no2.catalogue.var.cloud_radiance_fraction.vmax : 1.0 -cso.s5p.no2.catalogue.var.cloud_radiance_fraction.colors : ['blue','cyan','white'] - - -!----------------------------------------------------------- -! catalogue index -!----------------------------------------------------------- - -! target location: -cso.s5p.no2.catalogue-index.outdir : ${my.work}/CSO-data-catalogue/${my.region}/S5p/NO2/${my.s5p.no2.selection} - -! title: -cso.s5p.no2.catalogue-index.header : CSO catalogue -! show info on created page? -cso.s5p.no2.catalogue-index.info : True - -! create new page for each value? -cso.s5p.no2.catalogue-index.newpage : True - -! content type: -cso.s5p.no2.catalogue-index.type : list -! define row values: -cso.s5p.no2.catalogue-index.name : date -cso.s5p.no2.catalogue-index.values : TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' ) - -! create new page for each value: -cso.s5p.no2.catalogue-index.date.newpage : True -! content type: -cso.s5p.no2.catalogue-index.date.type : table-row -! define row values: -cso.s5p.no2.catalogue-index.date.name : orbit -cso.s5p.no2.catalogue-index.date.values : CsvFile( '%{date[0:4]}/%{date[4:6]}/%{date[6:8]}/orbits.csv' ) - -! content type: -cso.s5p.no2.catalogue-index.date.orbit.type : table-col -! define column values: -cso.s5p.no2.catalogue-index.date.orbit.name : var -cso.s5p.no2.catalogue-index.date.orbit.values : ${cso.s5p.no2.catalogue.vars} - -! content type: -cso.s5p.no2.catalogue-index.date.orbit.var.type : img -! define image: -cso.s5p.no2.catalogue-index.date.orbit.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{orbit}__%{var}.png -cso.s5p.no2.catalogue-index.date.orbit.var.kwargs : height=300 - - - -!====================================================================== -!=== -!=== gridded orbits -!=== -!====================================================================== - - -!----------------------------------------------------------- -! gridded orbits -!----------------------------------------------------------- - -! time range: -cso.s5p.no2.gridded.timerange.start : ${my.timerange.start} -cso.s5p.no2.gridded.timerange.end : ${my.timerange.end} -! create one gridded file per hour: -cso.s5p.no2.gridded.timerange.step : hour - -! renew existing files? -cso.s5p.no2.gridded.renew : True - -! target directory, incl. subdir for resolution and filters: -my.gridded.dir : ${my.work}/CSO-gridded/${my.region}__r01x01/S5p/NO2/${my.s5p.no2.selection}__${my.s5p.no2.gridded-selection} - -!~ - -! grid definition: -!~ same as pixel selection on conversion: -cso.s5p.no2.gridded.grid.west : ${my.region.west} -cso.s5p.no2.gridded.grid.east : ${my.region.east} -cso.s5p.no2.gridded.grid.south : ${my.region.south} -cso.s5p.no2.gridded.grid.north : ${my.region.north} -! resolution: -cso.s5p.no2.gridded.grid.dlon : 0.1 -cso.s5p.no2.gridded.grid.dlat : 0.1 - -! level of recursive splitting of footprint into triangles, -! and assignment of centroids to grid cells; -! for 4-corner footprints, number of centroids is: -! 1 (levels=0), 4 (1), 8 (2), 16 (3), 64 (5), 256 (7) -cso.s5p.no2.gridded.mapping.levels : 5 - -!~ - -! keywords for source files; -! the first one should have the footprints; -! here the only source are the converted files: -cso.s5p.no2.gridded.sources : data - -! input files for each source type: -!~ here: specify listing file, -! this is only supported for a single source -cso.s5p.no2.gridded.source.data.listing : ${cso.s5p.no2.listing.file} -!!~ alternative: filename patterns with time templates, -!! but here the source files do not have time stamps -!cso.s5p.no2.gridded.source.data.filenames : ${my.work}/CSO-data/S5p/RPRO/NO2/${my.region}/%Y/%m/S5p_*.nc - -!~ - -! filter description: -my.s5p.no2.gridded-selection : qa08 - -! switch: -#if "${my.s5p.no2.gridded-selection}" == "qa08" - -! keywords for filters: -cso.s5p.no2.gridded.filters : quality - -! minimum quality value required: -cso.s5p.no2.gridded.filter.quality.var : qa_value -cso.s5p.no2.gridded.filter.quality.type : min -cso.s5p.no2.gridded.filter.quality.min : 0.8 -cso.s5p.no2.gridded.filter.quality.units : 1 - -#else -#error unsupported my.s5p.no2.gridded-selection "${my.s5p.no2.gridded-selection}" -#endif - - -!~ - -! target file, might contain templates: -! %Y,%m,etc : time values -cso.s5p.no2.gridded.output.file : ${my.gridded.dir}/%Y/%m/S5p_NO2_%Y%m%d_%H%M_gridded.nc - -! pack floats as short values? -cso.s5p.no2.gridded.output.packed : True -! zlib compression level (default 1, 0 for no compression): -cso.s5p.no2.gridded.output.complevel : 1 - -! data variables to be created: -cso.s5p.no2.gridded.output.vars : yr - -! input variables: -! data:vcd : variable "vcd" from source "data" -cso.s5p.no2.gridded.output.yr.source : data:vcd - - -!----------------------------------------------------------- -! catalogue of gridded fields -!----------------------------------------------------------- - -! time range: -cso.s5p.no2.gridded-catalogue.timerange.start : ${my.timerange.start} -cso.s5p.no2.gridded-catalogue.timerange.end : ${my.timerange.end} -! create one gridded file per hour: -cso.s5p.no2.gridded-catalogue.timerange.step : hour - -! renew existing files? -cso.s5p.no2.gridded-catalogue.renew : True -!cso.s5p.no2.gridded-catalogue.renew : False - -! target directory for catalogue: -my.no2.gridded-catalogue.output.dir : ${my.gridded.dir}/catalogue - -! input files: -cso.s5p.no2.gridded-catalogue.input.file : ${my.gridded.dir}/%Y/%m/S5p_NO2_%Y%m%d_%H%M_gridded.nc -!!~ idem for daily average: -!cso.s5p.no2.gridded-catalogue.input.file : ${my.gridded.dir}/S5p_NO2_%Y%m%d_aver_gridded.nc - -! target files, time tempates and variable name are replaced: -cso.s5p.no2.gridded-catalogue.output.file : ${my.no2.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_gridded_%{var}.png -!!~ idem for daily average: -!cso.s5p.no2.gridded-catalogue.output.file : ${my.no2.gridded-catalogue.output.dir}/%Y/%m/%d/S5p_NO2_%Y%m%d_aver_gridded_%{var}.png - -! figure size (inches), default is (8,6): -cso.s5p.no2.gridded-catalogue.figsize : (6,6) - -! variables to be plotted: -cso.s5p.no2.gridded-catalogue.vars : yr - -! variable: -cso.s5p.no2.gridded-catalogue.var.yr.source : yr -! convert units: -cso.s5p.no2.gridded-catalogue.var.yr.units : umol/m2 -! style: -cso.s5p.no2.gridded-catalogue.var.yr.vmin : 0.0 -cso.s5p.no2.gridded-catalogue.var.yr.vmax : 100.0 - - - - -!----------------------------------------------------------- -! gridded catalogue index -!----------------------------------------------------------- - -! target location: -cso.s5p.no2.gridded-catalogue-index.outdir : ${my.no2.gridded-catalogue.output.dir} - -! title: -cso.s5p.no2.gridded-catalogue-index.header : CSO catalogue -! show info on created page? -cso.s5p.no2.gridded-catalogue-index.info : True - -! create new page for each value? -cso.s5p.no2.gridded-catalogue-index.newpage : True - -! content type: -cso.s5p.no2.gridded-catalogue-index.type : list -! define row values: -cso.s5p.no2.gridded-catalogue-index.name : date -cso.s5p.no2.gridded-catalogue-index.values : TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' ) - -! create new page for each value: -cso.s5p.no2.gridded-catalogue-index.date.newpage : True -! content type: -cso.s5p.no2.gridded-catalogue-index.date.type : table-row -! define row values: -cso.s5p.no2.gridded-catalogue-index.date.name : time -cso.s5p.no2.gridded-catalogue-index.date.values : Range( 0, 23, 1, '%2.2i00' ) - -! content type: -cso.s5p.no2.gridded-catalogue-index.date.time.type : table-col -! define column values: -cso.s5p.no2.gridded-catalogue-index.date.time.name : var -cso.s5p.no2.gridded-catalogue-index.date.time.values : ${cso.s5p.no2.gridded-catalogue.vars} - -! content type: -cso.s5p.no2.gridded-catalogue-index.date.time.var.type : img -! define image: -cso.s5p.no2.gridded-catalogue-index.date.time.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_gridded_%{var}.png -cso.s5p.no2.gridded-catalogue-index.date.time.var.kwargs : height=300 - - - -!====================================================================== -!=== -!=== simulation catalogue -!=== -!====================================================================== - - -!----------------------------------------------------------- -! simulation catalogue -!----------------------------------------------------------- - -! time range: -cso.s5p.no2.sim-catalogue.timerange.start : ${my.timerange.start} -cso.s5p.no2.sim-catalogue.timerange.end : ${my.timerange.end} -cso.s5p.no2.sim-catalogue.timerange.step : hour - -! input files: -cso.s5p.no2.sim-catalogue.input.data.file : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc -cso.s5p.no2.sim-catalogue.input.state.file : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc - -! target files, time tempates and variable name are replaced: -cso.s5p.no2.sim-catalogue.output.file : ${my.work}/CSO-data-sim-catalogue/S5p/NO2/${my.region}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_%{var}.png - -! map domain used for simulations (west east south north): -cso.s5p.no2.sim-catalogue.domain : ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north} - -! figure size (inches), default is (8,6): -cso.s5p.no2.sim-catalogue.figsize : ${my.region.figsize} - -! renew existing files? -cso.s5p.no2.sim-catalogue.renew : False -!cso.s5p.no2.sim-catalogue.renew : True - -! variables to be plotted: -cso.s5p.no2.sim-catalogue.vars : yr ys yr_m ys_m - -! variable: -cso.s5p.no2.sim-catalogue.var.yr.source : data:yr -! convert units: -cso.s5p.no2.sim-catalogue.var.yr.units : umol/m2 -! style: -cso.s5p.no2.sim-catalogue.var.yr.vmin : 0.0 -cso.s5p.no2.sim-catalogue.var.yr.vmax : 100.0 - -! variable: -cso.s5p.no2.sim-catalogue.var.ys.source : state:ys -! convert units: -cso.s5p.no2.sim-catalogue.var.ys.units : umol/m2 -! style: -cso.s5p.no2.sim-catalogue.var.ys.vmin : 0.0 -cso.s5p.no2.sim-catalogue.var.ys.vmax : 100.0 - -! variable: -cso.s5p.no2.sim-catalogue.var.yr_m.source : state:yr_m -! convert units: -cso.s5p.no2.sim-catalogue.var.yr_m.units : umol/m2 -! style: -cso.s5p.no2.sim-catalogue.var.yr_m.vmin : 0.0 -cso.s5p.no2.sim-catalogue.var.yr_m.vmax : 100.0 - -! variable: -cso.s5p.no2.sim-catalogue.var.ys_m.source : state:ys_m -! convert units: -cso.s5p.no2.sim-catalogue.var.ys_m.units : umol/m2 -! style: -cso.s5p.no2.sim-catalogue.var.ys_m.vmin : 0.0 -cso.s5p.no2.sim-catalogue.var.ys_m.vmax : 100.0 - - - -!----------------------------------------------------------- -! sim catalogue index -!----------------------------------------------------------- - -! target location: -cso.s5p.no2.sim-catalogue-index.outdir : ${my.work}/CSO-sim-catalogue/S5p/NO2/${my.region} - -! title: -cso.s5p.no2.sim-catalogue-index.header : CSO catalogue -! show info on created page? -cso.s5p.no2.sim-catalogue-index.info : True - -! create new page for each value? -cso.s5p.no2.sim-catalogue-index.newpage : True - -! content type: -cso.s5p.no2.sim-catalogue-index.type : list -! define row values: -cso.s5p.no2.sim-catalogue-index.name : date -cso.s5p.no2.sim-catalogue-index.values : TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' ) - -! create new page for each value: -cso.s5p.no2.sim-catalogue-index.date.newpage : True -! content type: -cso.s5p.no2.sim-catalogue-index.date.type : table-row -! define row values: -cso.s5p.no2.sim-catalogue-index.date.name : time -cso.s5p.no2.sim-catalogue-index.date.values : Range( 0, 23, 1, '%2.2i00' ) - -! content type: -cso.s5p.no2.sim-catalogue-index.date.time.type : table-col -! define column values: -cso.s5p.no2.sim-catalogue-index.date.time.name : var -cso.s5p.no2.sim-catalogue-index.date.time.values : ${cso.s5p.no2.sim-catalogue.vars} - -! content type: -cso.s5p.no2.sim-catalogue-index.date.time.var.type : img -! define image: -cso.s5p.no2.sim-catalogue-index.date.time.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_%{var}.png -cso.s5p.no2.sim-catalogue-index.date.time.var.kwargs : height=300 - - - -!====================================================================== -!=== -!=== gridded simulated orbits -!=== -!====================================================================== - - -!----------------------------------------------------------- -! gridded orbits -!----------------------------------------------------------- - -! time range: -cso.s5p.no2.sim-gridded.timerange.start : ${my.timerange.start} -cso.s5p.no2.sim-gridded.timerange.end : ${my.timerange.end} -! create one gridded file per hour: -cso.s5p.no2.sim-gridded.timerange.step : hour - -! renew existing files? -cso.s5p.no2.sim-gridded.renew : True -!cso.s5p.no2.sim-gridded.renew : False - -! target directory, incl. subdir for resolution and filters: -my.sim-gridded.dir : ${my.work}/CSO-sim-gridded/${my.region}__r01x01__qa08 - -!~ - -! testing .. -my.gridded.region : ${my.region} -! grid definition: -!!~ same as pixel selection on conversion: -!cso.s5p.no2.sim-gridded.grid.west : ${my.region.west} -!cso.s5p.no2.sim-gridded.grid.east : ${my.region.east} -!cso.s5p.no2.sim-gridded.grid.south : ${my.region.south} -!cso.s5p.no2.sim-gridded.grid.north : ${my.region.north} -!~ observation operator tutorial: -cso.s5p.no2.sim-gridded.grid.west : -10 -cso.s5p.no2.sim-gridded.grid.east : 30 -cso.s5p.no2.sim-gridded.grid.south : 35 -cso.s5p.no2.sim-gridded.grid.north : 65 -! resolution: -cso.s5p.no2.sim-gridded.grid.dlon : 0.1 -cso.s5p.no2.sim-gridded.grid.dlat : 0.1 - -! level of recursive splitting of footprint into triangles, -! and assignment of centroids to grid cells; -! for 4-corner footprints, number of centroids is: -! 1 (levels=0), 4 (1), 8 (2), 16 (3), 64 (5), 256 (7) -cso.s5p.no2.sim-gridded.mapping.levels : 5 - -!~ - -! keywords for source files; -! the first one should have the footprints: -cso.s5p.no2.sim-gridded.sources : data state - -! input files for each source type: -cso.s5p.no2.sim-gridded.source.data.filenames : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc -cso.s5p.no2.sim-gridded.source.state.filenames : ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc - -!~ - -! keywords for filters: -cso.s5p.no2.sim-gridded.filters : quality - -! minimum quality value required: -cso.s5p.no2.sim-gridded.filter.quality.var : qa_value -cso.s5p.no2.sim-gridded.filter.quality.type : min -cso.s5p.no2.sim-gridded.filter.quality.min : 0.8 -cso.s5p.no2.sim-gridded.filter.quality.units : 1 - -!~ - -! target file, might contain templates: -! %Y,%m,etc : time values -! %{basename} : basename (without extension) of first source file -cso.s5p.no2.sim-gridded.output.file : ${my.sim-gridded.dir}/%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc - -! pack variables on output: -cso.s5p.no2.sim-gridded.output.packed : True -! zlib compression level, 0 for no compression: -cso.s5p.no2.sim-gridded.output.complevel : 1 - -! data variables to be created: -cso.s5p.no2.sim-gridded.output.vars : yr ys yr_m ys_m - -! input variables: -! data:yr : from data file -! state:ys : from state file -cso.s5p.no2.sim-gridded.output.yr.source : data:yr -cso.s5p.no2.sim-gridded.output.ys.source : state:ys -cso.s5p.no2.sim-gridded.output.yr_m.source : state:yr_m -cso.s5p.no2.sim-gridded.output.ys_m.source : state:ys_m - - -!----------------------------------------------------------- -! catalogue of gridded simulations -!----------------------------------------------------------- - -! time range: -cso.s5p.no2.sim-gridded-catalogue.timerange.start : ${my.timerange.start} -cso.s5p.no2.sim-gridded-catalogue.timerange.end : ${my.timerange.end} -! hourly fields: -cso.s5p.no2.sim-gridded-catalogue.timerange.step : hour - -! renew existing files? -cso.s5p.no2.sim-gridded-catalogue.renew : True -!cso.s5p.no2.sim-gridded-catalogue.renew : False - -! input files: -cso.s5p.no2.sim-gridded-catalogue.input.file : ${my.sim-gridded.dir}/%Y/%m/CSO_output_%Y%m%d_%H%M_gridded.nc -!!~ idem for daily average: -!cso.s5p.no2.sim-gridded-catalogue.input.file : ${my.sim-gridded.dir}/CSO_output_%Y%m%d_aver_gridded.nc - -! target files, time tempates and variable name are replaced: -cso.s5p.no2.sim-gridded-catalogue.output.file : ${my.sim-gridded.dir}/catalogue/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_gridded_%{var}.png -!!~ idem for daily average: -!cso.s5p.no2.sim-gridded-catalogue.output.file : ${my.sim-gridded.dir}/catalogue/%Y/%m/%d/S5p_NO2_%Y%m%d_aver_gridded_%{var}.png - -! figure size (inches), default is (8,6): -cso.s5p.no2.sim-gridded-catalogue.figsize : (6,6) - -! variables to be plotted: -cso.s5p.no2.sim-gridded-catalogue.vars : yr ys yr_m ys_m - -! variable: -cso.s5p.no2.sim-gridded-catalogue.var.yr.source : yr -! convert units: -cso.s5p.no2.sim-gridded-catalogue.var.yr.units : umol/m2 -! style: -cso.s5p.no2.sim-gridded-catalogue.var.yr.vmin : 0.0 -cso.s5p.no2.sim-gridded-catalogue.var.yr.vmax : 100.0 - -! variable: -cso.s5p.no2.sim-gridded-catalogue.var.ys.source : ys -! convert units: -cso.s5p.no2.sim-gridded-catalogue.var.ys.units : umol/m2 -! style: -cso.s5p.no2.sim-gridded-catalogue.var.ys.vmin : 0.0 -cso.s5p.no2.sim-gridded-catalogue.var.ys.vmax : 100.0 - -! variable: -cso.s5p.no2.sim-gridded-catalogue.var.yr_m.source : yr_m -! convert units: -cso.s5p.no2.sim-gridded-catalogue.var.yr_m.units : umol/m2 -! style: -cso.s5p.no2.sim-gridded-catalogue.var.yr_m.vmin : 0.0 -cso.s5p.no2.sim-gridded-catalogue.var.yr_m.vmax : 100.0 - -! variable: -cso.s5p.no2.sim-gridded-catalogue.var.ys_m.source : ys_m -! convert units: -cso.s5p.no2.sim-gridded-catalogue.var.ys_m.units : umol/m2 -! style: -cso.s5p.no2.sim-gridded-catalogue.var.ys_m.vmin : 0.0 -cso.s5p.no2.sim-gridded-catalogue.var.ys_m.vmax : 100.0 - - - -!----------------------------------------------------------- -! gridded catalogue index -!----------------------------------------------------------- - -! target location: -cso.s5p.no2.sim-gridded-catalogue-index.outdir : ${my.sim-gridded.dir}/catalogue - -! title: -cso.s5p.no2.sim-gridded-catalogue-index.header : CSO catalogue -! show info on created page? -cso.s5p.no2.sim-gridded-catalogue-index.info : True - -! create new page for each value? -cso.s5p.no2.sim-gridded-catalogue-index.newpage : True - -! content type: -cso.s5p.no2.sim-gridded-catalogue-index.type : list -! define row values: -cso.s5p.no2.sim-gridded-catalogue-index.name : date -cso.s5p.no2.sim-gridded-catalogue-index.values : TimeSeries( ${my.timerange.start}, ${my.timerange.end}, '1 day', '%Y%m%d' ) - -! create new page for each value: -cso.s5p.no2.sim-gridded-catalogue-index.date.newpage : True -! content type: -cso.s5p.no2.sim-gridded-catalogue-index.date.type : table-row -! define row values: -cso.s5p.no2.sim-gridded-catalogue-index.date.name : time -cso.s5p.no2.sim-gridded-catalogue-index.date.values : Range( 0, 23, 1, '%2.2i00' ) - -! content type: -cso.s5p.no2.sim-gridded-catalogue-index.date.time.type : table-col -! define column values: -cso.s5p.no2.sim-gridded-catalogue-index.date.time.name : var -cso.s5p.no2.sim-gridded-catalogue-index.date.time.values : ${cso.s5p.no2.sim-gridded-catalogue.vars} - -! content type: -cso.s5p.no2.sim-gridded-catalogue-index.date.time.var.type : img -! define image: -cso.s5p.no2.sim-gridded-catalogue-index.date.time.var.img : %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_gridded_%{var}.png -cso.s5p.no2.sim-gridded-catalogue-index.date.time.var.kwargs : height=300 - - -!====================================================================== -!=== -!=== end -!=== -!====================================================================== - - diff --git a/config/_test-colhub/cso-user-settings.rc b/config/_test-colhub/cso-user-settings.rc deleted file mode 100644 index 10bc13f..0000000 --- a/config/_test-colhub/cso-user-settings.rc +++ /dev/null @@ -1,169 +0,0 @@ -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!!! -!!! CSO common configuration -!!! -!!! Base settings that are used by multiple tasks: -!!! - time range(s) -!!! - target domain(s) -!!! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - -!----------------------------------------------------------- -! id's -!----------------------------------------------------------- - -! file format: -my.cso.format : 1.0 - -! file format convention: -my.cso.convention : CF-1.7 - - -!----------------------------------------------------------- -! domain -!----------------------------------------------------------- - -! -! Used for: -! - orbit selection durning download -! - pixel selection -! - map plots -! - -! region name: -!my.region : globe -!my.region : CAMS -my.region : xEMEP - -! switch: -!.................................... -#if "${my.region}" == "globe" -!.................................... - -! global domain: -my.region.west : -180.0 -my.region.east : 180.0 -my.region.south : -90.0 -my.region.north : 90.0 - -! size of map figures for this region, default size is (8,6) -my.region.figsize : (8,6) - -!.................................... -#elif "${my.region}" == "CAMS" -!.................................... - -! CAMS regional ensemble domain: -my.region.west : -30.0 -my.region.east : 45.0 -my.region.south : 30.0 -my.region.north : 76.0 - -! size of map figures for this region, default size is (8,6) -my.region.figsize : (8,6) - -!.................................... -#elif "${my.region}" == "xEMEP" -!.................................... - -! outer bound of all known domains: -my.region.west : -50.0 -my.region.east : 90.0 -my.region.south : 25.0 -my.region.north : 85.0 - -! size of map figures for this region, default size is (8,6) -my.region.figsize : (8,5) - -!.................................... -#else -#error unsupported my.region "${my.region}" -#endif -!.................................... - - - - -!---------------------------------------------------------- -! timerange -!---------------------------------------------------------- - -!! full range, used for inquiry jobs: -!my.full-timerange.start : 2018-01-01 00:00 -!my.full-timerange.end : 2024-12-31 23:59 - -! TESTING: 1 month ... -my.full-timerange.start : 2023-01-01 00:00 -my.full-timerange.end : 2023-01-31 23:59 - -! processing for selected period: -my.timerange.start : 2018-06-01 00:00 -my.timerange.end : 2018-06-01 23:59 - -!! SO2-COBRA -!my.timerange.start : 2023-06-01 00:00 -!my.timerange.end : 2023-06-01 23:59 - - -!---------------------------------------------------------- -! user specific settings: -!---------------------------------------------------------- - -!.............................. -#if "${USER}" == "you" -!.............................. - -! Attributes written to output files. -my.attr.author : Your Name -my.attr.institution : CSO -my.attr.email : Your.Name@cso.org - -! base location for work directories: -my.work : /Scratch/${USER}/CSO-Copernicus - -! storage for downloaded observations: -my.observations : /Scratch/${USER}/observations - -!.............................. -#elif "${USER}" == "arjos" -!.............................. - -! Attributes written to output files. -my.attr.author : Arjo Segers -my.attr.institution : MET Norway -my.attr.email : arjos@met.no - -! base location for work directories: -my.work : ${WORK}/projects/SESAM/test-colhub - -! storage for downloaded observations: -my.observations : ${WORK}/../observations - -!.............................. -#elif "${USER}" == "segersaj" -!.............................. - -! Attributes written to output files. -my.attr.author : Arjo Segers -my.attr.institution : TNO -my.attr.email : Arjo.Segers@tno.nl - -! base location for work directories: -my.work : ${SCRATCH}/CSO-Copernicus - -! storage for downloaded observations: -my.observations : ${SCRATCH}/observations - -!.............................. -#else -#error unsupported USER "${USER}" -#endif -!.............................. - - -!---------------------------------------------------------- -! end -!---------------------------------------------------------- - - diff --git a/config/_test-colhub/cso.rc b/config/_test-colhub/cso.rc deleted file mode 100644 index 3de54af..0000000 --- a/config/_test-colhub/cso.rc +++ /dev/null @@ -1,311 +0,0 @@ -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!!! -!!! CSO - CAMS Satellite Operator -!!! -!!! Settings for project. -!!! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -! dummy keys, actual values defined in environment by "cso" script: -CSO_RCFILE : -CSO_RCDIR : - - -!---------------------------------------------------------- -! user specific settings: -!---------------------------------------------------------- - -! include user specfic settings: -#include cso-user-settings.rc - -! selected tracers: -!my.tracers : no2 so2 so2-cobra hcho co chocho -!~ one by one .. -my.tracers : no2 -!my.tracers : so2 -!my.tracers : so2-cobra -!my.tracers : hcho -!my.tracers : co -!my.tracers : o3-col -!my.tracers : o3-pr -!my.tracers : chocho - - -!---------------------------------------------------------- -! job tree -!---------------------------------------------------------- - -! class to create a job tree: -cso.class : utopya.UtopyaJobTree -! list of sub-elements: -cso.elements : copy s5p - -! class to create a job tree: -cso.s5p.class : utopya.UtopyaJobTree -! list of sub-elements: -cso.s5p.elements : ${my.tracers} - - -! * - -#for TRACER in ${my.tracers} - -! class to create a job tree: -cso.s5p.TRACER.class : utopya.UtopyaJobTree - -! list of sub-elements: -!cso.s5p.TRACER.elements : inquire \ -! convert listing \ -! catalogue \ -! gridded gridded-catalogue -!~ preprocessor steps one by one ... -cso.s5p.TRACER.elements : inquire -!cso.s5p.TRACER.elements : convert listing catalogue -!cso.s5p.TRACER.elements : convert -!cso.s5p.TRACER.elements : listing -!cso.s5p.TRACER.elements : catalogue -!cso.s5p.TRACER.elements : gridded -!cso.s5p.TRACER.elements : gridded-catalogue - -!!~ downloading only, no convert: -!cso.s5p.TRACER.elements : download -! -!~ process simulator output: -!cso.s5p.TRACER.elements : sim-catalogue -!cso.s5p.TRACER.elements : sim-gridded -!cso.s5p.TRACER.elements : sim-gridded-catalogue - -! * - -! single step: -cso.s5p.TRACER.inquire.class : utopya.UtopyaJobStep -! two or more tasks: -#if "TRACER" in ["no2","so2","hcho","co","o3-pr","o3-col"] -!cso.s5p.TRACER.inquire.tasks : table-dataspace plot -cso.s5p.TRACER.inquire.tasks : table-dataspace -!cso.s5p.TRACER.inquire.tasks : table-mirror -!cso.s5p.TRACER.inquire.tasks : table-missing -!cso.s5p.TRACER.inquire.tasks : plot -#elif "TRACER" in ["so2-cobra"] -cso.s5p.TRACER.inquire.tasks : table-pal plot -#elif "TRACER" in ["chocho"] -cso.s5p.TRACER.inquire.tasks : table-glyretro table-pal plot -#else -#error unsupported tracer "TRACER" -#endif -!~ inquire files available on DataSpace: -cso.s5p.TRACER.inquire.table-dataspace.class : cso.CSO_DataSpace_Inquire -cso.s5p.TRACER.inquire.table-dataspace.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.inquire-table-dataspace' -!~ inquire files available on ColHub: -cso.s5p.TRACER.inquire.table-colhub.class : cso.CSO_ColHub_Inquire -cso.s5p.TRACER.inquire.table-colhub.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.inquire-table-colhub' -!~ inquire file archive: -cso.s5p.TRACER.inquire.table-mirror.class : cso.CSO_ColHubMirror_Inquire -cso.s5p.TRACER.inquire.table-mirror.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.inquire-table-mirror' -!~ create table with files that are missing: -cso.s5p.TRACER.inquire.table-missing.class : cso.CSO_ColHubMirror_Missing -cso.s5p.TRACER.inquire.table-missing.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.inquire-table-missing' -!~ inquire files available on PAL: -cso.s5p.TRACER.inquire.table-pal.class : cso.CSO_PAL_Inquire -cso.s5p.TRACER.inquire.table-pal.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.inquire-table-pal' -!~ inquire files downloaded from GlyRetro: -cso.s5p.TRACER.inquire.table-glyretro.class : cso.CSO_S5p_Download_Listing -cso.s5p.TRACER.inquire.table-glyretro.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.inquire-table-glyretro' -!~ create plot of available versions: -cso.s5p.TRACER.inquire.plot.class : cso.CSO_Inquire_Plot -cso.s5p.TRACER.inquire.plot.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.inquire-plot' - -! * - -! single step: -cso.s5p.TRACER.download.class : utopya.UtopyaJobStep -! convert task: -cso.s5p.TRACER.download.task.class : cso.CSO_S5p_Download -cso.s5p.TRACER.download.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.download' -! single step: -cso.s5p.TRACER.convert.class : utopya.UtopyaJobStep -! convert task: -cso.s5p.TRACER.convert.task.class : cso.CSO_S5p_Convert -cso.s5p.TRACER.convert.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.convert' -! single step: -cso.s5p.TRACER.listing.class : utopya.UtopyaJobStep -! listing task: -cso.s5p.TRACER.listing.task.class : cso.CSO_S5p_Listing -cso.s5p.TRACER.listing.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.listing' - -! single step: -cso.s5p.TRACER.catalogue.class : utopya.UtopyaJobStep -! two tasks: -cso.s5p.TRACER.catalogue.tasks : figs index - -! catalogue creation task: -cso.s5p.TRACER.catalogue.figs.class : cso.CSO_Catalogue -cso.s5p.TRACER.catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.catalogue' -! indexer task: -cso.s5p.TRACER.catalogue.index.class : utopya.Indexer -cso.s5p.TRACER.catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.catalogue-index' - -! * - -! single step: -cso.s5p.TRACER.gridded.class : utopya.UtopyaJobStep -! catalogue creation task: -cso.s5p.TRACER.gridded.task.class : cso.CSO_GriddedAverage -cso.s5p.TRACER.gridded.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.gridded' - -! single step: -cso.s5p.TRACER.gridded-catalogue.class : utopya.UtopyaJobStep -! two tasks: -cso.s5p.TRACER.gridded-catalogue.tasks : figs index -! catalogue creation task: -cso.s5p.TRACER.gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue -cso.s5p.TRACER.gridded-catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.gridded-catalogue' -! indexer task: -cso.s5p.TRACER.gridded-catalogue.index.class : utopya.Indexer -cso.s5p.TRACER.gridded-catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.gridded-catalogue-index' - -! * - -! single step: -cso.s5p.TRACER.sim-catalogue.class : utopya.UtopyaJobStep -! two tasks: -cso.s5p.TRACER.sim-catalogue.tasks : figs index - -! catalogue creation task: -cso.s5p.TRACER.sim-catalogue.figs.class : cso.CSO_SimCatalogue -cso.s5p.TRACER.sim-catalogue.figs.args : , \ - rcbase='cso.s5p.TRACER.sim-catalogue' -! indexer task: -cso.s5p.TRACER.sim-catalogue.index.class : utopya.Indexer -cso.s5p.TRACER.sim-catalogue.index.args : , \ - rcbase='cso.s5p.TRACER.sim-catalogue-index' - -! -! * -! - -! single step: -cso.s5p.TRACER.sim-gridded.class : utopya.UtopyaJobStep -! catalogue creation task: -cso.s5p.TRACER.sim-gridded.task.class : cso.CSO_GriddedAverage -cso.s5p.TRACER.sim-gridded.task.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.sim-gridded' - -! single step: -cso.s5p.TRACER.sim-gridded-catalogue.class : utopya.UtopyaJobStep -! two tasks: -cso.s5p.TRACER.sim-gridded-catalogue.tasks : figs index -! catalogue creation task: -cso.s5p.TRACER.sim-gridded-catalogue.figs.class : cso.CSO_GriddedCatalogue -cso.s5p.TRACER.sim-gridded-catalogue.figs.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.sim-gridded-catalogue' -! indexer task: -cso.s5p.TRACER.sim-gridded-catalogue.index.class : utopya.Indexer -cso.s5p.TRACER.sim-gridded-catalogue.index.args : '${my.work}/rc/cso-s5p-TRACER.rc', \ - rcbase='cso.s5p.TRACER.sim-gridded-catalogue-index' - -#endfor - - -!---------------------------------------------------------- -! job step defaults -!---------------------------------------------------------- - -! run jobs in foreground: -*.script.class : utopya.UtopyaJobScriptForeground - -! search path for python modules: -*.pypath : ${my.work}/py - -! work directory for jobs; -! here path including subdirectories for job name elements: -*.workdir : ${my.work}/__NAME2PATH__ -!! TESTING ... -!*.workdir : ${my.work}/test-difflist-__NAME2PATH__ - -! jobtree settings from this file: -*.rcfile : ${my.work}/rc/cso.rc - -! script interpreter: -*.shell : /usr/bin/env python3 - - -!---------------------------------------------------------- -! copy -! - create run directory -! - copy scripts and settings for jobtree -!---------------------------------------------------------- - -! no sub list: -cso.copy.class : utopya.UtopyaJobStep - -! default class is defined in machine specific settings; -! copy is always done in foreground to avoid that files are -! changed before the job is started: -cso.copy.script.class : utopya.UtopyaJobScriptForeground - -! utopya jobstep settings: this file, -! this task will copy it to the workdir: -cso.copy.rcfile : ${CSO_RCFILE} - -! search path to utopya modules; -! the 'build.utopya' job will create a copy on the workdir -! for which the path is included in the '*.pypath' setting: -cso.copy.pypath : ${CSO_PREFIX}/py - -! configure and build: -cso.copy.task.class : utopya.UtopyaCopy -cso.copy.task.args : '${CSO_RCFILE}', rcbase='main' - -! ~ - -! prefix for destination of source and script files -! (base path for subdirectories src, py, etc) ; -! this should be an absolute path, -! use ${PWD} for the present dir if necessary: -main.copy.prefix : ${my.work} - -! space separated extensions: -main.copy.prefix.extensions : - -! remove existing build directory if present ? -main.copy.new : False - -! sub directories to be included ; -! leave empty if no subdirs are defined: -main.copy.subdirs : py rc - -! directories to be inlcuded in copy, -! otherwise only files are copied: -main.copy.incdirs : - -! list of source directories to be copied; -! files in latest directory overwrite previously copied versions: -main.copy.dirs : ${CSO_PREFIX} - -! extra directories to be copied into "rc": -main.copy.rc.dirs : ${CSO_RCDIR} - -! skip files matching these filename patterns -! (tempoary editor files, compiled modules, etc) -main.copy.skip : .#* *~ .DS* *.pyc - - -!---------------------------------------------------------- -! end -!---------------------------------------------------------- -- GitLab From 3dc563d8e5c661e17a0dd5e3c74a527d1f7a7b38 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Thu, 30 Jan 2025 10:43:24 +0100 Subject: [PATCH 4/4] Updated documentation for ColHub archive. --- doc/source/index.rst | 1 + doc/source/portals.rst | 20 + doc/source/pymod-cso_colhub.rst | 5 + doc/source/pymods.rst | 3 +- doc/source/s5p-no2.rst | 2 +- py/cso_colhub.py | 1034 +------------------------------ 6 files changed, 44 insertions(+), 1021 deletions(-) create mode 100644 doc/source/portals.rst create mode 100644 doc/source/pymod-cso_colhub.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index 579b0a4..16d2c87 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -42,6 +42,7 @@ Contents gridding superobs colocate + portals pymods history documentation diff --git a/doc/source/portals.rst b/doc/source/portals.rst new file mode 100644 index 0000000..6e72cbf --- /dev/null +++ b/doc/source/portals.rst @@ -0,0 +1,20 @@ + +.. Label between '.. _' and ':' ; use :ref:`text