diff --git a/config/Copernicus/cso-s5p-co.rc b/config/Copernicus/cso-s5p-co.rc index 7bb95d233a5193b394738b3f3234cfd7c4e499e5..fa6e72cac9669014b32976d32fe4f7e0c8998b46 100644 --- a/config/Copernicus/cso-s5p-co.rc +++ b/config/Copernicus/cso-s5p-co.rc @@ -34,16 +34,11 @@ cso.s5p.co.inquire-table-dataspace.timerange.start : ${my.full-timerange cso.s5p.co.inquire-table-dataspace.timerange.end : ${my.full-timerange.end} ! API url: -cso.s5p.co.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api +cso.s5p.co.inquire-table-dataspace.url : https://stac.dataspace.copernicus.eu/v1 -! collection name: -cso.s5p.co.inquire-table-dataspace.collection : Sentinel5P - -! product type, always 10 characters! -! L2__CO___ -! L2__CO____ -! ... -cso.s5p.co.inquire-table-dataspace.producttype : L2__CO____ +! collection names (space seperated list): +cso.s5p.co.inquire-table-dataspace.collections : sentinel-5p-l2-co-rpro \ + sentinel-5p-l2-co-offl ! target area; !!~ empty for no limitation: @@ -51,9 +46,6 @@ cso.s5p.co.inquire-table-dataspace.producttype : L2__CO____ !~ domain specified as: west,south,east,north cso.s5p.co.inquire-table-dataspace.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} -! template for download url given "{product_id}": -cso.s5p.co.inquire-table-dataspace.download_url : https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value - ! output table, date of today: cso.s5p.co.inquire-table-dataspace.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_CO_dataspace__%Y-%m-%d.csv @@ -160,7 +152,7 @@ my.s5p.co.selection : C03 ! (%{collection} == '03') and (%{processing} == 'OFFL') ! #if "${my.s5p.co.selection}" == "C03" -cso.s5p.co.convert.selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ +cso.s5p.co.convert.selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ (%{collection} == '03') and (%{processing} == 'OFFL') #else #error unsupported my.s5p.co.selection "${my.s5p.co.selection}" diff --git a/config/Copernicus/cso-s5p-hcho.rc b/config/Copernicus/cso-s5p-hcho.rc index 77026f10a1632fdcc0f87b2b6ba1439adf4f68f8..20b7bea0b39c5480ffee5d76f2287ed94cfc6d99 100644 --- a/config/Copernicus/cso-s5p-hcho.rc +++ b/config/Copernicus/cso-s5p-hcho.rc @@ -34,16 +34,11 @@ cso.s5p.hcho.inquire-table-dataspace.timerange.start : ${my.full-timeran cso.s5p.hcho.inquire-table-dataspace.timerange.end : ${my.full-timerange.end} ! API url: -cso.s5p.hcho.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api +cso.s5p.hcho.inquire-table-dataspace.url : https://stac.dataspace.copernicus.eu/v1 -! collection name: -cso.s5p.hcho.inquire-table-dataspace.collection : Sentinel5P - -! product type, always 10 characters! -! L2__HCHO___ -! L2__CO____ -! ... -cso.s5p.hcho.inquire-table-dataspace.producttype : L2__HCHO__ +! collection names (space seperated list): +cso.s5p.hcho.inquire-table-dataspace.collections : sentinel-5p-l2-hcho-rpro \ + sentinel-5p-l2-hcho-offl ! target area; !!~ empty for no limitation: @@ -51,9 +46,6 @@ cso.s5p.hcho.inquire-table-dataspace.producttype : L2__HCHO__ !~ domain specified as: west,south,east,north cso.s5p.hcho.inquire-table-dataspace.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} -! template for download url given "{product_id}": -cso.s5p.hcho.inquire-table-dataspace.download_url : https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value - ! output table, date of today: cso.s5p.hcho.inquire-table-dataspace.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_HCHO_dataspace__%Y-%m-%d.csv diff --git a/config/Copernicus/cso-s5p-no2.rc b/config/Copernicus/cso-s5p-no2.rc index 5ae5685046af3119a3ba8774578776f0ad58494f..4cc36ed827b6f87cb8d23dc1936690963dca302e 100644 --- a/config/Copernicus/cso-s5p-no2.rc +++ b/config/Copernicus/cso-s5p-no2.rc @@ -34,16 +34,11 @@ cso.s5p.no2.inquire-table-dataspace.timerange.start : ${my.full-timerang cso.s5p.no2.inquire-table-dataspace.timerange.end : ${my.full-timerange.end} ! API url: -cso.s5p.no2.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api +cso.s5p.no2.inquire-table-dataspace.url : https://stac.dataspace.copernicus.eu/v1 -! collection name: -cso.s5p.no2.inquire-table-dataspace.collection : Sentinel5P - -! product type, always 10 characters! -! L2__NO2___ -! L2__CO____ -! ... -cso.s5p.no2.inquire-table-dataspace.producttype : L2__NO2___ +! collection names (space seperated list): +cso.s5p.no2.inquire-table-dataspace.collections : sentinel-5p-l2-no2-rpro \ + sentinel-5p-l2-no2-offl ! target area; !!~ empty for no limitation: @@ -51,9 +46,6 @@ cso.s5p.no2.inquire-table-dataspace.producttype : L2__NO2___ !~ domain specified as: west,south,east,north cso.s5p.no2.inquire-table-dataspace.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} -! template for download url given "{product_id}": -cso.s5p.no2.inquire-table-dataspace.download_url : https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value - ! output table, date of today: cso.s5p.no2.inquire-table-dataspace.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv diff --git a/config/Copernicus/cso-s5p-o3-pr.rc b/config/Copernicus/cso-s5p-o3-pr.rc index 257706e68a384007227b5c52c32f7146254b21f9..6d01be99fdb0b1b291d4c99b6bfba17085e75dde 100644 --- a/config/Copernicus/cso-s5p-o3-pr.rc +++ b/config/Copernicus/cso-s5p-o3-pr.rc @@ -34,20 +34,11 @@ cso.s5p.o3-pr.inquire-table-dataspace.timerange.start : ${my.full-timera cso.s5p.o3-pr.inquire-table-dataspace.timerange.end : ${my.full-timerange.end} ! API url: -cso.s5p.o3-pr.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api +cso.s5p.o3-pr.inquire-table-dataspace.url : https://stac.dataspace.copernicus.eu/v1 -! collection name: -cso.s5p.o3-pr.inquire-table-dataspace.collection : Sentinel5P - -! product type, always 10 characters! -! L2__O3____ : ozone total column -! L2__O3_TCL : ozone tropospheric column -! Level3 product? -! 3-days running average on 1.0 x 0.5 (lon x lat) grid -! L2__O3__PR : ozone profile -! L2__CO____ -! ... -cso.s5p.o3-pr.inquire-table-dataspace.producttype : L2__O3__PR +! collection names (space seperated list): +cso.s5p.o3-pr.inquire-table-dataspace.collections : sentinel-5p-l2-o3-pr-rpro \ + sentinel-5p-l2-o3-pr-offl ! target area; !!~ empty for no limitation: @@ -55,9 +46,6 @@ cso.s5p.o3-pr.inquire-table-dataspace.producttype : L2__O3__PR !~ domain specified as: west,south,east,north cso.s5p.o3-pr.inquire-table-dataspace.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} -! template for download url given "{product_id}": -cso.s5p.o3-pr.inquire-table-dataspace.download_url : https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value - ! output table, date of today: cso.s5p.o3-pr.inquire-table-dataspace.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_O3-PR_dataspace__%Y-%m-%d.csv diff --git a/config/Copernicus/cso-s5p-so2.rc b/config/Copernicus/cso-s5p-so2.rc index e3e00a5f2771ed7a1de822b664c1debc96f45d27..4a6c7584814e5c737d96eec01936c54154a84f62 100644 --- a/config/Copernicus/cso-s5p-so2.rc +++ b/config/Copernicus/cso-s5p-so2.rc @@ -34,16 +34,11 @@ cso.s5p.so2.inquire-table-dataspace.timerange.start : ${my.full-timerang cso.s5p.so2.inquire-table-dataspace.timerange.end : ${my.full-timerange.end} ! API url: -cso.s5p.so2.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api +cso.s5p.so2.inquire-table-dataspace.url : https://stac.dataspace.copernicus.eu/v1 -! collection name: -cso.s5p.so2.inquire-table-dataspace.collection : Sentinel5P - -! product type, always 10 characters! -! L2__SO2___ -! L2__CO____ -! ... -cso.s5p.so2.inquire-table-dataspace.producttype : L2__SO2___ +! collection names (space seperated list): +cso.s5p.so2.inquire-table-dataspace.collections : sentinel-5p-l2-so2-rpro \ + sentinel-5p-l2-so2-offl ! target area; !!~ empty for no limitation: @@ -51,9 +46,6 @@ cso.s5p.so2.inquire-table-dataspace.producttype : L2__SO2___ !~ domain specified as: west,south,east,north cso.s5p.so2.inquire-table-dataspace.area : ${my.region.west},${my.region.south},${my.region.east},${my.region.north} -! template for download url given "{product_id}": -cso.s5p.so2.inquire-table-dataspace.download_url : https://zipper.dataspace.copernicus.eu/odata/v1/Products({product_id})/$value - ! output table, date of today: cso.s5p.so2.inquire-table-dataspace.output.file : ${my.work}/Copernicus-inquire/Copernicus_S5p_SO2_dataspace__%Y-%m-%d.csv @@ -107,8 +99,10 @@ my.s5p.so2.download.selection : C03 ! (%{collection} == '03') and (%{processing} == 'OFFL') ! #if "${my.s5p.so2.download.selection}" == "C03" -cso.s5p.so2.download.selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ - (%{collection} == '03') and (%{processing} == 'OFFL') +!~ SO2 on DataSpace has a reprocessing with v2.7.0 (using COBRA algorithm?), +! ignore the previous processings: +cso.s5p.so2.download.selection : (%{collection} == '03') and (%{processing} == 'RPRO') & (%{processor_version} >= '020700') ; \ + (%{collection} == '03') and (%{processing} == 'OFFL') & (%{processor_version} >= '020700') #else #error unsupported my.s5p.so2.download.selection "${my.s5p.so2.download.selection}" #endif diff --git a/doc/source/figs/SO2/Copernicus_S5p_SO2.png b/doc/source/figs/SO2/Copernicus_S5p_SO2.png index 65b0435fc8b9de5c3105ef1417a54ead945a3042..48d701593a5e316e089f102ef950246c5ec63ff8 100644 Binary files a/doc/source/figs/SO2/Copernicus_S5p_SO2.png and b/doc/source/figs/SO2/Copernicus_S5p_SO2.png differ diff --git a/doc/source/s5p-so2.rst b/doc/source/s5p-so2.rst index c519d8d615ae43700d3d1781d4444528be080051..e002f15368db331813820a454ead1df7f77b8f70 100644 --- a/doc/source/s5p-so2.rst +++ b/doc/source/s5p-so2.rst @@ -141,8 +141,8 @@ The result is a csv file which with columns for keywords such as orbit number an as well as the filename of the data and the url that should be used to actually download the data:: orbit;start_time;end_time;processing;collection;processor_version;filename;href - 21497;2021-12-06 14:05:54;2021-12-06 15:47:24;OFFL;02;020301;S5P_OFFL_L2__SO2____20211206T140554_20211206T154724_21497_02_020301_20211208T043331.nc;https://zipper.dataspace.copernicus.eu/odata/v1/Products('d9d33ffa-9fe5-43cc-b5a1-b65c22e874ad')/$value - 21852;2021-12-31 14:37:39;2021-12-31 16:19:09;OFFL;02;020301;S5P_OFFL_L2__SO2____20211231T143739_20211231T161909_21852_02_020301_20220102T064010.nc;https://zipper.dataspace.copernicus.eu/odata/v1/Products('ff5c922c-450c-43db-97e4-f46bdd55ffb2')/$value + 02818;2018-04-30 00:18:51;2018-04-30 02:02:19;RPRO;01;010102;S5P_RPRO_L2__SO2____20180430T001851_20180430T020219_02818_01_010102_20180903T080822.nc;s3://eodata/Sentinel-5P/TROPOMI/L2__SO2___/2018/04/30/S5P_RPRO_L2__SO2____20180430T001851_20180430T020219_02818_01_010102_20180903T080822.nc + 02818;2018-04-30 00:18:51;2018-04-30 02:02:19;RPRO;01;010105;S5P_RPRO_L2__SO2____20180430T001851_20180430T020219_02818_01_010105_20190201T205638.nc;s3://eodata/Sentinel-5P/TROPOMI/L2__SO2___/2018/04/30/S5P_RPRO_L2__SO2____20180430T001851_20180430T020219_02818_01_010105_20190201T205638.nc : See the section on *File name convention* in the *Product User Manual* for the meaning of all @@ -214,6 +214,9 @@ For the S5P files a useful property is also the *collection number*, a 2-digit n (or actually processor versions) that together form a contineous series. The *collection number* is extracted from the filename, and stored as a column of the listing file. +Specific for the SO\ :sub:`2` product, processor version ``v2.7.0`` had a major upgrade by incorporating the COBRA algorithm; +therefore, orbits should be selected from this processor version onwards. + The following setting is used to select specific files from the archive based on the properities stored in the listing file:: @@ -225,75 +228,77 @@ in the listing file:: ! (%{collection} == '03') and (%{processing} == 'RPRO') ; \ ! (%{collection} == '03') and (%{processing} == 'OFFL') ! - cso.s5p.so2.convert.selection : (%{collection} == '03') and (%{processing} == 'RPRO') ; \ - (%{collection} == '03') and (%{processing} == 'OFFL') + cso.s5p.so2.convert.selection : (%{collection} == '03') and (%{processing} == 'RPRO') & (%{processor_version} >= '020700') ; \ + (%{collection} == '03') and (%{processing} == 'OFFL') & (%{processor_version} >= '020700') -Pixel selection ---------------- +.. SPECIC FOR EARLIER VERSIONS ... -The :py:class:`.CSO_S5p_Convert` class calls the :py:meth:`.S5p_File.SelectPixels` method -to create a pixel selection mask for the input file. -The selection is done using one or more filters. -First provide a list of filter names:: + Pixel selection + --------------- - cso.s5p.so2.convert.filters : lons lats valid quality sza ground_pixel cloud_fraction + The :py:class:`.CSO_S5p_Convert` class calls the :py:meth:`.S5p_File.SelectPixels` method + to create a pixel selection mask for the input file. + The selection is done using one or more filters. + First provide a list of filter names:: -Then provide for each filter the the input variable to be used for testing, -as a path name in the input file. -The next settings is the type of filter to be used, see the :py:meth:`.S5p_File.SelectPixels` for supported types, -and the other settings required by the type. -The following is an example of a selection on longitude:: + cso.s5p.so2.convert.filters : lons lats valid quality sza ground_pixel cloud_fraction - cso.s5p.so2.convert.filter.lons.var : Geolocation Fields/Longitude - cso.s5p.so2.convert.filter.lons.type : minmax - cso.s5p.so2.convert.filter.lons.minmax : -30.0 45.0 - cso.s5p.so2.convert.filter.lons.units : degrees_east - -Extension to the product guide ------------------------------- - -Several publications report extended data criteria beyond the PUMs quality_flags that should be used to ensure data quality. -Examples of such are the emission source detection paper by *(Fioletov et al., 2020)*, -and the volcanic SO\ :sub:`2` monitoring paper by Theys et al., 2019. -Both publications mention the poor quality of observations at the edges of TROPOMI observation swath -as well as the reduced quality at high Solar Zenith Angles. -The integration time of pixels toward the edge of the swath has been reduced to decrease the pixel size, -however this also reduces the overall quality of the observation (SNR). -Therefor we advise to only select the pixels with id's 25-425, and for really strict cases only 50-400. -Examples of both the edge pixels and SZA effects are shown in the figures below, -with 2 figures from Fioletov et al., 2020 and an example for the SZA based on the CSO results. - -.. topic:: Figure from Fioletov et al. (2020) - - .. figure:: figs/SO2/Example_fig1_fioletov_2020.png - :scale: 85 % - :align: center - :alt: Figure 1 from Fioletov (2020) - -.. topic:: Figure from Fioletov et al. (2020) - - .. figure:: figs/SO2/Example_fig2_fioletov_2020.png - :scale: 85 % - :align: center - :alt: Figure 2 from Fioletov (2020) - -.. figure:: figs/SO2/overpass_example_filters_so2.png - :scale: 50 % - :align: center - :alt: Examples of column, error, and quality number for two orbits. - - *Examples of retrieved SO2 columns, retrieval errors, and quality flags.* - -Furthermore, Fioletov et al., 2020 (Fig below) report large-scale biases in the current TROPOMI product, with TROPOMI showing significantly higher total columns, which can be expected to introduce a potential high bias throughout the domain. -A solution advised by Fioletov et al is to remove the bias by comparing up- and down-wind values around an SO\ :sub:`2` emissions source, but that will not be included in this algorithm. + Then provide for each filter the the input variable to be used for testing, + as a path name in the input file. + The next settings is the type of filter to be used, see the :py:meth:`.S5p_File.SelectPixels` for supported types, + and the other settings required by the type. + The following is an example of a selection on longitude:: + + cso.s5p.so2.convert.filter.lons.var : Geolocation Fields/Longitude + cso.s5p.so2.convert.filter.lons.type : minmax + cso.s5p.so2.convert.filter.lons.minmax : -30.0 45.0 + cso.s5p.so2.convert.filter.lons.units : degrees_east + + Extension to the product guide + ------------------------------ + + Several publications report extended data criteria beyond the PUMs quality_flags that should be used to ensure data quality. + Examples of such are the emission source detection paper by *(Fioletov et al., 2020)*, + and the volcanic SO\ :sub:`2` monitoring paper by Theys et al., 2019. + Both publications mention the poor quality of observations at the edges of TROPOMI observation swath + as well as the reduced quality at high Solar Zenith Angles. + The integration time of pixels toward the edge of the swath has been reduced to decrease the pixel size, + however this also reduces the overall quality of the observation (SNR). + Therefor we advise to only select the pixels with id's 25-425, and for really strict cases only 50-400. + Examples of both the edge pixels and SZA effects are shown in the figures below, + with 2 figures from Fioletov et al., 2020 and an example for the SZA based on the CSO results. + + .. topic:: Figure from Fioletov et al. (2020) + + .. figure:: figs/SO2/Example_fig1_fioletov_2020.png + :scale: 85 % + :align: center + :alt: Figure 1 from Fioletov (2020) + + .. topic:: Figure from Fioletov et al. (2020) + + .. figure:: figs/SO2/Example_fig2_fioletov_2020.png + :scale: 85 % + :align: center + :alt: Figure 2 from Fioletov (2020) + + .. figure:: figs/SO2/overpass_example_filters_so2.png + :scale: 50 % + :align: center + :alt: Examples of column, error, and quality number for two orbits. + + *Examples of retrieved SO2 columns, retrieval errors, and quality flags.* + + Furthermore, Fioletov et al., 2020 (Fig below) report large-scale biases in the current TROPOMI product, with TROPOMI showing significantly higher total columns, which can be expected to introduce a potential high bias throughout the domain. + A solution advised by Fioletov et al is to remove the bias by comparing up- and down-wind values around an SO\ :sub:`2` emissions source, but that will not be included in this algorithm. -.. topic:: Figure from Fioletov et al. (2020) + .. topic:: Figure from Fioletov et al. (2020) - .. figure:: figs/SO2/Example_fig5_fioletov_2020.png - :scale: 85 % - :align: center - :alt: Example of mean SO\ :sub:`2` columns for TROPOMI OMI and OMPS over several regions (Fioletov et al., 2020). + .. figure:: figs/SO2/Example_fig5_fioletov_2020.png + :scale: 85 % + :align: center + :alt: Example of mean SO\ :sub:`2` columns for TROPOMI OMI and OMPS over several regions (Fioletov et al., 2020). Variable specification