TNO Intern

Observation operator with time selection.

Arjo Segersrequested to merge
update-oper into master

Update observation operator code:

  • Select eventually multiple files that overlap with simulation time interval.
  • Select pixels based on time.

Merge request reports

config/tutorial/tutorial.rc

+29 −20
Changes for config/tutorial/tutorial.rc: 29 added lines, 20 removed lines.
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ cso.tutorial.elements : inquire \ 


! ... preprocessor steps one by one:

!cso.tutorial.elements                          :  inquire

!cso.tutorial.elements                          :  convert

!cso.tutorial.elements                          :  convert listing

!cso.tutorial.elements                          :  listing

!cso.tutorial.elements                          :  catalogue

!
@@ -260,7 +260,8 @@ my.attr.email : Your.Name@cso.org 


! base location for work directories:

!my.work                         :  /work/${USER}/CSO-Tutorial

my.work                         :  /Scratch/${USER}/CSO-Tutorial

!my.work                         :  /Scratch/${USER}/CSO-Tutorial

my.work                         :  ${WORK}/CSO-Tutorial





!----------------------------------------------------------
@@ -298,9 +299,12 @@ VIRTUAL_ENV : 
! Obtain names of all available S5p files.

! Stored as csv with processing type, processor version, filenames, etc.



! full time range:

cso.tutorial.inquire-table-dataspace.timerange.start        :  2018-04-30 00:00:00

cso.tutorial.inquire-table-dataspace.timerange.end          :  2025-01-01 00:00:00

!! full time range:

!cso.tutorial.inquire-table-dataspace.timerange.start        :  2018-04-30 00:00:00

!cso.tutorial.inquire-table-dataspace.timerange.end          :  2025-01-01 00:00:00

! testing ...

cso.tutorial.inquire-table-dataspace.timerange.start        :  2018-06-01 00:00:00

cso.tutorial.inquire-table-dataspace.timerange.end          :  2018-07-01 00:00:00



! API url:

cso.tutorial.inquire-table-dataspace.url                    :  https://stac.dataspace.copernicus.eu/v1
@@ -347,8 +351,8 @@ cso.tutorial.inquire-plot.output.file : ${my.work}/Copernicus/Cope 




! renew existing files (True|False) ?

cso.tutorial.convert.renew                  :  True

!cso.tutorial.convert.renew                  :  False

!cso.tutorial.convert.renew                  :  True

cso.tutorial.convert.renew                  :  False



! time range:

cso.tutorial.convert.timerange.start        :  ${my.timerange.start}
@@ -1148,13 +1152,18 @@ cso.tutorial.sim-catalogue.timerange.start : ${my.timerange.start} 
cso.tutorial.sim-catalogue.timerange.end          :  ${my.timerange.end}

cso.tutorial.sim-catalogue.timerange.step         :  hour



! input files:

cso.tutorial.sim-catalogue.input.data.file        :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_data.nc

cso.tutorial.sim-catalogue.input.state.file       :  ${my.work}/CSO-oper/CSO_output_%Y%m%d_%H%M_state.nc

! listing with "data" and "state" files:

cso.tutorial.sim-catalogue.input.listing          :  ${my.work}/CSO-oper/listing.csv



! target files, time tempates and variable name are replaced:

cso.tutorial.sim-catalogue.output.file            :  ${my.work}/CSO-sim-catalogue/S5p/NO2/${my.region}/%Y/%m/%d/S5p_NO2_%Y%m%d_%H%M_%{var}.png

! target directory for figures:

! - time values  : average time of pixels in file

! full filenames includes "basename" of simulation files (without "_data.nc" or "_state.nc")

! and the variable name: 

!   <output.dir>/<basename>_<var>.png

cso.tutorial.sim-catalogue.output.dir            :  ${my.work}/CSO-sim-catalogue/S5p/NO2/${my.region}/%Y/%m/%d



!! domain of converted data:

!cso.tutorial.sim-catalogue.domain                 :  ${my.region.west} ${my.region.east} ${my.region.south} ${my.region.north}

! map domain used for simulations (west east south north):

cso.tutorial.sim-catalogue.domain                 :  -10 30 35 65

!!~ globe:
@@ -1238,20 +1247,20 @@ cso.tutorial.sim-catalogue-index.date.newpage : True 
! content type:

cso.tutorial.sim-catalogue-index.date.type                 :  table-row

! define row values:

cso.tutorial.sim-catalogue-index.date.name                 :  time

cso.tutorial.sim-catalogue-index.date.values               :  Range( 0, 23, 1, '%2.2i00' )

cso.tutorial.sim-catalogue-index.date.name                 :  basename

cso.tutorial.sim-catalogue-index.date.values               :  CsvFile( '%{date[0:4]}/%{date[4:6]}/%{date[6:8]}/basenames.csv' )



! content type:

cso.tutorial.sim-catalogue-index.date.time.type            :  table-col

cso.tutorial.sim-catalogue-index.date.basename.type        :  table-col

! define column values:

cso.tutorial.sim-catalogue-index.date.time.name            :  var

cso.tutorial.sim-catalogue-index.date.time.values          :  ${cso.tutorial.sim-catalogue.vars}

cso.tutorial.sim-catalogue-index.date.basename.name        :  var

cso.tutorial.sim-catalogue-index.date.basename.values      :  ${cso.tutorial.sim-catalogue.vars}



! content type:

cso.tutorial.sim-catalogue-index.date.time.var.type        :  img

cso.tutorial.sim-catalogue-index.date.basename.var.type    :  img

! define image:

cso.tutorial.sim-catalogue-index.date.time.var.img         :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/S5p_NO2_%{date}_%{time}_%{var}.png

cso.tutorial.sim-catalogue-index.date.time.var.kwargs      :  height=300

cso.tutorial.sim-catalogue-index.date.basename.var.img     :  %{date[0:4]}/%{date[4:6]}/%{date[6:8]}/%{basename}_%{var}.png

cso.tutorial.sim-catalogue-index.date.basename.var.kwargs  :  height=300

doc/source/history.rst

+5 −0
Changes for doc/source/history.rst: 5 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -138,6 +138,11 @@ A summary of the versions and changes. 
  | Convert multiple files if orbit is stored in patches. Check on duplicate output files.

  | *(2026-05)*



* | *v2.18*

  | Updated oservation operator code: select multiple data files that overlap with simulation time, use time selection for pixels.

  | *(2026-07)*





To be done

==========

doc/source/pymod-cso_listing.rst

0 → 100644
+5 −0
Changes for doc/source/pymod-cso_listing.rst: 5 added lines, 0 removed lines.
Original line number Diff line number Diff line 
.. Documentation for module.



.. Import documentation from ".py" file:

.. automodule:: cso.cso_listing

doc/source/pymods.rst

+1 −0
Changes for doc/source/pymods.rst: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ Overview of the Python module(s) and classes. 
   pymod-cso_s5p

   pymod-cso_viirs

   pymod-cso_file

   pymod-cso_listing

   pymod-cso_gridded

   pymod-cso_superobs

   pymod-cso_colocate

oper/src/Makefile_config_metno-nebula→oper/src/Makefile_config_metno-fahrenheit

+6 −12
Changes for oper/src/Makefile_config_metno-fahrenheit: 6 added lines, 12 removed lines.
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ else 
endif



# default flags, similar to what is used in EMEP:

FFLAGS_DEFAULT = -extend-source -fpp -recursive -r8 -convert big_endian -shared-intel -assume noold_maxminloc -ftz

FFLAGS_DEFAULT = -extend-source -recursive -r8



# optimization flags:

FFLAGS_OPTIM   = -O0 -g -traceback -check all -check noarg_temp_created -debug-parameters all -ftrapuv -fpe0
@@ -25,16 +25,10 @@ FFLAGS_OPTIM = -O0 -g -traceback -check all -check noarg_temp_created -debug-p 
FFLAGS  = $(FFLAGS_DEFAULT) $(FFLAGS_OPTIM)

LDFLAGS =



# library:

NETCDF_FORTRAN_HOME = ${EBROOTNETCDF}

NETCDF_FORTRAN_INCS = -I${NETCDF_FORTRAN_HOME}/include

NETCDF_FORTRAN_LIBS = -L${NETCDF_FORTRAN_HOME}/lib -lnetcdff -Wl,-rpath -Wl,${NETCDF_FORTRAN_HOME}/lib



# library:

NETCDF_C_HOME = ${EBROOTNETCDF}

NETCDF_C_INCS =

NETCDF_C_LIBS = -L${NETCDF_C_HOME}/lib -lnetcdf -Wl,-rpath -Wl,${NETCDF_C_HOME}/lib

# single source for NETCDF libs:

NETCDF_INCS = -I${NETCDF_DIR}/include

NETCDF_LIBS = -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -Wl,-rpath -Wl,${NETCDF_DIR}/lib



# collect:

INCS = $(NETCDF_FORTRAN_INCS) $(NETCDF_C_INCS)

LIBS = $(NETCDF_FORTRAN_LIBS) $(NETCDF_C_LIBS)
 
INCS = $(NETCDF_INCS)

LIBS = $(NETCDF_LIBS)