Specify output file template for catalogue figures rather than output... (616e8bad) · Commits · CAMS / CSO

py/cso_catalogue.py

+175 −120

Original line number	Diff line number	Diff line
		@@ -4,6 +4,11 @@
		# 2022-09, Arjo Segers, Met-Norway
		# Trap non-existing or empty listing files when creating catalogue of converted data.
		#
		# 2022-09, Arjo Segers, Met-Norway
		# Specify output file template for catalogue figures rather than output directory.
		# Support steps over year when plotting series of gridded observations.
		# Define full output filename in settings instead of path only.
		#

		########################################################################
		###
		@@ -130,6 +135,12 @@ class CSO_CatalogueBase( utopya.UtopyaRc ) :
		# apply factor:
		values = values * 1e6

		# vertical column density:
		elif conversion == 'mol m-2 -> mmol/m2' :

		# apply factor:
		values = values * 1e3

		# vertical column density:
		elif conversion == 'mol m-2 -> 1e15 mlc/cm2' :

		@@ -140,10 +151,16 @@ class CSO_CatalogueBase( utopya.UtopyaRc ) :
		# vertical column density error covariance:
		elif conversion == '(mol m-2)**2 -> 1e15 mlc/cm2' :

		# apply factor:
		# take square root and apply factor:
		# mol/m2 mlc/mol m2/cm2
		values = numpy.sqrt(values) * cso_constants.Avog * 1e-4 / 1e15 # 1e15 mlc/cm2

		# vertical column density error covariance:
		elif conversion == '(mol m-2)**2 -> umol/m2' :

		# take square root and apply factor:
		values = numpy.sqrt(values) * 1e6


		else :
		logging.error( 'unsupported conversion "%s"' % conversion )
		@@ -185,6 +202,7 @@ class CSO_Catalogue( CSO_CatalogueBase ) :

		A time range for which images should be created is specified with::

		! specifiy timerange:
		<rcbase>.timerange.start : 2012-01-01 00:00
		<rcbase>.timerange.end : 2012-12-31 23:59

		@@ -216,6 +234,11 @@ class CSO_Catalogue( CSO_CatalogueBase ) :

		<rcbase>.var.qa_flag.colors : ['red','yellow','green']

		The label below the colorbar will by default show the variable name,
		unless a ``long_name`` is defined::

		<rcbase>.var.vcd.long_name : retrieved vertical column density

		The created images will have filenames including subdirectories for year/month/day::

		2018/06/01/S5p_RPRO_NO2_03278__vcd.png
		@@ -247,7 +270,6 @@ class CSO_Catalogue( CSO_CatalogueBase ) :
		# modules:
		import os
		import pandas
		import fnmatch
		import numpy
		import matplotlib
		import matplotlib.pyplot as plt
		@@ -274,18 +296,8 @@ class CSO_Catalogue( CSO_CatalogueBase ) :
		tfmt = '%Y-%m-%d %H:%M'
		logging.info( indent+'timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )

		# target directory:
		workdir = self.GetSetting( 'output.dir' )

		# info ..
		logging.info( indent+'work directory:' )
		logging.info( indent+' %s' % workdir )
		# create?
		if not os.path.isdir(workdir) : os.makedirs( workdir )
		# store current:
		owd = os.getcwd()
		# change to:
		os.chdir( workdir )
		# target files:
		figfile_template = self.GetSetting( 'output.file' )

		# listing file:
		lstfile = self.GetSetting( 'input.listing' )
		@@ -345,6 +357,9 @@ class CSO_Catalogue( CSO_CatalogueBase ) :
		# storage for csv files with orbits per subdir:
		orbits = {}

		# base path, filenames are relative to listing file:
		inputdir = os.path.dirname( lstfile )

		# info ..
		logging.info( indent+'loop over orbit files ...' )
		# loop:
		@@ -367,16 +382,17 @@ class CSO_Catalogue( CSO_CatalogueBase ) :
		taver = tr1 + 0.5*(tr2-tr1)

		# subdir:
		figdir = taver.strftime('%Y/%m/%d')
		# create if necessary:
		if not os.path.isdir( figdir ) : os.makedirs( figdir )
		figdir = os.path.dirname( taver.strftime( figfile_template ) )
		## create if necessary:
		#if not os.path.isdir( figdir ) : os.makedirs( figdir )

		# create storage for orbit lists:
		if figdir not in orbits.keys() : orbits[figdir] = []
		# store current:
		orbits[figdir].append( orbit )

		# basename of file without extension:
		bname,ext = os.path.splitext( os.path.basename(filename) )
		## basename of file without extension:
		#bname,ext = os.path.splitext( os.path.basename(filename) )

		# no orbit file read yet ..
		orb = None
		@@ -385,7 +401,10 @@ class CSO_Catalogue( CSO_CatalogueBase ) :
		for varname in varnames :

		# target file:
		figfile = '%s/%s__%s.png' % (figdir,bname,varname)
		#figfile = '%s/%s__%s.png' % (figdir,bname,varname)
		figfile = taver.strftime( figfile_template )
		figfile = figfile.replace('%{orbit}' ,orbit )
		figfile = figfile.replace('%{varname}',varname)

		# renew?
		if (not os.path.isfile(figfile)) or renew :
		@@ -413,6 +432,8 @@ class CSO_Catalogue( CSO_CatalogueBase ) :
		vname = self.GetSetting( vkey+'.source', default=varname )
		# target units:
		vunits = self.GetSetting( vkey+'.units' , default='None' )
		# long name used in labels:
		long_name = self.GetSetting( vkey+'.long_name', default=varname )

		# switch:
		if ptype == 'map' :
		@@ -452,7 +473,7 @@ class CSO_Catalogue( CSO_CatalogueBase ) :
		values,units = self._ConvertUnits( values, attrs, vunits )

		# annote:
		label = '%s [%s]' % (vname,units)
		label = '%s [%s]' % (long_name,units)

		# map properties:
		bmp = dict( resolution='i', countries=True, domain=domain, title=title )
		@@ -474,11 +495,16 @@ class CSO_Catalogue( CSO_CatalogueBase ) :
		raise Exception
		#endif # dim types

		## testing ...
		#logging.warning( 'break after first variable' )
		#break

		#endif # renew

		#endfor # variables

		## testing ...
		#logging.warning( 'break after first orbit' )
		#break

		#endfor # orbit files
		@@ -493,9 +519,6 @@ class CSO_Catalogue( CSO_CatalogueBase ) :

		#endif # nrec > 0

		# back:
		os.chdir( owd )

		#enddef __init__


		@@ -525,8 +548,11 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :

		A time range for which images should be created is specified with::

		! specifiy timerange:
		<rcbase>.timerange.start : 2012-01-01 00:00
		<rcbase>.timerange.end : 2012-12-31 23:59
		! step is one of: hour \| day \| month
		<rcbase>.timerange.step : hour

		The time range is used to scan for output files from the satellite observation operator.
		Both a data file with for example footprints and observations,
		@@ -559,6 +585,11 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :

		<rcbase>.var.qa_flag.colors : ['red','yellow','green']

		The label below the colorbar will by default show the variable name,
		unless a ``long_name`` is defined::

		<rcbase>.var.vcd.long_name : retrieved vertical column density

		The name of the created image files is read from::

		! target files, time tempates are replaced:
		@@ -593,9 +624,9 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		# modules:
		import os
		import pandas
		import fnmatch
		import numpy
		import datetime
		import calendar
		import matplotlib
		import matplotlib.pyplot as plt

		@@ -617,22 +648,10 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		# time range:
		t1 = self.GetSetting( 'timerange.start', totype='datetime' )
		t2 = self.GetSetting( 'timerange.end' , totype='datetime' )
		tstep = self.GetSetting( 'timerange.step' )
		# info ...
		tfmt = '%Y-%m-%d %H:%M'
		logging.info( indent+'timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )

		# target directory:
		workdir = self.GetSetting( 'output.dir' )

		# info ..
		logging.info( indent+'work directory:' )
		logging.info( indent+' %s' % workdir )
		# create?
		if not os.path.isdir(workdir) : os.makedirs( workdir )
		# store current:
		owd = os.getcwd()
		# change to:
		os.chdir( workdir )
		logging.info( indent+'timerange: [%s,%s] by %s' % (t1.strftime(tfmt),t2.strftime(tfmt),tstep) )

		# target file template:
		figfile_template = self.GetSetting( 'output.file' )
		@@ -653,6 +672,11 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		# figure size:
		figsize = eval( self.GetSetting('figsize') )

		# no-data color:
		color_nan = self.GetSetting( 'color_nan', default='0.80' )
		# extra map properties:
		bmp_kwargs = self.GetSetting( 'map', totype='dict', default=dict() )

		# info ..
		logging.info( indent+'time loop ...' )
		# time loop:
		@@ -661,6 +685,23 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		# info ..
		logging.info( indent+' %s ...' % t.strftime('%Y-%m-%d %H:%M') )

		# timestep:
		if tstep in ['hour','hourly'] :
		long_timestamp = t.strftime('%Y-%m-%d %H:%M')
		dt = datetime.timedelta(0,3600)
		elif tstep in ['day','daily'] :
		long_timestamp = t.strftime('%Y-%m-%d')
		dt = datetime.timedelta(1)
		elif tstep in ['month','monthly'] :
		t = datetime.datetime(t.year,t.month,1,0,0)
		long_timestamp = t.strftime('%Y-%m')
		weekday,nday = calendar.monthrange(t.year,t.month)
		dt = datetime.timedelta(nday)
		else :
		logging.error( 'unsupported time step "%s"' % tstep )
		raise Exception
		#endif

		# input files:
		datafile = t.strftime( datafile_template )
		statefile = t.strftime( statefile_template )
		@@ -703,7 +744,7 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		#endif

		# annote:
		title = t.strftime('%Y-%m-%d %H:%M')
		title = long_timestamp

		# settings for this variable:
		vkey = 'var.%s' % varname
		@@ -713,6 +754,8 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		vunits = self.GetSetting( vkey+'.units' , default='None' )
		# plot type:
		ptype = self.GetSetting( vkey+'.type', default='map' )
		# long name used in labels:
		long_name = self.GetSetting( vkey+'.long_name', default=varname )

		# switch:
		if ptype == 'map' :
		@@ -763,13 +806,16 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		values,units = self._ConvertUnits( values, attrs, vunits )

		# annote:
		label = '%s [%s]' % (varname,units)
		label = '%s [%s]' % (long_name,units)

		# map properties:
		bmp = dict( resolution='i', countries=True, domain=domain, title=title )
		bmp.update( bmp_kwargs )

		# create map figure:
		fig = cso_plot.QuickMap( values, xx=xx, yy=yy, vmin=vmin, vmax=vmax,
		cmap=dict(colors=colors),
		bmp=dict(resolution='i',countries=True,domain=domain,title=title),
		cbar=dict(label=label),
		cmap=dict(colors=colors,color_bad=color_nan),
		bmp=bmp, cbar=dict(label=label),
		figsize=figsize )
		# save:
		fig.Export( figfile )
		@@ -782,6 +828,9 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		raise Exception
		#endif # ptype

		else :
		# info ..
		logging.info( indent+' keep %s ...' % figfile )
		#endif # renew

		#endfor # variables
		@@ -794,13 +843,10 @@ class CSO_SimCatalogue( CSO_CatalogueBase ) :
		#endif # state file present

		# next:
		t = t + datetime.timedelta(0,3600)
		t = t + dt

		#endfor # time loop

		# back:
		os.chdir( owd )

		#enddef __init__


		@@ -830,30 +876,29 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :

		A time range for which images should be created is specified with::

		! specifiy timerange:
		<rcbase>.timerange.start : 2012-01-01 00:00
		<rcbase>.timerange.end : 2012-12-31 23:59
		! step is one of: hour \| day \| month
		<rcbase>.timerange.step : hour

		The time range is used to scan for output files from the :py:class:`CSO_GriddedAverage` class.
		Both a data file with for example footprints and observations,
		as well as a state file with simulations is needed::
		The time range is used to scan for output files from the :py:class:`CSO_GriddedAverage` class::

		<rcbase>.input.data.file : /Scratch/model/output/CSO_output_%Y%m%d_%H%M_data.nc
		<rcbase>.input.state.file : /Scratch/model/output/CSO_output_%Y%m%d_%H%M_state.nc
		<rcbase>.input.file : /Scratch/model/output/CSO_output_%Y%m%d_%H%M_gridded.nc

		Specify a list of variables to be plotted, for example the retrieved and simulated column::

		<rcbase>.vars : yr ys

		By default the ``var`` is expected to be a variable in the data file;
		use the following source specifications to explicitly define the origin::
		For each variable, define the name of the source variable::

		<rcbase>.var.yr.source :: data:vcd
		<rcbase>.var.ys.source :: state:vcd
		<rcbase>.var.yr.source : yr
		<rcbase>.var.yr.source : ys

		Optionally specify target units that are different from the input::

		! convert units:
		<rcbase>.var.vcd.units : 1e15 mlc/cm2
		<rcbase>.var.yr.units : 1e15 mlc/cm2

		The value range of the colorbar could be tunes using (default limits are based on data values)::

		@@ -864,13 +909,21 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :

		<rcbase>.var.qa_flag.colors : ['red','yellow','green']

		The label below the colorbar will by default show the variable name,
		unless a ``long_name`` is defined::

		<rcbase>.var.vcd.long_name : retrieved vertical column density

		The name of the created image files is read from::

		! target files, time tempates are replaced:
		<rcbase>.output.file : %Y/%m/%d/S5p_RPRO_NO2_%Y%m%d_%H%M_%{var}.png
		<rcbase>.output.file : %Y/%m/%d/S5p_RPRO_NO2_%Y%m%d_%H%M_gridded_%{var}.png

		2018/06/01/S5p_RPRO_NO2_03278__vcd.png
		S5p_RPRO_NO2_03278__qa_value.png
		which will give::

		2020/01/01/CSO_output_20200101_0100_gridded_yr.png
		CSO_output_20200101_0100_gridded_ys.png
		:

		Enable the following flag to re-create existing files,
		by default only non-existing files are created::
		@@ -882,11 +935,6 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		! figure size (inches), default is (8,6):
		<rcbase>.figsize : (6,6)

		Specify the domain of the map, projection is regular longitude/latitude:

		! map domain (west east south north):
		<rcbase>.domain : -30 45 35 75

		"""

		def __init__( self, rcfile, rcbase='', env={}, indent='' ) :
		@@ -898,9 +946,9 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		# modules:
		import os
		import pandas
		import fnmatch
		import numpy
		import datetime
		import calendar
		import matplotlib
		import matplotlib.pyplot as plt

		@@ -922,22 +970,10 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		# time range:
		t1 = self.GetSetting( 'timerange.start', totype='datetime' )
		t2 = self.GetSetting( 'timerange.end' , totype='datetime' )
		tstep = self.GetSetting( 'timerange.step' )
		# info ...
		tfmt = '%Y-%m-%d %H:%M'
		logging.info( indent+'timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )

		# target directory:
		workdir = self.GetSetting( 'output.dir' )

		# info ..
		logging.info( indent+'work directory:' )
		logging.info( indent+' %s' % workdir )
		# create?
		if not os.path.isdir(workdir) : os.makedirs( workdir )
		# store current:
		owd = os.getcwd()
		# change to:
		os.chdir( workdir )
		logging.info( indent+'timerange: [%s,%s] by %s' % (t1.strftime(tfmt),t2.strftime(tfmt),tstep) )

		# target file template:
		figfile_template = self.GetSetting( 'output.file' )
		@@ -952,6 +988,9 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		# figure size:
		figsize = eval( self.GetSetting('figsize') )

		# extra map properties:
		bmp_kwargs = self.GetSetting( 'map', totype='dict', default=dict() )

		# info ..
		logging.info( indent+'time loop ...' )
		# time loop:
		@@ -960,21 +999,29 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		# info ..
		logging.info( indent+' %s ...' % t.strftime('%Y-%m-%d %H:%M') )

		# no data read yet:
		indata = {}

		# loop over variables ...
		for varname in varnames :

		# settings for this variable:
		vkey = 'var.%s' % varname
		# originating variable:
		vsource = self.GetSetting( vkey+'.source' )
		# variable source: all:ys
		vfile,vname = vsource.split(':')
		# timestep:
		if tstep in ['hour','hourly'] :
		long_timestamp = t.strftime('%Y-%m-%d %H:%M')
		dt = datetime.timedelta(0,3600)
		elif tstep in ['day','daily'] :
		long_timestamp = t.strftime('%Y-%m-%d')
		dt = datetime.timedelta(1)
		elif tstep in ['month','monthly'] :
		t = datetime.datetime(t.year,t.month,1,0,0)
		long_timestamp = t.strftime('%Y-%m')
		weekday,nday = calendar.monthrange(t.year,t.month)
		dt = datetime.timedelta(nday)
		elif tstep in ['year','yearly'] :
		t = datetime.datetime(t.year,1,1,0,0)
		long_timestamp = t.strftime('%Y')
		dt = datetime.timedelta(366)
		else :
		logging.error( 'unsupported time step "%s"' % tstep )
		raise Exception
		#endif

		# input file:
		infile = self.GetSetting( 'input.%s.file' % vfile )
		infile = self.GetSetting( 'input.file' )
		infile = t.strftime( infile )

		# input file present?
		@@ -983,6 +1030,12 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		# info ...
		logging.info( indent+' found input file: %s' % infile )

		# no data read yet:
		indata = None

		# loop over variables ...
		for varname in varnames :

		# target file:
		figfile = t.strftime( figfile_template )
		figfile = figfile.replace( '%{var}', varname )
		@@ -993,21 +1046,26 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		logging.info( indent+' create %s ...' % figfile )

		# read?
		if vfile not in indata.keys() :
		if indata is None :
		# info ...
		logging.info( indent+' read ...' )
		# init storage and read:
		indata[vkey] = cso_file.CSO_File( filename=infile )
		indata = cso_file.CSO_File( filename=infile )
		#endif

		# annote:
		title = t.strftime('%Y-%m-%d %H:%M')
		#title = t.strftime('%Y-%m-%d')
		title = long_timestamp

		# settings for this variable:
		vkey = 'var.%s' % varname
		# originating variable:
		vsource = self.GetSetting( vkey+'.source' )
		# target units:
		vunits = self.GetSetting( vkey+'.units' , default='None' )
		# plot type:
		ptype = self.GetSetting( vkey+'.type', default='map' )
		# long name used in labels:
		long_name = self.GetSetting( vkey+'.long_name', default=varname )

		# switch:
		if ptype == 'map' :
		@@ -1018,7 +1076,7 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		colors = eval( self.GetSetting( vkey+'.colors', default='None' ) )

		# extract values:
		vda = indata[vkey].ds[vname]
		vda = indata.ds[vsource]

		# grid:
		if 'longitude' in vda.coords.keys() :
		@@ -1039,13 +1097,16 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :
		values,units = self._ConvertUnits( values, vda.attrs, vunits )

		# annote:
		label = '%s [%s]' % (varname,units)
		label = '%s [%s]' % (long_name,units)

		# map properties:
		bmp = dict( resolution='i', countries=True, title=title )
		bmp.update( bmp_kwargs )

		# create map figure:
		fig = cso_plot.QuickMap( values, xx=xm, yy=ym, vmin=vmin, vmax=vmax,
		fig = cso_plot.QuickMap( values, xm=xm, ym=ym, vmin=vmin, vmax=vmax,
		cmap=dict(colors=colors),
		bmp=dict(resolution='i',countries=True,title=title),
		cbar=dict(label=label),
		bmp=bmp, cbar=dict(label=label),
		figsize=figsize )
		# save:
		fig.Export( figfile )
		@@ -1060,24 +1121,20 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :

		#endif # renew

		else :
		logging.info( indent+' no input file: %s' % infile )
		#endif # state file present

		## testing ...
		#break

		#endfor # variables

		else :
		logging.info( indent+' no input file: %s' % infile )
		#endif # state file present

		# next:
		t = t + datetime.timedelta(0,3600) # hourly
		#t = t + datetime.timedelta(1) # daily
		t = t + dt

		#endfor # time loop

		# back:
		os.chdir( owd )

		#enddef __init__


		@@ -1085,8 +1142,6 @@ class CSO_GriddedCatalogue( CSO_CatalogueBase ) :





		########################################################################
		###
		### end