TNO Intern

Commit 984e4115 authored by Arjo Segers's avatar Arjo Segers
Browse files

Write files with packing and compression.

parent 51c31053

py/cso_file.py

+678 −33

File changed.

Preview size limit exceeded, changes collapsed.

py/cso_s5p.py

+65 −38
Original line number Diff line number Diff line
@@ -7,6 +7,10 @@ 
# 2022-09, Arjo Segers, Met-Norway

#   Download files during conversion, eventually remove them when done.

# 

# 2022-09, Arjo Segers, Met-Norway

#   Added flags to disable compression and packing of variables on output, 

#   by default both are enabled.

#



########################################################################

###
@@ -699,13 +703,13 @@ class CSO_S5p_File( cso_file.CSO_File ) : 
        # selected scan lines:

        is0,is1 = iis.min(),iis.max()+1

        # scan line coordinate in track:

        track_scan = xarray.DataArray( numpy.arange(is0,is1,dtype='i4'), dims=('track_scan'), 

        track_scan = xarray.DataArray( numpy.arange(is0,is1,dtype='i2'), dims=('track_scan'), 

                                         attrs={ 'units' : '1', 'long_name' : 'scan line index within original track (0-based)' } )

        # store:

        self.AddCoord( 'track_scan', track_scan )

        

        # track pixel coordinate:

        track_pixel = xarray.DataArray( numpy.arange(track_npix,dtype='i4'), dims=('track_pixel'),

        track_pixel = xarray.DataArray( numpy.arange(track_npix,dtype='i2'), dims=('track_pixel'),

                                         attrs={ 'units' : '1', 'long_name' : 'pixel index within original track (0-based)' } )

        # store:

        self.AddCoord( 'track_pixel', track_pixel )
@@ -1064,6 +1068,11 @@ class CSO_S5p_File( cso_file.CSO_File ) : 
                da = xarray.DataArray( values, dims=vdims, 

                                         coords={ 'pixel' : pixel }, 

                                         attrs=oda.attrs )

                # ensure corrrect output:

                da.attrs['standard_name'] = 'time'

                da.attrs['long_name'    ] = 'time'

                da.encoding['calendar'] = 'standard'

                



            #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

            # full track, centers:
@@ -1898,6 +1907,18 @@ class CSO_S5p_Convert( utopya.UtopyaRc ) : 
        <rcbase>.output.attr.institution    :  CSO

        <rcbase>.output.attr.email          :  t.emplate@cso.org

        

    To reduce file size, by default all variables are written to file as short-integers (2 bytes) 

    accompanied by ``add_offset`` and ``scale_factor`` attributes. 

    A flag is available to disable packing.

    In addition, ``zlib`` compression is enabled.

    The default compression level is 1 (out of 9); set the following flag to a higher level

    to have stronger compression (on expense of computation time), or set to 0 to disable compression::

    

        ! pack floats as shorts:

        <rcbase>.output.packed              :  True

        ! zlib compression level (default 1, 0 for no compression):

        <rcbase>.output.complevel           :  1



    For testing an (optional) whitelist could be provided with output filenames (no path);

    if defined, only the listed files will be created::
@@ -1995,6 +2016,11 @@ class CSO_S5p_Convert( utopya.UtopyaRc ) : 
            attrs[key] = self.GetSetting( 'output.attr.%s' % (key) )

        #endfor

        

        # pack floats as shorts?

        packed = self.GetSetting( 'output.packed', 'bool' )

        # compression level:

        complevel = self.GetSetting( 'output.complevel', 'int' )



        ## max layers:

        #nlayer = self.GetSetting( 'leip.prod.xomi.max_nlayer_trop', 'int' )
@@ -2258,7 +2284,8 @@ class CSO_S5p_Convert( utopya.UtopyaRc ) : 
                    #endfor

                    attrs['orbit_file'] = input_filename_descr

                    # write:

                    csf.Write( filename=output_filename, attrs=attrs, history=history )

                    csf.Write( filename=output_filename, attrs=attrs, history=history, 

                                 packed=packed, complevel=complevel )

                    

                    ## testing ...

                    #logging.warning( indent+'break after first created file ...' )