Support ESA `PAL` archive.

########################################################################

###

### help

###

########################################################################

"""

.. _cso-pal:

******************

``cso_pal`` module

******************

The :py:mod:`cso_pal` module provides classes for accessing  data from the 

`Product Algorithm Laboratory <https://www.s5p-pal.com/>`_.

Data is available from a special portal: `S5P-PAL Data Portal <https://data-portal.s5p-pal.com/>`_.

To browse through the data, use the `browser interface <https://data-portal.s5p-pal.com/browser/>`_.

.. _pal-api:

PAL API

=======

See the PAL `API info <https://data-portal.s5p-pal.com/cat-doc>`_ for latest info.

S5P-PAL product files can be selected and downloaded using the *Spatio Temporal Asset Catalog* (STAC).

The `PySTAC <https://pystac-client.readthedocs.io/en/latest/` Python interface is used for access.

Class hierchy

=============

The classes and are defined according to the following hierchy:

* :py:class:`.UtopyaRc`

  * :py:class:`.CSO_PAL_Inquire`

Classes

=======

"""

########################################################################

###

### modules

###

########################################################################

# modules:

import logging

# tools:

import utopya

########################################################################

###

### OpenSearch inquire

###

########################################################################

class CSO_PAL_Inquire( utopya.UtopyaRc ) :

    """

    Inquire available Sentinel data from the `Product Algorithm Laboratory <https://www.s5p-pal.com/>`_.

    A query is sent to search for products that are available 

    for a certain time and overlap with a specified region. 

    The result is a list with orbit files and instructions on how to download them.

    In the settings, specify the url of the portal::

      <rcbase>.url              :  https://data-portal.s5p-pal.com/cat/sentinel-5p/catalog.json

    Specify the time range over which files should be downloaded::

      <rcbase>.timerange.start  :  2018-07-01 00:00

      <rcbase>.timerange.end    :  2018-07-01 23:59

    Provide a product type::

        ! product type (always 10 characters!):

        <rcbase>.producttype            :  L2__NO2___

    Eventually specify a target area, only orbits with some pixels within the defined box will be downloaded::

      ! target area, leave empty for globe; format:  west,south:east,north

      <rcbase>.area             :  

      !<rcbase>.area             :  -30,30:35,76

    Name of output csv file::

        ! output table, date of today:

        cso.s5p.no2.inquire-s5phub.output.file            :  ${my.work}/PAL_S5P_NO2_%Y-%m-%d.csv

    Example records::

        orbit;         start_time;           end_time;processing;collection;processor_version;filename                                                                              ;href

        02832;2018-05-01 00:00:52;2018-05-01 01:42:22;PAL_      ;01        ;020301           ;S5P_PAL__L2__NO2____20180501T000052_20180501T014222_02832_01_020301_20211108T132200.nc;https://data-portal.s5p-pal.com/cat/sentinel-5p/download/c3e90b62-c1f6-47b6-9ed7-bd70aa6f46ce

        02833;2018-05-01 01:42:22;2018-05-01 03:23:52;PAL_      ;01        ;020301           ;S5P_PAL__L2__NO2____20180501T014222_20180501T032352_02833_01_020301_20211108T163508.nc;https://data-portal.s5p-pal.com/cat/sentinel-5p/download/a08cda20-0018-434e-bdcc-800dd580db0f

        :

    """

    def __init__( self, rcfile, rcbase='', env={}, indent='' ) :

        """

        Inquire oribt files.

        """

        # modules:

        import os

        import datetime

        import calendar

        import pystac

        import pystac_client

        import pandas

        # info ...

        logging.info( indent+'' )

        logging.info( indent+'** Inquire files available on PAL' )

        logging.info( indent+'' )

        # init base object:

        utopya.UtopyaRc.__init__( self, rcfile=rcfile, rcbase=rcbase, env=env )

        # domain:

        url = self.GetSetting( 'url' )

        # info ...

        logging.info( indent+'url          : %s' % url )

        # time range:

        t1 = self.GetSetting( 'timerange.start', totype='datetime' )

        t2 = self.GetSetting( 'timerange.end'  , totype='datetime' )

        # info ...

        tfmt = '%Y-%m-%d %H:%M'

        logging.info( indent+'timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )

        # product type (always 10 characters!):

        #    L2__NO2___

        producttype = self.GetSetting( 'producttype' )

        # filter for ItemSearch:

        sfilter = "s5p:file_type='%s'" % producttype

        # area of interest: west,south:east,north

        area = self.GetSetting( 'area' )

        # defined?

        if len(area) > 0 :

            # convert from format for "dhusget.sh":

            #    west,south:east,north

            west,south,east,north = map( float, area.replace(':',' ').replace(',',' ').split() )

            # info ...

            logging.info( indent+'area         : [%8.2f,%8.2f] x [%8.2f,%8.2f]' % (west,east,south,north) )

            # intersection description:

            intersects = { 'type' : 'Polygon',

                           'coordinates' : [[[west,south],[east,south],[east,north],[west,north],[west,south]]] }

        else :

            # info ...

            logging.info( indent+'area         : no' )

            # no intersection:

            intersects = None

        #endif

        # target file, might include time templates:

        output_file__template = self.GetSetting( 'output.file' )

        # current time:

        output_file = datetime.datetime.now().strftime( output_file__template )

        # new output table:

        output_df = pandas.DataFrame()

        # info ...

        logging.info( indent+'search all items in timerange ...' )

        # loop over months to follow progress ..

        xt2 = t1

        while xt2 < t2 :

            # start:

            xt1 = xt2

            # end of month:

            weekday,nday = calendar.monthrange(xt1.year,xt1.month)

            xt2 = datetime.datetime(xt1.year,xt1.month,1) + datetime.timedelta(nday)

            # limit:

            xt2 = min( xt2, t2 )

            # info ..

            logging.info( indent+'  %s ..' % xt1.strftime('%Y-%m') )

            # collection:

            collection = pystac.Collection.from_file( url )

            # end point for item search:

            endpoint_url = collection.get_single_link("search").target

            # search:

            item_collection = pystac_client.ItemSearch( endpoint_url, 

                                                        datetime=(xt1,xt2),

                                                        intersects=intersects,

                                                        filter=sfilter ).get_all_items()

            # info ..

            logging.info( indent+'    found %i items' % len(item_collection) )

            # testing ..

            for item in item_collection :

                # target file:

                filename = item.properties["physical_name"]

                ## info ...

                #logging.info( indent+'    file : %s' % filename )

                #

                # filenames:

                #

                #    S5P_OFFL_L2__NO2____20180701T005930_20180701T024100_03698_01_010002_20180707T022838.nc

                #    plt proc [product-] [starttime....] [endtime......] orbit cl prvers [prodtime.....]

                #

                bname = os.path.basename(filename).replace('.nc','')

                # split:

                platform_name,rest = bname.split('_',1)

                processing   = rest[0:4]

                product_type = rest[5:15]

                start_time,end_time,orbit,collection,processor_version,production_time = rest[16:].split('_')

                # convert:

                tfmt = '%Y%m%dT%H%M%S'

                ts = datetime.datetime.strptime(start_time,tfmt)

                te = datetime.datetime.strptime(end_time  ,tfmt)

                # extract download link:

                href = item.assets['download'].href

                # row:

                rec = { 'orbit'               : [orbit],

                        'start_time'          : [ts],

                        'end_time'            : [te],

                        'processing'          : [processing],

                        'collection'          : [collection],

                        'processor_version'   : [processor_version],

                        'filename'            : [filename],

                        'href'                : [href] }

                # add record:

                output_df = output_df.append( pandas.DataFrame(rec), ignore_index=True )

            #endfor # entries

        #endwhile  # months

        # info ..

        logging.info( 'save to: %s ...' % output_file )

        # create directory:

        dirname = os.path.dirname( output_file )

        if len(dirname) > 0 :

            if not os.path.isdir(dirname) : os.makedirs( dirname )

        #endif

        # write:

        output_df.to_csv( output_file, sep=';', index=False )

        # info ...

        logging.info( indent+'' )

        logging.info( indent+'** end inquire' )

        logging.info( indent+'' )

    #enddef __init__

#endclass CSO_PAL_Inquire

########################################################################

###

### end

###

########################################################################