#
# CHANGES
#
# 2023, Lewis Blake, Arjo Segers
#   Formatted using "black".
#
# 2024-01, Arjo Segers
#   Added 'CSO_PAL_Downloader' class.
#

########################################################################
###
### help
###
########################################################################

"""
.. _cso-pal:

******************
``cso_pal`` module
******************

The :py:mod:`cso_pal` module provides classes for accessing  data from the
`Product Algorithm Laboratory <https://www.s5p-pal.com/>`_.

Data is available from a special portal: `S5P-PAL Data Portal <https://data-portal.s5p-pal.com/>`_.

To browse through the data, use the `browser interface <https://data-portal.s5p-pal.com/browser/>`_.


.. _pal-api:

PAL API
=======

See the PAL `API info <https://data-portal.s5p-pal.com/cat-doc>`_ for latest info.

S5P-PAL product files can be selected and downloaded using the *Spatio Temporal Asset Catalog* (STAC).
The `PySTAC <https://pystac-client.readthedocs.io/en/latest/>`_ Python interface is used for access.


Class hierchy
=============

The classes and are defined according to the following hierchy:


* :py:class:`.UtopyaRc`

  * :py:class:`.CSO_PAL_Inquire`
  * :py:class:`.CSO_PAL_Downloader`


Classes
=======


"""


########################################################################
###
### modules
###
########################################################################

# modules:
import logging

# tools:
import utopya


########################################################################
###
### OpenSearch inquire
###
########################################################################


class CSO_PAL_Inquire(utopya.UtopyaRc):

    """
    Inquire available Sentinel data from the `Product Algorithm Laboratory <https://www.s5p-pal.com/>`_.

    A query is sent to search for products that are available
    for a certain time and overlap with a specified region.
    The result is a list with orbit files and instructions on how to download them.

    In the settings, specify the url of the portal::

      <rcbase>.url              :  https://data-portal.s5p-pal.com/api/s5p-l2/collection.json

    Specify the time range over which files should be downloaded::

      <rcbase>.timerange.start  :  2018-07-01 00:00
      <rcbase>.timerange.end    :  2018-07-01 23:59

    Provide a product type::

        ! product type (always 10 characters!):
        <rcbase>.producttype            :  L2__NO2___

    Eventually specify a target area, only orbits with some pixels within the defined box will be downloaded::

      ! target area, leave empty for globe; format:  west,south:east,north
      <rcbase>.area             :
      !<rcbase>.area             :  -30,30:35,76

    Name of output csv file::

        ! output table, date of today:
        <rcbase>.output.file            :  ${my.work}/PAL_S5P_NO2_%Y-%m-%d.csv

    Example records::

        orbit;         start_time;           end_time;processing;collection;processor_version;filename                                                                              ;href
        02832;2018-05-01 00:00:52;2018-05-01 01:42:22;PAL_      ;01        ;020301           ;S5P_PAL__L2__NO2____20180501T000052_20180501T014222_02832_01_020301_20211108T132200.nc;https://data-portal.s5p-pal.com/cat/sentinel-5p/download/c3e90b62-c1f6-47b6-9ed7-bd70aa6f46ce
        02833;2018-05-01 01:42:22;2018-05-01 03:23:52;PAL_      ;01        ;020301           ;S5P_PAL__L2__NO2____20180501T014222_20180501T032352_02833_01_020301_20211108T163508.nc;https://data-portal.s5p-pal.com/cat/sentinel-5p/download/a08cda20-0018-434e-bdcc-800dd580db0f
        :

    """

    def __init__(self, rcfile, rcbase="", env={}, indent=""):

        """
        Inquire oribt files.
        """

        # modules:
        import os
        import datetime
        import calendar
        import pystac
        import pystac_client
        import pandas

        # info ...
        logging.info(indent + "")
        logging.info(indent + "** Inquire files available on PAL")
        logging.info(indent + "")

        # init base object:
        utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env)

        # domain:
        url = self.GetSetting("url")
        # info ...
        logging.info(indent + "url          : %s" % url)

        # time range:
        t1 = self.GetSetting("timerange.start", totype="datetime")
        t2 = self.GetSetting("timerange.end", totype="datetime")
        # info ...
        tfmt = "%Y-%m-%d %H:%M"
        logging.info(indent + "timerange: [%s,%s]" % (t1.strftime(tfmt), t2.strftime(tfmt)))

        # product type (always 10 characters!):
        #    L2__NO2___
        producttype = self.GetSetting("producttype")
        # filter for ItemSearch:
        sfilter = "s5p:file_type='%s'" % producttype

        # area of interest: west,south:east,north
        area = self.GetSetting("area")
        # defined?
        if len(area) > 0:
            # convert from format for "dhusget.sh":
            #    west,south:east,north
            west, south, east, north = map(float, area.replace(":", " ").replace(",", " ").split())
            # info ...
            logging.info(
                indent
                + "area         : [%8.2f,%8.2f] x [%8.2f,%8.2f]" % (west, east, south, north)
            )
            # intersection description:
            intersects = {
                "type": "Polygon",
                "coordinates": [
                    [[west, south], [east, south], [east, north], [west, north], [west, south]]
                ],
            }
        else:
            # info ...
            logging.info(indent + "area         : no")
            # no intersection:
            intersects = None
        # endif

        # target file, might include time templates:
        output_file__template = self.GetSetting("output.file")
        # current time:
        output_file = datetime.datetime.now().strftime(output_file__template)

        # new output table:
        output_df = pandas.DataFrame()

        # info ...
        logging.info(indent + "search all items in timerange ...")

        # loop over months to follow progress ..
        xt2 = t1
        while xt2 < t2:
            # start:
            xt1 = xt2
            # end of month:
            weekday, nday = calendar.monthrange(xt1.year, xt1.month)
            xt2 = datetime.datetime(xt1.year, xt1.month, 1) + datetime.timedelta(nday)
            # limit:
            xt2 = min(xt2, t2)
            # info ..
            logging.info(indent + "  %s .." % xt1.strftime("%Y-%m"))

            # collection:
            collection = pystac.Collection.from_file(url)
            # end point for item search:
            endpoint_url = collection.get_single_link("search").target
            # search:
            item_collection = pystac_client.ItemSearch(
                endpoint_url, datetime=(xt1, xt2), intersects=intersects, filter=sfilter
            ).get_all_items()
            # info ..
            logging.info(indent + "    found %i items" % len(item_collection))

            # testing ..
            for item in item_collection:

                # target file:
                filename = item.properties["physical_name"]
                ## info ...
                # logging.info( indent+'    file : %s' % filename )

                #
                # filenames:
                #
                #    S5P_OFFL_L2__NO2____20180701T005930_20180701T024100_03698_01_010002_20180707T022838.nc
                #    plt proc [product-] [starttime....] [endtime......] orbit cl prvers [prodtime.....]
                #
                bname = os.path.basename(filename).replace(".nc", "")
                # split:
                platform_name, rest = bname.split("_", 1)
                processing = rest[0:4]
                product_type = rest[5:15]
                (
                    start_time,
                    end_time,
                    orbit,
                    collection,
                    processor_version,
                    production_time,
                ) = rest[16:].split("_")

                # convert:
                tfmt = "%Y%m%dT%H%M%S"
                ts = datetime.datetime.strptime(start_time, tfmt)
                te = datetime.datetime.strptime(end_time, tfmt)

                # extract download link:
                href = item.assets["download"].href

                # row:
                rec = {
                    "orbit": [orbit],
                    "start_time": [ts],
                    "end_time": [te],
                    "processing": [processing],
                    "collection": [collection],
                    "processor_version": [processor_version],
                    "filename": [filename],
                    "href": [href],
                }
                # add record:
                output_df = pandas.concat([output_df, pandas.DataFrame(rec)], ignore_index=True)

            # endfor # entries

        # endwhile  # months

        # info ..
        logging.info("save to: %s ..." % output_file)
        # create directory:
        dirname = os.path.dirname(output_file)
        if len(dirname) > 0:
            if not os.path.isdir(dirname):
                os.makedirs(dirname)
        # endif
        # write:
        output_df.to_csv(output_file, sep=";", index=False)

        # info ...
        logging.info(indent + "")
        logging.info(indent + "** end inquire")
        logging.info(indent + "")

    # enddef __init__


# endclass CSO_PAL_Inquire


########################################################################
###
### PAL download
###
########################################################################


class CSO_PAL_Downloader(object):

    """
    Class to download single file from the `Product Algorithm Laboratory <https://www.s5p-pal.com/>`_.

    The :py:class:`DownloadFile` method should be used to
    actually download a file.

    Usage::

        # initialize downloader:
        downloader = CSO_PAL_Downloader()
        # download file, store in specified file:
        downloader.DownloadFile( "https://data-portal.s5p-pal.com/cat/sentinel-5p/download/88c15681-db43-4219-b391-c8567e39cccf", "orbit.nc" )

    """

    def __init__(self):

        """
        Initialize downloader.
        """

    # enddef __init__

    # *

    def DownloadFile(self, href, output_file, maxtry=10, nsec_wait=60, indent=""):

        """
        Download file from PAL.

        If a request fails it is tried again up to a maximum of ``maxtry`` times,
        with a delay of ``nsec_wait`` between requsts.

        Arguments:

        * ``href`` : download url, for example::

            https://data-portal.s5p-pal.com/cat/sentinel-5p/download/88c15681-db43-4219-b391-c8567e39cccf

        * ``output_file`` : target file

        Optional arguments:

        * ``maxtry`` : number of times to try again if download fails
        * ``nsec_wait`` : delay in seconds between requests

        """

        # modules:
        import sys
        import os
        import time
        import requests

        # tools:
        import cso_file

        # retry loop ..
        ntry = 1
        while True:
            # try to download and save:
            try:

                # get data:
                r = requests.get(href)
                # check status, raise error if request failed:
                r.raise_for_status()

                # product is netcdf file, use base name of target file:
                product_file = os.path.basename(output_file)
                # info ..
                logging.info(f"{indent}write to {product_file} ...")
                # write to temporary target first ..
                tmpfile = product_file + ".tmp"
                # open destination file for binary write:
                with open(tmpfile, "wb") as fd:
                    # prefered way to write content following:
                    #   https://docs.python-requests.org/en/master/user/quickstart/
                    for chunk in r.iter_content(chunk_size=128):
                        fd.write(chunk)
                    # endfor
                # endwith
                # rename:
                os.rename(tmpfile, product_file)

                # create target dir if necessary:
                cso_file.CheckDir(output_file)
                # move to destination:
                os.rename(product_file, output_file)

                # all ok, leave retry loop:
                break

            except requests.exceptions.HTTPError as err:
                # info ..
                msg = str(err)
                logging.warning(f"{indent}exception from download; message received:")
                logging.warning(f"{indent}  %s" % msg)
                # catch known problem ...
                if msg.startswith("401 Client Error: Unauthorized for url:"):
                    logging.warning(f"{indent}renew token ...")
                    self.CreateToken(href, indent=indent)
                # endif

            except MemoryError as err:
                logging.error("memory error from download; increase resources?")
                # quit with error:
                raise

            except Exception as err:
                # info ..
                logging.error("from download; message received:")
                logging.error("  %s" % str(err))
                # catch known problem ...
                if msg.startswith("File is not a zip file"):
                    logging.warning(f"{indent}maybe download was interrupted, try again  ...")
                else:
                    # quit with error:
                    raise
                # endif

            # endtry

            # increase counter:
            ntry += 1
            # switch:
            if ntry == maxtry:
                logging.warning(f"{indent}tried {maxtry} times; exit ...")
                raise Exception
            else:
                logging.warning(f"{indent}wait {nsec_wait} seconds ...")
                time.sleep(nsec_wait)
                logging.warning(f"{indent}attempt {ntry} / {maxtry} ...")
                continue  # while-loop
            # endif

        # endwhile # retry

    # enddef DownloadFile


# endclass CSO_PAL_Downloader


########################################################################
###
### end
###
########################################################################
