TNO Intern

Commit 1d6a48ad authored by Arjo Segers's avatar Arjo Segers
Browse files

Optionally search in multiple mirror archives for missing files.

parent e6ff5d2e

py/cso_colhub.py

+31 −13
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#

# 2025-02, Arjo Segers

#   Fixed type conversion of "dmode" setting.

#   Optionally search in multiple mirror archives for missing files.

#



########################################################################
@@ -322,10 +323,11 @@ class CSO_ColHubMirror_Missing(utopya.UtopyaRc): 
        <rcbase>.all.file           :  /work/inquire/Copernicus_S5p_NO2_dataspace__%Y-%m-%d.csv

        !<rcbase>.all.filedate       :  2025-01-24

        

    Similar specify the name of the file that is listing the current mirror, 

    Similar specify the name (or ";" seperated list of names of the file that is listing the current mirror(s), 

    probably the output of the :py:class:`CSO_ColHubMirror_Inquire` class::



        <rcbase>.curr.file           :  /work/inquire/Copernicus_S5p_NO2_colhub-mirror__%Y-%m-%d.csv

        <rcbase>.curr.file           :  /work/inquire/Copernicus_S5p_NO2_colhub-mirror__%Y-%m-%d.csv ; \\

                                        /work/inquire/Copernicus_S5p_NO2_colhub-mirror2__%Y-%m-%d.csv

        !<rcbase>.curr.filedate       :  2025-01-24

        

    Specify a selection filter; this defines which of the orbit files are actually needed::
@@ -413,14 +415,20 @@ class CSO_ColHubMirror_Missing(utopya.UtopyaRc): 
            listing_all = cso_file.CSO_Listing(listfile_all)



            # table with currently already available files:

            listfile_curr = self.GetSetting("curr.file")

            listfiles = self.GetSetting("curr.file").split(";")

            # evaluate time?

            filedate = self.GetSetting(

                "curr.filedate", totype="datetime", default=datetime.datetime.now()

            )

            listfile_curr = filedate.strftime(listfile_curr)

            # read:

            listing_curr = cso_file.CSO_Listing(listfile_curr)

            listings_curr = []

            for listfile in listfiles :

                # evaluate time templates if necessary:

                listfile = filedate.strftime(listfile)

                listfile = listfile.strip()

                # read:

                listings_curr.append( cso_file.CSO_Listing(listfile) )

            #endfor



            # extract orbits:

            orbits = listing_all.GetValues("orbit").unique()
@@ -456,11 +464,22 @@ class CSO_ColHubMirror_Missing(utopya.UtopyaRc): 
                    rec = xlst.GetRecord(irec)

                    ## info ..

                    # logging.info( f"{indent}      {fname}" )

                    # set flag:

                    found = False

                    # loop over mirror listings:

                    for listing_curr in listings_curr :

                        # check if already available:

                    if rec["filename"] in listing_curr:

                        found = rec["filename"] in listing_curr

                        # found?

                        if found :

                            # info ...

                            logging.info(f"{indent}    file already present ...")

                    else:

                            # leave:

                            break

                        # endif

                    # endfor

                    # not found?

                    if not found:

                        # info ...

                        logging.info(f"{indent}    file not present yet, add to list ...")

                        # add record to list:
@@ -468,8 +487,7 @@ class CSO_ColHubMirror_Missing(utopya.UtopyaRc): 
                    # endif

                # endfor



                # testing ...

                # break

                ## TESTING ...

                #if len(listing) > 0 :

                #    logging.warning(f"BREAK!")

                #    break