TNO Intern

Commit eae4aa3e authored by Arjo Segers's avatar Arjo Segers
Browse files

Support scanning of multiple archive directories.

parent fbc2af60

py/cso_colhub.py

+107 −95
Original line number Diff line number Diff line
@@ -100,11 +100,11 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): 


    This file could be used to scan for available versions and how they were produced.



    In the settings, define the base directory of the archive::

    In the settings, define one or more base directories of the archive::



        <rcbase>.dir           :  /archive/mirror

        <rcbase>.dir           :  /archive/mirror  /scratch/mirror2



    This directory is recursively scanned using the :py:class:`os.walk` class on files with a specified form::

    The directories are recursively scanned using the :py:class:`os.walk` class on files that match a filename pattern:::



        ! search S5P CH4 files:

        <rcbase>.pattern       :  S5P_*_L2_CH4___*.nc
@@ -149,6 +149,9 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): 
        # init base object:

        utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env)



        # directory creation mode:

        dmode = self.GetSetting("dmode", totype=int, default=None)



        # renew output?

        renew = self.GetSetting("renew", totype="bool")
@@ -165,8 +168,12 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): 
            # initiallize for (re)creation:

            listing = cso_file.CSO_Listing()



            # archive directory:

            archive_dir = self.GetSetting("dir")

            # archive directories

            archive_dirs = self.GetSetting("dir").split()

            # loop over paths:

            for archive_dir in archive_dirs:

                # info ..

                logging.info(f"{indent}walk through %s ..." % archive_dir)

                # check ..

                if not os.path.isdir(archive_dir):

                    logging.error(f"archive directory not found: {archive_dir}")
@@ -258,10 +265,12 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): 


                # endfor # walk



            # endfor # archive_dir



            # sort:

            listing.Sort(by="orbit")

            # save:

            listing.Save(lst_file, indent=f"{indent}  ")

            listing.Save(lst_file, dmode=dmode, indent=f"{indent}  ")



        else:

            # info ..
@@ -361,6 +370,9 @@ class CSO_ColHubMirror_Missing(utopya.UtopyaRc): 
        # init base object:

        utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env)



        # directory creation mode:

        dmode = self.GetSetting("dmode", totype=int, default=None)



        # renew output?

        renew = self.GetSetting("renew", totype="bool")
@@ -452,7 +464,7 @@ class CSO_ColHubMirror_Missing(utopya.UtopyaRc): 
            # endfor



            # save:

            listing.Save(lst_file, indent=f"{indent}  ")

            listing.Save(lst_file, dmode=dmode, indent=f"{indent}  ")



        else:

            # info ..