TNO Intern

Commit e7e95cfe authored by Arjo Segers's avatar Arjo Segers
Browse files

Skip files that cannot be accessed.

parent 3f56562b

src/cso/cso_colhub.py

+17 −5
Original line number Diff line number Diff line
@@ -32,6 +32,9 @@ 
#   Added production time of data files to listings.

#   Select file with latest production time if multiple are found.

#

# 2026-04, Arjo Segers

#   Skip files that cannot be accessed, for example a broken link.

#



########################################################################

###
@@ -209,13 +212,22 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): 
                    for fname in files:

                        # data file?

                        if fnmatch.fnmatch(fname, fpattern):

                            # already in table?

                            if fname in listing:

                            # full path:

                            href = os.path.join(root, fname)

                            #~ sometimes links to non-existing targets ...

                            if not os.path.isfile(href):

                                # info ...

                                logging.info(f"{indent}  keep entry %s ..." % fname)

                                logging.info(f"{indent}  WARNING - invalid file (broken link?): {href}")

                            #

                            #~ already in table?

                            elif fname in listing:

                                # info ...

                                logging.info(f"{indent}  keep entry {fname} ...")

                            #

                            #~ new entry:

                            else:

                                # info ...

                                logging.info(f"{indent}  add entry %s ..." % fname)

                                logging.info(f"{indent}  add entry {fname} ...")

                                # Example filename:

                                #   S5P_RPRO_L2__CH4____20180430T001851_20180430T020219_02818_01_010301_20190513T141133.nc

                                #
@@ -270,7 +282,7 @@ class CSO_ColHubMirror_Inquire(utopya.UtopyaRc): 
                                data["collection"] = collection

                                data["processor_version"] = processor_version

                                data["production_time"] = production_time

                                data["href"] = os.path.join(root, fname)

                                data["href"] = href

                                # update record:

                                listing.UpdateRecord(fname, data, indent=f"{indent}    ")

                            # endif  # new record?