TNO Intern

Commit f535af85 authored by Arjo Segers's avatar Arjo Segers
Browse files

Increase waiting time after failed download.

parent ed7cb7a5

py/cso_dataspace.py

+14 −6
Original line number Diff line number Diff line
@@ -21,8 +21,9 @@ 
#   Sort inquiry table (in place) by orbit number.

#

# 2024-08, Arjo Segers

#   Changed number of attemps and delay based on latest experiences.

#   Trap "404" error on files that cannot be downloaded.

#   Increase waiting time after failed download.

#   Trap "404 Client Error", display messate that inquire table might need an update.

#   Sort listing inplace.

#



########################################################################
@@ -702,7 +703,9 @@ class CSO_DataSpace_Downloader(object): 
        Optional arguments:



        * ``maxtry`` : number of times to try again if download fails

        * ``nsec_wait`` : delay in seconds between requests

        * ``nsec_wait`` : initial delay in seconds between requests; 

          with every new attempt, this will be increased with a maximum of:

        * ``nsec_wait_max``



        """
@@ -725,6 +728,8 @@ class CSO_DataSpace_Downloader(object): 
            self.CreateToken(href, indent=indent)

        # endif



        # info ..

        logging.info(f"{indent}download ...")

        # retry loop ..

        ntry = 1

        while True:
@@ -734,7 +739,7 @@ class CSO_DataSpace_Downloader(object): 
                headers = {"Authorization": f"Bearer {self.access_token}"}

                # ensure that "~/.netrc" is ignored by passing null-authorization,

                # otherwise the token in the header is overwritten by a token formed

                # from the login/password in the rcfile if that is found:

                # from the login/password in the netrc file if that is found:

                r = requests.get(href, auth=NullAuth(), headers=headers)

                # check status, raise error if request failed:

                r.raise_for_status()
@@ -803,8 +808,9 @@ class CSO_DataSpace_Downloader(object): 
                    logging.warning(f"{indent}renew token ...")

                    self.CreateToken(href, indent=indent)

                elif msg.startswith("404 Client Error: Not Found for url:"):

                    logging.warning(f"Could not download file {output_file}. Skipping file")

                    break

                    logging.error(f"url seems not available anymore; maybe need to update inquiry table?")

                    # quit with error:

                    raise

                # endif



            except MemoryError as err:
@@ -837,6 +843,8 @@ class CSO_DataSpace_Downloader(object): 
            else:

                logging.warning(f"{indent}wait {nsec_wait} seconds ...")

                time.sleep(nsec_wait)

                # next time, wait a bit longer, but not too long ...

                nsec_wait = min( nsec_wait*2, nsec_wait_max )

                logging.warning(f"{indent}attempt {ntry} / {maxtry} ...")

                continue  # while-loop

            # endif