Improved re-try loop when downloading from PAL. (16f6fcce) · Commits · CAMS / CSO

py/cso_pal.py

+21 −7

Original line number	Diff line number	Diff line
		@@ -13,6 +13,8 @@
		#
		# 2025-02, Arjo Segers
		# Fixed error message.
		# Retry download if failed, with increasing waiting time between attempts.
		# Trap errors on write permissions.
		#

		########################################################################
		@@ -358,12 +360,14 @@ class CSO_PAL_Downloader(object):

		# *

		def DownloadFile(self, href, output_file, maxtry=10, nsec_wait=60, indent=""):
		def DownloadFile(
		self, href, output_file, maxtry=10, nsec_wait=5, nsec_wait_max=600, dmode=None, indent=""
		):
		"""
		Download file from PAL.

		If a request fails it is tried again up to a maximum of ``maxtry`` times,
		with a delay of ``nsec_wait`` between requsts.
		with an initial delay of ``nsec_wait`` between requsts.

		Arguments:

		@@ -376,7 +380,9 @@ class CSO_PAL_Downloader(object):
		Optional arguments:

		* ``maxtry`` : number of times to try again if download fails
		* ``nsec_wait`` : delay in seconds between requests
		* ``nsec_wait`` : initial delay in seconds between requests;
		with every new attempt, this will be increased up to a maximum of ``nsec_wait_max``
		* ``dmode``: directory creation mode, e.g. ``777``

		"""

		@@ -417,7 +423,7 @@ class CSO_PAL_Downloader(object):
		os.rename(tmpfile, product_file)

		# create target dir if necessary:
		cso_file.CheckDir(output_file)
		cso_file.CheckDir(output_file, dmode=dmode)
		# move to destination:
		os.rename(product_file, output_file)

		@@ -446,12 +452,18 @@ class CSO_PAL_Downloader(object):
		# info ..
		logging.error(f"{indent}from download; message received:")
		logging.error(f"{indent} {msg}")
		# catch known problem ...
		# catch known problems ...
		if msg.startswith("File is not a zip file"):
		logging.warning(f"{indent}maybe download was interrupted, try again ...")
		elif "Permission denied" in msg:
		logging.error(f"no permission to write in target directory:")
		logging.error(f" {os.path.dirname(output_file)}")
		raise Exception
		else:
		# quit with error:
		raise
		## quit with error:
		# raise
		# try again ...
		logging.warning(f"{indent}try again ...")
		# endif

		# endtry
		@@ -465,6 +477,8 @@ class CSO_PAL_Downloader(object):
		else:
		logging.warning(f"{indent}wait {nsec_wait} seconds ...")
		time.sleep(nsec_wait)
		# next time, wait a bit longer, but not too long ...
		nsec_wait = min(nsec_wait * 2, nsec_wait_max)
		logging.warning(f"{indent}attempt {ntry} / {maxtry} ...")
		continue # while-loop
		# endif