Extended error traps. (4e820add) · Commits · CAMS / CSO

py/cso_dataspace.py

+146 −122

Original line number	Diff line number	Diff line
		@@ -4,8 +4,8 @@
		# 2023-10, Arjo Segers
		# Tools to access Copernicus DataSpace.
		#
		# 2023-10, Arjo Segers
		# Trap errors from "requests.get" during inquire.
		# 2023-11, Arjo Segers
		# Extended error traps.
		#

		########################################################################
		@@ -191,6 +191,9 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc):
		# init base object:
		utopya.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env)

		# number of seconds to wait in retry loop:
		nsec_wait = 10

		# url of API:
		api_url = self.GetSetting("url")
		# info ...
		@@ -309,8 +312,8 @@ class CSO_DataSpace_Inquire(utopya.UtopyaRc):
		logging.error(f"{indent} tried {ntry} times now, exit ...")
		raise Exception
		else:
		logging.error(f"{indent} wait ..")
		time.sleep(10)
		logging.error(f"{indent} wait {nsec_wait} seconds ..")
		time.sleep(nsec_wait)
		logging.error(f"{indent} try again ...")
		ntry += 1
		continue
		@@ -543,7 +546,9 @@ class CSO_DataSpace_DownloadFile(object):
		"""

		# modules:
		import sys
		import os
		import time
		import urllib.parse
		import requests
		import zipfile
		@@ -552,6 +557,9 @@ class CSO_DataSpace_DownloadFile(object):
		# tools:
		import cso_file

		# number of seconds to wait in retry loop:
		nsec_wait = 10

		#
		# On linux system, login/passwords for websites and ftp can be stored in "~/.netrc" file:
		# ---[~/.netrc]-----------------------------------------------
		@@ -587,40 +595,59 @@ class CSO_DataSpace_DownloadFile(object):
		# identity server:
		domain = "identity.dataspace.copernicus.eu"
		url = f"https://{domain}/auth/realms/CDSE/protocol/openid-connect/token"
		# retry loop ..
		ntry = 0
		while True:
		# try to obtain token:
		try:
		# send request:
		r = requests.post(url, data=data)
		# check status, raise error if request failed:
		r.raise_for_status()
		# extract token from response:
		access_token = r.json()["access_token"]
		# all ok, leave try loop:
		break
		except requests.exceptions.HTTPError as err:
		# info ..
		msg = str(err)
		logging.error(f"exception from download; message received:")
		logging.error(f" {msg}")
		logging.error(f"{indent} exception from token creation; message received:")
		logging.error(f"{indent} {msg}")
		# catch known problem ...
		if msg.startswith("401 Client Error: Unauthorized for url:"):
		logging.error(f"Interpretation: the (username,password) received from")
		logging.error(f"your '~/.netrc' file are incorrect.")
		logging.error(f"For the Copernicus DataSpace, the file should contain:")
		logging.error(f" machine {p.netloc} login ** password **")
		logging.error(f"If the machine was not found, a default might have been received")
		raise Exception
		logging.error(f"{indent} Possible causes:")
		logging.error(f"{indent} * Just a random failure ...")
		logging.error(f"{indent} * The (login,password) pair received from your '~/.netrc' file are incorrect.")
		logging.error(f"{indent} For the Copernicus DataSpace, the file should contain:")
		logging.error(f"{indent} machine {p.netloc} login ** password **")
		logging.error(f"{indent} If the machine was not found, a default might have been received.")
		logging.error(f"{indent} Login received: {username}")
		logging.error(f"{indent} * System maintenance? Check the Copernicus DataSpace website.")
		else:
		raise Exception(f"Access token creation failed; server response: {r.json()}")
		logging.error(f"{indent} Access token creation failed; server response: {r.json()}")
		# endif
		except:
		raise Exception(f"Access token creation failed; server response: {r.json()}")
		# endtry # get access token
		# extract token from response:
		access_token = r.json()["access_token"]
		# info ...
		logging.error(f"{indent} Access token creation failed; server response: {r.json()}")
		# end try
		# increase counter:
		ntry += 1
		# switch:
		if ntry == maxtry:
		logging.warning(f"{indent} tried {maxtry} times; exit ...")
		raise Exception
		else:
		logging.warning(f"{indent} exception from token creation; wait {nsec_wait} seconds ...")
		time.sleep(nsec_wait)
		logging.warning(f"{indent} try again ...")
		continue # while-loop
		# endif
		# endwhile # retry

		# retry loop ..
		ntry = 0
		while True:
		# try to download and save:
		try:

		# try to download:
		try:

		# fill authorization token in header:
		@@ -683,6 +710,8 @@ class CSO_DataSpace_DownloadFile(object):
		logging.info(f"{indent} remove product file ...")
		# remove package:
		os.remove(product_file)
		# all ok, leave retry loop:
		break

		except requests.exceptions.HTTPError as err:
		# info ..
		@@ -704,23 +733,18 @@ class CSO_DataSpace_DownloadFile(object):

		# endtry

		# error from download or save:
		except:
		# increase counter:
		ntry += 1
		# switch:
		if ntry == maxtry:
		logging.warning(f"{indent} tried {maxtry} times ...")
		logging.warning(f"{indent} tried {maxtry} times; exit ...")
		raise Exception
		else:
		logging.warning(f"{indent} exception from download; try again ...")
		logging.warning(f"{indent} exception from download; wait {nsec_wait} seconds ...")
		time.sleep(nsec_wait)
		logging.warning(f"{indent} try again ...")
		continue # while-loop
		# endif
		# endtry
		# leave retry loop,
		# either because download was ok,
		# or because maximum number of retries was reached:
		break

		# endwhile # retry