Support zip files downloaded from Copernicus DataSpace that are actually netcdf files. (721c4b34) · Commits · CAMS / CSO

py/cso_dataspace.py

+62 −46

Original line number	Diff line number	Diff line
		@@ -24,17 +24,21 @@
		# Increase waiting time after failed download.
		# Trap "404 Client Error", display messate that inquire table might need an update.
		# Sort listing inplace.
		# s
		#
		# 2024-11, Arjo Segers
		# Inquire Dataspace per month after change in allowed maximum number of records.
		# Sort listing files by processing and processor version.
		# Increased maximum number of records and introduced sort order to avoid that
		# search request return a different result when repeated.
		#
		# 2025-01, Arjo Segers
		# Support download products that are netCDF files rather than zip archives.
		#
		# 2025-02, Arjo Segers
		# Support user defined directory creation mode.
		#


		########################################################################
		###
		### help
		@@ -795,8 +799,8 @@ class CSO_DataSpace_Downloader(object):
		# check status, raise error if request failed:
		r.raise_for_status()

		# product is a zip-file:
		product_file = "product.zip"
		# product is either a zip-file or a netcdf file ...
		product_file = "product.dat"
		# info ..
		logging.info(f"{indent}write to {product_file} ...")
		# write to temporary target first ..
		@@ -812,7 +816,10 @@ class CSO_DataSpace_Downloader(object):
		# rename:
		os.rename(tmpfile, product_file)

		# open product file:
		# try to open product file;
		# first try if it is a zipfile:
		try:
		# open as zipfile:
		arch = zipfile.ZipFile(product_file, mode="r")
		# loop over members, probably two files in a directory:
		# S5P_RPRO_L2__CH4____20200101T005246_etc/S5P_RPRO_L2__CH4____20200101T005246_etc.cdl
		@@ -833,7 +840,7 @@ class CSO_DataSpace_Downloader(object):
		# info ..
		logging.info(f"{indent}store ...")
		# create target dir if necessary:
		cso_file.CheckDir(output_file, dmode=dmode)
		cso_file.CheckDir(output_file)
		# move to destination:
		os.rename(member, output_file)
		# remove directory tree:
		@@ -846,6 +853,28 @@ class CSO_DataSpace_Downloader(object):
		logging.info(f"{indent}remove product file ...")
		# remove package:
		os.remove(product_file)
		#
		except Exception as err:
		# info ..
		msg = str(err)
		# logging.error("from download; message received:")
		# logging.error(" %s" % msg)
		# catch known problem ...
		if "File is not a zip file" in msg:
		# logging.warning(f"{indent}maybe download was interrupted, try again ...")
		# info ..
		logging.info(f"{indent}product is no zipfile, rename to output file ...")
		# this is probably the target file already;
		# create target dir if necessary:
		cso_file.CheckDir(output_file)
		# rename to destination:
		os.rename(product_file, output_file)
		else:
		# quit with error:
		raise
		# endif
		# endtry

		# all ok, leave retry loop:
		break

		@@ -872,19 +901,6 @@ class CSO_DataSpace_Downloader(object):
		# quit with error:
		raise

		except Exception as err:
		# info ..
		msg = str(err)
		logging.error("from download; message received:")
		logging.error(" %s" % msg)
		# catch known problem ...
		if msg.startswith("File is not a zip file"):
		logging.warning(f"{indent}maybe download was interrupted, try again ...")
		else:
		# quit with error:
		raise
		# endif

		# endtry

		# increase counter: