Removed depricated DHUS downloads. (acd47b00) · Commits · CAMS / CSO

py/cso_scihub.py

+1 −212

Original line number	Diff line number	Diff line
		@@ -185,10 +185,8 @@ The classes and are defined according to the following hierchy:
		* :py:class:`.UtopyaRc`

		* :py:class:`.CSO_SciHub_Inquire`
		* :py:class:`.CSO_SciHub_DownloadFile`
		* :py:class:`.CSO_SciHub_Download`
		* :py:class:`.CSO_SciHub_Download_DHuS`
		* :py:class:`.CSO_SciHub_Listing`
		* :py:class:`.CSO_SciHub_ListingPlot`



		@@ -998,215 +996,6 @@ class CSO_SciHub_Download( utopya.UtopyaRc ) :
		#endclass CSO_SciHub_Download


		########################################################################
		###
		### DHuS download
		###
		########################################################################

		class CSO_SciHub_Download_DHuS( utopya.UtopyaRc ) :

		"""
		Download Sentinel data from the `Copernicus Open Access Hub <https://scihub.copernicus.eu/>`_
		using DHuS ("Data Hub Software").

		A download script ``dhusget.sh`` is available from:

		* `Copernicus Open Access Hub <https://scihub.copernicus.eu/>`_

		* `User Guide <https://scihub.copernicus.eu/userguide/>`_

		* `8 Batch Scripting <https://scihub.copernicus.eu/userguide/BatchScripting>`_

		A copy of the script is included with CSO, with some minor modifications.
		The settings should specify the location of the local copy::

		! location of script:
		<rcbase>.script : ${PWD}/bin/dhusget.sh

		The script is called by this class using arguments from the settings.
		First specify the time range over which files should be downloaded::

		<rcbase>.timerange.start : 2018-07-01 00:00
		<rcbase>.timerange.end : 2018-07-01 23:59

		Then specify the url of the hub, which is either the Open Data hub or the S5-P hub::

		! server url; provide login/password in ~/.netrc:
		<rcbase>.url : https://s5phub.copernicus.eu/dhus

		Specify a target area, only orbits with some pixels within the defined box will be downloaded::

		! target area, leave empty for globe; format: west,south:east,north
		<rcbase>.area :
		!<rcbase>.area : -30,30:35,76

		A query is used to select the required data.
		The search box on the hub could be used for inspiration on the format.
		Note that the '``producttype``' should have exactly 10 characters,
		with the first 3 used for the retrieval level, and the last 6 for the product;
		empty characters should have an underscore instead::

		! search query, obtained from interactive download:
		<rcbase>.query : platformname:Sentinel-5 AND \\
		producttype:L2__NO2___ AND \\
		processinglevel:L2 AND \\
		processingmode:Offline

		The target directory for downloaded file could include templates for time values::

		! output archive, store per month:
		<rcbase>.output.dir : /data/Copernicus/S5P/OFFL/NO2/%Y/%m

		Also specify a temporary work directory where the ``dhusget.sh`` script will actually
		run and create its log files etc::

		! work directory, will contain log files etc:
		<rcbase>.work.dir : /scratch/tmp.DHuS

		"""

		def __init__( self, rcfile, rcbase='', env={}, indent='' ) :

		"""
		Call dhusget script.
		"""

		# modules:
		import sys
		import os
		import datetime
		import subprocess

		# info ...
		logging.info( indent+'' )
		logging.info( indent+'** download from Copernicus Data Hub' )
		logging.info( indent+'' )

		# init base object:
		utopya.UtopyaRc.__init__( self, rcfile=rcfile, rcbase=rcbase, env=env )

		# domain:
		url = self.GetSetting( 'url' )
		# info ...
		logging.info( indent+'url : %s' % url )

		# area of interest: west,south:east,north
		area = self.GetSetting( 'area' )
		# info ...
		logging.info( indent+'area : %s' % area )

		# query, probably obtained from interactive download page:
		query = self.GetSetting( 'query' )
		# cleanup ..
		query = ' '.join( query.split() )
		# info ...
		logging.info( indent+'query : %s' % query )

		# work dir, might contain time templates:
		work_dir__template = self.GetSetting( 'work.dir' )

		# output dir, might contain time templates:
		output_dir__template = self.GetSetting( 'output.dir' )

		# full path to main script:
		script = self.GetSetting( 'script' )
		# check ..
		if not os.path.isfile(script) :
		logging.error( 'download script not found: %s' % script )
		raise Exception
		#endif

		# time range:
		t1 = self.GetSetting( 'timerange.start', totype='datetime' )
		t2 = self.GetSetting( 'timerange.end' , totype='datetime' )
		# info ...
		tfmt = '%Y-%m-%d %H:%M'
		logging.info( indent+'timerange: [%s,%s]' % (t1.strftime(tfmt),t2.strftime(tfmt)) )

		# loop over days:
		t = t1
		while t < t2 :

		# info ...
		logging.info( indent+'%s ...' % t.strftime('%Y-%m-%d') )

		# current:
		work_dir = t.strftime( work_dir__template )
		# info ..
		logging.info( indent+' work directory: %s' % work_dir )
		# create?
		if not os.path.isdir( work_dir ) : os.makedirs( work_dir )
		# goto:
		os.chdir( work_dir )

		# current:
		output_dir = t.strftime( output_dir__template )
		# info ..
		logging.info( indent+' output directory: %s' % output_dir )
		# create?
		if not os.path.isdir( output_dir ) : os.makedirs( output_dir )

		# start and end time:
		ts = t
		# add 24h:
		tx = t + datetime.timedelta(1)
		# convert to midnight:
		tx24 = datetime.datetime(tx.year,tx.month,tx.day,0,0)
		# not after t2 ...
		te = min( tx24, t2 )

		# full command:
		command = [script]
		# add url:
		command.extend( ['-d',url] )
		# time range:
		tfmt = '%Y-%m-%dT%H:%M:%S.%fZ'
		command.extend( ['-S', ts.strftime(tfmt)] )
		command.extend( ['-E', te.strftime(tfmt)] )
		# area of interest:
		if len(area) > 0 : command.extend( ['-c', area] )
		# query:
		if len(query) > 0 : command.extend( ['-F', query] )
		# download products:
		command.extend( ['-o','product'] )
		# output directory:
		command.extend( ['-O',output_dir] )
		# temporary lock dir:
		command.extend( ['-L',os.path.join(work_dir,'lock')] )
		## ... something wrong in script for flags without argument ...
		## remove files with failed MD5 check:
		#command.append( '-D' )

		# info ...
		line = ''
		for c in command :
		if ' ' in c :
		line = line+" '"+c+"'"
		else :
		line = line+" "+c
		#endif
		#endfor
		logging.info( indent+' command:' )
		logging.info( indent+' '+line )

		# run:
		subprocess.check_call( command )

		# start of next day:
		t = te
		#endwhile # day loop

		# info ...
		logging.info( indent+'' )
		logging.info( indent+'** end download' )
		logging.info( indent+'' )

		#enddef __init__

		#endclass CSO_SciHub_Download_DHuS


		########################################################################
		###
		### create listing file