From 36349836990db378ca37520c25c256aca7d6075b Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Thu, 24 Apr 2025 15:10:08 +0200 Subject: [PATCH 1/8] Updated paths to new source file locations. --- bin/cso | 4 +++- config/tutorial/tutorial.rc | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/cso b/bin/cso index 867cb97..151e42d 100755 --- a/bin/cso +++ b/bin/cso @@ -36,7 +36,9 @@ import logging # prefix of CSO installation: prefix = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), os.pardir)) # extend path: -sys.path.insert(0, os.path.join(prefix, "py")) +sys.path.insert(0, os.path.join(prefix, "src")) +sys.path.insert(0, os.path.join(prefix, "src", "utopya")) +sys.path.insert(0, os.path.join(prefix, "src", "cso")) # tools: import utopya diff --git a/config/tutorial/tutorial.rc b/config/tutorial/tutorial.rc index e324e22..5795f57 100644 --- a/config/tutorial/tutorial.rc +++ b/config/tutorial/tutorial.rc @@ -271,7 +271,7 @@ my.work : /Scratch/${USER}/CSO-Tutorial *.script.class : utopya.UtopyaJobScriptForeground ! search path for python modules: -*.pypath : ${CSO_PREFIX}/py +*.pypath : ${CSO_PREFIX}/src:${CSO_PREFIX}/src/utopya:${CSO_PREFIX}/src/cso ! work directory for jobs; *.workdir : ${my.work}/__NAME2PATH__ @@ -292,7 +292,10 @@ my.work : /Scratch/${USER}/CSO-Tutorial ! full time range: cso.tutorial.inquire-table-dataspace.timerange.start : 2018-01-01 00:00:00 -cso.tutorial.inquire-table-dataspace.timerange.end : 2024-01-01 00:00:00 +cso.tutorial.inquire-table-dataspace.timerange.end : 2025-01-01 00:00:00 +!! ... testing ... +!cso.tutorial.inquire-table-dataspace.timerange.start : 2018-06-01 00:00:00 +!cso.tutorial.inquire-table-dataspace.timerange.end : 2018-07-01 00:00:00 ! API url: cso.tutorial.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api -- GitLab From de7d092a08120735aced3868141930d548a46088 Mon Sep 17 00:00:00 2001 From: Arjo Segers Date: Thu, 24 Apr 2025 15:12:00 +0200 Subject: [PATCH 2/8] Changes in documentation and doc strings such that autodoc can import the cso and utopya packages. --- doc/source/conf.py | 8 +- doc/source/history.rst | 3 + doc/source/pymod-cso_catalogue.rst | 2 +- doc/source/pymod-cso_colhub.rst | 2 +- doc/source/pymod-cso_colocate.rst | 2 +- doc/source/pymod-cso_constants.rst | 2 +- doc/source/pymod-cso_dataspace.rst | 2 +- doc/source/pymod-cso_earthaccess.rst | 2 +- doc/source/pymod-cso_file.rst | 2 +- doc/source/pymod-cso_gridded.rst | 2 +- doc/source/pymod-cso_inquire.rst | 2 +- doc/source/pymod-cso_mapping.rst | 2 +- doc/source/pymod-cso_pal.rst | 2 +- doc/source/pymod-cso_plot.rst | 2 +- doc/source/pymod-cso_regions.rst | 2 +- doc/source/pymod-cso_s5p.rst | 2 +- doc/source/pymod-cso_superobs.rst | 2 +- doc/source/pymod-cso_tools.rst | 2 +- doc/source/pymod-utopya_base.rst | 2 +- doc/source/pymod-utopya_build.rst | 2 +- doc/source/pymod-utopya_index.rst | 2 +- doc/source/pymod-utopya_jobscript.rst | 2 +- doc/source/pymod-utopya_jobtree.rst | 2 +- doc/source/pymod-utopya_post.rst | 2 +- doc/source/pymod-utopya_rc.rst | 2 +- doc/source/pymod-utopya_runscript.rst | 2 +- doc/source/pymod-utopya_tools.rst | 2 +- doc/source/pymods.rst | 44 +++---- pyproject.toml | 1 + src/cso/__init__.py | 127 ++++++++++++++++++- src/cso/cso.py | 135 --------------------- src/cso/cso_catalogue.py | 2 +- src/cso/cso_colhub.py | 2 +- src/cso/cso_colocate.py | 2 +- src/cso/cso_earthaccess.py | 2 +- src/cso/cso_gridded.py | 2 +- src/cso/cso_inquire.py | 2 +- src/cso/cso_pal.py | 2 +- src/cso/cso_regions.py | 2 +- src/cso/cso_s5p.py | 2 +- src/cso/cso_superobs.py | 2 +- src/utopya/__init__.py | 157 +++++++++++++++++++++++- src/utopya/utopya.py | 168 -------------------------- src/utopya/utopya_build.py | 4 +- src/utopya/utopya_index.py | 35 +++--- src/utopya/utopya_jobtree.py | 6 +- src/utopya/utopya_post.py | 2 +- 47 files changed, 364 insertions(+), 396 deletions(-) delete mode 100644 src/cso/cso.py delete mode 100644 src/utopya/utopya.py diff --git a/doc/source/conf.py b/doc/source/conf.py index 11c49ab..850aa67 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -12,8 +12,10 @@ # import os import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, "py"))) +# +sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, "src"))) +sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, "src", "utopya"))) +sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, "src", "cso"))) # -- Project information ----------------------------------------------------- @@ -23,7 +25,7 @@ copyright = "2020-2025, Arjo Segers" author = "Arjo Segers" # The full version, including alpha/beta/rc tags -release = "v2.11" +release = "v2.12" # -- General configuration --------------------------------------------------- diff --git a/doc/source/history.rst b/doc/source/history.rst index 7daf338..990ef51 100644 --- a/doc/source/history.rst +++ b/doc/source/history.rst @@ -115,6 +115,9 @@ A summary of the versions and changes. * | *v2.11* | Support inquiry and download from NASA EarthData archive. + +* | *v2.12* + | Support running in Python virtual environment. To be included diff --git a/doc/source/pymod-cso_catalogue.rst b/doc/source/pymod-cso_catalogue.rst index 5aa1fa6..495cb62 100644 --- a/doc/source/pymod-cso_catalogue.rst +++ b/doc/source/pymod-cso_catalogue.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_catalogue +.. automodule:: cso.cso_catalogue diff --git a/doc/source/pymod-cso_colhub.rst b/doc/source/pymod-cso_colhub.rst index ab95fb9..19c7247 100644 --- a/doc/source/pymod-cso_colhub.rst +++ b/doc/source/pymod-cso_colhub.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_colhub +.. automodule:: cso.cso_colhub diff --git a/doc/source/pymod-cso_colocate.rst b/doc/source/pymod-cso_colocate.rst index 162b779..63018c5 100644 --- a/doc/source/pymod-cso_colocate.rst +++ b/doc/source/pymod-cso_colocate.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_colocate +.. automodule:: cso.cso_colocate diff --git a/doc/source/pymod-cso_constants.rst b/doc/source/pymod-cso_constants.rst index 0f5b7d2..0a024a2 100644 --- a/doc/source/pymod-cso_constants.rst +++ b/doc/source/pymod-cso_constants.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_constants +.. automodule:: cso.cso_constants diff --git a/doc/source/pymod-cso_dataspace.rst b/doc/source/pymod-cso_dataspace.rst index bb01d70..199438f 100644 --- a/doc/source/pymod-cso_dataspace.rst +++ b/doc/source/pymod-cso_dataspace.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_dataspace +.. automodule:: cso.cso_dataspace diff --git a/doc/source/pymod-cso_earthaccess.rst b/doc/source/pymod-cso_earthaccess.rst index f8695ad..50c48c9 100644 --- a/doc/source/pymod-cso_earthaccess.rst +++ b/doc/source/pymod-cso_earthaccess.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_earthaccess +.. automodule:: cso.cso_earthaccess diff --git a/doc/source/pymod-cso_file.rst b/doc/source/pymod-cso_file.rst index a5c6d35..a57b404 100644 --- a/doc/source/pymod-cso_file.rst +++ b/doc/source/pymod-cso_file.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_file +.. automodule:: cso.cso_file diff --git a/doc/source/pymod-cso_gridded.rst b/doc/source/pymod-cso_gridded.rst index 5b9e140..cf636b4 100644 --- a/doc/source/pymod-cso_gridded.rst +++ b/doc/source/pymod-cso_gridded.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_gridded +.. automodule:: cso.cso_gridded diff --git a/doc/source/pymod-cso_inquire.rst b/doc/source/pymod-cso_inquire.rst index eeb04ca..bd87867 100644 --- a/doc/source/pymod-cso_inquire.rst +++ b/doc/source/pymod-cso_inquire.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_inquire +.. automodule:: cso.cso_inquire diff --git a/doc/source/pymod-cso_mapping.rst b/doc/source/pymod-cso_mapping.rst index d04a5dd..d87d83b 100644 --- a/doc/source/pymod-cso_mapping.rst +++ b/doc/source/pymod-cso_mapping.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_mapping +.. automodule:: cso.cso_mapping diff --git a/doc/source/pymod-cso_pal.rst b/doc/source/pymod-cso_pal.rst index a90d777..d3e0d9b 100644 --- a/doc/source/pymod-cso_pal.rst +++ b/doc/source/pymod-cso_pal.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_pal +.. automodule:: cso.cso_pal diff --git a/doc/source/pymod-cso_plot.rst b/doc/source/pymod-cso_plot.rst index 33ef1a1..6ddefa9 100644 --- a/doc/source/pymod-cso_plot.rst +++ b/doc/source/pymod-cso_plot.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_plot +.. automodule:: cso.cso_plot diff --git a/doc/source/pymod-cso_regions.rst b/doc/source/pymod-cso_regions.rst index 5e00fe6..584710c 100644 --- a/doc/source/pymod-cso_regions.rst +++ b/doc/source/pymod-cso_regions.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_regions +.. automodule:: cso.cso_regions diff --git a/doc/source/pymod-cso_s5p.rst b/doc/source/pymod-cso_s5p.rst index 93fae98..e188431 100644 --- a/doc/source/pymod-cso_s5p.rst +++ b/doc/source/pymod-cso_s5p.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_s5p +.. automodule:: cso.cso_s5p diff --git a/doc/source/pymod-cso_superobs.rst b/doc/source/pymod-cso_superobs.rst index c35e32a..b892a33 100644 --- a/doc/source/pymod-cso_superobs.rst +++ b/doc/source/pymod-cso_superobs.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_superobs +.. automodule:: cso.cso_superobs diff --git a/doc/source/pymod-cso_tools.rst b/doc/source/pymod-cso_tools.rst index 6f9e3f7..5e18b25 100644 --- a/doc/source/pymod-cso_tools.rst +++ b/doc/source/pymod-cso_tools.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_tools +.. automodule:: cso.cso_tools diff --git a/doc/source/pymod-utopya_base.rst b/doc/source/pymod-utopya_base.rst index 48f8102..45b79ac 100644 --- a/doc/source/pymod-utopya_base.rst +++ b/doc/source/pymod-utopya_base.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_base +.. automodule:: utopya.utopya_base diff --git a/doc/source/pymod-utopya_build.rst b/doc/source/pymod-utopya_build.rst index adc79cb..58bb84f 100644 --- a/doc/source/pymod-utopya_build.rst +++ b/doc/source/pymod-utopya_build.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_build +.. automodule:: utopya.utopya_build diff --git a/doc/source/pymod-utopya_index.rst b/doc/source/pymod-utopya_index.rst index 16a89cc..6125c64 100644 --- a/doc/source/pymod-utopya_index.rst +++ b/doc/source/pymod-utopya_index.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_index +.. automodule:: utopya.utopya_index diff --git a/doc/source/pymod-utopya_jobscript.rst b/doc/source/pymod-utopya_jobscript.rst index 63db5a2..192457f 100644 --- a/doc/source/pymod-utopya_jobscript.rst +++ b/doc/source/pymod-utopya_jobscript.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_jobscript +.. automodule:: utopya.utopya_jobscript diff --git a/doc/source/pymod-utopya_jobtree.rst b/doc/source/pymod-utopya_jobtree.rst index 27c6d6a..74e18e4 100644 --- a/doc/source/pymod-utopya_jobtree.rst +++ b/doc/source/pymod-utopya_jobtree.rst @@ -1,4 +1,4 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_jobtree \ No newline at end of file +.. automodule:: utopya.utopya_jobtree diff --git a/doc/source/pymod-utopya_post.rst b/doc/source/pymod-utopya_post.rst index 788c7c7..27ead7a 100644 --- a/doc/source/pymod-utopya_post.rst +++ b/doc/source/pymod-utopya_post.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_post +.. automodule:: utopya.utopya_post diff --git a/doc/source/pymod-utopya_rc.rst b/doc/source/pymod-utopya_rc.rst index df67c5f..4fcf23f 100644 --- a/doc/source/pymod-utopya_rc.rst +++ b/doc/source/pymod-utopya_rc.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_rc +.. automodule:: utopya.utopya_rc diff --git a/doc/source/pymod-utopya_runscript.rst b/doc/source/pymod-utopya_runscript.rst index 1b84299..e6d5943 100644 --- a/doc/source/pymod-utopya_runscript.rst +++ b/doc/source/pymod-utopya_runscript.rst @@ -1,4 +1,4 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_runscript +.. automodule:: utopya.utopya_runscript diff --git a/doc/source/pymod-utopya_tools.rst b/doc/source/pymod-utopya_tools.rst index ac26014..7993d31 100644 --- a/doc/source/pymod-utopya_tools.rst +++ b/doc/source/pymod-utopya_tools.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_tools +.. automodule:: utopya.utopya_tools diff --git a/doc/source/pymods.rst b/doc/source/pymods.rst index 216503e..d738385 100644 --- a/doc/source/pymods.rst +++ b/doc/source/pymods.rst @@ -8,24 +8,25 @@ Python modules Overview of the Python module(s) and classes. -CSO modules -=========== +* :py:mod:`cso` -The ``cso`` module provides the classes that do the work in the CSO pre- and post- processing. + Top level module that provides the classes that do the work in the CSO pre- and post- processing. -.. The following are names of '.rst' files in the 'doc/sources' directory ; - this ensures that each sub-module is given a seperate page in the documentation. +* :py:mod:`utopya` -.. toctree:: - :maxdepth: 1 + Toolbox to perform general tasks as copying source codes to a work directory, + creation of job files, generation of index pages, etc. - pymod-cso +* :py:mod:`rc` + + Base module to process configuration files. -Classes used for specific tasks are implemented in the ``cso_*`` modules. +.. Hidden table-of-contents to ensure that documents are included in a toctree. .. toctree:: - :maxdepth: 1 + :hidden: + pymod-cso pymod-cso_inquire pymod-cso_dataspace pymod-cso_pal @@ -42,21 +43,16 @@ Classes used for specific tasks are implemented in the ``cso_*`` modules. pymod-cso_plot pymod-cso_constants pymod-cso_tools - - - -Tools -===== - -The following modules provide generic classes used for building the application. - -.. The following are names of '.rst' files in the 'doc/sources' directory ; - this ensures that each sub-module is given a seperate page in the documentation. - -.. toctree:: - :maxdepth: 1 - pymod-utopya + pymod-utopya_base + pymod-utopya_rc + pymod-utopya_jobscript + pymod-utopya_jobtree + pymod-utopya_runscript + pymod-utopya_tools + pymod-utopya_build + pymod-utopya_post + pymod-utopya_index pymod-rc diff --git a/pyproject.toml b/pyproject.toml index b0e7ae6..8778da3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "scipy >=1.1.0", "requests", "typer", + "sphinx", ] [project.urls] diff --git a/src/cso/__init__.py b/src/cso/__init__.py index 85a1b92..020d635 100644 --- a/src/cso/__init__.py +++ b/src/cso/__init__.py @@ -1,15 +1,124 @@ -from importlib import metadata +# +# Changes +# +# 2022-09, Arjo Segers +# Added cso_superob module. +# Added cso_colocate module. +# +# 2023, Lewis Blake, Arjo Segers +# Formatted using "black". +# +# 2025-01, Arjo Segers +# Added support for ColHub mirror. +# + + +# ------------------------------------------------- +# help +# ------------------------------------------------- + +""" + +.. Label, use :ref:`text