diff --git a/CHANGELOG b/CHANGELOG index e383a41e48c4e61ca9b9a3b71eecf2b9539f3191..5a0154cc5854851d605781f5d146a1dcd1a299eb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -528,3 +528,8 @@ Optionally search in multiple mirror archives for missing files in ColHub mirror py/cso_colhub.py +v2.12 +----- + +Support running CSO within a virtual environment. + diff --git a/README.md b/README.md index 0b75cba541c8f49a6b49dac3b19072ff78761ba8..a22bda9c240ecece4285e3377818fb8cc0647be5 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ A documentation is available online via the *Gitlab Pages*: It is updated automatically when changes are pushed. +For first introduction, see in particular the [Tutorial](https://cams.ci.tno.nl/cso/tutorial.html) chapter. + Eventually create a local version of the documentation from its source files using: make docu @@ -37,40 +39,56 @@ Note that the documenation requires 'sphinx', which is usually part of your python distribution. -Installation from source ------------------------- +Running CSO within your own Python environment +---------------------------------------------- + +The CSO tools could be started within your own Python environment. +You should then ensure that the required (versions of) packages have been installed correctly. + +To test CSO within your environment, try for example run the example from the Tutorial: + + ./bin/cso config/tutorial/tutorial.rc -CSO can be installed from source in either conda or virtual environments. -**Pip** -The file py_project.toml contains the dependencies needed by CSO. + +Running CSO within a virtual environment +---------------------------------------- + +A *virtual environment* could be created that has all required (versions) of Python packages installed. +For this, the `py_project.toml` file is included which contains the dependencies needed by CSO. + +To create a new virtual enviroment run: python3 -m venv --prompt cso .venv -This will create a new Python evironment folder called `cso`. To activate it run +This will create a new Python evironment folder called `.venv`. +To activate it run: source .venv/bin/activate -You may need to updated pip with the command +The terminal prompt will then be preceded by `(cso)` which holds the name specified above +with the `--prompt` argument. + +You may need to upgrade `pip` with the command: - pip install -U pip + pip install --upgrade pip -We can then install CSO and its dependencies simply using pip +We can then install CSO and its dependencies into the virtual environment using: - pip install -e . + pip install --editable . -The `-e` flag installs CSO in editable mode, meaning that the user can change the source code and run CSO anywhere using these changes. +The `--editable` flag installs CSO in editable mode, meaning that the user can change the source code +in the `src/` directory and run CSO anywhere using these changes. +Test the installation using the Tutorial settings: -Tutorial --------- + ./bin/cso config/tutorial/tutorial.rc -A tutorial is included in the documentation. +When finished, leave from the virtual environment using: -For the impatient: + deactivate - ./bin/cso rc/tutorial.rc diff --git a/bin/cso b/bin/cso index 867cb971efe99c79a10bfd06dc9e1e04e9f5e07d..151e42de03893c2e8c3093b3f83cbf262240494d 100755 --- a/bin/cso +++ b/bin/cso @@ -36,7 +36,9 @@ import logging # prefix of CSO installation: prefix = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), os.pardir)) # extend path: -sys.path.insert(0, os.path.join(prefix, "py")) +sys.path.insert(0, os.path.join(prefix, "src")) +sys.path.insert(0, os.path.join(prefix, "src", "utopya")) +sys.path.insert(0, os.path.join(prefix, "src", "cso")) # tools: import utopya diff --git a/config/tutorial/tutorial.rc b/config/tutorial/tutorial.rc index e324e2242abc1246fcac97eda913723a6abe9584..8af4063d05a89473931e3242389bce5f5f1ead92 100644 --- a/config/tutorial/tutorial.rc +++ b/config/tutorial/tutorial.rc @@ -270,9 +270,17 @@ my.work : /Scratch/${USER}/CSO-Tutorial ! run jobs in foreground: *.script.class : utopya.UtopyaJobScriptForeground +! dummy value, will be defined when running in virtrual environment: +VIRTUAL_ENV : +! running in virutual envionment? +#if "${VIRTUAL_ENV}" > "" +! interpretor from virtual enviornment, the system path is set automatically: +*.shell : ${VIRTUAL_ENV}/bin/python3 +#else ! search path for python modules: -*.pypath : ${CSO_PREFIX}/py - +*.pypath : ${CSO_PREFIX}/src:${CSO_PREFIX}/src/utopya:${CSO_PREFIX}/src/cso +#endif + ! work directory for jobs; *.workdir : ${my.work}/__NAME2PATH__ @@ -292,7 +300,10 @@ my.work : /Scratch/${USER}/CSO-Tutorial ! full time range: cso.tutorial.inquire-table-dataspace.timerange.start : 2018-01-01 00:00:00 -cso.tutorial.inquire-table-dataspace.timerange.end : 2024-01-01 00:00:00 +cso.tutorial.inquire-table-dataspace.timerange.end : 2025-01-01 00:00:00 +!! ... testing ... +!cso.tutorial.inquire-table-dataspace.timerange.start : 2018-06-01 00:00:00 +!cso.tutorial.inquire-table-dataspace.timerange.end : 2018-07-01 00:00:00 ! API url: cso.tutorial.inquire-table-dataspace.url : https://catalogue.dataspace.copernicus.eu/resto/api diff --git a/doc/source/conf.py b/doc/source/conf.py index 11c49ab4683be0e99053ea14825f934942af6d59..3e6cc4cfd2c05ac8a992a66e200482ebc28041b6 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -12,8 +12,10 @@ # import os import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, "py"))) +# +sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, "src"))) +sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, "src", "utopya"))) +sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, "src", "cso"))) # -- Project information ----------------------------------------------------- @@ -22,8 +24,23 @@ project = "CSO" copyright = "2020-2025, Arjo Segers" author = "Arjo Segers" -# The full version, including alpha/beta/rc tags -release = "v2.11" +# extract the version id from the project file: +ppfile = os.path.join(os.pardir, os.pardir, "pyproject.toml") +# check .. +if not os.path.isfile(ppfile): + # info ... + print(f"WARNING - could not find '{ppfile}' to extract release tag ...") + # dummy: + release = "vx.y" +else: + # modules: + import toml + # open file: + with open(ppfile, "r") as f: + config = toml.load(f) + # extrract: + release = config["project"]["version"] +#endif # -- General configuration --------------------------------------------------- diff --git a/doc/source/history.rst b/doc/source/history.rst index 7daf3385946d100ddabe59e8b330274e3131eef3..990ef51c92bc3dff217664bdb323005e4a7702db 100644 --- a/doc/source/history.rst +++ b/doc/source/history.rst @@ -115,6 +115,9 @@ A summary of the versions and changes. * | *v2.11* | Support inquiry and download from NASA EarthData archive. + +* | *v2.12* + | Support running in Python virtual environment. To be included diff --git a/doc/source/pymod-cso_catalogue.rst b/doc/source/pymod-cso_catalogue.rst index 5aa1fa60a1c2b4851c5fbf1deefd18c177853c10..495cb622b88abc8aa75ab4e7141693c551afdb78 100644 --- a/doc/source/pymod-cso_catalogue.rst +++ b/doc/source/pymod-cso_catalogue.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_catalogue +.. automodule:: cso.cso_catalogue diff --git a/doc/source/pymod-cso_colhub.rst b/doc/source/pymod-cso_colhub.rst index ab95fb93f08e1a9a6efd8590d393b52adb979171..19c7247557ffbbc37ac7ed18ed1ecae214cd9764 100644 --- a/doc/source/pymod-cso_colhub.rst +++ b/doc/source/pymod-cso_colhub.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_colhub +.. automodule:: cso.cso_colhub diff --git a/doc/source/pymod-cso_colocate.rst b/doc/source/pymod-cso_colocate.rst index 162b7791ed3f76440cd3ce1d3a28325e7d650fec..63018c5147d2fbdf2f7a6b90903c9376b2613c10 100644 --- a/doc/source/pymod-cso_colocate.rst +++ b/doc/source/pymod-cso_colocate.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_colocate +.. automodule:: cso.cso_colocate diff --git a/doc/source/pymod-cso_constants.rst b/doc/source/pymod-cso_constants.rst index 0f5b7d2668866aed82968e4eee92e411d5c5abb1..0a024a224d21de16656555c41fdf1c2d4480b4b7 100644 --- a/doc/source/pymod-cso_constants.rst +++ b/doc/source/pymod-cso_constants.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_constants +.. automodule:: cso.cso_constants diff --git a/doc/source/pymod-cso_dataspace.rst b/doc/source/pymod-cso_dataspace.rst index bb01d7060de5790041cdf38634c18b35ea6f4b0a..199438f04383a9a13cee513a1f3e24a8dea47646 100644 --- a/doc/source/pymod-cso_dataspace.rst +++ b/doc/source/pymod-cso_dataspace.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_dataspace +.. automodule:: cso.cso_dataspace diff --git a/doc/source/pymod-cso_earthaccess.rst b/doc/source/pymod-cso_earthaccess.rst index f8695ad90bf92c35a1af2db82940e4879c85ef67..50c48c9c0defb55710d54973ccf12dab82bef862 100644 --- a/doc/source/pymod-cso_earthaccess.rst +++ b/doc/source/pymod-cso_earthaccess.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_earthaccess +.. automodule:: cso.cso_earthaccess diff --git a/doc/source/pymod-cso_file.rst b/doc/source/pymod-cso_file.rst index a5c6d3539fff044f125eeecd85b8751561c0a607..a57b404dd35aefd96667aa828c1a4657bf844633 100644 --- a/doc/source/pymod-cso_file.rst +++ b/doc/source/pymod-cso_file.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_file +.. automodule:: cso.cso_file diff --git a/doc/source/pymod-cso_gridded.rst b/doc/source/pymod-cso_gridded.rst index 5b9e14085d12c847a7829be8860733adfb658be4..cf636b4aab934e0134258307ad4fe0c9cf38f9ee 100644 --- a/doc/source/pymod-cso_gridded.rst +++ b/doc/source/pymod-cso_gridded.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_gridded +.. automodule:: cso.cso_gridded diff --git a/doc/source/pymod-cso_inquire.rst b/doc/source/pymod-cso_inquire.rst index eeb04cad9fd34589a7401aa8345a3302ca3e7842..bd87867109e8ba275ccef57dded88e8ce3b5b021 100644 --- a/doc/source/pymod-cso_inquire.rst +++ b/doc/source/pymod-cso_inquire.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_inquire +.. automodule:: cso.cso_inquire diff --git a/doc/source/pymod-cso_mapping.rst b/doc/source/pymod-cso_mapping.rst index d04a5dd72231b1979ac2c6e1439ae6ac4b7f4c60..d87d83bea0de9cfb39fc9f42617b8d99d2591fe4 100644 --- a/doc/source/pymod-cso_mapping.rst +++ b/doc/source/pymod-cso_mapping.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_mapping +.. automodule:: cso.cso_mapping diff --git a/doc/source/pymod-cso_pal.rst b/doc/source/pymod-cso_pal.rst index a90d777c2d68f4a718a4e19c318b9309a3fabac5..d3e0d9bd0e45b13b78b5f90ccea96398bbebaa66 100644 --- a/doc/source/pymod-cso_pal.rst +++ b/doc/source/pymod-cso_pal.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_pal +.. automodule:: cso.cso_pal diff --git a/doc/source/pymod-cso_plot.rst b/doc/source/pymod-cso_plot.rst index 33ef1a1cc2885935f17adf5b9976fa52acc1c911..6ddefa9a9b919851f25d81932611503c5de92c14 100644 --- a/doc/source/pymod-cso_plot.rst +++ b/doc/source/pymod-cso_plot.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_plot +.. automodule:: cso.cso_plot diff --git a/doc/source/pymod-cso_regions.rst b/doc/source/pymod-cso_regions.rst index 5e00fe6374fa95a9dd676bbafa06303bc395b34d..584710c112338646b729dcfdd898deb94cf3a454 100644 --- a/doc/source/pymod-cso_regions.rst +++ b/doc/source/pymod-cso_regions.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_regions +.. automodule:: cso.cso_regions diff --git a/doc/source/pymod-cso_s5p.rst b/doc/source/pymod-cso_s5p.rst index 93fae98883b1e387cb784cc7c495ca78214778c2..e188431d6d22d326b1bb6568cb55f49bac408401 100644 --- a/doc/source/pymod-cso_s5p.rst +++ b/doc/source/pymod-cso_s5p.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_s5p +.. automodule:: cso.cso_s5p diff --git a/doc/source/pymod-cso_superobs.rst b/doc/source/pymod-cso_superobs.rst index c35e32ad01196f1bbed94e49568557f483f469b4..b892a33844379b0c7cf20b9ee88f597e4c6055c1 100644 --- a/doc/source/pymod-cso_superobs.rst +++ b/doc/source/pymod-cso_superobs.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_superobs +.. automodule:: cso.cso_superobs diff --git a/doc/source/pymod-cso_tools.rst b/doc/source/pymod-cso_tools.rst index 6f9e3f7d2aea614f740b5cd4051f5a6b0f6fd8c8..5e18b258b059a417c9b7f22f46ba337b0e09e729 100644 --- a/doc/source/pymod-cso_tools.rst +++ b/doc/source/pymod-cso_tools.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: cso_tools +.. automodule:: cso.cso_tools diff --git a/doc/source/pymod-utopya_base.rst b/doc/source/pymod-utopya_base.rst index 48f81028ad803d3fdaf32c9248317b8dd5c97a00..45b79ac4d827412657923f6cb0fa9cc7554952a1 100644 --- a/doc/source/pymod-utopya_base.rst +++ b/doc/source/pymod-utopya_base.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_base +.. automodule:: utopya.utopya_base diff --git a/doc/source/pymod-utopya_build.rst b/doc/source/pymod-utopya_build.rst index adc79cb4cb6abf509cf821ae6de4372a1d39ec82..58bb84f35f354e8674854afdaf25c8726e5260b0 100644 --- a/doc/source/pymod-utopya_build.rst +++ b/doc/source/pymod-utopya_build.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_build +.. automodule:: utopya.utopya_build diff --git a/doc/source/pymod-utopya_index.rst b/doc/source/pymod-utopya_index.rst index 16a89cccaeb33cff8b0f0810c0e1f3b1ae0b1002..6125c64919f1eb2cea535d3ba7fc1c8ee6b125c3 100644 --- a/doc/source/pymod-utopya_index.rst +++ b/doc/source/pymod-utopya_index.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_index +.. automodule:: utopya.utopya_index diff --git a/doc/source/pymod-utopya_jobscript.rst b/doc/source/pymod-utopya_jobscript.rst index 63db5a224a889b541f081c16e1f7aaca8323a94c..192457fffecabd1789447f5a36e2626b896cfd41 100644 --- a/doc/source/pymod-utopya_jobscript.rst +++ b/doc/source/pymod-utopya_jobscript.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_jobscript +.. automodule:: utopya.utopya_jobscript diff --git a/doc/source/pymod-utopya_jobtree.rst b/doc/source/pymod-utopya_jobtree.rst index 27c6d6a528858dddf1387dc1c4dedea4fa687c25..74e18e44960b44d0b4769fa705a23524a2c0099a 100644 --- a/doc/source/pymod-utopya_jobtree.rst +++ b/doc/source/pymod-utopya_jobtree.rst @@ -1,4 +1,4 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_jobtree \ No newline at end of file +.. automodule:: utopya.utopya_jobtree diff --git a/doc/source/pymod-utopya_post.rst b/doc/source/pymod-utopya_post.rst index 788c7c76b06f797dd6e7867bdee11cb6fa577700..27ead7a593dc69ceed65cb1c47ae79de80f67f83 100644 --- a/doc/source/pymod-utopya_post.rst +++ b/doc/source/pymod-utopya_post.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_post +.. automodule:: utopya.utopya_post diff --git a/doc/source/pymod-utopya_rc.rst b/doc/source/pymod-utopya_rc.rst index df67c5f07a3f113b4589d2433468b01aea7bf439..4fcf23f466827a4e5e965cd4962a13df0c48354f 100644 --- a/doc/source/pymod-utopya_rc.rst +++ b/doc/source/pymod-utopya_rc.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_rc +.. automodule:: utopya.utopya_rc diff --git a/doc/source/pymod-utopya_runscript.rst b/doc/source/pymod-utopya_runscript.rst index 1b84299b1099b05893d3443be1be817ee159d2dc..e6d59434c29abf23fe72cccfeb59864c5e470887 100644 --- a/doc/source/pymod-utopya_runscript.rst +++ b/doc/source/pymod-utopya_runscript.rst @@ -1,4 +1,4 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_runscript +.. automodule:: utopya.utopya_runscript diff --git a/doc/source/pymod-utopya_tools.rst b/doc/source/pymod-utopya_tools.rst index ac26014571079eef9de8c3cfd6d32e1e3fa0e01c..7993d314a6dc62c2656b3739d8e5ce5569337e23 100644 --- a/doc/source/pymod-utopya_tools.rst +++ b/doc/source/pymod-utopya_tools.rst @@ -1,5 +1,5 @@ .. Documentation for module. .. Import documentation from ".py" file: -.. automodule:: utopya_tools +.. automodule:: utopya.utopya_tools diff --git a/doc/source/pymods.rst b/doc/source/pymods.rst index 216503e8311a6eb4360cba630ecbd0ca3a10c6c5..d73838598816b3d4cfb218696c2032e8ddcf84e9 100644 --- a/doc/source/pymods.rst +++ b/doc/source/pymods.rst @@ -8,24 +8,25 @@ Python modules Overview of the Python module(s) and classes. -CSO modules -=========== +* :py:mod:`cso` -The ``cso`` module provides the classes that do the work in the CSO pre- and post- processing. + Top level module that provides the classes that do the work in the CSO pre- and post- processing. -.. The following are names of '.rst' files in the 'doc/sources' directory ; - this ensures that each sub-module is given a seperate page in the documentation. +* :py:mod:`utopya` -.. toctree:: - :maxdepth: 1 + Toolbox to perform general tasks as copying source codes to a work directory, + creation of job files, generation of index pages, etc. - pymod-cso +* :py:mod:`rc` + + Base module to process configuration files. -Classes used for specific tasks are implemented in the ``cso_*`` modules. +.. Hidden table-of-contents to ensure that documents are included in a toctree. .. toctree:: - :maxdepth: 1 + :hidden: + pymod-cso pymod-cso_inquire pymod-cso_dataspace pymod-cso_pal @@ -42,21 +43,16 @@ Classes used for specific tasks are implemented in the ``cso_*`` modules. pymod-cso_plot pymod-cso_constants pymod-cso_tools - - - -Tools -===== - -The following modules provide generic classes used for building the application. - -.. The following are names of '.rst' files in the 'doc/sources' directory ; - this ensures that each sub-module is given a seperate page in the documentation. - -.. toctree:: - :maxdepth: 1 - pymod-utopya + pymod-utopya_base + pymod-utopya_rc + pymod-utopya_jobscript + pymod-utopya_jobtree + pymod-utopya_runscript + pymod-utopya_tools + pymod-utopya_build + pymod-utopya_post + pymod-utopya_index pymod-rc diff --git a/pyproject.toml b/pyproject.toml index b0e7ae6d445196c72585a30132d658f2e38a1836..a67b47de8d2478264ade0761b1f89ea14d1fa3de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "cso" -version = "0.1.0" -authors = [{ name = "Arjo Segers" }] +version = "2.12" +authors = [ + { name = "Arjo Segers" }, + { name = "Lewis Blake" }, +] description = "CAMS Satellite Operator (CSO) tool." readme = "README.md" classifiers = [ @@ -41,7 +44,10 @@ dependencies = [ Home = "https://ci.tno.nl/gitlab/cams/cso" [project.optional-dependencies] -docs = ["pre-commit"] +docs = [ + "sphinx", + "toml", +] [project.scripts] cso = "cso.scripts.cli:main" diff --git a/src/cso/__init__.py b/src/cso/__init__.py index 85a1b92969568095e9eae48104131927260e32cc..020d635cabd5bc099117166b09361e6c0283c4df 100644 --- a/src/cso/__init__.py +++ b/src/cso/__init__.py @@ -1,15 +1,124 @@ -from importlib import metadata +# +# Changes +# +# 2022-09, Arjo Segers +# Added cso_superob module. +# Added cso_colocate module. +# +# 2023, Lewis Blake, Arjo Segers +# Formatted using "black". +# +# 2025-01, Arjo Segers +# Added support for ColHub mirror. +# + + +# ------------------------------------------------- +# help +# ------------------------------------------------- + +""" + +.. Label, use :ref:`text