From 1e2a12ab32d5e1c695b17d02efaaa55db94dd834 Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Fri, 11 Oct 2024 08:18:32 +0000 Subject: [PATCH 01/17] gitignore cso-user-settings --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 3132557..9331141 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ ENV/ env.bak/ venv.bak/ .vscode + +config/Copernicus/cso-user-settings.rc -- GitLab From faa6413785bf07f3ad335d827a084b633932d41f Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Fri, 11 Oct 2024 08:19:15 +0000 Subject: [PATCH 02/17] gitignore user settings --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9331141..9e34872 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ env.bak/ venv.bak/ .vscode + config/Copernicus/cso-user-settings.rc -- GitLab From ea0259943d6134de16b1e417eccb228003ac20f2 Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Fri, 11 Oct 2024 08:19:58 +0000 Subject: [PATCH 03/17] user setttings ignore --- config/Copernicus/cso-user-settings.rc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/Copernicus/cso-user-settings.rc b/config/Copernicus/cso-user-settings.rc index b07f313..ccf052a 100644 --- a/config/Copernicus/cso-user-settings.rc +++ b/config/Copernicus/cso-user-settings.rc @@ -93,19 +93,19 @@ my.timerange.end : 2018-06-01 23:59 !---------------------------------------------------------- !.............................. -#if "${USER}" == "you" +#if "${USER}" == "lewisb" !.............................. ! Attributes written to output files. -my.attr.author : Your Name -my.attr.institution : CSO -my.attr.email : Your.Name@cso.org +my.attr.author : Lewis Blake +my.attr.institution : MET Norway +my.attr.email : lewisb@met.no ! base location for work directories: -my.work : /Scratch/${USER}/CSO-Copernicus +my.work : /lustre/storeB/project/fou/kl/sesam/work/users/${USER}/release-v24r2 ! storage for downloaded observations: -my.observations : /Scratch/${USER}/observations +my.observations : /lustre/storeB/project/fou/kl/sesam/archive/v24r2 !.............................. #elif "${USER}" == "arjos" -- GitLab From c0d71f5b30d3814716a41781cca8f56e92b17f67 Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Thu, 17 Oct 2024 07:51:56 +0000 Subject: [PATCH 04/17] pyproject.toml and README.md --- README.md | 26 +++++++++++++++++++++++ pyproject.toml | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 pyproject.toml diff --git a/README.md b/README.md index 1be27c3..ea670b2 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,32 @@ Then use the following command to clone the repository: git clone git@ci.tno.nl:cams/cso.git +Installation from source +------------------------ + +CSO can be installed from source in either conda or virtual environments. + +**Pip** + +The file py_project.toml contains the dependencies needed by CSO. + + python3 -m venv --prompt cso .venv + +This will create a new Python evironment folder called `cso`. To activate it run + + source .venv/bin/activate + +You may need to updated pip with the command + + pip install -U pip + +We can then install CSO and its dependencies simply using pip + + pip install -e . + +The `-e` flag installs CSO in editable mode, meaning that the user can change the source code and run CSO anywhere using these changes. + + Tutorial -------- diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e1de9d6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = [ + "setuptools >= 40.9.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "cso" +version = "0.1.0" +authors = [{name = "Arjo Segers"}] +description = "CAMS Satellite Operator (CSO) tool." +readme = "README.md" +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: MIT", + "Operating System :: OS Independent", + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "Intended Audience :: Education", + "Topic :: Scientific/Engineering :: Atmospheric Science", +] +requires-python = ">=3.10" + +dependencies = [ + "cartopy>=0.20", + "shapely", + "numpy", + "matplotlib", + "numba", + "pandas", + "tqdm", + "xarray", + "netcdf4", + "scipy >=1.1.0", + "requests" +] + +[project.urls] +Home = "https://ci.tno.nl/gitlab/cams/cso" + +[project.optional-dependencies] +docs = [ + "pre-commit", + "sphinx>=4.2.0", + "sphinxcontrib-napoleon", + "sphinx_rtd_theme", + "sphinx-argparse", + "nbsphinx", +] + +[tool.black] +target-version = ['py310'] +extend_skip = ["doc"] +line-length = 99 \ No newline at end of file -- GitLab From 272b6409c30c5a4467aba95554df1250cc334b8e Mon Sep 17 00:00:00 2001 From: Lewis Blake Date: Thu, 17 Oct 2024 07:52:11 +0000 Subject: [PATCH 05/17] src and change imports --- src/cso/__init__.py | 14 ++++++++++++++ {py => src/cso}/cso.py | 0 {py => src/cso}/cso_catalogue.py | 14 +++++++------- {py => src/cso}/cso_colocate.py | 6 +++--- {py => src/cso}/cso_constants.py | 0 {py => src/cso}/cso_dataspace.py | 2 +- {py => src/cso}/cso_file.py | 0 {py => src/cso}/cso_gridded.py | 6 +++--- {py => src/cso}/cso_inquire.py | 0 {py => src/cso}/cso_mapping.py | 0 {py => src/cso}/cso_pal.py | 2 +- {py => src/cso}/cso_plot.py | 0 {py => src/cso}/cso_regions.py | 14 +++++++------- {py => src/cso}/cso_s5p.py | 18 +++++++++--------- {py => src/cso}/cso_superobs.py | 4 ++-- {py => src/cso}/cso_tools.py | 0 {py => src}/rc.py | 0 src/utopya/__init__.py | 10 ++++++++++ {py => src/utopya}/utopya.py | 0 {py => src/utopya}/utopya_base.py | 0 {py => src/utopya}/utopya_build.py | 0 {py => src/utopya}/utopya_index.py | 0 {py => src/utopya}/utopya_jobscript.py | 0 {py => src/utopya}/utopya_jobtree.py | 0 {py => src/utopya}/utopya_post.py | 0 {py => src/utopya}/utopya_rc.py | 0 {py => src/utopya}/utopya_runscript.py | 0 {py => src/utopya}/utopya_tools.py | 0 28 files changed, 57 insertions(+), 33 deletions(-) create mode 100644 src/cso/__init__.py rename {py => src/cso}/cso.py (100%) rename {py => src/cso}/cso_catalogue.py (99%) rename {py => src/cso}/cso_colocate.py (99%) rename {py => src/cso}/cso_constants.py (100%) rename {py => src/cso}/cso_dataspace.py (99%) rename {py => src/cso}/cso_file.py (100%) rename {py => src/cso}/cso_gridded.py (99%) rename {py => src/cso}/cso_inquire.py (100%) rename {py => src/cso}/cso_mapping.py (100%) rename {py => src/cso}/cso_pal.py (99%) rename {py => src/cso}/cso_plot.py (100%) rename {py => src/cso}/cso_regions.py (99%) rename {py => src/cso}/cso_s5p.py (99%) rename {py => src/cso}/cso_superobs.py (99%) rename {py => src/cso}/cso_tools.py (100%) rename {py => src}/rc.py (100%) create mode 100644 src/utopya/__init__.py rename {py => src/utopya}/utopya.py (100%) rename {py => src/utopya}/utopya_base.py (100%) rename {py => src/utopya}/utopya_build.py (100%) rename {py => src/utopya}/utopya_index.py (100%) rename {py => src/utopya}/utopya_jobscript.py (100%) rename {py => src/utopya}/utopya_jobtree.py (100%) rename {py => src/utopya}/utopya_post.py (100%) rename {py => src/utopya}/utopya_rc.py (100%) rename {py => src/utopya}/utopya_runscript.py (100%) rename {py => src/utopya}/utopya_tools.py (100%) diff --git a/src/cso/__init__.py b/src/cso/__init__.py new file mode 100644 index 0000000..14f5a77 --- /dev/null +++ b/src/cso/__init__.py @@ -0,0 +1,14 @@ +from .cso_file import * +from .cso_catalogue import * +from .cso_inquire import * +from .cso_scihub import * +from .cso_pal import * +from .cso_s5p import * +from .cso_s5p_superobs import * +from .cso_gridded import * +from .cso_colocate import * +from .cso_superobs import * +from .cso_regions import * +from .cso_constants import * +from .cso_mapping import * +from .cso_plot import * \ No newline at end of file diff --git a/py/cso.py b/src/cso/cso.py similarity index 100% rename from py/cso.py rename to src/cso/cso.py diff --git a/py/cso_catalogue.py b/src/cso/cso_catalogue.py similarity index 99% rename from py/cso_catalogue.py rename to src/cso/cso_catalogue.py index 4498b56..aafd8e5 100644 --- a/py/cso_catalogue.py +++ b/src/cso/cso_catalogue.py @@ -110,7 +110,7 @@ class CSO_CatalogueBase(utopya.UtopyaRc): import numpy # tools: - import cso_constants + from . import cso_constants # no target units defined? if (vunits is None) or (vunits == "None"): @@ -274,8 +274,8 @@ class CSO_Catalogue(CSO_CatalogueBase): import matplotlib.pyplot as plt # tools: - import cso_file - import cso_plot + from . import cso_file + from . import cso_plot # setup graphics back-end: matplotlib.use("Agg") @@ -656,8 +656,8 @@ class CSO_SimCatalogue(CSO_CatalogueBase): import matplotlib.pyplot as plt # tools: - import cso_file - import cso_plot + from . import cso_file + from . import cso_plot # setup graphics back-end: matplotlib.use("Agg") @@ -1004,8 +1004,8 @@ class CSO_GriddedCatalogue(CSO_CatalogueBase): import matplotlib.pyplot as plt # tools: - import cso_file - import cso_plot + from . import cso_file + from . import cso_plot # setup graphics back-end: matplotlib.use("Agg") diff --git a/py/cso_colocate.py b/src/cso/cso_colocate.py similarity index 99% rename from py/cso_colocate.py rename to src/cso/cso_colocate.py index 1e80961..129fe8e 100644 --- a/py/cso_colocate.py +++ b/src/cso/cso_colocate.py @@ -56,7 +56,7 @@ import logging # tools: import utopya -import cso_catalogue +from . import cso_catalogue ######################################################################## @@ -191,8 +191,8 @@ class CSO_CoLocate(utopya.UtopyaRc): import xarray # tools: - import cso_file - import cso_mapping + from . import cso_file + from . import cso_mapping # init base object: utopya.UtopyaRc.__init__(self, rcfile, rcbase=rcbase, env=env) diff --git a/py/cso_constants.py b/src/cso/cso_constants.py similarity index 100% rename from py/cso_constants.py rename to src/cso/cso_constants.py diff --git a/py/cso_dataspace.py b/src/cso/cso_dataspace.py similarity index 99% rename from py/cso_dataspace.py rename to src/cso/cso_dataspace.py index 6a87347..fe98e09 100644 --- a/py/cso_dataspace.py +++ b/src/cso/cso_dataspace.py @@ -718,7 +718,7 @@ class CSO_DataSpace_Downloader(object): import shutil # tools: - import cso_file + from . import cso_file # no token yet? if self.access_token is None: diff --git a/py/cso_file.py b/src/cso/cso_file.py similarity index 100% rename from py/cso_file.py rename to src/cso/cso_file.py diff --git a/py/cso_gridded.py b/src/cso/cso_gridded.py similarity index 99% rename from py/cso_gridded.py rename to src/cso/cso_gridded.py index b5f0182..ca7b4c9 100644 --- a/py/cso_gridded.py +++ b/src/cso/cso_gridded.py @@ -267,8 +267,8 @@ class CSO_GriddedAverage(utopya.UtopyaRc): import pandas # tools: - import cso_file - import cso_mapping + from . import cso_file + from . import cso_mapping # init base object: utopya.UtopyaRc.__init__(self, rcfile, rcbase=rcbase, env=env) @@ -1210,7 +1210,7 @@ class CSO_GriddedAverageMeans(utopya.UtopyaRc): import pandas # tools: - import cso_file + from . import cso_file # init base object: utopya.UtopyaRc.__init__(self, rcfile, rcbase=rcbase, env=env) diff --git a/py/cso_inquire.py b/src/cso/cso_inquire.py similarity index 100% rename from py/cso_inquire.py rename to src/cso/cso_inquire.py diff --git a/py/cso_mapping.py b/src/cso/cso_mapping.py similarity index 100% rename from py/cso_mapping.py rename to src/cso/cso_mapping.py diff --git a/py/cso_pal.py b/src/cso/cso_pal.py similarity index 99% rename from py/cso_pal.py rename to src/cso/cso_pal.py index af0777c..a0d0644 100644 --- a/py/cso_pal.py +++ b/src/cso/cso_pal.py @@ -363,7 +363,7 @@ class CSO_PAL_Downloader(object): import requests # tools: - import cso_file + from . import cso_file # retry loop .. ntry = 1 diff --git a/py/cso_plot.py b/src/cso/cso_plot.py similarity index 100% rename from py/cso_plot.py rename to src/cso/cso_plot.py diff --git a/py/cso_regions.py b/src/cso/cso_regions.py similarity index 99% rename from py/cso_regions.py rename to src/cso/cso_regions.py index d8db155..1424620 100644 --- a/py/cso_regions.py +++ b/src/cso/cso_regions.py @@ -74,7 +74,7 @@ import logging # tools: import utopya -import cso_catalogue +from . import cso_catalogue ######################################################################## @@ -1703,9 +1703,9 @@ class CSO_Catalogue_RegionsMaps(cso_catalogue.CSO_CatalogueBase): import matplotlib.pyplot as plt # tools: - import cso_file - import cso_plot - import cso_regions + from . import cso_file + from . import cso_plot + from . import cso_regions # setup graphics back-end: matplotlib.use("Agg") @@ -2063,9 +2063,9 @@ class CSO_Catalogue_RegionsTimeSeries(cso_catalogue.CSO_CatalogueBase): import xarray # tools: - import cso_file - import cso_plot - import cso_regions + from . import cso_file + from . import cso_plot + from . import cso_regions # setup graphics back-end: matplotlib.use("Agg") diff --git a/py/cso_s5p.py b/src/cso/cso_s5p.py similarity index 99% rename from py/cso_s5p.py rename to src/cso/cso_s5p.py index 9d678e7..5e0f17b 100644 --- a/py/cso_s5p.py +++ b/src/cso/cso_s5p.py @@ -110,7 +110,7 @@ import logging # tools: import utopya -import cso_file +from . import cso_file ######################################################################## @@ -2670,10 +2670,10 @@ class CSO_S5p_Convert(utopya.UtopyaRc): import pandas # tools: - import cso_file - import cso_dataspace - import cso_pal - import utopya + from . import cso_file + from . import cso_dataspace + from . import cso_pal + from . import utopya # info ... logging.info(f"{indent}") @@ -3240,8 +3240,8 @@ class CSO_S5p_Download(utopya.UtopyaRc): import pandas # tools: - import cso_dataspace - import cso_pal + from . import cso_dataspace + from . import cso_pal # info ... logging.info(f"{indent}") @@ -3563,7 +3563,7 @@ class CSO_S5p_Listing(utopya.UtopyaRc): import glob # tools: - import cso_file + from . import cso_file # info ... logging.info(f"{indent}") @@ -3787,7 +3787,7 @@ class CSO_S5p_Download_Listing(utopya.UtopyaRc): import collections # tools: - import cso_file + from . import cso_file # info ... logging.info(f"{indent}") diff --git a/py/cso_superobs.py b/src/cso/cso_superobs.py similarity index 99% rename from py/cso_superobs.py rename to src/cso/cso_superobs.py index a465ef6..9394b3e 100644 --- a/py/cso_superobs.py +++ b/src/cso/cso_superobs.py @@ -187,8 +187,8 @@ class CSO_SuperObs(utopya.UtopyaRc): import pandas # tools: - import cso_file - import cso_mapping + from . import cso_file + from . import cso_mapping # init base object: utopya.UtopyaRc.__init__(self, rcfile, rcbase=rcbase, env=env) diff --git a/py/cso_tools.py b/src/cso/cso_tools.py similarity index 100% rename from py/cso_tools.py rename to src/cso/cso_tools.py diff --git a/py/rc.py b/src/rc.py similarity index 100% rename from py/rc.py rename to src/rc.py diff --git a/src/utopya/__init__.py b/src/utopya/__init__.py new file mode 100644 index 0000000..dced124 --- /dev/null +++ b/src/utopya/__init__.py @@ -0,0 +1,10 @@ +from .utopya_base import * +from .utopya_rc import * +from .utopya_jobscript import * +from .utopya_jobtree import * +from .utopya_runscript import * + +# from utopya_install import * +from .utopya_tools import * +from .utopya_index import * +from .utopya_post import * \ No newline at end of file diff --git a/py/utopya.py b/src/utopya/utopya.py similarity index 100% rename from py/utopya.py rename to src/utopya/utopya.py diff --git a/py/utopya_base.py b/src/utopya/utopya_base.py similarity index 100% rename from py/utopya_base.py rename to src/utopya/utopya_base.py diff --git a/py/utopya_build.py b/src/utopya/utopya_build.py similarity index 100% rename from py/utopya_build.py rename to src/utopya/utopya_build.py diff --git a/py/utopya_index.py b/src/utopya/utopya_index.py similarity index 100% rename from py/utopya_index.py rename to src/utopya/utopya_index.py diff --git a/py/utopya_jobscript.py b/src/utopya/utopya_jobscript.py similarity index 100% rename from py/utopya_jobscript.py rename to src/utopya/utopya_jobscript.py diff --git a/py/utopya_jobtree.py b/src/utopya/utopya_jobtree.py similarity index 100% rename from py/utopya_jobtree.py rename to src/utopya/utopya_jobtree.py diff --git a/py/utopya_post.py b/src/utopya/utopya_post.py similarity index 100% rename from py/utopya_post.py rename to src/utopya/utopya_post.py diff --git a/py/utopya_rc.py b/src/utopya/utopya_rc.py similarity index 100% rename from py/utopya_rc.py rename to src/utopya/utopya_rc.py diff --git a/py/utopya_runscript.py b/src/utopya/utopya_runscript.py similarity index 100% rename from py/utopya_runscript.py rename to src/utopya/utopya_runscript.py diff --git a/py/utopya_tools.py b/src/utopya/utopya_tools.py similarity index 100% rename from py/utopya_tools.py rename to src/utopya/utopya_tools.py -- GitLab From c1332f24c035a9ee31baa90ea899482f29420b78 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Thu, 17 Oct 2024 09:44:06 +0000 Subject: [PATCH 06/17] pre-commit-config.yaml --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1e75fd8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 + hooks: + - id: ruff-format + - id: ruff # linter \ No newline at end of file -- GitLab From 6f7567961335f59c7fc1193f8a9afd4c7f82dd03 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Thu, 17 Oct 2024 09:46:12 +0000 Subject: [PATCH 07/17] update .gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 9e34872..52e9cdc 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,9 @@ venv.bak/ config/Copernicus/cso-user-settings.rc +.vscode +cso.egg* +src/cso.egg* + +# doc +doc/ -- GitLab From 922387e0079109a26371cdccff885fa00ebe07d5 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Thu, 17 Oct 2024 09:58:13 +0000 Subject: [PATCH 08/17] cli --- pyproject.toml | 3 +- scripts/cli.py | 101 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 scripts/cli.py diff --git a/pyproject.toml b/pyproject.toml index e1de9d6..773488b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,8 @@ dependencies = [ "xarray", "netcdf4", "scipy >=1.1.0", - "requests" + "requests", + "typer", ] [project.urls] diff --git a/scripts/cli.py b/scripts/cli.py new file mode 100644 index 0000000..4579afe --- /dev/null +++ b/scripts/cli.py @@ -0,0 +1,101 @@ + +from typing import Optional + +import typer +import sys +import os +import logging + +from cso import __package__, __version__ +import utopya + +# prefix of CSO installation: +prefix = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), os.pardir)) +# extend path: +sys.path.insert(0, os.path.join(prefix, "src/cso")) +sys.path.insert(0, os.path.join(prefix, "src/utopya")) + +main = typer.Typer() + + +def version_callback(value: bool): + if not value: + return + + typer.echo(f" {__package__} {__version__}") + raise typer.Exit() + + +@main.callback() +def callback( + version: Optional[bool] = typer.Option(None, "--version", "-V", callback=version_callback), +): + """ CSO Command Line Interface""" + + +@main.command() +def run(rc_file: str = typer.Argument(..., help="Provide .rc file")): + """Run cso based on settings provided in .rc file (e.g., cso run /tutorial/tutorial.rc)""" + # set environment: + os.environ["CSO_PREFIX"] = prefix + # dummy values to ensure that settings could be read first to setup logging: + os.environ['CSO_RCFILE'] = 'None' + os.environ['CSO_RCDIR'] = 'None' + + # init script: + utos = utopya.UtopyaRunScriptRc() + + # setup standard command line arguments, and also enable rcfile arguments: + utos.ArgumentsSetup(description="CAMS Satellite Operator", rcbase="cso") + + # evaluate known arguments, store the other ; + # might show help text and exit: + args, xargs = utos.ArgumentsParse() + + # store full path to settings: + os.environ['CSO_RCFILE'] = os.path.abspath( args.rcfile ) + # location of settings file: + os.environ['CSO_RCDIR'] = os.path.dirname( os.path.abspath( args.rcfile ) ) + + # start, shout info: + logging.info(f"") + logging.info(f"** CSO - CAMS Satellite Operator **") + logging.info(f"") + logging.info(f"installation prefix : {prefix}") + logging.info(f"input rcfile : {args.rcfile}") + logging.info(f"settings base : {args.rcbase}") + logging.info(f"") + logging.info(f"environment:") + for key in ["CSO_PREFIX","CSO_RCFILE","CSO_RCDIR"]: + logging.info(f" {key} = {os.environ[key]}") + # endfor + logging.info(f"") + + # info ... + logging.info(f"start job tree ...") + # import class as defined in settings by 'cso.class' key; + # should be (derived from) 'utopya.JobStep' : + jbs_cls = utos.ImportClass("class") + # init job step object, use the 'rcbase' as name: + jbs = jbs_cls(args.rcbase, args.rcfile) + # start first: + jbs.Start(single=args.single) + + # info: + logging.info(f"") + logging.info(f"** end **") + logging.info(f"") + + +# ------------------------------------------------- +# end +# ------------------------------------------------- + + + + + + + +if __name__ == "__main__": + main() \ No newline at end of file -- GitLab From 289e67a83d6d180b7aebce1018db4571a5d7df63 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Thu, 17 Oct 2024 09:58:30 +0000 Subject: [PATCH 09/17] relative import for some utpoya --- src/utopya/utopya_jobscript.py | 2 +- src/utopya/utopya_rc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utopya/utopya_jobscript.py b/src/utopya/utopya_jobscript.py index dc93201..d2afe2d 100644 --- a/src/utopya/utopya_jobscript.py +++ b/src/utopya/utopya_jobscript.py @@ -147,7 +147,7 @@ Classes import logging # tools: -import utopya_base +from . import utopya_base # ====================================================================== diff --git a/src/utopya/utopya_rc.py b/src/utopya/utopya_rc.py index 697c1ba..db5eaf2 100644 --- a/src/utopya/utopya_rc.py +++ b/src/utopya/utopya_rc.py @@ -44,7 +44,7 @@ Classes import logging # tools: -from utopya_base import UtopyaBase +from .utopya_base import UtopyaBase # ------------------------------------------------- -- GitLab From 2533a6d1019433061dd16b73456eda4f32d46350 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Fri, 18 Oct 2024 06:55:17 +0000 Subject: [PATCH 10/17] from . import utopya --- src/utopya/utopya_build.py | 12 ++++++------ src/utopya/utopya_index.py | 4 ++-- src/utopya/utopya_jobtree.py | 14 +++++++------- src/utopya/utopya_post.py | 2 +- src/utopya/utopya_rc.py | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/utopya/utopya_build.py b/src/utopya/utopya_build.py index 69f7f08..8231fc9 100644 --- a/src/utopya/utopya_build.py +++ b/src/utopya/utopya_build.py @@ -106,7 +106,7 @@ Classes import logging # tools: -import utopya_rc +from . import utopya_rc # ------------------------------------------------- @@ -303,7 +303,7 @@ class UtopyaCopy(utopya_rc.UtopyaRc): import os # tools: - import utopya_tools + from . import utopya_tools # init base object: utopya_rc.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) @@ -797,7 +797,7 @@ class UtopyaConfigure(utopya_rc.UtopyaRc): import glob # tools: - import utopya_tools + from . import utopya_tools # info ... logging.info("configure source code ...") @@ -1236,7 +1236,7 @@ class UtopyaConfigure(utopya_rc.UtopyaRc): """ # tools: - import utopya_tools + from . import utopya_tools # info... logging.info(" macro definitions ...") @@ -2496,7 +2496,7 @@ class UtopyaDependencies(utopya_rc.UtopyaRc): """ # tools: - import utopya_tools + from . import utopya_tools # external program: prog = "makedepf90" @@ -2689,7 +2689,7 @@ class UtopyaMake(utopya_rc.UtopyaRc): import os # tools: - import utopya_tools + from . import utopya_tools # init base object: utopya_rc.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) diff --git a/src/utopya/utopya_index.py b/src/utopya/utopya_index.py index cb89cf2..63aa491 100644 --- a/src/utopya/utopya_index.py +++ b/src/utopya/utopya_index.py @@ -267,7 +267,7 @@ Classes and methods import logging # tools: -import utopya +from . import utopya # ----------------------------------------------------------------------- @@ -925,7 +925,7 @@ class IndexPart(utopya.UtopyaRc): """ # tools: - import utopya_tools + from . import utopya_tools # read list from settings: line = self.GetSetting(key) diff --git a/src/utopya/utopya_jobtree.py b/src/utopya/utopya_jobtree.py index 7ae1ff2..8c6dbb0 100644 --- a/src/utopya/utopya_jobtree.py +++ b/src/utopya/utopya_jobtree.py @@ -105,7 +105,7 @@ Classes import logging # tools: -import utopya_rc +from . import utopya_rc # ====================================================================== @@ -470,7 +470,7 @@ class UtopyaJobStep(utopya_rc.UtopyaRc): import datetime # tools: - import utopya_tools + from . import utopya_tools # defined ? if len(self.logfile) > 0: @@ -754,7 +754,7 @@ class UtopyaJobStep(utopya_rc.UtopyaRc): # add lines to import tools: self.Append("") self.Append("# tools:") - self.Append("import utopya") + self.Append("from . import utopya") self.Append("") # enddef AddModules @@ -3345,7 +3345,7 @@ class UtopyaJobTaskSubmit(UtopyaJobTask): import os # tools: - import utopya + from . import utopya # parent init: UtopyaJobTask.__init__(self, msg=msg) @@ -3526,7 +3526,7 @@ class UtopyaRcCommand(utopya_rc.UtopyaRc): """ # tools: - import utopya_tools + from . import utopya_tools # init base object: utopya_rc.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) @@ -3593,8 +3593,8 @@ class UtopyaJobTreeTiming(utopya_rc.UtopyaRc): import os # tools: - import utopya_tools - import utopya_timing + from . import utopya_tools + from . import utopya_timing # init base object: utopya_rc.UtopyaRc.__init__(self, rcfile=rcfile, rcbase=rcbase, env=env) diff --git a/src/utopya/utopya_post.py b/src/utopya/utopya_post.py index 0be7490..2c58c62 100644 --- a/src/utopya/utopya_post.py +++ b/src/utopya/utopya_post.py @@ -52,7 +52,7 @@ Classes import logging # tools: -import utopya_base +from . import utopya_base ############################################################ diff --git a/src/utopya/utopya_rc.py b/src/utopya/utopya_rc.py index db5eaf2..61c2eb0 100644 --- a/src/utopya/utopya_rc.py +++ b/src/utopya/utopya_rc.py @@ -298,7 +298,7 @@ class UtopyaRc(UtopyaBase): import sys # tools: - import utopya_tools + from . import utopya_tools # name of model module and class: moduleclass = self.GetSetting(name) -- GitLab From e0ca9c879ca632306b7e054ae92442b5ba74b091 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Fri, 18 Oct 2024 14:34:15 +0000 Subject: [PATCH 11/17] clean up empty code --- scripts/cli.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/cli.py b/scripts/cli.py index 4579afe..5d4a63d 100644 --- a/scripts/cli.py +++ b/scripts/cli.py @@ -92,10 +92,5 @@ def run(rc_file: str = typer.Argument(..., help="Provide .rc file")): # ------------------------------------------------- - - - - - if __name__ == "__main__": main() \ No newline at end of file -- GitLab From 6d7340a366b8abaea772c1d318d5dc283754759c Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sat, 19 Oct 2024 07:23:28 +0000 Subject: [PATCH 12/17] register cli in pyrpoject.toml --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 773488b..3d243b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ docs = [ "nbsphinx", ] +[project.scripts] +cso = "scripts.cli:main" + [tool.black] target-version = ['py310'] extend_skip = ["doc"] -- GitLab From 948d7c8be7e288d06544610e8c6afd13e1f6d15e Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sat, 19 Oct 2024 10:23:22 +0000 Subject: [PATCH 13/17] CLI WIP --- .gitignore | 2 +- cli.py | 98 ++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- scripts/__init__.py | 0 scripts/cli.py | 96 --------------------------------- src/cso/__init__.py | 7 +-- src/utopya/utopya.py | 18 +++---- src/utopya/utopya_runscript.py | 6 +-- 8 files changed, 115 insertions(+), 114 deletions(-) create mode 100644 cli.py create mode 100644 scripts/__init__.py delete mode 100644 scripts/cli.py diff --git a/.gitignore b/.gitignore index 52e9cdc..98c66c4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,7 @@ ENV/ env.bak/ venv.bak/ .vscode - +.cso2 config/Copernicus/cso-user-settings.rc .vscode diff --git a/cli.py b/cli.py new file mode 100644 index 0000000..6db96f4 --- /dev/null +++ b/cli.py @@ -0,0 +1,98 @@ +from typing import Optional + +import typer +import sys +import os +import logging + +# from cso import __package__, __version__ + +# import utopya + +# prefix of CSO installation: +prefix = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), os.pardir)) +# extend path: +sys.path.insert(0, os.path.join(prefix, "src/cso")) +sys.path.insert(0, os.path.join(prefix, "src/utopya")) + +main = typer.Typer() + + +def version_callback(value: bool): + if not value: + return + typer.echo("Not sure yet!") + # typer.echo(f" {__package__} {__version__}") + raise typer.Exit() + + +@main.callback() +def callback( + version: Optional[bool] = typer.Option(None, "--version", "-V", callback=version_callback), +): + """🛰️ CSO Command Line Interface""" + typer.echo("This is the CLI") + + +@main.command() +def run(rc_file: str = typer.Argument(..., help="Provide .rc file")): + """Run cso based on settings provided in .rc file (e.g., cso run /tutorial/tutorial.rc)""" + raise NotImplementedError + # # set environment: + # os.environ["CSO_PREFIX"] = prefix + # # dummy values to ensure that settings could be read first to setup logging: + # os.environ["CSO_RCFILE"] = "None" + # os.environ["CSO_RCDIR"] = "None" + + # # init script: + # utos = utopya.UtopyaRunScriptRc() + + # # setup standard command line arguments, and also enable rcfile arguments: + # utos.ArgumentsSetup(description="CAMS Satellite Operator", rcbase="cso") + + # # evaluate known arguments, store the other ; + # # might show help text and exit: + # args, xargs = utos.ArgumentsParse() + + # # store full path to settings: + # os.environ["CSO_RCFILE"] = os.path.abspath(args.rcfile) + # # location of settings file: + # os.environ["CSO_RCDIR"] = os.path.dirname(os.path.abspath(args.rcfile)) + + # # start, shout info: + # logging.info(f"") + # logging.info(f"** CSO - CAMS Satellite Operator **") + # logging.info(f"") + # logging.info(f"installation prefix : {prefix}") + # logging.info(f"input rcfile : {args.rcfile}") + # logging.info(f"settings base : {args.rcbase}") + # logging.info(f"") + # logging.info(f"environment:") + # for key in ["CSO_PREFIX", "CSO_RCFILE", "CSO_RCDIR"]: + # logging.info(f" {key} = {os.environ[key]}") + # # endfor + # logging.info(f"") + + # # info ... + # logging.info(f"start job tree ...") + # # import class as defined in settings by 'cso.class' key; + # # should be (derived from) 'utopya.JobStep' : + # jbs_cls = utos.ImportClass("class") + # # init job step object, use the 'rcbase' as name: + # jbs = jbs_cls(args.rcbase, args.rcfile) + # # start first: + # jbs.Start(single=args.single) + + # # info: + # logging.info(f"") + # logging.info(f"** end **") + # logging.info(f"") + + +# ------------------------------------------------- +# end +# ------------------------------------------------- + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 3d243b6..c5f6586 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ docs = [ ] [project.scripts] -cso = "scripts.cli:main" +cso = "cli:main" [tool.black] target-version = ['py310'] diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/cli.py b/scripts/cli.py deleted file mode 100644 index 5d4a63d..0000000 --- a/scripts/cli.py +++ /dev/null @@ -1,96 +0,0 @@ - -from typing import Optional - -import typer -import sys -import os -import logging - -from cso import __package__, __version__ -import utopya - -# prefix of CSO installation: -prefix = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), os.pardir)) -# extend path: -sys.path.insert(0, os.path.join(prefix, "src/cso")) -sys.path.insert(0, os.path.join(prefix, "src/utopya")) - -main = typer.Typer() - - -def version_callback(value: bool): - if not value: - return - - typer.echo(f" {__package__} {__version__}") - raise typer.Exit() - - -@main.callback() -def callback( - version: Optional[bool] = typer.Option(None, "--version", "-V", callback=version_callback), -): - """ CSO Command Line Interface""" - - -@main.command() -def run(rc_file: str = typer.Argument(..., help="Provide .rc file")): - """Run cso based on settings provided in .rc file (e.g., cso run /tutorial/tutorial.rc)""" - # set environment: - os.environ["CSO_PREFIX"] = prefix - # dummy values to ensure that settings could be read first to setup logging: - os.environ['CSO_RCFILE'] = 'None' - os.environ['CSO_RCDIR'] = 'None' - - # init script: - utos = utopya.UtopyaRunScriptRc() - - # setup standard command line arguments, and also enable rcfile arguments: - utos.ArgumentsSetup(description="CAMS Satellite Operator", rcbase="cso") - - # evaluate known arguments, store the other ; - # might show help text and exit: - args, xargs = utos.ArgumentsParse() - - # store full path to settings: - os.environ['CSO_RCFILE'] = os.path.abspath( args.rcfile ) - # location of settings file: - os.environ['CSO_RCDIR'] = os.path.dirname( os.path.abspath( args.rcfile ) ) - - # start, shout info: - logging.info(f"") - logging.info(f"** CSO - CAMS Satellite Operator **") - logging.info(f"") - logging.info(f"installation prefix : {prefix}") - logging.info(f"input rcfile : {args.rcfile}") - logging.info(f"settings base : {args.rcbase}") - logging.info(f"") - logging.info(f"environment:") - for key in ["CSO_PREFIX","CSO_RCFILE","CSO_RCDIR"]: - logging.info(f" {key} = {os.environ[key]}") - # endfor - logging.info(f"") - - # info ... - logging.info(f"start job tree ...") - # import class as defined in settings by 'cso.class' key; - # should be (derived from) 'utopya.JobStep' : - jbs_cls = utos.ImportClass("class") - # init job step object, use the 'rcbase' as name: - jbs = jbs_cls(args.rcbase, args.rcfile) - # start first: - jbs.Start(single=args.single) - - # info: - logging.info(f"") - logging.info(f"** end **") - logging.info(f"") - - -# ------------------------------------------------- -# end -# ------------------------------------------------- - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/src/cso/__init__.py b/src/cso/__init__.py index 14f5a77..2fd68bf 100644 --- a/src/cso/__init__.py +++ b/src/cso/__init__.py @@ -1,14 +1,15 @@ from .cso_file import * from .cso_catalogue import * from .cso_inquire import * -from .cso_scihub import * + +# from .cso_scihub import * from .cso_pal import * from .cso_s5p import * -from .cso_s5p_superobs import * +from .cso_superobs import * from .cso_gridded import * from .cso_colocate import * from .cso_superobs import * from .cso_regions import * from .cso_constants import * from .cso_mapping import * -from .cso_plot import * \ No newline at end of file +from .cso_plot import * diff --git a/src/utopya/utopya.py b/src/utopya/utopya.py index 277ea0f..5fc65dc 100644 --- a/src/utopya/utopya.py +++ b/src/utopya/utopya.py @@ -152,15 +152,15 @@ and are defined according to the following hierchy: # imports from sub modules # ------------------------------------------------- -from utopya_base import * -from utopya_rc import * -from utopya_jobscript import * -from utopya_jobtree import * -from utopya_runscript import * -from utopya_tools import * -from utopya_build import * -from utopya_index import * -from utopya_post import * +from .utopya_base import * +from .utopya_rc import * +from .utopya_jobscript import * +from .utopya_jobtree import * +from .utopya_runscript import * +from .utopya_tools import * +from .utopya_build import * +from .utopya_index import * +from .utopya_post import * # ------------------------------------------------- diff --git a/src/utopya/utopya_runscript.py b/src/utopya/utopya_runscript.py index 8612e74..6b3eda5 100644 --- a/src/utopya/utopya_runscript.py +++ b/src/utopya/utopya_runscript.py @@ -53,8 +53,8 @@ Classes import logging # tools: -from utopya_base import UtopyaBase -from utopya_rc import UtopyaRc +from .utopya_base import UtopyaBase +from .utopya_rc import UtopyaRc # ------------------------------------------------- @@ -63,7 +63,6 @@ from utopya_rc import UtopyaRc class UtopyaRunScript(UtopyaBase): - """ Base class for script object, to be used in user defined scripts to quickly perform some common tasks: @@ -210,7 +209,6 @@ class UtopyaRunScript(UtopyaBase): class UtopyaRunScriptRc(UtopyaRc): - """ Base class for script object, to be used in user defined scripts that need a rcfile argument. -- GitLab From be09b776a0212b0a05ad696d6dbc95f4f855fb19 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sat, 19 Oct 2024 11:04:17 +0000 Subject: [PATCH 14/17] needed cli in src/cso. now kinda works --- pyproject.toml | 2 +- {scripts => src/cso/scripts}/__init__.py | 0 cli.py => src/cso/scripts/cli.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {scripts => src/cso/scripts}/__init__.py (100%) rename cli.py => src/cso/scripts/cli.py (100%) diff --git a/pyproject.toml b/pyproject.toml index c5f6586..5e2087f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ docs = [ ] [project.scripts] -cso = "cli:main" +cso = "cso.scripts.cli:main" [tool.black] target-version = ['py310'] diff --git a/scripts/__init__.py b/src/cso/scripts/__init__.py similarity index 100% rename from scripts/__init__.py rename to src/cso/scripts/__init__.py diff --git a/cli.py b/src/cso/scripts/cli.py similarity index 100% rename from cli.py rename to src/cso/scripts/cli.py -- GitLab From 07d1137f8c93bb963a8e2b2f219160dde09c1ec7 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sat, 19 Oct 2024 11:11:20 +0000 Subject: [PATCH 15/17] version cli working --- pyproject.toml | 2 +- src/cso/__init__.py | 4 ++++ src/cso/scripts/cli.py | 9 ++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5e2087f..0a2750d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "cso" -version = "0.1.0" +version = "0.1.dev0" authors = [{name = "Arjo Segers"}] description = "CAMS Satellite Operator (CSO) tool." readme = "README.md" diff --git a/src/cso/__init__.py b/src/cso/__init__.py index 2fd68bf..90818bb 100644 --- a/src/cso/__init__.py +++ b/src/cso/__init__.py @@ -1,3 +1,7 @@ +from importlib import metadata + +__version__ = metadata.version(__package__) + from .cso_file import * from .cso_catalogue import * from .cso_inquire import * diff --git a/src/cso/scripts/cli.py b/src/cso/scripts/cli.py index 6db96f4..3e3a78c 100644 --- a/src/cso/scripts/cli.py +++ b/src/cso/scripts/cli.py @@ -5,7 +5,7 @@ import sys import os import logging -# from cso import __package__, __version__ +from cso import __package__, __version__ # import utopya @@ -21,8 +21,7 @@ main = typer.Typer() def version_callback(value: bool): if not value: return - typer.echo("Not sure yet!") - # typer.echo(f" {__package__} {__version__}") + typer.echo(f"🛰️ {__package__} {__version__}") raise typer.Exit() @@ -30,13 +29,13 @@ def version_callback(value: bool): def callback( version: Optional[bool] = typer.Option(None, "--version", "-V", callback=version_callback), ): - """🛰️ CSO Command Line Interface""" + """🛰️ CAMS Satellite Operator (CSO) Command Line Interface""" typer.echo("This is the CLI") @main.command() def run(rc_file: str = typer.Argument(..., help="Provide .rc file")): - """Run cso based on settings provided in .rc file (e.g., cso run /tutorial/tutorial.rc)""" + """Run cso based on settings provided in .rc file (e.g., cso run config/tutorial/tutorial.rc)""" raise NotImplementedError # # set environment: # os.environ["CSO_PREFIX"] = prefix -- GitLab From d6526578002611818bd561f93f694e768d959d1c Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sat, 19 Oct 2024 12:05:25 +0000 Subject: [PATCH 16/17] git ignore build --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 98c66c4..af1c39c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,9 +31,14 @@ venv.bak/ .cso2 config/Copernicus/cso-user-settings.rc + + .vscode cso.egg* src/cso.egg* # doc doc/ + +#packaging +build/ \ No newline at end of file -- GitLab From 3136d2c3ae0567ce08e7fa04d22ae41ec8fd0b25 Mon Sep 17 00:00:00 2001 From: lewisblake Date: Sat, 19 Oct 2024 12:05:38 +0000 Subject: [PATCH 17/17] build docs from cli beta --- src/cso/scripts/cli.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/cso/scripts/cli.py b/src/cso/scripts/cli.py index 3e3a78c..9d0c584 100644 --- a/src/cso/scripts/cli.py +++ b/src/cso/scripts/cli.py @@ -1,9 +1,10 @@ +import subprocess from typing import Optional import typer import sys import os -import logging +import webbrowser from cso import __package__, __version__ @@ -30,7 +31,37 @@ def callback( version: Optional[bool] = typer.Option(None, "--version", "-V", callback=version_callback), ): """🛰️ CAMS Satellite Operator (CSO) Command Line Interface""" - typer.echo("This is the CLI") + + +@main.command() +def docs(): + """BETA: Build the documentation""" + docs_dir = "doc" # Define your docs directory + make_command = "make html" # command to generate docs + output_link = "docs/build/index.html" + try: + # Change to the docs directory + os.chdir(docs_dir) + typer.echo(f"Changed directory to {docs_dir}") + + # Run the make docs command + typer.echo("Running 'make html'...") + result = subprocess.run(make_command, shell=True, check=True) + + if result.returncode == 0: + typer.echo("'make html' command executed successfully.") + + # Construct the link to the documentation + docs_path = os.path.abspath(output_link) + typer.echo(f"Opening docs at: {docs_path}") + + # Open the generated docs in a browser + webbrowser.open(f"file://{docs_path}") + else: + typer.echo("Failed to generate the documentation.", err=True) + + except Exception as e: + typer.echo(f"Error: {e}", err=True) @main.command() -- GitLab