From 300e5c6db00f8c05245c68614381b4be3fc26564 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Tue, 27 Oct 2026 11:48:21 +0100 Subject: [PATCH 1/2] Move cli, runner, verification and their helpers into pipeline/ The YAML-driven pipeline (cli, runner, verification, transformations and the config/dates/save helpers it alone uses) now lives in mxalign.pipeline. The console script points at mxalign.pipeline.cli:main. Public re-exports from mxalign/__init__.py are unchanged. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- pyproject.toml | 2 +- src/mxalign/__init__.py | 11 +++++++---- src/mxalign/pipeline/__init__.py | 5 +++++ src/mxalign/{ => pipeline}/cli.py | 0 src/mxalign/{utils => pipeline}/config.py | 0 src/mxalign/{utils => pipeline}/dates.py | 0 src/mxalign/{ => pipeline}/runner.py | 8 ++++---- src/mxalign/{utils => pipeline}/save.py | 0 .../{ => pipeline}/transformations/__init__.py | 0 src/mxalign/{ => pipeline}/transformations/base.py | 0 .../{ => pipeline}/transformations/external.py | 0 .../{ => pipeline}/transformations/registry.py | 0 .../{ => pipeline}/transformations/transform.py | 0 src/mxalign/{ => pipeline}/verification.py | 0 tests/test_dates.py | 6 +++--- tests/test_runner.py | 2 +- tests/test_transformations.py | 4 ++-- 18 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 src/mxalign/pipeline/__init__.py rename src/mxalign/{ => pipeline}/cli.py (100%) rename src/mxalign/{utils => pipeline}/config.py (100%) rename src/mxalign/{utils => pipeline}/dates.py (100%) rename src/mxalign/{ => pipeline}/runner.py (97%) rename src/mxalign/{utils => pipeline}/save.py (100%) rename src/mxalign/{ => pipeline}/transformations/__init__.py (100%) rename src/mxalign/{ => pipeline}/transformations/base.py (100%) rename src/mxalign/{ => pipeline}/transformations/external.py (100%) rename src/mxalign/{ => pipeline}/transformations/registry.py (100%) rename src/mxalign/{ => pipeline}/transformations/transform.py (100%) rename src/mxalign/{ => pipeline}/verification.py (100%) diff --git a/README.md b/README.md index 6f4da9b..b5c57ff 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ mxalign slurm path/to/config.yaml --account your_account --queue your_queue --co You can also execute the entire configuration-driven pipeline directly from Python using the `Runner` class. ```python -from mxalign.runner import Runner +from mxalign.pipeline.runner import Runner # Initialize the runner with a YAML config file or a dictionary runner = Runner("path/to/config.yaml") diff --git a/pyproject.toml b/pyproject.toml index 81cd43a..f1a9cee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ ] [project.scripts] -mxalign = "mxalign.cli:main" +mxalign = "mxalign.pipeline.cli:main" [project.optional-dependencies] earthkit = [ diff --git a/src/mxalign/__init__.py b/src/mxalign/__init__.py index d104858..c4549b8 100644 --- a/src/mxalign/__init__.py +++ b/src/mxalign/__init__.py @@ -1,5 +1,8 @@ -from .transformations.transform import transform -from .transformations.registry import available_transformations, register_transformation +from .pipeline.transformations.transform import transform +from .pipeline.transformations.registry import ( + available_transformations, + register_transformation, +) from .alignment.interpolations.interpolate import interpolate from .alignment.interpolations.registry import ( available_interpolations, @@ -10,7 +13,7 @@ from . import accessor from . import alignment -from . import transformations +from . import pipeline __all__ = [ "transform", @@ -23,5 +26,5 @@ "align_space", "accessor", "alignment", - "transformations", + "pipeline", ] diff --git a/src/mxalign/pipeline/__init__.py b/src/mxalign/pipeline/__init__.py new file mode 100644 index 0000000..50dece2 --- /dev/null +++ b/src/mxalign/pipeline/__init__.py @@ -0,0 +1,5 @@ +from . import transformations + +__all__ = [ + "transformations", +] diff --git a/src/mxalign/cli.py b/src/mxalign/pipeline/cli.py similarity index 100% rename from src/mxalign/cli.py rename to src/mxalign/pipeline/cli.py diff --git a/src/mxalign/utils/config.py b/src/mxalign/pipeline/config.py similarity index 100% rename from src/mxalign/utils/config.py rename to src/mxalign/pipeline/config.py diff --git a/src/mxalign/utils/dates.py b/src/mxalign/pipeline/dates.py similarity index 100% rename from src/mxalign/utils/dates.py rename to src/mxalign/pipeline/dates.py diff --git a/src/mxalign/runner.py b/src/mxalign/pipeline/runner.py similarity index 97% rename from src/mxalign/runner.py rename to src/mxalign/pipeline/runner.py index 3b21117..57014b2 100644 --- a/src/mxalign/runner.py +++ b/src/mxalign/pipeline/runner.py @@ -3,11 +3,11 @@ from mlwp_data_loaders import load_and_validate_dataset -from .utils.config import Config +from .config import Config from .transformations.transform import transform -from .alignment.time import align_time -from .alignment.nans import broadcast_nans -from .utils.save import save_dataset, save_metrics +from ..alignment.time import align_time +from ..alignment.nans import broadcast_nans +from .save import save_dataset, save_metrics from .verification import Metric diff --git a/src/mxalign/utils/save.py b/src/mxalign/pipeline/save.py similarity index 100% rename from src/mxalign/utils/save.py rename to src/mxalign/pipeline/save.py diff --git a/src/mxalign/transformations/__init__.py b/src/mxalign/pipeline/transformations/__init__.py similarity index 100% rename from src/mxalign/transformations/__init__.py rename to src/mxalign/pipeline/transformations/__init__.py diff --git a/src/mxalign/transformations/base.py b/src/mxalign/pipeline/transformations/base.py similarity index 100% rename from src/mxalign/transformations/base.py rename to src/mxalign/pipeline/transformations/base.py diff --git a/src/mxalign/transformations/external.py b/src/mxalign/pipeline/transformations/external.py similarity index 100% rename from src/mxalign/transformations/external.py rename to src/mxalign/pipeline/transformations/external.py diff --git a/src/mxalign/transformations/registry.py b/src/mxalign/pipeline/transformations/registry.py similarity index 100% rename from src/mxalign/transformations/registry.py rename to src/mxalign/pipeline/transformations/registry.py diff --git a/src/mxalign/transformations/transform.py b/src/mxalign/pipeline/transformations/transform.py similarity index 100% rename from src/mxalign/transformations/transform.py rename to src/mxalign/pipeline/transformations/transform.py diff --git a/src/mxalign/verification.py b/src/mxalign/pipeline/verification.py similarity index 100% rename from src/mxalign/verification.py rename to src/mxalign/pipeline/verification.py diff --git a/tests/test_dates.py b/tests/test_dates.py index c8e6380..c37a6bb 100644 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -2,9 +2,9 @@ import pytest -from mxalign.utils.config import Config -from mxalign.utils.dates import Dates -from mxalign.utils.save import DatasetPath +from mxalign.pipeline.config import Config +from mxalign.pipeline.dates import Dates +from mxalign.pipeline.save import DatasetPath # 3 reference times (daily) × 3 lead times (0, 12, 24h) DATES = dict(start="2020-01-01", end="2020-01-03", period="1D", range="24h", step="12h") diff --git a/tests/test_runner.py b/tests/test_runner.py index a4c52be..b3aa11d 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,6 +1,6 @@ import pytest -from mxalign.runner import Runner +from mxalign.pipeline.runner import Runner def _runner(datasets: dict, alignment: dict) -> Runner: diff --git a/tests/test_transformations.py b/tests/test_transformations.py index fd737c3..ec85306 100644 --- a/tests/test_transformations.py +++ b/tests/test_transformations.py @@ -1,11 +1,11 @@ import pytest import xarray as xr -from mxalign.transformations.registry import ( +from mxalign.pipeline.transformations.registry import ( available_transformations, register_transformation, ) -from mxalign.transformations.transform import transform +from mxalign.pipeline.transformations.transform import transform @register_transformation("_test_double") From 6d5aca535ca40847697d404d85bba217e3cb65dd Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Tue, 27 Oct 2026 12:25:09 +0100 Subject: [PATCH 2/2] Update Runner import in advanced example notebook Co-Authored-By: Claude Opus 5.5 --- examples/advanced.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/advanced.ipynb b/examples/advanced.ipynb index 2c15b56..c577565 100644 --- a/examples/advanced.ipynb +++ b/examples/advanced.ipynb @@ -384,7 +384,7 @@ "metadata": {}, "outputs": [], "source": [ - "from mxalign.runner import Runner\n", + "from mxalign.pipeline.runner import Runner\n", "\n", "runner = Runner(\"./config.yaml\")" ]