Finish loader separation: remove leftover loaders package (for v0.2.0)

.pre-commit-config.yaml

@@ -16,7 +16,9 @@ repos:
     rev: v2.4.2
     hooks:
     - id: codespell
-      args: [--ignore-words-list=COO, --ignore-regex=^\s*"image\/png":\s.*]
+      args:
+        - --ignore-words-list=COO,obstable
+        - --ignore-regex=^\s*"image\/png":\s.*
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.15.20

CHANGELOG.md

@@ -9,9 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - Remove dataset loading and dataset validation and outsource it to [mlwp-data-loaders](https://github.com/mlwp-tools/mlwp-data-loaders) and [mlwp-data-specs](https://github.com/mlwp-tools/mlwp-data-specs) @michielv
+  The optional `ifs` extra is removed; GRIB/IFS dependencies are now handled by `mlwp-data-loaders`.
+  **Migration:** `mxalign.loaders` is gone. In YAML configs, `loader:` is now a dotted module path from `mlwp_data_loaders` (e.g. `mlwp_data_loaders.loaders.anemoi.anemoi_inference`, `...anemoi.anemoi_datasets`, `...harp.obstable`, `...ifs.forecast`) or a path to a local `.py` file defining `load_dataset(paths, **kwargs)`. In Python, use `mlwp_data_loaders.load_and_validate_dataset(paths, loader=...)`. The `grid_mapping:` key is still applied by `Runner` after loading.
 - Store dataset properties as netCDF-safe individual attributes while keeping read compatibility with legacy `attrs["properties"]` dict/JSON data. [\#21](https://github.com/mlwp-tools/mxalign/pull/21) @observingClouds
 - Added CI test workflow with first unit tests. [\#21](https://github.com/mlwp-tools/mxalign/pull/21) @observingClouds
-- Added optional `ifs` dependency group with `cfgrib`, `eccodes`, and `eccodeslib`. [\#21](https://github.com/mlwp-tools/mxalign/pull/21) @observingClouds
 - Added CI action for package build and upload to pypi.org on releases. [\#28](https://github.com/mlwp-tools/mxalign/pull/28) @leifdenby
 
 ## [0.1.0](https://github.com/mlwp-tools/mxalign/releases/tag/v0.1.0)

README.md

@@ -23,9 +23,15 @@ These traits are defined and validated by [`mlwp-data-specs`](https://github.com
 import mlwp_data_loaders as dl
 import mxalign as mx
 
-# Load datasets — traits are attached by the loader
-ds_obs = dl.load("observations_loader", files=["obs.nc"])
-ds_fcst = dl.load("anemoi_inference", files=["forecast.nc"])
+# Load datasets — traits are attached by the loader.
+# The `harp.obstable` loader reads SQLite observation databases (netCDF observation
+# files are no longer supported by a built-in loader).
+ds_obs = dl.load_and_validate_dataset(
+    ["obs.sqlite"], loader="mlwp_data_loaders.loaders.harp.obstable"
+)
+ds_fcst = dl.load_and_validate_dataset(
+    ["forecast.nc"], loader="mlwp_data_loaders.loaders.anemoi.anemoi_inference"
+)
 
 # Align the forecast spatially to match the observation reference
 ds_fcst_aligned = mx.align_space(ds_fcst, reference=ds_obs, method="interpolation")
@@ -49,10 +55,10 @@ The configuration file is divided into several main sections:
 datasets:
   # Define datasets to load, specifying the loader, files, and variables
   obs_data:
-    loader: observations_loader
-    files: ["obs.nc"]
+    loader: mlwp_data_loaders.loaders.harp.obstable
+    files: ["obs.sqlite"]
   fcst_data:
-    loader: anemoi_inference
+    loader: mlwp_data_loaders.loaders.anemoi.anemoi_inference
     files: ["forecast.nc"]
 
 transformations:

examples/config.yaml

@@ -7,17 +7,17 @@ dates:
 
 datasets:
   la-1024-ea-00:
-    loader: anemoi-inference
+    loader: mlwp_data_loaders.loaders.anemoi.anemoi_inference
     files: /scratch/project_465000527/vandenbl/sg-la-comparison/inference-out/la-1024-01-ea-00/{reference_time:strftime(%Y%m%d%H)}.nc
     #grid_mapping: CERRA
     variables: ["2t","10u","10v","msl"]
   cerra:
-    loader: anemoi-datasets
+    loader: mlwp_data_loaders.loaders.anemoi.anemoi_datasets
     files: /scratch/project_465002133/datasets/cerra-rr-an-oper-0001-mars-5p5km-1984-2020-3h-v2-rmi.zarr
     #grid_mapping: CERRA
     variables: ["2t_2", "10u_10", "10v_10", "msl_0"]
   # synop:
-  #   loader: mxalign
+  #   loader: mlwp_data_loaders.loaders.harp.obstable
   #   files: /scratch/project_465002133/datasets/observations/cerra_synops_2020.nc
   #   variables: ["2t","10si","msl"]
 transformations:

pyproject.toml

@@ -31,11 +31,6 @@ mxalign = "mxalign.cli:main"
 earthkit = [
     "earthkit-meteo>=0.6.1",
 ]
-ifs = [
-    "cfgrib>=0.9.15.1",
-    "eccodes>=2.45.0",
-    "eccodeslib>=2.46.2.19",
-]
 verification = [
     "xskillscore>=0.0.29",
 ]

src/mxalign/__init__.py

@@ -19,7 +19,6 @@
     "align_time",
     "align_space",
     "accessors",
-    "loaders",
     "transformations",
     "interpolations",
 ]

src/mxalign/runner.py

@@ -1,8 +1,8 @@
 import os
 import xarray as xr
+from mlwp_data_loaders import load_and_validate_dataset
 
 from .utils.config import Config
-from .loaders.loader import load
 from .transformations.transform import transform
 from .align.time import align_time
 from .align.space import align_space
@@ -42,13 +42,12 @@ def load_datasets(self):
                     files.append(file)
                 else:
                     print(f"File: {file} is missing, skipping.")
-            self.datasets[name] = load(
-                name=loader,
-                files=files,
-                variables=variables,
-                grid_mapping=grid_mapping,
-                **config_ds,
-            )
+            if variables is not None:
+                config_ds["variables"] = variables
+            ds = load_and_validate_dataset(files, loader=loader, **config_ds)
+            if grid_mapping is not None:
+                ds = ds.mx.add_grid_mapping(grid_mapping)
+            self.datasets[name] = ds
 
     def transform_datasets(self):
         config = self.config["transformations"]