From 2de9155c13061372cf8e9c77f0372e074f3de885 Mon Sep 17 00:00:00 2001 From: Bhaskar Gurram Date: Fri, 9 Oct 2026 14:38:02 +0000 Subject: [PATCH 01/22] feat(ogc): accept date, datetime and Timestamp values in date arguments A datetime.date, datetime.datetime or pandas.Timestamp passed to a date argument (time=[df.index.min(), None]) failed inside the formatter with AttributeError ('endswith') or, for a lone datetime, TypeError (not iterable), naming no argument. Read them like the equivalent string: aware values convert to UTC, naive ones use the local zone as naive strings do, a date is midnight of that day, and NaT is an open bound. Any other type raises ValueError naming the argument. Follow-up named in #441. --- NEWS.md | 2 + dataretrieval/ogc/dates.py | 70 +++++++++++++++++++++--------- tests/waterdata_test.py | 12 ++++++ tests/waterdata_utils_test.py | 80 +++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 19 deletions(-) diff --git a/NEWS.md b/NEWS.md index f683c9d2..2933de8c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**10/09/2026:** Date arguments of the OGC getters (`get_daily()`, `get_continuous()`, `waterdata.get_ratings()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument) accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. They are read like the equivalent string: an aware value is converted to UTC, a naive one is read in the local time zone as a naive string is, a `date` is midnight of that day, and `NaT` is an open bound like `None`. **Bug fix:** these values used to fail inside the package with `AttributeError: ... has no attribute 'endswith'`, or `TypeError: ... is not iterable` for a lone `datetime`, which named no argument. Any other type, such as a number, now raises `ValueError` naming the argument and the types it accepts. + **10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter. **10/07/2026:** **Bug fix:** a `".."` endpoint in a two-value date range -- `time=["2024-01-01", ".."]`, the open-ended form shown in the `waterdata.get_ratings()` docstring -- discarded the whole range, because `..` was not recognized as an open bound and failed to parse as a date. `waterdata.get_ratings()` then searched with no `datetime` at all and returned every rating regardless of date (59 instead of 44 for the docstring's bounding box from 2026-09-20 on), and the OGC getters (`get_daily()`, `get_continuous()`, `get_field_measurements()`, and the other `time`, `begin`, `end`, and `last_modified` arguments) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format". `".."` is now an open bound like `None`, so `["2024-01-01", ".."]` sends the same range as `"2024-01-01/.."` and `["2024-01-01", None]`. diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 0fe5cd9e..319df8be 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -8,7 +8,8 @@ from __future__ import annotations import re -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence +from datetime import date as _date from datetime import datetime from zoneinfo import ZoneInfo @@ -65,7 +66,13 @@ def _parse_datetime(value: str) -> datetime | None: _OPEN_BOUND = ".." -def _is_blank(dt: str | None) -> bool: +# One element of a date argument: an ISO 8601 string, or a ``date``, +# ``datetime`` or ``pandas.Timestamp`` (both subclasses of ``datetime.date``), +# with ``None`` (or ``NaN`` / ``NaT``) for an open bound. +_DateLike = str | _date | None + + +def _is_blank(dt: _DateLike) -> bool: """True for a None, NaN, empty-string, or ``..`` element. Each is a spelling of an open bound, so ``["2024-01-01", ".."]`` means the @@ -74,22 +81,42 @@ def _is_blank(dt: str | None) -> bool: return dt is None or bool(pd.isna(dt)) or dt in ("", _OPEN_BOUND) -def _format_one(dt: str | None, *, date: bool, name: str) -> str: - """Format a single datetime element for inclusion in the API time arg. +def _to_datetime(dt: str | _date, *, name: str) -> datetime: + """Read one non-blank element as a ``datetime`` (naive iff it has no zone). - Raises ``ValueError`` naming *name* when the element is not blank and - matches no supported format. + A ``date`` is midnight of that day, like the string ``"2024-01-01"``. + Raises ``ValueError`` naming *name* for an unreadable string or a value of + another type. """ - if dt is None or _is_blank(dt): - return _OPEN_BOUND - parsed = _parse_datetime(dt) + if isinstance(dt, pd.Timestamp): + # A plain ``datetime``: ``Timestamp.astimezone()`` needs an explicit zone. + converted: datetime = dt.to_pydatetime(warn=False) + return converted + if isinstance(dt, datetime): + return dt + if isinstance(dt, _date): + return datetime(dt.year, dt.month, dt.day) # noqa: DTZ001 + parsed = _parse_datetime(dt) if isinstance(dt, str) else None if parsed is None: raise ValueError( f"{name} could not be read as a date or datetime: {dt!r}. " "Pass an ISO 8601 date or datetime such as '2024-01-01' or " - "'2024-01-01T12:00:00Z', and None for an open end of a range " + "'2024-01-01T12:00:00Z' (a datetime.date, datetime.datetime or " + "pandas.Timestamp also works), and None for an open end of a range " "(['2024-01-01', None])." ) + return parsed + + +def _format_one(dt: _DateLike, *, date: bool, name: str) -> str: + """Format a single datetime element for inclusion in the API time arg. + + Raises ``ValueError`` naming *name* when the element is not blank and + cannot be read as a date or datetime. + """ + if dt is None or _is_blank(dt): + return _OPEN_BOUND + parsed = _to_datetime(dt, name=name) if date: return parsed.strftime("%Y-%m-%d") # Naive inputs are interpreted in the system local zone (for backwards @@ -101,11 +128,14 @@ def _format_one(dt: str | None, *, date: bool, name: str) -> str: def _coerce_to_list( - datetime_input: str | Sequence[str | None], + datetime_input: str | _date | Sequence[_DateLike], name: str = "date input", -) -> list[str | None]: +) -> list[_DateLike]: """Normalize datetime input to a list, raising on invalid shapes.""" - if isinstance(datetime_input, str): + if isinstance(datetime_input, (str, _date)) or not isinstance( + datetime_input, Iterable + ): + # A lone value; anything that is not a date is rejected per element. return [datetime_input] if isinstance(datetime_input, Mapping): raise TypeError( @@ -120,13 +150,13 @@ def _is_passthrough(single: str) -> bool: return bool(_DURATION_RE.match(single) or "/" in single) -def _all_blank(items: list[str | None]) -> bool: +def _all_blank(items: list[_DateLike]) -> bool: """True when every element is None, NaN, the empty string, or ``..``.""" return all(_is_blank(dt) for dt in items) def _format_api_dates( - datetime_input: str | Sequence[str | None] | None, + datetime_input: str | _date | Sequence[_DateLike] | None, date: bool = False, *, name: str = "date input", @@ -140,9 +170,11 @@ def _format_api_dates( Parameters ---------- - datetime_input : Union[str, List[Optional[str]], None] - A single date/datetime string or a list of one or two date/datetime - strings. Accepts formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or + datetime_input : Union[str, date, List[Optional[Union[str, date]]], None] + A single date/datetime or a list of one or two of them. Each may be a + ``datetime.date``, ``datetime.datetime`` or ``pandas.Timestamp`` (a + naive one is read in the local time zone, as a naive string is), or a + string. Strings accept formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or without ``Z``/numeric offset), or relative periods (e.g., "P7D" / "PT36H"). Range endpoints may be ``None``/``NaN``/empty or ``".."`` to denote a half-bounded range. @@ -174,7 +206,7 @@ def _format_api_dates( ------ ValueError If `datetime_input` contains more than two values, or an element that - is not blank matches no supported format. + is not blank matches no supported format or is of another type. Notes ----- diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 541f7a24..ae6ed660 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -619,6 +619,18 @@ def test_construct_api_requests_two_element_date_list_becomes_interval(): assert "time=2024-01-01%2F2024-01-31" in str(req.url) +def test_construct_api_requests_accepts_timestamp_bounds(): + """A ``pandas.Timestamp`` bound, as in ``time=[df.index.min(), None]``, + builds the same request as its string spelling instead of raising + ``AttributeError``.""" + req = _construct_api_requests( + "daily", + monitoring_location_id="USGS-05427718", + time=[pd.Timestamp("2024-01-01", tz="UTC"), None], + ) + assert "time=2024-01-01%2F.." in str(req.url) + + # --- mocked getter smoke tests ------------------------------------------------ # These replace what used to be ~34 live calls to the Water Data API. Each one # serves a committed fixture (``tests/data/waterdata_ogc_fixtures.json``, two diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 281744c0..157970e0 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1049,6 +1049,86 @@ def test_format_api_dates_rejects_mapping(): _format_api_dates({"2024-01-01": "ignored"}) +@pytest.mark.parametrize( + "value, date, expected", + [ + (datetime.date(2024, 1, 1), True, "2024-01-01"), + ([datetime.date(2024, 1, 1), None], True, "2024-01-01/.."), + ( + [datetime.date(2024, 1, 1), datetime.date(2024, 2, 1)], + True, + "2024-01-01/2024-02-01", + ), + (pd.Timestamp("2024-01-01T10:30:00", tz="UTC"), True, "2024-01-01"), + ( + pd.Timestamp("2024-01-01T10:30:00", tz="UTC"), + False, + "2024-01-01T10:30:00Z", + ), + ( + datetime.datetime( + 2024, 1, 1, 6, 0, tzinfo=datetime.timezone(datetime.timedelta(hours=-4)) + ), + False, + "2024-01-01T10:00:00Z", + ), + ( + [pd.Timestamp("2024-01-01", tz="UTC"), pd.NaT], + False, + "2024-01-01T00:00:00Z/..", + ), + ( + ["2024-01-01T00:00:00Z", pd.Timestamp("2024-02-01", tz="UTC")], + False, + "2024-01-01T00:00:00Z/2024-02-01T00:00:00Z", + ), + ], + ids=[ + "date", + "date_open_end", + "date_pair", + "aware_timestamp_date_only", + "aware_timestamp_to_utc", + "aware_datetime_offset_to_utc", + "nat_is_an_open_bound", + "mixed_string_and_timestamp", + ], +) +def test_format_api_dates_accepts_date_and_datetime_objects(value, date, expected): + """``datetime.date``, ``datetime.datetime`` and ``pandas.Timestamp`` used to + raise ``AttributeError`` (no ``endswith``) or ``TypeError`` (a lone datetime + is not iterable) from inside the formatter. They are read like the + equivalent string, and ``NaT`` is an open bound like ``None``.""" + assert _format_api_dates(value, date=date) == expected + + +@pytest.mark.parametrize("date", [True, False], ids=["date_only", "datetime"]) +def test_format_api_dates_reads_naive_objects_like_naive_strings(date): + """A naive ``datetime``/``Timestamp`` is local time, the same rule a naive + string follows, so both spellings of one instant build the same filter in + any time zone. A ``date`` is midnight of that day, like ``"2024-01-01"``.""" + as_string = _format_api_dates(["2024-01-01T10:00:00", None], date=date) + naive_datetime = [datetime.datetime(2024, 1, 1, 10), None] # noqa: DTZ001 + assert _format_api_dates(naive_datetime, date=date) == as_string + naive_timestamp = [pd.Timestamp("2024-01-01T10:00:00"), None] + assert _format_api_dates(naive_timestamp, date=date) == as_string + assert _format_api_dates(datetime.date(2024, 1, 1), date=date) == ( + _format_api_dates("2024-01-01", date=date) + ) + + +@pytest.mark.parametrize("value", [20240101, [20240101, None], 1.5]) +def test_format_api_dates_rejects_other_types_naming_the_argument(value): + """A number is not a date. It used to fail with an ``AttributeError`` from + inside the formatter; the message now names the caller's argument and the + types it accepts.""" + with pytest.raises(ValueError) as excinfo: + _format_api_dates(value, name="time") + message = str(excinfo.value) + assert message.startswith("time could not be read as a date") + assert "pandas.Timestamp" in message + + def _make_response(status, body, reason=None, content_type="text/html"): headers = {"Content-Type": content_type} extensions = {} From 3bb00743be86430c59a041d33596ae5d0c38e856 Mon Sep 17 00:00:00 2001 From: Bhaskar Gurram Date: Fri, 9 Oct 2026 16:00:27 +0000 Subject: [PATCH 02/22] fix(ogc): read each side of a "start/end" interval string like the list form _format_api_dates sent any single string containing "/" unchanged. The same range spelled as the documented string ("2024-01-01T10:00:00/..") and as the list (["2024-01-01T10:00:00", None]) could select different data: the string skipped the local-to-UTC conversion of naive times, the offset-to-Z conversion, and the date truncation for date-only collections, and "not-a-date/also-bad" reached the service unvalidated. Split the string on its "/" and format each side as a list element is formatted, so an unreadable side raises ValueError naming the caller's argument before any request. An empty side becomes "..", and "../.." means no filter, as [None, None] does. One side may still be an ISO 8601 duration paired with an instant ("2024-01-01/P7D"), kept unchanged; any other shape (three sides, two durations, a duration with an open end) raises ValueError naming the argument. A lone duration ("P7D") is unchanged. Follow-up named in #441. --- NEWS.md | 2 + dataretrieval/ogc/dates.py | 45 ++++++++++++++-- tests/waterdata_test.py | 4 +- tests/waterdata_utils_test.py | 97 +++++++++++++++++++++++++++++++++-- 4 files changed, 140 insertions(+), 8 deletions(-) diff --git a/NEWS.md b/NEWS.md index f683c9d2..18e2dc12 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**10/09/2026:** **Behavior change:** a date argument given as one `"start/end"` string -- `time="2024-01-01T10:00:00/.."`, the half-bounded form shown in the getter docstrings -- is now read side by side like the two-value list form `["2024-01-01T10:00:00", None]`, for the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument) and `waterdata.get_ratings()`. **Bug fix:** the string used to be sent unchanged, so the same range spelled two documented ways could select different data: a naive time was not converted from local time to UTC (off by 7 hours in Denver in January), a time with an offset was not converted to `Z`, and a date-only collection such as `daily` received times. A side that cannot be read as a date -- `time="not-a-date/also-bad"` or `"2024-01-01/tomorrow"` -- now raises `ValueError` naming the argument before any request is sent, as a bad list element already does, and so does a string that is not two sides separated by one `/`, or that pairs a duration with an open end or a second duration. An empty side is an open bound like `..` (the service rejects `"2024-01-01T00:00:00Z/"`), and `"../.."` means no date filter, as `[None, None]` does. A lone duration such as `"P7D"` is unchanged, and one side may still be a duration paired with a date (`"2024-01-01/P7D"`), which is sent as before. + **10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter. **10/07/2026:** **Bug fix:** a `".."` endpoint in a two-value date range -- `time=["2024-01-01", ".."]`, the open-ended form shown in the `waterdata.get_ratings()` docstring -- discarded the whole range, because `..` was not recognized as an open bound and failed to parse as a date. `waterdata.get_ratings()` then searched with no `datetime` at all and returned every rating regardless of date (59 instead of 44 for the docstring's bounding box from 2026-09-20 on), and the OGC getters (`get_daily()`, `get_continuous()`, `get_field_measurements()`, and the other `time`, `begin`, `end`, and `last_modified` arguments) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format". `".."` is now an open bound like `None`, so `["2024-01-01", ".."]` sends the same range as `"2024-01-01/.."` and `["2024-01-01", None]`. diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 0fe5cd9e..65e1a52a 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -117,7 +117,38 @@ def _coerce_to_list( def _is_passthrough(single: str) -> bool: """True when a single-element input should be returned as-is.""" - return bool(_DURATION_RE.match(single) or "/" in single) + return bool(_DURATION_RE.match(single)) + + +def _format_interval(interval: str, *, date: bool, name: str) -> str | None: + """Format each side of a pre-formatted ``"start/end"`` interval string. + + Each side is formatted like an element of the two-value list form, so + ``"2024-01-01T10:00:00/.."`` sends the same range as + ``["2024-01-01T10:00:00", None]``. One side may instead be an ISO 8601 + duration paired with an instant (``"2024-01-01/P7D"``), which is kept + unchanged. Returns None when both sides are open. + + Raises ``ValueError`` naming *name* when the string is not two sides + separated by one ``"/"``, or a side cannot be read. + """ + sides = interval.split("/") + durations = [bool(_DURATION_RE.match(side)) for side in sides] + if len(sides) != 2 or ( + any(durations) and (all(durations) or any(_is_blank(s) for s in sides)) + ): + raise ValueError( + f"{name} is not a valid interval: {interval!r}. Pass a start and " + "an end separated by '/', such as '2024-01-01/2024-12-31', with " + "'..' for an open end ('2024-01-01/..')." + ) + formatted = [ + side if is_duration else _format_one(side, date=date, name=name) + for side, is_duration in zip(sides, durations, strict=True) + ] + if formatted == [_OPEN_BOUND, _OPEN_BOUND]: + return None + return "/".join(formatted) def _all_blank(items: list[str | None]) -> bool: @@ -182,8 +213,11 @@ def _format_api_dates( or ``".."`` endpoint is rendered as ``".."`` to denote an open bound (e.g. ``"2024-01-01/.."``); the range is only None when *every* element is blank/NA/``".."``. - - Supports ISO 8601 durations such as "P7D" and "PT36H" and pre-formatted - intervals containing ``"/"``; both are passed through unchanged. + - Supports ISO 8601 durations such as "P7D" and "PT36H", which are passed + through unchanged. + - A single string containing ``"/"`` is an interval: each side is formatted + like an element of the two-value form, and may also be an ISO 8601 + duration paired with an instant (``"2024-01-01/P7D"``). - Converts datetimes to UTC and formats as ISO 8601 with 'Z' suffix when `date` is False. Inputs with an explicit offset (``Z`` or ``+HH:MM``) are converted from that offset to UTC; naive inputs are interpreted in the @@ -204,7 +238,10 @@ def _format_api_dates( "or two for a closed interval ('2020-01-01', '2020-12-31')." ) - # Pass through duration ("P7D", "PT36H") and pre-formatted interval ("a/b") + if len(items) == 1 and isinstance(items[0], str) and "/" in items[0]: + return _format_interval(items[0], date=date, name=name) + + # Pass through duration ("P7D", "PT36H") if len(items) == 1 and isinstance(items[0], str) and _is_passthrough(items[0]): return items[0] diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 541f7a24..c1fee7c6 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -801,7 +801,9 @@ def test_get_daily_keeps_a_dotdot_open_bound_in_time(httpx_mock): @pytest.mark.parametrize( - "time", ["2025-13-45", ["2025-01-01", "yesterday"]], ids=["single", "range"] + "time", + ["2025-13-45", ["2025-01-01", "yesterday"], "not-a-date/also-bad"], + ids=["single", "range", "interval_string"], ) def test_get_daily_rejects_an_unreadable_time_before_any_request(httpx_mock, time): """A bound that matches no date format used to drop the whole filter, so diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 281744c0..02a847d4 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -948,7 +948,7 @@ def test_type_cols_warning_is_singular_for_one_value(): "fractional_seconds", "offset_to_utc", "iso8601_pair_to_interval", - "passthrough_interval", + "utc_interval_string", "passthrough_duration", "time_only_duration", "date_only", @@ -964,8 +964,8 @@ def test_type_cols_warning_is_singular_for_one_value(): def test_format_api_dates(value, date, expected): """``_format_api_dates`` normalizes ISO 8601 datetimes to UTC (dropping fractional seconds, converting offsets), joins a pair into an interval, - passes durations / intervals through unchanged, and renders a None - endpoint as ``..``.""" + passes durations through unchanged, and renders a None endpoint as + ``..``.""" assert _format_api_dates(value, date=date) == expected @@ -982,6 +982,97 @@ def test_format_api_dates_treats_an_all_blank_sequence_as_no_filter(): assert _format_api_dates(["..", ".."]) is None +@pytest.mark.parametrize( + "interval, pair, date", + [ + ("2024-01-01T10:00:00/..", ["2024-01-01T10:00:00", None], False), + ("../2024-01-01T10:00:00", [None, "2024-01-01T10:00:00"], False), + ("2018-02-12T19:20:50-04:00/..", ["2018-02-12T19:20:50-04:00", ".."], False), + ( + "2024-01-01 10:00:00/2024-01-02", + ["2024-01-01 10:00:00", "2024-01-02"], + False, + ), + ( + "2024-01-01T10:00:00Z/2024-02-01T00:00:00Z", + ["2024-01-01T10:00:00Z", "2024-02-01T00:00:00Z"], + True, + ), + ("2024-01-01T10:00:00Z/", ["2024-01-01T10:00:00Z", ""], False), + ], + ids=[ + "naive_start", + "naive_end", + "offset", + "space_separated", + "date_only_truncates", + "empty_end", + ], +) +def test_format_api_dates_formats_an_interval_string_like_the_pair( + interval, pair, date +): + """A ``"start/end"`` string and the two-value list are two spellings of the + same range, so they send the same value. The string used to be sent + unchanged: a naive time was not converted from local time to UTC, and a + date-only collection received times.""" + assert _format_api_dates(interval, date=date) == _format_api_dates(pair, date=date) + + +@pytest.mark.parametrize( + "interval, date, expected", + [ + ("2018-02-12T19:20:50-04:00/..", False, "2018-02-12T23:20:50Z/.."), + ( + "2024-01-01T10:00:00Z/2024-02-01T00:00:00Z", + True, + "2024-01-01/2024-02-01", + ), + ("2024-01-01T10:00:00+02:00/PT36H", False, "2024-01-01T08:00:00Z/PT36H"), + ("P7D/2024-01-08T00:00:00Z", True, "P7D/2024-01-08"), + ], + ids=["offset_to_utc", "date_only", "start_duration", "duration_end"], +) +def test_format_api_dates_formats_each_side_of_an_interval_string( + interval, date, expected +): + """Each side is formatted on its own; an ISO 8601 duration paired with an + instant is kept unchanged.""" + assert _format_api_dates(interval, date=date) == expected + + +def test_format_api_dates_treats_an_all_open_interval_string_as_no_filter(): + """``"../.."`` is the string spelling of ``[None, None]``.""" + assert _format_api_dates("../..") is None + + +@pytest.mark.parametrize( + "interval, message", + [ + ("not-a-date/also-bad", "time could not be read as a date or datetime"), + ("2024-01-01/tomorrow", "time could not be read as a date or datetime"), + ("2024-01-01/2024-02-01/2024-03-01", "time is not a valid interval"), + ("P7D/P1D", "time is not a valid interval"), + ("P7D/..", "time is not a valid interval"), + ("../P7D", "time is not a valid interval"), + ], + ids=[ + "both_sides", + "one_side", + "three_sides", + "two_durations", + "duration_open_end", + "open_start_duration", + ], +) +def test_format_api_dates_rejects_an_unreadable_interval_string(interval, message): + """An interval string used to be sent unchanged, so ``time="garbage/.."`` + reached the service. It is now rejected like an unreadable list element, + naming the caller's argument.""" + with pytest.raises(ValueError, match=f"^{message}"): + _format_api_dates(interval, name="time") + + @pytest.mark.parametrize( "value", ["2024-13-45", "Jan 1 2024", "Apr", ["2024-01-01", "garbage"], ["garbage", None]], From 07836576dce2f34bd9e801e25d6a8d517dc43b59 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 08:12:45 -0500 Subject: [PATCH 03/22] fix(ratings): accept a lone date or Timestamp in get_ratings(time=...) _validate_time_no_duration iterated a lone Timestamp/date and raised TypeError before the formatter could read it; it now normalizes with _coerce_to_list and checks only string elements. Also: drop the redundant date check in _coerce_to_list (a date is not iterable), update the Mapping TypeError to name the accepted types, and tighten the NEWS entry. --- NEWS.md | 2 +- dataretrieval/ogc/dates.py | 12 ++++++------ dataretrieval/waterdata/ratings.py | 7 +++++-- tests/waterdata_ratings_test.py | 22 ++++++++++++++++++++++ tests/waterdata_utils_test.py | 6 ++++-- 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index 2933de8c..ee48b480 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -**10/09/2026:** Date arguments of the OGC getters (`get_daily()`, `get_continuous()`, `waterdata.get_ratings()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument) accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. They are read like the equivalent string: an aware value is converted to UTC, a naive one is read in the local time zone as a naive string is, a `date` is midnight of that day, and `NaT` is an open bound like `None`. **Bug fix:** these values used to fail inside the package with `AttributeError: ... has no attribute 'endswith'`, or `TypeError: ... is not iterable` for a lone `datetime`, which named no argument. Any other type, such as a number, now raises `ValueError` naming the argument and the types it accepts. +**10/09/2026:** Date arguments of the `waterdata` and `ngwmn` OGC getters (`time`, `begin`, `end`, `datetime`, `last_modified`) and of `waterdata.get_ratings()` accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. Each is read like the equivalent string: an aware value is converted to UTC, a naive one is read in local time, a `date` is midnight, and `NaT` is an open bound. **Bug fix:** these values used to raise `AttributeError` or `TypeError` without naming the argument. Any other type, such as a number, now raises `ValueError` naming the argument and the accepted types. **10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter. diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 319df8be..3b2f1bc6 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -89,7 +89,8 @@ def _to_datetime(dt: str | _date, *, name: str) -> datetime: another type. """ if isinstance(dt, pd.Timestamp): - # A plain ``datetime``: ``Timestamp.astimezone()`` needs an explicit zone. + # ``Timestamp.astimezone`` is ``tz_convert`` and requires a zone, so + # hand ``_format_one`` a plain ``datetime``. converted: datetime = dt.to_pydatetime(warn=False) return converted if isinstance(dt, datetime): @@ -132,14 +133,13 @@ def _coerce_to_list( name: str = "date input", ) -> list[_DateLike]: """Normalize datetime input to a list, raising on invalid shapes.""" - if isinstance(datetime_input, (str, _date)) or not isinstance( - datetime_input, Iterable - ): - # A lone value; anything that is not a date is rejected per element. + if isinstance(datetime_input, str) or not isinstance(datetime_input, Iterable): + # A lone value (a ``date`` is not iterable); a non-date is rejected + # per element. return [datetime_input] if isinstance(datetime_input, Mapping): raise TypeError( - f"{name} must be a string or sequence of strings, " + f"{name} must be a date, a string, or a sequence of them, " f"not {type(datetime_input).__name__}." ) return list(datetime_input) diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index f78ea2fc..5c864906 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -20,7 +20,7 @@ from dataretrieval._validation import render_options from dataretrieval.exceptions import DataRetrievalError, SkippedRatingWarning -from dataretrieval.ogc.dates import _DURATION_RE, _format_api_dates +from dataretrieval.ogc.dates import _DURATION_RE, _coerce_to_list, _format_api_dates from dataretrieval.ogc.errors import _raise_for_non_200 from dataretrieval.ogc.filters import _quote_cql_str from dataretrieval.ogc.requests import _check_monitoring_location_id @@ -221,7 +221,10 @@ def _validate_time_no_duration(time: str | list[str] | None) -> None: """Raise ValueError if ``time`` contains an ISO 8601 duration.""" if time is None: return - if any(_DURATION_RE.match(str(v)) for v in _as_list(time)): + # ``_coerce_to_list`` keeps a lone ``date``/``Timestamp`` whole, where + # ``_as_list`` would try to iterate it. + values = _coerce_to_list(time, "time") + if any(isinstance(v, str) and _DURATION_RE.match(v) for v in values): raise ValueError( "ISO 8601 durations (e.g. 'P7D') are not supported in `time` " "for the rating-curve service. Provide a date or interval instead." diff --git a/tests/waterdata_ratings_test.py b/tests/waterdata_ratings_test.py index f0c96967..cdae9538 100644 --- a/tests/waterdata_ratings_test.py +++ b/tests/waterdata_ratings_test.py @@ -190,6 +190,28 @@ def test_get_ratings_keeps_a_dotdot_open_bound_in_time(httpx_mock): assert params["datetime"] == ["2026-04-29T00:00:00Z/.."] +@pytest.mark.parametrize( + "time", + [ + pd.Timestamp("2026-04-29", tz="UTC"), + [pd.Timestamp("2026-04-29", tz="UTC"), None], + ], + ids=["lone", "range"], +) +def test_get_ratings_accepts_a_timestamp_time(httpx_mock, time): + """The duration check iterated a lone ``Timestamp`` and raised + ``TypeError`` before the date formatter could read it.""" + httpx_mock.add_response( + method="GET", url=STAC_SEARCH_RE, json=_stub_search_response() + ) + get_ratings( + monitoring_location_id="USGS-01104475", time=time, download_and_parse=False + ) + (request,) = httpx_mock.get_requests() + datetime_param = parse_qs(urlsplit(str(request.url)).query)["datetime"][0] + assert datetime_param.startswith("2026-04-29T00:00:00Z") + + def test_get_ratings_rejects_an_unreadable_time_before_any_request(httpx_mock): """An unreadable bound used to drop the ``datetime`` filter, so the search silently returned every rating regardless of date.""" diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 157970e0..43844d73 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1037,7 +1037,7 @@ def test_format_api_dates_names_the_callers_parameter(): ``datetime_input`` -- a local of this private helper that no public getter accepts, so correcting the argument the message named sent an unrecognized parameter.""" - with pytest.raises(TypeError, match="^time must be a string"): + with pytest.raises(TypeError, match="^time must be a date, a string"): _format_api_dates({"2024-01-01": "ignored"}, name="time") @@ -1045,7 +1045,9 @@ def test_format_api_dates_rejects_mapping(): """`time={"2024-01-01": "x"}` would materialize as the keys list, accepting input the user clearly didn't intend. """ - with pytest.raises(TypeError, match="date input must be a string or sequence"): + with pytest.raises( + TypeError, match="date input must be a date, a string, or a sequence" + ): _format_api_dates({"2024-01-01": "ignored"}) From 6647ff5816c8792545e8274976dcacc664bd57c6 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 08:15:12 -0500 Subject: [PATCH 04/22] fix(ogc): reject a duration side in an interval string The Water Data and NGWMN services answer '2024-01-01/P7D' and 'P7D/2024-01-08' with HTTP 400, so passing them through only deferred the failure to an error that names no argument. _split_interval now rejects them and hands the two sides to the list path, which removes _format_interval's duration bookkeeping and the one-line _is_passthrough wrapper. Tighten the NEWS entry. --- NEWS.md | 2 +- dataretrieval/ogc/dates.py | 49 ++++++++++++----------------------- tests/waterdata_utils_test.py | 22 ++++++---------- 3 files changed, 26 insertions(+), 47 deletions(-) diff --git a/NEWS.md b/NEWS.md index 18e2dc12..3c7e24e9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -**10/09/2026:** **Behavior change:** a date argument given as one `"start/end"` string -- `time="2024-01-01T10:00:00/.."`, the half-bounded form shown in the getter docstrings -- is now read side by side like the two-value list form `["2024-01-01T10:00:00", None]`, for the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument) and `waterdata.get_ratings()`. **Bug fix:** the string used to be sent unchanged, so the same range spelled two documented ways could select different data: a naive time was not converted from local time to UTC (off by 7 hours in Denver in January), a time with an offset was not converted to `Z`, and a date-only collection such as `daily` received times. A side that cannot be read as a date -- `time="not-a-date/also-bad"` or `"2024-01-01/tomorrow"` -- now raises `ValueError` naming the argument before any request is sent, as a bad list element already does, and so does a string that is not two sides separated by one `/`, or that pairs a duration with an open end or a second duration. An empty side is an open bound like `..` (the service rejects `"2024-01-01T00:00:00Z/"`), and `"../.."` means no date filter, as `[None, None]` does. A lone duration such as `"P7D"` is unchanged, and one side may still be a duration paired with a date (`"2024-01-01/P7D"`), which is sent as before. +**10/09/2026:** **Behavior change:** a date range passed as one `"start/end"` string, such as `time="2024-01-01T10:00:00/.."`, is now formatted like the list form `["2024-01-01T10:00:00", None]`, in the `waterdata` and `ngwmn` OGC getters and `waterdata.get_ratings()`. **Bug fix:** the string used to be sent unchanged, so the two spellings could select different data: a naive time was not converted from local time to UTC, an offset was not converted to `Z`, and a date-only collection such as `daily` received times. A side that is not a date (`time="2024-01-01/tomorrow"`) now raises `ValueError` before any request is sent, as does a string with more than one `/` or with a duration as either side (`"2024-01-01/P7D"`), which the services answered with HTTP 400. An empty side is an open bound, and `"../.."` means no date filter. A lone duration such as `"P7D"` is unchanged. **10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter. diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 65e1a52a..71def011 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -115,40 +115,26 @@ def _coerce_to_list( return list(datetime_input) -def _is_passthrough(single: str) -> bool: - """True when a single-element input should be returned as-is.""" - return bool(_DURATION_RE.match(single)) +def _split_interval(interval: str, *, name: str) -> list[str | None]: + """Split a ``"start/end"`` string into the two-value list form. - -def _format_interval(interval: str, *, date: bool, name: str) -> str | None: - """Format each side of a pre-formatted ``"start/end"`` interval string. - - Each side is formatted like an element of the two-value list form, so + Each side is then formatted like a list element, so ``"2024-01-01T10:00:00/.."`` sends the same range as - ``["2024-01-01T10:00:00", None]``. One side may instead be an ISO 8601 - duration paired with an instant (``"2024-01-01/P7D"``), which is kept - unchanged. Returns None when both sides are open. + ``["2024-01-01T10:00:00", None]``. - Raises ``ValueError`` naming *name* when the string is not two sides - separated by one ``"/"``, or a side cannot be read. + Raises ``ValueError`` naming *name* unless the string is two sides + separated by one ``"/"``, neither of them a duration: the Water Data and + NGWMN services answer ``"2024-01-01/P7D"`` and ``"P7D/2024-01-08"`` with + HTTP 400. """ sides = interval.split("/") - durations = [bool(_DURATION_RE.match(side)) for side in sides] - if len(sides) != 2 or ( - any(durations) and (all(durations) or any(_is_blank(s) for s in sides)) - ): + if len(sides) != 2 or any(_DURATION_RE.match(side) for side in sides): raise ValueError( f"{name} is not a valid interval: {interval!r}. Pass a start and " "an end separated by '/', such as '2024-01-01/2024-12-31', with " "'..' for an open end ('2024-01-01/..')." ) - formatted = [ - side if is_duration else _format_one(side, date=date, name=name) - for side, is_duration in zip(sides, durations, strict=True) - ] - if formatted == [_OPEN_BOUND, _OPEN_BOUND]: - return None - return "/".join(formatted) + return [sides[0], sides[1]] def _all_blank(items: list[str | None]) -> bool: @@ -215,9 +201,9 @@ def _format_api_dates( is blank/NA/``".."``. - Supports ISO 8601 durations such as "P7D" and "PT36H", which are passed through unchanged. - - A single string containing ``"/"`` is an interval: each side is formatted - like an element of the two-value form, and may also be an ISO 8601 - duration paired with an instant (``"2024-01-01/P7D"``). + - A single string containing ``"/"`` is an interval: it is split into the + two-value form, so ``"2024-01-01/.."`` and ``["2024-01-01", None]`` are + formatted alike. A duration is not accepted as either side. - Converts datetimes to UTC and formats as ISO 8601 with 'Z' suffix when `date` is False. Inputs with an explicit offset (``Z`` or ``+HH:MM``) are converted from that offset to UTC; naive inputs are interpreted in the @@ -227,6 +213,8 @@ def _format_api_dates( return None items = _coerce_to_list(datetime_input, name) + if len(items) == 1 and isinstance(items[0], str) and "/" in items[0]: + items = _split_interval(items[0], name=name) if _all_blank(items): return None @@ -238,11 +226,8 @@ def _format_api_dates( "or two for a closed interval ('2020-01-01', '2020-12-31')." ) - if len(items) == 1 and isinstance(items[0], str) and "/" in items[0]: - return _format_interval(items[0], date=date, name=name) - - # Pass through duration ("P7D", "PT36H") - if len(items) == 1 and isinstance(items[0], str) and _is_passthrough(items[0]): + # Pass through a duration ("P7D", "PT36H") + if len(items) == 1 and isinstance(items[0], str) and _DURATION_RE.match(items[0]): return items[0] return "/".join(_format_one(dt, date=date, name=name) for dt in items) diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 02a847d4..80218d5d 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1023,21 +1023,14 @@ def test_format_api_dates_formats_an_interval_string_like_the_pair( "interval, date, expected", [ ("2018-02-12T19:20:50-04:00/..", False, "2018-02-12T23:20:50Z/.."), - ( - "2024-01-01T10:00:00Z/2024-02-01T00:00:00Z", - True, - "2024-01-01/2024-02-01", - ), - ("2024-01-01T10:00:00+02:00/PT36H", False, "2024-01-01T08:00:00Z/PT36H"), - ("P7D/2024-01-08T00:00:00Z", True, "P7D/2024-01-08"), + ("2024-01-01T10:00:00Z/2024-02-01T00:00:00Z", True, "2024-01-01/2024-02-01"), ], - ids=["offset_to_utc", "date_only", "start_duration", "duration_end"], + ids=["offset_to_utc", "date_only"], ) def test_format_api_dates_formats_each_side_of_an_interval_string( interval, date, expected ): - """Each side is formatted on its own; an ISO 8601 duration paired with an - instant is kept unchanged.""" + """Each side is formatted on its own, as a list element is.""" assert _format_api_dates(interval, date=date) == expected @@ -1052,17 +1045,18 @@ def test_format_api_dates_treats_an_all_open_interval_string_as_no_filter(): ("not-a-date/also-bad", "time could not be read as a date or datetime"), ("2024-01-01/tomorrow", "time could not be read as a date or datetime"), ("2024-01-01/2024-02-01/2024-03-01", "time is not a valid interval"), - ("P7D/P1D", "time is not a valid interval"), + # The Water Data and NGWMN services answer a duration side with HTTP 400. + ("2024-01-01T10:00:00Z/PT36H", "time is not a valid interval"), + ("P7D/2024-01-08", "time is not a valid interval"), ("P7D/..", "time is not a valid interval"), - ("../P7D", "time is not a valid interval"), ], ids=[ "both_sides", "one_side", "three_sides", - "two_durations", + "start_duration", + "duration_end", "duration_open_end", - "open_start_duration", ], ) def test_format_api_dates_rejects_an_unreadable_interval_string(interval, message): From 638374d67143d639149e830e8a1cffd3061b2780 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 08:16:57 -0500 Subject: [PATCH 05/22] refactor(ogc): return the interval sides as a tuple A tuple keeps _split_interval assignable to items once #442 widens the element type, where a list[str | None] would trip list invariance. --- dataretrieval/ogc/dates.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 71def011..c57a48d4 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -115,8 +115,8 @@ def _coerce_to_list( return list(datetime_input) -def _split_interval(interval: str, *, name: str) -> list[str | None]: - """Split a ``"start/end"`` string into the two-value list form. +def _split_interval(interval: str, *, name: str) -> tuple[str, str]: + """Split a ``"start/end"`` string into its two sides. Each side is then formatted like a list element, so ``"2024-01-01T10:00:00/.."`` sends the same range as @@ -134,7 +134,7 @@ def _split_interval(interval: str, *, name: str) -> list[str | None]: "an end separated by '/', such as '2024-01-01/2024-12-31', with " "'..' for an open end ('2024-01-01/..')." ) - return [sides[0], sides[1]] + return sides[0], sides[1] def _all_blank(items: list[str | None]) -> bool: @@ -214,7 +214,7 @@ def _format_api_dates( items = _coerce_to_list(datetime_input, name) if len(items) == 1 and isinstance(items[0], str) and "/" in items[0]: - items = _split_interval(items[0], name=name) + items = list(_split_interval(items[0], name=name)) if _all_blank(items): return None From 5125381cf8a93e2bc0620222ce4aae875e88d68a Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 08:21:16 -0500 Subject: [PATCH 06/22] fix(ogc): say in the interval error that a side cannot be a duration --- dataretrieval/ogc/dates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index c57a48d4..1e173354 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -132,7 +132,8 @@ def _split_interval(interval: str, *, name: str) -> tuple[str, str]: raise ValueError( f"{name} is not a valid interval: {interval!r}. Pass a start and " "an end separated by '/', such as '2024-01-01/2024-12-31', with " - "'..' for an open end ('2024-01-01/..')." + "'..' for an open end ('2024-01-01/..'). Neither side can be a " + "duration." ) return sides[0], sides[1] From f6cbbbbea682f7f7d924b9445514399ca8c7760c Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 09:01:13 -0500 Subject: [PATCH 07/22] fix(ogc): spell an open range start the way each API accepts The NGWMN and STAC firewalls answer any datetime containing '../' with HTTP 403, so get_water_level(datetime=[None, end]) and get_ratings(time=[None, end]) never worked; and since #443 formats a '/end' string, the empty-start workaround stopped working too. The Water Data API is the reverse: it accepts '../end' and answers '/end' with HTTP 400. OgcDialect.open_start records the spelling (default '..'); NGWMN sets '', and get_ratings passes '' to _format_api_dates directly since STAC has no dialect. An open end stays '..', which all three accept. --- NEWS.md | 2 +- dataretrieval/ngwmn.py | 11 +++++++---- dataretrieval/ogc/dates.py | 12 +++++++++++- dataretrieval/ogc/policy.py | 7 +++++++ dataretrieval/ogc/requests.py | 1 + dataretrieval/waterdata/ratings.py | 7 ++++++- tests/ngwmn_test.py | 14 ++++++++++++++ tests/waterdata_ratings_test.py | 17 +++++++++++++++++ tests/waterdata_test.py | 9 +++++++++ tests/waterdata_utils_test.py | 27 +++++++++++++++++++++++++++ 10 files changed, 100 insertions(+), 7 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3c7e24e9..2fa2bea4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -**10/09/2026:** **Behavior change:** a date range passed as one `"start/end"` string, such as `time="2024-01-01T10:00:00/.."`, is now formatted like the list form `["2024-01-01T10:00:00", None]`, in the `waterdata` and `ngwmn` OGC getters and `waterdata.get_ratings()`. **Bug fix:** the string used to be sent unchanged, so the two spellings could select different data: a naive time was not converted from local time to UTC, an offset was not converted to `Z`, and a date-only collection such as `daily` received times. A side that is not a date (`time="2024-01-01/tomorrow"`) now raises `ValueError` before any request is sent, as does a string with more than one `/` or with a duration as either side (`"2024-01-01/P7D"`), which the services answered with HTTP 400. An empty side is an open bound, and `"../.."` means no date filter. A lone duration such as `"P7D"` is unchanged. +**10/09/2026:** **Behavior change:** a date range passed as one `"start/end"` string, such as `time="2024-01-01T10:00:00/.."`, is now formatted like the list form `["2024-01-01T10:00:00", None]`, in the `waterdata` and `ngwmn` OGC getters and `waterdata.get_ratings()`. **Bug fix:** the string used to be sent unchanged, so the two spellings could select different data: a naive time was not converted from local time to UTC, an offset was not converted to `Z`, and a date-only collection such as `daily` received times. A side that is not a date (`time="2024-01-01/tomorrow"`) now raises `ValueError` before any request is sent, as does a string with more than one `/` or with a duration as either side (`"2024-01-01/P7D"`), which the services answered with HTTP 400. An empty side is an open bound, and `"../.."` means no date filter. A lone duration such as `"P7D"` is unchanged. **Bug fix:** a range with an open start, such as `[None, "2016-01-01"]` or `"../2016-01-01"`, failed with HTTP 403 in `ngwmn.get_water_level()` and `waterdata.get_ratings()`, because those services' firewalls reject `../`. Both now send an empty start (`"/2016-01-01"`), which both services read as open. The other Water Data getters still send `..`, because that service rejects an empty start. **10/07/2026:** **Behavior change:** a date argument the package cannot read as a date -- a typo such as `time="2024-13-45"`, an unsupported format such as `"Jan 1 2024"`, or one bad end of a range such as `time=["2024-01-01", "tomorrow"]` -- now raises `ValueError` naming the argument and the value, before any request is sent. It used to drop the date filter: `waterdata.get_ratings()` searched with no `datetime` and silently returned every rating, and the OGC getters (`get_daily()`, `get_continuous()`, `ngwmn.get_water_level()`, and every `time`, `begin`, `end`, `datetime`, and `last_modified` argument, plus the deprecated `begin_utc` and `end_utc`) sent an empty `time=` that the service rejected with HTTP 400 "Invalid datetime format", which named no argument. To leave one end of a range open, pass `None` (`["2024-01-01", None]`) or `".."`; the getter docstrings now show this list form. A range whose ends are all open still means no date filter. diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index 7fc351ba..a5e3d998 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -80,10 +80,12 @@ # (there is no service-specific id name as there is for the main collections). _NGWMN_OUTPUT_ID = "id" -# NGWMN's request shape matches the generic OGC default (no CQL2-only or -# date-only collections), but its result columns need their own coercion and -# sort vocabulary: water-level observations are timestamped by ``sample_time`` -# (not the Water Data ``time``) and report depths/levels in feet. +# NGWMN has no CQL2-only or date-only collections, but its result columns need +# their own coercion and sort vocabulary: water-level observations are +# timestamped by ``sample_time`` (not the Water Data ``time``) and report +# depths/levels in feet. Its firewall answers ``datetime=../2016-01-01`` with +# HTTP 403, so an open start is sent empty (``/2016-01-01``), which it reads as +# open. NGWMN_DIALECT = OgcDialect( time_cols=frozenset({"sample_time"}), numerical_cols=frozenset( @@ -94,6 +96,7 @@ } ), sort_cols=("sample_time", "monitoring_location_id"), + open_start="", ) diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 1e173354..3e6c2236 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -149,6 +149,7 @@ def _format_api_dates( *, name: str = "date input", single_value_hint: str = "an instant or a duration ('2020-01-01', 'P7D')", + open_start: str = _OPEN_BOUND, ) -> str | None: """ Formats date or datetime input(s) for use with an API. @@ -178,6 +179,12 @@ def _format_api_dates( forms (``get_ratings`` rejects durations) enforces that itself and passes a hint naming only what it accepts, so the remedy does not direct a caller to a value that is rejected. + open_start : str, optional + How the target API spells the open start of a range (``".."`` + by default). OGC API Features and STAC also allow an empty string, + ``"/2024-01-01"``, which an API whose firewall rejects ``"../"`` needs + instead; see :attr:`OgcDialect.open_start`. An open end is always + ``".."``. Returns ------- @@ -231,4 +238,7 @@ def _format_api_dates( if len(items) == 1 and isinstance(items[0], str) and _DURATION_RE.match(items[0]): return items[0] - return "/".join(_format_one(dt, date=date, name=name) for dt in items) + formatted = [_format_one(dt, date=date, name=name) for dt in items] + if len(formatted) == 2 and formatted[0] == _OPEN_BOUND: + formatted[0] = open_start + return "/".join(formatted) diff --git a/dataretrieval/ogc/policy.py b/dataretrieval/ogc/policy.py index c92c7da1..807fd48a 100644 --- a/dataretrieval/ogc/policy.py +++ b/dataretrieval/ogc/policy.py @@ -60,6 +60,12 @@ class OgcDialect: Columns to sort the combined result by, in priority order. Sorting is applied only when the first (primary) column is present; any later columns also present are added as secondary keys. + open_start : str + How the API spells the open start of a date range: ``".."`` + (``"../2024-01-01"``, the default) or ``""`` (``"/2024-01-01"``). + OGC API Features allows both, but deployments differ in which they + accept: the Water Data API rejects the empty start with HTTP 400, and + the NGWMN firewall rejects ``"../"`` with HTTP 403. """ cql2_services: frozenset[str] = field(default_factory=frozenset) @@ -67,6 +73,7 @@ class OgcDialect: time_cols: frozenset[str] = field(default_factory=frozenset) numerical_cols: frozenset[str] = field(default_factory=frozenset) sort_cols: tuple[str, ...] = field(default_factory=tuple) + open_start: str = ".." # Default dialect: a plain OGC API with no CQL2-only collections and no diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 1427dc38..87e032ed 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -184,6 +184,7 @@ def _construct_api_requests( date=( collection in dialect.date_only_services and key != "last_modified" ), + open_start=dialect.open_start, ) params, post_params = _partition_request_params( kwargs, use_cql2=collection in dialect.cql2_services diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index f78ea2fc..6ceb2a53 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -179,7 +179,12 @@ def get_ratings( _validate_time_no_duration(time) time_str = ( _format_api_dates( - time, name="time", single_value_hint="an instant ('2020-01-01')" + time, + name="time", + single_value_hint="an instant ('2020-01-01')", + # The STAC firewall answers ``datetime=../2026-01-01`` with HTTP + # 403; STAC reads an empty start (``/2026-01-01``) as open. + open_start="", ) if time is not None else None diff --git a/tests/ngwmn_test.py b/tests/ngwmn_test.py index d73ae0f2..deeabb7a 100644 --- a/tests/ngwmn_test.py +++ b/tests/ngwmn_test.py @@ -460,6 +460,20 @@ def test_get_water_level_datetime_subsets(httpx_mock): assert qs["datetime"] == ["2022-01-01T00:00:00Z/2024-01-01T00:00:00Z"] +@pytest.mark.parametrize( + "datetime", [[None, "2016-01-01T00:00:00Z"], "../2016-01-01T00:00:00Z"] +) +def test_get_water_level_sends_an_open_start_empty(httpx_mock, datetime): + """The NGWMN firewall answers ``datetime=../2016-01-01`` with HTTP 403, so + an open start is sent as an empty string, which NGWMN reads as open.""" + _mock(httpx_mock, "waterLevelObs", _WATER_LEVELS) + + ngwmn.get_water_level(monitoring_location_id=_SITE, datetime=datetime) + + qs = _queries(httpx_mock, "waterLevelObs")[0] + assert qs["datetime"] == ["/2016-01-01T00:00:00Z"] + + def test_get_lithology(httpx_mock): _mock(httpx_mock, "lithologyObs", _LITHOLOGY) diff --git a/tests/waterdata_ratings_test.py b/tests/waterdata_ratings_test.py index f0c96967..aeedff0e 100644 --- a/tests/waterdata_ratings_test.py +++ b/tests/waterdata_ratings_test.py @@ -169,6 +169,23 @@ def test_get_ratings_download_and_parse_false_returns_features(httpx_mock): assert features[0]["id"] == "USGS-01104475.exsa.rdb" +@pytest.mark.parametrize( + "time", [[None, "2026-04-29T00:00:00Z"], "../2026-04-29T00:00:00Z"] +) +def test_get_ratings_sends_an_open_start_empty(httpx_mock, time): + """The STAC firewall answers ``datetime=../2026-04-29`` with HTTP 403, so an + open start is sent as an empty string, which STAC reads as open.""" + httpx_mock.add_response( + method="GET", url=STAC_SEARCH_RE, json=_stub_search_response() + ) + get_ratings( + monitoring_location_id="USGS-01104475", time=time, download_and_parse=False + ) + (request,) = httpx_mock.get_requests() + params = parse_qs(urlsplit(str(request.url)).query) + assert params["datetime"] == ["/2026-04-29T00:00:00Z"] + + def test_get_ratings_keeps_a_dotdot_open_bound_in_time(httpx_mock): """``time=[start, ".."]`` is the documented open-ended range. The ``..`` endpoint used to fail to parse, which discarded the whole range, so the diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index c1fee7c6..966e0505 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -604,6 +604,15 @@ def test_get_args_materializes_numpy_and_series_numeric_params(): assert "bbox=-92.8%2C44.2%2C-88.9%2C46.0" in str(req.url) +def test_construct_api_requests_keeps_dotdot_for_an_open_start(): + """The Water Data API answers an empty start (``time=/2024-01-31``) with + HTTP 400, so its dialect keeps the default ``..``.""" + req = _construct_api_requests( + "daily", monitoring_location_id="USGS-05427718", time=[None, "2024-01-31"] + ) + assert "time=..%2F2024-01-31" in str(req.url) + + def test_construct_api_requests_two_element_date_list_becomes_interval(): """A two-element date list is interpreted as start/end of an OGC datetime interval (joined with '/'), NOT as two discrete dates. The OGC `datetime` diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 80218d5d..6c22f040 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1039,6 +1039,33 @@ def test_format_api_dates_treats_an_all_open_interval_string_as_no_filter(): assert _format_api_dates("../..") is None +@pytest.mark.parametrize( + "value", + [ + [None, "2024-01-01T00:00:00Z"], + ["..", "2024-01-01T00:00:00Z"], + "../2024-01-01T00:00:00Z", + "/2024-01-01T00:00:00Z", + ], + ids=["none", "dotdot", "dotdot_string", "empty_string"], +) +def test_format_api_dates_spells_an_open_start_as_the_api_does(value): + """Every spelling of an open start becomes ``..`` by default and the + caller's ``open_start`` otherwise: NGWMN and STAC reject ``"../"`` (HTTP + 403), and the Water Data API rejects an empty start (HTTP 400).""" + assert _format_api_dates(value) == "../2024-01-01T00:00:00Z" + assert _format_api_dates(value, open_start="") == "/2024-01-01T00:00:00Z" + + +def test_format_api_dates_open_start_leaves_an_open_end_and_no_filter_alone(): + """``open_start`` changes only the start: an open end is ``..`` on every API, + and a range with both ends open is still no filter.""" + open_end = _format_api_dates(["2024-01-01T00:00:00Z", None], open_start="") + assert open_end == "2024-01-01T00:00:00Z/.." + assert _format_api_dates([None, None], open_start="") is None + assert _format_api_dates("../..", open_start="") is None + + @pytest.mark.parametrize( "interval, message", [ From cc0cead37591a9ac66dbdae3f09cdd7b97193ad0 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 12:55:09 -0500 Subject: [PATCH 08/22] docs(adr): propose 0014, declare services, endpoints, and dialects as values Replaces the OGC-only proposal in #444. Six abstractions (service, endpoint, protocol, dialect, execution, adapter), the rule that shared code names no service, dates read once and written per protocol, and values-not-clients. Proposed and revised as each implementation step lands. Adds the Service declaration, Endpoint, and Protocol glossary entries and restates Adapter and Dialect. --- CONTEXT.md | 35 ++- .../0014-services-endpoints-dialects.rst | 220 ++++++++++++++++++ docs/source/architecture/decisions/index.rst | 1 + 3 files changed, 247 insertions(+), 9 deletions(-) create mode 100644 docs/source/architecture/decisions/0014-services-endpoints-dialects.rst diff --git a/CONTEXT.md b/CONTEXT.md index f76a9f7e..5450ca68 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -253,15 +253,32 @@ value. Where the distinction matters — reporting what a call will use ## Boundaries -**Adapter** — A module owning one service's conventions: its URLs, parameters, -error shapes, and response quirks. Adapters may use shared code; shared code -may not reference adapters. - -**Dialect** — The per-API quirks the shared OGC code needs in order to -serve two services from one code path: which collections must be POSTed as -CQL2, which render dates date-only, which columns to coerce and sort by. An -adapter supplies one and the shared code reads it, which is how protocol code -stays free of service names. +**Adapter** — A module owning one service's conventions: its getters, public +parameters, and their translation, error shapes, and response quirks. Adapters +may use shared code; shared code may not reference adapters. An adapter +declares its service, the endpoints it calls, and their dialects as values, +and hands those to shared code (ADR 0014). + +**Service declaration** — The value an adapter declares for its service: the +name configuration resolves its settings by, and the URL a `base_url` setting +replaces. The one place that name is written outside the configuration +roster. In code, `Service`. + +**Endpoint** — One API of a service, at one URL, speaking one protocol. Water +Data is one service with four endpoints: OGC API Features, the ratings STAC +catalog, Samples, and Statistics. An endpoint resolves its URL once when a call +starts and carries its dialect. + +**Protocol** — The rules for talking to one kind of standard API, implemented +once as functions that receive an endpoint: OGC API Features, or the ISO 8601 +interval grammar OGC and STAC share. A protocol gets its own module only when +two endpoints speak it. + +**Dialect** — How one endpoint departs from its protocol's standard, as data +the protocol code reads. Each field is one extension, restriction, or renaming, +and its default is the standard, so a default dialect describes an endpoint +that follows the standard. Fields are grouped by the code that reads them; the +date group, `DateDialect`, is shared by the protocols that write intervals. **Single-shot adapter** — An adapter whose query is always exactly one request: WQP, NLDI, StreamStats, and deprecated NWIS. Nothing divides and nothing diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst new file mode 100644 index 00000000..0fc70b96 --- /dev/null +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -0,0 +1,220 @@ +ADR 0014: Declare services, endpoints, and dialects as values +============================================================= + +Status +------ + +Proposed. This record is revised as its implementation lands, one pull request +per step; ``Notes`` records each revision. It is accepted when the last step +merges and the record describes what was built. + +Context +------- + +An adapter tells shared code about its service in three unconnected ways. + +**A name.** Seven adapters pass a string such as ``adapter="waterdata"`` to +configuration, retry policy, pagination, and fan-out -- about forty call sites, +each spelling the name again. The same name is the key of the adapter's table +in the configuration file and of its ``Configuration`` class. + +**A URL.** Each adapter resolves its base URL from configuration in its own +way, and Water Data alone has four APIs under one root: OGC API Features, a +STAC catalog (ratings), Samples, and Statistics. Nothing in the code says these +are four endpoints of one service. + +**Hard-coded facts.** The shared OGC code lists Water Data's parameter names as +"the OGC date parameters", exempts ``last_modified`` by name from date-only +rendering, passes durations through for every service because one accepts +them, and assumes every service spells an open range start ``..``. ADR 0003 +keeps shared code from *importing* a service; it says nothing about *naming* +one, and a string literal is not an import. The services differ: + +.. list-table:: + :header-rows: 1 + + * - Behavior (observed 2026-10-11) + - Water Data OGC + - NGWMN + - Water Data STAC + * - Date parameters + - ``time``, ``begin``, ``end``, ``last_modified``, ``datetime`` + - ``datetime`` + - ``datetime`` + * - A lone duration, ``P7D`` + - accepted + - HTTP 400 + - HTTP 400 + * - An open start, ``../2016-01-01`` + - accepted + - HTTP 403 from the firewall + - HTTP 403 from the firewall + * - An empty start, ``/2016-01-01`` + - HTTP 400 (500 on ``continuous``) + - accepted + - accepted + +Pull request #443 formatted interval strings to the one open-start spelling the +OGC code knew, which broke those requests on NGWMN and STAC. Its review missed +it because no rule said which facts belong to a service. + +Dates are also read and written in one function. Reading what a caller passed +(a string, a ``date``, a ``Timestamp``, ``None``, local-time rules) is package +policy; writing it as an ISO 8601 interval is protocol grammar. Every date +defect fixed in #440 through #443 was where those two meet. + +Decision +-------- + +Six abstractions, each answering one question: + +.. list-table:: + :header-rows: 1 + + * - Abstraction + - Answers + - Kind + * - **Service** + - Which external system: its name, and the URL its ``base_url`` setting + replaces. + - a value + * - **Endpoint** + - Which API of that service, at what URL, speaking which protocol with + which dialect. + - a value + * - **Protocol** + - How to talk to a standard API: request grammar, paging, filters, + chunking. + - functions + * - **Dialect** + - Where one endpoint departs from its protocol's standard. + - a value, one type per protocol + * - **Execution** + - How requests run: retry, fan-out, progress, quota (ADR 0006). + - functions + * - **Adapter** + - What the public sees: getters, parameter names, translation, docs. + - a module + +**An adapter declares one** ``Service`` **.** It holds the adapter's name and +its default URL, and it is the only place the adapter writes either. +Configuration and execution keep resolving settings by the service's name, +because that is the key a configuration file uses; they receive it from the +declaration, never as a literal. + +**Each API an adapter calls is an** ``Endpoint`` **of its service.** An +endpoint names its service, resolves its URL from configuration once when a +call starts -- so every chunk and a later resume use the same URL -- and carries +the dialect of its protocol. Water Data is one service with four endpoints. + +**A protocol is implemented as the standard is written.** Protocol code is +functions that receive an endpoint. Each dialect field records one way an +endpoint *extends* the standard (Water Data accepts a lone duration), +*restricts* it (NGWMN refuses ``../``), or *renames* it (Water Data filters on +``time``), and its default is the standard. A default dialect therefore +describes an endpoint that follows the standard. A field's docstring records the +observed server behavior that justifies it, and when it was observed. + +**Shared code names no service.** Nothing a ``Service``, ``Endpoint``, or +dialect declares -- a name, a parameter, a collection, a column -- appears as a +literal in protocol or execution code; that code reads it from the declaration. +A protocol's own standard names (``datetime``, ``bbox``, ``limit``) are not +deviations. A comment may cite a service as evidence for why the code tolerates +a variation; the behavior may not depend on which service it is. + +**Dialect fields are grouped by the code that reads them.** OGC's dialect has a +date group, a request-construction group, and a result-shaping group. A group is +created when one part of the code reads several fields together. + +**A date range is read once, at the public boundary, and written by each +protocol.** One package-wide value holds what the caller meant. The ISO 8601 +interval grammar shared by OGC and STAC is written by one module, adjusted by a +``DateDialect``; another protocol writes the value in its own format. + +**Values are data; protocols and execution are functions.** ``Service``, +``Endpoint``, and the dialect types are frozen dataclasses. They may resolve a +URL from configuration; they do not send requests, and nothing subclasses them +or a protocol's code. A method that sends requests, or an override, could +change behavior for one service without declaring a field, which is the problem +this record exists to prevent. + +**Facts about one collection travel with the call.** Which column the wire +``id`` is renamed to, and whether a collection has geometry, are arguments of +each call, not fields of an endpoint: reference tables and ``get_cql`` reach +collections outside any adapter's map. + +**A protocol module exists only when two endpoints share the protocol.** OGC +API Features has two (Water Data, NGWMN). The interval grammar has three (those +two and STAC). WQP, NLDI, NWDC, and StreamStats keep their request code in their +adapters until a second endpoint shares it. + +Consequences +------------ + +- Adding a service means declaring values, not editing shared code, and a new + endpoint of an existing protocol starts from the standard's behavior. +- Everything shared code knows about a service is in one place, next to the + evidence for each deviation. +- Inside ``ogc``, one endpoint argument replaces the ``base_url``, ``dialect``, + and ``adapter`` arguments that are passed together today. +- Resolving a URL is the one behavior a value has. It only reads + configuration, so it cannot change what protocol code does with the URL. +- A deviation the protocol code does not model needs a new field first. That + costs one field and its default, deliberately. +- Behavior change, when the date step lands: a lone duration sent to NGWMN + raises ``ValueError`` before sending instead of returning HTTP 400. + +Alternatives rejected: + +- *Per-service conditionals in shared code.* The current state. +- *A client class per service, or methods such as* ``WATERDATA.get_data()``. + A method that sends requests puts the protocol behind the value, which would + make the module that defines dialects import the OGC engine -- forbidden for + the modules that need the type -- and invites per-service overrides. +- *One OGC-specific service object holding name, URL, and dialect.* Name and + default URL are not OGC-specific: every adapter has them. One service can have + endpoints in several protocols. +- *A flat dialect.* The date fields are read together and are used without the + rest by STAC. +- *Probing a service at runtime for what it accepts.* That adds a request per + call to learn a fact that changes rarely, and some of these behaviors come + from a firewall the API does not describe. + +Compliance +---------- + +``tests/architecture_test.py`` fails if a service's name appears as a string +literal anywhere in the package except the configuration roster and that +service's declaration. Once endpoints and dialects land, it also fails if a +parameter, collection, or column a shipped dialect lists appears as a literal +in protocol code. Docstrings and comments are excluded. + +Each dialect field that records a server behavior has a ``@pytest.mark.live`` +test that sends the request the field prevents and the request it produces, so +the scheduled live workflow fails when a deployment changes. + +``CONTEXT.md`` defines *Endpoint*, *Protocol*, and *Dialect*, and says how a +*Service* is declared. + +Deviations still in the OGC protocol code, each removed when another endpoint +needs a different value or the code holding it next changes: + +- ``monitoring_location_id`` is validated by name against the ``AGENCY-ID`` form + both OGC services use. +- ``skipGeometry`` is pygeoapi's spelling; the standard has no such parameter. +- ``limit`` is capped at 50000, Water Data's largest page. +- An unquoted number in a CQL text filter raises, because every Water Data + queryable is typed as a string. + +Notes +----- + +Prompted by the review of #442 and #443. Supersedes the narrower proposal in +#444, which bundled name, URL, and dialect into one OGC-specific value. The +behaviors in the Context table were observed by sending each request to each +service on 2026-10-11. + +Implementation steps, each its own pull request: declare a ``Service`` per +adapter; endpoints for the OGC getters; endpoints for the remaining adapters; +the package-wide date range; the shared interval grammar and ``DateDialect``; +the OGC code naming nothing a dialect declares; acceptance. diff --git a/docs/source/architecture/decisions/index.rst b/docs/source/architecture/decisions/index.rst index 29825505..104ad58a 100644 --- a/docs/source/architecture/decisions/index.rst +++ b/docs/source/architecture/decisions/index.rst @@ -32,4 +32,5 @@ belong here rather than in a docstring, an inline comment, or a commit message. 0011-configuration-profiles 0012-deprecation-horizons 0013-core-and-domain-terms + 0014-services-endpoints-dialects template From 1a5933665dd001502b60fb001e723960f6394ff7 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 13:04:42 -0500 Subject: [PATCH 09/22] refactor: declare one Service per adapter (ADR 0014) Each adapter now declares Service(name, default_url) once, in the module that owns its URL, and every read site takes the name from it: the Configuration class's adapter, retry/fan-out/pagination adapter=, and the base URL through configuration.service_url(). Water Data declares one service in waterdata.endpoints for its four APIs. No behavior change. A fitness function fails if a service's name is written as a literal anywhere but its declaration and the configuration roster; the read-site check now follows the declaration. --- .importlinter | 3 + dataretrieval/_service.py | 27 +++++++ dataretrieval/configuration.py | 10 ++- dataretrieval/ngwmn.py | 10 ++- dataretrieval/nldi.py | 10 ++- dataretrieval/nwdc.py | 10 ++- dataretrieval/streamstats.py | 12 ++-- dataretrieval/waterdata/configuration.py | 3 +- dataretrieval/waterdata/endpoints.py | 10 ++- dataretrieval/waterdata/ratings.py | 8 +-- dataretrieval/waterdata/stats.py | 4 +- dataretrieval/waterdata/utils.py | 3 +- dataretrieval/wqp.py | 14 +++- .../0014-services-endpoints-dialects.rst | 11 +++ tests/architecture_test.py | 72 +++++++++++++++++++ tests/configuration_test.py | 39 +++++++--- 16 files changed, 209 insertions(+), 37 deletions(-) create mode 100644 dataretrieval/_service.py diff --git a/.importlinter b/.importlinter index 7dc23e52..7c64bca7 100644 --- a/.importlinter +++ b/.importlinter @@ -38,6 +38,9 @@ layers = credentials configuration _configuration_core +; The service declaration type is a dependency-free leaf, so every layer +; above may accept one (ADR 0014). + _service exceptions ; Pure dependency-free mechanisms sit together at the floor: none imports ; first-party code, so every layer can use them without reaching sideways or diff --git a/dataretrieval/_service.py b/dataretrieval/_service.py new file mode 100644 index 00000000..7eb2873f --- /dev/null +++ b/dataretrieval/_service.py @@ -0,0 +1,27 @@ +"""The value an adapter declares for its service (ADR 0014). + +A dependency-free leaf, so configuration, execution, and protocol code can all +accept a :class:`Service` without importing an adapter. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Service: + """One external system this package retrieves from. + + Attributes + ---------- + name : str + The adapter's name: the key of its configuration table and its + ``Configuration`` class, and the name settings are resolved by. It must + be listed in the configuration roster. + default_url : str + The URL used when no ``base_url`` setting replaces it. + """ + + name: str + default_url: str diff --git a/dataretrieval/configuration.py b/dataretrieval/configuration.py index 6437d789..2eca1c2a 100644 --- a/dataretrieval/configuration.py +++ b/dataretrieval/configuration.py @@ -64,11 +64,14 @@ from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import TextIO, overload +from typing import TYPE_CHECKING, TextIO, overload # Explicit same-name aliases preserve the facade's public and private compatibility # symbols for runtime users and static analyzers. Ruff otherwise expands these # aliases into one import statement each, obscuring the boundary inventory. +if TYPE_CHECKING: + from dataretrieval._service import Service + # isort: off from dataretrieval._configuration_core import ( ADAPTERS as ADAPTERS, @@ -658,6 +661,11 @@ def base_url(*, adapter: str | None = None, default: str | None = None) -> str | return _parse_base_url(raw, label) +def service_url(service: Service) -> str: + """*service*'s base URL: its ``base_url`` setting, or its default.""" + return base_url(adapter=service.name, default=service.default_url) + + @overload def api_version(*, adapter: str | None = ...) -> str | None: ... diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index 7fc351ba..c11394c3 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -24,6 +24,7 @@ import pandas as pd from dataretrieval import configuration as _configuration +from dataretrieval._service import Service from dataretrieval.codes.states import apply_state from dataretrieval.configuration import ( BaseConfiguration, @@ -57,6 +58,9 @@ # separate, unversioned base. NGWMN_OGC_API_URL = f"{BASE_URL}/ngwmn/ogcapi" +#: This adapter's service (ADR 0014). +_SERVICE = Service("ngwmn", NGWMN_OGC_API_URL) + # --- state-filter shim ------------------------------------------------------- # NGWMN's collections expose different state queryables: ``sites`` filters on # the full ``state_name`` (e.g. "Wisconsin"), while ``providers`` filters on the @@ -114,10 +118,10 @@ def _get(service: str, local_vars: dict[str, Any]) -> tuple[pd.DataFrame, BaseMe # A ``NgwmnConfiguration(base_url=...)`` from an enclosing block, or # this service's own base. Read here because this is the one place the # NGWMN base is named (ADR 0011). - base_url=_configuration.base_url(adapter="ngwmn", default=NGWMN_OGC_API_URL), + base_url=_configuration.service_url(_SERVICE), spatial=service == "sites", dialect=NGWMN_DIALECT, - adapter="ngwmn", + adapter=_SERVICE.name, ) @@ -479,7 +483,7 @@ class NgwmnConfiguration( rate-limit quota, so raise it only for pulls you know are large. """ - adapter: ClassVar[str] = "ngwmn" + adapter: ClassVar[str] = _SERVICE.name _register(NgwmnConfiguration) diff --git a/dataretrieval/nldi.py b/dataretrieval/nldi.py index 33b92027..264546a9 100644 --- a/dataretrieval/nldi.py +++ b/dataretrieval/nldi.py @@ -16,6 +16,7 @@ from dataretrieval import configuration as _configuration from dataretrieval._querying import _query_with_retry +from dataretrieval._service import Service from dataretrieval._validation import ( reject_together, render_options, @@ -51,6 +52,9 @@ ) from err NLDI_API_BASE_URL = "https://api.water.usgs.gov/nldi/linked-data" + +#: This adapter's service (ADR 0014). +_SERVICE = Service("nldi", NLDI_API_BASE_URL) _AVAILABLE_DATA_SOURCES = None _CRS = "EPSG:4326" _VALID_NAVIGATION_MODES = ("UM", "DM", "UT", "DD") @@ -73,7 +77,7 @@ def _api_base() -> str: :data:`NLDI_API_BASE_URL` directly, so a ``NldiConfiguration(base_url=...)`` applies to every navigation, basin, and catalog request alike (ADR 0011). """ - return _configuration.base_url(adapter="nldi", default=NLDI_API_BASE_URL) + return _configuration.service_url(_SERVICE) def _query_nldi( @@ -83,7 +87,7 @@ def _query_nldi( # A helper function to query the NLDI API. ``query()`` already raises a typed # ``DataRetrievalError`` for any HTTP error response, so a returned response is a # success that only needs parsing. - response = _query_with_retry(url, payload=query_params, adapter="nldi") + response = _query_with_retry(url, payload=query_params, adapter=_SERVICE.name) response_data: dict[str, Any] | list[Any] = {} try: response_data = response.json() @@ -734,7 +738,7 @@ class registers itself later than the rest -- which is why environment refuse it. """ - adapter: ClassVar[str] = "nldi" + adapter: ClassVar[str] = _SERVICE.name _register(NldiConfiguration) diff --git a/dataretrieval/nwdc.py b/dataretrieval/nwdc.py index a42b8254..f3d06d5d 100644 --- a/dataretrieval/nwdc.py +++ b/dataretrieval/nwdc.py @@ -48,6 +48,7 @@ from dataretrieval._csv import read_code_csv from dataretrieval._querying import _raise_for_status, to_str from dataretrieval._response_metadata import BaseMetadata +from dataretrieval._service import Service from dataretrieval._validation import render_options, require_exactly_one from dataretrieval.codes.states import to_state from dataretrieval.configuration import ( @@ -72,6 +73,9 @@ ] WATERUSE_URL = "https://api.water.usgs.gov/nwaa-data/data" + +#: This adapter's service (ADR 0014). +_SERVICE = Service("nwdc", WATERUSE_URL) _WATERUSE_HOST = httpx.URL(WATERUSE_URL).host # Hosts a ``rel="next"`` cursor may name for this same service; each is # rewritten to :data:`_WATERUSE_HOST` rather than followed as given. @@ -244,7 +248,7 @@ def get_wateruse( # An ``NwdcConfiguration(base_url=...)`` from an enclosing block, or this # service's own endpoint, passed to every request and the page walk # (ADR 0011). - service_url = _configuration.base_url(adapter="nwdc", default=WATERUSE_URL) + service_url = _configuration.service_url(_SERVICE) # The NWDC queries one location per request, so fan a multi-value selector # out into one request per location, each handled by shared transport @@ -380,7 +384,7 @@ def finalize( client_options={"verify": ssl_check}, default_concurrent=DEFAULT_CONCURRENT_REQUESTS, service="nwdc", - adapter="nwdc", + adapter=_SERVICE.name, ) @@ -488,7 +492,7 @@ class NwdcConfiguration(_Concurrent, _Redirectable, _Retrying, BaseConfiguration Cap on simultaneous sub-requests, or ``"unbounded"``. """ - adapter: ClassVar[str] = "nwdc" + adapter: ClassVar[str] = _SERVICE.name _register(NwdcConfiguration) diff --git a/dataretrieval/streamstats.py b/dataretrieval/streamstats.py index f8c14e2c..a99944a4 100644 --- a/dataretrieval/streamstats.py +++ b/dataretrieval/streamstats.py @@ -14,6 +14,7 @@ from dataretrieval import configuration as _configuration from dataretrieval._querying import _get_with_retry +from dataretrieval._service import Service from dataretrieval.configuration import ( BaseConfiguration, _Redirectable, @@ -32,6 +33,9 @@ STREAMSTATS_URL = "https://streamstats.usgs.gov/streamstatsservices" +#: This adapter's service (ADR 0014). +_SERVICE = Service("streamstats", STREAMSTATS_URL) + def _service_base() -> str: """The StreamStats base this call targets: a block's redirect, or its own. @@ -40,7 +44,7 @@ def _service_base() -> str: ``StreamstatsConfiguration(base_url=...)`` redirects the whole service rather than one endpoint (ADR 0011). """ - return _configuration.base_url(adapter="streamstats", default=STREAMSTATS_URL) + return _configuration.service_url(_SERVICE) def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: @@ -65,7 +69,7 @@ def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: payload = {"workspaceID": workspaceID, "format": format} url = f"{_service_base()}/download" - r = _get_with_retry(url, params=payload, adapter="streamstats", **HTTPX_DEFAULTS) + r = _get_with_retry(url, params=payload, adapter=_SERVICE.name, **HTTPX_DEFAULTS) return r # data = r.raw.read() @@ -168,7 +172,7 @@ def get_watershed( } url = f"{_service_base()}/watershed.geojson" - r = _get_with_retry(url, params=payload, adapter="streamstats", **HTTPX_DEFAULTS) + r = _get_with_retry(url, params=payload, adapter=_SERVICE.name, **HTTPX_DEFAULTS) if format == "geojson": return r @@ -264,7 +268,7 @@ class StreamstatsConfiguration(_Redirectable, _Retrying, BaseConfiguration): the file and the environment refuse it. """ - adapter: ClassVar[str] = "streamstats" + adapter: ClassVar[str] = _SERVICE.name _register(StreamstatsConfiguration) diff --git a/dataretrieval/waterdata/configuration.py b/dataretrieval/waterdata/configuration.py index b82bff1d..da54f601 100644 --- a/dataretrieval/waterdata/configuration.py +++ b/dataretrieval/waterdata/configuration.py @@ -20,6 +20,7 @@ _Retrying, _Versioned, ) +from dataretrieval.waterdata.endpoints import SERVICE __all__ = ["WaterdataConfiguration"] @@ -71,7 +72,7 @@ class WaterdataConfiguration( # executed concurrently -- both fan-out settings. Each group declares the # setting itself once, in :mod:`dataretrieval.configuration`, which also # defines its grammar and its coercion. - adapter: ClassVar[str] = "waterdata" + adapter: ClassVar[str] = SERVICE.name _register(WaterdataConfiguration) diff --git a/dataretrieval/waterdata/endpoints.py b/dataretrieval/waterdata/endpoints.py index 978a009f..96205504 100644 --- a/dataretrieval/waterdata/endpoints.py +++ b/dataretrieval/waterdata/endpoints.py @@ -9,8 +9,13 @@ from __future__ import annotations from dataretrieval import configuration as _configuration +from dataretrieval._service import Service from dataretrieval.credentials import WATERDATA_BASE_URL +#: The Water Data service (ADR 0014). Its OGC, Samples, Statistics, and STAC +#: APIs are endpoints below this one root, resolved by the functions below. +SERVICE = Service("waterdata", WATERDATA_BASE_URL) + #: The version of the Water Data OGC API this release requests by default and #: shapes responses for. OGC_API_VERSION = "v1" @@ -34,8 +39,7 @@ def _endpoint(path: str) -> str: """Return *path* beneath the effective Water Data root for this call.""" - root = _configuration.base_url(adapter="waterdata", default=WATERDATA_BASE_URL) - return f"{root}{path}" + return f"{_configuration.service_url(SERVICE)}{path}" def ogc_api_url(*, api_version: str | None = None) -> str: @@ -52,7 +56,7 @@ def ogc_api_url(*, api_version: str | None = None) -> str: """ if api_version is None: api_version = _configuration.api_version( - adapter="waterdata", default=OGC_API_VERSION + adapter=SERVICE.name, default=OGC_API_VERSION ) return _endpoint(f"{_OGC_API_PATH}/{api_version}") diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index f78ea2fc..220a61a1 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -32,7 +32,7 @@ from dataretrieval.transport.links import resolve_next_url from dataretrieval.transport.pagination import run_paginated from dataretrieval.transport.retry import RetryPolicy -from dataretrieval.waterdata.endpoints import ratings_catalog_url +from dataretrieval.waterdata.endpoints import SERVICE, ratings_catalog_url __all__ = ["get_ratings"] @@ -313,7 +313,7 @@ async def follow_up(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: raise_for_status=_raise_for_non_200, client_options={"verify": ssl_check}, service="ratings", - adapter="waterdata", + adapter=SERVICE.name, ) # Every page frame is built with a ``feature`` column, and the combine # helpers preserve it, so the empty case needs no special branch. @@ -438,12 +438,12 @@ async def fetch(feature: dict[str, Any]) -> tuple[pd.DataFrame, httpx.Response]: FanOut( features, fetch, - RetryPolicy.from_configuration(adapter="waterdata"), + RetryPolicy.from_configuration(adapter=SERVICE.name), client_options={"verify": ssl_check}, # No single URL expresses "all of these assets" -- the aggregate # reports the first, matching what a single-feature call would show. canonical_url=_asset_href(features[0]), service="ratings", - adapter="waterdata", + adapter=SERVICE.name, ).resume() return out diff --git a/dataretrieval/waterdata/stats.py b/dataretrieval/waterdata/stats.py index 0087641d..be08b00f 100644 --- a/dataretrieval/waterdata/stats.py +++ b/dataretrieval/waterdata/stats.py @@ -27,7 +27,7 @@ ) from dataretrieval.transport.http import default_headers from dataretrieval.transport.pagination import run_paginated -from dataretrieval.waterdata.endpoints import statistics_api_url +from dataretrieval.waterdata.endpoints import SERVICE, statistics_api_url __all__ = ["get_data"] @@ -282,7 +282,7 @@ async def follow_up(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: raise_for_status=_raise_for_non_200, client=client, service=service, - adapter="waterdata", + adapter=SERVICE.name, ) if expand_percentiles: diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index 13ddadb7..7eb8d46a 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -34,6 +34,7 @@ _DEFAULT_BASE_URL, _DEFAULT_OGC_API_URL, _DEFAULT_SAMPLES_URL, + SERVICE, ogc_api_url, ) @@ -282,7 +283,7 @@ def get_ogc_data( # wrapper every Water Data getter goes through, rather than derived # from ``base_url``: NGWMN is served from the same host, so a URL # does not distinguish the two adapters (ADR 0010). - adapter="waterdata", + adapter=SERVICE.name, ) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index fedbba40..917f51dd 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -17,6 +17,7 @@ from dataretrieval import configuration as _configuration from dataretrieval._csv import read_code_csv from dataretrieval._response_metadata import BaseMetadata +from dataretrieval._service import Service from dataretrieval._validation import require_one_of from dataretrieval.configuration import ( BaseConfiguration, @@ -57,6 +58,9 @@ #: this is only the part they share, and the part a redirect replaces. _WQP_BASE_URL = "https://www.waterqualitydata.us" +#: This adapter's service (ADR 0014). +_SERVICE = Service("wqp", _WQP_BASE_URL) + result_profiles_wqx3 = ["basicPhysChem", "fullPhysChem", "narrow"] result_profiles_legacy = ["biological", "narrowResult", "resultPhysChem"] activity_profiles_legacy = ["activity", "activityAll"] @@ -150,7 +154,11 @@ def _query_wqp( delimiter = None response = _query_with_retry( - url, payload=kwargs, delimiter=delimiter, ssl_check=ssl_check, adapter="wqp" + url, + payload=kwargs, + delimiter=delimiter, + ssl_check=ssl_check, + adapter=_SERVICE.name, ) df = read_code_csv(response.text) # Only get_results documents the appended DateTime columns and the @@ -652,7 +660,7 @@ def _service_base() -> str: different paths, so a ``WqpConfiguration(base_url=...)`` names that root and both use it (ADR 0011). """ - return _configuration.base_url(adapter="wqp", default=_WQP_BASE_URL) + return _configuration.service_url(_SERVICE) def wqp_url(service: str) -> str: @@ -832,7 +840,7 @@ class WqpConfiguration(_Redirectable, _Retrying, BaseConfiguration): the environment refuse it. """ - adapter: ClassVar[str] = "wqp" + adapter: ClassVar[str] = _SERVICE.name _register(WqpConfiguration) diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index 0fc70b96..7bbf6e74 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -218,3 +218,14 @@ Implementation steps, each its own pull request: declare a ``Service`` per adapter; endpoints for the OGC getters; endpoints for the remaining adapters; the package-wide date range; the shared interval grammar and ``DateDialect``; the OGC code naming nothing a dialect declares; acceptance. + +Revisions: + +- *Service declarations.* Each adapter declares its ``Service`` in the module + that owns its URL (Water Data's in ``waterdata.endpoints``, shared by its four + APIs). The type is a dependency-free leaf, ``dataretrieval._service``. + Configuration, execution, and the OGC code keep taking the service's *name* + rather than the value: settings are keyed by name, and a value would add + nothing they read. ``configuration.service_url`` resolves a service's URL. The + name check excludes ``__all__``, whose entries name modules that share a + service's spelling. diff --git a/tests/architecture_test.py b/tests/architecture_test.py index c098eb4d..a4c5874a 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -667,3 +667,75 @@ def test_adr_references_resolve_to_a_record() -> None: assert not dangling, "citations name no such decision record:\n " + "\n ".join( dangling ) + + +def _service_declarations() -> dict[str, list[Path]]: + """Each ``Service("name", ...)`` call in the package, by name.""" + found: dict[str, list[Path]] = {} + for path in sorted(PACKAGE_ROOT.rglob("*.py")): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "Service" + and node.args + and isinstance(node.args[0], ast.Constant) + ): + found.setdefault(node.args[0].value, []).append(path) + return found + + +def test_service_names_are_written_only_where_declared() -> None: + """ADR 0014: an adapter declares its service once, and everything else reads + the name from that declaration. A name written as a literal anywhere else is + a second spelling of the declaration, which is how shared code ends up + naming a service. The configuration roster is the one other place, because + a file table must be validated before its adapter is imported.""" + from dataretrieval._configuration_core import ADAPTERS + + declarations = _service_declarations() + assert sorted(declarations) == sorted(ADAPTERS), ( + "declare exactly one Service per configuration roster entry" + ) + duplicated = {name: paths for name, paths in declarations.items() if len(paths) > 1} + assert not duplicated, f"a service is declared once: {duplicated}" + + roster = PACKAGE_ROOT / "_configuration_core.py" + found = [ + f"{path.relative_to(PACKAGE_ROOT)}:{line} {value!r}" + for path in sorted(PACKAGE_ROOT.rglob("*.py")) + if path != roster + for line, value in _code_string_literals(path) + if value in declarations and path not in declarations[value] + ] + assert not found, f"read the name from the service declaration: {found}" + + +def _code_string_literals(path: Path) -> list[tuple[int, str]]: + """String constants in *path*, excluding docstrings and ``__all__``, whose + entries name modules and symbols rather than services.""" + tree = ast.parse(path.read_text(encoding="utf-8")) + exports = { + id(element) + for node in ast.walk(tree) + if isinstance(node, ast.Assign) + and any(isinstance(t, ast.Name) and t.id == "__all__" for t in node.targets) + for element in ast.walk(node.value) + } + docstrings = exports | { + id(node.body[0].value) + for node in ast.walk(tree) + if isinstance( + node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) + ) + and node.body + and isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, ast.Constant) + } + return [ + (node.lineno, node.value) + for node in ast.walk(tree) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and id(node) not in docstrings + ] diff --git a/tests/configuration_test.py b/tests/configuration_test.py index 01abc7f4..4393626c 100644 --- a/tests/configuration_test.py +++ b/tests/configuration_test.py @@ -1291,17 +1291,24 @@ def test_every_adapter_is_actually_wired_to_a_read_site(): package-wide without an error. """ import pathlib - - source = "\n".join( - p.read_text(encoding="utf-8") - for p in pathlib.Path(configuration.__file__).parent.rglob("*.py") - if p.name != "configuration.py" - ) - missing = [a for a in configuration.ADAPTERS if f'adapter="{a}"' not in source] + import re + + root = pathlib.Path(configuration.__file__).parent + # Each adapter declares its service once (ADR 0014) and reads the name from + # that declaration, so a read site is ``adapter=.name`` in the + # adapter's own module or package. + declared = re.compile(r'(_?SERVICE) = Service\("([a-z]+)"') + missing = list(configuration.ADAPTERS) + for path in root.rglob("*.py"): + for variable, name in declared.findall(path.read_text(encoding="utf-8")): + scope = path.parent.glob("*.py") if path.parent != root else [path] + source = "\n".join(p.read_text(encoding="utf-8") for p in scope) + if f"adapter={variable}.name" in source and name in missing: + missing.remove(name) assert not missing, ( f"adapters with a schema but no read site: {missing}. Either pass " - 'adapter="" where that adapter builds its policy or fan-out, or ' - "drop it from configuration.ADAPTERS." + "adapter=.name where that adapter builds its policy or " + "fan-out, or drop it from configuration.ADAPTERS." ) @@ -2228,3 +2235,17 @@ def test_show_configuration_reports_an_unresolvable_path_as_the_file_row( text = out.getvalue() assert "config file Date: Sun, 11 Oct 2026 13:19:50 -0500 Subject: [PATCH 10/22] fix: review findings on the service declarations Correct the ADR's call-site counts (six adapters, 18 sites in 10 files) and say plainly that execution takes service.name; keep the TYPE_CHECKING import out of configuration's alias inventory; rename nwdc's local service_url, which shadowed the new function; note why Water Data's declaration is package-public; mark DateDialect in the glossary as proposed. --- CONTEXT.md | 3 ++- dataretrieval/configuration.py | 6 +++--- dataretrieval/nwdc.py | 8 ++++---- dataretrieval/waterdata/endpoints.py | 1 + .../decisions/0014-services-endpoints-dialects.rst | 14 ++++++++------ 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 5450ca68..4e26933e 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -278,7 +278,8 @@ two endpoints speak it. the protocol code reads. Each field is one extension, restriction, or renaming, and its default is the standard, so a default dialect describes an endpoint that follows the standard. Fields are grouped by the code that reads them; the -date group, `DateDialect`, is shared by the protocols that write intervals. +date group, `DateDialect`, is meant to be shared by the protocols that write +intervals (ADR 0014, proposed). **Single-shot adapter** — An adapter whose query is always exactly one request: WQP, NLDI, StreamStats, and deprecated NWIS. Nothing divides and nothing diff --git a/dataretrieval/configuration.py b/dataretrieval/configuration.py index 2eca1c2a..bb4b9aee 100644 --- a/dataretrieval/configuration.py +++ b/dataretrieval/configuration.py @@ -69,9 +69,6 @@ # Explicit same-name aliases preserve the facade's public and private compatibility # symbols for runtime users and static analyzers. Ruff otherwise expands these # aliases into one import statement each, obscuring the boundary inventory. -if TYPE_CHECKING: - from dataretrieval._service import Service - # isort: off from dataretrieval._configuration_core import ( ADAPTERS as ADAPTERS, @@ -125,6 +122,9 @@ # isort: on from dataretrieval.exceptions import ConfigurationError as ConfigurationError +if TYPE_CHECKING: + from dataretrieval._service import Service + __all__ = [ "ADAPTERS", # The package-wide configuration, and the base every adapter subclasses. diff --git a/dataretrieval/nwdc.py b/dataretrieval/nwdc.py index f3d06d5d..44e54b15 100644 --- a/dataretrieval/nwdc.py +++ b/dataretrieval/nwdc.py @@ -248,22 +248,22 @@ def get_wateruse( # An ``NwdcConfiguration(base_url=...)`` from an enclosing block, or this # service's own endpoint, passed to every request and the page walk # (ADR 0011). - service_url = _configuration.service_url(_SERVICE) + root = _configuration.service_url(_SERVICE) # The NWDC queries one location per request, so fan a multi-value selector # out into one request per location, each handled by shared transport # pagination, and concatenate the results. - headers = default_headers(service_url) + headers = default_headers(root) requests = [ httpx.Request( "GET", - service_url, + root, params={**base_params, "location": location}, headers=headers, ) for location in _resolve_locations(state, county, huc) ] - return _fan_out(requests, headers, ssl_check, host=httpx.URL(service_url).host) + return _fan_out(requests, headers, ssl_check, host=httpx.URL(root).host) # Valid HUC code lengths (digits) → the hydrologic-unit level they query. diff --git a/dataretrieval/waterdata/endpoints.py b/dataretrieval/waterdata/endpoints.py index 96205504..e3384b62 100644 --- a/dataretrieval/waterdata/endpoints.py +++ b/dataretrieval/waterdata/endpoints.py @@ -14,6 +14,7 @@ #: The Water Data service (ADR 0014). Its OGC, Samples, Statistics, and STAC #: APIs are endpoints below this one root, resolved by the functions below. +#: Public within the package because each of those families reads it. SERVICE = Service("waterdata", WATERDATA_BASE_URL) #: The version of the Water Data OGC API this release requests by default and diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index 7bbf6e74..9f6cb1b8 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -13,10 +13,11 @@ Context An adapter tells shared code about its service in three unconnected ways. -**A name.** Seven adapters pass a string such as ``adapter="waterdata"`` to -configuration, retry policy, pagination, and fan-out -- about forty call sites, -each spelling the name again. The same name is the key of the adapter's table -in the configuration file and of its ``Configuration`` class. +**A name.** Six adapters pass a string such as ``adapter="waterdata"`` to +configuration, retry policy, pagination, and fan-out -- 18 call sites in 10 +files, each spelling the name again, plus the ``adapter`` attribute of each +``Configuration`` class. The same name is the key of the adapter's table in the +configuration file. **A URL.** Each adapter resolves its base URL from configuration in its own way, and Water Data alone has four APIs under one root: OGC API Features, a @@ -99,8 +100,9 @@ Six abstractions, each answering one question: **An adapter declares one** ``Service`` **.** It holds the adapter's name and its default URL, and it is the only place the adapter writes either. Configuration and execution keep resolving settings by the service's name, -because that is the key a configuration file uses; they receive it from the -declaration, never as a literal. +because that is the key a configuration file uses: they take ``service.name``, +read from the declaration, never a literal. Only resolving the URL takes the +value itself. **Each API an adapter calls is an** ``Endpoint`` **of its service.** An endpoint names its service, resolves its URL from configuration once when a From 8a09052360c15d3b2e5a8db82956d0cb5f1b5f34 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 13:33:23 -0500 Subject: [PATCH 11/22] refactor(ogc): reach the OGC getters' API through one Endpoint (ADR 0014) Endpoint(service, url, dialect) is built by the Water Data and NGWMN adapters when a call starts. get_ogc_data, _construct_api_requests, and _finalize_ogc take it in place of the base_url, dialect, and adapter they used to receive separately; settings resolve by endpoint.service.name. DEFAULT_DIALECT, now unused, is removed. Tests: the engine reads URL, dialect, and settings from the endpoint; a call interrupted inside configure(base_url=...) resumes against the same mirror after the block exits. ADR and glossary record that an endpoint holds a resolved URL rather than resolving one. --- CONTEXT.md | 5 +- dataretrieval/_service.py | 28 ++++++++ dataretrieval/ngwmn.py | 13 ++-- dataretrieval/ogc/engine.py | 34 ++++------ dataretrieval/ogc/policy.py | 5 -- dataretrieval/ogc/requests.py | 27 ++++---- dataretrieval/ogc/shaping.py | 19 +++--- dataretrieval/waterdata/utils.py | 17 ++--- .../0014-services-endpoints-dialects.rst | 23 ++++--- tests/configuration_test.py | 11 +++- tests/headers_host_scoping_test.py | 8 ++- tests/waterdata_chunking_test.py | 6 +- tests/waterdata_filters_test.py | 6 +- tests/waterdata_test.py | 5 +- tests/waterdata_utils_test.py | 66 ++++++++++++++++++- 15 files changed, 185 insertions(+), 88 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 4e26933e..fca8a8e6 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -266,8 +266,9 @@ roster. In code, `Service`. **Endpoint** — One API of a service, at one URL, speaking one protocol. Water Data is one service with four endpoints: OGC API Features, the ratings STAC -catalog, Samples, and Statistics. An endpoint resolves its URL once when a call -starts and carries its dialect. +catalog, Samples, and Statistics. The adapter builds an endpoint when a call +starts, so the URL it holds is the one in effect then; it also carries the +dialect. In code, `Endpoint`. **Protocol** — The rules for talking to one kind of standard API, implemented once as functions that receive an endpoint: OGC API Features, or the ISO 8601 diff --git a/dataretrieval/_service.py b/dataretrieval/_service.py index 7eb2873f..a1b5c0ce 100644 --- a/dataretrieval/_service.py +++ b/dataretrieval/_service.py @@ -7,6 +7,7 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Generic, TypeVar @dataclass(frozen=True) @@ -25,3 +26,30 @@ class Service: name: str default_url: str + + +DialectT = TypeVar("DialectT") + + +@dataclass(frozen=True) +class Endpoint(Generic[DialectT]): + """One API of a service, as a call reaches it (ADR 0014). + + An adapter builds one when a call starts, so its URL is the one the + configuration in effect at that moment resolves; every chunk, and a later + resume, use the same value. + + Attributes + ---------- + service : Service + The service this API belongs to; settings resolve by its name. + url : str + The API's base URL for this call. + dialect : DialectT + How this API departs from its protocol's standard. The type is the + protocol's own dialect type. + """ + + service: Service + url: str + dialect: DialectT diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index c11394c3..5e3801d4 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -24,7 +24,7 @@ import pandas as pd from dataretrieval import configuration as _configuration -from dataretrieval._service import Service +from dataretrieval._service import Endpoint, Service from dataretrieval.codes.states import apply_state from dataretrieval.configuration import ( BaseConfiguration, @@ -115,13 +115,12 @@ def _get(service: str, local_vars: dict[str, Any]) -> tuple[pd.DataFrame, BaseMe args, service, output_id=_NGWMN_OUTPUT_ID, - # A ``NgwmnConfiguration(base_url=...)`` from an enclosing block, or - # this service's own base. Read here because this is the one place the - # NGWMN base is named (ADR 0011). - base_url=_configuration.service_url(_SERVICE), + # Built when the call starts, so an enclosing + # ``NgwmnConfiguration(base_url=...)`` applies (ADR 0011, ADR 0014). + endpoint=Endpoint( + _SERVICE, _configuration.service_url(_SERVICE), NGWMN_DIALECT + ), spatial=service == "sites", - dialect=NGWMN_DIALECT, - adapter=_SERVICE.name, ) diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index 0694fc4a..d385e586 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -16,10 +16,9 @@ * ``output_id`` — the user-facing column the wire ``id`` is renamed to, passed explicitly (no collection map is defined here). -* ``base_url`` — the OGC API base to target. +* ``endpoint`` — the API to target, its service, and its + :class:`OgcDialect` (ADR 0014). * ``extra_id_cols`` — synthetic id columns to move to the end of a result. -* ``dialect`` — an :class:`OgcDialect` describing which collections need - POST/CQL2 and which use date-only (vs. full datetime) time arguments. """ from __future__ import annotations @@ -37,7 +36,6 @@ import dataretrieval.ogc.chunking as chunking from dataretrieval.ogc.errors import _raise_for_non_200 from dataretrieval.ogc.policy import ( - DEFAULT_DIALECT, OgcDialect, _require_positive_int, ) @@ -58,6 +56,7 @@ if TYPE_CHECKING: from dataretrieval._response_metadata import BaseMetadata + from dataretrieval._service import Endpoint # Set up logger for this module logger = logging.getLogger(__name__) @@ -219,13 +218,11 @@ def get_ogc_data( collection: str, output_id: str, *, - base_url: str, + endpoint: Endpoint[OgcDialect], spatial: bool, max_rows: int | None = None, extra_id_cols: frozenset[str] | set[str] = frozenset(), - dialect: OgcDialect | None = None, cql_body: str | None = None, - adapter: str | None = None, ) -> tuple[pd.DataFrame, BaseMetadata]: """ Retrieves OGC (Open Geospatial Consortium) data as a DataFrame with metadata. @@ -249,10 +246,10 @@ def get_ogc_data( truncate the result to exactly ``max_rows``. ``None`` (default) fetches the full result. Intended for few-request previews of large, un-chunked tables (e.g. :func:`get_reference_table`). - base_url : str - OGC API base URL to target. Required: this package is API-neutral and - names no API of its own, so each adapter passes its own base (e.g. - ``waterdata.utils.OGC_API_URL``, ``ngwmn.NGWMN_OGC_API_URL``). + endpoint : Endpoint[OgcDialect] + The API to target, built by the adapter when the call starts: its URL, + the service whose settings apply, and how it departs from the standard + (ADR 0014). This package names no API of its own. spatial : bool Whether this collection's result contract includes feature geometry. The adapter supplies this semantic fact; ``skip_geometry`` and @@ -260,9 +257,6 @@ def get_ogc_data( extra_id_cols : set or frozenset, optional Synthetic id columns to move to the end of a result frame (see :func:`_arrange_cols`). Defaults to an empty set. - dialect : OgcDialect, optional - Per-API request quirks (CQL2-only collections, date-only collections). - Defaults to a plain OGC API with neither. cql_body : str, optional A verbatim CQL2 JSON body to POST against ``collection`` instead of building the query from ``args``. With a body, only the @@ -291,8 +285,7 @@ def get_ogc_data( if max_rows is not None: _require_positive_int(max_rows, "max_rows") - if dialect is None: - dialect = DEFAULT_DIALECT + adapter = endpoint.service.name args = args.copy() args["collection"] = collection args = _switch_arg_id(args, id_name=output_id, collection=collection) @@ -329,8 +322,7 @@ def get_ogc_data( include_geometry=include_geometry, max_rows=max_rows, extra_id_cols=extra_id_cols, - dialect=dialect, - base_url=base_url, + endpoint=endpoint, ) if cql_body is not None: @@ -340,7 +332,7 @@ def get_ogc_data( req = _construct_cql_request( collection, cql_body, - base_url=base_url, + base_url=endpoint.url, properties=args.get("properties"), bbox=args.get("bbox"), limit=args.get("limit"), @@ -372,9 +364,7 @@ def get_ogc_data( # rebuilds them through these same bound callables, so the values the # call was created with apply to every chunk — even a resume invoked long # after this function returned — without any ambient state to snapshot. - build_request = functools.partial( - _construct_api_requests, base_url=base_url, dialect=dialect - ) + build_request = functools.partial(_construct_api_requests, endpoint=endpoint) fetch = functools.partial( _fetch_once, build_request=build_request, diff --git a/dataretrieval/ogc/policy.py b/dataretrieval/ogc/policy.py index c92c7da1..8583551f 100644 --- a/dataretrieval/ogc/policy.py +++ b/dataretrieval/ogc/policy.py @@ -67,8 +67,3 @@ class OgcDialect: time_cols: frozenset[str] = field(default_factory=frozenset) numerical_cols: frozenset[str] = field(default_factory=frozenset) sort_cols: tuple[str, ...] = field(default_factory=tuple) - - -# Default dialect: a plain OGC API with no CQL2-only collections and no -# date-only collections (every time argument rendered as a full UTC datetime). -DEFAULT_DIALECT = OgcDialect() diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 1427dc38..a8f1aaa4 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -1,7 +1,7 @@ """OGC argument normalization and HTTP request construction. -The API to target and its quirks are explicit parameters (``base_url``, -``dialect``) -- construction takes everything it needs as parameters. +The API to target and its quirks are one explicit parameter, the endpoint +(ADR 0014) -- construction takes everything it needs as parameters. Queryables and schema execution are in :mod:`dataretrieval.ogc.schema` and are not re-exported here, so this module does not depend on the part of OGC that executes HTTP (ADR 0007). @@ -12,14 +12,17 @@ import json import re from collections.abc import Iterable, Mapping -from typing import Any +from typing import TYPE_CHECKING, Any import httpx from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS, _format_api_dates -from dataretrieval.ogc.policy import DEFAULT_DIALECT, OgcDialect from dataretrieval.transport.http import default_headers as _default_headers +if TYPE_CHECKING: + from dataretrieval._service import Endpoint + from dataretrieval.ogc.policy import OgcDialect + # --------------------------------------------------------------------------- # Monitoring location ID validation # --------------------------------------------------------------------------- @@ -163,19 +166,16 @@ def _construct_api_requests( limit: int | None = None, skip_geometry: bool | None = None, *, - base_url: str, - dialect: OgcDialect | None = None, + endpoint: Endpoint[OgcDialect], **kwargs: Any, ) -> httpx.Request: """Construct an HTTP request object for the specified OGC API collection. - ``base_url`` is required: this package is API-neutral and names no API of - its own, so the adapter naming the collection also names the API it targets. - ``dialect`` defaults to a plain OGC API with no per-collection quirks. + ``endpoint`` is required: this package names no API of its own, so the + adapter naming the collection also names the API it targets. """ - service_url = _items_url(collection, base_url) - if dialect is None: - dialect = DEFAULT_DIALECT + service_url = _items_url(collection, endpoint.url) + dialect = endpoint.dialect for key in _DATE_RANGE_PARAMS: if key in kwargs: kwargs[key] = _format_api_dates( @@ -224,8 +224,7 @@ def _construct_cql_request( ) -> httpx.Request: """Build a POST/CQL2 request from a verbatim CQL2 body. - ``base_url`` is required for the same reason as in - :func:`_construct_api_requests`. + ``base_url`` is required: this package names no API of its own. """ service_url = _items_url(collection, base_url) params = _ogc_query_params( diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index 8e874606..06973542 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -13,13 +13,16 @@ import logging import re import warnings -from typing import Any +from typing import TYPE_CHECKING, Any import httpx import pandas as pd from dataretrieval._response_metadata import BaseMetadata -from dataretrieval.ogc.policy import DEFAULT_DIALECT, OgcDialect + +if TYPE_CHECKING: + from dataretrieval._service import Endpoint + from dataretrieval.ogc.policy import OgcDialect try: import geopandas as gpd @@ -399,8 +402,7 @@ def _finalize_ogc( include_geometry: bool, max_rows: int | None = None, extra_id_cols: frozenset[str] | set[str] = frozenset(), - dialect: OgcDialect | None = None, - base_url: str, + endpoint: Endpoint[OgcDialect], ) -> tuple[pd.DataFrame, BaseMetadata]: """Shape a combined OGC result into the user-facing ``(df, md)``. @@ -418,16 +420,15 @@ def _finalize_ogc( rather than only per-chunk, so a chunked call's total is bounded to exactly ``max_rows`` and a resumed call applies the cap too. The per-page ``row_cap`` bound in the engine is only an early-stop download - bound. ``base_url`` is required and captured with the finalizer so resumed - calls query the same API's schema when their combined result is empty. + bound. ``endpoint`` is captured with the finalizer so resumed calls query + the same API's schema when their combined result is empty. """ - if dialect is None: - dialect = DEFAULT_DIALECT + dialect = endpoint.dialect frame = _deal_with_empty( frame, properties, collection, - base_url=base_url, + base_url=endpoint.url, geopd=geopd, include_geometry=include_geometry, ) diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index 7eb8d46a..89d830f8 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -23,6 +23,7 @@ import pandas as pd from dataretrieval._deprecation import warn_deprecated +from dataretrieval._service import Endpoint from dataretrieval.codes.states import apply_state from dataretrieval.credentials import refuse_credential_keywords from dataretrieval.ogc import OgcDialect, prepare_request_args @@ -271,19 +272,15 @@ def get_ogc_data( collection, output_id, max_rows=max_rows, - # The endpoint is resolved from the active ContextVar at request time; - # the documented ``OGC_API_URL`` constant remains the default-value - # compatibility path rather than a production request destination. - base_url=ogc_api_url(api_version=api_version), + # Built when the call starts, from the configuration in effect; the + # documented ``OGC_API_URL`` constant remains a default-value + # compatibility path, not a request destination (ADR 0014). + endpoint=Endpoint( + SERVICE, ogc_api_url(api_version=api_version), WATERDATA_DIALECT + ), spatial=spatial, extra_id_cols=_EXTRA_ID_COLS, - dialect=WATERDATA_DIALECT, cql_body=cql_body, - # Which settings table these calls read. Declared here, in the one - # wrapper every Water Data getter goes through, rather than derived - # from ``base_url``: NGWMN is served from the same host, so a URL - # does not distinguish the two adapters (ADR 0010). - adapter=SERVICE.name, ) diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index 9f6cb1b8..fc165892 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -105,9 +105,10 @@ read from the declaration, never a literal. Only resolving the URL takes the value itself. **Each API an adapter calls is an** ``Endpoint`` **of its service.** An -endpoint names its service, resolves its URL from configuration once when a -call starts -- so every chunk and a later resume use the same URL -- and carries -the dialect of its protocol. Water Data is one service with four endpoints. +endpoint names its service, its URL, and the dialect of its protocol. The +adapter builds it when a call starts, from the configuration in effect, so +every chunk and a later resume use the same URL. Water Data is one service with +four endpoints. **A protocol is implemented as the standard is written.** Protocol code is functions that receive an endpoint. Each dialect field records one way an @@ -134,9 +135,9 @@ interval grammar shared by OGC and STAC is written by one module, adjusted by a ``DateDialect``; another protocol writes the value in its own format. **Values are data; protocols and execution are functions.** ``Service``, -``Endpoint``, and the dialect types are frozen dataclasses. They may resolve a -URL from configuration; they do not send requests, and nothing subclasses them -or a protocol's code. A method that sends requests, or an override, could +``Endpoint``, and the dialect types are frozen dataclasses with no behavior. +They do not send requests or read configuration, and nothing subclasses them or +a protocol's code. A method that sends requests, or an override, could change behavior for one service without declaring a field, which is the problem this record exists to prevent. @@ -159,8 +160,6 @@ Consequences evidence for each deviation. - Inside ``ogc``, one endpoint argument replaces the ``base_url``, ``dialect``, and ``adapter`` arguments that are passed together today. -- Resolving a URL is the one behavior a value has. It only reads - configuration, so it cannot change what protocol code does with the URL. - A deviation the protocol code does not model needs a new field first. That costs one field and its default, deliberately. - Behavior change, when the date step lands: a lone duration sent to NGWMN @@ -231,3 +230,11 @@ Revisions: nothing they read. ``configuration.service_url`` resolves a service's URL. The name check excludes ``__all__``, whose entries name modules that share a service's spelling. +- *Endpoints for the OGC getters.* An endpoint holds a resolved URL rather than + a function that resolves one: the adapter builds it at the start of each + call, which gives the same once-per-call guarantee with no behavior in the + value. The OGC engine, request builder, and result finalizer take one + ``endpoint`` where they took ``base_url``, ``dialect``, and ``adapter``; + functions that need only the URL (schema and CQL requests) still take it + alone. ``DEFAULT_DIALECT`` is removed, since every call now supplies a + dialect through its endpoint. diff --git a/tests/configuration_test.py b/tests/configuration_test.py index 4393626c..d92c2f51 100644 --- a/tests/configuration_test.py +++ b/tests/configuration_test.py @@ -1295,15 +1295,20 @@ def test_every_adapter_is_actually_wired_to_a_read_site(): root = pathlib.Path(configuration.__file__).parent # Each adapter declares its service once (ADR 0014) and reads the name from - # that declaration, so a read site is ``adapter=.name`` in the - # adapter's own module or package. + # that declaration, so a read site is ``adapter=.name`` or an + # ``Endpoint(, ...)`` in the adapter's own module or package. declared = re.compile(r'(_?SERVICE) = Service\("([a-z]+)"') missing = list(configuration.ADAPTERS) for path in root.rglob("*.py"): for variable, name in declared.findall(path.read_text(encoding="utf-8")): scope = path.parent.glob("*.py") if path.parent != root else [path] source = "\n".join(p.read_text(encoding="utf-8") for p in scope) - if f"adapter={variable}.name" in source and name in missing: + reads = ( + f"adapter={variable}.name", + f"Endpoint(\n {variable},", + f"Endpoint({variable},", + ) + if any(r in source for r in reads) and name in missing: missing.remove(name) assert not missing, ( f"adapters with a schema but no read site: {missing}. Either pass " diff --git a/tests/headers_host_scoping_test.py b/tests/headers_host_scoping_test.py index f5aeeb03..28e844c2 100644 --- a/tests/headers_host_scoping_test.py +++ b/tests/headers_host_scoping_test.py @@ -117,11 +117,13 @@ def handler(request: httpx.Request) -> httpx.Response: def test_generic_ogc_request_excludes_key_for_custom_host(self): """A caller-supplied OGC base URL never receives Water Data auth.""" + from dataretrieval._service import Endpoint, Service + from dataretrieval.ogc.policy import OgcDialect from dataretrieval.ogc.requests import _construct_api_requests - request = _construct_api_requests( - "things", base_url="https://features.example.org/ogcapi" - ) + url = "https://features.example.org/ogcapi" + endpoint = Endpoint(Service("ngwmn", url), url, OgcDialect()) + request = _construct_api_requests("things", endpoint=endpoint) assert "X-Api-Key" not in request.headers def test_rating_download_scopes_headers_to_asset_url(self): diff --git a/tests/waterdata_chunking_test.py b/tests/waterdata_chunking_test.py index b9689ee7..b7bbbf7a 100644 --- a/tests/waterdata_chunking_test.py +++ b/tests/waterdata_chunking_test.py @@ -34,6 +34,7 @@ import dataretrieval from dataretrieval import configuration as _configuration +from dataretrieval._service import Endpoint from dataretrieval.combining import ( _QUOTA_HEADER, _combine_chunk_frames, @@ -72,6 +73,7 @@ _request_bytes, _safe_request_bytes, ) +from dataretrieval.ogc.policy import OgcDialect from dataretrieval.ogc.requests import ( _construct_api_requests as _construct_api_requests_explicit, ) @@ -84,12 +86,14 @@ retry_async as _retry, ) from dataretrieval.utils import HTTPX_DEFAULTS +from dataretrieval.waterdata.endpoints import SERVICE from dataretrieval.waterdata.utils import OGC_API_URL # The joint-planner stress test uses the real request builder; bind the # target API the way ``get_ogc_data`` does — explicitly, via ``partial``. _construct_api_requests = functools.partial( - _construct_api_requests_explicit, base_url=OGC_API_URL + _construct_api_requests_explicit, + endpoint=Endpoint(SERVICE, OGC_API_URL, OgcDialect()), ) diff --git a/tests/waterdata_filters_test.py b/tests/waterdata_filters_test.py index 1f3bfee8..9e99f916 100644 --- a/tests/waterdata_filters_test.py +++ b/tests/waterdata_filters_test.py @@ -7,21 +7,25 @@ import pandas as pd import pytest +from dataretrieval._service import Endpoint from dataretrieval.ogc.filters import ( _check_numeric_filter_pitfall, _quote_cql_str, _split_top_level_or, ) +from dataretrieval.ogc.policy import OgcDialect from dataretrieval.ogc.requests import ( _construct_api_requests as _construct_api_requests_explicit, ) from dataretrieval.waterdata import get_continuous +from dataretrieval.waterdata.endpoints import SERVICE from dataretrieval.waterdata.utils import OGC_API_URL # Bind the target API the way ``get_ogc_data`` does — explicitly — so the # direct construction tests below exercise the real builder. _construct_api_requests = functools.partial( - _construct_api_requests_explicit, base_url=OGC_API_URL + _construct_api_requests_explicit, + endpoint=Endpoint(SERVICE, OGC_API_URL, OgcDialect()), ) diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 541f7a24..2a649ed4 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -15,6 +15,7 @@ import dataretrieval from dataretrieval import configuration +from dataretrieval._service import Endpoint from dataretrieval.ogc.requests import ( _check_monitoring_location_id, _normalize_str_iterable, @@ -45,6 +46,7 @@ get_stats_por, get_time_series_metadata, ) +from dataretrieval.waterdata.endpoints import SERVICE from dataretrieval.waterdata.types import _check_profiles from dataretrieval.waterdata.utils import ( OGC_API_URL, @@ -76,7 +78,8 @@ # the real Water Data behavior (monitoring-locations -> POST/CQL2; daily -> # date-only time args). _construct_api_requests = functools.partial( - _construct_api_requests_explicit, base_url=OGC_API_URL, dialect=WATERDATA_DIALECT + _construct_api_requests_explicit, + endpoint=Endpoint(SERVICE, OGC_API_URL, WATERDATA_DIALECT), ) _construct_cql_request = functools.partial( _construct_cql_request_explicit, base_url=OGC_API_URL diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 281744c0..d1bb4103 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -3,6 +3,7 @@ import functools import json import logging +import re import warnings from unittest import mock @@ -13,6 +14,7 @@ import dataretrieval.ogc.shaping as _shaping_module import dataretrieval.waterdata.stats as _stats_module import dataretrieval.waterdata.utils as _utils_module +from dataretrieval._service import Endpoint, Service from dataretrieval.exceptions import ( DataRetrievalError, HTTPError, @@ -42,6 +44,7 @@ ) from dataretrieval.ogc.shaping import _finalize_ogc as _ogc_finalize from dataretrieval.waterdata import get_stats_date_range, get_stats_por +from dataretrieval.waterdata.endpoints import SERVICE from dataretrieval.waterdata.stats import _handle_nesting, get_data from dataretrieval.waterdata.utils import ( _EXTRA_ID_COLS, @@ -56,8 +59,7 @@ _finalize_ogc = functools.partial( _ogc_finalize, extra_id_cols=_EXTRA_ID_COLS, - dialect=WATERDATA_DIALECT, - base_url=OGC_API_URL, + endpoint=Endpoint(SERVICE, OGC_API_URL, WATERDATA_DIALECT), geopd=False, include_geometry=True, ) @@ -1581,3 +1583,63 @@ def test_a_json_scalar_body_yields_no_detail(self): assert _json_error_detail(httpx.Response(400, json="just a string")) is None assert _json_error_detail(httpx.Response(400, json=[1, 2, 3])) is None + + +def test_get_ogc_data_reads_everything_about_the_service_from_its_endpoint( + httpx_mock, +): + """ADR 0014: an endpoint is all the OGC engine is told about a service. The + request goes to its URL, its dialect decides the request shape (a + ``cql2_services`` collection is POSTed), and settings resolve by its + service's name.""" + import dataretrieval + from dataretrieval.ngwmn import NgwmnConfiguration + from dataretrieval.ogc.engine import get_ogc_data + + endpoint = Endpoint( + Service("ngwmn", "https://unused.example"), + "https://mirror.example/ogc", + OgcDialect(cql2_services=frozenset({"things"})), + ) + httpx_mock.add_response( + method="POST", + url=re.compile(r"^https://mirror\.example/ogc/collections/things/items"), + json={"type": "FeatureCollection", "features": [], "numberReturned": 0}, + ) + with dataretrieval.configure(NgwmnConfiguration(retries=0)): + df, _ = get_ogc_data( + {"properties": ["a"], "site": ["A", "B"]}, + "things", + "thing_id", + endpoint=endpoint, + spatial=False, + ) + assert df.empty + (request,) = httpx_mock.get_requests() + assert request.method == "POST" + + +def test_a_resumed_call_keeps_the_url_its_endpoint_was_built_with(httpx_mock): + """The endpoint is built when the call starts (ADR 0014), so a call + interrupted inside a ``configure(base_url=...)`` block and resumed after the + block has exited still reaches the mirror it started on.""" + import dataretrieval + from dataretrieval.interruptions import FanOutInterrupted + from dataretrieval.waterdata import WaterdataConfiguration, get_daily + + mirror = re.compile(r"^https://mirror\.example/ogcapi/v1/collections/daily/items") + empty = {"type": "FeatureCollection", "features": [], "numberReturned": 0} + httpx_mock.add_response(url=mirror, status_code=429) + httpx_mock.add_response(url=mirror, json=empty) + + with dataretrieval.configure( + WaterdataConfiguration(base_url="https://mirror.example") + ): + with pytest.raises(FanOutInterrupted) as excinfo: + get_daily(monitoring_location_id="USGS-01646500", properties=["value"]) + excinfo.value.call.resume() + + assert [r.url.host for r in httpx_mock.get_requests()] == [ + "mirror.example", + "mirror.example", + ] From 0860fd0cdc0f7e9666960bb215eee7adcda0020c Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 13:45:42 -0500 Subject: [PATCH 12/22] docs(ogc): policy no longer says calls are targeted by base_url --- dataretrieval/ogc/policy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataretrieval/ogc/policy.py b/dataretrieval/ogc/policy.py index 8583551f..255485c8 100644 --- a/dataretrieval/ogc/policy.py +++ b/dataretrieval/ogc/policy.py @@ -5,9 +5,9 @@ It depends only on the stdlib, so any OGC submodule can import it without creating cycles. -It names no endpoint: which API an OGC call targets is the *adapter's* -policy, supplied per call as ``base_url``. A default here would -direct every generic OGC caller to one API. +It names no API: which one an OGC call targets is the *adapter's* policy, +supplied per call as an endpoint (ADR 0014). A default here would direct every +generic OGC caller to one API. It must not import engine, shaping, or any collection adapter. """ From d9d658cefa93caa4fd05e4235cab74790742fb67 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 13:46:23 -0500 Subject: [PATCH 13/22] docs(adr): 0014, an Endpoint value exists only where protocol code reads one Records why the remaining adapters get no Endpoint: without a protocol module there is no dialect to carry, and their URLs already resolve through service_url and waterdata.endpoints. --- .../decisions/0014-services-endpoints-dialects.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index fc165892..0c977d9e 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -104,7 +104,7 @@ because that is the key a configuration file uses: they take ``service.name``, read from the declaration, never a literal. Only resolving the URL takes the value itself. -**Each API an adapter calls is an** ``Endpoint`` **of its service.** An +**Each API protocol code reaches is an** ``Endpoint`` **of its service.** An endpoint names its service, its URL, and the dialect of its protocol. The adapter builds it when a call starts, from the configuration in effect, so every chunk and a later resume use the same URL. Water Data is one service with @@ -238,3 +238,11 @@ Revisions: functions that need only the URL (schema and CQL requests) still take it alone. ``DEFAULT_DIALECT`` is removed, since every call now supplies a dialect through its endpoint. +- *Endpoints for the remaining adapters* -- not built. An ``Endpoint`` value + exists only where protocol code receives one, because its point is to carry a + dialect to code shared between endpoints. WQP, NLDI, NWDC, StreamStats, and + Water Data's Samples and Statistics APIs have no protocol module, so an + endpoint there would carry no dialect and be read only for its URL, which + ``configuration.service_url`` and ``waterdata.endpoints`` already resolve. They + are still endpoints in the glossary's sense. The STAC ratings search gets an + ``Endpoint`` with the interval grammar, where its ``DateDialect`` is read. From 997d91f0c4ffbc202e0564ee084fb1229f77dbc9 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 17:21:08 -0500 Subject: [PATCH 14/22] refactor: read a date range once, at the public boundary (ADR 0014) A getter now reads each date argument into a DateRange when it is called: prepare_request_args does it for every OGC date parameter, and get_ratings for its time. The value records what the caller meant -- an instant, a range with open bounds, or a duration -- and the OGC request builder only writes it. Reading moves to the dataretrieval._dates leaf; writing stays in ogc.dates. Error messages are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- .importlinter | 2 +- dataretrieval/_dates.py | 229 ++++++++++++++ dataretrieval/ogc/dates.py | 282 +++--------------- dataretrieval/ogc/requests.py | 41 +-- dataretrieval/waterdata/ratings.py | 25 +- .../0014-services-endpoints-dialects.rst | 9 + tests/_dates_test.py | 50 ++++ tests/contracts/README.md | 2 +- tests/waterdata_test.py | 31 +- tests/waterdata_utils_test.py | 106 +++---- 10 files changed, 437 insertions(+), 340 deletions(-) create mode 100644 dataretrieval/_dates.py create mode 100644 tests/_dates_test.py diff --git a/.importlinter b/.importlinter index 7c64bca7..376d5a0e 100644 --- a/.importlinter +++ b/.importlinter @@ -45,7 +45,7 @@ layers = ; Pure dependency-free mechanisms sit together at the floor: none imports ; first-party code, so every layer can use them without reaching sideways or ; creating artificial dependencies among them. - _ambient | _csv | _deprecation | _validation + _ambient | _csv | _dates | _deprecation | _validation ; Every top-level module must be placed in the stack deliberately. A new ; top-level module fails this contract until someone decides where it sits. exhaustive = True diff --git a/dataretrieval/_dates.py b/dataretrieval/_dates.py new file mode 100644 index 00000000..37204c64 --- /dev/null +++ b/dataretrieval/_dates.py @@ -0,0 +1,229 @@ +"""What a caller meant by a date argument, read once at the public boundary. + +A dependency-free leaf (ADR 0014): a getter reads its date arguments into a +:class:`DateRange` when it is called, and the protocol code that sends the +request writes that value in its own wire format. Reading -- which types and +string formats are accepted, what spells an open bound, and that a naive value +is local time -- is package policy, so it lives here and names no service. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import date as _date +from datetime import datetime + +import pandas as pd + +_DATETIME_FORMATS = ( + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S.%f", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", +) + + +# Anchored to ``[Pp]\d`` so a normal word containing ``p`` (e.g. ``"Apr"``) +# doesn't get mis-classified as an ISO 8601 duration; the optional ``T`` +# admits time-only forms like ``PT36H``. +DURATION_RE = re.compile(r"^[Pp]T?\d") + + +# The ISO 8601 spelling of an open interval bound, as in ``"2024-01-01/.."``. +OPEN_BOUND = ".." + + +# One element of a date argument: an ISO 8601 string, or a ``date``, +# ``datetime`` or ``pandas.Timestamp`` (both subclasses of ``datetime.date``), +# with ``None`` (or ``NaN`` / ``NaT``) for an open bound. +DateLike = str | _date | None + + +@dataclass(frozen=True) +class DateRange: + """One date argument as the caller meant it. + + Exactly one shape is set: an instant (``instant=True``, ``start`` only), a + range of two bounds where ``None`` is an open bound (``start``, ``end``), + or an ISO 8601 duration (``duration``, such as ``"P7D"``). A bound is + naive exactly when the caller gave no zone, and a naive bound is local + time. + """ + + start: datetime | None = None + end: datetime | None = None + instant: bool = False + duration: str | None = None + + +def _parse_datetime(value: str) -> datetime | None: + """Parse a single datetime string against the supported formats. + + Returns a ``datetime`` (tz-aware iff the input included a UTC offset), + or ``None`` if no format matched. + """ + # ``datetime.strptime`` accepts a numeric offset like ``+00:00`` but not + # the ``Z`` shorthand, so normalize trailing ``Z`` first. + candidate = value[:-1] + "+00:00" if value.endswith("Z") else value + for fmt in _DATETIME_FORMATS: + try: + # DTZ007: naive is the documented outcome for a naive input -- + # ``_DATETIME_FORMATS`` carries both the ``%z`` and the bare forms. + return datetime.strptime(candidate, fmt) # noqa: DTZ007 + except ValueError: + continue + return None + + +def _is_blank(dt: DateLike) -> bool: + """True for a None, NaN, empty-string, or ``..`` element. + + Each is a spelling of an open bound, so ``["2024-01-01", ".."]`` means the + same range as ``["2024-01-01", None]`` and the string ``"2024-01-01/.."``. + """ + return dt is None or bool(pd.isna(dt)) or dt in ("", OPEN_BOUND) + + +def _to_datetime(dt: str | _date, *, name: str) -> datetime: + """Read one non-blank element as a ``datetime`` (naive iff it has no zone). + + A ``date`` is midnight of that day, like the string ``"2024-01-01"``. + Raises ``ValueError`` naming *name* for an unreadable string or a value of + another type. + """ + if isinstance(dt, pd.Timestamp): + # A writer converts with ``astimezone``, which on a ``Timestamp`` is + # ``tz_convert`` and requires a zone, so keep a plain ``datetime``. + converted: datetime = dt.to_pydatetime(warn=False) + return converted + if isinstance(dt, datetime): + return dt + if isinstance(dt, _date): + return datetime(dt.year, dt.month, dt.day) # noqa: DTZ001 + parsed = _parse_datetime(dt) if isinstance(dt, str) else None + if parsed is None: + raise ValueError( + f"{name} could not be read as a date or datetime: {dt!r}. " + "Pass an ISO 8601 date or datetime such as '2024-01-01' or " + "'2024-01-01T12:00:00Z' (a datetime.date, datetime.datetime or " + "pandas.Timestamp also works), and None for an open end of a range " + "(['2024-01-01', None])." + ) + return parsed + + +def _read_bound(dt: DateLike, *, name: str) -> datetime | None: + """One element as a bound: ``None`` when it is blank (an open bound).""" + return None if dt is None or _is_blank(dt) else _to_datetime(dt, name=name) + + +def coerce_to_list( + value: str | _date | Sequence[DateLike], + name: str = "date input", +) -> list[DateLike]: + """Normalize a date argument to a list of elements, raising on a mapping.""" + if isinstance(value, str) or not isinstance(value, Iterable): + # A lone value (a ``date`` is not iterable); a non-date is rejected + # per element. + return [value] + if isinstance(value, Mapping): + raise TypeError( + f"{name} must be a date, a string, or a sequence of them, " + f"not {type(value).__name__}." + ) + return list(value) + + +def _split_interval(interval: str, *, name: str) -> tuple[str, str]: + """Split a ``"start/end"`` string into its two sides. + + Each side is then read like a list element, so ``"2024-01-01T10:00:00/.."`` + means the same range as ``["2024-01-01T10:00:00", None]``. + + Raises ``ValueError`` naming *name* unless the string is two sides + separated by one ``"/"``, neither of them a duration: the Water Data and + NGWMN services answer ``"2024-01-01/P7D"`` and ``"P7D/2024-01-08"`` with + HTTP 400. + """ + sides = interval.split("/") + if len(sides) != 2 or any(DURATION_RE.match(side) for side in sides): + raise ValueError( + f"{name} is not a valid interval: {interval!r}. Pass a start and " + "an end separated by '/', such as '2024-01-01/2024-12-31', with " + "'..' for an open end ('2024-01-01/..'). Neither side can be a " + "duration." + ) + return sides[0], sides[1] + + +def read_date_range( + value: str | _date | Sequence[DateLike] | None, + *, + name: str = "date input", + single_value_hint: str = "an instant or a duration ('2020-01-01', 'P7D')", +) -> DateRange | None: + """Read a date argument as the caller meant it. + + Parameters + ---------- + value : str, date, sequence of them, or None + A single date/datetime, a ``"start/end"`` interval string, an ISO 8601 + duration (``"P7D"``, ``"PT36H"``), or a list of one or two elements. + Each element may be a ``datetime.date``, ``datetime.datetime`` or + ``pandas.Timestamp``, or a string in ISO 8601 (with or without ``Z`` or + a numeric offset) or ``"%Y-%m-%d %H:%M:%S"`` form. ``None``, ``NaN``, + ``NaT``, ``""`` and ``".."`` are open bounds. + name : str, optional + The caller's own spelling of this argument, used as the subject of + every message raised here, so a caller correcting the error edits an + argument their getter accepts. + single_value_hint : str, optional + How the "too many values" message describes an acceptable single + value. Wording only: a getter that rejects some of the default's forms + (``get_ratings`` rejects durations) passes a hint naming only what it + accepts. + + Returns + ------- + DateRange or None + ``None`` when *value* is ``None`` or every element is an open bound, + meaning no filter. + + Raises + ------ + ValueError + For more than two elements, an unreadable interval string, or an + element that is not blank and matches no supported format or type. + TypeError + For a mapping. + """ + if value is None: + return None + + items = coerce_to_list(value, name) + if len(items) == 1 and isinstance(items[0], str) and "/" in items[0]: + items = list(_split_interval(items[0], name=name)) + + if all(_is_blank(dt) for dt in items): + return None + + if len(items) > 2: + raise ValueError( + f"{name} takes at most 2 values, got {len(items)}: {items!r}. " + f"Pass one value for {single_value_hint}, " + "or two for a closed interval ('2020-01-01', '2020-12-31')." + ) + + if len(items) == 1: + (item,) = items + if isinstance(item, str) and DURATION_RE.match(item): + return DateRange(duration=item) + return DateRange(start=_read_bound(item, name=name), instant=True) + return DateRange( + start=_read_bound(items[0], name=name), end=_read_bound(items[1], name=name) + ) diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py index 8e4df6c3..b877d765 100644 --- a/dataretrieval/ogc/dates.py +++ b/dataretrieval/ogc/dates.py @@ -1,276 +1,70 @@ -"""Date and datetime marshalling for OGC time parameters. +"""Writing a date range as an OGC time parameter. -Pure helpers that render the OGC getters' time-shaped arguments — single -instants, two-element ``[start, end]`` ranges, ISO-8601 durations, and open -``..`` bounds — into the wire form the API expects. No I/O, no engine state. +A getter reads each date argument into a :class:`~dataretrieval._dates.DateRange` +when it is called; this module writes that value in the wire form the API +expects -- an instant, a ``start/end`` interval with ``..`` open bounds, or a +duration. No I/O, no engine state. """ from __future__ import annotations -import re -from collections.abc import Iterable, Mapping, Sequence -from datetime import date as _date from datetime import datetime from zoneinfo import ZoneInfo -import pandas as pd - -_DATETIME_FORMATS = ( - "%Y-%m-%dT%H:%M:%S.%f%z", - "%Y-%m-%dT%H:%M:%S%z", - "%Y-%m-%dT%H:%M:%S.%f", - "%Y-%m-%dT%H:%M:%S", - "%Y-%m-%d %H:%M:%S.%f", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%d", -) - - -# Anchored to ``[Pp]\d`` so a normal word containing ``p`` (e.g. ``"Apr"``) -# doesn't get mis-classified as an ISO 8601 duration; the optional ``T`` -# admits time-only forms like ``PT36H``. -_DURATION_RE = re.compile(r"^[Pp]T?\d") - +from dataretrieval._dates import OPEN_BOUND, DateRange # OGC API parameters that take a date/datetime value (single string, # two-element range, or interval/duration string) rather than a multi-value -# string list. Used by ``_construct_api_requests`` to keep them out of the -# POST/CQL2 multi-value path and to route them through ``_format_api_dates``, -# and by the default ``_get_args`` no-normalize set to bypass string-iterable -# normalization. +# string list. ``prepare_request_args`` reads them into a ``DateRange``, +# ``_construct_api_requests`` writes them with ``_format_date_range``, and the +# chunk planner never splits them. _DATE_RANGE_PARAMS = frozenset( {"datetime", "last_modified", "begin", "begin_utc", "end", "end_utc", "time"} ) -def _parse_datetime(value: str) -> datetime | None: - """Parse a single datetime string against the supported formats. - - Returns a ``datetime`` (tz-aware iff the input included a UTC offset), - or ``None`` if no format matched. - """ - # ``datetime.strptime`` accepts a numeric offset like ``+00:00`` but not - # the ``Z`` shorthand, so normalize trailing ``Z`` first. - candidate = value[:-1] + "+00:00" if value.endswith("Z") else value - for fmt in _DATETIME_FORMATS: - try: - # DTZ007: naive is the documented outcome for a naive input -- - # ``_DATETIME_FORMATS`` carries both the ``%z`` and the bare forms. - return datetime.strptime(candidate, fmt) # noqa: DTZ007 - except ValueError: - continue - return None - - -# The OGC API spelling of an open interval bound, as in ``"2024-01-01/.."``. -_OPEN_BOUND = ".." - - -# One element of a date argument: an ISO 8601 string, or a ``date``, -# ``datetime`` or ``pandas.Timestamp`` (both subclasses of ``datetime.date``), -# with ``None`` (or ``NaN`` / ``NaT``) for an open bound. -_DateLike = str | _date | None - - -def _is_blank(dt: _DateLike) -> bool: - """True for a None, NaN, empty-string, or ``..`` element. - - Each is a spelling of an open bound, so ``["2024-01-01", ".."]`` means the - same range as ``["2024-01-01", None]`` and the string ``"2024-01-01/.."``. - """ - return dt is None or bool(pd.isna(dt)) or dt in ("", _OPEN_BOUND) - - -def _to_datetime(dt: str | _date, *, name: str) -> datetime: - """Read one non-blank element as a ``datetime`` (naive iff it has no zone). - - A ``date`` is midnight of that day, like the string ``"2024-01-01"``. - Raises ``ValueError`` naming *name* for an unreadable string or a value of - another type. - """ - if isinstance(dt, pd.Timestamp): - # ``Timestamp.astimezone`` is ``tz_convert`` and requires a zone, so - # hand ``_format_one`` a plain ``datetime``. - converted: datetime = dt.to_pydatetime(warn=False) - return converted - if isinstance(dt, datetime): - return dt - if isinstance(dt, _date): - return datetime(dt.year, dt.month, dt.day) # noqa: DTZ001 - parsed = _parse_datetime(dt) if isinstance(dt, str) else None - if parsed is None: - raise ValueError( - f"{name} could not be read as a date or datetime: {dt!r}. " - "Pass an ISO 8601 date or datetime such as '2024-01-01' or " - "'2024-01-01T12:00:00Z' (a datetime.date, datetime.datetime or " - "pandas.Timestamp also works), and None for an open end of a range " - "(['2024-01-01', None])." - ) - return parsed - - -def _format_one(dt: _DateLike, *, date: bool, name: str) -> str: - """Format a single datetime element for inclusion in the API time arg. - - Raises ``ValueError`` naming *name* when the element is not blank and - cannot be read as a date or datetime. - """ - if dt is None or _is_blank(dt): - return _OPEN_BOUND - parsed = _to_datetime(dt, name=name) +def _format_bound(bound: datetime | None, *, date: bool) -> str: + """Write one bound: ``YYYY-MM-DD``, a UTC ``...Z`` datetime, or ``..``.""" + if bound is None: + return OPEN_BOUND if date: - return parsed.strftime("%Y-%m-%d") - # Naive inputs are interpreted in the system local zone (for backwards - # compatibility). Use ``.astimezone()`` rather than a fixed offset so each - # value is resolved against the DST rules for its own date — a fixed - # ``datetime.now()`` offset shifted off-season inputs by an hour. - aware = parsed if parsed.tzinfo is not None else parsed.astimezone() + return bound.strftime("%Y-%m-%d") + # A naive bound is local time. Use ``.astimezone()`` rather than a fixed + # offset so each value is resolved against the DST rules for its own date. + aware = bound if bound.tzinfo is not None else bound.astimezone() return aware.astimezone(ZoneInfo("UTC")).strftime("%Y-%m-%dT%H:%M:%SZ") -def _coerce_to_list( - datetime_input: str | _date | Sequence[_DateLike], - name: str = "date input", -) -> list[_DateLike]: - """Normalize datetime input to a list, raising on invalid shapes.""" - if isinstance(datetime_input, str) or not isinstance(datetime_input, Iterable): - # A lone value (a ``date`` is not iterable); a non-date is rejected - # per element. - return [datetime_input] - if isinstance(datetime_input, Mapping): - raise TypeError( - f"{name} must be a date, a string, or a sequence of them, " - f"not {type(datetime_input).__name__}." - ) - return list(datetime_input) - - -def _split_interval(interval: str, *, name: str) -> tuple[str, str]: - """Split a ``"start/end"`` string into its two sides. - - Each side is then formatted like a list element, so - ``"2024-01-01T10:00:00/.."`` sends the same range as - ``["2024-01-01T10:00:00", None]``. - - Raises ``ValueError`` naming *name* unless the string is two sides - separated by one ``"/"``, neither of them a duration: the Water Data and - NGWMN services answer ``"2024-01-01/P7D"`` and ``"P7D/2024-01-08"`` with - HTTP 400. - """ - sides = interval.split("/") - if len(sides) != 2 or any(_DURATION_RE.match(side) for side in sides): - raise ValueError( - f"{name} is not a valid interval: {interval!r}. Pass a start and " - "an end separated by '/', such as '2024-01-01/2024-12-31', with " - "'..' for an open end ('2024-01-01/..'). Neither side can be a " - "duration." - ) - return sides[0], sides[1] - - -def _all_blank(items: list[_DateLike]) -> bool: - """True when every element is None, NaN, the empty string, or ``..``.""" - return all(_is_blank(dt) for dt in items) - - -def _format_api_dates( - datetime_input: str | _date | Sequence[_DateLike] | None, - date: bool = False, - *, - name: str = "date input", - single_value_hint: str = "an instant or a duration ('2020-01-01', 'P7D')", - open_start: str = _OPEN_BOUND, -) -> str | None: - """ - Formats date or datetime input(s) for use with an API. - - Handles single values or ranges, converting to ISO 8601 or date-only - formats as needed. +def _format_date_range( + value: DateRange, *, date: bool = False, open_start: str = OPEN_BOUND +) -> str: + """Write a date range as an OGC API time parameter. Parameters ---------- - datetime_input : Union[str, date, List[Optional[Union[str, date]]], None] - A single date/datetime or a list of one or two of them. Each may be a - ``datetime.date``, ``datetime.datetime`` or ``pandas.Timestamp`` (a - naive one is read in the local time zone, as a naive string is), or a - string. Strings accept formats like "%Y-%m-%d %H:%M:%S", ISO 8601 (with or - without ``Z``/numeric offset), or relative periods (e.g., "P7D" / - "PT36H"). Range endpoints may be ``None``/``NaN``/empty or ``".."`` - to denote a half-bounded range. + value : DateRange + What the caller meant, as read by + :func:`~dataretrieval._dates.read_date_range`. date : bool, optional - If True, uses only the date portion ("YYYY-MM-DD"). If False (default), - returns full datetime in UTC ISO 8601 format ("YYYY-MM-DDTHH:MM:SSZ"). - name : str, optional - The caller's own spelling of this argument, used as the subject of - every message raised here. Defaults to a generic "date input"; pass - the real parameter name (``"time"``, ``"last_modified"``) so a caller - correcting the error edits an argument their getter accepts. - single_value_hint : str, optional - How the "too many values" message describes an acceptable single - value. Wording only -- a getter that rejects some of the default's - forms (``get_ratings`` rejects durations) enforces that itself and - passes a hint naming only what it accepts, so the remedy does not - direct a caller to a value that is rejected. + If True, write only the date (``YYYY-MM-DD``) of each bound. If False + (default), write each bound as a UTC datetime (``YYYY-MM-DDTHH:MM:SSZ``), + reading a naive bound in the local time zone. open_start : str, optional - How the target API spells the open start of a range (``".."`` - by default). OGC API Features and STAC also allow an empty string, + How the target API spells the open start of a range (``".."`` by + default). OGC API Features and STAC also allow an empty string, ``"/2024-01-01"``, which an API whose firewall rejects ``"../"`` needs instead; see :attr:`OgcDialect.open_start`. An open end is always ``".."``. Returns ------- - Union[str, None] - - If input is a single value, returns the formatted date/datetime string. - - If input is a list of two values, returns a date/datetime range string - separated by "/" (e.g., "YYYY-MM-DD/YYYY-MM-DD" or - "YYYY-MM-DDTHH:MM:SSZ/YYYY-MM-DDTHH:MM:SSZ"). - - Returns None if input is None, empty, or every element is blank. - - Raises - ------ - ValueError - If `datetime_input` contains more than two values, or an element that - is not blank matches no supported format or is of another type. - - Notes - ----- - - A single blank/NA value returns None. In a two-value range, a blank/NA - or ``".."`` endpoint is rendered as ``".."`` to denote an open bound - (e.g. ``"2024-01-01/.."``); the range is only None when *every* element - is blank/NA/``".."``. - - Supports ISO 8601 durations such as "P7D" and "PT36H", which are passed - through unchanged. - - A single string containing ``"/"`` is an interval: it is split into the - two-value form, so ``"2024-01-01/.."`` and ``["2024-01-01", None]`` are - formatted alike. A duration is not accepted as either side. - - Converts datetimes to UTC and formats as ISO 8601 with 'Z' suffix when - `date` is False. Inputs with an explicit offset (``Z`` or ``+HH:MM``) are - converted from that offset to UTC; naive inputs are interpreted in the - local time zone for backwards compatibility. + str + The duration unchanged, a single bound for an instant, or two bounds + joined by ``"/"``. """ - if datetime_input is None: - return None - - items = _coerce_to_list(datetime_input, name) - if len(items) == 1 and isinstance(items[0], str) and "/" in items[0]: - items = list(_split_interval(items[0], name=name)) - - if _all_blank(items): - return None - - if len(items) > 2: - raise ValueError( - f"{name} takes at most 2 values, got {len(items)}: {items!r}. " - f"Pass one value for {single_value_hint}, " - "or two for a closed interval ('2020-01-01', '2020-12-31')." - ) - - # Pass through a duration ("P7D", "PT36H") - if len(items) == 1 and isinstance(items[0], str) and _DURATION_RE.match(items[0]): - return items[0] - - formatted = [_format_one(dt, date=date, name=name) for dt in items] - if len(formatted) == 2 and formatted[0] == _OPEN_BOUND: - formatted[0] = open_start - return "/".join(formatted) + if value.duration is not None: + return value.duration + if value.instant: + return _format_bound(value.start, date=date) + start = open_start if value.start is None else _format_bound(value.start, date=date) + return f"{start}/{_format_bound(value.end, date=date)}" diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 85d1795a..e9dce155 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -16,7 +16,8 @@ import httpx -from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS, _format_api_dates +from dataretrieval._dates import read_date_range +from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS, _format_date_range from dataretrieval.transport.http import default_headers as _default_headers if TYPE_CHECKING: @@ -176,16 +177,12 @@ def _construct_api_requests( """ service_url = _items_url(collection, endpoint.url) dialect = endpoint.dialect - for key in _DATE_RANGE_PARAMS: - if key in kwargs: - kwargs[key] = _format_api_dates( - kwargs[key], - name=key, - date=( - collection in dialect.date_only_services and key != "last_modified" - ), - open_start=dialect.open_start, - ) + for key in _DATE_RANGE_PARAMS & kwargs.keys(): + kwargs[key] = _format_date_range( + kwargs[key], + date=(collection in dialect.date_only_services and key != "last_modified"), + open_start=dialect.open_start, + ) params, post_params = _partition_request_params( kwargs, use_cql2=collection in dialect.cql2_services ) @@ -243,11 +240,10 @@ def _construct_cql_request( # --------------------------------------------------------------------------- # Iterable params that ``_get_args`` must not pass through -# ``_normalize_str_iterable`` (date-range params may contain ``pd.NaT``/None or -# interval strings; ``bbox`` is ``list[float]``). These apply to every OGC caller; -# an adapter with extra numeric params names only its extras via +# ``_normalize_str_iterable`` (``bbox`` is ``list[float]``). These apply to every +# OGC caller; an adapter with extra numeric params names only its extras via # ``prepare_request_args(..., extra_no_normalize=...)``. -_NO_NORMALIZE_PARAMS = _DATE_RANGE_PARAMS | {"bbox"} +_NO_NORMALIZE_PARAMS = frozenset({"bbox"}) def _normalize_str_iterable( @@ -319,6 +315,8 @@ def _normalize_arg( return _check_monitoring_location_id(value) if key == "properties": return _as_str_list(value, key) + if key in _DATE_RANGE_PARAMS: + return read_date_range(value, name=key) if ( key in no_normalize and isinstance(value, Iterable) @@ -339,13 +337,14 @@ def prepare_request_args( """Build OGC request kwargs from a getter's ``locals()``. Internal control keys, caller-supplied exclusions, and ``None`` values - are omitted. Identifiers and properties are validated; other iterables are - normalized unless exempted. + are omitted. Identifiers and properties are validated, date arguments are + read into a ``DateRange`` (and omitted when every bound is open), and other + iterables are normalized unless exempted. ``extra_no_normalize`` *adds* to the engine's own :data:`_NO_NORMALIZE_PARAMS` rather than replacing it, so an adapter names - only the params it owns and cannot drop the date-range exemptions - by omitting them. + only the params it owns and cannot drop the ``bbox`` exemption by + omitting it. """ no_normalize = _NO_NORMALIZE_PARAMS | frozenset(extra_no_normalize) to_exclude = {"collection", "service", "output_id"} @@ -356,5 +355,7 @@ def prepare_request_args( for k, v in local_vars.items(): if k in to_exclude or v is None: continue - args[k] = _normalize_arg(k, v, no_normalize) + normalized = _normalize_arg(k, v, no_normalize) + if normalized is not None: + args[k] = normalized return args diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index 815913de..e71f1ba8 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -18,9 +18,10 @@ import httpx import pandas as pd +from dataretrieval._dates import DURATION_RE, coerce_to_list, read_date_range from dataretrieval._validation import render_options from dataretrieval.exceptions import DataRetrievalError, SkippedRatingWarning -from dataretrieval.ogc.dates import _DURATION_RE, _coerce_to_list, _format_api_dates +from dataretrieval.ogc.dates import _format_date_range from dataretrieval.ogc.errors import _raise_for_non_200 from dataretrieval.ogc.filters import _quote_cql_str from dataretrieval.ogc.requests import _check_monitoring_location_id @@ -177,16 +178,14 @@ def get_ratings( file_types = _as_list(file_type) _validate_file_types(file_types) _validate_time_no_duration(time) + time_range = read_date_range( + time, name="time", single_value_hint="an instant ('2020-01-01')" + ) time_str = ( - _format_api_dates( - time, - name="time", - single_value_hint="an instant ('2020-01-01')", - # The STAC firewall answers ``datetime=../2026-01-01`` with HTTP - # 403; STAC reads an empty start (``/2026-01-01``) as open. - open_start="", - ) - if time is not None + # The STAC firewall answers ``datetime=../2026-01-01`` with HTTP 403; + # STAC reads an empty start (``/2026-01-01``) as open. + _format_date_range(time_range, open_start="") + if time_range is not None else None ) @@ -226,10 +225,10 @@ def _validate_time_no_duration(time: str | list[str] | None) -> None: """Raise ValueError if ``time`` contains an ISO 8601 duration.""" if time is None: return - # ``_coerce_to_list`` keeps a lone ``date``/``Timestamp`` whole, where + # ``coerce_to_list`` keeps a lone ``date``/``Timestamp`` whole, where # ``_as_list`` would try to iterate it. - values = _coerce_to_list(time, "time") - if any(isinstance(v, str) and _DURATION_RE.match(v) for v in values): + values = coerce_to_list(time, "time") + if any(isinstance(v, str) and DURATION_RE.match(v) for v in values): raise ValueError( "ISO 8601 durations (e.g. 'P7D') are not supported in `time` " "for the rating-curve service. Provide a date or interval instead." diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index 0c977d9e..be854c58 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -246,3 +246,12 @@ Revisions: ``configuration.service_url`` and ``waterdata.endpoints`` already resolve. They are still endpoints in the glossary's sense. The STAC ratings search gets an ``Endpoint`` with the interval grammar, where its ``DateDialect`` is read. +- *The package-wide date range.* ``dataretrieval._dates`` is a dependency-free + leaf holding :class:`DateRange` and ``read_date_range``: an instant, a range + whose ``None`` bound is open, or a duration, with a naive bound meaning local + time. ``prepare_request_args`` reads every OGC date parameter into one when + a getter is called, and drops a range whose bounds are all open, as it drops + ``None``; ``get_ratings`` reads its ``time`` the same way. The OGC request + builder writes the value with ``ogc.dates._format_date_range``. Error + messages are unchanged; they are raised when the getter is called rather + than when the first request is built. diff --git a/tests/_dates_test.py b/tests/_dates_test.py new file mode 100644 index 00000000..60d4cf0f --- /dev/null +++ b/tests/_dates_test.py @@ -0,0 +1,50 @@ +"""Reading a date argument into a ``DateRange`` (ADR 0014). + +How each shape is written for the wire is tested with the OGC request builder +in ``waterdata_utils_test.py``. +""" + +import datetime + +import pandas as pd +import pytest + +from dataretrieval._dates import DateRange, read_date_range +from dataretrieval.ogc import prepare_request_args + +NAIVE = datetime.datetime(2024, 1, 1) # noqa: DTZ001 -- a naive bound is the case +UTC = datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) + + +@pytest.mark.parametrize( + "value, expected", + [ + ("2024-01-01", DateRange(start=NAIVE, instant=True)), + (datetime.date(2024, 1, 1), DateRange(start=NAIVE, instant=True)), + ("P7D", DateRange(duration="P7D")), + (["2024-01-01T00:00:00Z", None], DateRange(start=UTC)), + ("../2024-01-01", DateRange(end=NAIVE)), + ([pd.NaT, pd.Timestamp("2024-01-01")], DateRange(end=NAIVE)), + ], +) +def test_read_date_range_keeps_the_shape_the_caller_passed(value, expected): + """An instant, a duration, and a range with open bounds stay distinct: a + one-element instant is not a range with an open end.""" + assert read_date_range(value) == expected + + +@pytest.mark.parametrize("value", [None, pd.NaT, [None, ".."], "../.."]) +def test_read_date_range_returns_none_for_no_filter(value): + assert read_date_range(value) is None + + +def test_prepare_request_args_reads_date_arguments_on_entry(): + """A getter's date arguments are read when it is called, so an unreadable + one raises before any request is planned, and an all-open one is dropped + like ``None``.""" + args = prepare_request_args( + {"time": ["2024-01-01", None], "last_modified": [None, ".."]} + ) + assert args == {"time": DateRange(start=NAIVE)} + with pytest.raises(ValueError, match="^time could not be read"): + prepare_request_args({"time": "yesterday"}) diff --git a/tests/contracts/README.md b/tests/contracts/README.md index c8d24e1d..0fe4c68c 100644 --- a/tests/contracts/README.md +++ b/tests/contracts/README.md @@ -10,7 +10,7 @@ The suite uses four dependency-oriented layers without moving established tests: response parsing, and documented protocol behavior. - **Component** (`transport_test.py`, `waterdata_chunking_test.py`, `waterdata_queryables_test.py`, `waterdata_endpoints_test.py`, - `waterdata_properties_test.py`, `rdb_test.py`, `_csv_test.py`): one internal + `waterdata_properties_test.py`, `rdb_test.py`, `_csv_test.py`, `_dates_test.py`): one internal responsibility in isolation. - **Cross-component** (`architecture_test.py`, `headers_host_scoping_test.py`, `waterdata_progress_test.py`): dependency fitness functions and behavior that diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index d87929f4..8b0e6eea 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -15,7 +15,9 @@ import dataretrieval from dataretrieval import configuration +from dataretrieval._dates import read_date_range from dataretrieval._service import Endpoint +from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS from dataretrieval.ogc.requests import ( _check_monitoring_location_id, _normalize_str_iterable, @@ -71,16 +73,23 @@ ) -# The direct request-construction unit tests below bypass ``get_ogc_data``, -# which is what normally binds the Water Data base URL and dialect (the OGC -# package names no collection of its own). Bind them here the same way the -# engine does — explicitly, via ``functools.partial`` — so the tests exercise -# the real Water Data behavior (monitoring-locations -> POST/CQL2; daily -> -# date-only time args). -_construct_api_requests = functools.partial( - _construct_api_requests_explicit, - endpoint=Endpoint(SERVICE, OGC_API_URL, WATERDATA_DIALECT), -) +# The direct request-construction unit tests below bypass the getters and +# ``get_ogc_data``, which normally read the date arguments and bind the Water +# Data base URL and dialect (the OGC package names no collection of its own). +# Do both here so the tests exercise the real Water Data behavior +# (monitoring-locations -> POST/CQL2; daily -> date-only time args). +def _construct_api_requests(collection, **kwargs): + for key in _DATE_RANGE_PARAMS & kwargs.keys(): + read = read_date_range(kwargs.pop(key), name=key) + if read is not None: + kwargs[key] = read + return _construct_api_requests_explicit( + collection, + endpoint=Endpoint(SERVICE, OGC_API_URL, WATERDATA_DIALECT), + **kwargs, + ) + + _construct_cql_request = functools.partial( _construct_cql_request_explicit, base_url=OGC_API_URL ) @@ -627,7 +636,7 @@ def test_construct_api_requests_two_element_date_list_becomes_interval(): time=["2024-01-01", "2024-01-31"], ) assert req.method == "GET" - # `/` URL-encodes to %2F. Confirms _format_api_dates ran before the join. + # `/` URL-encodes to %2F. Confirms the date range was written before the join. assert "time=2024-01-01%2F2024-01-31" in str(req.url) diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 39991d0b..bba02d11 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -14,6 +14,7 @@ import dataretrieval.ogc.shaping as _shaping_module import dataretrieval.waterdata.stats as _stats_module import dataretrieval.waterdata.utils as _utils_module +from dataretrieval._dates import read_date_range from dataretrieval._service import Endpoint, Service from dataretrieval.exceptions import ( DataRetrievalError, @@ -23,7 +24,7 @@ TransientError, ) from dataretrieval.interruptions import ServiceInterrupted -from dataretrieval.ogc.dates import _format_api_dates +from dataretrieval.ogc.dates import _format_date_range from dataretrieval.ogc.engine import ( _next_req_url, _walk_pages, @@ -915,7 +916,16 @@ def test_type_cols_warning_is_singular_for_one_value(): _type_cols(df, _TYPING_DIALECT) -# --- _format_api_dates ------------------------------------------------------- +# --- read_date_range, then _format_date_range ------------------------------- + + +def _wire_dates(value, date=False, *, name="date input", open_start=".."): + """Read a date argument and write it for OGC, as a getter and the request + builder do in turn; ``None`` when there is no filter.""" + read = read_date_range(value, name=name) + if read is None: + return None + return _format_date_range(read, date=date, open_start=open_start) @pytest.mark.parametrize( @@ -963,25 +973,25 @@ def test_type_cols_warning_is_singular_for_one_value(): "open_ended_dotdot_datetime", ], ) -def test_format_api_dates(value, date, expected): - """``_format_api_dates`` normalizes ISO 8601 datetimes to UTC (dropping +def test_wire_dates(value, date, expected): + """``_wire_dates`` normalizes ISO 8601 datetimes to UTC (dropping fractional seconds, converting offsets), joins a pair into an interval, passes durations through unchanged, and renders a None endpoint as ``..``.""" - assert _format_api_dates(value, date=date) == expected + assert _wire_dates(value, date=date) == expected -def test_format_api_dates_passes_none_through(): +def test_wire_dates_passes_none_through(): """``None`` means "no date filter", not an empty one.""" - assert _format_api_dates(None) is None + assert _wire_dates(None) is None -def test_format_api_dates_treats_an_all_blank_sequence_as_no_filter(): +def test_wire_dates_treats_an_all_blank_sequence_as_no_filter(): """``[None, None]`` is an interval with no endpoints, which is no filter at all -- sending ``../..`` would be a query the service has to reject.""" - assert _format_api_dates([None, None]) is None - assert _format_api_dates(["", ""]) is None - assert _format_api_dates(["..", ".."]) is None + assert _wire_dates([None, None]) is None + assert _wire_dates(["", ""]) is None + assert _wire_dates(["..", ".."]) is None @pytest.mark.parametrize( @@ -1011,14 +1021,12 @@ def test_format_api_dates_treats_an_all_blank_sequence_as_no_filter(): "empty_end", ], ) -def test_format_api_dates_formats_an_interval_string_like_the_pair( - interval, pair, date -): +def test_wire_dates_formats_an_interval_string_like_the_pair(interval, pair, date): """A ``"start/end"`` string and the two-value list are two spellings of the same range, so they send the same value. The string used to be sent unchanged: a naive time was not converted from local time to UTC, and a date-only collection received times.""" - assert _format_api_dates(interval, date=date) == _format_api_dates(pair, date=date) + assert _wire_dates(interval, date=date) == _wire_dates(pair, date=date) @pytest.mark.parametrize( @@ -1029,16 +1037,14 @@ def test_format_api_dates_formats_an_interval_string_like_the_pair( ], ids=["offset_to_utc", "date_only"], ) -def test_format_api_dates_formats_each_side_of_an_interval_string( - interval, date, expected -): +def test_wire_dates_formats_each_side_of_an_interval_string(interval, date, expected): """Each side is formatted on its own, as a list element is.""" - assert _format_api_dates(interval, date=date) == expected + assert _wire_dates(interval, date=date) == expected -def test_format_api_dates_treats_an_all_open_interval_string_as_no_filter(): +def test_wire_dates_treats_an_all_open_interval_string_as_no_filter(): """``"../.."`` is the string spelling of ``[None, None]``.""" - assert _format_api_dates("../..") is None + assert _wire_dates("../..") is None @pytest.mark.parametrize( @@ -1051,21 +1057,21 @@ def test_format_api_dates_treats_an_all_open_interval_string_as_no_filter(): ], ids=["none", "dotdot", "dotdot_string", "empty_string"], ) -def test_format_api_dates_spells_an_open_start_as_the_api_does(value): +def test_wire_dates_spells_an_open_start_as_the_api_does(value): """Every spelling of an open start becomes ``..`` by default and the caller's ``open_start`` otherwise: NGWMN and STAC reject ``"../"`` (HTTP 403), and the Water Data API rejects an empty start (HTTP 400).""" - assert _format_api_dates(value) == "../2024-01-01T00:00:00Z" - assert _format_api_dates(value, open_start="") == "/2024-01-01T00:00:00Z" + assert _wire_dates(value) == "../2024-01-01T00:00:00Z" + assert _wire_dates(value, open_start="") == "/2024-01-01T00:00:00Z" -def test_format_api_dates_open_start_leaves_an_open_end_and_no_filter_alone(): +def test_wire_dates_open_start_leaves_an_open_end_and_no_filter_alone(): """``open_start`` changes only the start: an open end is ``..`` on every API, and a range with both ends open is still no filter.""" - open_end = _format_api_dates(["2024-01-01T00:00:00Z", None], open_start="") + open_end = _wire_dates(["2024-01-01T00:00:00Z", None], open_start="") assert open_end == "2024-01-01T00:00:00Z/.." - assert _format_api_dates([None, None], open_start="") is None - assert _format_api_dates("../..", open_start="") is None + assert _wire_dates([None, None], open_start="") is None + assert _wire_dates("../..", open_start="") is None @pytest.mark.parametrize( @@ -1088,12 +1094,12 @@ def test_format_api_dates_open_start_leaves_an_open_end_and_no_filter_alone(): "duration_open_end", ], ) -def test_format_api_dates_rejects_an_unreadable_interval_string(interval, message): +def test_read_date_range_rejects_an_unreadable_interval_string(interval, message): """An interval string used to be sent unchanged, so ``time="garbage/.."`` reached the service. It is now rejected like an unreadable list element, naming the caller's argument.""" with pytest.raises(ValueError, match=f"^{message}"): - _format_api_dates(interval, name="time") + read_date_range(interval, name="time") @pytest.mark.parametrize( @@ -1108,24 +1114,24 @@ def test_format_api_dates_rejects_an_unreadable_interval_string(interval, messag "range_start", ], ) -def test_format_api_dates_rejects_an_unreadable_bound(value): +def test_read_date_range_rejects_an_unreadable_bound(value): """An element that is not blank and matches no format used to return None, which the callers send as no filter at all. The message names the caller's argument and the bad value, and shows the forms that are accepted.""" with pytest.raises(ValueError) as excinfo: - _format_api_dates(value, name="last_modified") + read_date_range(value, name="last_modified") message = str(excinfo.value) assert message.startswith("last_modified could not be read as a date") assert "'2024-01-01'" in message assert "None for an open end" in message -def test_format_api_dates_rejects_more_than_two_values(): +def test_read_date_range_rejects_more_than_two_values(): """A date filter is an instant, a duration, or a closed interval. Three values is a caller who meant something else, and the message says which shapes exist rather than truncating without an error.""" with pytest.raises(ValueError) as excinfo: - _format_api_dates(["2024-01-01", "2024-06-01", "2024-12-31"], name="time") + read_date_range(["2024-01-01", "2024-06-01", "2024-12-31"], name="time") message = str(excinfo.value) assert "time takes at most 2 values, got 3" in message assert "closed interval" in message @@ -1135,9 +1141,9 @@ def test_the_duration_example_is_withheld_where_durations_are_rejected(): """``get_ratings`` refuses ISO 8601 durations, so the shared message must not offer 'P7D' on that path -- a caller following it would receive a second rejection.""" with pytest.raises(ValueError) as allowed: - _format_api_dates(["a", "b", "c"], name="time") + read_date_range(["a", "b", "c"], name="time") with pytest.raises(ValueError) as refused: - _format_api_dates( + read_date_range( ["a", "b", "c"], name="time", single_value_hint="an instant ('2020-01-01')" ) @@ -1146,23 +1152,23 @@ def test_the_duration_example_is_withheld_where_durations_are_rejected(): assert "an instant ('2020-01-01')" in str(refused.value) -def test_format_api_dates_names_the_callers_parameter(): +def test_read_date_range_names_the_callers_parameter(): """The subject must be the argument the caller passed. It used to be ``datetime_input`` -- a local of this private helper that no public getter accepts, so correcting the argument the message named sent an unrecognized parameter.""" with pytest.raises(TypeError, match="^time must be a date, a string"): - _format_api_dates({"2024-01-01": "ignored"}, name="time") + read_date_range({"2024-01-01": "ignored"}, name="time") -def test_format_api_dates_rejects_mapping(): +def test_read_date_range_rejects_mapping(): """`time={"2024-01-01": "x"}` would materialize as the keys list, accepting input the user clearly didn't intend. """ with pytest.raises( TypeError, match="date input must be a date, a string, or a sequence" ): - _format_api_dates({"2024-01-01": "ignored"}) + read_date_range({"2024-01-01": "ignored"}) @pytest.mark.parametrize( @@ -1210,36 +1216,36 @@ def test_format_api_dates_rejects_mapping(): "mixed_string_and_timestamp", ], ) -def test_format_api_dates_accepts_date_and_datetime_objects(value, date, expected): +def test_wire_dates_accepts_date_and_datetime_objects(value, date, expected): """``datetime.date``, ``datetime.datetime`` and ``pandas.Timestamp`` used to raise ``AttributeError`` (no ``endswith``) or ``TypeError`` (a lone datetime is not iterable) from inside the formatter. They are read like the equivalent string, and ``NaT`` is an open bound like ``None``.""" - assert _format_api_dates(value, date=date) == expected + assert _wire_dates(value, date=date) == expected @pytest.mark.parametrize("date", [True, False], ids=["date_only", "datetime"]) -def test_format_api_dates_reads_naive_objects_like_naive_strings(date): +def test_wire_dates_reads_naive_objects_like_naive_strings(date): """A naive ``datetime``/``Timestamp`` is local time, the same rule a naive string follows, so both spellings of one instant build the same filter in any time zone. A ``date`` is midnight of that day, like ``"2024-01-01"``.""" - as_string = _format_api_dates(["2024-01-01T10:00:00", None], date=date) + as_string = _wire_dates(["2024-01-01T10:00:00", None], date=date) naive_datetime = [datetime.datetime(2024, 1, 1, 10), None] # noqa: DTZ001 - assert _format_api_dates(naive_datetime, date=date) == as_string + assert _wire_dates(naive_datetime, date=date) == as_string naive_timestamp = [pd.Timestamp("2024-01-01T10:00:00"), None] - assert _format_api_dates(naive_timestamp, date=date) == as_string - assert _format_api_dates(datetime.date(2024, 1, 1), date=date) == ( - _format_api_dates("2024-01-01", date=date) + assert _wire_dates(naive_timestamp, date=date) == as_string + assert _wire_dates(datetime.date(2024, 1, 1), date=date) == ( + _wire_dates("2024-01-01", date=date) ) @pytest.mark.parametrize("value", [20240101, [20240101, None], 1.5]) -def test_format_api_dates_rejects_other_types_naming_the_argument(value): +def test_read_date_range_rejects_other_types_naming_the_argument(value): """A number is not a date. It used to fail with an ``AttributeError`` from inside the formatter; the message now names the caller's argument and the types it accepts.""" with pytest.raises(ValueError) as excinfo: - _format_api_dates(value, name="time") + read_date_range(value, name="time") message = str(excinfo.value) assert message.startswith("time could not be read as a date") assert "pandas.Timestamp" in message From c3a6f6337defaf523d46c846cf7754e882752719 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 17:27:02 -0500 Subject: [PATCH 15/22] refactor: one interval grammar for OGC and STAC, adjusted by DateDialect (ADR 0014) dataretrieval._intervals writes a DateRange as the ISO 8601 interval both standards define, adjusted by the endpoint's DateDialect: which parameters carry dates, date-only collections, parameters that keep a time, the open-start spelling, and whether a lone duration is sent. It replaces ogc.dates. OgcDialect holds the group as `dates`, and prepare_request_args takes it. get_ratings reaches STAC through an Endpoint with its own DateDialect and drops its duration check. Behavior change: a lone duration sent to NGWMN now raises ValueError before sending instead of failing with HTTP 400, because durations default to off, as in the standards. Live tests check open_start and durations against Water Data OGC, NGWMN, and STAC. Co-Authored-By: Claude Opus 5.5 (1M context) --- .importlinter | 3 + NEWS.md | 2 + dataretrieval/_intervals.py | 129 ++++++++++++++++++ dataretrieval/ngwmn.py | 7 +- dataretrieval/ogc/dates.py | 70 ---------- dataretrieval/ogc/engine.py | 3 +- dataretrieval/ogc/policy.py | 22 +-- dataretrieval/ogc/requests.py | 26 ++-- dataretrieval/waterdata/ratings.py | 41 +++--- dataretrieval/waterdata/utils.py | 23 +++- .../0014-services-endpoints-dialects.rst | 16 +++ docs/source/architecture/index.rst | 5 +- tests/_dates_test.py | 6 +- tests/ngwmn_test.py | 39 ++++++ tests/waterdata_chunking_test.py | 18 +-- tests/waterdata_ratings_test.py | 36 ++++- tests/waterdata_test.py | 32 ++++- tests/waterdata_utils_test.py | 16 ++- 18 files changed, 343 insertions(+), 151 deletions(-) create mode 100644 dataretrieval/_intervals.py delete mode 100644 dataretrieval/ogc/dates.py diff --git a/.importlinter b/.importlinter index 376d5a0e..473e3b40 100644 --- a/.importlinter +++ b/.importlinter @@ -27,6 +27,9 @@ layers = wateruse ngwmn | nldi | nwdc | nwis | streamstats | waterdata | wqp ogc +; The ISO 8601 interval grammar is a protocol OGC shares with STAC, which is +; not OGC, so it sits below ``ogc`` rather than inside it (ADR 0014). + _intervals utils _querying transport diff --git a/NEWS.md b/NEWS.md index ffaf162d..373b71ef 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**10/11/2026:** **Behavior change:** `ngwmn.get_water_level()` raises `ValueError` for a lone ISO 8601 duration such as `datetime="P7D"` before sending, instead of failing with the service's HTTP 400. Pass a start and an end, such as `datetime=["2024-01-01", "2024-01-08"]`. `waterdata.get_ratings()` already refused a duration; its message now names `time` and suggests that same form. The Water Data OGC getters still accept durations. + **10/09/2026:** Date arguments of the `waterdata` and `ngwmn` OGC getters (`time`, `begin`, `end`, `datetime`, `last_modified`) and of `waterdata.get_ratings()` accept `datetime.date`, `datetime.datetime`, and `pandas.Timestamp` values, alone or as either end of a range, such as `time=[df.index.min(), None]`. Each is read like the equivalent string: an aware value is converted to UTC, a naive one is read in local time, a `date` is midnight, and `NaT` is an open bound. **Bug fix:** these values used to raise `AttributeError` or `TypeError` without naming the argument. Any other type, such as a number, now raises `ValueError` naming the argument and the accepted types. **10/09/2026:** **Behavior change:** a date range passed as one `"start/end"` string, such as `time="2024-01-01T10:00:00/.."`, is now formatted like the list form `["2024-01-01T10:00:00", None]`, in the `waterdata` and `ngwmn` OGC getters and `waterdata.get_ratings()`. **Bug fix:** the string used to be sent unchanged, so the two spellings could select different data: a naive time was not converted from local time to UTC, an offset was not converted to `Z`, and a date-only collection such as `daily` received times. A side that is not a date (`time="2024-01-01/tomorrow"`) now raises `ValueError` before any request is sent, as does a string with more than one `/` or with a duration as either side (`"2024-01-01/P7D"`), which the services answered with HTTP 400. An empty side is an open bound, and `"../.."` means no date filter. A lone duration such as `"P7D"` is unchanged. **Bug fix:** a range with an open start, such as `[None, "2016-01-01"]` or `"../2016-01-01"`, failed with HTTP 403 in `ngwmn.get_water_level()` and `waterdata.get_ratings()`, because those services' firewalls reject `../`. Both now send an empty start (`"/2016-01-01"`), which both services read as open. The other Water Data getters still send `..`, because that service rejects an empty start. diff --git a/dataretrieval/_intervals.py b/dataretrieval/_intervals.py new file mode 100644 index 00000000..c6dd326b --- /dev/null +++ b/dataretrieval/_intervals.py @@ -0,0 +1,129 @@ +"""The ISO 8601 interval grammar OGC API Features and STAC share (ADR 0014). + +A getter reads a date argument into a :class:`~dataretrieval._dates.DateRange`; +this module writes that value as the ``datetime`` parameter of both standards +-- an instant, a ``start/end`` interval with ``..`` open bounds, or a duration +-- adjusted by the endpoint's :class:`DateDialect`. It names no service. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from zoneinfo import ZoneInfo + +from dataretrieval._dates import OPEN_BOUND, DateRange + + +@dataclass(frozen=True) +class DateDialect: + """How one endpoint writes dates differently from the interval standard. + + Each default is the standard, so ``DateDialect()`` describes an endpoint + that follows it. + + Attributes + ---------- + params : frozenset[str] + The parameters that carry a date range. The standard has one, + ``datetime``; an endpoint that filters other fields by date (Water Data's + ``time``, ``begin``, ``last_modified``) lists them all. + date_only_collections : frozenset[str] + Collections whose dates are written as ``YYYY-MM-DD`` rather than as a + UTC datetime. This is a choice rather than a server restriction: Water + Data's ``daily`` accepts either, but its values are calendar days, and + a naive bound converted to UTC could move to the neighboring day. + timed_params : frozenset[str] + Parameters written as a UTC datetime even in a date-only collection, + because they filter a timestamp (Water Data's ``last_modified``). + open_start : str + How the open start of a range is written: ``".."`` (``"../2024-01-01"``, + the default) or ``""`` (``"/2024-01-01"``). Both standards allow + either, but deployments differ. Observed 2026-10-11: Water Data OGC + answers the empty start with HTTP 400; the NGWMN and STAC firewalls + answer ``"../"`` with HTTP 403. + durations : bool + Whether a lone ISO 8601 duration (``"P7D"``, the last seven days) is + sent. Neither standard defines one, so the default is ``False`` and a + duration raises before sending. Observed 2026-10-11: Water Data OGC + accepts it; NGWMN and STAC answer HTTP 400. + """ + + params: frozenset[str] = frozenset({"datetime"}) + date_only_collections: frozenset[str] = frozenset() + timed_params: frozenset[str] = frozenset() + open_start: str = OPEN_BOUND + durations: bool = False + + +def single_value_hint(dialect: DateDialect) -> str: + """How a "too many values" message describes one acceptable value.""" + if dialect.durations: + return "an instant or a duration ('2020-01-01', 'P7D')" + return "an instant ('2020-01-01')" + + +def _write_bound(bound: datetime | None, *, date: bool) -> str: + """One bound: ``YYYY-MM-DD``, a UTC ``...Z`` datetime, or ``..``.""" + if bound is None: + return OPEN_BOUND + if date: + return bound.strftime("%Y-%m-%d") + # A naive bound is local time. Use ``.astimezone()`` rather than a fixed + # offset so each value is resolved against the DST rules for its own date. + aware = bound if bound.tzinfo is not None else bound.astimezone() + return aware.astimezone(ZoneInfo("UTC")).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def write_interval( + value: DateRange, + dialect: DateDialect, + *, + param: str, + collection: str | None = None, + name: str | None = None, +) -> str: + """Write a date range as the endpoint's *param* expects it. + + Parameters + ---------- + value : DateRange + What the caller meant, as read by + :func:`~dataretrieval._dates.read_date_range`. + dialect : DateDialect + The endpoint's date dialect. + param : str + The request parameter being written; decides, with *collection*, + whether its bounds are written date-only. + collection : str, optional + The collection the request targets, if any. + name : str, optional + The caller's spelling of the argument, the subject of the message when + a duration is refused. Defaults to *param*. + + Raises + ------ + ValueError + For a duration, unless the dialect allows durations. + """ + if value.duration is not None: + if not dialect.durations: + raise ValueError( + f"{name or param} cannot be an ISO 8601 duration for this " + f"service: {value.duration!r}. Pass a start and an end instead, " + "such as ['2024-01-01', '2024-01-08'], with None for an open " + "end (['2024-01-01', None])." + ) + return value.duration + date = ( + collection in dialect.date_only_collections + and param not in dialect.timed_params + ) + if value.instant: + return _write_bound(value.start, date=date) + start = ( + dialect.open_start + if value.start is None + else _write_bound(value.start, date=date) + ) + return f"{start}/{_write_bound(value.end, date=date)}" diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index 79cdab4f..d4c33c3d 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -24,6 +24,7 @@ import pandas as pd from dataretrieval import configuration as _configuration +from dataretrieval._intervals import DateDialect from dataretrieval._service import Endpoint, Service from dataretrieval.codes.states import apply_state from dataretrieval.configuration import ( @@ -89,7 +90,7 @@ # timestamped by ``sample_time`` (not the Water Data ``time``) and report # depths/levels in feet. Its firewall answers ``datetime=../2016-01-01`` with # HTTP 403, so an open start is sent empty (``/2016-01-01``), which it reads as -# open. +# open; see ``DateDialect.open_start``. NGWMN_DIALECT = OgcDialect( time_cols=frozenset({"sample_time"}), numerical_cols=frozenset( @@ -100,7 +101,7 @@ } ), sort_cols=("sample_time", "monitoring_location_id"), - open_start="", + dates=DateDialect(open_start=""), ) @@ -113,7 +114,7 @@ def _get(service: str, local_vars: dict[str, Any]) -> tuple[pd.DataFrame, BaseMe queryable = _STATE_QUERYABLE.get(service) if queryable is not None: apply_state(local_vars, to=queryable["to"], into=queryable["into"]) - args = prepare_request_args(local_vars) + args = prepare_request_args(local_vars, dates=NGWMN_DIALECT.dates) return get_ogc_data( args, service, diff --git a/dataretrieval/ogc/dates.py b/dataretrieval/ogc/dates.py deleted file mode 100644 index b877d765..00000000 --- a/dataretrieval/ogc/dates.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Writing a date range as an OGC time parameter. - -A getter reads each date argument into a :class:`~dataretrieval._dates.DateRange` -when it is called; this module writes that value in the wire form the API -expects -- an instant, a ``start/end`` interval with ``..`` open bounds, or a -duration. No I/O, no engine state. -""" - -from __future__ import annotations - -from datetime import datetime -from zoneinfo import ZoneInfo - -from dataretrieval._dates import OPEN_BOUND, DateRange - -# OGC API parameters that take a date/datetime value (single string, -# two-element range, or interval/duration string) rather than a multi-value -# string list. ``prepare_request_args`` reads them into a ``DateRange``, -# ``_construct_api_requests`` writes them with ``_format_date_range``, and the -# chunk planner never splits them. -_DATE_RANGE_PARAMS = frozenset( - {"datetime", "last_modified", "begin", "begin_utc", "end", "end_utc", "time"} -) - - -def _format_bound(bound: datetime | None, *, date: bool) -> str: - """Write one bound: ``YYYY-MM-DD``, a UTC ``...Z`` datetime, or ``..``.""" - if bound is None: - return OPEN_BOUND - if date: - return bound.strftime("%Y-%m-%d") - # A naive bound is local time. Use ``.astimezone()`` rather than a fixed - # offset so each value is resolved against the DST rules for its own date. - aware = bound if bound.tzinfo is not None else bound.astimezone() - return aware.astimezone(ZoneInfo("UTC")).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _format_date_range( - value: DateRange, *, date: bool = False, open_start: str = OPEN_BOUND -) -> str: - """Write a date range as an OGC API time parameter. - - Parameters - ---------- - value : DateRange - What the caller meant, as read by - :func:`~dataretrieval._dates.read_date_range`. - date : bool, optional - If True, write only the date (``YYYY-MM-DD``) of each bound. If False - (default), write each bound as a UTC datetime (``YYYY-MM-DDTHH:MM:SSZ``), - reading a naive bound in the local time zone. - open_start : str, optional - How the target API spells the open start of a range (``".."`` by - default). OGC API Features and STAC also allow an empty string, - ``"/2024-01-01"``, which an API whose firewall rejects ``"../"`` needs - instead; see :attr:`OgcDialect.open_start`. An open end is always - ``".."``. - - Returns - ------- - str - The duration unchanged, a single bound for an instant, or two bounds - joined by ``"/"``. - """ - if value.duration is not None: - return value.duration - if value.instant: - return _format_bound(value.start, date=date) - start = open_start if value.start is None else _format_bound(value.start, date=date) - return f"{start}/{_format_bound(value.end, date=date)}" diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index d385e586..e55cc349 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -5,8 +5,7 @@ pagination and sync dispatch are in :mod:`dataretrieval.transport`; request construction is in :mod:`~dataretrieval.ogc.requests`. The surrounding concerns are in sibling modules this one composes, each with its own reason -to change: :mod:`~dataretrieval.ogc.dates` (time-parameter marshalling), -:mod:`~dataretrieval.ogc.errors` (HTTP error mapping), and +to change: :mod:`~dataretrieval.ogc.errors` (HTTP error mapping) and :mod:`~dataretrieval.ogc.shaping` (GeoJSON features to DataFrame and result finalization). It is deliberately free of any Water-Data-specific constants so a sibling package (e.g. NGWMN) can use it without importing diff --git a/dataretrieval/ogc/policy.py b/dataretrieval/ogc/policy.py index 06fdf3ce..63d2a13c 100644 --- a/dataretrieval/ogc/policy.py +++ b/dataretrieval/ogc/policy.py @@ -2,8 +2,8 @@ This module is the one definition of the :class:`OgcDialect` type (per-API quirks the generic request builder needs) and OGC control validation. -It depends only on the stdlib, so any OGC submodule can import it without -creating cycles. +It depends only on the stdlib and the interval grammar's dialect type, so any +OGC submodule can import it without creating cycles. It names no API: which one an OGC call targets is the *adapter's* policy, supplied per call as an endpoint (ADR 0014). A default here would direct every @@ -17,6 +17,8 @@ import numbers from dataclasses import dataclass, field +from dataretrieval._intervals import DateDialect + def _require_positive_int( value: int, name: str, *, examples: str | None = None @@ -45,11 +47,6 @@ class OgcDialect: cql2_services : frozenset[str] Collections that don't accept comma-separated multi-value GET parameters and so must be queried via POST with a CQL2 JSON body. - date_only_services : frozenset[str] - Collections whose time arguments are rendered date-only - (``YYYY-MM-DD``) rather than as a full UTC datetime. The - ``last_modified`` parameter is always rendered as a full datetime - regardless of this set. time_cols : frozenset[str] Result columns to coerce to datetime when ``convert_type`` is set. Empty by default, so the generic engine holds no API-specific column list; each API @@ -60,17 +57,12 @@ class OgcDialect: Columns to sort the combined result by, in priority order. Sorting is applied only when the first (primary) column is present; any later columns also present are added as secondary keys. - open_start : str - How the API spells the open start of a date range: ``".."`` - (``"../2024-01-01"``, the default) or ``""`` (``"/2024-01-01"``). - OGC API Features allows both, but deployments differ in which they - accept: the Water Data API rejects the empty start with HTTP 400, and - the NGWMN firewall rejects ``"../"`` with HTTP 403. + dates : DateDialect + How the API's date parameters are read and written (ADR 0014). """ cql2_services: frozenset[str] = field(default_factory=frozenset) - date_only_services: frozenset[str] = field(default_factory=frozenset) time_cols: frozenset[str] = field(default_factory=frozenset) numerical_cols: frozenset[str] = field(default_factory=frozenset) sort_cols: tuple[str, ...] = field(default_factory=tuple) - open_start: str = ".." + dates: DateDialect = field(default_factory=DateDialect) diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index e9dce155..3e508a0d 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -17,7 +17,7 @@ import httpx from dataretrieval._dates import read_date_range -from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS, _format_date_range +from dataretrieval._intervals import DateDialect, single_value_hint, write_interval from dataretrieval.transport.http import default_headers as _default_headers if TYPE_CHECKING: @@ -177,11 +177,9 @@ def _construct_api_requests( """ service_url = _items_url(collection, endpoint.url) dialect = endpoint.dialect - for key in _DATE_RANGE_PARAMS & kwargs.keys(): - kwargs[key] = _format_date_range( - kwargs[key], - date=(collection in dialect.date_only_services and key != "last_modified"), - open_start=dialect.open_start, + for key in dialect.dates.params & kwargs.keys(): + kwargs[key] = write_interval( + kwargs[key], dialect.dates, param=key, collection=collection ) params, post_params = _partition_request_params( kwargs, use_cql2=collection in dialect.cql2_services @@ -309,14 +307,17 @@ def _normalize_arg( key: str, value: Any, no_normalize: frozenset[str], + dates: DateDialect, ) -> Any: """Normalize a single request argument value based on its key.""" if key == "monitoring_location_id": return _check_monitoring_location_id(value) if key == "properties": return _as_str_list(value, key) - if key in _DATE_RANGE_PARAMS: - return read_date_range(value, name=key) + if key in dates.params: + return read_date_range( + value, name=key, single_value_hint=single_value_hint(dates) + ) if ( key in no_normalize and isinstance(value, Iterable) @@ -332,14 +333,15 @@ def prepare_request_args( local_vars: dict[str, Any], exclude: set[str] | None = None, *, + dates: DateDialect, extra_no_normalize: frozenset[str] | set[str] = frozenset(), ) -> dict[str, Any]: """Build OGC request kwargs from a getter's ``locals()``. Internal control keys, caller-supplied exclusions, and ``None`` values - are omitted. Identifiers and properties are validated, date arguments are - read into a ``DateRange`` (and omitted when every bound is open), and other - iterables are normalized unless exempted. + are omitted. Identifiers and properties are validated, the parameters + ``dates`` lists are read into a ``DateRange`` (and omitted when every bound + is open), and other iterables are normalized unless exempted. ``extra_no_normalize`` *adds* to the engine's own :data:`_NO_NORMALIZE_PARAMS` rather than replacing it, so an adapter names @@ -355,7 +357,7 @@ def prepare_request_args( for k, v in local_vars.items(): if k in to_exclude or v is None: continue - normalized = _normalize_arg(k, v, no_normalize) + normalized = _normalize_arg(k, v, no_normalize, dates) if normalized is not None: args[k] = normalized return args diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index e71f1ba8..858031ff 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -18,10 +18,11 @@ import httpx import pandas as pd -from dataretrieval._dates import DURATION_RE, coerce_to_list, read_date_range +from dataretrieval._dates import read_date_range +from dataretrieval._intervals import DateDialect, single_value_hint, write_interval +from dataretrieval._service import Endpoint from dataretrieval._validation import render_options from dataretrieval.exceptions import DataRetrievalError, SkippedRatingWarning -from dataretrieval.ogc.dates import _format_date_range from dataretrieval.ogc.errors import _raise_for_non_200 from dataretrieval.ogc.filters import _quote_cql_str from dataretrieval.ogc.requests import _check_monitoring_location_id @@ -38,6 +39,13 @@ __all__ = ["get_ratings"] +# The STAC search's departures from the interval standard (ADR 0014). Its +# firewall answers ``datetime=../2026-01-01T00:00:00Z`` with HTTP 403, and it +# reads an empty start (``/2026-01-01T00:00:00Z``) as open; it refuses a lone +# duration with HTTP 400, as the standard does. +STAC_DATES = DateDialect(open_start="") + + RATING_FILE_TYPE = Literal["exsa", "base", "corr"] _VALID_FILE_TYPES = get_args(RATING_FILE_TYPE) @@ -177,14 +185,14 @@ def get_ratings( monitoring_location_id = _check_monitoring_location_id(monitoring_location_id) file_types = _as_list(file_type) _validate_file_types(file_types) - _validate_time_no_duration(time) + # Built when the call starts, so an enclosing ``configure(base_url=...)`` + # applies to the search (ADR 0014). + endpoint = Endpoint(SERVICE, ratings_catalog_url(), STAC_DATES) time_range = read_date_range( - time, name="time", single_value_hint="an instant ('2020-01-01')" + time, name="time", single_value_hint=single_value_hint(endpoint.dialect) ) time_str = ( - # The STAC firewall answers ``datetime=../2026-01-01`` with HTTP 403; - # STAC reads an empty start (``/2026-01-01``) as open. - _format_date_range(time_range, open_start="") + write_interval(time_range, endpoint.dialect, param="datetime", name="time") if time_range is not None else None ) @@ -193,7 +201,7 @@ def get_ratings( server_file_type = file_types[0] if len(file_types) == 1 else None filter_str = _build_filter(monitoring_location_id, server_file_type) - features = _search(filter_str, time_str, bbox, limit, ssl_check) + features = _search(endpoint.url, filter_str, time_str, bbox, limit, ssl_check) if not download_and_parse: return features @@ -221,20 +229,6 @@ def _validate_file_types(file_types: list[str]) -> None: ) -def _validate_time_no_duration(time: str | list[str] | None) -> None: - """Raise ValueError if ``time`` contains an ISO 8601 duration.""" - if time is None: - return - # ``coerce_to_list`` keeps a lone ``date``/``Timestamp`` whole, where - # ``_as_list`` would try to iterate it. - values = coerce_to_list(time, "time") - if any(isinstance(v, str) and DURATION_RE.match(v) for v in values): - raise ValueError( - "ISO 8601 durations (e.g. 'P7D') are not supported in `time` " - "for the rating-curve service. Provide a date or interval instead." - ) - - def _filter_features_by_type( features: list[dict[str, Any]], file_types: list[str] ) -> list[dict[str, Any]]: @@ -264,6 +258,7 @@ def _build_filter( def _search( + catalog_url: str, filter_str: str | None, time_str: str | None, bbox: list[float] | None, @@ -288,7 +283,7 @@ def _search( if bbox is not None: query_params["bbox"] = ",".join(map(str, bbox)) - url = f"{ratings_catalog_url()}/search" + url = f"{catalog_url}/search" req = httpx.Request("GET", url, params=query_params, headers=_default_headers(url)) def parse_response(resp: httpx.Response) -> tuple[pd.DataFrame, str | None]: diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index 89d830f8..2ca816fe 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -23,6 +23,7 @@ import pandas as pd from dataretrieval._deprecation import warn_deprecated +from dataretrieval._intervals import DateDialect from dataretrieval._service import Endpoint from dataretrieval.codes.states import apply_state from dataretrieval.credentials import refuse_credential_keywords @@ -86,7 +87,22 @@ # ``2005``), and no one datetime parse keeps all three, so it stays as sent. WATERDATA_DIALECT = OgcDialect( cql2_services=frozenset({"combined-metadata", "monitoring-locations"}), - date_only_services=frozenset({"daily"}), + dates=DateDialect( + params=frozenset( + { + "datetime", + "last_modified", + "begin", + "begin_utc", + "end", + "end_utc", + "time", + } + ), + date_only_collections=frozenset({"daily"}), + timed_params=frozenset({"last_modified"}), + durations=True, + ), time_cols=frozenset( { "begin", @@ -183,7 +199,10 @@ def _get_args( """ _flatten_queryables(local_vars) args = prepare_request_args( - local_vars, exclude, extra_no_normalize=_NO_NORMALIZE_PARAMS + local_vars, + exclude, + dates=WATERDATA_DIALECT.dates, + extra_no_normalize=_NO_NORMALIZE_PARAMS, ) for name in _JSON_ARRAY_PARAMS & args.keys(): # An empty list is dropped from the request downstream, as for any filter. diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index be854c58..862378a1 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -255,3 +255,19 @@ Revisions: builder writes the value with ``ogc.dates._format_date_range``. Error messages are unchanged; they are raised when the getter is called rather than when the first request is built. +- *The shared interval grammar and* ``DateDialect``. ``dataretrieval._intervals`` + holds ``DateDialect`` and ``write_interval``, and sits below ``ogc`` because + STAC is not OGC; ``ogc.dates`` is removed. ``DateDialect`` has five fields: + ``params``, ``date_only_collections``, ``timed_params``, ``open_start``, and + ``durations``, whose default is ``False`` because neither standard defines a + lone duration. ``OgcDialect.dates`` holds the group, and + ``prepare_request_args`` takes it, so the reading side knows which parameters + carry dates. ``get_ratings`` builds an ``Endpoint`` whose dialect is a + ``DateDialect`` alone; STAC has no other dialect fields. Live tests cover + ``open_start`` and ``durations`` for each of the three endpoints. + ``date_only_collections`` and ``timed_params`` have none: they record what a + ``daily`` value means rather than what a server refuses, since ``daily`` + accepts full datetimes too (observed 2026-10-11). Also observed: STAC reads + an empty start as open only before an RFC 3339 datetime + (``/2026-01-01T00:00:00Z``); ``/2026-01-01`` is HTTP 400. ``get_ratings`` + always writes datetimes. diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index a71518f0..e02c6526 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -118,8 +118,9 @@ Shared components ``planning`` determines chunk boundaries; ``chunking`` connects those plans to the shared fan-out executor and retains compatibility aliases; ``interruptions`` retains its deprecated public import path; and - ``shaping``, ``dates``, ``filters``, and ``errors`` isolate their named - protocol concerns. The full runtime OGC graph, including the facade, is + ``shaping``, ``filters``, and ``errors`` isolate their named protocol + concerns. Date parameters are written by ``dataretrieval._intervals``, the + interval grammar OGC shares with STAC. The full runtime OGC graph, including the facade, is acyclic -- enforced by the package-wide fitness function in ``tests/architecture_test.py``. diff --git a/tests/_dates_test.py b/tests/_dates_test.py index 60d4cf0f..d323d68a 100644 --- a/tests/_dates_test.py +++ b/tests/_dates_test.py @@ -10,6 +10,7 @@ import pytest from dataretrieval._dates import DateRange, read_date_range +from dataretrieval._intervals import DateDialect from dataretrieval.ogc import prepare_request_args NAIVE = datetime.datetime(2024, 1, 1) # noqa: DTZ001 -- a naive bound is the case @@ -42,9 +43,10 @@ def test_prepare_request_args_reads_date_arguments_on_entry(): """A getter's date arguments are read when it is called, so an unreadable one raises before any request is planned, and an all-open one is dropped like ``None``.""" + dates = DateDialect(params=frozenset({"time", "last_modified"})) args = prepare_request_args( - {"time": ["2024-01-01", None], "last_modified": [None, ".."]} + {"time": ["2024-01-01", None], "last_modified": [None, ".."]}, dates=dates ) assert args == {"time": DateRange(start=NAIVE)} with pytest.raises(ValueError, match="^time could not be read"): - prepare_request_args({"time": "yesterday"}) + prepare_request_args({"time": "yesterday"}, dates=dates) diff --git a/tests/ngwmn_test.py b/tests/ngwmn_test.py index deeabb7a..a3166dd2 100644 --- a/tests/ngwmn_test.py +++ b/tests/ngwmn_test.py @@ -11,6 +11,7 @@ cannot tell us, because the mock is what would need updating. """ +import datetime import re from urllib.parse import parse_qs, urlsplit @@ -648,3 +649,41 @@ def queryables(collection): providers_q = queryables("providers") assert "state_name" in sites_q and "state" not in sites_q, sites_q assert "state" in providers_q and "state_name" not in providers_q, providers_q + + +def test_get_water_level_refuses_a_duration_before_sending(httpx_mock): + """NGWMN answers a lone duration with HTTP 400, so it raises locally.""" + with pytest.raises(ValueError, match=r"^datetime cannot be an ISO 8601 duration"): + ngwmn.get_water_level(monitoring_location_id=_SITE, datetime="P7D") + assert httpx_mock.get_requests() == [] + + +@pytest.mark.live +def test_date_dialect_matches_the_service(): + """Each ``NGWMN_DIALECT.dates`` field against the running service: the + request the field prevents fails, and the one it produces succeeds. + + Kept live on purpose: the 403 comes from a firewall the API does not + describe, so only the running service can show it has changed. + """ + import httpx + + from dataretrieval._dates import DateRange + from dataretrieval._intervals import write_interval + from dataretrieval.ngwmn import NGWMN_DIALECT, NGWMN_OGC_API_URL + from dataretrieval.transport.http import default_headers + + url = f"{NGWMN_OGC_API_URL}/collections/waterLevelObs/items" + + def status(value): + params = {"monitoring_location_id": _OTHER_SITE, "limit": 1, "datetime": value} + return httpx.get( + url, params=params, headers=default_headers(url), timeout=60 + ).status_code + + # open_start: the standard's "../" is refused; the empty start is read as open. + end = DateRange(end=datetime.datetime(2016, 1, 1, tzinfo=datetime.timezone.utc)) + assert status("../2016-01-01T00:00:00Z") == 403 + assert status(write_interval(end, NGWMN_DIALECT.dates, param="datetime")) == 200 + # durations=False: a lone duration is refused, so none is sent. + assert status("P7D") == 400 diff --git a/tests/waterdata_chunking_test.py b/tests/waterdata_chunking_test.py index b7bbbf7a..5e6669c4 100644 --- a/tests/waterdata_chunking_test.py +++ b/tests/waterdata_chunking_test.py @@ -63,7 +63,6 @@ multi_value_chunked, parallel_chunks, ) -from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS from dataretrieval.ogc.planning import ( _LIST_SEP, _NEVER_CHUNK, @@ -87,7 +86,7 @@ ) from dataretrieval.utils import HTTPX_DEFAULTS from dataretrieval.waterdata.endpoints import SERVICE -from dataretrieval.waterdata.utils import OGC_API_URL +from dataretrieval.waterdata.utils import OGC_API_URL, WATERDATA_DIALECT # The joint-planner stress test uses the real request builder; bind the # target API the way ``get_ogc_data`` does — explicitly, via ``partial``. @@ -153,17 +152,12 @@ def _fake_build(*, base=200, **kwargs): def test_never_chunk_covers_all_date_range_params(): - """``_NEVER_CHUNK`` and ``_DATE_RANGE_PARAMS`` are maintained in - separate modules (chunker vs request builder) for layering reasons, - but every date-range param MUST be excluded from chunking — a - range value isn't an enumerable set to split. Guard against drift: - adding a new param to ``_DATE_RANGE_PARAMS`` without also adding - it to ``_NEVER_CHUNK`` would let the chunker try to - comma-join an interval string.""" - missing = _DATE_RANGE_PARAMS - _NEVER_CHUNK + """Every parameter the Water Data dialect reads as a date range MUST be + excluded from chunking -- a range value isn't an enumerable set to split.""" + missing = WATERDATA_DIALECT.dates.params - _NEVER_CHUNK assert not missing, ( - f"_DATE_RANGE_PARAMS contains entries not in _NEVER_CHUNK: " - f"{sorted(missing)}. Add them to planning._NEVER_CHUNK." + f"Date parameters not in _NEVER_CHUNK: {sorted(missing)}. " + "Add them to planning._NEVER_CHUNK." ) diff --git a/tests/waterdata_ratings_test.py b/tests/waterdata_ratings_test.py index ccc2ebce..186b3e40 100644 --- a/tests/waterdata_ratings_test.py +++ b/tests/waterdata_ratings_test.py @@ -44,8 +44,9 @@ def test_get_ratings_rejects_invalid_file_type(): def test_get_ratings_rejects_iso_8601_duration_in_time(): - """STAC ratings doesn't accept ISO 8601 durations; raise a clear error.""" - with pytest.raises(ValueError, match=r"durations.*not supported"): + """STAC answers a lone duration with HTTP 400, so it raises before sending, + naming the caller's argument rather than the wire's ``datetime``.""" + with pytest.raises(ValueError, match=r"^time cannot be an ISO 8601 duration"): get_ratings( monitoring_location_id="USGS-01104475", time="P7D", @@ -438,3 +439,34 @@ def test_stac_next_link_strips_embedded_credentials(httpx_mock): followed = httpx_mock.get_requests()[1] assert followed.url.userinfo == b"" assert followed.headers.get("Authorization") is None + + +@pytest.mark.live +def test_stac_date_dialect_matches_the_service(): + """Each ``STAC_DATES`` field against the running search: the request the + field prevents fails, and the one it produces succeeds. The 403 comes from + a firewall the API does not describe, so only the running service shows it.""" + import datetime + + import httpx + + from dataretrieval._dates import DateRange + from dataretrieval._intervals import write_interval + from dataretrieval.transport.http import default_headers + from dataretrieval.waterdata.endpoints import ratings_catalog_url + from dataretrieval.waterdata.ratings import STAC_DATES + + url = f"{ratings_catalog_url()}/search" + + def status(value): + params = {"limit": 1, "datetime": value} + return httpx.get( + url, params=params, headers=default_headers(url), timeout=60 + ).status_code + + # open_start: the standard's "../" is refused; the empty start is read as open. + end = DateRange(end=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)) + assert status("../2026-01-01T00:00:00Z") == 403 + assert status(write_interval(end, STAC_DATES, param="datetime")) == 200 + # durations=False: a lone duration is refused, so none is sent. + assert status("P7D") == 400 diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 8b0e6eea..2d31026d 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -17,7 +17,6 @@ from dataretrieval import configuration from dataretrieval._dates import read_date_range from dataretrieval._service import Endpoint -from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS from dataretrieval.ogc.requests import ( _check_monitoring_location_id, _normalize_str_iterable, @@ -79,7 +78,7 @@ # Do both here so the tests exercise the real Water Data behavior # (monitoring-locations -> POST/CQL2; daily -> date-only time args). def _construct_api_requests(collection, **kwargs): - for key in _DATE_RANGE_PARAMS & kwargs.keys(): + for key in WATERDATA_DIALECT.dates.params & kwargs.keys(): read = read_date_range(kwargs.pop(key), name=key) if read is not None: kwargs[key] = read @@ -1915,3 +1914,32 @@ def test_get_reference_table_docstring_lists_every_collection(): doc = get_reference_table.__doc__ or "" missing = [c for c in get_args(METADATA_COLLECTIONS) if f'"{c}"' not in doc] assert not missing, f"collections served but undocumented: {missing}" + + +@pytest.mark.live +def test_ogc_date_dialect_matches_the_service(): + """The ``WATERDATA_DIALECT.dates`` fields that record server behavior, + against the running API: it accepts the request each field produces and + refuses the one ``open_start`` prevents.""" + import httpx + + from dataretrieval._dates import DateRange + from dataretrieval._intervals import write_interval + from dataretrieval.transport.http import default_headers + from dataretrieval.waterdata.endpoints import ogc_api_url + + url = f"{ogc_api_url()}/collections/daily/items" + dates = WATERDATA_DIALECT.dates + + def status(value): + params = {"monitoring_location_id": "USGS-05427718", "limit": 1, "time": value} + return httpx.get( + url, params=params, headers=default_headers(url), timeout=60 + ).status_code + + # open_start: the default ".." is accepted; the empty start is refused. + end = DateRange(end=datetime.datetime(2024, 1, 3, tzinfo=datetime.timezone.utc)) + assert status(write_interval(end, dates, param="time", collection="daily")) == 200 + assert status("/2024-01-03") == 400 + # durations=True: a lone duration is accepted, so it is sent. + assert status(write_interval(DateRange(duration="P7D"), dates, param="time")) == 200 diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index bba02d11..6e2271fb 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -15,6 +15,7 @@ import dataretrieval.waterdata.stats as _stats_module import dataretrieval.waterdata.utils as _utils_module from dataretrieval._dates import read_date_range +from dataretrieval._intervals import DateDialect, write_interval from dataretrieval._service import Endpoint, Service from dataretrieval.exceptions import ( DataRetrievalError, @@ -24,7 +25,6 @@ TransientError, ) from dataretrieval.interruptions import ServiceInterrupted -from dataretrieval.ogc.dates import _format_date_range from dataretrieval.ogc.engine import ( _next_req_url, _walk_pages, @@ -920,12 +920,20 @@ def test_type_cols_warning_is_singular_for_one_value(): def _wire_dates(value, date=False, *, name="date input", open_start=".."): - """Read a date argument and write it for OGC, as a getter and the request - builder do in turn; ``None`` when there is no filter.""" + """Read a date argument and write it as an interval, as a getter and the + request builder do in turn; ``None`` when there is no filter. ``date`` + targets a date-only collection.""" read = read_date_range(value, name=name) if read is None: return None - return _format_date_range(read, date=date, open_start=open_start) + dialect = DateDialect( + date_only_collections=frozenset({"days"}), + open_start=open_start, + durations=True, + ) + return write_interval( + read, dialect, param=name, collection="days" if date else "instants" + ) @pytest.mark.parametrize( From aef4fce522a6951a3938f682a7c63561093d5e4d Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 17:31:03 -0500 Subject: [PATCH 16/22] refactor(ogc): the OGC code names nothing a dialect declares (ADR 0014) OgcDialect groups its fields by the code that reads them: `dates` and `shaping` (ShapingDialect: time, numeric, sort, and extra id columns). extra_id_cols moves from a get_ogc_data argument into the shaping group, and cql2_services is renamed cql2_collections, since it lists collections. The chunk planner drops its list of Water Data date parameters: a date argument is a DateRange, never a list, so it is never split. A fitness function fails if a string a shipped dialect lists appears as a literal in ogc or _intervals. monitoring_location_id, a recorded deviation, is the one allowlisted entry. The normalizers' "value" default name is removed; every caller passes its argument's name. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 2 +- CONTEXT.md | 6 +- dataretrieval/ngwmn.py | 27 +++++--- dataretrieval/ogc/__init__.py | 6 +- dataretrieval/ogc/engine.py | 6 -- dataretrieval/ogc/planning.py | 13 +--- dataretrieval/ogc/policy.py | 45 ++++++++++---- dataretrieval/ogc/requests.py | 6 +- dataretrieval/ogc/shaping.py | 15 +++-- dataretrieval/waterdata/utils.py | 54 ++++++++-------- .../0014-services-endpoints-dialects.rst | 16 ++++- tests/architecture_test.py | 61 +++++++++++++++++++ tests/waterdata_chunking_test.py | 21 ++++--- tests/waterdata_utils_test.py | 8 +-- 14 files changed, 187 insertions(+), 99 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7579e89f..ccde668c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,7 +132,7 @@ raise states the problem and then the action that fixes it, in that order. wrote `None` into the URL and returned an empty frame. Run the corrected call against the real service; wording review does not catch these. - Shared checks take the caller's argument name. `_validate_data_source`, - `_format_api_dates`, and `require_one_of` all accept a `name=` so the subject + `read_date_range`, and `require_one_of` all accept a `name=` so the subject of the message is the argument that was passed. A helper that hard-codes one noun reports the wrong parameter the moment a second call site reuses it. - Prefer raising over returning something empty when the library cannot distinguish diff --git a/CONTEXT.md b/CONTEXT.md index fca8a8e6..156172ff 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -331,9 +331,9 @@ entry (ADR 0013). OGC internals, the Water Data wrappers, and all eleven typed getters now say `collection`; `waterdata.get_cql` takes `collection`; and the type alias is `WATERDATA_COLLECTIONS`. `service=` on `get_cql` and the `WATERDATA_SERVICES` - alias remain — a deprecated keyword and a permanent alias respectively. Two - `OgcDialect` fields also still say it: `cql2_services` and - `date_only_services` are keyed by collection. + alias remain — a deprecated keyword and a permanent alias respectively. The + dialect fields keyed by collection now say so too: `cql2_collections` and + `DateDialect.date_only_collections`. `service` still means the external system in `transport` and `progress`, where it labels a progress line. That usage is correct. diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index d4c33c3d..dd92c9e6 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -36,7 +36,12 @@ _Retrying, ) from dataretrieval.credentials import WATERDATA_BASE_URL -from dataretrieval.ogc import OgcDialect, get_ogc_data, prepare_request_args +from dataretrieval.ogc import ( + OgcDialect, + ShapingDialect, + get_ogc_data, + prepare_request_args, +) if TYPE_CHECKING: from dataretrieval._response_metadata import BaseMetadata @@ -92,16 +97,18 @@ # HTTP 403, so an open start is sent empty (``/2016-01-01``), which it reads as # open; see ``DateDialect.open_start``. NGWMN_DIALECT = OgcDialect( - time_cols=frozenset({"sample_time"}), - numerical_cols=frozenset( - { - "water_depth_below_land_surface_ft", - "water_level_above_site_datum_ft", - "water_level_above_navd88_ft", - } - ), - sort_cols=("sample_time", "monitoring_location_id"), dates=DateDialect(open_start=""), + shaping=ShapingDialect( + time_cols=frozenset({"sample_time"}), + numerical_cols=frozenset( + { + "water_depth_below_land_surface_ft", + "water_level_above_site_datum_ft", + "water_level_above_navd88_ft", + } + ), + sort_cols=("sample_time", "monitoring_location_id"), + ), ) diff --git a/dataretrieval/ogc/__init__.py b/dataretrieval/ogc/__init__.py index 1f43cb92..0e70e72b 100644 --- a/dataretrieval/ogc/__init__.py +++ b/dataretrieval/ogc/__init__.py @@ -2,7 +2,8 @@ The public facade exposes only the minimal collection-adapter seam: -- :class:`OgcDialect` — per-API request/response quirks. +- :class:`OgcDialect` — how an API departs from OGC API Features, with its + :class:`ShapingDialect` group of result-shaping fields. - :func:`prepare_request_args` — normalize caller kwargs for the engine. - :func:`get_ogc_data` — full orchestrated OGC fetch (chunking + pagination), including verbatim-CQL2 queries via its ``cql_body`` parameter. @@ -14,11 +15,12 @@ """ from dataretrieval.ogc.engine import get_ogc_data -from dataretrieval.ogc.policy import OgcDialect +from dataretrieval.ogc.policy import OgcDialect, ShapingDialect from dataretrieval.ogc.requests import prepare_request_args __all__ = [ "OgcDialect", + "ShapingDialect", "get_ogc_data", "prepare_request_args", ] diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index e55cc349..47027b69 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -17,7 +17,6 @@ passed explicitly (no collection map is defined here). * ``endpoint`` — the API to target, its service, and its :class:`OgcDialect` (ADR 0014). -* ``extra_id_cols`` — synthetic id columns to move to the end of a result. """ from __future__ import annotations @@ -220,7 +219,6 @@ def get_ogc_data( endpoint: Endpoint[OgcDialect], spatial: bool, max_rows: int | None = None, - extra_id_cols: frozenset[str] | set[str] = frozenset(), cql_body: str | None = None, ) -> tuple[pd.DataFrame, BaseMetadata]: """ @@ -253,9 +251,6 @@ def get_ogc_data( Whether this collection's result contract includes feature geometry. The adapter supplies this semantic fact; ``skip_geometry`` and geopandas availability then select the concrete frame representation. - extra_id_cols : set or frozenset, optional - Synthetic id columns to move to the end of a result frame (see - :func:`_arrange_cols`). Defaults to an empty set. cql_body : str, optional A verbatim CQL2 JSON body to POST against ``collection`` instead of building the query from ``args``. With a body, only the @@ -320,7 +315,6 @@ def get_ogc_data( geopd=geopd, include_geometry=include_geometry, max_rows=max_rows, - extra_id_cols=extra_id_cols, endpoint=endpoint, ) diff --git a/dataretrieval/ogc/planning.py b/dataretrieval/ogc/planning.py index 0f7ae045..193502d4 100644 --- a/dataretrieval/ogc/planning.py +++ b/dataretrieval/ogc/planning.py @@ -37,20 +37,13 @@ # Any list-shaped kwarg with >1 element is chunked (comma-joined per sub-list in the # URL); ~90 OGC params qualify, so the few exceptions are denylisted rather than # maintaining a growing allowlist. Excluded because: ``properties`` defines the column -# schema; ``bbox`` is a fixed coord tuple; date/time params are intervals, not -# enumerable sets; ``filter`` is handled as its own OR-axis in ``_extract_axes``; and -# ``limit`` / ``skip_geometry`` / ``filter_lang`` are scalar by contract. +# schema; ``bbox`` is a fixed coord tuple; ``filter`` is handled as its own OR-axis in +# ``_extract_axes``; and ``limit`` / ``skip_geometry`` / ``filter_lang`` are scalar by +# contract. A date parameter needs no entry: it arrives as a ``DateRange``, not a list. _NEVER_CHUNK = frozenset( { "properties", "bbox", - "datetime", - "last_modified", - "begin", - "begin_utc", - "end", - "end_utc", - "time", "filter", "filter_lang", "limit", diff --git a/dataretrieval/ogc/policy.py b/dataretrieval/ogc/policy.py index 63d2a13c..9847f3ca 100644 --- a/dataretrieval/ogc/policy.py +++ b/dataretrieval/ogc/policy.py @@ -39,30 +39,49 @@ def _require_positive_int( @dataclass(frozen=True) -class OgcDialect: - """Per-API differences the generic request builder must handle. +class ShapingDialect: + """How an API's results are shaped into a frame: the fields the result + finalizer reads together. Each default leaves the result as sent. Attributes ---------- - cql2_services : frozenset[str] - Collections that don't accept comma-separated multi-value GET - parameters and so must be queried via POST with a CQL2 JSON body. time_cols : frozenset[str] - Result columns to coerce to datetime when ``convert_type`` is set. Empty by - default, so the generic engine holds no API-specific column list; each API - supplies its own. + Result columns to coerce to datetime when ``convert_type`` is set. numerical_cols : frozenset[str] Result columns to coerce to numeric when ``convert_type`` is set. sort_cols : tuple[str, ...] Columns to sort the combined result by, in priority order. Sorting is applied only when the first (primary) column is present; any later columns also present are added as secondary keys. + extra_id_cols : frozenset[str] + Synthetic per-record id columns, with no meaning to a user, moved to + the end of a result frame when ``properties`` was not given. + """ + + time_cols: frozenset[str] = frozenset() + numerical_cols: frozenset[str] = frozenset() + sort_cols: tuple[str, ...] = () + extra_id_cols: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class OgcDialect: + """How one OGC API departs from OGC API Features (ADR 0014). + + Each default is the standard, so ``OgcDialect()`` describes an API that + follows it. + + Attributes + ---------- + cql2_collections : frozenset[str] + Collections that don't accept comma-separated multi-value GET + parameters and so must be queried via POST with a CQL2 JSON body. dates : DateDialect - How the API's date parameters are read and written (ADR 0014). + How the API's date parameters are read and written. + shaping : ShapingDialect + How the API's results are shaped into a frame. """ - cql2_services: frozenset[str] = field(default_factory=frozenset) - time_cols: frozenset[str] = field(default_factory=frozenset) - numerical_cols: frozenset[str] = field(default_factory=frozenset) - sort_cols: tuple[str, ...] = field(default_factory=tuple) + cql2_collections: frozenset[str] = frozenset() dates: DateDialect = field(default_factory=DateDialect) + shaping: ShapingDialect = field(default_factory=ShapingDialect) diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 3e508a0d..1576b334 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -182,7 +182,7 @@ def _construct_api_requests( kwargs[key], dialect.dates, param=key, collection=collection ) params, post_params = _partition_request_params( - kwargs, use_cql2=collection in dialect.cql2_services + kwargs, use_cql2=collection in dialect.cql2_collections ) _ogc_query_params( @@ -246,7 +246,7 @@ def _construct_cql_request( def _normalize_str_iterable( value: str | Iterable[str] | None, - param_name: str = "value", + param_name: str, ) -> str | list[str] | None: """Validate that ``value`` is None, a string, or an iterable of strings.""" if value is None: @@ -271,7 +271,7 @@ def _normalize_str_iterable( def _as_str_list( value: str | Iterable[str] | None, - param_name: str = "value", + param_name: str, ) -> list[str] | None: """Normalize ``value`` to ``list[str]`` (``None`` passes through).""" normalized = _normalize_str_iterable(value, param_name) diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index 06973542..72c0aa8c 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: from dataretrieval._service import Endpoint - from dataretrieval.ogc.policy import OgcDialect + from dataretrieval.ogc.policy import OgcDialect, ShapingDialect try: import geopandas as gpd @@ -274,7 +274,7 @@ def _arrange_cols( return df.loc[:, [col for col in local_properties if col in df.columns]] -def _type_cols(df: pd.DataFrame, dialect: OgcDialect) -> pd.DataFrame: +def _type_cols(df: pd.DataFrame, dialect: ShapingDialect) -> pd.DataFrame: """ Casts columns into appropriate types per the API ``dialect``. @@ -282,7 +282,7 @@ def _type_cols(df: pd.DataFrame, dialect: OgcDialect) -> pd.DataFrame: ---------- df : pd.DataFrame The input DataFrame. - dialect : OgcDialect + dialect : ShapingDialect Supplies ``time_cols`` / ``numerical_cols`` — which columns to coerce to datetime/numeric. The engine itself holds no API-specific column list. @@ -336,7 +336,7 @@ def _warn_unparsed(raw: pd.Series, parsed: pd.Series, col: str, kind: str) -> No ) -def _sort_rows(df: pd.DataFrame, dialect: OgcDialect) -> pd.DataFrame: +def _sort_rows(df: pd.DataFrame, dialect: ShapingDialect) -> pd.DataFrame: """ Sorts rows by the API ``dialect``'s ``sort_cols`` (in priority order). @@ -350,7 +350,7 @@ def _sort_rows(df: pd.DataFrame, dialect: OgcDialect) -> pd.DataFrame: ---------- df : pd.DataFrame The input DataFrame. - dialect : OgcDialect + dialect : ShapingDialect Supplies ``sort_cols``. Returns @@ -401,7 +401,6 @@ def _finalize_ogc( geopd: bool, include_geometry: bool, max_rows: int | None = None, - extra_id_cols: frozenset[str] | set[str] = frozenset(), endpoint: Endpoint[OgcDialect], ) -> tuple[pd.DataFrame, BaseMetadata]: """Shape a combined OGC result into the user-facing ``(df, md)``. @@ -423,7 +422,7 @@ def _finalize_ogc( bound. ``endpoint`` is captured with the finalizer so resumed calls query the same API's schema when their combined result is empty. """ - dialect = endpoint.dialect + dialect = endpoint.dialect.shaping frame = _deal_with_empty( frame, properties, @@ -446,7 +445,7 @@ def _finalize_ogc( frame = frame.rename(columns=renames) if convert_type: frame = _type_cols(frame, dialect) - frame = _arrange_cols(frame, properties, output_id, extra_id_cols) + frame = _arrange_cols(frame, properties, output_id, dialect.extra_id_cols) frame = _sort_rows(frame, dialect) if max_rows is not None: frame = frame.head(max_rows) diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index 2ca816fe..e9baaa90 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -27,7 +27,7 @@ from dataretrieval._service import Endpoint from dataretrieval.codes.states import apply_state from dataretrieval.credentials import refuse_credential_keywords -from dataretrieval.ogc import OgcDialect, prepare_request_args +from dataretrieval.ogc import OgcDialect, ShapingDialect, prepare_request_args from dataretrieval.ogc import get_ogc_data as _facade_get_ogc_data # Default endpoint constants remain at their documented compatibility paths. @@ -80,13 +80,13 @@ # ``combined-metadata`` don't interpret comma-separated GET params as multiple # values (they return empty results, so multi-value filters must POST CQL2 JSON), # ``daily`` renders its time arguments date-only (``YYYY-MM-DD``), and the -# ``time_cols``/``numerical_cols``/``sort_cols`` are the Water-Data column -# vocabulary used to coerce datetime/numeric columns and to sort results. +# shaping fields are the Water-Data column vocabulary used to coerce +# datetime/numeric columns, sort results, and move synthetic ids to the end. # ``construction_date`` is deliberately not a time column: the service sends it # as a string at day, month, or year precision (``19950812``, ``199508``, # ``2005``), and no one datetime parse keeps all three, so it stays as sent. WATERDATA_DIALECT = OgcDialect( - cql2_services=frozenset({"combined-metadata", "monitoring-locations"}), + cql2_collections=frozenset({"combined-metadata", "monitoring-locations"}), dates=DateDialect( params=frozenset( { @@ -103,28 +103,31 @@ timed_params=frozenset({"last_modified"}), durations=True, ), - time_cols=frozenset( - { - "begin", - "begin_utc", - "end", - "end_utc", - "last_modified", - "time", - } - ), - numerical_cols=frozenset( - { - "altitude", - "altitude_accuracy", - "contributing_drainage_area", - "drainage_area", - "hole_constructed_depth", - "value", - "well_constructed_depth", - } + shaping=ShapingDialect( + time_cols=frozenset( + { + "begin", + "begin_utc", + "end", + "end_utc", + "last_modified", + "time", + } + ), + numerical_cols=frozenset( + { + "altitude", + "altitude_accuracy", + "contributing_drainage_area", + "drainage_area", + "hole_constructed_depth", + "value", + "well_constructed_depth", + } + ), + sort_cols=("time", "monitoring_location_id"), + extra_id_cols=_EXTRA_ID_COLS, ), - sort_cols=("time", "monitoring_location_id"), ) # The Water-Data-specific extras in addition to the engine's own no-normalize set @@ -298,7 +301,6 @@ def get_ogc_data( SERVICE, ogc_api_url(api_version=api_version), WATERDATA_DIALECT ), spatial=spatial, - extra_id_cols=_EXTRA_ID_COLS, cql_body=cql_body, ) diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index 862378a1..1b8795e6 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -126,8 +126,9 @@ deviations. A comment may cite a service as evidence for why the code tolerates a variation; the behavior may not depend on which service it is. **Dialect fields are grouped by the code that reads them.** OGC's dialect has a -date group, a request-construction group, and a result-shaping group. A group is -created when one part of the code reads several fields together. +date group and a result-shaping group; request construction reads one field, +which stays on the dialect itself. A group is created when one part of the code +reads several fields together. **A date range is read once, at the public boundary, and written by each protocol.** One package-wide value holds what the caller meant. The ISO 8601 @@ -271,3 +272,14 @@ Revisions: an empty start as open only before an RFC 3339 datetime (``/2026-01-01T00:00:00Z``); ``/2026-01-01`` is HTTP 400. ``get_ratings`` always writes datetimes. +- *The OGC code names nothing a dialect declares.* ``OgcDialect`` has the + date group, ``dates``, and the result-shaping group, ``shaping`` + (``ShapingDialect``: ``time_cols``, ``numerical_cols``, ``sort_cols``, and + ``extra_id_cols``, which moved there from an argument of ``get_ogc_data``). + Request construction reads one field, ``cql2_collections`` (renamed from + ``cql2_services``), so it has no group of its own. The chunk planner lists + no date parameter: a date argument arrives as a ``DateRange``, which is + never split. ``tests/architecture_test.py`` fails if a string a shipped + dialect lists appears as a literal in ``ogc`` or ``_intervals``. Of the four + recorded deviations, only ``monitoring_location_id`` is such a literal, so + it is the one allowlisted entry. diff --git a/tests/architecture_test.py b/tests/architecture_test.py index a4c5874a..c4e661df 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -739,3 +739,64 @@ def _code_string_literals(path: Path) -> list[tuple[int, str]]: and isinstance(node.value, str) and id(node) not in docstrings ] + + +# Protocol code: the modules shared by more than one endpoint (ADR 0014). +_PROTOCOL_PATHS = ( + *sorted((PACKAGE_ROOT / "ogc").glob("*.py")), + PACKAGE_ROOT / "_intervals.py", +) + +# The deviations ADR 0014 records as still written in the OGC protocol code. +# Only this one is a literal a dialect also lists; the other three (the +# ``skipGeometry`` spelling, the 50000 ``limit`` cap, and the unquoted-number +# CQL check) name nothing a dialect declares. +_RECORDED_DEVIATIONS = frozenset({"monitoring_location_id"}) + + +def _declared_strings(value: object) -> set[str]: + """Every string a dialect value lists, through nested groups.""" + import dataclasses + + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return set().union( + *( + _declared_strings(getattr(value, f.name)) + for f in dataclasses.fields(value) + ) + ) + if isinstance(value, str): + return {value} if value else set() + if isinstance(value, (frozenset, set, tuple, list)): + return set().union(*(_declared_strings(v) for v in value)) + return set() + + +def test_protocol_code_names_nothing_a_dialect_declares() -> None: + """ADR 0014: shared protocol code reads every parameter, collection, and + column a service departs on from that service's dialect. A literal of one + in protocol code is behavior that depends on which service it is, without + a field saying so. The standard's own names (each field's default) are not + deviations.""" + from dataretrieval._intervals import DateDialect + from dataretrieval.ngwmn import NGWMN_DIALECT + from dataretrieval.ogc import OgcDialect + from dataretrieval.waterdata.ratings import STAC_DATES + from dataretrieval.waterdata.utils import WATERDATA_DIALECT + + declared = ( + ( + _declared_strings(WATERDATA_DIALECT) + | _declared_strings(NGWMN_DIALECT) + | _declared_strings(STAC_DATES) + ) + - _declared_strings(OgcDialect()) + - _declared_strings(DateDialect()) + ) + found = [ + f"{path.relative_to(PACKAGE_ROOT)}:{line} {value!r}" + for path in _PROTOCOL_PATHS + for line, value in _code_string_literals(path) + if value in declared - _RECORDED_DEVIATIONS + ] + assert not found, f"declare it as a dialect field and read it there: {found}" diff --git a/tests/waterdata_chunking_test.py b/tests/waterdata_chunking_test.py index 5e6669c4..67e71071 100644 --- a/tests/waterdata_chunking_test.py +++ b/tests/waterdata_chunking_test.py @@ -34,6 +34,7 @@ import dataretrieval from dataretrieval import configuration as _configuration +from dataretrieval._dates import read_date_range from dataretrieval._service import Endpoint from dataretrieval.combining import ( _QUOTA_HEADER, @@ -65,7 +66,6 @@ ) from dataretrieval.ogc.planning import ( _LIST_SEP, - _NEVER_CHUNK, _OR_SEP, ChunkPlan, _extract_axes, @@ -86,7 +86,7 @@ ) from dataretrieval.utils import HTTPX_DEFAULTS from dataretrieval.waterdata.endpoints import SERVICE -from dataretrieval.waterdata.utils import OGC_API_URL, WATERDATA_DIALECT +from dataretrieval.waterdata.utils import OGC_API_URL # The joint-planner stress test uses the real request builder; bind the # target API the way ``get_ogc_data`` does — explicitly, via ``partial``. @@ -151,14 +151,15 @@ def _fake_build(*, base=200, **kwargs): return _FakeReq("x" * bytes_) -def test_never_chunk_covers_all_date_range_params(): - """Every parameter the Water Data dialect reads as a date range MUST be - excluded from chunking -- a range value isn't an enumerable set to split.""" - missing = WATERDATA_DIALECT.dates.params - _NEVER_CHUNK - assert not missing, ( - f"Date parameters not in _NEVER_CHUNK: {sorted(missing)}. " - "Add them to planning._NEVER_CHUNK." - ) +def test_extract_axes_never_splits_a_date_range(): + """A date argument arrives as a ``DateRange``, so it is never an axis, + whatever the dialect names the parameter -- the planner lists no date + parameter of its own.""" + args = { + "monitoring_location_id": ["USGS-A", "USGS-B"], + "any_date_param": read_date_range(["2024-01-01", "2024-12-31"]), + } + assert [axis.arg_key for axis in _extract_axes(args)] == ["monitoring_location_id"] def test_extract_axes_picks_up_list_dims_and_filter(): diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 6e2271fb..5030a1d7 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -34,7 +34,7 @@ _parse_retry_after, _raise_for_non_200, ) -from dataretrieval.ogc.policy import OgcDialect +from dataretrieval.ogc.policy import OgcDialect, ShapingDialect from dataretrieval.ogc.schema import _check_ogc_requests from dataretrieval.ogc.shaping import ( _arrange_cols, @@ -48,7 +48,6 @@ from dataretrieval.waterdata.endpoints import SERVICE from dataretrieval.waterdata.stats import _handle_nesting, get_data from dataretrieval.waterdata.utils import ( - _EXTRA_ID_COLS, OGC_API_URL, WATERDATA_DIALECT, _flatten_queryables, @@ -59,7 +58,6 @@ # exercise the same result shape the typed getters produce. _finalize_ogc = functools.partial( _ogc_finalize, - extra_id_cols=_EXTRA_ID_COLS, endpoint=Endpoint(SERVICE, OGC_API_URL, WATERDATA_DIALECT), geopd=False, include_geometry=True, @@ -863,7 +861,7 @@ def test_arrange_cols_keeps_geometry_when_present(): # --- _type_cols -------------------------------------------------------------- -_TYPING_DIALECT = OgcDialect( +_TYPING_DIALECT = ShapingDialect( time_cols=frozenset({"time"}), numerical_cols=frozenset({"value"}) ) @@ -1807,7 +1805,7 @@ def test_get_ogc_data_reads_everything_about_the_service_from_its_endpoint( endpoint = Endpoint( Service("ngwmn", "https://unused.example"), "https://mirror.example/ogc", - OgcDialect(cql2_services=frozenset({"things"})), + OgcDialect(cql2_collections=frozenset({"things"})), ) httpx_mock.add_response( method="POST", From f9935632ead41c9616d4b91180f4ec33e57b750d Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 17:36:49 -0500 Subject: [PATCH 17/22] refactor: simplify the date reader and interval writer (ADR 0014) Review cleanups on the date-range steps: - read_date_range takes the endpoint's `durations` and refuses a duration there, when the getter is called, and words its "too many values" message from the same flag. The single_value_hint helper and parameter, and write_interval's refusal and `name`, are gone. - write_interval takes `date_only` instead of `param` and `collection`; the OGC request builder decides it, since only OGC has collections. - The Samples and Statistics getters read no parameter as a date range (`_get_args(dates=_NO_DATES)`). - _DURATION_RE and _coerce_to_list are private again; _EXTRA_ID_COLS leaves __all__; the test request builder normalizes through prepare_request_args; the fitness function drops a redundant subtraction; stale names in docs and tests are fixed. Co-Authored-By: Claude Opus 5.5 (1M context) --- dataretrieval/_dates.py | 67 +++++++++++++------ dataretrieval/_intervals.py | 58 ++++------------ dataretrieval/ogc/requests.py | 16 ++--- dataretrieval/ogc/shaping.py | 2 +- dataretrieval/waterdata/ratings.py | 6 +- dataretrieval/waterdata/samples.py | 3 +- dataretrieval/waterdata/time_series.py | 3 + dataretrieval/waterdata/utils.py | 17 ++++- .../0014-services-endpoints-dialects.rst | 13 ++-- tests/_dates_test.py | 12 +++- tests/architecture_test.py | 16 ++--- tests/ngwmn_test.py | 2 +- tests/waterdata_chunking_test.py | 2 +- tests/waterdata_ratings_test.py | 2 +- tests/waterdata_test.py | 21 +++--- tests/waterdata_utils_test.py | 39 +++++------ 16 files changed, 144 insertions(+), 135 deletions(-) diff --git a/dataretrieval/_dates.py b/dataretrieval/_dates.py index 37204c64..02a891e1 100644 --- a/dataretrieval/_dates.py +++ b/dataretrieval/_dates.py @@ -31,7 +31,7 @@ # Anchored to ``[Pp]\d`` so a normal word containing ``p`` (e.g. ``"Apr"``) # doesn't get mis-classified as an ISO 8601 duration; the optional ``T`` # admits time-only forms like ``PT36H``. -DURATION_RE = re.compile(r"^[Pp]T?\d") +_DURATION_RE = re.compile(r"^[Pp]T?\d") # The ISO 8601 spelling of an open interval bound, as in ``"2024-01-01/.."``. @@ -122,7 +122,7 @@ def _read_bound(dt: DateLike, *, name: str) -> datetime | None: return None if dt is None or _is_blank(dt) else _to_datetime(dt, name=name) -def coerce_to_list( +def _coerce_to_list( value: str | _date | Sequence[DateLike], name: str = "date input", ) -> list[DateLike]: @@ -151,7 +151,7 @@ def _split_interval(interval: str, *, name: str) -> tuple[str, str]: HTTP 400. """ sides = interval.split("/") - if len(sides) != 2 or any(DURATION_RE.match(side) for side in sides): + if len(sides) != 2 or any(_DURATION_RE.match(side) for side in sides): raise ValueError( f"{name} is not a valid interval: {interval!r}. Pass a start and " "an end separated by '/', such as '2024-01-01/2024-12-31', with " @@ -165,7 +165,7 @@ def read_date_range( value: str | _date | Sequence[DateLike] | None, *, name: str = "date input", - single_value_hint: str = "an instant or a duration ('2020-01-01', 'P7D')", + durations: bool, ) -> DateRange | None: """Read a date argument as the caller meant it. @@ -182,11 +182,10 @@ def read_date_range( The caller's own spelling of this argument, used as the subject of every message raised here, so a caller correcting the error edits an argument their getter accepts. - single_value_hint : str, optional - How the "too many values" message describes an acceptable single - value. Wording only: a getter that rejects some of the default's forms - (``get_ratings`` rejects durations) passes a hint naming only what it - accepts. + durations : bool + Whether the endpoint accepts a lone ISO 8601 duration; see + ``DateDialect.durations``. When it does not, a duration raises, and no + message offers one. Returns ------- @@ -197,15 +196,16 @@ def read_date_range( Raises ------ ValueError - For more than two elements, an unreadable interval string, or an - element that is not blank and matches no supported format or type. + For more than two elements, an unreadable interval string, a duration + where *durations* is false, or an element that is not blank and matches + no supported format or type. TypeError For a mapping. """ if value is None: return None - items = coerce_to_list(value, name) + items = _coerce_to_list(value, name) if len(items) == 1 and isinstance(items[0], str) and "/" in items[0]: items = list(_split_interval(items[0], name=name)) @@ -213,17 +213,40 @@ def read_date_range( return None if len(items) > 2: - raise ValueError( - f"{name} takes at most 2 values, got {len(items)}: {items!r}. " - f"Pass one value for {single_value_hint}, " - "or two for a closed interval ('2020-01-01', '2020-12-31')." - ) - + raise _too_many_values(items, name=name, durations=durations) if len(items) == 1: - (item,) = items - if isinstance(item, str) and DURATION_RE.match(item): - return DateRange(duration=item) - return DateRange(start=_read_bound(item, name=name), instant=True) + return _read_single(items[0], name=name, durations=durations) return DateRange( start=_read_bound(items[0], name=name), end=_read_bound(items[1], name=name) ) + + +def _too_many_values( + items: list[DateLike], *, name: str, durations: bool +) -> ValueError: + """The error for more than two elements; it offers a duration only where + one is accepted.""" + single = ( + "an instant or a duration ('2020-01-01', 'P7D')" + if durations + else "an instant ('2020-01-01')" + ) + return ValueError( + f"{name} takes at most 2 values, got {len(items)}: {items!r}. " + f"Pass one value for {single}, " + "or two for a closed interval ('2020-01-01', '2020-12-31')." + ) + + +def _read_single(item: DateLike, *, name: str, durations: bool) -> DateRange: + """One non-blank element: a duration, or an instant.""" + if isinstance(item, str) and _DURATION_RE.match(item): + if not durations: + raise ValueError( + f"{name} cannot be an ISO 8601 duration for this service: " + f"{item!r}. Pass a start and an end instead, such as " + "['2024-01-01', '2024-01-08'], with None for an open end " + "(['2024-01-01', None])." + ) + return DateRange(duration=item) + return DateRange(start=_read_bound(item, name=name), instant=True) diff --git a/dataretrieval/_intervals.py b/dataretrieval/_intervals.py index c6dd326b..e98f5f24 100644 --- a/dataretrieval/_intervals.py +++ b/dataretrieval/_intervals.py @@ -45,7 +45,7 @@ class DateDialect: durations : bool Whether a lone ISO 8601 duration (``"P7D"``, the last seven days) is sent. Neither standard defines one, so the default is ``False`` and a - duration raises before sending. Observed 2026-10-11: Water Data OGC + duration raises when the getter is called. Observed 2026-10-11: Water Data OGC accepts it; NGWMN and STAC answer HTTP 400. """ @@ -56,13 +56,6 @@ class DateDialect: durations: bool = False -def single_value_hint(dialect: DateDialect) -> str: - """How a "too many values" message describes one acceptable value.""" - if dialect.durations: - return "an instant or a duration ('2020-01-01', 'P7D')" - return "an instant ('2020-01-01')" - - def _write_bound(bound: datetime | None, *, date: bool) -> str: """One bound: ``YYYY-MM-DD``, a UTC ``...Z`` datetime, or ``..``.""" if bound is None: @@ -75,55 +68,28 @@ def _write_bound(bound: datetime | None, *, date: bool) -> str: return aware.astimezone(ZoneInfo("UTC")).strftime("%Y-%m-%dT%H:%M:%SZ") -def write_interval( - value: DateRange, - dialect: DateDialect, - *, - param: str, - collection: str | None = None, - name: str | None = None, -) -> str: - """Write a date range as the endpoint's *param* expects it. +def write_interval(value: DateRange, dialect: DateDialect, *, date_only: bool) -> str: + """Write a date range as the endpoint's date parameter expects it. Parameters ---------- value : DateRange What the caller meant, as read by - :func:`~dataretrieval._dates.read_date_range`. + :func:`~dataretrieval._dates.read_date_range`, which refuses a duration + the dialect does not accept. dialect : DateDialect - The endpoint's date dialect. - param : str - The request parameter being written; decides, with *collection*, - whether its bounds are written date-only. - collection : str, optional - The collection the request targets, if any. - name : str, optional - The caller's spelling of the argument, the subject of the message when - a duration is refused. Defaults to *param*. - - Raises - ------ - ValueError - For a duration, unless the dialect allows durations. + The endpoint's date dialect; supplies ``open_start``. + date_only : bool + Write each bound as ``YYYY-MM-DD`` rather than as a UTC datetime. The + caller decides from ``date_only_collections`` and ``timed_params``. """ if value.duration is not None: - if not dialect.durations: - raise ValueError( - f"{name or param} cannot be an ISO 8601 duration for this " - f"service: {value.duration!r}. Pass a start and an end instead, " - "such as ['2024-01-01', '2024-01-08'], with None for an open " - "end (['2024-01-01', None])." - ) return value.duration - date = ( - collection in dialect.date_only_collections - and param not in dialect.timed_params - ) if value.instant: - return _write_bound(value.start, date=date) + return _write_bound(value.start, date=date_only) start = ( dialect.open_start if value.start is None - else _write_bound(value.start, date=date) + else _write_bound(value.start, date=date_only) ) - return f"{start}/{_write_bound(value.end, date=date)}" + return f"{start}/{_write_bound(value.end, date=date_only)}" diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 1576b334..6321835a 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -17,7 +17,7 @@ import httpx from dataretrieval._dates import read_date_range -from dataretrieval._intervals import DateDialect, single_value_hint, write_interval +from dataretrieval._intervals import DateDialect, write_interval from dataretrieval.transport.http import default_headers as _default_headers if TYPE_CHECKING: @@ -177,10 +177,12 @@ def _construct_api_requests( """ service_url = _items_url(collection, endpoint.url) dialect = endpoint.dialect - for key in dialect.dates.params & kwargs.keys(): - kwargs[key] = write_interval( - kwargs[key], dialect.dates, param=key, collection=collection + dates = dialect.dates + for key in dates.params & kwargs.keys(): + date_only = ( + collection in dates.date_only_collections and key not in dates.timed_params ) + kwargs[key] = write_interval(kwargs[key], dates, date_only=date_only) params, post_params = _partition_request_params( kwargs, use_cql2=collection in dialect.cql2_collections ) @@ -315,9 +317,7 @@ def _normalize_arg( if key == "properties": return _as_str_list(value, key) if key in dates.params: - return read_date_range( - value, name=key, single_value_hint=single_value_hint(dates) - ) + return read_date_range(value, name=key, durations=dates.durations) if ( key in no_normalize and isinstance(value, Iterable) @@ -355,7 +355,7 @@ def prepare_request_args( args: dict[str, Any] = {} for k, v in local_vars.items(): - if k in to_exclude or v is None: + if k in to_exclude: continue normalized = _normalize_arg(k, v, no_normalize, dates) if normalized is not None: diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index 72c0aa8c..0a5511d4 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -223,7 +223,7 @@ def _arrange_cols( df: pd.DataFrame, properties: list[str] | None, output_id: str, - extra_id_cols: frozenset[str] | set[str] = frozenset(), + extra_id_cols: frozenset[str] = frozenset(), ) -> pd.DataFrame: """ Rearranges and renames DataFrame columns per ``properties`` and the output id. diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index 858031ff..faf27bf0 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -19,7 +19,7 @@ import pandas as pd from dataretrieval._dates import read_date_range -from dataretrieval._intervals import DateDialect, single_value_hint, write_interval +from dataretrieval._intervals import DateDialect, write_interval from dataretrieval._service import Endpoint from dataretrieval._validation import render_options from dataretrieval.exceptions import DataRetrievalError, SkippedRatingWarning @@ -189,10 +189,10 @@ def get_ratings( # applies to the search (ADR 0014). endpoint = Endpoint(SERVICE, ratings_catalog_url(), STAC_DATES) time_range = read_date_range( - time, name="time", single_value_hint=single_value_hint(endpoint.dialect) + time, name="time", durations=endpoint.dialect.durations ) time_str = ( - write_interval(time_range, endpoint.dialect, param="datetime", name="time") + write_interval(time_range, endpoint.dialect, date_only=False) if time_range is not None else None ) diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py index 7e70f6a2..a471f371 100644 --- a/dataretrieval/waterdata/samples.py +++ b/dataretrieval/waterdata/samples.py @@ -40,6 +40,7 @@ _check_profiles, ) from dataretrieval.waterdata.utils import ( + _NO_DATES, _accept_legacy_kwargs, _get_args, ) @@ -356,7 +357,7 @@ def get_samples( # Build argument dictionary, omitting None values. Parameters are the # public snake_case names here; translate them to the camelCase names the # Samples API expects just before building the request. - args = _get_args(locals(), exclude={"ssl_check", "profile"}) + args = _get_args(locals(), exclude={"ssl_check", "profile"}, dates=_NO_DATES) params = {_SAMPLES_PARAM_TO_API.get(key, key): value for key, value in args.items()} params.update({"mimeType": "text/csv"}) diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py index 96b2ff9f..c4896ec8 100644 --- a/dataretrieval/waterdata/time_series.py +++ b/dataretrieval/waterdata/time_series.py @@ -18,6 +18,7 @@ from dataretrieval.waterdata import stats from dataretrieval.waterdata.utils import ( + _NO_DATES, _get_args, _with_state, get_ogc_data, @@ -1044,6 +1045,7 @@ def get_stats_por( params = _get_args( _with_state(locals(), to="fips_us", into="state_code"), exclude={"expand_percentiles"}, + dates=_NO_DATES, ) return stats.get_data( @@ -1191,6 +1193,7 @@ def get_stats_date_range( params = _get_args( _with_state(locals(), to="fips_us", into="state_code"), exclude={"expand_percentiles"}, + dates=_NO_DATES, ) return stats.get_data( diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index e9baaa90..75052a4e 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -190,11 +190,22 @@ def _flatten_queryables(local_vars: dict[str, Any]) -> dict[str, Any]: return local_vars +# For a getter whose API is not OGC: no parameter is read as a date range. +_NO_DATES = DateDialect(params=frozenset()) + + def _get_args( - local_vars: dict[str, Any], exclude: set[str] | None = None + local_vars: dict[str, Any], + exclude: set[str] | None = None, + *, + dates: DateDialect = WATERDATA_DIALECT.dates, ) -> dict[str, Any]: """Water-Data wrapper over :func:`~dataretrieval.ogc.prepare_request_args`. + ``dates`` names the parameters read as date ranges: the OGC API's by + default, and none (:data:`_NO_DATES`) for the Samples and Statistics + getters, whose APIs take dates in their own form. + Adds the Water Data API's extra no-normalize params (numeric params such as ``water_year``, ``thresholds``, ``boundingBox``) so they keep their element types. Also flattens any ``**queryables`` passthrough (see @@ -204,7 +215,7 @@ def _get_args( args = prepare_request_args( local_vars, exclude, - dates=WATERDATA_DIALECT.dates, + dates=dates, extra_no_normalize=_NO_NORMALIZE_PARAMS, ) for name in _JSON_ARRAY_PARAMS & args.keys(): @@ -376,7 +387,7 @@ def wrapper(*args: Any, **kwargs: Any) -> _R: "OGC_API_URL", "SAMPLES_URL", "WATERDATA_DIALECT", - "_EXTRA_ID_COLS", + "_NO_DATES", "_NO_NORMALIZE_PARAMS", "_OUTPUT_ID_BY_COLLECTION", "_accept_legacy_kwargs", diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index 1b8795e6..a7df00c9 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -253,9 +253,9 @@ Revisions: time. ``prepare_request_args`` reads every OGC date parameter into one when a getter is called, and drops a range whose bounds are all open, as it drops ``None``; ``get_ratings`` reads its ``time`` the same way. The OGC request - builder writes the value with ``ogc.dates._format_date_range``. Error - messages are unchanged; they are raised when the getter is called rather - than when the first request is built. + builder only writes the value. Error messages are unchanged; they are + raised when the getter is called rather than when the first request is + built. - *The shared interval grammar and* ``DateDialect``. ``dataretrieval._intervals`` holds ``DateDialect`` and ``write_interval``, and sits below ``ogc`` because STAC is not OGC; ``ogc.dates`` is removed. ``DateDialect`` has five fields: @@ -263,7 +263,12 @@ Revisions: ``durations``, whose default is ``False`` because neither standard defines a lone duration. ``OgcDialect.dates`` holds the group, and ``prepare_request_args`` takes it, so the reading side knows which parameters - carry dates. ``get_ratings`` builds an ``Endpoint`` whose dialect is a + carry dates and whether a duration is accepted: an endpoint that refuses one + raises when the getter is called. ``write_interval`` never raises; the + protocol code decides from ``date_only_collections`` and ``timed_params`` + whether it writes date-only bounds, since only OGC has collections. The + Samples and Statistics getters, which share Water Data's argument helper, + read no parameter as a date range. ``get_ratings`` builds an ``Endpoint`` whose dialect is a ``DateDialect`` alone; STAC has no other dialect fields. Live tests cover ``open_start`` and ``durations`` for each of the three endpoints. ``date_only_collections`` and ``timed_params`` have none: they record what a diff --git a/tests/_dates_test.py b/tests/_dates_test.py index d323d68a..3c3c57d7 100644 --- a/tests/_dates_test.py +++ b/tests/_dates_test.py @@ -31,12 +31,12 @@ def test_read_date_range_keeps_the_shape_the_caller_passed(value, expected): """An instant, a duration, and a range with open bounds stay distinct: a one-element instant is not a range with an open end.""" - assert read_date_range(value) == expected + assert read_date_range(value, durations=True) == expected @pytest.mark.parametrize("value", [None, pd.NaT, [None, ".."], "../.."]) def test_read_date_range_returns_none_for_no_filter(value): - assert read_date_range(value) is None + assert read_date_range(value, durations=True) is None def test_prepare_request_args_reads_date_arguments_on_entry(): @@ -50,3 +50,11 @@ def test_prepare_request_args_reads_date_arguments_on_entry(): assert args == {"time": DateRange(start=NAIVE)} with pytest.raises(ValueError, match="^time could not be read"): prepare_request_args({"time": "yesterday"}, dates=dates) + + +def test_read_date_range_refuses_a_duration_where_none_is_accepted(): + """A duration the endpoint refuses raises when the getter is called, naming + the caller's argument; where durations are accepted it is kept as given.""" + with pytest.raises(ValueError, match=r"^time cannot be an ISO 8601 duration"): + read_date_range("P7D", name="time", durations=False) + assert read_date_range("P7D", durations=True) == DateRange(duration="P7D") diff --git a/tests/architecture_test.py b/tests/architecture_test.py index c4e661df..320540ff 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -27,6 +27,7 @@ import ast import configparser +import dataclasses import functools import re import sys @@ -756,8 +757,6 @@ def _code_string_literals(path: Path) -> list[tuple[int, str]]: def _declared_strings(value: object) -> set[str]: """Every string a dialect value lists, through nested groups.""" - import dataclasses - if dataclasses.is_dataclass(value) and not isinstance(value, type): return set().union( *( @@ -778,21 +777,16 @@ def test_protocol_code_names_nothing_a_dialect_declares() -> None: in protocol code is behavior that depends on which service it is, without a field saying so. The standard's own names (each field's default) are not deviations.""" - from dataretrieval._intervals import DateDialect from dataretrieval.ngwmn import NGWMN_DIALECT from dataretrieval.ogc import OgcDialect from dataretrieval.waterdata.ratings import STAC_DATES from dataretrieval.waterdata.utils import WATERDATA_DIALECT declared = ( - ( - _declared_strings(WATERDATA_DIALECT) - | _declared_strings(NGWMN_DIALECT) - | _declared_strings(STAC_DATES) - ) - - _declared_strings(OgcDialect()) - - _declared_strings(DateDialect()) - ) + _declared_strings(WATERDATA_DIALECT) + | _declared_strings(NGWMN_DIALECT) + | _declared_strings(STAC_DATES) + ) - _declared_strings(OgcDialect()) found = [ f"{path.relative_to(PACKAGE_ROOT)}:{line} {value!r}" for path in _PROTOCOL_PATHS diff --git a/tests/ngwmn_test.py b/tests/ngwmn_test.py index a3166dd2..0dd9af85 100644 --- a/tests/ngwmn_test.py +++ b/tests/ngwmn_test.py @@ -684,6 +684,6 @@ def status(value): # open_start: the standard's "../" is refused; the empty start is read as open. end = DateRange(end=datetime.datetime(2016, 1, 1, tzinfo=datetime.timezone.utc)) assert status("../2016-01-01T00:00:00Z") == 403 - assert status(write_interval(end, NGWMN_DIALECT.dates, param="datetime")) == 200 + assert status(write_interval(end, NGWMN_DIALECT.dates, date_only=False)) == 200 # durations=False: a lone duration is refused, so none is sent. assert status("P7D") == 400 diff --git a/tests/waterdata_chunking_test.py b/tests/waterdata_chunking_test.py index 67e71071..ccd267db 100644 --- a/tests/waterdata_chunking_test.py +++ b/tests/waterdata_chunking_test.py @@ -157,7 +157,7 @@ def test_extract_axes_never_splits_a_date_range(): parameter of its own.""" args = { "monitoring_location_id": ["USGS-A", "USGS-B"], - "any_date_param": read_date_range(["2024-01-01", "2024-12-31"]), + "any_date_param": read_date_range(["2024-01-01", "2024-12-31"], durations=True), } assert [axis.arg_key for axis in _extract_axes(args)] == ["monitoring_location_id"] diff --git a/tests/waterdata_ratings_test.py b/tests/waterdata_ratings_test.py index 186b3e40..29fb6183 100644 --- a/tests/waterdata_ratings_test.py +++ b/tests/waterdata_ratings_test.py @@ -467,6 +467,6 @@ def status(value): # open_start: the standard's "../" is refused; the empty start is read as open. end = DateRange(end=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)) assert status("../2026-01-01T00:00:00Z") == 403 - assert status(write_interval(end, STAC_DATES, param="datetime")) == 200 + assert status(write_interval(end, STAC_DATES, date_only=False)) == 200 # durations=False: a lone duration is refused, so none is sent. assert status("P7D") == 400 diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 2d31026d..b6c91789 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -15,11 +15,11 @@ import dataretrieval from dataretrieval import configuration -from dataretrieval._dates import read_date_range from dataretrieval._service import Endpoint from dataretrieval.ogc.requests import ( _check_monitoring_location_id, _normalize_str_iterable, + prepare_request_args, ) from dataretrieval.ogc.requests import ( _construct_api_requests as _construct_api_requests_explicit, @@ -49,6 +49,9 @@ ) from dataretrieval.waterdata.endpoints import SERVICE from dataretrieval.waterdata.types import _check_profiles +from dataretrieval.waterdata.utils import ( + _NO_NORMALIZE_PARAMS as _WATERDATA_NO_NORMALIZE, +) from dataretrieval.waterdata.utils import ( OGC_API_URL, WATERDATA_DIALECT, @@ -78,14 +81,14 @@ # Do both here so the tests exercise the real Water Data behavior # (monitoring-locations -> POST/CQL2; daily -> date-only time args). def _construct_api_requests(collection, **kwargs): - for key in WATERDATA_DIALECT.dates.params & kwargs.keys(): - read = read_date_range(kwargs.pop(key), name=key) - if read is not None: - kwargs[key] = read return _construct_api_requests_explicit( collection, endpoint=Endpoint(SERVICE, OGC_API_URL, WATERDATA_DIALECT), - **kwargs, + **prepare_request_args( + kwargs, + dates=WATERDATA_DIALECT.dates, + extra_no_normalize=_WATERDATA_NO_NORMALIZE, + ), ) @@ -1939,7 +1942,9 @@ def status(value): # open_start: the default ".." is accepted; the empty start is refused. end = DateRange(end=datetime.datetime(2024, 1, 3, tzinfo=datetime.timezone.utc)) - assert status(write_interval(end, dates, param="time", collection="daily")) == 200 + assert status(write_interval(end, dates, date_only=True)) == 200 assert status("/2024-01-03") == 400 # durations=True: a lone duration is accepted, so it is sent. - assert status(write_interval(DateRange(duration="P7D"), dates, param="time")) == 200 + assert ( + status(write_interval(DateRange(duration="P7D"), dates, date_only=False)) == 200 + ) diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 5030a1d7..ab51eb89 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -914,24 +914,17 @@ def test_type_cols_warning_is_singular_for_one_value(): _type_cols(df, _TYPING_DIALECT) -# --- read_date_range, then _format_date_range ------------------------------- +# --- read_date_range, then write_interval ----------------------------------- def _wire_dates(value, date=False, *, name="date input", open_start=".."): """Read a date argument and write it as an interval, as a getter and the request builder do in turn; ``None`` when there is no filter. ``date`` - targets a date-only collection.""" - read = read_date_range(value, name=name) + writes date-only bounds.""" + read = read_date_range(value, name=name, durations=True) if read is None: return None - dialect = DateDialect( - date_only_collections=frozenset({"days"}), - open_start=open_start, - durations=True, - ) - return write_interval( - read, dialect, param=name, collection="days" if date else "instants" - ) + return write_interval(read, DateDialect(open_start=open_start), date_only=date) @pytest.mark.parametrize( @@ -1105,7 +1098,7 @@ def test_read_date_range_rejects_an_unreadable_interval_string(interval, message reached the service. It is now rejected like an unreadable list element, naming the caller's argument.""" with pytest.raises(ValueError, match=f"^{message}"): - read_date_range(interval, name="time") + read_date_range(interval, name="time", durations=True) @pytest.mark.parametrize( @@ -1125,7 +1118,7 @@ def test_read_date_range_rejects_an_unreadable_bound(value): which the callers send as no filter at all. The message names the caller's argument and the bad value, and shows the forms that are accepted.""" with pytest.raises(ValueError) as excinfo: - read_date_range(value, name="last_modified") + read_date_range(value, name="last_modified", durations=True) message = str(excinfo.value) assert message.startswith("last_modified could not be read as a date") assert "'2024-01-01'" in message @@ -1137,21 +1130,21 @@ def test_read_date_range_rejects_more_than_two_values(): values is a caller who meant something else, and the message says which shapes exist rather than truncating without an error.""" with pytest.raises(ValueError) as excinfo: - read_date_range(["2024-01-01", "2024-06-01", "2024-12-31"], name="time") + read_date_range( + ["2024-01-01", "2024-06-01", "2024-12-31"], name="time", durations=True + ) message = str(excinfo.value) assert "time takes at most 2 values, got 3" in message assert "closed interval" in message def test_the_duration_example_is_withheld_where_durations_are_rejected(): - """``get_ratings`` refuses ISO 8601 durations, so the shared message must not offer + """Where durations are refused (STAC, NGWMN), the message must not offer 'P7D' on that path -- a caller following it would receive a second rejection.""" with pytest.raises(ValueError) as allowed: - read_date_range(["a", "b", "c"], name="time") + read_date_range(["a", "b", "c"], name="time", durations=True) with pytest.raises(ValueError) as refused: - read_date_range( - ["a", "b", "c"], name="time", single_value_hint="an instant ('2020-01-01')" - ) + read_date_range(["a", "b", "c"], name="time", durations=False) assert "'P7D'" in str(allowed.value) assert "'P7D'" not in str(refused.value) @@ -1164,7 +1157,7 @@ def test_read_date_range_names_the_callers_parameter(): accepts, so correcting the argument the message named sent an unrecognized parameter.""" with pytest.raises(TypeError, match="^time must be a date, a string"): - read_date_range({"2024-01-01": "ignored"}, name="time") + read_date_range({"2024-01-01": "ignored"}, name="time", durations=True) def test_read_date_range_rejects_mapping(): @@ -1174,7 +1167,7 @@ def test_read_date_range_rejects_mapping(): with pytest.raises( TypeError, match="date input must be a date, a string, or a sequence" ): - read_date_range({"2024-01-01": "ignored"}) + read_date_range({"2024-01-01": "ignored"}, durations=True) @pytest.mark.parametrize( @@ -1251,7 +1244,7 @@ def test_read_date_range_rejects_other_types_naming_the_argument(value): inside the formatter; the message now names the caller's argument and the types it accepts.""" with pytest.raises(ValueError) as excinfo: - read_date_range(value, name="time") + read_date_range(value, name="time", durations=True) message = str(excinfo.value) assert message.startswith("time could not be read as a date") assert "pandas.Timestamp" in message @@ -1796,7 +1789,7 @@ def test_get_ogc_data_reads_everything_about_the_service_from_its_endpoint( ): """ADR 0014: an endpoint is all the OGC engine is told about a service. The request goes to its URL, its dialect decides the request shape (a - ``cql2_services`` collection is POSTed), and settings resolve by its + ``cql2_collections`` collection is POSTed), and settings resolve by its service's name.""" import dataretrieval from dataretrieval.ngwmn import NgwmnConfiguration From 83af0ed809c8cdd0297a688546af03f135e4f387 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 17:46:50 -0500 Subject: [PATCH 18/22] fix: review findings on the date-range and dialect steps (ADR 0014) - The interval and date modules name no service: each field's observed server behavior is recorded where an adapter sets it. - A refused duration is refused "for this API": durations depend on the endpoint, and Water Data's OGC API accepts one where its STAC search does not. - read_date_range requires `name`, as every caller passes it. - Docstrings: get_water_level says a duration raises; get_ratings no longer says its time is passed through verbatim; ogc.policy describes both dialect types. - ADR 0014 notes record the two changed messages, and why cql2_collections has no live test: on 2026-10-11 a comma-joined GET returned the same rows as the CQL2 POST, in v0 and v1. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HBt3Ee7eneKtdKQWq2T3YK --- dataretrieval/_dates.py | 11 +++++----- dataretrieval/_intervals.py | 21 ++++++++----------- dataretrieval/ngwmn.py | 6 ++++-- dataretrieval/ogc/policy.py | 7 ++++--- dataretrieval/waterdata/ratings.py | 4 ++-- dataretrieval/waterdata/utils.py | 9 +++++--- .../0014-services-endpoints-dialects.rst | 19 ++++++++++++----- tests/_dates_test.py | 8 ++++--- tests/waterdata_chunking_test.py | 4 +++- tests/waterdata_utils_test.py | 10 --------- 10 files changed, 52 insertions(+), 47 deletions(-) diff --git a/dataretrieval/_dates.py b/dataretrieval/_dates.py index 02a891e1..afe4ce03 100644 --- a/dataretrieval/_dates.py +++ b/dataretrieval/_dates.py @@ -124,7 +124,7 @@ def _read_bound(dt: DateLike, *, name: str) -> datetime | None: def _coerce_to_list( value: str | _date | Sequence[DateLike], - name: str = "date input", + name: str, ) -> list[DateLike]: """Normalize a date argument to a list of elements, raising on a mapping.""" if isinstance(value, str) or not isinstance(value, Iterable): @@ -146,9 +146,8 @@ def _split_interval(interval: str, *, name: str) -> tuple[str, str]: means the same range as ``["2024-01-01T10:00:00", None]``. Raises ``ValueError`` naming *name* unless the string is two sides - separated by one ``"/"``, neither of them a duration: the Water Data and - NGWMN services answer ``"2024-01-01/P7D"`` and ``"P7D/2024-01-08"`` with - HTTP 400. + separated by one ``"/"``, neither of them a duration, which the services + this package reaches answer with HTTP 400. """ sides = interval.split("/") if len(sides) != 2 or any(_DURATION_RE.match(side) for side in sides): @@ -164,7 +163,7 @@ def _split_interval(interval: str, *, name: str) -> tuple[str, str]: def read_date_range( value: str | _date | Sequence[DateLike] | None, *, - name: str = "date input", + name: str, durations: bool, ) -> DateRange | None: """Read a date argument as the caller meant it. @@ -243,7 +242,7 @@ def _read_single(item: DateLike, *, name: str, durations: bool) -> DateRange: if isinstance(item, str) and _DURATION_RE.match(item): if not durations: raise ValueError( - f"{name} cannot be an ISO 8601 duration for this service: " + f"{name} cannot be an ISO 8601 duration for this API: " f"{item!r}. Pass a start and an end instead, such as " "['2024-01-01', '2024-01-08'], with None for an open end " "(['2024-01-01', None])." diff --git a/dataretrieval/_intervals.py b/dataretrieval/_intervals.py index e98f5f24..c5b1e1fb 100644 --- a/dataretrieval/_intervals.py +++ b/dataretrieval/_intervals.py @@ -26,27 +26,24 @@ class DateDialect: ---------- params : frozenset[str] The parameters that carry a date range. The standard has one, - ``datetime``; an endpoint that filters other fields by date (Water Data's - ``time``, ``begin``, ``last_modified``) lists them all. + ``datetime``; an endpoint that filters other fields by date lists them + all. date_only_collections : frozenset[str] Collections whose dates are written as ``YYYY-MM-DD`` rather than as a - UTC datetime. This is a choice rather than a server restriction: Water - Data's ``daily`` accepts either, but its values are calendar days, and - a naive bound converted to UTC could move to the neighboring day. + UTC datetime, because their values are calendar days and a naive bound + converted to UTC could move to the neighboring day. timed_params : frozenset[str] Parameters written as a UTC datetime even in a date-only collection, - because they filter a timestamp (Water Data's ``last_modified``). + because they filter a timestamp. open_start : str How the open start of a range is written: ``".."`` (``"../2024-01-01"``, the default) or ``""`` (``"/2024-01-01"``). Both standards allow - either, but deployments differ. Observed 2026-10-11: Water Data OGC - answers the empty start with HTTP 400; the NGWMN and STAC firewalls - answer ``"../"`` with HTTP 403. + either, but deployments refuse one or the other; each declaration + records what its service was observed to refuse. durations : bool Whether a lone ISO 8601 duration (``"P7D"``, the last seven days) is - sent. Neither standard defines one, so the default is ``False`` and a - duration raises when the getter is called. Observed 2026-10-11: Water Data OGC - accepts it; NGWMN and STAC answer HTTP 400. + accepted. Neither standard defines one, so the default is ``False`` and + a duration raises when the getter is called. """ params: frozenset[str] = frozenset({"datetime"}) diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index dd92c9e6..cfd6e100 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -95,7 +95,8 @@ # timestamped by ``sample_time`` (not the Water Data ``time``) and report # depths/levels in feet. Its firewall answers ``datetime=../2016-01-01`` with # HTTP 403, so an open start is sent empty (``/2016-01-01``), which it reads as -# open; see ``DateDialect.open_start``. +# open, and it answers a lone duration (``P7D``) with HTTP 400, so durations are +# off, as in the standard (both observed 2026-10-11). NGWMN_DIALECT = OgcDialect( dates=DateDialect(open_start=""), shaping=ShapingDialect( @@ -280,7 +281,8 @@ def get_water_level( datetime : str or iterable of str, optional Temporal filter — a single instant or a two-element ``[start, end]`` range (ISO-8601 dates/datetimes); ``None`` or ``".."`` denotes an - open end. + open end. A duration such as ``"P7D"`` raises ``ValueError``, because + the service refuses one. properties : str or iterable of str, optional Subset of columns to return. ``None`` (default) returns all columns. limit : int, optional diff --git a/dataretrieval/ogc/policy.py b/dataretrieval/ogc/policy.py index 9847f3ca..fca521d8 100644 --- a/dataretrieval/ogc/policy.py +++ b/dataretrieval/ogc/policy.py @@ -1,7 +1,8 @@ -"""Low-level OGC policy: the dialect type and control validation. +"""Low-level OGC policy: the dialect types and control validation. -This module is the one definition of the :class:`OgcDialect` type -(per-API quirks the generic request builder needs) and OGC control validation. +This module is the one definition of :class:`OgcDialect` (how an API departs +from OGC API Features, read by the request builder and the result finalizer), +its :class:`ShapingDialect` group, and OGC control validation. It depends only on the stdlib and the interval grammar's dialect type, so any OGC submodule can import it without creating cycles. diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index faf27bf0..04ba4c5e 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -42,7 +42,7 @@ # The STAC search's departures from the interval standard (ADR 0014). Its # firewall answers ``datetime=../2026-01-01T00:00:00Z`` with HTTP 403, and it # reads an empty start (``/2026-01-01T00:00:00Z``) as open; it refuses a lone -# duration with HTTP 400, as the standard does. +# duration with HTTP 400, as the standard does (observed 2026-10-11). STAC_DATES = DateDialect(open_start="") @@ -91,7 +91,7 @@ def get_ratings( persisting the bytes to disk; ``df.attrs["url"]`` still records where each rating came from. time : string or list of strings, optional - STAC ``datetime`` filter (passed through verbatim under that name) + STAC ``datetime`` filter, sent as a UTC datetime or interval — a single date / datetime, an interval (``"start/end"``, optionally half-bounded with ``..``), or a list of two values with ``None`` for an open end (``["2026-04-29", None]``). A value diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index 75052a4e..49e9fdf9 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -79,9 +79,12 @@ # The Water Data API dialect: ``monitoring-locations`` and # ``combined-metadata`` don't interpret comma-separated GET params as multiple # values (they return empty results, so multi-value filters must POST CQL2 JSON), -# ``daily`` renders its time arguments date-only (``YYYY-MM-DD``), and the -# shaping fields are the Water-Data column vocabulary used to coerce -# datetime/numeric columns, sort results, and move synthetic ids to the end. +# ``daily`` renders its time arguments date-only (``YYYY-MM-DD``); the API +# accepts a lone duration (``time=P7D``) and answers an empty open start +# (``time=/2024-01-03``) with HTTP 400, so it keeps ``..`` (both observed +# 2026-10-11); and the shaping fields are the Water-Data column vocabulary +# used to coerce datetime/numeric columns, sort results, and move synthetic ids +# to the end. # ``construction_date`` is deliberately not a time column: the service sends it # as a string at day, month, or year precision (``19950812``, ``199508``, # ``2005``), and no one datetime parse keeps all three, so it stays as sent. diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index a7df00c9..94582518 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -253,9 +253,9 @@ Revisions: time. ``prepare_request_args`` reads every OGC date parameter into one when a getter is called, and drops a range whose bounds are all open, as it drops ``None``; ``get_ratings`` reads its ``time`` the same way. The OGC request - builder only writes the value. Error messages are unchanged; they are - raised when the getter is called rather than when the first request is - built. + builder only writes the value. Error messages are unchanged by this step; + they are raised when the getter is called rather than when the first + request is built. - *The shared interval grammar and* ``DateDialect``. ``dataretrieval._intervals`` holds ``DateDialect`` and ``write_interval``, and sits below ``ogc`` because STAC is not OGC; ``ogc.dates`` is removed. ``DateDialect`` has five fields: @@ -268,7 +268,12 @@ Revisions: protocol code decides from ``date_only_collections`` and ``timed_params`` whether it writes date-only bounds, since only OGC has collections. The Samples and Statistics getters, which share Water Data's argument helper, - read no parameter as a date range. ``get_ratings`` builds an ``Endpoint`` whose dialect is a + read no parameter as a date range. Two messages change: the duration refusal + is one shared message naming the caller's argument (``get_ratings`` had its + own), and the "too many values" message offers a duration only where one is + accepted. A field's observed server behavior is recorded where the field is + set, in each adapter's declaration, since the type is shared code and names + no service. ``get_ratings`` builds an ``Endpoint`` whose dialect is a ``DateDialect`` alone; STAC has no other dialect fields. Live tests cover ``open_start`` and ``durations`` for each of the three endpoints. ``date_only_collections`` and ``timed_params`` have none: they record what a @@ -287,4 +292,8 @@ Revisions: never split. ``tests/architecture_test.py`` fails if a string a shipped dialect lists appears as a literal in ``ogc`` or ``_intervals``. Of the four recorded deviations, only ``monitoring_location_id`` is such a literal, so - it is the one allowlisted entry. + it is the one allowlisted entry. ``cql2_collections`` has no live test: on + 2026-10-11 a comma-joined GET returned the same rows as the CQL2 POST for + two ids on ``monitoring-locations`` and ``combined-metadata``, in v0 and + v1, so the behavior it records did not reproduce. Whether to keep the field + is left to a separate change. diff --git a/tests/_dates_test.py b/tests/_dates_test.py index 3c3c57d7..2b8c758e 100644 --- a/tests/_dates_test.py +++ b/tests/_dates_test.py @@ -31,12 +31,12 @@ def test_read_date_range_keeps_the_shape_the_caller_passed(value, expected): """An instant, a duration, and a range with open bounds stay distinct: a one-element instant is not a range with an open end.""" - assert read_date_range(value, durations=True) == expected + assert read_date_range(value, name="time", durations=True) == expected @pytest.mark.parametrize("value", [None, pd.NaT, [None, ".."], "../.."]) def test_read_date_range_returns_none_for_no_filter(value): - assert read_date_range(value, durations=True) is None + assert read_date_range(value, name="time", durations=True) is None def test_prepare_request_args_reads_date_arguments_on_entry(): @@ -57,4 +57,6 @@ def test_read_date_range_refuses_a_duration_where_none_is_accepted(): the caller's argument; where durations are accepted it is kept as given.""" with pytest.raises(ValueError, match=r"^time cannot be an ISO 8601 duration"): read_date_range("P7D", name="time", durations=False) - assert read_date_range("P7D", durations=True) == DateRange(duration="P7D") + assert read_date_range("P7D", name="time", durations=True) == DateRange( + duration="P7D" + ) diff --git a/tests/waterdata_chunking_test.py b/tests/waterdata_chunking_test.py index ccd267db..445c1b56 100644 --- a/tests/waterdata_chunking_test.py +++ b/tests/waterdata_chunking_test.py @@ -157,7 +157,9 @@ def test_extract_axes_never_splits_a_date_range(): parameter of its own.""" args = { "monitoring_location_id": ["USGS-A", "USGS-B"], - "any_date_param": read_date_range(["2024-01-01", "2024-12-31"], durations=True), + "any_date_param": read_date_range( + ["2024-01-01", "2024-12-31"], name="time", durations=True + ), } assert [axis.arg_key for axis in _extract_axes(args)] == ["monitoring_location_id"] diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index ab51eb89..62b15fb0 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1160,16 +1160,6 @@ def test_read_date_range_names_the_callers_parameter(): read_date_range({"2024-01-01": "ignored"}, name="time", durations=True) -def test_read_date_range_rejects_mapping(): - """`time={"2024-01-01": "x"}` would materialize as the keys list, - accepting input the user clearly didn't intend. - """ - with pytest.raises( - TypeError, match="date input must be a date, a string, or a sequence" - ): - read_date_range({"2024-01-01": "ignored"}, durations=True) - - @pytest.mark.parametrize( "value, date, expected", [ From 630a9f033f5a95f3e58cf44dd40d188f34d39aef Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 11 Oct 2026 20:23:52 -0500 Subject: [PATCH 19/22] docs(adr): 0014, cql2_collections records the wrong behavior The earlier note said the behavior behind cql2_collections did not reproduce. That was from two identifier lists, the one kind of parameter the API reads correctly as a comma-joined list. Probing every list-valued parameter showed the behavior varies by parameter, not collection; #446 fixes it with a per-API switch. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HBt3Ee7eneKtdKQWq2T3YK --- .../decisions/0014-services-endpoints-dialects.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index 94582518..616fc697 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -292,8 +292,9 @@ Revisions: never split. ``tests/architecture_test.py`` fails if a string a shipped dialect lists appears as a literal in ``ogc`` or ``_intervals``. Of the four recorded deviations, only ``monitoring_location_id`` is such a literal, so - it is the one allowlisted entry. ``cql2_collections`` has no live test: on - 2026-10-11 a comma-joined GET returned the same rows as the CQL2 POST for - two ids on ``monitoring-locations`` and ``combined-metadata``, in v0 and - v1, so the behavior it records did not reproduce. Whether to keep the field - is left to a separate change. + it is the one allowlisted entry. ``cql2_collections`` has no live test + because it records the wrong behavior: whether the Water Data API reads a + comma-joined GET value as a list depends on the parameter, not the + collection, and it does so only for a few identifiers and codes (probed + 2026-10-11, v0 and v1). #446 replaces the field with a per-API switch, + ``multi_value_cql2``; this record takes that field when #446 merges. From 4b583d972bc511f9ac37296c21f8acb704b617b4 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Mon, 12 Oct 2026 09:35:37 -0500 Subject: [PATCH 20/22] refactor: simplify pass after merging #446 (ADR 0014) - The NGWMN and Water Data dialect comments state only the fields the code cannot show (lists, open start, durations), one line each. - ngwmn's module docstring names the endpoint, not the removed base_url. - ADR 0014 and CONTEXT.md wording left over from the merge. - The API-key host test reads literals through _code_string_literals, so one rule excludes docstrings. - A live_status fixture replaces three copies of the dialect probe. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HBt3Ee7eneKtdKQWq2T3YK --- CONTEXT.md | 2 +- dataretrieval/ngwmn.py | 16 ++++----- dataretrieval/waterdata/utils.py | 11 ++---- .../0014-services-endpoints-dialects.rst | 7 ++-- tests/architecture_test.py | 34 +++++-------------- tests/conftest.py | 15 ++++++++ tests/ngwmn_test.py | 12 +++---- tests/waterdata_ratings_test.py | 10 ++---- tests/waterdata_test.py | 12 +++---- tests/waterdata_utils_test.py | 2 +- 10 files changed, 48 insertions(+), 73 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 7d6bd98e..f23d0607 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -332,7 +332,7 @@ entry (ADR 0013). `collection`; `waterdata.get_cql` takes `collection`; and the type alias is `WATERDATA_COLLECTIONS`. `service=` on `get_cql` and the `WATERDATA_SERVICES` alias remain — a deprecated keyword and a permanent alias respectively. The - dialect field keyed by collection now says so too: + dialect field keyed by collection says so too: `DateDialect.date_only_collections`. `service` still means the external system in `transport` and `progress`, diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index 2b60dd15..b1e8ade3 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -4,7 +4,8 @@ (``https://api.waterdata.usgs.gov/ngwmn/ogcapi``) with five collections: ``sites``, ``waterLevelObs``, ``lithologyObs``, ``constructionObs``, and ``providers``. Each getter below delegates to the shared OGC facade -(:func:`~dataretrieval.ogc.get_ogc_data`) with ``base_url=NGWMN_OGC_API_URL``. +(:func:`~dataretrieval.ogc.get_ogc_data`) with an endpoint carrying +``NGWMN_DIALECT``. Multi-value chunking, pagination, retry/resume, and result shaping therefore behave as they do for the main Water Data getters. @@ -90,15 +91,10 @@ # (there is no service-specific id name as there is for the main collections). _NGWMN_OUTPUT_ID = "id" -# NGWMN reads comma-joined GET values as lists, and its firewall refuses a text -# filter naming a site, so a list could not be folded into one; lists stay -# comma-joined. It has no date-only collections, but its result columns need -# their own coercion and sort vocabulary: water-level observations are -# timestamped by ``sample_time`` (not the Water Data ``time``) and report -# depths/levels in feet. Its firewall answers ``datetime=../2016-01-01`` with -# HTTP 403, so an open start is sent empty (``/2016-01-01``), which it reads as -# open, and it answers a lone duration (``P7D``) with HTTP 400, so durations are -# off, as in the standard (both observed 2026-10-11). +# Lists stay comma-joined: NGWMN reads them as lists, and its firewall refuses a +# text filter naming a site, so a list could not be folded into one. +# ``open_start=""``: the firewall answers ``datetime=../2016-01-01`` with HTTP +# 403 and reads ``/2016-01-01`` as open (observed 2026-10-11). NGWMN_DIALECT = OgcDialect( dates=DateDialect(open_start=""), shaping=ShapingDialect( diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index 20d9d98b..2d0ca6b7 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -76,14 +76,9 @@ - {"monitoring_location_id", "time_series_id"} ) -# The Water Data API dialect: multi-value filters are CQL2 predicates, because -# the API reads a comma-joined GET value as one value for all but a few -# identifiers and codes; ``daily`` renders its time arguments date-only -# (``YYYY-MM-DD``); the API accepts a lone duration (``time=P7D``) and answers -# an empty open start (``time=/2024-01-03``) with HTTP 400, so it keeps ``..`` -# (both observed 2026-10-11); and the shaping fields are the Water-Data column -# vocabulary used to coerce datetime/numeric columns, sort results, and move -# synthetic ids to the end. +# ``lists_as_cql2``: the API reads a comma-joined GET value as one value for all +# but a few identifiers and codes. Dates keep ``..`` for an open start (an empty +# one is HTTP 400) and allow a lone duration (both observed 2026-10-11). # ``construction_date`` is deliberately not a time column: the service sends it # as a string at day, month, or year precision (``19950812``, ``199508``, # ``2005``), and no one datetime parse keeps all three, so it stays as sent. diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index af46cd6a..43f7c040 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -286,10 +286,9 @@ Revisions: date group, ``dates``, and the result-shaping group, ``shaping`` (``ShapingDialect``: ``time_cols``, ``numerical_cols``, ``sort_cols``, and ``extra_id_cols``, which moved there from an argument of ``get_ogc_data``). - Request construction reads one field, ``lists_as_cql2`` (which replaced the - per-collection ``cql2_services`` in #446), so it has no group of its own. The chunk planner lists - no date parameter: a date argument arrives as a ``DateRange``, which is - never split. ``tests/architecture_test.py`` fails if a string a shipped + Request construction reads one switch, ``lists_as_cql2`` (#446), so it has + no group of its own. The chunk planner lists no date parameter: a date + argument arrives as a ``DateRange``, which is never split. ``tests/architecture_test.py`` fails if a string a shipped dialect lists appears as a literal in ``ogc`` or ``_intervals``. Of the four recorded deviations, only ``monitoring_location_id`` is such a literal, so it is the one allowlisted entry. The switch is per API because whether the diff --git a/tests/architecture_test.py b/tests/architecture_test.py index 320540ff..5a1911c1 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -372,31 +372,15 @@ def test_credential_policy_has_one_definition() -> None: ``transport.http`` re-exports the predicate, it does not restate it. """ host = "api.waterdata.usgs.gov" - # Walked as AST string *values*, not as source text. A line-substring match - # produces both false negatives and false positives: it missed the - # ``"https://…"`` form three modules use to write the same host, and it - # flagged docstring prose that only names the service. Docstrings are - # excluded here (they are documentation, not a second definition) - # while every other literal -- bare host or full base URL -- counts. - offenders: list[str] = [] - for path in sorted(PACKAGE_ROOT.rglob("*.py")): - if path.name == "credentials.py": - continue - tree = ast.parse(path.read_text(encoding="utf-8")) - docstrings = { - text - for node in ast.walk(tree) - if isinstance( - node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) - ) - for text in [ast.get_docstring(node, clean=False)] - if text is not None - } - for node in ast.walk(tree): - if not isinstance(node, ast.Constant) or not isinstance(node.value, str): - continue - if host in node.value and node.value not in docstrings: - offenders.append(f"{_module_name(path)}:{node.lineno}") + # String *values*, not source text: a line match missed the ``"https://…"`` + # form and flagged docstring prose that only names the service. + offenders = [ + f"{_module_name(path)}:{line}" + for path in sorted(PACKAGE_ROOT.rglob("*.py")) + if path.name != "credentials.py" + for line, value in _code_string_literals(path) + if host in value + ] assert not offenders, ( "The API-key host must come from dataretrieval.credentials, " f"not be restated at: {offenders}" diff --git a/tests/conftest.py b/tests/conftest.py index 793dfb06..69155c81 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,3 +64,18 @@ def _pin_chunker_env(monkeypatch, tmp_path): # pointing the variable at a file they wrote. monkeypatch.setenv("DATARETRIEVAL_CONFIG", str(tmp_path / "no-such-config.toml")) configuration._reset_file_cache() + + +@pytest.fixture +def live_status(): + """``live_status(url, **params)``: the status a running service answers a GET + with, for the live tests that pin a dialect field to server behavior.""" + import httpx + + from dataretrieval.transport.http import default_headers + + def status(url: str, **params: object) -> int: + headers = default_headers(url) + return httpx.get(url, params=params, headers=headers, timeout=60).status_code + + return status diff --git a/tests/ngwmn_test.py b/tests/ngwmn_test.py index 0dd9af85..bf5dc3df 100644 --- a/tests/ngwmn_test.py +++ b/tests/ngwmn_test.py @@ -659,27 +659,23 @@ def test_get_water_level_refuses_a_duration_before_sending(httpx_mock): @pytest.mark.live -def test_date_dialect_matches_the_service(): +def test_date_dialect_matches_the_service(live_status): """Each ``NGWMN_DIALECT.dates`` field against the running service: the request the field prevents fails, and the one it produces succeeds. Kept live on purpose: the 403 comes from a firewall the API does not describe, so only the running service can show it has changed. """ - import httpx - from dataretrieval._dates import DateRange from dataretrieval._intervals import write_interval from dataretrieval.ngwmn import NGWMN_DIALECT, NGWMN_OGC_API_URL - from dataretrieval.transport.http import default_headers url = f"{NGWMN_OGC_API_URL}/collections/waterLevelObs/items" def status(value): - params = {"monitoring_location_id": _OTHER_SITE, "limit": 1, "datetime": value} - return httpx.get( - url, params=params, headers=default_headers(url), timeout=60 - ).status_code + return live_status( + url, monitoring_location_id=_OTHER_SITE, limit=1, datetime=value + ) # open_start: the standard's "../" is refused; the empty start is read as open. end = DateRange(end=datetime.datetime(2016, 1, 1, tzinfo=datetime.timezone.utc)) diff --git a/tests/waterdata_ratings_test.py b/tests/waterdata_ratings_test.py index 2fc263bb..a188386b 100644 --- a/tests/waterdata_ratings_test.py +++ b/tests/waterdata_ratings_test.py @@ -444,27 +444,21 @@ def test_stac_next_link_strips_embedded_credentials(httpx_mock): @pytest.mark.live -def test_stac_date_dialect_matches_the_service(): +def test_stac_date_dialect_matches_the_service(live_status): """Each ``STAC_DATES`` field against the running search: the request the field prevents fails, and the one it produces succeeds. The 403 comes from a firewall the API does not describe, so only the running service shows it.""" import datetime - import httpx - from dataretrieval._dates import DateRange from dataretrieval._intervals import write_interval - from dataretrieval.transport.http import default_headers from dataretrieval.waterdata.endpoints import ratings_catalog_url from dataretrieval.waterdata.ratings import STAC_DATES url = f"{ratings_catalog_url()}/search" def status(value): - params = {"limit": 1, "datetime": value} - return httpx.get( - url, params=params, headers=default_headers(url), timeout=60 - ).status_code + return live_status(url, limit=1, datetime=value) # open_start: the standard's "../" is refused; the empty start is read as open. end = DateRange(end=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)) diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index a9c3e454..3a6819b3 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -2057,25 +2057,21 @@ def test_get_reference_table_docstring_lists_every_collection(): @pytest.mark.live -def test_ogc_date_dialect_matches_the_service(): +def test_ogc_date_dialect_matches_the_service(live_status): """The ``WATERDATA_DIALECT.dates`` fields that record server behavior, against the running API: it accepts the request each field produces and refuses the one ``open_start`` prevents.""" - import httpx - from dataretrieval._dates import DateRange from dataretrieval._intervals import write_interval - from dataretrieval.transport.http import default_headers from dataretrieval.waterdata.endpoints import ogc_api_url url = f"{ogc_api_url()}/collections/daily/items" dates = WATERDATA_DIALECT.dates def status(value): - params = {"monitoring_location_id": "USGS-05427718", "limit": 1, "time": value} - return httpx.get( - url, params=params, headers=default_headers(url), timeout=60 - ).status_code + return live_status( + url, monitoring_location_id="USGS-05427718", limit=1, time=value + ) # open_start: the default ".." is accepted; the empty start is refused. end = DateRange(end=datetime.datetime(2024, 1, 3, tzinfo=datetime.timezone.utc)) diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 176b16d6..1b855c1f 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1779,7 +1779,7 @@ def test_get_ogc_data_reads_everything_about_the_service_from_its_endpoint( ): """ADR 0014: an endpoint is all the OGC engine is told about a service. The request goes to its URL, its dialect decides the request shape (a - ``lists_as_cql2`` list is POSTed), and settings resolve by its + list is POSTed under ``lists_as_cql2``), and settings resolve by its service's name.""" import dataretrieval from dataretrieval.ngwmn import NgwmnConfiguration From 6b10a3664e0c76eb54e90955c05cba1f42dc4c5a Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Mon, 12 Oct 2026 09:43:15 -0500 Subject: [PATCH 21/22] fix(ogc): read a date in get_reference_table's query (ADR 0014) get_reference_table(query={"last_modified": "2024-01-01"}) raised AttributeError: a query dict skips prepare_request_args, so the request builder received a raw string where it expects a DateRange. get_ogc_data now reads any date parameter still raw with the endpoint's DateDialect, so the planner and the builder see a DateRange on every path. Also: read_date_range's name is required, not optional, in its docstring. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HBt3Ee7eneKtdKQWq2T3YK --- dataretrieval/_dates.py | 2 +- dataretrieval/ogc/engine.py | 2 ++ dataretrieval/ogc/requests.py | 16 +++++++++++++++- tests/waterdata_test.py | 17 +++++++++++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/dataretrieval/_dates.py b/dataretrieval/_dates.py index afe4ce03..72f02734 100644 --- a/dataretrieval/_dates.py +++ b/dataretrieval/_dates.py @@ -177,7 +177,7 @@ def read_date_range( ``pandas.Timestamp``, or a string in ISO 8601 (with or without ``Z`` or a numeric offset) or ``"%Y-%m-%d %H:%M:%S"`` form. ``None``, ``NaN``, ``NaT``, ``""`` and ``".."`` are open bounds. - name : str, optional + name : str The caller's own spelling of this argument, used as the subject of every message raised here, so a caller correcting the error edits an argument their getter accepts. diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index a21a31fb..e8efba49 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -43,6 +43,7 @@ from dataretrieval.ogc.requests import ( _construct_api_requests, _construct_cql_request, + _read_raw_dates, _switch_arg_id, _switch_properties_id, ) @@ -295,6 +296,7 @@ def get_ogc_data( # The APIs answer a blank filter with HTTP 400. del args["filter"] args.pop("filter_lang", None) + args = _read_raw_dates(args, endpoint.dialect.dates) # Choose one semantic frame shape for the whole request. Every page and # the all-empty finalizer receive these same values, so frame type never diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 6b11dde1..91f59e21 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -16,7 +16,7 @@ import httpx -from dataretrieval._dates import read_date_range +from dataretrieval._dates import DateRange, read_date_range from dataretrieval._intervals import DateDialect, write_interval from dataretrieval.ogc.filters import _cql_in_clause from dataretrieval.transport.http import default_headers as _default_headers @@ -337,6 +337,20 @@ def _check_monitoring_location_id( return value +def _read_raw_dates(args: dict[str, Any], dates: DateDialect) -> dict[str, Any]: + """``args`` with each date parameter not yet read (a caller's verbatim + ``query``, which skips :func:`prepare_request_args`) read into a + ``DateRange``, and dropped when every bound is open.""" + out = dict(args) + for key in dates.params & out.keys(): + if isinstance(out[key], DateRange): + continue + value = read_date_range(out.pop(key), name=key, durations=dates.durations) + if value is not None: + out[key] = value + return out + + def _normalize_arg( key: str, value: Any, diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 3a6819b3..7d4913a8 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -1734,6 +1734,23 @@ def test_get_reference_table_serves_countries(httpx_mock): assert "country" in df.columns +@pytest.mark.parametrize( + ("value", "sent"), + [ + ("2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z"), + (["2024-01-01T00:00:00Z", None], "2024-01-01T00:00:00Z/.."), + ], +) +def test_get_reference_table_reads_a_date_in_its_query(httpx_mock, value, sent): + """A ``query`` dict skips the getters' argument reading, so the engine reads + its date parameters; a raw string used to raise AttributeError.""" + _mock_items(httpx_mock, "agency-codes") + + get_reference_table("agency-codes", query={"last_modified": value}) + + assert _sent(httpx_mock, "agency-codes")[0]["last_modified"] == [sent] + + def test_get_reference_table_with_query(httpx_mock): """A ``query`` dict is merged into the request's query params.""" _mock_items(httpx_mock, "agency-codes") From c33757cb0fe14fe53100219559960252728f15ca Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Mon, 12 Oct 2026 09:54:41 -0500 Subject: [PATCH 22/22] refactor: one dialect reads and writes a call's dates (ADR 0014) - DateDialect holds only the interval grammar STAC and OGC share (open_start, durations). The fields only OGC reads -- date_params, date_only_collections, timed_params -- move to OgcDialect, so STAC's dialect no longer carries collection fields it never reads. - prepare_request_args takes the OgcDialect its endpoint carries rather than a DateDialect, and given none reads no date. The Samples and Statistics getters call _get_query_args, which passes none; the _NO_DATES sentinel they had to remember is gone. - get_ogc_data raises when a DateRange arrives under a key the endpoint's dialect does not list, so arguments read with one dialect are not sent with another as a repr. - Date arguments are annotated DateArgument (a date, datetime, Timestamp, string, or a list of them with None for an open end), matching what the getters accept since the 10/09 NEWS entry; a typed caller passing time=[df.index.min(), None] failed mypy. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HBt3Ee7eneKtdKQWq2T3YK --- CONTEXT.md | 6 +- dataretrieval/_dates.py | 10 ++- dataretrieval/_intervals.py | 18 +---- dataretrieval/ngwmn.py | 5 +- dataretrieval/ogc/engine.py | 2 +- dataretrieval/ogc/policy.py | 16 ++++- dataretrieval/ogc/requests.py | 69 ++++++++++++------- dataretrieval/waterdata/measurements.py | 13 ++-- dataretrieval/waterdata/metadata.py | 23 ++++--- dataretrieval/waterdata/ratings.py | 4 +- dataretrieval/waterdata/samples.py | 5 +- dataretrieval/waterdata/time_series.py | 25 ++++--- dataretrieval/waterdata/utils.py | 55 ++++++++------- .../0014-services-endpoints-dialects.rst | 50 ++++++++------ tests/_dates_test.py | 24 +++++-- tests/waterdata_test.py | 2 +- 16 files changed, 189 insertions(+), 138 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index f23d0607..57c5d08e 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -279,8 +279,8 @@ two endpoints speak it. the protocol code reads. Each field is one extension, restriction, or renaming, and its default is the standard, so a default dialect describes an endpoint that follows the standard. Fields are grouped by the code that reads them; the -date group, `DateDialect`, is meant to be shared by the protocols that write -intervals (ADR 0014, proposed). +date group, `DateDialect`, holds the interval grammar the protocols that write +intervals share (ADR 0014, proposed). **Single-shot adapter** — An adapter whose query is always exactly one request: WQP, NLDI, StreamStats, and deprecated NWIS. Nothing divides and nothing @@ -333,7 +333,7 @@ entry (ADR 0013). `WATERDATA_COLLECTIONS`. `service=` on `get_cql` and the `WATERDATA_SERVICES` alias remain — a deprecated keyword and a permanent alias respectively. The dialect field keyed by collection says so too: - `DateDialect.date_only_collections`. + `OgcDialect.date_only_collections`. `service` still means the external system in `transport` and `progress`, where it labels a progress line. That usage is correct. diff --git a/dataretrieval/_dates.py b/dataretrieval/_dates.py index 72f02734..32b27673 100644 --- a/dataretrieval/_dates.py +++ b/dataretrieval/_dates.py @@ -10,7 +10,7 @@ from __future__ import annotations import re -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping from dataclasses import dataclass from datetime import date as _date from datetime import datetime @@ -43,6 +43,10 @@ # with ``None`` (or ``NaN`` / ``NaT``) for an open bound. DateLike = str | _date | None +# What a getter accepts as one date argument: one element, an interval string, +# or a list of one or two elements (a range whose ``None`` end is open). +DateArgument = DateLike | Iterable[DateLike] + @dataclass(frozen=True) class DateRange: @@ -123,7 +127,7 @@ def _read_bound(dt: DateLike, *, name: str) -> datetime | None: def _coerce_to_list( - value: str | _date | Sequence[DateLike], + value: DateArgument, name: str, ) -> list[DateLike]: """Normalize a date argument to a list of elements, raising on a mapping.""" @@ -161,7 +165,7 @@ def _split_interval(interval: str, *, name: str) -> tuple[str, str]: def read_date_range( - value: str | _date | Sequence[DateLike] | None, + value: DateArgument | None, *, name: str, durations: bool, diff --git a/dataretrieval/_intervals.py b/dataretrieval/_intervals.py index c5b1e1fb..91d9e95c 100644 --- a/dataretrieval/_intervals.py +++ b/dataretrieval/_intervals.py @@ -24,17 +24,6 @@ class DateDialect: Attributes ---------- - params : frozenset[str] - The parameters that carry a date range. The standard has one, - ``datetime``; an endpoint that filters other fields by date lists them - all. - date_only_collections : frozenset[str] - Collections whose dates are written as ``YYYY-MM-DD`` rather than as a - UTC datetime, because their values are calendar days and a naive bound - converted to UTC could move to the neighboring day. - timed_params : frozenset[str] - Parameters written as a UTC datetime even in a date-only collection, - because they filter a timestamp. open_start : str How the open start of a range is written: ``".."`` (``"../2024-01-01"``, the default) or ``""`` (``"/2024-01-01"``). Both standards allow @@ -46,9 +35,6 @@ class DateDialect: a duration raises when the getter is called. """ - params: frozenset[str] = frozenset({"datetime"}) - date_only_collections: frozenset[str] = frozenset() - timed_params: frozenset[str] = frozenset() open_start: str = OPEN_BOUND durations: bool = False @@ -77,8 +63,8 @@ def write_interval(value: DateRange, dialect: DateDialect, *, date_only: bool) - dialect : DateDialect The endpoint's date dialect; supplies ``open_start``. date_only : bool - Write each bound as ``YYYY-MM-DD`` rather than as a UTC datetime. The - caller decides from ``date_only_collections`` and ``timed_params``. + Write each bound as ``YYYY-MM-DD`` rather than as a UTC datetime; the + caller's protocol decides which values are calendar days. """ if value.duration is not None: return value.duration diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index b1e8ade3..017267aa 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -45,6 +45,7 @@ ) if TYPE_CHECKING: + from dataretrieval._dates import DateArgument from dataretrieval._response_metadata import BaseMetadata __all__ = [ @@ -120,7 +121,7 @@ def _get(service: str, local_vars: dict[str, Any]) -> tuple[pd.DataFrame, BaseMe queryable = _STATE_QUERYABLE.get(service) if queryable is not None: apply_state(local_vars, to=queryable["to"], into=queryable["into"]) - args = prepare_request_args(local_vars, dates=NGWMN_DIALECT.dates) + args = prepare_request_args(local_vars, dialect=NGWMN_DIALECT) return get_ogc_data( args, service, @@ -250,7 +251,7 @@ def get_water_level( water_level_above_site_datum_ft: str | Iterable[str] | None = None, monitoring_location_vertical_datum: str | Iterable[str] | None = None, water_level_above_navd88_ft: str | Iterable[str] | None = None, - datetime: str | Iterable[str] | None = None, + datetime: DateArgument | None = None, properties: str | Iterable[str] | None = None, limit: int | None = None, convert_type: bool = True, diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index e8efba49..cef1ed45 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -296,7 +296,7 @@ def get_ogc_data( # The APIs answer a blank filter with HTTP 400. del args["filter"] args.pop("filter_lang", None) - args = _read_raw_dates(args, endpoint.dialect.dates) + args = _read_raw_dates(args, endpoint.dialect) # Choose one semantic frame shape for the whole request. Every page and # the all-empty finalizer receive these same values, so frame type never diff --git a/dataretrieval/ogc/policy.py b/dataretrieval/ogc/policy.py index bd927b1e..7fed18c7 100644 --- a/dataretrieval/ogc/policy.py +++ b/dataretrieval/ogc/policy.py @@ -78,12 +78,26 @@ class OgcDialect: Whether a filter with more than one value is sent as a CQL2 ``in`` predicate, rather than as a comma-joined GET value (``site=A,B``, the default) that the API would read as one value. + date_params : frozenset[str] + The parameters that carry a date range. The standard has one, + ``datetime``; an API that filters other fields by date lists them all. + date_only_collections : frozenset[str] + Collections whose dates are written as ``YYYY-MM-DD`` rather than as a + UTC datetime, because their values are calendar days and a naive bound + converted to UTC could move to the neighboring day. + timed_params : frozenset[str] + Parameters written as a UTC datetime even in a date-only collection, + because they filter a timestamp. dates : DateDialect - How the API's date parameters are read and written. + How the API's date values are read and written: the interval grammar + it shares with STAC. shaping : ShapingDialect How the API's results are shaped into a frame. """ lists_as_cql2: bool = False + date_params: frozenset[str] = frozenset({"datetime"}) + date_only_collections: frozenset[str] = frozenset() + timed_params: frozenset[str] = frozenset() dates: DateDialect = field(default_factory=DateDialect) shaping: ShapingDialect = field(default_factory=ShapingDialect) diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 91f59e21..60305ddc 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -17,7 +17,7 @@ import httpx from dataretrieval._dates import DateRange, read_date_range -from dataretrieval._intervals import DateDialect, write_interval +from dataretrieval._intervals import write_interval from dataretrieval.ogc.filters import _cql_in_clause from dataretrieval.transport.http import default_headers as _default_headers @@ -204,12 +204,12 @@ def _construct_api_requests( """ service_url = _items_url(collection, endpoint.url) dialect = endpoint.dialect - dates = dialect.dates - for key in dates.params & kwargs.keys(): + for key in dialect.date_params & kwargs.keys(): date_only = ( - collection in dates.date_only_collections and key not in dates.timed_params + collection in dialect.date_only_collections + and key not in dialect.timed_params ) - kwargs[key] = write_interval(kwargs[key], dates, date_only=date_only) + kwargs[key] = write_interval(kwargs[key], dialect.dates, date_only=date_only) json_filter = _pop_json_filter(kwargs) params, post_params = _partition_request_params( kwargs, use_cql2=dialect.lists_as_cql2 @@ -337,17 +337,34 @@ def _check_monitoring_location_id( return value -def _read_raw_dates(args: dict[str, Any], dates: DateDialect) -> dict[str, Any]: - """``args`` with each date parameter not yet read (a caller's verbatim - ``query``, which skips :func:`prepare_request_args`) read into a - ``DateRange``, and dropped when every bound is open.""" - out = dict(args) - for key in dates.params & out.keys(): - if isinstance(out[key], DateRange): - continue - value = read_date_range(out.pop(key), name=key, durations=dates.durations) - if value is not None: - out[key] = value +def _read_date(key: str, value: Any, dialect: OgcDialect) -> DateRange | None: + """A date argument as a ``DateRange``, read by the dialect of the endpoint + it is sent to; ``None`` when every bound is open.""" + if isinstance(value, DateRange): + return value + return read_date_range(value, name=key, durations=dialect.dates.durations) + + +def _read_raw_dates(args: dict[str, Any], dialect: OgcDialect) -> dict[str, Any]: + """``args`` with every date parameter of ``dialect`` read, so a verbatim + ``query`` that skipped :func:`prepare_request_args` is read too. + + A ``DateRange`` under a key the dialect does not list would be sent as its + repr, so it raises: the arguments were read with another API's dialect. + """ + out: dict[str, Any] = {} + for key, value in args.items(): + if key in dialect.date_params: + value = _read_date(key, value, dialect) + if value is None: + continue + elif isinstance(value, DateRange): + raise TypeError( + f"{key} was read as a date range, but this endpoint's dialect " + f"has no date parameter {key!r}. Read the arguments with the " + f"dialect of the endpoint they are sent to." + ) + out[key] = value return out @@ -355,15 +372,15 @@ def _normalize_arg( key: str, value: Any, no_normalize: frozenset[str], - dates: DateDialect, + dialect: OgcDialect | None, ) -> Any: """Normalize a single request argument value based on its key.""" if key == "monitoring_location_id": return _check_monitoring_location_id(value) if key == "properties": return _as_str_list(value, key) - if key in dates.params: - return read_date_range(value, name=key, durations=dates.durations) + if dialect is not None and key in dialect.date_params: + return _read_date(key, value, dialect) if ( key in no_normalize and isinstance(value, Iterable) @@ -379,15 +396,17 @@ def prepare_request_args( local_vars: dict[str, Any], exclude: set[str] | None = None, *, - dates: DateDialect, + dialect: OgcDialect | None = None, extra_no_normalize: frozenset[str] | set[str] = frozenset(), ) -> dict[str, Any]: - """Build OGC request kwargs from a getter's ``locals()``. + """Build request kwargs from a getter's ``locals()``. Internal control keys, caller-supplied exclusions, and ``None`` values - are omitted. Identifiers and properties are validated, the parameters - ``dates`` lists are read into a ``DateRange`` (and omitted when every bound - is open), and other iterables are normalized unless exempted. + are omitted. Identifiers and properties are validated, the date parameters + of ``dialect`` -- the one the call's endpoint carries -- are read into a + ``DateRange`` (and omitted when every bound is open), and other iterables + are normalized unless exempted. Without a dialect no argument is read as a + date, as for an API that is not OGC. ``extra_no_normalize`` *adds* to the engine's own :data:`_NO_NORMALIZE_PARAMS` rather than replacing it, so an adapter names @@ -403,7 +422,7 @@ def prepare_request_args( for k, v in local_vars.items(): if k in to_exclude: continue - normalized = _normalize_arg(k, v, no_normalize, dates) + normalized = _normalize_arg(k, v, no_normalize, dialect) if normalized is not None: args[k] = normalized return args diff --git a/dataretrieval/waterdata/measurements.py b/dataretrieval/waterdata/measurements.py index 51b184b0..906b0dea 100644 --- a/dataretrieval/waterdata/measurements.py +++ b/dataretrieval/waterdata/measurements.py @@ -18,6 +18,7 @@ ) if TYPE_CHECKING: + from dataretrieval._dates import DateArgument from dataretrieval._response_metadata import BaseMetadata from dataretrieval.ogc.filters import FILTER_LANG @@ -32,12 +33,12 @@ def get_field_measurements( unit_of_measure: str | Iterable[str] | None = None, qualifier: str | Iterable[str] | None = None, value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, + last_modified: DateArgument | None = None, observing_procedure: str | Iterable[str] | None = None, vertical_datum: str | Iterable[str] | None = None, measuring_agency: str | Iterable[str] | None = None, skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, + time: DateArgument | None = None, bbox: list[float] | None = None, limit: int | None = None, filter: str | None = None, @@ -243,8 +244,8 @@ def get_peaks( parameter_code: str | Iterable[str] | None = None, time_series_id: str | Iterable[str] | None = None, unit_of_measure: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, + time: DateArgument | None = None, + last_modified: DateArgument | None = None, water_year: int | list[int] | None = None, year: int | list[int] | None = None, month: int | list[int] | None = None, @@ -382,7 +383,7 @@ def get_channel( monitoring_location_id: str | Iterable[str] | None = None, field_visit_id: str | Iterable[str] | None = None, measurement_number: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, + time: DateArgument | None = None, channel_name: str | Iterable[str] | None = None, channel_flow: str | Iterable[str] | None = None, channel_flow_unit: str | Iterable[str] | None = None, @@ -401,7 +402,7 @@ def get_channel( vertical_velocity_description: str | Iterable[str] | None = None, longitudinal_velocity_description: str | Iterable[str] | None = None, measurement_type: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, + last_modified: DateArgument | None = None, channel_measurement_type: str | Iterable[str] | None = None, properties: str | Iterable[str] | None = None, skip_geometry: bool | None = None, diff --git a/dataretrieval/waterdata/metadata.py b/dataretrieval/waterdata/metadata.py index 3f2d14a7..92c1b463 100644 --- a/dataretrieval/waterdata/metadata.py +++ b/dataretrieval/waterdata/metadata.py @@ -21,6 +21,7 @@ ) if TYPE_CHECKING: + from dataretrieval._dates import DateArgument from dataretrieval._response_metadata import BaseMetadata from dataretrieval.ogc.filters import FILTER_LANG @@ -400,11 +401,11 @@ def get_time_series_metadata( hydrologic_unit_code: str | Iterable[str] | None = None, state: str | Iterable[str] | None = None, state_name: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, - begin_utc: str | Iterable[str] | None = None, - end_utc: str | Iterable[str] | None = None, + last_modified: DateArgument | None = None, + begin: DateArgument | None = None, + end: DateArgument | None = None, + begin_utc: DateArgument | None = None, + end_utc: DateArgument | None = None, unit_of_measure: str | Iterable[str] | None = None, computation_period_identifier: str | Iterable[str] | None = None, computation_identifier: str | Iterable[str] | None = None, @@ -674,9 +675,9 @@ def get_combined_metadata( primary: str | Iterable[str] | None = None, parent_time_series_id: str | Iterable[str] | None = None, web_description: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, + last_modified: DateArgument | None = None, + begin: DateArgument | None = None, + end: DateArgument | None = None, agency_code: str | Iterable[str] | None = None, agency_name: str | Iterable[str] | None = None, monitoring_location_number: str | Iterable[str] | None = None, @@ -927,9 +928,9 @@ def get_field_measurements_metadata( parameter_code: str | Iterable[str] | None = None, parameter_name: str | Iterable[str] | None = None, parameter_description: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, + begin: DateArgument | None = None, + end: DateArgument | None = None, + last_modified: DateArgument | None = None, properties: str | Iterable[str] | None = None, skip_geometry: bool | None = None, bbox: list[float] | None = None, diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index 4843e383..2e88e709 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -18,7 +18,7 @@ import httpx import pandas as pd -from dataretrieval._dates import read_date_range +from dataretrieval._dates import DateArgument, read_date_range from dataretrieval._intervals import DateDialect, write_interval from dataretrieval._service import Endpoint from dataretrieval._validation import render_options @@ -54,7 +54,7 @@ def get_ratings( monitoring_location_id: str | Iterable[str] | None = None, file_type: RATING_FILE_TYPE | list[RATING_FILE_TYPE] = "exsa", file_path: str | None = None, - time: str | list[str] | None = None, + time: DateArgument | None = None, bbox: list[float] | None = None, limit: int = 10000, download_and_parse: bool = True, diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py index a471f371..4048fa15 100644 --- a/dataretrieval/waterdata/samples.py +++ b/dataretrieval/waterdata/samples.py @@ -40,9 +40,8 @@ _check_profiles, ) from dataretrieval.waterdata.utils import ( - _NO_DATES, _accept_legacy_kwargs, - _get_args, + _get_query_args, ) logger = logging.getLogger(__name__) @@ -357,7 +356,7 @@ def get_samples( # Build argument dictionary, omitting None values. Parameters are the # public snake_case names here; translate them to the camelCase names the # Samples API expects just before building the request. - args = _get_args(locals(), exclude={"ssl_check", "profile"}, dates=_NO_DATES) + args = _get_query_args(locals(), exclude={"ssl_check", "profile"}) params = {_SAMPLES_PARAM_TO_API.get(key, key): value for key, value in args.items()} params.update({"mimeType": "text/csv"}) diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py index c4896ec8..c5282b9d 100644 --- a/dataretrieval/waterdata/time_series.py +++ b/dataretrieval/waterdata/time_series.py @@ -18,13 +18,14 @@ from dataretrieval.waterdata import stats from dataretrieval.waterdata.utils import ( - _NO_DATES, _get_args, + _get_query_args, _with_state, get_ogc_data, ) if TYPE_CHECKING: + from dataretrieval._dates import DateArgument from dataretrieval._response_metadata import BaseMetadata from dataretrieval.ogc.filters import FILTER_LANG @@ -40,9 +41,9 @@ def get_daily( unit_of_measure: str | Iterable[str] | None = None, qualifier: str | Iterable[str] | None = None, value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, + last_modified: DateArgument | None = None, skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, + time: DateArgument | None = None, bbox: list[float] | None = None, limit: int | None = None, filter: str | None = None, @@ -276,8 +277,8 @@ def get_continuous( qualifier: str | Iterable[str] | None = None, method_category: str | Iterable[str] | None = None, value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, + last_modified: DateArgument | None = None, + time: DateArgument | None = None, limit: int | None = None, filter: str | None = None, filter_lang: FILTER_LANG | None = None, @@ -487,9 +488,9 @@ def get_latest_continuous( unit_of_measure: str | Iterable[str] | None = None, qualifier: str | Iterable[str] | None = None, value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, + last_modified: DateArgument | None = None, skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, + time: DateArgument | None = None, bbox: list[float] | None = None, limit: int | None = None, filter: str | None = None, @@ -704,9 +705,9 @@ def get_latest_daily( unit_of_measure: str | Iterable[str] | None = None, qualifier: str | Iterable[str] | None = None, value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, + last_modified: DateArgument | None = None, skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, + time: DateArgument | None = None, bbox: list[float] | None = None, limit: int | None = None, filter: str | None = None, @@ -1042,10 +1043,9 @@ def get_stats_por( ... ) """ # Build argument dictionary, omitting None values - params = _get_args( + params = _get_query_args( _with_state(locals(), to="fips_us", into="state_code"), exclude={"expand_percentiles"}, - dates=_NO_DATES, ) return stats.get_data( @@ -1190,10 +1190,9 @@ def get_stats_date_range( ... ) """ # Build argument dictionary, omitting None values - params = _get_args( + params = _get_query_args( _with_state(locals(), to="fips_us", into="state_code"), exclude={"expand_percentiles"}, - dates=_NO_DATES, ) return stats.get_data( diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index 2d0ca6b7..eb40f3ab 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -84,22 +84,20 @@ # ``2005``), and no one datetime parse keeps all three, so it stays as sent. WATERDATA_DIALECT = OgcDialect( lists_as_cql2=True, - dates=DateDialect( - params=frozenset( - { - "datetime", - "last_modified", - "begin", - "begin_utc", - "end", - "end_utc", - "time", - } - ), - date_only_collections=frozenset({"daily"}), - timed_params=frozenset({"last_modified"}), - durations=True, + date_params=frozenset( + { + "datetime", + "last_modified", + "begin", + "begin_utc", + "end", + "end_utc", + "time", + } ), + date_only_collections=frozenset({"daily"}), + timed_params=frozenset({"last_modified"}), + dates=DateDialect(durations=True), shaping=ShapingDialect( time_cols=frozenset( { @@ -187,21 +185,26 @@ def _flatten_queryables(local_vars: dict[str, Any]) -> dict[str, Any]: return local_vars -# For a getter whose API is not OGC: no parameter is read as a date range. -_NO_DATES = DateDialect(params=frozenset()) +def _get_args( + local_vars: dict[str, Any], exclude: set[str] | None = None +) -> dict[str, Any]: + """Request arguments for a Water Data OGC getter, its date parameters read + by :data:`WATERDATA_DIALECT`, the dialect its endpoint carries. + See :func:`_get_query_args` for what else is done. + """ + return _get_query_args(local_vars, exclude, dialect=WATERDATA_DIALECT) -def _get_args( + +def _get_query_args( local_vars: dict[str, Any], exclude: set[str] | None = None, *, - dates: DateDialect = WATERDATA_DIALECT.dates, + dialect: OgcDialect | None = None, ) -> dict[str, Any]: - """Water-Data wrapper over :func:`~dataretrieval.ogc.prepare_request_args`. - - ``dates`` names the parameters read as date ranges: the OGC API's by - default, and none (:data:`_NO_DATES`) for the Samples and Statistics - getters, whose APIs take dates in their own form. + """Request arguments for a Water Data getter: the Samples and Statistics + getters call this directly, since their APIs are not OGC and read no date + range; the OGC getters call :func:`_get_args`. Adds the Water Data API's extra no-normalize params (numeric params such as ``water_year``, ``thresholds``, ``boundingBox``) so they keep their @@ -212,7 +215,7 @@ def _get_args( args = prepare_request_args( local_vars, exclude, - dates=dates, + dialect=dialect, extra_no_normalize=_NO_NORMALIZE_PARAMS, ) for name in _JSON_ARRAY_PARAMS & args.keys(): @@ -384,7 +387,7 @@ def wrapper(*args: Any, **kwargs: Any) -> _R: "OGC_API_URL", "SAMPLES_URL", "WATERDATA_DIALECT", - "_NO_DATES", + "_get_query_args", "_NO_NORMALIZE_PARAMS", "_OUTPUT_ID_BY_COLLECTION", "_accept_legacy_kwargs", diff --git a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst index 43f7c040..e28cbe0c 100644 --- a/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst +++ b/docs/source/architecture/decisions/0014-services-endpoints-dialects.rst @@ -250,25 +250,32 @@ Revisions: - *The package-wide date range.* ``dataretrieval._dates`` is a dependency-free leaf holding :class:`DateRange` and ``read_date_range``: an instant, a range whose ``None`` bound is open, or a duration, with a naive bound meaning local - time. ``prepare_request_args`` reads every OGC date parameter into one when - a getter is called, and drops a range whose bounds are all open, as it drops - ``None``; ``get_ratings`` reads its ``time`` the same way. The OGC request - builder only writes the value. Error messages are unchanged by this step; - they are raised when the getter is called rather than when the first - request is built. + time. ``prepare_request_args`` reads every date parameter of the + ``OgcDialect`` it is given into one when a getter is called, and drops a + range whose bounds are all open, as it drops ``None``; ``get_ratings`` reads + its ``time`` the same way. ``get_ogc_data`` reads any date parameter still + raw -- a verbatim ``query`` skips ``prepare_request_args`` -- with the + endpoint's dialect, and raises when a ``DateRange`` arrives under a key that + dialect does not list, so arguments read with one dialect are not sent with + another. The OGC request builder only writes the value. Error messages are + unchanged by this step; they are raised when the getter is called rather + than when the first request is built. - *The shared interval grammar and* ``DateDialect``. ``dataretrieval._intervals`` holds ``DateDialect`` and ``write_interval``, and sits below ``ogc`` because - STAC is not OGC; ``ogc.dates`` is removed. ``DateDialect`` has five fields: - ``params``, ``date_only_collections``, ``timed_params``, ``open_start``, and - ``durations``, whose default is ``False`` because neither standard defines a - lone duration. ``OgcDialect.dates`` holds the group, and - ``prepare_request_args`` takes it, so the reading side knows which parameters - carry dates and whether a duration is accepted: an endpoint that refuses one - raises when the getter is called. ``write_interval`` never raises; the - protocol code decides from ``date_only_collections`` and ``timed_params`` - whether it writes date-only bounds, since only OGC has collections. The - Samples and Statistics getters, which share Water Data's argument helper, - read no parameter as a date range. Two messages change: the duration refusal + STAC is not OGC; ``ogc.dates`` is removed. ``DateDialect`` holds the grammar + both protocols read, two fields: ``open_start``, and ``durations``, whose + default is ``False`` because neither standard defines a lone duration. The + fields only OGC reads -- ``date_params``, ``date_only_collections``, and + ``timed_params`` -- are fields of ``OgcDialect``, since only OGC has + collections and STAC names its one date parameter in ``get_ratings``. + ``OgcDialect.dates`` holds the grammar, and ``prepare_request_args`` takes + the ``OgcDialect``, so the reading side knows which parameters carry dates + and whether a duration is accepted: an endpoint that refuses one raises when + the getter is called. Given no dialect it reads no date, which is how the + Samples and Statistics getters call it, through their own argument helper. + ``write_interval`` never raises; the OGC request builder decides from + ``date_only_collections`` and ``timed_params`` whether it writes date-only + bounds. Two messages change: the duration refusal is one shared message naming the caller's argument (``get_ratings`` had its own), and the "too many values" message offers a duration only where one is accepted. A field's observed server behavior is recorded where the field is @@ -286,10 +293,11 @@ Revisions: date group, ``dates``, and the result-shaping group, ``shaping`` (``ShapingDialect``: ``time_cols``, ``numerical_cols``, ``sort_cols``, and ``extra_id_cols``, which moved there from an argument of ``get_ogc_data``). - Request construction reads one switch, ``lists_as_cql2`` (#446), so it has - no group of its own. The chunk planner lists no date parameter: a date - argument arrives as a ``DateRange``, which is never split. ``tests/architecture_test.py`` fails if a string a shipped - dialect lists appears as a literal in ``ogc`` or ``_intervals``. Of the four + Request construction reads ``lists_as_cql2`` (#446) and the three OGC date + fields, which sit on ``OgcDialect`` itself. The chunk planner lists no date + parameter: a date argument arrives as a ``DateRange``, which is never split. + ``tests/architecture_test.py`` fails if a string a shipped dialect lists + appears as a literal in ``ogc`` or ``_intervals``. Of the four recorded deviations, only ``monitoring_location_id`` is such a literal, so it is the one allowlisted entry. The switch is per API because whether the Water Data API reads a comma-joined GET value as a list depends on the diff --git a/tests/_dates_test.py b/tests/_dates_test.py index 2b8c758e..ae75ba2f 100644 --- a/tests/_dates_test.py +++ b/tests/_dates_test.py @@ -10,8 +10,9 @@ import pytest from dataretrieval._dates import DateRange, read_date_range -from dataretrieval._intervals import DateDialect from dataretrieval.ogc import prepare_request_args +from dataretrieval.ogc.policy import OgcDialect +from dataretrieval.ogc.requests import _read_raw_dates NAIVE = datetime.datetime(2024, 1, 1) # noqa: DTZ001 -- a naive bound is the case UTC = datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) @@ -43,13 +44,28 @@ def test_prepare_request_args_reads_date_arguments_on_entry(): """A getter's date arguments are read when it is called, so an unreadable one raises before any request is planned, and an all-open one is dropped like ``None``.""" - dates = DateDialect(params=frozenset({"time", "last_modified"})) + dialect = OgcDialect(date_params=frozenset({"time", "last_modified"})) args = prepare_request_args( - {"time": ["2024-01-01", None], "last_modified": [None, ".."]}, dates=dates + {"time": ["2024-01-01", None], "last_modified": [None, ".."]}, + dialect=dialect, ) assert args == {"time": DateRange(start=NAIVE)} with pytest.raises(ValueError, match="^time could not be read"): - prepare_request_args({"time": "yesterday"}, dates=dates) + prepare_request_args({"time": "yesterday"}, dialect=dialect) + + +def test_prepare_request_args_reads_no_date_without_a_dialect(): + """An API that is not OGC (Samples, Statistics) gets its arguments as given.""" + assert prepare_request_args({"time": "2024-01-01"}) == {"time": "2024-01-01"} + + +def test_a_date_range_the_endpoint_does_not_list_raises(): + """Arguments read with one dialect and sent with another would put a + ``DateRange`` repr in the URL.""" + args = {"time": DateRange(start=UTC)} + assert _read_raw_dates(args, OgcDialect(date_params=frozenset({"time"}))) == args + with pytest.raises(TypeError, match="^time was read as a date range"): + _read_raw_dates(args, OgcDialect()) def test_read_date_range_refuses_a_duration_where_none_is_accepted(): diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index 7d4913a8..6f3b4ddb 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -92,7 +92,7 @@ def _construct_api_requests(collection, **kwargs): endpoint=Endpoint(SERVICE, OGC_API_URL, WATERDATA_DIALECT), **prepare_request_args( kwargs, - dates=WATERDATA_DIALECT.dates, + dialect=WATERDATA_DIALECT, extra_no_normalize=_WATERDATA_NO_NORMALIZE, ), )