From 7d8919296a0aa120106adced711d3a35c1730bce Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 1 Nov 2026 21:04:25 +0900 Subject: [PATCH] native: keep float64 array doubles and floor epoch millis in integer arithmetic A1: a float64 ndarray bound by name, inside CONTENT :row or set on a record went through the float32 side buffer in binary mode (and the "F" tag, a float[] in Java, in text mode), so 0.1 was stored as 0.10000000149011612 while the JVM build stores the double. Only float32 arrays take the side buffer now; any other float array is its tolist(). A float array among the positional parameters stays a query vector sent as float32 (encode_params narrows it first, as Database._convert_args does on the JVM build), as documented. C1: json_native computed epoch milliseconds as int(total_seconds() * 1000), which rounds toward zero before 1970, rolled 9999-12-31 23:59:59.999999 into year 10000 and could land 1 ms low on exact milliseconds. Integer days/seconds/ microseconds arithmetic, floored like Instant.toEpochMilli. Tests are shared by both builds. Docs: the query vector note now says named, CONTENT and record float64 arrays keep their doubles on both builds. Against the main 26.11.1.dev0 wheels the native wheel fails the three float64 tests and the DATETIME_MICROS epoch test and the JVM wheel passes all nine. Closes #175 Co-Authored-By: Claude Opus 5.5 --- .../pages/how-to/databases-and-queries.adoc | 2 +- src/arcadedb/_native/codec.py | 46 +++++++++++--- tests/test_numpy_support.py | 63 +++++++++++++++++++ tests/test_resultset.py | 45 +++++++++++++ 4 files changed, 145 insertions(+), 11 deletions(-) diff --git a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc index 165ac4c..8c644be 100644 --- a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc +++ b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc @@ -226,7 +226,7 @@ A query vector for `vectorNeighbors` can be a numpy array or a list. The convers * A `float32` array crosses as one buffer on both builds: about 5 to 15 us. This is the fast path. `np.asarray(v, dtype=np.float32)` makes it from a list or from another array. * A list that holds only Python floats (no `int`, `bool`, `None`, `str` or numpy scalar) also crosses as one buffer, and so does a `float64` array on the JVM build. The statement reads the same list of doubles it always read: no value changes, and an `int` in a list stays an `int`. A list shorter than 32 numbers (JVM build) or 64 numbers (native build) is not worth a buffer and goes number by number. On the native build the list crosses as a buffer only when `float32` holds every number exactly, which is true of a list from a float32 model (`array.tolist()`); a list of other doubles, `NaN` or an infinity is sent as text, because a buffer would round it. -* A `float64` array is a `float32` array to the engine on the native build and when it is the one positional parameter on the JVM build, so its numbers are rounded to `float32` (a vector index stores `float32`). As a named parameter on the JVM build it keeps its doubles. +* A `float64` array among the positional parameters (`?`) is a `float32` array to the engine on both builds, so its numbers are rounded to `float32` (a vector index stores `float32`). As a named parameter, inside `CONTENT :row` or as a property of a record you save, it keeps its doubles on both builds: `0.1` is stored as `0.1`. Such a value goes as text, so give a `float32` array when you want the buffer. === Results diff --git a/src/arcadedb/_native/codec.py b/src/arcadedb/_native/codec.py index 4cc74a9..929f548 100644 --- a/src/arcadedb/_native/codec.py +++ b/src/arcadedb/_native/codec.py @@ -14,6 +14,7 @@ TAG = "\u0001" _TAG_ESCAPED = "\\u0001" # TAG as it appears in reply text: NativeCodec.quote writes it as a \u escape _EPOCH_NAIVE = datetime(1970, 1, 1) +_EPOCH_AWARE = datetime(1970, 1, 1, tzinfo=timezone.utc) # A list of this many Python floats or more may cross in the float32 side buffer (see _exact_float_list). A shorter one is cheaper # as JSON text, and a short IN list is the common use of a list parameter. @@ -93,6 +94,11 @@ def _default(value, _bin=None): if _np is not None: if isinstance(value, _np.ndarray): if value.dtype.kind == "f": + if value.dtype != _np.float32: + # a float64 (or float16, longdouble) array keeps its doubles, as on the JVM build and in row_default. The + # float32 side buffer and the "F" tag (a float[] in Java) would both round it to float32 (0.1 -> 0.10000000149011612). + # Only a lone positional ``?`` is a query vector sent as float32: encode_params narrows that one before it gets here. + return value.tolist() if _bin is not None: arr = _np.ascontiguousarray(value.ravel(), dtype=_np.float32) off = _bin[1] @@ -152,6 +158,22 @@ def _prep(value, binary=False): return value +def _query_vector(value): + """A float ndarray among the positional parameters (``?``) is a query vector: the engine stores vectors as float32, so it is + sent as float32 (the fast side buffer), as ``Database._convert_args`` does on the JVM build and as documented in "A query + vector". A named parameter, ``CONTENT :row`` and a record's properties are values, not query vectors: a float64 array there + keeps its doubles (see :func:`_default`). + """ + if ( + _np is not None + and isinstance(value, _np.ndarray) + and value.dtype.kind == "f" + and value.dtype != _np.float32 + ): + return value.astype(_np.float32) + return value + + def encode_params(args, binary=False): """The ``params`` text for ``arcadedb_n_exec``: '' for none, else ``{"named":{..}}`` or ``{"pos":[..]}``. @@ -183,7 +205,7 @@ def encode_params(args, binary=False): named = {k: _prep(v, binary) for k, v in named.items()} return _dump({"named": dict(named)}, binary) if not all(type(v) in plain for v in values): - values = [_prep(v, binary) for v in values] + values = [_prep(_query_vector(v), binary) for v in values] return _dump({"pos": list(values)}, binary) @@ -364,19 +386,23 @@ def hook(obj): # --- JSON-native form of a row (ResultSet.to_json_list): DATE/DATETIME as epoch millis, DECIMAL as float. +def _epoch_millis(delta): + """Whole milliseconds of a timedelta since the epoch, in integer arithmetic and floored toward negative infinity: the + ``Instant.toEpochMilli`` of the JVM build. ``days`` floors and ``seconds`` and ``microseconds`` are never negative, so a + time before 1970 with microseconds (``.999999``) is not rounded toward zero, and no float product can fall just under an + integer (``total_seconds() * 1000`` did both). + """ + return (delta.days * 86400 + delta.seconds) * 1000 + delta.microseconds // 1000 + + def json_native(value): if isinstance(value, datetime): - return ( - int((value.replace(tzinfo=None) - _EPOCH_NAIVE).total_seconds() * 1000) - if value.tzinfo is None - else int(value.timestamp() * 1000) + return _epoch_millis( + value - _EPOCH_AWARE if value.tzinfo is not None else value - _EPOCH_NAIVE ) if isinstance(value, date): - return int( - ( - datetime.combine(value, datetime.min.time()) - _EPOCH_NAIVE - ).total_seconds() - * 1000 + return _epoch_millis( + datetime.combine(value, datetime.min.time()) - _EPOCH_NAIVE ) if isinstance(value, Decimal): return float(value) diff --git a/tests/test_numpy_support.py b/tests/test_numpy_support.py index d7c359a..abfb2cf 100644 --- a/tests/test_numpy_support.py +++ b/tests/test_numpy_support.py @@ -157,3 +157,66 @@ def test_int_ndarray_as_named_param_and_document_property(temp_db): for row in db.query("sql", "SELECT small, big FROM IntDoc"): assert list(row.get("small")) == [4, 5, 6] assert list(row.get("big")) == [2**53 + 1, 8] + + +# A float64 array is a value, not a query vector, on every path but a positional ``?``: it keeps its doubles. The native build sent +# it through the float32 side buffer on the named, CONTENT and record paths, so 0.1 was stored as 0.10000000149011612. + + +def _write_named(db, t, arr): + with db.transaction(): + sql = f"INSERT INTO {t} SET v = :v" # nosec B608 - test-owned + db.command("sql", sql, {"v": arr}) + + +def _write_content_named(db, t, arr): + with db.transaction(): + sql = f"INSERT INTO {t} CONTENT :row" # nosec B608 - test-owned + db.command("sql", sql, {"row": {"v": arr}}) + + +def _write_record(db, t, arr): + with db.transaction(): + db.new_document(t).set("v", arr).save() + + +def _stored(db, t): + """The one stored value of ``v`` read two ways: the result row and a lookup of the record by its rid.""" + sql = f"SELECT @rid AS rid, v FROM {t}" # nosec B608 - test-owned + row = db.query("sql", sql).first() + by_rid = db.lookup_by_rid(str(row.get("rid"))).get("v") + return list(row.get("v")), list(by_rid) + + +@pytest.mark.parametrize("write", [_write_named, _write_content_named, _write_record]) +def test_float64_array_keeps_its_doubles(temp_db, write): + temp_db.command("sql", "CREATE DOCUMENT TYPE F64") + write(temp_db, "F64", np.array([0.1, 0.2, 0.3], dtype=np.float64)) + from_row, from_rid = _stored(temp_db, "F64") + assert from_row == [0.1, 0.2, 0.3] + assert from_rid == [0.1, 0.2, 0.3] + + +@pytest.mark.parametrize("write", [_write_named, _write_content_named, _write_record]) +def test_float32_array_reads_back_as_its_float32_values(temp_db, write): + temp_db.command("sql", "CREATE DOCUMENT TYPE F32") + arr = np.array([0.1, 0.2, 0.3], dtype=np.float32) + write(temp_db, "F32", arr) + expected = arr.astype( + np.float64 + ).tolist() # float32 widened: 0.10000000149011612, ... + from_row, from_rid = _stored(temp_db, "F32") + assert from_row == expected + assert from_rid == expected + + +def test_float64_array_as_a_lone_positional_parameter_is_a_query_vector(temp_db): + """The documented rule (databases-and-queries, "A query vector"): a float array among the positional parameters is sent as + float32 on both builds. Pinned beside the named case so a change of either side is a decision. + """ + temp_db.command("sql", "CREATE DOCUMENT TYPE F64Pos") + arr = np.array([0.1, 0.2, 0.3], dtype=np.float64) + with temp_db.transaction(): + temp_db.command("sql", "INSERT INTO F64Pos SET v = ?", arr) + from_row, _ = _stored(temp_db, "F64Pos") + assert from_row == arr.astype(np.float32).astype(np.float64).tolist() diff --git a/tests/test_resultset.py b/tests/test_resultset.py index 30c03b8..a49785c 100644 --- a/tests/test_resultset.py +++ b/tests/test_resultset.py @@ -2,6 +2,8 @@ Tests for enhanced ResultSet and Result functionality. """ +from datetime import datetime + import pytest import arcadedb @@ -751,3 +753,46 @@ def test_the_bridge_releases_the_engine_cursor(self, temp_db): page = getattr(temp_db.query("sql", q), read)() assert len(page) == 5000 last = str(page[-1]["rid"]) + + +def _epoch_millis(value): + """Epoch milliseconds of a naive UTC datetime in integer arithmetic, floored toward negative infinity (``Instant.toEpochMilli``).""" + delta = value - datetime(1970, 1, 1) + return (delta.days * 86400 + delta.seconds) * 1000 + delta.microseconds // 1000 + + +_EPOCH_MILLIS_VALUES = [ + datetime( + 1969, 12, 31, 23, 59, 59, 999999 + ), # before 1970 with microseconds: 1 ms low when rounded toward zero + datetime(1950, 6, 1, 12, 0, 0, 123000), # an exact millisecond before 1970 + datetime( + 9999, 12, 31, 23, 59, 59, 999000 + ), # the last whole millisecond of year 9999 + datetime(9999, 12, 31, 23, 59, 59, 999999), # .999999 must not roll into year 10000 + datetime( + 2026, 11, 1, 12, 0, 0, 123000 + ), # exact milliseconds: a float product can land just under the integer + datetime(2026, 10, 3, 12, 34, 56, 123456), +] + + +@pytest.mark.parametrize("declared", ["DATETIME_MICROS", "DATETIME"]) +def test_to_json_list_datetime_is_the_floored_epoch_millis(temp_db, declared): + """DATETIME_MICROS keeps every value above; DATETIME keeps milliseconds, so only the millisecond-exact ones are compared there.""" + values = [ + v + for v in _EPOCH_MILLIS_VALUES + if declared == "DATETIME_MICROS" or v.microsecond % 1000 == 0 + ] + temp_db.command("sql", "CREATE DOCUMENT TYPE Stamp") + temp_db.command( + "sql", f"CREATE PROPERTY Stamp.at {declared}" + ) # nosec B608 - test-owned + with temp_db.transaction(): + for i, v in enumerate(values): + temp_db.command( + "sql", "INSERT INTO Stamp SET i = :i, at = :at", {"i": i, "at": v} + ) + rows = temp_db.query("sql", "SELECT i, at FROM Stamp ORDER BY i").to_json_list() + assert [r["at"] for r in rows] == [_epoch_millis(v) for v in values]