diff --git a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc index 165ac4c..8c644be 100644 --- a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc +++ b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc @@ -226,7 +226,7 @@ A query vector for `vectorNeighbors` can be a numpy array or a list. The convers * A `float32` array crosses as one buffer on both builds: about 5 to 15 us. This is the fast path. `np.asarray(v, dtype=np.float32)` makes it from a list or from another array. * A list that holds only Python floats (no `int`, `bool`, `None`, `str` or numpy scalar) also crosses as one buffer, and so does a `float64` array on the JVM build. The statement reads the same list of doubles it always read: no value changes, and an `int` in a list stays an `int`. A list shorter than 32 numbers (JVM build) or 64 numbers (native build) is not worth a buffer and goes number by number. On the native build the list crosses as a buffer only when `float32` holds every number exactly, which is true of a list from a float32 model (`array.tolist()`); a list of other doubles, `NaN` or an infinity is sent as text, because a buffer would round it. -* A `float64` array is a `float32` array to the engine on the native build and when it is the one positional parameter on the JVM build, so its numbers are rounded to `float32` (a vector index stores `float32`). As a named parameter on the JVM build it keeps its doubles. +* A `float64` array among the positional parameters (`?`) is a `float32` array to the engine on both builds, so its numbers are rounded to `float32` (a vector index stores `float32`). As a named parameter, inside `CONTENT :row` or as a property of a record you save, it keeps its doubles on both builds: `0.1` is stored as `0.1`. Such a value goes as text, so give a `float32` array when you want the buffer. === Results diff --git a/src/arcadedb/_native/codec.py b/src/arcadedb/_native/codec.py index 4cc74a9..929f548 100644 --- a/src/arcadedb/_native/codec.py +++ b/src/arcadedb/_native/codec.py @@ -14,6 +14,7 @@ TAG = "\u0001" _TAG_ESCAPED = "\\u0001" # TAG as it appears in reply text: NativeCodec.quote writes it as a \u escape _EPOCH_NAIVE = datetime(1970, 1, 1) +_EPOCH_AWARE = datetime(1970, 1, 1, tzinfo=timezone.utc) # A list of this many Python floats or more may cross in the float32 side buffer (see _exact_float_list). A shorter one is cheaper # as JSON text, and a short IN list is the common use of a list parameter. @@ -93,6 +94,11 @@ def _default(value, _bin=None): if _np is not None: if isinstance(value, _np.ndarray): if value.dtype.kind == "f": + if value.dtype != _np.float32: + # a float64 (or float16, longdouble) array keeps its doubles, as on the JVM build and in row_default. The + # float32 side buffer and the "F" tag (a float[] in Java) would both round it to float32 (0.1 -> 0.10000000149011612). + # Only a lone positional ``?`` is a query vector sent as float32: encode_params narrows that one before it gets here. + return value.tolist() if _bin is not None: arr = _np.ascontiguousarray(value.ravel(), dtype=_np.float32) off = _bin[1] @@ -152,6 +158,22 @@ def _prep(value, binary=False): return value +def _query_vector(value): + """A float ndarray among the positional parameters (``?``) is a query vector: the engine stores vectors as float32, so it is + sent as float32 (the fast side buffer), as ``Database._convert_args`` does on the JVM build and as documented in "A query + vector". A named parameter, ``CONTENT :row`` and a record's properties are values, not query vectors: a float64 array there + keeps its doubles (see :func:`_default`). + """ + if ( + _np is not None + and isinstance(value, _np.ndarray) + and value.dtype.kind == "f" + and value.dtype != _np.float32 + ): + return value.astype(_np.float32) + return value + + def encode_params(args, binary=False): """The ``params`` text for ``arcadedb_n_exec``: '' for none, else ``{"named":{..}}`` or ``{"pos":[..]}``. @@ -183,7 +205,7 @@ def encode_params(args, binary=False): named = {k: _prep(v, binary) for k, v in named.items()} return _dump({"named": dict(named)}, binary) if not all(type(v) in plain for v in values): - values = [_prep(v, binary) for v in values] + values = [_prep(_query_vector(v), binary) for v in values] return _dump({"pos": list(values)}, binary) @@ -364,19 +386,23 @@ def hook(obj): # --- JSON-native form of a row (ResultSet.to_json_list): DATE/DATETIME as epoch millis, DECIMAL as float. +def _epoch_millis(delta): + """Whole milliseconds of a timedelta since the epoch, in integer arithmetic and floored toward negative infinity: the + ``Instant.toEpochMilli`` of the JVM build. ``days`` floors and ``seconds`` and ``microseconds`` are never negative, so a + time before 1970 with microseconds (``.999999``) is not rounded toward zero, and no float product can fall just under an + integer (``total_seconds() * 1000`` did both). + """ + return (delta.days * 86400 + delta.seconds) * 1000 + delta.microseconds // 1000 + + def json_native(value): if isinstance(value, datetime): - return ( - int((value.replace(tzinfo=None) - _EPOCH_NAIVE).total_seconds() * 1000) - if value.tzinfo is None - else int(value.timestamp() * 1000) + return _epoch_millis( + value - _EPOCH_AWARE if value.tzinfo is not None else value - _EPOCH_NAIVE ) if isinstance(value, date): - return int( - ( - datetime.combine(value, datetime.min.time()) - _EPOCH_NAIVE - ).total_seconds() - * 1000 + return _epoch_millis( + datetime.combine(value, datetime.min.time()) - _EPOCH_NAIVE ) if isinstance(value, Decimal): return float(value) diff --git a/tests/test_numpy_support.py b/tests/test_numpy_support.py index d7c359a..abfb2cf 100644 --- a/tests/test_numpy_support.py +++ b/tests/test_numpy_support.py @@ -157,3 +157,66 @@ def test_int_ndarray_as_named_param_and_document_property(temp_db): for row in db.query("sql", "SELECT small, big FROM IntDoc"): assert list(row.get("small")) == [4, 5, 6] assert list(row.get("big")) == [2**53 + 1, 8] + + +# A float64 array is a value, not a query vector, on every path but a positional ``?``: it keeps its doubles. The native build sent +# it through the float32 side buffer on the named, CONTENT and record paths, so 0.1 was stored as 0.10000000149011612. + + +def _write_named(db, t, arr): + with db.transaction(): + sql = f"INSERT INTO {t} SET v = :v" # nosec B608 - test-owned + db.command("sql", sql, {"v": arr}) + + +def _write_content_named(db, t, arr): + with db.transaction(): + sql = f"INSERT INTO {t} CONTENT :row" # nosec B608 - test-owned + db.command("sql", sql, {"row": {"v": arr}}) + + +def _write_record(db, t, arr): + with db.transaction(): + db.new_document(t).set("v", arr).save() + + +def _stored(db, t): + """The one stored value of ``v`` read two ways: the result row and a lookup of the record by its rid.""" + sql = f"SELECT @rid AS rid, v FROM {t}" # nosec B608 - test-owned + row = db.query("sql", sql).first() + by_rid = db.lookup_by_rid(str(row.get("rid"))).get("v") + return list(row.get("v")), list(by_rid) + + +@pytest.mark.parametrize("write", [_write_named, _write_content_named, _write_record]) +def test_float64_array_keeps_its_doubles(temp_db, write): + temp_db.command("sql", "CREATE DOCUMENT TYPE F64") + write(temp_db, "F64", np.array([0.1, 0.2, 0.3], dtype=np.float64)) + from_row, from_rid = _stored(temp_db, "F64") + assert from_row == [0.1, 0.2, 0.3] + assert from_rid == [0.1, 0.2, 0.3] + + +@pytest.mark.parametrize("write", [_write_named, _write_content_named, _write_record]) +def test_float32_array_reads_back_as_its_float32_values(temp_db, write): + temp_db.command("sql", "CREATE DOCUMENT TYPE F32") + arr = np.array([0.1, 0.2, 0.3], dtype=np.float32) + write(temp_db, "F32", arr) + expected = arr.astype( + np.float64 + ).tolist() # float32 widened: 0.10000000149011612, ... + from_row, from_rid = _stored(temp_db, "F32") + assert from_row == expected + assert from_rid == expected + + +def test_float64_array_as_a_lone_positional_parameter_is_a_query_vector(temp_db): + """The documented rule (databases-and-queries, "A query vector"): a float array among the positional parameters is sent as + float32 on both builds. Pinned beside the named case so a change of either side is a decision. + """ + temp_db.command("sql", "CREATE DOCUMENT TYPE F64Pos") + arr = np.array([0.1, 0.2, 0.3], dtype=np.float64) + with temp_db.transaction(): + temp_db.command("sql", "INSERT INTO F64Pos SET v = ?", arr) + from_row, _ = _stored(temp_db, "F64Pos") + assert from_row == arr.astype(np.float32).astype(np.float64).tolist() diff --git a/tests/test_resultset.py b/tests/test_resultset.py index 30c03b8..a49785c 100644 --- a/tests/test_resultset.py +++ b/tests/test_resultset.py @@ -2,6 +2,8 @@ Tests for enhanced ResultSet and Result functionality. """ +from datetime import datetime + import pytest import arcadedb @@ -751,3 +753,46 @@ def test_the_bridge_releases_the_engine_cursor(self, temp_db): page = getattr(temp_db.query("sql", q), read)() assert len(page) == 5000 last = str(page[-1]["rid"]) + + +def _epoch_millis(value): + """Epoch milliseconds of a naive UTC datetime in integer arithmetic, floored toward negative infinity (``Instant.toEpochMilli``).""" + delta = value - datetime(1970, 1, 1) + return (delta.days * 86400 + delta.seconds) * 1000 + delta.microseconds // 1000 + + +_EPOCH_MILLIS_VALUES = [ + datetime( + 1969, 12, 31, 23, 59, 59, 999999 + ), # before 1970 with microseconds: 1 ms low when rounded toward zero + datetime(1950, 6, 1, 12, 0, 0, 123000), # an exact millisecond before 1970 + datetime( + 9999, 12, 31, 23, 59, 59, 999000 + ), # the last whole millisecond of year 9999 + datetime(9999, 12, 31, 23, 59, 59, 999999), # .999999 must not roll into year 10000 + datetime( + 2026, 11, 1, 12, 0, 0, 123000 + ), # exact milliseconds: a float product can land just under the integer + datetime(2026, 10, 3, 12, 34, 56, 123456), +] + + +@pytest.mark.parametrize("declared", ["DATETIME_MICROS", "DATETIME"]) +def test_to_json_list_datetime_is_the_floored_epoch_millis(temp_db, declared): + """DATETIME_MICROS keeps every value above; DATETIME keeps milliseconds, so only the millisecond-exact ones are compared there.""" + values = [ + v + for v in _EPOCH_MILLIS_VALUES + if declared == "DATETIME_MICROS" or v.microsecond % 1000 == 0 + ] + temp_db.command("sql", "CREATE DOCUMENT TYPE Stamp") + temp_db.command( + "sql", f"CREATE PROPERTY Stamp.at {declared}" + ) # nosec B608 - test-owned + with temp_db.transaction(): + for i, v in enumerate(values): + temp_db.command( + "sql", "INSERT INTO Stamp SET i = :i, at = :at", {"i": i, "at": v} + ) + rows = temp_db.query("sql", "SELECT i, at FROM Stamp ORDER BY i").to_json_list() + assert [r["at"] for r in rows] == [_epoch_millis(v) for v in values]