diff --git a/docs/modules/ROOT/pages/how-to/vectors.adoc b/docs/modules/ROOT/pages/how-to/vectors.adoc index bf956e6..563819f 100644 --- a/docs/modules/ROOT/pages/how-to/vectors.adoc +++ b/docs/modules/ROOT/pages/how-to/vectors.adoc @@ -247,10 +247,11 @@ print(meta["quantization"], meta["add_hierarchy"]) INT8 True ---- -Two rules hold for the quantization arguments of the method: +These rules hold for the quantization arguments of the method: * The `pq_*` and `fused_pq` arguments need `quantization="PRODUCT"`. `rerank_oversample` needs `"PRODUCT"` or, on ArcadeDB 26.11.1 and later, `"BINARY"`: a `BINARY` search scores the one-bit codes and re-ranks `k * rerank_oversample` candidates on the stored vectors (default 10), or the whole `efSearch` beam when that is larger (https://github.com/ArcadeData/arcadedb/issues/10968[ArcadeData/arcadedb#10968]). The call raises `ValueError` otherwise. The same keys are `METADATA` keys in a statement (`pqSubspaces`, `pqClusters`, `pqCenterGlobally`, `pqTrainingLimit`, `rerankOversample`, `fusedPQ`). With `PRODUCT`, every graph rebuild retrains the codebook and re-encodes every vector (https://github.com/ArcadeData/arcadedb/issues/10847[ArcadeData/arcadedb#10847]), so measure recall and rebuild time on your own data before you choose it. * `encoding="INT8"` is for a `BINARY` property that already holds INT8 bytes. Set `quantization` to `"NONE"` with it. The call raises `ValueError` when both are `INT8`, because the vector would be quantized twice. +* A bucket that holds fewer vectors than `pq_clusters` is too small to train a codebook. The index still builds: the engine skips PQ for that bucket and searches it exactly, so `find_nearest_approximate` returns the exact neighbours with exact scores from it. A bucket with `pq_clusters` vectors or more is trained and answers approximately. == Rebuild the graph diff --git a/src/arcadedb/_native/schema_objects.py b/src/arcadedb/_native/schema_objects.py index ceb647a..9dda03f 100644 --- a/src/arcadedb/_native/schema_objects.py +++ b/src/arcadedb/_native/schema_objects.py @@ -118,7 +118,7 @@ def engine_info(self) -> dict: return info def getMetadata(self): - """Just what the PRODUCT-readiness check reads: the index's quantization and pq_clusters.""" + """The index's quantization and pq_clusters, shaped like the Java metadata object.""" from types import SimpleNamespace info = self.engine_info() @@ -272,7 +272,6 @@ def find_nearest_approximate(self, query_vector, k=10, allowed_rids=None): """PQ approximate search over PRODUCT quantization, the engine's own (``findNeighborsFromVectorApproximate``).""" if self.get_quantization() != "PRODUCT": raise ArcadeDBError("Approximate search requires quantization=PRODUCT (PQ)") - self._ensure_product_quantization_ready() try: return self._run_search( query_vector=query_vector, diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index 47955e8..e4c13f3 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -1530,9 +1530,9 @@ def create_vector_index( Options: "INT8", "BINARY", "PRODUCT" (PQ). Reduces memory usage and speeds up search at the cost of some precision. "PRODUCT" enables PQ data for approximate search (zero-disk-I/O path). - In current ArcadeDB engine builds, PRODUCT also requires enough indexed - vectors per bucket for PQ training; for tiny corpora, set `pq_clusters` - explicitly to a small value or prefer INT8/BINARY/NONE. + A bucket with fewer vectors than `pq_clusters` has too few points to + train a codebook: the engine skips PQ for that bucket and searches it + exactly, so approximate search returns exact neighbours there. With PRODUCT, every graph rebuild retrains the codebook and re-encodes every vector (ArcadeDB #10847); measure recall and rebuild time on your data before choosing it. @@ -1568,8 +1568,7 @@ def create_vector_index( the threshold is exactly this value. pq_subspaces: Number of PQ subspaces (M). Requires quantization="PRODUCT". pq_clusters: Clusters per subspace (K). Requires quantization="PRODUCT". - In current ArcadeDB engine builds, this should not exceed the - number of indexed vectors available for PQ training in a bucket. + A bucket with fewer vectors than this is searched exactly, without PQ. pq_center_globally: Whether to globally center vectors before PQ. Requires quantization="PRODUCT". pq_training_limit: Max vectors to use for PQ training. Requires diff --git a/src/arcadedb/vector.py b/src/arcadedb/vector.py index 5ac925f..f2c69dc 100644 --- a/src/arcadedb/vector.py +++ b/src/arcadedb/vector.py @@ -283,13 +283,11 @@ def __init__(self, java_index, database): """ self._java_index = java_index self._database = database - # Resolved once per wrapper: the underlying LSM indexes and the - # PRODUCT-quantization readiness check are stable for the lifetime of - # this wrapper, but were previously re-derived on every search (several - # JPype crossings per query). They are reset when _check_live() swaps in - # a rebuilt Java index. + # Resolved once per wrapper: the underlying LSM indexes are stable for + # the lifetime of this wrapper, but were previously re-derived on every + # search (several JPype crossings per query). Reset when _check_live() + # swaps in a rebuilt Java index. self._lsm_indexes = None - self._pq_ready = False def _check_live(self): """Re-resolve the Java index by name when SQL ``REBUILD INDEX`` (or a drop) invalidated it. @@ -314,7 +312,6 @@ def _check_live(self): ) from e self._java_index = fresh self._lsm_indexes = None - self._pq_ready = False def _iter_lsm_indexes(self): if self._lsm_indexes is None: @@ -339,35 +336,6 @@ def _get_primary_lsm_index(self): return index return None - def _ensure_product_quantization_ready(self): - # Once the check confirms enough vectors it stays valid (indexed vectors - # only accumulate), so don't repeat the metadata/countEntries crossings - # on every search. An empty PRODUCT bucket is inconclusive (data may - # arrive later), so it does not memoize. - if self._pq_ready: - return - all_confirmed = True - for index in self._iter_lsm_indexes(): - meta = index.getMetadata() - if str(meta.quantizationType) != "PRODUCT": - continue - - vector_count = int(index.countEntries()) - pq_clusters = int(meta.pqClusters) - - if 0 < vector_count < pq_clusters: - raise ArcadeDBError( - "PRODUCT quantization in this ArcadeDB version requires at least " - "pq_clusters indexed vectors per bucket before graph build. " - f"Current bucket has {vector_count} vector(s) but " - f"pq_clusters={pq_clusters}. " - "Use a smaller pq_clusters value, insert more vectors, " - "or use INT8/BINARY/NONE." - ) - if vector_count == 0: - all_confirmed = False - self._pq_ready = all_confirmed - def _normalize_ef_search(self, ef_search): if ef_search is None: return None @@ -651,7 +619,6 @@ def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids): effective_ef_search = self._normalize_ef_search(ef_search) try: - self._ensure_product_quantization_ready() return self._run_search( query_vector=query_vector, k=k, @@ -831,8 +798,6 @@ def find_nearest_approximate( "Approximate search requires quantization=PRODUCT (PQ)" ) - self._ensure_product_quantization_ready() - return self._run_search( query_vector=query_vector, k=k, @@ -909,8 +874,6 @@ def build_graph_now(self): """ self._check_live() try: - self._ensure_product_quantization_ready() - built_any = False for index in self._iter_lsm_indexes(): index.buildVectorGraphNow() diff --git a/tests/test_vector.py b/tests/test_vector.py index c3cf868..da5b4ed 100644 --- a/tests/test_vector.py +++ b/tests/test_vector.py @@ -487,28 +487,6 @@ def test_lsm_vector_search_approximate_fallback(self, test_db): with pytest.raises(arcadedb.ArcadeDBError): index.find_nearest_approximate([0.9, 0.1, 0.0], k=1) - def test_lsm_vector_search_approximate_product_requires_enough_vectors( - self, test_db - ): - """PRODUCT quantization should fail early on tiny corpora.""" - test_db.command("sql", "CREATE VERTEX TYPE Doc") - test_db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") - - index = test_db.create_vector_index( - "Doc", "embedding", dimensions=3, quantization="PRODUCT" - ) - - with test_db.transaction(): - for vec in ([1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]): - test_db.command( - "sql", - "INSERT INTO Doc SET embedding = ?", - arcadedb.to_java_float_array(vec), - ) - - with pytest.raises(arcadedb.ArcadeDBError, match="pq_clusters"): - index.build_graph_now() - def test_lsm_vector_search_approximate_returns_k(self, test_db): """Approximate search should return exactly k results.""" test_db.command("sql", "CREATE VERTEX TYPE Doc") diff --git a/tests/test_vector_product_quantization.py b/tests/test_vector_product_quantization.py index 3b85657..7d55cfe 100644 --- a/tests/test_vector_product_quantization.py +++ b/tests/test_vector_product_quantization.py @@ -19,6 +19,8 @@ only on a build from source (`engine-from-source: true`). """ +import math + import numpy as np import pytest @@ -368,6 +370,91 @@ def test_rerank_oversample_refuses_unquantized_and_int8( ) +SMALL_BUCKETS = 4 +SMALL_PQ_CLUSTERS = 16 + + +def _cosine_ranking(vectors, query): + """Ids sorted by cosine similarity to the query, best first, in plain Python.""" + + def cosine(vec): + dot = sum(a * b for a, b in zip(vec, query)) + return dot / ( + math.sqrt(sum(a * a for a in vec)) * math.sqrt(sum(b * b for b in query)) + ) + + return sorted(range(len(vectors)), key=lambda i: cosine(vectors[i]), reverse=True) + + +class TestProductQuantizationSmallBuckets: + """A bucket with fewer vectors than `pq_clusters` cannot train a PQ codebook. + + The engine skips PQ for such a bucket and searches it exactly (ArcadeData/arcadedb#5417). The + Python layer used to refuse the build on the JVM build before the engine was asked, with a + message that said PRODUCT needs `pq_clusters` vectors per bucket; the native build, which + checked the whole type, let a spread type through. Both builds now answer as the engine does. + """ + + @pytest.mark.parametrize("per_bucket", [5, 15]) + def test_buckets_below_pq_clusters_build_and_search_exactly( + self, test_db, per_bucket + ): + dims = 8 + rng = np.random.default_rng(100 + per_bucket) + count = per_bucket * SMALL_BUCKETS + vectors = [[float(x) for x in rng.standard_normal(dims)] for _ in range(count)] + queries = [[float(x) for x in rng.standard_normal(dims)] for _ in range(10)] + + test_db.command("sql", f"CREATE VERTEX TYPE T BUCKETS {SMALL_BUCKETS}") + test_db.command("sql", "CREATE PROPERTY T.id INTEGER") + test_db.command("sql", "CREATE PROPERTY T.v ARRAY_OF_FLOATS") + index = test_db.create_vector_index( + "T", + "v", + dimensions=dims, + quantization="PRODUCT", + pq_subspaces=4, + pq_clusters=SMALL_PQ_CLUSTERS, + build_graph_now=False, + ) + with test_db.transaction(): + for i, vec in enumerate(vectors): + test_db.command("sql", "INSERT INTO T SET id = ?, v = ?", i, vec) + + # the premise: every bucket is below pq_clusters (the inserts spread over the buckets) + in_bucket = [ + test_db.query( + "sql", + f"SELECT count(*) AS c FROM bucket:T_{b}", # nosec B608 - b from range() + ) + .first() + .get("c") + for b in range(SMALL_BUCKETS) + ] + assert sum(in_bucket) == count + assert max(in_bucket) < SMALL_PQ_CLUSTERS, in_bucket + + index.build_graph_now() + + for query in queries: + ranking = _cosine_ranking(vectors, query) + approximate = index.find_nearest_approximate(query, k=5) + exact = index.find_nearest(query, k=5) + approximate_ids = [int(r.get("id")) for r, _ in approximate] + exact_ids = [int(r.get("id")) for r, _ in exact] + + assert approximate_ids[0] == ranking[0] + assert exact_ids[0] == ranking[0] + assert approximate_ids == ranking[:5] + assert exact_ids == ranking[:5] + # no PQ estimate is in force: the approximate scores are the exact scores + exact_score = {int(r.get("id")): s for r, s in exact} + for record, score in approximate: + assert score == pytest.approx( + exact_score[int(record.get("id"))], abs=1e-5 + ) + + class TestBinaryRerankOversample: """`rerank_oversample` on a BINARY index (ArcadeData/arcadedb#10968, ArcadeDB 26.11.1).