vectors: remove the stale per-bucket PRODUCT guard; the engine searches small buckets exactly

docs/modules/ROOT/pages/how-to/vectors.adoc

@@ -247,10 +247,11 @@ print(meta["quantization"], meta["add_hierarchy"])
 INT8 True
 ----
 
-Two rules hold for the quantization arguments of the method:
+These rules hold for the quantization arguments of the method:
 
 * The `pq_*` and `fused_pq` arguments need `quantization="PRODUCT"`. `rerank_oversample` needs `"PRODUCT"` or, on ArcadeDB 26.11.1 and later, `"BINARY"`: a `BINARY` search scores the one-bit codes and re-ranks `k * rerank_oversample` candidates on the stored vectors (default 10), or the whole `efSearch` beam when that is larger (https://github.com/ArcadeData/arcadedb/issues/10968[ArcadeData/arcadedb#10968]). The call raises `ValueError` otherwise. The same keys are `METADATA` keys in a statement (`pqSubspaces`, `pqClusters`, `pqCenterGlobally`, `pqTrainingLimit`, `rerankOversample`, `fusedPQ`). With `PRODUCT`, every graph rebuild retrains the codebook and re-encodes every vector (https://github.com/ArcadeData/arcadedb/issues/10847[ArcadeData/arcadedb#10847]), so measure recall and rebuild time on your own data before you choose it.
 * `encoding="INT8"` is for a `BINARY` property that already holds INT8 bytes. Set `quantization` to `"NONE"` with it. The call raises `ValueError` when both are `INT8`, because the vector would be quantized twice.
+* A bucket that holds fewer vectors than `pq_clusters` is too small to train a codebook. The index still builds: the engine skips PQ for that bucket and searches it exactly, so `find_nearest_approximate` returns the exact neighbours with exact scores from it. A bucket with `pq_clusters` vectors or more is trained and answers approximately.
 
 == Rebuild the graph
 

src/arcadedb/_native/schema_objects.py

@@ -118,7 +118,7 @@ def engine_info(self) -> dict:
         return info
 
     def getMetadata(self):
-        """Just what the PRODUCT-readiness check reads: the index's quantization and pq_clusters."""
+        """The index's quantization and pq_clusters, shaped like the Java metadata object."""
         from types import SimpleNamespace
 
         info = self.engine_info()
@@ -272,7 +272,6 @@ def find_nearest_approximate(self, query_vector, k=10, allowed_rids=None):
         """PQ approximate search over PRODUCT quantization, the engine's own (``findNeighborsFromVectorApproximate``)."""
         if self.get_quantization() != "PRODUCT":
             raise ArcadeDBError("Approximate search requires quantization=PRODUCT (PQ)")
-        self._ensure_product_quantization_ready()
         try:
             return self._run_search(
                 query_vector=query_vector,

src/arcadedb/core.py

@@ -1530,9 +1530,9 @@ def create_vector_index(
                 Options: "INT8", "BINARY", "PRODUCT" (PQ).
                 Reduces memory usage and speeds up search at the cost of some precision.
                 "PRODUCT" enables PQ data for approximate search (zero-disk-I/O path).
-                In current ArcadeDB engine builds, PRODUCT also requires enough indexed
-                vectors per bucket for PQ training; for tiny corpora, set `pq_clusters`
-                explicitly to a small value or prefer INT8/BINARY/NONE.
+                A bucket with fewer vectors than `pq_clusters` has too few points to
+                train a codebook: the engine skips PQ for that bucket and searches it
+                exactly, so approximate search returns exact neighbours there.
                 With PRODUCT, every graph rebuild retrains the codebook and re-encodes
                 every vector (ArcadeDB #10847); measure recall and rebuild time on
                 your data before choosing it.
@@ -1568,8 +1568,7 @@ def create_vector_index(
                 the threshold is exactly this value.
             pq_subspaces: Number of PQ subspaces (M). Requires quantization="PRODUCT".
             pq_clusters: Clusters per subspace (K). Requires quantization="PRODUCT".
-                In current ArcadeDB engine builds, this should not exceed the
-                number of indexed vectors available for PQ training in a bucket.
+                A bucket with fewer vectors than this is searched exactly, without PQ.
             pq_center_globally: Whether to globally center vectors before PQ.
                 Requires quantization="PRODUCT".
             pq_training_limit: Max vectors to use for PQ training. Requires

src/arcadedb/vector.py

@@ -283,13 +283,11 @@ def __init__(self, java_index, database):
         """
         self._java_index = java_index
         self._database = database
-        # Resolved once per wrapper: the underlying LSM indexes and the
-        # PRODUCT-quantization readiness check are stable for the lifetime of
-        # this wrapper, but were previously re-derived on every search (several
-        # JPype crossings per query). They are reset when _check_live() swaps in
-        # a rebuilt Java index.
+        # Resolved once per wrapper: the underlying LSM indexes are stable for
+        # the lifetime of this wrapper, but were previously re-derived on every
+        # search (several JPype crossings per query). Reset when _check_live()
+        # swaps in a rebuilt Java index.
         self._lsm_indexes = None
-        self._pq_ready = False
 
     def _check_live(self):
         """Re-resolve the Java index by name when SQL ``REBUILD INDEX`` (or a drop) invalidated it.
@@ -314,7 +312,6 @@ def _check_live(self):
             ) from e
         self._java_index = fresh
         self._lsm_indexes = None
-        self._pq_ready = False
 
     def _iter_lsm_indexes(self):
         if self._lsm_indexes is None:
@@ -339,35 +336,6 @@ def _get_primary_lsm_index(self):
             return index
         return None
 
-    def _ensure_product_quantization_ready(self):
-        # Once the check confirms enough vectors it stays valid (indexed vectors
-        # only accumulate), so don't repeat the metadata/countEntries crossings
-        # on every search. An empty PRODUCT bucket is inconclusive (data may
-        # arrive later), so it does not memoize.
-        if self._pq_ready:
-            return
-        all_confirmed = True
-        for index in self._iter_lsm_indexes():
-            meta = index.getMetadata()
-            if str(meta.quantizationType) != "PRODUCT":
-                continue
-
-            vector_count = int(index.countEntries())
-            pq_clusters = int(meta.pqClusters)
-
-            if 0 < vector_count < pq_clusters:
-                raise ArcadeDBError(
-                    "PRODUCT quantization in this ArcadeDB version requires at least "
-                    "pq_clusters indexed vectors per bucket before graph build. "
-                    f"Current bucket has {vector_count} vector(s) but "
-                    f"pq_clusters={pq_clusters}. "
-                    "Use a smaller pq_clusters value, insert more vectors, "
-                    "or use INT8/BINARY/NONE."
-                )
-            if vector_count == 0:
-                all_confirmed = False
-        self._pq_ready = all_confirmed
-
     def _normalize_ef_search(self, ef_search):
         if ef_search is None:
             return None
@@ -651,7 +619,6 @@ def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids):
         effective_ef_search = self._normalize_ef_search(ef_search)
 
         try:
-            self._ensure_product_quantization_ready()
             return self._run_search(
                 query_vector=query_vector,
                 k=k,
@@ -831,8 +798,6 @@ def find_nearest_approximate(
                     "Approximate search requires quantization=PRODUCT (PQ)"
                 )
 
-            self._ensure_product_quantization_ready()
-
             return self._run_search(
                 query_vector=query_vector,
                 k=k,
@@ -909,8 +874,6 @@ def build_graph_now(self):
         """
         self._check_live()
         try:
-            self._ensure_product_quantization_ready()
-
             built_any = False
             for index in self._iter_lsm_indexes():
                 index.buildVectorGraphNow()

tests/test_vector.py

@@ -487,28 +487,6 @@ def test_lsm_vector_search_approximate_fallback(self, test_db):
         with pytest.raises(arcadedb.ArcadeDBError):
             index.find_nearest_approximate([0.9, 0.1, 0.0], k=1)
 
-    def test_lsm_vector_search_approximate_product_requires_enough_vectors(
-        self, test_db
-    ):
-        """PRODUCT quantization should fail early on tiny corpora."""
-        test_db.command("sql", "CREATE VERTEX TYPE Doc")
-        test_db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS")
-
-        index = test_db.create_vector_index(
-            "Doc", "embedding", dimensions=3, quantization="PRODUCT"
-        )
-
-        with test_db.transaction():
-            for vec in ([1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]):
-                test_db.command(
-                    "sql",
-                    "INSERT INTO Doc SET embedding = ?",
-                    arcadedb.to_java_float_array(vec),
-                )
-
-        with pytest.raises(arcadedb.ArcadeDBError, match="pq_clusters"):
-            index.build_graph_now()
-
     def test_lsm_vector_search_approximate_returns_k(self, test_db):
         """Approximate search should return exactly k results."""
         test_db.command("sql", "CREATE VERTEX TYPE Doc")

tests/test_vector_product_quantization.py

@@ -19,6 +19,8 @@
 only on a build from source (`engine-from-source: true`).
 """
 
+import math
+
 import numpy as np
 import pytest
 
@@ -368,6 +370,91 @@ def test_rerank_oversample_refuses_unquantized_and_int8(
             )
 
 
+SMALL_BUCKETS = 4
+SMALL_PQ_CLUSTERS = 16
+
+
+def _cosine_ranking(vectors, query):
+    """Ids sorted by cosine similarity to the query, best first, in plain Python."""
+
+    def cosine(vec):
+        dot = sum(a * b for a, b in zip(vec, query))
+        return dot / (
+            math.sqrt(sum(a * a for a in vec)) * math.sqrt(sum(b * b for b in query))
+        )
+
+    return sorted(range(len(vectors)), key=lambda i: cosine(vectors[i]), reverse=True)
+
+
+class TestProductQuantizationSmallBuckets:
+    """A bucket with fewer vectors than `pq_clusters` cannot train a PQ codebook.
+
+    The engine skips PQ for such a bucket and searches it exactly (ArcadeData/arcadedb#5417). The
+    Python layer used to refuse the build on the JVM build before the engine was asked, with a
+    message that said PRODUCT needs `pq_clusters` vectors per bucket; the native build, which
+    checked the whole type, let a spread type through. Both builds now answer as the engine does.
+    """
+
+    @pytest.mark.parametrize("per_bucket", [5, 15])
+    def test_buckets_below_pq_clusters_build_and_search_exactly(
+        self, test_db, per_bucket
+    ):
+        dims = 8
+        rng = np.random.default_rng(100 + per_bucket)
+        count = per_bucket * SMALL_BUCKETS
+        vectors = [[float(x) for x in rng.standard_normal(dims)] for _ in range(count)]
+        queries = [[float(x) for x in rng.standard_normal(dims)] for _ in range(10)]
+
+        test_db.command("sql", f"CREATE VERTEX TYPE T BUCKETS {SMALL_BUCKETS}")
+        test_db.command("sql", "CREATE PROPERTY T.id INTEGER")
+        test_db.command("sql", "CREATE PROPERTY T.v ARRAY_OF_FLOATS")
+        index = test_db.create_vector_index(
+            "T",
+            "v",
+            dimensions=dims,
+            quantization="PRODUCT",
+            pq_subspaces=4,
+            pq_clusters=SMALL_PQ_CLUSTERS,
+            build_graph_now=False,
+        )
+        with test_db.transaction():
+            for i, vec in enumerate(vectors):
+                test_db.command("sql", "INSERT INTO T SET id = ?, v = ?", i, vec)
+
+        # the premise: every bucket is below pq_clusters (the inserts spread over the buckets)
+        in_bucket = [
+            test_db.query(
+                "sql",
+                f"SELECT count(*) AS c FROM bucket:T_{b}",  # nosec B608 - b from range()
+            )
+            .first()
+            .get("c")
+            for b in range(SMALL_BUCKETS)
+        ]
+        assert sum(in_bucket) == count
+        assert max(in_bucket) < SMALL_PQ_CLUSTERS, in_bucket
+
+        index.build_graph_now()
+
+        for query in queries:
+            ranking = _cosine_ranking(vectors, query)
+            approximate = index.find_nearest_approximate(query, k=5)
+            exact = index.find_nearest(query, k=5)
+            approximate_ids = [int(r.get("id")) for r, _ in approximate]
+            exact_ids = [int(r.get("id")) for r, _ in exact]
+
+            assert approximate_ids[0] == ranking[0]
+            assert exact_ids[0] == ranking[0]
+            assert approximate_ids == ranking[:5]
+            assert exact_ids == ranking[:5]
+            # no PQ estimate is in force: the approximate scores are the exact scores
+            exact_score = {int(r.get("id")): s for r, s in exact}
+            for record, score in approximate:
+                assert score == pytest.approx(
+                    exact_score[int(record.get("id"))], abs=1e-5
+                )
+
+
 class TestBinaryRerankOversample:
     """`rerank_oversample` on a BINARY index (ArcadeData/arcadedb#10968, ArcadeDB 26.11.1).