From 7c45635c4021fe990c8821e916f562cdfd24cf12 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 12:32:21 +0800 Subject: [PATCH 01/14] feat(table): maintain primary-key indexes during compaction --- crates/paimon/src/spec/pk_index_source.rs | 70 ++ crates/paimon/src/spec/schema.rs | 48 +- crates/paimon/src/table/compact.rs | 94 +- crates/paimon/src/table/compact/automatic.rs | 8 + .../paimon/src/table/compact/primary_key.rs | 1 + .../src/table/compact/primary_key/indexes.rs | 445 +++++++ .../compact/primary_key/indexes/tests.rs | 155 +++ .../compact/primary_key/indexes/writer.rs | 390 ++++++ .../paimon/src/table/global_index_options.rs | 51 +- crates/paimon/src/table/mod.rs | 1 + crates/paimon/src/table/pk_index_options.rs | 167 +++ .../paimon/src/table/pk_sorted_index_scan.rs | 47 +- .../src/table/pk_vector_search_params.rs | 9 +- .../table/pk_vector_search_params/tests.rs | 39 + .../sorted_global_index_build_builder.rs | 11 +- .../extraction.rs | 4 +- .../sorted_global_index_build_builder/sort.rs | 14 +- .../validation.rs | 5 +- crates/paimon/src/table/table_commit.rs | 30 +- .../table/table_commit/auto_maintenance.rs | 1 + .../src/table/vindex_index_build_builder.rs | 4 +- .../vindex_index_build_builder/extraction.rs | 12 +- .../vindex_index_build_builder/writer.rs | 2 +- .../paimon/tests/pk_compact/dynamic_bucket.rs | 19 +- crates/paimon/tests/pk_compact/indexes.rs | 1119 +++++++++++++++++ crates/paimon/tests/pk_compact_test.rs | 3 + docs/src/compaction.md | 43 +- 27 files changed, 2678 insertions(+), 114 deletions(-) create mode 100644 crates/paimon/src/table/compact/primary_key/indexes.rs create mode 100644 crates/paimon/src/table/compact/primary_key/indexes/tests.rs create mode 100644 crates/paimon/src/table/compact/primary_key/indexes/writer.rs create mode 100644 crates/paimon/src/table/pk_index_options.rs create mode 100644 crates/paimon/tests/pk_compact/indexes.rs diff --git a/crates/paimon/src/spec/pk_index_source.rs b/crates/paimon/src/spec/pk_index_source.rs index a821bc6dd..e17f25ea0 100644 --- a/crates/paimon/src/spec/pk_index_source.rs +++ b/crates/paimon/src/spec/pk_index_source.rs @@ -117,6 +117,40 @@ impl PrimaryKeyIndexSourceMeta { &self.source_files } + /// Encode the Java DataOutput frame, including modified UTF-8 file names. + pub fn serialize(&self) -> crate::Result> { + let count = i32::try_from(self.source_files.len()) + .map_err(|_| data_invalid("too many primary-key index source files"))?; + let mut bytes = Vec::new(); + bytes.extend_from_slice(&SOURCE_META_VERSION.to_be_bytes()); + bytes.extend_from_slice(&self.data_level.to_be_bytes()); + bytes.extend_from_slice(&count.to_be_bytes()); + for file in &self.source_files { + let mut name = Vec::new(); + for ch in file.file_name.encode_utf16() { + match ch { + 1..=0x7f => name.push(ch as u8), + 0..=0x7ff => { + name.push(0xc0 | (ch >> 6) as u8); + name.push(0x80 | (ch & 0x3f) as u8); + } + _ => { + name.push(0xe0 | (ch >> 12) as u8); + name.push(0x80 | ((ch >> 6) & 0x3f) as u8); + name.push(0x80 | (ch & 0x3f) as u8); + } + } + } + let length = u16::try_from(name.len()).map_err(|_| { + data_invalid("primary-key index source name exceeds Java writeUTF limit") + })?; + bytes.extend_from_slice(&length.to_be_bytes()); + bytes.extend_from_slice(&name); + bytes.extend_from_slice(&file.row_count.to_be_bytes()); + } + Ok(bytes) + } + /// Build from the `_SOURCE_META` bytes carried on a [`GlobalIndexMeta`]. /// Errors if the metadata carries no source blob. pub fn from_global_index_meta(meta: &GlobalIndexMeta) -> crate::Result { @@ -501,4 +535,40 @@ mod tests { let mut cursor = DataInputCursor::new(&bytes); assert!(read_java_utf(&mut cursor).is_err()); } + + #[test] + fn serialize_matches_java_frame_and_modified_utf8() { + let ascii = PrimaryKeyIndexSourceMeta::new( + 2, + vec![ + PrimaryKeyIndexSourceFile::new("a.parquet".into(), 3).unwrap(), + PrimaryKeyIndexSourceFile::new("b.parquet".into(), 0).unwrap(), + ], + ) + .unwrap(); + assert_eq!( + ascii.serialize().unwrap(), + frame(2, &[("a.parquet", 3), ("b.parquet", 0)]) + ); + let source = PrimaryKeyIndexSourceMeta::new( + 1, + vec![PrimaryKeyIndexSourceFile::new("\0中😀".into(), 7).unwrap()], + ) + .unwrap(); + let bytes = source.serialize().unwrap(); + assert_eq!( + &bytes[12..25], + &[0, 11, 0xc0, 0x80, 0xe4, 0xb8, 0xad, 0xed, 0xa0, 0xbd, 0xed, 0xb8, 0x80] + ); + assert_eq!( + PrimaryKeyIndexSourceMeta::deserialize(&bytes).unwrap(), + source + ); + let oversized = PrimaryKeyIndexSourceMeta::new( + 1, + vec![PrimaryKeyIndexSourceFile::new("中".repeat(22000), 1).unwrap()], + ) + .unwrap(); + assert!(oversized.serialize().is_err()); + } } diff --git a/crates/paimon/src/spec/schema.rs b/crates/paimon/src/spec/schema.rs index 85dd36a5a..91f5b41eb 100644 --- a/crates/paimon/src/spec/schema.rs +++ b/crates/paimon/src/spec/schema.rs @@ -879,7 +879,14 @@ fn assert_not_updating_primary_key_index_column( .iter() .any(|column| column == field_name); - if is_vector_index_column || is_full_text_index_column { + let is_scalar_index_column = ["btree", "bitmap", "multivalue", "fm"] + .iter() + .any(|family| { + options + .get(&format!("pk-{family}.index.columns")) + .is_some_and(|columns| columns.split(',').any(|column| column.trim() == field_name)) + }); + if is_vector_index_column || is_full_text_index_column || is_scalar_index_column { return Err(crate::Error::Unsupported { message: format!("Cannot {operation} primary-key index column: [{field_name}]"), }); @@ -5764,6 +5771,45 @@ mod tests { } } + #[test] + fn test_rejects_destructive_primary_key_scalar_index_column_changes() { + for family in ["btree", "bitmap", "multivalue", "fm"] { + let data_type = match family { + "fm" => DataType::VarChar(VarCharType::string_type()), + "multivalue" => DataType::Array(ArrayType::new(DataType::Int(IntType::new()))), + _ => DataType::Int(IntType::new()), + }; + let table_schema = TableSchema::new( + 0, + &Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", data_type) + .primary_key(["id"]) + .option("bucket", "1") + .option(format!("pk-{family}.index.columns"), "value") + .build() + .unwrap(), + ); + assert_primary_key_index_column_changes_rejected( + &table_schema, + "value", + DataType::BigInt(crate::spec::BigIntType::new()), + ); + let error = table_schema + .apply_changes(vec![ + crate::spec::SchemaChange::remove_option(format!("pk-{family}.index.columns")), + crate::spec::SchemaChange::rename_column("value".into(), "renamed".into()), + ]) + .unwrap_err(); + assert!( + error + .to_string() + .contains("Cannot rename primary-key index column"), + "{error}" + ); + } + } + #[test] fn test_rejects_destructive_primary_key_full_text_index_column_changes() { let table_schema = TableSchema::new( diff --git a/crates/paimon/src/table/compact.rs b/crates/paimon/src/table/compact.rs index ad86a5047..56077585c 100644 --- a/crates/paimon/src/table/compact.rs +++ b/crates/paimon/src/table/compact.rs @@ -370,7 +370,8 @@ pub struct CompactExecutor { emit_changelog: bool, } -/// Prepared files owned by the caller until submitted to `TableCommit::commit_compact`. +/// Prepared data, changelog and index files owned by the caller until submitted +/// to `TableCommit::commit_compact`. /// A failed or uncertain commit must retain these files for retry. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CompactResult { @@ -380,9 +381,16 @@ pub struct CompactResult { changelog_files: Vec, #[serde(default)] deletion_vector_updates: Option>, + #[serde(default)] + index_entries: Option>, } impl CompactResult { + /// Atomic index additions and retirements prepared with these data files. + pub fn index_entries(&self) -> Option<&[crate::spec::IndexManifestEntry]> { + self.index_entries.as_deref() + } + pub fn files(&self) -> &[DataFileMeta] { &self.files } @@ -491,6 +499,7 @@ impl CompactExecutor { files, changelog_files: vec![], deletion_vector_updates: None, + index_entries: None, }); } if task.is_primary_key() { @@ -547,10 +556,33 @@ impl CompactExecutor { }); } } + let index_entries = match primary_key::indexes::prepare( + &table, + task, + &split, + &snapshot, + &active, + &files, + self.resources.clone(), + ) + .await + { + Ok(entries) => entries, + Err(error) => { + let mut message = + super::CommitMessage::new(task.partition.clone(), task.bucket, files); + message.new_changelog_files = changelog_files; + let _ = super::TableCommit::new(table.clone(), "compact-cleanup".into()) + .abort(&[message]) + .await; + return Err(error); + } + }; return Ok(CompactResult { task: task.clone(), files, changelog_files, + index_entries: Some(index_entries), deletion_vector_updates: options .deletion_vectors_enabled() .then_some(deletion_vector_updates), @@ -606,6 +638,7 @@ impl CompactExecutor { files, changelog_files: vec![], deletion_vector_updates: None, + index_entries: None, }), Err(error) => { writer.abort().await; @@ -816,6 +849,7 @@ pub(super) struct PreparedCompact { pub entries: Vec, pub changelog_entries: Vec, pub deletion_vector_updates: Option>, + pub index_entries: Option>, pub schema_id: i64, pub snapshot_id: i64, pub read_snapshot: Option>, @@ -834,6 +868,7 @@ pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Resul let mut entries = Vec::new(); let mut changelog_entries = Vec::new(); let mut deletion_vector_updates = Some(Vec::new()); + let mut index_entries = Some(Vec::new()); let mut metrics = CompactMetrics::default(); for result in results { let task = &result.task; @@ -847,6 +882,29 @@ pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Resul return Err(invalid("Compaction results must belong to one plan")); } ids.push(task.task_id); + if task.is_primary_key() { + match (&mut index_entries, &result.index_entries) { + (Some(all), Some(entries)) => { + if entries + .iter() + .any(|e| e.partition != task.partition || e.bucket != task.bucket) + { + return Err(invalid("PK index output belongs to a different bucket")); + } + all.extend(entries.iter().cloned()); + } + (_, None) => index_entries = None, + _ => {} + } + } else if result + .index_entries + .as_ref() + .is_some_and(|entries| !entries.is_empty()) + { + return Err(invalid( + "Only PK compaction can prepare source-backed indexes", + )); + } for file in &task.files { if !inputs.insert((task.partition.clone(), task.bucket, file.file_name.clone())) { return Err(invalid("Overlapping compaction results")); @@ -1008,6 +1066,7 @@ pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Resul entries, changelog_entries, deletion_vector_updates, + index_entries, schema_id: first.task.context.schema.id(), snapshot_id: first.task.snapshot_id(), read_snapshot: first @@ -1024,6 +1083,13 @@ pub(super) fn ensure_supported(table: &Table) -> Result<()> { table.ensure_not_branch_reference_for_write()?; let options = table.schema().core_options(); let properties = table.schema().options(); + if !table.schema().primary_keys().is_empty() { + super::pk_index_options::definitions(table.schema()).map_err(|error| { + Error::ConfigInvalid { + message: format!("Invalid primary-key index declaration: {error}"), + } + })?; + } if options.data_evolution_enabled() { data_evolution::ensure_supported(table)?; return options.validate_data_file_path_directory(); @@ -1138,8 +1204,7 @@ fn ensure_snapshot_supported(table: &Table, snapshot: &Snapshot) -> Result<()> { if snapshot.index_manifest().is_some() && !table.schema().core_options().data_evolution_enabled() && !table.schema().core_options().deletion_vectors_enabled() - && (table.schema().primary_keys().is_empty() - || table.schema().core_options().bucket() != -1) + && table.schema().primary_keys().is_empty() { return Err(Error::Unsupported { message: "Compaction only supports dynamic-bucket HASH index manifests".into(), @@ -1149,8 +1214,8 @@ fn ensure_snapshot_supported(table: &Table, snapshot: &Snapshot) -> Result<()> { } /// HASH indexes contain key-to-bucket assignments, independent of data files. -/// Preserve their manifest verbatim, including hashes of deleted keys. Other -/// index types may address rewritten files or row positions and cannot be reused. +/// Preserve their manifest verbatim, including hashes of deleted keys. PK source +/// indexes and deletion vectors are validated here and maintained at commit. pub(super) async fn validate_snapshot_indexes( table: &Table, snapshot: &Snapshot, @@ -1165,17 +1230,24 @@ pub(super) async fn validate_snapshot_indexes( data_evolution::validate_indexes(&entries)?; return Ok(Some(name.to_string())); } - if entries.iter().any(|entry| { - !(entry.index_file.index_type == "HASH" + for entry in &entries { + if !table.schema().primary_keys().is_empty() && primary_key::indexes::is_source_index(entry) + { + primary_key::indexes::source(entry)?; + continue; + } + if !(entry.index_file.index_type == "HASH" || (table.schema().core_options().deletion_vectors_enabled() && entry.index_file.index_type == "DELETION_VECTORS")) || (entry.index_file.index_type == "HASH" && entry.index_file.deletion_vectors_ranges.is_some()) || entry.index_file.global_index_meta.is_some() - }) { - return Err(Error::Unsupported { - message: "Compaction only supports dynamic-bucket HASH index manifests".into(), - }); + { + return Err(Error::Unsupported { + message: "Compaction requires HASH, deletion-vector or source-backed PK indexes" + .into(), + }); + } } Ok(Some(name.to_string())) } diff --git a/crates/paimon/src/table/compact/automatic.rs b/crates/paimon/src/table/compact/automatic.rs index ba42bd1c7..e497b72bc 100644 --- a/crates/paimon/src/table/compact/automatic.rs +++ b/crates/paimon/src/table/compact/automatic.rs @@ -59,6 +59,14 @@ pub(in crate::table) async fn execute_auto_plan( result.files.clone(), ); message.new_changelog_files = result.changelog_files.clone(); + message.new_index_files = result + .index_entries + .as_deref() + .unwrap_or_default() + .iter() + .filter(|e| e.kind == crate::spec::FileKind::Add) + .map(|e| e.index_file.clone()) + .collect(); message }) .collect::>(); diff --git a/crates/paimon/src/table/compact/primary_key.rs b/crates/paimon/src/table/compact/primary_key.rs index a0826e58c..005cffca7 100644 --- a/crates/paimon/src/table/compact/primary_key.rs +++ b/crates/paimon/src/table/compact/primary_key.rs @@ -16,6 +16,7 @@ // under the License. pub(in crate::table) mod deletion_vectors; +pub(in crate::table) mod indexes; pub(super) mod lookup; pub(super) mod lookup_file; pub(super) mod lookup_store; diff --git a/crates/paimon/src/table/compact/primary_key/indexes.rs b/crates/paimon/src/table/compact/primary_key/indexes.rs new file mode 100644 index 000000000..b81ca77cd --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/indexes.rs @@ -0,0 +1,445 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Source-backed index maintenance for primary-key compaction. + +mod writer; + +#[cfg(test)] +mod tests; + +use super::super::{invalid, CompactTask}; +use crate::resource::ResourceContext; +use crate::spec::{ + should_read_pk_index_source, DataFileMeta, FileKind, IndexManifest, IndexManifestEntry, + ManifestEntry, PrimaryKeyIndexSourceFile, PrimaryKeyIndexSourceMeta, Snapshot, +}; +use crate::table::pk_index_options::{definitions, Definition}; +use crate::table::{CommitMessage, DataSplit, Table, TableCommit}; +use crate::Result; +use std::collections::{BTreeMap, HashSet}; + +type Unit = (Vec, i32, i32, i32); +type Levels = BTreeMap>; + +pub(in crate::table) fn is_source_index(entry: &IndexManifestEntry) -> bool { + entry.index_file.global_index_meta.is_some() +} + +pub(in crate::table) fn source(entry: &IndexManifestEntry) -> Result { + let file = &entry.index_file; + if !matches!( + file.index_type.as_str(), + "btree" | "bitmap" | "multivalue" | "fm" | "full-text" + ) && !crate::vindex::is_vindex_index_type(&file.index_type) + { + return Err(invalid("Unsupported primary-key index type")); + } + let meta = file + .global_index_meta + .as_ref() + .ok_or_else(|| invalid("Missing PK index metadata"))?; + if meta + .extra_field_ids + .as_ref() + .is_some_and(|ids| !ids.is_empty()) + || file.deletion_vectors_ranges.is_some() + { + return Err(invalid("Invalid primary-key index metadata")); + } + PrimaryKeyIndexSourceMeta::from_global_index_meta(meta) +} + +fn unit(entry: &IndexManifestEntry) -> Result { + Ok(( + entry.partition.clone(), + entry.bucket, + source(entry)?.data_level(), + entry + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_field_id, + )) +} + +async fn read_indexes(table: &Table, snapshot: &Snapshot) -> Result> { + match snapshot.index_manifest() { + Some(name) => { + IndexManifest::read( + table.file_io(), + &table.snapshot_manager().manifest_path(name), + ) + .await + } + None => Ok(vec![]), + } +} + +fn levels( + active: &[ManifestEntry], + changes: &[ManifestEntry], + partition: &[u8], + bucket: i32, +) -> Levels { + let removed = changes + .iter() + .filter(|e| { + *e.kind() == FileKind::Delete && e.partition() == partition && e.bucket() == bucket + }) + .map(|e| e.file().file_name.as_str()) + .collect::>(); + let mut levels = Levels::new(); + for entry in active + .iter() + .filter(|e| !removed.contains(e.file().file_name.as_str())) + .chain(changes.iter().filter(|e| *e.kind() == FileKind::Add)) + .filter(|e| { + e.partition() == partition + && e.bucket() == bucket + && should_read_pk_index_source(e.file()) + }) + { + levels + .entry(entry.file().level) + .or_default() + .push(entry.file().clone()); + } + for files in levels.values_mut() { + files.sort_by(|a, b| a.file_name.cmp(&b.file_name)); + } + levels +} + +fn source_meta(level: i32, files: &[DataFileMeta]) -> Result { + PrimaryKeyIndexSourceMeta::new( + level, + files + .iter() + .map(|f| PrimaryKeyIndexSourceFile::new(f.file_name.clone(), f.row_count)) + .collect::>()?, + ) +} + +fn row_count(files: &[DataFileMeta]) -> Result { + files.iter().try_fold(0i64, |sum, f| { + sum.checked_add(f.row_count) + .ok_or_else(|| invalid("PK index row count overflow")) + }) +} + +fn compatible(table: &Table, entry: &IndexManifestEntry, definition: &Definition) -> Result { + if entry.index_file.index_type != definition.index_type { + return Ok(false); + } + if definition.index_type == "full-text" + || crate::vindex::is_vindex_index_type(&definition.index_type) + { + let expected = writer::native_options(table, definition)?; + let recorded = entry + .index_file + .global_index_meta + .as_ref() + .and_then(|m| m.index_meta.as_deref()) + .and_then(|bytes| serde_json::from_slice::>(bytes).ok()); + return Ok(recorded.as_ref() == Some(&expected)); + } + Ok(true) +} + +struct Maintenance<'a> { + deletes: Vec, + builds: Vec<(i32, &'a Definition)>, +} + +fn plan<'a>( + table: &Table, + indexes: &[IndexManifestEntry], + levels: &Levels, + defs: &'a [Definition], + partition: &[u8], + bucket: i32, +) -> Result> { + let mut existing = BTreeMap::<(i32, i32), Vec<&IndexManifestEntry>>::new(); + for entry in indexes + .iter() + .filter(|e| e.partition == partition && e.bucket == bucket && is_source_index(e)) + { + let meta = source(entry)?; + existing + .entry(( + meta.data_level(), + entry + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_field_id, + )) + .or_default() + .push(entry); + } + let mut deletes = Vec::new(); + let mut builds = Vec::new(); + for ((level, field), entries) in &existing { + let expected = levels + .get(level) + .map(|files| source_meta(*level, files)) + .transpose()?; + let definition = defs.iter().find(|d| d.field.id() == *field); + let valid = entries.len() == 1 + && expected.as_ref() == Some(&source(entries[0])?) + && match definition { + Some(d) => compatible(table, entries[0], d)?, + None => true, + }; + if !valid { + for entry in entries { + let mut deleted = (*entry).clone(); + deleted.kind = FileKind::Delete; + deletes.push(deleted); + } + } + } + for (level, files) in levels { + if row_count(files)? == 0 { + continue; + } + for definition in defs { + let valid = existing + .get(&(*level, definition.field.id())) + .is_some_and(|entries| { + entries.len() == 1 + && !deletes + .iter() + .any(|d| d.index_file.file_name == entries[0].index_file.file_name) + }); + if !valid { + builds.push((*level, definition)); + } + } + } + Ok(Maintenance { deletes, builds }) +} + +pub(in crate::table::compact) async fn prepare( + table: &Table, + task: &CompactTask, + split: &DataSplit, + snapshot: &Snapshot, + active: &[ManifestEntry], + files: &[DataFileMeta], + resources: Option, +) -> Result> { + let defs = definitions(table.schema())?; + let indexes = read_indexes(table, snapshot).await?; + let changes = task + .files + .iter() + .map(|f| { + ManifestEntry::new( + FileKind::Delete, + task.partition.clone(), + task.bucket, + task.total_buckets, + f.clone(), + 2, + ) + }) + .chain(files.iter().map(|f| { + ManifestEntry::new( + FileKind::Add, + task.partition.clone(), + task.bucket, + task.total_buckets, + f.clone(), + 2, + ) + })) + .collect::>(); + let after = levels(active, &changes, &task.partition, task.bucket); + let maintenance = plan(table, &indexes, &after, &defs, &task.partition, task.bucket)?; + let mut entries = maintenance.deletes; + for (level, definition) in maintenance.builds { + match writer::build(table, split, &after[&level], definition, resources.clone()).await { + Ok(Some(index_file)) => entries.push(IndexManifestEntry { + kind: FileKind::Add, + partition: task.partition.clone(), + bucket: task.bucket, + index_file, + version: 1, + }), + Ok(None) => {} + Err(error) => { + abort(table, &entries).await; + return Err(error); + } + } + } + Ok(entries) +} + +pub(in crate::table) async fn abort(table: &Table, entries: &[IndexManifestEntry]) { + let messages = entries + .iter() + .filter(|e| e.kind == FileKind::Add) + .map(|e| { + let mut message = CommitMessage::new(e.partition.clone(), e.bucket, vec![]); + message.new_index_files.push(e.index_file.clone()); + message + }) + .collect::>(); + let _ = TableCommit::new(table.clone(), "pk-index-cleanup".into()) + .abort(&messages) + .await; +} + +/// Rebase unrelated indexes while checking the captured membership of changed units. +pub(in crate::table) async fn resolve( + table: &Table, + captured: Option<&Snapshot>, + latest: &Snapshot, + active: &[ManifestEntry], + changes: &[ManifestEntry], + prepared: Option<&[IndexManifestEntry]>, +) -> Result>> { + if table.schema().primary_keys().is_empty() { + return Ok(None); + } + let defs = definitions(table.schema())?; + let current = read_indexes(table, latest).await?; + let captured = match captured { + Some(snapshot) => read_indexes(table, snapshot).await?, + None => return Err(invalid("PK compaction requires its captured snapshot")), + }; + let buckets = changes + .iter() + .map(|e| (e.partition().to_vec(), e.bucket())) + .collect::>(); + if prepared.is_none() + && (!defs.is_empty() + || captured + .iter() + .any(|e| is_source_index(e) && buckets.contains(&(e.partition.clone(), e.bucket)))) + { + return Err(invalid( + "Primary-key compaction results are missing index maintenance; execute the plan again", + )); + } + let prepared = prepared.unwrap_or_default(); + let mut required_deletes = Vec::new(); + let mut required_builds = HashSet::new(); + let mut after_levels = BTreeMap::new(); + for (partition, bucket) in &buckets { + let after = levels(active, changes, partition, *bucket); + let maintenance = plan(table, &captured, &after, &defs, partition, *bucket)?; + required_deletes.extend(maintenance.deletes); + for (level, definition) in maintenance.builds { + required_builds.insert((partition.clone(), *bucket, level, definition.field.id())); + } + after_levels.insert((partition.clone(), *bucket), after); + } + let actual_deletes = prepared + .iter() + .filter(|e| e.kind == FileKind::Delete) + .collect::>(); + if actual_deletes.len() != required_deletes.len() + || required_deletes + .iter() + .any(|e| !actual_deletes.contains(&e)) + { + return Err(invalid( + "PK compaction index retirements do not match captured sources", + )); + } + let mut touched = required_deletes + .iter() + .map(unit) + .collect::>>()?; + touched.extend(required_builds.iter().cloned()); + for key in &touched { + let before = captured + .iter() + .filter(|e| is_source_index(e) && unit(e).is_ok_and(|u| &u == key)) + .collect::>(); + let now = current + .iter() + .filter(|e| is_source_index(e) && unit(e).is_ok_and(|u| &u == key)) + .collect::>(); + if before != now { + return Err(invalid( + "PK index changed concurrently; execute compaction again", + )); + } + } + let mut names = HashSet::new(); + for entry in prepared.iter().filter(|e| e.kind == FileKind::Add) { + if !names.insert(entry.index_file.file_name.as_str()) + || current + .iter() + .any(|old| old.index_file.file_name == entry.index_file.file_name) + { + return Err(invalid("PK index output must have a new unique file name")); + } + let key = unit(entry)?; + if !required_builds.remove(&key) { + return Err(invalid("Unexpected or duplicate PK index output")); + } + let definition = defs + .iter() + .find(|d| d.field.id() == key.3) + .ok_or_else(|| invalid("Undeclared PK index output"))?; + let files = &after_levels[&(key.0.clone(), key.1)][&key.2]; + let meta = entry.index_file.global_index_meta.as_ref().unwrap(); + let count = row_count(files)?; + if source(entry)? != source_meta(key.2, files)? + || meta.row_range_start != 0 + || meta.row_range_end != count - 1 + || entry.index_file.row_count != count + || entry.index_file.file_size < 0 + || !compatible(table, entry, definition)? + { + return Err(invalid( + "PK index output does not match complete compacted level", + )); + } + } + // All-NULL vector levels intentionally have no payload. + if required_builds.iter().any(|key| { + !defs + .iter() + .any(|d| d.field.id() == key.3 && crate::vindex::is_vindex_index_type(&d.index_type)) + }) { + return Err(invalid("Missing PK index output")); + } + if prepared.is_empty() { + return Ok(None); + } + let mut merged = current; + for deleted in actual_deletes { + let mut added = deleted.clone(); + added.kind = FileKind::Add; + let position = merged + .iter() + .position(|e| e == &added) + .ok_or_else(|| invalid("PK index changed concurrently"))?; + merged.remove(position); + } + merged.extend(prepared.iter().filter(|e| e.kind == FileKind::Add).cloned()); + Ok(Some(merged)) +} diff --git a/crates/paimon/src/table/compact/primary_key/indexes/tests.rs b/crates/paimon/src/table/compact/primary_key/indexes/tests.rs new file mode 100644 index 000000000..13f9a3a81 --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/indexes/tests.rs @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::io::FileIOBuilder; +use crate::spec::{ + ArrayType, DataType, Datum, IntType, PredicateBuilder, Schema, TableSchema, VarCharType, +}; +use crate::table::global_index_scanner::GlobalIndexScanner; +use crate::table::RowRange; +use arrow_array::{ + builder::{Int32Builder, ListBuilder}, + ArrayRef, Int32Array, Int8Array, RecordBatch, StringArray, +}; +use std::sync::Arc; + +/// Read the actual native payload instead of relying on a query's raw fallback. +#[tokio::test] +async fn scalar_payloads_address_nulls_and_tombstones_by_physical_ordinal() { + for kind in ["btree", "bitmap", "multivalue", "fm"] { + let data_type = match kind { + "fm" => DataType::VarChar(VarCharType::string_type()), + "multivalue" => DataType::Array(ArrayType::new(DataType::Int(IntType::new()))), + _ => DataType::Int(IntType::new()), + }; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", data_type) + .primary_key(["id"]) + .option("bucket", "1") + .option("write-only", "true") + .option("index-file-in-data-file-dir", "false") + .option( + "fields.value.pk-fm.index.options", + r#"{"sa-sample-rate":1}"#, + ) + .option(format!("pk-{kind}.index.columns"), "value") + .build() + .unwrap(); + let location = format!("memory:/pk-index-native/{}", uuid::Uuid::new_v4()); + let io = FileIOBuilder::new("memory").build().unwrap(); + let table = Table::new( + io.clone(), + crate::catalog::Identifier::new("default", "native"), + location.clone(), + TableSchema::new(0, &schema), + None, + ); + io.mkdirs(&format!("{location}/schema")).await.unwrap(); + io.mkdirs(&format!("{location}/snapshot")).await.unwrap(); + io.new_output(&format!("{location}/schema/schema-0")) + .unwrap() + .write(bytes::Bytes::from( + serde_json::to_vec(table.schema()).unwrap(), + )) + .await + .unwrap(); + let values: ArrayRef = match kind { + "fm" => Arc::new(StringArray::from(vec![Some("ten"), None, Some("thirty")])), + "multivalue" => { + let mut builder = ListBuilder::new(Int32Builder::new()).with_field(Arc::new( + arrow_schema::Field::new("element", arrow_schema::DataType::Int32, true), + )); + builder.values().append_value(10); + builder.append(true); + builder.append(false); + builder.values().append_slice(&[30, 30]); + builder.append(true); + Arc::new(builder.finish()) + } + _ => Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])), + }; + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef), + ("value", values), + ( + "_VALUE_KIND", + Arc::new(Int8Array::from(vec![0, 0, 3])) as ArrayRef, + ), + ]) + .unwrap(); + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + write + .new_commit() + .commit(writer.prepare_commit().await.unwrap()) + .await + .unwrap(); + let compact = table.new_compact_builder(); + let plan = compact.new_planner().unwrap().plan().await.unwrap(); + let result = compact + .new_executor() + .unwrap() + .execute(&plan.tasks()[0]) + .await + .unwrap(); + assert_eq!(result.files()[0].delete_row_count, Some(1)); + let entries = result.index_entries().unwrap(); + assert_eq!(entries.len(), 1); + let scanner = GlobalIndexScanner::create_with_fm_options( + &io, + &location, + 1, + i64::MAX, + 128 * 1024 * 1024, + i64::MAX, + entries, + table.schema().fields(), + crate::fm_index::FMReadOptions { + locate_cost_ratio: 1.0, + ..Default::default() + }, + ) + .unwrap() + .unwrap(); + let predicates = PredicateBuilder::new(table.schema().fields()); + let predicate = match kind { + "fm" => predicates + .contains("value", Datum::String("irty".into())) + .unwrap(), + "multivalue" => predicates.array_contains("value", Datum::Int(30)).unwrap(), + _ => predicates.equal("value", Datum::Int(30)).unwrap(), + }; + assert_eq!( + scanner.matching_ranges(&predicate).await.unwrap(), + Some(vec![RowRange::new(2, 2)]), + "{kind}" + ); + if matches!(kind, "btree" | "bitmap") { + assert_eq!( + scanner + .matching_ranges(&predicates.is_null("value").unwrap()) + .await + .unwrap(), + Some(vec![RowRange::new(1, 1)]), + "{kind}" + ); + } + } +} diff --git a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs new file mode 100644 index 000000000..9fa61b794 --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs @@ -0,0 +1,390 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native writers shared with global indexes, fed physical PK file ordinals. + +use super::super::super::invalid; +use crate::resource::ResourceContext; +use crate::spec::{ + extract_datum_from_arrow, DataFileMeta, DataType, Datum, GlobalIndexMeta, IndexFileMeta, + ROW_ID_FIELD_NAME, +}; +use crate::table::global_index_build_common::copy_local_file_to_output; +use crate::table::index_file_path::IndexFileLocation; +use crate::table::pk_index_options::Definition; +use crate::table::sorted_global_index_build_builder::{ + extraction::{emit_multivalue_rows, emit_scalar_rows}, + make_index_key_codec, + sort::IndexSorter, + validation::index_key_type, +}; +use crate::table::sorted_global_index_options::SortedIndexWriteOptions; +use crate::table::vindex_index_build_builder::{ + extraction::validate_vector_batch, writer::train_and_add_spilled_vectors, +}; +use crate::table::{DataSplit, Table}; +use crate::{Error, Result}; +use arrow_array::{Int64Array, RecordBatch}; +use arrow_schema::{DataType as ArrowType, Field, Schema}; +use futures::TryStreamExt; +use std::collections::BTreeMap; +use std::future::Future; +use std::sync::Arc; +use tokio::io::AsyncWriteExt; + +fn native_error(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::DataInvalid { + message: "PK index build failed".into(), + source: Some(Box::new(error)), + } +} + +async fn blocking(f: impl FnOnce() -> Result + Send + 'static) -> Result { + tokio::task::spawn_blocking(f).await.map_err(native_error)? +} + +pub(super) fn native_options( + table: &Table, + definition: &Definition, +) -> Result> { + if crate::vindex::is_vindex_index_type(&definition.index_type) { + return Ok(crate::vindex::VindexVectorIndexOptions::new( + table.schema().options(), + &definition.options, + &definition.index_type, + &definition.field, + )? + .native_options + .into_iter() + .collect()); + } + let mut options = table.schema().options().clone(); + options.extend(definition.options.clone()); + Ok(options + .into_iter() + .filter_map(|(key, value)| { + key.strip_prefix("full-text.") + .map(|key| (key.to_string(), value)) + }) + .collect()) +} + +/// Keep every physical row, including tombstones, NULLs, and DV-hidden rows. +/// Native extractors use a synthetic consecutive row ID local to this level. +async fn batches( + table: &Table, + split: &DataSplit, + files: &[DataFileMeta], + resources: Option, + mut state: S, + mut consume: F, +) -> Result +where + F: FnMut(S, RecordBatch, i64) -> Fut, + Fut: Future>, +{ + let mut offset = 0i64; + for file in files { + let reader = super::super::new_reader(table, resources.clone())?; + let mut stream = reader.read_physical_file(split.clone(), file.clone())?; + while let Some(batch) = stream.try_next().await? { + let count = batch.num_rows() as i64; + let mut fields = batch.schema().fields().iter().cloned().collect::>(); + fields.push(Arc::new(Field::new( + ROW_ID_FIELD_NAME, + ArrowType::Int64, + false, + ))); + let mut columns = batch.columns().to_vec(); + columns.push(Arc::new(Int64Array::from_iter_values( + offset..offset + count, + ))); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) + .map_err(native_error)?; + state = consume(state, batch, offset).await?; + offset = offset + .checked_add(count) + .ok_or_else(|| invalid("PK index ordinal overflow"))?; + } + } + if offset != super::row_count(files)? { + return Err(invalid("PK index source row count changed")); + } + Ok(state) +} + +pub(super) async fn build( + table: &Table, + split: &DataSplit, + files: &[DataFileMeta], + definition: &Definition, + resources: Option, +) -> Result> { + let kind = definition.index_type.as_str(); + let count = super::row_count(files)?; + let file_name = format!("{kind}-pk-index-{}.index", uuid::Uuid::new_v4()); + let in_data = table.schema().core_options().index_file_in_data_file_dir(); + let relative = split + .bucket_path() + .strip_prefix(&table.data_file_location()) + .unwrap_or(split.bucket_path()) + .trim_start_matches('/'); + let external_path = crate::table::external_path::new_index_external_path( + table.schema().options(), + in_data, + relative, + &file_name, + )?; + let location = IndexFileLocation::BucketLocal { + table_path: table.location(), + bucket_path: split.bucket_path(), + index_file_in_data_file_dir: in_data, + }; + let path = location.resolve(&file_name, external_path.as_deref()); + if let Some((directory, _)) = path.rsplit_once('/') { + table.file_io().mkdirs(directory).await?; + } + let mut options = table.schema().options().clone(); + options.extend(definition.options.clone()); + let written: Result>> = async { + if crate::vindex::is_vindex_index_type(kind) { + vector(table,split,files,definition,resources,&path).await + } else if kind == "full-text" { + full_text(table,split,files,definition,resources,&path).await.map(Some) + } else { + let field = definition.field.clone(); + let codec_type = if kind == "btree" { "btree" } else { "bitmap" }; + let key_type = index_key_type(kind,&field)?; + let (cmp,serialize) = make_index_key_codec(codec_type,key_type); + let output = table.file_io().new_output(&path)?.writer().await?; + if kind == "fm" { + let writer = crate::fm_index::FMGlobalIndexWriter::new(output,crate::fm_index::FMOptions::from_options(&options)?.write).map_err(native_error)?; + let writer = batches(table,split,files,resources,writer,|mut native,batch,offset| { + let field = field.clone(); + async move { + let index = batch.schema().index_of(field.name()).map_err(native_error)?; + for row in 0..batch.num_rows() { + let key = extract_datum_from_arrow(&batch,row,index,field.data_type())?.map(|d| serialize(&d,field.data_type())); + native.write(key.as_deref(),(offset+row as i64) as u64).await.map_err(native_error)?; + } + Ok(native) + } + }).await?; + return Ok(Some(writer.finish().await.map_err(native_error)?.index_meta)); + } + let sorter = IndexSorter::new(crate::spec::CoreOptions::new(&options).write_buffer_size()?,cmp); + let sorter = batches(table,split,files,resources,sorter,|mut native,batch,offset| { + let field = field.clone(); let kind = kind.to_string(); + async move { + blocking(move || { + let mut emit = |row| native.push(row); + if kind == "multivalue" { + let DataType::Array(array) = field.data_type() else { unreachable!() }; + emit_multivalue_rows(std::slice::from_ref(&batch),field.name(),array.element_type(),0,batch.num_rows() as i64,serialize,offset,&mut emit)?; + } else { + emit_scalar_rows(std::slice::from_ref(&batch),field.name(),field.data_type(),0,batch.num_rows() as i64,serialize,offset,&mut emit)?; + } + Ok(native) + }).await + } + }).await?; + let mut rows = blocking(move || sorter.finish()).await?; + let options = SortedIndexWriteOptions::from_options(kind,&options)?; + let cmp = make_index_key_codec(codec_type,key_type).0; + if kind == "btree" { + let mut writer = crate::btree::BTreeIndexWriter::with_comparator_and_options(output,options.block_size,options.compression_type,options.compression_level,options.bloom_filter_enabled,cmp) + .with_file_version(options.btree_file_version).map_err(native_error)?; + while let Some((key,id)) = rows.next().await? { writer.write(key.as_deref(),id).await.map_err(native_error)?; } + Ok(Some(writer.finish().await.map_err(native_error)?.meta.serialize())) + } else { + let mut writer = crate::table::bitmap_global_index_writer::BitmapGlobalIndexWriter::with_compression_level(output,options.block_size,options.compression_type,options.compression_level,cmp); + while let Some((key,id)) = rows.next().await? { + if kind == "multivalue" { writer.write_posting(key.as_deref().unwrap(),id).map_err(native_error)?; } + else { writer.write(key.as_deref(),id).map_err(native_error)?; } + } + let result = if kind == "multivalue" { writer.finish_with_source_row_count(count as u64).await } + else { writer.finish().await }.map_err(native_error)?; + Ok(Some(result.meta.serialize())) + } + } + }.await; + let index_meta = match written { + Ok(Some(meta)) => meta, + Ok(None) => return Ok(None), + Err(error) => { + let _ = table.file_io().delete_file(&path).await; + return Err(error); + } + }; + let metadata = async { + let file_size = + i64::try_from(table.file_io().get_status(&path).await?.size).map_err(native_error)?; + Ok(Some(IndexFileMeta { + index_type: kind.to_string(), + file_name, + file_size, + row_count: count, + deletion_vectors_ranges: None, + external_path, + global_index_meta: Some(GlobalIndexMeta { + row_range_start: 0, + row_range_end: count - 1, + index_field_id: definition.field.id(), + extra_field_ids: None, + source_meta: Some(super::source_meta(files[0].level, files)?.serialize()?), + index_meta: Some(index_meta), + }), + })) + } + .await; + if metadata.is_err() { + let _ = table.file_io().delete_file(&path).await; + } + metadata +} + +async fn full_text( + table: &Table, + split: &DataSplit, + files: &[DataFileMeta], + definition: &Definition, + resources: Option, + path: &str, +) -> Result> { + let native = native_options(table, definition)?; + let config = paimon_ftindex_core::FullTextIndexConfig::from_options( + &native.clone().into_iter().collect(), + ) + .map_err(native_error)?; + let writer = blocking(move || { + paimon_ftindex_core::FullTextIndexWriter::new(config).map_err(native_error) + }) + .await?; + let mut writer = batches( + table, + split, + files, + resources, + writer, + |mut native, batch, offset| { + let field = definition.field.clone(); + async move { + blocking(move || { + let column = batch + .schema() + .index_of(field.name()) + .map_err(native_error)?; + for row in 0..batch.num_rows() { + if let Some(Datum::String(text)) = + extract_datum_from_arrow(&batch, row, column, field.data_type())? + { + native + .add_document(offset + row as i64, text) + .map_err(native_error)?; + } + } + Ok(native) + }) + .await + } + }, + ) + .await?; + let file = blocking(move || { + let mut file = tempfile::NamedTempFile::new().map_err(native_error)?; + writer + .write(&mut paimon_ftindex_core::io::PosWriter::new( + file.as_file_mut(), + )) + .map_err(native_error)?; + Ok(file) + }) + .await?; + copy_local_file_to_output(file.path(), table.file_io().new_output(path)?).await?; + serde_json::to_vec(&native).map_err(native_error) +} + +async fn vector( + table: &Table, + split: &DataSplit, + files: &[DataFileMeta], + definition: &Definition, + resources: Option, + path: &str, +) -> Result>> { + let options = crate::vindex::VindexVectorIndexOptions::new( + table.schema().options(), + &definition.options, + &definition.index_type, + &definition.field, + )?; + let dimension = options.dimension(); + let raw = tokio::io::BufWriter::new(tokio::fs::File::from_std( + tempfile::tempfile().map_err(native_error)?, + )); + let ids = tokio::io::BufWriter::new(tokio::fs::File::from_std( + tempfile::tempfile().map_err(native_error)?, + )); + let (mut raw, mut ids, count) = batches( + table, + split, + files, + resources, + (raw, ids, 0usize), + |(mut raw, mut ids, count), batch, offset| { + let column = definition.field.name(); + async move { + let mut expected = offset; + let vectors = validate_vector_batch(&batch, column, dimension, &mut expected)?; + raw.write_all(vectors.values.inner().as_slice()) + .await + .map_err(native_error)?; + ids.write_all(vectors.row_ids.inner().as_slice()) + .await + .map_err(native_error)?; + Ok((raw, ids, count + vectors.vector_count)) + } + }, + ) + .await?; + if count == 0 { + return Ok(None); + } + raw.flush().await.map_err(native_error)?; + ids.flush().await.map_err(native_error)?; + let raw = raw.into_inner().into_std().await; + let ids = ids.into_inner().into_std().await; + let config = options.training_config(count)?; + let ratio = options.train_sample_ratio; + let file = blocking(move || { + let (mut writer, _, _, _) = + train_and_add_spilled_vectors(raw, ids, config, count, dimension, ratio, false) + .map_err(native_error)?; + let mut file = tempfile::NamedTempFile::new().map_err(native_error)?; + writer + .write(&mut paimon_vindex_core::io::PosWriter::new( + file.as_file_mut(), + )) + .map_err(native_error)?; + Ok(file) + }) + .await?; + copy_local_file_to_output(file.path(), table.file_io().new_output(path)?).await?; + serde_json::to_vec(&native_options(table, definition)?) + .map(Some) + .map_err(native_error) +} diff --git a/crates/paimon/src/table/global_index_options.rs b/crates/paimon/src/table/global_index_options.rs index 14f8cb18f..bdaf383e6 100644 --- a/crates/paimon/src/table/global_index_options.rs +++ b/crates/paimon/src/table/global_index_options.rs @@ -96,18 +96,48 @@ pub(super) fn field_index_options( "vector" | "full-text" => format!("{index_type}."), _ => format!("{family}-index."), }; + let options_key = format!("{field_prefix}index-options"); + let mut options = parse_field_index_options( + table_options, + column, + &options_key, + &native_prefix, + family == "vector", + )?; + if family == "vector" { + let metric_key = format!("{field_prefix}distance-metric"); + let metric = table_options + .get(&metric_key) + .map(String::as_str) + .unwrap_or("inner_product"); + crate::vindex::pkvector::metric::VectorSearchMetric::parse(metric)?; + options.insert( + format!("{native_prefix}metric"), + metric.to_ascii_lowercase().replace('-', "_"), + ); + } + Ok(options) +} + +pub(super) fn parse_field_index_options( + table_options: &HashMap, + column: &str, + options_key: &str, + native_prefix: &str, + vector: bool, +) -> Result> { let mut options = HashMap::new(); let mut insert = |key: &str, value: String| -> Result<()> { if key.trim().is_empty() { return Err(crate::Error::ConfigInvalid { - message: format!("{field_prefix}index-options contains an empty option key"), + message: format!("{options_key} contains an empty option key"), }); } - let qualified = if key.starts_with(&native_prefix) + let qualified = if key.starts_with(native_prefix) || key.starts_with("fields.") || key.starts_with("global-index.") || key.starts_with("sorted-index.") - || (family == "vector" && matches!(key, "index.type" | "vindex.build.granule.enabled")) + || (vector && matches!(key, "index.type" | "vindex.build.granule.enabled")) { key.into() } else { @@ -125,9 +155,8 @@ pub(super) fn field_index_options( options.insert(qualified, value); Ok(()) }; - let options_key = format!("{field_prefix}index-options"); if let Some(raw) = table_options - .get(&options_key) + .get(options_key) .filter(|raw| !raw.trim().is_empty()) { let parsed: serde_json::Map = serde_json::from_str(raw) @@ -149,18 +178,6 @@ pub(super) fn field_index_options( insert(&key, value)?; } } - if family == "vector" { - let metric_key = format!("{field_prefix}distance-metric"); - let metric = table_options - .get(&metric_key) - .map(String::as_str) - .unwrap_or("inner_product"); - crate::vindex::pkvector::metric::VectorSearchMetric::parse(metric)?; - options.insert( - format!("{native_prefix}metric"), - metric.to_ascii_lowercase().replace('-', "_"), - ); - } Ok(options) } diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 2557a51f1..ca35f6db4 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -116,6 +116,7 @@ mod pk_full_text_bucket_search; mod pk_full_text_bucket_state; mod pk_full_text_read; mod pk_full_text_scan; +mod pk_index_options; mod pk_search_position; mod pk_search_ranker; mod pk_sorted_index_scan; diff --git a/crates/paimon/src/table/pk_index_options.rs b/crates/paimon/src/table/pk_index_options.rs new file mode 100644 index 000000000..cca938a4a --- /dev/null +++ b/crates/paimon/src/table/pk_index_options.rs @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Java-compatible declarations for source-backed primary-key indexes. + +use crate::spec::{CoreOptions, DataField, DataType, TableSchema}; +use crate::table::global_index_options::parse_field_index_options; +use crate::table::global_index_types::normalize_global_index_type; +use crate::{Error, Result}; +use std::collections::{HashMap, HashSet}; + +pub(super) struct Definition { + pub field: DataField, + pub index_type: String, + pub options: HashMap, +} + +pub(super) fn field_options( + options: &HashMap, + column: &str, + family: &str, + index_type: &str, +) -> Result> { + let key = format!("fields.{column}.pk-{family}.index.options"); + let prefix = if matches!(family, "vector" | "full-text") { + format!("{index_type}.") + } else { + format!("{family}-index.") + }; + let mut resolved = + parse_field_index_options(options, column, &key, &prefix, family == "vector")?; + if family == "vector" { + let metric = CoreOptions::new(options).primary_key_vector_distance_metric(column)?; + crate::vindex::pkvector::metric::VectorSearchMetric::parse(&metric)?; + resolved.insert(format!("{index_type}.metric"), metric); + } + Ok(resolved) +} + +pub(super) fn definitions(schema: &TableSchema) -> Result> { + let options = schema.options(); + let mut owners = HashMap::new(); + for family in ["btree", "bitmap", "multivalue", "fm", "full-text", "vector"] { + let key = format!("pk-{family}.index.columns"); + if let Some(columns) = options.get(&key) { + let mut unique = HashSet::new(); + for column in columns.split(',').map(str::trim) { + if column.is_empty() || !unique.insert(column) { + return Err(Error::ConfigInvalid { + message: format!("{key} contains an empty or duplicate column '{column}'"), + }); + } + if !schema.fields().iter().any(|field| field.name() == column) { + return Err(Error::ConfigInvalid { + message: format!("{key} contains unknown column '{column}'"), + }); + } + if owners.insert(column, family).is_some() { + return Err(Error::ConfigInvalid { + message: format!("Column '{column}' can own at most one primary-key index"), + }); + } + } + } + } + if options.contains_key("pk-vector.index.columns") { + schema.core_options().primary_key_vector_index_column()?; + } + let mut result = Vec::new(); + for field in schema.fields() { + let Some(&family) = owners.get(field.name()) else { + continue; + }; + let kind = if family == "vector" { + let configured = schema + .core_options() + .primary_key_vector_index_type(field.name())?; + normalize_global_index_type(configured.trim()) + .filter(|kind| crate::vindex::is_vindex_index_type(kind)) + .ok_or_else(|| Error::ConfigInvalid { + message: format!("Unsupported primary-key vector index type: {configured}"), + })? + } else { + family + }; + let configured = field_options(options, field.name(), family, kind)?; + let mut merged = options.clone(); + merged.extend(configured.clone()); + match family { + "vector" => { + match field.data_type() { + DataType::Vector(_) => {} + DataType::Array(array) + if matches!(array.element_type(), DataType::Float(_)) => {} + _ => { + return Err(Error::Unsupported { + message: format!( + "PK vector index requires VECTOR or ARRAY: {}", + field.name() + ), + }) + } + } + crate::vindex::VindexVectorIndexOptions::new(options, &configured, kind, field)?; + } + "full-text" => { + if !matches!(field.data_type(), DataType::Char(_) | DataType::VarChar(_)) { + return Err(Error::Unsupported { + message: format!( + "PK full-text index requires a string column: {}", + field.name() + ), + }); + } + let native = merged + .iter() + .filter_map(|(k, v)| { + k.strip_prefix("full-text.") + .map(|key| (key.to_string(), v.clone())) + }) + .collect(); + paimon_ftindex_core::FullTextIndexConfig::from_options(&native).map_err(invalid)?; + } + "fm" => { + crate::table::sorted_global_index_build_builder::validation::index_key_type( + kind, field, + )?; + crate::fm_index::FMOptions::from_options(&merged)?; + } + _ => { + crate::table::sorted_global_index_build_builder::validation::index_key_type( + kind, field, + )?; + crate::table::sorted_global_index_options::SortedIndexWriteOptions::from_options( + kind, &merged, + )?; + schema.core_options().write_buffer_size()?; + } + } + result.push(Definition { + field: field.clone(), + index_type: kind.into(), + options: configured, + }); + } + Ok(result) +} + +fn invalid(error: impl std::fmt::Display) -> Error { + Error::ConfigInvalid { + message: format!("Invalid primary-key index options: {error}"), + } +} diff --git a/crates/paimon/src/table/pk_sorted_index_scan.rs b/crates/paimon/src/table/pk_sorted_index_scan.rs index 75a17ab28..a4c56bfd2 100644 --- a/crates/paimon/src/table/pk_sorted_index_scan.rs +++ b/crates/paimon/src/table/pk_sorted_index_scan.rs @@ -86,50 +86,9 @@ fn definition_options(options: &Options, column: &str, family: &str) -> Result = - serde_json::from_str(raw).map_err(|_| { - config_invalid(format!( - "{option_key} must be a JSON object of option key-value pairs" - )) - })?; - let prefix = format!("{family}-index."); - for (key, value) in parsed { - if key.trim().is_empty() { - return Err(config_invalid(format!( - "{option_key} contains an empty option key" - ))); - } - let value = match value { - serde_json::Value::String(value) => value, - serde_json::Value::Number(_) | serde_json::Value::Bool(_) => value.to_string(), - _ => { - return Err(config_invalid(format!( - "{option_key} contains an invalid option value for {key}" - ))) - } - }; - let qualified = if key.starts_with(&prefix) || key.starts_with("fields.") { - key - } else { - format!("{prefix}{key}") - }; - if resolved - .get(&qualified) - .is_some_and(|previous| previous != &value) - { - return Err(config_invalid(format!( - "{option_key} defines conflicting values for {qualified}" - ))); - } - resolved.insert(qualified, value); - } + resolved.extend(super::pk_index_options::field_options( + options, column, family, family, + )?); Ok(resolved) } diff --git a/crates/paimon/src/table/pk_vector_search_params.rs b/crates/paimon/src/table/pk_vector_search_params.rs index 9058853aa..1a158fc8e 100644 --- a/crates/paimon/src/table/pk_vector_search_params.rs +++ b/crates/paimon/src/table/pk_vector_search_params.rs @@ -194,6 +194,12 @@ fn pk_vector_query_dimension( } } let mut table_dimension_options = table_options.clone(); + let declared_options = super::pk_index_options::field_options( + table_options, + vector_field.name(), + "vector", + index_type, + )?; if let Some(value) = table_options.get("dimension") { if !table_options.contains_key(&format!("{index_type}.dimension")) && !table_options.contains_key(&format!("{index_type}.index.dimension")) @@ -203,8 +209,9 @@ fn pk_vector_query_dimension( } } Ok(Some( - VindexVectorIndexOptions::new( + VindexVectorIndexOptions::new_with_defaults( &table_dimension_options, + &declared_options, &dimension_options, index_type, vector_field, diff --git a/crates/paimon/src/table/pk_vector_search_params/tests.rs b/crates/paimon/src/table/pk_vector_search_params/tests.rs index f01a42740..d5cf66789 100644 --- a/crates/paimon/src/table/pk_vector_search_params/tests.rs +++ b/crates/paimon/src/table/pk_vector_search_params/tests.rs @@ -73,3 +73,42 @@ fn raw_query_dimensions_remain_supported_with_java_build_option_mapping() { ); } } + +#[test] +fn declared_array_dimension_matches_writes_and_preserves_query_override() { + let field = DataField::new( + 1, + "embedding".into(), + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ); + for index_type in ["ivf-flat", "ivf-pq", "diskann"] { + let table = HashMap::from([ + ("dimension".into(), "8".into()), + ("fields.embedding.index.dimension".into(), "32".into()), + ( + "fields.embedding.pk-vector.index.options".into(), + r#"{"dimension":4}"#.into(), + ), + ]); + let declared = crate::table::pk_index_options::field_options( + &table, + "embedding", + "vector", + index_type, + ) + .unwrap(); + let write_dimension = VindexVectorIndexOptions::new(&table, &declared, index_type, &field) + .unwrap() + .dimension(); + assert_eq!(write_dimension, 4); + assert_eq!( + pk_vector_query_dimension(&table, &HashMap::new(), index_type, &field).unwrap(), + Some(write_dimension) + ); + let query = HashMap::from([("dimension".into(), "2".into())]); + assert_eq!( + pk_vector_query_dimension(&table, &query, index_type, &field).unwrap(), + Some(2) + ); + } +} diff --git a/crates/paimon/src/table/sorted_global_index_build_builder.rs b/crates/paimon/src/table/sorted_global_index_build_builder.rs index bd386d170..c581fa41f 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -mod extraction; +pub(super) mod extraction; mod planning; -mod sort; -mod validation; +pub(super) mod sort; +pub(super) mod validation; mod writer; #[cfg(test)] @@ -53,7 +53,10 @@ enum GlobalIndexWriteOptions { FM(FMWriteOptions), } -fn make_index_key_codec(index_type: &str, data_type: &DataType) -> (KeyComparator, SerializeKeyFn) { +pub(super) fn make_index_key_codec( + index_type: &str, + data_type: &DataType, +) -> (KeyComparator, SerializeKeyFn) { match index_type { BTREE_GLOBAL_INDEX_TYPE => (make_key_comparator(data_type), serialize_datum), BITMAP_GLOBAL_INDEX_TYPE => ( diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/extraction.rs b/crates/paimon/src/table/sorted_global_index_build_builder/extraction.rs index 64314fc2c..a148127be 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/extraction.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/extraction.rs @@ -256,7 +256,7 @@ pub(super) fn build_read_splits_for_shard( } #[allow(clippy::too_many_arguments)] -fn emit_scalar_rows( +pub(in crate::table) fn emit_scalar_rows( batches: &[RecordBatch], index_column: &str, data_type: &DataType, @@ -331,7 +331,7 @@ fn emit_scalar_rows( } #[allow(clippy::too_many_arguments)] -fn emit_multivalue_rows( +pub(in crate::table) fn emit_multivalue_rows( batches: &[RecordBatch], index_column: &str, element_type: &DataType, diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/sort.rs b/crates/paimon/src/table/sorted_global_index_build_builder/sort.rs index dee452556..92b14c888 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/sort.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/sort.rs @@ -28,7 +28,7 @@ use tokio::io::AsyncReadExt; /// One run per binary level bounds open files during merging to two inputs. /// Temporary files are private scratch state and are removed on failure or cancellation. -pub(super) struct IndexSorter { +pub(in crate::table) struct IndexSorter { rows: Vec, bytes: usize, budget: usize, @@ -37,7 +37,7 @@ pub(super) struct IndexSorter { } impl IndexSorter { - pub(super) fn new(budget: usize, cmp: KeyComparator) -> Self { + pub(in crate::table) fn new(budget: usize, cmp: KeyComparator) -> Self { Self { rows: Vec::new(), bytes: 0, @@ -47,7 +47,7 @@ impl IndexSorter { } } - pub(super) fn push(&mut self, row: SortedIndexKeyRow) -> Result<()> { + pub(in crate::table) fn push(&mut self, row: SortedIndexKeyRow) -> Result<()> { // Charge spare Vec slots as well as the actual serialized-key allocation. // The unstable sort uses no additional row-sized scratch allocation. let bytes = row @@ -109,7 +109,7 @@ impl IndexSorter { } } - pub(super) fn finish(mut self) -> Result { + pub(in crate::table) fn finish(mut self) -> Result { if self.runs.is_empty() { sort_index_rows(&mut self.rows, &self.cmp)?; return Ok(SortedRows::Memory(self.rows.into_iter())); @@ -131,7 +131,7 @@ impl IndexSorter { } } -pub(super) enum SortedRows { +pub(in crate::table) enum SortedRows { Memory(std::vec::IntoIter), Disk { _run: NamedTempFile, @@ -140,7 +140,7 @@ pub(super) enum SortedRows { } impl SortedRows { - pub(super) async fn next(&mut self) -> Result> { + pub(in crate::table) async fn next(&mut self) -> Result> { match self { Self::Memory(rows) => Ok(rows.next()), Self::Disk { reader, .. } => { @@ -208,7 +208,7 @@ fn read_row(input: &mut impl Read) -> Result> { Ok(Some((key, i64::from_le_bytes(row_id)))) } -pub(super) fn compare_rows( +pub(in crate::table) fn compare_rows( left: &SortedIndexKeyRow, right: &SortedIndexKeyRow, cmp: &DynKeyComparator<'_>, diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/validation.rs b/crates/paimon/src/table/sorted_global_index_build_builder/validation.rs index ae90987ad..c8a8ed183 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/validation.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/validation.rs @@ -49,7 +49,10 @@ pub(super) fn validate_btree_field(field: &DataField) -> Result<()> { Ok(()) } -pub(super) fn index_key_type<'a>(index_type: &str, field: &'a DataField) -> Result<&'a DataType> { +pub(in crate::table) fn index_key_type<'a>( + index_type: &str, + field: &'a DataField, +) -> Result<&'a DataType> { if index_type == MULTIVALUE_GLOBAL_INDEX_TYPE { let DataType::Array(array_type) = field.data_type() else { return Err(Error::Unsupported { diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index 0f3de339d..fba808f72 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -465,6 +465,7 @@ impl TableCommit { entries: prepared.entries, changelog_entries: prepared.changelog_entries, deletion_vector_updates: prepared.deletion_vector_updates, + index_entries: prepared.index_entries, schema_id: prepared.schema_id, read_snapshot: prepared.read_snapshot, planned_snapshot_id: prepared.snapshot_id, @@ -1303,10 +1304,15 @@ impl TableCommit { CommitEntriesPlan::Compact { entries, changelog_entries, + index_entries, .. } => { return self - .check_recovery_entries(entries, changelog_entries, &[]) + .check_recovery_entries( + entries, + changelog_entries, + index_entries.as_deref().unwrap_or_default(), + ) .await; } }; @@ -1452,7 +1458,7 @@ impl TableCommit { if self.deletion_vectors_enabled && resolved.kind == CommitKind::COMPACT { if let Some(snapshot) = latest_snapshot { - if let Some(indexes) = super::compact::deletion_vectors::rewrite_indexes( + if let Some(mut indexes) = super::compact::deletion_vectors::rewrite_indexes( &self.table, snapshot, &resolved.entries, @@ -1461,6 +1467,11 @@ impl TableCommit { ) .await? { + if let Some(prepared) = &resolved.new_index_manifest_entries { + let is_pk = super::compact::primary_key::indexes::is_source_index; + indexes.retain(|e| !is_pk(e)); + indexes.extend(prepared.iter().filter(|e| is_pk(e)).cloned()); + } resolved.index_manifest_changed = true; resolved.new_index_manifest_entries = Some(indexes); } @@ -1987,6 +1998,7 @@ impl TableCommit { entries, changelog_entries, deletion_vector_updates, + index_entries, schema_id, read_snapshot, planned_snapshot_id, @@ -2169,14 +2181,23 @@ impl TableCommit { ) .await?; } + let new_indexes = super::compact::primary_key::indexes::resolve( + &self.table, + read_snapshot.as_deref(), + latest_snapshot.as_ref().expect("validated latest snapshot"), + base_data_files.as_deref().unwrap_or_default(), + entries, + index_entries.as_deref(), + ) + .await?; Ok(ResolvedCommit { entries: entries.clone(), changelog_entries: changelog_entries.clone(), deletion_vector_updates: deletion_vector_updates.to_vec(), kind: CommitKind::COMPACT, index_manifest_name, - index_manifest_changed: false, - new_index_manifest_entries: None, + index_manifest_changed: new_indexes.is_some(), + new_index_manifest_entries: new_indexes, base_data_files, }) } @@ -4004,6 +4025,7 @@ enum CommitEntriesPlan { changelog_entries: Vec, deletion_vector_updates: Option>, + index_entries: Option>, schema_id: i64, read_snapshot: Option>, planned_snapshot_id: i64, diff --git a/crates/paimon/src/table/table_commit/auto_maintenance.rs b/crates/paimon/src/table/table_commit/auto_maintenance.rs index 19c6a847e..e2719c710 100644 --- a/crates/paimon/src/table/table_commit/auto_maintenance.rs +++ b/crates/paimon/src/table/table_commit/auto_maintenance.rs @@ -345,6 +345,7 @@ impl TableCommit { entries: prepared.entries, changelog_entries: prepared.changelog_entries, deletion_vector_updates: prepared.deletion_vector_updates, + index_entries: prepared.index_entries, schema_id: prepared.schema_id, read_snapshot: prepared.read_snapshot, planned_snapshot_id: prepared.snapshot_id, diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs b/crates/paimon/src/table/vindex_index_build_builder.rs index 3b7210944..e6ed47a7c 100644 --- a/crates/paimon/src/table/vindex_index_build_builder.rs +++ b/crates/paimon/src/table/vindex_index_build_builder.rs @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -mod extraction; +pub(super) mod extraction; mod pipeline; mod planning; pub(super) mod timing; mod validation; -mod writer; +pub(super) mod writer; use planning::plan_vindex_shards; use validation::{checked_i32, find_index_field, validate_vector_field}; diff --git a/crates/paimon/src/table/vindex_index_build_builder/extraction.rs b/crates/paimon/src/table/vindex_index_build_builder/extraction.rs index 49a4351ae..e14fd205c 100644 --- a/crates/paimon/src/table/vindex_index_build_builder/extraction.rs +++ b/crates/paimon/src/table/vindex_index_build_builder/extraction.rs @@ -48,12 +48,12 @@ pub(super) fn data_split_for_shard_ranges( .build() } -pub(super) struct ValidatedVectorBatch { - pub(super) values: ScalarBuffer, - pub(super) row_ids: ScalarBuffer, +pub(in crate::table) struct ValidatedVectorBatch { + pub(in crate::table) values: ScalarBuffer, + pub(in crate::table) row_ids: ScalarBuffer, /// Number of non-null vectors, not the number of source rows. - pub(super) vector_count: usize, - pub(super) source_rows: usize, + pub(in crate::table) vector_count: usize, + pub(in crate::table) source_rows: usize, } impl ValidatedVectorBatch { @@ -76,7 +76,7 @@ pub(super) fn extract_vector_batch( validate_vector_batch_with(batch, index_column, dimension, |_| Ok(())) } -pub(super) fn validate_vector_batch( +pub(in crate::table) fn validate_vector_batch( batch: &RecordBatch, index_column: &str, dimension: usize, diff --git a/crates/paimon/src/table/vindex_index_build_builder/writer.rs b/crates/paimon/src/table/vindex_index_build_builder/writer.rs index 7b77cff4b..4d2294c3f 100644 --- a/crates/paimon/src/table/vindex_index_build_builder/writer.rs +++ b/crates/paimon/src/table/vindex_index_build_builder/writer.rs @@ -417,7 +417,7 @@ fn temporary_vector_file(kind: &str) -> Result { } /// Java samples the compacted non-null vector stream, then adds the original relative IDs. -fn train_and_add_spilled_vectors( +pub(in crate::table) fn train_and_add_spilled_vectors( mut raw_file: std::fs::File, mut id_file: std::fs::File, config: VectorIndexConfig, diff --git a/crates/paimon/tests/pk_compact/dynamic_bucket.rs b/crates/paimon/tests/pk_compact/dynamic_bucket.rs index 4a9b69549..8586bcce6 100644 --- a/crates/paimon/tests/pk_compact/dynamic_bucket.rs +++ b/crates/paimon/tests/pk_compact/dynamic_bucket.rs @@ -449,24 +449,19 @@ async fn dynamic_compaction_rejects_other_index_types_at_plan_and_commit() { .await .unwrap(); let error = builder.new_planner().unwrap().plan().await.unwrap_err(); - assert!( - error - .to_string() - .contains("only supports dynamic-bucket HASH"), - "{error}" - ); + let expected = if index_type == "HASH_WITH_GLOBAL_METADATA" { + "Unsupported primary-key index type" + } else { + "Compaction requires HASH, deletion-vector or source-backed PK indexes" + }; + assert!(error.to_string().contains(expected), "{error}"); let error = builder .new_commit() .unwrap() .commit_compact(results) .await .unwrap_err(); - assert!( - error - .to_string() - .contains("only supports dynamic-bucket HASH"), - "{error}" - ); + assert!(error.to_string().contains(expected), "{error}"); assert_eq!(latest(&table).await.id(), snapshot.id()); } } diff --git a/crates/paimon/tests/pk_compact/indexes.rs b/crates/paimon/tests/pk_compact/indexes.rs new file mode 100644 index 000000000..71fb0835c --- /dev/null +++ b/crates/paimon/tests/pk_compact/indexes.rs @@ -0,0 +1,1119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use paimon::spec::{should_read_pk_index_source, IndexManifest, PrimaryKeyIndexSourceMeta}; +use std::collections::HashSet; + +#[tokio::test] +async fn independently_committed_buckets_rebase_without_conflicting_local_ordinals() { + let table = table(&[("bucket", "2"), ("pk-btree.index.columns", "value")]).await; + write_batch(&table, &make_batch((0..100).collect(), (0..100).collect())).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + assert_eq!(plan.tasks().len(), 2); + let results = execute(&table, &plan).await; + let commit = builder.new_commit().unwrap(); + commit + .commit_compact(vec![results[0].clone()]) + .await + .unwrap(); + let first = indexes(&table).await; + commit + .commit_compact(vec![results[1].clone()]) + .await + .unwrap(); + let active = indexes(&table).await; + assert_eq!(active.len(), 2); + assert!( + active.contains(&first[0]), + "concurrent unrelated bucket index must survive" + ); + assert_ne!(active[0].bucket, active[1].bucket); + assert!(active.iter().all(|e| e + .index_file + .global_index_meta + .as_ref() + .unwrap() + .row_range_start + == 0)); + for value in [0, 42, 99] { + assert_eq!(filtered_rows(&table, value).await, vec![(value, value)]); + } + commit + .commit_compact(vec![results[0].clone()]) + .await + .unwrap(); + commit + .commit_compact(vec![results[1].clone()]) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 3); +} + +#[tokio::test] +async fn concurrently_added_compacted_source_requires_a_new_index_build() { + let table = table(&[("pk-btree.index.columns", "value")]).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer + .write_arrow_batch(&make_batch(vec![256], vec![2560])) + .await + .unwrap(); + let mut messages = writer.prepare_commit().await.unwrap(); + for message in &mut messages { + for file in &mut message.new_files { + file.level = results[0].files()[0].level; + file.file_source = Some(1); + } + } + write.new_commit().commit(messages).await.unwrap(); + let error = builder + .new_commit() + .unwrap() + .commit_compact(results.clone()) + .await + .unwrap_err(); + assert!( + error.to_string().contains("complete compacted level"), + "{error}" + ); + assert_eq!(latest(&table).await.id(), 2); + assert!(indexes(&table).await.is_empty()); + for entry in results[0].index_entries().unwrap() { + assert!(table + .file_io() + .exists(&index_path(&table, entry)) + .await + .unwrap()); + } +} + +#[tokio::test] +async fn removed_declaration_still_retires_indexes_whose_sources_are_rewritten() { + let table = table(&[("pk-btree.index.columns", "value")]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + compact(&table).await; + let old = indexes(&table).await[0].clone(); + let schema = table + .schema() + .apply_changes(vec![paimon::spec::SchemaChange::RemoveOption { + key: "pk-btree.index.columns".into(), + }]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &schema).await; + let current = table.copy_with_resolved_schema(schema, "main").unwrap(); + write_batch(¤t, &make_batch(vec![1, 2], vec![11, 22])).await; + compact(¤t).await; + assert!(indexes(¤t).await.is_empty()); + assert!(table + .file_io() + .exists(&index_path(&table, &old)) + .await + .unwrap()); + assert_eq!(rows(¤t).await, vec![(1, 11), (2, 22)]); +} + +#[tokio::test] +async fn failed_index_stat_cleans_current_prior_indexes_and_private_data_outputs() { + use std::collections::{BTreeSet, HashMap}; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }; + #[derive(Debug)] + struct FailIndexStat { + root: String, + op: opendal::Operator, + calls: Mutex>, + failed: AtomicBool, + } + #[async_trait::async_trait] + impl paimon::io::FileIOProvider for FailIndexStat { + async fn create(&self, path: &str) -> paimon::Result<(opendal::Operator, String)> { + if path.contains("bitmap-pk-index-") { + let mut calls = self.calls.lock().unwrap(); + let count = calls.entry(path.into()).or_default(); + *count += 1; + if *count == 2 { + self.failed.store(true, Ordering::SeqCst); + return Err(paimon::Error::UnexpectedError { + message: "injected PK payload stat failure".into(), + source: None, + }); + } + } + Ok(( + self.op.clone(), + path.strip_prefix(&self.root) + .unwrap() + .trim_start_matches('/') + .into(), + )) + } + } + fn files(root: &std::path::Path) -> BTreeSet { + fn visit(path: &std::path::Path, output: &mut BTreeSet) { + for entry in std::fs::read_dir(path).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(&path, output); + } else { + output.insert(path); + } + } + } + let mut output = BTreeSet::new(); + visit(root, &mut output); + output + } + let dir = tempfile::tempdir().unwrap(); + let location = dir.path().to_str().unwrap().to_string(); + let mut config = opendal_service_fs::FsConfig::default(); + config.root = Some(location.clone()); + let provider = Arc::new(FailIndexStat { + root: location.clone(), + op: opendal::Operator::from_config(config).unwrap(), + calls: Mutex::new(HashMap::new()), + failed: AtomicBool::new(false), + }); + let io = paimon::io::FileIOBuilder::new("file") + .with_provider(provider.clone()) + .build() + .unwrap(); + let seed_io = paimon::io::FileIOBuilder::new("file").build().unwrap(); + let table = Table::new( + seed_io.clone(), + paimon::catalog::Identifier::new("default", "cleanup"), + location.clone(), + pk_schema(&[ + ("write-only", "true"), + ("pk-btree.index.columns", "id"), + ("pk-bitmap.index.columns", "value"), + ]), + None, + ); + setup_dirs(&seed_io, &location).await; + persist_table_schema(&seed_io, &location, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let table = Table::new( + io, + table.identifier().clone(), + location, + table.schema().clone(), + None, + ); + let before = files(dir.path()); + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let error = builder + .new_executor() + .unwrap() + .execute(&plan.tasks()[0]) + .await + .expect_err("stat must fail"); + assert!( + error + .to_string() + .contains("injected PK payload stat failure"), + "{error}" + ); + assert!(provider.failed.load(Ordering::SeqCst)); + assert_eq!( + files(dir.path()), + before, + "all unpublished data and index outputs must be removed" + ); + assert_eq!(latest(&table).await.id(), 1); + assert_eq!(rows(&table).await, vec![(1, 10), (2, 20)]); +} + +#[tokio::test] +async fn retained_history_publishes_new_pk_index_and_deletion_vectors_together() { + let table = table(&[ + ("pk-btree.index.columns", "value"), + ("deletion-vectors.enabled", "true"), + ("deletion-vectors.merge-on-read", "false"), + ]) + .await; + write_batch( + &table, + &make_batch((0..1000).collect(), (0..1000).collect()), + ) + .await; + compact(&table).await; + let original = indexes(&table).await[0].clone(); + let historical = table.copy_with_pinned_snapshot(Some(&latest(&table).await)); + write_batch(&table, &make_batch(vec![42], vec![42000])).await; + compact(&table).await; + let active = indexes(&table).await; + assert_eq!( + active + .iter() + .filter(|e| e.index_file.index_type == "btree") + .count(), + 2 + ); + assert!(active.contains(&original)); + assert!(active + .iter() + .any(|e| e.index_file.index_type == "DELETION_VECTORS")); + assert_eq!(filtered_rows(&table, 42).await, vec![]); + assert_eq!(filtered_rows(&table, 42000).await, vec![(42, 42000)]); + assert_eq!(filtered_rows(&historical, 42).await, vec![(42, 42)]); +} + +#[tokio::test] +async fn missing_payload_and_tampered_source_cannot_publish_and_do_not_abort_outputs() { + for case in ["missing", "omitted", "source"] { + let table = table(&[("pk-btree.index.columns", "value")]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + let path = index_path(&table, &results[0].index_entries().unwrap()[0]); + let results = if case == "missing" { + table.file_io().delete_file(&path).await.unwrap(); + results + } else { + let mut json = serde_json::to_value(&results).unwrap(); + if case == "omitted" { + json[0]["index_entries"] = serde_json::json!([]); + } else { + let source = PrimaryKeyIndexSourceMeta::new( + results[0].files()[0].level, + vec![paimon::spec::PrimaryKeyIndexSourceFile::new( + "wrong-source.parquet".into(), + 2, + ) + .unwrap()], + ) + .unwrap(); + json[0]["index_entries"][0]["_GLOBAL_INDEX"]["_SOURCE_META"] = + serde_json::to_value(source.serialize().unwrap()).unwrap(); + } + serde_json::from_value(json).unwrap() + }; + let error = builder + .new_commit() + .unwrap() + .commit_compact(results.clone()) + .await + .unwrap_err(); + let expected = match case { + "missing" => "does not exist", + "omitted" => "Missing PK index output", + _ => "complete compacted level", + }; + assert!(error.to_string().contains(expected), "{error}"); + assert_eq!(latest(&table).await.id(), 1); + for file in results[0].files() { + assert!(table + .file_io() + .exists(&format!("{}/bucket-0/{}", table.location(), file.file_name)) + .await + .unwrap()); + } + assert_eq!( + table.file_io().exists(&path).await.unwrap(), + case != "missing" + ); + } +} + +#[tokio::test] +async fn malformed_declarations_fail_before_automatic_data_publication() { + for (key, value) in [ + ("pk-btree.index.columns", "value,value"), + ("pk-bitmap.index.columns", "missing"), + ( + "fields.value.pk-btree.index.options", + r#"{"block-size":[]}"#, + ), + ( + "fields.value.pk-btree.index.options", + r#"{"btree-index.block-size":"1kb","block-size":"2kb"}"#, + ), + ] { + let original = table(&[("pk-btree.index.columns", "value"), (key, value)]).await; + let table = original + .copy_with_resolved_schema( + original + .schema() + .copy_with_options(std::collections::HashMap::from([( + "write-only".into(), + "false".into(), + )])), + "main", + ) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), table.schema()).await; + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer + .write_arrow_batch(&make_batch(vec![1], vec![10])) + .await + .unwrap(); + let error = write + .new_commit() + .commit(writer.prepare_commit().await.unwrap()) + .await + .unwrap_err(); + assert!(error.to_string().contains("index"), "{error}"); + assert_eq!( + table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(), + None + ); + } +} + +#[tokio::test] +async fn all_native_families_build_with_physical_null_gaps_and_field_options() { + use arrow_array::{ + builder::{Float32Builder, Int32Builder, ListBuilder}, + ArrayRef, + }; + use paimon::spec::{ArrayType, FloatType}; + use std::sync::Arc; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .column( + "tags", + DataType::Array(ArrayType::new(DataType::Int(IntType::new()))), + ) + .column("body", DataType::VarChar(VarCharType::string_type())) + .column("text", DataType::VarChar(VarCharType::string_type())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .primary_key(["id"]) + .option("bucket", "1") + .option("deletion-vectors.enabled", "true") + .option("deletion-vectors.merge-on-read", "false") + .option("pk-btree.index.columns", "value") + .option("pk-multivalue.index.columns", "tags") + .option("pk-fm.index.columns", "body") + .option("pk-full-text.index.columns", "text") + .option("pk-vector.index.columns", "embedding") + .option("fields.embedding.pk-vector.index.type", "ivf-flat") + .option("fields.embedding.pk-vector.distance.metric", "l2") + .option( + "fields.embedding.pk-vector.index.options", + r#"{"dimension":2,"nlist":1}"#, + ) + .option( + "fields.text.pk-full-text.index.options", + r#"{"lower-case":false}"#, + ) + .option("global-index.search-mode", "fast") + .option("target-file-row-num", "1") + .build() + .unwrap(); + let table = table_with_schema(TableSchema::new(0, &schema)).await; + let mut tags = ListBuilder::new(Int32Builder::new()).with_field(Arc::new( + arrow_schema::Field::new("element", arrow_schema::DataType::Int32, true), + )); + let mut vectors = ListBuilder::new(Float32Builder::new()).with_field(Arc::new( + arrow_schema::Field::new("element", arrow_schema::DataType::Float32, true), + )); + for row in 0..3 { + if row == 1 { + tags.append(false); + vectors.append(false); + } else { + tags.values().append_value(row); + tags.values().append_value(row); + tags.append(true); + vectors.values().append_value(row as f32); + vectors.values().append_value(0.); + vectors.append(true); + } + } + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef), + ( + "value", + Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef, + ), + ("tags", Arc::new(tags.finish()) as ArrayRef), + ( + "body", + Arc::new(StringArray::from(vec![Some("alpha"), None, Some("beta")])) as ArrayRef, + ), + ( + "text", + Arc::new(StringArray::from(vec![Some("Needle"), None, Some("hay")])) as ArrayRef, + ), + ("embedding", Arc::new(vectors.finish()) as ArrayRef), + ]) + .unwrap(); + write_batch(&table, &batch).await; + compact(&table).await; + let active = indexes(&table).await; + assert_eq!( + active + .iter() + .map(|e| e.index_file.index_type.as_str()) + .collect::>(), + HashSet::from(["btree", "multivalue", "fm", "full-text", "ivf-flat"]) + ); + for entry in &active { + let source = PrimaryKeyIndexSourceMeta::from_global_index_meta( + entry.index_file.global_index_meta.as_ref().unwrap(), + ) + .unwrap(); + assert_eq!(entry.index_file.row_count, 3); + assert_eq!(source.source_files().len(), 3); + assert!(table + .file_io() + .exists(&index_path(&table, entry)) + .await + .unwrap()); + } + let vector = table + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![2., 0.]) + .with_limit(2) + .execute() + .await + .unwrap(); + let batches: Vec = vector + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let ids = batches + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + assert_eq!( + ids, + vec![3, 1], + "NULL vector must leave a physical ordinal gap" + ); + for (term, expected) in [("Needle", vec![1]), ("needle", vec![])] { + let query = serde_json::json!({"match":{"query":term}}).to_string(); + let text = table + .new_full_text_search_builder() + .with_text_column("text") + .with_query_text(&query) + .with_limit(3) + .execute() + .await + .unwrap(); + let batches: Vec = text + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let ids = batches + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + assert_eq!( + ids, expected, + "field tokenizer options must survive native serialization" + ); + } +} + +#[tokio::test] +async fn vector_definition_changes_replace_old_payloads_atomically() { + use arrow_array::{ + builder::{Float32Builder, ListBuilder}, + ArrayRef, + }; + use paimon::spec::{ArrayType, FloatType}; + use std::sync::Arc; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .primary_key(["id"]) + .option("bucket", "1") + .option("pk-vector.index.columns", "embedding") + .option("deletion-vectors.enabled", "true") + .option("deletion-vectors.merge-on-read", "false") + .option("fields.embedding.pk-vector.index.type", "ivf-flat") + .option("fields.embedding.pk-vector.distance.metric", "l2") + .option( + "fields.embedding.pk-vector.index.options", + r#"{"dimension":2,"nlist":1}"#, + ) + .build() + .unwrap(); + for (kind, metric) in [("ivf-flat", "inner_product"), ("ivf-sq", "l2")] { + let table = table_with_schema(TableSchema::new(0, &schema)).await; + let batch = || { + let mut vectors = ListBuilder::new(Float32Builder::new()).with_field(Arc::new( + arrow_schema::Field::new("element", arrow_schema::DataType::Float32, true), + )); + for value in [1., 100.] { + vectors.values().append_slice(&[value, 0.]); + vectors.append(true); + } + RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef), + ("embedding", Arc::new(vectors.finish()) as ArrayRef), + ]) + .unwrap() + }; + write_batch(&table, &batch()).await; + compact(&table).await; + let old = indexes(&table).await; + let evolved = table + .schema() + .apply_changes(vec![ + paimon::spec::SchemaChange::set_option( + "fields.embedding.pk-vector.index.type".into(), + kind.into(), + ), + paimon::spec::SchemaChange::set_option( + "fields.embedding.pk-vector.distance.metric".into(), + metric.into(), + ), + ]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &evolved).await; + let current = table.copy_with_resolved_schema(evolved, "main").unwrap(); + write_batch(¤t, &batch()).await; + compact(¤t).await; + let active = indexes(¤t).await; + assert_eq!(active.len(), 1); + assert_eq!(active[0].index_file.index_type, kind); + assert_ne!(active[0].index_file.file_name, old[0].index_file.file_name); + assert!(table + .file_io() + .exists(&index_path(&table, &old[0])) + .await + .unwrap()); + let result = current + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(1) + .execute() + .await + .unwrap(); + let batches: Vec = result + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let id = batches[0] + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + assert_eq!(id, if metric == "inner_product" { 2 } else { 1 }); + } +} + +#[tokio::test] +async fn automatic_compaction_maintains_indexes_and_recovers_callback_failure() { + use paimon::table::{CommitCallback, CommitCallbackContext}; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + struct FailOnce(AtomicBool); + #[async_trait::async_trait] + impl CommitCallback for FailOnce { + async fn call(&self, context: &CommitCallbackContext) -> paimon::Result<()> { + if context.snapshot.commit_kind() == &CommitKind::COMPACT + && !self.0.swap(true, Ordering::SeqCst) + { + return Err(paimon::Error::UnexpectedError { + message: "pk-index callback failure".into(), + source: None, + }); + } + Ok(()) + } + } + let original = table(&[ + ("pk-bitmap.index.columns", "value"), + ("compaction.force-up-level-0", "true"), + ]) + .await; + let table = original + .copy_with_resolved_schema( + original + .schema() + .copy_with_options(std::collections::HashMap::from([( + "write-only".into(), + "false".into(), + )])), + "main", + ) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), table.schema()).await; + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer + .write_arrow_batch(&make_batch(vec![1, 2], vec![10, 20])) + .await + .unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + let mut commit = write.new_commit(); + commit + .add_commit_callback(Arc::new(FailOnce(AtomicBool::new(false)))) + .unwrap(); + assert!(commit + .commit_with_identifier(messages.clone(), 7) + .await + .unwrap_err() + .to_string() + .contains("pk-index callback failure")); + let snapshot = latest(&table).await; + assert_eq!(snapshot.commit_kind(), &CommitKind::COMPACT); + assert_eq!(snapshot.id(), 2); + let active = indexes(&table).await; + assert_eq!(active.len(), 1); + commit + .filter_and_commit_with_identifier(messages, 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 2); + assert_eq!(indexes(&table).await, active); + assert_eq!(filtered_rows(&table, 20).await, vec![(2, 20)]); +} + +async fn latest(table: &Table) -> paimon::spec::Snapshot { + table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap() +} + +async fn indexes(table: &Table) -> Vec { + let snapshot = latest(table).await; + match snapshot.index_manifest() { + Some(name) => IndexManifest::read( + table.file_io(), + &table.snapshot_manager().manifest_path(name), + ) + .await + .unwrap(), + None => vec![], + } +} + +async fn compact(table: &Table) -> Vec { + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + assert!(!plan.tasks().is_empty()); + let results = execute(table, &plan).await; + builder + .new_commit() + .unwrap() + .commit_compact(results.clone()) + .await + .unwrap(); + results +} + +fn index_path(table: &Table, entry: &paimon::spec::IndexManifestEntry) -> String { + if let Some(path) = &entry.index_file.external_path { + return path.clone(); + } + if table.schema().core_options().index_file_in_data_file_dir() { + format!( + "{}/bucket-{}/{}", + table.location(), + entry.bucket, + entry.index_file.file_name + ) + } else { + format!("{}/index/{}", table.location(), entry.index_file.file_name) + } +} + +async fn filtered_rows(table: &Table, value: i32) -> Vec<(i32, i32)> { + let mut builder = table.new_read_builder(); + builder.with_filter( + PredicateBuilder::new(table.schema().fields()) + .equal("value", Datum::Int(value)) + .unwrap(), + ); + let plan = builder.new_scan().plan().await.unwrap(); + let batches: Vec = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut rows = Vec::new(); + for batch in batches { + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + rows.extend((0..batch.num_rows()).map(|row| (ids.value(row), values.value(row)))); + } + rows.sort_unstable(); + rows +} + +#[tokio::test] +async fn indexes_preserve_nulls_tombstones_file_order_and_incremental_history() { + for family in ["btree", "bitmap"] { + for in_data in ["true", "false"] { + let key = format!("pk-{family}.index.columns"); + let table = table(&[ + (&key, "value"), + ("index-file-in-data-file-dir", in_data), + ("target-file-row-num", "13"), + ("read.batch-size", "7"), + ("write-buffer-size", "1kb"), + ]) + .await; + let batch = RecordBatch::try_from_iter([ + ( + "id", + std::sync::Arc::new(Int32Array::from_iter_values(0..150)) + as arrow_array::ArrayRef, + ), + ( + "value", + std::sync::Arc::new(Int32Array::from( + (0..150).map(|v| (v != 4).then_some(v)).collect::>(), + )) as arrow_array::ArrayRef, + ), + ]) + .unwrap(); + write_batch(&table, &batch).await; + let first = compact(&table).await; + assert!(first[0].files().len() > 1); + let historical = table.copy_with_pinned_snapshot(Some(&latest(&table).await)); + let original = indexes(&table).await; + let source = PrimaryKeyIndexSourceMeta::from_global_index_meta( + original[0].index_file.global_index_meta.as_ref().unwrap(), + ) + .unwrap(); + assert!(source + .source_files() + .windows(2) + .all(|p| p[0].file_name() < p[1].file_name())); + for value in [0, 13, 100, 149] { + assert_eq!(filtered_rows(&table, value).await, vec![(value, value)]); + } + write_batch( + &table, + &make_batch_with_kinds(vec![13, 100], vec![1300, 100], vec![0, 3]), + ) + .await; + compact(&table).await; + assert_eq!(filtered_rows(&table, 13).await, vec![]); + assert_eq!(filtered_rows(&table, 100).await, vec![]); + assert_eq!(filtered_rows(&table, 1300).await, vec![(13, 1300)]); + assert_eq!(filtered_rows(&historical, 13).await, vec![(13, 13)]); + let active = indexes(&table).await; + assert!( + active.iter().any(|entry| entry == &original[0]), + "unselected higher-level source and index must survive" + ); + assert_eq!(active.len(), 2); + assert!(active.iter().all(|e| e.index_file.index_type == family)); + for entry in original { + assert!(table + .file_io() + .exists(&index_path(&table, &entry)) + .await + .unwrap()); + } + } + } +} + +#[tokio::test] +async fn legacy_results_require_reexecution_but_completed_results_recover_without_sources() { + let table = table(&[("pk-btree.index.columns", "value")]).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + let builder = table + .new_compact_builder() + .with_commit_user("pk-recovery") + .unwrap(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + let mut legacy = serde_json::to_value(&results).unwrap(); + for result in legacy.as_array_mut().unwrap() { + result.as_object_mut().unwrap().remove("index_entries"); + } + let legacy: Vec = serde_json::from_value(legacy).unwrap(); + let commit = builder.new_commit().unwrap(); + let error = commit.commit_compact(legacy.clone()).await.unwrap_err(); + assert!( + error.to_string().contains("missing index maintenance"), + "{error}" + ); + assert_eq!(latest(&table).await.id(), 1); + commit.commit_compact(results.clone()).await.unwrap(); + let before = indexes(&table).await; + table + .file_io() + .delete_file(&table.snapshot_manager().snapshot_path(1)) + .await + .unwrap(); + commit.commit_compact(legacy).await.unwrap(); + commit.commit_compact(results).await.unwrap(); + assert_eq!(latest(&table).await.id(), 2); + assert_eq!(indexes(&table).await, before); +} + +#[tokio::test] +async fn concurrent_l0_append_rebases_but_changed_index_membership_is_rejected() { + for conflict in [false, true] { + let table = table(&[("pk-btree.index.columns", "value")]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + if conflict { + // Index-only concurrent publication for the same bucket/field/level. + let prepared = results[0].index_entries().unwrap()[0].clone(); + let mut message = + paimon::table::CommitMessage::new(prepared.partition, prepared.bucket, vec![]); + message.new_index_files.push(prepared.index_file); + table + .new_write_builder() + .new_commit() + .commit(vec![message]) + .await + .unwrap(); + } else { + write_batch(&table, &make_batch(vec![1, 3], vec![11, 30])).await; + } + let outcome = builder + .new_commit() + .unwrap() + .commit_compact(results.clone()) + .await; + if conflict { + let error = outcome.unwrap_err(); + assert!( + error.to_string().contains("snapshot was replaced") + || error.to_string().contains("index changed concurrently"), + "{error}" + ); + for entry in results[0].index_entries().unwrap() { + assert!( + table + .file_io() + .exists(&index_path(&table, entry)) + .await + .unwrap(), + "failed commit must retain prepared output" + ); + } + } else { + outcome.unwrap(); + assert_eq!(filtered_rows(&table, 10).await, vec![]); + assert_eq!(filtered_rows(&table, 11).await, vec![(1, 11)]); + assert_eq!(rows(&table).await, vec![(1, 11), (2, 20), (3, 30)]); + assert_eq!(indexes(&table).await.len(), 1); + } + } +} + +#[tokio::test] +async fn compaction_retires_old_indexes_and_preserves_hash_and_deletion_vectors() { + for mode in ["fixed", "dynamic", "dv"] { + let mut options = vec![("pk-bitmap.index.columns", "value")]; + if mode == "dynamic" { + options.push(("bucket", "-1")); + } + if mode == "dv" { + options.extend([ + ("deletion-vectors.enabled", "true"), + ("deletion-vectors.merge-on-read", "false"), + ]); + } + let table = table(&options).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + compact(&table).await; + let old = indexes(&table).await; + let old_pk = old + .iter() + .find(|e| e.index_file.index_type == "bitmap") + .unwrap() + .clone(); + let old_snapshot = table.copy_with_pinned_snapshot(Some(&latest(&table).await)); + write_batch(&table, &make_batch(vec![1, 2], vec![11, 22])).await; + compact(&table).await; + let active = indexes(&table).await; + assert!(!active + .iter() + .any(|e| e.index_file.file_name == old_pk.index_file.file_name)); + assert_eq!( + active + .iter() + .filter(|e| e.index_file.index_type == "bitmap") + .count(), + 1 + ); + if mode == "dynamic" { + assert!(active.iter().any(|e| e.index_file.index_type == "HASH")); + } + assert_eq!(rows(&table).await, vec![(1, 11), (2, 22)]); + assert_eq!(rows(&old_snapshot).await, vec![(1, 10), (2, 20)]); + assert!(table + .file_io() + .exists(&index_path(&table, &old_pk)) + .await + .unwrap()); + } +} + +#[tokio::test] +async fn compact_atomically_maintains_source_backed_primary_key_indexes() { + let table = table(&[ + ("pk-btree.index.columns", "value"), + ("target-file-row-num", "2"), + ]) + .await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + write_batch(&table, &make_batch(vec![2, 3], vec![22, 30])).await; + let builder = table + .new_compact_builder() + .with_commit_user("pk-index") + .unwrap(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let executor = builder.new_executor().unwrap(); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + let commit = builder.new_commit().unwrap(); + commit.commit_compact(results.clone()).await.unwrap(); + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + assert_eq!(*snapshot.commit_kind(), CommitKind::COMPACT); + assert_eq!( + snapshot.id(), + 3, + "data and indexes must publish in the same transaction" + ); + let indexes = IndexManifest::read( + table.file_io(), + &table.snapshot_manager().manifest_path( + snapshot + .index_manifest() + .expect("PK compaction must publish the declared index"), + ), + ) + .await + .unwrap(); + assert_eq!(indexes.len(), 1); + assert_eq!(indexes[0].index_file.index_type, "btree"); + let meta = indexes[0].index_file.global_index_meta.as_ref().unwrap(); + let source = PrimaryKeyIndexSourceMeta::from_global_index_meta(meta).unwrap(); + let data = table + .new_read_builder() + .new_scan() + .with_scan_all_files() + .plan() + .await + .unwrap(); + let active = data + .splits() + .iter() + .flat_map(|split| split.data_files()) + .filter(|file| should_read_pk_index_source(file)) + .collect::>(); + assert_eq!( + source + .source_files() + .iter() + .map(|file| (file.file_name(), file.row_count())) + .collect::>(), + active + .iter() + .map(|file| (file.file_name.as_str(), file.row_count)) + .collect() + ); + assert_eq!(meta.row_range_start, 0); + assert_eq!(meta.row_range_end, 2); + assert_eq!(rows(&table).await, vec![(1, 10), (2, 22), (3, 30)]); + commit.commit_compact(results).await.unwrap(); + assert_eq!( + table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap() + .id(), + 3 + ); +} diff --git a/crates/paimon/tests/pk_compact_test.rs b/crates/paimon/tests/pk_compact_test.rs index 6f7edeb15..3ca49c8ec 100644 --- a/crates/paimon/tests/pk_compact_test.rs +++ b/crates/paimon/tests/pk_compact_test.rs @@ -17,6 +17,9 @@ mod common; +#[path = "pk_compact/indexes.rs"] +mod index_tests; + #[path = "pk_compact/merge_engines.rs"] mod engine_tests; diff --git a/docs/src/compaction.md b/docs/src/compaction.md index 11c1637bb..bc3987e7c 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -252,6 +252,46 @@ snapshot expiration and storage reclamation are separate maintenance work. ## Primary-key compaction +Primary-key compaction also maintains declared source-backed indexes, using +Java's primary-key options: + +| Option | Meaning | +| --- | --- | +| `pk-btree.index.columns` | Comma-separated BTree columns. | +| `pk-bitmap.index.columns` | Comma-separated bitmap columns. | +| `pk-multivalue.index.columns` | Comma-separated array columns. | +| `pk-fm.index.columns` | Comma-separated FM string columns. | +| `pk-full-text.index.columns` | Comma-separated full-text string columns. | +| `pk-vector.index.columns` | Vector column, subject to the existing PK vector constraints. | +| `fields..pk-.index.options` | JSON object of native writer options. | +| `fields..pk-vector.index.type` | Vector algorithm, such as `ivf-flat`. | +| `fields..pk-vector.distance.metric` | Vector metric: `l2`, `cosine`, or `inner_product`. | + +Declarations are unset by default. With `write-only=false`, the normal automatic +compaction round maintains them. Explicit compaction also maintains them with +`write-only=true`, for dedicated compactors. This uses the same native writers +as global indexes. PK vector and full-text search retain their existing +deletion-vector and search-mode requirements. + +Each payload covers the complete compacted file set for one partition, bucket, +positive level and column. Source files are sorted by name; payload ordinals +concatenate their physical rows, preserving NULL and tombstone positions. +Unchanged coverage is retained. Changed coverage and incompatible vector +definitions replace the previous references in the same COMPACT snapshot as +data, changelogs and deletion vectors. Old payload files remain for historical +snapshots. L0 files remain outside these indexes and use the existing read path. + +Commit validates the complete source set and captured index membership before +publishing. Concurrent L0 appends and unrelated index changes can be rebased; +changes to a maintained unit require a new plan. A failed build cleans its +private outputs. A failed or uncertain commit retains prepared outputs for +retry with the same results and commit user. Older serialized results lacking +index maintenance must be re-executed before first publication when indexes +need maintenance; already published results remain recoverable without their +original source snapshot. +As in Java, declared indexed columns cannot be renamed, dropped or have their +type changed while the declaration remains in effect. + The same API compacts primary-key tables with `deduplicate`, `first-row`, `partial-update` or `aggregation` merge engines. Changelog producers `none`, `input` and `lookup` are supported; `first-row` supports `none` or `lookup`. @@ -537,7 +577,8 @@ Ordered bucketed append, append incremental clustering, primary-key clustering override (`pk-clustering-override`) and record-level expiration (`record-level.expire-time`) are also unsupported. Snapshot index manifests may contain dynamic-bucket HASH indexes and primary-key -deletion vectors, or deletion vectors and global indexes in Data Evolution mode. +deletion vectors and source-backed PK indexes, or deletion vectors and global +indexes in Data Evolution mode. Embedded and sidecar file indexes are regenerated according to the captured table options. The automatic behavior also applies to language bindings using the table committer. The manual compaction API and background compaction scheduling From 4b4d1365a09961141a21ddd0cbfa5f32f53252a3 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 13:22:27 +0800 Subject: [PATCH 02/14] refactor(index): share native builds between PK and DE --- .../compact/primary_key/indexes/writer.rs | 439 ++++----------- .../table/full_text_index_build_builder.rs | 304 ++--------- .../src/table/global_index_build_common.rs | 61 --- crates/paimon/src/table/index_build.rs | 181 +++++++ .../src/table/index_build/extraction.rs | 512 ++++++++++++++++++ crates/paimon/src/table/index_build/fields.rs | 94 ++++ .../paimon/src/table/index_build/full_text.rs | 244 +++++++++ crates/paimon/src/table/index_build/scalar.rs | 273 ++++++++++ .../sort.rs | 2 +- crates/paimon/src/table/index_build/tests.rs | 170 ++++++ crates/paimon/src/table/index_build/vector.rs | 367 +++++++++++++ .../table/index_build/vector_extraction.rs | 346 ++++++++++++ .../table/index_build/vector_validation.rs | 102 ++++ crates/paimon/src/table/mod.rs | 1 + crates/paimon/src/table/pk_index_options.rs | 32 +- .../sorted_global_index_build_builder.rs | 62 +-- .../extraction.rs | 511 +---------------- .../tests.rs | 94 +++- .../validation.rs | 107 +--- .../writer.rs | 410 ++------------ .../src/table/vindex_index_build_builder.rs | 4 +- .../vindex_index_build_builder/extraction.rs | 328 +---------- .../vindex_index_build_builder/validation.rs | 91 +--- .../vindex_index_build_builder/writer.rs | 331 ++--------- 24 files changed, 2679 insertions(+), 2387 deletions(-) create mode 100644 crates/paimon/src/table/index_build.rs create mode 100644 crates/paimon/src/table/index_build/extraction.rs create mode 100644 crates/paimon/src/table/index_build/fields.rs create mode 100644 crates/paimon/src/table/index_build/full_text.rs create mode 100644 crates/paimon/src/table/index_build/scalar.rs rename crates/paimon/src/table/{sorted_global_index_build_builder => index_build}/sort.rs (99%) create mode 100644 crates/paimon/src/table/index_build/tests.rs create mode 100644 crates/paimon/src/table/index_build/vector.rs create mode 100644 crates/paimon/src/table/index_build/vector_extraction.rs create mode 100644 crates/paimon/src/table/index_build/vector_validation.rs diff --git a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs index 9fa61b794..92c8324fb 100644 --- a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs +++ b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs @@ -15,54 +15,33 @@ // specific language governing permissions and limitations // under the License. -//! Native writers shared with global indexes, fed physical PK file ordinals. - +//! Adapt a complete PK level to the shared native index build pipeline. use super::super::super::invalid; use crate::resource::ResourceContext; -use crate::spec::{ - extract_datum_from_arrow, DataFileMeta, DataType, Datum, GlobalIndexMeta, IndexFileMeta, - ROW_ID_FIELD_NAME, +use crate::spec::{DataFileMeta, GlobalIndexMeta, IndexFileMeta, ROW_ID_FIELD_NAME}; +use crate::table::index_build::{ + self, full_text, + scalar::{self, ScalarIndexOptions}, + vector, IndexFileWriter, }; -use crate::table::global_index_build_common::copy_local_file_to_output; use crate::table::index_file_path::IndexFileLocation; use crate::table::pk_index_options::Definition; -use crate::table::sorted_global_index_build_builder::{ - extraction::{emit_multivalue_rows, emit_scalar_rows}, - make_index_key_codec, - sort::IndexSorter, - validation::index_key_type, -}; -use crate::table::sorted_global_index_options::SortedIndexWriteOptions; -use crate::table::vindex_index_build_builder::{ - extraction::validate_vector_batch, writer::train_and_add_spilled_vectors, -}; -use crate::table::{DataSplit, Table}; -use crate::{Error, Result}; +use crate::table::{ArrowRecordBatchStream, DataSplit, RowRange, Table}; +use crate::vindex::VindexVectorIndexOptions; +use crate::Result; use arrow_array::{Int64Array, RecordBatch}; use arrow_schema::{DataType as ArrowType, Field, Schema}; +use async_stream::try_stream; use futures::TryStreamExt; use std::collections::BTreeMap; -use std::future::Future; use std::sync::Arc; -use tokio::io::AsyncWriteExt; - -fn native_error(error: impl std::error::Error + Send + Sync + 'static) -> Error { - Error::DataInvalid { - message: "PK index build failed".into(), - source: Some(Box::new(error)), - } -} - -async fn blocking(f: impl FnOnce() -> Result + Send + 'static) -> Result { - tokio::task::spawn_blocking(f).await.map_err(native_error)? -} pub(super) fn native_options( table: &Table, definition: &Definition, ) -> Result> { if crate::vindex::is_vindex_index_type(&definition.index_type) { - return Ok(crate::vindex::VindexVectorIndexOptions::new( + Ok(VindexVectorIndexOptions::new( table.schema().options(), &definition.options, &definition.index_type, @@ -70,61 +49,53 @@ pub(super) fn native_options( )? .native_options .into_iter() - .collect()); - } - let mut options = table.schema().options().clone(); - options.extend(definition.options.clone()); - Ok(options - .into_iter() - .filter_map(|(key, value)| { - key.strip_prefix("full-text.") - .map(|key| (key.to_string(), value)) - }) .collect()) + } else { + let mut options = table.schema().options().clone(); + options.extend(definition.options.clone()); + Ok(full_text::native_full_text_options(&options) + .into_iter() + .collect()) + } } -/// Keep every physical row, including tombstones, NULLs, and DV-hidden rows. -/// Native extractors use a synthetic consecutive row ID local to this level. -async fn batches( - table: &Table, - split: &DataSplit, - files: &[DataFileMeta], +/// Physical rows include NULLs, tombstones and DV-hidden positions. The common +/// writers receive dense, level-local ordinals, not a merged or filtered read. +fn batches( + table: Table, + split: DataSplit, + files: Vec, resources: Option, - mut state: S, - mut consume: F, -) -> Result -where - F: FnMut(S, RecordBatch, i64) -> Fut, - Fut: Future>, -{ - let mut offset = 0i64; - for file in files { - let reader = super::super::new_reader(table, resources.clone())?; - let mut stream = reader.read_physical_file(split.clone(), file.clone())?; - while let Some(batch) = stream.try_next().await? { - let count = batch.num_rows() as i64; - let mut fields = batch.schema().fields().iter().cloned().collect::>(); - fields.push(Arc::new(Field::new( - ROW_ID_FIELD_NAME, - ArrowType::Int64, - false, - ))); - let mut columns = batch.columns().to_vec(); - columns.push(Arc::new(Int64Array::from_iter_values( - offset..offset + count, - ))); - let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) - .map_err(native_error)?; - state = consume(state, batch, offset).await?; - offset = offset - .checked_add(count) - .ok_or_else(|| invalid("PK index ordinal overflow"))?; +) -> ArrowRecordBatchStream { + Box::pin(try_stream! { + let mut offset = 0i64; + for file in &files { + let reader = super::super::new_reader(&table, resources.clone())?; + let mut stream = reader.read_physical_file(split.clone(), file.clone())?; + while let Some(batch) = stream.try_next().await? { + let count = batch.num_rows() as i64; + let mut fields = batch.schema().fields().iter().cloned().collect::>(); + fields.push(Arc::new(Field::new( + ROW_ID_FIELD_NAME, + ArrowType::Int64, + false, + ))); + let mut columns = batch.columns().to_vec(); + columns.push(Arc::new(Int64Array::from_iter_values( + offset..offset + count, + ))); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) + .map_err(index_build::error)?; + yield batch; + offset = offset + .checked_add(count) + .ok_or_else(|| invalid("PK index ordinal overflow"))?; + } } - } - if offset != super::row_count(files)? { - return Err(invalid("PK index source row count changed")); - } - Ok(state) + if offset != super::row_count(&files)? { + Err(invalid("PK index source row count changed"))?; + } + }) } pub(super) async fn build( @@ -134,257 +105,83 @@ pub(super) async fn build( definition: &Definition, resources: Option, ) -> Result> { - let kind = definition.index_type.as_str(); let count = super::row_count(files)?; - let file_name = format!("{kind}-pk-index-{}.index", uuid::Uuid::new_v4()); + let range = RowRange::new(0, count - 1); + let meta = GlobalIndexMeta { + row_range_start: range.from(), + row_range_end: range.to(), + index_field_id: definition.field.id(), + extra_field_ids: None, + source_meta: Some(super::source_meta(files[0].level, files)?.serialize()?), + index_meta: None, + }; let in_data = table.schema().core_options().index_file_in_data_file_dir(); let relative = split .bucket_path() .strip_prefix(&table.data_file_location()) .unwrap_or(split.bucket_path()) .trim_start_matches('/'); - let external_path = crate::table::external_path::new_index_external_path( - table.schema().options(), - in_data, + let file = IndexFileWriter::new( + table, + &definition.index_type, + format!( + "{}-pk-index-{}.index", + definition.index_type, + uuid::Uuid::new_v4() + ), + IndexFileLocation::BucketLocal { + table_path: table.location().trim_end_matches('/'), + bucket_path: split.bucket_path(), + index_file_in_data_file_dir: in_data, + }, relative, - &file_name, - )?; - let location = IndexFileLocation::BucketLocal { - table_path: table.location(), - bucket_path: split.bucket_path(), - index_file_in_data_file_dir: in_data, - }; - let path = location.resolve(&file_name, external_path.as_deref()); - if let Some((directory, _)) = path.rsplit_once('/') { - table.file_io().mkdirs(directory).await?; - } - let mut options = table.schema().options().clone(); - options.extend(definition.options.clone()); - let written: Result>> = async { - if crate::vindex::is_vindex_index_type(kind) { - vector(table,split,files,definition,resources,&path).await - } else if kind == "full-text" { - full_text(table,split,files,definition,resources,&path).await.map(Some) - } else { - let field = definition.field.clone(); - let codec_type = if kind == "btree" { "btree" } else { "bitmap" }; - let key_type = index_key_type(kind,&field)?; - let (cmp,serialize) = make_index_key_codec(codec_type,key_type); - let output = table.file_io().new_output(&path)?.writer().await?; - if kind == "fm" { - let writer = crate::fm_index::FMGlobalIndexWriter::new(output,crate::fm_index::FMOptions::from_options(&options)?.write).map_err(native_error)?; - let writer = batches(table,split,files,resources,writer,|mut native,batch,offset| { - let field = field.clone(); - async move { - let index = batch.schema().index_of(field.name()).map_err(native_error)?; - for row in 0..batch.num_rows() { - let key = extract_datum_from_arrow(&batch,row,index,field.data_type())?.map(|d| serialize(&d,field.data_type())); - native.write(key.as_deref(),(offset+row as i64) as u64).await.map_err(native_error)?; - } - Ok(native) - } - }).await?; - return Ok(Some(writer.finish().await.map_err(native_error)?.index_meta)); - } - let sorter = IndexSorter::new(crate::spec::CoreOptions::new(&options).write_buffer_size()?,cmp); - let sorter = batches(table,split,files,resources,sorter,|mut native,batch,offset| { - let field = field.clone(); let kind = kind.to_string(); - async move { - blocking(move || { - let mut emit = |row| native.push(row); - if kind == "multivalue" { - let DataType::Array(array) = field.data_type() else { unreachable!() }; - emit_multivalue_rows(std::slice::from_ref(&batch),field.name(),array.element_type(),0,batch.num_rows() as i64,serialize,offset,&mut emit)?; - } else { - emit_scalar_rows(std::slice::from_ref(&batch),field.name(),field.data_type(),0,batch.num_rows() as i64,serialize,offset,&mut emit)?; - } - Ok(native) - }).await - } - }).await?; - let mut rows = blocking(move || sorter.finish()).await?; - let options = SortedIndexWriteOptions::from_options(kind,&options)?; - let cmp = make_index_key_codec(codec_type,key_type).0; - if kind == "btree" { - let mut writer = crate::btree::BTreeIndexWriter::with_comparator_and_options(output,options.block_size,options.compression_type,options.compression_level,options.bloom_filter_enabled,cmp) - .with_file_version(options.btree_file_version).map_err(native_error)?; - while let Some((key,id)) = rows.next().await? { writer.write(key.as_deref(),id).await.map_err(native_error)?; } - Ok(Some(writer.finish().await.map_err(native_error)?.meta.serialize())) - } else { - let mut writer = crate::table::bitmap_global_index_writer::BitmapGlobalIndexWriter::with_compression_level(output,options.block_size,options.compression_type,options.compression_level,cmp); - while let Some((key,id)) = rows.next().await? { - if kind == "multivalue" { writer.write_posting(key.as_deref().unwrap(),id).map_err(native_error)?; } - else { writer.write(key.as_deref(),id).map_err(native_error)?; } - } - let result = if kind == "multivalue" { writer.finish_with_source_row_count(count as u64).await } - else { writer.finish().await }.map_err(native_error)?; - Ok(Some(result.meta.serialize())) - } - } - }.await; - let index_meta = match written { - Ok(Some(meta)) => meta, - Ok(None) => return Ok(None), - Err(error) => { - let _ = table.file_io().delete_file(&path).await; - return Err(error); - } - }; - let metadata = async { - let file_size = - i64::try_from(table.file_io().get_status(&path).await?.size).map_err(native_error)?; - Ok(Some(IndexFileMeta { - index_type: kind.to_string(), - file_name, - file_size, - row_count: count, - deletion_vectors_ranges: None, - external_path, - global_index_meta: Some(GlobalIndexMeta { - row_range_start: 0, - row_range_end: count - 1, - index_field_id: definition.field.id(), - extra_field_ids: None, - source_meta: Some(super::source_meta(files[0].level, files)?.serialize()?), - index_meta: Some(index_meta), - }), - })) - } - .await; - if metadata.is_err() { - let _ = table.file_io().delete_file(&path).await; - } - metadata -} - -async fn full_text( - table: &Table, - split: &DataSplit, - files: &[DataFileMeta], - definition: &Definition, - resources: Option, - path: &str, -) -> Result> { - let native = native_options(table, definition)?; - let config = paimon_ftindex_core::FullTextIndexConfig::from_options( - &native.clone().into_iter().collect(), + meta, ) - .map_err(native_error)?; - let writer = blocking(move || { - paimon_ftindex_core::FullTextIndexWriter::new(config).map_err(native_error) - }) .await?; - let mut writer = batches( - table, - split, - files, - resources, - writer, - |mut native, batch, offset| { - let field = definition.field.clone(); - async move { - blocking(move || { - let column = batch - .schema() - .index_of(field.name()) - .map_err(native_error)?; - for row in 0..batch.num_rows() { - if let Some(Datum::String(text)) = - extract_datum_from_arrow(&batch, row, column, field.data_type())? - { - native - .add_document(offset + row as i64, text) - .map_err(native_error)?; - } - } - Ok(native) - }) + let stream = batches(table.clone(), split.clone(), files.to_vec(), resources); + let mut options = table.schema().options().clone(); + options.extend(definition.options.clone()); + file.write(table, |output| async move { + match definition.index_type.as_str() { + "full-text" => { + let native = full_text::native_full_text_options(&options); + full_text::write_payload( + stream, + output, + definition.field.name(), + range, + &native, + index_build::serialize_options(&native)?, + ) .await } - }, - ) - .await?; - let file = blocking(move || { - let mut file = tempfile::NamedTempFile::new().map_err(native_error)?; - writer - .write(&mut paimon_ftindex_core::io::PosWriter::new( - file.as_file_mut(), - )) - .map_err(native_error)?; - Ok(file) - }) - .await?; - copy_local_file_to_output(file.path(), table.file_io().new_output(path)?).await?; - serde_json::to_vec(&native).map_err(native_error) -} - -async fn vector( - table: &Table, - split: &DataSplit, - files: &[DataFileMeta], - definition: &Definition, - resources: Option, - path: &str, -) -> Result>> { - let options = crate::vindex::VindexVectorIndexOptions::new( - table.schema().options(), - &definition.options, - &definition.index_type, - &definition.field, - )?; - let dimension = options.dimension(); - let raw = tokio::io::BufWriter::new(tokio::fs::File::from_std( - tempfile::tempfile().map_err(native_error)?, - )); - let ids = tokio::io::BufWriter::new(tokio::fs::File::from_std( - tempfile::tempfile().map_err(native_error)?, - )); - let (mut raw, mut ids, count) = batches( - table, - split, - files, - resources, - (raw, ids, 0usize), - |(mut raw, mut ids, count), batch, offset| { - let column = definition.field.name(); - async move { - let mut expected = offset; - let vectors = validate_vector_batch(&batch, column, dimension, &mut expected)?; - raw.write_all(vectors.values.inner().as_slice()) - .await - .map_err(native_error)?; - ids.write_all(vectors.row_ids.inner().as_slice()) - .await - .map_err(native_error)?; - Ok((raw, ids, count + vectors.vector_count)) + kind if crate::vindex::is_vindex_index_type(kind) => { + let native = VindexVectorIndexOptions::new( + table.schema().options(), + &definition.options, + kind, + &definition.field, + )?; + vector::write_payload( + stream, + output, + definition.field.name(), + &native, + range, + index_build::serialize_options(&native.native_options)?, + ) + .await } - }, - ) - .await?; - if count == 0 { - return Ok(None); - } - raw.flush().await.map_err(native_error)?; - ids.flush().await.map_err(native_error)?; - let raw = raw.into_inner().into_std().await; - let ids = ids.into_inner().into_std().await; - let config = options.training_config(count)?; - let ratio = options.train_sample_ratio; - let file = blocking(move || { - let (mut writer, _, _, _) = - train_and_add_spilled_vectors(raw, ids, config, count, dimension, ratio, false) - .map_err(native_error)?; - let mut file = tempfile::NamedTempFile::new().map_err(native_error)?; - writer - .write(&mut paimon_vindex_core::io::PosWriter::new( - file.as_file_mut(), - )) - .map_err(native_error)?; - Ok(file) + kind => scalar::write_payload( + stream, + output, + &[&definition.field], + range, + &ScalarIndexOptions::from_options(kind, &options)?, + ) + .await + .map(Some), + } }) - .await?; - copy_local_file_to_output(file.path(), table.file_io().new_output(path)?).await?; - serde_json::to_vec(&native_options(table, definition)?) - .map(Some) - .map_err(native_error) + .await } diff --git a/crates/paimon/src/table/full_text_index_build_builder.rs b/crates/paimon/src/table/full_text_index_build_builder.rs index 96045b373..5e78c54c9 100644 --- a/crates/paimon/src/table/full_text_index_build_builder.rs +++ b/crates/paimon/src/table/full_text_index_build_builder.rs @@ -31,22 +31,19 @@ use crate::table::global_index_build_common::vector::{ find_index_field, plan_vector_index_shards, VectorIndexShard, }; use crate::table::global_index_build_common::{ - copy_local_file_to_output, indexed_row_ranges, validate_existing_index_overlap, + indexed_row_ranges, validate_existing_index_overlap, }; use crate::table::global_index_types::FULL_TEXT_GLOBAL_INDEX_TYPE; +#[cfg(test)] +use crate::table::index_build::full_text::ShardCursor; +use crate::table::index_build::{ + full_text::{self, full_text_config, native_full_text_options, serialize_index_meta}, + IndexFileWriter, +}; +use crate::table::index_file_path::IndexFileLocation; use crate::table::{CommitMessage, DataSplitBuilder, RowRange, Table}; use crate::{Error, Result}; -use arrow_array::cast::AsArray; -use arrow_array::{Array, Int64Array, LargeStringArray, RecordBatch, StringArray, StringViewArray}; -use arrow_schema::DataType as ArrowDataType; -use futures::TryStreamExt; -use paimon_ftindex_core::io::PosWriter; -use paimon_ftindex_core::{FullTextIndexConfig, FullTextIndexWriter}; -use std::collections::{BTreeMap, HashMap}; - -/// Java `NativeFullTextIndexOptions.FULL_TEXT_PREFIX`: only these options reach -/// the native writer, with the prefix removed. -const FULL_TEXT_OPTION_PREFIX: &str = "full-text."; +use std::collections::HashMap; pub(super) struct FullTextIndexBuildBuilder<'a> { table: &'a Table, @@ -217,206 +214,43 @@ impl<'a> FullTextIndexBuildBuilder<'a> { let mut read_builder = self.table.new_read_builder(); read_builder.with_projection(&[index_column, ROW_ID_FIELD_NAME])?; let read = read_builder.new_read()?; - let mut stream = read.to_arrow(&[split])?; - - let config = full_text_config(native_options)?; - let mut writer = - run_blocking(move || FullTextIndexWriter::new(config).map_err(full_text_error)).await?; - let mut cursor = ShardCursor::new(shard.row_range_start, shard.row_range_end); - while let Some(batch) = stream.try_next().await? { - let documents = cursor.documents(&batch, index_column)?; - writer = run_blocking(move || { - for (row_id, text) in documents { - writer.add_document(row_id, text).map_err(full_text_error)?; - } - Ok(writer) - }) - .await?; - } - if cursor.row_count == 0 { - return Ok(None); - } - - let local_index = run_blocking(move || { - let mut file = tempfile::NamedTempFile::new().map_err(|e| Error::UnexpectedError { - message: format!("Failed to create temporary full-text index file: {e}"), - source: None, - })?; - writer - .write(&mut PosWriter::new(file.as_file_mut())) - .map_err(full_text_error)?; - file.as_file_mut() - .sync_all() - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to flush temporary full-text index file: {e}"), - source: None, - })?; - Ok(file) - }) - .await?; - - let file_name = format!( - "{FULL_TEXT_GLOBAL_INDEX_TYPE}-global-index-{}.index", - uuid::Uuid::new_v4() - ); - let (index_path, external_path) = - super::global_index_build_common::prepare_index_file_path(self.table, &file_name) - .await?; - let upload: Result = async { - copy_local_file_to_output( - local_index.path(), - self.table.file_io().new_output(&index_path)?, - ) - .await?; - let status = self.table.file_io().get_status(&index_path).await?; - i64::try_from(status.size).map_err(|_| Error::DataInvalid { - message: format!( - "Full-text index file '{file_name}' is too large: {} bytes", - status.size - ), - source: None, - }) - } - .await; - let file_size = match upload { - Ok(file_size) => file_size, - Err(error) => { - let _ = self.table.file_io().delete_file(&index_path).await; - return Err(error); - } + let stream = read.to_arrow(&[split])?; + + let meta = GlobalIndexMeta { + row_range_start: shard.row_range_start, + row_range_end: shard.row_range_end, + index_field_id, + extra_field_ids: None, + source_meta: Some( + crate::spec::DataEvolutionIndexSourceMeta::new(shard.snapshot_id)?.serialize(), + ), + index_meta: None, }; - - Ok(Some(IndexFileMeta { - index_type: FULL_TEXT_GLOBAL_INDEX_TYPE.to_string(), - file_name, - file_size, - row_count: cursor.row_count, - deletion_vectors_ranges: None, - external_path, - global_index_meta: Some(GlobalIndexMeta { - row_range_start: shard.row_range_start, - row_range_end: shard.row_range_end, - index_field_id, - extra_field_ids: None, - source_meta: Some( - crate::spec::DataEvolutionIndexSourceMeta::new(shard.snapshot_id)?.serialize(), - ), - index_meta: Some(index_meta.to_vec()), - }), - })) - } -} - -/// Walks one shard's read batches in row-id order and turns them into -/// shard-relative `(row_id, text)` documents, like the loop in Java -/// `GenericIndexTopoBuilder.BuildIndexOperator#processElement`. -struct ShardCursor { - row_range_start: i64, - row_range_end: i64, - last_row_id: Option, - /// Rows inside the shard range, NULL text included (Java `rowCount`). - row_count: i64, -} - -impl ShardCursor { - fn new(row_range_start: i64, row_range_end: i64) -> Self { - Self { - row_range_start, - row_range_end, - last_row_id: None, - row_count: 0, - } - } - - fn documents(&mut self, batch: &RecordBatch, index_column: &str) -> Result> { - let text_index = batch - .schema() - .index_of(index_column) - .map_err(|e| Error::DataInvalid { - message: format!("Full-text column '{index_column}' not found in read batch: {e}"), - source: None, - })?; - let row_id_index = - batch - .schema() - .index_of(ROW_ID_FIELD_NAME) - .map_err(|e| Error::DataInvalid { - message: format!("_ROW_ID column not found in read batch: {e}"), - source: None, - })?; - let row_ids = batch - .column(row_id_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "Full-text index build requires Int64 _ROW_ID".to_string(), - source: None, - })?; - let texts = batch.column(text_index); - let texts = TextColumn::try_new(texts.as_ref())?; - - let mut documents = Vec::with_capacity(batch.num_rows()); - for row in 0..batch.num_rows() { - if row_ids.is_null(row) { - return Err(Error::DataInvalid { - message: "Full-text index build found null _ROW_ID".to_string(), - source: None, - }); - } - let row_id = row_ids.value(row); - if let Some(last_row_id) = self.last_row_id { - if row_id < last_row_id { - return Err(Error::DataInvalid { - message: format!( - "Row IDs are not monotonically increasing: previous={last_row_id}, \ - current={row_id} in shard [{}, {}]", - self.row_range_start, self.row_range_end - ), - source: None, - }); - } - } - self.last_row_id = Some(row_id); - if row_id < self.row_range_start || row_id > self.row_range_end { - continue; - } - self.row_count += 1; - if let Some(text) = texts.value(row) { - documents.push((row_id - self.row_range_start, text.to_string())); - } - } - Ok(documents) - } -} - -/// The Arrow string layouts a CHAR/VARCHAR column can be read as. -enum TextColumn<'a> { - Utf8(&'a StringArray), - LargeUtf8(&'a LargeStringArray), - Utf8View(&'a StringViewArray), -} - -impl<'a> TextColumn<'a> { - fn try_new(array: &'a dyn Array) -> Result { - match array.data_type() { - ArrowDataType::Utf8 => Ok(Self::Utf8(array.as_string::())), - ArrowDataType::LargeUtf8 => Ok(Self::LargeUtf8(array.as_string::())), - ArrowDataType::Utf8View => Ok(Self::Utf8View(array.as_string_view())), - other => Err(Error::DataInvalid { - message: format!( - "Full-text index build requires a string column, got Arrow type {other}" - ), - source: None, - }), - } - } - - fn value(&self, row: usize) -> Option<&'a str> { - match self { - Self::Utf8(array) => array.is_valid(row).then(|| array.value(row)), - Self::LargeUtf8(array) => array.is_valid(row).then(|| array.value(row)), - Self::Utf8View(array) => array.is_valid(row).then(|| array.value(row)), - } + IndexFileWriter::new( + self.table, + FULL_TEXT_GLOBAL_INDEX_TYPE, + format!( + "{FULL_TEXT_GLOBAL_INDEX_TYPE}-global-index-{}.index", + uuid::Uuid::new_v4() + ), + IndexFileLocation::Global { + table_path: self.table.location().trim_end_matches('/'), + }, + "", + meta, + ) + .await? + .write(self.table, |output| { + full_text::write_payload( + stream, + output, + index_column, + RowRange::new(shard.row_range_start, shard.row_range_end), + native_options, + index_meta.to_vec(), + ) + }) + .await } } @@ -446,53 +280,5 @@ fn validate_text_field(field: &DataField) -> Result<()> { Ok(()) } -/// Java `options.removePrefix("full-text.")`. -fn native_full_text_options(options: &HashMap) -> HashMap { - options - .iter() - .filter_map(|(key, value)| { - key.strip_prefix(FULL_TEXT_OPTION_PREFIX) - .map(|native_key| (native_key.to_string(), value.clone())) - }) - .collect() -} - -fn full_text_config(native_options: &HashMap) -> Result { - FullTextIndexConfig::from_options(native_options).map_err(|e| Error::ConfigInvalid { - message: format!("Invalid full-text index options: {e}"), - }) -} - -/// Java `NativeFullTextIndexOptions#serialize`: the native options as a flat -/// JSON object. Keys are sorted so the metadata is deterministic. -fn serialize_index_meta(native_options: &HashMap) -> Result> { - let sorted = native_options.iter().collect::>(); - serde_json::to_vec(&sorted).map_err(|e| Error::DataInvalid { - message: format!("Failed to serialize full-text index metadata: {e}"), - source: Some(Box::new(e)), - }) -} - -fn full_text_error(error: impl std::fmt::Display) -> Error { - Error::UnexpectedError { - message: format!("Full-text index error: {error}"), - source: None, - } -} - -/// The native writer is synchronous and CPU-bound; keep it off the async workers. -async fn run_blocking(task: F) -> Result -where - F: FnOnce() -> Result + Send + 'static, - T: Send + 'static, -{ - tokio::task::spawn_blocking(task) - .await - .map_err(|e| Error::UnexpectedError { - message: format!("Full-text index build task failed: {e}"), - source: None, - })? -} - #[cfg(test)] mod tests; diff --git a/crates/paimon/src/table/global_index_build_common.rs b/crates/paimon/src/table/global_index_build_common.rs index 0a937d7f3..ad3585a40 100644 --- a/crates/paimon/src/table/global_index_build_common.rs +++ b/crates/paimon/src/table/global_index_build_common.rs @@ -25,20 +25,14 @@ //! `index_type` string, so it lives here once rather than being copied into //! each builder. -use crate::io::OutputFile; use crate::spec::{DataField, FileKind, IndexManifest, IndexManifestEntry, Snapshot}; use crate::table::{merge_row_ranges, RowRange, Table}; use crate::{Error, Result}; -use bytes::Bytes; -use std::path::Path; use std::sync::Arc; -use tokio::io::AsyncReadExt; pub(crate) mod preparation; pub(crate) mod vector; -const COPY_BUFFER_SIZE: usize = 1024 * 1024; - /// One source snapshot and its effective index coverage. Builders use this same /// immutable view for both gap planning and overlap validation. #[derive(Clone, Default)] @@ -126,38 +120,6 @@ pub(crate) fn validate_build_table(table: &Table) -> Result<()> { Ok(()) } -/// Stream a locally built full-text index to its table location. The native -/// writer serializes to a temporary file because it needs synchronous I/O. -pub(crate) async fn copy_local_file_to_output( - source_path: &Path, - output: OutputFile, -) -> Result<()> { - let mut source = - tokio::fs::File::open(source_path) - .await - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to open temporary index file: {e}"), - source: None, - })?; - let mut writer = output.writer().await?; - let mut buffer = vec![0u8; COPY_BUFFER_SIZE]; - - loop { - let len = source - .read(&mut buffer) - .await - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to read temporary index file: {e}"), - source: None, - })?; - if len == 0 { - break; - } - writer.write(Bytes::copy_from_slice(&buffer[..len])).await?; - } - writer.close().await -} - pub(crate) enum IndexColumns { Column(String), Columns(Vec), @@ -303,26 +265,3 @@ pub(crate) fn validate_existing_index_overlap( } Ok(()) } - -/// Resolve and create the table's global index directory, including external placement. -pub(crate) async fn prepare_index_file_path( - table: &Table, - file_name: &str, -) -> Result<(String, Option)> { - let external_path = super::external_path::new_index_external_path( - table.schema().options(), - false, - "", - file_name, - )?; - let path = external_path.clone().unwrap_or_else(|| { - format!( - "{}/index/{file_name}", - table.location().trim_end_matches('/') - ) - }); - if let Some((directory, _)) = path.rsplit_once('/') { - table.file_io().mkdirs(&format!("{directory}/")).await?; - } - Ok((path, external_path)) -} diff --git a/crates/paimon/src/table/index_build.rs b/crates/paimon/src/table/index_build.rs new file mode 100644 index 000000000..a751604f3 --- /dev/null +++ b/crates/paimon/src/table/index_build.rs @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared native payload construction and private output ownership. +//! +//! Callers provide indexed batches and source metadata. PK/DE source planning, +//! replacement policy and publication/recovery stay in their maintenance layers. + +pub(in crate::table) mod extraction; +pub(in crate::table) mod fields; +pub(in crate::table) mod full_text; +pub(in crate::table) mod scalar; +pub(in crate::table) mod sort; +pub(in crate::table) mod vector; +pub(in crate::table) mod vector_extraction; +pub(in crate::table) mod vector_validation; + +#[cfg(test)] +mod tests; + +use crate::io::OutputFile; +use crate::spec::{GlobalIndexMeta, IndexFileMeta}; +use crate::table::index_file_path::IndexFileLocation; +use crate::table::{RowRange, Table}; +use crate::{Error, Result}; +use bytes::Bytes; +use std::future::Future; +use std::path::Path; +use tokio::io::AsyncReadExt; +const COPY_BUFFER_SIZE: usize = 1024 * 1024; + +pub(in crate::table) struct IndexPayload { + pub row_count: i64, + pub index_meta: Vec, +} + +pub(in crate::table) fn error(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::DataInvalid { + message: "Index build failed".into(), + source: Some(Box::new(error)), + } +} + +pub(in crate::table) fn row_count(range: &RowRange) -> Result { + range + .to() + .checked_sub(range.from()) + .and_then(|count| count.checked_add(1)) + .filter(|count| *count > 0) + .ok_or_else(|| Error::DataInvalid { + message: format!("Invalid index row range [{}, {}]", range.from(), range.to()), + source: None, + }) +} + +/// A private file until its caller submits the returned metadata to a commit. +/// Every write/stat/size-conversion failure removes this file, never old files. +pub(in crate::table) struct IndexFileWriter { + path: String, + meta: IndexFileMeta, +} + +impl IndexFileWriter { + pub async fn new( + table: &Table, + index_type: &str, + file_name: String, + location: IndexFileLocation<'_>, + relative_bucket: &str, + meta: GlobalIndexMeta, + ) -> Result { + let in_data = matches!( + &location, + IndexFileLocation::BucketLocal { + index_file_in_data_file_dir: true, + .. + } + ); + let external_path = crate::table::external_path::new_index_external_path( + table.schema().options(), + in_data, + relative_bucket, + &file_name, + )?; + let path = location.resolve(&file_name, external_path.as_deref()); + if let Some((directory, _)) = path.rsplit_once('/') { + table.file_io().mkdirs(directory).await?; + } + Ok(Self { + path, + meta: IndexFileMeta { + index_type: index_type.into(), + file_name, + file_size: 0, + row_count: 0, + deletion_vectors_ranges: None, + external_path, + global_index_meta: Some(meta), + }, + }) + } + + pub async fn write(mut self, table: &Table, build: F) -> Result> + where + F: FnOnce(OutputFile) -> Fut, + Fut: Future>>, + { + let result = async { + let Some(payload) = build(table.file_io().new_output(&self.path)?).await? else { + return Ok(None); + }; + self.meta.file_size = + i64::try_from(table.file_io().get_status(&self.path).await?.size).map_err(error)?; + self.meta.row_count = payload.row_count; + self.meta + .global_index_meta + .as_mut() + .expect("native index metadata") + .index_meta = Some(payload.index_meta); + Ok(Some(self.meta)) + } + .await; + if result.is_err() { + let _ = table.file_io().delete_file(&self.path).await; + } + result + } +} + +pub(in crate::table) fn serialize_options( + options: &std::collections::HashMap, +) -> Result> { + let sorted = options.iter().collect::>(); + serde_json::to_vec(&sorted).map_err(error) +} + +/// Stream a locally built full-text index to its table location. The native +/// writer serializes to a temporary file because it needs synchronous I/O. +pub(in crate::table) async fn copy_local_file_to_output( + source_path: &Path, + output: OutputFile, +) -> Result<()> { + let mut source = + tokio::fs::File::open(source_path) + .await + .map_err(|e| Error::UnexpectedError { + message: format!("Failed to open temporary index file: {e}"), + source: None, + })?; + let mut writer = output.writer().await?; + let mut buffer = vec![0u8; COPY_BUFFER_SIZE]; + + loop { + let len = source + .read(&mut buffer) + .await + .map_err(|e| Error::UnexpectedError { + message: format!("Failed to read temporary index file: {e}"), + source: None, + })?; + if len == 0 { + break; + } + writer.write(Bytes::copy_from_slice(&buffer[..len])).await?; + } + writer.close().await +} diff --git a/crates/paimon/src/table/index_build/extraction.rs b/crates/paimon/src/table/index_build/extraction.rs new file mode 100644 index 000000000..99e7f08ce --- /dev/null +++ b/crates/paimon/src/table/index_build/extraction.rs @@ -0,0 +1,512 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared conversion of indexed batches into bounded, sorted postings. + +use super::scalar::{SerializeKeyFn, SortedIndexKeyRow}; +use crate::btree::key_serde::DynKeyComparator; +use crate::spec::{ + extract_datum_from_array, extract_datum_from_arrow, DataField, DataType, ROW_ID_FIELD_NAME, +}; +use crate::table::global_index_types::MULTIVALUE_GLOBAL_INDEX_TYPE; +use crate::table::{ArrowRecordBatchStream, RowRange}; +use crate::{Error, Result}; +use arrow_array::{Array, FixedSizeListArray, Int64Array, LargeListArray, ListArray, RecordBatch}; +use futures::TryStreamExt; +use std::cmp::Ordering; + +#[allow(clippy::too_many_arguments)] +pub(in crate::table) async fn collect_sorted_rows( + mut batches: ArrowRecordBatchStream, + fields: &[&DataField], + index_type: &str, + range: RowRange, + serialize_key: SerializeKeyFn, + budget: usize, + cmp: crate::btree::key_serde::KeyComparator, +) -> Result { + let fields = std::sync::Arc::new( + fields + .iter() + .map(|field| (*field).clone()) + .collect::>(), + ); + let mut next_row_id = range.from(); + let mut sorter = super::sort::IndexSorter::new(budget, cmp); + while let Some(batch) = batches.try_next().await? { + let fields = fields.clone(); + let index_type = index_type.to_string(); + let row_range_start = range.from(); + let row_count = i64::try_from(batch.num_rows()).map_err(|_| Error::DataInvalid { + message: "Index batch row count overflow".into(), + source: None, + })?; + // Sorting and local scratch-file I/O must not block the async reader. + sorter = tokio::task::spawn_blocking(move || { + let refs = fields.iter().collect::>(); + let mut emit = |row| sorter.push(row); + let batches = std::slice::from_ref(&batch); + if refs.len() > 1 { + emit_composite_rows( + batches, + &refs, + row_range_start, + row_count, + next_row_id, + &mut emit, + )?; + } else if index_type == MULTIVALUE_GLOBAL_INDEX_TYPE { + let DataType::Array(array_type) = refs[0].data_type() else { + unreachable!("validated multivalue type") + }; + emit_multivalue_rows( + batches, + refs[0].name(), + array_type.element_type(), + row_range_start, + row_count, + serialize_key, + next_row_id, + &mut emit, + )?; + } else { + emit_scalar_rows( + batches, + refs[0].name(), + refs[0].data_type(), + row_range_start, + row_count, + serialize_key, + next_row_id, + &mut emit, + )?; + } + Ok::<_, Error>(sorter) + }) + .await + .map_err(sort_task_error)??; + next_row_id = next_row_id + .checked_add(row_count) + .ok_or_else(|| Error::DataInvalid { + message: "Index row ID overflow".into(), + source: None, + })?; + } + let expected = super::row_count(&range)?; + if next_row_id - range.from() != expected { + return Err(Error::DataInvalid { + message: format!( + "Sorted index expected {expected} source rows, got {}", + next_row_id - range.from() + ), + source: None, + }); + } + tokio::task::spawn_blocking(move || sorter.finish()) + .await + .map_err(sort_task_error)? +} + +fn sort_task_error(error: tokio::task::JoinError) -> Error { + Error::UnexpectedError { + message: format!("Sorted index sort task failed: {error}"), + source: None, + } +} + +#[allow(clippy::too_many_arguments)] +fn emit_composite_rows( + batches: &[RecordBatch], + fields: &[&DataField], + row_range_start: i64, + expected_row_count: i64, + next_row_id: i64, + emit: &mut dyn FnMut(SortedIndexKeyRow) -> Result<()>, +) -> Result<()> { + let fields = fields + .iter() + .map(|field| (*field).clone()) + .collect::>(); + let codec = crate::btree::CompositeKeyCodec::new(&fields); + let mut expected_row_id = next_row_id; + for batch in batches { + let indices = fields + .iter() + .map(|field| batch.schema().index_of(field.name())) + .collect::, _>>() + .map_err(|error| Error::DataInvalid { + message: format!("Composite index column missing: {error}"), + source: None, + })?; + let row_id_index = batch + .schema() + .index_of(ROW_ID_FIELD_NAME) + .map_err(|error| Error::DataInvalid { + message: format!("_ROW_ID column missing: {error}"), + source: None, + })?; + let row_ids = batch + .column(row_id_index) + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::DataInvalid { + message: "Composite index requires Int64 _ROW_ID".into(), + source: None, + })?; + for row in 0..batch.num_rows() { + if row_ids.is_null(row) || row_ids.value(row) != expected_row_id { + return Err(Error::DataInvalid { + message: format!("Composite index expected non-null _ROW_ID {expected_row_id}"), + source: None, + }); + } + let values = fields + .iter() + .zip(&indices) + .map(|(field, index)| { + extract_datum_from_arrow(batch, row, *index, field.data_type()) + }) + .collect::>>()?; + emit(( + Some(codec.serialize(&values)?), + expected_row_id - row_range_start, + ))?; + expected_row_id = expected_row_id + .checked_add(1) + .ok_or_else(|| Error::DataInvalid { + message: "Composite index row ID overflow".into(), + source: None, + })?; + } + } + if expected_row_id - next_row_id != expected_row_count { + return Err(Error::DataInvalid { + message: format!( + "Composite index expected {expected_row_count} rows, got {}", + expected_row_id - next_row_id + ), + source: None, + }); + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::table) fn emit_scalar_rows( + batches: &[RecordBatch], + index_column: &str, + data_type: &DataType, + row_range_start: i64, + expected_row_count: i64, + serialize_key: SerializeKeyFn, + next_row_id: i64, + emit: &mut dyn FnMut(SortedIndexKeyRow) -> Result<()>, +) -> Result<()> { + let mut expected_row_id = next_row_id; + for batch in batches { + let value_index = + batch + .schema() + .index_of(index_column) + .map_err(|e| Error::DataInvalid { + message: format!("Index column '{index_column}' not found in read batch: {e}"), + source: None, + })?; + let row_id_index = + batch + .schema() + .index_of(ROW_ID_FIELD_NAME) + .map_err(|e| Error::DataInvalid { + message: format!("_ROW_ID column not found in read batch: {e}"), + source: None, + })?; + let row_ids = batch + .column(row_id_index) + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::DataInvalid { + message: "Sorted global index build requires non-null Int64 _ROW_ID".to_string(), + source: None, + })?; + + for row in 0..batch.num_rows() { + if row_ids.is_null(row) { + return Err(Error::DataInvalid { + message: "Sorted global index build found null _ROW_ID".to_string(), + source: None, + }); + } + let row_id = row_ids.value(row); + if row_id != expected_row_id { + return Err(Error::DataInvalid { + message: format!( + "Sorted global index build expected _ROW_ID {}, got {}", + expected_row_id, row_id + ), + source: None, + }); + } + expected_row_id += 1; + + let key = extract_datum_from_arrow(batch, row, value_index, data_type)? + .map(|datum| serialize_key(&datum, data_type)); + emit((key, row_id - row_range_start))?; + } + } + let actual_row_count = expected_row_id - next_row_id; + if actual_row_count != expected_row_count { + return Err(Error::DataInvalid { + message: format!( + "Sorted global index build expected {} rows, got {}", + expected_row_count, actual_row_count + ), + source: None, + }); + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::table) fn emit_multivalue_rows( + batches: &[RecordBatch], + index_column: &str, + element_type: &DataType, + row_range_start: i64, + expected_row_count: i64, + serialize_key: SerializeKeyFn, + next_row_id: i64, + emit: &mut dyn FnMut(SortedIndexKeyRow) -> Result<()>, +) -> Result<()> { + let mut expected_row_id = next_row_id; + for batch in batches { + let value_index = + batch + .schema() + .index_of(index_column) + .map_err(|e| Error::DataInvalid { + message: format!("Index column '{index_column}' not found in read batch: {e}"), + source: None, + })?; + let row_id_index = + batch + .schema() + .index_of(ROW_ID_FIELD_NAME) + .map_err(|e| Error::DataInvalid { + message: format!("_ROW_ID column not found in read batch: {e}"), + source: None, + })?; + let row_ids = batch + .column(row_id_index) + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::DataInvalid { + message: "Multivalue global index build requires non-null Int64 _ROW_ID" + .to_string(), + source: None, + })?; + + #[derive(Clone, Copy)] + enum ArrayLayout<'a> { + List(&'a ListArray), + LargeList(&'a LargeListArray), + Fixed(&'a FixedSizeListArray), + } + let column = batch.column(value_index); + let layout = if let Some(array) = column.as_any().downcast_ref::() { + ArrayLayout::List(array) + } else if let Some(array) = column.as_any().downcast_ref::() { + ArrayLayout::LargeList(array) + } else if let Some(array) = column.as_any().downcast_ref::() { + ArrayLayout::Fixed(array) + } else { + return Err(Error::DataInvalid { + message: format!( + "Multivalue global index extraction requires an Arrow list column, got {:?}", + column.data_type() + ), + source: None, + }); + }; + let values = match layout { + ArrayLayout::List(array) => array.values(), + ArrayLayout::LargeList(array) => array.values(), + ArrayLayout::Fixed(array) => array.values(), + }; + + for row in 0..batch.num_rows() { + if row_ids.is_null(row) { + return Err(Error::DataInvalid { + message: "Multivalue global index build found null _ROW_ID".to_string(), + source: None, + }); + } + let row_id = row_ids.value(row); + if row_id != expected_row_id { + return Err(Error::DataInvalid { + message: format!( + "Multivalue global index build expected _ROW_ID {}, got {}", + expected_row_id, row_id + ), + source: None, + }); + } + expected_row_id += 1; + + let is_null = match layout { + ArrayLayout::List(array) => array.is_null(row), + ArrayLayout::LargeList(array) => array.is_null(row), + ArrayLayout::Fixed(array) => array.is_null(row), + }; + if is_null { + continue; + } + let (start, end) = match layout { + ArrayLayout::List(array) => { + let offsets = array.value_offsets(); + ( + usize::try_from(offsets[row]), + usize::try_from(offsets[row + 1]), + ) + } + ArrayLayout::LargeList(array) => { + let offsets = array.value_offsets(); + ( + usize::try_from(offsets[row]), + usize::try_from(offsets[row + 1]), + ) + } + ArrayLayout::Fixed(array) => { + let start = usize::try_from(array.value_offset(row)); + let end = usize::try_from(array.value_offset(row) + array.value_length()); + (start, end) + } + }; + let (start, end) = match (start, end) { + (Ok(start), Ok(end)) => (start, end), + _ => { + return Err(Error::DataInvalid { + message: "Multivalue global index found a negative array offset" + .to_string(), + source: None, + }) + } + }; + for element_index in start..end { + if let Some(datum) = + extract_datum_from_array(values, element_index, value_index, element_type)? + { + emit(( + Some(serialize_key(&datum, element_type)), + row_id - row_range_start, + ))?; + } + } + } + } + + let actual_row_count = expected_row_id - next_row_id; + if actual_row_count != expected_row_count { + return Err(Error::DataInvalid { + message: format!( + "Multivalue global index build expected {} source rows, got {}", + expected_row_count, actual_row_count + ), + source: None, + }); + } + Ok(()) +} + +/// Sort the extracted rows with the index's key comparator. +/// +/// Both keys come from [`super::scalar::make_index_key_codec`] for the column's current type, so +/// a comparison failure is a real defect rather than an index built before a type change; +/// it aborts the build instead of degrading. +pub(in crate::table) fn sort_index_rows( + rows: &mut [SortedIndexKeyRow], + cmp: &DynKeyComparator<'_>, +) -> Result<()> { + let mut failure = None; + rows.sort_unstable_by(|left, right| match (&left.0, &right.0) { + (None, None) => left.1.cmp(&right.1), + (None, Some(_)) => Ordering::Less, + (Some(_), None) => Ordering::Greater, + (Some(left_key), Some(right_key)) => match cmp(left_key, right_key) { + Ok(order) => order.then_with(|| left.1.cmp(&right.1)), + Err(error) => { + failure.get_or_insert(error); + left.1.cmp(&right.1) + } + }, + }); + match failure { + Some(error) => Err(error), + None => Ok(()), + } +} + +#[cfg(test)] +pub(in crate::table) fn extract_index_rows_from_batches( + batches: &[RecordBatch], + column: &str, + data_type: &DataType, + start: i64, + count: i64, + serialize: SerializeKeyFn, +) -> Result> { + let mut rows = Vec::new(); + emit_scalar_rows( + batches, + column, + data_type, + start, + count, + serialize, + start, + &mut |row| { + rows.push(row); + Ok(()) + }, + )?; + Ok(rows) +} + +#[cfg(test)] +pub(in crate::table) fn extract_multivalue_index_rows_from_batches( + batches: &[RecordBatch], + column: &str, + data_type: &DataType, + start: i64, + count: i64, + serialize: SerializeKeyFn, +) -> Result> { + let mut rows = Vec::new(); + emit_multivalue_rows( + batches, + column, + data_type, + start, + count, + serialize, + start, + &mut |row| { + rows.push(row); + Ok(()) + }, + )?; + Ok(rows) +} diff --git a/crates/paimon/src/table/index_build/fields.rs b/crates/paimon/src/table/index_build/fields.rs new file mode 100644 index 000000000..f52dd9842 --- /dev/null +++ b/crates/paimon/src/table/index_build/fields.rs @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Scalar field validation used by PK and Data Evolution builders. +use crate::spec::{DataField, DataType}; +use crate::table::global_index_types::{FM_GLOBAL_INDEX_TYPE, MULTIVALUE_GLOBAL_INDEX_TYPE}; +use crate::{Error, Result}; +pub(in crate::table) fn validate_btree_field(field: &DataField) -> Result<()> { + if !is_btree_supported_data_type(field.data_type()) { + return Err(Error::Unsupported { + message: format!( + "Sorted global index only supports scalar columns, got {:?} for column '{}'", + field.data_type(), + field.name() + ), + }); + } + Ok(()) +} + +pub(in crate::table) fn index_key_type<'a>( + index_type: &str, + field: &'a DataField, +) -> Result<&'a DataType> { + if index_type == MULTIVALUE_GLOBAL_INDEX_TYPE { + let DataType::Array(array_type) = field.data_type() else { + return Err(Error::Unsupported { + message: format!( + "Multivalue global index requires an ARRAY column, got {:?} for column '{}'", + field.data_type(), + field.name() + ), + }); + }; + if !is_btree_supported_data_type(array_type.element_type()) { + return Err(Error::Unsupported { + message: format!( + "Multivalue global index does not support array element type {:?} for column '{}'", + array_type.element_type(), + field.name() + ), + }); + } + Ok(array_type.element_type()) + } else if index_type == FM_GLOBAL_INDEX_TYPE { + if !matches!(field.data_type(), DataType::Char(_) | DataType::VarChar(_)) { + return Err(Error::Unsupported { + message: format!( + "FM global index requires a character string column, got {:?} for column '{}'", + field.data_type(), + field.name() + ), + }); + } + Ok(field.data_type()) + } else { + validate_btree_field(field)?; + Ok(field.data_type()) + } +} + +pub(in crate::table) fn is_btree_supported_data_type(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Boolean(_) + | DataType::TinyInt(_) + | DataType::SmallInt(_) + | DataType::Int(_) + | DataType::BigInt(_) + | DataType::Decimal(_) + | DataType::Double(_) + | DataType::Float(_) + | DataType::Char(_) + | DataType::VarChar(_) + | DataType::Date(_) + | DataType::LocalZonedTimestamp(_) + | DataType::Time(_) + | DataType::Timestamp(_) + ) +} diff --git a/crates/paimon/src/table/index_build/full_text.rs b/crates/paimon/src/table/index_build/full_text.rs new file mode 100644 index 000000000..be3f0c499 --- /dev/null +++ b/crates/paimon/src/table/index_build/full_text.rs @@ -0,0 +1,244 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native full-text archives over an ordered, possibly sparse row-ID stream. +use super::copy_local_file_to_output; +use super::IndexPayload; +use crate::io::OutputFile; +use crate::spec::ROW_ID_FIELD_NAME; +use crate::table::{ArrowRecordBatchStream, RowRange}; +use crate::{Error, Result}; +use arrow_array::cast::AsArray; +use arrow_array::{Array, Int64Array, LargeStringArray, RecordBatch, StringArray, StringViewArray}; +use arrow_schema::DataType as ArrowDataType; +use futures::TryStreamExt; +use paimon_ftindex_core::io::PosWriter; +use paimon_ftindex_core::{FullTextIndexConfig, FullTextIndexWriter}; +use std::collections::HashMap; +const FULL_TEXT_OPTION_PREFIX: &str = "full-text."; + +pub(in crate::table) async fn write_payload( + mut stream: ArrowRecordBatchStream, + output: OutputFile, + index_column: &str, + range: RowRange, + native_options: &HashMap, + index_meta: Vec, +) -> Result> { + let config = full_text_config(native_options)?; + let mut writer = + run_blocking(move || FullTextIndexWriter::new(config).map_err(full_text_error)).await?; + let mut cursor = ShardCursor::new(range.from(), range.to()); + while let Some(batch) = stream.try_next().await? { + let documents = cursor.documents(&batch, index_column)?; + writer = run_blocking(move || { + for (row_id, text) in documents { + writer.add_document(row_id, text).map_err(full_text_error)?; + } + Ok(writer) + }) + .await?; + } + if cursor.row_count == 0 { + return Ok(None); + } + + let local_index = run_blocking(move || { + let mut file = tempfile::NamedTempFile::new().map_err(|e| Error::UnexpectedError { + message: format!("Failed to create temporary full-text index file: {e}"), + source: None, + })?; + writer + .write(&mut PosWriter::new(file.as_file_mut())) + .map_err(full_text_error)?; + file.as_file_mut() + .sync_all() + .map_err(|e| Error::UnexpectedError { + message: format!("Failed to flush temporary full-text index file: {e}"), + source: None, + })?; + Ok(file) + }) + .await?; + + copy_local_file_to_output(local_index.path(), output).await?; + Ok(Some(IndexPayload { + row_count: cursor.row_count, + index_meta, + })) +} +/// Walks one shard's read batches in row-id order and turns them into +/// shard-relative `(row_id, text)` documents, like the loop in Java +/// `GenericIndexTopoBuilder.BuildIndexOperator#processElement`. +pub(in crate::table) struct ShardCursor { + row_range_start: i64, + row_range_end: i64, + last_row_id: Option, + /// Rows inside the shard range, NULL text included (Java `rowCount`). + pub(in crate::table) row_count: i64, +} + +impl ShardCursor { + pub(in crate::table) fn new(row_range_start: i64, row_range_end: i64) -> Self { + Self { + row_range_start, + row_range_end, + last_row_id: None, + row_count: 0, + } + } + + pub(in crate::table) fn documents( + &mut self, + batch: &RecordBatch, + index_column: &str, + ) -> Result> { + let text_index = batch + .schema() + .index_of(index_column) + .map_err(|e| Error::DataInvalid { + message: format!("Full-text column '{index_column}' not found in read batch: {e}"), + source: None, + })?; + let row_id_index = + batch + .schema() + .index_of(ROW_ID_FIELD_NAME) + .map_err(|e| Error::DataInvalid { + message: format!("_ROW_ID column not found in read batch: {e}"), + source: None, + })?; + let row_ids = batch + .column(row_id_index) + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::DataInvalid { + message: "Full-text index build requires Int64 _ROW_ID".to_string(), + source: None, + })?; + let texts = batch.column(text_index); + let texts = TextColumn::try_new(texts.as_ref())?; + + let mut documents = Vec::with_capacity(batch.num_rows()); + for row in 0..batch.num_rows() { + if row_ids.is_null(row) { + return Err(Error::DataInvalid { + message: "Full-text index build found null _ROW_ID".to_string(), + source: None, + }); + } + let row_id = row_ids.value(row); + if let Some(last_row_id) = self.last_row_id { + if row_id < last_row_id { + return Err(Error::DataInvalid { + message: format!( + "Row IDs are not monotonically increasing: previous={last_row_id}, \ + current={row_id} in shard [{}, {}]", + self.row_range_start, self.row_range_end + ), + source: None, + }); + } + } + self.last_row_id = Some(row_id); + if row_id < self.row_range_start || row_id > self.row_range_end { + continue; + } + self.row_count += 1; + if let Some(text) = texts.value(row) { + documents.push((row_id - self.row_range_start, text.to_string())); + } + } + Ok(documents) + } +} + +/// The Arrow string layouts a CHAR/VARCHAR column can be read as. +enum TextColumn<'a> { + Utf8(&'a StringArray), + LargeUtf8(&'a LargeStringArray), + Utf8View(&'a StringViewArray), +} + +impl<'a> TextColumn<'a> { + fn try_new(array: &'a dyn Array) -> Result { + match array.data_type() { + ArrowDataType::Utf8 => Ok(Self::Utf8(array.as_string::())), + ArrowDataType::LargeUtf8 => Ok(Self::LargeUtf8(array.as_string::())), + ArrowDataType::Utf8View => Ok(Self::Utf8View(array.as_string_view())), + other => Err(Error::DataInvalid { + message: format!( + "Full-text index build requires a string column, got Arrow type {other}" + ), + source: None, + }), + } + } + + fn value(&self, row: usize) -> Option<&'a str> { + match self { + Self::Utf8(array) => array.is_valid(row).then(|| array.value(row)), + Self::LargeUtf8(array) => array.is_valid(row).then(|| array.value(row)), + Self::Utf8View(array) => array.is_valid(row).then(|| array.value(row)), + } + } +} + +pub(in crate::table) fn native_full_text_options( + options: &HashMap, +) -> HashMap { + options + .iter() + .filter_map(|(key, value)| { + key.strip_prefix(FULL_TEXT_OPTION_PREFIX) + .map(|native_key| (native_key.to_string(), value.clone())) + }) + .collect() +} + +pub(in crate::table) fn full_text_config( + native_options: &HashMap, +) -> Result { + FullTextIndexConfig::from_options(native_options).map_err(|e| Error::ConfigInvalid { + message: format!("Invalid full-text index options: {e}"), + }) +} + +/// Java `NativeFullTextIndexOptions#serialize`: the native options as a flat +/// JSON object. Keys are sorted so the metadata is deterministic. +pub(in crate::table) use super::serialize_options as serialize_index_meta; + +fn full_text_error(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::UnexpectedError { + message: "Full-text index error".into(), + source: Some(Box::new(error)), + } +} + +/// The native writer is synchronous and CPU-bound; keep it off the async workers. +async fn run_blocking(task: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + tokio::task::spawn_blocking(task) + .await + .map_err(|e| Error::UnexpectedError { + message: format!("Full-text index build task failed: {e}"), + source: None, + })? +} diff --git a/crates/paimon/src/table/index_build/scalar.rs b/crates/paimon/src/table/index_build/scalar.rs new file mode 100644 index 000000000..0018b70d7 --- /dev/null +++ b/crates/paimon/src/table/index_build/scalar.rs @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native scalar payload writing, independent of PK/DE source planning. +use super::extraction::collect_sorted_rows; +use super::{error, IndexPayload}; +use crate::btree::key_serde::KeyComparator; +use crate::btree::{make_key_comparator, serialize_datum, BTreeIndexWriter}; +use crate::fm_index::{FMGlobalIndexWriter, FMOptions, FMWriteOptions}; +use crate::io::OutputFile; +use crate::spec::{ + extract_datum_from_arrow, CoreOptions, DataField, DataType, Datum, RowType, ROW_ID_FIELD_NAME, +}; +use crate::table::bitmap_global_index_format::{ + make_bitmap_key_comparator, serialize_bitmap_datum, +}; +use crate::table::bitmap_global_index_writer::BitmapGlobalIndexWriter; +use crate::table::sorted_global_index_options::SortedIndexWriteOptions; +use crate::table::{ArrowRecordBatchStream, RowRange}; +use crate::{Error, Result}; +use arrow_array::{Array, Int64Array}; +use futures::TryStreamExt; +use std::collections::HashMap; + +pub(in crate::table) type SortedIndexKeyRow = (Option>, i64); +pub(in crate::table) type SerializeKeyFn = fn(&Datum, &DataType) -> Vec; + +enum WriteOptions { + Sorted(SortedIndexWriteOptions), + FM(FMWriteOptions), +} +pub(in crate::table) struct ScalarIndexOptions { + kind: &'static str, + write: WriteOptions, + buffer_size: usize, +} +impl ScalarIndexOptions { + pub fn from_options(kind: &str, options: &HashMap) -> Result { + let kind = crate::table::global_index_types::normalize_queryable_global_index_type(kind) + .ok_or_else(|| Error::Unsupported { + message: format!("Unsupported scalar index type: {kind}"), + })?; + if kind == "fm" { + Ok(Self { + kind, + write: WriteOptions::FM(FMOptions::from_options(options)?.write), + buffer_size: 0, + }) + } else { + Ok(Self { + kind, + write: WriteOptions::Sorted(SortedIndexWriteOptions::from_options(kind, options)?), + buffer_size: CoreOptions::new(options).write_buffer_size()?, + }) + } + } + + pub fn index_type(&self) -> &'static str { + self.kind + } +} + +pub(in crate::table) fn make_index_key_codec( + kind: &str, + data_type: &DataType, +) -> (KeyComparator, SerializeKeyFn) { + match kind { + "btree" => (make_key_comparator(data_type), serialize_datum), + "bitmap" => ( + make_bitmap_key_comparator(data_type), + serialize_bitmap_datum, + ), + _ => unreachable!("normalized scalar codec"), + } +} + +pub(in crate::table) async fn write_payload( + batches: ArrowRecordBatchStream, + output: OutputFile, + fields: &[&DataField], + range: RowRange, + options: &ScalarIndexOptions, +) -> Result { + let kind = options.index_type(); + let count = super::row_count(&range)?; + let composite = DataType::Row(RowType::new(fields.iter().map(|f| (*f).clone()).collect())); + let key_type = if fields.len() > 1 { + &composite + } else { + super::fields::index_key_type(kind, fields[0])? + }; + let codec = if kind == "btree" { "btree" } else { "bitmap" }; + let (cmp, serialize) = make_index_key_codec(codec, key_type); + let (written, index_meta) = match &options.write { + WriteOptions::FM(options) => { + write_fm(batches, output, fields[0], range, serialize, *options).await? + } + WriteOptions::Sorted(write) => { + let mut rows = collect_sorted_rows( + batches, + fields, + kind, + range, + serialize, + options.buffer_size, + cmp, + ) + .await?; + let cmp = make_index_key_codec(codec, key_type).0; + let output = output.writer().await?; + if kind == "btree" { + let mut writer = BTreeIndexWriter::with_comparator_and_options( + output, + write.block_size, + write.compression_type, + write.compression_level, + write.bloom_filter_enabled, + cmp, + ) + .with_file_version(write.btree_file_version) + .map_err(error)?; + while let Some((key, id)) = rows.next().await? { + writer.write(key.as_deref(), id).await.map_err(error)?; + } + let result = writer.finish().await.map_err(error)?; + (result.row_count, result.meta.serialize()) + } else { + let mut writer = BitmapGlobalIndexWriter::with_compression_level( + output, + write.block_size, + write.compression_type, + write.compression_level, + cmp, + ); + while let Some((key, id)) = rows.next().await? { + if kind == "multivalue" { + writer + .write_posting(key.as_deref().expect("nonnull multivalue posting"), id) + .map_err(error)?; + } else { + writer.write(key.as_deref(), id).map_err(error)?; + } + } + let result = if kind == "multivalue" { + writer.finish_with_source_row_count(count as u64).await + } else { + writer.finish().await + } + .map_err(error)?; + (result.row_count, result.meta.serialize()) + } + } + }; + if written != count as u64 { + return Err(Error::DataInvalid { + message: format!("Sorted index expected {count} rows, wrote {written}"), + source: None, + }); + } + Ok(IndexPayload { + row_count: count, + index_meta, + }) +} + +async fn write_fm( + mut batches: ArrowRecordBatchStream, + output: OutputFile, + index_field: &DataField, + range: RowRange, + serialize_key: SerializeKeyFn, + options: FMWriteOptions, +) -> Result<(u64, Vec)> { + let mut writer = FMGlobalIndexWriter::new(output.writer().await?, options).map_err(error)?; + let index_column = index_field.name(); + let expected_row_count = super::row_count(&range)?; + let mut expected_row_id = range.from(); + + while let Some(batch) = batches.try_next().await? { + let value_index = + batch + .schema() + .index_of(index_column) + .map_err(|error| Error::DataInvalid { + message: format!( + "Index column '{index_column}' not found in FM read batch: {error}" + ), + source: None, + })?; + let row_id_index = batch + .schema() + .index_of(ROW_ID_FIELD_NAME) + .map_err(|error| Error::DataInvalid { + message: format!("_ROW_ID column not found in FM read batch: {error}"), + source: None, + })?; + let row_ids = batch + .column(row_id_index) + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::DataInvalid { + message: "FM global index build requires non-null Int64 _ROW_ID".to_string(), + source: None, + })?; + + for row in 0..batch.num_rows() { + if row_ids.is_null(row) { + return Err(Error::DataInvalid { + message: "FM global index build found null _ROW_ID".to_string(), + source: None, + }); + } + let row_id = row_ids.value(row); + if row_id != expected_row_id { + return Err(Error::DataInvalid { + message: format!( + "FM global index build expected _ROW_ID {expected_row_id}, got {row_id}" + ), + source: None, + }); + } + expected_row_id = expected_row_id + .checked_add(1) + .ok_or_else(|| Error::DataInvalid { + message: "FM global index row ID overflow".to_string(), + source: None, + })?; + let local_row_id = + u64::try_from(row_id - range.from()).map_err(|_| Error::DataInvalid { + message: "FM index build has a negative local row ID".into(), + source: None, + })?; + let key = extract_datum_from_arrow(&batch, row, value_index, index_field.data_type())? + .map(|datum| serialize_key(&datum, index_field.data_type())); + writer + .write(key.as_deref(), local_row_id) + .await + .map_err(|error| Error::DataInvalid { + message: "Failed to write FM index file".into(), + source: Some(Box::new(error)), + })?; + } + } + + let actual_row_count = expected_row_id - range.from(); + if actual_row_count != expected_row_count { + return Err(Error::DataInvalid { + message: format!( + "FM global index build expected {expected_row_count} rows, got {actual_row_count}" + ), + source: None, + }); + } + let result = writer.finish().await.map_err(|error| Error::DataInvalid { + message: "Failed to finish FM index file".into(), + source: Some(Box::new(error)), + })?; + Ok((result.row_count, result.index_meta)) +} diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/sort.rs b/crates/paimon/src/table/index_build/sort.rs similarity index 99% rename from crates/paimon/src/table/sorted_global_index_build_builder/sort.rs rename to crates/paimon/src/table/index_build/sort.rs index 92b14c888..5671807be 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/sort.rs +++ b/crates/paimon/src/table/index_build/sort.rs @@ -18,7 +18,7 @@ //! Bounded sort buffers and pairwise disk merges. Disk work runs on blocking workers. use super::extraction::sort_index_rows; -use super::SortedIndexKeyRow; +use super::scalar::SortedIndexKeyRow; use crate::btree::key_serde::{DynKeyComparator, KeyComparator}; use crate::{Error, Result}; use std::cmp::Ordering; diff --git a/crates/paimon/src/table/index_build/tests.rs b/crates/paimon/src/table/index_build/tests.rs new file mode 100644 index 000000000..064da8eb5 --- /dev/null +++ b/crates/paimon/src/table/index_build/tests.rs @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::io::{FileIOBuilder, FileIOProvider}; +use crate::spec::{ + DataEvolutionIndexSourceMeta, DataType, IntType, PrimaryKeyIndexSourceFile, + PrimaryKeyIndexSourceMeta, Schema, TableSchema, +}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +#[derive(Debug)] +struct StatFailure { + operator: opendal::Operator, + calls: AtomicUsize, + fail_stat: bool, +} +#[async_trait::async_trait] +impl FileIOProvider for StatFailure { + async fn create(&self, path: &str) -> Result<(opendal::Operator, String)> { + if path.ends_with("/new.index") + && self.calls.fetch_add(1, Ordering::SeqCst) == 1 + && self.fail_stat + { + return Err(Error::UnexpectedError { + message: "injected index stat failure".into(), + source: None, + }); + } + Ok(( + self.operator.clone(), + path.strip_prefix("index-test:/").unwrap().into(), + )) + } +} + +/// Both maintenance paths share output ownership, while supplying their own +/// source frames and placement. Never delete a previous snapshot's payload. +#[tokio::test] +async fn output_ownership_and_source_metadata_are_identical_for_pk_and_de() { + for bucket_local in [false, true] { + for failure in ["none", "write", "stat"] { + let provider = Arc::new(StatFailure { + operator: + opendal::Operator::from_config(opendal::services::MemoryConfig::default()) + .unwrap(), + calls: AtomicUsize::new(0), + fail_stat: failure == "stat", + }); + let io = FileIOBuilder::new("memory") + .with_provider(provider) + .build() + .unwrap(); + let schema = Schema::builder() + .column("value", DataType::Int(IntType::new())) + .option("index-file-in-data-file-dir", "true") + .build() + .unwrap(); + let table = Table::new( + io.clone(), + crate::catalog::Identifier::new("default", "output"), + "index-test:/table".into(), + TableSchema::new(0, &schema), + None, + ); + let source = if bucket_local { + PrimaryKeyIndexSourceMeta::new( + 1, + vec![PrimaryKeyIndexSourceFile::new("data.parquet".into(), 3).unwrap()], + ) + .unwrap() + .serialize() + .unwrap() + } else { + DataEvolutionIndexSourceMeta::new(3).unwrap().serialize() + }; + let start = if bucket_local { 0 } else { 11 }; + let location = if bucket_local { + IndexFileLocation::BucketLocal { + table_path: table.location(), + bucket_path: "index-test:/table/custom/p=a/bucket-3", + index_file_in_data_file_dir: true, + } + } else { + IndexFileLocation::Global { + table_path: table.location(), + } + }; + let path = location.resolve("new.index", None); + let old_path = location.resolve("old.index", None); + let file = IndexFileWriter::new( + &table, + "btree", + "new.index".into(), + location, + "custom/p=a/bucket-3", + GlobalIndexMeta { + row_range_start: start, + row_range_end: start + 2, + index_field_id: 0, + extra_field_ids: None, + source_meta: Some(source.clone()), + index_meta: None, + }, + ) + .await + .unwrap(); + io.new_output(&old_path) + .unwrap() + .write(Bytes::from_static(b"old")) + .await + .unwrap(); + let outcome = file + .write(&table, |output| async move { + output.write(Bytes::from_static(b"payload")).await?; + if failure == "write" { + return Err(Error::UnexpectedError { + message: "injected index write failure".into(), + source: None, + }); + } + Ok(Some(IndexPayload { + row_count: 3, + index_meta: vec![1, 2, 3], + })) + }) + .await; + assert_eq!( + io.new_input(&old_path).unwrap().read().await.unwrap(), + Bytes::from_static(b"old") + ); + if failure == "none" { + let file = outcome.unwrap().unwrap(); + assert_eq!(file.file_size, 7); + assert_eq!(file.row_count, 3); + let meta = file.global_index_meta.unwrap(); + assert_eq!(meta.source_meta, Some(source)); + assert_eq!( + (meta.row_range_start, meta.row_range_end), + (start, start + 2) + ); + assert_eq!(meta.index_meta, Some(vec![1, 2, 3])); + assert!(io.exists(&path).await.unwrap()); + } else { + assert!(outcome.unwrap_err().to_string().contains("injected index")); + assert!( + !io.exists(&path).await.unwrap(), + "private payload must be removed on {failure} failure" + ); + } + } + } +} diff --git a/crates/paimon/src/table/index_build/vector.rs b/crates/paimon/src/table/index_build/vector.rs new file mode 100644 index 000000000..844e4188e --- /dev/null +++ b/crates/paimon/src/table/index_build/vector.rs @@ -0,0 +1,367 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared sparse-vector spill, training, and native serialization. +use super::vector_extraction::{local_ids, validate_vector_batch}; +use super::vector_validation::{ + checked_std_vector_bytes, checked_training_sample_index, checked_training_vector_count, + checked_vector_bytes, +}; +use super::{error, IndexPayload}; +use crate::io::OutputFile; +use crate::table::{ArrowRecordBatchStream, RowRange}; +use crate::vindex::VindexVectorIndexOptions; +use crate::{Error, Result}; +use arrow_array::RecordBatch; +use arrow_buffer::MutableBuffer; +use futures::TryStreamExt; +use paimon_vindex_core::autotune::default_training_vector_count; +use paimon_vindex_core::index::{VectorIndexConfig, VectorIndexTrainer, VectorIndexWriter}; +use paimon_vindex_core::io::PosWriter; +use std::io::{Read, Seek, SeekFrom}; +use std::time::{Duration, Instant}; +use tokio::io::AsyncWriteExt; +use tokio_util::io::SyncIoBridge; +const VECTOR_BUFFER_BYTES: usize = 8 * 1024 * 1024; + +pub(in crate::table) struct VectorSpill { + raw: tokio::fs::File, + ids: tokio::fs::File, + range: RowRange, + dimension: usize, + next_id: i64, + rows: usize, + vectors: usize, + bytes: usize, +} + +impl VectorSpill { + pub fn new(range: RowRange, dimension: usize) -> Result { + let count = usize::try_from(super::row_count(&range)?).map_err(error)?; + if dimension == 0 { + return Err(Error::DataInvalid { + message: "vindex vector dimension must be positive".into(), + source: None, + }); + } + checked_vector_bytes(count, dimension)?; + Ok(Self { + raw: temporary_vector_file("vectors")?, + ids: temporary_vector_file("row IDs")?, + next_id: range.from(), + range, + dimension, + rows: 0, + vectors: 0, + bytes: 0, + }) + } + pub fn bytes_written(&self) -> usize { + self.bytes + } + + pub async fn push_batch( + &mut self, + batch: &RecordBatch, + column: &str, + timing_enabled: bool, + ) -> Result { + let vectors = validate_vector_batch(batch, column, self.dimension, &mut self.next_id)?; + self.rows = + self.rows + .checked_add(vectors.source_rows) + .ok_or_else(|| Error::DataInvalid { + message: "vindex streamed row count overflows usize".into(), + source: None, + })?; + self.vectors = self + .vectors + .checked_add(vectors.vector_count) + .ok_or_else(|| Error::DataInvalid { + message: "vindex streamed vector count overflows usize".into(), + source: None, + })?; + let count = usize::try_from(super::row_count(&self.range)?).map_err(error)?; + let ids: arrow_buffer::ScalarBuffer = + local_ids(&vectors.row_ids, self.range.from(), count)?.into(); + let start = timing_enabled.then(Instant::now); + self.raw + .write_all(vectors.bytes()) + .await + .map_err(spill_error)?; + self.ids + .write_all(ids.inner().as_slice()) + .await + .map_err(spill_error)?; + self.bytes = self + .bytes + .checked_add(vectors.bytes().len()) + .ok_or_else(|| Error::DataInvalid { + message: "vindex spilled byte count overflows usize".into(), + source: None, + })?; + Ok(start.map_or(Duration::ZERO, |s| s.elapsed())) + } + + pub async fn finish(mut self) -> Result> { + let rows = usize::try_from(super::row_count(&self.range)?).map_err(error)?; + let end = self + .range + .to() + .checked_add(1) + .ok_or_else(|| Error::DataInvalid { + message: "vindex row range end overflows i64".into(), + source: None, + })?; + if self.rows != rows + || self.next_id != end + || self.bytes != checked_vector_bytes(self.vectors, self.dimension)? + { + return Err(Error::DataInvalid { + message: format!( + "vindex streamed data mismatch: rows={}/{rows}, vectors={}, bytes={}", + self.rows, self.vectors, self.bytes + ), + source: None, + }); + } + if self.vectors == 0 { + return Ok(None); + } + self.raw.flush().await.map_err(spill_error)?; + self.ids.flush().await.map_err(spill_error)?; + if self.raw.metadata().await.map_err(spill_error)?.len() != self.bytes as u64 + || self.ids.metadata().await.map_err(spill_error)?.len() + != checked_vector_bytes(self.vectors, 2)? as u64 + { + return Err(Error::DataInvalid { + message: "temporary vindex vector or row ID file size mismatch".into(), + source: None, + }); + } + Ok(Some(SpilledVectors { + raw: self.raw.into_std().await, + ids: self.ids.into_std().await, + count: self.vectors, + dimension: self.dimension, + })) + } +} + +pub(in crate::table) struct SpilledVectors { + raw: std::fs::File, + ids: std::fs::File, + count: usize, + dimension: usize, +} +pub(in crate::table) struct TrainedVectors { + pub writer: VectorIndexWriter, + pub train_finish: Duration, + pub raw_temp_reread: Duration, + pub index_add: Duration, + pub training_vector_count: usize, + pub training_rows_retained: usize, +} +impl SpilledVectors { + pub async fn train( + self, + options: &VindexVectorIndexOptions, + timing_enabled: bool, + ) -> Result { + let training_vector_count = + checked_training_vector_count(self.count, options.train_sample_ratio)?; + let config = options.training_config(self.count)?; + let training_rows_retained = if timing_enabled { + default_training_vector_count(training_vector_count, config.nlist()).unwrap_or(0) + } else { + 0 + }; + let ratio = options.train_sample_ratio; + let (writer, train_finish, raw_temp_reread, index_add) = + tokio::task::spawn_blocking(move || { + train_and_add_spilled_vectors( + self.raw, + self.ids, + config, + self.count, + self.dimension, + ratio, + timing_enabled, + ) + }) + .await + .map_err(|e| Error::UnexpectedError { + message: format!("vindex training task failed: {e}"), + source: None, + })? + .map_err(|e| Error::UnexpectedError { + message: "Failed to train or add vectors to vindex index".into(), + source: Some(Box::new(e)), + })?; + Ok(TrainedVectors { + writer, + train_finish, + raw_temp_reread, + index_add, + training_vector_count, + training_rows_retained, + }) + } +} + +pub(in crate::table) async fn serialize_writer( + mut writer: VectorIndexWriter, + output: OutputFile, +) -> Result<()> { + let mut output = SyncIoBridge::new(output.async_writer().await?); + tokio::task::spawn_blocking(move || -> std::io::Result<()> { + writer.write(&mut PosWriter::new(&mut output))?; + output.shutdown() + }) + .await + .map_err(|e| Error::UnexpectedError { + message: format!("vindex serialization task failed: {e}"), + source: None, + })? + .map_err(|e| Error::UnexpectedError { + message: "Failed to stream vindex index".into(), + source: Some(Box::new(e)), + }) +} + +pub(in crate::table) async fn write_payload( + mut batches: ArrowRecordBatchStream, + output: OutputFile, + column: &str, + options: &VindexVectorIndexOptions, + range: RowRange, + index_meta: Vec, +) -> Result> { + let row_count = super::row_count(&range)?; + let mut spill = VectorSpill::new(range, options.dimension())?; + while let Some(batch) = batches.try_next().await? { + spill.push_batch(&batch, column, false).await?; + } + let Some(spill) = spill.finish().await? else { + return Ok(None); + }; + let trained = spill.train(options, false).await?; + serialize_writer(trained.writer, output).await?; + Ok(Some(IndexPayload { + row_count, + index_meta, + })) +} + +fn spill_error(error: std::io::Error) -> Error { + Error::UnexpectedError { + message: format!("Failed to spill vindex vectors: {error}"), + source: Some(Box::new(error)), + } +} + +fn temporary_vector_file(kind: &str) -> Result { + tempfile::tempfile() + .map(tokio::fs::File::from_std) + .map_err(|error| Error::UnexpectedError { + message: format!("Failed to create temporary vindex {kind} file: {error}"), + source: Some(Box::new(error)), + }) +} + +/// Java samples the compacted non-null vector stream, then adds the original relative IDs. +fn train_and_add_spilled_vectors( + mut raw_file: std::fs::File, + mut id_file: std::fs::File, + config: VectorIndexConfig, + vector_count: usize, + dimension: usize, + ratio: f64, + timing_enabled: bool, +) -> std::io::Result<(VectorIndexWriter, Duration, Duration, Duration)> { + let train_start = timing_enabled.then(Instant::now); + let samples = + checked_training_vector_count(vector_count, ratio).map_err(std::io::Error::other)?; + let batch_rows = (VECTOR_BUFFER_BYTES / checked_std_vector_bytes(1, dimension)?) + .max(1) + .min(vector_count); + let mut buffer = MutableBuffer::new(checked_std_vector_bytes(batch_rows, dimension)?); + let mut sample_buffer = Vec::with_capacity(batch_rows * dimension); + let mut trainer = VectorIndexTrainer::new(config)?; + raw_file.seek(SeekFrom::Start(0))?; + let mut rows_seen = 0; + let mut selected = 0; + while rows_seen < vector_count { + let rows = batch_rows.min(vector_count - rows_seen); + buffer.resize(checked_std_vector_bytes(rows, dimension)?, 0); + raw_file.read_exact(buffer.as_slice_mut())?; + let values = buffer.typed_data::(); + while selected < samples { + let sample = checked_training_sample_index(selected, vector_count, samples) + .map_err(std::io::Error::other)?; + if sample >= rows_seen + rows { + break; + } + let offset = (sample - rows_seen) * dimension; + sample_buffer.extend_from_slice(&values[offset..offset + dimension]); + selected += 1; + } + if !sample_buffer.is_empty() { + trainer.add_training_vectors_mut(&sample_buffer, sample_buffer.len() / dimension)?; + sample_buffer.clear(); + } + rows_seen += rows; + } + if selected != samples { + return Err(std::io::Error::other( + "vindex training sample count mismatch", + )); + } + let training = trainer.finish()?; + let train_finish = train_start.map_or(Duration::ZERO, |start| start.elapsed()); + let mut writer = VectorIndexWriter::new(training); + let mut raw_temp_reread = Duration::ZERO; + let mut index_add = Duration::ZERO; + raw_file.seek(SeekFrom::Start(0))?; + id_file.seek(SeekFrom::Start(0))?; + let mut ids = MutableBuffer::new(batch_rows * std::mem::size_of::()); + let mut rows_added = 0; + while rows_added < vector_count { + let rows = batch_rows.min(vector_count - rows_added); + buffer.resize(checked_std_vector_bytes(rows, dimension)?, 0); + ids.resize(rows * std::mem::size_of::(), 0); + let read_start = timing_enabled.then(Instant::now); + raw_file.read_exact(buffer.as_slice_mut())?; + id_file.read_exact(ids.as_slice_mut())?; + if let Some(start) = read_start { + raw_temp_reread = raw_temp_reread.saturating_add(start.elapsed()); + } + let add_start = timing_enabled.then(Instant::now); + writer.add_vectors(ids.typed_data::(), buffer.typed_data::(), rows)?; + if let Some(start) = add_start { + index_add = index_add.saturating_add(start.elapsed()); + } + rows_added += rows; + } + let mut trailing = [0u8; 1]; + if raw_file.read(&mut trailing)? != 0 || id_file.read(&mut trailing)? != 0 { + return Err(std::io::Error::other( + "temporary vindex vector or row ID file contains trailing bytes", + )); + } + Ok((writer, train_finish, raw_temp_reread, index_add)) +} diff --git a/crates/paimon/src/table/index_build/vector_extraction.rs b/crates/paimon/src/table/index_build/vector_extraction.rs new file mode 100644 index 000000000..1784c85a1 --- /dev/null +++ b/crates/paimon/src/table/index_build/vector_extraction.rs @@ -0,0 +1,346 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::vector_validation::checked_vector_bytes; +use crate::spec::ROW_ID_FIELD_NAME; +use crate::table::RowRange; +use crate::{Error, Result}; +use arrow_array::{Array, FixedSizeListArray, Float32Array, Int64Array, ListArray, RecordBatch}; +use arrow_buffer::ScalarBuffer; +pub(in crate::table) struct ValidatedVectorBatch { + pub(in crate::table) values: ScalarBuffer, + pub(in crate::table) row_ids: ScalarBuffer, + /// Number of non-null vectors, not the number of source rows. + pub(in crate::table) vector_count: usize, + pub(in crate::table) source_rows: usize, +} + +impl ValidatedVectorBatch { + pub(in crate::table) fn bytes(&self) -> &[u8] { + self.values.inner().as_slice() + } +} + +pub(in crate::table) fn contains_null_vectors(batch: &RecordBatch, index_column: &str) -> bool { + batch + .column_by_name(index_column) + .is_some_and(|column| column.null_count() != 0) +} + +pub(in crate::table) fn extract_vector_batch( + batch: &RecordBatch, + index_column: &str, + dimension: usize, +) -> Result { + validate_vector_batch_with(batch, index_column, dimension, |_| Ok(())) +} + +pub(in crate::table) fn validate_vector_batch( + batch: &RecordBatch, + index_column: &str, + dimension: usize, + expected_row_id: &mut i64, +) -> Result { + validate_vector_batch_with(batch, index_column, dimension, |row_id| { + if row_id != *expected_row_id { + return Err(Error::DataInvalid { + message: format!( + "vindex vector extraction expected _ROW_ID {}, got {}", + expected_row_id, row_id + ), + source: None, + }); + } + *expected_row_id = expected_row_id + .checked_add(1) + .ok_or_else(|| Error::DataInvalid { + message: "vindex expected row id overflows i64".to_string(), + source: None, + })?; + Ok(()) + }) +} + +pub(in crate::table) fn validate_vector_batch_ranges( + batch: &RecordBatch, + index_column: &str, + dimension: usize, + ranges: &[RowRange], + range_index: &mut usize, + expected_row_id: &mut i64, +) -> Result { + validate_vector_batch_with(batch, index_column, dimension, |row_id| { + let range = ranges.get(*range_index).ok_or_else(|| Error::DataInvalid { + message: format!("vindex vector extraction got unexpected _ROW_ID {row_id}"), + source: None, + })?; + if row_id != *expected_row_id { + return Err(Error::DataInvalid { + message: format!( + "vindex vector extraction expected _ROW_ID {}, got {}", + expected_row_id, row_id + ), + source: None, + }); + } + if row_id == range.to() { + *range_index += 1; + *expected_row_id = match ranges.get(*range_index) { + Some(next) => next.from(), + None => row_id.checked_add(1).ok_or_else(|| Error::DataInvalid { + message: "vindex expected row id overflows i64".to_string(), + source: None, + })?, + }; + } else { + *expected_row_id = row_id.checked_add(1).ok_or_else(|| Error::DataInvalid { + message: "vindex expected row id overflows i64".to_string(), + source: None, + })?; + } + Ok(()) + }) +} + +fn validate_vector_batch_with( + batch: &RecordBatch, + index_column: &str, + dimension: usize, + mut validate_row_id: impl FnMut(i64) -> Result<()>, +) -> Result { + let vector_index = batch + .schema() + .index_of(index_column) + .map_err(|e| Error::DataInvalid { + message: format!("Vector column '{index_column}' not found in read batch: {e}"), + source: None, + })?; + let row_id_index = + batch + .schema() + .index_of(ROW_ID_FIELD_NAME) + .map_err(|e| Error::DataInvalid { + message: format!("_ROW_ID column not found in read batch: {e}"), + source: None, + })?; + let row_ids = batch + .column(row_id_index) + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::DataInvalid { + message: "vindex vector extraction requires non-null Int64 _ROW_ID".to_string(), + source: None, + })?; + if row_ids.null_count() != 0 { + return Err(Error::DataInvalid { + message: "vindex vector extraction found null _ROW_ID".to_string(), + source: None, + }); + } + for row_id in row_ids.values() { + validate_row_id(*row_id)?; + } + + let column = batch.column(vector_index); + if column.null_count() != 0 { + return compact_nullable_vectors(column.as_ref(), row_ids, dimension, batch.num_rows()); + } + + let (values, start, end) = if let Some(array) = column.as_any().downcast_ref::() { + let offsets = array.value_offsets(); + for offsets in offsets.windows(2) { + let actual = offsets[1] - offsets[0]; + if actual != dimension as i32 { + return Err(Error::DataInvalid { + message: format!( + "vindex vector dimension mismatch: expected {dimension}, got {actual}" + ), + source: None, + }); + } + } + let start = usize::try_from(offsets[0]).map_err(|e| Error::DataInvalid { + message: "vindex vector offset is negative".to_string(), + source: Some(Box::new(e)), + })?; + let end = usize::try_from(offsets[offsets.len() - 1]).map_err(|e| Error::DataInvalid { + message: "vindex vector offset is negative".to_string(), + source: Some(Box::new(e)), + })?; + (array.values(), start, end) + } else if let Some(array) = column.as_any().downcast_ref::() { + let actual = usize::try_from(array.value_length()).map_err(|e| Error::DataInvalid { + message: format!( + "Invalid vindex FixedSizeList dimension: {}", + array.value_length() + ), + source: Some(Box::new(e)), + })?; + if actual != dimension { + return Err(Error::DataInvalid { + message: format!( + "vindex vector dimension mismatch: expected {dimension}, got {actual}" + ), + source: None, + }); + } + let end = batch + .num_rows() + .checked_mul(dimension) + .ok_or_else(|| Error::DataInvalid { + message: "vindex batch vector length overflows usize".to_string(), + source: None, + })?; + (array.values(), 0, end) + } else { + return Err(Error::DataInvalid { + message: + "vindex vector extraction requires Arrow List or FixedSizeList" + .to_string(), + source: None, + }); + }; + let values = values + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::DataInvalid { + message: "vindex vector extraction requires Float32 vector elements".to_string(), + source: None, + })?; + if values.null_count() != 0 + && values + .nulls() + .is_some_and(|nulls| nulls.slice(start, end - start).null_count() != 0) + { + return Err(Error::DataInvalid { + message: "vindex vector extraction found null vector element".to_string(), + source: None, + }); + } + checked_vector_bytes(end - start, 1)?; + Ok(ValidatedVectorBatch { + values: values.values().slice(start, end - start), + row_ids: row_ids.values().clone(), + vector_count: batch.num_rows(), + source_rows: batch.num_rows(), + }) +} + +fn compact_nullable_vectors( + column: &dyn Array, + row_ids: &Int64Array, + dimension: usize, + source_rows: usize, +) -> Result { + let list = column.as_any().downcast_ref::(); + let fixed = column.as_any().downcast_ref::(); + if list.is_none() && fixed.is_none() { + return Err(Error::DataInvalid { + message: + "vindex vector extraction requires Arrow List or FixedSizeList" + .into(), + source: None, + }); + } + if let Some(fixed) = fixed { + if fixed.value_length() as usize != dimension { + return Err(Error::DataInvalid { + message: format!( + "vindex vector dimension mismatch: expected {dimension}, got {}", + fixed.value_length() + ), + source: None, + }); + } + } + let child = list + .map(|list| list.values()) + .or_else(|| fixed.map(|list| list.values())) + .unwrap(); + if !child.as_any().is::() { + return Err(Error::DataInvalid { + message: "vindex vector extraction requires Float32 vector elements".into(), + source: None, + }); + } + let count = source_rows - column.null_count(); + checked_vector_bytes(count, dimension)?; + let mut values = Vec::with_capacity(count * dimension); + let mut ids = Vec::with_capacity(count); + for row in 0..source_rows { + if column.is_null(row) { + continue; + } + let vector = match list { + Some(list) => list.value(row), + None => fixed.unwrap().value(row), + }; + if vector.len() != dimension { + return Err(Error::DataInvalid { + message: format!( + "vindex vector dimension mismatch: expected {dimension}, got {}", + vector.len() + ), + source: None, + }); + } + let vector = vector.as_any().downcast_ref::().unwrap(); + if vector.null_count() != 0 { + return Err(Error::DataInvalid { + message: "vindex vector extraction found null vector element".into(), + source: None, + }); + } + values.extend_from_slice(vector.values()); + ids.push(row_ids.value(row)); + } + Ok(ValidatedVectorBatch { + values: values.into(), + row_ids: ids.into(), + vector_count: count, + source_rows, + }) +} + +pub(in crate::table) fn local_ids( + row_ids: &[i64], + start: i64, + row_count: usize, +) -> Result> { + let end = start + .checked_add(i64::try_from(row_count).map_err(|e| Error::DataInvalid { + message: "vindex row count does not fit i64".to_string(), + source: Some(Box::new(e)), + })?) + .ok_or_else(|| Error::DataInvalid { + message: "vindex row range overflows i64".to_string(), + source: None, + })?; + row_ids + .iter() + .map(|row_id| { + if *row_id < start || *row_id >= end { + Err(Error::DataInvalid { + message: format!("vindex row id {row_id} is outside shard [{start}, {end})"), + source: None, + }) + } else { + Ok(*row_id - start) + } + }) + .collect() +} diff --git a/crates/paimon/src/table/index_build/vector_validation.rs b/crates/paimon/src/table/index_build/vector_validation.rs new file mode 100644 index 000000000..74c073272 --- /dev/null +++ b/crates/paimon/src/table/index_build/vector_validation.rs @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{Error, Result}; +pub(in crate::table) fn checked_vector_bytes(row_count: usize, dimension: usize) -> Result { + row_count + .checked_mul(dimension) + .and_then(|values| values.checked_mul(std::mem::size_of::())) + .ok_or_else(|| Error::DataInvalid { + message: format!( + "vindex vector byte length overflows: row_count={row_count}, dimension={dimension}" + ), + source: None, + }) +} + +pub(in crate::table) fn checked_std_vector_bytes( + row_count: usize, + dimension: usize, +) -> std::io::Result { + row_count + .checked_mul(dimension) + .and_then(|values| values.checked_mul(std::mem::size_of::())) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "vindex vector byte length overflows usize", + ) + }) +} + +pub(in crate::table) fn checked_training_vector_count( + row_count: usize, + ratio: f64, +) -> Result { + if row_count == 0 || !(ratio > 0.0 && ratio <= 1.0) { + return Err(Error::DataInvalid { + message: format!( + "Invalid vindex training sample: row_count={row_count}, ratio={ratio}; expected a positive row count and ratio in (0, 1]" + ), + source: None, + }); + } + Ok(((row_count as f64 * ratio).ceil() as usize).clamp(1, row_count)) +} + +pub(in crate::table) fn checked_training_sample_index( + sample: usize, + rows: usize, + samples: usize, +) -> Result { + sample + .checked_mul(rows / samples) + .and_then(|base| { + sample + .checked_mul(rows % samples) + .and_then(|remainder| base.checked_add(remainder / samples)) + }) + .ok_or_else(|| Error::DataInvalid { + message: "vindex training sample index overflows usize".to_string(), + source: None, + }) +} + +pub(in crate::table) fn checked_i32(value: u64, context: &str) -> Result { + i32::try_from(value).map_err(|_| Error::DataInvalid { + message: format!("{context}: {value}"), + source: None, + }) +} + +pub(in crate::table) fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result { + if row_range_end < row_range_start { + return Err(Error::DataInvalid { + message: format!("Invalid vindex row range [{row_range_start}, {row_range_end}]"), + source: None, + }); + } + row_range_end + .checked_sub(row_range_start) + .and_then(|count| count.checked_add(1)) + .ok_or_else(|| Error::DataInvalid { + message: format!( + "Row count overflows for row range [{row_range_start}, {row_range_end}]" + ), + source: None, + }) +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index ca35f6db4..d267e7cde 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -91,6 +91,7 @@ mod global_index_search_filter; mod global_index_types; mod hybrid_search_builder; mod incremental_scan; +mod index_build; pub(crate) mod index_file_path; mod inline_blob; mod instant; diff --git a/crates/paimon/src/table/pk_index_options.rs b/crates/paimon/src/table/pk_index_options.rs index cca938a4a..cbeb5f816 100644 --- a/crates/paimon/src/table/pk_index_options.rs +++ b/crates/paimon/src/table/pk_index_options.rs @@ -126,29 +126,13 @@ pub(super) fn definitions(schema: &TableSchema) -> Result> { ), }); } - let native = merged - .iter() - .filter_map(|(k, v)| { - k.strip_prefix("full-text.") - .map(|key| (key.to_string(), v.clone())) - }) - .collect(); - paimon_ftindex_core::FullTextIndexConfig::from_options(&native).map_err(invalid)?; - } - "fm" => { - crate::table::sorted_global_index_build_builder::validation::index_key_type( - kind, field, - )?; - crate::fm_index::FMOptions::from_options(&merged)?; + let native = + crate::table::index_build::full_text::native_full_text_options(&merged); + crate::table::index_build::full_text::full_text_config(&native)?; } _ => { - crate::table::sorted_global_index_build_builder::validation::index_key_type( - kind, field, - )?; - crate::table::sorted_global_index_options::SortedIndexWriteOptions::from_options( - kind, &merged, - )?; - schema.core_options().write_buffer_size()?; + crate::table::index_build::fields::index_key_type(kind, field)?; + crate::table::index_build::scalar::ScalarIndexOptions::from_options(kind, &merged)?; } } result.push(Definition { @@ -159,9 +143,3 @@ pub(super) fn definitions(schema: &TableSchema) -> Result> { } Ok(result) } - -fn invalid(error: impl std::fmt::Display) -> Error { - Error::ConfigInvalid { - message: format!("Invalid primary-key index options: {error}"), - } -} diff --git a/crates/paimon/src/table/sorted_global_index_build_builder.rs b/crates/paimon/src/table/sorted_global_index_build_builder.rs index c581fa41f..77e007bcb 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder.rs @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -pub(super) mod extraction; +mod extraction; mod planning; -pub(super) mod sort; -pub(super) mod validation; +mod validation; mod writer; #[cfg(test)] @@ -28,45 +27,24 @@ use planning::plan_sorted_index_shards; use planning::SortedGlobalIndexShard; use validation::*; -use super::bitmap_global_index_format::{make_bitmap_key_comparator, serialize_bitmap_datum}; use super::global_index_build_common::IndexColumns; -use super::global_index_types::{ - normalize_queryable_global_index_type, BITMAP_GLOBAL_INDEX_TYPE, BTREE_GLOBAL_INDEX_TYPE, - FM_GLOBAL_INDEX_TYPE, -}; -use super::sorted_global_index_options::SortedIndexWriteOptions; -use crate::btree::key_serde::KeyComparator; +use super::global_index_types::{normalize_queryable_global_index_type, BTREE_GLOBAL_INDEX_TYPE}; +#[cfg(test)] +use super::global_index_types::{BITMAP_GLOBAL_INDEX_TYPE, FM_GLOBAL_INDEX_TYPE}; +#[cfg(test)] use crate::btree::{make_key_comparator, serialize_datum}; -use crate::fm_index::{FMOptions, FMWriteOptions}; -use crate::spec::{CoreOptions, DataType, Datum, Predicate}; +use crate::spec::{CoreOptions, Predicate}; +#[cfg(test)] +use crate::spec::{DataType, Datum}; +#[cfg(test)] +use crate::table::bitmap_global_index_format::serialize_bitmap_datum; +#[cfg(test)] +use crate::table::index_build::scalar::make_index_key_codec; +use crate::table::index_build::scalar::ScalarIndexOptions; use crate::table::{CommitMessage, RowRange, Table, TableCommit}; use crate::{Error, Result}; use std::collections::HashMap; -const INDEX_DIR: &str = "index"; - -type SortedIndexKeyRow = (Option>, i64); -type SerializeKeyFn = fn(&Datum, &DataType) -> Vec; - -enum GlobalIndexWriteOptions { - Sorted(SortedIndexWriteOptions), - FM(FMWriteOptions), -} - -pub(super) fn make_index_key_codec( - index_type: &str, - data_type: &DataType, -) -> (KeyComparator, SerializeKeyFn) { - match index_type { - BTREE_GLOBAL_INDEX_TYPE => (make_key_comparator(data_type), serialize_datum), - BITMAP_GLOBAL_INDEX_TYPE => ( - make_bitmap_key_comparator(data_type), - serialize_bitmap_datum, - ), - _ => unreachable!("normalized sorted global index type"), - } -} - pub(super) struct SortedGlobalIndexBuildBuilder<'a> { table: &'a Table, index_column: Option, @@ -155,17 +133,7 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { let core_options = CoreOptions::new(&resolved_options); super::global_index_build_common::validate_build_table(self.table)?; let records_per_range = core_options.sorted_index_records_per_range()?; - if index_type != FM_GLOBAL_INDEX_TYPE { - core_options.write_buffer_size()?; - } - let write_options = if index_type == FM_GLOBAL_INDEX_TYPE { - GlobalIndexWriteOptions::FM(FMOptions::from_options(&resolved_options)?.write) - } else { - GlobalIndexWriteOptions::Sorted(SortedIndexWriteOptions::from_options( - index_type, - &resolved_options, - )?) - }; + let write_options = ScalarIndexOptions::from_options(index_type, &resolved_options)?; let index_fields = super::global_index_build_common::resolve_index_fields( self.table, diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/extraction.rs b/crates/paimon/src/table/sorted_global_index_build_builder/extraction.rs index a148127be..c4dc6ba63 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/extraction.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/extraction.rs @@ -15,208 +15,13 @@ // specific language governing permissions and limitations // under the License. -//! Projected table reads and conversion into serialized index rows. - +//! Data Evolution split adapter for the shared native index build input. use super::planning::SortedGlobalIndexShard; -use super::validation::checked_row_count; -use super::{SerializeKeyFn, SortedIndexKeyRow}; -use crate::btree::key_serde::DynKeyComparator; -use crate::spec::{ - extract_datum_from_array, extract_datum_from_arrow, DataField, DataType, ROW_ID_FIELD_NAME, -}; -use crate::table::global_index_types::MULTIVALUE_GLOBAL_INDEX_TYPE; +#[cfg(test)] +pub(super) use crate::table::index_build::extraction::*; use crate::table::stats_filter::group_by_overlapping_row_id; -use crate::table::{DataSplit, DataSplitBuilder, RowRange, Table}; -use crate::{Error, Result}; -use arrow_array::{Array, FixedSizeListArray, Int64Array, LargeListArray, ListArray, RecordBatch}; -use futures::TryStreamExt; -use std::cmp::Ordering; - -pub(super) async fn extract_index_rows( - table: &Table, - shard: &SortedGlobalIndexShard, - index_fields: &[&DataField], - index_type: &str, - serialize_key: SerializeKeyFn, - budget: usize, - cmp: crate::btree::key_serde::KeyComparator, -) -> Result { - let splits = build_read_splits_for_shard(shard)?; - let mut read_builder = table.new_read_builder(); - let mut projection = index_fields - .iter() - .map(|field| field.name()) - .collect::>(); - projection.push(ROW_ID_FIELD_NAME); - read_builder.with_projection(&projection)?; - let read = read_builder.new_read()?; - let mut batches = read.to_arrow(&splits)?; - let fields = std::sync::Arc::new( - index_fields - .iter() - .map(|field| (*field).clone()) - .collect::>(), - ); - let mut sorter = super::sort::IndexSorter::new(budget, cmp); - let mut next_row_id = shard.row_range_start; - while let Some(batch) = batches.try_next().await? { - let fields = fields.clone(); - let index_type = index_type.to_string(); - let row_range_start = shard.row_range_start; - let row_count = i64::try_from(batch.num_rows()).map_err(|_| Error::DataInvalid { - message: "Index batch row count overflow".into(), - source: None, - })?; - // Sorting and local scratch-file I/O must not block the async reader. - sorter = tokio::task::spawn_blocking(move || { - let refs = fields.iter().collect::>(); - let mut emit = |row| sorter.push(row); - let batches = std::slice::from_ref(&batch); - if refs.len() > 1 { - emit_composite_rows( - batches, - &refs, - row_range_start, - row_count, - next_row_id, - &mut emit, - )?; - } else if index_type == MULTIVALUE_GLOBAL_INDEX_TYPE { - let DataType::Array(array_type) = refs[0].data_type() else { - unreachable!("validated multivalue type") - }; - emit_multivalue_rows( - batches, - refs[0].name(), - array_type.element_type(), - row_range_start, - row_count, - serialize_key, - next_row_id, - &mut emit, - )?; - } else { - emit_scalar_rows( - batches, - refs[0].name(), - refs[0].data_type(), - row_range_start, - row_count, - serialize_key, - next_row_id, - &mut emit, - )?; - } - Ok::<_, Error>(sorter) - }) - .await - .map_err(sort_task_error)??; - next_row_id = next_row_id - .checked_add(row_count) - .ok_or_else(|| Error::DataInvalid { - message: "Index row ID overflow".into(), - source: None, - })?; - } - let expected = checked_row_count(shard.row_range_start, shard.row_range_end)?; - if next_row_id - shard.row_range_start != expected { - return Err(Error::DataInvalid { - message: format!( - "Sorted index expected {expected} source rows, got {}", - next_row_id - shard.row_range_start - ), - source: None, - }); - } - tokio::task::spawn_blocking(move || sorter.finish()) - .await - .map_err(sort_task_error)? -} - -fn sort_task_error(error: tokio::task::JoinError) -> Error { - Error::UnexpectedError { - message: format!("Sorted index sort task failed: {error}"), - source: None, - } -} - -#[allow(clippy::too_many_arguments)] -fn emit_composite_rows( - batches: &[RecordBatch], - fields: &[&DataField], - row_range_start: i64, - expected_row_count: i64, - next_row_id: i64, - emit: &mut dyn FnMut(SortedIndexKeyRow) -> Result<()>, -) -> Result<()> { - let fields = fields - .iter() - .map(|field| (*field).clone()) - .collect::>(); - let codec = crate::btree::CompositeKeyCodec::new(&fields); - let mut expected_row_id = next_row_id; - for batch in batches { - let indices = fields - .iter() - .map(|field| batch.schema().index_of(field.name())) - .collect::, _>>() - .map_err(|error| Error::DataInvalid { - message: format!("Composite index column missing: {error}"), - source: None, - })?; - let row_id_index = batch - .schema() - .index_of(ROW_ID_FIELD_NAME) - .map_err(|error| Error::DataInvalid { - message: format!("_ROW_ID column missing: {error}"), - source: None, - })?; - let row_ids = batch - .column(row_id_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "Composite index requires Int64 _ROW_ID".into(), - source: None, - })?; - for row in 0..batch.num_rows() { - if row_ids.is_null(row) || row_ids.value(row) != expected_row_id { - return Err(Error::DataInvalid { - message: format!("Composite index expected non-null _ROW_ID {expected_row_id}"), - source: None, - }); - } - let values = fields - .iter() - .zip(&indices) - .map(|(field, index)| { - extract_datum_from_arrow(batch, row, *index, field.data_type()) - }) - .collect::>>()?; - emit(( - Some(codec.serialize(&values)?), - expected_row_id - row_range_start, - ))?; - expected_row_id = expected_row_id - .checked_add(1) - .ok_or_else(|| Error::DataInvalid { - message: "Composite index row ID overflow".into(), - source: None, - })?; - } - } - if expected_row_id - next_row_id != expected_row_count { - return Err(Error::DataInvalid { - message: format!( - "Composite index expected {expected_row_count} rows, got {}", - expected_row_id - next_row_id - ), - source: None, - }); - } - Ok(()) -} - +use crate::table::{DataSplit, DataSplitBuilder, RowRange}; +use crate::Result; pub(super) fn build_read_splits_for_shard( shard: &SortedGlobalIndexShard, ) -> Result> { @@ -254,309 +59,3 @@ pub(super) fn build_read_splits_for_shard( }) .collect() } - -#[allow(clippy::too_many_arguments)] -pub(in crate::table) fn emit_scalar_rows( - batches: &[RecordBatch], - index_column: &str, - data_type: &DataType, - row_range_start: i64, - expected_row_count: i64, - serialize_key: SerializeKeyFn, - next_row_id: i64, - emit: &mut dyn FnMut(SortedIndexKeyRow) -> Result<()>, -) -> Result<()> { - let mut expected_row_id = next_row_id; - for batch in batches { - let value_index = - batch - .schema() - .index_of(index_column) - .map_err(|e| Error::DataInvalid { - message: format!("Index column '{index_column}' not found in read batch: {e}"), - source: None, - })?; - let row_id_index = - batch - .schema() - .index_of(ROW_ID_FIELD_NAME) - .map_err(|e| Error::DataInvalid { - message: format!("_ROW_ID column not found in read batch: {e}"), - source: None, - })?; - let row_ids = batch - .column(row_id_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "Sorted global index build requires non-null Int64 _ROW_ID".to_string(), - source: None, - })?; - - for row in 0..batch.num_rows() { - if row_ids.is_null(row) { - return Err(Error::DataInvalid { - message: "Sorted global index build found null _ROW_ID".to_string(), - source: None, - }); - } - let row_id = row_ids.value(row); - if row_id != expected_row_id { - return Err(Error::DataInvalid { - message: format!( - "Sorted global index build expected _ROW_ID {}, got {}", - expected_row_id, row_id - ), - source: None, - }); - } - expected_row_id += 1; - - let key = extract_datum_from_arrow(batch, row, value_index, data_type)? - .map(|datum| serialize_key(&datum, data_type)); - emit((key, row_id - row_range_start))?; - } - } - let actual_row_count = expected_row_id - next_row_id; - if actual_row_count != expected_row_count { - return Err(Error::DataInvalid { - message: format!( - "Sorted global index build expected {} rows, got {}", - expected_row_count, actual_row_count - ), - source: None, - }); - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub(in crate::table) fn emit_multivalue_rows( - batches: &[RecordBatch], - index_column: &str, - element_type: &DataType, - row_range_start: i64, - expected_row_count: i64, - serialize_key: SerializeKeyFn, - next_row_id: i64, - emit: &mut dyn FnMut(SortedIndexKeyRow) -> Result<()>, -) -> Result<()> { - let mut expected_row_id = next_row_id; - for batch in batches { - let value_index = - batch - .schema() - .index_of(index_column) - .map_err(|e| Error::DataInvalid { - message: format!("Index column '{index_column}' not found in read batch: {e}"), - source: None, - })?; - let row_id_index = - batch - .schema() - .index_of(ROW_ID_FIELD_NAME) - .map_err(|e| Error::DataInvalid { - message: format!("_ROW_ID column not found in read batch: {e}"), - source: None, - })?; - let row_ids = batch - .column(row_id_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "Multivalue global index build requires non-null Int64 _ROW_ID" - .to_string(), - source: None, - })?; - - #[derive(Clone, Copy)] - enum ArrayLayout<'a> { - List(&'a ListArray), - LargeList(&'a LargeListArray), - Fixed(&'a FixedSizeListArray), - } - let column = batch.column(value_index); - let layout = if let Some(array) = column.as_any().downcast_ref::() { - ArrayLayout::List(array) - } else if let Some(array) = column.as_any().downcast_ref::() { - ArrayLayout::LargeList(array) - } else if let Some(array) = column.as_any().downcast_ref::() { - ArrayLayout::Fixed(array) - } else { - return Err(Error::DataInvalid { - message: format!( - "Multivalue global index extraction requires an Arrow list column, got {:?}", - column.data_type() - ), - source: None, - }); - }; - let values = match layout { - ArrayLayout::List(array) => array.values(), - ArrayLayout::LargeList(array) => array.values(), - ArrayLayout::Fixed(array) => array.values(), - }; - - for row in 0..batch.num_rows() { - if row_ids.is_null(row) { - return Err(Error::DataInvalid { - message: "Multivalue global index build found null _ROW_ID".to_string(), - source: None, - }); - } - let row_id = row_ids.value(row); - if row_id != expected_row_id { - return Err(Error::DataInvalid { - message: format!( - "Multivalue global index build expected _ROW_ID {}, got {}", - expected_row_id, row_id - ), - source: None, - }); - } - expected_row_id += 1; - - let is_null = match layout { - ArrayLayout::List(array) => array.is_null(row), - ArrayLayout::LargeList(array) => array.is_null(row), - ArrayLayout::Fixed(array) => array.is_null(row), - }; - if is_null { - continue; - } - let (start, end) = match layout { - ArrayLayout::List(array) => { - let offsets = array.value_offsets(); - ( - usize::try_from(offsets[row]), - usize::try_from(offsets[row + 1]), - ) - } - ArrayLayout::LargeList(array) => { - let offsets = array.value_offsets(); - ( - usize::try_from(offsets[row]), - usize::try_from(offsets[row + 1]), - ) - } - ArrayLayout::Fixed(array) => { - let start = usize::try_from(array.value_offset(row)); - let end = usize::try_from(array.value_offset(row) + array.value_length()); - (start, end) - } - }; - let (start, end) = match (start, end) { - (Ok(start), Ok(end)) => (start, end), - _ => { - return Err(Error::DataInvalid { - message: "Multivalue global index found a negative array offset" - .to_string(), - source: None, - }) - } - }; - for element_index in start..end { - if let Some(datum) = - extract_datum_from_array(values, element_index, value_index, element_type)? - { - emit(( - Some(serialize_key(&datum, element_type)), - row_id - row_range_start, - ))?; - } - } - } - } - - let actual_row_count = expected_row_id - next_row_id; - if actual_row_count != expected_row_count { - return Err(Error::DataInvalid { - message: format!( - "Multivalue global index build expected {} source rows, got {}", - expected_row_count, actual_row_count - ), - source: None, - }); - } - Ok(()) -} - -/// Sort the extracted rows with the index's key comparator. -/// -/// Both keys come from [`super::make_index_key_codec`] for the column's current type, so -/// a comparison failure is a real defect rather than an index built before a type change; -/// it aborts the build instead of degrading. -pub(super) fn sort_index_rows( - rows: &mut [SortedIndexKeyRow], - cmp: &DynKeyComparator<'_>, -) -> Result<()> { - let mut failure = None; - rows.sort_unstable_by(|left, right| match (&left.0, &right.0) { - (None, None) => left.1.cmp(&right.1), - (None, Some(_)) => Ordering::Less, - (Some(_), None) => Ordering::Greater, - (Some(left_key), Some(right_key)) => match cmp(left_key, right_key) { - Ok(order) => order.then_with(|| left.1.cmp(&right.1)), - Err(error) => { - failure.get_or_insert(error); - left.1.cmp(&right.1) - } - }, - }); - match failure { - Some(error) => Err(error), - None => Ok(()), - } -} - -#[cfg(test)] -pub(super) fn extract_index_rows_from_batches( - batches: &[RecordBatch], - column: &str, - data_type: &DataType, - start: i64, - count: i64, - serialize: SerializeKeyFn, -) -> Result> { - let mut rows = Vec::new(); - emit_scalar_rows( - batches, - column, - data_type, - start, - count, - serialize, - start, - &mut |row| { - rows.push(row); - Ok(()) - }, - )?; - Ok(rows) -} - -#[cfg(test)] -pub(super) fn extract_multivalue_index_rows_from_batches( - batches: &[RecordBatch], - column: &str, - data_type: &DataType, - start: i64, - count: i64, - serialize: SerializeKeyFn, -) -> Result> { - let mut rows = Vec::new(); - emit_multivalue_rows( - batches, - column, - data_type, - start, - count, - serialize, - start, - &mut |row| { - rows.push(row); - Ok(()) - }, - )?; - Ok(rows) -} diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/tests.rs b/crates/paimon/src/table/sorted_global_index_build_builder/tests.rs index 37a0dd71f..e7cfd7298 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/tests.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/tests.rs @@ -16,6 +16,85 @@ // under the License. use super::*; + +#[tokio::test] +async fn direct_scalar_builder_normalizes_native_type_and_manifest_together() { + use crate::spec::IndexManifestEntry; + use crate::table::global_index_scanner::GlobalIndexScanner; + for (configured, canonical) in [("BTREE", "btree"), ("Bitmap", "bitmap"), ("FM", "fm")] { + let table = test_table_with_path( + &format!("memory:/scalar_type_case/{}", uuid::Uuid::new_v4()), + table_options("10"), + ); + setup_dirs(&table).await; + let mut write = TableWrite::new(&table, "case-source".into()).unwrap(); + write + .write_arrow_batch(&data_batch(vec![1, 2, 3], vec!["alice", "bob", "alice"])) + .await + .unwrap(); + TableCommit::new(table.clone(), "case-source".into()) + .commit(write.prepare_commit().await.unwrap()) + .await + .unwrap(); + let mut build = SortedGlobalIndexBuildBuilder::new(&table); + build + .with_index_columns(&["name"]) + .with_index_type(configured) + .with_options(HashMap::from([( + "fm-index.sa-sample-rate".into(), + "1".into(), + )])); + let (_, messages) = build.prepare().await.unwrap(); + let entries = messages + .iter() + .flat_map(|message| { + message.new_index_files.iter().map(|file| { + assert_eq!(file.index_type, canonical); + assert!(file + .file_name + .starts_with(&format!("{canonical}-global-index-"))); + IndexManifestEntry { + kind: FileKind::Add, + partition: message.partition.clone(), + bucket: message.bucket, + index_file: file.clone(), + version: 1, + } + }) + }) + .collect::>(); + let scanner = GlobalIndexScanner::create_with_fm_options( + table.file_io(), + table.location(), + 1, + i64::MAX, + 128 * 1024 * 1024, + i64::MAX, + &entries, + table.schema().fields(), + crate::fm_index::FMReadOptions { + locate_cost_ratio: 1.0, + ..Default::default() + }, + ) + .unwrap() + .unwrap(); + let predicates = PredicateBuilder::new(table.schema().fields()); + let predicate = if canonical == "fm" { + predicates + .contains("name", Datum::String("bo".into())) + .unwrap() + } else { + predicates + .equal("name", Datum::String("bob".into())) + .unwrap() + }; + assert_eq!( + scanner.matching_ranges(&predicate).await.unwrap(), + Some(vec![RowRange::new(1, 1)]) + ); + } +} use crate::btree::{BTreeFileFooter, BTreeIndexMeta}; use crate::catalog::Identifier; use crate::io::FileIOBuilder; @@ -42,14 +121,23 @@ use std::sync::Arc; /// rejected, and an inverted or overflowing range is still an error. #[test] fn checked_row_count_spans_beyond_i32() { + let checked_row_count = |start, end| { + // Persisted ranges can bypass RowRange::new's ordering assertion. + let range = serde_json::from_value::(serde_json::json!({ + "from": start, + "to": end, + })) + .unwrap(); + crate::table::index_build::row_count(&range) + }; let start = 0; let end = i64::from(i32::MAX) + 10; assert_eq!( - super::checked_row_count(start, end).unwrap(), + checked_row_count(start, end).unwrap(), i64::from(i32::MAX) + 11 ); - assert!(super::checked_row_count(5, 4).is_err()); - assert!(super::checked_row_count(i64::MIN, i64::MAX).is_err()); + assert!(checked_row_count(5, 4).is_err()); + assert!(checked_row_count(i64::MIN, i64::MAX).is_err()); } fn data_file(name: &str, first_row_id: Option, row_count: i64) -> DataFileMeta { diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/validation.rs b/crates/paimon/src/table/sorted_global_index_build_builder/validation.rs index c8a8ed183..78740c1e3 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/validation.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/validation.rs @@ -17,10 +17,12 @@ //! Build-option, field-type, and row-range validation. -use crate::spec::{DataField, DataType}; -use crate::table::global_index_types::{FM_GLOBAL_INDEX_TYPE, MULTIVALUE_GLOBAL_INDEX_TYPE}; +#[cfg(test)] +use crate::spec::DataField; +pub(super) use crate::table::index_build::fields::*; #[cfg(test)] use crate::table::Table; +#[cfg(test)] use crate::{Error, Result}; #[cfg(test)] @@ -36,107 +38,6 @@ pub(super) fn find_index_field<'a>(table: &'a Table, column: &str) -> Result<&'a }) } -pub(super) fn validate_btree_field(field: &DataField) -> Result<()> { - if !is_btree_supported_data_type(field.data_type()) { - return Err(Error::Unsupported { - message: format!( - "Sorted global index only supports scalar columns, got {:?} for column '{}'", - field.data_type(), - field.name() - ), - }); - } - Ok(()) -} - -pub(in crate::table) fn index_key_type<'a>( - index_type: &str, - field: &'a DataField, -) -> Result<&'a DataType> { - if index_type == MULTIVALUE_GLOBAL_INDEX_TYPE { - let DataType::Array(array_type) = field.data_type() else { - return Err(Error::Unsupported { - message: format!( - "Multivalue global index requires an ARRAY column, got {:?} for column '{}'", - field.data_type(), - field.name() - ), - }); - }; - if !is_btree_supported_data_type(array_type.element_type()) { - return Err(Error::Unsupported { - message: format!( - "Multivalue global index does not support array element type {:?} for column '{}'", - array_type.element_type(), - field.name() - ), - }); - } - Ok(array_type.element_type()) - } else if index_type == FM_GLOBAL_INDEX_TYPE { - if !matches!(field.data_type(), DataType::Char(_) | DataType::VarChar(_)) { - return Err(Error::Unsupported { - message: format!( - "FM global index requires a character string column, got {:?} for column '{}'", - field.data_type(), - field.name() - ), - }); - } - Ok(field.data_type()) - } else { - validate_btree_field(field)?; - Ok(field.data_type()) - } -} - -pub(super) fn is_btree_supported_data_type(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Boolean(_) - | DataType::TinyInt(_) - | DataType::SmallInt(_) - | DataType::Int(_) - | DataType::BigInt(_) - | DataType::Decimal(_) - | DataType::Double(_) - | DataType::Float(_) - | DataType::Char(_) - | DataType::VarChar(_) - | DataType::Date(_) - | DataType::LocalZonedTimestamp(_) - | DataType::Time(_) - | DataType::Timestamp(_) - ) -} - -pub(super) fn checked_i64(value: u64, context: &str) -> Result { - i64::try_from(value).map_err(|_| Error::DataInvalid { - message: format!("{context}: {value}"), - source: None, - }) -} - -pub(super) fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result { - if row_range_end < row_range_start { - return Err(Error::DataInvalid { - message: format!( - "Invalid sorted global index row range [{row_range_start}, {row_range_end}]" - ), - source: None, - }); - } - row_range_end - .checked_sub(row_range_start) - .and_then(|span| span.checked_add(1)) - .ok_or_else(|| Error::DataInvalid { - message: format!( - "Row count overflows for row range [{row_range_start}, {row_range_end}]" - ), - source: None, - }) -} - pub(super) fn ranges_overlap( left_start: i64, left_end: i64, diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/writer.rs b/crates/paimon/src/table/sorted_global_index_build_builder/writer.rs index 83ec21c1c..9c817a7ca 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/writer.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/writer.rs @@ -15,381 +15,59 @@ // specific language governing permissions and limitations // under the License. -//! Type-specific index-file writing and failure cleanup. - -use super::extraction::{build_read_splits_for_shard, extract_index_rows}; +//! Adapt a captured DE shard to the shared native index writer. +use super::extraction::build_read_splits_for_shard; use super::planning::SortedGlobalIndexShard; -use super::validation::{checked_i64, checked_row_count, index_key_type}; -use super::{ - make_index_key_codec, GlobalIndexWriteOptions, SerializeKeyFn, SortedGlobalIndexBuildBuilder, - INDEX_DIR, -}; -use crate::btree::BTreeIndexWriter; -use crate::fm_index::{FMGlobalIndexWriter, FMWriteOptions}; -use crate::io::FileWrite; -use crate::spec::{ - extract_datum_from_arrow, DataField, DataType, GlobalIndexMeta, IndexFileMeta, RowType, - ROW_ID_FIELD_NAME, -}; -use crate::table::bitmap_global_index_writer::{BitmapGlobalIndexWriter, BitmapWriteResult}; -use crate::table::global_index_types::{ - normalize_queryable_global_index_type, BITMAP_GLOBAL_INDEX_TYPE, BTREE_GLOBAL_INDEX_TYPE, - FM_GLOBAL_INDEX_TYPE, MULTIVALUE_GLOBAL_INDEX_TYPE, -}; -use crate::table::Table; -use crate::{Error, Result}; -use arrow_array::{Array, Int64Array}; -use futures::TryStreamExt; +use super::{ScalarIndexOptions, SortedGlobalIndexBuildBuilder}; +use crate::spec::{DataField, GlobalIndexMeta, IndexFileMeta, ROW_ID_FIELD_NAME}; +use crate::table::index_build::{scalar, IndexFileWriter}; +use crate::table::index_file_path::IndexFileLocation; +use crate::table::RowRange; +use crate::Result; impl SortedGlobalIndexBuildBuilder<'_> { pub(super) async fn build_index_file( &self, shard: &SortedGlobalIndexShard, - index_fields: &[&DataField], - write_options: &GlobalIndexWriteOptions, + fields: &[&DataField], + options: &ScalarIndexOptions, ) -> Result { - let index_type = normalize_queryable_global_index_type(&self.index_type).ok_or_else(|| { - Error::Unsupported { - message: format!( - "Scalar global index build only supports index_type => 'btree', 'bitmap', 'multivalue', or 'fm', got '{}'", - self.index_type - ), - } - })?; - let row_count = checked_row_count(shard.row_range_start, shard.row_range_end)?; - let index_field = index_fields[0]; - let index_column = index_field.name(); - let composite_type = DataType::Row(RowType::new( - index_fields.iter().map(|field| (*field).clone()).collect(), - )); - let key_type = if index_fields.len() > 1 { - &composite_type - } else { - index_key_type(index_type, index_field)? - }; - let codec_type = if matches!( - index_type, - MULTIVALUE_GLOBAL_INDEX_TYPE | FM_GLOBAL_INDEX_TYPE - ) { - BITMAP_GLOBAL_INDEX_TYPE - } else { - index_type - }; - let (cmp, serialize_key) = make_index_key_codec(codec_type, key_type); - let mut rows = if index_type == FM_GLOBAL_INDEX_TYPE { - super::sort::SortedRows::Memory(Vec::new().into_iter()) - } else { - let mut options = self.table.schema().options().clone(); - options.extend(self.options.clone()); - let budget = crate::spec::CoreOptions::new(&options).write_buffer_size()?; - extract_index_rows( - self.table, - shard, - index_fields, - index_type, - serialize_key, - budget, - make_index_key_codec(codec_type, key_type).0, - ) - .await? + let kind = options.index_type(); + let splits = build_read_splits_for_shard(shard)?; + let mut read = self.table.new_read_builder(); + let mut projection = fields.iter().map(|f| f.name()).collect::>(); + projection.push(ROW_ID_FIELD_NAME); + read.with_projection(&projection)?; + let stream = read.new_read()?.to_arrow(&splits)?; + let range = RowRange::new(shard.row_range_start, shard.row_range_end); + let meta = GlobalIndexMeta { + row_range_start: range.from(), + row_range_end: range.to(), + index_field_id: fields[0].id(), + extra_field_ids: (fields.len() > 1) + .then(|| fields.iter().skip(1).map(|f| f.id()).collect()), + source_meta: Some( + crate::spec::DataEvolutionIndexSourceMeta::new(shard.snapshot_id)?.serialize(), + ), + index_meta: None, }; - - let file_name = format!("{index_type}-global-index-{}.index", uuid::Uuid::new_v4()); - // Global index placement is a table property, as in Java GlobalIndexFileFactory. - let external_path = super::super::external_path::new_index_external_path( - self.table.schema().options(), - false, + IndexFileWriter::new( + self.table, + kind, + format!("{}-global-index-{}.index", kind, uuid::Uuid::new_v4()), + IndexFileLocation::Global { + table_path: self.table.location().trim_end_matches('/'), + }, "", - &file_name, - )?; - let index_path = external_path.clone().unwrap_or_else(|| { - format!( - "{}/{INDEX_DIR}/{}", - self.table.location().trim_end_matches('/'), - file_name, - ) - }); - if let Some((directory, _)) = index_path.rsplit_once('/') { - self.table.file_io().mkdirs(directory).await?; - } - let write_result: Result<(u64, Vec, i64)> = async { - let output = self.table.file_io().new_output(&index_path)?; - let writer = output.writer().await?; - let (written_row_count, index_meta) = match index_type { - BTREE_GLOBAL_INDEX_TYPE => { - let GlobalIndexWriteOptions::Sorted(write_options) = write_options else { - unreachable!("BTree uses sorted write options") - }; - let mut writer = BTreeIndexWriter::with_comparator_and_options( - writer, - write_options.block_size, - write_options.compression_type, - write_options.compression_level, - write_options.bloom_filter_enabled, - cmp, - ) - .with_file_version(write_options.btree_file_version) - .map_err(|error| Error::DataInvalid { - message: "Invalid BTree file version".to_string(), - source: Some(Box::new(error)), - })?; - while let Some((key, local_row_id)) = rows.next().await? { - writer - .write(key.as_deref(), local_row_id) - .await - .map_err(|e| Error::DataInvalid { - message: format!( - "Failed to write BTree global index file '{file_name}'" - ), - source: Some(Box::new(e)), - })?; - } - let write_result = writer.finish().await.map_err(|e| Error::DataInvalid { - message: format!("Failed to finish BTree global index file '{file_name}'"), - source: Some(Box::new(e)), - })?; - (write_result.row_count, write_result.meta.serialize()) - } - BITMAP_GLOBAL_INDEX_TYPE => { - let GlobalIndexWriteOptions::Sorted(write_options) = write_options else { - unreachable!("bitmap uses sorted write options") - }; - let mut writer = BitmapGlobalIndexWriter::with_compression_level( - writer, - write_options.block_size, - write_options.compression_type, - write_options.compression_level, - cmp, - ); - while let Some((key, local_row_id)) = rows.next().await? { - writer.write(key.as_deref(), local_row_id).map_err(|e| { - Error::DataInvalid { - message: format!( - "Failed to write bitmap global index file '{file_name}'" - ), - source: Some(Box::new(e)), - } - })?; - } - let BitmapWriteResult { row_count, meta } = - writer.finish().await.map_err(|e| Error::DataInvalid { - message: format!( - "Failed to finish bitmap global index file '{file_name}'" - ), - source: Some(Box::new(e)), - })?; - (row_count, meta.serialize()) - } - MULTIVALUE_GLOBAL_INDEX_TYPE => { - let GlobalIndexWriteOptions::Sorted(write_options) = write_options else { - unreachable!("multivalue uses sorted write options") - }; - let mut writer = BitmapGlobalIndexWriter::with_compression_level( - writer, - write_options.block_size, - write_options.compression_type, - write_options.compression_level, - cmp, - ); - while let Some((key, local_row_id)) = rows.next().await? { - let key = key - .as_deref() - .expect("multivalue extraction skips null keys"); - writer.write_posting(key, local_row_id).map_err(|e| { - Error::DataInvalid { - message: format!( - "Failed to write multivalue global index file '{file_name}'" - ), - source: Some(Box::new(e)), - } - })?; - } - let BitmapWriteResult { row_count, meta } = writer - .finish_with_source_row_count(u64::try_from(row_count).unwrap()) - .await - .map_err(|e| Error::DataInvalid { - message: format!( - "Failed to finish multivalue global index file '{file_name}'" - ), - source: Some(Box::new(e)), - })?; - (row_count, meta.serialize()) - } - FM_GLOBAL_INDEX_TYPE => { - let GlobalIndexWriteOptions::FM(write_options) = write_options else { - unreachable!("FM uses FM write options") - }; - write_fm_index_streaming( - self.table, - shard, - index_column, - index_field, - serialize_key, - writer, - *write_options, - &file_name, - ) - .await? - } - _ => unreachable!("normalized queryable global index type"), - }; - - if written_row_count != u64::try_from(row_count).unwrap() { - return Err(Error::DataInvalid { - message: format!( - "Sorted global index expected {} rows, wrote {}", - row_count, written_row_count - ), - source: None, - }); - } - - let status = self.table.file_io().get_status(&index_path).await?; - let file_size = checked_i64( - status.size, - "Index file is too large for Rust IndexFileMeta", - )?; - Ok((written_row_count, index_meta, file_size)) - } - .await; - let (_, index_meta, file_size) = match write_result { - Ok(result) => result, - Err(error) => { - let _ = self.table.file_io().delete_file(&index_path).await; - return Err(error); - } - }; - Ok(IndexFileMeta { - index_type: index_type.to_string(), - file_name, - file_size, - row_count, - deletion_vectors_ranges: None, - external_path, - global_index_meta: Some(GlobalIndexMeta { - row_range_start: shard.row_range_start, - row_range_end: shard.row_range_end, - index_field_id: index_field.id(), - extra_field_ids: (index_fields.len() > 1).then(|| { - index_fields - .iter() - .skip(1) - .map(|field| field.id()) - .collect() - }), - source_meta: Some( - crate::spec::DataEvolutionIndexSourceMeta::new(shard.snapshot_id)?.serialize(), - ), - index_meta: Some(index_meta), - }), - }) - } -} - -#[allow(clippy::too_many_arguments)] -async fn write_fm_index_streaming( - table: &Table, - shard: &SortedGlobalIndexShard, - index_column: &str, - index_field: &DataField, - serialize_key: SerializeKeyFn, - output: Box, - write_options: FMWriteOptions, - file_name: &str, -) -> Result<(u64, Vec)> { - let mut writer = - FMGlobalIndexWriter::new(output, write_options).map_err(|error| Error::DataInvalid { - message: format!("Failed to create FM global index file '{file_name}'"), - source: Some(Box::new(error)), - })?; - let splits = build_read_splits_for_shard(shard)?; - let mut read_builder = table.new_read_builder(); - read_builder.with_projection(&[index_column, ROW_ID_FIELD_NAME])?; - let read = read_builder.new_read()?; - let mut batches = read.to_arrow(&splits)?; - let expected_row_count = checked_row_count(shard.row_range_start, shard.row_range_end)?; - let mut expected_row_id = shard.row_range_start; - - while let Some(batch) = batches.try_next().await? { - let value_index = - batch - .schema() - .index_of(index_column) - .map_err(|error| Error::DataInvalid { - message: format!( - "Index column '{index_column}' not found in FM read batch: {error}" - ), - source: None, - })?; - let row_id_index = batch - .schema() - .index_of(ROW_ID_FIELD_NAME) - .map_err(|error| Error::DataInvalid { - message: format!("_ROW_ID column not found in FM read batch: {error}"), - source: None, - })?; - let row_ids = batch - .column(row_id_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "FM global index build requires non-null Int64 _ROW_ID".to_string(), - source: None, - })?; - - for row in 0..batch.num_rows() { - if row_ids.is_null(row) { - return Err(Error::DataInvalid { - message: "FM global index build found null _ROW_ID".to_string(), - source: None, - }); - } - let row_id = row_ids.value(row); - if row_id != expected_row_id { - return Err(Error::DataInvalid { - message: format!( - "FM global index build expected _ROW_ID {expected_row_id}, got {row_id}" - ), - source: None, - }); - } - expected_row_id = expected_row_id - .checked_add(1) - .ok_or_else(|| Error::DataInvalid { - message: "FM global index row ID overflow".to_string(), - source: None, - })?; - let local_row_id = - u64::try_from(row_id - shard.row_range_start).map_err(|_| Error::DataInvalid { - message: format!( - "FM global index file '{file_name}' has a negative local row ID" - ), - source: None, - })?; - let key = extract_datum_from_arrow(&batch, row, value_index, index_field.data_type())? - .map(|datum| serialize_key(&datum, index_field.data_type())); - writer - .write(key.as_deref(), local_row_id) + meta, + ) + .await? + .write(self.table, |output| async move { + scalar::write_payload(stream, output, fields, range, options) .await - .map_err(|error| Error::DataInvalid { - message: format!("Failed to write FM global index file '{file_name}'"), - source: Some(Box::new(error)), - })?; - } - } - - let actual_row_count = expected_row_id - shard.row_range_start; - if actual_row_count != expected_row_count { - return Err(Error::DataInvalid { - message: format!( - "FM global index build expected {expected_row_count} rows, got {actual_row_count}" - ), - source: None, - }); + .map(Some) + }) + .await + .map(|file| file.expect("a scalar shard contains source rows")) } - let result = writer.finish().await.map_err(|error| Error::DataInvalid { - message: format!("Failed to finish FM global index file '{file_name}'"), - source: Some(Box::new(error)), - })?; - Ok((result.row_count, result.index_meta)) } diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs b/crates/paimon/src/table/vindex_index_build_builder.rs index e6ed47a7c..3b7210944 100644 --- a/crates/paimon/src/table/vindex_index_build_builder.rs +++ b/crates/paimon/src/table/vindex_index_build_builder.rs @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -pub(super) mod extraction; +mod extraction; mod pipeline; mod planning; pub(super) mod timing; mod validation; -pub(super) mod writer; +mod writer; use planning::plan_vindex_shards; use validation::{checked_i32, find_index_field, validate_vector_field}; diff --git a/crates/paimon/src/table/vindex_index_build_builder/extraction.rs b/crates/paimon/src/table/vindex_index_build_builder/extraction.rs index e14fd205c..b99ee7ced 100644 --- a/crates/paimon/src/table/vindex_index_build_builder/extraction.rs +++ b/crates/paimon/src/table/vindex_index_build_builder/extraction.rs @@ -16,13 +16,9 @@ // under the License. use super::planning::VindexIndexShard; -use super::validation::checked_vector_bytes; -use crate::spec::ROW_ID_FIELD_NAME; +pub(super) use crate::table::index_build::vector_extraction::*; use crate::table::{DataSplit, DataSplitBuilder, RowRange}; -use crate::{Error, Result}; -use arrow_array::{Array, FixedSizeListArray, Float32Array, Int64Array, ListArray, RecordBatch}; -use arrow_buffer::ScalarBuffer; - +use crate::Result; pub(super) fn data_split_for_shard(shard: &VindexIndexShard) -> Result { data_split_for_shard_ranges( shard, @@ -47,323 +43,3 @@ pub(super) fn data_split_for_shard_ranges( .with_row_ranges(row_ranges) .build() } - -pub(in crate::table) struct ValidatedVectorBatch { - pub(in crate::table) values: ScalarBuffer, - pub(in crate::table) row_ids: ScalarBuffer, - /// Number of non-null vectors, not the number of source rows. - pub(in crate::table) vector_count: usize, - pub(in crate::table) source_rows: usize, -} - -impl ValidatedVectorBatch { - pub(super) fn bytes(&self) -> &[u8] { - self.values.inner().as_slice() - } -} - -pub(super) fn contains_null_vectors(batch: &RecordBatch, index_column: &str) -> bool { - batch - .column_by_name(index_column) - .is_some_and(|column| column.null_count() != 0) -} - -pub(super) fn extract_vector_batch( - batch: &RecordBatch, - index_column: &str, - dimension: usize, -) -> Result { - validate_vector_batch_with(batch, index_column, dimension, |_| Ok(())) -} - -pub(in crate::table) fn validate_vector_batch( - batch: &RecordBatch, - index_column: &str, - dimension: usize, - expected_row_id: &mut i64, -) -> Result { - validate_vector_batch_with(batch, index_column, dimension, |row_id| { - if row_id != *expected_row_id { - return Err(Error::DataInvalid { - message: format!( - "vindex vector extraction expected _ROW_ID {}, got {}", - expected_row_id, row_id - ), - source: None, - }); - } - *expected_row_id = expected_row_id - .checked_add(1) - .ok_or_else(|| Error::DataInvalid { - message: "vindex expected row id overflows i64".to_string(), - source: None, - })?; - Ok(()) - }) -} - -pub(super) fn validate_vector_batch_ranges( - batch: &RecordBatch, - index_column: &str, - dimension: usize, - ranges: &[RowRange], - range_index: &mut usize, - expected_row_id: &mut i64, -) -> Result { - validate_vector_batch_with(batch, index_column, dimension, |row_id| { - let range = ranges.get(*range_index).ok_or_else(|| Error::DataInvalid { - message: format!("vindex vector extraction got unexpected _ROW_ID {row_id}"), - source: None, - })?; - if row_id != *expected_row_id { - return Err(Error::DataInvalid { - message: format!( - "vindex vector extraction expected _ROW_ID {}, got {}", - expected_row_id, row_id - ), - source: None, - }); - } - if row_id == range.to() { - *range_index += 1; - *expected_row_id = match ranges.get(*range_index) { - Some(next) => next.from(), - None => row_id.checked_add(1).ok_or_else(|| Error::DataInvalid { - message: "vindex expected row id overflows i64".to_string(), - source: None, - })?, - }; - } else { - *expected_row_id = row_id.checked_add(1).ok_or_else(|| Error::DataInvalid { - message: "vindex expected row id overflows i64".to_string(), - source: None, - })?; - } - Ok(()) - }) -} - -fn validate_vector_batch_with( - batch: &RecordBatch, - index_column: &str, - dimension: usize, - mut validate_row_id: impl FnMut(i64) -> Result<()>, -) -> Result { - let vector_index = batch - .schema() - .index_of(index_column) - .map_err(|e| Error::DataInvalid { - message: format!("Vector column '{index_column}' not found in read batch: {e}"), - source: None, - })?; - let row_id_index = - batch - .schema() - .index_of(ROW_ID_FIELD_NAME) - .map_err(|e| Error::DataInvalid { - message: format!("_ROW_ID column not found in read batch: {e}"), - source: None, - })?; - let row_ids = batch - .column(row_id_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "vindex vector extraction requires non-null Int64 _ROW_ID".to_string(), - source: None, - })?; - if row_ids.null_count() != 0 { - return Err(Error::DataInvalid { - message: "vindex vector extraction found null _ROW_ID".to_string(), - source: None, - }); - } - for row_id in row_ids.values() { - validate_row_id(*row_id)?; - } - - let column = batch.column(vector_index); - if column.null_count() != 0 { - return compact_nullable_vectors(column.as_ref(), row_ids, dimension, batch.num_rows()); - } - - let (values, start, end) = if let Some(array) = column.as_any().downcast_ref::() { - let offsets = array.value_offsets(); - for offsets in offsets.windows(2) { - let actual = offsets[1] - offsets[0]; - if actual != dimension as i32 { - return Err(Error::DataInvalid { - message: format!( - "vindex vector dimension mismatch: expected {dimension}, got {actual}" - ), - source: None, - }); - } - } - let start = usize::try_from(offsets[0]).map_err(|e| Error::DataInvalid { - message: "vindex vector offset is negative".to_string(), - source: Some(Box::new(e)), - })?; - let end = usize::try_from(offsets[offsets.len() - 1]).map_err(|e| Error::DataInvalid { - message: "vindex vector offset is negative".to_string(), - source: Some(Box::new(e)), - })?; - (array.values(), start, end) - } else if let Some(array) = column.as_any().downcast_ref::() { - let actual = usize::try_from(array.value_length()).map_err(|e| Error::DataInvalid { - message: format!( - "Invalid vindex FixedSizeList dimension: {}", - array.value_length() - ), - source: Some(Box::new(e)), - })?; - if actual != dimension { - return Err(Error::DataInvalid { - message: format!( - "vindex vector dimension mismatch: expected {dimension}, got {actual}" - ), - source: None, - }); - } - let end = batch - .num_rows() - .checked_mul(dimension) - .ok_or_else(|| Error::DataInvalid { - message: "vindex batch vector length overflows usize".to_string(), - source: None, - })?; - (array.values(), 0, end) - } else { - return Err(Error::DataInvalid { - message: - "vindex vector extraction requires Arrow List or FixedSizeList" - .to_string(), - source: None, - }); - }; - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::DataInvalid { - message: "vindex vector extraction requires Float32 vector elements".to_string(), - source: None, - })?; - if values.null_count() != 0 - && values - .nulls() - .is_some_and(|nulls| nulls.slice(start, end - start).null_count() != 0) - { - return Err(Error::DataInvalid { - message: "vindex vector extraction found null vector element".to_string(), - source: None, - }); - } - checked_vector_bytes(end - start, 1)?; - Ok(ValidatedVectorBatch { - values: values.values().slice(start, end - start), - row_ids: row_ids.values().clone(), - vector_count: batch.num_rows(), - source_rows: batch.num_rows(), - }) -} - -fn compact_nullable_vectors( - column: &dyn Array, - row_ids: &Int64Array, - dimension: usize, - source_rows: usize, -) -> Result { - let list = column.as_any().downcast_ref::(); - let fixed = column.as_any().downcast_ref::(); - if list.is_none() && fixed.is_none() { - return Err(Error::DataInvalid { - message: - "vindex vector extraction requires Arrow List or FixedSizeList" - .into(), - source: None, - }); - } - if let Some(fixed) = fixed { - if fixed.value_length() as usize != dimension { - return Err(Error::DataInvalid { - message: format!( - "vindex vector dimension mismatch: expected {dimension}, got {}", - fixed.value_length() - ), - source: None, - }); - } - } - let child = list - .map(|list| list.values()) - .or_else(|| fixed.map(|list| list.values())) - .unwrap(); - if !child.as_any().is::() { - return Err(Error::DataInvalid { - message: "vindex vector extraction requires Float32 vector elements".into(), - source: None, - }); - } - let count = source_rows - column.null_count(); - checked_vector_bytes(count, dimension)?; - let mut values = Vec::with_capacity(count * dimension); - let mut ids = Vec::with_capacity(count); - for row in 0..source_rows { - if column.is_null(row) { - continue; - } - let vector = match list { - Some(list) => list.value(row), - None => fixed.unwrap().value(row), - }; - if vector.len() != dimension { - return Err(Error::DataInvalid { - message: format!( - "vindex vector dimension mismatch: expected {dimension}, got {}", - vector.len() - ), - source: None, - }); - } - let vector = vector.as_any().downcast_ref::().unwrap(); - if vector.null_count() != 0 { - return Err(Error::DataInvalid { - message: "vindex vector extraction found null vector element".into(), - source: None, - }); - } - values.extend_from_slice(vector.values()); - ids.push(row_ids.value(row)); - } - Ok(ValidatedVectorBatch { - values: values.into(), - row_ids: ids.into(), - vector_count: count, - source_rows, - }) -} - -pub(super) fn local_ids(row_ids: &[i64], start: i64, row_count: usize) -> Result> { - let end = start - .checked_add(i64::try_from(row_count).map_err(|e| Error::DataInvalid { - message: "vindex row count does not fit i64".to_string(), - source: Some(Box::new(e)), - })?) - .ok_or_else(|| Error::DataInvalid { - message: "vindex row range overflows i64".to_string(), - source: None, - })?; - row_ids - .iter() - .map(|row_id| { - if *row_id < start || *row_id >= end { - Err(Error::DataInvalid { - message: format!("vindex row id {row_id} is outside shard [{start}, {end})"), - source: None, - }) - } else { - Ok(*row_id - start) - } - }) - .collect() -} diff --git a/crates/paimon/src/table/vindex_index_build_builder/validation.rs b/crates/paimon/src/table/vindex_index_build_builder/validation.rs index 298cbc427..76ecf381c 100644 --- a/crates/paimon/src/table/vindex_index_build_builder/validation.rs +++ b/crates/paimon/src/table/vindex_index_build_builder/validation.rs @@ -21,7 +21,7 @@ use crate::table::global_index_build_common::vector::{ validate_vector_field as validate_common_vector_field, }; use crate::table::Table; -use crate::{Error, Result}; +use crate::Result; pub(super) fn find_index_field<'a>(table: &'a Table, column: &str) -> Result<&'a DataField> { find_vector_index_field(table, column) @@ -31,91 +31,4 @@ pub(super) fn validate_vector_field(field: &DataField) -> Result<()> { validate_common_vector_field(field, "vindex") } -pub(super) fn checked_vector_bytes(row_count: usize, dimension: usize) -> Result { - row_count - .checked_mul(dimension) - .and_then(|values| values.checked_mul(std::mem::size_of::())) - .ok_or_else(|| Error::DataInvalid { - message: format!( - "vindex vector byte length overflows: row_count={row_count}, dimension={dimension}" - ), - source: None, - }) -} - -pub(super) fn checked_std_vector_bytes( - row_count: usize, - dimension: usize, -) -> std::io::Result { - row_count - .checked_mul(dimension) - .and_then(|values| values.checked_mul(std::mem::size_of::())) - .ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "vindex vector byte length overflows usize", - ) - }) -} - -pub(super) fn checked_training_vector_count(row_count: usize, ratio: f64) -> Result { - if row_count == 0 || !(ratio > 0.0 && ratio <= 1.0) { - return Err(Error::DataInvalid { - message: format!( - "Invalid vindex training sample: row_count={row_count}, ratio={ratio}; expected a positive row count and ratio in (0, 1]" - ), - source: None, - }); - } - Ok(((row_count as f64 * ratio).ceil() as usize).clamp(1, row_count)) -} - -pub(super) fn checked_training_sample_index( - sample: usize, - rows: usize, - samples: usize, -) -> Result { - sample - .checked_mul(rows / samples) - .and_then(|base| { - sample - .checked_mul(rows % samples) - .and_then(|remainder| base.checked_add(remainder / samples)) - }) - .ok_or_else(|| Error::DataInvalid { - message: "vindex training sample index overflows usize".to_string(), - source: None, - }) -} - -pub(super) fn checked_i32(value: u64, context: &str) -> Result { - i32::try_from(value).map_err(|_| Error::DataInvalid { - message: format!("{context}: {value}"), - source: None, - }) -} - -pub(super) fn checked_i64(value: u64, context: &str) -> Result { - i64::try_from(value).map_err(|_| Error::DataInvalid { - message: format!("{context}: {value}"), - source: None, - }) -} - -pub(super) fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result { - if row_range_end < row_range_start { - return Err(Error::DataInvalid { - message: format!("Invalid vindex row range [{row_range_start}, {row_range_end}]"), - source: None, - }); - } - row_range_end - .checked_sub(row_range_start) - .and_then(|count| count.checked_add(1)) - .ok_or_else(|| Error::DataInvalid { - message: format!( - "Row count overflows for row range [{row_range_start}, {row_range_end}]" - ), - source: None, - }) -} +pub(super) use crate::table::index_build::vector_validation::*; diff --git a/crates/paimon/src/table/vindex_index_build_builder/writer.rs b/crates/paimon/src/table/vindex_index_build_builder/writer.rs index 4d2294c3f..661e730fe 100644 --- a/crates/paimon/src/table/vindex_index_build_builder/writer.rs +++ b/crates/paimon/src/table/vindex_index_build_builder/writer.rs @@ -15,32 +15,28 @@ // specific language governing permissions and limitations // under the License. -use super::extraction::{data_split_for_shard, local_ids, validate_vector_batch}; +use super::extraction::data_split_for_shard; use super::pipeline::GranuleBuildOutcome; use super::planning::VindexIndexShard; use super::timing::{vector_index_build_timing_enabled, VectorIndexBuildTiming}; -use super::validation::{ - checked_i64, checked_row_count, checked_std_vector_bytes, checked_training_sample_index, - checked_training_vector_count, checked_vector_bytes, -}; +use super::validation::checked_row_count; use super::VindexIndexBuildBuilder; use crate::spec::{GlobalIndexMeta, IndexFileMeta, ROW_ID_FIELD_NAME}; use crate::table::data_file_reader::DataFileReadTiming; use crate::table::table_read::configured_parquet_read_budget; use crate::vindex::{VindexVectorIndexOptions, DISKANN_IDENTIFIER}; use crate::{Error, Result}; -use arrow_buffer::MutableBuffer; use futures::TryStreamExt; -use paimon_vindex_core::autotune::default_training_vector_count; -use paimon_vindex_core::index::{VectorIndexConfig, VectorIndexTrainer, VectorIndexWriter}; -use paimon_vindex_core::io::PosWriter; -use std::io::{Read, Seek, SeekFrom}; +use paimon_vindex_core::index::VectorIndexWriter; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::io::AsyncWriteExt; -use tokio_util::io::SyncIoBridge; -const VECTOR_BUFFER_BYTES: usize = 8 * 1024 * 1024; +use crate::table::index_build::{ + vector::{serialize_writer, TrainedVectors, VectorSpill}, + IndexFileWriter, IndexPayload, +}; +use crate::table::index_file_path::IndexFileLocation; +use crate::table::RowRange; pub(super) struct BuiltIndexFile { pub(super) meta: IndexFileMeta, @@ -140,15 +136,10 @@ impl<'a> VindexIndexBuildBuilder<'a> { message: format!("Invalid vindex dimension: {dimension}"), source: Some(Box::new(e)), })?; - if dimension_usize == 0 { - return Err(Error::DataInvalid { - message: "vindex vector dimension must be positive".to_string(), - source: None, - }); - } - checked_vector_bytes(row_count_usize, dimension_usize)?; - let mut raw_file = temporary_vector_file("vectors")?; - let mut id_file = temporary_vector_file("row IDs")?; + let mut spill = VectorSpill::new( + RowRange::new(shard.row_range_start, shard.row_range_end), + dimension_usize, + )?; let split = data_split_for_shard(shard)?; let mut read_builder = self.table.new_read_builder(); read_builder.with_projection(&[index_column, ROW_ID_FIELD_NAME])?; @@ -162,10 +153,6 @@ impl<'a> VindexIndexBuildBuilder<'a> { None => read, }; let mut batches = read.to_arrow(&[split])?; - let mut expected_row_id = shard.row_range_start; - let mut rows_seen = 0usize; - let mut vector_count = 0usize; - let mut bytes_written = 0usize; loop { let source_start = timing_enabled.then(Instant::now); let batch = batches.try_next().await?; @@ -174,105 +161,24 @@ impl<'a> VindexIndexBuildBuilder<'a> { } let Some(batch) = batch else { break }; batch_count += 1; - let vectors = - validate_vector_batch(&batch, index_column, dimension_usize, &mut expected_row_id)?; - rows_seen = - rows_seen - .checked_add(vectors.source_rows) - .ok_or_else(|| Error::DataInvalid { - message: "vindex streamed row count overflows usize".into(), - source: None, - })?; - vector_count = vector_count - .checked_add(vectors.vector_count) - .ok_or_else(|| Error::DataInvalid { - message: "vindex streamed vector count overflows usize".into(), - source: None, - })?; - let ids: arrow_buffer::ScalarBuffer = - local_ids(&vectors.row_ids, shard.row_range_start, row_count_usize)?.into(); - let raw_write_start = timing_enabled.then(Instant::now); - raw_file - .write_all(vectors.bytes()) - .await - .map_err(spill_error)?; - id_file - .write_all(ids.inner().as_slice()) - .await - .map_err(spill_error)?; - if let Some(start) = raw_write_start { - raw_temp_write = raw_temp_write.saturating_add(start.elapsed()); - } - bytes_written = bytes_written - .checked_add(vectors.bytes().len()) - .ok_or_else(|| Error::DataInvalid { - message: "vindex spilled byte count overflows usize".into(), - source: None, - })?; + raw_temp_write = raw_temp_write.saturating_add( + spill + .push_batch(&batch, index_column, timing_enabled) + .await?, + ); } - let expected_end = - shard - .row_range_end - .checked_add(1) - .ok_or_else(|| Error::DataInvalid { - message: "vindex row range end overflows i64".into(), - source: None, - })?; - if rows_seen != row_count_usize - || expected_row_id != expected_end - || bytes_written != checked_vector_bytes(vector_count, dimension_usize)? - { - return Err(Error::DataInvalid { - message: format!("vindex streamed data mismatch: rows={rows_seen}/{row_count_usize}, vectors={vector_count}, bytes={bytes_written}"), source: None, - }); - } - // Like Java NativeVectorGlobalIndexWriter.finish: no vectors means no file. - if vector_count == 0 { + let bytes_written = spill.bytes_written(); + let Some(spill) = spill.finish().await? else { return Ok(None); - } - raw_file.flush().await.map_err(spill_error)?; - id_file.flush().await.map_err(spill_error)?; - if raw_file.metadata().await.map_err(spill_error)?.len() != bytes_written as u64 - || id_file.metadata().await.map_err(spill_error)?.len() - != checked_vector_bytes(vector_count, 2)? as u64 - { - return Err(Error::DataInvalid { - message: "temporary vindex vector or row ID file size mismatch".into(), - source: None, - }); - } - let raw_file = raw_file.into_std().await; - let id_file = id_file.into_std().await; - let training_vector_count = - checked_training_vector_count(vector_count, options.train_sample_ratio)?; - let config = options.training_config(vector_count)?; - let training_rows_retained = if timing_enabled { - default_training_vector_count(training_vector_count, config.nlist()).unwrap_or(0) - } else { - 0 }; - let ratio = options.train_sample_ratio; - let (writer, train_finish, raw_temp_reread, index_add) = - tokio::task::spawn_blocking(move || { - train_and_add_spilled_vectors( - raw_file, - id_file, - config, - vector_count, - dimension_usize, - ratio, - timing_enabled, - ) - }) - .await - .map_err(|e| Error::UnexpectedError { - message: format!("vindex training task failed: {e}"), - source: None, - })? - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to train or add vectors to vindex index: {e}"), - source: Some(Box::new(e)), - })?; + let TrainedVectors { + writer, + train_finish, + raw_temp_reread, + index_add, + training_vector_count, + training_rows_retained, + } = spill.train(options, timing_enabled).await?; let serialize_upload_start = timing_enabled.then(Instant::now); let meta = self @@ -339,162 +245,35 @@ impl<'a> VindexIndexBuildBuilder<'a> { self.index_type, uuid::Uuid::new_v4() ); - let (index_path, external_path) = - crate::table::global_index_build_common::prepare_index_file_path( - self.table, &file_name, - ) - .await?; - let write_result = async { - let async_writer = self - .table - .file_io() - .new_output(&index_path)? - .async_writer() - .await?; - let mut output = SyncIoBridge::new(async_writer); - tokio::task::spawn_blocking(move || -> std::io::Result<()> { - let mut writer = writer; - writer.write(&mut PosWriter::new(&mut output))?; - output.shutdown() - }) - .await - .map_err(|e| Error::UnexpectedError { - message: format!("vindex serialization task failed: {e}"), - source: None, - })? - .map_err(|e| Error::UnexpectedError { - message: format!("Failed to stream vindex index: {e}"), - source: Some(Box::new(e)), - })?; - self.table.file_io().get_status(&index_path).await - } - .await; - let status = match write_result { - Ok(status) => status, - Err(error) => { - let _ = self.table.file_io().delete_file(&index_path).await; - return Err(error); - } + let meta = GlobalIndexMeta { + row_range_start: shard.row_range_start, + row_range_end: shard.row_range_end, + index_field_id, + extra_field_ids: None, + source_meta: Some( + crate::spec::DataEvolutionIndexSourceMeta::new(shard.snapshot_id)?.serialize(), + ), + index_meta: None, }; - Ok(IndexFileMeta { - index_type: self.index_type.clone(), + IndexFileWriter::new( + self.table, + &self.index_type, file_name, - file_size: checked_i64( - status.size, - "Index file is too large for Rust IndexFileMeta", - )?, - row_count, - deletion_vectors_ranges: None, - external_path, - global_index_meta: Some(GlobalIndexMeta { - row_range_start: shard.row_range_start, - row_range_end: shard.row_range_end, - index_field_id, - extra_field_ids: None, - source_meta: Some( - crate::spec::DataEvolutionIndexSourceMeta::new(shard.snapshot_id)?.serialize(), - ), - index_meta: Some(index_meta), - }), - }) - } -} - -fn spill_error(error: std::io::Error) -> Error { - Error::UnexpectedError { - message: format!("Failed to spill vindex vectors: {error}"), - source: Some(Box::new(error)), - } -} - -fn temporary_vector_file(kind: &str) -> Result { - tempfile::tempfile() - .map(tokio::fs::File::from_std) - .map_err(|error| Error::UnexpectedError { - message: format!("Failed to create temporary vindex {kind} file: {error}"), - source: Some(Box::new(error)), + IndexFileLocation::Global { + table_path: self.table.location().trim_end_matches('/'), + }, + "", + meta, + ) + .await? + .write(self.table, |output| async move { + serialize_writer(writer, output).await?; + Ok(Some(IndexPayload { + row_count, + index_meta, + })) }) -} - -/// Java samples the compacted non-null vector stream, then adds the original relative IDs. -pub(in crate::table) fn train_and_add_spilled_vectors( - mut raw_file: std::fs::File, - mut id_file: std::fs::File, - config: VectorIndexConfig, - vector_count: usize, - dimension: usize, - ratio: f64, - timing_enabled: bool, -) -> std::io::Result<(VectorIndexWriter, Duration, Duration, Duration)> { - let train_start = timing_enabled.then(Instant::now); - let samples = - checked_training_vector_count(vector_count, ratio).map_err(std::io::Error::other)?; - let batch_rows = (VECTOR_BUFFER_BYTES / checked_std_vector_bytes(1, dimension)?) - .max(1) - .min(vector_count); - let mut buffer = MutableBuffer::new(checked_std_vector_bytes(batch_rows, dimension)?); - let mut sample_buffer = Vec::with_capacity(batch_rows * dimension); - let mut trainer = VectorIndexTrainer::new(config)?; - raw_file.seek(SeekFrom::Start(0))?; - let mut rows_seen = 0; - let mut selected = 0; - while rows_seen < vector_count { - let rows = batch_rows.min(vector_count - rows_seen); - buffer.resize(checked_std_vector_bytes(rows, dimension)?, 0); - raw_file.read_exact(buffer.as_slice_mut())?; - let values = buffer.typed_data::(); - while selected < samples { - let sample = checked_training_sample_index(selected, vector_count, samples) - .map_err(std::io::Error::other)?; - if sample >= rows_seen + rows { - break; - } - let offset = (sample - rows_seen) * dimension; - sample_buffer.extend_from_slice(&values[offset..offset + dimension]); - selected += 1; - } - if !sample_buffer.is_empty() { - trainer.add_training_vectors_mut(&sample_buffer, sample_buffer.len() / dimension)?; - sample_buffer.clear(); - } - rows_seen += rows; - } - if selected != samples { - return Err(std::io::Error::other( - "vindex training sample count mismatch", - )); - } - let training = trainer.finish()?; - let train_finish = train_start.map_or(Duration::ZERO, |start| start.elapsed()); - let mut writer = VectorIndexWriter::new(training); - let mut raw_temp_reread = Duration::ZERO; - let mut index_add = Duration::ZERO; - raw_file.seek(SeekFrom::Start(0))?; - id_file.seek(SeekFrom::Start(0))?; - let mut ids = MutableBuffer::new(batch_rows * std::mem::size_of::()); - let mut rows_added = 0; - while rows_added < vector_count { - let rows = batch_rows.min(vector_count - rows_added); - buffer.resize(checked_std_vector_bytes(rows, dimension)?, 0); - ids.resize(rows * std::mem::size_of::(), 0); - let read_start = timing_enabled.then(Instant::now); - raw_file.read_exact(buffer.as_slice_mut())?; - id_file.read_exact(ids.as_slice_mut())?; - if let Some(start) = read_start { - raw_temp_reread = raw_temp_reread.saturating_add(start.elapsed()); - } - let add_start = timing_enabled.then(Instant::now); - writer.add_vectors(ids.typed_data::(), buffer.typed_data::(), rows)?; - if let Some(start) = add_start { - index_add = index_add.saturating_add(start.elapsed()); - } - rows_added += rows; - } - let mut trailing = [0u8; 1]; - if raw_file.read(&mut trailing)? != 0 || id_file.read(&mut trailing)? != 0 { - return Err(std::io::Error::other( - "temporary vindex vector or row ID file contains trailing bytes", - )); + .await + .map(|file| file.expect("trained vector payload")) } - Ok((writer, train_finish, raw_temp_reread, index_add)) } From 550dd81778a6a11089c4774504a45bd2d7762f8d Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 14:01:20 +0800 Subject: [PATCH 03/14] fix(index): align PK index maintenance and reads --- .../src/table/compact/primary_key/indexes.rs | 77 +++- .../compact/primary_key/indexes/writer.rs | 19 +- crates/paimon/src/table/index_build/vector.rs | 2 +- crates/paimon/src/table/mod.rs | 1 + crates/paimon/src/table/pk_index_options.rs | 40 +- crates/paimon/src/table/pk_index_schema.rs | 50 ++ .../paimon/src/table/pk_sorted_index_scan.rs | 52 ++- .../src/table/pk_vector_bucket_split.rs | 6 +- crates/paimon/src/table/pk_vector_scan.rs | 63 ++- .../src/table/pk_vector_search_params.rs | 4 +- crates/paimon/src/table/vector_scan.rs | 3 +- crates/paimon/src/vindex/pkvector/bucket.rs | 29 +- crates/paimon/tests/pk_compact/indexes.rs | 430 +++++++++++++++++- docs/src/compaction.md | 6 + 14 files changed, 748 insertions(+), 34 deletions(-) create mode 100644 crates/paimon/src/table/pk_index_schema.rs diff --git a/crates/paimon/src/table/compact/primary_key/indexes.rs b/crates/paimon/src/table/compact/primary_key/indexes.rs index b81ca77cd..b9ba5515d 100644 --- a/crates/paimon/src/table/compact/primary_key/indexes.rs +++ b/crates/paimon/src/table/compact/primary_key/indexes.rs @@ -147,6 +147,12 @@ fn compatible(table: &Table, entry: &IndexManifestEntry, definition: &Definition if entry.index_file.index_type != definition.index_type { return Ok(false); } + if crate::vindex::is_vindex_index_type(&definition.index_type) + && entry.index_file.row_count == 0 + && entry.index_file.file_size == 0 + { + return Ok(true); + } if definition.index_type == "full-text" || crate::vindex::is_vindex_index_type(&definition.index_type) { @@ -167,11 +173,36 @@ struct Maintenance<'a> { builds: Vec<(i32, &'a Definition)>, } +async fn unsupported_scalar_units( + table: &Table, + levels: &Levels, + defs: &[Definition], +) -> HashSet<(i32, i32)> { + let mut unsupported = HashSet::new(); + for (level, files) in levels { + let files = files.iter().collect::>(); + for definition in defs.iter().filter(|d| { + matches!( + d.index_type.as_str(), + "btree" | "bitmap" | "multivalue" | "fm" + ) + }) { + if !crate::table::pk_index_schema::source_types_match(table, &definition.field, &files) + .await + { + unsupported.insert((*level, definition.field.id())); + } + } + } + unsupported +} + fn plan<'a>( table: &Table, indexes: &[IndexManifestEntry], levels: &Levels, defs: &'a [Definition], + unsupported: &HashSet<(i32, i32)>, partition: &[u8], bucket: i32, ) -> Result> { @@ -205,7 +236,9 @@ fn plan<'a>( let valid = entries.len() == 1 && expected.as_ref() == Some(&source(entries[0])?) && match definition { - Some(d) => compatible(table, entries[0], d)?, + Some(d) => { + !unsupported.contains(&(*level, *field)) && compatible(table, entries[0], d)? + } None => true, }; if !valid { @@ -221,6 +254,9 @@ fn plan<'a>( continue; } for definition in defs { + if unsupported.contains(&(*level, definition.field.id())) { + continue; + } let valid = existing .get(&(*level, definition.field.id())) .is_some_and(|entries| { @@ -273,7 +309,16 @@ pub(in crate::table::compact) async fn prepare( })) .collect::>(); let after = levels(active, &changes, &task.partition, task.bucket); - let maintenance = plan(table, &indexes, &after, &defs, &task.partition, task.bucket)?; + let unsupported = unsupported_scalar_units(table, &after, &defs).await; + let maintenance = plan( + table, + &indexes, + &after, + &defs, + &unsupported, + &task.partition, + task.bucket, + )?; let mut entries = maintenance.deletes; for (level, definition) in maintenance.builds { match writer::build(table, split, &after[&level], definition, resources.clone()).await { @@ -347,7 +392,16 @@ pub(in crate::table) async fn resolve( let mut after_levels = BTreeMap::new(); for (partition, bucket) in &buckets { let after = levels(active, changes, partition, *bucket); - let maintenance = plan(table, &captured, &after, &defs, partition, *bucket)?; + let unsupported = unsupported_scalar_units(table, &after, &defs).await; + let maintenance = plan( + table, + &captured, + &after, + &defs, + &unsupported, + partition, + *bucket, + )?; required_deletes.extend(maintenance.deletes); for (level, definition) in maintenance.builds { required_builds.insert((partition.clone(), *bucket, level, definition.field.id())); @@ -407,10 +461,18 @@ pub(in crate::table) async fn resolve( let files = &after_levels[&(key.0.clone(), key.1)][&key.2]; let meta = entry.index_file.global_index_meta.as_ref().unwrap(); let count = row_count(files)?; + let indexed_rows = entry.index_file.row_count; + let valid_count = if crate::vindex::is_vindex_index_type(&definition.index_type) { + indexed_rows >= 0 + && indexed_rows <= count + && ((indexed_rows == 0) == (entry.index_file.file_size == 0)) + } else { + indexed_rows == count + }; if source(entry)? != source_meta(key.2, files)? || meta.row_range_start != 0 || meta.row_range_end != count - 1 - || entry.index_file.row_count != count + || !valid_count || entry.index_file.file_size < 0 || !compatible(table, entry, definition)? { @@ -419,12 +481,7 @@ pub(in crate::table) async fn resolve( )); } } - // All-NULL vector levels intentionally have no payload. - if required_builds.iter().any(|key| { - !defs - .iter() - .any(|d| d.field.id() == key.3 && crate::vindex::is_vindex_index_type(&d.index_type)) - }) { + if !required_builds.is_empty() { return Err(invalid("Missing PK index output")); } if prepared.is_empty() { diff --git a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs index 92c8324fb..68a477952 100644 --- a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs +++ b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs @@ -162,15 +162,28 @@ pub(super) async fn build( kind, &definition.field, )?; - vector::write_payload( + let payload = vector::write_payload( stream, - output, + output.clone(), definition.field.name(), &native, range, index_build::serialize_options(&native.native_options)?, ) - .await + .await?; + let payload = match payload { + Some(payload) => payload, + None => { + // Java persists an empty segment to record completed + // coverage; readers never open its native payload. + output.write(bytes::Bytes::new()).await?; + index_build::IndexPayload { + row_count: 0, + index_meta: Vec::new(), + } + } + }; + Ok(Some(payload)) } kind => scalar::write_payload( stream, diff --git a/crates/paimon/src/table/index_build/vector.rs b/crates/paimon/src/table/index_build/vector.rs index 844e4188e..6042ac850 100644 --- a/crates/paimon/src/table/index_build/vector.rs +++ b/crates/paimon/src/table/index_build/vector.rs @@ -251,7 +251,6 @@ pub(in crate::table) async fn write_payload( range: RowRange, index_meta: Vec, ) -> Result> { - let row_count = super::row_count(&range)?; let mut spill = VectorSpill::new(range, options.dimension())?; while let Some(batch) = batches.try_next().await? { spill.push_batch(&batch, column, false).await?; @@ -259,6 +258,7 @@ pub(in crate::table) async fn write_payload( let Some(spill) = spill.finish().await? else { return Ok(None); }; + let row_count = i64::try_from(spill.count).map_err(error)?; let trained = spill.train(options, false).await?; serialize_writer(trained.writer, output).await?; Ok(Some(IndexPayload { diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index d267e7cde..4c798d27f 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -118,6 +118,7 @@ mod pk_full_text_bucket_state; mod pk_full_text_read; mod pk_full_text_scan; mod pk_index_options; +mod pk_index_schema; mod pk_search_position; mod pk_search_ranker; mod pk_sorted_index_scan; diff --git a/crates/paimon/src/table/pk_index_options.rs b/crates/paimon/src/table/pk_index_options.rs index cbeb5f816..e41d8a35f 100644 --- a/crates/paimon/src/table/pk_index_options.rs +++ b/crates/paimon/src/table/pk_index_options.rs @@ -45,12 +45,41 @@ pub(super) fn field_options( parse_field_index_options(options, column, &key, &prefix, family == "vector")?; if family == "vector" { let metric = CoreOptions::new(options).primary_key_vector_distance_metric(column)?; - crate::vindex::pkvector::metric::VectorSearchMetric::parse(&metric)?; + let expected = crate::vindex::pkvector::metric::VectorSearchMetric::parse(&metric)?; resolved.insert(format!("{index_type}.metric"), metric); + // Native field aliases outrank the algorithm key within this map. Do + // not let them contradict the metric used for ANN/exact result merging. + for key in [ + format!("fields.{column}.metric"), + format!("fields.{column}.distance.metric"), + format!("{index_type}.distance.metric"), + ] { + if let Some(value) = resolved.get(&key) { + if crate::vindex::pkvector::metric::VectorSearchMetric::parse(value)? != expected { + return Err(Error::ConfigInvalid { + message: format!( + "{key} conflicts with fields.{column}.pk-vector.distance.metric" + ), + }); + } + } + } } Ok(resolved) } +pub(super) fn vector_index_type( + options: &HashMap, + column: &str, +) -> Result<&'static str> { + let configured = CoreOptions::new(options).primary_key_vector_index_type(column)?; + normalize_global_index_type(configured.trim()) + .filter(|kind| crate::vindex::is_vindex_index_type(kind)) + .ok_or_else(|| Error::ConfigInvalid { + message: format!("Unsupported primary-key vector index type: {configured}"), + }) +} + pub(super) fn definitions(schema: &TableSchema) -> Result> { let options = schema.options(); let mut owners = HashMap::new(); @@ -86,14 +115,7 @@ pub(super) fn definitions(schema: &TableSchema) -> Result> { continue; }; let kind = if family == "vector" { - let configured = schema - .core_options() - .primary_key_vector_index_type(field.name())?; - normalize_global_index_type(configured.trim()) - .filter(|kind| crate::vindex::is_vindex_index_type(kind)) - .ok_or_else(|| Error::ConfigInvalid { - message: format!("Unsupported primary-key vector index type: {configured}"), - })? + vector_index_type(options, field.name())? } else { family }; diff --git a/crates/paimon/src/table/pk_index_schema.rs b/crates/paimon/src/table/pk_index_schema.rs new file mode 100644 index 000000000..9bf038d1e --- /dev/null +++ b/crates/paimon/src/table/pk_index_schema.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Physical source schemas are the codec contract for source-backed scalar indexes. + +use crate::spec::{DataField, DataFileMeta}; +use crate::table::Table; +use std::collections::HashMap; + +/// A scalar payload does not record its key type. Never build cast keys on top +/// of unchanged sources, or later reverting a column type could misread them. +/// Reads use the same proof and fall back when any source schema is unknown. +pub(super) async fn source_types_match( + table: &Table, + field: &DataField, + files: &[&DataFileMeta], +) -> bool { + let manager = table.schema_manager(); + let mut schemas = HashMap::new(); + for file in files { + if file.schema_id == table.schema().id() { + continue; + } + if let std::collections::hash_map::Entry::Vacant(entry) = schemas.entry(file.schema_id) { + entry.insert(manager.schema(file.schema_id).await.ok()); + } + if !schemas[&file.schema_id].as_ref().is_some_and(|schema| { + schema.fields().iter().any(|old| { + old.id() == field.id() && old.data_type().equals_ignore_nullable(field.data_type()) + }) + }) { + return false; + } + } + true +} diff --git a/crates/paimon/src/table/pk_sorted_index_scan.rs b/crates/paimon/src/table/pk_sorted_index_scan.rs index a4c56bfd2..705ed631f 100644 --- a/crates/paimon/src/table/pk_sorted_index_scan.rs +++ b/crates/paimon/src/table/pk_sorted_index_scan.rs @@ -427,7 +427,57 @@ pub(super) async fn refine( definitions: &[Definition], predicates: &[Predicate], ) -> Result> { - let (groups, sources) = plan_groups(table, &splits, entries, definitions); + // Source frames contain file names and counts, but no key type. A declaration + // may have been removed before an allowed type change and then re-added. + // Equal-width encodings (for example INT/FLOAT) cannot be distinguished by + // the native reader. Writers prove one unchanged codec across the complete + // level. Check every visible active source here; retired/stats-pruned sources + // keep their ordinal offsets without preventing partial-coverage reads. + let files = splits + .iter() + .flat_map(|split| { + split.data_files().iter().map(move |file| { + ( + ( + split.partition().to_serialized_bytes(), + split.bucket(), + file.file_name.as_str(), + ), + file, + ) + }) + }) + .collect::>(); + let mut compatible = Vec::new(); + for entry in entries { + let Some(meta) = &entry.index_file.global_index_meta else { + continue; + }; + let Some(field) = table + .schema() + .fields() + .iter() + .find(|field| field.id() == meta.index_field_id) + else { + continue; + }; + let Ok(source) = PrimaryKeyIndexSourceMeta::from_global_index_meta(meta) else { + continue; + }; + let source_files = source + .source_files() + .iter() + .filter_map(|source| { + files + .get(&(entry.partition.clone(), entry.bucket, source.file_name())) + .copied() + }) + .collect::>(); + if super::pk_index_schema::source_types_match(table, field, &source_files).await { + compatible.push(entry.clone()); + } + } + let (groups, sources) = plan_groups(table, &splits, &compatible, definitions); if groups.is_empty() { return Ok(splits); } diff --git a/crates/paimon/src/table/pk_vector_bucket_split.rs b/crates/paimon/src/table/pk_vector_bucket_split.rs index 0b10e6fa5..e8117129f 100644 --- a/crates/paimon/src/table/pk_vector_bucket_split.rs +++ b/crates/paimon/src/table/pk_vector_bucket_split.rs @@ -130,8 +130,8 @@ impl BucketVectorPayload { /// Consume the payload into the pieces a planner needs, so its decoded metadata /// moves out of the payload rather than being cloned out of it. /// - /// Two decoded fields are deliberately left behind. `row_count` is the payload's - /// own row count, which the read path derives from the source metadata instead. + /// Preserve the payload's vector count: Java records empty segments with zero + /// rows, independently of the physical row range in their source metadata. /// `deletion_vectors_ranges` belongs to deletion-vector index files -- Java /// builds a vector payload through the overload that leaves it null, and a read /// takes its deletion vectors from the bucket's data split -- so a value here @@ -142,6 +142,7 @@ impl BucketVectorPayload { index_type: self.index_type, file_name: self.file_name, file_size: self.file_size, + row_count: self.row_count, external_path: self.external_path, global_index_meta: self.global_index_meta, } @@ -157,6 +158,7 @@ pub(crate) struct BucketVectorPayloadParts { /// As decoded: Java writes a signed length, so a negative value is possible on /// the wire and is rejected where it is converted, not here. pub(crate) file_size: i64, + pub(crate) row_count: i64, pub(crate) external_path: Option, pub(crate) global_index_meta: GlobalIndexMeta, } diff --git a/crates/paimon/src/table/pk_vector_scan.rs b/crates/paimon/src/table/pk_vector_scan.rs index 7e3921ce4..6d08f96b6 100644 --- a/crates/paimon/src/table/pk_vector_scan.rs +++ b/crates/paimon/src/table/pk_vector_scan.rs @@ -46,6 +46,7 @@ struct UnresolvedAnnSegment { file_name: String, external_path: Option, file_size: u64, + row_count: i64, index_meta: Vec, } @@ -410,6 +411,7 @@ impl Scan for PkVectorScan { file_name: entry.index_file.file_name.clone(), external_path: entry.index_file.external_path.clone(), file_size, + row_count: entry.index_file.row_count, index_meta: gim.index_meta.clone().unwrap_or_default(), }, )); @@ -551,6 +553,7 @@ fn plan_from_bucket_splits( file_name: parts.file_name, external_path: parts.external_path, file_size, + row_count: parts.row_count, index_meta, }, )); @@ -623,7 +626,7 @@ pub(super) fn plan_from_search_splits( ) -> crate::Result { let snapshot_id = snapshot.as_ref().map_or(0, Snapshot::id); let core = table.schema().core_options(); - let index_type = core.primary_key_vector_index_type(column)?; + let index_type = super::pk_index_options::vector_index_type(table.schema().options(), column)?; let field_id = crate::table::find_field_id_by_name(table.schema().fields(), column) .ok_or_else(|| data_invalid("PK-vector column does not exist"))?; let partition = filter @@ -732,6 +735,7 @@ pub(super) fn plan_from_search_splits( external_path: file.external_path.clone(), file_size: u64::try_from(file.file_size) .map_err(|_| data_invalid("Index file size must not be negative"))?, + row_count: file.row_count, index_meta: meta.index_meta.clone().unwrap_or_default(), }, )); @@ -855,6 +859,7 @@ fn plan_from_inputs( source_meta: p.source_meta, path: location.resolve(&p.file_name, p.external_path.as_deref()), file_size: p.file_size, + row_count: p.row_count, index_meta: p.index_meta, }) .collect(); @@ -982,6 +987,7 @@ mod tests { .unwrap(), path: path.to_string(), file_size: 1, + row_count: 1, index_meta: Vec::new(), } } @@ -997,6 +1003,7 @@ mod tests { file_name: file_name.to_string(), external_path: external_path.map(str::to_string), file_size: 10, + row_count: 1, index_meta: gim.index_meta.clone().unwrap_or_default(), } } @@ -1604,6 +1611,60 @@ mod tests { ) } + #[tokio::test] + async fn java_empty_vector_payload_covers_sources_after_transport_without_io() { + // Use Java's zero-row payload, whose source still has six physical rows. + let decoded = BucketVectorSearchSplit::deserialize(include_bytes!( + "goldens/bucket_vector_search_split_v1_deletion_vectors.bin" + )) + .unwrap(); + let empty = &decoded.payload_files()[1]; + let mut meta = empty.global_index_meta().clone(); + meta.row_range_end = 5; + let payload = BucketVectorPayload::new_for_test( + "ivf-flat", + empty.file_name(), + empty.file_size(), + empty.row_count(), + None, + None, + meta, + ); + let split = engine_split( + 11, + 0, + BinaryRow::new(0), + vec![dfm("data-1.orc", 6, 1, Some(1))], + vec![payload], + &[], + ); + let plan = + plan_from_bucket_splits("ivf-flat", 1, None, "/tbl", false, vec![split]).unwrap(); + let split = &plan.splits[0]; + assert_eq!(split.ann_segments[0].row_count, 0); + assert_eq!(split.active_files[0].row_count, 6); + let result = crate::vindex::pkvector::bucket::bucket_search( + None, + &split.ann_segments, + &split.active_files, + &Default::default(), + &|_, _, _, _, _| unreachable!("empty segment covers its sources without an exact read"), + &[0., 0.], + crate::vindex::pkvector::metric::VectorSearchMetric::L2, + 1, + 1, + &Default::default(), + false, + None, + 1, + None, + ) + .await + .unwrap(); + assert!(result.indexed.is_empty()); + assert!(result.exact.is_empty()); + } + fn engine_payload(meta: GlobalIndexMeta) -> BucketVectorPayload { BucketVectorPayload::new_for_test("ivf-pq", "seg0", 1, 4, None, None, meta) } diff --git a/crates/paimon/src/table/pk_vector_search_params.rs b/crates/paimon/src/table/pk_vector_search_params.rs index 1a158fc8e..b469b92a1 100644 --- a/crates/paimon/src/table/pk_vector_search_params.rs +++ b/crates/paimon/src/table/pk_vector_search_params.rs @@ -96,7 +96,9 @@ impl PkVectorSearchParams { // Fan-out limit for bucket orchestration plus ANN and exact-file leaves (Java // `GLOBAL_INDEX_THREAD_NUM`); `1` reproduces strictly sequential execution. let concurrency = core.global_index_thread_num()?; - let index_type = core.primary_key_vector_index_type(pk_col)?; + let index_type = + super::pk_index_options::vector_index_type(table.schema().options(), pk_col)? + .to_string(); let vector_field = table .schema() .fields() diff --git a/crates/paimon/src/table/vector_scan.rs b/crates/paimon/src/table/vector_scan.rs index 77ef44015..262077913 100644 --- a/crates/paimon/src/table/vector_scan.rs +++ b/crates/paimon/src/table/vector_scan.rs @@ -276,7 +276,8 @@ impl VectorScan { VectorScanKind::PrimaryKey(Box::new(PkVectorScan::new( table, field_id, - core.primary_key_vector_index_type(&column)?, + super::pk_index_options::vector_index_type(table.schema().options(), &column)? + .to_string(), filter.cloned(), ))) } else { diff --git a/crates/paimon/src/vindex/pkvector/bucket.rs b/crates/paimon/src/vindex/pkvector/bucket.rs index 62f5a1acc..e5deeeea2 100644 --- a/crates/paimon/src/vindex/pkvector/bucket.rs +++ b/crates/paimon/src/vindex/pkvector/bucket.rs @@ -59,6 +59,8 @@ pub(crate) struct BucketAnnSegment { /// Resolved index-file path (globally unique; the key the loader reads by). pub path: String, pub file_size: u64, + /// Indexed vectors; zero is Java's completed, empty source unit. + pub row_count: i64, pub index_meta: Vec, } @@ -71,6 +73,7 @@ impl BucketAnnSegment { source_meta, path: "seg".to_string(), file_size: 0, + row_count: 1, index_meta: Vec::new(), } } @@ -424,6 +427,11 @@ pub(crate) async fn bucket_search( let mut segments_by_path: HashMap<&str, usize> = HashMap::new(); let mut source_to_segment: HashMap<&str, &str> = HashMap::new(); for (idx, segment) in ann_segments.iter().enumerate() { + if segment.row_count < 0 { + return Err(data_invalid( + "ANN segment vector count must not be negative", + )); + } if segments_by_path .insert(segment.path.as_str(), idx) .is_some() @@ -485,7 +493,7 @@ pub(crate) async fn bucket_search( } } } - let searcher = if ann_segments.is_empty() { + let searcher = if ann_segments.iter().all(|segment| segment.row_count == 0) { None } else { Some( @@ -560,7 +568,7 @@ pub(crate) async fn bucket_search( query_owned, )) = ann_shared { - for segment in ann_segments { + for segment in ann_segments.iter().filter(|segment| segment.row_count > 0) { let searcher = searcher.clone(); let segment = segment.clone(); let active_source_files = active_source_files.clone(); @@ -735,6 +743,11 @@ pub(crate) async fn bucket_search_batch( let mut segments_by_path: HashMap<&str, usize> = HashMap::new(); let mut source_to_segment: HashMap<&str, &str> = HashMap::new(); for (idx, segment) in ann_segments.iter().enumerate() { + if segment.row_count < 0 { + return Err(data_invalid( + "ANN segment vector count must not be negative", + )); + } if segments_by_path .insert(segment.path.as_str(), idx) .is_some() @@ -788,7 +801,7 @@ pub(crate) async fn bucket_search_batch( } } } - let searcher = if ann_segments.is_empty() { + let searcher = if ann_segments.iter().all(|segment| segment.row_count == 0) { None } else { Some( @@ -852,7 +865,7 @@ pub(crate) async fn bucket_search_batch( queries_owned, )) = ann_shared { - for segment in ann_segments { + for segment in ann_segments.iter().filter(|segment| segment.row_count > 0) { let searcher = searcher.clone(); let segment = segment.clone(); let active_source_files = active_source_files.clone(); @@ -1618,12 +1631,14 @@ mod tests { source_meta: meta(&[("data-1", 2)]), path: "duplicate-path".to_string(), file_size: 100, + row_count: 1, index_meta: vec![1, 2, 3], }; let seg2 = BucketAnnSegment { source_meta: meta(&[("data-2", 2)]), path: "duplicate-path".to_string(), file_size: 200, + row_count: 1, index_meta: vec![4, 5, 6], }; let ann = FakeAnnSearcher { result: vec![] }; @@ -1658,12 +1673,14 @@ mod tests { source_meta: meta(&[("data-1", 2)]), path: "segment-1".to_string(), file_size: 100, + row_count: 1, index_meta: vec![1, 2, 3], }; let seg2 = BucketAnnSegment { source_meta: meta(&[("data-1", 2), ("data-2", 2)]), path: "segment-2".to_string(), file_size: 200, + row_count: 1, index_meta: vec![4, 5, 6], }; let ann = FakeAnnSearcher { result: vec![] }; @@ -2602,6 +2619,7 @@ mod tests { source_meta: meta(&[(src.as_str(), 2)]), path: format!("seg-{i}"), file_size: 0, + row_count: 1, index_meta: Vec::new(), }); actives.push(active(&src, 2)); @@ -2765,6 +2783,7 @@ mod tests { source_meta: meta(&[("cov-1", 99)]), path: "seg-1".to_string(), file_size: 0, + row_count: 1, index_meta: Vec::new(), }; let searcher: Arc = probe; @@ -3248,12 +3267,14 @@ mod tests { source_meta: meta(&[("cov-a", 2)]), path: "seg-a".to_string(), file_size: 0, + row_count: 1, index_meta: Vec::new(), }; let seg_b = BucketAnnSegment { source_meta: meta(&[("cov-b", 2)]), path: "seg-b".to_string(), file_size: 0, + row_count: 1, index_meta: Vec::new(), }; let factory = unreachable_search(); diff --git a/crates/paimon/tests/pk_compact/indexes.rs b/crates/paimon/tests/pk_compact/indexes.rs index 71fb0835c..a42f32b95 100644 --- a/crates/paimon/tests/pk_compact/indexes.rs +++ b/crates/paimon/tests/pk_compact/indexes.rs @@ -19,6 +19,427 @@ use super::*; use paimon::spec::{should_read_pk_index_source, IndexManifest, PrimaryKeyIndexSourceMeta}; use std::collections::HashSet; +async fn vector_table_for_indexes(options: &[(&str, &str)]) -> Table { + use paimon::spec::{ArrayType, FloatType}; + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .primary_key(["id"]) + .option("bucket", "1") + .option("pk-vector.index.columns", "embedding") + .option("deletion-vectors.enabled", "true") + .option("deletion-vectors.merge-on-read", "false") + .option("fields.embedding.pk-vector.index.type", "ivf-flat") + .option("fields.embedding.pk-vector.distance.metric", "l2") + .option( + "fields.embedding.pk-vector.index.options", + r#"{"dimension":2,"nlist":1}"#, + ) + .option("global-index.search-mode", "fast"); + for (key, value) in options { + schema = schema.option(*key, *value); + } + table_with_schema(TableSchema::new(0, &schema.build().unwrap())).await +} + +fn vector_index_batch(ids: Vec, values: Vec>) -> RecordBatch { + use arrow_array::{ + builder::{Float32Builder, ListBuilder}, + ArrayRef, + }; + use std::sync::Arc; + let mut vectors = ListBuilder::new(Float32Builder::new()).with_field(Arc::new( + arrow_schema::Field::new("element", arrow_schema::DataType::Float32, true), + )); + for value in values { + if let Some(value) = value { + vectors.values().append_slice(&value); + vectors.append(true); + } else { + vectors.append(false); + } + } + RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(ids)) as ArrayRef), + ("embedding", Arc::new(vectors.finish()) as ArrayRef), + ]) + .unwrap() +} + +#[tokio::test] +async fn empty_vector_level_is_persisted_reused_and_searchable() { + let table = vector_table_for_indexes(&[]).await; + write_batch( + &table, + &vector_index_batch((0..1000).collect(), vec![None; 1000]), + ) + .await; + let builder = table + .new_compact_builder() + .with_commit_user("empty-vector-level") + .unwrap(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let first = execute(&table, &plan).await; + let commit = builder.new_commit().unwrap(); + commit.commit_compact(first.clone()).await.unwrap(); + let empty = indexes(&table) + .await + .into_iter() + .find(|e| e.index_file.index_type == "ivf-flat") + .expect("Java records all-NULL sources as an explicit empty segment"); + assert_eq!(empty.index_file.row_count, 0); + assert_eq!(empty.index_file.file_size, 0); + assert_eq!( + table + .file_io() + .new_input(&index_path(&table, &empty)) + .unwrap() + .read() + .await + .unwrap() + .len(), + 0 + ); + commit.commit_compact(first).await.unwrap(); + let result = table + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![0., 0.]) + .with_limit(1) + .execute() + .await + .unwrap(); + let batches: Vec = result + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 0); + let mut batch = table.new_batch_vector_search_builder(); + for result in batch + .with_vector_column("embedding") + .with_query_vectors(vec![vec![0., 0.], vec![1., 0.]]) + .with_limit(1) + .execute() + .await + .unwrap() + { + let batches: Vec = result + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 0); + } + // A smaller level must not rebuild the retained all-NULL level. + for id in [42, 43] { + write_batch( + &table, + &vector_index_batch(vec![id], vec![Some([id as f32, 0.])]), + ) + .await; + let prepared = compact(&table).await; + assert!(indexes(&table).await.contains(&empty)); + assert!(prepared + .iter() + .flat_map(|r| r.index_entries().unwrap()) + .all(|entry| entry.index_file.file_name != empty.index_file.file_name)); + let result = table + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![id as f32, 0.]) + .with_limit(1) + .execute() + .await + .unwrap(); + let batches: Vec = result + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); + assert_eq!( + batches[0] + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + id + ); + let mut batch = table.new_batch_vector_search_builder(); + let results = batch + .with_vector_column("embedding") + .with_query_vectors(vec![vec![id as f32, 0.], vec![id as f32, 0.]]) + .with_limit(1) + .execute() + .await + .unwrap(); + for result in results { + let batches: Vec = result + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); + } + } +} + +#[tokio::test] +async fn vector_output_cannot_be_omitted_and_counts_non_null_vectors() { + let table = vector_table_for_indexes(&[]).await; + write_batch( + &table, + &vector_index_batch(vec![1, 2], vec![None, Some([2., 0.])]), + ) + .await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + let entry = &results[0].index_entries().unwrap()[0]; + assert_eq!( + entry.index_file.row_count, 1, + "Java records indexed vectors separately from the physical source range" + ); + assert_eq!( + entry + .index_file + .global_index_meta + .as_ref() + .unwrap() + .row_range_end, + 1 + ); + let path = index_path(&table, entry); + let mut wire = serde_json::to_value(&results).unwrap(); + wire[0]["index_entries"] = serde_json::json!([]); + let error = builder + .new_commit() + .unwrap() + .commit_compact(serde_json::from_value(wire).unwrap()) + .await + .unwrap_err(); + assert!( + error.to_string().contains("Missing PK index output"), + "{error}" + ); + assert_eq!(latest(&table).await.id(), 1); + assert!(table.file_io().exists(&path).await.unwrap()); + builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); +} + +#[tokio::test] +async fn vector_type_is_canonical_in_build_and_query() { + let table = + vector_table_for_indexes(&[("fields.embedding.pk-vector.index.type", " IVF-FLAT ")]).await; + write_batch( + &table, + &vector_index_batch(vec![1, 2], vec![Some([1., 0.]), Some([100., 0.])]), + ) + .await; + compact(&table).await; + assert!(indexes(&table) + .await + .iter() + .any(|e| e.index_file.index_type == "ivf-flat")); + let result = table + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(1) + .execute() + .await + .unwrap(); + let batches: Vec = result + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + batches[0] + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 1 + ); +} + +#[tokio::test] +async fn conflicting_field_metric_is_rejected_before_build() { + for native in [ + r#"{"dimension":2,"nlist":1,"fields.embedding.metric":"cosine"}"#, + r#"{"dimension":2,"nlist":1,"fields.embedding.distance.metric":"cosine"}"#, + r#"{"dimension":2,"nlist":1,"distance.metric":"cosine"}"#, + ] { + let table = vector_table_for_indexes(&[ + ("fields.embedding.pk-vector.index.options", native), + ("fields.embedding.metric", "cosine"), + ]) + .await; + assert!(table.new_compact_builder().new_planner().is_err()); + } +} + +#[tokio::test] +async fn redeclared_scalar_index_after_type_evolution_falls_back() { + use paimon::spec::{FloatType, SchemaChange}; + for family in ["btree", "bitmap"] { + let key = format!("pk-{family}.index.columns"); + let table = table(&[(&key, "value"), ("target-file-row-num", "1")]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + compact(&table).await; + let removed = table + .schema() + .apply_changes(vec![SchemaChange::remove_option(key.clone())]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &removed).await; + let changed = removed + .apply_changes(vec![SchemaChange::UpdateColumnType { + field_names: vec!["value".into()], + new_data_type: DataType::Float(FloatType::new()), + keep_nullability: true, + }]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &changed).await; + let declared = changed + .apply_changes(vec![SchemaChange::set_option(key, "value".into())]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &declared).await; + let current = table.copy_with_resolved_schema(declared, "main").unwrap(); + for enabled in ["false", "true"] { + let read_table = current.copy_with_options(std::collections::HashMap::from([( + "global-index.enabled".into(), + enabled.into(), + )])); + let mut read = read_table.new_read_builder(); + read.with_filter( + PredicateBuilder::new(read_table.schema().fields()) + .equal("value", Datum::Float(10.)) + .unwrap(), + ); + let plan = read.new_scan().plan().await.unwrap(); + let batches: Vec = read + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + 1, + "{family}, enabled={enabled}" + ); + } + } +} + +#[tokio::test] +async fn scalar_type_roundtrip_never_builds_cast_keys_on_retained_sources() { + use arrow_array::{ArrayRef, Float32Array}; + use paimon::spec::{FloatType, SchemaChange}; + use std::sync::Arc; + for family in ["btree", "bitmap"] { + let key = format!("pk-{family}.index.columns"); + let table = table(&[ + ("deletion-vectors.enabled", "true"), + ("deletion-vectors.merge-on-read", "false"), + ]) + .await; + write_batch(&table, &make_batch((0..1000).collect(), vec![10; 1000])).await; + let old = compact(&table).await[0].files().to_vec(); + let changed = table + .schema() + .apply_changes(vec![ + SchemaChange::UpdateColumnType { + field_names: vec!["value".into()], + new_data_type: DataType::Float(FloatType::new()), + keep_nullability: true, + }, + SchemaChange::set_option(key.clone(), "value".into()), + ]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &changed).await; + let changed = table.copy_with_resolved_schema(changed, "main").unwrap(); + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![1001])) as ArrayRef), + ("value", Arc::new(Float32Array::from(vec![10.])) as ArrayRef), + ]) + .unwrap(); + write_batch(&changed, &batch).await; + compact(&changed).await; + for entry in indexes(&changed) + .await + .into_iter() + .filter(|e| e.index_file.index_type == family) + { + let source = PrimaryKeyIndexSourceMeta::from_global_index_meta( + entry.index_file.global_index_meta.as_ref().unwrap(), + ) + .unwrap(); + assert!( + source + .source_files() + .iter() + .all(|source| !old.iter().any(|file| file.file_name == source.file_name())), + "retained INT sources must not receive FLOAT index keys" + ); + } + let removed = changed + .schema() + .apply_changes(vec![SchemaChange::remove_option(key.clone())]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &removed).await; + let reverted = removed + .apply_changes(vec![SchemaChange::UpdateColumnType { + field_names: vec!["value".into()], + new_data_type: DataType::Int(IntType::new()), + keep_nullability: true, + }]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &reverted).await; + let restored = reverted + .apply_changes(vec![SchemaChange::set_option(key, "value".into())]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &restored).await; + let restored = table.copy_with_resolved_schema(restored, "main").unwrap(); + assert_eq!(filtered_rows(&restored, 10).await.len(), 1001, "{family}"); + // Once sources are rewritten under the current codec they can be indexed. + write_batch(&restored, &make_batch(vec![1002], vec![10])).await; + compact(&restored).await; + assert_eq!(filtered_rows(&restored, 10).await.len(), 1002, "{family}"); + } +} + #[tokio::test] async fn independently_committed_buckets_rebase_without_conflicting_local_ordinals() { let table = table(&[("bucket", "2"), ("pk-btree.index.columns", "value")]).await; @@ -486,7 +907,14 @@ async fn all_native_families_build_with_physical_null_gaps_and_field_options() { entry.index_file.global_index_meta.as_ref().unwrap(), ) .unwrap(); - assert_eq!(entry.index_file.row_count, 3); + assert_eq!( + entry.index_file.row_count, + if entry.index_file.index_type == "ivf-flat" { + 2 + } else { + 3 + } + ); assert_eq!(source.source_files().len(), 3); assert!(table .file_io() diff --git a/docs/src/compaction.md b/docs/src/compaction.md index bc3987e7c..c2bf07904 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -280,6 +280,9 @@ Unchanged coverage is retained. Changed coverage and incompatible vector definitions replace the previous references in the same COMPACT snapshot as data, changelogs and deletion vectors. Old payload files remain for historical snapshots. L0 files remain outside these indexes and use the existing read path. +All-NULL vector levels retain an explicit empty segment, as in Java, so later +maintenance reuses their coverage without rescanning or opening a native index. +Vector payload row counts count indexed vectors; source ranges retain all physical positions. Commit validates the complete source set and captured index membership before publishing. Concurrent L0 appends and unrelated index changes can be rebased; @@ -291,6 +294,9 @@ need maintenance; already published results remain recoverable without their original source snapshot. As in Java, declared indexed columns cannot be renamed, dropped or have their type changed while the declaration remains in effect. +If a declaration is removed to change a scalar column's type, retained files with +different source types use the normal read path until rewritten. Maintenance and +reads share this codec check, including when a type is changed back later. The same API compacts primary-key tables with `deduplicate`, `first-row`, `partial-update` or `aggregation` merge engines. Changelog producers From 6517d8d4739388ac21e031e60d799f58b7f9954e Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 14:57:05 +0800 Subject: [PATCH 04/14] perf(table): reuse automatic maintenance metadata --- crates/paimon/src/table/compact.rs | 89 +++++++++-- crates/paimon/src/table/compact/automatic.rs | 8 +- .../table/compact/automatic/metadata_tests.rs | 151 ++++++++++++++++++ .../compact/primary_key/deletion_vectors.rs | 12 +- .../src/table/compact/primary_key/indexes.rs | 5 +- .../table/compact/primary_key/lookup_store.rs | 10 +- .../table/full_text_index_build_builder.rs | 10 +- .../src/table/global_index_build_common.rs | 57 ++++++- .../src/table/maintenance_test_utils.rs | 105 ++++++++++++ crates/paimon/src/table/mod.rs | 2 + .../sorted_global_index_build_builder.rs | 10 +- .../src/table/table_commit/auto_index.rs | 96 +++++++++-- .../table/table_commit/auto_index/planning.rs | 38 ++++- .../table/table_commit/auto_index_tests.rs | 61 ++++++- .../src/table/vindex_index_build_builder.rs | 10 +- docs/src/compaction.md | 6 + 16 files changed, 602 insertions(+), 68 deletions(-) create mode 100644 crates/paimon/src/table/compact/automatic/metadata_tests.rs create mode 100644 crates/paimon/src/table/maintenance_test_utils.rs diff --git a/crates/paimon/src/table/compact.rs b/crates/paimon/src/table/compact.rs index 56077585c..c05ae0a45 100644 --- a/crates/paimon/src/table/compact.rs +++ b/crates/paimon/src/table/compact.rs @@ -41,7 +41,8 @@ use super::{DataSplit, Plan, Table, TableCommit, TableScan}; use crate::resource::ResourceContext; use crate::spec::{ bucket_path_under, BinaryRow, ChangelogProducer, DataFileMeta, FileKind, IndexManifest, - ManifestEntry, MergeEngine, PartitionComputer, Predicate, Snapshot, TableSchema, + IndexManifestEntry, ManifestEntry, MergeEngine, PartitionComputer, Predicate, Snapshot, + TableSchema, }; use crate::{Error, Result}; @@ -370,6 +371,14 @@ pub struct CompactExecutor { emit_changelog: bool, } +// Valid only for one immutable automatic round. Publication still reads and +// validates current metadata; manual execution captures fresh inputs each call. +struct CompactInputs { + snapshot: Snapshot, + data: HashMap<(Vec, i32), Vec>, + indexes: HashMap<(Vec, i32), Vec>, +} + /// Prepared data, changelog and index files owned by the caller until submitted /// to `TableCommit::commit_compact`. /// A failed or uncertain commit must retain these files for retry. @@ -442,6 +451,11 @@ impl CompactExecutor { } pub async fn execute(&self, task: &CompactTask) -> Result { + let inputs = self.capture_inputs(task).await?; + self.execute_captured(task, &inputs).await + } + + async fn capture_inputs(&self, task: &CompactTask) -> Result { ensure_supported(&self.table)?; task.validate_target(&self.table)?; let manager = self.table.snapshot_manager(); @@ -449,14 +463,60 @@ impl CompactExecutor { if !task.context.snapshot.has_same_read_metadata(&snapshot) { return Err(invalid("Compaction snapshot was replaced")); } - validate_snapshot_indexes(&self.table, &snapshot).await?; + let indexes = read_snapshot_indexes(&self.table, &snapshot).await?; + if !self.table.schema().primary_keys().is_empty() + && self + .table + .schema() + .core_options() + .deletion_vectors_enabled() + { + primary_key::deletion_vectors::validate_index_references(&indexes)?; + } let active = read_snapshot_entries(&manager, &snapshot, None).await?; validate_dv_schemas(&self.table, active.iter().map(|entry| entry.file())).await?; - task.validate_inputs(&active)?; + let mut data = HashMap::<_, Vec<_>>::new(); + for entry in active { + data.entry((entry.partition().to_vec(), entry.bucket())) + .or_default() + .push(entry); + } + let mut by_bucket = HashMap::<_, Vec<_>>::new(); + for entry in indexes { + by_bucket + .entry((entry.partition.clone(), entry.bucket)) + .or_default() + .push(entry); + } + Ok(CompactInputs { + snapshot, + data, + indexes: by_bucket, + }) + } + + async fn execute_captured( + &self, + task: &CompactTask, + inputs: &CompactInputs, + ) -> Result { + task.validate_target(&self.table)?; + let snapshot = &inputs.snapshot; + if !task.context.snapshot.has_same_read_metadata(snapshot) { + return Err(invalid("Compaction snapshot was replaced")); + } + let key = (task.partition.clone(), task.bucket); + let active = inputs.data.get(&key).map(Vec::as_slice).unwrap_or_default(); + let indexes = inputs + .indexes + .get(&key) + .map(Vec::as_slice) + .unwrap_or_default(); + task.validate_inputs(active)?; // validate_target proved the complete schema and branch are identical. // Preserve already resolved catalog schemas in the shared manager; // rebuilding it here would require REST-only schemas to exist on disk. - let table = self.table.copy_with_pinned_snapshot(Some(&snapshot)); + let table = self.table.copy_with_pinned_snapshot(Some(snapshot)); let options = table.schema().core_options(); let partition = BinaryRow::from_serialized_bytes(&task.partition)?; let computer = PartitionComputer::new( @@ -481,7 +541,7 @@ impl CompactExecutor { .with_data_files(task.files.clone()) .build()?; let split = if options.deletion_vectors_enabled() && task.is_primary_key() { - primary_key::deletion_vectors::with_deletion_files(&table, &split, &snapshot).await? + primary_key::deletion_vectors::with_deletion_entries(&table, &split, indexes)? } else { split }; @@ -524,6 +584,7 @@ impl CompactExecutor { history, cache.clone(), self.resources.clone(), + indexes, ) .await?; deletions = collected; @@ -560,8 +621,8 @@ impl CompactExecutor { &table, task, &split, - &snapshot, - &active, + indexes, + active, &files, self.resources.clone(), ) @@ -1220,15 +1281,23 @@ pub(super) async fn validate_snapshot_indexes( table: &Table, snapshot: &Snapshot, ) -> Result> { + read_snapshot_indexes(table, snapshot).await?; + Ok(snapshot.index_manifest().map(str::to_owned)) +} + +async fn read_snapshot_indexes( + table: &Table, + snapshot: &Snapshot, +) -> Result> { ensure_snapshot_supported(table, snapshot)?; let Some(name) = snapshot.index_manifest() else { - return Ok(None); + return Ok(vec![]); }; let path = table.snapshot_manager().manifest_path(name); let entries = IndexManifest::read(table.file_io(), &path).await?; if table.schema().core_options().data_evolution_enabled() { data_evolution::validate_indexes(&entries)?; - return Ok(Some(name.to_string())); + return Ok(entries); } for entry in &entries { if !table.schema().primary_keys().is_empty() && primary_key::indexes::is_source_index(entry) @@ -1249,7 +1318,7 @@ pub(super) async fn validate_snapshot_indexes( }); } } - Ok(Some(name.to_string())) + Ok(entries) } fn validate_file(file: &DataFileMeta, primary_key: bool) -> Result<()> { diff --git a/crates/paimon/src/table/compact/automatic.rs b/crates/paimon/src/table/compact/automatic.rs index e497b72bc..6b29c9682 100644 --- a/crates/paimon/src/table/compact/automatic.rs +++ b/crates/paimon/src/table/compact/automatic.rs @@ -23,6 +23,8 @@ use futures::stream::{FuturesUnordered, StreamExt}; use super::{prepare_results, CompactExecutor, CompactPlan, PreparedCompact}; use crate::table::{CommitMessage, Table, TableCommit}; use crate::{Error, Result}; +#[cfg(test)] +mod metadata_tests; pub(in crate::table) async fn execute_auto_plan( table: &Table, @@ -39,8 +41,12 @@ pub(in crate::table) async fn execute_auto_plan( .or_default() .push(task); } + let inputs = match plan.tasks().first() { + Some(task) => Some(executor.capture_inputs(task).await?), + None => None, + }; let (results, error) = run_buckets(buckets.into_values(), parallelism, |task| { - executor.execute(task) + executor.execute_captured(task, inputs.as_ref().expect("nonempty task plan")) }) .await; let prepared = match error { diff --git a/crates/paimon/src/table/compact/automatic/metadata_tests.rs b/crates/paimon/src/table/compact/automatic/metadata_tests.rs new file mode 100644 index 000000000..93a259db8 --- /dev/null +++ b/crates/paimon/src/table/compact/automatic/metadata_tests.rs @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{DataType, IntType, Schema, TableSchema}; +use crate::table::maintenance_test_utils::{source_manifests, table}; +use arrow_array::{ArrayRef, Int32Array, RecordBatch}; +use std::sync::Arc; + +#[tokio::test] +async fn automatic_tasks_read_captured_metadata_once_and_keep_bucket_indexes() { + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .primary_key(["id"]) + .option("bucket", "4") + .option("write-only", "true") + .option("num-sorted-run.compaction-trigger", "2") + .option("compaction.force-up-level0", "true") + .option("deletion-vectors.enabled", "true") + .option("deletion-vectors.merge-on-read", "false") + .option("pk-btree.index.columns", "value") + .build() + .unwrap(); + let (table, counter) = table(TableSchema::new(0, &schema)).await; + for value in [10, 20] { + let batch = RecordBatch::try_from_iter([ + ( + "id", + Arc::new(Int32Array::from((0..64).collect::>())) as ArrayRef, + ), + ( + "value", + Arc::new(Int32Array::from(vec![value; 64])) as ArrayRef, + ), + ]) + .unwrap(); + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + write + .new_commit() + .commit(writer.prepare_commit().await.unwrap()) + .await + .unwrap(); + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + assert_eq!(plan.tasks().len(), 4); + let executor = builder.new_auto_executor().unwrap(); + if value == 10 { + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); + continue; + } + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + let mut watched = source_manifests(&table, &snapshot).await; + watched.push( + table + .snapshot_manager() + .manifest_path(snapshot.index_manifest().unwrap()), + ); + let snapshot_path = table.snapshot_manager().snapshot_path(snapshot.id()); + watched.push(snapshot_path.clone()); + counter.watch(watched); + let prepared = execute_auto_plan(&table, &plan, &executor, 2) + .await + .unwrap(); + let counts = counter.counts(); + assert!(counts.values().any(|count| *count > 0), "{counts:?}"); + assert!( + counts.iter().all(|(path, count)| *count <= if path == &snapshot_path { 2 } else { 1 }), + "one automatic round must read captured metadata once (snapshot existence + read): {counts:?}" + ); + assert_eq!( + prepared + .entries + .iter() + .filter(|e| *e.kind() == crate::spec::FileKind::Add) + .count(), + 4 + ); + let additions = prepared + .index_entries + .unwrap() + .into_iter() + .filter(|e| e.kind == crate::spec::FileKind::Add) + .collect::>(); + assert_eq!(additions.len(), 4); + assert!(additions.iter().all(|e| e.index_file.index_type == "btree")); + assert_eq!( + additions + .iter() + .map(|e| e.index_file.row_count) + .sum::(), + 64 + ); + // A later execution must not reuse this round's metadata after rollback + // or snapshot replacement, even when it reuses the same executor. + let mut replaced = serde_json::to_value(&snapshot).unwrap(); + replaced["deltaManifestList"] = serde_json::json!("replaced-manifest-list"); + let replaced: crate::spec::Snapshot = serde_json::from_value(replaced).unwrap(); + assert_eq!(replaced.delta_manifest_list(), "replaced-manifest-list"); + table + .file_io() + .new_output(&snapshot_path) + .unwrap() + .write(serde_json::to_vec(&replaced).unwrap().into()) + .await + .unwrap(); + let error = executor.execute(&plan.tasks()[0]).await.unwrap_err(); + assert!( + error.to_string().contains("snapshot was replaced"), + "{error}" + ); + let error = execute_auto_plan(&table, &plan, &executor, 2) + .await + .err() + .unwrap(); + assert!( + error.to_string().contains("snapshot was replaced"), + "{error}" + ); + } +} diff --git a/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs b/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs index 8f742844b..1b7033fa6 100644 --- a/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs +++ b/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs @@ -78,6 +78,12 @@ fn vector_references(indexes: &[IndexManifestEntry]) -> Result Result<()> { + vector_references(indexes).map(|_| ()) +} + fn resolve_vector(table: &Table, bucket_path: &str, reference: &VectorReference) -> DeletionFile { let location = IndexFileLocation::BucketLocal { table_path: table.location().trim_end_matches('/'), @@ -126,12 +132,12 @@ fn bucket_path(table: &Table, partition: &[u8], bucket: i32) -> Result { } /// Apply captured physical masks before merging, including masks on L0 files. -pub(in crate::table::compact) async fn with_deletion_files( +pub(in crate::table::compact) fn with_deletion_entries( table: &Table, split: &DataSplit, - snapshot: &Snapshot, + indexes: &[IndexManifestEntry], ) -> Result { - let references = vector_references(&read_indexes(table, snapshot).await?)?; + let references = vector_references(indexes)?; let partition = split.partition().to_serialized_bytes(); let files = split .data_files() diff --git a/crates/paimon/src/table/compact/primary_key/indexes.rs b/crates/paimon/src/table/compact/primary_key/indexes.rs index b9ba5515d..cf4f7c64b 100644 --- a/crates/paimon/src/table/compact/primary_key/indexes.rs +++ b/crates/paimon/src/table/compact/primary_key/indexes.rs @@ -277,13 +277,12 @@ pub(in crate::table::compact) async fn prepare( table: &Table, task: &CompactTask, split: &DataSplit, - snapshot: &Snapshot, + indexes: &[IndexManifestEntry], active: &[ManifestEntry], files: &[DataFileMeta], resources: Option, ) -> Result> { let defs = definitions(table.schema())?; - let indexes = read_indexes(table, snapshot).await?; let changes = task .files .iter() @@ -312,7 +311,7 @@ pub(in crate::table::compact) async fn prepare( let unsupported = unsupported_scalar_units(table, &after, &defs).await; let maintenance = plan( table, - &indexes, + indexes, &after, &defs, &unsupported, diff --git a/crates/paimon/src/table/compact/primary_key/lookup_store.rs b/crates/paimon/src/table/compact/primary_key/lookup_store.rs index c561cdd30..50feb677f 100644 --- a/crates/paimon/src/table/compact/primary_key/lookup_store.rs +++ b/crates/paimon/src/table/compact/primary_key/lookup_store.rs @@ -64,6 +64,7 @@ pub(in crate::table::compact) async fn provider( files: Vec, cache: Arc, resources: Option, + indexes: &[crate::spec::IndexManifestEntry], ) -> Result<(Arc, Option)> { let fields = super::read_type(table); let keys = table @@ -101,14 +102,7 @@ pub(in crate::table::compact) async fn provider( .with_data_files(files.clone()) .build()?; let history = if dv { - super::deletion_vectors::with_deletion_files( - table, - &history, - table - .travel_snapshot() - .ok_or_else(|| invalid("Lookup needs a captured snapshot"))?, - ) - .await? + super::deletion_vectors::with_deletion_entries(table, &history, indexes)? } else { history }; diff --git a/crates/paimon/src/table/full_text_index_build_builder.rs b/crates/paimon/src/table/full_text_index_build_builder.rs index 5e78c54c9..d2907ece5 100644 --- a/crates/paimon/src/table/full_text_index_build_builder.rs +++ b/crates/paimon/src/table/full_text_index_build_builder.rs @@ -118,14 +118,8 @@ impl<'a> FullTextIndexBuildBuilder<'a> { return Ok((None, vec![])); }; - let mut read_builder = self.table.new_read_builder(); - if let Some(filter) = &self.partition_filter { - read_builder.with_filter(filter.clone()); - } - let manifest_entries = read_builder - .new_scan() - .with_scan_all_files() - .plan_manifest_entries(snapshot) + let manifest_entries = view + .manifest_entries(self.table, self.partition_filter.as_ref()) .await?; let indexed = indexed_row_ranges( view.index_entries(), diff --git a/crates/paimon/src/table/global_index_build_common.rs b/crates/paimon/src/table/global_index_build_common.rs index ad3585a40..3553c797d 100644 --- a/crates/paimon/src/table/global_index_build_common.rs +++ b/crates/paimon/src/table/global_index_build_common.rs @@ -25,7 +25,9 @@ //! `index_type` string, so it lives here once rather than being copied into //! each builder. -use crate::spec::{DataField, FileKind, IndexManifest, IndexManifestEntry, Snapshot}; +use crate::spec::{ + DataField, FileKind, IndexManifest, IndexManifestEntry, ManifestEntry, Predicate, Snapshot, +}; use crate::table::{merge_row_ranges, RowRange, Table}; use crate::{Error, Result}; use std::sync::Arc; @@ -39,6 +41,7 @@ pub(crate) mod vector; pub(crate) struct GlobalIndexBuildView { snapshot: Option, index_entries: Arc<[IndexManifestEntry]>, + source_entries: Option>, } impl GlobalIndexBuildView { @@ -59,6 +62,7 @@ impl GlobalIndexBuildView { Ok(Self { snapshot: Some(snapshot), index_entries: entries.into(), + source_entries: None, }) } @@ -80,6 +84,36 @@ impl GlobalIndexBuildView { &self.index_entries } + /// Per-round immutable metadata, shared by all declared columns. Never + /// retained across retries or snapshots. + pub(crate) fn with_source_entries(mut self, entries: Arc<[ManifestEntry]>) -> Self { + self.source_entries = Some(entries); + self + } + + pub(super) async fn manifest_entries( + &self, + table: &Table, + filter: Option<&Predicate>, + ) -> Result> { + let Some(snapshot) = self.snapshot() else { + return Ok(vec![]); + }; + if filter.is_none() { + if let Some(entries) = &self.source_entries { + return Ok(entries.to_vec()); + } + } + let mut read = table.new_read_builder(); + if let Some(filter) = filter { + read.with_filter(filter.clone()); + } + read.new_scan() + .with_scan_all_files() + .plan_manifest_entries(snapshot) + .await + } + pub(crate) fn filter_indexes(&self, mut keep: impl FnMut(&IndexManifestEntry) -> bool) -> Self { Self { snapshot: self.snapshot.clone(), @@ -89,6 +123,27 @@ impl GlobalIndexBuildView { .filter(|entry| keep(entry)) .cloned() .collect(), + source_entries: self.source_entries.clone(), + } + } + + /// Source filtering belongs to the caller's maintenance plan. Native + /// builders still plan complete shards, with all overlapping column files. + pub(crate) fn filter_sources(&self, mut keep: impl FnMut(&ManifestEntry) -> bool) -> Self { + let Some(entries) = &self.source_entries else { + return self.clone(); + }; + let selected = entries + .iter() + .filter(|entry| keep(entry)) + .collect::>(); + if selected.len() == entries.len() { + return self.clone(); + } + Self { + snapshot: self.snapshot.clone(), + index_entries: self.index_entries.clone(), + source_entries: Some(selected.into_iter().cloned().collect()), } } } diff --git a/crates/paimon/src/table/maintenance_test_utils.rs b/crates/paimon/src/table/maintenance_test_utils.rs new file mode 100644 index 000000000..a9b68b38f --- /dev/null +++ b/crates/paimon/src/table/maintenance_test_utils.rs @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Storage I/O counters for automatic maintenance regression tests. + +use crate::io::{FileIOBuilder, FileIOProvider}; +use crate::spec::{ManifestList, Snapshot, TableSchema}; +use crate::table::Table; +use crate::Result; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +#[derive(Debug)] +pub(super) struct MetadataIo { + operator: opendal::Operator, + watched: Mutex>, +} +#[async_trait::async_trait] +impl FileIOProvider for MetadataIo { + async fn create(&self, path: &str) -> Result<(opendal::Operator, String)> { + if let Some(reads) = self.watched.lock().unwrap().get_mut(path) { + *reads += 1; + } + Ok(( + self.operator.clone(), + path.strip_prefix("memory:/") + .unwrap() + .trim_start_matches('/') + .into(), + )) + } +} +impl MetadataIo { + pub fn watch(&self, paths: Vec) { + *self.watched.lock().unwrap() = paths.into_iter().map(|path| (path, 0)).collect(); + } + pub fn counts(&self) -> BTreeMap { + self.watched.lock().unwrap().clone() + } +} +pub(super) async fn table(schema: TableSchema) -> (Table, Arc) { + let counter = Arc::new(MetadataIo { + operator: opendal::Operator::from_config(opendal::services::MemoryConfig::default()) + .unwrap(), + watched: Mutex::new(BTreeMap::new()), + }); + let io = FileIOBuilder::new("memory") + .with_provider(counter.clone()) + .build() + .unwrap(); + let path = format!("memory:/maintenance-io/{}", uuid::Uuid::new_v4()); + let table = Table::new( + io, + crate::catalog::Identifier::new("default", "maintenance"), + path.clone(), + schema, + None, + ); + for directory in ["schema", "snapshot"] { + table + .file_io() + .mkdirs(&format!("{path}/{directory}")) + .await + .unwrap(); + } + table + .file_io() + .new_output(&table.schema_manager().schema_path(table.schema().id())) + .unwrap() + .write(serde_json::to_vec(table.schema()).unwrap().into()) + .await + .unwrap(); + (table, counter) +} +pub(super) async fn source_manifests(table: &Table, snapshot: &Snapshot) -> Vec { + let manager = table.snapshot_manager(); + let mut paths = Vec::new(); + for list in [ + snapshot.base_manifest_list(), + snapshot.delta_manifest_list(), + ] { + paths.push(manager.manifest_path(list)); + for manifest in ManifestList::read(table.file_io(), &manager.manifest_path(list)) + .await + .unwrap() + { + paths.push(manager.manifest_path(manifest.file_name())); + } + } + paths +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 4c798d27f..a22177d66 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -98,6 +98,8 @@ mod instant; mod kv_data_file_writer; mod kv_file_reader; mod kv_file_writer; +#[cfg(test)] +mod maintenance_test_utils; mod managed_blob_reader; mod managed_blob_reference; #[cfg(test)] diff --git a/crates/paimon/src/table/sorted_global_index_build_builder.rs b/crates/paimon/src/table/sorted_global_index_build_builder.rs index 77e007bcb..40e66e72f 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder.rs @@ -160,14 +160,8 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { return Ok((None, vec![])); }; - let mut read_builder = self.table.new_read_builder(); - if let Some(filter) = &self.partition_filter { - read_builder.with_filter(filter.clone()); - } - let manifest_entries = read_builder - .new_scan() - .with_scan_all_files() - .plan_manifest_entries(snapshot) + let manifest_entries = view + .manifest_entries(self.table, self.partition_filter.as_ref()) .await?; let indexed = crate::table::global_index_build_common::indexed_row_ranges( view.index_entries(), diff --git a/crates/paimon/src/table/table_commit/auto_index.rs b/crates/paimon/src/table/table_commit/auto_index.rs index 511f0051a..baafe4be9 100644 --- a/crates/paimon/src/table/table_commit/auto_index.rs +++ b/crates/paimon/src/table/table_commit/auto_index.rs @@ -95,23 +95,33 @@ impl TableCommit { if definitions.is_empty() { return Ok(()); } - let data = crate::table::snapshot_manifest::read_snapshot_entries( - &self.snapshot_manager, - snapshot, - None, - ) - .await?; - let view = GlobalIndexBuildView::capture(&table, Some(snapshot.clone())).await?; + let data: Arc<[ManifestEntry]> = + crate::table::snapshot_manifest::read_snapshot_entries( + &self.snapshot_manager, + snapshot, + None, + ) + .await? + .into(); + let view = GlobalIndexBuildView::capture(&table, Some(snapshot.clone())) + .await? + .with_source_entries(data.clone()); let mut messages = Vec::new(); let prepared: Result<()> = async { for definition in definitions { let plan = planning::plan(&table, &definition, &data, view.index_entries()).await?; - let built = definition - .builder(&table) - .build_with_view(plan.build_view(&view)) - .await?; - messages.extend(built); + if plan.needs_build { + let built = definition + .builder(&table) + .build_with_view(plan.build_view(&view)) + .await?; + messages.extend(built); + } else { + // Preserve validation for a refreshed schema without + // planning already covered shards. + definition.builder(&table).validate().await?; + } messages.extend(plan.deletions); } Ok(()) @@ -179,3 +189,65 @@ pub(super) fn auto_index_error(identifier: i64, error: crate::Error) -> crate::E source: Some(Box::new(error)), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::table::table_commit::auto_index_tests::{assert_complete, latest, messages, table}; + #[tokio::test] + async fn builders_only_plan_sources_overlapping_changed_units() { + use crate::table::global_index_options::definitions; + let table = table(Some(true), false).await; + TableCommit::new(table.clone(), "initial".into()) + .commit(messages(&table, vec![1, 2]).await) + .await + .unwrap(); + let raw = table.copy_with_options(HashMap::from([("write-only".into(), "true".into())])); + TableCommit::new(raw.clone(), "append".into()) + .commit(messages(&raw, vec![3, 4]).await) + .await + .unwrap(); + let snapshot = latest(&table).await; + let data: Arc<[ManifestEntry]> = crate::table::snapshot_manifest::read_snapshot_entries( + &table.snapshot_manager(), + &snapshot, + None, + ) + .await + .unwrap() + .into(); + assert_eq!(data.len(), 2); + let view = GlobalIndexBuildView::capture(&table, Some(snapshot)) + .await + .unwrap() + .with_source_entries(data.clone()); + let mut messages = Vec::new(); + for definition in definitions(&table).unwrap() { + let plan = planning::plan(&table, &definition, &data, view.index_entries()) + .await + .unwrap(); + assert!(plan.needs_build); + let build_view = plan.build_view(&view); + let sources = build_view.manifest_entries(&table, None).await.unwrap(); + assert_eq!( + sources.len(), + 1, + "covered units must not enter per-column shard planning" + ); + assert_eq!(sources[0].file().first_row_id, Some(2)); + messages.extend( + definition + .builder(&table) + .build_with_view(build_view) + .await + .unwrap(), + ); + } + assert_eq!(messages.len(), 2); + TableCommit::new(table.clone(), "build-new".into()) + .commit(messages) + .await + .unwrap(); + assert_complete(&table).await; + } +} diff --git a/crates/paimon/src/table/table_commit/auto_index/planning.rs b/crates/paimon/src/table/table_commit/auto_index/planning.rs index 3e2f24669..63db6dda0 100644 --- a/crates/paimon/src/table/table_commit/auto_index/planning.rs +++ b/crates/paimon/src/table/table_commit/auto_index/planning.rs @@ -41,11 +41,30 @@ fn index_file_key(entry: &IndexManifestEntry) -> IndexFileKey { pub(super) struct MaintenancePlan { replaced: HashSet, pub deletions: Vec, + pub needs_build: bool, + build_ranges: HashMap<(Vec, i32), Vec>, } impl MaintenancePlan { pub(super) fn build_view(&self, source: &GlobalIndexBuildView) -> GlobalIndexBuildView { - source.filter_indexes(|entry| !self.replaced.contains(&index_file_key(entry))) + let view = if self.replaced.is_empty() { + source.clone() + } else { + source.filter_indexes(|entry| !self.replaced.contains(&index_file_key(entry))) + }; + view.filter_sources(|entry| { + let Some((start, end)) = entry.file().row_id_range() else { + return false; + }; + let Some(ranges) = self + .build_ranges + .get(&(partition_key(entry.partition()), entry.bucket())) + else { + return false; + }; + let first = ranges.partition_point(|range| range.to() < start); + ranges.get(first).is_some_and(|range| range.from() <= end) + }) } } @@ -119,7 +138,7 @@ pub(super) async fn plan( .push(RowRange::new(start.max(from), end.min(to))); } } - let desired = desired + let mut desired = desired .into_iter() .map(|(key, ranges)| (key, merge_row_ranges(ranges))) .collect::>(); @@ -193,6 +212,8 @@ pub(super) async fn plan( let wanted = desired.get(&key).map(Vec::as_slice).unwrap_or(&[]); if coverage.as_slice() != wanted || files.len() != wanted.len() { replaced.extend(files.iter().map(|entry| index_file_key(entry))); + } else { + desired.remove(&key); } } let mut deletions: BTreeMap<(Vec, i32), CommitMessage> = BTreeMap::new(); @@ -207,8 +228,21 @@ pub(super) async fn plan( .push(entry.index_file.clone()); } } + let mut build_ranges: HashMap<(Vec, i32), Vec> = HashMap::new(); + for ((partition, bucket, _), ranges) in desired { + build_ranges + .entry((partition, bucket)) + .or_default() + .extend(ranges); + } + let build_ranges = build_ranges + .into_iter() + .map(|(key, ranges)| (key, merge_row_ranges(ranges))) + .collect::>(); Ok(MaintenancePlan { replaced, deletions: deletions.into_values().collect(), + needs_build: !build_ranges.is_empty(), + build_ranges, }) } diff --git a/crates/paimon/src/table/table_commit/auto_index_tests.rs b/crates/paimon/src/table/table_commit/auto_index_tests.rs index 101d677f7..1587efa65 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests.rs @@ -26,7 +26,60 @@ use std::sync::atomic::{AtomicBool, Ordering}; mod search; -async fn table(enabled: Option, compact: bool) -> Table { +#[tokio::test] +async fn index_columns_share_source_scan_and_covered_rounds_do_not_rescan_builders() { + use crate::table::maintenance_test_utils::{source_manifests, table as counted_table}; + let mut totals = Vec::new(); + for multiple in [false, true] { + let schema = TableSchemaBuilder::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("write-only", "true") + .option("global-index.btree.index-columns", "id"); + let schema = if multiple { + schema.option("global-index.bitmap.index-columns", "value") + } else { + schema + }; + let (raw, counter) = counted_table(TableSchema::new(0, &schema.build().unwrap())).await; + TableCommit::new(raw.clone(), "source".into()) + .commit(messages(&raw, vec![1, 2]).await) + .await + .unwrap(); + let written = latest(&raw).await; + let table = raw.copy_with_options(HashMap::from([("write-only".into(), "false".into())])); + let paths = source_manifests(&table, &written).await; + counter.watch(paths.clone()); + TableCommit::new(table.clone(), "first-indexer".into()) + .index_after_write(&written) + .await + .unwrap(); + totals.push(counter.counts().values().sum::()); + assert_eq!(indexes(&table).await.len(), if multiple { 2 } else { 1 }); + let indexed = latest(&table).await; + let indexed_id = indexed.id(); + counter.watch(source_manifests(&table, &indexed).await); + // Another round owns a different identity but sees fully indexed data. + TableCommit::new(table.clone(), "other-indexer".into()) + .index_after_write(&written) + .await + .unwrap(); + assert!( + counter.counts().values().all(|count| *count == 1), + "fully covered round should only inspect coverage: {:?}", + counter.counts() + ); + assert_eq!(latest(&table).await.id(), indexed_id); + } + assert_eq!( + totals[0], totals[1], + "adding index columns must not reread source manifests" + ); +} + +pub(super) async fn table(enabled: Option, compact: bool) -> Table { let mut schema = TableSchemaBuilder::builder() .column("id", DataType::Int(IntType::new())) .column("value", DataType::Int(IntType::new())) @@ -72,7 +125,7 @@ async fn table(enabled: Option, compact: bool) -> Table { table } -async fn messages(table: &Table, ids: Vec) -> Vec { +pub(super) async fn messages(table: &Table, ids: Vec) -> Vec { let values = ids.iter().map(|id| id * 10).collect::>(); let batch = RecordBatch::try_from_iter(vec![ ( @@ -91,7 +144,7 @@ async fn messages(table: &Table, ids: Vec) -> Vec { write.prepare_commit().await.unwrap() } -async fn latest(table: &Table) -> Snapshot { +pub(super) async fn latest(table: &Table) -> Snapshot { table .snapshot_manager() .get_latest_snapshot() @@ -113,7 +166,7 @@ async fn indexes(table: &Table) -> Vec { } } -async fn assert_complete(table: &Table) { +pub(super) async fn assert_complete(table: &Table) { for (column, kind) in [("id", "btree"), ("value", "bitmap")] { assert!(table .new_global_index_build_builder() diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs b/crates/paimon/src/table/vindex_index_build_builder.rs index 3b7210944..3f243edc5 100644 --- a/crates/paimon/src/table/vindex_index_build_builder.rs +++ b/crates/paimon/src/table/vindex_index_build_builder.rs @@ -128,14 +128,8 @@ impl<'a> VindexIndexBuildBuilder<'a> { )); }; - let mut read_builder = self.table.new_read_builder(); - if let Some(filter) = &self.partition_filter { - read_builder.with_filter(filter.clone()); - } - let manifest_entries = read_builder - .new_scan() - .with_scan_all_files() - .plan_manifest_entries(snapshot) + let manifest_entries = view + .manifest_entries(self.table, self.partition_filter.as_ref()) .await?; let indexed = crate::table::global_index_build_common::indexed_row_ranges( view.index_entries(), diff --git a/docs/src/compaction.md b/docs/src/compaction.md index c2bf07904..7e29fc58d 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -59,6 +59,12 @@ Overwrite compaction is pinned to its own committed data snapshot, so suppressin overwrite changelogs cannot consume a later append's pending lookup events. As in Java, overwriting partitions being consumed by streaming readers is unsupported; lookup changelogs do not describe the overwrite operation itself. +Automatic tasks share one captured snapshot and bucket-grouped source/index +metadata per round. Automatic index fields also share captured source metadata; +fully covered fields skip shard planning, and changed fields only plan files +overlapping their missing or replaced units. Captured metadata is released after +the round. Publication and retries still validate current sources and index +membership, and manual task execution captures fresh inputs on every call. Automatic lookup compaction supports the default `lookup-compact=radical` policy. `gentle` requires writer-lifetime trigger intervals and is explicitly unsupported here; use `write-only=true` and schedule dedicated compaction separately. From 282b22b8f29dbe82955d8dffee5d0893fba23f88 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 15:40:17 +0800 Subject: [PATCH 05/14] fix(index): share filtered automatic build metadata --- .../src/table/global_index_build_common.rs | 40 ++++- .../global_index_build_common/source_tests.rs | 137 ++++++++++++++++++ 2 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 crates/paimon/src/table/global_index_build_common/source_tests.rs diff --git a/crates/paimon/src/table/global_index_build_common.rs b/crates/paimon/src/table/global_index_build_common.rs index 3553c797d..8c148eab2 100644 --- a/crates/paimon/src/table/global_index_build_common.rs +++ b/crates/paimon/src/table/global_index_build_common.rs @@ -33,6 +33,8 @@ use crate::{Error, Result}; use std::sync::Arc; pub(crate) mod preparation; +#[cfg(test)] +mod source_tests; pub(crate) mod vector; /// One source snapshot and its effective index coverage. Builders use this same @@ -42,6 +44,8 @@ pub(crate) struct GlobalIndexBuildView { snapshot: Option, index_entries: Arc<[IndexManifestEntry]>, source_entries: Option>, + /// Original positions in `source_entries`; `None` selects every entry. + source_selection: Option>, } impl GlobalIndexBuildView { @@ -63,6 +67,7 @@ impl GlobalIndexBuildView { snapshot: Some(snapshot), index_entries: entries.into(), source_entries: None, + source_selection: None, }) } @@ -88,6 +93,7 @@ impl GlobalIndexBuildView { /// retained across retries or snapshots. pub(crate) fn with_source_entries(mut self, entries: Arc<[ManifestEntry]>) -> Self { self.source_entries = Some(entries); + self.source_selection = None; self } @@ -101,7 +107,13 @@ impl GlobalIndexBuildView { }; if filter.is_none() { if let Some(entries) = &self.source_entries { - return Ok(entries.to_vec()); + return Ok(match &self.source_selection { + Some(selected) => selected + .iter() + .map(|&index| entries[index].clone()) + .collect(), + None => entries.to_vec(), + }); } } let mut read = table.new_read_builder(); @@ -124,6 +136,7 @@ impl GlobalIndexBuildView { .cloned() .collect(), source_entries: self.source_entries.clone(), + source_selection: self.source_selection.clone(), } } @@ -133,17 +146,30 @@ impl GlobalIndexBuildView { let Some(entries) = &self.source_entries else { return self.clone(); }; - let selected = entries - .iter() - .filter(|entry| keep(entry)) - .collect::>(); - if selected.len() == entries.len() { + let selected: Vec = match &self.source_selection { + Some(selected) => selected + .iter() + .copied() + .filter(|&index| keep(&entries[index])) + .collect(), + None => entries + .iter() + .enumerate() + .filter_map(|(index, entry)| keep(entry).then_some(index)) + .collect(), + }; + let current_len = self + .source_selection + .as_ref() + .map_or(entries.len(), |selected| selected.len()); + if selected.len() == current_len { return self.clone(); } Self { snapshot: self.snapshot.clone(), index_entries: self.index_entries.clone(), - source_entries: Some(selected.into_iter().cloned().collect()), + source_entries: Some(entries.clone()), + source_selection: Some(selected.into()), } } } diff --git a/crates/paimon/src/table/global_index_build_common/source_tests.rs b/crates/paimon/src/table/global_index_build_common/source_tests.rs new file mode 100644 index 000000000..3a6e3d744 --- /dev/null +++ b/crates/paimon/src/table/global_index_build_common/source_tests.rs @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{DataType, IntType, Schema, TableSchema}; +use crate::table::maintenance_test_utils::{source_manifests, table}; +use crate::table::Table; +use arrow_array::{ArrayRef, Int32Array, RecordBatch}; + +async fn captured_sources() -> ( + Table, + GlobalIndexBuildView, + Arc<[ManifestEntry]>, + Arc, +) { + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .option("row-tracking.enabled", "true") + .option("write-only", "true") + .build() + .unwrap(); + let (table, counter) = table(TableSchema::new(0, &schema)).await; + for id in 0..3 { + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer + .write_arrow_batch( + &RecordBatch::try_from_iter([( + "id", + Arc::new(Int32Array::from(vec![id])) as ArrayRef, + )]) + .unwrap(), + ) + .await + .unwrap(); + write + .new_commit() + .commit(writer.prepare_commit().await.unwrap()) + .await + .unwrap(); + } + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + let mut entries = crate::table::snapshot_manifest::read_snapshot_entries( + &table.snapshot_manager(), + &snapshot, + None, + ) + .await + .unwrap(); + entries.sort_by_key(|entry| entry.file().first_row_id); + for (id, entry) in entries.iter_mut().enumerate() { + // Opaque inline index/stat payloads make deep clones materially costly. + entry.file.embedded_index = Some(vec![id as u8; 32 * 1024]); + } + let source: Arc<[ManifestEntry]> = entries.into(); + let view = GlobalIndexBuildView::capture(&table, Some(snapshot.clone())) + .await + .unwrap() + .with_source_entries(source.clone()); + counter.watch(source_manifests(&table, &snapshot).await); + (table, view, source, counter) +} + +#[tokio::test] +async fn source_filters_share_metadata_and_materialize_only_selected_entries() { + let (table, view, original, counter) = captured_sources().await; + let filtered = view.filter_sources(|entry| entry.file().first_row_id.unwrap() >= 1); + assert!(Arc::ptr_eq(filtered.source_entries.as_ref().unwrap(), &original), + "partial views must share the captured metadata instead of retaining a deep clone during native builds"); + let first = filtered.manifest_entries(&table, None).await.unwrap(); + assert_eq!( + first + .iter() + .map(|entry| entry.file().first_row_id.unwrap()) + .collect::>(), + [1, 2] + ); + let twice = filtered + .filter_sources(|entry| entry.file().first_row_id == Some(2)) + .filter_indexes(|_| true); + assert!(Arc::ptr_eq( + twice.source_entries.as_ref().unwrap(), + &original + )); + assert_eq!( + twice.snapshot().unwrap().id(), + view.snapshot().unwrap().id() + ); + let selected = twice.manifest_entries(&table, None).await.unwrap(); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].file().first_row_id, Some(2)); + assert_eq!( + selected[0].file().embedded_index.as_deref(), + Some(vec![2; 32 * 1024].as_slice()) + ); + let empty = twice.filter_sources(|_| false); + assert!(empty + .filter_sources(|_| true) + .manifest_entries(&table, None) + .await + .unwrap() + .is_empty()); + // A refreshed capture owns a new source buffer and must drop old positions. + let reset = twice.with_source_entries(vec![original[0].clone()].into()); + let reset_entries = reset.manifest_entries(&table, None).await.unwrap(); + assert_eq!(reset_entries.len(), 1); + assert_eq!(reset_entries[0].file().first_row_id, Some(0)); + assert_eq!(view.manifest_entries(&table, None).await.unwrap().len(), 3); + assert_eq!( + filtered.manifest_entries(&table, None).await.unwrap().len(), + 2 + ); + assert!( + counter.counts().values().all(|reads| *reads == 0), + "{:?}", + counter.counts() + ); +} From 55f06516a3031f44f066f6baf329651300e2c4a4 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 17:16:49 +0800 Subject: [PATCH 06/14] refactor(index): unify PK and global index commit handling --- crates/paimon/src/table/compact.rs | 5 +- .../paimon/src/table/compact/primary_key.rs | 2 +- .../src/table/compact/primary_key/indexes.rs | 400 +---------------- .../compact/primary_key/indexes/writer.rs | 31 +- crates/paimon/src/table/mod.rs | 1 + .../paimon/src/table/pk_index_maintenance.rs | 409 ++++++++++++++++++ crates/paimon/src/table/pk_index_options.rs | 29 +- crates/paimon/src/table/table_commit.rs | 338 ++------------- .../src/table/table_commit/index_commit.rs | 374 ++++++++++++++++ .../table/table_commit/index_commit/tests.rs | 137 ++++++ .../table/table_commit/index_delete_tests.rs | 10 +- 11 files changed, 1011 insertions(+), 725 deletions(-) create mode 100644 crates/paimon/src/table/pk_index_maintenance.rs create mode 100644 crates/paimon/src/table/table_commit/index_commit.rs create mode 100644 crates/paimon/src/table/table_commit/index_commit/tests.rs diff --git a/crates/paimon/src/table/compact.rs b/crates/paimon/src/table/compact.rs index c05ae0a45..3456e80d6 100644 --- a/crates/paimon/src/table/compact.rs +++ b/crates/paimon/src/table/compact.rs @@ -1300,9 +1300,10 @@ async fn read_snapshot_indexes( return Ok(entries); } for entry in &entries { - if !table.schema().primary_keys().is_empty() && primary_key::indexes::is_source_index(entry) + if !table.schema().primary_keys().is_empty() + && super::pk_index_maintenance::is_source_index(entry) { - primary_key::indexes::source(entry)?; + super::pk_index_maintenance::source(entry)?; continue; } if !(entry.index_file.index_type == "HASH" diff --git a/crates/paimon/src/table/compact/primary_key.rs b/crates/paimon/src/table/compact/primary_key.rs index 005cffca7..500247549 100644 --- a/crates/paimon/src/table/compact/primary_key.rs +++ b/crates/paimon/src/table/compact/primary_key.rs @@ -16,7 +16,7 @@ // under the License. pub(in crate::table) mod deletion_vectors; -pub(in crate::table) mod indexes; +pub(super) mod indexes; pub(super) mod lookup; pub(super) mod lookup_file; pub(super) mod lookup_store; diff --git a/crates/paimon/src/table/compact/primary_key/indexes.rs b/crates/paimon/src/table/compact/primary_key/indexes.rs index cf4f7c64b..1cc00d7da 100644 --- a/crates/paimon/src/table/compact/primary_key/indexes.rs +++ b/crates/paimon/src/table/compact/primary_key/indexes.rs @@ -15,263 +15,20 @@ // specific language governing permissions and limitations // under the License. -//! Source-backed index maintenance for primary-key compaction. +//! Execute source-backed index maintenance for primary-key compaction. mod writer; #[cfg(test)] mod tests; -use super::super::{invalid, CompactTask}; +use super::super::CompactTask; use crate::resource::ResourceContext; -use crate::spec::{ - should_read_pk_index_source, DataFileMeta, FileKind, IndexManifest, IndexManifestEntry, - ManifestEntry, PrimaryKeyIndexSourceFile, PrimaryKeyIndexSourceMeta, Snapshot, -}; -use crate::table::pk_index_options::{definitions, Definition}; +use crate::spec::{DataFileMeta, FileKind, IndexManifestEntry, ManifestEntry}; +use crate::table::pk_index_maintenance::{levels, plan, unsupported_scalar_units}; +use crate::table::pk_index_options::definitions; use crate::table::{CommitMessage, DataSplit, Table, TableCommit}; use crate::Result; -use std::collections::{BTreeMap, HashSet}; - -type Unit = (Vec, i32, i32, i32); -type Levels = BTreeMap>; - -pub(in crate::table) fn is_source_index(entry: &IndexManifestEntry) -> bool { - entry.index_file.global_index_meta.is_some() -} - -pub(in crate::table) fn source(entry: &IndexManifestEntry) -> Result { - let file = &entry.index_file; - if !matches!( - file.index_type.as_str(), - "btree" | "bitmap" | "multivalue" | "fm" | "full-text" - ) && !crate::vindex::is_vindex_index_type(&file.index_type) - { - return Err(invalid("Unsupported primary-key index type")); - } - let meta = file - .global_index_meta - .as_ref() - .ok_or_else(|| invalid("Missing PK index metadata"))?; - if meta - .extra_field_ids - .as_ref() - .is_some_and(|ids| !ids.is_empty()) - || file.deletion_vectors_ranges.is_some() - { - return Err(invalid("Invalid primary-key index metadata")); - } - PrimaryKeyIndexSourceMeta::from_global_index_meta(meta) -} - -fn unit(entry: &IndexManifestEntry) -> Result { - Ok(( - entry.partition.clone(), - entry.bucket, - source(entry)?.data_level(), - entry - .index_file - .global_index_meta - .as_ref() - .unwrap() - .index_field_id, - )) -} - -async fn read_indexes(table: &Table, snapshot: &Snapshot) -> Result> { - match snapshot.index_manifest() { - Some(name) => { - IndexManifest::read( - table.file_io(), - &table.snapshot_manager().manifest_path(name), - ) - .await - } - None => Ok(vec![]), - } -} - -fn levels( - active: &[ManifestEntry], - changes: &[ManifestEntry], - partition: &[u8], - bucket: i32, -) -> Levels { - let removed = changes - .iter() - .filter(|e| { - *e.kind() == FileKind::Delete && e.partition() == partition && e.bucket() == bucket - }) - .map(|e| e.file().file_name.as_str()) - .collect::>(); - let mut levels = Levels::new(); - for entry in active - .iter() - .filter(|e| !removed.contains(e.file().file_name.as_str())) - .chain(changes.iter().filter(|e| *e.kind() == FileKind::Add)) - .filter(|e| { - e.partition() == partition - && e.bucket() == bucket - && should_read_pk_index_source(e.file()) - }) - { - levels - .entry(entry.file().level) - .or_default() - .push(entry.file().clone()); - } - for files in levels.values_mut() { - files.sort_by(|a, b| a.file_name.cmp(&b.file_name)); - } - levels -} - -fn source_meta(level: i32, files: &[DataFileMeta]) -> Result { - PrimaryKeyIndexSourceMeta::new( - level, - files - .iter() - .map(|f| PrimaryKeyIndexSourceFile::new(f.file_name.clone(), f.row_count)) - .collect::>()?, - ) -} - -fn row_count(files: &[DataFileMeta]) -> Result { - files.iter().try_fold(0i64, |sum, f| { - sum.checked_add(f.row_count) - .ok_or_else(|| invalid("PK index row count overflow")) - }) -} - -fn compatible(table: &Table, entry: &IndexManifestEntry, definition: &Definition) -> Result { - if entry.index_file.index_type != definition.index_type { - return Ok(false); - } - if crate::vindex::is_vindex_index_type(&definition.index_type) - && entry.index_file.row_count == 0 - && entry.index_file.file_size == 0 - { - return Ok(true); - } - if definition.index_type == "full-text" - || crate::vindex::is_vindex_index_type(&definition.index_type) - { - let expected = writer::native_options(table, definition)?; - let recorded = entry - .index_file - .global_index_meta - .as_ref() - .and_then(|m| m.index_meta.as_deref()) - .and_then(|bytes| serde_json::from_slice::>(bytes).ok()); - return Ok(recorded.as_ref() == Some(&expected)); - } - Ok(true) -} - -struct Maintenance<'a> { - deletes: Vec, - builds: Vec<(i32, &'a Definition)>, -} - -async fn unsupported_scalar_units( - table: &Table, - levels: &Levels, - defs: &[Definition], -) -> HashSet<(i32, i32)> { - let mut unsupported = HashSet::new(); - for (level, files) in levels { - let files = files.iter().collect::>(); - for definition in defs.iter().filter(|d| { - matches!( - d.index_type.as_str(), - "btree" | "bitmap" | "multivalue" | "fm" - ) - }) { - if !crate::table::pk_index_schema::source_types_match(table, &definition.field, &files) - .await - { - unsupported.insert((*level, definition.field.id())); - } - } - } - unsupported -} - -fn plan<'a>( - table: &Table, - indexes: &[IndexManifestEntry], - levels: &Levels, - defs: &'a [Definition], - unsupported: &HashSet<(i32, i32)>, - partition: &[u8], - bucket: i32, -) -> Result> { - let mut existing = BTreeMap::<(i32, i32), Vec<&IndexManifestEntry>>::new(); - for entry in indexes - .iter() - .filter(|e| e.partition == partition && e.bucket == bucket && is_source_index(e)) - { - let meta = source(entry)?; - existing - .entry(( - meta.data_level(), - entry - .index_file - .global_index_meta - .as_ref() - .unwrap() - .index_field_id, - )) - .or_default() - .push(entry); - } - let mut deletes = Vec::new(); - let mut builds = Vec::new(); - for ((level, field), entries) in &existing { - let expected = levels - .get(level) - .map(|files| source_meta(*level, files)) - .transpose()?; - let definition = defs.iter().find(|d| d.field.id() == *field); - let valid = entries.len() == 1 - && expected.as_ref() == Some(&source(entries[0])?) - && match definition { - Some(d) => { - !unsupported.contains(&(*level, *field)) && compatible(table, entries[0], d)? - } - None => true, - }; - if !valid { - for entry in entries { - let mut deleted = (*entry).clone(); - deleted.kind = FileKind::Delete; - deletes.push(deleted); - } - } - } - for (level, files) in levels { - if row_count(files)? == 0 { - continue; - } - for definition in defs { - if unsupported.contains(&(*level, definition.field.id())) { - continue; - } - let valid = existing - .get(&(*level, definition.field.id())) - .is_some_and(|entries| { - entries.len() == 1 - && !deletes - .iter() - .any(|d| d.index_file.file_name == entries[0].index_file.file_name) - }); - if !valid { - builds.push((*level, definition)); - } - } - } - Ok(Maintenance { deletes, builds }) -} pub(in crate::table::compact) async fn prepare( table: &Table, @@ -352,150 +109,3 @@ pub(in crate::table) async fn abort(table: &Table, entries: &[IndexManifestEntry .abort(&messages) .await; } - -/// Rebase unrelated indexes while checking the captured membership of changed units. -pub(in crate::table) async fn resolve( - table: &Table, - captured: Option<&Snapshot>, - latest: &Snapshot, - active: &[ManifestEntry], - changes: &[ManifestEntry], - prepared: Option<&[IndexManifestEntry]>, -) -> Result>> { - if table.schema().primary_keys().is_empty() { - return Ok(None); - } - let defs = definitions(table.schema())?; - let current = read_indexes(table, latest).await?; - let captured = match captured { - Some(snapshot) => read_indexes(table, snapshot).await?, - None => return Err(invalid("PK compaction requires its captured snapshot")), - }; - let buckets = changes - .iter() - .map(|e| (e.partition().to_vec(), e.bucket())) - .collect::>(); - if prepared.is_none() - && (!defs.is_empty() - || captured - .iter() - .any(|e| is_source_index(e) && buckets.contains(&(e.partition.clone(), e.bucket)))) - { - return Err(invalid( - "Primary-key compaction results are missing index maintenance; execute the plan again", - )); - } - let prepared = prepared.unwrap_or_default(); - let mut required_deletes = Vec::new(); - let mut required_builds = HashSet::new(); - let mut after_levels = BTreeMap::new(); - for (partition, bucket) in &buckets { - let after = levels(active, changes, partition, *bucket); - let unsupported = unsupported_scalar_units(table, &after, &defs).await; - let maintenance = plan( - table, - &captured, - &after, - &defs, - &unsupported, - partition, - *bucket, - )?; - required_deletes.extend(maintenance.deletes); - for (level, definition) in maintenance.builds { - required_builds.insert((partition.clone(), *bucket, level, definition.field.id())); - } - after_levels.insert((partition.clone(), *bucket), after); - } - let actual_deletes = prepared - .iter() - .filter(|e| e.kind == FileKind::Delete) - .collect::>(); - if actual_deletes.len() != required_deletes.len() - || required_deletes - .iter() - .any(|e| !actual_deletes.contains(&e)) - { - return Err(invalid( - "PK compaction index retirements do not match captured sources", - )); - } - let mut touched = required_deletes - .iter() - .map(unit) - .collect::>>()?; - touched.extend(required_builds.iter().cloned()); - for key in &touched { - let before = captured - .iter() - .filter(|e| is_source_index(e) && unit(e).is_ok_and(|u| &u == key)) - .collect::>(); - let now = current - .iter() - .filter(|e| is_source_index(e) && unit(e).is_ok_and(|u| &u == key)) - .collect::>(); - if before != now { - return Err(invalid( - "PK index changed concurrently; execute compaction again", - )); - } - } - let mut names = HashSet::new(); - for entry in prepared.iter().filter(|e| e.kind == FileKind::Add) { - if !names.insert(entry.index_file.file_name.as_str()) - || current - .iter() - .any(|old| old.index_file.file_name == entry.index_file.file_name) - { - return Err(invalid("PK index output must have a new unique file name")); - } - let key = unit(entry)?; - if !required_builds.remove(&key) { - return Err(invalid("Unexpected or duplicate PK index output")); - } - let definition = defs - .iter() - .find(|d| d.field.id() == key.3) - .ok_or_else(|| invalid("Undeclared PK index output"))?; - let files = &after_levels[&(key.0.clone(), key.1)][&key.2]; - let meta = entry.index_file.global_index_meta.as_ref().unwrap(); - let count = row_count(files)?; - let indexed_rows = entry.index_file.row_count; - let valid_count = if crate::vindex::is_vindex_index_type(&definition.index_type) { - indexed_rows >= 0 - && indexed_rows <= count - && ((indexed_rows == 0) == (entry.index_file.file_size == 0)) - } else { - indexed_rows == count - }; - if source(entry)? != source_meta(key.2, files)? - || meta.row_range_start != 0 - || meta.row_range_end != count - 1 - || !valid_count - || entry.index_file.file_size < 0 - || !compatible(table, entry, definition)? - { - return Err(invalid( - "PK index output does not match complete compacted level", - )); - } - } - if !required_builds.is_empty() { - return Err(invalid("Missing PK index output")); - } - if prepared.is_empty() { - return Ok(None); - } - let mut merged = current; - for deleted in actual_deletes { - let mut added = deleted.clone(); - added.kind = FileKind::Add; - let position = merged - .iter() - .position(|e| e == &added) - .ok_or_else(|| invalid("PK index changed concurrently"))?; - merged.remove(position); - } - merged.extend(prepared.iter().filter(|e| e.kind == FileKind::Add).cloned()); - Ok(Some(merged)) -} diff --git a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs index 68a477952..bcd07473a 100644 --- a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs +++ b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs @@ -25,6 +25,7 @@ use crate::table::index_build::{ vector, IndexFileWriter, }; use crate::table::index_file_path::IndexFileLocation; +use crate::table::pk_index_maintenance::{row_count, source_meta}; use crate::table::pk_index_options::Definition; use crate::table::{ArrowRecordBatchStream, DataSplit, RowRange, Table}; use crate::vindex::VindexVectorIndexOptions; @@ -33,32 +34,8 @@ use arrow_array::{Int64Array, RecordBatch}; use arrow_schema::{DataType as ArrowType, Field, Schema}; use async_stream::try_stream; use futures::TryStreamExt; -use std::collections::BTreeMap; use std::sync::Arc; -pub(super) fn native_options( - table: &Table, - definition: &Definition, -) -> Result> { - if crate::vindex::is_vindex_index_type(&definition.index_type) { - Ok(VindexVectorIndexOptions::new( - table.schema().options(), - &definition.options, - &definition.index_type, - &definition.field, - )? - .native_options - .into_iter() - .collect()) - } else { - let mut options = table.schema().options().clone(); - options.extend(definition.options.clone()); - Ok(full_text::native_full_text_options(&options) - .into_iter() - .collect()) - } -} - /// Physical rows include NULLs, tombstones and DV-hidden positions. The common /// writers receive dense, level-local ordinals, not a merged or filtered read. fn batches( @@ -92,7 +69,7 @@ fn batches( .ok_or_else(|| invalid("PK index ordinal overflow"))?; } } - if offset != super::row_count(&files)? { + if offset != row_count(&files)? { Err(invalid("PK index source row count changed"))?; } }) @@ -105,14 +82,14 @@ pub(super) async fn build( definition: &Definition, resources: Option, ) -> Result> { - let count = super::row_count(files)?; + let count = row_count(files)?; let range = RowRange::new(0, count - 1); let meta = GlobalIndexMeta { row_range_start: range.from(), row_range_end: range.to(), index_field_id: definition.field.id(), extra_field_ids: None, - source_meta: Some(super::source_meta(files[0].level, files)?.serialize()?), + source_meta: Some(source_meta(files[0].level, files)?.serialize()?), index_meta: None, }; let in_data = table.schema().core_options().index_file_in_data_file_dir(); diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index a22177d66..1ae508124 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -119,6 +119,7 @@ mod pk_full_text_bucket_search; mod pk_full_text_bucket_state; mod pk_full_text_read; mod pk_full_text_scan; +mod pk_index_maintenance; mod pk_index_options; mod pk_index_schema; mod pk_search_position; diff --git a/crates/paimon/src/table/pk_index_maintenance.rs b/crates/paimon/src/table/pk_index_maintenance.rs new file mode 100644 index 000000000..3d636b646 --- /dev/null +++ b/crates/paimon/src/table/pk_index_maintenance.rs @@ -0,0 +1,409 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared source planning and commit validation for primary-key index maintenance. +//! +//! Compaction builds the planned units. Commit validation checks the same source +//! and compatibility rules against captured and current snapshot metadata. + +use crate::spec::{ + should_read_pk_index_source, DataFileMeta, FileKind, IndexManifest, IndexManifestEntry, + ManifestEntry, PrimaryKeyIndexSourceFile, PrimaryKeyIndexSourceMeta, Snapshot, +}; +use crate::table::pk_index_options::{definitions, native_options, Definition}; +use crate::table::Table; +use crate::{Error, Result}; +use std::collections::{BTreeMap, HashSet}; + +type Unit = (Vec, i32, i32, i32); +type Levels = BTreeMap>; + +pub(super) fn is_source_index(entry: &IndexManifestEntry) -> bool { + entry.index_file.global_index_meta.is_some() +} + +pub(super) fn source(entry: &IndexManifestEntry) -> Result { + let file = &entry.index_file; + if !matches!( + file.index_type.as_str(), + "btree" | "bitmap" | "multivalue" | "fm" | "full-text" + ) && !crate::vindex::is_vindex_index_type(&file.index_type) + { + return Err(invalid("Unsupported primary-key index type")); + } + let meta = file + .global_index_meta + .as_ref() + .ok_or_else(|| invalid("Missing PK index metadata"))?; + if meta + .extra_field_ids + .as_ref() + .is_some_and(|ids| !ids.is_empty()) + || file.deletion_vectors_ranges.is_some() + { + return Err(invalid("Invalid primary-key index metadata")); + } + PrimaryKeyIndexSourceMeta::from_global_index_meta(meta) +} + +fn unit(entry: &IndexManifestEntry) -> Result { + Ok(( + entry.partition.clone(), + entry.bucket, + source(entry)?.data_level(), + entry + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_field_id, + )) +} + +async fn read_indexes(table: &Table, snapshot: &Snapshot) -> Result> { + match snapshot.index_manifest() { + Some(name) => { + IndexManifest::read( + table.file_io(), + &table.snapshot_manager().manifest_path(name), + ) + .await + } + None => Ok(vec![]), + } +} + +pub(super) fn levels( + active: &[ManifestEntry], + changes: &[ManifestEntry], + partition: &[u8], + bucket: i32, +) -> Levels { + let removed = changes + .iter() + .filter(|e| { + *e.kind() == FileKind::Delete && e.partition() == partition && e.bucket() == bucket + }) + .map(|e| e.file().file_name.as_str()) + .collect::>(); + let mut levels = Levels::new(); + for entry in active + .iter() + .filter(|e| !removed.contains(e.file().file_name.as_str())) + .chain(changes.iter().filter(|e| *e.kind() == FileKind::Add)) + .filter(|e| { + e.partition() == partition + && e.bucket() == bucket + && should_read_pk_index_source(e.file()) + }) + { + levels + .entry(entry.file().level) + .or_default() + .push(entry.file().clone()); + } + for files in levels.values_mut() { + files.sort_by(|a, b| a.file_name.cmp(&b.file_name)); + } + levels +} + +pub(super) fn source_meta(level: i32, files: &[DataFileMeta]) -> Result { + PrimaryKeyIndexSourceMeta::new( + level, + files + .iter() + .map(|f| PrimaryKeyIndexSourceFile::new(f.file_name.clone(), f.row_count)) + .collect::>()?, + ) +} + +pub(super) fn row_count(files: &[DataFileMeta]) -> Result { + files.iter().try_fold(0i64, |sum, f| { + sum.checked_add(f.row_count) + .ok_or_else(|| invalid("PK index row count overflow")) + }) +} + +fn compatible(table: &Table, entry: &IndexManifestEntry, definition: &Definition) -> Result { + if entry.index_file.index_type != definition.index_type { + return Ok(false); + } + if crate::vindex::is_vindex_index_type(&definition.index_type) + && entry.index_file.row_count == 0 + && entry.index_file.file_size == 0 + { + return Ok(true); + } + if definition.index_type == "full-text" + || crate::vindex::is_vindex_index_type(&definition.index_type) + { + let expected = native_options(table, definition)?; + let recorded = entry + .index_file + .global_index_meta + .as_ref() + .and_then(|m| m.index_meta.as_deref()) + .and_then(|bytes| serde_json::from_slice::>(bytes).ok()); + return Ok(recorded.as_ref() == Some(&expected)); + } + Ok(true) +} + +pub(super) struct Maintenance<'a> { + pub(super) deletes: Vec, + pub(super) builds: Vec<(i32, &'a Definition)>, +} + +pub(super) async fn unsupported_scalar_units( + table: &Table, + levels: &Levels, + defs: &[Definition], +) -> HashSet<(i32, i32)> { + let mut unsupported = HashSet::new(); + for (level, files) in levels { + let files = files.iter().collect::>(); + for definition in defs.iter().filter(|d| { + matches!( + d.index_type.as_str(), + "btree" | "bitmap" | "multivalue" | "fm" + ) + }) { + if !crate::table::pk_index_schema::source_types_match(table, &definition.field, &files) + .await + { + unsupported.insert((*level, definition.field.id())); + } + } + } + unsupported +} + +pub(super) fn plan<'a>( + table: &Table, + indexes: &[IndexManifestEntry], + levels: &Levels, + defs: &'a [Definition], + unsupported: &HashSet<(i32, i32)>, + partition: &[u8], + bucket: i32, +) -> Result> { + let mut existing = BTreeMap::<(i32, i32), Vec<&IndexManifestEntry>>::new(); + for entry in indexes + .iter() + .filter(|e| e.partition == partition && e.bucket == bucket && is_source_index(e)) + { + let meta = source(entry)?; + existing + .entry(( + meta.data_level(), + entry + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_field_id, + )) + .or_default() + .push(entry); + } + let mut deletes = Vec::new(); + let mut builds = Vec::new(); + for ((level, field), entries) in &existing { + let expected = levels + .get(level) + .map(|files| source_meta(*level, files)) + .transpose()?; + let definition = defs.iter().find(|d| d.field.id() == *field); + let valid = entries.len() == 1 + && expected.as_ref() == Some(&source(entries[0])?) + && match definition { + Some(d) => { + !unsupported.contains(&(*level, *field)) && compatible(table, entries[0], d)? + } + None => true, + }; + if !valid { + for entry in entries { + let mut deleted = (*entry).clone(); + deleted.kind = FileKind::Delete; + deletes.push(deleted); + } + } + } + for (level, files) in levels { + if row_count(files)? == 0 { + continue; + } + for definition in defs { + if unsupported.contains(&(*level, definition.field.id())) { + continue; + } + let valid = existing + .get(&(*level, definition.field.id())) + .is_some_and(|entries| { + entries.len() == 1 + && !deletes + .iter() + .any(|d| d.index_file.file_name == entries[0].index_file.file_name) + }); + if !valid { + builds.push((*level, definition)); + } + } + } + Ok(Maintenance { deletes, builds }) +} + +/// Validate prepared outputs and the captured membership of changed PK units. +pub(super) async fn validate( + table: &Table, + captured: Option<&Snapshot>, + current: &[IndexManifestEntry], + active: &[ManifestEntry], + changes: &[ManifestEntry], + prepared: Option<&[IndexManifestEntry]>, +) -> Result<()> { + if table.schema().primary_keys().is_empty() { + return Ok(()); + } + let defs = definitions(table.schema())?; + let captured = match captured { + Some(snapshot) => read_indexes(table, snapshot).await?, + None => return Err(invalid("PK compaction requires its captured snapshot")), + }; + let buckets = changes + .iter() + .map(|e| (e.partition().to_vec(), e.bucket())) + .collect::>(); + if prepared.is_none() + && (!defs.is_empty() + || captured + .iter() + .any(|e| is_source_index(e) && buckets.contains(&(e.partition.clone(), e.bucket)))) + { + return Err(invalid( + "Primary-key compaction results are missing index maintenance; execute the plan again", + )); + } + let prepared = prepared.unwrap_or_default(); + let mut required_deletes = Vec::new(); + let mut required_builds = HashSet::new(); + let mut after_levels = BTreeMap::new(); + for (partition, bucket) in &buckets { + let after = levels(active, changes, partition, *bucket); + let unsupported = unsupported_scalar_units(table, &after, &defs).await; + let maintenance = plan( + table, + &captured, + &after, + &defs, + &unsupported, + partition, + *bucket, + )?; + required_deletes.extend(maintenance.deletes); + for (level, definition) in maintenance.builds { + required_builds.insert((partition.clone(), *bucket, level, definition.field.id())); + } + after_levels.insert((partition.clone(), *bucket), after); + } + let actual_deletes = prepared + .iter() + .filter(|e| e.kind == FileKind::Delete) + .collect::>(); + if actual_deletes.len() != required_deletes.len() + || required_deletes + .iter() + .any(|e| !actual_deletes.contains(&e)) + { + return Err(invalid( + "PK compaction index retirements do not match captured sources", + )); + } + let mut touched = required_deletes + .iter() + .map(unit) + .collect::>>()?; + touched.extend(required_builds.iter().cloned()); + for key in &touched { + let before = captured + .iter() + .filter(|e| is_source_index(e) && unit(e).is_ok_and(|u| &u == key)) + .collect::>(); + let now = current + .iter() + .filter(|e| is_source_index(e) && unit(e).is_ok_and(|u| &u == key)) + .collect::>(); + if before != now { + return Err(invalid( + "PK index changed concurrently; execute compaction again", + )); + } + } + let mut names = HashSet::new(); + for entry in prepared.iter().filter(|e| e.kind == FileKind::Add) { + if !names.insert(entry.index_file.file_name.as_str()) + || current + .iter() + .any(|old| old.index_file.file_name == entry.index_file.file_name) + { + return Err(invalid("PK index output must have a new unique file name")); + } + let key = unit(entry)?; + if !required_builds.remove(&key) { + return Err(invalid("Unexpected or duplicate PK index output")); + } + let definition = defs + .iter() + .find(|d| d.field.id() == key.3) + .ok_or_else(|| invalid("Undeclared PK index output"))?; + let files = &after_levels[&(key.0.clone(), key.1)][&key.2]; + let meta = entry.index_file.global_index_meta.as_ref().unwrap(); + let count = row_count(files)?; + let indexed_rows = entry.index_file.row_count; + let valid_count = if crate::vindex::is_vindex_index_type(&definition.index_type) { + indexed_rows >= 0 + && indexed_rows <= count + && ((indexed_rows == 0) == (entry.index_file.file_size == 0)) + } else { + indexed_rows == count + }; + if source(entry)? != source_meta(key.2, files)? + || meta.row_range_start != 0 + || meta.row_range_end != count - 1 + || !valid_count + || entry.index_file.file_size < 0 + || !compatible(table, entry, definition)? + { + return Err(invalid( + "PK index output does not match complete compacted level", + )); + } + } + if !required_builds.is_empty() { + return Err(invalid("Missing PK index output")); + } + Ok(()) +} + +fn invalid(message: &str) -> Error { + Error::DataInvalid { + message: message.into(), + source: None, + } +} diff --git a/crates/paimon/src/table/pk_index_options.rs b/crates/paimon/src/table/pk_index_options.rs index e41d8a35f..0dfeb2f42 100644 --- a/crates/paimon/src/table/pk_index_options.rs +++ b/crates/paimon/src/table/pk_index_options.rs @@ -20,8 +20,11 @@ use crate::spec::{CoreOptions, DataField, DataType, TableSchema}; use crate::table::global_index_options::parse_field_index_options; use crate::table::global_index_types::normalize_global_index_type; +use crate::table::index_build::full_text; +use crate::table::Table; +use crate::vindex::VindexVectorIndexOptions; use crate::{Error, Result}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; pub(super) struct Definition { pub field: DataField, @@ -165,3 +168,27 @@ pub(super) fn definitions(schema: &TableSchema) -> Result> { } Ok(result) } + +/// Canonical persisted options used to validate PK payload compatibility. +pub(super) fn native_options( + table: &Table, + definition: &Definition, +) -> Result> { + if crate::vindex::is_vindex_index_type(&definition.index_type) { + Ok(VindexVectorIndexOptions::new( + table.schema().options(), + &definition.options, + &definition.index_type, + &definition.field, + )? + .native_options + .into_iter() + .collect()) + } else { + let mut options = table.schema().options().clone(); + options.extend(definition.options.clone()); + Ok(full_text::native_full_text_options(&options) + .into_iter() + .collect()) + } +} diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index fba808f72..bc1f49355 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -54,6 +54,9 @@ mod auto_index; mod auto_index_tests; mod auto_maintenance; mod global_index_source_check; +mod index_commit; + +use index_commit::IndexChanges; mod row_tracking; const DELETION_VECTORS_INDEX_TYPE: &str = "DELETION_VECTORS"; @@ -1468,7 +1471,7 @@ impl TableCommit { .await? { if let Some(prepared) = &resolved.new_index_manifest_entries { - let is_pk = super::compact::primary_key::indexes::is_source_index; + let is_pk = super::pk_index_maintenance::is_source_index; indexes.retain(|e| !is_pk(e)); indexes.extend(prepared.iter().filter(|e| is_pk(e)).cloned()); } @@ -1990,9 +1993,6 @@ impl TableCommit { retry_state: Option<&RetryState>, check_append_files: bool, ) -> Result { - let file_io = self.snapshot_manager.file_io(); - let manifest_dir = self.snapshot_manager.manifest_dir(); - match plan { CommitEntriesPlan::Compact { entries, @@ -2033,12 +2033,9 @@ impl TableCommit { }; // Concurrent writes may replace HASH indexes, deletion vectors // or global indexes. Rebase on the latest manifest every retry. - let index_manifest_name = match latest_snapshot { - Some(snapshot) => { - super::compact::validate_snapshot_indexes(&self.table, snapshot).await? - } - None => None, - }; + if let Some(snapshot) = latest_snapshot { + super::compact::validate_snapshot_indexes(&self.table, snapshot).await?; + } validate_file_entries(entries.iter())?; validate_file_entries(changelog_entries.iter())?; // This runs after duplicate-commit filtering: an uncertain commit @@ -2181,23 +2178,25 @@ impl TableCommit { ) .await?; } - let new_indexes = super::compact::primary_key::indexes::resolve( - &self.table, - read_snapshot.as_deref(), - latest_snapshot.as_ref().expect("validated latest snapshot"), - base_data_files.as_deref().unwrap_or_default(), - entries, - index_entries.as_deref(), - ) - .await?; + let indexes = self + .resolve_indexes( + latest_snapshot, + entries, + IndexChanges::Compact { + captured: read_snapshot.as_deref(), + active: base_data_files.as_deref().unwrap_or_default(), + prepared: index_entries.as_deref(), + }, + ) + .await?; Ok(ResolvedCommit { entries: entries.clone(), changelog_entries: changelog_entries.clone(), deletion_vector_updates: deletion_vector_updates.to_vec(), kind: CommitKind::COMPACT, - index_manifest_name, - index_manifest_changed: new_indexes.is_some(), - new_index_manifest_entries: new_indexes, + index_manifest_name: indexes.manifest_name, + index_manifest_changed: indexes.entries.is_some(), + new_index_manifest_entries: indexes.entries, base_data_files, }) } @@ -2241,37 +2240,22 @@ impl TableCommit { None }; - let previous = - Self::read_prev_index_entries(file_io, &manifest_dir, latest_snapshot).await?; - self.check_global_index_sources(latest_snapshot, entries, new_index_entries) + let indexes = self + .resolve_indexes( + latest_snapshot, + entries, + IndexChanges::Direct(new_index_entries), + ) .await?; - let mut index_entries = new_index_entries.clone(); - index_entries.extend( - self.global_index_update_entries(&previous, entries, new_index_entries) - .await?, - ); - let all = Self::merge_index_entries(&previous, &index_entries, false)?; - self.check_deletion_vector_references( - latest_snapshot, - entries, - &index_entries, - &all, - ) - .await?; - let index_manifest_changed = all != previous; - let index_manifest_name = latest_snapshot - .as_ref() - .and_then(|s| s.index_manifest().map(str::to_string)); - let new_index_manifest_entries = index_manifest_changed.then_some(all); Ok(ResolvedCommit { entries: entries.clone(), changelog_entries: changelog_entries.clone(), deletion_vector_updates: vec![], kind, - index_manifest_name, - index_manifest_changed, - new_index_manifest_entries, + index_manifest_name: indexes.manifest_name, + index_manifest_changed: indexes.entries.is_some(), + new_index_manifest_entries: indexes.entries, base_data_files, }) } @@ -2306,172 +2290,31 @@ impl TableCommit { ) .await?; - let previous = - Self::read_prev_index_entries(file_io, &manifest_dir, latest_snapshot).await?; - self.check_global_index_sources(latest_snapshot, &entries, &new_index_entries) + let indexes = self + .resolve_indexes( + latest_snapshot, + &entries, + IndexChanges::Overwrite { + entries: &new_index_entries, + partition_filter: partition_filter.as_ref(), + }, + ) .await?; - let mut all = previous.clone(); - match partition_filter.as_ref() { - None => all.clear(), - Some(filter) => { - let mut retained = Vec::new(); - for e in all { - if !filter.matches_entry(&e.partition)? { - retained.push(e); - } - } - all = retained; - } - } - let all = Self::merge_index_entries(&all, &new_index_entries, false)?; - self.check_deletion_vector_references( - latest_snapshot, - &entries, - &new_index_entries, - &all, - ) - .await?; - let index_manifest_changed = all != previous; - let index_manifest_name = latest_snapshot - .as_ref() - .and_then(|s| s.index_manifest().map(str::to_string)); - let new_index_manifest_entries = index_manifest_changed.then_some(all); Ok(ResolvedCommit { entries, changelog_entries: vec![], deletion_vector_updates: vec![], kind: CommitKind::OVERWRITE, - index_manifest_name, - index_manifest_changed, - new_index_manifest_entries, + index_manifest_name: indexes.manifest_name, + index_manifest_changed: indexes.entries.is_some(), + new_index_manifest_entries: indexes.entries, base_data_files, }) } } } - fn merge_index_entries( - previous_entries: &[IndexManifestEntry], - new_index_entries: &[IndexManifestEntry], - drop_previous_global_indexes: bool, - ) -> Result> { - let mut all = if drop_previous_global_indexes { - previous_entries - .iter() - .filter(|entry| entry.index_file.global_index_meta.is_none()) - .cloned() - .collect::>() - } else { - previous_entries.to_vec() - }; - let deletions = new_index_entries - .iter() - .filter(|entry| entry.kind == FileKind::Delete) - .collect::>(); - for deletion in deletions { - let global_index = !matches!( - deletion.index_file.index_type.as_str(), - "HASH" | DELETION_VECTORS_INDEX_TYPE - ); - // Java combines global indexes by type and file name, then checks - // the complete current metadata before removing an entry. A stale - // removal must not discard metadata-only row-id reassignment. - let position = all.iter().position(|entry| { - if global_index { - entry.index_file.index_type == deletion.index_file.index_type - && entry.index_file.file_name == deletion.index_file.file_name - } else { - same_index_file_entry(entry, deletion) - } - }); - if let Some(position) = position { - let previous = &all[position]; - if global_index - && (!same_index_partition(&previous.partition, &deletion.partition) - || previous.bucket != deletion.bucket - || previous.index_file != deletion.index_file) - { - return Err(crate::Error::DataInvalid { - message: format!( - "Index file {} metadata does not match the current entry.", - deletion.index_file.file_name - ), - source: None, - }); - } - all.remove(position); - } else if global_index { - return Err(crate::Error::DataInvalid { - message: format!( - "Trying to delete global index file {} which does not exist.", - deletion.index_file.file_name - ), - source: None, - }); - } else if deletion.index_file.index_type == DELETION_VECTORS_INDEX_TYPE { - return Err(crate::Error::DataInvalid { - message: format!("Cannot replace missing deletion vector index '{}'; prepare the DELETE again from the latest snapshot", deletion.index_file.file_name), - source: None, - }); - } - } - - let additions = new_index_entries - .iter() - .filter(|entry| entry.kind == FileKind::Add) - .cloned() - .collect::>(); - let new_hash_keys: HashSet<(Vec, i32)> = additions - .iter() - .filter(|e| e.index_file.index_type == "HASH") - .map(|e| (e.partition.clone(), e.bucket)) - .collect(); - all.retain(|e| { - if e.index_file.index_type == "HASH" { - !new_hash_keys.contains(&(e.partition.clone(), e.bucket)) - } else { - true - } - }); - Self::validate_global_index_overlap(&all, &additions)?; - Self::validate_added_global_index_overlap(&additions)?; - let mut dv_files = HashSet::new(); - let mut dv_names = HashSet::new(); - for entry in all - .iter() - .chain(&additions) - .filter(|entry| entry.index_file.index_type == DELETION_VECTORS_INDEX_TYPE) - { - let partition = if is_empty_partition(&entry.partition) { - &[][..] - } else { - entry.partition.as_slice() - }; - if !dv_names.insert((partition, entry.bucket, entry.index_file.file_name.as_str())) { - return Err(crate::Error::DataInvalid { - message: format!( - "Duplicate deletion vector index '{}'", - entry.index_file.file_name - ), - source: None, - }); - } - if let Some(ranges) = &entry.index_file.deletion_vectors_ranges { - for file in ranges.keys() { - if !dv_files.insert((partition, entry.bucket, file.as_str())) { - return Err(crate::Error::DataInvalid { - message: format!("Conflicting deletion vectors for data file '{file}'; prepare the DELETE again from the latest snapshot"), - source: None, - }); - } - } - } - } - all.extend(additions); - Ok(all) - } - async fn global_index_update_entries( &self, previous_entries: &[IndexManifestEntry], @@ -2618,103 +2461,6 @@ impl TableCommit { } } - fn validate_global_index_overlap( - retained_entries: &[IndexManifestEntry], - added_entries: &[IndexManifestEntry], - ) -> Result<()> { - for retained in retained_entries { - if retained.kind == FileKind::Delete { - continue; - } - let Some(retained_meta) = retained.index_file.global_index_meta.as_ref() else { - continue; - }; - for added in added_entries { - if added.kind == FileKind::Delete { - continue; - } - let Some(added_meta) = added.index_file.global_index_meta.as_ref() else { - continue; - }; - if retained_meta.index_field_id == added_meta.index_field_id - && retained.index_file.index_type == added.index_file.index_type - && same_extra_field_ids( - retained_meta.extra_field_ids.as_deref(), - added_meta.extra_field_ids.as_deref(), - ) - && ranges_overlap( - retained_meta.row_range_start, - retained_meta.row_range_end, - added_meta.row_range_start, - added_meta.row_range_end, - ) - { - return Err(global_index_overlap_error( - retained, - retained_meta, - added, - added_meta, - )); - } - } - } - Ok(()) - } - - fn validate_added_global_index_overlap(added_entries: &[IndexManifestEntry]) -> Result<()> { - for (left_index, left) in added_entries.iter().enumerate() { - if left.kind == FileKind::Delete { - continue; - } - let Some(left_meta) = left.index_file.global_index_meta.as_ref() else { - continue; - }; - for right in added_entries.iter().skip(left_index + 1) { - if right.kind == FileKind::Delete { - continue; - } - let Some(right_meta) = right.index_file.global_index_meta.as_ref() else { - continue; - }; - if left_meta.index_field_id == right_meta.index_field_id - && left.index_file.index_type == right.index_file.index_type - && same_extra_field_ids( - left_meta.extra_field_ids.as_deref(), - right_meta.extra_field_ids.as_deref(), - ) - && ranges_overlap( - left_meta.row_range_start, - left_meta.row_range_end, - right_meta.row_range_start, - right_meta.row_range_end, - ) - { - return Err(global_index_overlap_error( - left, left_meta, right, right_meta, - )); - } - } - } - Ok(()) - } - - /// Read index entries from the previous snapshot's index manifest. - async fn read_prev_index_entries( - file_io: &FileIO, - manifest_dir: &str, - latest_snapshot: &Option, - ) -> Result> { - if let Some(snap) = latest_snapshot { - if let Some(prev_index_manifest) = snap.index_manifest() { - let prev_path = format!("{manifest_dir}/{prev_index_manifest}"); - return Ok(normalize_index_entries( - IndexManifest::read(file_io, &prev_path).await?, - )); - } - } - Ok(vec![]) - } - /// Stateful overwrite provider mirroring Python `OverwriteChangesProvider`. async fn provide_overwrite_entries( &self, diff --git a/crates/paimon/src/table/table_commit/index_commit.rs b/crates/paimon/src/table/table_commit/index_commit.rs new file mode 100644 index 000000000..0732ec3ab --- /dev/null +++ b/crates/paimon/src/table/table_commit/index_commit.rs @@ -0,0 +1,374 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Resolve index deltas for compaction, ordinary writes and overwrites. +//! +//! Source validation belongs to each layout. Every validated delta uses the +//! same manifest merge and the caller's snapshot publication/recovery protocol. + +use super::*; +use std::borrow::Cow; + +#[derive(Clone, Copy)] +pub(super) enum IndexChanges<'a> { + Compact { + captured: Option<&'a Snapshot>, + active: &'a [ManifestEntry], + prepared: Option<&'a [IndexManifestEntry]>, + }, + Direct(&'a [IndexManifestEntry]), + Overwrite { + entries: &'a [IndexManifestEntry], + partition_filter: Option<&'a PartitionFilter>, + }, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum IndexDomain { + GlobalRowIds, + PrimaryKeyLevel, +} + +pub(super) struct ResolvedIndexes { + pub(super) manifest_name: Option, + pub(super) entries: Option>, +} + +impl TableCommit { + pub(super) async fn resolve_indexes( + &self, + latest_snapshot: &Option, + data_entries: &[ManifestEntry], + changes: IndexChanges<'_>, + ) -> Result { + let manifest_name = latest_snapshot + .as_ref() + .and_then(|snapshot| snapshot.index_manifest().map(str::to_owned)); + // DE compaction preserves global row IDs. Its index maintenance runs + // through Direct after data publication; compaction retains the manifest. + if matches!(changes, IndexChanges::Compact { .. }) + && self.table.schema().primary_keys().is_empty() + { + return Ok(ResolvedIndexes { + manifest_name, + entries: None, + }); + } + let file_io = self.snapshot_manager.file_io(); + let manifest_dir = self.snapshot_manager.manifest_dir(); + let previous = if matches!(changes, IndexChanges::Compact { .. }) { + Self::read_index_entries(file_io, &manifest_dir, latest_snapshot).await? + } else { + Self::read_prev_index_entries(file_io, &manifest_dir, latest_snapshot).await? + }; + if let IndexChanges::Compact { + captured, + active, + prepared, + } = changes + { + super::super::pk_index_maintenance::validate( + &self.table, + captured, + &previous, + active, + data_entries, + prepared, + ) + .await?; + let prepared = prepared.unwrap_or_default(); + let entries = if prepared.is_empty() { + None + } else { + Some(Self::merge_index_entries( + previous, + prepared, + IndexDomain::PrimaryKeyLevel, + )?) + }; + return Ok(ResolvedIndexes { + manifest_name, + entries, + }); + } + let (explicit, overwrite, partition_filter) = match changes { + IndexChanges::Direct(entries) => (entries, false, None), + IndexChanges::Overwrite { + entries, + partition_filter, + } => (entries, true, partition_filter), + IndexChanges::Compact { .. } => unreachable!("compaction handled above"), + }; + self.check_global_index_sources(latest_snapshot, data_entries, explicit) + .await?; + let mut updates = Cow::Borrowed(explicit); + if !overwrite { + let invalidated = self + .global_index_update_entries(&previous, data_entries, explicit) + .await?; + if !invalidated.is_empty() { + updates.to_mut().extend(invalidated); + } + } + let mut retained = previous.clone(); + if overwrite { + match partition_filter { + None => retained.clear(), + Some(filter) => { + let mut matching = Vec::new(); + for entry in retained { + if !filter.matches_entry(&entry.partition)? { + matching.push(entry); + } + } + retained = matching; + } + } + } + let merged = Self::merge_index_entries(retained, &updates, IndexDomain::GlobalRowIds)?; + self.check_deletion_vector_references(latest_snapshot, data_entries, &updates, &merged) + .await?; + Ok(ResolvedIndexes { + manifest_name, + entries: (merged != previous).then_some(merged), + }) + } + + async fn read_index_entries( + file_io: &FileIO, + manifest_dir: &str, + snapshot: &Option, + ) -> Result> { + if let Some(name) = snapshot.as_ref().and_then(Snapshot::index_manifest) { + return IndexManifest::read(file_io, &format!("{manifest_dir}/{name}")).await; + } + Ok(vec![]) + } + + /// Global index commits consume active entries; PK validation also needs + /// the original manifest membership before applying any changes. + pub(super) async fn read_prev_index_entries( + file_io: &FileIO, + manifest_dir: &str, + snapshot: &Option, + ) -> Result> { + Ok(normalize_index_entries( + Self::read_index_entries(file_io, manifest_dir, snapshot).await?, + )) + } + + /// Apply a validated index delta. PK ordinals belong to a source level; + /// global row IDs share a table-wide coordinate space. + pub(super) fn merge_index_entries( + mut all: Vec, + new_index_entries: &[IndexManifestEntry], + domain: IndexDomain, + ) -> Result> { + let deletions = new_index_entries + .iter() + .filter(|entry| entry.kind == FileKind::Delete) + .collect::>(); + for deletion in deletions { + let global_index = !matches!( + deletion.index_file.index_type.as_str(), + "HASH" | DELETION_VECTORS_INDEX_TYPE + ); + // Java combines global indexes by type and file name, then checks + // the complete current metadata before removing an entry. A stale + // removal must not discard metadata-only row-id reassignment. + let position = all.iter().position(|entry| { + if global_index { + entry.index_file.index_type == deletion.index_file.index_type + && entry.index_file.file_name == deletion.index_file.file_name + } else { + same_index_file_entry(entry, deletion) + } + }); + if let Some(position) = position { + let previous = &all[position]; + if global_index + && (!same_index_partition(&previous.partition, &deletion.partition) + || previous.bucket != deletion.bucket + || previous.index_file != deletion.index_file) + { + return Err(crate::Error::DataInvalid { + message: format!( + "Index file {} metadata does not match the current entry.", + deletion.index_file.file_name + ), + source: None, + }); + } + all.remove(position); + } else if global_index { + return Err(crate::Error::DataInvalid { + message: format!( + "Trying to delete global index file {} which does not exist.", + deletion.index_file.file_name + ), + source: None, + }); + } else if deletion.index_file.index_type == DELETION_VECTORS_INDEX_TYPE { + return Err(crate::Error::DataInvalid { + message: format!("Cannot replace missing deletion vector index '{}'; prepare the DELETE again from the latest snapshot", deletion.index_file.file_name), + source: None, + }); + } + } + + let additions = new_index_entries + .iter() + .filter(|entry| entry.kind == FileKind::Add) + .cloned() + .collect::>(); + let new_hash_keys: HashSet<(Vec, i32)> = additions + .iter() + .filter(|e| e.index_file.index_type == "HASH") + .map(|e| (e.partition.clone(), e.bucket)) + .collect(); + all.retain(|e| { + if e.index_file.index_type == "HASH" { + !new_hash_keys.contains(&(e.partition.clone(), e.bucket)) + } else { + true + } + }); + if domain == IndexDomain::GlobalRowIds { + Self::validate_global_index_overlap(&all, &additions)?; + Self::validate_added_global_index_overlap(&additions)?; + } + let mut dv_files = HashSet::new(); + let mut dv_names = HashSet::new(); + for entry in all + .iter() + .chain(&additions) + .filter(|entry| entry.index_file.index_type == DELETION_VECTORS_INDEX_TYPE) + { + let partition = if is_empty_partition(&entry.partition) { + &[][..] + } else { + entry.partition.as_slice() + }; + if !dv_names.insert((partition, entry.bucket, entry.index_file.file_name.as_str())) { + return Err(crate::Error::DataInvalid { + message: format!( + "Duplicate deletion vector index '{}'", + entry.index_file.file_name + ), + source: None, + }); + } + if let Some(ranges) = &entry.index_file.deletion_vectors_ranges { + for file in ranges.keys() { + if !dv_files.insert((partition, entry.bucket, file.as_str())) { + return Err(crate::Error::DataInvalid { + message: format!("Conflicting deletion vectors for data file '{file}'; prepare the DELETE again from the latest snapshot"), + source: None, + }); + } + } + } + } + all.extend(additions); + Ok(all) + } + + pub(super) fn validate_global_index_overlap( + retained_entries: &[IndexManifestEntry], + added_entries: &[IndexManifestEntry], + ) -> Result<()> { + for retained in retained_entries { + if retained.kind == FileKind::Delete { + continue; + } + let Some(retained_meta) = retained.index_file.global_index_meta.as_ref() else { + continue; + }; + for added in added_entries { + if added.kind == FileKind::Delete { + continue; + } + let Some(added_meta) = added.index_file.global_index_meta.as_ref() else { + continue; + }; + if retained_meta.index_field_id == added_meta.index_field_id + && retained.index_file.index_type == added.index_file.index_type + && same_extra_field_ids( + retained_meta.extra_field_ids.as_deref(), + added_meta.extra_field_ids.as_deref(), + ) + && ranges_overlap( + retained_meta.row_range_start, + retained_meta.row_range_end, + added_meta.row_range_start, + added_meta.row_range_end, + ) + { + return Err(global_index_overlap_error( + retained, + retained_meta, + added, + added_meta, + )); + } + } + } + Ok(()) + } + + pub(super) fn validate_added_global_index_overlap( + added_entries: &[IndexManifestEntry], + ) -> Result<()> { + for (left_index, left) in added_entries.iter().enumerate() { + if left.kind == FileKind::Delete { + continue; + } + let Some(left_meta) = left.index_file.global_index_meta.as_ref() else { + continue; + }; + for right in added_entries.iter().skip(left_index + 1) { + if right.kind == FileKind::Delete { + continue; + } + let Some(right_meta) = right.index_file.global_index_meta.as_ref() else { + continue; + }; + if left_meta.index_field_id == right_meta.index_field_id + && left.index_file.index_type == right.index_file.index_type + && same_extra_field_ids( + left_meta.extra_field_ids.as_deref(), + right_meta.extra_field_ids.as_deref(), + ) + && ranges_overlap( + left_meta.row_range_start, + left_meta.row_range_end, + right_meta.row_range_start, + right_meta.row_range_end, + ) + { + return Err(global_index_overlap_error( + left, left_meta, right, right_meta, + )); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/paimon/src/table/table_commit/index_commit/tests.rs b/crates/paimon/src/table/table_commit/index_commit/tests.rs new file mode 100644 index 000000000..1420bb7e6 --- /dev/null +++ b/crates/paimon/src/table/table_commit/index_commit/tests.rs @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{ + GlobalIndexMeta, IndexFileMeta, PrimaryKeyIndexSourceFile, PrimaryKeyIndexSourceMeta, +}; + +fn entry(domain: IndexDomain, name: &str, field: i32, level: i32) -> IndexManifestEntry { + let source_meta = match domain { + IndexDomain::GlobalRowIds => None, + IndexDomain::PrimaryKeyLevel => Some( + PrimaryKeyIndexSourceMeta::new( + level, + vec![PrimaryKeyIndexSourceFile::new(format!("{name}.parquet"), 10).unwrap()], + ) + .unwrap() + .serialize() + .unwrap(), + ), + }; + IndexManifestEntry { + kind: FileKind::Add, + partition: vec![], + bucket: 0, + index_file: IndexFileMeta { + index_type: "btree".into(), + file_name: name.into(), + file_size: 17, + row_count: 10, + deletion_vectors_ranges: None, + external_path: None, + global_index_meta: Some(GlobalIndexMeta { + row_range_start: 0, + row_range_end: 9, + index_field_id: field, + extra_field_ids: None, + source_meta, + index_meta: None, + }), + }, + version: 1, + } +} + +#[test] +fn exact_replacements_preserve_unrelated_indexes_for_both_layouts() { + for domain in [IndexDomain::GlobalRowIds, IndexDomain::PrimaryKeyLevel] { + let old = entry(domain, "old.index", 0, 1); + let other = entry(domain, "other.index", 1, 1); + let new = entry(domain, "new.index", 0, 1); + let mut deleted = old.clone(); + deleted.kind = FileKind::Delete; + let merged = TableCommit::merge_index_entries( + vec![old, other.clone()], + &[deleted, new.clone()], + domain, + ) + .unwrap(); + assert_eq!(merged, vec![other, new]); + } +} + +#[test] +fn retirements_check_current_metadata_and_membership_for_both_layouts() { + for domain in [IndexDomain::GlobalRowIds, IndexDomain::PrimaryKeyLevel] { + let old = entry(domain, "old.index", 0, 1); + for mismatch in ["size", "source", "missing"] { + let mut deleted = old.clone(); + deleted.kind = FileKind::Delete; + match mismatch { + "size" => deleted.index_file.file_size += 1, + "source" => { + deleted + .index_file + .global_index_meta + .as_mut() + .unwrap() + .source_meta = Some(vec![0]) + } + "missing" => deleted.index_file.file_name = "missing.index".into(), + _ => unreachable!(), + } + let error = TableCommit::merge_index_entries(vec![old.clone()], &[deleted], domain) + .expect_err("a stale index retirement must fail"); + let expected = if mismatch == "missing" { + "does not exist" + } else { + "metadata does not match" + }; + assert!(error.to_string().contains(expected), "{mismatch}: {error}"); + } + } +} + +#[test] +fn pk_level_ordinals_can_overlap_while_global_row_id_ranges_cannot() { + let first = entry(IndexDomain::PrimaryKeyLevel, "level-1.index", 0, 1); + let second = entry(IndexDomain::PrimaryKeyLevel, "level-2.index", 0, 2); + let merged = TableCommit::merge_index_entries( + vec![first.clone()], + std::slice::from_ref(&second), + IndexDomain::PrimaryKeyLevel, + ) + .unwrap(); + assert_eq!(merged, vec![first, second]); + let first = entry(IndexDomain::GlobalRowIds, "global-1.index", 0, 1); + let second = entry(IndexDomain::GlobalRowIds, "global-2.index", 0, 2); + for retained in [true, false] { + let (previous, additions) = if retained { + (vec![first.clone()], vec![second.clone()]) + } else { + (vec![], vec![first.clone(), second.clone()]) + }; + let error = + TableCommit::merge_index_entries(previous, &additions, IndexDomain::GlobalRowIds) + .expect_err("global indexes cannot overlap within or across commits"); + assert!( + error.to_string().contains("overlapping row range"), + "{error}" + ); + } +} diff --git a/crates/paimon/src/table/table_commit/index_delete_tests.rs b/crates/paimon/src/table/table_commit/index_delete_tests.rs index aeeb7e46c..ed8779964 100644 --- a/crates/paimon/src/table/table_commit/index_delete_tests.rs +++ b/crates/paimon/src/table/table_commit/index_delete_tests.rs @@ -142,9 +142,13 @@ fn stale_hash_index_deletes_remain_tolerated() { file.global_index_meta = None; let mut deletion = add_index_entry(file); deletion.kind = FileKind::Delete; - assert!(TableCommit::merge_index_entries(&[], &[deletion], false) - .unwrap() - .is_empty()); + assert!(TableCommit::merge_index_entries( + vec![], + &[deletion], + index_commit::IndexDomain::GlobalRowIds + ) + .unwrap() + .is_empty()); } #[tokio::test] From a5be57a84debbc57eef9105dd5a32c4f4e185f44 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 18:51:55 +0800 Subject: [PATCH 07/14] feat(index): tier DE index runs with shared LSM selection --- .../src/table/compact/primary_key/planner.rs | 98 +---- .../src/table/global_index_build_common.rs | 2 +- .../paimon/src/table/global_index_options.rs | 20 +- crates/paimon/src/table/index_build/vector.rs | 51 ++- .../src/table/maintenance_test_utils.rs | 8 +- crates/paimon/src/table/mod.rs | 1 + .../src/table/table_commit/auto_index.rs | 33 +- .../table/table_commit/auto_index/planning.rs | 182 +++++++-- .../table_commit/auto_index/planning/tests.rs | 163 ++++++++ .../table/table_commit/auto_index_tests.rs | 25 +- .../table_commit/auto_index_tests/lsm.rs | 324 +++++++++++++++ .../auto_index_tests/lsm/native.rs | 372 ++++++++++++++++++ .../paimon/src/table/universal_compaction.rs | 187 +++++++++ crates/paimon/src/vindex/mod.rs | 3 + docs/src/compaction.md | 62 ++- 15 files changed, 1385 insertions(+), 146 deletions(-) create mode 100644 crates/paimon/src/table/table_commit/auto_index/planning/tests.rs create mode 100644 crates/paimon/src/table/table_commit/auto_index_tests/lsm.rs create mode 100644 crates/paimon/src/table/table_commit/auto_index_tests/lsm/native.rs create mode 100644 crates/paimon/src/table/universal_compaction.rs diff --git a/crates/paimon/src/table/compact/primary_key/planner.rs b/crates/paimon/src/table/compact/primary_key/planner.rs index 7128b94bd..991e80db6 100644 --- a/crates/paimon/src/table/compact/primary_key/planner.rs +++ b/crates/paimon/src/table/compact/primary_key/planner.rs @@ -23,7 +23,8 @@ use crate::spec::{ AggregationConfig, ChangelogProducer, DataField, DataFileMeta, DataType, MergeEngine, PartialUpdateConfig, TableSchema, }; -use crate::{Error, Result}; +use crate::table::universal_compaction::{integer_option, Run, UniversalCompaction}; +use crate::Result; use std::collections::{BTreeMap, HashSet}; pub(in crate::table::compact) fn enabled(schema: &TableSchema) -> Result { @@ -140,28 +141,6 @@ fn mergeable_field(schema: &TableSchema, field: &DataField, function: &str) -> b } } -fn number( - options: &std::collections::HashMap, - key: &str, - default: i32, - minimum: i32, -) -> Result { - let value = options - .get(key) - .map(|value| value.parse::()) - .transpose() - .map_err(|_| Error::ConfigInvalid { - message: format!("Invalid {key}"), - })? - .unwrap_or(default); - if value < minimum { - return Err(Error::ConfigInvalid { - message: format!("{key} must be at least {minimum}"), - }); - } - Ok(value) -} - pub(in crate::table::compact) fn select( files: Vec, properties: &std::collections::HashMap, @@ -176,8 +155,13 @@ pub(in crate::table::compact) fn select_with_force_level_zero( properties: &std::collections::HashMap, force_level_zero: bool, ) -> Result, i32)>> { - let trigger = number(properties, "num-sorted-run.compaction-trigger", 5, 1)?; - let configured_levels = number(properties, "num-levels", trigger.saturating_add(1), 2)?; + let strategy = UniversalCompaction::from_options(properties, "")?; + let configured_levels = integer_option( + properties, + "num-levels", + strategy.trigger.saturating_add(1), + 2, + )?; let levels = configured_levels.max( files .iter() @@ -185,13 +169,6 @@ pub(in crate::table::compact) fn select_with_force_level_zero( .max() .unwrap_or(2), ); - let ratio = number(properties, "compaction.size-ratio", 1, 0)?; - let amplification = number( - properties, - "compaction.max-size-amplification-percent", - 200, - 0, - )?; let mut l0 = files .iter() .filter(|file| file.level == 0) @@ -224,36 +201,14 @@ pub(in crate::table::compact) fn select_with_force_level_zero( if runs.is_empty() { return Ok(None); } - let sizes = runs + let candidates = runs .iter() - .map(|(_, files)| { - files - .iter() - .map(|file| file.file_size as u128) - .sum::() + .map(|(level, files)| Run { + size: files.iter().map(|file| file.file_size as u128).sum(), + stop_before_large: *level > 1, }) .collect::>(); - let mut count = None; - if runs.len() >= trigger as usize { - let additional = sizes[..sizes.len() - 1].iter().sum::(); - if additional * 100 > amplification as u128 * sizes[sizes.len() - 1] { - count = Some(runs.len()); - } else { - count = pick_for_size_ratio(&runs, &sizes, ratio, 1, false); - } - if count.is_none() && runs.len() > trigger as usize { - count = pick_for_size_ratio( - &runs, - &sizes, - ratio, - runs.len() - trigger as usize + 1, - false, - ); - } - } - if count.is_none() && l0_count > 0 && force_level_zero { - count = pick_for_size_ratio(&runs, &sizes, ratio, l0_count, true); - } + let count = strategy.pick(&candidates, if force_level_zero { l0_count } else { 0 }); let Some(count) = count else { return Ok(None); }; @@ -272,34 +227,11 @@ pub(in crate::table::compact) fn select_with_force_level_zero( Ok(Some((selected, output_level))) } -fn pick_for_size_ratio( - runs: &[(i32, Vec)], - sizes: &[u128], - ratio: i32, - mut count: usize, - force: bool, -) -> Option { - let mut size = sizes[..count].iter().sum::(); - let mut triggered = force || count > 1; - while count < runs.len() { - // Match Java's double arithmetic, including threshold rounding for - // large runs, rather than comparing exact integer cross-products. - if size as f64 * (100.0 + f64::from(ratio)) / 100.0 < sizes[count] as f64 - && (!triggered || runs[count].0 > 1) - { - break; - } - size += sizes[count]; - count += 1; - triggered = true; - } - triggered.then_some(count) -} - #[cfg(test)] mod tests { use super::*; use crate::spec::stats::BinaryTableStats; + use crate::Error; use std::collections::HashMap; fn file(name: &str, level: i32, size: i64) -> DataFileMeta { DataFileMeta { diff --git a/crates/paimon/src/table/global_index_build_common.rs b/crates/paimon/src/table/global_index_build_common.rs index 8c148eab2..1704d6072 100644 --- a/crates/paimon/src/table/global_index_build_common.rs +++ b/crates/paimon/src/table/global_index_build_common.rs @@ -141,7 +141,7 @@ impl GlobalIndexBuildView { } /// Source filtering belongs to the caller's maintenance plan. Native - /// builders still plan complete shards, with all overlapping column files. + /// builders plan ranges with all overlapping column files. pub(crate) fn filter_sources(&self, mut keep: impl FnMut(&ManifestEntry) -> bool) -> Self { let Some(entries) = &self.source_entries else { return self.clone(); diff --git a/crates/paimon/src/table/global_index_options.rs b/crates/paimon/src/table/global_index_options.rs index bdaf383e6..9dbe0a90c 100644 --- a/crates/paimon/src/table/global_index_options.rs +++ b/crates/paimon/src/table/global_index_options.rs @@ -76,11 +76,18 @@ pub(super) fn definitions(table: &Table) -> Result> { } else { family }; - result.push(Definition { + let definition = Definition { column: column.into(), index_type: index_type.into(), options: field_index_options(options, column, family, index_type)?, - }); + }; + let mut merged = options.clone(); + merged.extend(definition.options.clone()); + crate::table::universal_compaction::UniversalCompaction::from_options( + &merged, + "global-index.", + )?; + result.push(definition); } Ok(result) } @@ -145,7 +152,14 @@ pub(super) fn parse_field_index_options( }; if options .get(&qualified) - .or_else(|| table_options.get(&qualified)) + .or_else(|| match qualified.as_str() { + // Maintenance policies are column overrides; native writer + // options still reject conflicting table and field definitions. + "global-index.num-sorted-run.compaction-trigger" + | "global-index.compaction.size-ratio" + | "global-index.compaction.max-size-amplification-percent" => None, + _ => table_options.get(&qualified), + }) .is_some_and(|previous| previous != &value) { return Err(crate::Error::ConfigInvalid { diff --git a/crates/paimon/src/table/index_build/vector.rs b/crates/paimon/src/table/index_build/vector.rs index 6042ac850..a3640d6be 100644 --- a/crates/paimon/src/table/index_build/vector.rs +++ b/crates/paimon/src/table/index_build/vector.rs @@ -158,6 +158,7 @@ impl VectorSpill { ids: self.ids.into_std().await, count: self.vectors, dimension: self.dimension, + range: self.range, })) } } @@ -167,6 +168,45 @@ pub(in crate::table) struct SpilledVectors { ids: std::fs::File, count: usize, dimension: usize, + range: RowRange, +} + +/// A data-dependent training configuration failure, distinct from native I/O +/// or training execution errors. Maintenance may retry a larger source unit. +#[derive(Debug)] +struct TrainingRequirement { + range: RowRange, + vectors: usize, + cause: Error, +} + +impl std::fmt::Display for TrainingRequirement { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} vectors in row range [{}, {}]: {}", + self.vectors, + self.range.from(), + self.range.to(), + self.cause + ) + } +} + +impl std::error::Error for TrainingRequirement { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.cause) + } +} + +pub(in crate::table) fn training_range(error: &Error) -> Option { + let mut cause: &(dyn std::error::Error + 'static) = error; + loop { + if let Some(requirement) = cause.downcast_ref::() { + return Some(requirement.range.clone()); + } + cause = cause.source()?; + } } pub(in crate::table) struct TrainedVectors { pub writer: VectorIndexWriter, @@ -184,7 +224,16 @@ impl SpilledVectors { ) -> Result { let training_vector_count = checked_training_vector_count(self.count, options.train_sample_ratio)?; - let config = options.training_config(self.count)?; + let config = options + .training_config(self.count) + .map_err(|cause| Error::DataInvalid { + message: cause.to_string(), + source: Some(Box::new(TrainingRequirement { + range: self.range.clone(), + vectors: self.count, + cause, + })), + })?; let training_rows_retained = if timing_enabled { default_training_vector_count(training_vector_count, config.nlist()).unwrap_or(0) } else { diff --git a/crates/paimon/src/table/maintenance_test_utils.rs b/crates/paimon/src/table/maintenance_test_utils.rs index a9b68b38f..38b52c60a 100644 --- a/crates/paimon/src/table/maintenance_test_utils.rs +++ b/crates/paimon/src/table/maintenance_test_utils.rs @@ -21,17 +21,19 @@ use crate::io::{FileIOBuilder, FileIOProvider}; use crate::spec::{ManifestList, Snapshot, TableSchema}; use crate::table::Table; use crate::Result; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex}; #[derive(Debug)] pub(super) struct MetadataIo { operator: opendal::Operator, watched: Mutex>, + observed: Mutex>, } #[async_trait::async_trait] impl FileIOProvider for MetadataIo { async fn create(&self, path: &str) -> Result<(opendal::Operator, String)> { + self.observed.lock().unwrap().insert(path.into()); if let Some(reads) = self.watched.lock().unwrap().get_mut(path) { *reads += 1; } @@ -51,12 +53,16 @@ impl MetadataIo { pub fn counts(&self) -> BTreeMap { self.watched.lock().unwrap().clone() } + pub fn observed_paths(&self) -> BTreeSet { + self.observed.lock().unwrap().clone() + } } pub(super) async fn table(schema: TableSchema) -> (Table, Arc) { let counter = Arc::new(MetadataIo { operator: opendal::Operator::from_config(opendal::services::MemoryConfig::default()) .unwrap(), watched: Mutex::new(BTreeMap::new()), + observed: Mutex::new(BTreeSet::new()), }); let io = FileIOBuilder::new("memory") .with_provider(counter.clone()) diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 1ae508124..f2c3a4b7d 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -191,6 +191,7 @@ mod table_upsert; pub(crate) mod table_write; mod tag_manager; pub(crate) mod time_travel; +mod universal_compaction; mod update_assignment; mod update_input; mod upsert_key_matcher; diff --git a/crates/paimon/src/table/table_commit/auto_index.rs b/crates/paimon/src/table/table_commit/auto_index.rs index baafe4be9..2d31fa3aa 100644 --- a/crates/paimon/src/table/table_commit/auto_index.rs +++ b/crates/paimon/src/table/table_commit/auto_index.rs @@ -109,13 +109,40 @@ impl TableCommit { let mut messages = Vec::new(); let prepared: Result<()> = async { for definition in definitions { - let plan = + let mut plan = planning::plan(&table, &definition, &data, view.index_entries()).await?; if plan.needs_build { - let built = definition + let built = match definition .builder(&table) .build_with_view(plan.build_view(&view)) - .await?; + .await + { + Ok(built) => built, + Err(error) => { + let expandable = + crate::table::index_build::vector::training_range(&error) + .is_some_and(|range| { + plan.can_expand_training_range(&range) + }); + if !expandable { + return Err(error); + } + // A small/sparse leaf may not meet a persisted-size + // objective. Retry once with larger pending groups; + // keep the same codec, metric and source snapshot. + plan = planning::plan_for_training( + &table, + &definition, + &data, + view.index_entries(), + ) + .await?; + definition + .builder(&table) + .build_with_view(plan.build_view(&view)) + .await? + } + }; messages.extend(built); } else { // Preserve validation for a refreshed schema without diff --git a/crates/paimon/src/table/table_commit/auto_index/planning.rs b/crates/paimon/src/table/table_commit/auto_index/planning.rs index 63db6dda0..0c402beae 100644 --- a/crates/paimon/src/table/table_commit/auto_index/planning.rs +++ b/crates/paimon/src/table/table_commit/auto_index/planning.rs @@ -15,18 +15,22 @@ // specific language governing permissions and limitations // under the License. -//! Compare desired global row-ID units with their current index payloads. -//! Like Java PrimaryKeyIndexLevels, rebuild changed units and retire their old -//! payloads atomically. Global indexes use row-ID shards rather than PK levels: -//! a physical compaction preserving logical row IDs does not invalidate them. +//! Append missing row-ID coverage as independent index runs. Select adjacent +//! runs with the shared UniversalCompaction policy and rebuild selected ranges +//! from current data, atomically retiring their old payloads. use super::*; use crate::table::global_index_build_common::{ resolve_index_fields, GlobalIndexBuildView, IndexColumns, }; +use crate::table::source::exclude_row_ranges; +use crate::table::universal_compaction::{Run, UniversalCompaction}; use crate::table::{merge_row_ranges, RowRange}; use std::collections::BTreeMap; +#[cfg(test)] +mod tests; + type UnitKey = (Vec, i32, i64); type IndexFileKey = (Vec, i32, String); @@ -43,9 +47,16 @@ pub(super) struct MaintenancePlan { pub deletions: Vec, pub needs_build: bool, build_ranges: HashMap<(Vec, i32), Vec>, + training_groups: Vec, } impl MaintenancePlan { + pub(super) fn can_expand_training_range(&self, range: &RowRange) -> bool { + self.training_groups + .iter() + .any(|group| group.from() <= range.from() && range.to() <= group.to() && group != range) + } + pub(super) fn build_view(&self, source: &GlobalIndexBuildView) -> GlobalIndexBuildView { let view = if self.replaced.is_empty() { source.clone() @@ -81,10 +92,32 @@ pub(super) async fn plan( definition: &Definition, data: &[ManifestEntry], indexes: &[IndexManifestEntry], +) -> Result { + plan_impl(table, definition, data, indexes, false).await +} + +/// A native training constraint may require a larger unit than an independent +/// leaf. Retry once with complete pending groups, preserving the declared codec. +pub(super) async fn plan_for_training( + table: &Table, + definition: &Definition, + data: &[ManifestEntry], + indexes: &[IndexManifestEntry], +) -> Result { + plan_impl(table, definition, data, indexes, true).await +} + +async fn plan_impl( + table: &Table, + definition: &Definition, + data: &[ManifestEntry], + indexes: &[IndexManifestEntry], + force_training_merge: bool, ) -> Result { let mut options = table.schema().options().clone(); options.extend(definition.options.clone()); let core = CoreOptions::new(&options); + let strategy = UniversalCompaction::from_options(&options, "global-index.")?; let width = if matches!( definition.index_type.as_str(), "btree" | "bitmap" | "multivalue" | "fm" @@ -138,7 +171,7 @@ pub(super) async fn plan( .push(RowRange::new(start.max(from), end.min(to))); } } - let mut desired = desired + let desired = desired .into_iter() .map(|(key, ranges)| (key, merge_row_ranges(ranges))) .collect::>(); @@ -161,7 +194,7 @@ pub(super) async fn plan( .collect::>(); let mut current: HashMap> = HashMap::new(); let mut replaced = HashSet::new(); - for entry in &selected { + for entry in selected.iter().copied() { let meta = entry.index_file.global_index_meta.as_ref().unwrap(); if let Some(metric) = vector_metric { // A vector column owns one algorithm and scoring metric across all shards. @@ -189,31 +222,87 @@ pub(super) async fn plan( }); } let shard = meta.row_range_start / width; - if shard != meta.row_range_end / width { - // A shard-size change can invalidate a former unit's boundaries. + let key = (partition_key(&entry.partition), entry.bucket, shard); + let contained = desired.get(&key).is_some_and(|groups| { + groups.iter().any(|group| { + group.from() <= meta.row_range_start && meta.row_range_end <= group.to() + }) + }); + if shard != meta.row_range_end / width || !contained { + // Row removal or shard-boundary changes cannot leave stale coverage. replaced.insert(index_file_key(entry)); } else { - current - .entry((partition_key(&entry.partition), entry.bucket, shard)) - .or_default() - .push(entry); + current.entry(key).or_default().push(entry); } } - for (key, files) in current { - let coverage = merge_row_ranges( - files + for files in current.values_mut() { + retire_overlaps(files, &mut replaced); + } + let mut build_ranges: HashMap<(Vec, i32), Vec> = HashMap::new(); + let mut training_groups = Vec::new(); + for ((partition, bucket, shard), groups) in desired { + let files = current + .remove(&(partition.clone(), bucket, shard)) + .unwrap_or_default(); + for group in groups { + let existing = files .iter() - .map(|entry| { + .copied() + .filter(|entry| { let meta = entry.index_file.global_index_meta.as_ref().unwrap(); - RowRange::new(meta.row_range_start, meta.row_range_end) + group.from() <= meta.row_range_start && meta.row_range_end <= group.to() }) - .collect(), - ); - let wanted = desired.get(&key).map(Vec::as_slice).unwrap_or(&[]); - if coverage.as_slice() != wanted || files.len() != wanted.len() { - replaced.extend(files.iter().map(|entry| index_file_key(entry))); - } else { - desired.remove(&key); + .collect::>(); + let covered = + merge_row_ranges(existing.iter().map(|entry| index_range(entry)).collect()); + let missing = exclude_row_ranges(std::slice::from_ref(&group), &covered); + let mut runs = existing + .iter() + .map(|&entry| IndexRun { + range: index_range(entry), + index: Some(entry), + }) + .chain( + missing + .iter() + .cloned() + .map(|range| IndexRun { range, index: None }), + ) + .collect::>(); + // Increasing row IDs act as append sequence: newest runs first, as + // in PK L0 selection. Only adjacent ranges of one group may merge. + runs.sort_by_key(|run| std::cmp::Reverse(run.range.from())); + let candidates = runs + .iter() + .map(|run| Run { + size: run.range.to() as u128 - run.range.from() as u128 + 1, + stop_before_large: true, + }) + .collect::>(); + let mut pending = missing; + if let Some(count) = strategy.pick(&candidates, 0).filter(|&count| count >= 2) { + for run in &runs[..count] { + if let Some(entry) = run.index { + replaced.insert(index_file_key(entry)); + } + pending.push(run.range.clone()); + } + } + let mut pending = merge_row_ranges(pending); + if pending.is_empty() { + continue; + } + if pending.as_slice() != std::slice::from_ref(&group) && vector_metric.is_some() { + training_groups.push(group.clone()); + if force_training_merge { + replaced.extend(existing.iter().map(|entry| index_file_key(entry))); + pending = vec![group]; + } + } + build_ranges + .entry((partition.clone(), bucket)) + .or_default() + .extend(pending); } } let mut deletions: BTreeMap<(Vec, i32), CommitMessage> = BTreeMap::new(); @@ -228,13 +317,6 @@ pub(super) async fn plan( .push(entry.index_file.clone()); } } - let mut build_ranges: HashMap<(Vec, i32), Vec> = HashMap::new(); - for ((partition, bucket, _), ranges) in desired { - build_ranges - .entry((partition, bucket)) - .or_default() - .extend(ranges); - } let build_ranges = build_ranges .into_iter() .map(|(key, ranges)| (key, merge_row_ranges(ranges))) @@ -244,5 +326,43 @@ pub(super) async fn plan( deletions: deletions.into_values().collect(), needs_build: !build_ranges.is_empty(), build_ranges, + training_groups, }) } + +struct IndexRun<'a> { + range: RowRange, + index: Option<&'a IndexManifestEntry>, +} + +fn index_range(entry: &IndexManifestEntry) -> RowRange { + let meta = entry.index_file.global_index_meta.as_ref().unwrap(); + RowRange::new(meta.row_range_start, meta.row_range_end) +} + +/// Adjacent fragments are legitimate runs; overlapping coverage must be +/// replaced before incremental builds can safely reuse the remaining files. +fn retire_overlaps(files: &mut Vec<&IndexManifestEntry>, replaced: &mut HashSet) { + files.sort_by_key(|entry| { + entry + .index_file + .global_index_meta + .as_ref() + .unwrap() + .row_range_start + }); + let mut first = 0; + while first < files.len() { + let mut end = index_range(files[first]).to(); + let mut next = first + 1; + while next < files.len() && index_range(files[next]).from() <= end { + end = end.max(index_range(files[next]).to()); + next += 1; + } + if next > first + 1 { + replaced.extend(files[first..next].iter().map(|entry| index_file_key(entry))); + } + first = next; + } + files.retain(|entry| !replaced.contains(&index_file_key(entry))); +} diff --git a/crates/paimon/src/table/table_commit/auto_index/planning/tests.rs b/crates/paimon/src/table/table_commit/auto_index/planning/tests.rs new file mode 100644 index 000000000..9c0dcd421 --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index/planning/tests.rs @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{GlobalIndexMeta, IndexFileMeta}; + +async fn source(rows: i32) -> (Table, Definition, Vec) { + let table = crate::table::table_commit::auto_index_tests::table(Some(false), false) + .await + .copy_with_options(HashMap::from([( + "sorted-index.records-per-range".into(), + "64".into(), + )])); + TableCommit::new(table.clone(), "metadata-source".into()) + .commit( + crate::table::table_commit::auto_index_tests::messages(&table, (0..rows).collect()) + .await, + ) + .await + .unwrap(); + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + let data = crate::table::snapshot_manifest::read_snapshot_entries( + &table.snapshot_manager(), + &snapshot, + None, + ) + .await + .unwrap(); + let definition = definitions(&table) + .unwrap() + .into_iter() + .find(|definition| definition.column == "id") + .unwrap(); + (table, definition, data) +} + +fn entry(data: &ManifestEntry, name: &str, start: i64, end: i64) -> IndexManifestEntry { + IndexManifestEntry { + kind: FileKind::Add, + partition: data.partition().to_vec(), + bucket: data.bucket(), + version: 1, + index_file: IndexFileMeta { + index_type: "btree".into(), + file_name: name.into(), + file_size: 17, + row_count: end - start + 1, + deletion_vectors_ranges: None, + external_path: None, + global_index_meta: Some(GlobalIndexMeta { + row_range_start: start, + row_range_end: end, + index_field_id: 0, + extra_field_ids: None, + source_meta: None, + index_meta: None, + }), + }, + } +} + +#[tokio::test] +async fn missing_ranges_leave_compatible_fragments_untouched_below_the_trigger() { + let (table, definition, data) = source(4).await; + let indexes = vec![ + entry(&data[0], "first.index", 0, 0), + entry(&data[0], "middle.index", 2, 2), + ]; + let plan = plan(&table, &definition, &data, &indexes).await.unwrap(); + assert!(plan.needs_build); + assert!(plan.deletions.is_empty()); + assert_eq!( + plan.build_ranges.values().next().unwrap(), + &vec![RowRange::new(1, 1), RowRange::new(3, 3)] + ); +} + +#[tokio::test] +async fn compaction_does_not_bridge_missing_logical_rows() { + let (table, definition, mut data) = source(4).await; + data[0].file.row_count = 2; + let mut other = data[0].clone(); + other.file.file_name = "other.parquet".into(); + other.file.first_row_id = Some(10); + data.push(other); + let indexes = vec![ + entry(&data[0], "zero.index", 0, 0), + entry(&data[0], "one.index", 1, 1), + entry(&data[0], "ten.index", 10, 10), + entry(&data[0], "eleven.index", 11, 11), + ]; + let table = table.copy_with_options(HashMap::from([( + "global-index.num-sorted-run.compaction-trigger".into(), + "3".into(), + )])); + let plan = plan(&table, &definition, &data, &indexes).await.unwrap(); + assert!( + !plan.needs_build, + "two runs in each connected group cannot become one index across a row-ID hole" + ); + assert!(plan.deletions.is_empty()); +} + +#[tokio::test] +async fn unchanged_logical_sources_reuse_runs_but_removed_rows_and_overlaps_are_rebuilt() { + let (table, definition, mut data) = source(4).await; + let indexes = vec![ + entry(&data[0], "first.index", 0, 1), + entry(&data[0], "second.index", 2, 3), + ]; + data[0].file.file_name = "compacted.parquet".into(); + let unchanged = plan(&table, &definition, &data, &indexes).await.unwrap(); + assert!( + !unchanged.needs_build, + "physical compaction alone must not invalidate logical index runs" + ); + assert!(unchanged.deletions.is_empty()); + let overlapping = vec![ + entry(&data[0], "wide.index", 0, 2), + entry(&data[0], "overlap.index", 1, 3), + ]; + let repaired = plan(&table, &definition, &data, &overlapping) + .await + .unwrap(); + assert_eq!(repaired.deletions[0].deleted_index_files.len(), 2); + assert_eq!( + repaired.build_ranges.values().next().unwrap(), + &vec![RowRange::new(0, 3)] + ); + data[0].file.row_count = 2; + let removed = plan( + &table, + &definition, + &data, + &[entry(&data[0], "stale.index", 0, 3)], + ) + .await + .unwrap(); + assert_eq!(removed.deletions[0].deleted_index_files.len(), 1); + assert_eq!( + removed.build_ranges.values().next().unwrap(), + &vec![RowRange::new(0, 1)] + ); +} diff --git a/crates/paimon/src/table/table_commit/auto_index_tests.rs b/crates/paimon/src/table/table_commit/auto_index_tests.rs index 1587efa65..498680478 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests.rs @@ -24,6 +24,7 @@ use arrow_array::{Int32Array, RecordBatch}; use futures::TryStreamExt; use std::sync::atomic::{AtomicBool, Ordering}; +mod lsm; mod search; #[tokio::test] @@ -332,13 +333,16 @@ async fn append_only_adds_missing_ranges_and_empty_commits_do_nothing() { } #[tokio::test] -async fn changed_shards_replace_fragmented_indexes_and_preserve_historical_payloads() { +async fn triggered_merges_replace_fragments_and_preserve_historical_payloads() { let table = table(Some(true), false) .await - .copy_with_options(HashMap::from([( - "sorted-index.records-per-range".into(), - "4".into(), - )])); + .copy_with_options(HashMap::from([ + ("sorted-index.records-per-range".into(), "4".into()), + ( + "global-index.num-sorted-run.compaction-trigger".into(), + "2".into(), + ), + ])); let commit = TableCommit::new(table.clone(), "writer".into()); commit .commit_with_identifier(messages(&table, vec![1]).await, 6) @@ -806,7 +810,12 @@ async fn definite_conflict_replans_coverage_and_preserves_competing_indexes() { #[tokio::test] async fn conflicting_replacement_replans_instead_of_deleting_a_competitors_payloads() { - let table = table(Some(true), false).await; + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([( + "global-index.num-sorted-run.compaction-trigger".into(), + "2".into(), + )])); TableCommit::new(table.clone(), "seed".into()) .commit(messages(&table, vec![1]).await) .await @@ -1203,8 +1212,8 @@ async fn native_builders_share_automatic_incremental_and_captured_snapshot_contr let files = indexes(&table).await; assert_eq!( files.len(), - 2, - "each native index should have one current shard after maintenance" + 4, + "each native index retains two append runs below the merge trigger" ); let full_text = files .iter() diff --git a/crates/paimon/src/table/table_commit/auto_index_tests/lsm.rs b/crates/paimon/src/table/table_commit/auto_index_tests/lsm.rs new file mode 100644 index 000000000..de05ede4a --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index_tests/lsm.rs @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{Datum, PredicateBuilder}; +use crate::table::global_index_scanner::GlobalIndexScanner; +use crate::table::RowRange; + +mod native; + +async fn scalar_hits(table: &Table, files: &[IndexManifestEntry], id: i32) -> Vec { + let scanner = GlobalIndexScanner::create( + table.file_io(), + table.location(), + 1, + i64::MAX, + i64::MAX, + files, + table.schema().fields(), + ) + .unwrap() + .unwrap(); + scanner + .matching_ranges( + &PredicateBuilder::new(table.schema().fields()) + .equal("id", Datum::Int(id)) + .unwrap(), + ) + .await + .unwrap() + .unwrap() +} + +#[tokio::test] +async fn small_appends_keep_leaf_indexes_until_the_default_merge_trigger() { + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([( + "sorted-index.records-per-range".into(), + "64".into(), + )])); + let commit = TableCommit::new(table.clone(), "lsm-writer".into()); + let mut historical = Vec::new(); + let mut last_input = Vec::new(); + for id in 1..=5 { + let previous = if id == 1 { + vec![] + } else { + indexes(&table).await + }; + let input = messages(&table, vec![id]).await; + commit + .commit_with_identifier(input.clone(), i64::from(id)) + .await + .unwrap(); + let current = indexes(&table).await; + assert_eq!( + scalar_hits(&table, ¤t, id).await, + vec![RowRange::new(i64::from(id - 1), i64::from(id - 1))] + ); + if id < 5 { + assert_eq!( + current.len(), + 2 * id as usize, + "each append must add a leaf per column" + ); + assert!( + previous.iter().all(|file| current.contains(file)), + "small writes must retain existing payloads" + ); + for kind in ["btree", "bitmap"] { + let mut ranges = current + .iter() + .filter(|entry| entry.index_file.index_type == kind) + .map(|entry| { + let meta = entry.index_file.global_index_meta.as_ref().unwrap(); + (meta.row_range_start, meta.row_range_end) + }) + .collect::>(); + ranges.sort_unstable(); + assert_eq!( + ranges, + (0..i64::from(id)).map(|row| (row, row)).collect::>() + ); + } + } else { + assert_eq!( + current.len(), + 2, + "the fifth adjacent run triggers one merged payload per column" + ); + assert!(previous.iter().all(|file| !current.contains(file))); + for file in ¤t { + let meta = file.index_file.global_index_meta.as_ref().unwrap(); + assert_eq!((meta.row_range_start, meta.row_range_end), (0, 4)); + } + } + historical.extend(previous); + last_input = input; + } + for file in historical { + assert!( + table + .file_io() + .exists(&format!( + "{}/index/{}", + table.location(), + file.index_file.file_name + )) + .await + .unwrap(), + "merged inputs remain available to historical snapshots" + ); + } + let indexed = latest(&table).await; + commit + .filter_and_commit_with_identifier(last_input, 5) + .await + .unwrap(); + assert_eq!( + latest(&table).await, + indexed, + "recovery cannot publish duplicate leaf or merged indexes" + ); + assert_complete(&table).await; +} + +#[tokio::test] +async fn tiered_selection_merges_small_tail_runs_without_rebuilding_the_large_base() { + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([ + ("sorted-index.records-per-range".into(), "64".into()), + ( + "global-index.num-sorted-run.compaction-trigger".into(), + "3".into(), + ), + ])); + let commit = TableCommit::new(table.clone(), "tiered-writer".into()); + commit + .commit(messages(&table, (0..8).collect()).await) + .await + .unwrap(); + let base = indexes(&table).await; + commit + .commit(messages(&table, vec![8]).await) + .await + .unwrap(); + let second = indexes(&table).await; + assert_eq!(second.len(), 4); + assert!(base.iter().all(|file| second.contains(file))); + commit + .commit(messages(&table, vec![9]).await) + .await + .unwrap(); + let current = indexes(&table).await; + assert_eq!( + current.len(), + 4, + "retain the large run and merge the two small adjacent runs" + ); + assert!( + base.iter().all(|file| current.contains(file)), + "size-ratio selection must not pull in the large base" + ); + for file in current.iter().filter(|file| !base.contains(file)) { + let meta = file.index_file.global_index_meta.as_ref().unwrap(); + assert_eq!((meta.row_range_start, meta.row_range_end), (8, 9)); + } + for id in [0, 8, 9] { + assert_eq!( + scalar_hits(&table, ¤t, id).await, + vec![RowRange::new(i64::from(id), i64::from(id))] + ); + } + assert_complete(&table).await; +} + +#[tokio::test] +async fn invalid_index_lsm_parameters_fail_before_data_publication() { + for (key, value) in [ + ("global-index.num-sorted-run.compaction-trigger", "0"), + ("global-index.num-sorted-run.compaction-trigger", "invalid"), + ("global-index.compaction.size-ratio", "-1"), + ( + "global-index.compaction.max-size-amplification-percent", + "-1", + ), + ] { + let original = table(Some(true), false).await; + let table = original.copy_with_options(HashMap::from([(key.into(), value.into())])); + let error = TableCommit::new(table.clone(), "invalid-lsm".into()) + .commit(messages(&table, vec![1]).await) + .await + .expect_err("validate LSM policy before writing the data snapshot"); + assert!(error.to_string().contains(key), "{error}"); + assert!(table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .is_none()); + } +} + +#[tokio::test] +async fn column_specific_merge_triggers_and_shard_boundaries_are_respected() { + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([ + ("sorted-index.records-per-range".into(), "4".into()), + ("global-index.num-sorted-run.compaction-trigger".into(), "5".into()), + ("global-index.compaction.size-ratio".into(), "99".into()), + ("global-index.compaction.max-size-amplification-percent".into(), "9".into()), + ( + "fields.id.global-index.btree.index-options".into(), + r#"{"global-index.num-sorted-run.compaction-trigger":2,"global-index.compaction.size-ratio":1,"global-index.compaction.max-size-amplification-percent":200}"#.into(), + ), + ])); + let commit = TableCommit::new(table.clone(), "column-triggers".into()); + for id in [0, 1] { + commit + .commit(messages(&table, vec![id]).await) + .await + .unwrap(); + } + let current = indexes(&table).await; + assert_eq!( + current + .iter() + .filter(|file| file.index_file.index_type == "btree") + .count(), + 1 + ); + assert_eq!( + current + .iter() + .filter(|file| file.index_file.index_type == "bitmap") + .count(), + 2 + ); + let before = current; + // One physical source crosses the row-ID shard boundary. + commit + .commit(messages(&table, vec![2, 3, 4]).await) + .await + .unwrap(); + let current = indexes(&table).await; + for file in current + .iter() + .filter(|file| file.index_file.index_type == "bitmap") + { + let meta = file.index_file.global_index_meta.as_ref().unwrap(); + assert_eq!(meta.row_range_start / 4, meta.row_range_end / 4); + } + assert!(before + .iter() + .filter(|file| file.index_file.index_type == "bitmap") + .all(|file| current.contains(file))); + for id in 0..5 { + assert_eq!( + scalar_hits(&table, ¤t, id).await, + vec![RowRange::new(i64::from(id), i64::from(id))] + ); + } + assert_complete(&table).await; +} + +#[tokio::test] +async fn merged_runs_recover_an_uncertain_publication_without_duplicate_outputs() { + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([ + ("sorted-index.records-per-range".into(), "64".into()), + ( + "global-index.num-sorted-run.compaction-trigger".into(), + "3".into(), + ), + ])); + let seed = TableCommit::new(table.clone(), "merge-recovery".into()); + for id in 1..=2 { + seed.commit_with_identifier(messages(&table, vec![id]).await, i64::from(id)) + .await + .unwrap(); + } + let before = indexes(&table).await; + let input = messages(&table, vec![3]).await; + let mut commit = TableCommit::new(table.clone(), "merge-recovery".into()); + commit.snapshot_commit = Arc::new(InjectIndexPublication { + table: table.clone(), + mode: Publication::LoseResponse, + fired: AtomicBool::new(false), + }); + commit + .commit_with_identifier(input.clone(), 3) + .await + .unwrap(); + let completed = latest(&table).await; + let current = indexes(&table).await; + assert_eq!(current.len(), 2); + assert!(before.iter().all(|file| !current.contains(file))); + for _ in 0..2 { + seed.filter_and_commit_with_identifier(input.clone(), 3) + .await + .unwrap(); + assert_eq!(latest(&table).await, completed); + assert_eq!(indexes(&table).await, current); + } + assert_complete(&table).await; +} diff --git a/crates/paimon/src/table/table_commit/auto_index_tests/lsm/native.rs b/crates/paimon/src/table/table_commit/auto_index_tests/lsm/native.rs new file mode 100644 index 000000000..3630625d4 --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index_tests/lsm/native.rs @@ -0,0 +1,372 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{ArrayType, FloatType, VarCharType}; +use crate::table::maintenance_test_utils::{table as counted_table, MetadataIo}; +use arrow_array::builder::{Float32Builder, ListBuilder}; +use arrow_array::{ArrayRef, StringArray}; +use std::collections::BTreeSet; + +async fn native_table(kind: &str, options: &str) -> (Table, Arc) { + let mut schema = TableSchemaBuilder::builder() + .column("id", DataType::Int(IntType::new())) + .column("text", DataType::VarChar(VarCharType::string_type())) + .column("folded_text", DataType::VarChar(VarCharType::string_type())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("global-index.enabled", "true") + .option("global-index.row-count-per-shard", "4096") + .option("compaction.min.file-num", "1000") + .option("commit.min-retry-wait", "0 ms") + .option("commit.max-retry-wait", "0 ms"); + schema = if kind == "full-text" { + schema.option("global-index.full-text.index-columns", "text") + } else { + schema + .option("global-index.vector.index-columns", "embedding") + .option("fields.embedding.global-index.vector.index-type", kind) + .option( + "fields.embedding.global-index.vector.index-options", + options, + ) + }; + counted_table(TableSchema::new(0, &schema.build().unwrap())).await +} + +async fn native_messages(table: &Table, vectors: &[Option<[f32; 2]>]) -> Vec { + let start = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .and_then(|snapshot| snapshot.next_row_id()) + .unwrap_or(0); + let ids = (start..start + vectors.len() as i64) + .map(|id| i32::try_from(id).unwrap()) + .collect::>(); + let mut list = ListBuilder::new(Float32Builder::new()); + for vector in vectors { + if let Some(vector) = vector { + list.values().append_slice(vector); + list.append(true); + } else { + list.append(false); + } + } + let texts = Arc::new(StringArray::from_iter_values( + ids.iter().map(|id| format!("sentinel row {id}")), + )) as ArrayRef; + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(ids)) as ArrayRef), + ("text", texts.clone()), + ("folded_text", texts), + ("embedding", Arc::new(list.finish()) as ArrayRef), + ]) + .unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + writer.prepare_commit().await.unwrap() +} + +async fn top_vector(table: &Table, mode: &str) -> u64 { + table + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(1) + .with_options(HashMap::from([( + "vector-index.search-mode".into(), + mode.into(), + )])) + .execute() + .await + .unwrap() + .row_ids() + .unwrap() + .row_ids[0] +} + +fn new_payloads(table: &Table, counter: &MetadataIo, before: &BTreeSet) -> usize { + let prefix = format!("{}/index/", table.location()); + counter + .observed_paths() + .difference(before) + .filter(|path| path.starts_with(&prefix)) + .count() +} + +#[tokio::test] +async fn vector_leaf_training_uses_non_null_counts_and_preserves_global_row_offsets() { + for (kind, options) in [ + ( + "ivf-flat", + r#"{"dimension":2,"global-index.num-sorted-run.compaction-trigger":5,"global-index.compaction.size-ratio":1,"global-index.compaction.max-size-amplification-percent":200}"#, + ), + ("ivf-flat", r#"{"dimension":2,"nlist":64}"#), + ("ivf-pq", r#"{"dimension":2,"nlist":1,"pq.m":1}"#), + ] { + let (table, counter) = native_table(kind, options).await; + let commit = TableCommit::new(table.clone(), "vector-leaves".into()); + commit + .commit(native_messages(&table, &[Some([0., 1.]); 8]).await) + .await + .unwrap(); + let base = indexes(&table).await; + let base_sources = counter + .observed_paths() + .into_iter() + .filter(|path| path.ends_with(".parquet")) + .collect::>(); + assert!(!base_sources.is_empty()); + counter.watch(base_sources); + let paths = counter.observed_paths(); + commit + .commit(native_messages(&table, &[Some([1., 0.])]).await) + .await + .unwrap(); + assert!( + counter.counts().values().all(|count| *count == 0), + "independent leaves must not read base data: {:?}", + counter.counts() + ); + assert_eq!(new_payloads(&table, &counter, &paths), 1); + let current = indexes(&table).await; + assert_eq!(current.len(), 2, "{kind}"); + assert!(base.iter().all(|file| current.contains(file))); + assert_eq!(top_vector(&table, "fast").await, 8, "{kind}"); + let mut sparse = vec![None; 9]; + sparse[8] = Some([2., 0.]); + commit + .commit(native_messages(&table, &sparse).await) + .await + .unwrap(); + let current = indexes(&table).await; + assert_eq!(current.len(), 3, "{kind}"); + let leaf = current + .iter() + .find(|file| { + file.index_file + .global_index_meta + .as_ref() + .unwrap() + .row_range_start + == 9 + }) + .unwrap(); + assert_eq!( + leaf.index_file.row_count, 9, + "global metadata counts physical rows, not vectors" + ); + assert_eq!( + leaf.index_file + .global_index_meta + .as_ref() + .unwrap() + .row_range_end, + 17 + ); + assert_eq!(top_vector(&table, "fast").await, 17, "{kind}"); + assert!(current + .iter() + .all(|file| file.index_file.index_type == kind)); + } +} + +#[tokio::test] +async fn vector_size_budgets_expand_dense_and_sparse_leaves_before_native_training() { + for sparse in [false, true] { + let (table, counter) = + native_table("ivf-flat", r#"{"dimension":2,"max-bytes-per-vector":32}"#).await; + let commit = TableCommit::new(table.clone(), "budget-leaf".into()); + commit + .commit(native_messages(&table, &vec![Some([0., 1.]); 1000]).await) + .await + .unwrap(); + let base = indexes(&table).await; + let paths = counter.observed_paths(); + let mut tail = vec![None; if sparse { 9 } else { 1 }]; + let last = tail.len() - 1; + tail[last] = Some([2., 0.]); + commit + .commit(native_messages(&table, &tail).await) + .await + .unwrap(); + assert_eq!( + new_payloads(&table, &counter, &paths), + 1, + "an untrainable leaf must not produce an intermediate native index" + ); + let current = indexes(&table).await; + assert_eq!(current.len(), 1); + assert!(!current.contains(&base[0])); + let file = ¤t[0].index_file; + assert_eq!(file.index_type, "ivf-flat"); + assert_eq!(file.row_count, 1000 + tail.len() as i64); + let meta = file.global_index_meta.as_ref().unwrap(); + assert_eq!( + (meta.row_range_start, meta.row_range_end), + (0, 999 + tail.len() as i64) + ); + for mode in ["fast", "full", "detail"] { + assert_eq!(top_vector(&table, mode).await, 999 + tail.len() as u64); + } + assert!(table + .file_io() + .exists(&format!( + "{}/index/{}", + table.location(), + base[0].index_file.file_name + )) + .await + .unwrap()); + } +} + +#[tokio::test] +async fn all_null_vector_leaves_keep_java_empty_build_semantics_and_recover() { + let (table, counter) = native_table("ivf-flat", r#"{"dimension":2}"#).await; + let commit = TableCommit::new(table.clone(), "null-leaf".into()); + commit + .commit_with_identifier(native_messages(&table, &[Some([0., 1.]); 8]).await, 1) + .await + .unwrap(); + let base = indexes(&table).await; + let paths = counter.observed_paths(); + let input = native_messages(&table, &[None; 3]).await; + commit + .commit_with_identifier(input.clone(), 2) + .await + .unwrap(); + let completed = latest(&table).await; + assert_eq!(indexes(&table).await, base); + assert_eq!( + new_payloads(&table, &counter, &paths), + 0, + "Java does not publish zero-byte DE markers" + ); + commit + .filter_and_commit_with_identifier(input, 2) + .await + .unwrap(); + assert_eq!(latest(&table).await, completed); + assert_eq!(indexes(&table).await, base); + commit + .commit(native_messages(&table, &[Some([2., 0.])]).await) + .await + .unwrap(); + let current = indexes(&table).await; + assert_eq!(current.len(), 2); + assert!(base.iter().all(|file| current.contains(file))); + let leaf = current + .iter() + .find(|file| { + file.index_file + .global_index_meta + .as_ref() + .unwrap() + .row_range_start + == 8 + }) + .unwrap(); + assert_eq!(leaf.index_file.row_count, 4); + assert_eq!(top_vector(&table, "fast").await, 11); +} + +async fn text_rows(table: &Table) -> BTreeSet { + table + .new_full_text_search_builder() + .with_text_column("text") + .with_query_text("sentinel") + .with_limit(100) + .execute() + .await + .unwrap() + .row_ids() + .unwrap() + .row_ids + .iter() + .copied() + .collect() +} + +#[tokio::test] +async fn full_text_leaves_and_tiered_merges_preserve_hits_filters_and_history() { + let (table, counter) = native_table("full-text", "{}").await; + let commit = TableCommit::new(table.clone(), "text-lsm".into()); + commit + .commit(native_messages(&table, &[None; 8]).await) + .await + .unwrap(); + let base = indexes(&table).await; + let mut history = None; + for row in 8..12 { + let paths = counter.observed_paths(); + commit + .commit(native_messages(&table, &[None]).await) + .await + .unwrap(); + assert_eq!( + new_payloads(&table, &counter, &paths), + 1, + "one final payload per append or chosen merge" + ); + assert_eq!(text_rows(&table).await, (0..=row).collect()); + if row == 10 { + history = Some(latest(&table).await); + } + } + let current = indexes(&table).await; + assert_eq!(current.len(), 2); + assert!(base.iter().all(|file| current.contains(file))); + let pinned = table.copy_with_pinned_snapshot(history.as_ref()); + assert_eq!(text_rows(&pinned).await, (0..11).collect()); + let predicate = PredicateBuilder::new(table.schema().fields()) + .equal("id", Datum::Int(11)) + .unwrap(); + for (mode, expected) in [("fast", vec![]), ("full", vec![11]), ("detail", vec![11])] { + let filtered = table.copy_with_options(HashMap::from([( + "scalar-index.search-mode".into(), + mode.into(), + )])); + let result = filtered + .new_full_text_search_builder() + .with_text_column("text") + .with_query_text("sentinel") + .with_filter(predicate.clone()) + .with_limit(1) + .execute() + .await + .unwrap(); + assert_eq!(result.row_ids().unwrap().row_ids, expected, "{mode}"); + } + let result = table + .new_full_text_search_builder() + .with_text_column("text") + .with_query_text("sentinel") + .with_include_row_ids(std::iter::once(8_u64).collect()) + .with_limit(1) + .execute() + .await + .unwrap(); + assert_eq!(result.row_ids().unwrap().row_ids, vec![8]); +} diff --git a/crates/paimon/src/table/universal_compaction.rs b/crates/paimon/src/table/universal_compaction.rs new file mode 100644 index 000000000..db657707d --- /dev/null +++ b/crates/paimon/src/table/universal_compaction.rs @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared Java UniversalCompaction prefix selection for data and index runs. +//! +//! Callers supply ordered runs and their own cost unit: PK uses data bytes, +//! global indexes use logical rows. Source layout and output levels stay local. + +use crate::{Error, Result}; +use std::collections::HashMap; + +pub(super) struct Run { + pub size: u128, + /// PK L0/L1 must be consumed once a merge starts; other boundaries may stop + /// before a larger run. Global index runs allow stopping at every boundary. + pub stop_before_large: bool, +} + +pub(super) struct UniversalCompaction { + pub trigger: i32, + ratio: i32, + amplification: i32, +} + +impl UniversalCompaction { + pub fn from_options(options: &HashMap, prefix: &str) -> Result { + Ok(Self { + trigger: integer_option( + options, + &format!("{prefix}num-sorted-run.compaction-trigger"), + 5, + 1, + )?, + ratio: integer_option(options, &format!("{prefix}compaction.size-ratio"), 1, 0)?, + amplification: integer_option( + options, + &format!("{prefix}compaction.max-size-amplification-percent"), + 200, + 0, + )?, + }) + } + + pub fn pick(&self, runs: &[Run], forced_minimum: usize) -> Option { + if runs.is_empty() { + return None; + } + let mut count = None; + if runs.len() >= self.trigger as usize { + let additional = runs[..runs.len() - 1] + .iter() + .map(|run| run.size) + .sum::(); + if additional * 100 > self.amplification as u128 * runs[runs.len() - 1].size { + count = Some(runs.len()); + } else { + count = self.pick_for_size_ratio(runs, 1, false); + } + if count.is_none() && runs.len() > self.trigger as usize { + count = + self.pick_for_size_ratio(runs, runs.len() - self.trigger as usize + 1, false); + } + } + if count.is_none() && forced_minimum > 0 { + count = self.pick_for_size_ratio(runs, forced_minimum, true); + } + count + } + + fn pick_for_size_ratio(&self, runs: &[Run], mut count: usize, force: bool) -> Option { + let mut size = runs[..count].iter().map(|run| run.size).sum::(); + let mut triggered = force || count > 1; + while count < runs.len() { + // Preserve Java's double arithmetic, including rounding for large runs. + if size as f64 * (100.0 + f64::from(self.ratio)) / 100.0 < runs[count].size as f64 + && (!triggered || runs[count].stop_before_large) + { + break; + } + size += runs[count].size; + count += 1; + triggered = true; + } + triggered.then_some(count) + } +} + +pub(super) fn integer_option( + options: &HashMap, + key: &str, + default: i32, + minimum: i32, +) -> Result { + let value = options + .get(key) + .map(|value| value.parse::()) + .transpose() + .map_err(|_| Error::ConfigInvalid { + message: format!("Invalid {key}"), + })? + .unwrap_or(default); + if value < minimum { + return Err(Error::ConfigInvalid { + message: format!("{key} must be at least {minimum}"), + }); + } + Ok(value) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn runs(sizes: &[u128]) -> Vec { + sizes + .iter() + .map(|&size| Run { + size, + stop_before_large: true, + }) + .collect() + } + + #[test] + fn global_prefix_is_independent_of_data_compaction_and_stops_before_a_large_base() { + let policy = UniversalCompaction::from_options( + &HashMap::from([ + ("num-sorted-run.compaction-trigger".into(), "1".into()), + ( + "global-index.num-sorted-run.compaction-trigger".into(), + "3".into(), + ), + ]), + "global-index.", + ) + .unwrap(); + assert!(policy.pick(&runs(&[1, 1000]), 0).is_none()); + assert_eq!(policy.pick(&runs(&[1, 1, 1000]), 0), Some(2)); + assert_eq!(policy.pick(&runs(&[1, 10, 100, 1000]), 0), Some(2)); + assert_eq!(policy.pick(&runs(&[1000, 1000, 100]), 0), Some(3)); + } + + #[test] + fn pk_force_consumes_l1_but_does_not_force_global_index_merges() { + let policy = UniversalCompaction::from_options(&HashMap::new(), "").unwrap(); + let runs = vec![ + Run { + size: 1, + stop_before_large: false, + }, + Run { + size: 100, + stop_before_large: false, + }, + Run { + size: 10000, + stop_before_large: true, + }, + ]; + assert!(policy.pick(&runs, 0).is_none()); + assert_eq!(policy.pick(&runs, 1), Some(2)); + assert_eq!( + policy.pick( + &[Run { + size: 1, + stop_before_large: false + }], + 1 + ), + Some(1) + ); + } +} diff --git a/crates/paimon/src/vindex/mod.rs b/crates/paimon/src/vindex/mod.rs index a93b4e671..3fede5f1d 100644 --- a/crates/paimon/src/vindex/mod.rs +++ b/crates/paimon/src/vindex/mod.rs @@ -279,6 +279,9 @@ fn validate_user_option_keys( "index.type" | "global-index.row-count-per-shard" | "global-index.build.parallelism" + | "global-index.num-sorted-run.compaction-trigger" + | "global-index.compaction.size-ratio" + | "global-index.compaction.max-size-amplification-percent" ) { return false; } diff --git a/docs/src/compaction.md b/docs/src/compaction.md index 7e29fc58d..deb3558f2 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -61,9 +61,10 @@ As in Java, overwriting partitions being consumed by streaming readers is unsupp lookup changelogs do not describe the overwrite operation itself. Automatic tasks share one captured snapshot and bucket-grouped source/index metadata per round. Automatic index fields also share captured source metadata; -fully covered fields skip shard planning, and changed fields only plan files -overlapping their missing or replaced units. Captured metadata is released after -the round. Publication and retries still validate current sources and index +native builds only plan files overlapping missing or selected merge ranges. +Covered fields can still consolidate runs when their thresholds are met. +Captured metadata is released after the round. Publication and retries still +validate current sources and index membership, and manual task execution captures fresh inputs on every call. Automatic lookup compaction supports the default `lookup-compact=radical` policy. `gentle` requires writer-lifetime trigger intervals and is explicitly unsupported @@ -147,9 +148,9 @@ Per-column parameters use `fields..global-index..index-options`, a JSON object containing string, boolean, or numeric values. Short keys are qualified with the native writer prefix, such as `block-size` to `btree-index.block-size`. Fully qualified -writer keys and shared shard options are accepted. Conflicting values are -rejected, matching primary-key option resolution. Vector columns additionally -require `fields..global-index.vector.index-type`; the optional +writer keys and shared shard options are accepted. Conflicting native writer +values are rejected, matching primary-key option resolution. Vector columns +additionally require `fields..global-index.vector.index-type`; the optional `distance-metric` defaults to `inner_product`. Explicit single-column builds also use these declarations as defaults; explicit builder options take precedence over them. @@ -157,14 +158,44 @@ FULL/DETAIL raw fallback uses the same per-column parameters as automatic builds, including vector metric defaults and full-text tokenization. A declared full-text column can use raw fallback before its first index is published. -Each round captures the current schema and one source snapshot. Like Java -primary-key index maintenance, it compares the desired sources with current -index units and replaces changed units atomically. Global units use partition, -bucket, and row-ID shard boundaries instead of primary-key LSM levels. Valid -units are retained, missing ranges are built, and partial or fragmented shards -are rebuilt from current rows. A physical compaction that preserves row IDs and -values can retain existing global payloads. Additions and retired references -publish together; old files remain available to historical snapshots. +Each round captures the current schema and one source snapshot. Missing row-ID +coverage is built as independent index files, retaining compatible existing +payloads. Adjacent index ranges form runs within one partition, bucket and row-ID +shard; logical row-ID holes are separate groups. The UniversalCompaction run +selector is shared with PK data compaction. Global index runs use logical row +counts as their size measure, so new ranges can participate in selection before +encoding; PK data runs continue to use bytes and their existing level rules. + +Index merging has independent options with the same defaults as PK compaction: + +| Option | Default | Meaning | +| --- | --- | --- | +| `global-index.num-sorted-run.compaction-trigger` | `5` | Run count at which size-based selection becomes eligible; must be positive. | +| `global-index.compaction.size-ratio` | `1` | Percentage tolerance when extending a merge to the next run; must be nonnegative. | +| `global-index.compaction.max-size-amplification-percent` | `200` | Select all runs when newer runs exceed this percentage of the oldest run; must be nonnegative. | + +These options can also appear, fully qualified, in per-column `index-options` +JSON and override the corresponding table-level index policy. They do not change +the table's data-compaction options. Above the run-count limit, the selector +also makes progress when adjacent run sizes differ too much +for the normal size ratio. A single run is never rebuilt just to compact itself. + +For example, with a 1000-row-ID shard, an existing `[0, 599]` index and a new +`[600, 799]` batch stay as two files below the trigger. Once enough adjacent runs +accumulate, selected ranges are rebuilt together from current data and sidecar +values. A large older run can remain untouched while smaller tail runs merge. +The implementation rebuilds selected data ranges rather than concatenating old +native index payloads. A physical compaction preserving row IDs and values can +retain global payloads. Shard-boundary changes, removed rows and incompatible +vector definitions retire affected payloads immediately. Additions and retired +references publish together; historical snapshots retain their old files. + +Vector training still uses the actual non-NULL count. If a small or sparse run +cannot satisfy its native training configuration, such as a size-per-vector +budget, maintenance may retry once with complete pending connected groups. This +keeps the declared algorithm, metric and budget. I/O and training execution +errors do not trigger this expansion. All-NULL vector builds retain Java's +existing empty-build behavior: they publish no zero-byte DE index marker. Manual `commit_compact` also maintains declared indexes when `write-only=false`. An empty maintenance plan publishes no snapshot. Empty data commits, index-only commits, and truncation do not trigger a separate round. Indexed-column updates @@ -173,7 +204,8 @@ the data update and automatically rebuild the affected coverage. Changing a vector algorithm or distance metric retires all incompatible payloads for that column and publishes their replacements together. Other writer parameters alone do not rebuild unchanged coverage; changing shard boundaries can regroup it. -This synchronous mode has no minimum-row or time threshold. +Missing coverage is processed synchronously without a minimum-row or time +delay; index consolidation follows the run thresholds above. A concurrent snapshot or schema change causes private outputs to be discarded and coverage to be replanned, bounded by `commit.max-retries` and From e2d2c49ee6f9fdf8cb4a9243707df3df27fc78fa Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 20:37:45 +0800 Subject: [PATCH 08/14] fix(index): validate retained index semantics and vector sources Rebuild incompatible DE scalar and full-text runs using actual builder-schema provenance and parsed analyzer options. Check vector dimensions alongside algorithms and metrics before reusing coverage. Project PK indexed fields and apply vector deletion masks before schema casts, preserving source ordinals with bounded null batches. Reject publication when concurrent deletion-vector changes restore positions omitted by a new payload. Add reproducing schema, analyzer, dimension, projection, ordinal and concurrent deletion regressions, and document the maintenance contracts. --- crates/paimon/src/table/compact.rs | 1 + .../paimon/src/table/compact/primary_key.rs | 9 +- .../compact/primary_key/deletion_vectors.rs | 42 + .../src/table/compact/primary_key/indexes.rs | 38 +- .../compact/primary_key/indexes/writer.rs | 115 ++- .../primary_key/indexes/writer/tests.rs | 544 +++++++++++++ .../paimon/src/table/deletion_vector_index.rs | 118 ++- .../src/table/deletion_vector_index/tests.rs | 91 +++ .../table/full_text_index_build_builder.rs | 6 +- .../paimon/src/table/global_index_schema.rs | 131 ++++ crates/paimon/src/table/kv_file_reader.rs | 17 +- crates/paimon/src/table/mod.rs | 1 + .../paimon/src/table/pk_index_maintenance.rs | 20 +- .../writer.rs | 2 +- .../table/table_commit/auto_index/planning.rs | 60 +- .../table/table_commit/auto_index_tests.rs | 2 + .../table_commit/auto_index_tests/schema.rs | 719 ++++++++++++++++++ .../auto_index_tests/vector_dimension.rs | 138 ++++ .../paimon/src/table/vector_search_common.rs | 180 ++++- crates/paimon/tests/pk_compact/indexes.rs | 397 ++++++++++ docs/src/compaction.md | 37 +- 21 files changed, 2605 insertions(+), 63 deletions(-) create mode 100644 crates/paimon/src/table/compact/primary_key/indexes/writer/tests.rs create mode 100644 crates/paimon/src/table/deletion_vector_index/tests.rs create mode 100644 crates/paimon/src/table/global_index_schema.rs create mode 100644 crates/paimon/src/table/table_commit/auto_index_tests/schema.rs create mode 100644 crates/paimon/src/table/table_commit/auto_index_tests/vector_dimension.rs diff --git a/crates/paimon/src/table/compact.rs b/crates/paimon/src/table/compact.rs index 3456e80d6..20728db4b 100644 --- a/crates/paimon/src/table/compact.rs +++ b/crates/paimon/src/table/compact.rs @@ -624,6 +624,7 @@ impl CompactExecutor { indexes, active, &files, + &deletion_vector_updates, self.resources.clone(), ) .await diff --git a/crates/paimon/src/table/compact/primary_key.rs b/crates/paimon/src/table/compact/primary_key.rs index 500247549..f8e42480e 100644 --- a/crates/paimon/src/table/compact/primary_key.rs +++ b/crates/paimon/src/table/compact/primary_key.rs @@ -324,10 +324,17 @@ fn read_type(table: &Table) -> Vec { } fn new_reader(table: &Table, resources: Option) -> Result { + new_reader_with_projection(table, resources, read_type(table)) +} + +fn new_reader_with_projection( + table: &Table, + resources: Option, + read_type: Vec, +) -> Result { let schema = table.schema(); let options = schema.core_options(); let merge_engine = options.merge_engine()?; - let read_type = read_type(table); let budget = configured_parquet_read_budget(table)?; let budget = match &resources { Some(resources) => Arc::new(budget.with_resources(resources.clone()).without_prefetch()), diff --git a/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs b/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs index 1b7033fa6..41249572d 100644 --- a/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs +++ b/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs @@ -159,6 +159,48 @@ pub(in crate::table::compact) fn with_deletion_entries( .build() } +/// Vector indexes exclude captured deletions and lookup hits discovered by this +/// compaction. Decode each complete-level source mask once for all columns. +pub(in crate::table::compact) async fn index_source_deletions( + table: &Table, + split: &DataSplit, + indexes: &[IndexManifestEntry], + updates: &[DeletionVectorUpdate], + files: &[&DataFileMeta], +) -> Result> { + let references = vector_references(indexes)?; + let partition = split.partition().to_serialized_bytes(); + let mut pending = HashMap::<&str, Vec<_>>::new(); + for update in updates + .iter() + .filter(|update| update.partition == partition && update.bucket == split.bucket()) + { + pending.entry(&update.file_name).or_default().push(update); + } + let mut masks = HashMap::new(); + for file in files { + let mut mask = read_bitmap( + table, + split.bucket_path(), + references.get(&(partition.clone(), split.bucket(), file.file_name.clone())), + ) + .await?; + for update in pending.get(file.file_name.as_str()).into_iter().flatten() { + mask |= RoaringBitmap::deserialize_from(update.bitmap.as_slice()) + .map_err(|error| super::super::invalid(&error.to_string()))?; + } + if !mask.is_empty() { + if i64::from(mask.max().unwrap()) >= file.row_count { + return Err(super::super::invalid( + "PK index deletion exceeds source rows", + )); + } + masks.insert(file.file_name.clone(), mask); + } + } + Ok(masks) +} + /// Selected masks must be unchanged. Actual point-lookup hits must still be /// live; unrelated deletions in their files can safely be rebased by union. pub(in crate::table) async fn validate_dependencies( diff --git a/crates/paimon/src/table/compact/primary_key/indexes.rs b/crates/paimon/src/table/compact/primary_key/indexes.rs index 1cc00d7da..78ab4e6f4 100644 --- a/crates/paimon/src/table/compact/primary_key/indexes.rs +++ b/crates/paimon/src/table/compact/primary_key/indexes.rs @@ -29,7 +29,10 @@ use crate::table::pk_index_maintenance::{levels, plan, unsupported_scalar_units} use crate::table::pk_index_options::definitions; use crate::table::{CommitMessage, DataSplit, Table, TableCommit}; use crate::Result; +use std::collections::HashSet; +use std::sync::Arc; +#[allow(clippy::too_many_arguments)] pub(in crate::table::compact) async fn prepare( table: &Table, task: &CompactTask, @@ -37,6 +40,7 @@ pub(in crate::table::compact) async fn prepare( indexes: &[IndexManifestEntry], active: &[ManifestEntry], files: &[DataFileMeta], + deletion_updates: &[super::super::deletion_vectors::DeletionVectorUpdate], resources: Option, ) -> Result> { let defs = definitions(table.schema())?; @@ -75,9 +79,41 @@ pub(in crate::table::compact) async fn prepare( &task.partition, task.bucket, )?; + let vector_levels = maintenance + .builds + .iter() + .filter(|(_, definition)| crate::vindex::is_vindex_index_type(&definition.index_type)) + .map(|(level, _)| *level) + .collect::>(); + let vector_sources = after + .iter() + .filter(|(level, _)| vector_levels.contains(level)) + .flat_map(|(_, files)| files) + .collect::>(); + let masks = Arc::new(if vector_sources.is_empty() { + Default::default() + } else { + super::deletion_vectors::index_source_deletions( + table, + split, + indexes, + deletion_updates, + &vector_sources, + ) + .await? + }); let mut entries = maintenance.deletes; for (level, definition) in maintenance.builds { - match writer::build(table, split, &after[&level], definition, resources.clone()).await { + match writer::build( + table, + split, + &after[&level], + definition, + masks.clone(), + resources.clone(), + ) + .await + { Ok(Some(index_file)) => entries.push(IndexManifestEntry { kind: FileKind::Add, partition: task.partition.clone(), diff --git a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs index bcd07473a..5d84e0fc2 100644 --- a/crates/paimon/src/table/compact/primary_key/indexes/writer.rs +++ b/crates/paimon/src/table/compact/primary_key/indexes/writer.rs @@ -18,7 +18,7 @@ //! Adapt a complete PK level to the shared native index build pipeline. use super::super::super::invalid; use crate::resource::ResourceContext; -use crate::spec::{DataFileMeta, GlobalIndexMeta, IndexFileMeta, ROW_ID_FIELD_NAME}; +use crate::spec::{DataField, DataFileMeta, GlobalIndexMeta, IndexFileMeta, ROW_ID_FIELD_NAME}; use crate::table::index_build::{ self, full_text, scalar::{self, ScalarIndexOptions}, @@ -30,44 +30,98 @@ use crate::table::pk_index_options::Definition; use crate::table::{ArrowRecordBatchStream, DataSplit, RowRange, Table}; use crate::vindex::VindexVectorIndexOptions; use crate::Result; -use arrow_array::{Int64Array, RecordBatch}; +use arrow_array::{new_null_array, Int64Array, RecordBatch}; use arrow_schema::{DataType as ArrowType, Field, Schema}; use async_stream::try_stream; use futures::TryStreamExt; +use roaring::RoaringBitmap; +use std::collections::HashMap; use std::sync::Arc; -/// Physical rows include NULLs, tombstones and DV-hidden positions. The common -/// writers receive dense, level-local ordinals, not a merged or filtered read. +#[cfg(test)] +mod tests; + +fn with_ordinals(batch: RecordBatch, start: i64) -> Result { + let end = start + .checked_add(batch.num_rows() as i64) + .ok_or_else(|| invalid("PK index ordinal overflow"))?; + let mut fields = batch.schema().fields().iter().cloned().collect::>(); + fields.push(Arc::new(Field::new( + ROW_ID_FIELD_NAME, + ArrowType::Int64, + false, + ))); + let mut columns = batch.columns().to_vec(); + columns.push(Arc::new(Int64Array::from_iter_values(start..end))); + RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).map_err(index_build::error) +} + +/// Select live vector positions before casts/native validation. Reinsert bounded +/// NULL batches for excluded positions so every native writer keeps the same +/// dense physical ordinal and source-count contract. fn batches( table: Table, split: DataSplit, files: Vec, + field: DataField, + deletions: Arc>, resources: Option, ) -> ArrowRecordBatchStream { Box::pin(try_stream! { let mut offset = 0i64; + let read_size = table.schema().core_options().read_batch_size()?.max(1); + let null_schema = crate::arrow::build_read_arrow_schema(std::slice::from_ref(&field))?; + let null_schema = Arc::new(Schema::new(vec![null_schema.field(0).clone().with_nullable(true)])); for file in &files { - let reader = super::super::new_reader(&table, resources.clone())?; - let mut stream = reader.read_physical_file(split.clone(), file.clone())?; - while let Some(batch) = stream.try_next().await? { - let count = batch.num_rows() as i64; - let mut fields = batch.schema().fields().iter().cloned().collect::>(); - fields.push(Arc::new(Field::new( - ROW_ID_FIELD_NAME, - ArrowType::Int64, - false, - ))); - let mut columns = batch.columns().to_vec(); - columns.push(Arc::new(Int64Array::from_iter_values( - offset..offset + count, - ))); - let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) + let whole = if file.row_count == 0 { vec![] } else { vec![RowRange::new(0, file.row_count - 1)] }; + let ranges = if let Some(mask) = deletions.get(&file.file_name) { + let mut deleted = Vec::new(); + let mut positions = mask.iter(); + while let Some(range) = positions.next_range() { + deleted.push(RowRange::new(i64::from(*range.start()), i64::from(*range.end()))); + } + crate::table::source::exclude_row_ranges(&whole, &deleted) + } else { whole }; + let reader = super::super::new_reader_with_projection( + &table, resources.clone(), vec![field.clone()], + )?; + let selection = deletions.contains_key(&file.file_name).then(|| ranges.clone()); + let mut stream = reader.read_physical_file_with_ranges(split.clone(), file.clone(), selection)?; + let mut pending: Option = None; + let mut position = 0i64; + for range in ranges { + while position < range.from() { + let rows = (range.from() - position).min(read_size as i64) as usize; + let batch = RecordBatch::try_new(null_schema.clone(), vec![new_null_array(null_schema.field(0).data_type(), rows)]) + .map_err(index_build::error)?; + yield with_ordinals(batch, offset + position)?; + position += rows as i64; + } + while position <= range.to() { + let batch = match pending.take() { + Some(batch) => batch, + None => stream.try_next().await?.ok_or_else(|| invalid("PK index source ended before its selected rows"))?, + }; + if batch.num_rows() == 0 { continue; } + let rows = batch.num_rows().min((range.to() - position + 1) as usize); + yield with_ordinals(batch.slice(0, rows), offset + position)?; + position += rows as i64; + if rows < batch.num_rows() { + pending = Some(batch.slice(rows, batch.num_rows() - rows)); + } + } + } + while position < file.row_count { + let rows = (file.row_count - position).min(read_size as i64) as usize; + let batch = RecordBatch::try_new(null_schema.clone(), vec![new_null_array(null_schema.field(0).data_type(), rows)]) .map_err(index_build::error)?; - yield batch; - offset = offset - .checked_add(count) - .ok_or_else(|| invalid("PK index ordinal overflow"))?; + yield with_ordinals(batch, offset + position)?; + position += rows as i64; + } + if pending.is_some() || stream.try_next().await?.is_some() { + Err(invalid("PK index source exceeds its selected rows"))?; } + offset = offset.checked_add(position).ok_or_else(|| invalid("PK index ordinal overflow"))?; } if offset != row_count(&files)? { Err(invalid("PK index source row count changed"))?; @@ -80,6 +134,7 @@ pub(super) async fn build( split: &DataSplit, files: &[DataFileMeta], definition: &Definition, + deletions: Arc>, resources: Option, ) -> Result> { let count = row_count(files)?; @@ -115,7 +170,19 @@ pub(super) async fn build( meta, ) .await?; - let stream = batches(table.clone(), split.clone(), files.to_vec(), resources); + let deletions = if crate::vindex::is_vindex_index_type(&definition.index_type) { + deletions + } else { + Arc::default() + }; + let stream = batches( + table.clone(), + split.clone(), + files.to_vec(), + definition.field.clone(), + deletions, + resources, + ); let mut options = table.schema().options().clone(); options.extend(definition.options.clone()); file.write(table, |output| async move { diff --git a/crates/paimon/src/table/compact/primary_key/indexes/writer/tests.rs b/crates/paimon/src/table/compact/primary_key/indexes/writer/tests.rs new file mode 100644 index 000000000..55e52cd81 --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/indexes/writer/tests.rs @@ -0,0 +1,544 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{ + DataType, Datum, FileKind, IndexManifestEntry, IntType, PredicateBuilder, + Schema as TableSchema, SchemaChange, TableSchema as ResolvedSchema, VarCharType, +}; +use crate::table::global_index_scanner::GlobalIndexScanner; +use arrow_array::{ArrayRef, Int32Array, StringArray}; + +#[tokio::test] +async fn masked_sources_preserve_dense_ordinals_across_files_and_single_row_batches() { + use crate::spec::{ArrayType, BinaryRow, FloatType}; + use arrow_array::{ + builder::{Float32Builder, ListBuilder}, + Array, Float32Array, ListArray, + }; + let schema = TableSchema::builder() + .column("id", DataType::Int(IntType::new())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .primary_key(["id"]) + .option("bucket", "1") + .option("write-only", "true") + .option("read.batch-size", "1") + .build() + .unwrap(); + let (table, counter) = + crate::table::maintenance_test_utils::table(ResolvedSchema::new(0, &schema)).await; + let write = table.new_write_builder(); + let mut files = Vec::new(); + for values in [vec![2, 3, 4], vec![10, 11]] { + let mut vectors = ListBuilder::new(Float32Builder::new()).with_field(Arc::new(Field::new( + "element", + ArrowType::Float32, + true, + ))); + for &value in &values { + vectors.values().append_slice(&[value as f32, 0.]); + vectors.append(true); + } + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(values)) as ArrayRef), + ("embedding", Arc::new(vectors.finish()) as ArrayRef), + ]) + .unwrap(); + let mut writer = write.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].new_files.len(), 1); + files.extend(messages[0].new_files.clone()); + write.new_commit().commit(messages).await.unwrap(); + } + let path = format!("{}/bucket-0", table.data_file_location()); + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + let split = DataSplit::builder() + .with_snapshot(snapshot.id()) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_total_buckets(1) + .with_bucket_path(path.clone()) + .with_data_files(files.clone()) + .build() + .unwrap(); + let field = table.schema().fields()[1].clone(); + for read_size in [1, 8] { + let table = table.copy_with_options(HashMap::from([( + "read.batch-size".into(), + read_size.to_string(), + )])); + for (first, second, expected) in [ + ( + vec![], + vec![], + vec![Some(2.), Some(3.), Some(4.), Some(10.), Some(11.)], + ), + ( + vec![0], + vec![], + vec![None, Some(3.), Some(4.), Some(10.), Some(11.)], + ), + ( + vec![1], + vec![1], + vec![Some(2.), None, Some(4.), Some(10.), None], + ), + (vec![0, 1, 2], vec![0, 1], vec![None; 5]), + ] { + let masks = [(&files[0], first), (&files[1], second)] + .into_iter() + .filter(|(_, positions)| !positions.is_empty()) + .map(|(file, positions)| (file.file_name.clone(), positions.into_iter().collect())) + .collect::>(); + counter.watch( + files + .iter() + .map(|file| format!("{path}/{}", file.file_name)) + .collect(), + ); + let output = batches( + table.clone(), + split.clone(), + files.clone(), + field.clone(), + Arc::new(masks), + None, + ) + .try_collect::>() + .await + .unwrap(); + let mut actual = Vec::new(); + for batch in output { + assert!( + batch.num_rows() <= read_size, + "NULL gaps must obey the read batch bound" + ); + let positions = batch + .column_by_name(ROW_ID_FIELD_NAME) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let vectors = batch + .column_by_name("embedding") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + let value = (!vectors.is_null(row)).then(|| { + let vector = vectors.value(row); + vector + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + }); + actual.push((positions.value(row), value)); + } + } + assert_eq!( + actual, + expected + .iter() + .enumerate() + .map(|(row, &value)| (row as i64, value)) + .collect::>() + ); + if expected.iter().all(Option::is_none) { + assert!( + counter.counts().values().all(|&opens| opens == 0), + "fully deleted sources need no data I/O" + ); + } + } + } +} + +#[tokio::test] +async fn scalar_source_projection_does_not_decode_unrelated_evolved_columns() { + let schema = TableSchema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .column("text", DataType::VarChar(VarCharType::string_type())) + .primary_key(["id"]) + .option("bucket", "1") + .option("write-only", "true") + .option("index-file-in-data-file-dir", "false") + .build() + .unwrap(); + let (table, _) = + crate::table::maintenance_test_utils::table(ResolvedSchema::new(0, &schema)).await; + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef), + ( + "value", + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + ), + ( + "text", + Arc::new(StringArray::from(vec!["not-a-number", "also-not-a-number"])) as ArrayRef, + ), + ]) + .unwrap(); + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + write + .new_commit() + .commit(writer.prepare_commit().await.unwrap()) + .await + .unwrap(); + + // Use a real compacted KV source, rather than changing an L0 file's metadata. + let compact = table.new_compact_builder(); + let plan = compact.new_planner().unwrap().plan().await.unwrap(); + let result = compact + .new_executor() + .unwrap() + .execute(&plan.tasks()[0]) + .await + .unwrap(); + compact + .new_commit() + .unwrap() + .commit_compact(vec![result]) + .await + .unwrap(); + let source_plan = table + .new_read_builder() + .new_scan() + .with_scan_all_files() + .plan() + .await + .unwrap(); + assert_eq!(source_plan.splits().len(), 1); + let split = source_plan.splits()[0].clone(); + let files = split.data_files().to_vec(); + assert_eq!(files.len(), 1); + assert!(crate::spec::should_read_pk_index_source(&files[0])); + + // This legal explicit type change cannot decode the old text values as INT. + // Building an index on the unchanged value column must not perform that cast. + let evolved = table + .schema() + .apply_changes(vec![ + SchemaChange::update_column_type("text".into(), DataType::Int(IntType::new())), + SchemaChange::set_option("pk-btree.index.columns".into(), "value".into()), + ]) + .unwrap(); + table + .file_io() + .new_output(&table.schema_manager().schema_path(evolved.id())) + .unwrap() + .write(serde_json::to_vec(&evolved).unwrap().into()) + .await + .unwrap(); + let current = table.copy_with_resolved_schema(evolved, "main").unwrap(); + crate::table::compact::primary_key::new_reader(¤t, None) + .unwrap() + .read_physical_file(split.clone(), files[0].clone()) + .unwrap() + .try_collect::>() + .await + .expect_err("the full-column control must decode the invalid text-to-INT cast"); + + let definition = crate::table::pk_index_options::definitions(current.schema()) + .unwrap() + .remove(0); + let source_batches = batches( + current.clone(), + split.clone(), + files.clone(), + definition.field.clone(), + Arc::default(), + None, + ) + .try_collect::>() + .await + .unwrap(); + let mut positions = Vec::new(); + for batch in source_batches { + assert_eq!( + batch + .schema() + .fields() + .iter() + .map(|field| field.name().as_str()) + .collect::>(), + vec!["value", ROW_ID_FIELD_NAME] + ); + let values = batch + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let ordinals = batch + .column_by_name(ROW_ID_FIELD_NAME) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + positions.extend((0..batch.num_rows()).map(|row| (values.value(row), ordinals.value(row)))); + } + assert_eq!(positions, vec![(10, 0), (20, 1)]); + + let payload = build(¤t, &split, &files, &definition, Arc::default(), None) + .await + .unwrap() + .unwrap(); + let entry = IndexManifestEntry { + kind: FileKind::Add, + partition: split.partition().to_serialized_bytes(), + bucket: split.bucket(), + index_file: payload, + version: 1, + }; + let scanner = GlobalIndexScanner::create_with_fm_options( + current.file_io(), + current.location(), + 1, + i64::MAX, + 128 * 1024 * 1024, + i64::MAX, + &[entry], + current.schema().fields(), + crate::fm_index::FMReadOptions::default(), + ) + .unwrap() + .unwrap(); + let predicate = PredicateBuilder::new(current.schema().fields()) + .equal("value", Datum::Int(20)) + .unwrap(); + assert_eq!( + scanner.matching_ranges(&predicate).await.unwrap(), + Some(vec![RowRange::new(1, 1)]) + ); +} + +#[tokio::test] +async fn vector_source_deletions_skip_schema_casts_without_renumbering_live_rows() { + use crate::spec::{ArrayType, FloatType, PrimaryKeyIndexSourceMeta}; + use crate::vector_search::{GlobalIndexIOMeta, VectorSearch}; + use crate::vindex::reader::VindexVectorGlobalIndexReader; + use arrow_array::{ + builder::ListBuilder, builder::StringBuilder, Array, Float32Array, ListArray, + }; + use std::io::Cursor; + + let schema = TableSchema::builder() + .column("id", DataType::Int(IntType::new())) + .column( + "embedding", + DataType::Array(ArrayType::new( + DataType::VarChar(VarCharType::string_type()), + )), + ) + .primary_key(["id"]) + .option("bucket", "1") + .option("write-only", "true") + .option("deletion-vectors.enabled", "true") + .option("deletion-vectors.merge-on-read", "false") + .option("index-file-in-data-file-dir", "false") + .build() + .unwrap(); + let (table, _) = + crate::table::maintenance_test_utils::table(ResolvedSchema::new(0, &schema)).await; + let mut vectors = ListBuilder::new(StringBuilder::new()).with_field(Arc::new( + arrow_schema::Field::new("element", ArrowType::Utf8, true), + )); + for vector in [["not-a-float", "0"], ["1", "0"]] { + vectors.values().append_value(vector[0]); + vectors.values().append_value(vector[1]); + vectors.append(true); + } + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef), + ("embedding", Arc::new(vectors.finish()) as ArrayRef), + ]) + .unwrap(); + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + write + .new_commit() + .commit(writer.prepare_commit().await.unwrap()) + .await + .unwrap(); + let compact = table.new_compact_builder(); + let plan = compact.new_planner().unwrap().plan().await.unwrap(); + let result = compact + .new_auto_executor() + .unwrap() + .execute(&plan.tasks()[0]) + .await + .unwrap(); + compact + .new_commit() + .unwrap() + .commit_compact(vec![result]) + .await + .unwrap(); + let source_plan = table + .new_read_builder() + .new_scan() + .with_scan_all_files() + .plan() + .await + .unwrap(); + assert_eq!(source_plan.splits().len(), 1); + let split = source_plan.splits()[0].clone(); + let files = split.data_files().to_vec(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].row_count, 2); + assert!(crate::spec::should_read_pk_index_source(&files[0])); + + let evolved = table + .schema() + .apply_changes(vec![ + SchemaChange::update_column_type( + "embedding".into(), + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ), + SchemaChange::set_option("pk-vector.index.columns".into(), "embedding".into()), + SchemaChange::set_option( + "fields.embedding.pk-vector.index.type".into(), + "ivf-flat".into(), + ), + SchemaChange::set_option( + "fields.embedding.pk-vector.distance.metric".into(), + "l2".into(), + ), + SchemaChange::set_option( + "fields.embedding.pk-vector.index.options".into(), + r#"{"dimension":2,"nlist":1}"#.into(), + ), + ]) + .unwrap(); + table + .file_io() + .new_output(&table.schema_manager().schema_path(evolved.id())) + .unwrap() + .write(serde_json::to_vec(&evolved).unwrap().into()) + .await + .unwrap(); + let current = table.copy_with_resolved_schema(evolved, "main").unwrap(); + let definition = crate::table::pk_index_options::definitions(current.schema()) + .unwrap() + .remove(0); + batches( + current.clone(), + split.clone(), + files.clone(), + definition.field.clone(), + Arc::default(), + None, + ) + .try_collect::>() + .await + .expect_err("an unmasked source must reject the invalid STRING-to-FLOAT element"); + + // Deletion positions refer to the old file, before decoding/casting values. + // The surviving vector must retain ordinal 1 even though native count is 1. + let deletions = Arc::new(HashMap::from([( + files[0].file_name.clone(), + RoaringBitmap::from_iter([0]), + )])); + let source_batches = batches( + current.clone(), + split.clone(), + files.clone(), + definition.field.clone(), + deletions.clone(), + None, + ) + .try_collect::>() + .await + .unwrap(); + let mut rows = Vec::new(); + for batch in source_batches { + let vectors = batch + .column_by_name("embedding") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let ordinals = batch + .column_by_name(ROW_ID_FIELD_NAME) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + let vector = (!vectors.is_null(row)).then(|| { + vectors + .value(row) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }); + rows.push((ordinals.value(row), vector)); + } + } + assert_eq!(rows, vec![(0, None), (1, Some(vec![1., 0.]))]); + + let payload = build(¤t, &split, &files, &definition, deletions, None) + .await + .unwrap() + .unwrap(); + assert_eq!(payload.row_count, 1); + let meta = payload.global_index_meta.as_ref().unwrap(); + assert_eq!((meta.row_range_start, meta.row_range_end), (0, 1)); + let source = PrimaryKeyIndexSourceMeta::from_global_index_meta(meta).unwrap(); + assert_eq!(source.source_files().len(), 1); + assert_eq!(source.source_files()[0].row_count(), 2); + let path = format!("{}/index/{}", current.location(), payload.file_name); + let bytes = current + .file_io() + .new_input(&path) + .unwrap() + .read() + .await + .unwrap() + .to_vec(); + let io_meta = GlobalIndexIOMeta::new( + path, + payload.file_size as u64, + meta.index_meta.clone().unwrap(), + ); + let mut reader = VindexVectorGlobalIndexReader::new(io_meta, HashMap::new()); + let query = VectorSearch::new(vec![1., 0.], 1, "embedding".into()).unwrap(); + let hits = reader + .visit_vector_search(&query, |_| Ok(Cursor::new(bytes))) + .unwrap() + .unwrap(); + assert_eq!(hits.len(), 1); + assert!(hits.contains_key(&1)); +} diff --git a/crates/paimon/src/table/deletion_vector_index.rs b/crates/paimon/src/table/deletion_vector_index.rs index 1471fd142..b96560e3d 100644 --- a/crates/paimon/src/table/deletion_vector_index.rs +++ b/crates/paimon/src/table/deletion_vector_index.rs @@ -17,7 +17,7 @@ //! Deletion-vector index I/O shared by table writers and compaction. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use bytes::Bytes; use indexmap::IndexMap; @@ -37,6 +37,9 @@ const DELETION_VECTORS_INDEX_TYPE: &str = "DELETION_VECTORS"; const DELETION_VECTORS_INDEX_VERSION_V1: u8 = 1; const MANIFEST_DIR: &str = "manifest"; +#[cfg(test)] +mod tests; + /// One bucket's deletion-vector location, owning the strings the shared resolver /// borrows. Built once per bucket so the read that merges existing vectors and the /// write that follows it cannot disagree about where the file goes. @@ -181,6 +184,119 @@ pub(super) async fn read_containers( Ok(bitmaps) } +/// An index may omit positions deleted in its captured snapshot. Those +/// positions must remain deleted when publishing that index: a concurrent +/// rollback can restore masks without changing any source data-file metadata. +/// New deletions and immutable-container repacking remain safe. Both manifests +/// are supplied by the commit attempt, so this never rereads index manifests. +pub(super) async fn validate_source_deletion_growth( + table: &Table, + captured: &[IndexManifestEntry], + current: &[IndexManifestEntry], + sources: HashSet<(Vec, i32, String)>, +) -> Result<()> { + type Source = (Vec, i32, String); + type Reference<'a> = (&'a IndexFileMeta, &'a DeletionVectorMeta); + + fn references<'a>( + table: &Table, + entries: &'a [IndexManifestEntry], + sources: &HashSet, + ) -> Result>> { + let mut result = HashMap::new(); + for entry in entries.iter().filter(|entry| { + entry.kind == FileKind::Add + && entry.index_file.index_type == DELETION_VECTORS_INDEX_TYPE + }) { + let partition = if table.schema().partition_keys().is_empty() { + vec![] + } else { + entry.partition.clone() + }; + for (name, meta) in entry.index_file.deletion_vectors_ranges.iter().flatten() { + let source = (partition.clone(), entry.bucket, name.clone()); + if sources.contains(&source) + && result.insert(source, (&entry.index_file, meta)).is_some() + { + return Err(crate::Error::DataInvalid { + message: "Multiple deletion vectors reference one index source".into(), + source: None, + }); + } + } + } + Ok(result) + } + + async fn bitmap( + table: &Table, + source: &Source, + reference: Reference<'_>, + ) -> Result { + let (file, meta) = reference; + let path = file_path(table, &source.0, source.1, file)?; + let deletion_file = crate::DeletionFile::new( + path, + i64::from(meta.offset), + i64::from(meta.length), + meta.cardinality, + ); + Ok(DeletionVectorFactory::read(table.file_io(), &deletion_file) + .await? + .to_bitmap()) + } + + if sources.is_empty() { + return Ok(()); + } + let sources = sources + .into_iter() + .map(|(partition, bucket, name)| { + let partition = if table.schema().partition_keys().is_empty() { + vec![] + } else { + partition + }; + (partition, bucket, name) + }) + .collect(); + let before = references(table, captured, &sources)?; + // No captured mask means the payload omitted no positions because of a + // deletion vector. Current masks need not be read in that case. + if before.is_empty() { + return Ok(()); + } + let after = references(table, current, &sources)?; + for (source, previous) in before { + let next = after.get(&source).copied(); + if next.is_some_and(|(file, meta)| { + file.file_name == previous.0.file_name + && file.external_path == previous.0.external_path + && meta == previous.1 + }) { + continue; + } + let excluded = bitmap(table, &source, previous).await?; + if excluded.is_empty() { + continue; + } + let deleted = match next { + Some(reference) => bitmap(table, &source, reference).await?, + None => RoaringBitmap::new(), + }; + if !excluded.is_subset(&deleted) { + return Err(crate::Error::DataInvalid { + message: format!( + "Index source deletions were restored for '{}'; rebuild the index from a new snapshot", + source.2 + ), + source: None, + }); + } + } + Ok(()) +} + pub(super) fn file_path( table: &Table, partition: &[u8], diff --git a/crates/paimon/src/table/deletion_vector_index/tests.rs b/crates/paimon/src/table/deletion_vector_index/tests.rs new file mode 100644 index 000000000..eeebb0616 --- /dev/null +++ b/crates/paimon/src/table/deletion_vector_index/tests.rs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{DataType, IntType, Schema, TableSchema}; + +#[tokio::test] +async fn unchanged_source_masks_and_new_masks_do_not_open_payloads() { + for in_data in ["false", "true"] { + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .primary_key(["id"]) + .option("bucket", "1") + .option("deletion-vectors.enabled", "true") + .option("deletion-vectors.merge-on-read", "false") + .option("index-file-in-data-file-dir", in_data) + .build() + .unwrap(); + let (table, counter) = + crate::table::maintenance_test_utils::table(TableSchema::new(0, &schema)).await; + let file = IndexFileMeta { + index_type: DELETION_VECTORS_INDEX_TYPE.into(), + file_name: "unopened-dv".into(), + file_size: 100, + row_count: 1, + deletion_vectors_ranges: Some(IndexMap::from([( + "source.parquet".into(), + DeletionVectorMeta { + offset: 1, + length: 20, + cardinality: Some(1), + }, + )])), + external_path: None, + global_index_meta: None, + }; + let partition = BinaryRow::new(0).to_serialized_bytes(); + let path = file_path(&table, &partition, 0, &file).unwrap(); + let entry = IndexManifestEntry { + kind: FileKind::Add, + partition, + bucket: 0, + index_file: file, + version: 1, + }; + // These files deliberately do not exist. Reopening an unchanged mask + // would fail instead of merely adding an invisible extra I/O. + let mut unrelated = entry.clone(); + unrelated.bucket = 1; + unrelated.index_file.file_name = "unrelated-unopened-dv".into(); + let unrelated_path = file_path( + &table, + &unrelated.partition, + unrelated.bucket, + &unrelated.index_file, + ) + .unwrap(); + counter.watch(vec![path, unrelated_path]); + let sources = HashSet::from([(vec![], 0, "source.parquet".into())]); + validate_source_deletion_growth( + &table, + std::slice::from_ref(&entry), + &[entry.clone(), unrelated], + sources.clone(), + ) + .await + .unwrap(); + validate_source_deletion_growth(&table, &[], &[entry], sources) + .await + .unwrap(); + assert!( + counter.counts().values().all(|reads| *reads == 0), + "{in_data}: {:?}", + counter.counts() + ); + } +} diff --git a/crates/paimon/src/table/full_text_index_build_builder.rs b/crates/paimon/src/table/full_text_index_build_builder.rs index d2907ece5..019008e68 100644 --- a/crates/paimon/src/table/full_text_index_build_builder.rs +++ b/crates/paimon/src/table/full_text_index_build_builder.rs @@ -223,9 +223,9 @@ impl<'a> FullTextIndexBuildBuilder<'a> { IndexFileWriter::new( self.table, FULL_TEXT_GLOBAL_INDEX_TYPE, - format!( - "{FULL_TEXT_GLOBAL_INDEX_TYPE}-global-index-{}.index", - uuid::Uuid::new_v4() + crate::table::global_index_schema::index_file_name( + FULL_TEXT_GLOBAL_INDEX_TYPE, + self.table.schema().id(), ), IndexFileLocation::Global { table_path: self.table.location().trim_end_matches('/'), diff --git a/crates/paimon/src/table/global_index_schema.rs b/crates/paimon/src/table/global_index_schema.rs new file mode 100644 index 000000000..bdedb9f1a --- /dev/null +++ b/crates/paimon/src/table/global_index_schema.rs @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Prove the logical field type of a global payload before reusing its coverage. +//! +//! Native scalar keys and full-text tokens do not record their logical source +//! type. New file names identify the actual builder schema, which can differ +//! from its captured data snapshot. +//! Java opens manifest file names unchanged, so no native or manifest format +//! changes are needed. Legacy payloads require an unchanged complete history. + +use crate::spec::DataField; +use crate::table::Table; +use std::collections::HashMap; + +pub(super) fn index_file_name(index_type: &str, schema_id: i64) -> String { + format!( + "{index_type}-global-index-schema-{schema_id}-{}.index", + uuid::Uuid::new_v4() + ) +} + +enum BuildSchema { + Tagged(i64), + Legacy, + Invalid, +} + +fn build_schema(file_name: &str, index_type: &str) -> BuildSchema { + let prefix = format!("{index_type}-global-index-schema-"); + let Some(tag) = file_name.strip_prefix(&prefix) else { + return BuildSchema::Legacy; + }; + let Some((id, uuid)) = tag.split_once('-') else { + return BuildSchema::Invalid; + }; + let Some(id) = id.parse::().ok().filter(|id| *id >= 0) else { + return BuildSchema::Invalid; + }; + if uuid + .strip_suffix(".index") + .and_then(|uuid| uuid::Uuid::parse_str(uuid).ok()) + .is_none() + { + return BuildSchema::Invalid; + } + BuildSchema::Tagged(id) +} + +/// One definition and maintenance round share successful and failed lookups. +/// In particular, a legacy history is not scanned once per payload file. +pub(super) struct GlobalIndexSchema<'a> { + table: &'a Table, + field: &'a DataField, + schemas: HashMap, + legacy: Option, +} + +impl<'a> GlobalIndexSchema<'a> { + pub(super) fn new(table: &'a Table, field: &'a DataField) -> Self { + Self { + table, + field, + schemas: HashMap::new(), + legacy: None, + } + } + + pub(super) async fn compatible(&mut self, file_name: &str, index_type: &str) -> bool { + match build_schema(file_name, index_type) { + BuildSchema::Tagged(id) => self.schema_matches(id).await, + BuildSchema::Invalid => false, + BuildSchema::Legacy => { + if let Some(compatible) = self.legacy { + return compatible; + } + let current = self.table.schema().id(); + let mut compatible = current >= 0; + for id in 0..=current { + if !self.schema_matches(id).await { + compatible = false; + break; + } + } + self.legacy = Some(compatible); + compatible + } + } + } + + async fn schema_matches(&mut self, id: i64) -> bool { + if let Some(&compatible) = self.schemas.get(&id) { + return compatible; + } + let compatible = if id == self.table.schema().id() { + // Includes catalog-resolved schemas without filesystem metadata. + true + } else { + self.table + .schema_manager() + .schema(id) + .await + .ok() + .is_some_and(|schema| { + schema.id() == id + && schema.fields().iter().any(|field| { + field.id() == self.field.id() + && field + .data_type() + .equals_ignore_nullable(self.field.data_type()) + }) + }) + }; + self.schemas.insert(id, compatible); + compatible + } +} diff --git a/crates/paimon/src/table/kv_file_reader.rs b/crates/paimon/src/table/kv_file_reader.rs index 9c7a575b2..2183e6568 100644 --- a/crates/paimon/src/table/kv_file_reader.rs +++ b/crates/paimon/src/table/kv_file_reader.rs @@ -384,6 +384,17 @@ impl KeyValueFileReader { self, split: DataSplit, file: DataFileMeta, + ) -> crate::Result { + self.read_physical_file_with_ranges(split, file, None) + } + + /// Select physical positions before decoding schema-evolution casts. The + /// caller retains the selected ranges to reconstruct source ordinals. + pub(super) fn read_physical_file_with_ranges( + self, + split: DataSplit, + file: DataFileMeta, + ranges: Option>, ) -> crate::Result { Ok(Box::pin(try_stream! { let config = self.config; @@ -405,9 +416,11 @@ impl KeyValueFileReader { .with_parquet_read_budget(config.parquet_read_budget) .with_table_options(config.table_options) .with_mosaic_prefetch(config.mosaic_prefetch); - let expected = file.row_count; + let expected = ranges.as_ref().map_or(file.row_count, |ranges| { + ranges.iter().map(super::RowRange::count).sum() + }); let mut stream = reader.read_single_file_stream_with_schema_and_local_ranges( - &split, file, old.map(|s| s.fields().to_vec()), data_schema, None, None, + &split, file, old.map(|s| s.fields().to_vec()), data_schema, None, ranges, )?; let mut rows = 0i64; while let Some(batch) = stream.next().await { diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index f2c3a4b7d..e3ed23746 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -87,6 +87,7 @@ mod global_index_freshness; mod global_index_options; mod global_index_result; pub(crate) mod global_index_scanner; +mod global_index_schema; mod global_index_search_filter; mod global_index_types; mod hybrid_search_builder; diff --git a/crates/paimon/src/table/pk_index_maintenance.rs b/crates/paimon/src/table/pk_index_maintenance.rs index 3d636b646..09f7005f0 100644 --- a/crates/paimon/src/table/pk_index_maintenance.rs +++ b/crates/paimon/src/table/pk_index_maintenance.rs @@ -356,6 +356,7 @@ pub(super) async fn validate( } } let mut names = HashSet::new(); + let mut vector_sources = HashSet::new(); for entry in prepared.iter().filter(|e| e.kind == FileKind::Add) { if !names.insert(entry.index_file.file_name.as_str()) || current @@ -383,7 +384,8 @@ pub(super) async fn validate( } else { indexed_rows == count }; - if source(entry)? != source_meta(key.2, files)? + let sources = source(entry)?; + if sources != source_meta(key.2, files)? || meta.row_range_start != 0 || meta.row_range_end != count - 1 || !valid_count @@ -394,10 +396,26 @@ pub(super) async fn validate( "PK index output does not match complete compacted level", )); } + if crate::vindex::is_vindex_index_type(&definition.index_type) { + vector_sources.extend(sources.source_files().iter().map(|file| { + ( + entry.partition.clone(), + entry.bucket, + file.file_name().into(), + ) + })); + } } if !required_builds.is_empty() { return Err(invalid("Missing PK index output")); } + super::deletion_vector_index::validate_source_deletion_growth( + table, + &captured, + current, + vector_sources, + ) + .await?; Ok(()) } diff --git a/crates/paimon/src/table/sorted_global_index_build_builder/writer.rs b/crates/paimon/src/table/sorted_global_index_build_builder/writer.rs index 9c817a7ca..a0d0d8270 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder/writer.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder/writer.rs @@ -54,7 +54,7 @@ impl SortedGlobalIndexBuildBuilder<'_> { IndexFileWriter::new( self.table, kind, - format!("{}-global-index-{}.index", kind, uuid::Uuid::new_v4()), + crate::table::global_index_schema::index_file_name(kind, self.table.schema().id()), IndexFileLocation::Global { table_path: self.table.location().trim_end_matches('/'), }, diff --git a/crates/paimon/src/table/table_commit/auto_index/planning.rs b/crates/paimon/src/table/table_commit/auto_index/planning.rs index 0c402beae..9beaf0646 100644 --- a/crates/paimon/src/table/table_commit/auto_index/planning.rs +++ b/crates/paimon/src/table/table_commit/auto_index/planning.rs @@ -23,6 +23,7 @@ use super::*; use crate::table::global_index_build_common::{ resolve_index_fields, GlobalIndexBuildView, IndexColumns, }; +use crate::table::index_build::full_text::{full_text_config, native_full_text_options}; use crate::table::source::exclude_row_ranges; use crate::table::universal_compaction::{Run, UniversalCompaction}; use crate::table::{merge_row_ranges, RowRange}; @@ -132,16 +133,24 @@ async fn plan_impl( &definition.index_type, )?; let field_id = fields[0].id(); - let vector_metric = if crate::vindex::is_vindex_index_type(&definition.index_type) { + let vector_semantics = if crate::vindex::is_vindex_index_type(&definition.index_type) { let options = crate::vindex::VindexVectorIndexOptions::new( table.schema().options(), &definition.options, &definition.index_type, fields[0], )?; - Some(crate::vindex::pkvector::metric::VectorSearchMetric::parse( - &options.native_options["metric"], - )?) + Some(( + crate::vindex::pkvector::metric::VectorSearchMetric::parse( + &options.native_options["metric"], + )?, + options.dimension(), + )) + } else { + None + }; + let full_text_definition = if definition.index_type == "full-text" { + Some(full_text_config(&native_full_text_options(&options))?) } else { None }; @@ -180,7 +189,7 @@ async fn plan_impl( .filter(|entry| { entry.kind == FileKind::Add && (entry.index_file.index_type == definition.index_type - || vector_metric.is_some() + || vector_semantics.is_some() && crate::vindex::is_vindex_index_type(&entry.index_file.index_type)) && entry .index_file @@ -194,21 +203,50 @@ async fn plan_impl( .collect::>(); let mut current: HashMap> = HashMap::new(); let mut replaced = HashSet::new(); + let mut index_schema = matches!( + definition.index_type.as_str(), + "btree" | "bitmap" | "multivalue" | "fm" | "full-text" + ) + .then(|| crate::table::global_index_schema::GlobalIndexSchema::new(table, fields[0])); for entry in selected.iter().copied() { let meta = entry.index_file.global_index_meta.as_ref().unwrap(); - if let Some(metric) = vector_metric { - // A vector column owns one algorithm and scoring metric across all shards. + if let Some(proof) = &mut index_schema { + if !proof + .compatible(&entry.index_file.file_name, &definition.index_type) + .await + { + replaced.insert(index_file_key(entry)); + continue; + } + } + if let Some(expected) = &full_text_definition { + // Analyzer options change matching semantics. Independent leaves + // must agree after applying native defaults and option aliases. + let recorded = meta + .index_meta + .as_deref() + .and_then(|bytes| serde_json::from_slice::>(bytes).ok()) + .and_then(|options| full_text_config(&options).ok()); + if recorded.as_ref() != Some(expected) { + replaced.insert(index_file_key(entry)); + continue; + } + } + if let Some((metric, dimension)) = vector_semantics { + // A vector column owns one algorithm, metric and dimension across all shards. // Coverage equality cannot make a previous definition compatible. let compatible = entry.index_file.index_type == definition.index_type - && crate::table::vector_search_common::read_index_vector_metric( + && crate::table::vector_search_common::read_index_vector_semantics( table.file_io(), table.location(), entry, ) .await? .is_some_and(|persisted| { - crate::vindex::pkvector::metric::VectorSearchMetric::from_vindex(persisted) - == metric + crate::vindex::pkvector::metric::VectorSearchMetric::from_vindex( + persisted.metric, + ) == metric + && persisted.dimension == dimension }); if !compatible { replaced.insert(index_file_key(entry)); @@ -292,7 +330,7 @@ async fn plan_impl( if pending.is_empty() { continue; } - if pending.as_slice() != std::slice::from_ref(&group) && vector_metric.is_some() { + if pending.as_slice() != std::slice::from_ref(&group) && vector_semantics.is_some() { training_groups.push(group.clone()); if force_training_merge { replaced.extend(existing.iter().map(|entry| index_file_key(entry))); diff --git a/crates/paimon/src/table/table_commit/auto_index_tests.rs b/crates/paimon/src/table/table_commit/auto_index_tests.rs index 498680478..80dbb729c 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests.rs @@ -25,7 +25,9 @@ use futures::TryStreamExt; use std::sync::atomic::{AtomicBool, Ordering}; mod lsm; +mod schema; mod search; +mod vector_dimension; #[tokio::test] async fn index_columns_share_source_scan_and_covered_rounds_do_not_rescan_builders() { diff --git a/crates/paimon/src/table/table_commit/auto_index_tests/schema.rs b/crates/paimon/src/table/table_commit/auto_index_tests/schema.rs new file mode 100644 index 000000000..afb132bb1 --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index_tests/schema.rs @@ -0,0 +1,719 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{FloatType, PredicateBuilder, SchemaChange, VarCharType}; +use arrow_array::{ArrayRef, Float32Array, StringArray}; + +async fn evolve(table: &Table, changes: Vec) -> Table { + let schema = table.schema().apply_changes(changes).unwrap(); + table + .file_io() + .new_output(&table.schema_manager().schema_path(schema.id())) + .unwrap() + .write(serde_json::to_vec(&schema).unwrap().into()) + .await + .unwrap(); + table + .copy_with_resolved_schema(schema, table.branch()) + .unwrap() +} + +async fn scalar_table(family: &str) -> Table { + evolve( + &table(Some(true), false).await, + vec![ + SchemaChange::remove_option("global-index.btree.index-columns".into()), + SchemaChange::remove_option("global-index.bitmap.index-columns".into()), + SchemaChange::set_option( + format!("global-index.{family}.index-columns"), + "value".into(), + ), + SchemaChange::set_option("sorted-index.records-per-range".into(), "64".into()), + SchemaChange::set_option("global-index.enabled".into(), "true".into()), + ], + ) + .await +} + +async fn float_messages(table: &Table, id: i32, value: f32) -> Vec { + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![id])) as ArrayRef), + ( + "value", + Arc::new(Float32Array::from(vec![value])) as ArrayRef, + ), + ]) + .unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + writer.prepare_commit().await.unwrap() +} + +async fn matching_ids(table: &Table, value: Datum, indexed: bool) -> Vec { + let table = table.copy_with_options(HashMap::from([( + "global-index.enabled".into(), + indexed.to_string(), + )])); + let mut read = table.new_read_builder(); + read.with_filter( + PredicateBuilder::new(table.schema().fields()) + .equal("value", value) + .unwrap(), + ); + let plan = read.new_scan().plan().await.unwrap(); + let batches: Vec = read + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect() +} + +async fn assert_scalar_complete(table: &Table, family: &str) { + assert!(table + .new_global_index_build_builder() + .with_index_column("value") + .with_index_type(family) + .build() + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn scalar_type_changes_retire_same_shard_payloads_in_both_directions() { + for family in ["btree", "bitmap"] { + let original = scalar_table(family).await; + TableCommit::new(original.clone(), "int-source".into()) + .commit(messages(&original, vec![1]).await) + .await + .unwrap(); + let integer_payloads = indexes(&original).await; + assert_eq!(integer_payloads.len(), 1); + let changed = evolve( + &original, + vec![SchemaChange::UpdateColumnType { + field_names: vec!["value".into()], + new_data_type: DataType::Float(FloatType::new()), + keep_nullability: true, + }], + ) + .await; + TableCommit::new(changed.clone(), "float-source".into()) + .commit(float_messages(&changed, 2, 20.).await) + .await + .unwrap(); + for indexed in [false, true] { + assert_eq!( + matching_ids(&changed, Datum::Float(10.), indexed).await, + vec![1], + "{family}, indexed={indexed}: INT keys cannot be read as FLOAT keys" + ); + } + let float_payloads = indexes(&changed).await; + assert!(integer_payloads + .iter() + .all(|entry| !float_payloads.contains(entry))); + assert_scalar_complete(&changed, family).await; + + // Remove and restore the declaration around the reverse type change. + // A type guard on currently declared columns alone cannot protect it. + let key = format!("global-index.{family}.index-columns"); + let removed = evolve(&changed, vec![SchemaChange::remove_option(key.clone())]).await; + let reverted = evolve( + &removed, + vec![SchemaChange::UpdateColumnType { + field_names: vec!["value".into()], + new_data_type: DataType::Int(IntType::new()), + keep_nullability: true, + }], + ) + .await; + let restored = evolve( + &reverted, + vec![SchemaChange::set_option(key, "value".into())], + ) + .await; + TableCommit::new(restored.clone(), "int-again".into()) + .commit(messages(&restored, vec![3]).await) + .await + .unwrap(); + for indexed in [false, true] { + assert_eq!( + matching_ids(&restored, Datum::Int(10), indexed).await, + vec![1], + "{family}, indexed={indexed}: FLOAT keys cannot survive a type round trip" + ); + } + let current = indexes(&restored).await; + assert!(float_payloads.iter().all(|entry| !current.contains(entry))); + assert_scalar_complete(&restored, family).await; + } +} + +#[tokio::test] +async fn scalar_writer_option_changes_preserve_compatible_leaf_payloads() { + for family in ["btree", "bitmap"] { + let original = scalar_table(family).await; + TableCommit::new(original.clone(), "original-options".into()) + .commit(messages(&original, vec![1]).await) + .await + .unwrap(); + let previous = indexes(&original).await; + let changed = evolve( + &original, + vec![SchemaChange::set_option( + format!("fields.value.global-index.{family}.index-options"), + r#"{"compression":"zstd"}"#.into(), + )], + ) + .await; + TableCommit::new(changed.clone(), "changed-options".into()) + .commit(messages(&changed, vec![2]).await) + .await + .unwrap(); + let current = indexes(&changed).await; + assert_eq!(current.len(), 2); + assert!(previous.iter().all(|entry| current.contains(entry))); + assert_eq!(matching_ids(&changed, Datum::Int(10), true).await, vec![1]); + assert_eq!(matching_ids(&changed, Datum::Int(20), true).await, vec![2]); + assert_scalar_complete(&changed, family).await; + } +} + +#[tokio::test] +async fn stale_scalar_build_keys_record_the_actual_builder_schema_and_are_retired() { + for family in ["btree", "bitmap"] { + let original = scalar_table(family).await; + let raw = original.copy_with_options(HashMap::from([("write-only".into(), "true".into())])); + TableCommit::new(raw.clone(), "raw-int-source".into()) + .commit(messages(&raw, vec![1]).await) + .await + .unwrap(); + let source = latest(&original).await; + let changed = evolve( + &original, + vec![SchemaChange::UpdateColumnType { + field_names: vec!["value".into()], + new_data_type: DataType::Float(FloatType::new()), + keep_nullability: true, + }], + ) + .await; + let raw = changed.copy_with_options(HashMap::from([("write-only".into(), "true".into())])); + TableCommit::new(raw.clone(), "raw-float-source".into()) + .commit(float_messages(&raw, 2, 20.).await) + .await + .unwrap(); + assert!(indexes(&changed).await.is_empty()); + let prepared = original + .new_global_index_build_builder() + .with_index_column("value") + .with_index_type(family) + .build() + .await + .unwrap(); + assert_eq!(prepared.len(), 1); + assert!(prepared[0].new_index_files[0] + .file_name + .starts_with(&format!( + "{family}-global-index-schema-{}-", + original.schema().id() + ))); + TableCommit::new(changed.clone(), "publish-stale-keys".into()) + .commit(prepared) + .await + .unwrap(); + let stale = indexes(&changed).await; + TableCommit::new(changed.clone(), "replace-stale-keys".into()) + .commit(float_messages(&changed, 3, 30.).await) + .await + .unwrap(); + let current = indexes(&changed).await; + assert!(stale.iter().all(|entry| !current.contains(entry))); + assert_eq!( + matching_ids(&changed, Datum::Float(10.), true).await, + vec![1] + ); + assert_eq!( + matching_ids(&changed, Datum::Float(20.), true).await, + vec![2] + ); + + // A current handle may still intentionally build cast keys from an + // older captured snapshot. Its name records the FLOAT builder schema, + // not the INT source snapshot's schema. + let prepared = changed + .new_global_index_build_builder() + .with_index_column("value") + .with_index_type(family) + .build_with_view( + GlobalIndexBuildView::capture(&changed, Some(source)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(prepared.len(), 1); + assert_eq!(prepared[0].new_index_files[0].row_count, 1); + assert!(prepared[0].new_index_files[0] + .file_name + .starts_with(&format!( + "{family}-global-index-schema-{}-", + changed.schema().id() + ))); + TableCommit::new(changed.clone(), "abort-cast-probe".into()) + .abort(&prepared) + .await + .unwrap(); + } +} + +#[tokio::test] +async fn scalar_build_schema_proof_does_not_depend_on_source_snapshot_retention() { + for family in ["btree", "bitmap"] { + let original = scalar_table(family).await; + TableCommit::new(original.clone(), "retained-payload".into()) + .commit(messages(&original, vec![1]).await) + .await + .unwrap(); + let previous = indexes(&original).await; + let source = crate::spec::DataEvolutionIndexSourceMeta::deserialize( + previous[0] + .index_file + .global_index_meta + .as_ref() + .unwrap() + .source_meta + .as_deref() + .unwrap(), + ) + .unwrap(); + let source_path = original + .snapshot_manager() + .snapshot_path(source.scan_snapshot_id()); + original.file_io().delete_file(&source_path).await.unwrap(); + assert!(!original.file_io().exists(&source_path).await.unwrap()); + let changed = evolve( + &original, + vec![SchemaChange::set_option( + "read.batch-size".into(), + "2".into(), + )], + ) + .await; + TableCommit::new(changed.clone(), "append-after-expiration".into()) + .commit(messages(&changed, vec![2]).await) + .await + .unwrap(); + let current = indexes(&changed).await; + assert_eq!(current.len(), 2); + assert!(previous.iter().all(|entry| current.contains(entry))); + assert_eq!(matching_ids(&changed, Datum::Int(10), true).await, vec![1]); + assert_eq!(matching_ids(&changed, Datum::Int(20), true).await, vec![2]); + } +} + +async fn legacy_payloads(table: &Table) -> Vec { + let mut messages = Vec::new(); + for entry in indexes(table).await { + let old = entry.index_file; + let mut legacy = old.clone(); + legacy.file_name = format!( + "{}-global-index-{}.index", + old.index_type, + uuid::Uuid::new_v4() + ); + table + .file_io() + .copy_file( + &format!("{}/index/{}", table.location(), old.file_name), + &format!("{}/index/{}", table.location(), legacy.file_name), + ) + .await + .unwrap(); + let mut message = CommitMessage::new(entry.partition, entry.bucket, vec![]); + message.deleted_index_files.push(old); + message.new_index_files.push(legacy); + messages.push(message); + } + TableCommit::new(table.clone(), "legacy-file-names".into()) + .commit(messages) + .await + .unwrap(); + indexes(table).await +} + +#[tokio::test] +async fn legacy_scalar_files_require_a_complete_unchanged_type_history() { + for family in ["btree", "bitmap"] { + for history in ["same-type", "round-trip", "missing"] { + let original = scalar_table(family).await; + TableCommit::new(original.clone(), "legacy-source".into()) + .commit(messages(&original, vec![1]).await) + .await + .unwrap(); + let previous = legacy_payloads(&original).await; + let changed = if history == "round-trip" { + let float = evolve( + &original, + vec![SchemaChange::UpdateColumnType { + field_names: vec!["value".into()], + new_data_type: DataType::Float(FloatType::new()), + keep_nullability: true, + }], + ) + .await; + evolve( + &float, + vec![SchemaChange::UpdateColumnType { + field_names: vec!["value".into()], + new_data_type: DataType::Int(IntType::new()), + keep_nullability: true, + }], + ) + .await + } else { + if history == "missing" { + original + .file_io() + .delete_file(&original.schema_manager().schema_path(0)) + .await + .unwrap(); + } + evolve( + &original, + vec![SchemaChange::set_option( + format!("fields.value.global-index.{family}.index-options"), + r#"{"compression":"zstd"}"#.into(), + )], + ) + .await + }; + TableCommit::new(changed.clone(), "legacy-maintenance".into()) + .commit(messages(&changed, vec![2]).await) + .await + .unwrap(); + let current = indexes(&changed).await; + assert_eq!( + previous.iter().all(|entry| current.contains(entry)), + history == "same-type", + "{family}, history={history}" + ); + assert_eq!(matching_ids(&changed, Datum::Int(10), true).await, vec![1]); + assert_eq!(matching_ids(&changed, Datum::Int(20), true).await, vec![2]); + let before = current; + TableCommit::new(changed.clone(), "stable-provenance".into()) + .commit(messages(&changed, vec![3]).await) + .await + .unwrap(); + let after = indexes(&changed).await; + assert!(before.iter().all(|entry| after.contains(entry))); + } + } +} + +async fn text_messages(table: &Table, id: i32, text: &str) -> Vec { + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![id])) as ArrayRef), + ( + "value", + Arc::new(Int32Array::from(vec![id * 10])) as ArrayRef, + ), + ("text", Arc::new(StringArray::from(vec![text])) as ArrayRef), + ]) + .unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + writer.prepare_commit().await.unwrap() +} + +async fn text_matches(table: &Table, query: &str, mode: &str) -> Vec { + let table = table.copy_with_options(HashMap::from([( + "full-text-index.search-mode".into(), + mode.into(), + )])); + let mut ids = table + .new_full_text_search_builder() + .with_text_column("text") + .with_query_text(query) + .with_limit(10) + .execute() + .await + .unwrap() + .row_ids() + .unwrap() + .row_ids + .clone(); + ids.sort_unstable(); + ids +} + +#[tokio::test] +async fn full_text_narrowing_rebuilds_same_shard_tokens_to_match_current_text() { + let original = evolve( + &table(Some(true), false).await, + vec![ + SchemaChange::remove_option("global-index.btree.index-columns".into()), + SchemaChange::remove_option("global-index.bitmap.index-columns".into()), + SchemaChange::add_column( + "text".into(), + DataType::VarChar(VarCharType::new(100).unwrap()), + ), + SchemaChange::set_option("global-index.full-text.index-columns".into(), "text".into()), + SchemaChange::set_option("global-index.row-count-per-shard".into(), "64".into()), + ], + ) + .await; + TableCommit::new(original.clone(), "wide-text".into()) + .commit(text_messages(&original, 1, "sentinel").await) + .await + .unwrap(); + let previous = indexes(&original).await; + assert_eq!(text_matches(&original, "sentinel", "fast").await, vec![0]); + let changed = evolve( + &original, + vec![SchemaChange::UpdateColumnType { + field_names: vec!["text".into()], + new_data_type: DataType::VarChar(VarCharType::new(3).unwrap()), + keep_nullability: true, + }], + ) + .await; + TableCommit::new(changed.clone(), "narrow-text".into()) + .commit(text_messages(&changed, 2, "new").await) + .await + .unwrap(); + let native = [ + ("sentinel", text_matches(&changed, "sentinel", "fast").await), + ("sen", text_matches(&changed, "sen", "fast").await), + ]; + assert!( + native[0].1.is_empty(), + "old tokens must not match truncated text" + ); + assert_eq!( + native[1].1, + vec![0], + "current text must be indexed as 'sen'" + ); + let current = indexes(&changed).await; + assert!(previous.iter().all(|entry| !current.contains(entry))); + + // Remove index references to force the existing raw FULL/DETAIL paths. + // Their document text is read through the same real VARCHAR narrowing. + changed + .new_global_index_drop_builder() + .with_index_column("text") + .with_index_type("full-text") + .execute() + .await + .unwrap(); + assert!(indexes(&changed).await.is_empty()); + for mode in ["full", "detail"] { + for (query, expected) in &native { + assert_eq!( + text_matches(&changed, query, mode).await, + *expected, + "{query}, {mode}" + ); + } + } +} + +async fn analyzer_table() -> Table { + evolve( + &table(Some(true), false).await, + vec![ + SchemaChange::remove_option("global-index.btree.index-columns".into()), + SchemaChange::remove_option("global-index.bitmap.index-columns".into()), + SchemaChange::add_column("text".into(), DataType::VarChar(VarCharType::string_type())), + SchemaChange::set_option("global-index.full-text.index-columns".into(), "text".into()), + SchemaChange::set_option("global-index.row-count-per-shard".into(), "64".into()), + ], + ) + .await +} + +#[tokio::test] +async fn full_text_analyzer_change_rebuilds_same_shard_tokens() { + let original = analyzer_table().await; + TableCommit::new(original.clone(), "default-analyzer".into()) + .commit(text_messages(&original, 1, "MiXeD").await) + .await + .unwrap(); + let previous = indexes(&original).await; + assert_eq!(previous.len(), 1); + let persisted: HashMap = serde_json::from_slice( + previous[0] + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_meta + .as_ref() + .unwrap(), + ) + .unwrap(); + assert!( + persisted.is_empty(), + "lower-case=true is the native default" + ); + assert_eq!(text_matches(&original, "mixed", "fast").await, vec![0]); + let historical = original.copy_with_pinned_snapshot(Some(&latest(&original).await)); + + let changed = evolve( + &original, + vec![SchemaChange::set_option( + "fields.text.global-index.full-text.index-options".into(), + r#"{"lower-case":false}"#.into(), + )], + ) + .await; + // Both rows remain in the same 64-row shard, and two runs are below the + // default merge trigger. Definition compatibility must still retire the old analyzer. + TableCommit::new(changed.clone(), "case-sensitive-analyzer".into()) + .commit(text_messages(&changed, 2, "new").await) + .await + .unwrap(); + let native = [ + ("mixed", text_matches(&changed, "mixed", "fast").await), + ("MiXeD", text_matches(&changed, "MiXeD", "fast").await), + ("new", text_matches(&changed, "new", "fast").await), + ]; + assert!( + native[0].1.is_empty(), + "the old lower-case analyzer must not produce a false positive" + ); + assert_eq!(native[1].1, vec![0]); + assert_eq!(native[2].1, vec![1]); + let current = indexes(&changed).await; + assert!(previous.iter().all(|entry| !current.contains(entry))); + assert_eq!(text_matches(&historical, "mixed", "fast").await, vec![0]); + + changed + .new_global_index_drop_builder() + .with_index_column("text") + .with_index_type("full-text") + .execute() + .await + .unwrap(); + assert!(indexes(&changed).await.is_empty()); + for mode in ["full", "detail"] { + for (query, expected) in &native { + assert_eq!( + text_matches(&changed, query, mode).await, + *expected, + "{query}, {mode}" + ); + } + } +} + +#[tokio::test] +async fn full_text_explicit_default_analyzer_preserves_compatible_leaf() { + let original = analyzer_table().await; + TableCommit::new(original.clone(), "implicit-default-analyzer".into()) + .commit(text_messages(&original, 1, "MiXeD").await) + .await + .unwrap(); + let previous = indexes(&original).await; + let changed = evolve( + &original, + vec![SchemaChange::set_option( + "fields.text.global-index.full-text.index-options".into(), + r#"{"lower-case":true}"#.into(), + )], + ) + .await; + TableCommit::new(changed.clone(), "explicit-default-analyzer".into()) + .commit(text_messages(&changed, 2, "new").await) + .await + .unwrap(); + let current = indexes(&changed).await; + assert_eq!( + current.len(), + 2, + "equivalent defaults must retain independent runs" + ); + assert!(previous.iter().all(|entry| current.contains(entry))); + assert_eq!(text_matches(&changed, "mixed", "fast").await, vec![0]); + assert_eq!(text_matches(&changed, "new", "fast").await, vec![1]); +} + +#[tokio::test] +async fn full_text_unproven_analyzer_metadata_requires_rebuilding() { + for index_meta in [None, Some(b"invalid-options".to_vec())] { + let table = analyzer_table().await; + TableCommit::new(table.clone(), "known-analyzer".into()) + .commit(text_messages(&table, 1, "MiXeD").await) + .await + .unwrap(); + let previous = indexes(&table).await.remove(0); + let mut legacy = previous.index_file.clone(); + legacy.file_name = + crate::table::global_index_schema::index_file_name("full-text", table.schema().id()); + legacy.global_index_meta.as_mut().unwrap().index_meta = index_meta; + table + .file_io() + .copy_file( + &format!( + "{}/index/{}", + table.location(), + previous.index_file.file_name + ), + &format!("{}/index/{}", table.location(), legacy.file_name), + ) + .await + .unwrap(); + let mut replacement = CommitMessage::new(previous.partition, previous.bucket, vec![]); + replacement.deleted_index_files.push(previous.index_file); + replacement.new_index_files.push(legacy.clone()); + TableCommit::new(table.clone(), "unproven-analyzer".into()) + .commit(vec![replacement]) + .await + .unwrap(); + TableCommit::new(table.clone(), "append-to-unproven-analyzer".into()) + .commit(text_messages(&table, 2, "new").await) + .await + .unwrap(); + assert!(indexes(&table) + .await + .iter() + .all(|entry| entry.index_file.file_name != legacy.file_name)); + assert_eq!(text_matches(&table, "mixed", "fast").await, vec![0]); + assert_eq!(text_matches(&table, "new", "fast").await, vec![1]); + } +} diff --git a/crates/paimon/src/table/table_commit/auto_index_tests/vector_dimension.rs b/crates/paimon/src/table/table_commit/auto_index_tests/vector_dimension.rs new file mode 100644 index 000000000..5aa33c5db --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index_tests/vector_dimension.rs @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::spec::{ArrayType, FloatType, SchemaChange}; +use arrow_array::builder::{Float32Builder, ListBuilder}; +use arrow_array::ArrayRef; + +async fn vector_messages(table: &Table, id: i32, vector: &[f32]) -> Vec { + let mut vectors = ListBuilder::new(Float32Builder::new()); + vectors.values().append_slice(vector); + vectors.append(true); + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(vec![id])) as ArrayRef), + ("embedding", Arc::new(vectors.finish()) as ArrayRef), + ]) + .unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + writer.prepare_commit().await.unwrap() +} + +#[tokio::test] +async fn vector_dimension_change_cannot_publish_incompatible_leaf_indexes() { + let schema = TableSchemaBuilder::builder() + .column("id", DataType::Int(IntType::new())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("global-index.enabled", "true") + .option("global-index.row-count-per-shard", "64") + .option("global-index.vector.index-columns", "embedding") + .option( + "fields.embedding.global-index.vector.index-type", + "ivf-flat", + ) + .option( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":2,"nlist":1}"#, + ) + .option("compaction.min.file-num", "1000") + .build() + .unwrap(); + let (table, _) = + crate::table::maintenance_test_utils::table(TableSchema::new(0, &schema)).await; + TableCommit::new(table.clone(), "two-dimensional-source".into()) + .commit(vector_messages(&table, 1, &[1., 0.]).await) + .await + .unwrap(); + let indexed = latest(&table).await; + let original = indexes(&table).await; + assert_eq!(original.len(), 1); + let previous = &original[0].index_file; + let original_path = previous + .external_path + .clone() + .unwrap_or_else(|| format!("{}/index/{}", table.location(), previous.file_name)); + let original_bytes = table + .file_io() + .new_input(&original_path) + .unwrap() + .read() + .await + .unwrap(); + + // ARRAY permits different lengths without a schema type change. + // Two runs are below the default merge trigger, but their dimensions are + // still incompatible even when algorithm, metric and shard remain equal. + let evolved = table + .schema() + .apply_changes(vec![SchemaChange::set_option( + "fields.embedding.global-index.vector.index-options".into(), + r#"{"dimension":3,"nlist":1}"#.into(), + )]) + .unwrap(); + table + .file_io() + .new_output(&table.schema_manager().schema_path(evolved.id())) + .unwrap() + .write(serde_json::to_vec(&evolved).unwrap().into()) + .await + .unwrap(); + let current = table.copy_with_resolved_schema(evolved, "main").unwrap(); + let input = vector_messages(¤t, 2, &[0., 1., 0.]).await; + let commit = TableCommit::new(current.clone(), "changed-vector-dimension".into()); + let error = commit + .filter_and_commit_with_identifier(input.clone(), 77) + .await + .expect_err("dimension changes cannot silently retain an incompatible old leaf"); + assert!(error.to_string().contains("dimension"), "{error}"); + + let committed_data = latest(¤t).await; + assert_eq!(committed_data.id(), indexed.id() + 1); + assert_eq!(*committed_data.commit_kind(), CommitKind::APPEND); + assert_eq!(committed_data.next_row_id(), Some(2)); + assert_eq!(indexes(¤t).await, original); + assert_eq!( + current + .file_io() + .new_input(&original_path) + .unwrap() + .read() + .await + .unwrap(), + original_bytes + ); + + for _ in 0..2 { + let error = commit + .filter_and_commit_with_identifier(input.clone(), 77) + .await + .expect_err("recovery must retry maintenance without publishing mixed dimensions"); + assert!(error.to_string().contains("dimension"), "{error}"); + let recovered = latest(¤t).await; + assert_eq!(recovered.id(), committed_data.id()); + assert_eq!(recovered.next_row_id(), Some(2)); + assert_eq!(indexes(¤t).await, original); + assert!(current.file_io().exists(&original_path).await.unwrap()); + } +} diff --git a/crates/paimon/src/table/vector_search_common.rs b/crates/paimon/src/table/vector_search_common.rs index 5d5adf3fa..4f2425aab 100644 --- a/crates/paimon/src/table/vector_search_common.rs +++ b/crates/paimon/src/table/vector_search_common.rs @@ -37,7 +37,15 @@ use std::io::Cursor; pub(super) use super::partition_filter::validate_partition_filter; -/// Read persisted scoring semantics for both raw queries and definition compatibility checks. +/// Persisted vector properties which must agree across all runs of one column. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct IndexVectorSemantics { + pub metric: MetricType, + pub dimension: usize, +} + +/// Read the scoring metric without requiring dimension metadata or a payload read. +/// Older callers and JSON-only metadata need only this scoring property. pub(super) async fn read_index_vector_metric( file_io: &FileIO, table_path: &str, @@ -51,19 +59,63 @@ pub(super) async fn read_index_vector_metric( } if let Some(index_meta) = global_meta.index_meta.as_ref() { if let Ok(options) = serde_json::from_slice::>(index_meta) { - if let Some(metric) = options.get("metric") { - let metric = match normalize_metric(metric).as_str() { - "l2" => Some(MetricType::L2), - "cosine" => Some(MetricType::Cosine), - "inner_product" => Some(MetricType::InnerProduct), - _ => None, - }; - if metric.is_some() { - return Ok(metric); - } + if let Some(metric) = metric_from_index_options(&options) { + return Ok(Some(metric)); + } + } + } + Ok(Some( + read_index_vector_header(file_io, table_path, entry) + .await? + .metric, + )) +} + +/// Read the structural definition of an existing run before reusing its coverage. +/// Java and older metadata may omit these options, so fall back to the native header. +pub(super) async fn read_index_vector_semantics( + file_io: &FileIO, + table_path: &str, + entry: &IndexManifestEntry, +) -> crate::Result> { + let Some(global_meta) = entry.index_file.global_index_meta.as_ref() else { + return Ok(None); + }; + if !is_vindex_index_type(&entry.index_file.index_type) { + return Ok(None); + } + if let Some(index_meta) = global_meta.index_meta.as_ref() { + if let Ok(options) = serde_json::from_slice::>(index_meta) { + let dimension = options + .get("dimension") + .and_then(|value| value.parse::().ok()) + .filter(|dimension| *dimension > 0); + if let (Some(metric), Some(dimension)) = + (metric_from_index_options(&options), dimension) + { + return Ok(Some(IndexVectorSemantics { metric, dimension })); } } } + Ok(Some( + read_index_vector_header(file_io, table_path, entry).await?, + )) +} + +fn metric_from_index_options(options: &HashMap) -> Option { + match normalize_metric(options.get("metric")?).as_str() { + "l2" => Some(MetricType::L2), + "cosine" => Some(MetricType::Cosine), + "inner_product" => Some(MetricType::InnerProduct), + _ => None, + } +} + +async fn read_index_vector_header( + file_io: &FileIO, + table_path: &str, + entry: &IndexManifestEntry, +) -> crate::Result { let path = IndexFileLocation::Global { table_path }.resolve( &entry.index_file.file_name, entry.index_file.external_path.as_deref(), @@ -95,7 +147,11 @@ pub(super) async fn read_index_vector_metric( ), source: Some(Box::new(e)), })?; - Ok(Some(reader.metadata().metric)) + let metadata = reader.metadata(); + Ok(IndexVectorSemantics { + metric: metadata.metric, + dimension: metadata.dimension, + }) } pub(super) fn add_filter(current: &mut Option, filter: Predicate) { @@ -513,3 +569,103 @@ pub(super) fn targets_primary_key_column(core: &CoreOptions<'_>, column: &str) - .ok() .is_some_and(|columns| columns.iter().any(|c| c == column)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::io::FileIOBuilder; + use crate::spec::{GlobalIndexMeta, IndexFileMeta}; + + fn entry(index_meta: Option>) -> IndexManifestEntry { + IndexManifestEntry { + version: 1, + kind: FileKind::Add, + partition: Vec::new(), + bucket: 0, + index_file: IndexFileMeta { + index_type: "ivf-flat".into(), + file_name: "vectors.index".into(), + file_size: 0, + row_count: 3, + deletion_vectors_ranges: None, + external_path: None, + global_index_meta: Some(GlobalIndexMeta { + row_range_start: 0, + row_range_end: 2, + index_field_id: 1, + extra_field_ids: None, + index_meta, + source_meta: None, + }), + }, + } + } + + #[tokio::test] + async fn metric_only_json_does_not_require_dimension_or_payload() { + let file_io = FileIOBuilder::new("memory").build().unwrap(); + let entry = entry(Some(br#"{"metric":"L2"}"#.to_vec())); + let table_path = "memory:/metric-only-vector-metadata"; + assert!(!file_io + .exists(&format!("{table_path}/index/vectors.index")) + .await + .unwrap()); + assert_eq!( + read_index_vector_metric(&file_io, table_path, &entry) + .await + .unwrap(), + Some(MetricType::L2) + ); + } + + #[tokio::test] + async fn complete_vector_semantics_json_does_not_read_payload() { + let file_io = FileIOBuilder::new("memory").build().unwrap(); + let entry = entry(Some( + br#"{"metric":"inner-product","dimension":"3"}"#.to_vec(), + )); + let table_path = "memory:/complete-vector-metadata"; + assert!(!file_io + .exists(&format!("{table_path}/index/vectors.index")) + .await + .unwrap()); + assert_eq!( + read_index_vector_semantics(&file_io, table_path, &entry) + .await + .unwrap(), + Some(IndexVectorSemantics { + metric: MetricType::InnerProduct, + dimension: 3, + }) + ); + } + + #[tokio::test] + async fn legacy_vector_semantics_read_dimension_from_native_header() { + let file_io = FileIOBuilder::new("memory").build().unwrap(); + let table_path = "memory:/legacy-vector-metadata"; + let payload = crate::table::vector_search_test_utils::build_vindex_segment_bytes("l2"); + file_io + .new_output(&format!("{table_path}/index/vectors.index")) + .unwrap() + .write(payload.into()) + .await + .unwrap(); + for index_meta in [ + None, + Some(br#"{}"#.to_vec()), + Some(br#"{"metric":"l2"}"#.to_vec()), + Some(br#"{"metric":"l2","dimension":"0"}"#.to_vec()), + ] { + assert_eq!( + read_index_vector_semantics(&file_io, table_path, &entry(index_meta)) + .await + .unwrap(), + Some(IndexVectorSemantics { + metric: MetricType::L2, + dimension: 2, + }) + ); + } + } +} diff --git a/crates/paimon/tests/pk_compact/indexes.rs b/crates/paimon/tests/pk_compact/indexes.rs index a42f32b95..40beaf250 100644 --- a/crates/paimon/tests/pk_compact/indexes.rs +++ b/crates/paimon/tests/pk_compact/indexes.rs @@ -202,6 +202,403 @@ async fn empty_vector_level_is_persisted_reused_and_searchable() { } } +#[tokio::test] +async fn vector_indexes_skip_invalid_values_hidden_by_captured_and_new_deletion_vectors() { + use arrow_array::{ + builder::{Float32Builder, ListBuilder}, + ArrayRef, + }; + use paimon::spec::SchemaChange; + use std::sync::Arc; + + for captured_mask in [true, false] { + let case = if captured_mask { "captured" } else { "new" }; + let original = vector_table_for_indexes(&[]).await; + let schema = original + .schema() + .apply_changes(vec![SchemaChange::remove_option( + "pk-vector.index.columns".into(), + )]) + .unwrap(); + persist_table_schema(original.file_io(), original.location(), &schema).await; + let table = original.copy_with_resolved_schema(schema, "main").unwrap(); + + // A nullable ARRAY can contain a NULL child before an index is + // declared. The only non-NULL vector in this base will later be hidden. + let mut vectors = ListBuilder::new(Float32Builder::new()).with_field(Arc::new( + arrow_schema::Field::new("element", arrow_schema::DataType::Float32, true), + )); + for id in 0..1000 { + if id == 42 { + vectors.values().append_value(1.); + vectors.values().append_null(); + vectors.append(true); + } else { + vectors.append(false); + } + } + let batch = RecordBatch::try_from_iter([ + ( + "id", + Arc::new(Int32Array::from_iter_values(0..1000)) as ArrayRef, + ), + ("embedding", Arc::new(vectors.finish()) as ArrayRef), + ]) + .unwrap(); + write_batch(&table, &batch).await; + let first = compact(&table).await; + assert_eq!(first.len(), 1, "{case}"); + assert_eq!(first[0].files().len(), 1, "{case}"); + let base = first[0].files()[0].clone(); + assert_eq!(base.row_count, 1000, "{case}"); + assert!(base.level > 1, "{case}"); + assert!(indexes(&table).await.is_empty(), "{case}"); + + if captured_mask { + write_batch(&table, &vector_index_batch(vec![42], vec![Some([42., 0.])])).await; + compact(&table).await; + assert!(indexes(&table).await.iter().any(|entry| { + entry.index_file.index_type == "DELETION_VECTORS" + && entry + .index_file + .deletion_vectors_ranges + .as_ref() + .is_some_and(|ranges| ranges.contains_key(&base.file_name)) + })); + } + + let schema = table + .schema() + .apply_changes(vec![SchemaChange::set_option( + "pk-vector.index.columns".into(), + "embedding".into(), + )]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &schema).await; + let current = table.copy_with_resolved_schema(schema, "main").unwrap(); + let batch = if captured_mask { + vector_index_batch(vec![1001], vec![Some([1001., 0.])]) + } else { + // This compaction discovers the old position through lookup and + // publishes its mask together with the newly built indexes. + vector_index_batch(vec![42, 1001], vec![Some([42., 0.]), Some([1001., 0.])]) + }; + write_batch(¤t, &batch).await; + let builder = current.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + assert!(!plan.tasks().is_empty(), "{case}"); + assert!( + plan.tasks() + .iter() + .flat_map(CompactTask::files) + .all(|file| file.file_name != base.file_name), + "{case}: the hidden value must remain in unselected source history" + ); + let results = execute(¤t, &plan).await; + builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); + + let entries = indexes(¤t).await; + let empty = entries + .iter() + .find(|entry| { + entry.index_file.index_type == "ivf-flat" + && PrimaryKeyIndexSourceMeta::from_global_index_meta( + entry.index_file.global_index_meta.as_ref().unwrap(), + ) + .unwrap() + .source_files() + .iter() + .any(|source| source.file_name() == base.file_name) + }) + .unwrap(); + let meta = empty.index_file.global_index_meta.as_ref().unwrap(); + let source = PrimaryKeyIndexSourceMeta::from_global_index_meta(meta).unwrap(); + assert_eq!(source.source_files().len(), 1, "{case}"); + assert_eq!(source.source_files()[0].row_count(), 1000, "{case}"); + assert_eq!( + (meta.row_range_start, meta.row_range_end), + (0, 999), + "{case}" + ); + assert_eq!(empty.index_file.row_count, 0, "{case}"); + assert_eq!(empty.index_file.file_size, 0, "{case}"); + assert!(current + .file_io() + .exists(&index_path(¤t, empty)) + .await + .unwrap()); + let deletion = entries + .iter() + .find_map(|entry| { + entry + .index_file + .deletion_vectors_ranges + .as_ref() + .and_then(|ranges| ranges.get(&base.file_name)) + }) + .unwrap(); + assert_eq!(deletion.cardinality, Some(1), "{case}"); + + for id in [42, 1001] { + let result = current + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![id as f32, 0.]) + .with_limit(1) + .execute() + .await + .unwrap(); + let batches: Vec = result + .new_read_builder() + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); + assert_eq!( + batches[0] + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + id, + "{case}" + ); + } + } +} + +#[tokio::test] +async fn retained_vector_sources_reject_restored_deletions_but_allow_repacking_and_growth() { + use bytes::Bytes; + use indexmap::IndexMap; + use paimon::spec::{DeletionVectorMeta, IndexFileMeta, SchemaChange}; + use paimon::table::CommitMessage; + use roaring::RoaringBitmap; + + // Publish a normal index-only DV transition. Removing this entry restores + // the same physical data, as rollback-to-as-latest does without a data delta. + async fn write_mask(table: &Table, source: &str, positions: &[u32]) -> IndexFileMeta { + let bitmap: RoaringBitmap = positions.iter().copied().collect(); + let mut payload = 1581511376u32.to_be_bytes().to_vec(); + bitmap.serialize_into(&mut payload).unwrap(); + let mut bytes = vec![1]; + bytes.extend_from_slice(&(payload.len() as i32).to_be_bytes()); + bytes.extend_from_slice(&payload); + bytes.extend_from_slice(&crc32fast::hash(&payload).to_be_bytes()); + let file = IndexFileMeta { + index_type: "DELETION_VECTORS".into(), + file_name: format!("index-{}-1", uuid::Uuid::new_v4()), + file_size: bytes.len() as i64, + row_count: 1, + deletion_vectors_ranges: Some(IndexMap::from([( + source.into(), + DeletionVectorMeta { + offset: 1, + length: payload.len() as i32, + cardinality: Some(bitmap.len() as i64), + }, + )])), + external_path: None, + global_index_meta: None, + }; + let path = if table.schema().core_options().index_file_in_data_file_dir() { + format!("{}/bucket-0/{}", table.location(), file.file_name) + } else { + format!("{}/index/{}", table.location(), file.file_name) + }; + table + .file_io() + .mkdirs(path.rsplit_once('/').unwrap().0) + .await + .unwrap(); + table + .file_io() + .new_output(&path) + .unwrap() + .write(Bytes::from(bytes)) + .await + .unwrap(); + file + } + + for change in ["removed", "shrunk", "repacked", "grown"] { + let original = vector_table_for_indexes(&[]).await; + let schema = original + .schema() + .apply_changes(vec![SchemaChange::remove_option( + "pk-vector.index.columns".into(), + )]) + .unwrap(); + persist_table_schema(original.file_io(), original.location(), &schema).await; + let table = original.copy_with_resolved_schema(schema, "main").unwrap(); + write_batch( + &table, + &vector_index_batch( + (0..1000).collect(), + (0..1000).map(|id| Some([id as f32, 0.])).collect(), + ), + ) + .await; + let first = compact(&table).await; + let base = first[0].files()[0].clone(); + assert!(base.level > 1); + assert!(indexes(&table).await.is_empty()); + + let schema = table + .schema() + .apply_changes(vec![SchemaChange::set_option( + "pk-vector.index.columns".into(), + "embedding".into(), + )]) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), &schema).await; + let current = table.copy_with_resolved_schema(schema, "main").unwrap(); + write_batch( + ¤t, + &vector_index_batch(vec![1001], vec![Some([1001., 0.])]), + ) + .await; + let captured_mask = write_mask(¤t, &base.file_name, &[42]).await; + let partition = paimon::spec::BinaryRow::new(0).to_serialized_bytes(); + let mut deletion = CommitMessage::new(partition.clone(), 0, vec![]); + deletion.new_index_files.push(captured_mask.clone()); + current + .new_write_builder() + .new_commit() + .commit(vec![deletion]) + .await + .unwrap(); + + let builder = current.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + assert!( + plan.tasks() + .iter() + .flat_map(CompactTask::files) + .all(|file| file.file_name != base.file_name), + "{change}: source must be retained history" + ); + let results = execute(¤t, &plan).await; + let retained = results + .iter() + .flat_map(|result| result.index_entries().unwrap()) + .find(|entry| { + entry.kind == paimon::spec::FileKind::Add + && entry.index_file.index_type == "ivf-flat" + && PrimaryKeyIndexSourceMeta::from_global_index_meta( + entry.index_file.global_index_meta.as_ref().unwrap(), + ) + .unwrap() + .source_files() + .iter() + .any(|source| source.file_name() == base.file_name) + }) + .unwrap(); + assert_eq!( + retained.index_file.row_count, 999, + "{change}: captured deletion must be omitted" + ); + + let mut concurrent = CommitMessage::new(partition, 0, vec![]); + concurrent.deleted_index_files.push(captured_mask); + let replacement = match change { + "removed" => None, + "shrunk" => Some(vec![43]), + "repacked" => Some(vec![42]), + "grown" => Some(vec![42, 43]), + _ => unreachable!(), + }; + if let Some(positions) = replacement { + concurrent + .new_index_files + .push(write_mask(¤t, &base.file_name, &positions).await); + } + current + .new_write_builder() + .new_commit() + .commit(vec![concurrent]) + .await + .unwrap(); + let before = latest(¤t).await; + let published = builder + .new_commit() + .unwrap() + .commit_compact(results.clone()) + .await; + if matches!(change, "removed" | "shrunk") { + let error = + published.expect_err("a prepared vector payload cannot omit a restored live row"); + assert!( + error.to_string().contains("source deletions were restored"), + "{change}: {error}" + ); + assert_eq!(latest(¤t).await, before, "{change}"); + for result in &results { + for entry in result + .index_entries() + .unwrap() + .iter() + .filter(|entry| entry.kind == paimon::spec::FileKind::Add) + { + assert!( + current + .file_io() + .exists(&index_path(¤t, entry)) + .await + .unwrap(), + "{change}: failed commit owns no prepared files" + ); + } + for file in result.files() { + assert!( + current + .file_io() + .exists(&format!( + "{}/bucket-0/{}", + current.location(), + file.file_name + )) + .await + .unwrap(), + "{change}: prepared data must remain recoverable" + ); + } + } + } else { + published.unwrap(); + assert_eq!(latest(¤t).await.id(), before.id() + 1, "{change}"); + let batches = read(¤t, Some(&["id"])).await; + let ids = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect::>(); + assert!(!ids.contains(&42), "{change}"); + assert_eq!(ids.contains(&43), change == "repacked", "{change}"); + assert!(ids.contains(&1001), "{change}"); + } + } +} + #[tokio::test] async fn vector_output_cannot_be_omitted_and_counts_non_null_vectors() { let table = vector_table_for_indexes(&[]).await; diff --git a/docs/src/compaction.md b/docs/src/compaction.md index deb3558f2..1785346ec 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -190,6 +190,18 @@ retain global payloads. Shard-boundary changes, removed rows and incompatible vector definitions retire affected payloads immediately. Additions and retired references publish together; historical snapshots retain their old files. +Scalar and full-text payload reuse also checks the indexed field's type. New file +names record the actual builder schema ID; this remains valid after its source +snapshot expires and permits current-schema builds over older data. Java opens +the manifest's file name unchanged, so native and manifest formats are unchanged. +Legacy names require a complete schema history with an unchanged field ID and +type. Missing history or incompatible types cause atomic retirement and rebuild; +scalar layout-option changes alone retain compatible runs. Full-text type +changes, such as a shorter VARCHAR width, rebuild tokens from current logical +text. Full-text runs also require an equivalent parsed native configuration; +analyzer changes rebuild coverage, while explicit defaults preserve compatible +leaves. Missing or invalid analyzer metadata is rebuilt conservatively. + Vector training still uses the actual non-NULL count. If a small or sparse run cannot satisfy its native training configuration, such as a size-per-vector budget, maintenance may retry once with complete pending connected groups. This @@ -201,9 +213,13 @@ An empty maintenance plan publishes no snapshot. Empty data commits, index-only commits, and truncation do not trigger a separate round. Indexed-column updates retain `global-index.column-update-action`: use `DROP_PARTITION_INDEX` to allow the data update and automatically rebuild the affected coverage. Changing a -vector algorithm or distance metric retires all incompatible payloads for that -column and publishes their replacements together. Other writer parameters alone -do not rebuild unchanged coverage; changing shard boundaries can regroup it. +vector algorithm, distance metric or dimension retires all incompatible payloads +for that column and publishes their replacements together. Other vector training +parameters alone do not rebuild unchanged coverage; changing shard boundaries +can regroup it. +Source vectors must satisfy a changed dimension. If they do not, maintenance +fails without publishing mixed-dimension payloads; fix the source values and +retry maintenance with the same originating commit identity. Missing coverage is processed synchronously without a minimum-row or time delay; index consolidation follows the run thresholds above. @@ -314,6 +330,12 @@ deletion-vector and search-mode requirements. Each payload covers the complete compacted file set for one partition, bucket, positive level and column. Source files are sorted by name; payload ordinals concatenate their physical rows, preserving NULL and tombstone positions. +Each native build projects its indexed column rather than decoding every value +column. As in Java, vector builds exclude captured DV positions and lookup +deletions discovered by the current compaction, while keeping their physical +positions in the source frame. Selection precedes schema casts; bounded NULL +batches fill ordinal gaps for the shared native writer. Scalar and full-text +builds retain physical rows. Unchanged coverage is retained. Changed coverage and incompatible vector definitions replace the previous references in the same COMPACT snapshot as data, changelogs and deletion vectors. Old payload files remain for historical @@ -324,9 +346,12 @@ Vector payload row counts count indexed vectors; source ranges retain all physic Commit validates the complete source set and captured index membership before publishing. Concurrent L0 appends and unrelated index changes can be rebased; -changes to a maintained unit require a new plan. A failed build cleans its -private outputs. A failed or uncertain commit retains prepared outputs for -retry with the same results and commit user. Older serialized results lacking +changes to a maintained unit require a new plan. +For newly built vector payloads, captured deleted positions must remain deleted +at commit. Concurrent DV additions and container repacking can be rebased; +restoring positions, for example through rollback-as-latest, requires re-execution. +A failed build cleans its private outputs. A failed or uncertain commit retains +prepared outputs for retry with the same results and commit user. Older serialized results lacking index maintenance must be re-executed before first publication when indexes need maintenance; already published results remain recoverable without their original source snapshot. From 1ee7fb34396a9140213ee88004e2bf2181346f71 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 00:44:44 +0800 Subject: [PATCH 09/14] feat(index): unify PK and global index configuration Share CoreOptions declaration and field-JSON resolution between PK and append/Data Evolution indexes, following apache/paimon#10914. Retain legacy PK and Rust aliases alongside DLF singular declarations and flat parameters. Apply effective field options to native construction, compaction, indexed and raw reads, query widths and refinement. Preserve table-mode routing and PK's compaction-only index lifecycle. Share read caches by budget across fields. Cover precedence, insertion order, invalid defaults, schema guards, native build/read consistency, shard overrides and recovery with regression tests. --- crates/paimon/Cargo.toml | 2 +- crates/paimon/src/spec/core_options.rs | 50 +- .../src/spec/core_options/index_options.rs | 675 ++++++++++++++++++ crates/paimon/src/spec/schema.rs | 36 +- crates/paimon/src/table/de_vector_read.rs | 23 +- .../src/table/full_text_search_builder.rs | 22 +- .../table/full_text_search_builder/filter.rs | 15 +- .../table/full_text_search_builder/scan.rs | 3 +- .../paimon/src/table/global_index_options.rs | 240 +++---- .../paimon/src/table/global_index_scanner.rs | 194 ++++- .../table/global_index_scanner/composite.rs | 11 +- .../src/table/global_index_scanner/reader.rs | 39 +- .../src/table/global_index_scanner/tests.rs | 149 ++++ .../paimon/src/table/hybrid_search_builder.rs | 11 +- crates/paimon/src/table/pk_index_options.rs | 88 +-- .../paimon/src/table/pk_sorted_index_scan.rs | 34 +- crates/paimon/src/table/pk_vector_read.rs | 11 +- .../src/table/table_commit/auto_index.rs | 5 +- .../table_commit/auto_index_tests/search.rs | 114 +++ crates/paimon/src/table/table_scan.rs | 12 +- crates/paimon/src/table/vector_read.rs | 2 +- crates/paimon/src/table/vector_scan.rs | 4 +- .../paimon/src/table/vector_search_common.rs | 99 ++- crates/paimon/src/vindex/mod.rs | 14 + crates/paimon/tests/pk_compact/indexes.rs | 283 ++++++-- docs/src/compaction.md | 76 +- 26 files changed, 1826 insertions(+), 386 deletions(-) create mode 100644 crates/paimon/src/spec/core_options/index_options.rs diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index 3d11744c3..2d0e42260 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -110,7 +110,7 @@ opendal-service-oss = { version = "0.58.0", optional = true } opendal-service-s3 = { version = "0.58.0", optional = true } pretty_assertions = "1" apache-avro = { version = "0.21", features = ["snappy", "zstandard", "bzip", "xz"] } -indexmap = "2.5.0" +indexmap = { version = "2.5.0", features = ["serde"] } lru = "0.18.2" roaring = "0.11" datasketches = "0.2" diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index 9a0135e8e..390adeac6 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -19,6 +19,8 @@ use std::collections::{HashMap, HashSet}; use crate::spec::TableType; +mod index_options; + const DELETION_VECTORS_ENABLED_OPTION: &str = "deletion-vectors.enabled"; const DELETION_VECTORS_MERGE_ON_READ_OPTION: &str = "deletion-vectors.merge-on-read"; pub(crate) const QUERY_AUTH_ENABLED_OPTION: &str = "query-auth.enabled"; @@ -191,8 +193,6 @@ pub(crate) const BLOB_DESCRIPTOR_FIELD_FALLBACK: &str = "blob.stored-descriptor- pub(crate) const BLOB_VIEW_FIELD_OPTION: &str = "blob-view-field"; pub(crate) const VIDEO_FRAME_FIELD_OPTION: &str = "video-frame-field"; pub const BLOB_VIEW_RESOLVE_ENABLED_OPTION: &str = "blob-view.resolve.enabled"; -const PK_VECTOR_INDEX_COLUMNS_OPTION: &str = "pk-vector.index.columns"; -const PK_FULL_TEXT_INDEX_COLUMNS_OPTION: &str = "pk-full-text.index.columns"; /// Merge engine for primary-key tables. /// @@ -2169,21 +2169,21 @@ impl<'a> CoreOptions<'a> { .unwrap_or_default() } - /// True when the PK-vector index column option key is present (regardless of value). + /// Whether a shared or legacy vector declaration is present. pub fn primary_key_vector_index_enabled(&self) -> bool { - self.options.contains_key(PK_VECTOR_INDEX_COLUMNS_OPTION) + !self.vector_index_columns().is_empty() } /// The configured PK-vector index columns, split on ',' and trimmed. Blank tokens /// are preserved so schema validation can reject malformed column lists. pub fn primary_key_vector_index_columns(&self) -> crate::Result> { - let raw = self - .options - .get(PK_VECTOR_INDEX_COLUMNS_OPTION) - .ok_or_else(|| crate::Error::ConfigInvalid { + let columns = self.vector_index_columns(); + if columns.is_empty() { + return Err(crate::Error::ConfigInvalid { message: "pk-vector.index.columns is not set".to_string(), - })?; - Ok(raw.split(',').map(|c| c.trim().to_string()).collect()) + }); + } + Ok(columns) } /// The single PK-vector index column. The first release supports exactly one. @@ -2203,10 +2203,7 @@ impl<'a> CoreOptions<'a> { /// The index type for a PK-vector column. Required — planning and the index reader /// both need it, so an absent value is a hard error rather than a guessed default. pub fn primary_key_vector_index_type(&self, col: &str) -> crate::Result { - self.options - .get(&format!("fields.{col}.pk-vector.index.type")) - .map(|v| v.trim().to_string()) - .filter(|v| !v.is_empty()) + self.vector_index_type(col)? .ok_or_else(|| crate::Error::ConfigInvalid { message: format!("fields.{col}.pk-vector.index.type is required but not set"), }) @@ -2215,6 +2212,22 @@ impl<'a> CoreOptions<'a> { /// The distance metric name for a PK-vector column, defaulting to inner_product. /// Validated against the supported metrics; an unknown value is a hard error. pub fn primary_key_vector_distance_metric(&self, col: &str) -> crate::Result { + if let Some(kind) = self.vector_index_type(col)? { + return self + .vector_index_options(&kind, col, true)? + .get(&format!("{kind}.metric")) + .cloned() + .ok_or_else(|| crate::Error::ConfigInvalid { + message: format!("Unsupported vector index type: {kind}"), + }); + } + self.legacy_primary_key_vector_distance_metric(col) + } + + pub(crate) fn legacy_primary_key_vector_distance_metric( + &self, + col: &str, + ) -> crate::Result { let raw = self .options .get(&format!("fields.{col}.pk-vector.distance.metric")) @@ -2227,7 +2240,7 @@ impl<'a> CoreOptions<'a> { /// True when the PK full-text index column option key is present (any value). pub fn primary_key_full_text_index_enabled(&self) -> bool { - self.options.contains_key(PK_FULL_TEXT_INDEX_COLUMNS_OPTION) + !self.primary_key_full_text_index_columns().is_empty() } /// Configured PK full-text index columns: split on ',' and trim each token, @@ -2236,10 +2249,7 @@ impl<'a> CoreOptions<'a> { /// absent. Never returns a `Result`, never errors (do not copy the fail-loud /// shape of `primary_key_vector_index_columns`). pub fn primary_key_full_text_index_columns(&self) -> Vec { - match self.options.get(PK_FULL_TEXT_INDEX_COLUMNS_OPTION) { - None => Vec::new(), - Some(raw) => raw.split(',').map(|c| c.trim().to_string()).collect(), - } + self.global_index_columns("full-text") } } @@ -4343,7 +4353,7 @@ mod tests { ); assert_eq!( co.primary_key_vector_distance_metric("embedding").unwrap(), - "Inner-Product" + "inner_product" ); } diff --git a/crates/paimon/src/spec/core_options/index_options.rs b/crates/paimon/src/spec/core_options/index_options.rs new file mode 100644 index 000000000..4fba8e8bf --- /dev/null +++ b/crates/paimon/src/spec/core_options/index_options.rs @@ -0,0 +1,675 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Configuration contracts shared by PK and Data Evolution index consumers. + +use super::CoreOptions; +use crate::vindex::pkvector::metric::normalize_metric as normalize_metric_name; +use crate::{Error, Result}; +use std::collections::HashMap; + +fn normalize_metric(value: &str) -> String { + normalize_metric_name(value.trim()) +} + +fn vector(kind: &str) -> bool { + !matches!(kind, "btree" | "bitmap" | "multivalue" | "fm" | "full-text") +} + +fn columns(raw: Option<&String>) -> Vec { + raw.map(|raw| raw.split(',').map(|s| s.trim().to_string()).collect()) + .unwrap_or_default() +} + +fn invalid(message: impl Into) -> Error { + Error::ConfigInvalid { + message: message.into(), + } +} + +impl CoreOptions<'_> { + /// Declared implementation identifiers, in deterministic order. + pub fn global_index_types(&self) -> Vec { + let mut types = self + .options + .keys() + .filter_map(|key| { + key.strip_prefix("global-index.") + .and_then(|s| { + s.strip_suffix(".index-columns") + .or_else(|| s.strip_suffix(".index-column")) + }) + .map(str::to_string) + }) + .collect::>(); + types.sort(); + types.dedup(); + types + } + + /// Canonical plural, singular DLF alias, then the legacy PK declaration. + /// Preserve empty tokens so validation cannot silently disable malformed declarations. + pub fn global_index_columns(&self, kind: &str) -> Vec { + for key in [ + format!("global-index.{kind}.index-columns"), + format!("global-index.{kind}.index-column"), + format!("pk-{kind}.index.columns"), + ] { + if let Some(raw) = self.options.get(&key) { + return columns(Some(raw)); + } + } + if vector(kind) && kind != "vector" { + return columns(self.options.get("pk-vector.index.columns")) + .into_iter() + .filter(|col| { + self.options + .get(&format!("fields.{col}.pk-vector.index.type")) + .is_some_and(|v| v.trim() == kind) + }) + .collect(); + } + Vec::new() + } + + pub fn vector_index_columns(&self) -> Vec { + let types = self + .global_index_types() + .into_iter() + .filter(|t| vector(t) && t != "vector") + .collect::>(); + if types.is_empty() { + self.global_index_columns("vector") + } else { + types + .iter() + .flat_map(|kind| self.global_index_columns(kind)) + .collect() + } + } + + /// Concrete declarations select the implementation; retain old PK and Rust + /// field declarations and the flat implementation aliases used by DLF. + pub fn vector_index_type(&self, column: &str) -> Result> { + let declared = self + .global_index_types() + .into_iter() + .filter(|kind| { + vector(kind) + && kind != "vector" + && self.global_index_columns(kind).iter().any(|c| c == column) + }) + .collect::>(); + if declared.len() > 1 { + return Err(invalid(format!( + "Multiple vector index types declared for '{column}'" + ))); + } + if let Some(kind) = declared.into_iter().next() { + return Ok(Some(kind)); + } + Ok([ + format!("fields.{column}.global-index.vector.index-type"), + format!("fields.{column}.pk-vector.index.type"), + "index_type".into(), + "index-type".into(), + "vector.index-type".into(), + format!("fields.{column}.index-type"), + ] + .iter() + .find_map(|key| self.options.get(key)) + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty())) + } + + /// Resolve implementation defaults and one field's JSON without mutating + /// table options. Metric aliases are resolved separately as one parameter. + pub fn global_index_options( + &self, + kind: &str, + column: &str, + ) -> Result> { + let is_vector = vector(kind); + let family = if is_vector { "vector" } else { kind }; + let prefix = if is_vector || kind == "full-text" { + format!("{kind}.") + } else { + format!("{kind}-index.") + }; + let field = format!("fields.{column}."); + let shared_key = format!("{field}index-options"); + let rust_key = format!("{field}global-index.{family}.index-options"); + let legacy_key = format!("{field}pk-{family}.index.options"); + let shared = self.options.contains_key(&shared_key); + let key = if shared { + &shared_key + } else if self.options.contains_key(&rust_key) { + &rust_key + } else { + &legacy_key + }; + let mut resolved = self.options.clone(); + if let Some(value) = self + .options + .get(&format!("{field}global-index.row-count-per-shard")) + { + resolved.insert("global-index.row-count-per-shard".into(), value.clone()); + } + if let Some(raw) = self.options.get(key).filter(|s| !s.trim().is_empty()) { + let parsed: indexmap::IndexMap = serde_json::from_str(raw) + .map_err(|_| { + invalid(format!( + "{key} must be a JSON object of option key-value pairs" + )) + })?; + for (name, value) in parsed { + if name.trim().is_empty() { + return Err(invalid(format!("{key} contains an empty option key"))); + } + let value = match value { + serde_json::Value::Null => { + return Err(invalid(format!( + "{key} contains an invalid option value for {name}" + ))) + } + serde_json::Value::String(value) => value, + serde_json::Value::Array(values) if shared && kind == "full-text" => values + .into_iter() + .map(|v| { + v.as_str() + .map(str::to_string) + .unwrap_or_else(|| v.to_string()) + }) + .collect::>() + .join(";"), + value if shared || value.is_number() || value.is_boolean() => value.to_string(), + _ => { + return Err(invalid(format!( + "{key} contains an invalid option value for {name}" + ))) + } + }; + let qualified = if name.starts_with(&prefix) + || name.starts_with("fields.") + || name.starts_with("global-index.") + || name.starts_with("sorted-index.") + || (is_vector + && matches!(name.as_str(), "index.type" | "vindex.build.granule.enabled")) + { + name + } else { + format!("{prefix}{name}") + }; + if !shared + && !qualified.starts_with("global-index.") + && resolved + .get(&qualified) + .is_some_and(|previous| previous != &value) + { + return Err(invalid(format!( + "Conflicting field index options for {column}: {qualified}" + ))); + } + resolved.insert(qualified.clone(), value.clone()); + if shared && is_vector { + if let Some(short) = qualified.strip_prefix(&prefix) { + resolved.insert(format!("{field}{short}"), value); + } + } + } + } + Ok(resolved) + } + + /// Merge vector field parameters before normalizing metric aliases, like + /// Java VectorIndexOptions. Readers can retain an unspecified metric. + pub fn vector_index_options( + &self, + kind: &str, + column: &str, + default_metric: bool, + ) -> Result> { + if !vector(kind) { + return Err(invalid(format!("Unsupported vector index type: {kind}"))); + } + let field = format!("fields.{column}."); + let prefix = format!("{kind}."); + let aliases = [ + format!("{field}distance.metric"), + format!("{field}metric"), + format!("{prefix}distance.metric"), + format!("{prefix}metric"), + ]; + let mut metric = [ + format!("{field}distance.metric"), + format!("{field}metric"), + format!("{field}global-index.vector.distance-metric"), + format!("{field}pk-vector.distance.metric"), + format!("{prefix}distance.metric"), + format!("{prefix}metric"), + "distance.metric".into(), + "metric".into(), + ] + .iter() + .find_map(|key| self.options.get(key).cloned()); + let mut defaults = self.options.clone(); + for key in &aliases { + defaults.remove(key); + } + let mut resolved = CoreOptions::new(&defaults).global_index_options(kind, column)?; + metric = aliases + .iter() + .find_map(|key| resolved.get(key).cloned()) + .or(metric); + // Preserve the former Rust family declaration's metric/default priority. + let legacy_rust = !self.options.contains_key(&format!("{field}index-options")) + && (self + .options + .contains_key("global-index.vector.index-columns") + || self + .options + .contains_key("global-index.vector.index-column")) + && !self + .global_index_types() + .iter() + .any(|kind| vector(kind) && kind != "vector"); + if legacy_rust && default_metric { + metric = Some( + self.options + .get(&format!("{field}global-index.vector.distance-metric")) + .cloned() + .unwrap_or_else(|| "inner_product".into()), + ); + } + if metric.is_some() || default_metric { + let metric = normalize_metric(metric.as_deref().unwrap_or("inner_product")); + crate::vindex::pkvector::metric::VectorSearchMetric::parse(&metric)?; + for key in aliases { + resolved.insert(key, metric.clone()); + } + } + Ok(resolved) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn map(values: &[(&str, &str)]) -> HashMap { + values + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn shared_schema_guards_apply_to_pk_but_allow_append_and_de_type_evolution() { + use crate::spec::{BigIntType, DataType, IntType, Schema, SchemaChange, TableSchema}; + for suffix in ["index-columns", "index-column"] { + for mode in ["pk", "append", "de"] { + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option(format!("global-index.btree.{suffix}"), "value"); + if mode == "pk" { + schema = schema + .primary_key(["id"]) + .option("bucket", "1") + .option("deletion-vectors.enabled", "true"); + } else { + schema = schema + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", (mode == "de").to_string()); + } + let schema = TableSchema::new(0, &schema.build().unwrap()); + let change = SchemaChange::update_column_type( + "value".into(), + DataType::BigInt(BigIntType::new()), + ); + let result = schema.apply_changes(vec![change]); + if mode == "pk" { + assert!(result + .unwrap_err() + .to_string() + .contains("primary-key index column")); + } else { + result.unwrap(); + } + } + } + } + + #[test] + fn dlf_dimension_aliases_have_stable_field_precedence() { + use crate::spec::{ArrayType, DataField, DataType, FloatType}; + let field = DataField::new( + 1, + "embedding".into(), + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ); + let values = map(&[ + ("ivf-flat.dimension", "4"), + ("ivf-flat.index.dimension", "5"), + ("fields.embedding.dimension", "2"), + ("fields.embedding.index.dimension", "3"), + ]); + let native = crate::vindex::VindexVectorIndexOptions::new( + &values, + &HashMap::new(), + "ivf-flat", + &field, + ) + .unwrap(); + assert_eq!(native.dimension(), 2); + let values = map(&[ + ("ivf-flat.dimension", "4"), + ("ivf-flat.index.dimension", "5"), + ]); + let native = crate::vindex::VindexVectorIndexOptions::new( + &values, + &HashMap::new(), + "ivf-flat", + &field, + ) + .unwrap(); + assert_eq!(native.dimension(), 4); + let values = map(&[ + ("ivf-flat.index.dimension", "5"), + ("fields.embedding.index.dimension", "3"), + ]); + let native = crate::vindex::VindexVectorIndexOptions::new( + &values, + &HashMap::new(), + "ivf-flat", + &field, + ) + .unwrap(); + assert_eq!(native.dimension(), 3); + } + + #[test] + fn declaration_precedence_and_legacy_vector_selection() { + for kind in [ + "btree", + "bitmap", + "multivalue", + "fm", + "full-text", + "ivf-flat", + ] { + let family = if vector(kind) { "vector" } else { kind }; + let mut values = map(&[ + (&format!("pk-{family}.index.columns"), "old"), + ("fields.old.pk-vector.index.type", "ivf-flat"), + ]); + assert_eq!( + CoreOptions::new(&values).global_index_columns(kind), + vec!["old"] + ); + values.insert( + format!("global-index.{kind}.index-column"), + "singular".into(), + ); + assert_eq!( + CoreOptions::new(&values).global_index_columns(kind), + vec!["singular"] + ); + values.insert( + format!("global-index.{kind}.index-columns"), + " first, second ".into(), + ); + assert_eq!( + CoreOptions::new(&values).global_index_columns(kind), + vec!["first", "second"] + ); + if vector(kind) { + let core = CoreOptions::new(&values); + assert_eq!( + core.primary_key_vector_index_columns().unwrap(), + vec!["first", "second"] + ); + assert_eq!(core.primary_key_vector_index_type("first").unwrap(), kind); + } + values.insert(format!("global-index.{kind}.index-columns"), "".into()); + assert_eq!( + CoreOptions::new(&values).global_index_columns(kind), + vec![""] + ); + } + } + + #[test] + fn scalar_algorithm_in_vector_declaration_is_an_error() { + for kind in ["btree", "bitmap", "multivalue", "fm", "full-text"] { + let values = map(&[ + ("pk-vector.index.columns", "embedding"), + ("fields.embedding.pk-vector.index.type", kind), + ]); + assert!(CoreOptions::new(&values) + .primary_key_vector_distance_metric("embedding") + .unwrap_err() + .to_string() + .contains("Unsupported vector index type")); + } + } + + #[test] + fn shared_json_aliases_follow_java_insertion_order() { + for (json, expected) in [ + ( + r#"{"block-size":"4 kb","btree-index.block-size":"8 kb"}"#, + "8 kb", + ), + ( + r#"{"btree-index.block-size":"8 kb","block-size":"4 kb"}"#, + "4 kb", + ), + ] { + let values = map(&[("fields.amount.index-options", json)]); + assert_eq!( + CoreOptions::new(&values) + .global_index_options("btree", "amount") + .unwrap()["btree-index.block-size"], + expected + ); + } + } + + #[test] + fn vector_json_alias_priority_is_independent_of_json_key_order() { + for suffix in ["index-options", "pk-vector.index.options"] { + for json in [ + r#"{"metric":"L2","distance.metric":"COSINE"}"#, + r#"{"distance.metric":"COSINE","metric":"L2"}"#, + ] { + let values = map(&[ + ("fields.embedding.pk-vector.distance.metric", "l2"), + (&format!("fields.embedding.{suffix}"), json), + ]); + let resolved = CoreOptions::new(&values) + .vector_index_options("ivf-pq", "embedding", true) + .unwrap(); + for key in [ + "fields.embedding.distance.metric", + "fields.embedding.metric", + "ivf-pq.distance.metric", + "ivf-pq.metric", + ] { + assert_eq!(resolved[key], "cosine"); + } + assert_eq!( + CoreOptions::new(&resolved) + .vector_index_options("ivf-pq", "embedding", true) + .unwrap(), + resolved + ); + } + } + } + + #[test] + fn shared_field_json_overrides_defaults_and_replaces_invalid_legacy_json() { + for (kind, prefix, key) in [ + ("btree", "btree-index.", "block-size"), + ("bitmap", "bitmap-index.", "dictionary-block-size"), + ("multivalue", "multivalue-index.", "dictionary-block-size"), + ("fm", "fm-index.", "sa-sample-rate"), + ("full-text", "full-text.", "tokenizer"), + ("ivf-pq", "ivf-pq.", "nlist"), + ] { + let values = map(&[ + (&format!("{prefix}{key}"), "32"), + ( + "fields.payload.index-options", + &format!(r#"{{"{key}":"64"}}"#), + ), + ( + &format!( + "fields.payload.pk-{}.index.options", + if vector(kind) { "vector" } else { kind } + ), + "invalid", + ), + ("sorted-index.records-per-file", "20"), + ]); + let resolved = CoreOptions::new(&values) + .global_index_options(kind, "payload") + .unwrap(); + assert_eq!(resolved[&format!("{prefix}{key}")], "64"); + assert_eq!(resolved["sorted-index.records-per-file"], "20"); + assert_eq!( + CoreOptions::new(&resolved) + .global_index_options(kind, "payload") + .unwrap(), + resolved + ); + assert_eq!(values[&format!("{prefix}{key}")], "32"); + } + } + + #[test] + fn dlf_flat_vector_parameters_and_shared_json_keep_metric_aliases_consistent() { + let mut values = map(&[ + ("global-index.ivf-pq.index-column", "embedding"), + ("ivf-pq.dimension", "768"), + ("ivf-pq.distance.metric", "l2"), + ("ivf-pq.nlist", "32"), + ("fields.embedding.nlist", "48"), + ("fields.embedding.distance.metric", "cosine"), + ]); + assert_eq!( + CoreOptions::new(&values) + .primary_key_vector_distance_metric("embedding") + .unwrap(), + "cosine" + ); + values.insert( + "fields.embedding.index-options".into(), + r#"{"nlist":64,"distance.metric":"Inner-Product"}"#.into(), + ); + let resolved = CoreOptions::new(&values) + .vector_index_options("ivf-pq", "embedding", true) + .unwrap(); + for key in [ + "ivf-pq.metric", + "ivf-pq.distance.metric", + "fields.embedding.metric", + "fields.embedding.distance.metric", + ] { + assert_eq!(resolved[key], "inner_product"); + } + assert_eq!(resolved["ivf-pq.dimension"], "768"); + assert_eq!(resolved["ivf-pq.nlist"], "64"); + assert_eq!(resolved["fields.embedding.nlist"], "64"); + assert_eq!( + CoreOptions::new(&values) + .primary_key_vector_distance_metric("embedding") + .unwrap(), + "inner_product" + ); + } + + #[test] + fn full_text_arrays_and_boolean_values_use_native_encoding() { + let values = map(&[( + "fields.text.index-options", + r#"{"lower-case":false,"stop-words":["paimon","lake"]}"#, + )]); + let resolved = CoreOptions::new(&values) + .global_index_options("full-text", "text") + .unwrap(); + assert_eq!(resolved["full-text.lower-case"], "false"); + assert_eq!(resolved["full-text.stop-words"], "paimon;lake"); + } + + #[test] + fn malformed_shared_json_is_not_hidden_by_valid_legacy_values() { + for raw in ["[]", r#"{"nlist":null}"#, r#"{"":64}"#, "invalid"] { + let values = map(&[ + ("fields.embedding.index-options", raw), + ("fields.embedding.pk-vector.index.options", r#"{"nlist":1}"#), + ]); + assert!(CoreOptions::new(&values) + .vector_index_options("ivf-flat", "embedding", true) + .unwrap_err() + .to_string() + .contains("fields.embedding.index-options")); + } + let values = map(&[("fields.embedding.index-options", r#"{"nlist":64}"#)]); + let resolved = CoreOptions::new(&values) + .vector_index_options("ivf-pq", "embedding", false) + .unwrap(); + assert!(!resolved.contains_key("ivf-pq.metric")); + } + + #[test] + fn legacy_json_conflicts_and_rust_de_aliases_remain_supported() { + let mut values = map(&[ + ("btree-index.block-size", "4 kb"), + ( + "fields.amount.pk-btree.index.options", + r#"{"block-size":"8 kb"}"#, + ), + ]); + assert!(CoreOptions::new(&values) + .global_index_options("btree", "amount") + .is_err()); + values = map(&[ + ("global-index.vector.index-columns", "embedding"), + ( + "fields.embedding.global-index.vector.index-type", + "ivf-flat", + ), + ("fields.embedding.global-index.vector.distance-metric", "l2"), + ( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":2,"nlist":1}"#, + ), + ]); + let core = CoreOptions::new(&values); + assert_eq!( + core.primary_key_vector_index_type("embedding").unwrap(), + "ivf-flat" + ); + let resolved = core + .vector_index_options("ivf-flat", "embedding", true) + .unwrap(); + assert_eq!(resolved["ivf-flat.dimension"], "2"); + assert_eq!(resolved["ivf-flat.metric"], "l2"); + } +} diff --git a/crates/paimon/src/spec/schema.rs b/crates/paimon/src/spec/schema.rs index 91f5b41eb..e9576a2d7 100644 --- a/crates/paimon/src/spec/schema.rs +++ b/crates/paimon/src/spec/schema.rs @@ -479,7 +479,7 @@ impl TableSchema { message: format!("Cannot rename partition column: [{name}]"), }); } - assert_not_updating_primary_key_index_column(&self.options, name, "rename")?; + assert_not_updating_primary_key_index_column(self, name, "rename")?; let idx = field_index(&fields, name).ok_or_else(|| crate::Error::ColumnNotExist { full_name: full_name.to_string(), @@ -533,7 +533,7 @@ impl TableSchema { ), }); } - assert_not_updating_primary_key_index_column(&self.options, name, "drop")?; + assert_not_updating_primary_key_index_column(self, name, "drop")?; assert_not_updating_bucket_key_column( &old_bucket_key_field_ids, &fields[idx], @@ -621,11 +621,7 @@ impl TableSchema { message: "Cannot update primary key".to_string(), }); } - assert_not_updating_primary_key_index_column( - &self.options, - name, - "update type of", - )?; + assert_not_updating_primary_key_index_column(self, name, "update type of")?; let idx = field_index(&fields, name).ok_or_else(|| crate::Error::ColumnNotExist { full_name: full_name.to_string(), @@ -864,11 +860,15 @@ fn assert_nullability_change( /// name and type. Mirrors Java /// `SchemaManager.assertNotUpdatingPrimaryKeyIndexColumn`. fn assert_not_updating_primary_key_index_column( - options: &HashMap, + schema: &TableSchema, field_name: &str, operation: &str, ) -> crate::Result<()> { + let options = schema.options(); let core_options = CoreOptions::new(options); + if schema.primary_keys().is_empty() || core_options.data_evolution_enabled() { + return Ok(()); + } let is_vector_index_column = core_options.primary_key_vector_index_enabled() && core_options .primary_key_vector_index_columns()? @@ -882,9 +882,10 @@ fn assert_not_updating_primary_key_index_column( let is_scalar_index_column = ["btree", "bitmap", "multivalue", "fm"] .iter() .any(|family| { - options - .get(&format!("pk-{family}.index.columns")) - .is_some_and(|columns| columns.split(',').any(|column| column.trim() == field_name)) + core_options + .global_index_columns(family) + .iter() + .any(|column| column == field_name) }); if is_vector_index_column || is_full_text_index_column || is_scalar_index_column { return Err(crate::Error::Unsupported { @@ -1433,8 +1434,15 @@ impl Schema { if !core_options.is_format_table() { core_options.target_file_row_num()?; } - Self::validate_primary_key_vector_index(fields, primary_keys, options)?; - Self::validate_primary_key_full_text_index(fields, primary_keys, options)?; + if !core_options.data_evolution_enabled() + && (!primary_keys.is_empty() + || options + .keys() + .any(|key| key.starts_with("pk-") && key.ends_with(".index.columns"))) + { + Self::validate_primary_key_vector_index(fields, primary_keys, options)?; + Self::validate_primary_key_full_text_index(fields, primary_keys, options)?; + } Ok(()) } @@ -2327,7 +2335,7 @@ impl Schema { }); } - core.primary_key_vector_distance_metric(&column) + core.legacy_primary_key_vector_distance_metric(&column) .map(|_| ()) .map_err(Self::options_error_to_config_invalid) } diff --git a/crates/paimon/src/table/de_vector_read.rs b/crates/paimon/src/table/de_vector_read.rs index cdfa9a33b..53ff69ff0 100644 --- a/crates/paimon/src/table/de_vector_read.rs +++ b/crates/paimon/src/table/de_vector_read.rs @@ -323,8 +323,12 @@ async fn execute_vector_index_work( let index_meta_bytes = global_meta.index_meta.clone().unwrap_or_default(); let row_range_start = global_meta.row_range_start; let vector_searches = work.queries; - let mut options = evaluation.table_options.clone(); - options.extend(vector_searches[0].options.clone()); + let options = super::vector_search_common::vector_read_options( + evaluation.table_options, + &vector_searches[0].options, + &vector_searches[0].field_name, + &work.entry.index_file.index_type, + )?; let input = evaluation.file_io.new_input(&path); if vector_searches.iter().all(|search| { search @@ -574,7 +578,6 @@ async fn scalar_matched_row_ids( vector_ranges: &[RowRange], filter: &Predicate, ) -> crate::Result> { - let core = CoreOptions::new(evaluation.table_options); // Java search pre-filters use per-column coverage and exclude multi-field // definitions, even though ordinary table scans can use composite indexes. let scalar_entries: Vec<_> = entries @@ -588,6 +591,12 @@ async fn scalar_matched_row_ids( }) .cloned() .collect(); + let resolved = GlobalIndexScanner::read_defaults( + evaluation.table_options, + &scalar_entries, + evaluation.schema_fields, + )?; + let core = CoreOptions::new(&resolved); let scanner = GlobalIndexScanner::create_with_fm_options( evaluation.file_io, evaluation.table_path, @@ -601,7 +610,7 @@ async fn scalar_matched_row_ids( .iter() .any(|entry| entry.index_file.index_type.eq_ignore_ascii_case("fm")) { - crate::fm_index::FMOptions::from_options(evaluation.table_options)?.read + crate::fm_index::FMOptions::from_options(&resolved)?.read } else { crate::fm_index::FMReadOptions::default() }, @@ -609,6 +618,7 @@ async fn scalar_matched_row_ids( let Some(scanner) = scanner else { return Ok(None); }; + let scanner = scanner.with_field_options(evaluation.table_options, &scalar_entries)?; let Some((ranges, exact)) = scanner.matching_ranges_with_exactness(filter).await? else { return Ok(None); }; @@ -1750,8 +1760,9 @@ fn vector_metric_override( ) -> crate::Result> { // A known index family determines its own defaults and option namespace. // Only a search without a family may infer a metric from suffixes. - if index_type.is_some() { - direct_raw_vector_metric_override(options, field_name, index_type) + if let Some(kind) = index_type { + let resolved = CoreOptions::new(options).vector_index_options(kind, field_name, false)?; + direct_raw_vector_metric_override(&resolved, field_name, Some(kind)) } else { configured_raw_vector_metric_override(options, field_name) } diff --git a/crates/paimon/src/table/full_text_search_builder.rs b/crates/paimon/src/table/full_text_search_builder.rs index c02cc4122..c6fce865c 100644 --- a/crates/paimon/src/table/full_text_search_builder.rs +++ b/crates/paimon/src/table/full_text_search_builder.rs @@ -216,7 +216,11 @@ impl<'a> FullTextSearchBuilder<'a> { message: "Text column must be set via with_text_column()".to_string(), })?; - if !resolves_to_pk_full_text_path(&core, text_column) { + if !resolves_to_pk_full_text_path( + &core, + text_column, + !self.table.schema().primary_keys().is_empty(), + ) { return Err(crate::Error::Unsupported { message: "materialized full-text read (execute_read) is only supported on the \ primary-key full-text path (data-evolution disabled and the column \ @@ -240,8 +244,13 @@ impl<'a> FullTextSearchBuilder<'a> { /// data-evolution is DISABLED and the column is a configured /// `pk-full-text.index.columns` entry (membership via the non-erroring accessor so /// a malformed config cannot abort an unrelated append/DE query). -fn resolves_to_pk_full_text_path(core: &CoreOptions<'_>, text_column: &str) -> bool { - !core.data_evolution_enabled() +fn resolves_to_pk_full_text_path( + core: &CoreOptions<'_>, + text_column: &str, + primary_key: bool, +) -> bool { + primary_key + && !core.data_evolution_enabled() && core .primary_key_full_text_index_columns() .iter() @@ -1899,15 +1908,16 @@ mod tests { ]); let core = CoreOptions::new(&de_on); assert!( - !resolves_to_pk_full_text_path(&core, "body"), + !resolves_to_pk_full_text_path(&core, "body", true), "data-evolution on must not resolve to the PK full-text path" ); // Off + configured column -> PK path; a non-configured column -> not PK. let de_off = HashMap::from([("pk-full-text.index.columns".to_string(), "body".to_string())]); let core_off = CoreOptions::new(&de_off); - assert!(resolves_to_pk_full_text_path(&core_off, "body")); - assert!(!resolves_to_pk_full_text_path(&core_off, "other")); + assert!(resolves_to_pk_full_text_path(&core_off, "body", true)); + assert!(!resolves_to_pk_full_text_path(&core_off, "other", true)); + assert!(!resolves_to_pk_full_text_path(&core_off, "body", false)); } // A non-positive limit must fail loud at construction, even on an empty table diff --git a/crates/paimon/src/table/full_text_search_builder/filter.rs b/crates/paimon/src/table/full_text_search_builder/filter.rs index bfca94b43..2d0dbd0f9 100644 --- a/crates/paimon/src/table/full_text_search_builder/filter.rs +++ b/crates/paimon/src/table/full_text_search_builder/filter.rs @@ -169,6 +169,12 @@ pub(super) async fn matching_indexed_rows( .is_some_and(|meta| meta.extra_field_ids.as_ref().is_none_or(Vec::is_empty)) }) .collect::>(); + let resolved = GlobalIndexScanner::read_defaults( + evaluation.table_options, + &scalar_entries, + evaluation.schema_fields, + )?; + let core = CoreOptions::new(&resolved); let scanner = GlobalIndexScanner::create_with_fm_options( evaluation.file_io, evaluation.table_path, @@ -182,13 +188,18 @@ pub(super) async fn matching_indexed_rows( .iter() .any(|entry| entry.index_file.index_type.eq_ignore_ascii_case("fm")) { - crate::fm_index::FMOptions::from_options(evaluation.table_options)?.read + crate::fm_index::FMOptions::from_options(&resolved)?.read } else { crate::fm_index::FMReadOptions::default() }, )?; let answer = match scanner { - Some(scanner) => scanner.matching_ranges_with_exactness(filter).await?, + Some(scanner) => { + scanner + .with_field_options(evaluation.table_options, &scalar_entries)? + .matching_ranges_with_exactness(filter) + .await? + } None => None, }; match answer { diff --git a/crates/paimon/src/table/full_text_search_builder/scan.rs b/crates/paimon/src/table/full_text_search_builder/scan.rs index a03f86a7b..6d1fde5d5 100644 --- a/crates/paimon/src/table/full_text_search_builder/scan.rs +++ b/crates/paimon/src/table/full_text_search_builder/scan.rs @@ -80,7 +80,8 @@ impl PlanContext { } } let primary_key_field_id = - resolves_to_pk_full_text_path(&core, column).then_some(field.id()); + resolves_to_pk_full_text_path(&core, column, !table.schema().primary_keys().is_empty()) + .then_some(field.id()); if primary_key_field_id.is_some() && config.include_row_ids.is_some() { return Err(crate::Error::Unsupported { message: diff --git a/crates/paimon/src/table/global_index_options.rs b/crates/paimon/src/table/global_index_options.rs index 9dbe0a90c..ad7c58529 100644 --- a/crates/paimon/src/table/global_index_options.rs +++ b/crates/paimon/src/table/global_index_options.rs @@ -17,7 +17,7 @@ //! Shared field declarations and native options for global-index maintenance and raw reads. -use crate::table::global_index_build_common::{resolve_index_fields, IndexColumns}; +use crate::spec::CoreOptions; use crate::table::global_index_types::normalize_global_index_type; use crate::table::{GlobalIndexBuildBuilder, Table}; use crate::Result; @@ -40,46 +40,87 @@ impl Definition { } } -pub(super) fn definitions(table: &Table) -> Result> { - let options = table.schema().options(); - let mut owners = HashMap::new(); +/// Ownership is shared by maintenance and both table modes' read paths. +pub(super) fn declarations(options: &HashMap) -> Result> { + declarations_impl(options, true) +} + +/// Read planning can encounter unresolved legacy schema properties. Ignore +/// blank/unmatched declarations, while still rejecting competing column owners. +pub(super) fn declarations_for_read( + options: &HashMap, +) -> Result> { + declarations_impl(options, false) +} + +fn declarations_impl( + options: &HashMap, + strict: bool, +) -> Result> { + let core = CoreOptions::new(options); + for kind in core.global_index_types() { + if strict && kind != "vector" && normalize_global_index_type(&kind).is_none() { + return Err(crate::Error::ConfigInvalid { + message: format!("Unsupported global index type: {kind}"), + }); + } + } + let mut result = Vec::new(); + let mut owners = HashSet::new(); for family in ["btree", "bitmap", "multivalue", "fm", "full-text", "vector"] { - let key = format!("global-index.{family}.index-columns"); - let Some(columns) = options.get(&key) else { - continue; + let columns = if family == "vector" { + core.vector_index_columns() + } else { + core.global_index_columns(family) }; let mut unique = HashSet::new(); - for column in columns.split(',').map(str::trim) { - if column.is_empty() || !unique.insert(column) { - return Err(crate::Error::ConfigInvalid { - message: format!("{key} contains an empty or duplicate column '{column}'"), - }); + for column in columns { + if column.is_empty() && !strict { + continue; + } + if column.is_empty() || !unique.insert(column.clone()) { + return Err(crate::Error::ConfigInvalid { message: format!("global-index.{family}.index-columns (legacy pk-{family}.index.columns) contains an empty or duplicate column '{column}'") }); } - if owners.insert(column, family).is_some() { - return Err(crate::Error::ConfigInvalid { - message: format!("Column '{column}' can own at most one automatically maintained global index"), - }); + if !owners.insert(column.clone()) { + return Err(crate::Error::ConfigInvalid { message: format!("Column '{column}' can own at most one primary-key index or automatically maintained global index") }); } - resolve_index_fields(table, &IndexColumns::Columns(vec![column.into()]), "btree")?; + let kind = if family == "vector" && strict { + vector_index_type(options, &column)?.ok_or_else(|| crate::Error::ConfigInvalid { message: format!("fields.{column}.global-index.vector.index-type must specify a supported vector index type") })? + } else { + family + }; + result.push((column, kind.into())); + } + } + Ok(result) +} + +pub(super) fn definitions(table: &Table) -> Result> { + let options = table.schema().options(); + let owners = declarations(options)? + .into_iter() + .collect::>(); + for column in owners.keys() { + if !table.schema().fields().iter().any(|f| f.name() == column) { + return Err(crate::Error::ConfigInvalid { + message: format!("Global index contains unknown column '{column}'"), + }); } } let mut result = Vec::with_capacity(owners.len()); for field in table.schema().fields() { - let column = field.name(); - let Some(&family) = owners.get(column) else { + let Some(kind) = owners.get(field.name()) else { continue; }; - let index_type = if family == "vector" { - vector_index_type(options, column)?.ok_or_else(|| crate::Error::ConfigInvalid { - message: format!("fields.{column}.global-index.vector.index-type must specify a supported vector index type"), - })? + let family = if crate::vindex::is_vindex_index_type(kind) { + "vector" } else { - family + kind }; let definition = Definition { - column: column.into(), - index_type: index_type.into(), - options: field_index_options(options, column, family, index_type)?, + column: field.name().into(), + index_type: kind.clone(), + options: field_index_options(options, field.name(), family, kind)?, }; let mut merged = options.clone(); merged.extend(definition.options.clone()); @@ -92,123 +133,38 @@ pub(super) fn definitions(table: &Table) -> Result> { Ok(result) } +/// Strict native adapters receive resolved overrides without unrelated table properties. pub(super) fn field_index_options( table_options: &HashMap, column: &str, - family: &str, + _family: &str, index_type: &str, ) -> Result> { - let field_prefix = format!("fields.{column}.global-index.{family}."); - let native_prefix = match family { - "vector" | "full-text" => format!("{index_type}."), - _ => format!("{family}-index."), - }; - let options_key = format!("{field_prefix}index-options"); - let mut options = parse_field_index_options( - table_options, - column, - &options_key, - &native_prefix, - family == "vector", - )?; - if family == "vector" { - let metric_key = format!("{field_prefix}distance-metric"); - let metric = table_options - .get(&metric_key) - .map(String::as_str) - .unwrap_or("inner_product"); - crate::vindex::pkvector::metric::VectorSearchMetric::parse(metric)?; - options.insert( - format!("{native_prefix}metric"), - metric.to_ascii_lowercase().replace('-', "_"), - ); - } - Ok(options) -} - -pub(super) fn parse_field_index_options( - table_options: &HashMap, - column: &str, - options_key: &str, - native_prefix: &str, - vector: bool, -) -> Result> { - let mut options = HashMap::new(); - let mut insert = |key: &str, value: String| -> Result<()> { - if key.trim().is_empty() { - return Err(crate::Error::ConfigInvalid { - message: format!("{options_key} contains an empty option key"), - }); - } - let qualified = if key.starts_with(native_prefix) - || key.starts_with("fields.") - || key.starts_with("global-index.") - || key.starts_with("sorted-index.") - || (vector && matches!(key, "index.type" | "vindex.build.granule.enabled")) - { - key.into() - } else { - format!("{native_prefix}{key}") - }; - if options - .get(&qualified) - .or_else(|| match qualified.as_str() { - // Maintenance policies are column overrides; native writer - // options still reject conflicting table and field definitions. - "global-index.num-sorted-run.compaction-trigger" - | "global-index.compaction.size-ratio" - | "global-index.compaction.max-size-amplification-percent" => None, - _ => table_options.get(&qualified), - }) - .is_some_and(|previous| previous != &value) - { - return Err(crate::Error::ConfigInvalid { - message: format!("Conflicting field index options for {column}: {qualified}"), - }); - } - options.insert(qualified, value); - Ok(()) + let core = CoreOptions::new(table_options); + let resolved = if crate::vindex::is_vindex_index_type(index_type) { + core.vector_index_options(index_type, column, true)? + } else { + core.global_index_options(index_type, column)? }; - if let Some(raw) = table_options - .get(options_key) - .filter(|raw| !raw.trim().is_empty()) - { - let parsed: serde_json::Map = serde_json::from_str(raw) - .map_err(|_| crate::Error::ConfigInvalid { - message: format!("{options_key} must be a JSON object of option key-value pairs"), - })?; - for (key, value) in parsed { - let value = match value { - serde_json::Value::String(value) => value, - serde_json::Value::Number(_) | serde_json::Value::Bool(_) => value.to_string(), - _ => { - return Err(crate::Error::ConfigInvalid { - message: format!( - "{options_key} contains an invalid option value for {key}" - ), - }) - } - }; - insert(&key, value)?; - } - } - Ok(options) + Ok(resolved + .into_iter() + .filter(|(key, value)| table_options.get(key) != Some(value)) + .collect()) } pub(super) fn vector_index_type( options: &HashMap, column: &str, ) -> Result> { - let key = format!("fields.{column}.global-index.vector.index-type"); - match options.get(&key) { - Some(value) => normalize_global_index_type(value.trim()) - .filter(|kind| crate::vindex::is_vindex_index_type(kind)) - .map(Some) - .ok_or_else(|| crate::Error::ConfigInvalid { - message: format!("{key} must specify a supported vector index type"), - }), - None => Ok(None), - } + let Some(kind) = CoreOptions::new(options).vector_index_type(column)? else { + return Ok(None); + }; + normalize_global_index_type(&kind) + .filter(|kind| crate::vindex::is_vindex_index_type(kind)) + .map(Some) + .ok_or_else(|| crate::Error::ConfigInvalid { + message: format!("Unsupported vector index type for {column}: {kind}"), + }) } pub(super) fn declares_column( @@ -216,9 +172,13 @@ pub(super) fn declares_column( column: &str, family: &str, ) -> bool { - options - .get(&format!("global-index.{family}.index-columns")) - .is_some_and(|columns| columns.split(',').any(|name| name.trim() == column)) + let core = CoreOptions::new(options); + let columns = if family == "vector" { + core.vector_index_columns() + } else { + core.global_index_columns(family) + }; + columns.iter().any(|c| c == column) } pub(super) fn declared_options( @@ -227,9 +187,15 @@ pub(super) fn declared_options( family: &str, index_type: &str, ) -> Result> { - let prefix = format!("fields.{column}.global-index.{family}."); + let prefixes = [ + format!("fields.{column}.global-index.{family}."), + format!("fields.{column}.pk-{family}."), + ]; if declares_column(options, column, family) - || options.keys().any(|key| key.starts_with(&prefix)) + || options.contains_key(&format!("fields.{column}.index-options")) + || options + .keys() + .any(|key| prefixes.iter().any(|prefix| key.starts_with(prefix))) { field_index_options(options, column, family, index_type) } else { diff --git a/crates/paimon/src/table/global_index_scanner.rs b/crates/paimon/src/table/global_index_scanner.rs index 397d37669..602edce73 100644 --- a/crates/paimon/src/table/global_index_scanner.rs +++ b/crates/paimon/src/table/global_index_scanner.rs @@ -49,7 +49,7 @@ use crate::spec::{DataField, FileKind, GlobalIndexSearchMode, IndexManifestEntry use crate::table::RowRange; use crate::{Error, Result}; use std::cmp::Ordering; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; use tokio::sync::Semaphore; @@ -121,9 +121,11 @@ pub(crate) struct GlobalIndexScanner { query_semaphore: Arc, btree_fallback_scan_max_size: i64, btree_data_block_cache: Arc, + btree_data_block_cache_size: usize, bitmap_fallback_scan_max_size: i64, fm_read_options: FMReadOptions, fm_read_context: Arc, + field_read_options: HashMap>, /// Global index entries grouped by field_id. entries_by_field: Vec<(i32, Vec)>, /// Composite BTree files grouped by the full ordered definition. These @@ -142,6 +144,14 @@ pub(crate) struct GlobalIndexScanner { query_io_probe: Option>, } +struct FieldReadOptions { + btree_fallback: i64, + bitmap_fallback: i64, + btree_cache: Arc, + fm_options: FMReadOptions, + fm_context: Arc, +} + impl GlobalIndexScanner { /// Create a scanner from index manifest entries. /// Returns `Ok(None)` if there are no global index entries. @@ -341,9 +351,11 @@ impl GlobalIndexScanner { query_semaphore: Arc::new(Semaphore::new(global_index_thread_num)), btree_fallback_scan_max_size, btree_data_block_cache: Arc::new(BTreeDataBlockCache::new(btree_data_block_cache_size)), + btree_data_block_cache_size, bitmap_fallback_scan_max_size, fm_read_options, fm_read_context: Arc::new(FMReadContext::new(fm_read_options.cache_size)), + field_read_options: HashMap::new(), entries_by_field: entries_by_field.into_iter().collect(), composite_entries: composite_entries.into_iter().collect(), #[cfg(test)] @@ -356,6 +368,163 @@ impl GlobalIndexScanner { })) } + /// Bootstrap shared reader resources from effective field settings. An + /// overridden invalid table default must not fail before field resolution. + pub(super) fn read_defaults( + options: &HashMap, + entries: &[IndexManifestEntry], + fields: &[DataField], + ) -> Result> { + let mut defaults = options.clone(); + let mut kinds = HashSet::new(); + for entry in entries.iter().filter(|entry| entry.kind == FileKind::Add) { + let Some(kind) = normalize_queryable_global_index_type(&entry.index_file.index_type) + else { + continue; + }; + let Some(meta) = entry.index_file.global_index_meta.as_ref() else { + continue; + }; + let Some(field) = fields + .iter() + .find(|field| field.id() == meta.index_field_id) + else { + continue; + }; + if !kinds.insert(kind) { + continue; + } + let resolved = + crate::spec::CoreOptions::new(options).global_index_options(kind, field.name())?; + if kind == "multivalue" { + if let Some(value) = resolved.get("multivalue-index.fallback-scan-max-size") { + defaults.insert("bitmap-index.fallback-scan-max-size".into(), value.clone()); + } + } + let prefix = format!("{kind}-index."); + defaults.extend( + resolved + .into_iter() + .filter(|(key, _)| key.starts_with(&prefix)), + ); + } + Ok(defaults) + } + + /// Resolve each column/profile once, sharing equal-budget caches across columns and shards. + pub(super) fn with_field_options( + mut self, + options: &HashMap, + entries: &[IndexManifestEntry], + ) -> Result { + let mut resolved = HashMap::new(); + let mut btree_caches = HashMap::from([( + self.btree_data_block_cache_size, + Arc::clone(&self.btree_data_block_cache), + )]); + let mut fm_contexts = HashMap::from([( + self.fm_read_options.cache_size, + Arc::clone(&self.fm_read_context), + )]); + for entry in entries.iter().filter(|entry| entry.kind == FileKind::Add) { + let Some(kind) = normalize_queryable_global_index_type(&entry.index_file.index_type) + else { + continue; + }; + let Some(meta) = entry.index_file.global_index_meta.as_ref() else { + continue; + }; + let Some(field) = self + .schema_fields + .iter() + .find(|f| f.id() == meta.index_field_id) + else { + continue; + }; + let keys = [ + format!("fields.{}.pk-{kind}.index.options", field.name()), + format!("fields.{}.global-index.{kind}.index-options", field.name()), + format!("fields.{}.index-options", field.name()), + ]; + let key = ( + keys.iter() + .any(|key| options.contains_key(key)) + .then_some(field.id()), + kind, + ); + let settings = match resolved.get(&key) { + Some(settings) => Arc::clone(settings), + None => { + let mut map = crate::spec::CoreOptions::new(options) + .global_index_options(kind, field.name())?; + if kind == "multivalue" { + if let Some(value) = + map.get("multivalue-index.fallback-scan-max-size").cloned() + { + map.insert("bitmap-index.fallback-scan-max-size".into(), value); + } + } + let core = crate::spec::CoreOptions::new(&map); + let fm = if kind == "fm" { + crate::fm_index::FMOptions::from_options(&map)?.read + } else { + self.fm_read_options + }; + let settings = Arc::new(FieldReadOptions { + btree_fallback: if kind == "btree" { + core.btree_index_fallback_scan_max_size()? + } else { + self.btree_fallback_scan_max_size + }, + bitmap_fallback: if matches!(kind, "bitmap" | "multivalue") { + core.bitmap_index_fallback_scan_max_size()? + } else { + self.bitmap_fallback_scan_max_size + }, + btree_cache: if kind == "btree" { + let size = core.btree_index_data_block_cache_size()?; + Arc::clone( + btree_caches + .entry(size) + .or_insert_with(|| Arc::new(BTreeDataBlockCache::new(size))), + ) + } else { + Arc::clone(&self.btree_data_block_cache) + }, + fm_options: fm, + fm_context: Arc::clone( + fm_contexts + .entry(fm.cache_size) + .or_insert_with(|| Arc::new(FMReadContext::new(fm.cache_size))), + ), + }); + resolved.insert(key, Arc::clone(&settings)); + settings + } + }; + let path = super::index_file_path::IndexFileLocation::Global { + table_path: &self.table_path, + } + .resolve( + &entry.index_file.file_name, + entry.index_file.external_path.as_deref(), + ); + self.field_read_options.insert(path, settings); + } + Ok(self) + } + + fn field_options(&self, entry: &GlobalIndexEntry) -> Option<&FieldReadOptions> { + self.field_read_options + .get(&entry.resolved_path(&self.table_path)) + .map(Arc::as_ref) + } + + fn btree_fallback_limit(&self, entry: &GlobalIndexEntry) -> i64 { + self.field_options(entry) + .map_or(self.btree_fallback_scan_max_size, |s| s.btree_fallback) + } + fn btree_file_lock(&self, entry: &GlobalIndexEntry) -> Arc> { let path = entry.resolved_path(&self.table_path); let mut locks = self.btree_file_locks.lock().unwrap(); @@ -442,8 +611,23 @@ pub(crate) struct GlobalIndexEvaluation<'a> { pub(crate) data_ranges: &'a [RowRange], } +#[cfg(test)] pub(crate) async fn evaluate_global_index( evaluation: GlobalIndexEvaluation<'_>, +) -> Result>> { + evaluate_global_index_impl(evaluation, None).await +} + +pub(crate) async fn evaluate_global_index_with_options( + evaluation: GlobalIndexEvaluation<'_>, + options: &HashMap, +) -> Result>> { + evaluate_global_index_impl(evaluation, Some(options)).await +} + +async fn evaluate_global_index_impl( + evaluation: GlobalIndexEvaluation<'_>, + options: Option<&HashMap>, ) -> Result>> { let scanner = match GlobalIndexScanner::create_with_fm_options( evaluation.file_io, @@ -456,7 +640,13 @@ pub(crate) async fn evaluate_global_index( evaluation.schema_fields, evaluation.fm_read_options, )? { - Some(s) => s, + Some(s) => { + if let Some(options) = options { + s.with_field_options(options, evaluation.index_entries)? + } else { + s + } + } None => return Ok(None), }; let combined = Predicate::and(evaluation.predicates.to_vec()); diff --git a/crates/paimon/src/table/global_index_scanner/composite.rs b/crates/paimon/src/table/global_index_scanner/composite.rs index c99926673..bcd8057ca 100644 --- a/crates/paimon/src/table/global_index_scanner/composite.rs +++ b/crates/paimon/src/table/global_index_scanner/composite.rs @@ -104,11 +104,14 @@ impl GlobalIndexScanner { false } }); - if self.btree_fallback_scan_max_size <= 0 + let fallback_limit = selected + .first() + .map_or(self.btree_fallback_scan_max_size, |entry| { + self.btree_fallback_limit(entry) + }); + if fallback_limit <= 0 || !valid - || sizes - .values() - .any(|size| *size > self.btree_fallback_scan_max_size) + || sizes.values().any(|size| *size > fallback_limit) { continue; } diff --git a/crates/paimon/src/table/global_index_scanner/reader.rs b/crates/paimon/src/table/global_index_scanner/reader.rs index ba3c42996..5fe6a68c6 100644 --- a/crates/paimon/src/table/global_index_scanner/reader.rs +++ b/crates/paimon/src/table/global_index_scanner/reader.rs @@ -266,7 +266,10 @@ impl GlobalIndexScanner { file_size, meta, cmp, - Arc::clone(&self.btree_data_block_cache), + Arc::clone( + self.field_options(entry) + .map_or(&self.btree_data_block_cache, |s| &s.btree_cache), + ), Arc::from(resolved_path.as_str()), ) .await @@ -356,8 +359,12 @@ impl GlobalIndexScanner { Box::new(file_reader), file_size, manifest_meta, - self.fm_read_options, - Arc::clone(&self.fm_read_context), + self.field_options(entry) + .map_or(self.fm_read_options, |s| s.fm_options), + Arc::clone( + self.field_options(entry) + .map_or(&self.fm_read_context, |s| &s.fm_context), + ), entry.file_name.clone(), ) .await @@ -420,14 +427,28 @@ impl GlobalIndexScanner { } } - plan.allow_btree = plan.selected_btree > 0 - && btree_valid - && self.btree_fallback_scan_max_size > 0 - && btree_total <= self.btree_fallback_scan_max_size; + let btree_limit = entries + .iter() + .find(|e| matches!(e.index_type, GlobalIndexFileKind::BTree)) + .map_or(self.btree_fallback_scan_max_size, |entry| { + self.btree_fallback_limit(entry) + }); + let bitmap_limit = entries + .iter() + .find(|e| { + matches!( + e.index_type, + GlobalIndexFileKind::Bitmap | GlobalIndexFileKind::Multivalue + ) + }) + .and_then(|entry| self.field_options(entry)) + .map_or(self.bitmap_fallback_scan_max_size, |s| s.bitmap_fallback); + plan.allow_btree = + plan.selected_btree > 0 && btree_valid && btree_limit > 0 && btree_total <= btree_limit; plan.allow_bitmap = plan.selected_bitmap > 0 && bitmap_valid - && self.bitmap_fallback_scan_max_size > 0 - && bitmap_total <= self.bitmap_fallback_scan_max_size; + && bitmap_limit > 0 + && bitmap_total <= bitmap_limit; plan } diff --git a/crates/paimon/src/table/global_index_scanner/tests.rs b/crates/paimon/src/table/global_index_scanner/tests.rs index 89773c87b..74e157502 100644 --- a/crates/paimon/src/table/global_index_scanner/tests.rs +++ b/crates/paimon/src/table/global_index_scanner/tests.rs @@ -35,6 +35,155 @@ use std::collections::HashSet; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use std::sync::Arc; +#[test] +fn multivalue_field_options_override_bitmap_reader_defaults() { + let options = HashMap::from([ + ( + "bitmap-index.fallback-scan-max-size".into(), + "invalid default".into(), + ), + ( + "fields.tags.index-options".into(), + r#"{"fallback-scan-max-size":"1 mb"}"#.into(), + ), + ]); + let fields = vec![DataField::new( + 1, + "tags".into(), + DataType::Array(crate::spec::ArrayType::new(DataType::Int( + crate::spec::IntType::new(), + ))), + )]; + let entry = make_global_index_entry_with_type( + MULTIVALUE_GLOBAL_INDEX_TYPE, + "unopened.index", + 1, + 0, + 1, + &BTreeIndexMeta::new(None, None, false), + ); + assert!(crate::spec::CoreOptions::new(&options) + .bitmap_index_fallback_scan_max_size() + .is_err()); + let resolved = GlobalIndexScanner::read_defaults(&options, &[entry], &fields).unwrap(); + assert_eq!( + crate::spec::CoreOptions::new(&resolved) + .bitmap_index_fallback_scan_max_size() + .unwrap(), + 1024 * 1024 + ); + assert_eq!( + options["bitmap-index.fallback-scan-max-size"], + "invalid default" + ); +} + +#[tokio::test] +async fn shared_field_options_control_range_read_fallback() { + let (io, path, name, tmp) = setup_testdata_table("btree_int_100_no_compress.bin"); + let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false); + let mut entry = make_global_index_entry(&name, 1, 0, 99, &meta); + entry.index_file.file_size = std::fs::metadata(tmp.path().join("index").join(&name)) + .unwrap() + .len() as i64; + let mut second = entry.clone(); + second.index_file.file_name = "other.index".into(); + second + .index_file + .global_index_meta + .as_mut() + .unwrap() + .index_field_id = 2; + io.copy_file( + &format!("{path}/index/{name}"), + &format!("{path}/index/other.index"), + ) + .await + .unwrap(); + let entries = vec![entry, second]; + let mut fields = int_schema_fields(); + fields.push(DataField::new( + 2, + "other".into(), + fields[0].data_type().clone(), + )); + let predicate = Predicate::Leaf { + column: "id".into(), + index: 0, + data_type: fields[0].data_type().clone(), + op: PredicateOperator::GtEq, + literals: vec![Datum::Int(50)], + }; + let make = || { + GlobalIndexScanner::create_with_fm_options( + &io, + &path, + 1, + 0, + 0, + 0, + &entries, + &fields, + FMReadOptions::default(), + ) + .unwrap() + .unwrap() + }; + assert!(make().evaluate(&predicate).await.unwrap().is_none()); + let options = HashMap::from([ + ( + "btree-index.fallback-scan-max-size".into(), + "invalid default".into(), + ), + ( + "fields.id.index-options".into(), + r#"{"fallback-scan-max-size":"1 mb"}"#.into(), + ), + ( + "fields.unrelated.index-options".into(), + "invalid JSON".into(), + ), + ( + "fields.other.index-options".into(), + r#"{"fallback-scan-max-size":0}"#.into(), + ), + ]); + let defaults = GlobalIndexScanner::read_defaults(&options, &entries, &fields).unwrap(); + let core = crate::spec::CoreOptions::new(&defaults); + let scanner = GlobalIndexScanner::create_with_fm_options( + &io, + &path, + 1, + core.btree_index_fallback_scan_max_size().unwrap(), + core.btree_index_data_block_cache_size().unwrap(), + 0, + &entries, + &fields, + FMReadOptions::default(), + ) + .unwrap() + .unwrap() + .with_field_options(&options, &entries) + .unwrap(); + assert_eq!( + scanner + .evaluate(&predicate) + .await + .unwrap() + .unwrap() + .row_ranges, + vec![RowRange::new(25, 99)] + ); + let other = Predicate::Leaf { + column: "other".into(), + index: 1, + data_type: fields[1].data_type().clone(), + op: PredicateOperator::GtEq, + literals: vec![Datum::Int(50)], + }; + assert!(scanner.evaluate(&other).await.unwrap().is_none()); +} + #[tokio::test] async fn test_try_fold_bounded_respects_concurrency_limit() { for limit in [1, 3] { diff --git a/crates/paimon/src/table/hybrid_search_builder.rs b/crates/paimon/src/table/hybrid_search_builder.rs index ee0170702..1e2e12b81 100644 --- a/crates/paimon/src/table/hybrid_search_builder.rs +++ b/crates/paimon/src/table/hybrid_search_builder.rs @@ -716,7 +716,7 @@ impl<'a> HybridSearchBuilder<'a> { let mut any_pk = false; let mut any_global = false; for route in &self.routes { - if route_is_primary_key(core, route) { + if route_is_primary_key(core, route, !self.table.schema().primary_keys().is_empty()) { any_pk = true; } else { any_global = true; @@ -1052,7 +1052,14 @@ enum HybridAddressSpace { /// non-erroring accessor so a malformed config cannot misclassify an unrelated /// route). Full-text: data-evolution disabled AND the column configured in /// `pk-full-text.index.columns`. -fn route_is_primary_key(core: &CoreOptions<'_>, route: &HybridSearchRoute) -> bool { +fn route_is_primary_key( + core: &CoreOptions<'_>, + route: &HybridSearchRoute, + primary_key: bool, +) -> bool { + if !primary_key || core.data_evolution_enabled() { + return false; + } match route.kind { HybridSearchRouteKind::Vector => { core.primary_key_vector_index_enabled() diff --git a/crates/paimon/src/table/pk_index_options.rs b/crates/paimon/src/table/pk_index_options.rs index 0dfeb2f42..26a42489c 100644 --- a/crates/paimon/src/table/pk_index_options.rs +++ b/crates/paimon/src/table/pk_index_options.rs @@ -18,13 +18,11 @@ //! Java-compatible declarations for source-backed primary-key indexes. use crate::spec::{CoreOptions, DataField, DataType, TableSchema}; -use crate::table::global_index_options::parse_field_index_options; -use crate::table::global_index_types::normalize_global_index_type; use crate::table::index_build::full_text; use crate::table::Table; use crate::vindex::VindexVectorIndexOptions; use crate::{Error, Result}; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; pub(super) struct Definition { pub field: DataField, @@ -38,89 +36,45 @@ pub(super) fn field_options( family: &str, index_type: &str, ) -> Result> { - let key = format!("fields.{column}.pk-{family}.index.options"); - let prefix = if matches!(family, "vector" | "full-text") { - format!("{index_type}.") - } else { - format!("{family}-index.") - }; - let mut resolved = - parse_field_index_options(options, column, &key, &prefix, family == "vector")?; - if family == "vector" { - let metric = CoreOptions::new(options).primary_key_vector_distance_metric(column)?; - let expected = crate::vindex::pkvector::metric::VectorSearchMetric::parse(&metric)?; - resolved.insert(format!("{index_type}.metric"), metric); - // Native field aliases outrank the algorithm key within this map. Do - // not let them contradict the metric used for ANN/exact result merging. - for key in [ - format!("fields.{column}.metric"), - format!("fields.{column}.distance.metric"), - format!("{index_type}.distance.metric"), - ] { - if let Some(value) = resolved.get(&key) { - if crate::vindex::pkvector::metric::VectorSearchMetric::parse(value)? != expected { - return Err(Error::ConfigInvalid { - message: format!( - "{key} conflicts with fields.{column}.pk-vector.distance.metric" - ), - }); - } - } - } - } - Ok(resolved) + crate::table::global_index_options::field_index_options(options, column, family, index_type) } pub(super) fn vector_index_type( options: &HashMap, column: &str, ) -> Result<&'static str> { - let configured = CoreOptions::new(options).primary_key_vector_index_type(column)?; - normalize_global_index_type(configured.trim()) - .filter(|kind| crate::vindex::is_vindex_index_type(kind)) - .ok_or_else(|| Error::ConfigInvalid { - message: format!("Unsupported primary-key vector index type: {configured}"), - }) + crate::table::global_index_options::vector_index_type(options, column)?.ok_or_else(|| { + Error::ConfigInvalid { + message: format!("fields.{column}.pk-vector.index.type is required but not set"), + } + }) } pub(super) fn definitions(schema: &TableSchema) -> Result> { let options = schema.options(); - let mut owners = HashMap::new(); - for family in ["btree", "bitmap", "multivalue", "fm", "full-text", "vector"] { - let key = format!("pk-{family}.index.columns"); - if let Some(columns) = options.get(&key) { - let mut unique = HashSet::new(); - for column in columns.split(',').map(str::trim) { - if column.is_empty() || !unique.insert(column) { - return Err(Error::ConfigInvalid { - message: format!("{key} contains an empty or duplicate column '{column}'"), - }); - } - if !schema.fields().iter().any(|field| field.name() == column) { - return Err(Error::ConfigInvalid { - message: format!("{key} contains unknown column '{column}'"), - }); - } - if owners.insert(column, family).is_some() { - return Err(Error::ConfigInvalid { - message: format!("Column '{column}' can own at most one primary-key index"), - }); - } - } + let owners = crate::table::global_index_options::declarations(options)? + .into_iter() + .collect::>(); + for column in owners.keys() { + if !schema.fields().iter().any(|field| field.name() == column) { + return Err(Error::ConfigInvalid { + message: format!("Primary-key index contains unknown column '{column}'"), + }); } } - if options.contains_key("pk-vector.index.columns") { + if CoreOptions::new(options).primary_key_vector_index_enabled() { schema.core_options().primary_key_vector_index_column()?; } let mut result = Vec::new(); for field in schema.fields() { - let Some(&family) = owners.get(field.name()) else { + let Some(kind) = owners.get(field.name()) else { continue; }; - let kind = if family == "vector" { - vector_index_type(options, field.name())? + let kind = kind.as_str(); + let family = if crate::vindex::is_vindex_index_type(kind) { + "vector" } else { - family + kind }; let configured = field_options(options, field.name(), family, kind)?; let mut merged = options.clone(); diff --git a/crates/paimon/src/table/pk_sorted_index_scan.rs b/crates/paimon/src/table/pk_sorted_index_scan.rs index 705ed631f..f34e25355 100644 --- a/crates/paimon/src/table/pk_sorted_index_scan.rs +++ b/crates/paimon/src/table/pk_sorted_index_scan.rs @@ -26,8 +26,8 @@ use crate::spec::{ should_read_pk_index_source, CoreOptions, DataField, DataFileMeta, FileKind, IndexManifestEntry, Predicate, PrimaryKeyIndexSourceMeta, }; -use crate::{Error, Result}; -use std::collections::{BTreeMap, HashMap, HashSet}; +use crate::Result; +use std::collections::{BTreeMap, HashMap}; use std::future::Future; use std::pin::Pin; @@ -45,30 +45,16 @@ pub(super) struct Definition { } pub(super) fn definitions(fields: &[DataField], options: &Options) -> Result> { - let mut owners = HashMap::new(); - for family in ["btree", "bitmap", "vector", "full-text", "multivalue", "fm"] { - let key = format!("pk-{family}.index.columns"); - if let Some(columns) = options.get(&key) { - let mut unique = HashSet::new(); - for column in columns.split(',').map(str::trim) { - if !unique.insert(column) { - return Err(config_invalid(format!( - "{key} contains duplicate column '{column}'" - ))); - } - if owners.insert(column, family).is_some() { - return Err(config_invalid(format!( - "Column '{column}' can own at most one primary-key index" - ))); - } - } - } - } + let owners = super::global_index_options::declarations_for_read(options)? + .into_iter() + .collect::>(); let mut result = Vec::new(); for field in fields { - let Some(&family @ ("btree" | "bitmap")) = owners.get(field.name()) else { + let Some(family @ ("btree" | "bitmap")) = owners.get(field.name()).map(String::as_str) + else { continue; }; + let family = if family == "btree" { "btree" } else { "bitmap" }; result.push(Definition { field_id: field.id(), index_type: family, @@ -78,10 +64,6 @@ pub(super) fn definitions(fields: &[DataField], options: &Options) -> Result Error { - Error::ConfigInvalid { message } -} - fn definition_options(options: &Options, column: &str, family: &str) -> Result { let mut resolved = options.clone(); resolved.remove("sorted-index.records-per-file"); diff --git a/crates/paimon/src/table/pk_vector_read.rs b/crates/paimon/src/table/pk_vector_read.rs index c6e3c0026..b41522f94 100644 --- a/crates/paimon/src/table/pk_vector_read.rs +++ b/crates/paimon/src/table/pk_vector_read.rs @@ -300,11 +300,12 @@ async fn search_pk_raw_candidates_batch_with_plan( // Real ANN scorer + loader. Each segment source is opened lazily inside its // bucket leaf and dropped after scoring. Range reads load only metadata // and the index data needed for the search. - let options = { - let mut o = table.schema().options().clone(); - o.extend(query_options.clone()); - o - }; + let options = super::vector_search_common::vector_read_options( + table.schema().options(), + query_options, + pk_col, + &index_type, + )?; let search_options = options.clone(); let field_name = pk_col.to_string(); diff --git a/crates/paimon/src/table/table_commit/auto_index.rs b/crates/paimon/src/table/table_commit/auto_index.rs index 2d31fa3aa..1404c10af 100644 --- a/crates/paimon/src/table/table_commit/auto_index.rs +++ b/crates/paimon/src/table/table_commit/auto_index.rs @@ -24,7 +24,10 @@ mod planning; impl TableCommit { pub(super) async fn auto_index_enabled(&self, has_new_files: bool) -> Result { - if !has_new_files || self.table.schema().core_options().write_only()? { + if !has_new_files + || !self.table.schema().primary_keys().is_empty() + || self.table.schema().core_options().write_only()? + { return Ok(false); } let definitions = definitions(&self.table)?; diff --git a/crates/paimon/src/table/table_commit/auto_index_tests/search.rs b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs index 9681bb404..8b770458d 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests/search.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs @@ -22,6 +22,120 @@ use crate::Options; use arrow_array::builder::{Float32Builder, ListBuilder}; use arrow_array::{ArrayRef, StringArray}; +#[tokio::test] +async fn unified_and_dlf_declarations_share_build_and_raw_search_options() { + for suffix in ["index-columns", "index-column"] { + let vector_key = format!("global-index.ivf-flat.{suffix}"); + let text_key = format!("global-index.full-text.{suffix}"); + let (_catalog, table) = search_table(&[ + (&vector_key, "embedding"), + (&text_key, "text"), + ("ivf-flat.dimension", "9"), + ("fields.embedding.dimension", "2"), + ("fields.embedding.nlist", "1"), + ("fields.embedding.global-index.row-count-per-shard", "1"), + ("fields.embedding.distance.metric", "cosine"), + ("fields.text.index-options", r#"{"lower-case":false}"#), + ]) + .await; + TableCommit::new(table.clone(), "unified-config".into()) + .commit(search_messages(&table, &[[1., 0.], [0., 1.]]).await) + .await + .unwrap(); + let entries = indexes(&table).await; + assert_eq!(entries.len(), 3); + let mut ranges = entries + .iter() + .filter(|entry| entry.index_file.index_type == "ivf-flat") + .map(|entry| { + let meta = entry.index_file.global_index_meta.as_ref().unwrap(); + (meta.row_range_start, meta.row_range_end) + }) + .collect::>(); + ranges.sort_unstable(); + assert_eq!(ranges, vec![(0, 0), (1, 1)]); + let vector = entries + .iter() + .find(|e| e.index_file.index_type == "ivf-flat") + .unwrap(); + let meta: HashMap = serde_json::from_slice( + vector + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_meta + .as_ref() + .unwrap(), + ) + .unwrap(); + assert_eq!(meta["dimension"], "2"); + assert_eq!(meta["metric"], "cosine"); + let indexed = vector_scores(&table, "fast").await; + let bad_read_options = table.copy_with_options(HashMap::from([( + "fields.embedding.index-options".into(), + r#"{"nprobe":"invalid"}"#.into(), + )])); + let error = bad_read_options + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(2) + .with_options(HashMap::from([( + "vector-index.search-mode".into(), + "fast".into(), + )])) + .execute() + .await + .expect_err("shared read parameters must reach the native reader"); + assert!(error.to_string().contains("ivf.nprobe"), "{error}"); + bad_read_options + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(2) + .with_options(HashMap::from([ + ("vector-index.search-mode".into(), "fast".into()), + ("ivf.nprobe".into(), "1".into()), + ])) + .execute() + .await + .unwrap(); + for kind in ["ivf-flat", "full-text"] { + let column = if kind == "full-text" { + "text" + } else { + "embedding" + }; + table + .new_global_index_drop_builder() + .with_index_type(kind) + .with_index_column(column) + .execute() + .await + .unwrap(); + } + for mode in ["full", "detail"] { + let raw = vector_scores(&table, mode).await; + assert_eq!(raw.row_ids, indexed.row_ids); + assert_eq!(raw.scores, indexed.scores); + let hits = table + .copy_with_options(HashMap::from([( + "global-index.search-mode".into(), + mode.into(), + )])) + .new_full_text_search_builder() + .with_text_column("text") + .with_query_text("FOO") + .with_limit(4) + .execute() + .await + .unwrap(); + assert_eq!(hits.row_ids().unwrap().row_ids, vec![0]); + } + } +} + #[tokio::test] async fn filtered_build_views_preserve_snapshot_and_unrelated_index_coverage() { use crate::table::global_index_build_common::{ diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 0cad7d6e9..617967e0d 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -2538,8 +2538,13 @@ impl<'a> PaimonTableScan<'a> { settings: GlobalIndexScanSettings, data_ranges: &[RowRange], ) -> crate::Result>> { - let core_options = CoreOptions::new(self.table.schema().options()); - super::global_index_scanner::evaluate_global_index( + let resolved = super::global_index_scanner::GlobalIndexScanner::read_defaults( + self.table.schema().options(), + index_entries, + self.table.schema().fields(), + )?; + let core_options = CoreOptions::new(&resolved); + super::global_index_scanner::evaluate_global_index_with_options( super::global_index_scanner::GlobalIndexEvaluation { file_io: self.table.file_io(), table_path: self.table.location().trim_end_matches('/'), @@ -2556,13 +2561,14 @@ impl<'a> PaimonTableScan<'a> { .iter() .any(|entry| entry.index_file.index_type.eq_ignore_ascii_case("fm")) { - crate::fm_index::FMOptions::from_options(self.table.schema().options())?.read + crate::fm_index::FMOptions::from_options(&resolved)?.read } else { crate::fm_index::FMReadOptions::default() }, next_row_id: snapshot.next_row_id(), data_ranges, }, + self.table.schema().options(), ) .await } diff --git a/crates/paimon/src/table/vector_read.rs b/crates/paimon/src/table/vector_read.rs index 91a1a7a72..fe99cba87 100644 --- a/crates/paimon/src/table/vector_read.rs +++ b/crates/paimon/src/table/vector_read.rs @@ -87,7 +87,7 @@ impl BatchVectorRead { context: PlanContext, ) -> crate::Result { let core = CoreOptions::new(table.schema().options()); - let reader = if targets_primary_key_column(&core, column) { + let reader = if targets_primary_key_column(table, column) { let pk_col = core.primary_key_vector_index_column()?; let params = PkVectorSearchParams::resolve(table, options, filter, &pk_col, queries, limit)?; diff --git a/crates/paimon/src/table/vector_scan.rs b/crates/paimon/src/table/vector_scan.rs index 262077913..c0201b2c1 100644 --- a/crates/paimon/src/table/vector_scan.rs +++ b/crates/paimon/src/table/vector_scan.rs @@ -222,7 +222,7 @@ impl PlanContext { message: "Vector column must be set via with_vector_column()".to_string(), }); } - if targets_primary_key_column(&core, column) + if targets_primary_key_column(table, column) && (include_row_ids.is_some() || prepared.is_some()) { return Err(crate::Error::DataInvalid { @@ -264,7 +264,7 @@ impl VectorScan { ) -> crate::Result { let context = PlanContext::new(table, column, filter, include_row_ids, prepared)?; let core = CoreOptions::new(table.schema().options()); - let scan = if targets_primary_key_column(&core, column) { + let scan = if targets_primary_key_column(table, column) { let column = core.primary_key_vector_index_column()?; let field_id = find_field_id_by_name(table.schema().fields(), &column).ok_or_else(|| { diff --git a/crates/paimon/src/table/vector_search_common.rs b/crates/paimon/src/table/vector_search_common.rs index 4f2425aab..bac1035d0 100644 --- a/crates/paimon/src/table/vector_search_common.rs +++ b/crates/paimon/src/table/vector_search_common.rs @@ -514,19 +514,51 @@ pub(super) fn configured_refine_factor( field_name: &str, index_type: &str, ) -> crate::Result { - if let Some(value) = - configured_refine_factor_from_options(search_options, field_name, index_type) - { + let query = + CoreOptions::new(search_options).vector_index_options(index_type, field_name, false)?; + if let Some(value) = configured_refine_factor_from_options(&query, field_name, index_type) { return parse_refine_factor(&value); } - if let Some(value) = - configured_refine_factor_from_options(table_options, field_name, index_type) - { + let resolved = + CoreOptions::new(table_options).vector_index_options(index_type, field_name, false)?; + if let Some(value) = configured_refine_factor_from_options(&resolved, field_name, index_type) { return parse_refine_factor(&value); } Ok(0) } +/// Resolve each source before merging so a broad query option outranks every +/// table field alias. Native readers consume their established query keys. +pub(super) fn vector_read_options( + table: &HashMap, + query: &HashMap, + column: &str, + kind: &str, +) -> crate::Result> { + let resolve = |source: &HashMap| -> crate::Result> { + let mut options = CoreOptions::new(source).vector_index_options(kind, column, false)?; + for (native, suffixes) in [ + ("ivf.nprobe", &["nprobe", "ivf.nprobe"][..]), + ("diskann.l_search", &["l_search", "diskann.l_search"][..]), + ] { + let value = indexed_type_prefixes(column, kind) + .iter() + .find_map(|prefix| { + suffixes + .iter() + .find_map(|suffix| options.get(&format!("{prefix}{suffix}")).cloned()) + }); + if let Some(value) = value { + options.insert(native.into(), value); + } + } + Ok(options) + }; + let mut options = resolve(table)?; + options.extend(resolve(query)?); + Ok(options) +} + fn configured_refine_factor_from_options( options: &HashMap, field_name: &str, @@ -562,8 +594,11 @@ fn parse_refine_factor(value: &str) -> crate::Result { } /// A malformed PK configuration must not reject an unrelated DE query. -pub(super) fn targets_primary_key_column(core: &CoreOptions<'_>, column: &str) -> bool { - core.primary_key_vector_index_enabled() +pub(super) fn targets_primary_key_column(table: &Table, column: &str) -> bool { + let core = table.schema().core_options(); + !table.schema().primary_keys().is_empty() + && !core.data_evolution_enabled() + && core.primary_key_vector_index_enabled() && core .primary_key_vector_index_columns() .ok() @@ -576,6 +611,54 @@ mod tests { use crate::io::FileIOBuilder; use crate::spec::{GlobalIndexMeta, IndexFileMeta}; + #[test] + fn shared_read_options_keep_query_precedence_and_refinement() { + let table = HashMap::from([ + ( + "fields.embedding.index-options".into(), + r#"{"nprobe":8,"refine-factor":3}"#.into(), + ), + ("fields.embedding.nprobe".into(), "4".into()), + ("ivf.nprobe".into(), "2".into()), + ]); + assert_eq!( + vector_read_options(&table, &HashMap::new(), "embedding", "ivf-flat").unwrap() + ["ivf.nprobe"], + "8" + ); + let query = HashMap::from([ + ("ivf.nprobe".into(), "1".into()), + ("refine-factor".into(), "2".into()), + ]); + assert_eq!( + vector_read_options(&table, &query, "embedding", "ivf-flat").unwrap()["ivf.nprobe"], + "1" + ); + assert_eq!( + configured_refine_factor(&HashMap::new(), &table, "embedding", "ivf-flat").unwrap(), + 3 + ); + assert_eq!( + configured_refine_factor(&query, &table, "embedding", "ivf-flat").unwrap(), + 2 + ); + let query_json = HashMap::from([( + "fields.embedding.index-options".into(), + r#"{"nprobe":1,"refine-factor":2}"#.into(), + )]); + assert_eq!( + vector_read_options(&table, &query_json, "embedding", "ivf-flat").unwrap() + ["ivf.nprobe"], + "1" + ); + for defaults in [&table, &HashMap::new()] { + assert_eq!( + configured_refine_factor(&query_json, defaults, "embedding", "ivf-flat").unwrap(), + 2 + ); + } + } + fn entry(index_meta: Option>) -> IndexManifestEntry { IndexManifestEntry { version: 1, diff --git a/crates/paimon/src/vindex/mod.rs b/crates/paimon/src/vindex/mod.rs index 3fede5f1d..b81e45339 100644 --- a/crates/paimon/src/vindex/mod.rs +++ b/crates/paimon/src/vindex/mod.rs @@ -292,6 +292,15 @@ fn validate_user_option_keys( .strip_prefix(&format!("{index_type}.")) .or_else(|| key.strip_prefix(&format!("fields.{field}."))) .unwrap_or(key); + if matches!( + suffix, + "refine-factor" | "refine_factor" | "rerank-factor" | "rerank_factor" + ) || (index_type != DISKANN_IDENTIFIER && matches!(suffix, "nprobe" | "ivf.nprobe")) + || (index_type == DISKANN_IDENTIFIER + && matches!(suffix, "l_search" | "diskann.l_search")) + { + return false; + } if suffix == "train.sample-ratio" { return key.as_str() == suffix; } @@ -415,6 +424,11 @@ fn collect_native_options( let Some(native_key) = native_option_key(suffix) else { continue; }; + // DLF prefers dimension over index.dimension within each source. + // Iterating alias keys in hash order must not choose the dimension. + if suffix == "index.dimension" && options.contains_key(&format!("{prefix}dimension")) { + continue; + } // Schema options follow Java. Explicit Rust/SQL builder options also // accept the documented native aliases. if prefix.is_empty() diff --git a/crates/paimon/tests/pk_compact/indexes.rs b/crates/paimon/tests/pk_compact/indexes.rs index 40beaf250..8e4938976 100644 --- a/crates/paimon/tests/pk_compact/indexes.rs +++ b/crates/paimon/tests/pk_compact/indexes.rs @@ -19,6 +19,142 @@ use super::*; use paimon::spec::{should_read_pk_index_source, IndexManifest, PrimaryKeyIndexSourceMeta}; use std::collections::HashSet; +#[tokio::test] +async fn unified_index_config_drives_pk_compaction_and_ranked_reads() { + use paimon::spec::{ArrayType, FloatType}; + use std::sync::Arc; + for suffix in ["index-columns", "index-column"] { + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .column("text", DataType::VarChar(VarCharType::string_type())) + .primary_key(["id"]) + .option("bucket", "1") + .option("deletion-vectors.enabled", "true") + .option("deletion-vectors.merge-on-read", "false") + .option(format!("global-index.btree.{suffix}"), "id") + .option(format!("global-index.full-text.{suffix}"), "text") + .option(format!("global-index.ivf-flat.{suffix}"), "embedding") + .option("ivf-flat.dimension", "9") + .option("fields.embedding.dimension", "2") + .option("fields.embedding.distance.metric", "cosine") + .option( + "fields.embedding.pk-vector.index.options", + "invalid legacy JSON", + ) + .option( + "fields.embedding.index-options", + r#"{"nlist":1,"metric":"L2"}"#, + ) + .option("full-text.lower-case", "true") + .option("fields.text.index-options", r#"{"lower-case":false}"#) + .option("global-index.enabled", "true") + .option("global-index.search-mode", "fast") + .build() + .unwrap(); + // Exercise normal schema validation as well as the compact/read adapters. + let schema = TableSchema::new(0, &schema); + let table = table_with_schema(schema).await; + let vectors = vector_index_batch(vec![1, 2], vec![Some([1., 0.]), Some([0., 1.])]); + let batch = RecordBatch::try_from_iter([ + ("id", vectors.column_by_name("id").unwrap().clone()), + ( + "embedding", + vectors.column_by_name("embedding").unwrap().clone(), + ), + ( + "text", + Arc::new(arrow_array::StringArray::from(vec!["MiXeD", "plain"])) + as arrow_array::ArrayRef, + ), + ]) + .unwrap(); + write_batch(&table, &batch).await; + let builder = table + .new_compact_builder() + .with_commit_user("unified-index-options") + .unwrap(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + builder + .new_commit() + .unwrap() + .commit_compact(execute(&table, &plan).await) + .await + .unwrap(); + let entries = indexes(&table).await; + assert_eq!(entries.len(), 3); + let vector = entries + .iter() + .find(|e| e.index_file.index_type == "ivf-flat") + .unwrap(); + let meta: std::collections::HashMap = serde_json::from_slice( + vector + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_meta + .as_ref() + .unwrap(), + ) + .unwrap(); + assert_eq!(meta["dimension"], "2"); + assert_eq!(meta["metric"], "l2"); + let result = table + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(1) + .execute() + .await + .unwrap(); + let batches: Vec = result + .new_read_builder() + .with_projection(&["id"]) + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + batches[0] + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 1 + ); + for (query, count) in [("MiXeD", 1), ("mixed", 0)] { + let result = table + .new_full_text_search_builder() + .with_query("text", &format!(r#"{{"match":{{"query":"{query}"}}}}"#)) + .with_limit(2) + .execute() + .await + .unwrap(); + let batches: Vec = result + .new_read_builder() + .with_projection(&["id"]) + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + count + ); + } + } +} + async fn vector_table_for_indexes(options: &[(&str, &str)]) -> Table { use paimon::spec::{ArrayType, FloatType}; let mut schema = Schema::builder() @@ -690,7 +826,7 @@ async fn vector_type_is_canonical_in_build_and_query() { } #[tokio::test] -async fn conflicting_field_metric_is_rejected_before_build() { +async fn legacy_field_metric_aliases_override_pk_metric_consistently() { for native in [ r#"{"dimension":2,"nlist":1,"fields.embedding.metric":"cosine"}"#, r#"{"dimension":2,"nlist":1,"fields.embedding.distance.metric":"cosine"}"#, @@ -701,7 +837,57 @@ async fn conflicting_field_metric_is_rejected_before_build() { ("fields.embedding.metric", "cosine"), ]) .await; - assert!(table.new_compact_builder().new_planner().is_err()); + assert_eq!( + table + .schema() + .core_options() + .primary_key_vector_distance_metric("embedding") + .unwrap(), + "cosine" + ); + write_batch( + &table, + &vector_index_batch(vec![1, 2], vec![Some([100., 1.]), Some([1., 1.])]), + ) + .await; + let builder = table + .new_compact_builder() + .with_commit_user("metric-alias-priority") + .unwrap(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + builder + .new_commit() + .unwrap() + .commit_compact(execute(&table, &plan).await) + .await + .unwrap(); + let result = table + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(1) + .execute() + .await + .unwrap(); + let batches: Vec = result + .new_read_builder() + .with_projection(&["id"]) + .read() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 1, + "cosine must win over the legacy L2 option in both build and query" + ); } } @@ -1510,52 +1696,53 @@ async fn automatic_compaction_maintains_indexes_and_recovers_callback_failure() Ok(()) } } - let original = table(&[ - ("pk-bitmap.index.columns", "value"), - ("compaction.force-up-level-0", "true"), - ]) - .await; - let table = original - .copy_with_resolved_schema( - original - .schema() - .copy_with_options(std::collections::HashMap::from([( - "write-only".into(), - "false".into(), - )])), - "main", - ) - .unwrap(); - persist_table_schema(table.file_io(), table.location(), table.schema()).await; - let write = table.new_write_builder(); - let mut writer = write.new_write().unwrap(); - writer - .write_arrow_batch(&make_batch(vec![1, 2], vec![10, 20])) - .await - .unwrap(); - let messages = writer.prepare_commit().await.unwrap(); - let mut commit = write.new_commit(); - commit - .add_commit_callback(Arc::new(FailOnce(AtomicBool::new(false)))) - .unwrap(); - assert!(commit - .commit_with_identifier(messages.clone(), 7) - .await - .unwrap_err() - .to_string() - .contains("pk-index callback failure")); - let snapshot = latest(&table).await; - assert_eq!(snapshot.commit_kind(), &CommitKind::COMPACT); - assert_eq!(snapshot.id(), 2); - let active = indexes(&table).await; - assert_eq!(active.len(), 1); - commit - .filter_and_commit_with_identifier(messages, 7) - .await - .unwrap(); - assert_eq!(latest(&table).await.id(), 2); - assert_eq!(indexes(&table).await, active); - assert_eq!(filtered_rows(&table, 20).await, vec![(2, 20)]); + for key in [ + "pk-bitmap.index.columns", + "global-index.bitmap.index-columns", + ] { + let original = table(&[(key, "value"), ("compaction.force-up-level-0", "true")]).await; + let table = original + .copy_with_resolved_schema( + original + .schema() + .copy_with_options(std::collections::HashMap::from([( + "write-only".into(), + "false".into(), + )])), + "main", + ) + .unwrap(); + persist_table_schema(table.file_io(), table.location(), table.schema()).await; + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer + .write_arrow_batch(&make_batch(vec![1, 2], vec![10, 20])) + .await + .unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + let mut commit = write.new_commit(); + commit + .add_commit_callback(Arc::new(FailOnce(AtomicBool::new(false)))) + .unwrap(); + assert!(commit + .commit_with_identifier(messages.clone(), 7) + .await + .unwrap_err() + .to_string() + .contains("pk-index callback failure")); + let snapshot = latest(&table).await; + assert_eq!(snapshot.commit_kind(), &CommitKind::COMPACT); + assert_eq!(snapshot.id(), 2); + let active = indexes(&table).await; + assert_eq!(active.len(), 1); + commit + .filter_and_commit_with_identifier(messages, 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 2); + assert_eq!(indexes(&table).await, active); + assert_eq!(filtered_rows(&table, 20).await, vec![(2, 20)]); + } } async fn latest(table: &Table) -> paimon::spec::Snapshot { diff --git a/docs/src/compaction.md b/docs/src/compaction.md index 1785346ec..b2b4b0b01 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -108,9 +108,11 @@ Rust, Python, and DataFusion writes share this behavior. Tables without index declarations retain their existing write behavior; no separate index switch is required and no background job is scheduled. -Index options follow the primary-key index convention: a comma-separated column -list for each family and optional per-column parameters. Supported families are -`btree`, `bitmap`, `multivalue`, `fm`, `full-text`, and `vector`: +Primary-key and append/Data Evolution tables share the same configuration: +`global-index..index-columns` declares comma-separated columns, and +`fields..index-options` supplies a JSON object of field parameters. +Supported types are `btree`, `bitmap`, `multivalue`, `fm`, `full-text`, +`ivf-flat`, `ivf-sq`, `ivf-pq`, `ivf-rq`, and `diskann`: ```rust let options = std::collections::HashMap::from([ @@ -120,15 +122,13 @@ let options = std::collections::HashMap::from([ ("global-index.btree.index-columns".into(), "id,age".into()), ("global-index.full-text.index-columns".into(), "text".into()), ( - "fields.text.global-index.full-text.index-options".into(), + "fields.text.index-options".into(), r#"{"lower-case":true}"#.into(), ), - ("global-index.vector.index-columns".into(), "embedding".into()), - ("fields.embedding.global-index.vector.index-type".into(), "diskann".into()), - ("fields.embedding.global-index.vector.distance-metric".into(), "cosine".into()), + ("global-index.diskann.index-columns".into(), "embedding".into()), ( - "fields.embedding.global-index.vector.index-options".into(), - r#"{"dimension":768}"#.into(), + "fields.embedding.index-options".into(), + r#"{"dimension":768,"distance.metric":"cosine"}"#.into(), ), ]); ``` @@ -142,16 +142,49 @@ are trimmed; empty names, duplicates, unknown fields, and assigning multiple families to one column are rejected before data publication. Each listed column has its own index. Use the explicit build API for composite BTree indexes. Update the lists and per-column option keys together with schema renames or drops. -`write-only=true` skips automatic index configuration validation. - -Per-column parameters use -`fields..global-index..index-options`, a JSON object containing -string, boolean, or numeric values. Short keys are qualified with the native -writer prefix, such as `block-size` to `btree-index.block-size`. Fully qualified -writer keys and shared shard options are accepted. Conflicting native writer -values are rejected, matching primary-key option resolution. Vector columns -additionally require `fields..global-index.vector.index-type`; the optional -`distance-metric` defaults to `inner_product`. +`write-only=true` skips automatic maintenance. Shared declarations and native +parameters are validated when the corresponding index path is used; existing +Rust PK schema prerequisites still apply. + +Short JSON keys are qualified with the implementation prefix: `block-size` +becomes `btree-index.block-size`, and vector `nlist` becomes `ivf-pq.nlist`. +String, boolean and numeric values are accepted; full-text arrays such as +`stop-words` use the native semicolon-separated encoding. The shared JSON +overrides table implementation defaults. Vector metrics default to +`inner_product`, and accepted metric aliases resolve consistently for builds, +indexed reads, raw scoring and refinement. + +Compatibility and precedence follow Java's shared configuration resolver: + +- Declarations prefer `global-index..index-columns`, then the DLF singular + `global-index..index-column`, then legacy `pk-.index.columns`. + An explicitly empty preferred declaration is invalid rather than falling back. +- Field JSON prefers `fields..index-options`, then the existing Rust + `fields..global-index..index-options`, then legacy + `fields..pk-.index.options`. Shared JSON replaces legacy JSON; + it can override defaults, while non-metric legacy conflicts remain errors. + Vector metric aliases are resolved together after JSON merging, so field JSON + overrides legacy metric properties and case-only aliases stay compatible. +- Legacy `pk-vector` declarations still select the algorithm through + `fields..pk-vector.index.type`. The former Rust + `global-index.vector` declaration and field algorithm/metric keys remain aliases. + That older Rust declaration retains its metric/default priority over flat + aliases until replaced by a concrete declaration or shared field JSON. +- DLF implementation parameters such as `ivf-pq.dimension` and flat field + overrides such as `fields.embedding.dimension`, `distance.metric`, `nlist` and + `pq.m` remain supported. Shared vector JSON overrides those flat parameters. + `fields..global-index.row-count-per-shard` overrides the shard size + for that column. Morax scheduling switches do not control Rust maintenance. + +The configuration is shared; table modes retain their validation and lifecycle. +PK indexes are maintained with data compaction, while append/DE indexes retain +global row-ID coverage and the incremental/consolidation policy below. + +Upgrade readers and writers before switching PK tables to shared-only keys. +During a mixed-version rollout, retain the legacy declarations and field values +alongside the shared keys and keep their effective definitions aligned; older +clients still depend on the legacy PK namespace. Existing stored options do not +need migration for the updated resolver. Explicit single-column builds also use these declarations as defaults; explicit builder options take precedence over them. FULL/DETAIL raw fallback uses the same per-column parameters as automatic @@ -306,8 +339,9 @@ snapshot expiration and storage reclamation are separate maintenance work. ## Primary-key compaction -Primary-key compaction also maintains declared source-backed indexes, using -Java's primary-key options: +Primary-key compaction maintains declared source-backed indexes using the same +`global-index..index-columns` and `fields..index-options` options +shown above. Existing Java primary-key configurations remain compatible: | Option | Meaning | | --- | --- | From c9982f6de643eefd3e4fdb639ac8030200c88566 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 11:22:08 +0800 Subject: [PATCH 10/14] feat(vector): add explicit native index type configuration Align PK/DE option resolution with Java PR #10914 and remove the four query implementation hint aliases. Preserve canonical, singular, legacy PK and DLF configuration, including replacement field JSON and short parameter keys. Carry per-column explicit implementations through Rust/Python single, batch and hybrid builders, captured scans/readers, raw source plans and workers. Use persisted metadata first, and use the explicit type only for raw fallback. Preserve Java PK metric behavior and eager query validation. Cover native fallback scores, metadata and empty-marker precedence, captured routes/readers, transport round trips, aliases and short JSON parameters. --- .../python/pypaimon_rust/datafusion.pyi | 3 + bindings/python/src/hybrid_search.rs | 13 + bindings/python/src/vector_search.rs | 29 + bindings/python/tests/test_vector_search.py | 53 ++ .../src/spec/core_options/index_options.rs | 66 ++- .../src/table/batch_vector_search_builder.rs | 17 +- crates/paimon/src/table/de_vector_read.rs | 111 ++-- .../paimon/src/table/de_vector_read/tests.rs | 18 +- .../table/full_text_search_builder/scan.rs | 1 + .../paimon/src/table/hybrid_search_builder.rs | 33 +- crates/paimon/src/table/pk_vector_read.rs | 29 +- .../src/table/pk_vector_search_params.rs | 93 ++- .../table/pk_vector_search_params/tests.rs | 15 + crates/paimon/src/table/search_scan_split.rs | 7 +- crates/paimon/src/table/vector_read.rs | 21 +- crates/paimon/src/table/vector_scan.rs | 8 + .../paimon/src/table/vector_search_builder.rs | 25 +- .../native_type_tests.rs | 529 ++++++++++++++++++ .../paimon/src/table/vector_search_common.rs | 33 +- .../src/table/vector_search_test_utils.rs | 14 +- crates/paimon/src/vector_search.rs | 3 + crates/paimon/tests/pk_compact/indexes.rs | 1 + .../paimon/tests/pk_vector_baseline_test.rs | 1 + docs/src/compaction.md | 15 + 24 files changed, 1032 insertions(+), 106 deletions(-) create mode 100644 crates/paimon/src/table/vector_search_builder/native_type_tests.rs diff --git a/bindings/python/python/pypaimon_rust/datafusion.pyi b/bindings/python/python/pypaimon_rust/datafusion.pyi index 0ede5a1fb..6e0a8819a 100644 --- a/bindings/python/python/pypaimon_rust/datafusion.pyi +++ b/bindings/python/python/pypaimon_rust/datafusion.pyi @@ -1139,6 +1139,7 @@ class SQLContext: class VectorSearchBuilder: + def with_vector_index_type(self, column: str, index_type: str) -> "VectorSearchBuilder": ... def with_vector_column(self, name: str) -> "VectorSearchBuilder": ... def with_limit(self, limit: int) -> "VectorSearchBuilder": ... def with_query_vector(self, vector: Sequence[float]) -> "VectorSearchBuilder": ... @@ -1153,6 +1154,7 @@ class VectorSearchBuilder: def execute(self) -> "GlobalIndexResult": ... class BatchVectorSearchBuilder: + def with_vector_index_type(self, column: str, index_type: str) -> "BatchVectorSearchBuilder": ... def with_vector_column(self, name: str) -> "BatchVectorSearchBuilder": ... def with_limit(self, limit: int) -> "BatchVectorSearchBuilder": ... def with_query_vectors(self, vectors: Sequence[Sequence[float]]) -> "BatchVectorSearchBuilder": ... @@ -1267,6 +1269,7 @@ class HybridSearchRouteResult: def result(self) -> GlobalIndexResult: ... class HybridSearchBuilder: + def with_vector_index_type(self, column: str, index_type: str) -> "HybridSearchBuilder": ... def with_limit(self, limit: int) -> HybridSearchBuilder: ... def with_snapshot(self, snapshot: Optional[Snapshot]) -> HybridSearchBuilder: """Pin the resolved snapshot object; None pins an empty read view.""" diff --git a/bindings/python/src/hybrid_search.rs b/bindings/python/src/hybrid_search.rs index 6888f21cc..cc52dce3d 100644 --- a/bindings/python/src/hybrid_search.rs +++ b/bindings/python/src/hybrid_search.rs @@ -278,6 +278,7 @@ pub struct PyHybridSearchBuilder { limit: Option, ranker: HybridSearchRanker, pinned_snapshot: Option>, + vector_index_types: HashMap, } impl PyHybridSearchBuilder { @@ -289,11 +290,15 @@ impl PyHybridSearchBuilder { limit: None, ranker: HybridSearchRanker::Rrf, pinned_snapshot: None, + vector_index_types: HashMap::new(), } } fn builder(&self) -> paimon::table::HybridSearchBuilder<'_> { let mut builder = self.table.new_hybrid_search_builder(); + for (column, index_type) in &self.vector_index_types { + builder.with_vector_index_type(column, index_type); + } for route in &self.routes { builder.add_route(route.clone()); } @@ -316,6 +321,14 @@ impl PyHybridSearchBuilder { #[pymethods] impl PyHybridSearchBuilder { + fn with_vector_index_type( + mut slf: PyRefMut<'_, Self>, + column: String, + index_type: String, + ) -> PyRefMut<'_, Self> { + slf.vector_index_types.insert(column, index_type); + slf + } #[pyo3(signature = (field_name, vector, limit, weight=1.0, options=None))] fn add_vector_route( mut slf: PyRefMut<'_, Self>, diff --git a/bindings/python/src/vector_search.rs b/bindings/python/src/vector_search.rs index 302b2e16d..2beffa261 100644 --- a/bindings/python/src/vector_search.rs +++ b/bindings/python/src/vector_search.rs @@ -38,6 +38,7 @@ struct SearchConfig { column: Option, limit: Option, options: Vec>, + vector_index_types: HashMap, filters: Vec, partition_filters: Vec, } @@ -48,6 +49,9 @@ impl SearchConfig { table: &'a Table, ) -> paimon::Result> { let mut builder = table.new_batch_vector_search_builder(); + for (column, index_type) in &self.vector_index_types { + builder.with_vector_index_type(column, index_type); + } if let Some(column) = &self.column { builder.with_vector_column(column); } @@ -93,6 +97,14 @@ impl PyVectorSearchBuilder { slf.inner.with_vector_column(&name); slf } + fn with_vector_index_type( + mut slf: PyRefMut<'_, Self>, + column: String, + index_type: String, + ) -> PyRefMut<'_, Self> { + slf.inner.with_vector_index_type(&column, &index_type); + slf + } fn with_limit(mut slf: PyRefMut<'_, Self>, limit: usize) -> PyRefMut<'_, Self> { slf.inner.with_limit(limit); slf @@ -169,6 +181,14 @@ impl PyBatchVectorSearchBuilder { slf.config.column = Some(name); slf } + fn with_vector_index_type( + mut slf: PyRefMut<'_, Self>, + column: String, + index_type: String, + ) -> PyRefMut<'_, Self> { + slf.config.vector_index_types.insert(column, index_type); + slf + } fn with_limit(mut slf: PyRefMut<'_, Self>, limit: usize) -> PyRefMut<'_, Self> { slf.config.limit = Some(limit); slf @@ -662,14 +682,23 @@ for factory, expected in ( raise AssertionError('Unconfigured query was accepted') single = table.new_vector_search_builder().with_vector_column('embedding').with_query_vector([1., 0.]).with_limit(1) +for factory in (table.new_vector_search_builder, table.new_batch_vector_search_builder, table.new_hybrid_search_builder): + configured = factory() + # This API must only store configuration: neither a query nor a snapshot + # is available, and a superseded invalid implementation is accepted here. + assert configured.with_vector_index_type(column='embedding', index_type='invalid') is configured + assert configured.with_vector_index_type('embedding', 'ivf-flat') is configured +assert single.with_vector_index_type('embedding', 'ivf-flat') is single assert isinstance(single.execute(), sdk.GlobalIndexResult) assert len(single.execute()) == 0 batch = table.new_batch_vector_search_builder().with_vector_column('embedding').with_query_vectors([[1., 0.], [0., 1.]]).with_limit(1) +assert batch.with_vector_index_type('embedding', 'diskann') is batch results = batch.execute() assert len(results) == 2 and all(isinstance(result, sdk.GlobalIndexResult) and len(result) == 0 for result in results) text = table.new_full_text_search_builder().with_query('body', 'needle').with_limit(1) assert isinstance(text.execute(), sdk.GlobalIndexResult) and len(text.execute()) == 0 hybrid = table.new_hybrid_search_builder().add_vector_route('embedding', [1., 0.], 1).add_full_text_route('body', 'needle', 1).with_limit(1) +assert hybrid.with_vector_index_type('embedding', 'ivf-flat') is hybrid assert isinstance(hybrid.execute(), sdk.GlobalIndexResult) and len(hybrid.execute()) == 0 try: batch.with_query_vectors([]).execute() diff --git a/bindings/python/tests/test_vector_search.py b/bindings/python/tests/test_vector_search.py index 70b77cfc7..881bbe86d 100644 --- a/bindings/python/tests/test_vector_search.py +++ b/bindings/python/tests/test_vector_search.py @@ -158,6 +158,59 @@ def test_single_vector_keyword_matches_stub(table): assert builder.execute().row_ids() == _single(table).with_limit(1).execute().row_ids() +@pytest.mark.parametrize('batch', [False, True]) +def test_explicit_native_type_survives_scan_read_and_uses_raw_metric_options(table, batch): + import pickle + builder = (table.new_batch_vector_search_builder().with_query_vectors([[1, 0], [1, 0]]) + if batch else table.new_vector_search_builder().with_query_vector([1, 0])) + assert builder.with_vector_index_type(column='embedding', index_type='ivf-flat') is builder + builder.with_vector_column('embedding').with_limit(3) + plan = pickle.loads(pickle.dumps(builder.new_vector_search_scan().scan())) + reader = builder.new_batch_vector_search_read() if batch else builder.new_vector_search_read() + # Mutating the builder must not change captured reader configuration. + builder.with_vector_index_type('embedding', 'diskann').with_option('metric', 'l2') + results = reader.read_batch_plan(plan) if batch else [reader.read_plan(plan)] + for result in results: + assert sorted(result.row_ids().values()) == pytest.approx([0, 1, 1]) + results = builder.execute() if batch else [builder.execute()] + for result in results: + assert sorted(result.row_ids().values()) == pytest.approx([1 / 3, 1, 1]) + + +def test_native_type_keeps_index_metadata_and_ignores_removed_option_hints(table): + expected = _single(table).execute().row_ids() + for key in ('index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type'): + assert _single(table.copy({key: 'diskann'})).with_option(key, 'ivf-flat').execute().row_ids() == expected + (table.new_global_index_build_builder().with_index_column('embedding') + .with_index_type('ivf-flat').with_options({'dimension': '2', 'nlist': '1', 'metric': 'l2'}).execute()) + expected = _single(table).execute().row_ids() + assert (_single(table).with_vector_index_type('embedding', 'invalid-unused-type') + .execute().row_ids()) == expected + # Table defaults are not explicit query metrics. Both JSON namespaces obey + # that boundary when the persisted index uses a different metric. + for key, value in ( + ('ivf-flat.metric', 'cosine'), + ('fields.embedding.index-options', '{"metric":"cosine"}'), + ('fields.embedding.pk-vector.index.options', '{"metric":"cosine"}'), + ): + assert (_single(table.copy({key: value})).with_vector_index_type('embedding', 'diskann') + .execute().row_ids()) == expected + with pytest.raises(Exception, match='does not match index metric'): + _single(table).with_option(key, value).execute() + + +def test_hybrid_native_type_is_captured_with_the_route_builder(table): + hybrid = table.new_hybrid_search_builder() + assert hybrid.with_vector_index_type('embedding', 'ivf-flat') is hybrid + hybrid.add_vector_route('embedding', [1, 0], 3, options={ + 'ivf-flat.metric': 'l2', 'diskann.metric': 'inner_product'}).with_limit(3) + captured = hybrid.route_builders()[0].search_builder() + hybrid.with_vector_index_type('embedding', 'diskann') + assert sorted(captured.execute().row_ids().values()) == pytest.approx([1 / 3, 1, 1]) + updated = hybrid.route_builders()[0].search_builder() + assert sorted(updated.execute().row_ids().values()) == pytest.approx([0, 1, 1]) + + def test_external_result_uses_same_scan_type_without_search_context(table): found = _single(table).with_limit(1).execute() external = GlobalIndexResult.from_scored_row_ids(found.row_ids()) diff --git a/crates/paimon/src/spec/core_options/index_options.rs b/crates/paimon/src/spec/core_options/index_options.rs index 4fba8e8bf..7cffc5a60 100644 --- a/crates/paimon/src/spec/core_options/index_options.rs +++ b/crates/paimon/src/spec/core_options/index_options.rs @@ -103,7 +103,7 @@ impl CoreOptions<'_> { } /// Concrete declarations select the implementation; retain old PK and Rust - /// field declarations and the flat implementation aliases used by DLF. + /// field declarations. Query implementation hints belong to the builder API. pub fn vector_index_type(&self, column: &str) -> Result> { let declared = self .global_index_types() @@ -125,10 +125,6 @@ impl CoreOptions<'_> { Ok([ format!("fields.{column}.global-index.vector.index-type"), format!("fields.{column}.pk-vector.index.type"), - "index_type".into(), - "index-type".into(), - "vector.index-type".into(), - format!("fields.{column}.index-type"), ] .iter() .find_map(|key| self.options.get(key)) @@ -672,4 +668,64 @@ mod tests { assert_eq!(resolved["ivf-flat.dimension"], "2"); assert_eq!(resolved["ivf-flat.metric"], "l2"); } + + #[test] + fn removed_vector_type_hints_do_not_select_an_implementation() { + for key in [ + "index_type", + "index-type", + "vector.index-type", + "fields.embedding.index-type", + ] { + let mut values = map(&[(key, "diskann")]); + assert_eq!( + CoreOptions::new(&values) + .vector_index_type("embedding") + .unwrap(), + None + ); + values.insert( + "global-index.ivf-flat.index-columns".into(), + "embedding".into(), + ); + assert_eq!( + CoreOptions::new(&values) + .vector_index_type("embedding") + .unwrap() + .as_deref(), + Some("ivf-flat") + ); + } + } + + #[test] + fn short_field_options_replace_old_json_and_override_prefixed_defaults_without_query_metric() { + let values = map(&[ + ("global-index.ivf-pq.index-column", "embedding"), + ("ivf-pq.dimension", "8"), + ("ivf-pq.nlist", "16"), + ("fields.embedding.dimension", "4"), + ( + "fields.embedding.pk-vector.index.options", + r#"{"pq.m":3,"nlist":32}"#, + ), + ( + "fields.embedding.index-options", + r#"{"dimension":2,"nlist":1}"#, + ), + ]); + let resolved = CoreOptions::new(&values) + .vector_index_options("ivf-pq", "embedding", false) + .unwrap(); + assert_eq!(resolved["ivf-pq.nlist"], "1"); + assert_eq!(resolved["fields.embedding.dimension"], "2"); + assert!(!resolved.contains_key("ivf-pq.pq.m")); + assert!(!resolved.contains_key("ivf-pq.metric")); + assert!(!resolved.contains_key("fields.embedding.metric")); + let defaults = CoreOptions::new(&values) + .vector_index_options("ivf-pq", "embedding", true) + .unwrap(); + assert_eq!(defaults["ivf-pq.metric"], "inner_product"); + assert_eq!(values["ivf-pq.dimension"], "8"); + } } diff --git a/crates/paimon/src/table/batch_vector_search_builder.rs b/crates/paimon/src/table/batch_vector_search_builder.rs index 4853a27f3..871c9b6c1 100644 --- a/crates/paimon/src/table/batch_vector_search_builder.rs +++ b/crates/paimon/src/table/batch_vector_search_builder.rs @@ -34,6 +34,7 @@ pub struct BatchVectorSearchBuilder<'a> { query_vectors: Option>>, limit: Option, options: HashMap, + vector_index_types: HashMap, filter: Option, include_row_ids: Option>, prepared_filter: Option, @@ -47,6 +48,7 @@ impl<'a> BatchVectorSearchBuilder<'a> { query_vectors: None, limit: None, options: HashMap::new(), + vector_index_types: HashMap::new(), filter: None, include_row_ids: None, prepared_filter: None, @@ -58,6 +60,13 @@ impl<'a> BatchVectorSearchBuilder<'a> { self } + /// Store a raw-fallback implementation without planning or executing a query. + pub fn with_vector_index_type(&mut self, column: &str, index_type: &str) -> &mut Self { + self.vector_index_types + .insert(column.into(), index_type.into()); + self + } + pub fn with_query_vectors(&mut self, vectors: Vec>) -> &mut Self { self.query_vectors = Some(vectors); self @@ -122,19 +131,23 @@ impl<'a> BatchVectorSearchBuilder<'a> { self.include_row_ids.as_ref(), self.prepared_filter.as_ref(), ) - .map(|scan| scan.with_options(&self.options)) + .map(|scan| { + scan.with_options(&self.options) + .with_vector_index_type(self.vector_index_types.get(column).cloned()) + }) } /// Create an owned batch reader; result i belongs to input query i. pub fn new_read(&self) -> crate::Result { let column = self.column()?; - let context = PlanContext::new( + let mut context = PlanContext::new( self.table, column, self.filter.as_ref(), self.include_row_ids.as_ref(), self.prepared_filter.as_ref(), )?; + context.vector_index_type = self.vector_index_types.get(column).cloned(); let queries = self .query_vectors .as_deref() diff --git a/crates/paimon/src/table/de_vector_read.rs b/crates/paimon/src/table/de_vector_read.rs index 53ff69ff0..d438350bb 100644 --- a/crates/paimon/src/table/de_vector_read.rs +++ b/crates/paimon/src/table/de_vector_read.rs @@ -37,9 +37,9 @@ use crate::table::search_task::{ }; use crate::table::vector_read::Read; use crate::table::vector_search_common::{ - configured_refine_factor, configured_vector_index_type, indexed_search_limit, - log_vindex_range_io_stats, normalize_metric, read_index_vector_metric, - resolve_materialize_read_type, validate_global_vector_index_types, vindex_concurrency_limits, + configured_refine_factor, indexed_search_limit, log_vindex_range_io_stats, normalize_metric, + raw_vector_index_type, read_index_vector_metric, resolve_materialize_read_type, + validate_global_vector_index_types, vindex_concurrency_limits, }; use crate::table::{ find_field_id_by_name, merge_row_ranges, ArrowRecordBatchStream, RowRange, Table, @@ -73,6 +73,13 @@ pub(super) struct DeVectorRead { } impl DeVectorRead { + pub(super) fn with_vector_index_type(mut self, index_type: Option) -> Self { + for search in &mut self.vector_searches { + search.index_type = index_type.clone(); + } + self + } + /// Validate query parameters before planning, including for empty tables and /// filters. Snapshot-dependent row-ID filters are supplied by the plan. pub(super) fn new( @@ -907,8 +914,24 @@ async fn evaluate_batch_vector_search_with_executor( .map(|entry| (entry, None)) .collect::>() }; - let configured_index_type = - configured_vector_index_type(evaluation.table_options, &search_options, field_name)?; + let raw_index_type = sources + .into_iter() + .flatten() + .find_map(|source| match source { + crate::table::SearchScanSplit::Raw { index_type, .. } => index_type.as_deref(), + _ => None, + }) + .or(vector_searches[0].index_type.as_deref()); + let configured_index_type = if vector_entries.is_empty() { + raw_vector_index_type( + evaluation.table_options, + &search_options, + field_name, + raw_index_type, + )? + } else { + None + }; let index_type = vector_entries .first() .map(|entry| entry.index_file.index_type.as_str()) @@ -1132,32 +1155,16 @@ async fn evaluate_batch_vector_search_with_executor( source: None, })?; let metric_start = timing_enabled.then(Instant::now); - let mut metric_options = vector_searches[0].options.clone(); - if configured_vector_index_type(evaluation.table_options, &metric_options, field_name)? - .is_none() - { - if let Some(index_type) = - sources - .into_iter() - .flatten() - .find_map(|source| match source { - crate::table::SearchScanSplit::Raw { index_type, .. } => { - index_type.as_ref() - } - _ => None, - }) - { - metric_options.insert("index_type".into(), index_type.clone()); - } - } + let metric_options = &vector_searches[0].options; let metric = resolve_raw_vector_metric( evaluation.file_io, table_path, evaluation.table_options, - &metric_options, + metric_options, index_entries, field_id, field_name, + raw_index_type, ) .await?; let metric_resolve = metric_start.map_or(Duration::ZERO, |start| start.elapsed()); @@ -1445,6 +1452,7 @@ async fn maybe_rerank_indexed_batch_results( index_entries, field_id, field_name, + vector_searches[0].index_type.as_deref(), ) .await?; metric_check.check(metric)?; @@ -1672,6 +1680,7 @@ impl RawVectorMetric { } } +#[allow(clippy::too_many_arguments)] async fn resolve_raw_vector_metric( file_io: &FileIO, table_path: &str, @@ -1680,9 +1689,9 @@ async fn resolve_raw_vector_metric( index_entries: &[IndexManifestEntry], field_id: i32, field_name: &str, + explicit_index_type: Option<&str>, ) -> crate::Result { - let configured_type = configured_vector_index_type(table_options, search_options, field_name)?; - let index_type = index_entries + let persisted_type = index_entries .iter() .filter(|entry| entry.kind == FileKind::Add) .find(|entry| { @@ -1693,8 +1702,18 @@ async fn resolve_raw_vector_metric( .as_ref() .is_some_and(|meta| meta.index_field_id == field_id) }) - .map(|entry| entry.index_file.index_type.as_str()) - .or(configured_type.as_deref()); + .map(|entry| entry.index_file.index_type.as_str()); + let configured_type = if persisted_type.is_none() { + raw_vector_index_type( + table_options, + search_options, + field_name, + explicit_index_type, + )? + } else { + None + }; + let index_type = persisted_type.or(configured_type.as_deref()); let check = VectorMetricCheck { persisted: Arc::new(std::sync::Mutex::new(None)), requested: vector_metric_override(search_options, field_name, index_type)?, @@ -1722,23 +1741,21 @@ async fn resolve_raw_vector_metric( if let Some(metric) = check.requested { return Ok(metric); } - if let Some(kind) = super::global_index_options::vector_index_type(table_options, field_name)? { - if index_type == Some(kind) { - let options = super::global_index_options::field_index_options( - table_options, - field_name, - "vector", - kind, - )?; - let native = crate::vindex::VindexVectorIndexOptions::resolve_native_options( - table_options, - &options, - &HashMap::new(), - kind, - field_name, - )?; - return RawVectorMetric::parse(&native["metric"]); - } + if let Some(kind) = index_type { + let options = super::global_index_options::field_index_options( + table_options, + field_name, + "vector", + kind, + )?; + let native = crate::vindex::VindexVectorIndexOptions::resolve_native_options( + table_options, + &options, + &HashMap::new(), + kind, + field_name, + )?; + return RawVectorMetric::parse(&native["metric"]); } let table_metric = vector_metric_override(table_options, field_name, index_type)?; if let Some(metric) = table_metric { @@ -2632,6 +2649,8 @@ struct TaskVectorSearch { limit: usize, field: String, options: HashMap, + #[serde(default)] + index_type: Option, include: Option, } #[derive(Clone, Serialize, Deserialize)] @@ -2668,6 +2687,7 @@ impl TaskVectorQueries { limit: search.limit, field: search.field_name.clone(), options: search.options.clone(), + index_type: search.index_type.clone(), include, }) }) @@ -2698,6 +2718,7 @@ impl TaskVectorQueries { query.field.clone(), )? .with_options(query.options.clone()); + search.index_type = query.index_type.clone(); if let Some(index) = query.include { let shared = filters.get(index).ok_or_else(|| { crate::table::search_task::invalid( diff --git a/crates/paimon/src/table/de_vector_read/tests.rs b/crates/paimon/src/table/de_vector_read/tests.rs index c3e80c85f..404dd7534 100644 --- a/crates/paimon/src/table/de_vector_read/tests.rs +++ b/crates/paimon/src/table/de_vector_read/tests.rs @@ -245,6 +245,7 @@ fn scheduled_vector_transport_restores_one_shared_raw_filter_per_bitmap() { .collect::>(); for search in &mut searches { search.set_shared_include_row_ids(Arc::clone(&shared)); + search.index_type = Some("diskann".into()); } let wire = TaskVectorQueries::capture(&searches).unwrap(); assert_eq!( @@ -253,6 +254,10 @@ fn scheduled_vector_transport_restores_one_shared_raw_filter_per_bitmap() { "encode a shared bitmap once per query batch" ); let restored = wire.restore().unwrap(); + assert!(restored + .iter() + .all(|search| search.index_type.as_deref() == Some("diskann") + && !search.options.contains_key("index_type"))); let restored_shared = shared_batch_include_row_ids(&restored).unwrap(); assert_eq!(restored_shared.as_ref(), shared.as_ref()); let scoring = RawScoringPlan::new(&restored, RawVectorMetric::L2); @@ -482,7 +487,7 @@ async fn raw_search_without_files_uses_the_configured_indexer_default() { ("diskann", RawVectorMetric::InnerProduct), ] { let options = HashMap::from([( - "fields.embedding.index-type".to_string(), + "fields.embedding.pk-vector.index.type".to_string(), index_type.to_string(), )]); assert_eq!( @@ -494,6 +499,7 @@ async fn raw_search_without_files_uses_the_configured_indexer_default() { &[], 2, "embedding", + None, ) .await .unwrap(), @@ -509,6 +515,7 @@ async fn raw_search_without_files_uses_the_configured_indexer_default() { &[], 2, "embedding", + None, ) .await .unwrap(), @@ -517,7 +524,7 @@ async fn raw_search_without_files_uses_the_configured_indexer_default() { } let mut table_options = HashMap::from([ ( - "fields.embedding.index-type".to_string(), + "fields.embedding.pk-vector.index.type".to_string(), "ivf-flat".to_string(), ), ("fields.other.distance.metric".to_string(), "l2".to_string()), @@ -542,6 +549,7 @@ async fn raw_search_without_files_uses_the_configured_indexer_default() { &[], 2, "embedding", + None, ) .await .unwrap(), @@ -610,6 +618,7 @@ async fn indexed_vector_metric_ignores_unrelated_column_options() { &[entry], 2, "embedding", + None, ) .await .unwrap(), @@ -641,6 +650,7 @@ async fn test_resolve_raw_vector_metric_uses_vindex_manifest_metadata() { &[entry], 2, "embedding", + None, ) .await .unwrap(); @@ -680,6 +690,7 @@ async fn test_resolve_raw_vector_metric_falls_back_to_vindex_header() { &[entry], 2, "embedding", + None, ) .await .unwrap(); @@ -1493,6 +1504,7 @@ async fn raw_query_metric_options_precede_table_options() { &[], 2, "embedding", + None, ) .await .unwrap(); @@ -1510,6 +1522,7 @@ async fn raw_query_metric_options_precede_table_options() { &[], 2, "embedding", + None, ) .await .unwrap(); @@ -1901,6 +1914,7 @@ async fn scheduled_vector_selects_global_candidates_before_rerank() { &entries, find_field_id_by_name(table.schema().fields(), "embedding").unwrap(), "embedding", + None, ) .await .unwrap(); diff --git a/crates/paimon/src/table/full_text_search_builder/scan.rs b/crates/paimon/src/table/full_text_search_builder/scan.rs index 6d1fde5d5..f1102fb17 100644 --- a/crates/paimon/src/table/full_text_search_builder/scan.rs +++ b/crates/paimon/src/table/full_text_search_builder/scan.rs @@ -246,6 +246,7 @@ impl FullTextScan { self.context.filter.as_ref(), &Default::default(), true, + None, ) .await? }; diff --git a/crates/paimon/src/table/hybrid_search_builder.rs b/crates/paimon/src/table/hybrid_search_builder.rs index 1e2e12b81..41051e9a8 100644 --- a/crates/paimon/src/table/hybrid_search_builder.rs +++ b/crates/paimon/src/table/hybrid_search_builder.rs @@ -392,6 +392,7 @@ pub struct HybridSearchBuilder<'a> { ranker: HybridSearchRanker, filter: Option, pinned_snapshot: Option>, + vector_index_types: HashMap, } impl<'a> HybridSearchBuilder<'a> { @@ -403,6 +404,7 @@ impl<'a> HybridSearchBuilder<'a> { ranker: HybridSearchRanker::Rrf, filter: None, pinned_snapshot: None, + vector_index_types: HashMap::new(), } } @@ -411,6 +413,13 @@ impl<'a> HybridSearchBuilder<'a> { self } + /// Store each vector route's raw-fallback implementation, even before routes exist. + pub fn with_vector_index_type(&mut self, column: &str, index_type: &str) -> &mut Self { + self.vector_index_types + .insert(column.into(), index_type.into()); + self + } + pub fn add_vector_route( &mut self, field_name: &str, @@ -621,6 +630,9 @@ impl<'a> HybridSearchBuilder<'a> { super::VectorSearchBuilder::from_owned(table.clone()), route, self.filter.as_ref(), + self.vector_index_types + .get(&route.field_name) + .map(String::as_str), )) } HybridSearchRouteKind::FullText => { @@ -645,7 +657,14 @@ impl<'a> HybridSearchBuilder<'a> { table: &'t Table, route: &HybridSearchRoute, ) -> super::VectorSearchBuilder<'t> { - vector_route_builder(table, route, self.filter.as_ref()) + configure_vector_route_builder( + table.new_vector_search_builder(), + route, + self.filter.as_ref(), + self.vector_index_types + .get(&route.field_name) + .map(String::as_str), + ) } fn validate_global_routes(&self) -> crate::Result<()> { @@ -1226,24 +1245,20 @@ fn build_hybrid_indexed_splits( Ok(out) } -fn vector_route_builder<'t>( - table: &'t Table, - route: &HybridSearchRoute, - filter: Option<&Predicate>, -) -> super::VectorSearchBuilder<'t> { - configure_vector_route_builder(table.new_vector_search_builder(), route, filter) -} - fn configure_vector_route_builder<'t>( mut builder: super::VectorSearchBuilder<'t>, route: &HybridSearchRoute, filter: Option<&Predicate>, + index_type: Option<&str>, ) -> super::VectorSearchBuilder<'t> { builder .with_vector_column(&route.field_name) .with_query_vector(route.vector.clone().expect("validated vector route")) .with_limit(route.limit) .with_options(route.options.clone()); + if let Some(index_type) = index_type { + builder.with_vector_index_type(&route.field_name, index_type); + } if let Some(filter) = filter { builder.with_filter(filter.clone()); } diff --git a/crates/paimon/src/table/pk_vector_read.rs b/crates/paimon/src/table/pk_vector_read.rs index b41522f94..e7e426ca3 100644 --- a/crates/paimon/src/table/pk_vector_read.rs +++ b/crates/paimon/src/table/pk_vector_read.rs @@ -62,6 +62,7 @@ pub(super) struct PkVectorRead { queries: Vec>, limit: usize, params: PkVectorSearchParams, + raw_index_type: Option, } impl PkVectorRead { @@ -83,8 +84,14 @@ impl PkVectorRead { queries: queries.iter().map(|query| query.to_vec()).collect(), limit, params, + raw_index_type: None, } } + + pub(super) fn with_vector_index_type(mut self, index_type: Option) -> Self { + self.raw_index_type = index_type; + self + } } impl Read for PkVectorRead { @@ -93,6 +100,24 @@ impl Read for PkVectorRead { async fn read(&self, plan: PkVectorScanPlan) -> crate::Result> { let core = CoreOptions::new(self.table.schema().options()); let queries: Vec<&[f32]> = self.queries.iter().map(Vec::as_slice).collect(); + let mut params = self.params.clone(); + if let Some(kind) = &self.raw_index_type { + let raw_type = plan + .splits + .iter() + .all(|split| split.ann_segments.is_empty()) + .then_some(kind.as_str()); + params = PkVectorSearchParams::resolve_for_type( + &self.table, + &self.options, + self.filter.as_ref(), + &self.vector_column, + &queries, + self.limit, + raw_type, + true, + )?; + } let candidates = search_pk_candidates_batch_with_plan( &self.table, &self.options, @@ -102,7 +127,7 @@ impl Read for PkVectorRead { &queries, self.limit, &plan, - &self.params, + ¶ms, ) .await?; let table = Arc::new(self.table.clone()); @@ -114,7 +139,7 @@ impl Read for PkVectorRead { plan.snapshot_id, candidates, &plan.splits, - self.params.metric, + params.metric, ) }) .collect() diff --git a/crates/paimon/src/table/pk_vector_search_params.rs b/crates/paimon/src/table/pk_vector_search_params.rs index b469b92a1..6019dbe78 100644 --- a/crates/paimon/src/table/pk_vector_search_params.rs +++ b/crates/paimon/src/table/pk_vector_search_params.rs @@ -30,6 +30,7 @@ use std::collections::HashMap; /// from the table schema, the options and the queries alone, independent of which /// splits planning yields. Resolved before planning so a malformed query or option /// fails loud even when the plan turns out empty. +#[derive(Clone)] pub(super) struct PkVectorSearchParams { pub(super) metric: VectorSearchMetric, /// Fan-out limit for bucket orchestration plus ANN and exact-file leaves (Java @@ -52,6 +53,29 @@ impl PkVectorSearchParams { pk_col: &str, queries: &[&[f32]], limit: usize, + ) -> crate::Result { + Self::resolve_for_type( + table, + query_options, + filter, + pk_col, + queries, + limit, + None, + true, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn resolve_for_type( + table: &Table, + query_options: &HashMap, + filter: Option<&Predicate>, + pk_col: &str, + queries: &[&[f32]], + limit: usize, + index_type: Option<&str>, + validate_dimension: bool, ) -> crate::Result { let core = CoreOptions::new(table.schema().options()); core.ensure_read_authorized()?; @@ -90,15 +114,46 @@ impl PkVectorSearchParams { source: None, }); } - // `primary_key_vector_distance_metric` returns a validated name; re-parse into - // the enum for the numeric semantics. - let metric = VectorSearchMetric::parse(&core.primary_key_vector_distance_metric(pk_col)?)?; // Fan-out limit for bucket orchestration plus ANN and exact-file leaves (Java // `GLOBAL_INDEX_THREAD_NUM`); `1` reproduces strictly sequential execution. let concurrency = core.global_index_thread_num()?; - let index_type = + let raw_type_override = index_type.is_some(); + let index_type = if let Some(kind) = index_type { + super::vector_search_common::raw_vector_index_type( + table.schema().options(), + query_options, + pk_col, + Some(kind), + )? + .unwrap() + } else { super::pk_index_options::vector_index_type(table.schema().options(), pk_col)? - .to_string(); + .to_string() + }; + let metric = if raw_type_override { + let mut configured_metric = None; + for options in [query_options, table.schema().options()] { + let resolved = + CoreOptions::new(options).vector_index_options(&index_type, pk_col, false)?; + configured_metric = [ + format!("fields.{pk_col}.distance.metric"), + format!("fields.{pk_col}.metric"), + format!("{index_type}.metric"), + "metric".into(), + "distance.metric".into(), + ] + .iter() + .find_map(|key| resolved.get(key).cloned()); + if configured_metric.is_some() { + break; + } + } + VectorSearchMetric::parse(configured_metric.as_deref().unwrap_or("inner_product"))? + } else { + // PK readers retain Java's table metric validation. The explicit + // implementation affects only plans with no ANN metadata at all. + VectorSearchMetric::parse(&core.primary_key_vector_distance_metric(pk_col)?)? + }; let vector_field = table .schema() .fields() @@ -141,14 +196,21 @@ impl PkVectorSearchParams { // ARRAY gets the index dimension from the same vindex option resolver // used by index reads. Both valid PK-vector column shapes must reject NaN/Inf // up front, not only after a non-empty plan opens readers. - if let Some(dimension) = pk_vector_query_dimension( - table.schema().options(), - query_options, - &index_type, - &vector_field, - )? { - for query in queries { - validate_query(query, dimension)?; + // Only ARRAY's implementation-dependent width waits for the source + // plan. Fixed schema widths, emptiness and finiteness remain eager. + for query in queries { + validate_query(query, query.len().max(1))?; + } + if validate_dimension || matches!(vector_field.data_type(), DataType::Vector(_)) { + if let Some(dimension) = pk_vector_query_dimension( + table.schema().options(), + query_options, + &index_type, + &vector_field, + )? { + for query in queries { + validate_query(query, dimension)?; + } } } @@ -178,6 +240,11 @@ fn pk_vector_query_dimension( } DataType::Array(array_type) if matches!(array_type.element_type(), DataType::Float(_)) => { // ARRAY carries its dimension in the vindex options. + let query_options = CoreOptions::new(query_options).vector_index_options( + index_type, + vector_field.name(), + false, + )?; let mut dimension_options = HashMap::new(); for key in [ "dimension".to_string(), diff --git a/crates/paimon/src/table/pk_vector_search_params/tests.rs b/crates/paimon/src/table/pk_vector_search_params/tests.rs index d5cf66789..0b7787ae5 100644 --- a/crates/paimon/src/table/pk_vector_search_params/tests.rs +++ b/crates/paimon/src/table/pk_vector_search_params/tests.rs @@ -58,6 +58,21 @@ fn raw_query_dimensions_remain_supported_with_java_build_option_mapping() { pk_vector_query_dimension(&table, &query, index_type, &field).unwrap(), Some(2) ); + let query = HashMap::from([ + ("dimension".into(), "7".into()), + ( + "fields.embedding.pk-vector.index.options".into(), + "invalid old JSON".into(), + ), + ( + "fields.embedding.index-options".into(), + r#"{"dimension":2}"#.into(), + ), + ]); + assert_eq!( + pk_vector_query_dimension(&table, &query, index_type, &field).unwrap(), + Some(2) + ); let table = HashMap::from([ ("dimension".into(), "8".into()), (format!("{index_type}.index.dimension"), "16".into()), diff --git a/crates/paimon/src/table/search_scan_split.rs b/crates/paimon/src/table/search_scan_split.rs index 8b3497f91..25f0672d4 100644 --- a/crates/paimon/src/table/search_scan_split.rs +++ b/crates/paimon/src/table/search_scan_split.rs @@ -21,7 +21,7 @@ use super::bucket_filter::split_partition_and_data_predicates; use super::global_index_scanner::unindexed_ranges_for_global_index_entries; use super::global_index_types::normalize_queryable_global_index_type; use super::row_id_predicate::intersect_sorted_ranges; -use super::vector_search_common::configured_vector_index_type; +use super::vector_search_common::raw_vector_index_type; use super::{find_field_id_by_name, merge_row_ranges, DataSplit, RowRange, Table}; use crate::spec::{ CoreOptions, FileKind, GlobalIndexSearchMode, IndexFileMeta, IndexManifestEntry, Predicate, @@ -60,6 +60,7 @@ pub(super) async fn global_search_splits( filter: Option<&Predicate>, options: &HashMap, full_text: bool, + vector_index_type: Option<&str>, ) -> crate::Result> { let field_id = find_field_id_by_name(table.schema().fields(), column).ok_or_else(|| { crate::Error::ConfigInvalid { @@ -211,7 +212,9 @@ pub(super) async fn global_search_splits( let index_type = match index_type { Some(kind) => Some(kind), None if full_text => None, - None => configured_vector_index_type(table.schema().options(), options, column)?, + None => { + raw_vector_index_type(table.schema().options(), options, column, vector_index_type)? + } }; result.push(SearchScanSplit::Raw { scalar_index_files: intersecting_scalar(&raw_ranges), diff --git a/crates/paimon/src/table/vector_read.rs b/crates/paimon/src/table/vector_read.rs index fe99cba87..a89338c7f 100644 --- a/crates/paimon/src/table/vector_read.rs +++ b/crates/paimon/src/table/vector_read.rs @@ -89,13 +89,22 @@ impl BatchVectorRead { let core = CoreOptions::new(table.schema().options()); let reader = if targets_primary_key_column(table, column) { let pk_col = core.primary_key_vector_index_column()?; - let params = - PkVectorSearchParams::resolve(table, options, filter, &pk_col, queries, limit)?; - VectorReadKind::PrimaryKey(Box::new(PkVectorRead::new( - table, options, filter, &pk_col, queries, limit, params, - ))) + let params = if context.vector_index_type.is_some() { + PkVectorSearchParams::resolve_for_type( + table, options, filter, &pk_col, queries, limit, None, false, + )? + } else { + PkVectorSearchParams::resolve(table, options, filter, &pk_col, queries, limit)? + }; + VectorReadKind::PrimaryKey(Box::new( + PkVectorRead::new(table, options, filter, &pk_col, queries, limit, params) + .with_vector_index_type(context.vector_index_type.clone()), + )) } else { - VectorReadKind::DataEvolution(DeVectorRead::new(column, queries, limit, options)?) + VectorReadKind::DataEvolution( + DeVectorRead::new(column, queries, limit, options)? + .with_vector_index_type(context.vector_index_type.clone()), + ) }; Ok(Self { context, diff --git a/crates/paimon/src/table/vector_scan.rs b/crates/paimon/src/table/vector_scan.rs index c0201b2c1..a7fb4f7a6 100644 --- a/crates/paimon/src/table/vector_scan.rs +++ b/crates/paimon/src/table/vector_scan.rs @@ -189,6 +189,7 @@ async fn primary_key_sources( #[derive(Clone)] pub(super) struct PlanContext { pub(super) column: String, + pub(super) vector_index_type: Option, pub(super) filter: Option, pub(super) include_row_ids: Option>, pub(super) prepared_snapshot: Option>, @@ -231,6 +232,7 @@ impl PlanContext { } Ok(Self { column: column.to_string(), + vector_index_type: None, filter: filter.cloned(), include_row_ids: prepared .map(PreparedVectorSearchFilter::include_row_ids) @@ -301,6 +303,11 @@ impl VectorScan { self } + pub(super) fn with_vector_index_type(mut self, index_type: Option) -> Self { + self.context.vector_index_type = index_type; + self + } + pub async fn plan(&self) -> crate::Result { match &self.scan { VectorScanKind::DataEvolution(scan) => { @@ -325,6 +332,7 @@ impl VectorScan { self.context.filter.as_ref(), &self.options, false, + self.context.vector_index_type.as_deref(), ) .await? }; diff --git a/crates/paimon/src/table/vector_search_builder.rs b/crates/paimon/src/table/vector_search_builder.rs index 6b119964f..a9c6036e2 100644 --- a/crates/paimon/src/table/vector_search_builder.rs +++ b/crates/paimon/src/table/vector_search_builder.rs @@ -43,6 +43,7 @@ struct VectorSearchConfig { query_vector: Option>, limit: Option, options: HashMap, + vector_index_types: HashMap, filter: Option, } @@ -74,6 +75,17 @@ impl<'a> VectorSearchBuilder<'a> { self } + /// Store the implementation used when no vector index metadata is available. + /// Does not plan or execute a query. Repeated calls replace the column's hint. + pub fn with_vector_index_type(&mut self, column: &str, index_type: &str) -> &mut Self { + self.config + .write() + .unwrap() + .vector_index_types + .insert(column.into(), index_type.into()); + self + } + pub fn with_query_vector(&mut self, vector: Vec) -> &mut Self { self.config.write().unwrap().query_vector = Some(vector); self @@ -126,8 +138,10 @@ impl<'a> VectorSearchBuilder<'a> { .ok_or_else(|| crate::Error::ConfigInvalid { message: "Vector column must be set via with_vector_column()".to_string(), })?; - VectorScan::new(&self.table, column, config.filter.as_ref(), None, None) - .map(|scan| scan.with_options(&config.options)) + VectorScan::new(&self.table, column, config.filter.as_ref(), None, None).map(|scan| { + scan.with_options(&config.options) + .with_vector_index_type(config.vector_index_types.get(column).cloned()) + }) } /// Create an owned reader; query errors are reported before planning. @@ -138,7 +152,9 @@ impl<'a> VectorSearchBuilder<'a> { fn read_with_config(&self, config: &VectorSearchConfig) -> crate::Result { let (column, query, limit) = self.query(config)?; - let context = PlanContext::new(&self.table, column, config.filter.as_ref(), None, None)?; + let mut context = + PlanContext::new(&self.table, column, config.filter.as_ref(), None, None)?; + context.vector_index_type = config.vector_index_types.get(column).cloned(); Ok(VectorRead { batch: BatchVectorRead::new( &self.table, @@ -192,3 +208,6 @@ impl<'a> VectorSearchBuilder<'a> { #[cfg(test)] mod tests; + +#[cfg(test)] +mod native_type_tests; diff --git a/crates/paimon/src/table/vector_search_builder/native_type_tests.rs b/crates/paimon/src/table/vector_search_builder/native_type_tests.rs new file mode 100644 index 000000000..c5640aca4 --- /dev/null +++ b/crates/paimon/src/table/vector_search_builder/native_type_tests.rs @@ -0,0 +1,529 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::table::vector_search_test_utils::{de_vector_table, de_vector_table_with_metric}; +use crate::table::{ + SearchScanSplit, SearchTask, SearchTaskExecutor, SearchTaskResult, Table, VectorScanPlan, +}; +use std::collections::HashMap; + +async fn raw_table() -> Table { + let table = de_vector_table().await; + let snapshot = table.snapshot_manager().get_snapshot(1).await.unwrap(); + table + .copy_with_pinned_snapshot(Some(&snapshot)) + .copy_with_options(HashMap::from([( + "vector-index.search-mode".into(), + "full".into(), + )])) +} + +async fn two_vector_table(pk: bool, declaration: &str) -> Table { + use crate::catalog::Identifier; + use crate::io::FileIOBuilder; + use crate::spec::{ArrayType, DataType, FloatType, IntType, Schema, TableSchema, VectorType}; + use crate::table::{TableCommit, TableWrite}; + use arrow_array::types::Float32Type; + use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array, ListArray, RecordBatch}; + use arrow_schema::{Field, Schema as ArrowSchema}; + use std::sync::Arc; + let vector_type = if pk { + DataType::Vector(VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap()) + } else { + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))) + }; + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("embedding", vector_type.clone()) + .column("other", vector_type) + .option("write-only", "true") + .option("vector-index.search-mode", "full") + .option("ivf-flat.dimension", "3") + .option("diskann.dimension", "2"); + if pk { + schema = schema + .primary_key(["id"]) + .option("bucket", "1") + .option("deletion-vectors.enabled", "true") + .option(declaration, "embedding") + .option("fields.embedding.pk-vector.index.type", "ivf-flat"); + } else { + schema = schema + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("global-index.enabled", "true"); + } + let table = Table::new( + FileIOBuilder::new("memory").build().unwrap(), + Identifier::new("default", "native_type"), + format!("memory:/native-type-{}", uuid::Uuid::new_v4()), + TableSchema::new(0, &schema.build().unwrap()), + None, + ); + let vectors = ListArray::from_iter_primitive::([ + Some(vec![Some(3.0), Some(0.0)]), + Some(vec![Some(1.0), Some(1.0)]), + ]); + let vectors: ArrayRef = if pk { + Arc::new(FixedSizeListArray::new( + Arc::new(Field::new("element", arrow_schema::DataType::Float32, true)), + 2, + vectors.values().clone(), + None, + )) + } else { + Arc::new(vectors) + }; + let columns: Vec = vec![ + Arc::new(Int32Array::from(vec![1, 2])), + vectors.clone(), + vectors, + ]; + let arrow = Arc::new(ArrowSchema::new(vec![ + Field::new("id", arrow_schema::DataType::Int32, false), + Field::new("embedding", columns[1].data_type().clone(), true), + Field::new("other", columns[2].data_type().clone(), true), + ])); + let mut writer = TableWrite::new(&table, "native-type".into()).unwrap(); + writer + .write_arrow_batch(&RecordBatch::try_new(arrow, columns).unwrap()) + .await + .unwrap(); + let mut messages = writer.prepare_commit().await.unwrap(); + if pk { + // Java PK raw fallback reads eligible compacted sources, never L0. + for file in messages + .iter_mut() + .flat_map(|message| &mut message.new_files) + { + file.level = 1; + file.file_source = Some(1); + } + } + TableCommit::new(table.clone(), "native-type".into()) + .commit(messages) + .await + .unwrap(); + table +} + +fn raw_type(plan: &VectorScanPlan) -> Option<&str> { + plan.splits().iter().find_map(|source| match source { + SearchScanSplit::Raw { index_type, .. } => index_type.as_deref(), + _ => None, + }) +} + +struct Worker { + table: Table, + expected_type: String, +} + +#[async_trait::async_trait] +impl SearchTaskExecutor for Worker { + async fn execute(&self, tasks: Vec) -> crate::Result> { + let (context, frames) = SearchTask::serialize_batch(&tasks)?; + let mut results = Vec::new(); + for frame in frames { + let task = SearchTask::deserialize_with_context(&frame, &context)?; + let wire: serde_json::Value = serde_json::from_slice(&task.serialize()?).unwrap(); + for query in wire["payload"]["queries"]["queries"].as_array().unwrap() { + assert_eq!(query["index_type"], self.expected_type); + assert!(query["options"].get("index_type").is_none()); + assert!(query["options"].get("metric").is_none()); + } + results.push(SearchTaskResult::deserialize( + &task.execute(&self.table).await?.serialize()?, + )?); + } + results.reverse(); + Ok(results) + } +} + +#[tokio::test] +async fn native_type_is_captured_by_single_batch_plan_reader_and_worker() { + let table = raw_table().await; + let mut builder = table.new_vector_search_builder(); + // Store before configuring a query; unrelated columns and superseded hints + // cannot influence the selected vector column. + builder + .with_vector_index_type("other", "invalid") + .with_vector_index_type("embedding", "invalid") + .with_vector_index_type("embedding", "ivf-flat") + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(3); + let scan = builder.new_scan().unwrap(); + let reader = builder.new_read().unwrap(); + builder.with_vector_index_type("embedding", "diskann"); + let plan = scan.plan().await.unwrap(); + assert_eq!(raw_type(&plan), Some("ivf-flat")); + let restored: VectorScanPlan = + serde_json::from_slice(&serde_json::to_vec(&plan).unwrap()).unwrap(); + let local = reader.read(restored.clone()).await.unwrap(); + assert_eq!(local.row_ids().unwrap().scores, vec![1.0, 1.0, 0.0]); + let mut fresh = table.new_vector_search_builder(); + fresh + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(3); + let result = fresh + .new_read() + .unwrap() + .read(restored.clone()) + .await + .unwrap(); + assert_eq!( + result.row_ids().unwrap().scores, + local.row_ids().unwrap().scores + ); + // A transported raw source remains authoritative even if a fresh reader's + // fallback hint names a different parameter namespace. + fresh + .with_vector_index_type("embedding", "diskann") + .with_options(HashMap::from([ + ("ivf-flat.metric".into(), "l2".into()), + ("diskann.metric".into(), "cosine".into()), + ])); + let result = fresh + .new_read() + .unwrap() + .read(restored.clone()) + .await + .unwrap(); + assert_eq!(result.row_ids().unwrap().scores, vec![1.0, 0.5, 1.0 / 3.0]); + let worker = Worker { + table: table.clone(), + expected_type: "ivf-flat".into(), + }; + let remote = reader.read_with_executor(restored, &worker).await.unwrap(); + assert_eq!( + remote.row_ids().unwrap().row_ids, + local.row_ids().unwrap().row_ids + ); + assert_eq!( + remote.row_ids().unwrap().scores, + local.row_ids().unwrap().scores + ); + + let mut batch = table.new_batch_vector_search_builder(); + batch + .with_vector_index_type("embedding", "ivf-flat") + .with_vector_column("embedding") + .with_query_vectors(vec![vec![1.0, 0.0]; 2]) + .with_limit(3); + let scan = batch.new_scan().unwrap(); + let reader = batch.new_read().unwrap(); + batch.with_vector_index_type("embedding", "diskann"); + let results = reader + .read_with_executor(scan.plan().await.unwrap(), &worker) + .await + .unwrap(); + assert_eq!(results.len(), 2); + for result in results { + assert_eq!( + result.row_ids().unwrap().scores, + local.row_ids().unwrap().scores + ); + } +} + +#[tokio::test] +async fn native_type_raw_fallback_preserves_declarations_and_metric_options() { + let raw = raw_table().await; + for declaration in [ + "global-index.ivf-flat.index-columns", + "global-index.ivf-flat.index-column", + "pk-vector.index.columns", + ] { + let table = raw.copy_with_options(HashMap::from([ + (declaration.into(), "embedding".into()), + ( + "fields.embedding.pk-vector.index.type".into(), + "ivf-flat".into(), + ), + ( + "fields.embedding.pk-vector.index.options".into(), + "invalid old JSON".into(), + ), + ( + "fields.embedding.index-options".into(), + r#"{"dimension":2,"nlist":1}"#.into(), + ), + ])); + let mut builder = table.new_vector_search_builder(); + builder + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(3); + assert_eq!( + raw_type(&builder.new_scan().unwrap().plan().await.unwrap()), + Some("ivf-flat") + ); + assert_eq!( + builder.execute().await.unwrap().row_ids().unwrap().scores, + vec![1.0, 1.0, 0.0] + ); + builder + .with_vector_index_type("embedding", "diskann") + .with_options(HashMap::from([ + ("ivf-flat.metric".into(), "l2".into()), + ("diskann.metric".into(), "cosine".into()), + ])); + let result = builder.execute().await.unwrap(); + assert_eq!(result.row_ids().unwrap().row_ids, vec![0, 2, 1]); + assert!((result.row_ids().unwrap().scores[1] - 1.0_f32 / 2.0_f32.sqrt()).abs() < 1e-6); + } +} + +#[tokio::test] +async fn removed_type_options_do_not_change_raw_defaults() { + let table = raw_table().await; + for key in [ + "index_type", + "index-type", + "vector.index-type", + "fields.embedding.index-type", + ] { + let table = table.copy_with_options(HashMap::from([(key.into(), "diskann".into())])); + let mut builder = table.new_vector_search_builder(); + builder + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(3) + .with_options(HashMap::from([(key.into(), "ivf-flat".into())])); + assert_eq!( + raw_type(&builder.new_scan().unwrap().plan().await.unwrap()), + None + ); + let result = builder.execute().await.unwrap(); + assert_eq!(result.row_ids().unwrap().row_ids, vec![0, 2, 1]); + assert_eq!(result.row_ids().unwrap().scores, vec![1.0, 0.5, 1.0 / 3.0]); + } +} + +#[tokio::test] +async fn metadata_precedes_native_type_even_with_partial_raw_coverage() { + let table = de_vector_table_with_metric(Some("l2")) + .await + .copy_with_options(HashMap::from([ + ("vector-index.search-mode".into(), "full".into()), + ("diskann.metric".into(), "cosine".into()), + ])); + let mut builder = table.new_vector_search_builder(); + builder + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(3) + .with_options(HashMap::from([("ivf.nprobe".into(), "2".into())])); + let expected = builder.execute().await.unwrap(); + assert_eq!( + expected.row_ids().unwrap().scores, + vec![1.0, 0.5, 1.0 / 3.0] + ); + for hint in ["diskann", "invalid-unused-type"] { + builder.with_vector_index_type("embedding", hint); + let plan = builder.new_scan().unwrap().plan().await.unwrap(); + let result = builder + .new_read() + .unwrap() + .read(plan.clone()) + .await + .unwrap(); + assert_eq!( + result.row_ids().unwrap().scores, + expected.row_ids().unwrap().scores + ); + // Keep one authoritative indexed source and add raw coverage: the raw + // range must use persisted semantics too, without interpreting the hint. + let mut splits = plan.splits().to_vec(); + splits.push(SearchScanSplit::Raw { + row_ranges: vec![crate::table::RowRange::new(0, 2)], + index_type: Some("ivf-flat".into()), + scalar_index_files: vec![], + }); + let result = builder + .new_read() + .unwrap() + .read(VectorScanPlan::new(plan.snapshot().cloned(), splits).unwrap()) + .await + .unwrap(); + assert_eq!( + result.row_ids().unwrap().row_ids, + expected.row_ids().unwrap().row_ids + ); + assert_eq!( + result.row_ids().unwrap().scores, + expected.row_ids().unwrap().scores + ); + } +} + +#[tokio::test] +async fn native_type_pk_raw_fallback_uses_its_dimension_metric_and_hybrid_route() { + use futures::TryStreamExt; + for declaration in [ + "global-index.ivf-flat.index-columns", + "global-index.ivf-flat.index-column", + "pk-vector.index.columns", + ] { + let table = two_vector_table(true, declaration).await; + for query in [ + vec![], + vec![1.0], + vec![f32::NAN, 0.0], + vec![0.0, f32::INFINITY], + ] { + let mut invalid = table.new_vector_search_builder(); + invalid + .with_vector_index_type("embedding", "diskann") + .with_vector_column("embedding") + .with_query_vector(query) + .with_limit(1); + assert!( + invalid.new_read().is_err(), + "fixed vector errors must precede planning" + ); + } + let options = HashMap::from([("diskann.metric".into(), "cosine".into())]); + let mut builder = table.new_vector_search_builder(); + builder + .with_vector_index_type("embedding", "diskann") + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(1); + let plan = builder.new_scan().unwrap().plan().await.unwrap(); + assert!(!plan.splits().is_empty()); + assert!(plan.splits().iter().all(|source| matches!(source, SearchScanSplit::PrimaryKey { payloads, .. } if payloads.is_empty()))); + let result = builder + .new_read() + .unwrap() + .read(plan.clone()) + .await + .unwrap(); + assert_eq!(result.positions().unwrap().len(), 1); + assert_eq!(result.positions().unwrap()[0].score, 3.0); + builder.with_options(options.clone()); + let result = builder.new_read().unwrap().read(plan).await.unwrap(); + assert_eq!(result.positions().unwrap().len(), 1); + assert_eq!(result.positions().unwrap()[0].score, 1.0); + let mut batch = table.new_batch_vector_search_builder(); + batch + .with_vector_index_type("embedding", "diskann") + .with_vector_column("embedding") + .with_query_vectors(vec![vec![1.0, 0.0]; 2]) + .with_limit(1) + .with_options(options.clone()); + for result in batch.execute().await.unwrap() { + assert_eq!(result.positions().unwrap()[0].score, 1.0); + } + let mut hybrid = table.new_hybrid_search_builder(); + hybrid.with_vector_index_type("embedding", "diskann"); + hybrid + .add_vector_route("embedding", vec![1.0, 0.0], 1, 1.0, options) + .unwrap() + .with_limit(1); + let batches = hybrid + .execute_read() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 1 + ); + assert_eq!( + batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 1 + ); + } +} + +#[test] +fn native_type_does_not_defer_array_query_emptiness_or_finiteness() { + let table = crate::table::vector_search_test_utils::pk_vector_table(&[ + ("pk-vector.index.columns", "embedding"), + ("fields.embedding.pk-vector.index.type", "ivf-flat"), + ]); + for query in [vec![], vec![f32::NAN, 0.0], vec![0.0, f32::INFINITY]] { + let mut builder = table.new_vector_search_builder(); + builder + .with_vector_index_type("embedding", "diskann") + .with_vector_column("embedding") + .with_query_vector(query) + .with_limit(1); + assert!(builder.new_read().is_err()); + } +} + +#[tokio::test] +async fn native_type_hybrid_is_per_column_and_captured_before_builder_changes() { + let table = two_vector_table(false, "unused").await; + let options = HashMap::from([ + ("ivf-flat.metric".into(), "l2".into()), + ("diskann.metric".into(), "cosine".into()), + ]); + let mut hybrid = table.new_hybrid_search_builder(); + hybrid + .with_vector_index_type("embedding", "ivf-flat") + .with_vector_index_type("other", "diskann") + .with_vector_index_type("embedding", "diskann"); + hybrid + .add_vector_route("other", vec![1.0, 0.0], 1, 1.0, options.clone()) + .unwrap(); + hybrid + .add_vector_route("embedding", vec![1.0, 0.0], 1, 1.0, options) + .unwrap() + .with_limit(2); + // Give embedding a different implementation, independent of route order. + hybrid.with_vector_index_type("embedding", "ivf-flat"); + let routes = hybrid.route_builders().await.unwrap(); + hybrid + .with_vector_index_type("embedding", "diskann") + .with_vector_index_type("other", "ivf-flat"); + for route in routes { + let builder = route.vector_search_builder().unwrap(); + let plan = builder.new_scan().unwrap().plan().await.unwrap(); + let (kind, row_id) = if route.route().field_name() == "other" { + ("diskann", 0) + } else { + ("ivf-flat", 1) + }; + assert_eq!(raw_type(&plan), Some(kind)); + let result = route.execute().await.unwrap(); + assert_eq!(result.row_ids().unwrap().row_ids, vec![row_id]); + let worker = Worker { + table: table.clone(), + expected_type: kind.into(), + }; + let remote = builder + .new_read() + .unwrap() + .read_with_executor(plan, &worker) + .await + .unwrap(); + assert_eq!(remote.row_ids().unwrap().row_ids, vec![row_id]); + } +} diff --git a/crates/paimon/src/table/vector_search_common.rs b/crates/paimon/src/table/vector_search_common.rs index bac1035d0..e1fd9bc55 100644 --- a/crates/paimon/src/table/vector_search_common.rs +++ b/crates/paimon/src/table/vector_search_common.rs @@ -202,25 +202,32 @@ pub(super) fn configured_vector_index_type( query_options: &HashMap, column: &str, ) -> crate::Result> { - let keys = [ - "index_type".to_string(), - "index-type".into(), - "vector.index-type".into(), - format!("fields.{column}.index-type"), - ]; - if let Some(value) = keys.iter().find_map(|key| query_options.get(key)) { - return Ok(Some(value.trim().to_ascii_lowercase())); - } if let Some(kind) = super::global_index_options::vector_index_type(query_options, column)? { return Ok(Some(kind.into())); } if let Some(kind) = super::global_index_options::vector_index_type(table_options, column)? { return Ok(Some(kind.into())); } - Ok(keys - .iter() - .find_map(|key| table_options.get(key)) - .map(|value| value.trim().to_ascii_lowercase())) + Ok(None) +} + +/// Resolve only after discovering that no persisted vector implementation exists. +pub(super) fn raw_vector_index_type( + table_options: &HashMap, + query_options: &HashMap, + column: &str, + explicit: Option<&str>, +) -> crate::Result> { + let Some(kind) = explicit else { + return configured_vector_index_type(table_options, query_options, column); + }; + let kind = kind.trim().to_ascii_lowercase(); + if !is_vindex_index_type(&kind) { + return Err(crate::Error::ConfigInvalid { + message: format!("Unsupported vector index type for {column}: {kind}"), + }); + } + Ok(Some(kind)) } pub(super) fn current_tokio_runtime_handle() -> crate::Result { diff --git a/crates/paimon/src/table/vector_search_test_utils.rs b/crates/paimon/src/table/vector_search_test_utils.rs index 257a18973..b8794a26b 100644 --- a/crates/paimon/src/table/vector_search_test_utils.rs +++ b/crates/paimon/src/table/vector_search_test_utils.rs @@ -122,8 +122,12 @@ pub(super) fn pk_vector_table(options: &[(&str, &str)]) -> Table { /// returned table has one committed batch of `(id, embedding)` rows and a real /// vindex index built end-to-end. pub(super) async fn de_vector_table() -> Table { + de_vector_table_with_metric(None).await +} + +pub(super) async fn de_vector_table_with_metric(metric: Option<&str>) -> Table { let table_path = "memory:/de_vector_search_test"; - let schema = Schema::builder() + let mut schema = Schema::builder() .column("id", DataType::Int(IntType::new())) .column( "embedding", @@ -134,9 +138,11 @@ pub(super) async fn de_vector_table() -> Table { .option("global-index.enabled", "true") .option("global-index.row-count-per-shard", "10") .option("ivf-flat.dimension", "2") - .option("ivf-flat.nlist", "2") - .build() - .unwrap(); + .option("ivf-flat.nlist", "2"); + if let Some(metric) = metric { + schema = schema.option("ivf-flat.metric", metric); + } + let schema = schema.build().unwrap(); let file_io = FileIOBuilder::new("memory").build().unwrap(); let table = Table::new( file_io.clone(), diff --git a/crates/paimon/src/vector_search.rs b/crates/paimon/src/vector_search.rs index dc5327857..7432499d1 100644 --- a/crates/paimon/src/vector_search.rs +++ b/crates/paimon/src/vector_search.rs @@ -27,6 +27,8 @@ pub struct VectorSearch { pub limit: usize, pub field_name: String, pub options: HashMap, + /// Implementation hint for raw fallback; persisted index metadata takes precedence. + pub(crate) index_type: Option, pub include_row_ids: Option, pub(crate) shared_include_row_ids: Option>, } @@ -56,6 +58,7 @@ impl VectorSearch { limit, field_name, options: HashMap::new(), + index_type: None, include_row_ids: None, shared_include_row_ids: None, }) diff --git a/crates/paimon/tests/pk_compact/indexes.rs b/crates/paimon/tests/pk_compact/indexes.rs index 8e4938976..2f141b2fc 100644 --- a/crates/paimon/tests/pk_compact/indexes.rs +++ b/crates/paimon/tests/pk_compact/indexes.rs @@ -242,6 +242,7 @@ async fn empty_vector_level_is_persisted_reused_and_searchable() { commit.commit_compact(first).await.unwrap(); let result = table .new_vector_search_builder() + .with_vector_index_type("embedding", "invalid-unused-type") .with_vector_column("embedding") .with_query_vector(vec![0., 0.]) .with_limit(1) diff --git a/crates/paimon/tests/pk_vector_baseline_test.rs b/crates/paimon/tests/pk_vector_baseline_test.rs index 66f3d7bfa..39a004d20 100644 --- a/crates/paimon/tests/pk_vector_baseline_test.rs +++ b/crates/paimon/tests/pk_vector_baseline_test.rs @@ -589,6 +589,7 @@ async fn pk_vector_end_to_end_returns_expected_row_ids_and_scores() { // Search-only PK results preserve local positions and metric scores. let scored = table .new_vector_search_builder() + .with_vector_index_type(VECTOR_COLUMN, "invalid-unused-type") .with_vector_column(VECTOR_COLUMN) .with_query_vector(query.to_vec()) .with_limit(3) diff --git a/docs/src/compaction.md b/docs/src/compaction.md index b2b4b0b01..caa5c8a16 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -154,6 +154,21 @@ overrides table implementation defaults. Vector metrics default to `inner_product`, and accepted metric aliases resolve consistently for builds, indexed reads, raw scoring and refinement. +Query builders expose `with_vector_index_type(column, index_type)` in Rust and +Python for single, batch and hybrid vector searches. This stores a per-column +implementation hint without planning a query. Existing index metadata takes +precedence; the hint selects the implementation for raw fallback when no vector +index metadata is available. Captured scans, readers, hybrid routes and worker +transport retain it. Ordinary parameters still use `with_options`, and readers +leave an unspecified metric unset so global index metadata can supply it. PK +readers retain Java's table metric validation. + +The option aliases `index_type`, `index-type`, `vector.index-type` and +`fields..index-type` no longer select vector implementations. Use the +shared declaration, the retained PK declaration, or the explicit builder API. +The native training parameter `index.type` and typed index metadata fields are +unchanged. + Compatibility and precedence follow Java's shared configuration resolver: - Declarations prefer `global-index..index-columns`, then the DLF singular From 0c00d1f85a71ad67c7540bcd6bc18f386d2a561f Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 11:45:06 +0800 Subject: [PATCH 11/14] refactor(index): keep legacy PK options as internal compatibility Keep PK-specific CoreOptions convenience accessors crate-internal and route public callers through the shared global/vector index option APIs. Resolve a legacy vector field type only when pk-vector.index.columns names that column. Preserve complete old configurations and canonical precedence, and retain the native explicit-type API and logical VectorType. Document the compatibility-only boundary and cover orphan field keys, complete declarations and raw fallback scoring. --- crates/paimon/src/spec/core_options.rs | 16 ++-- .../src/spec/core_options/index_options.rs | 80 ++++++++++++++++--- .../paimon/src/table/de_vector_read/tests.rs | 18 ++++- .../native_type_tests.rs | 1 + crates/paimon/tests/pk_compact/indexes.rs | 4 +- docs/src/compaction.md | 26 +++--- 6 files changed, 108 insertions(+), 37 deletions(-) diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index 390adeac6..55c7e8078 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -2169,14 +2169,16 @@ impl<'a> CoreOptions<'a> { .unwrap_or_default() } + // PK layout adapters remain internal. Public configuration uses the shared + // global_index_* and vector_index_* resolvers, including legacy fallback. /// Whether a shared or legacy vector declaration is present. - pub fn primary_key_vector_index_enabled(&self) -> bool { + pub(crate) fn primary_key_vector_index_enabled(&self) -> bool { !self.vector_index_columns().is_empty() } /// The configured PK-vector index columns, split on ',' and trimmed. Blank tokens /// are preserved so schema validation can reject malformed column lists. - pub fn primary_key_vector_index_columns(&self) -> crate::Result> { + pub(crate) fn primary_key_vector_index_columns(&self) -> crate::Result> { let columns = self.vector_index_columns(); if columns.is_empty() { return Err(crate::Error::ConfigInvalid { @@ -2187,7 +2189,7 @@ impl<'a> CoreOptions<'a> { } /// The single PK-vector index column. The first release supports exactly one. - pub fn primary_key_vector_index_column(&self) -> crate::Result { + pub(crate) fn primary_key_vector_index_column(&self) -> crate::Result { let mut columns = self.primary_key_vector_index_columns()?; if columns.len() != 1 { return Err(crate::Error::ConfigInvalid { @@ -2202,7 +2204,7 @@ impl<'a> CoreOptions<'a> { /// The index type for a PK-vector column. Required — planning and the index reader /// both need it, so an absent value is a hard error rather than a guessed default. - pub fn primary_key_vector_index_type(&self, col: &str) -> crate::Result { + pub(crate) fn primary_key_vector_index_type(&self, col: &str) -> crate::Result { self.vector_index_type(col)? .ok_or_else(|| crate::Error::ConfigInvalid { message: format!("fields.{col}.pk-vector.index.type is required but not set"), @@ -2211,7 +2213,7 @@ impl<'a> CoreOptions<'a> { /// The distance metric name for a PK-vector column, defaulting to inner_product. /// Validated against the supported metrics; an unknown value is a hard error. - pub fn primary_key_vector_distance_metric(&self, col: &str) -> crate::Result { + pub(crate) fn primary_key_vector_distance_metric(&self, col: &str) -> crate::Result { if let Some(kind) = self.vector_index_type(col)? { return self .vector_index_options(&kind, col, true)? @@ -2239,7 +2241,7 @@ impl<'a> CoreOptions<'a> { } /// True when the PK full-text index column option key is present (any value). - pub fn primary_key_full_text_index_enabled(&self) -> bool { + pub(crate) fn primary_key_full_text_index_enabled(&self) -> bool { !self.primary_key_full_text_index_columns().is_empty() } @@ -2248,7 +2250,7 @@ impl<'a> CoreOptions<'a> { /// filter them) so parsing matches Java exactly; `[]` only when the key is /// absent. Never returns a `Result`, never errors (do not copy the fail-loud /// shape of `primary_key_vector_index_columns`). - pub fn primary_key_full_text_index_columns(&self) -> Vec { + pub(crate) fn primary_key_full_text_index_columns(&self) -> Vec { self.global_index_columns("full-text") } } diff --git a/crates/paimon/src/spec/core_options/index_options.rs b/crates/paimon/src/spec/core_options/index_options.rs index 7cffc5a60..d6b8fa2ad 100644 --- a/crates/paimon/src/spec/core_options/index_options.rs +++ b/crates/paimon/src/spec/core_options/index_options.rs @@ -102,8 +102,9 @@ impl CoreOptions<'_> { } } - /// Concrete declarations select the implementation; retain old PK and Rust - /// field declarations. Query implementation hints belong to the builder API. + /// Concrete declarations select the implementation. Legacy PK field types + /// apply only to columns listed by the legacy PK declaration; they are not + /// standalone implementation hints. Query hints belong to the builder API. pub fn vector_index_type(&self, column: &str) -> Result> { let declared = self .global_index_types() @@ -122,14 +123,22 @@ impl CoreOptions<'_> { if let Some(kind) = declared.into_iter().next() { return Ok(Some(kind)); } - Ok([ - format!("fields.{column}.global-index.vector.index-type"), - format!("fields.{column}.pk-vector.index.type"), - ] - .iter() - .find_map(|key| self.options.get(key)) - .map(|s| s.trim().to_ascii_lowercase()) - .filter(|s| !s.is_empty())) + let legacy_pk = columns(self.options.get("pk-vector.index.columns")) + .iter() + .any(|declared| declared == column); + Ok(self + .options + .get(&format!("fields.{column}.global-index.vector.index-type")) + .or_else(|| { + legacy_pk + .then(|| { + self.options + .get(&format!("fields.{column}.pk-vector.index.type")) + }) + .flatten() + }) + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty())) } /// Resolve implementation defaults and one field's JSON without mutating @@ -698,6 +707,57 @@ mod tests { } } + #[test] + fn legacy_pk_vector_type_requires_the_complete_column_declaration() { + let mut values = map(&[("fields.embedding.pk-vector.index.type", " IVF-FLAT ")]); + for columns in [None, Some(""), Some("other")] { + if let Some(columns) = columns { + values.insert("pk-vector.index.columns".into(), columns.into()); + } else { + values.remove("pk-vector.index.columns"); + } + assert_eq!( + CoreOptions::new(&values) + .vector_index_type("embedding") + .unwrap(), + None + ); + } + values.insert( + "pk-vector.index.columns".into(), + " other, embedding ".into(), + ); + assert_eq!( + CoreOptions::new(&values) + .vector_index_type("embedding") + .unwrap() + .as_deref(), + Some("ivf-flat") + ); + values.remove("fields.embedding.pk-vector.index.type"); + assert_eq!( + CoreOptions::new(&values) + .vector_index_type("embedding") + .unwrap(), + None + ); + values.insert( + "fields.embedding.pk-vector.index.type".into(), + "diskann".into(), + ); + for suffix in ["index-columns", "index-column"] { + values.insert(format!("global-index.ivf-pq.{suffix}"), "embedding".into()); + assert_eq!( + CoreOptions::new(&values) + .vector_index_type("embedding") + .unwrap() + .as_deref(), + Some("ivf-pq") + ); + values.remove(&format!("global-index.ivf-pq.{suffix}")); + } + } + #[test] fn short_field_options_replace_old_json_and_override_prefixed_defaults_without_query_metric() { let values = map(&[ diff --git a/crates/paimon/src/table/de_vector_read/tests.rs b/crates/paimon/src/table/de_vector_read/tests.rs index 404dd7534..bf28502a4 100644 --- a/crates/paimon/src/table/de_vector_read/tests.rs +++ b/crates/paimon/src/table/de_vector_read/tests.rs @@ -486,10 +486,16 @@ async fn raw_search_without_files_uses_the_configured_indexer_default() { (" IVF-FLAT ", RawVectorMetric::InnerProduct), ("diskann", RawVectorMetric::InnerProduct), ] { - let options = HashMap::from([( - "fields.embedding.pk-vector.index.type".to_string(), - index_type.to_string(), - )]); + let options = HashMap::from([ + ( + "pk-vector.index.columns".to_string(), + "embedding".to_string(), + ), + ( + "fields.embedding.pk-vector.index.type".to_string(), + index_type.to_string(), + ), + ]); assert_eq!( resolve_raw_vector_metric( &file_io, @@ -523,6 +529,10 @@ async fn raw_search_without_files_uses_the_configured_indexer_default() { ); } let mut table_options = HashMap::from([ + ( + "pk-vector.index.columns".to_string(), + "embedding".to_string(), + ), ( "fields.embedding.pk-vector.index.type".to_string(), "ivf-flat".to_string(), diff --git a/crates/paimon/src/table/vector_search_builder/native_type_tests.rs b/crates/paimon/src/table/vector_search_builder/native_type_tests.rs index c5640aca4..2b368d98d 100644 --- a/crates/paimon/src/table/vector_search_builder/native_type_tests.rs +++ b/crates/paimon/src/table/vector_search_builder/native_type_tests.rs @@ -299,6 +299,7 @@ async fn removed_type_options_do_not_change_raw_defaults() { "index-type", "vector.index-type", "fields.embedding.index-type", + "fields.embedding.pk-vector.index.type", ] { let table = table.copy_with_options(HashMap::from([(key.into(), "diskann".into())])); let mut builder = table.new_vector_search_builder(); diff --git a/crates/paimon/tests/pk_compact/indexes.rs b/crates/paimon/tests/pk_compact/indexes.rs index 2f141b2fc..0f0ca87f3 100644 --- a/crates/paimon/tests/pk_compact/indexes.rs +++ b/crates/paimon/tests/pk_compact/indexes.rs @@ -842,8 +842,8 @@ async fn legacy_field_metric_aliases_override_pk_metric_consistently() { table .schema() .core_options() - .primary_key_vector_distance_metric("embedding") - .unwrap(), + .vector_index_options("ivf-flat", "embedding", true) + .unwrap()["ivf-flat.metric"], "cosine" ); write_batch( diff --git a/docs/src/compaction.md b/docs/src/compaction.md index caa5c8a16..7cc253877 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -181,7 +181,9 @@ Compatibility and precedence follow Java's shared configuration resolver: Vector metric aliases are resolved together after JSON merging, so field JSON overrides legacy metric properties and case-only aliases stay compatible. - Legacy `pk-vector` declarations still select the algorithm through - `fields..pk-vector.index.type`. The former Rust + `fields..pk-vector.index.type`, only for columns named by + `pk-vector.index.columns`. The field type alone does not select an + implementation. The former Rust `global-index.vector` declaration and field algorithm/metric keys remain aliases. That older Rust declaration retains its metric/default priority over flat aliases until replaced by a concrete declaration or shared field JSON. @@ -356,19 +358,15 @@ snapshot expiration and storage reclamation are separate maintenance work. Primary-key compaction maintains declared source-backed indexes using the same `global-index..index-columns` and `fields..index-options` options -shown above. Existing Java primary-key configurations remain compatible: - -| Option | Meaning | -| --- | --- | -| `pk-btree.index.columns` | Comma-separated BTree columns. | -| `pk-bitmap.index.columns` | Comma-separated bitmap columns. | -| `pk-multivalue.index.columns` | Comma-separated array columns. | -| `pk-fm.index.columns` | Comma-separated FM string columns. | -| `pk-full-text.index.columns` | Comma-separated full-text string columns. | -| `pk-vector.index.columns` | Vector column, subject to the existing PK vector constraints. | -| `fields..pk-.index.options` | JSON object of native writer options. | -| `fields..pk-vector.index.type` | Vector algorithm, such as `ivf-flat`. | -| `fields..pk-vector.distance.metric` | Vector metric: `l2`, `cosine`, or `inner_product`. | +shown above. New code uses the shared `CoreOptions` index resolvers; PK-specific +option accessors are internal implementation details. + +Existing Java `pk-.index.columns` declarations, field JSON and metric +properties remain compatibility aliases. The legacy vector configuration pairs +`pk-vector.index.columns` with `fields..pk-vector.index.type`; new +configuration selects the implementation directly in +`global-index..index-columns`. No new PK-specific configuration API is +provided. Declarations are unset by default. With `write-only=false`, the normal automatic compaction round maintains them. Explicit compaction also maintains them with From 69efb11560dd77890d7874f087cba115c2468f21 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 13:21:58 +0800 Subject: [PATCH 12/14] refactor(vector): capture resolved index configuration --- crates/paimon/src/spec/core_options.rs | 126 +++---- .../src/spec/core_options/index_options.rs | 324 +++++++++++++----- crates/paimon/src/spec/schema.rs | 93 ++++- crates/paimon/src/table/de_vector_read.rs | 177 +++++----- .../paimon/src/table/de_vector_read/tests.rs | 10 + .../src/table/full_text_search_builder.rs | 2 +- .../paimon/src/table/global_index_options.rs | 77 +++-- .../paimon/src/table/hybrid_search_builder.rs | 10 +- crates/paimon/src/table/pk_index_options.rs | 22 +- crates/paimon/src/table/pk_vector_read.rs | 78 ++--- .../paimon/src/table/pk_vector_read/tests.rs | 58 +++- crates/paimon/src/table/pk_vector_scan.rs | 278 ++++++++------- .../src/table/pk_vector_search_params.rs | 271 ++++++--------- crates/paimon/src/table/search_scan_split.rs | 12 +- .../table/table_commit/auto_index_tests.rs | 2 +- .../table_commit/auto_index_tests/search.rs | 1 + crates/paimon/src/table/vector_read.rs | 12 +- crates/paimon/src/table/vector_scan.rs | 4 +- .../paimon/src/table/vector_search_common.rs | 148 +++++--- crates/paimon/src/vindex/pkvector/bucket.rs | 10 + crates/paimon/tests/pk_compact/indexes.rs | 5 +- .../paimon/tests/pk_vector_baseline_test.rs | 56 ++- docs/src/compaction.md | 7 +- 23 files changed, 1053 insertions(+), 730 deletions(-) diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index 55c7e8078..ee8f42c75 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -20,6 +20,7 @@ use std::collections::{HashMap, HashSet}; use crate::spec::TableType; mod index_options; +pub use index_options::VectorIndexOptions; const DELETION_VECTORS_ENABLED_OPTION: &str = "deletion-vectors.enabled"; const DELETION_VECTORS_MERGE_ON_READ_OPTION: &str = "deletion-vectors.merge-on-read"; @@ -2172,25 +2173,13 @@ impl<'a> CoreOptions<'a> { // PK layout adapters remain internal. Public configuration uses the shared // global_index_* and vector_index_* resolvers, including legacy fallback. /// Whether a shared or legacy vector declaration is present. - pub(crate) fn primary_key_vector_index_enabled(&self) -> bool { + pub(crate) fn vector_index_enabled(&self) -> bool { !self.vector_index_columns().is_empty() } - /// The configured PK-vector index columns, split on ',' and trimmed. Blank tokens - /// are preserved so schema validation can reject malformed column lists. - pub(crate) fn primary_key_vector_index_columns(&self) -> crate::Result> { - let columns = self.vector_index_columns(); - if columns.is_empty() { - return Err(crate::Error::ConfigInvalid { - message: "pk-vector.index.columns is not set".to_string(), - }); - } - Ok(columns) - } - /// The single PK-vector index column. The first release supports exactly one. - pub(crate) fn primary_key_vector_index_column(&self) -> crate::Result { - let mut columns = self.primary_key_vector_index_columns()?; + pub(crate) fn single_vector_index_column(&self) -> crate::Result { + let mut columns = self.vector_index_columns(); if columns.len() != 1 { return Err(crate::Error::ConfigInvalid { message: format!( @@ -2202,55 +2191,17 @@ impl<'a> CoreOptions<'a> { Ok(columns.remove(0)) } - /// The index type for a PK-vector column. Required — planning and the index reader - /// both need it, so an absent value is a hard error rather than a guessed default. - pub(crate) fn primary_key_vector_index_type(&self, col: &str) -> crate::Result { - self.vector_index_type(col)? - .ok_or_else(|| crate::Error::ConfigInvalid { - message: format!("fields.{col}.pk-vector.index.type is required but not set"), - }) - } - - /// The distance metric name for a PK-vector column, defaulting to inner_product. - /// Validated against the supported metrics; an unknown value is a hard error. - pub(crate) fn primary_key_vector_distance_metric(&self, col: &str) -> crate::Result { - if let Some(kind) = self.vector_index_type(col)? { - return self - .vector_index_options(&kind, col, true)? - .get(&format!("{kind}.metric")) - .cloned() - .ok_or_else(|| crate::Error::ConfigInvalid { - message: format!("Unsupported vector index type: {kind}"), - }); - } - self.legacy_primary_key_vector_distance_metric(col) - } - - pub(crate) fn legacy_primary_key_vector_distance_metric( - &self, - col: &str, - ) -> crate::Result { - let raw = self - .options - .get(&format!("fields.{col}.pk-vector.distance.metric")) - .map(|v| v.trim().to_string()) - .unwrap_or_else(|| "inner_product".to_string()); - // Validate now (fail-loud) without exposing the crate-private metric enum. - crate::vindex::pkvector::metric::VectorSearchMetric::parse(&raw)?; - Ok(raw) - } - /// True when the PK full-text index column option key is present (any value). - pub(crate) fn primary_key_full_text_index_enabled(&self) -> bool { - !self.primary_key_full_text_index_columns().is_empty() + pub(crate) fn full_text_index_enabled(&self) -> bool { + !self.full_text_index_columns().is_empty() } /// Configured PK full-text index columns: split on ',' and trim each token, /// mirroring Java `split(",",-1).map(trim)`. Blank tokens are PRESERVED (do NOT /// filter them) so parsing matches Java exactly; `[]` only when the key is /// absent. Never returns a `Result`, never errors (do not copy the fail-loud - /// shape of `primary_key_vector_index_columns`). - pub(crate) fn primary_key_full_text_index_columns(&self) -> Vec { + /// shape of a fallible vector-column resolver). + pub(crate) fn full_text_index_columns(&self) -> Vec { self.global_index_columns("full-text") } } @@ -4327,7 +4278,7 @@ mod tests { #[test] fn test_pk_vector_index_disabled_by_default() { let opts = HashMap::new(); - assert!(!CoreOptions::new(&opts).primary_key_vector_index_enabled()); + assert!(!CoreOptions::new(&opts).vector_index_enabled()); } #[test] @@ -4347,14 +4298,20 @@ mod tests { ), ]); let co = CoreOptions::new(&opts); - assert!(co.primary_key_vector_index_enabled()); - assert_eq!(co.primary_key_vector_index_column().unwrap(), "embedding"); + assert!(co.vector_index_enabled()); + assert_eq!(co.single_vector_index_column().unwrap(), "embedding"); assert_eq!( - co.primary_key_vector_index_type("embedding").unwrap(), + co.vector_index_options(None, "embedding", false) + .unwrap() + .index_type + .as_deref() + .unwrap(), "ivf-flat" ); assert_eq!( - co.primary_key_vector_distance_metric("embedding").unwrap(), + co.vector_index_options(None, "embedding", true) + .map(|resolved| resolved.metric) + .unwrap(), "inner_product" ); } @@ -4364,7 +4321,8 @@ mod tests { let opts = HashMap::from([("pk-vector.index.columns".to_string(), "e".to_string())]); assert_eq!( CoreOptions::new(&opts) - .primary_key_vector_distance_metric("e") + .vector_index_options(None, "e", true) + .map(|resolved| resolved.metric) .unwrap(), "inner_product" ); @@ -4380,7 +4338,8 @@ mod tests { ), ]); assert!(CoreOptions::new(&opts) - .primary_key_vector_distance_metric("e") + .vector_index_options(None, "e", true) + .map(|resolved| resolved.metric) .is_err()); } @@ -4393,12 +4352,7 @@ mod tests { (" ", vec![""]), ] { let opts = HashMap::from([("pk-vector.index.columns".to_string(), value.to_string())]); - assert_eq!( - CoreOptions::new(&opts) - .primary_key_vector_index_columns() - .unwrap(), - expected - ); + assert_eq!(CoreOptions::new(&opts).vector_index_columns(), expected); } } @@ -4406,27 +4360,27 @@ mod tests { fn test_pk_vector_multiple_columns_unsupported() { let opts = HashMap::from([("pk-vector.index.columns".to_string(), "a,b".to_string())]); assert!(CoreOptions::new(&opts) - .primary_key_vector_index_column() + .single_vector_index_column() .is_err()); } #[test] - fn test_pk_vector_index_type_absent_errors() { + fn test_pk_vector_missing_implementation_stays_unresolved() { let opts = HashMap::from([("pk-vector.index.columns".to_string(), "e".to_string())]); - assert!(CoreOptions::new(&opts) - .primary_key_vector_index_type("e") - .is_err()); + let resolved = CoreOptions::new(&opts) + .vector_index_options(None, "e", false) + .unwrap(); + assert_eq!(resolved.index_type, None); + assert_eq!(resolved.metric, "inner_product"); + assert_eq!(resolved.options, opts); } #[test] fn test_pk_full_text_index_absent_is_disabled_and_empty() { let opts = HashMap::new(); let co = CoreOptions::new(&opts); - assert!(!co.primary_key_full_text_index_enabled()); - assert_eq!( - co.primary_key_full_text_index_columns(), - Vec::::new() - ); + assert!(!co.full_text_index_enabled()); + assert_eq!(co.full_text_index_columns(), Vec::::new()); } #[test] @@ -4436,9 +4390,9 @@ mod tests { "a, b ,c".to_string(), )]); let co = CoreOptions::new(&opts); - assert!(co.primary_key_full_text_index_enabled()); + assert!(co.full_text_index_enabled()); assert_eq!( - co.primary_key_full_text_index_columns(), + co.full_text_index_columns(), vec!["a".to_string(), "b".to_string(), "c".to_string()] ); } @@ -4449,9 +4403,9 @@ mod tests { // three columns with a blank in the middle. let opts = HashMap::from([("pk-full-text.index.columns".to_string(), "a,,b".to_string())]); let co = CoreOptions::new(&opts); - assert!(co.primary_key_full_text_index_enabled()); + assert!(co.full_text_index_enabled()); assert_eq!( - co.primary_key_full_text_index_columns(), + co.full_text_index_columns(), vec!["a".to_string(), "".to_string(), "b".to_string()] ); } @@ -4463,9 +4417,9 @@ mod tests { // `split(",",-1).map(trim)`, so " , " yields two empty tokens (NOT []). let opts = HashMap::from([("pk-full-text.index.columns".to_string(), " , ".to_string())]); let co = CoreOptions::new(&opts); - assert!(co.primary_key_full_text_index_enabled()); + assert!(co.full_text_index_enabled()); assert_eq!( - co.primary_key_full_text_index_columns(), + co.full_text_index_columns(), vec!["".to_string(), "".to_string()] ); } diff --git a/crates/paimon/src/spec/core_options/index_options.rs b/crates/paimon/src/spec/core_options/index_options.rs index d6b8fa2ad..b26d9185c 100644 --- a/crates/paimon/src/spec/core_options/index_options.rs +++ b/crates/paimon/src/spec/core_options/index_options.rs @@ -41,6 +41,15 @@ fn invalid(message: impl Into) -> Error { } } +/// One captured vector configuration. A supplied metadata implementation wins +/// over declarations; readers can leave an unspecified metric out of `options`. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct VectorIndexOptions { + pub index_type: Option, + pub options: HashMap, + pub metric: String, +} + impl CoreOptions<'_> { /// Declared implementation identifiers, in deterministic order. pub fn global_index_types(&self) -> Vec { @@ -102,43 +111,66 @@ impl CoreOptions<'_> { } } - /// Concrete declarations select the implementation. Legacy PK field types - /// apply only to columns listed by the legacy PK declaration; they are not - /// standalone implementation hints. Query hints belong to the builder API. - pub fn vector_index_type(&self, column: &str) -> Result> { - let declared = self - .global_index_types() - .into_iter() - .filter(|kind| { - vector(kind) - && kind != "vector" - && self.global_index_columns(kind).iter().any(|c| c == column) - }) - .collect::>(); - if declared.len() > 1 { - return Err(invalid(format!( - "Multiple vector index types declared for '{column}'" - ))); - } - if let Some(kind) = declared.into_iter().next() { - return Ok(Some(kind)); - } - let legacy_pk = columns(self.options.get("pk-vector.index.columns")) - .iter() - .any(|declared| declared == column); - Ok(self - .options - .get(&format!("fields.{column}.global-index.vector.index-type")) - .or_else(|| { - legacy_pk - .then(|| { - self.options - .get(&format!("fields.{column}.pk-vector.index.type")) + /// Resolve the implementation and its parameters together. Explicit types + /// from captured metadata take precedence without consulting declarations. + pub fn vector_index_options( + &self, + kind: Option<&str>, + column: &str, + default_metric: bool, + ) -> Result { + let selected = if let Some(kind) = kind { + Some(kind.trim().to_ascii_lowercase()) + } else { + let declared = self + .global_index_types() + .into_iter() + .filter(|kind| { + vector(kind) + && kind != "vector" + && self.global_index_columns(kind).iter().any(|c| c == column) + }) + .collect::>(); + if declared.len() > 1 { + return Err(invalid(format!( + "Multiple vector index types declared for '{column}'" + ))); + } + if let Some(kind) = declared.into_iter().next() { + Some(kind) + } else { + let legacy_pk = columns(self.options.get("pk-vector.index.columns")) + .iter() + .any(|declared| declared == column); + self.options + .get(&format!("fields.{column}.global-index.vector.index-type")) + .or_else(|| { + legacy_pk + .then(|| { + self.options + .get(&format!("fields.{column}.pk-vector.index.type")) + }) + .flatten() }) - .flatten() - }) - .map(|s| s.trim().to_ascii_lowercase()) - .filter(|s| !s.is_empty())) + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty()) + } + }; + let Some(kind) = selected else { + let metric = normalize_metric( + self.options + .get(&format!("fields.{column}.pk-vector.distance.metric")) + .map(String::as_str) + .unwrap_or("inner_product"), + ); + crate::vindex::pkvector::metric::VectorSearchMetric::parse(&metric)?; + return Ok(VectorIndexOptions { + index_type: None, + options: self.options.clone(), + metric, + }); + }; + self.resolve_vector_parameters(kind, column, default_metric) } /// Resolve implementation defaults and one field's JSON without mutating @@ -230,7 +262,7 @@ impl CoreOptions<'_> { ))); } resolved.insert(qualified.clone(), value.clone()); - if shared && is_vector { + if is_vector { if let Some(short) = qualified.strip_prefix(&prefix) { resolved.insert(format!("{field}{short}"), value); } @@ -240,15 +272,14 @@ impl CoreOptions<'_> { Ok(resolved) } - /// Merge vector field parameters before normalizing metric aliases, like - /// Java VectorIndexOptions. Readers can retain an unspecified metric. - pub fn vector_index_options( + /// Merge parameters only after the implementation has been selected. + fn resolve_vector_parameters( &self, - kind: &str, + kind: String, column: &str, default_metric: bool, - ) -> Result> { - if !vector(kind) { + ) -> Result { + if !vector(&kind) { return Err(invalid(format!("Unsupported vector index type: {kind}"))); } let field = format!("fields.{column}."); @@ -275,7 +306,7 @@ impl CoreOptions<'_> { for key in &aliases { defaults.remove(key); } - let mut resolved = CoreOptions::new(&defaults).global_index_options(kind, column)?; + let mut resolved = CoreOptions::new(&defaults).global_index_options(&kind, column)?; metric = aliases .iter() .find_map(|key| resolved.get(key).cloned()) @@ -292,13 +323,14 @@ impl CoreOptions<'_> { .global_index_types() .iter() .any(|kind| vector(kind) && kind != "vector"); - if legacy_rust && default_metric { - metric = Some( - self.options - .get(&format!("{field}global-index.vector.distance-metric")) - .cloned() - .unwrap_or_else(|| "inner_product".into()), - ); + let legacy_metric = legacy_rust.then(|| { + self.options + .get(&format!("{field}global-index.vector.distance-metric")) + .cloned() + .unwrap_or_else(|| "inner_product".into()) + }); + if default_metric && legacy_rust { + metric = legacy_metric.clone(); } if metric.is_some() || default_metric { let metric = normalize_metric(metric.as_deref().unwrap_or("inner_product")); @@ -307,7 +339,16 @@ impl CoreOptions<'_> { resolved.insert(key, metric.clone()); } } - Ok(resolved) + Ok(VectorIndexOptions { + index_type: Some(kind), + options: resolved, + metric: normalize_metric( + legacy_metric + .as_deref() + .or(metric.as_deref()) + .unwrap_or("inner_product"), + ), + }) } } @@ -322,6 +363,99 @@ mod tests { .collect() } + #[test] + fn vector_resolution_captures_type_options_and_metric_atomically() { + let values = map(&[ + ("global-index.ivf-pq.index-columns", "embedding"), + ("ivf-pq.dimension", "16"), + ("ivf-pq.nlist", "8"), + ("ivf-pq.metric", "l2"), + ( + "fields.embedding.pk-vector.index.options", + r#"{"dimension": 99, "m": 4}"#, + ), + ( + "fields.embedding.index-options", + r#"{"dimension": 2, "nlist": 1, "metric": "COSINE"}"#, + ), + ]); + let before = values.clone(); + let resolved = CoreOptions::new(&values) + .vector_index_options(None, "embedding", true) + .unwrap(); + assert_eq!(resolved.index_type.as_deref(), Some("ivf-pq")); + assert_eq!(resolved.metric, "cosine"); + assert_eq!(resolved.options["ivf-pq.dimension"], "2"); + assert_eq!(resolved.options["ivf-pq.nlist"], "1"); + assert_eq!(resolved.options["ivf-pq.metric"], "cosine"); + assert!( + !resolved.options.contains_key("ivf-pq.m"), + "new JSON replaces the whole legacy JSON" + ); + let restored: VectorIndexOptions = + serde_json::from_str(&serde_json::to_string(&resolved).unwrap()).unwrap(); + assert_eq!(restored, resolved); + assert_eq!(values, before); + } + + #[test] + fn captured_format_ignores_declaration_changes_without_injecting_query_metric() { + let values = map(&[ + ("global-index.ivf-pq.index-columns", "embedding"), + ("global-index.diskann.index-columns", "embedding"), + ("diskann.metric", "invalid"), + ("ivf-flat.nlist", "3"), + ("fields.embedding.index-options", r#"{"nlist": 1}"#), + ]); + assert!(CoreOptions::new(&values) + .vector_index_options(None, "embedding", false) + .is_err()); + let resolved = CoreOptions::new(&values) + .vector_index_options(Some("ivf-flat"), "embedding", false) + .unwrap(); + assert_eq!(resolved.index_type.as_deref(), Some("ivf-flat")); + assert_eq!(resolved.options["ivf-flat.nlist"], "1"); + assert_eq!(resolved.metric, "inner_product"); + for key in [ + "ivf-flat.metric", + "ivf-flat.distance.metric", + "fields.embedding.metric", + "fields.embedding.distance.metric", + ] { + assert!(!resolved.options.contains_key(key), "{key}"); + } + } + + #[test] + fn legacy_rust_result_preserves_build_metric_without_query_defaults() { + let mut values = map(&[ + ("global-index.vector.index-columns", "embedding"), + ( + "fields.embedding.global-index.vector.index-type", + "ivf-flat", + ), + ("ivf-flat.metric", "l2"), + ]); + let read = CoreOptions::new(&values) + .vector_index_options(None, "embedding", false) + .unwrap(); + let build = CoreOptions::new(&values) + .vector_index_options(None, "embedding", true) + .unwrap(); + assert_eq!(read.metric, build.metric); + assert_eq!(read.metric, "inner_product"); + assert_eq!( + read.options["ivf-flat.metric"], "l2", + "retain explicit read parameters" + ); + values.remove("ivf-flat.metric"); + let query = CoreOptions::new(&values) + .vector_index_options(None, "embedding", false) + .unwrap(); + assert!(!query.options.contains_key("ivf-flat.metric")); + assert_eq!(query.metric, "inner_product"); + } + #[test] fn shared_schema_guards_apply_to_pk_but_allow_append_and_de_type_evolution() { use crate::spec::{BigIntType, DataType, IntType, Schema, SchemaChange, TableSchema}; @@ -444,11 +578,18 @@ mod tests { ); if vector(kind) { let core = CoreOptions::new(&values); + assert_eq!(core.vector_index_columns(), vec!["first", "second"]); assert_eq!( - core.primary_key_vector_index_columns().unwrap(), - vec!["first", "second"] + core.vector_index_options(None, "first", false) + .map(|resolved| resolved.index_type.ok_or_else(|| { + crate::Error::ConfigInvalid { + message: "Vector index implementation is not configured".into(), + } + })) + .and_then(|kind| kind) + .unwrap(), + kind ); - assert_eq!(core.primary_key_vector_index_type("first").unwrap(), kind); } values.insert(format!("global-index.{kind}.index-columns"), "".into()); assert_eq!( @@ -466,7 +607,8 @@ mod tests { ("fields.embedding.pk-vector.index.type", kind), ]); assert!(CoreOptions::new(&values) - .primary_key_vector_distance_metric("embedding") + .vector_index_options(None, "embedding", true) + .map(|resolved| resolved.metric) .unwrap_err() .to_string() .contains("Unsupported vector index type")); @@ -507,8 +649,9 @@ mod tests { (&format!("fields.embedding.{suffix}"), json), ]); let resolved = CoreOptions::new(&values) - .vector_index_options("ivf-pq", "embedding", true) - .unwrap(); + .vector_index_options(Some("ivf-pq"), "embedding", true) + .unwrap() + .options; for key in [ "fields.embedding.distance.metric", "fields.embedding.metric", @@ -519,8 +662,9 @@ mod tests { } assert_eq!( CoreOptions::new(&resolved) - .vector_index_options("ivf-pq", "embedding", true) - .unwrap(), + .vector_index_options(Some("ivf-pq"), "embedding", true) + .unwrap() + .options, resolved ); } @@ -579,7 +723,8 @@ mod tests { ]); assert_eq!( CoreOptions::new(&values) - .primary_key_vector_distance_metric("embedding") + .vector_index_options(None, "embedding", true) + .map(|resolved| resolved.metric) .unwrap(), "cosine" ); @@ -588,8 +733,9 @@ mod tests { r#"{"nlist":64,"distance.metric":"Inner-Product"}"#.into(), ); let resolved = CoreOptions::new(&values) - .vector_index_options("ivf-pq", "embedding", true) - .unwrap(); + .vector_index_options(Some("ivf-pq"), "embedding", true) + .unwrap() + .options; for key in [ "ivf-pq.metric", "ivf-pq.distance.metric", @@ -603,7 +749,8 @@ mod tests { assert_eq!(resolved["fields.embedding.nlist"], "64"); assert_eq!( CoreOptions::new(&values) - .primary_key_vector_distance_metric("embedding") + .vector_index_options(None, "embedding", true) + .map(|resolved| resolved.metric) .unwrap(), "inner_product" ); @@ -630,15 +777,16 @@ mod tests { ("fields.embedding.pk-vector.index.options", r#"{"nlist":1}"#), ]); assert!(CoreOptions::new(&values) - .vector_index_options("ivf-flat", "embedding", true) + .vector_index_options(Some("ivf-flat"), "embedding", true) .unwrap_err() .to_string() .contains("fields.embedding.index-options")); } let values = map(&[("fields.embedding.index-options", r#"{"nlist":64}"#)]); let resolved = CoreOptions::new(&values) - .vector_index_options("ivf-pq", "embedding", false) - .unwrap(); + .vector_index_options(Some("ivf-pq"), "embedding", false) + .unwrap() + .options; assert!(!resolved.contains_key("ivf-pq.metric")); } @@ -668,12 +816,20 @@ mod tests { ]); let core = CoreOptions::new(&values); assert_eq!( - core.primary_key_vector_index_type("embedding").unwrap(), + core.vector_index_options(None, "embedding", false) + .map(|resolved| resolved + .index_type + .ok_or_else(|| crate::Error::ConfigInvalid { + message: "Vector index implementation is not configured".into() + })) + .and_then(|kind| kind) + .unwrap(), "ivf-flat" ); let resolved = core - .vector_index_options("ivf-flat", "embedding", true) - .unwrap(); + .vector_index_options(Some("ivf-flat"), "embedding", true) + .unwrap() + .options; assert_eq!(resolved["ivf-flat.dimension"], "2"); assert_eq!(resolved["ivf-flat.metric"], "l2"); } @@ -689,7 +845,8 @@ mod tests { let mut values = map(&[(key, "diskann")]); assert_eq!( CoreOptions::new(&values) - .vector_index_type("embedding") + .vector_index_options(None, "embedding", false) + .map(|resolved| resolved.index_type) .unwrap(), None ); @@ -699,7 +856,8 @@ mod tests { ); assert_eq!( CoreOptions::new(&values) - .vector_index_type("embedding") + .vector_index_options(None, "embedding", false) + .map(|resolved| resolved.index_type) .unwrap() .as_deref(), Some("ivf-flat") @@ -718,7 +876,8 @@ mod tests { } assert_eq!( CoreOptions::new(&values) - .vector_index_type("embedding") + .vector_index_options(None, "embedding", false) + .map(|resolved| resolved.index_type) .unwrap(), None ); @@ -729,7 +888,8 @@ mod tests { ); assert_eq!( CoreOptions::new(&values) - .vector_index_type("embedding") + .vector_index_options(None, "embedding", false) + .map(|resolved| resolved.index_type) .unwrap() .as_deref(), Some("ivf-flat") @@ -737,7 +897,8 @@ mod tests { values.remove("fields.embedding.pk-vector.index.type"); assert_eq!( CoreOptions::new(&values) - .vector_index_type("embedding") + .vector_index_options(None, "embedding", false) + .map(|resolved| resolved.index_type) .unwrap(), None ); @@ -749,7 +910,8 @@ mod tests { values.insert(format!("global-index.ivf-pq.{suffix}"), "embedding".into()); assert_eq!( CoreOptions::new(&values) - .vector_index_type("embedding") + .vector_index_options(None, "embedding", false) + .map(|resolved| resolved.index_type) .unwrap() .as_deref(), Some("ivf-pq") @@ -775,16 +937,18 @@ mod tests { ), ]); let resolved = CoreOptions::new(&values) - .vector_index_options("ivf-pq", "embedding", false) - .unwrap(); + .vector_index_options(Some("ivf-pq"), "embedding", false) + .unwrap() + .options; assert_eq!(resolved["ivf-pq.nlist"], "1"); assert_eq!(resolved["fields.embedding.dimension"], "2"); assert!(!resolved.contains_key("ivf-pq.pq.m")); assert!(!resolved.contains_key("ivf-pq.metric")); assert!(!resolved.contains_key("fields.embedding.metric")); let defaults = CoreOptions::new(&values) - .vector_index_options("ivf-pq", "embedding", true) - .unwrap(); + .vector_index_options(Some("ivf-pq"), "embedding", true) + .unwrap() + .options; assert_eq!(defaults["ivf-pq.metric"], "inner_product"); assert_eq!(values["ivf-pq.dimension"], "8"); } diff --git a/crates/paimon/src/spec/schema.rs b/crates/paimon/src/spec/schema.rs index e9576a2d7..f5ddab002 100644 --- a/crates/paimon/src/spec/schema.rs +++ b/crates/paimon/src/spec/schema.rs @@ -869,13 +869,13 @@ fn assert_not_updating_primary_key_index_column( if schema.primary_keys().is_empty() || core_options.data_evolution_enabled() { return Ok(()); } - let is_vector_index_column = core_options.primary_key_vector_index_enabled() + let is_vector_index_column = core_options.vector_index_enabled() && core_options - .primary_key_vector_index_columns()? + .vector_index_columns() .iter() .any(|column| column == field_name); let is_full_text_index_column = core_options - .primary_key_full_text_index_columns() + .full_text_index_columns() .iter() .any(|column| column == field_name); @@ -2301,17 +2301,36 @@ impl Schema { options: &HashMap, ) -> crate::Result<()> { let core = CoreOptions::new(options); - if !core.primary_key_vector_index_enabled() { + if !core.vector_index_enabled() { return Ok(()); } - let column = core.primary_key_vector_index_column()?; + let column = core.single_vector_index_column()?; if column.is_empty() { return Err(crate::Error::ConfigInvalid { message: "pk-vector.index.columns must name a non-empty column".to_string(), }); } - core.primary_key_vector_index_type(&column)?; + let declared = core.global_index_types().iter().any(|kind| { + !matches!( + kind.as_str(), + "vector" | "btree" | "bitmap" | "multivalue" | "fm" | "full-text" + ) && core.global_index_columns(kind).contains(&column) + }); + let legacy_type = options + .get(&format!("fields.{column}.global-index.vector.index-type")) + .is_some_and(|kind| !kind.trim().is_empty()) + || (options + .get("pk-vector.index.columns") + .is_some_and(|columns| columns.split(',').any(|c| c.trim() == column)) + && options + .get(&format!("fields.{column}.pk-vector.index.type")) + .is_some_and(|kind| !kind.trim().is_empty())); + if !declared && !legacy_type { + return Err(crate::Error::ConfigInvalid { + message: "Vector index implementation is not configured".into(), + }); + } Self::validate_primary_key_index_prerequisites("vector", primary_keys, &core)?; let field = fields @@ -2335,7 +2354,11 @@ impl Schema { }); } - core.legacy_primary_key_vector_distance_metric(&column) + let metric = options + .get(&format!("fields.{column}.pk-vector.distance.metric")) + .map(String::as_str) + .unwrap_or("inner_product"); + crate::vindex::pkvector::metric::VectorSearchMetric::parse(metric) .map(|_| ()) .map_err(Self::options_error_to_config_invalid) } @@ -2346,11 +2369,11 @@ impl Schema { options: &HashMap, ) -> crate::Result<()> { let core = CoreOptions::new(options); - if !core.primary_key_full_text_index_enabled() { + if !core.full_text_index_enabled() { return Ok(()); } - let columns = core.primary_key_full_text_index_columns(); + let columns = core.full_text_index_columns(); if columns.len() != 1 { return Err(crate::Error::ConfigInvalid { message: format!( @@ -2367,9 +2390,7 @@ impl Schema { } Self::validate_primary_key_index_prerequisites("full-text", primary_keys, &core)?; - if core.primary_key_vector_index_enabled() - && core.primary_key_vector_index_column()? == *column - { + if core.vector_index_enabled() && core.single_vector_index_column()? == *column { return Err(crate::Error::ConfigInvalid { message: format!( "Primary-key vector and full-text indexes cannot reference the same column: '{column}'." @@ -5426,7 +5447,7 @@ mod tests { build_vector_index_test_schema(vector_4f(), &["id"], |options| { options.remove("fields.embedding.pk-vector.index.type"); }), - "index.type is required", + "Vector index implementation is not configured", ); assert_config_invalid( build_vector_index_test_schema(vector_4f(), &["id"], |options| { @@ -5460,6 +5481,52 @@ mod tests { ); } + #[test] + fn pk_vector_family_declaration_still_requires_an_implementation() { + assert_config_invalid( + build_vector_index_test_schema(vector_4f(), &["id"], |options| { + options.remove("pk-vector.index.columns"); + options.insert( + "global-index.vector.index-columns".into(), + "embedding".into(), + ); + }), + "Vector index implementation is not configured", + ); + for field_type in [None, Some("ivf-flat")] { + let result = build_vector_index_test_schema(vector_4f(), &["id"], |options| { + options.remove("pk-vector.index.columns"); + options.remove("fields.embedding.pk-vector.index.type"); + options.insert( + "global-index.vector.index-columns".into(), + "embedding".into(), + ); + if let Some(kind) = field_type { + options.insert( + "fields.embedding.global-index.vector.index-type".into(), + kind.into(), + ); + } + }); + if field_type.is_some() { + result.unwrap(); + } else { + assert_config_invalid(result, "Vector index implementation is not configured"); + } + } + build_vector_index_test_schema(vector_4f(), &["id"], |options| { + options.remove("pk-vector.index.columns"); + options.remove("fields.embedding.pk-vector.index.type"); + options.insert( + "global-index.ivf-flat.index-columns".into(), + "embedding".into(), + ); + // Preserve deferred parameter validation; creation checks schema shape. + options.insert("fields.embedding.index-options".into(), "bad-json".into()); + }) + .unwrap(); + } + #[test] fn test_apply_changes_validates_primary_key_vector_index() { let table_schema = TableSchema::new( diff --git a/crates/paimon/src/table/de_vector_read.rs b/crates/paimon/src/table/de_vector_read.rs index d438350bb..21cfade25 100644 --- a/crates/paimon/src/table/de_vector_read.rs +++ b/crates/paimon/src/table/de_vector_read.rs @@ -37,8 +37,8 @@ use crate::table::search_task::{ }; use crate::table::vector_read::Read; use crate::table::vector_search_common::{ - configured_refine_factor, indexed_search_limit, log_vindex_range_io_stats, normalize_metric, - raw_vector_index_type, read_index_vector_metric, resolve_materialize_read_type, + indexed_search_limit, log_vindex_range_io_stats, normalize_metric, read_index_vector_metric, + resolve_materialize_read_type, resolve_vector_search_options, validate_global_vector_index_types, vindex_concurrency_limits, }; use crate::table::{ @@ -330,12 +330,21 @@ async fn execute_vector_index_work( let index_meta_bytes = global_meta.index_meta.clone().unwrap_or_default(); let row_range_start = global_meta.row_range_start; let vector_searches = work.queries; - let options = super::vector_search_common::vector_read_options( - evaluation.table_options, - &vector_searches[0].options, - &vector_searches[0].field_name, - &work.entry.index_file.index_type, - )?; + let options = if let Some(resolved) = work.resolved { + if resolved.index_type.as_deref() != Some(work.entry.index_file.index_type.as_str()) { + return Err(crate::table::search_task::invalid( + "Captured vector configuration does not match payload type", + )); + } + resolved.options + } else { + super::vector_search_common::vector_read_options( + evaluation.table_options, + &vector_searches[0].options, + &vector_searches[0].field_name, + &work.entry.index_file.index_type, + )? + }; let input = evaluation.file_io.new_input(&path); if vector_searches.iter().all(|search| { search @@ -744,6 +753,25 @@ async fn evaluate_batch_vector_search_with_executor( return Ok(vec![ScoredRowIds::empty(); vector_searches.len()]); } + let raw_index_type = sources + .into_iter() + .flatten() + .find_map(|source| match source { + crate::table::SearchScanSplit::Raw { index_type, .. } => index_type.as_deref(), + _ => None, + }) + .or(vector_searches[0].index_type.as_deref()); + let configuration = resolve_vector_search_options( + evaluation.table_options, + &search_options, + field_name, + vector_entries + .first() + .map(|entry| entry.index_file.index_type.as_str()) + .or(raw_index_type), + false, + )?; + let index_type = configuration.resolved.index_type.as_deref(); let vector_ranges = merge_row_ranges( vector_entries .iter() @@ -869,9 +897,9 @@ async fn evaluate_batch_vector_search_with_executor( .max() .unwrap_or(0); let refine_factor = match vector_entries.first() { - Some(entry) => configured_refine_factor( - &search_options, - evaluation.table_options, + Some(entry) => super::vector_search_common::resolved_refine_factor( + &configuration.query_options, + &configuration.resolved.options, field_name, &entry.index_file.index_type, )?, @@ -914,31 +942,13 @@ async fn evaluate_batch_vector_search_with_executor( .map(|entry| (entry, None)) .collect::>() }; - let raw_index_type = sources - .into_iter() - .flatten() - .find_map(|source| match source { - crate::table::SearchScanSplit::Raw { index_type, .. } => index_type.as_deref(), - _ => None, - }) - .or(vector_searches[0].index_type.as_deref()); - let configured_index_type = if vector_entries.is_empty() { - raw_vector_index_type( - evaluation.table_options, - &search_options, - field_name, - raw_index_type, - )? - } else { - None - }; - let index_type = vector_entries - .first() - .map(|entry| entry.index_file.index_type.as_str()) - .or(configured_index_type.as_deref()); let metric_check = VectorMetricCheck { persisted: Arc::new(std::sync::Mutex::new(None)), - requested: vector_metric_override(&search_options, field_name, index_type)?, + requested: resolved_vector_metric_override( + &configuration.query_options, + field_name, + index_type, + )?, }; let mut permit_wait = Duration::ZERO; let mut file_reader_open = Duration::ZERO; @@ -999,6 +1009,7 @@ async fn evaluate_batch_vector_search_with_executor( Ok::<_, crate::Error>(VectorIndexWork { entry: entry.clone(), queries: searches, + resolved: Some(configuration.resolved.clone()), }) }); let results = if let Some(executor) = executor { @@ -1015,6 +1026,7 @@ async fn evaluate_batch_vector_search_with_executor( let work = VectorIndexTask { entry: work.entry, queries: TaskVectorQueries::capture(&work.queries)?, + resolved: work.resolved, }; SearchTask::new( table, @@ -1084,6 +1096,7 @@ async fn evaluate_batch_vector_search_with_executor( merged, index_search_limit, executor, + &configuration, ) .await?; refine = refine_start.map_or(Duration::ZERO, |start| start.elapsed()); @@ -1155,16 +1168,14 @@ async fn evaluate_batch_vector_search_with_executor( source: None, })?; let metric_start = timing_enabled.then(Instant::now); - let metric_options = &vector_searches[0].options; - let metric = resolve_raw_vector_metric( + let metric = resolve_raw_vector_metric_resolved( evaluation.file_io, table_path, evaluation.table_options, - metric_options, + &configuration, index_entries, field_id, field_name, - raw_index_type, ) .await?; let metric_resolve = metric_start.map_or(Duration::ZERO, |start| start.elapsed()); @@ -1409,6 +1420,7 @@ async fn maybe_rerank_indexed_batch_results( results: Vec, index_search_limit: usize, executor: Option<&dyn SearchTaskExecutor>, + configuration: &super::vector_search_common::VectorSearchOptions, ) -> crate::Result> { let timing_enabled = vector_search_timing_enabled(); let total_start = timing_enabled.then(Instant::now); @@ -1444,15 +1456,14 @@ async fn maybe_rerank_indexed_batch_results( let raw_ranges = sorted_row_ids_to_row_ranges(union_candidates.iter())?; let metric_start = timing_enabled.then(Instant::now); let field_name = &vector_searches[0].field_name; - let metric = resolve_raw_vector_metric( + let metric = resolve_raw_vector_metric_resolved( evaluation.file_io, evaluation.table_path.trim_end_matches('/'), evaluation.table_options, - &vector_searches[0].options, + configuration, index_entries, field_id, field_name, - vector_searches[0].index_type.as_deref(), ) .await?; metric_check.check(metric)?; @@ -1680,6 +1691,7 @@ impl RawVectorMetric { } } +#[cfg(test)] #[allow(clippy::too_many_arguments)] async fn resolve_raw_vector_metric( file_io: &FileIO, @@ -1703,20 +1715,42 @@ async fn resolve_raw_vector_metric( .is_some_and(|meta| meta.index_field_id == field_id) }) .map(|entry| entry.index_file.index_type.as_str()); - let configured_type = if persisted_type.is_none() { - raw_vector_index_type( - table_options, - search_options, - field_name, - explicit_index_type, - )? - } else { - None - }; - let index_type = persisted_type.or(configured_type.as_deref()); + let configuration = resolve_vector_search_options( + table_options, + search_options, + field_name, + persisted_type.or(explicit_index_type), + false, + )?; + resolve_raw_vector_metric_resolved( + file_io, + table_path, + table_options, + &configuration, + index_entries, + field_id, + field_name, + ) + .await +} + +async fn resolve_raw_vector_metric_resolved( + file_io: &FileIO, + table_path: &str, + table_options: &HashMap, + configuration: &super::vector_search_common::VectorSearchOptions, + index_entries: &[IndexManifestEntry], + field_id: i32, + field_name: &str, +) -> crate::Result { + let index_type = configuration.resolved.index_type.as_deref(); let check = VectorMetricCheck { persisted: Arc::new(std::sync::Mutex::new(None)), - requested: vector_metric_override(search_options, field_name, index_type)?, + requested: resolved_vector_metric_override( + &configuration.query_options, + field_name, + index_type, + )?, }; let mut persisted = None; for entry in index_entries { @@ -1741,36 +1775,18 @@ async fn resolve_raw_vector_metric( if let Some(metric) = check.requested { return Ok(metric); } - if let Some(kind) = index_type { - let options = super::global_index_options::field_index_options( - table_options, - field_name, - "vector", - kind, - )?; - let native = crate::vindex::VindexVectorIndexOptions::resolve_native_options( - table_options, - &options, - &HashMap::new(), - kind, - field_name, - )?; - return RawVectorMetric::parse(&native["metric"]); + if index_type.is_some() { + return RawVectorMetric::parse(&configuration.resolved.metric); } - let table_metric = vector_metric_override(table_options, field_name, index_type)?; + let table_metric = configured_raw_vector_metric_override(table_options, field_name)?; if let Some(metric) = table_metric { return Ok(metric); } - // Before files exist, Java creates the configured indexer for raw search. - // The vindex option resolver defaults to inner product. - Ok(if index_type.is_some_and(is_vindex_index_type) { - RawVectorMetric::InnerProduct - } else { - RawVectorMetric::L2 - }) + // Preserve the historical untyped raw-search default. + Ok(RawVectorMetric::L2) } -fn vector_metric_override( +fn resolved_vector_metric_override( options: &HashMap, field_name: &str, index_type: Option<&str>, @@ -1778,8 +1794,7 @@ fn vector_metric_override( // A known index family determines its own defaults and option namespace. // Only a search without a family may infer a metric from suffixes. if let Some(kind) = index_type { - let resolved = CoreOptions::new(options).vector_index_options(kind, field_name, false)?; - direct_raw_vector_metric_override(&resolved, field_name, Some(kind)) + direct_raw_vector_metric_override(options, field_name, Some(kind)) } else { configured_raw_vector_metric_override(options, field_name) } @@ -2736,10 +2751,13 @@ impl TaskVectorQueries { struct VectorIndexTask { entry: IndexManifestEntry, queries: TaskVectorQueries, + #[serde(default)] + resolved: Option, } struct VectorIndexWork { entry: IndexManifestEntry, queries: Vec, + resolved: Option, } #[derive(Serialize, Deserialize)] struct VectorRowsTask { @@ -2825,6 +2843,7 @@ pub(super) async fn execute_search_task( VectorIndexWork { entry: work.entry, queries: work.queries.restore()?, + resolved: work.resolved, }, metric_check.clone(), RangeReadLimiter::new(range_concurrency), diff --git a/crates/paimon/src/table/de_vector_read/tests.rs b/crates/paimon/src/table/de_vector_read/tests.rs index bf28502a4..3df3f0c65 100644 --- a/crates/paimon/src/table/de_vector_read/tests.rs +++ b/crates/paimon/src/table/de_vector_read/tests.rs @@ -22,6 +22,7 @@ use crate::spec::{ IndexManifestEntry, IntType, ROW_ID_FIELD_NAME, }; use crate::table::pk_vector_position_read::{PKEY_VECTOR_POSITION_COLUMN, SEARCH_SCORE_COLUMN}; +use crate::table::vector_search_common::configured_refine_factor; use crate::table::vector_search_common::{collect_ranked_rows, reorder_and_strip_position}; use crate::table::vector_search_test_utils::{ build_vindex_segment_bytes, de_vector_table, id_gt_filter, pk_vector_table, @@ -1769,6 +1770,15 @@ impl crate::table::SearchTaskExecutor for WireSearchExecutor { let mut results = Vec::new(); for (task, frame) in tasks.iter().zip(frames) { let value: serde_json::Value = serde_json::from_slice(&task.serialize()?).unwrap(); + if value["kind"] == "vector-index" { + let resolved: crate::spec::VectorIndexOptions = + serde_json::from_value(value["payload"]["resolved"].clone()).unwrap(); + assert_eq!(resolved.index_type.as_deref(), Some("ivf-flat")); + assert_eq!( + resolved.options.get("refine_factor").map(String::as_str), + Some("2") + ); + } self.kinds .lock() .unwrap() diff --git a/crates/paimon/src/table/full_text_search_builder.rs b/crates/paimon/src/table/full_text_search_builder.rs index c6fce865c..60388875d 100644 --- a/crates/paimon/src/table/full_text_search_builder.rs +++ b/crates/paimon/src/table/full_text_search_builder.rs @@ -252,7 +252,7 @@ fn resolves_to_pk_full_text_path( primary_key && !core.data_evolution_enabled() && core - .primary_key_full_text_index_columns() + .full_text_index_columns() .iter() .any(|c| c == text_column) } diff --git a/crates/paimon/src/table/global_index_options.rs b/crates/paimon/src/table/global_index_options.rs index ad7c58529..032bbf9c0 100644 --- a/crates/paimon/src/table/global_index_options.rs +++ b/crates/paimon/src/table/global_index_options.rs @@ -41,7 +41,7 @@ impl Definition { } /// Ownership is shared by maintenance and both table modes' read paths. -pub(super) fn declarations(options: &HashMap) -> Result> { +pub(super) fn declarations(options: &HashMap) -> Result> { declarations_impl(options, true) } @@ -50,13 +50,13 @@ pub(super) fn declarations(options: &HashMap) -> Result, ) -> Result> { - declarations_impl(options, false) + Ok(declarations_impl(options, false)? + .into_iter() + .map(|definition| (definition.column, definition.index_type)) + .collect()) } -fn declarations_impl( - options: &HashMap, - strict: bool, -) -> Result> { +fn declarations_impl(options: &HashMap, strict: bool) -> Result> { let core = CoreOptions::new(options); for kind in core.global_index_types() { if strict && kind != "vector" && normalize_global_index_type(&kind).is_none() { @@ -84,12 +84,39 @@ fn declarations_impl( if !owners.insert(column.clone()) { return Err(crate::Error::ConfigInvalid { message: format!("Column '{column}' can own at most one primary-key index or automatically maintained global index") }); } - let kind = if family == "vector" && strict { - vector_index_type(options, &column)?.ok_or_else(|| crate::Error::ConfigInvalid { message: format!("fields.{column}.global-index.vector.index-type must specify a supported vector index type") })? + let (kind, resolved) = if family == "vector" && strict { + let resolved = core.vector_index_options(None, &column, true)?; + let kind = resolved + .index_type + .ok_or_else(|| crate::Error::ConfigInvalid { + message: format!( + "Vector index implementation is not configured for '{column}'" + ), + })?; + let kind = normalize_global_index_type(&kind) + .filter(|kind| crate::vindex::is_vindex_index_type(kind)) + .ok_or_else(|| crate::Error::ConfigInvalid { + message: format!("Unsupported vector index type for {column}: {kind}"), + })?; + (kind, resolved.options) } else { - family + ( + family, + if strict { + core.global_index_options(family, &column)? + } else { + HashMap::new() + }, + ) }; - result.push((column, kind.into())); + result.push(Definition { + column, + index_type: kind.into(), + options: resolved + .into_iter() + .filter(|(key, value)| options.get(key) != Some(value)) + .collect(), + }); } } Ok(result) @@ -99,6 +126,7 @@ pub(super) fn definitions(table: &Table) -> Result> { let options = table.schema().options(); let owners = declarations(options)? .into_iter() + .map(|definition| (definition.column.clone(), definition)) .collect::>(); for column in owners.keys() { if !table.schema().fields().iter().any(|f| f.name() == column) { @@ -109,18 +137,13 @@ pub(super) fn definitions(table: &Table) -> Result> { } let mut result = Vec::with_capacity(owners.len()); for field in table.schema().fields() { - let Some(kind) = owners.get(field.name()) else { + let Some(definition) = owners.get(field.name()) else { continue; }; - let family = if crate::vindex::is_vindex_index_type(kind) { - "vector" - } else { - kind - }; let definition = Definition { column: field.name().into(), - index_type: kind.clone(), - options: field_index_options(options, field.name(), family, kind)?, + index_type: definition.index_type.clone(), + options: definition.options.clone(), }; let mut merged = options.clone(); merged.extend(definition.options.clone()); @@ -142,7 +165,8 @@ pub(super) fn field_index_options( ) -> Result> { let core = CoreOptions::new(table_options); let resolved = if crate::vindex::is_vindex_index_type(index_type) { - core.vector_index_options(index_type, column, true)? + core.vector_index_options(Some(index_type), column, true)? + .options } else { core.global_index_options(index_type, column)? }; @@ -152,21 +176,6 @@ pub(super) fn field_index_options( .collect()) } -pub(super) fn vector_index_type( - options: &HashMap, - column: &str, -) -> Result> { - let Some(kind) = CoreOptions::new(options).vector_index_type(column)? else { - return Ok(None); - }; - normalize_global_index_type(&kind) - .filter(|kind| crate::vindex::is_vindex_index_type(kind)) - .map(Some) - .ok_or_else(|| crate::Error::ConfigInvalid { - message: format!("Unsupported vector index type for {column}: {kind}"), - }) -} - pub(super) fn declares_column( options: &HashMap, column: &str, diff --git a/crates/paimon/src/table/hybrid_search_builder.rs b/crates/paimon/src/table/hybrid_search_builder.rs index 41051e9a8..fcec26106 100644 --- a/crates/paimon/src/table/hybrid_search_builder.rs +++ b/crates/paimon/src/table/hybrid_search_builder.rs @@ -1081,16 +1081,16 @@ fn route_is_primary_key( } match route.kind { HybridSearchRouteKind::Vector => { - core.primary_key_vector_index_enabled() + core.vector_index_enabled() && core - .primary_key_vector_index_columns() - .ok() - .is_some_and(|cols| cols.iter().any(|c| c == route.field_name())) + .vector_index_columns() + .iter() + .any(|c| c == route.field_name()) } HybridSearchRouteKind::FullText => { !core.data_evolution_enabled() && core - .primary_key_full_text_index_columns() + .full_text_index_columns() .iter() .any(|c| c == route.field_name()) } diff --git a/crates/paimon/src/table/pk_index_options.rs b/crates/paimon/src/table/pk_index_options.rs index 26a42489c..0f64290c5 100644 --- a/crates/paimon/src/table/pk_index_options.rs +++ b/crates/paimon/src/table/pk_index_options.rs @@ -39,21 +39,11 @@ pub(super) fn field_options( crate::table::global_index_options::field_index_options(options, column, family, index_type) } -pub(super) fn vector_index_type( - options: &HashMap, - column: &str, -) -> Result<&'static str> { - crate::table::global_index_options::vector_index_type(options, column)?.ok_or_else(|| { - Error::ConfigInvalid { - message: format!("fields.{column}.pk-vector.index.type is required but not set"), - } - }) -} - pub(super) fn definitions(schema: &TableSchema) -> Result> { let options = schema.options(); let owners = crate::table::global_index_options::declarations(options)? .into_iter() + .map(|definition| (definition.column.clone(), definition)) .collect::>(); for column in owners.keys() { if !schema.fields().iter().any(|field| field.name() == column) { @@ -62,21 +52,21 @@ pub(super) fn definitions(schema: &TableSchema) -> Result> { }); } } - if CoreOptions::new(options).primary_key_vector_index_enabled() { - schema.core_options().primary_key_vector_index_column()?; + if CoreOptions::new(options).vector_index_enabled() { + schema.core_options().single_vector_index_column()?; } let mut result = Vec::new(); for field in schema.fields() { - let Some(kind) = owners.get(field.name()) else { + let Some(declaration) = owners.get(field.name()) else { continue; }; - let kind = kind.as_str(); + let kind = declaration.index_type.as_str(); let family = if crate::vindex::is_vindex_index_type(kind) { "vector" } else { kind }; - let configured = field_options(options, field.name(), family, kind)?; + let configured = declaration.options.clone(); let mut merged = options.clone(); merged.extend(configured.clone()); match family { diff --git a/crates/paimon/src/table/pk_vector_read.rs b/crates/paimon/src/table/pk_vector_read.rs index e7e426ca3..ed74ecfce 100644 --- a/crates/paimon/src/table/pk_vector_read.rs +++ b/crates/paimon/src/table/pk_vector_read.rs @@ -61,12 +61,11 @@ pub(super) struct PkVectorRead { vector_column: String, queries: Vec>, limit: usize, - params: PkVectorSearchParams, raw_index_type: Option, } impl PkVectorRead { - /// Construct a reader from parameters resolved and validated by the builder. + /// Capture query inputs; format-dependent options wait for the scan plan. pub(super) fn new( table: &Table, options: &HashMap, @@ -74,7 +73,6 @@ impl PkVectorRead { vector_column: &str, queries: &[&[f32]], limit: usize, - params: PkVectorSearchParams, ) -> Self { Self { table: table.clone(), @@ -83,7 +81,6 @@ impl PkVectorRead { vector_column: vector_column.to_string(), queries: queries.iter().map(|query| query.to_vec()).collect(), limit, - params, raw_index_type: None, } } @@ -100,28 +97,34 @@ impl Read for PkVectorRead { async fn read(&self, plan: PkVectorScanPlan) -> crate::Result> { let core = CoreOptions::new(self.table.schema().options()); let queries: Vec<&[f32]> = self.queries.iter().map(Vec::as_slice).collect(); - let mut params = self.params.clone(); - if let Some(kind) = &self.raw_index_type { - let raw_type = plan - .splits - .iter() - .all(|split| split.ann_segments.is_empty()) - .then_some(kind.as_str()); - params = PkVectorSearchParams::resolve_for_type( - &self.table, - &self.options, - self.filter.as_ref(), - &self.vector_column, - &queries, - self.limit, - raw_type, - true, - )?; + let mut metadata_type = None; + for segment in plan.splits.iter().flat_map(|split| &split.ann_segments) { + if let Some(previous) = metadata_type { + if previous != segment.index_type.as_str() { + return Err(crate::Error::DataInvalid { + message: format!( + "Vector column '{}' has multiple index types: {previous} and {}", + self.vector_column, segment.index_type + ), + source: None, + }); + } + } + metadata_type = Some(segment.index_type.as_str()); } - let candidates = search_pk_candidates_batch_with_plan( + let params = PkVectorSearchParams::resolve( &self.table, &self.options, self.filter.as_ref(), + &self.vector_column, + &queries, + self.limit, + metadata_type, + self.raw_index_type.as_deref(), + )?; + let candidates = search_pk_candidates_batch_with_plan( + &self.table, + self.filter.as_ref(), &core, &self.vector_column, &queries, @@ -264,7 +267,6 @@ fn intersect_row_ranges_by_split( #[allow(clippy::too_many_arguments)] async fn search_pk_raw_candidates_batch_with_plan( table: &Table, - query_options: &HashMap, filter: Option<&Predicate>, core: &CoreOptions<'_>, pk_col: &str, @@ -273,8 +275,8 @@ async fn search_pk_raw_candidates_batch_with_plan( plan: &PkVectorScanPlan, params: &PkVectorSearchParams, ) -> crate::Result> { - // An empty plan has nothing to search. Returned before the backend is resolved - // so a table with no searchable data never errors on an unrecognized index type. + // Options have been validated against the captured plan. An empty plan has + // no buckets to search, but must preserve one result per query. if plan.splits.is_empty() { return Ok(queries .iter() @@ -287,14 +289,14 @@ async fn search_pk_raw_candidates_batch_with_plan( let metric = params.metric; let concurrency = params.concurrency; - let index_type = params.index_type.clone(); + let index_type = params.vector_options.index_type.as_deref().unwrap_or(""); let vector_field = params.vector_field.clone(); let skip_exact_fallback = params.skip_exact_fallback; let indexed_limit = params.indexed_limit; - // All segments use the table's configured vindex type. An empty plan returns + // All valid segments use the format selected from the captured plan. An empty plan returns // above before opening readers or resolving concurrency limits. - if !is_vindex_index_type(&index_type) { + if !is_vindex_index_type(index_type) { return Err(crate::Error::DataInvalid { message: format!("unsupported PK vector index backend/type: '{index_type}'"), source: None, @@ -325,13 +327,8 @@ async fn search_pk_raw_candidates_batch_with_plan( // Real ANN scorer + loader. Each segment source is opened lazily inside its // bucket leaf and dropped after scoring. Range reads load only metadata // and the index data needed for the search. - let options = super::vector_search_common::vector_read_options( - table.schema().options(), - query_options, - pk_col, - &index_type, - )?; - let search_options = options.clone(); + let search_options = params.vector_options.options.clone(); + let options = search_options.clone(); let field_name = pk_col.to_string(); let loader_io = table.file_io().clone(); @@ -499,7 +496,6 @@ async fn search_pk_raw_candidates_batch_with_plan( #[allow(clippy::too_many_arguments)] async fn search_pk_candidates_batch_with_plan( table: &Table, - query_options: &HashMap, filter: Option<&Predicate>, core: &CoreOptions<'_>, pk_col: &str, @@ -509,15 +505,7 @@ async fn search_pk_candidates_batch_with_plan( params: &PkVectorSearchParams, ) -> crate::Result>> { let searches = search_pk_raw_candidates_batch_with_plan( - table, - query_options, - filter, - core, - pk_col, - queries, - limit, - plan, - params, + table, filter, core, pk_col, queries, limit, plan, params, ) .await?; diff --git a/crates/paimon/src/table/pk_vector_read/tests.rs b/crates/paimon/src/table/pk_vector_read/tests.rs index d3f9f8ffb..e60e803fb 100644 --- a/crates/paimon/src/table/pk_vector_read/tests.rs +++ b/crates/paimon/src/table/pk_vector_read/tests.rs @@ -947,15 +947,20 @@ async fn reader_uses_planned_splits_after_index_manifest_is_removed() { let table = build_committed_pk_vector_table(&[[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]).await; let options = HashMap::new(); let query = [0.0, 1.0]; - let params = - PkVectorSearchParams::resolve(&table, &options, None, "embedding", &[&query], 2).unwrap(); - let scan = crate::table::pk_vector_scan::PkVectorScan::new( + let params = PkVectorSearchParams::resolve( &table, - params.vector_field.id(), - params.index_type.clone(), + &options, None, - ); - let read = PkVectorRead::new(&table, &options, None, "embedding", &[&query], 2, params); + "embedding", + &[&query], + 2, + None, + None, + ) + .unwrap(); + let scan = + crate::table::pk_vector_scan::PkVectorScan::new(&table, params.vector_field.id(), None); + let read = PkVectorRead::new(&table, &options, None, "embedding", &[&query], 2); let plan = scan.plan().await.unwrap(); let snapshot_id = plan.snapshot_id; let manager = table.snapshot_manager(); @@ -1182,3 +1187,42 @@ fn residual_uses_only_data_conjuncts_of_the_filter() { "a mixed partition/data conjunct must stay whole in the residual" ); } + +#[tokio::test] +async fn reader_rejects_mixed_valid_payload_types_including_empty_markers() { + for cross_bucket in [false, true] { + let table = build_committed_pk_vector_table(&[[1.0, 0.0], [0.0, 1.0]]).await; + let scan = crate::table::pk_vector_scan::PkVectorScan::new(&table, 1, None); + let mut plan = scan.plan().await.unwrap(); + assert_eq!(plan.splits[0].ann_segments.len(), 1); + let mut other = plan.splits[0].ann_segments[0].clone(); + other.index_type = "ivf-pq".into(); + other.row_count = 0; + if cross_bucket { + let mut split = plan.splits[0].clone(); + split.data_split = DataSplitBuilder::new() + .with_snapshot(plan.snapshot_id) + .with_partition(split.data_split.partition().clone()) + .with_bucket(1) + .with_bucket_path("memory:/pk_vector_route_test/bucket-1".into()) + .with_total_buckets(2) + .with_data_files(split.data_split.data_files().to_vec()) + .build() + .unwrap(); + split.ann_segments = vec![other]; + plan.splits.push(split); + } else { + plan.splits[0].ann_segments.push(other); + } + let reader = PkVectorRead::new( + &table, + &HashMap::new(), + None, + "embedding", + &[&[1.0, 0.0]], + 1, + ); + let err = reader.read(plan).await.unwrap_err(); + assert!(err.to_string().contains("multiple index types"), "{err}"); + } +} diff --git a/crates/paimon/src/table/pk_vector_scan.rs b/crates/paimon/src/table/pk_vector_scan.rs index 6d08f96b6..8d55d35e0 100644 --- a/crates/paimon/src/table/pk_vector_scan.rs +++ b/crates/paimon/src/table/pk_vector_scan.rs @@ -42,6 +42,7 @@ use crate::vindex::pkvector::RowRangesByFile; /// A payload whose bucket-local path is resolved in planning Phase C, once the /// owning bucket's data split (and directory) is known. struct UnresolvedAnnSegment { + index_type: String, source_meta: PrimaryKeyIndexSourceMeta, file_name: String, external_path: Option, @@ -86,7 +87,9 @@ fn current_ann_segments( sources.sort_by(|a, b| a.file_name().cmp(b.file_name())); } - let mut segments_by_level: BTreeMap> = BTreeMap::new(); + // Duplicate matches invalidate one format's level. Distinct valid formats + // remain visible so the reader can reject a mixed-algorithm snapshot. + let mut segments_by_level: BTreeMap<(String, i32), Vec> = BTreeMap::new(); for segment in ann_segments { let source_meta = &segment.source_meta; let Some(desired) = sources_by_level.get(&source_meta.data_level()) else { @@ -96,7 +99,7 @@ fn current_ann_segments( && desired.as_slice() == source_meta.source_files() { segments_by_level - .entry(source_meta.data_level()) + .entry((segment.index_type.clone(), source_meta.data_level())) .or_default() .push(segment); } @@ -239,21 +242,14 @@ pub(crate) struct PkVectorScanPlan { pub(crate) struct PkVectorScan { table: Table, vector_field_id: i32, - index_type: String, filter: Option, } impl PkVectorScan { - pub(crate) fn new( - table: &Table, - vector_field_id: i32, - index_type: String, - filter: Option, - ) -> Self { + pub(crate) fn new(table: &Table, vector_field_id: i32, filter: Option) -> Self { Self { table: table.clone(), vector_field_id, - index_type, filter, } } @@ -288,7 +284,6 @@ impl PkVectorScan { }) }); plan_from_bucket_splits( - &self.index_type, self.vector_field_id, partition_filter.as_ref(), self.table.location().trim_end_matches('/'), @@ -377,7 +372,7 @@ impl Scan for PkVectorScan { entry.index_file.file_name, entry.kind ))); } - if entry.index_file.index_type != self.index_type { + if !crate::vindex::is_vindex_index_type(&entry.index_file.index_type) { continue; } let Some(gim) = entry.index_file.global_index_meta.clone() else { @@ -396,17 +391,15 @@ impl Scan for PkVectorScan { let partition = BinaryRow::from_serialized_bytes(&entry.partition)?; let file_size = u64::try_from(entry.index_file.file_size) .map_err(|_| data_invalid("index file size must not be negative"))?; - let source_meta = - PrimaryKeyIndexSourceMeta::from_global_index_meta(&gim).map_err(|_| { - data_invalid(format!( - "index file {} is not active", - entry.index_file.file_name - )) - })?; + let Ok(source_meta) = PrimaryKeyIndexSourceMeta::from_global_index_meta(&gim) + else { + continue; + }; entries.push(( partition, entry.bucket, UnresolvedAnnSegment { + index_type: entry.index_file.index_type.clone(), source_meta, file_name: entry.index_file.file_name.clone(), external_path: entry.index_file.external_path.clone(), @@ -444,7 +437,6 @@ impl Scan for PkVectorScan { /// so planning from engine-supplied splits is testable the same way planning from a /// manifest is. fn plan_from_bucket_splits( - index_type: &str, vector_field_id: i32, partition_filter: Option<&PartitionFilter>, table_path: &str, @@ -522,10 +514,10 @@ fn plan_from_bucket_splits( for payload in payload_files { let parts = payload.into_parts(); - // The same three filters the manifest route applies: this column's - // index type, this column's field id, and a payload that carries the + // The same three filters the manifest route applies: a supported + // vector format, this column's field id, and a payload that carries the // source metadata a search needs to map ordinals back to rows. - if parts.index_type != index_type + if !crate::vindex::is_vindex_index_type(&parts.index_type) || parts.global_index_meta.index_field_id != vector_field_id || parts.global_index_meta.source_meta.is_none() { @@ -535,11 +527,11 @@ fn plan_from_bucket_splits( // negative value the segment addressing cannot represent. let file_size = u64::try_from(parts.file_size) .map_err(|_| data_invalid("index file size must not be negative"))?; - let source_meta = + let Ok(source_meta) = PrimaryKeyIndexSourceMeta::from_global_index_meta(&parts.global_index_meta) - .map_err(|_| { - data_invalid(format!("index file {} is not active", parts.file_name)) - })?; + else { + continue; + }; let index_meta = parts .global_index_meta .index_meta @@ -549,6 +541,7 @@ fn plan_from_bucket_splits( data_split.partition().clone(), data_split.bucket(), UnresolvedAnnSegment { + index_type: parts.index_type, source_meta, file_name: parts.file_name, external_path: parts.external_path, @@ -626,7 +619,6 @@ pub(super) fn plan_from_search_splits( ) -> crate::Result { let snapshot_id = snapshot.as_ref().map_or(0, Snapshot::id); let core = table.schema().core_options(); - let index_type = super::pk_index_options::vector_index_type(table.schema().options(), column)?; let field_id = crate::table::find_field_id_by_name(table.schema().fields(), column) .ok_or_else(|| data_invalid("PK-vector column does not exist"))?; let partition = filter @@ -721,16 +713,22 @@ pub(super) fn plan_from_search_splits( .global_index_meta .as_ref() .ok_or_else(|| data_invalid("PK payload lacks global metadata"))?; - if file.index_type != index_type || meta.index_field_id != field_id { + if !crate::vindex::is_vindex_index_type(&file.index_type) + || meta.index_field_id != field_id + { return Err(data_invalid( "PK payload does not match the reader's vector column or index type", )); } + let Ok(source_meta) = PrimaryKeyIndexSourceMeta::from_global_index_meta(meta) else { + continue; + }; entries.push(( data_split.partition().clone(), data_split.bucket(), UnresolvedAnnSegment { - source_meta: PrimaryKeyIndexSourceMeta::from_global_index_meta(meta)?, + index_type: file.index_type.clone(), + source_meta, file_name: file.file_name.clone(), external_path: file.external_path.clone(), file_size: u64::try_from(file.file_size) @@ -856,6 +854,7 @@ fn plan_from_inputs( .unwrap_or_default() .into_iter() .map(|p| BucketAnnSegment { + index_type: p.index_type, source_meta: p.source_meta, path: location.resolve(&p.file_name, p.external_path.as_deref()), file_size: p.file_size, @@ -975,6 +974,7 @@ mod tests { fn ann_segment(data_level: i32, path: &str, source_files: &[(&str, i64)]) -> BucketAnnSegment { BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: PrimaryKeyIndexSourceMeta::new( data_level, source_files @@ -999,6 +999,7 @@ mod tests { external_path: Option<&str>, ) -> UnresolvedAnnSegment { UnresolvedAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: PrimaryKeyIndexSourceMeta::from_global_index_meta(&gim).unwrap(), file_name: file_name.to_string(), external_path: external_path.map(str::to_string), @@ -1130,6 +1131,65 @@ mod tests { assert!(current.is_empty()); } + #[test] + fn current_segments_keep_different_formats_and_ignore_stale_ones() { + let active = vec![dfm("a", 1, 5, Some(1))]; + let mut other = ann_segment(5, "pq", &[("a", 1)]); + other.index_type = "ivf-pq".into(); + other.row_count = 0; // A completed all-null unit still selects its format. + let mut stale = ann_segment(5, "stale", &[("a", 2)]); + stale.index_type = "diskann".into(); + let current = current_ann_segments( + &active, + vec![ann_segment(5, "flat", &[("a", 1)]), other, stale], + ) + .unwrap(); + assert_eq!( + current + .iter() + .map(|segment| (segment.index_type.as_str(), segment.row_count)) + .collect::>(), + vec![("ivf-flat", 1), ("ivf-pq", 0)] + ); + } + + #[test] + fn engine_plan_selects_valid_payload_independently_of_configured_format() { + let mut malformed = gim(2, 5, &[("d0", 4)]); + malformed.source_meta = Some(vec![0]); + let split = engine_split( + 11, + 0, + BinaryRow::new(0), + vec![dfm("d0", 4, 5, Some(1))], + vec![ + BucketVectorPayload::new_for_test( + "diskann", + "stale", + 1, + 4, + None, + None, + gim(2, 5, &[("old", 4)]), + ), + BucketVectorPayload::new_for_test( + "ivf-flat", + "malformed", + 1, + 4, + None, + None, + malformed, + ), + engine_payload(gim(2, 5, &[("d0", 4)])), + ], + &[], + ); + let plan = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]).unwrap(); + assert_eq!(plan.splits[0].ann_segments.len(), 1); + assert_eq!(plan.splits[0].ann_segments[0].index_type, "ivf-pq"); + } + #[test] fn rejects_data_split_with_wrong_snapshot() { let data = DataSplitBuilder::new() @@ -1222,9 +1282,6 @@ mod tests { const PRUNE_DIM: usize = 4; /// The primary-key vector column name. const PRUNE_VECTOR_COLUMN: &str = "embedding"; - /// vindex index type string; only used to route `PkVectorScan::new`, no index - /// segment is built for these tests. - const PRUNE_INDEX_TYPE: &str = "ivf-flat"; /// Number of rows written; `id`/`score` values live in `0..PRUNE_ROWS`. const PRUNE_ROWS: i32 = 4; /// A predicate literal guaranteed to fall outside the written `id`/`score` @@ -1393,31 +1450,23 @@ mod tests { )]), ] { let view = table.copy_with_options(extra); - assert!(PkVectorScan::new( - &view, - prune_vector_field_id(&view), - PRUNE_INDEX_TYPE.to_string(), - None - ) - .plan() - .await - .is_err()); + assert!(PkVectorScan::new(&view, prune_vector_field_id(&view), None) + .plan() + .await + .is_err()); } let empty_window = table.copy_with_options(HashMap::from([( "incremental-between-timestamp".to_string(), "0,1".to_string(), )])); - assert!(PkVectorScan::new( - &empty_window, - prune_vector_field_id(&empty_window), - PRUNE_INDEX_TYPE.to_string(), - None - ) - .plan() - .await - .unwrap() - .splits - .is_empty()); + assert!( + PkVectorScan::new(&empty_window, prune_vector_field_id(&empty_window), None) + .plan() + .await + .unwrap() + .splits + .is_empty() + ); } #[tokio::test] @@ -1430,15 +1479,10 @@ mod tests { options.insert("scan.snapshot-id".into(), snapshot.id().to_string()); } let view = table.copy_with_options(options); - let plan = PkVectorScan::new( - &view, - prune_vector_field_id(&view), - PRUNE_INDEX_TYPE.to_string(), - None, - ) - .plan() - .await - .unwrap(); + let plan = PkVectorScan::new(&view, prune_vector_field_id(&view), None) + .plan() + .await + .unwrap(); assert_eq!(plan.snapshot_id, snapshot.id()); assert!(!plan.splits.is_empty()); } @@ -1460,28 +1504,18 @@ mod tests { .await .unwrap(); for view in [pinned, tagged] { - let plan = PkVectorScan::new( - &view, - prune_vector_field_id(&view), - PRUNE_INDEX_TYPE.to_string(), - None, - ) - .plan() - .await - .unwrap(); + let plan = PkVectorScan::new(&view, prune_vector_field_id(&view), None) + .plan() + .await + .unwrap(); assert_eq!(plan.snapshot_id, snapshot.id()); assert_eq!(plan.splits.len(), 1); } let empty = table.copy_with_pinned_snapshot(None); - let plan = PkVectorScan::new( - &empty, - prune_vector_field_id(&empty), - PRUNE_INDEX_TYPE.to_string(), - None, - ) - .plan() - .await - .unwrap(); + let plan = PkVectorScan::new(&empty, prune_vector_field_id(&empty), None) + .plan() + .await + .unwrap(); assert_eq!(plan.snapshot_id, 0); assert!(plan.splits.is_empty()); } @@ -1494,11 +1528,10 @@ mod tests { let (_tmp, table) = build_pruning_test_table(false).await; let field_id = prune_vector_field_id(&table); - let unfiltered = - PkVectorScan::new(&table, field_id, PRUNE_INDEX_TYPE.to_string(), None) - .plan() - .await - .unwrap(); + let unfiltered = PkVectorScan::new(&table, field_id, None) + .plan() + .await + .unwrap(); assert_eq!( unfiltered.splits.len(), 1, @@ -1506,15 +1539,10 @@ mod tests { ); let out_of_range = prune_equal(&table, "id", OUT_OF_RANGE); - let filtered = PkVectorScan::new( - &table, - field_id, - PRUNE_INDEX_TYPE.to_string(), - Some(out_of_range), - ) - .plan() - .await - .unwrap(); + let filtered = PkVectorScan::new(&table, field_id, Some(out_of_range)) + .plan() + .await + .unwrap(); assert!( filtered.splits.is_empty(), "pk predicate stats-excludes the only file" @@ -1529,11 +1557,10 @@ mod tests { let field_id = prune_vector_field_id(&table); // Control: without a filter the file is present. - let unfiltered = - PkVectorScan::new(&table, field_id, PRUNE_INDEX_TYPE.to_string(), None) - .plan() - .await - .unwrap(); + let unfiltered = PkVectorScan::new(&table, field_id, None) + .plan() + .await + .unwrap(); assert_eq!( unfiltered.splits.len(), 1, @@ -1541,15 +1568,10 @@ mod tests { ); let out_of_range = prune_equal(&table, "score", OUT_OF_RANGE); - let filtered = PkVectorScan::new( - &table, - field_id, - PRUNE_INDEX_TYPE.to_string(), - Some(out_of_range), - ) - .plan() - .await - .unwrap(); + let filtered = PkVectorScan::new(&table, field_id, Some(out_of_range)) + .plan() + .await + .unwrap(); assert!( filtered.splits.is_empty(), "non-pk predicate stats-excludes the file under deletion vectors" @@ -1638,8 +1660,7 @@ mod tests { vec![payload], &[], ); - let plan = - plan_from_bucket_splits("ivf-flat", 1, None, "/tbl", false, vec![split]).unwrap(); + let plan = plan_from_bucket_splits(1, None, "/tbl", false, vec![split]).unwrap(); let split = &plan.splits[0]; assert_eq!(split.ann_segments[0].row_count, 0); assert_eq!(split.active_files[0].row_count, 6); @@ -1712,8 +1733,7 @@ mod tests { .unwrap() .planned(None); let split = BucketVectorSearchSplit::new_for_test(data_split, vec![], Default::default()); - let Err(err) = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, vec![split]) - else { + let Err(err) = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]) else { panic!("a marked split must not plan") }; assert!( @@ -1725,7 +1745,7 @@ mod tests { #[test] fn plans_the_java_golden_bucket_split() { let split = BucketVectorSearchSplit::deserialize(BUCKET_SPLIT_GOLDEN).unwrap(); - let plan = plan_from_bucket_splits("ivf-pq", 7, None, "/tbl", false, vec![split]).unwrap(); + let plan = plan_from_bucket_splits(7, None, "/tbl", false, vec![split]).unwrap(); // The snapshot the split pins, not one re-resolved from the table. assert_eq!(plan.snapshot_id, 11); @@ -1763,7 +1783,7 @@ mod tests { vec![engine_payload(gim(2, 5, &[("d0", i64::MAX)]))], &[], ); - let plan = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, vec![split]).unwrap(); + let plan = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]).unwrap(); let selections = plan .physical_row_ranges_by_split .expect("split-driven plan"); @@ -1786,11 +1806,9 @@ mod tests { ); // Derived from the fixture itself so the test cannot drift from it. let payload = &split.payload_files()[0]; - let index_type = payload.index_type().to_string(); let field_id = payload.global_index_meta().index_field_id; - let plan = plan_from_bucket_splits(&index_type, field_id, None, "/tbl", false, vec![split]) - .unwrap(); + let plan = plan_from_bucket_splits(field_id, None, "/tbl", false, vec![split]).unwrap(); let selections = plan .physical_row_ranges_by_split .expect("split-driven plan"); @@ -1811,13 +1829,11 @@ mod tests { // both return the same rows for this fixture. let split = BucketVectorSearchSplit::deserialize(BUCKET_SPLIT_NO_PREFILTER).unwrap(); let payload = &split.payload_files()[0]; - let index_type = payload.index_type().to_string(); let field_id = payload.global_index_meta().index_field_id; let source_meta = PrimaryKeyIndexSourceMeta::from_global_index_meta(payload.global_index_meta()).unwrap(); - let plan = plan_from_bucket_splits(&index_type, field_id, None, "/tbl", false, vec![split]) - .unwrap(); + let plan = plan_from_bucket_splits(field_id, None, "/tbl", false, vec![split]).unwrap(); let row_ranges_by_file = plan .physical_row_ranges_by_split .expect("split-driven plan") @@ -1843,7 +1859,7 @@ mod tests { #[test] fn rejects_empty_bucket_split_input() { - let error = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, Vec::new()) + let error = plan_from_bucket_splits(2, None, "/tbl", false, Vec::new()) .map(|_| ()) .expect_err("empty input pins no snapshot to report"); assert!( @@ -1855,7 +1871,6 @@ mod tests { #[test] fn rejects_bucket_splits_pinning_different_snapshots() { let error = plan_from_bucket_splits( - "ivf-pq", 2, None, "/tbl", @@ -1873,7 +1888,6 @@ mod tests { #[test] fn rejects_two_splits_for_one_bucket() { let error = plan_from_bucket_splits( - "ivf-pq", 2, None, "/tbl", @@ -1902,7 +1916,7 @@ mod tests { vec![engine_payload(gim(2, 5, &[("d0", 4)]))], IndexMap::new(), ); - let error = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, vec![split]) + let error = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]) .map(|_| ()) .expect_err("two row-range authorities may disagree"); assert!( @@ -1924,7 +1938,7 @@ mod tests { vec![engine_payload(gim(2, 5, &[("d0", 4), ("d1", 3)]))], &[("d0", &[])], ); - let plan = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, vec![split]).unwrap(); + let plan = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]).unwrap(); let ranges = plan .physical_row_ranges_by_split .expect("split-driven plan"); @@ -1952,7 +1966,7 @@ mod tests { )], &[], ); - let error = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, vec![split]) + let error = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]) .map(|_| ()) .expect_err("a signed wire size can be negative, segment addressing cannot"); assert!( @@ -1992,7 +2006,7 @@ mod tests { )], &[], ); - let plan = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, vec![split]).unwrap(); + let plan = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]).unwrap(); assert_eq!(plan.splits.len(), 1); assert_eq!(plan.splits[0].ann_segments.len(), 1); let ranges = plan @@ -2003,7 +2017,7 @@ mod tests { } #[test] - fn skips_payloads_for_another_column_or_index_type() { + fn skips_payloads_for_another_column_or_non_vector_type() { let split = engine_split( 11, 0, @@ -2025,7 +2039,7 @@ mod tests { ], &[], ); - let plan = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, vec![split]).unwrap(); + let plan = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]).unwrap(); assert_eq!(plan.splits.len(), 1); assert!(plan.splits[0].ann_segments.is_empty()); // Still exact-fallback eligible: no ANN segment covers the file. @@ -2049,7 +2063,6 @@ mod tests { // pruning first would hide an inconsistent input behind an empty plan. let filter = partition_filter_on_dt(3); let error = plan_from_bucket_splits( - "ivf-pq", 2, Some(&filter), "/tbl", @@ -2085,7 +2098,6 @@ mod tests { fn pruning_every_split_keeps_the_pinned_snapshot() { let filter = partition_filter_on_dt(3); let plan = plan_from_bucket_splits( - "ivf-pq", 2, Some(&filter), "/tbl", @@ -2112,11 +2124,11 @@ mod tests { #[test] fn resolves_a_payload_without_an_external_path_into_the_bucket_directory() { let split = one_file_split(11, 0, &[]); - let plan = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", true, vec![split]).unwrap(); + let plan = plan_from_bucket_splits(2, None, "/tbl", true, vec![split]).unwrap(); assert_eq!(plan.splits[0].ann_segments[0].path, "bucket-0/seg0"); let split = one_file_split(11, 0, &[]); - let plan = plan_from_bucket_splits("ivf-pq", 2, None, "/tbl", false, vec![split]).unwrap(); + let plan = plan_from_bucket_splits(2, None, "/tbl", false, vec![split]).unwrap(); assert_eq!(plan.splits[0].ann_segments[0].path, "/tbl/index/seg0"); } } diff --git a/crates/paimon/src/table/pk_vector_search_params.rs b/crates/paimon/src/table/pk_vector_search_params.rs index 6019dbe78..c638b29fb 100644 --- a/crates/paimon/src/table/pk_vector_search_params.rs +++ b/crates/paimon/src/table/pk_vector_search_params.rs @@ -15,28 +15,26 @@ // specific language governing permissions and limitations // under the License. -//! Resolve and validate PK vector-search parameters before scan planning. +//! Resolve PK search options once the captured plan selects a vector format. -use crate::spec::{CoreOptions, DataField, DataType, GlobalIndexSearchMode, Predicate}; +use crate::spec::{ + CoreOptions, DataField, DataType, GlobalIndexSearchMode, Predicate, VectorIndexOptions, +}; use crate::table::bucket_filter::split_partition_and_data_predicates; -use crate::table::vector_search_common::{configured_refine_factor, indexed_search_limit}; +use crate::table::vector_search_common::{ + indexed_search_limit, resolve_vector_search_options, resolved_refine_factor, +}; use crate::table::Table; use crate::vindex::pkvector::exact::validate_query; use crate::vindex::pkvector::metric::VectorSearchMetric; use crate::vindex::VindexVectorIndexOptions; use std::collections::HashMap; -/// Query-level parameters for a primary-key vector search: everything resolvable -/// from the table schema, the options and the queries alone, independent of which -/// splits planning yields. Resolved before planning so a malformed query or option -/// fails loud even when the plan turns out empty. #[derive(Clone)] pub(super) struct PkVectorSearchParams { pub(super) metric: VectorSearchMetric, - /// Fan-out limit for bucket orchestration plus ANN and exact-file leaves (Java - /// `GLOBAL_INDEX_THREAD_NUM`); `1` reproduces strictly sequential execution. pub(super) concurrency: usize, - pub(super) index_type: String, + pub(super) vector_options: VectorIndexOptions, pub(super) vector_field: DataField, pub(super) skip_exact_fallback: bool, pub(super) refine_factor: usize, @@ -44,168 +42,103 @@ pub(super) struct PkVectorSearchParams { } impl PkVectorSearchParams { - /// Resolve the query-level parameters and reject a query the search cannot answer - /// correctly, before any planning or read happens. - pub(super) fn resolve( - table: &Table, - query_options: &HashMap, - filter: Option<&Predicate>, - pk_col: &str, - queries: &[&[f32]], - limit: usize, - ) -> crate::Result { - Self::resolve_for_type( - table, - query_options, - filter, - pk_col, - queries, - limit, - None, - true, - ) - } - - #[allow(clippy::too_many_arguments)] - pub(super) fn resolve_for_type( + /// Validate inputs independent of the snapshot's actual index format. + pub(super) fn validate_inputs( table: &Table, - query_options: &HashMap, filter: Option<&Predicate>, - pk_col: &str, + column: &str, queries: &[&[f32]], limit: usize, - index_type: Option<&str>, - validate_dimension: bool, - ) -> crate::Result { - let core = CoreOptions::new(table.schema().options()); + ) -> crate::Result { + let core = table.schema().core_options(); core.ensure_read_authorized()?; - // Residual pre-filter guard, mirroring Java `PrimaryKeyVectorScan`. A DATA - // predicate set via `with_filter` is applied post-recall by re-reading each - // candidate file's physical rows during search. That physical-position filtering - // only agrees with the bucket search when the table exposes physical rows - // directly: deletion vectors enabled and merge-on-read disabled. Under - // merge-on-read (or without deletion vectors) a read merges multiple key - // versions, so a scalar filter could retain a stale version whose live version - // does not match — a silent wrong-read. Reject such queries rather than answer - // them incorrectly. - // - // Guard on the DATA conjuncts, not the whole filter: partition-only conjuncts - // are enforced entirely by scan planning (partition pruning) and produce no - // per-row residual, so they need no physical-row read. This mirrors Java, where - // `BatchVectorSearchBuilderImpl.withFilter` splits at the builder level and - // leaves `this.filter == null` for a partition-only filter — the scan guard is - // then skipped. No data predicate (partition-only or no filter) → nothing to - // guard, so the search-only and read paths are unaffected. - let physical_row_read = - core.deletion_vectors_enabled() && !core.deletion_vectors_merge_on_read(); let has_data_predicate = filter.is_some_and(|f| { - let (_partition, data) = split_partition_and_data_predicates( + let (_, data) = split_partition_and_data_predicates( f.clone(), table.schema().fields(), table.schema().partition_keys(), ); !data.is_empty() }); - if has_data_predicate && !physical_row_read { + if has_data_predicate + && !(core.deletion_vectors_enabled() && !core.deletion_vectors_merge_on_read()) + { return Err(crate::Error::DataInvalid { message: "primary-key vector pre-filter requires deletion vectors without merge-on-read" - .to_string(), + .into(), source: None, }); } - // Fan-out limit for bucket orchestration plus ANN and exact-file leaves (Java - // `GLOBAL_INDEX_THREAD_NUM`); `1` reproduces strictly sequential execution. - let concurrency = core.global_index_thread_num()?; - let raw_type_override = index_type.is_some(); - let index_type = if let Some(kind) = index_type { - super::vector_search_common::raw_vector_index_type( - table.schema().options(), - query_options, - pk_col, - Some(kind), - )? - .unwrap() - } else { - super::pk_index_options::vector_index_type(table.schema().options(), pk_col)? - .to_string() - }; - let metric = if raw_type_override { - let mut configured_metric = None; - for options in [query_options, table.schema().options()] { - let resolved = - CoreOptions::new(options).vector_index_options(&index_type, pk_col, false)?; - configured_metric = [ - format!("fields.{pk_col}.distance.metric"), - format!("fields.{pk_col}.metric"), - format!("{index_type}.metric"), - "metric".into(), - "distance.metric".into(), - ] - .iter() - .find_map(|key| resolved.get(key).cloned()); - if configured_metric.is_some() { - break; - } - } - VectorSearchMetric::parse(configured_metric.as_deref().unwrap_or("inner_product"))? - } else { - // PK readers retain Java's table metric validation. The explicit - // implementation affects only plans with no ANN metadata at all. - VectorSearchMetric::parse(&core.primary_key_vector_distance_metric(pk_col)?)? - }; - let vector_field = table + if limit == 0 { + return Err(crate::Error::DataInvalid { + message: "vector search limit must be positive".into(), + source: None, + }); + } + let field = table .schema() .fields() .iter() - .find(|f| f.name() == pk_col) + .find(|f| f.name() == column) .cloned() .ok_or_else(|| crate::Error::DataInvalid { - message: format!("PK-vector column '{pk_col}' not found in schema"), + message: format!("PK-vector column '{column}' not found in schema"), source: None, })?; - - let search_mode = core.vector_index_search_mode()?; - let skip_exact_fallback = search_mode == GlobalIndexSearchMode::Fast; - - // A non-positive limit is invalid regardless of the plan; reject it before - // planning so an empty plan cannot mask it with empty results. - if limit == 0 { - return Err(crate::Error::DataInvalid { - message: "vector search limit must be positive".to_string(), - source: None, - }); + for query in queries { + let dimension = match field.data_type() { + DataType::Vector(vector) => vector.length() as usize, + _ => query.len().max(1), + }; + validate_query(query, dimension)?; } + Ok(field) + } - // Resolve the refine factor from the query options first, then fall back to - // the table options; a positive factor over-fetches indexed (approximate) - // candidates so the reader's exact rerank has a wider pool to reorder. Factor 0 - // (unset) leaves `indexed_limit == limit`, byte-identical to the no-rerank - // path. The two option maps are kept distinct (query options passed - // separately from table options) so a broad query key cannot be overridden - // by a more specific table key: query options take precedence as a whole. - // Resolved before planning so an invalid factor (e.g. a non-numeric value) - // fails loud regardless of whether the table currently has searchable data. - let refine_factor = - configured_refine_factor(query_options, table.schema().options(), pk_col, &index_type)?; - let indexed_limit = indexed_search_limit(limit, refine_factor)?; - - // Validate every query against the vector column's dimension (and finiteness) - // before planning or any read, so a malformed query fails loud even when the - // plan turns out empty. VECTOR carries the dimension in its type; - // ARRAY gets the index dimension from the same vindex option resolver - // used by index reads. Both valid PK-vector column shapes must reject NaN/Inf - // up front, not only after a non-empty plan opens readers. - // Only ARRAY's implementation-dependent width waits for the source - // plan. Fixed schema widths, emptiness and finiteness remain eager. - for query in queries { - validate_query(query, query.len().max(1))?; + #[allow(clippy::too_many_arguments)] + pub(super) fn resolve( + table: &Table, + query_options: &HashMap, + filter: Option<&Predicate>, + column: &str, + queries: &[&[f32]], + limit: usize, + metadata_type: Option<&str>, + raw_type: Option<&str>, + ) -> crate::Result { + let vector_field = Self::validate_inputs(table, filter, column, queries, limit)?; + let core = CoreOptions::new(table.schema().options()); + let configuration = resolve_vector_search_options( + table.schema().options(), + query_options, + column, + metadata_type.or(raw_type), + true, + )?; + let mut vector_options = configuration.resolved; + // Like Java, persisted PK payloads are checked against the table metric. + // The Native hint affects only fallback when no valid payload exists. + if metadata_type.is_some() || raw_type.is_none() { + vector_options.metric = configuration.table_metric; } - if validate_dimension || matches!(vector_field.data_type(), DataType::Vector(_)) { - if let Some(dimension) = pk_vector_query_dimension( - table.schema().options(), - query_options, - &index_type, + let metric = VectorSearchMetric::parse(&vector_options.metric)?; + let refine_factor = if let Some(kind) = vector_options.index_type.as_deref() { + resolved_refine_factor( + &configuration.query_options, + &vector_options.options, + column, + kind, + )? + } else { + 0 + }; + let indexed_limit = indexed_search_limit(limit, refine_factor)?; + if let Some(kind) = vector_options.index_type.as_deref() { + if let Some(dimension) = resolved_query_dimension( + &vector_options.options, + &configuration.query_options, + kind, &vector_field, )? { for query in queries { @@ -213,38 +146,29 @@ impl PkVectorSearchParams { } } } - Ok(Self { metric, - concurrency, - index_type, + concurrency: core.global_index_thread_num()?, + vector_options, vector_field, - skip_exact_fallback, + skip_exact_fallback: core.vector_index_search_mode()? == GlobalIndexSearchMode::Fast, refine_factor, indexed_limit, }) } } -fn pk_vector_query_dimension( +fn resolved_query_dimension( table_options: &HashMap, query_options: &HashMap, index_type: &str, vector_field: &DataField, ) -> crate::Result> { match vector_field.data_type() { - DataType::Vector(vector_type) - if matches!(vector_type.element_type(), DataType::Float(_)) => - { - Ok(Some(vector_type.length() as usize)) + DataType::Vector(vector) if matches!(vector.element_type(), DataType::Float(_)) => { + Ok(Some(vector.length() as usize)) } - DataType::Array(array_type) if matches!(array_type.element_type(), DataType::Float(_)) => { - // ARRAY carries its dimension in the vindex options. - let query_options = CoreOptions::new(query_options).vector_index_options( - index_type, - vector_field.name(), - false, - )?; + DataType::Array(array) if matches!(array.element_type(), DataType::Float(_)) => { let mut dimension_options = HashMap::new(); for key in [ "dimension".to_string(), @@ -262,25 +186,18 @@ fn pk_vector_query_dimension( dimension_options.insert(key, value.clone()); } } - let mut table_dimension_options = table_options.clone(); - let declared_options = super::pk_index_options::field_options( - table_options, - vector_field.name(), - "vector", - index_type, - )?; + let mut defaults = table_options.clone(); if let Some(value) = table_options.get("dimension") { if !table_options.contains_key(&format!("{index_type}.dimension")) && !table_options.contains_key(&format!("{index_type}.index.dimension")) { - table_dimension_options - .insert(format!("{index_type}.dimension"), value.clone()); + defaults.insert(format!("{index_type}.dimension"), value.clone()); } } Ok(Some( VindexVectorIndexOptions::new_with_defaults( - &table_dimension_options, - &declared_options, + &defaults, + &HashMap::new(), &dimension_options, index_type, vector_field, @@ -292,5 +209,17 @@ fn pk_vector_query_dimension( } } +#[cfg(test)] +fn pk_vector_query_dimension( + table: &HashMap, + query: &HashMap, + kind: &str, + field: &DataField, +) -> crate::Result> { + let table = CoreOptions::new(table).vector_index_options(Some(kind), field.name(), false)?; + let query = CoreOptions::new(query).vector_index_options(Some(kind), field.name(), false)?; + resolved_query_dimension(&table.options, &query.options, kind, field) +} + #[cfg(test)] mod tests; diff --git a/crates/paimon/src/table/search_scan_split.rs b/crates/paimon/src/table/search_scan_split.rs index 25f0672d4..067f58dbe 100644 --- a/crates/paimon/src/table/search_scan_split.rs +++ b/crates/paimon/src/table/search_scan_split.rs @@ -21,7 +21,7 @@ use super::bucket_filter::split_partition_and_data_predicates; use super::global_index_scanner::unindexed_ranges_for_global_index_entries; use super::global_index_types::normalize_queryable_global_index_type; use super::row_id_predicate::intersect_sorted_ranges; -use super::vector_search_common::raw_vector_index_type; +use super::vector_search_common::resolve_vector_search_options; use super::{find_field_id_by_name, merge_row_ranges, DataSplit, RowRange, Table}; use crate::spec::{ CoreOptions, FileKind, GlobalIndexSearchMode, IndexFileMeta, IndexManifestEntry, Predicate, @@ -213,7 +213,15 @@ pub(super) async fn global_search_splits( Some(kind) => Some(kind), None if full_text => None, None => { - raw_vector_index_type(table.schema().options(), options, column, vector_index_type)? + resolve_vector_search_options( + table.schema().options(), + options, + column, + vector_index_type, + false, + )? + .resolved + .index_type } }; result.push(SearchScanSplit::Raw { diff --git a/crates/paimon/src/table/table_commit/auto_index_tests.rs b/crates/paimon/src/table/table_commit/auto_index_tests.rs index 80dbb729c..267dbe334 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests.rs @@ -594,7 +594,7 @@ async fn vector_definitions_require_a_supported_algorithm_and_metric() { .await .unwrap_err() .to_string() - .contains("fields.embedding.global-index.vector.index-type")); + .contains("Vector index implementation is not configured")); for (key, value) in [ ("fields.embedding.global-index.vector.index-type", "btree"), ( diff --git a/crates/paimon/src/table/table_commit/auto_index_tests/search.rs b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs index 8b770458d..17571856d 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests/search.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs @@ -514,6 +514,7 @@ async fn full_and_detail_vector_fallback_share_declared_metrics_with_indexes() { // A declared metric (including its default) has the same build-over-table // precedence as explicit native builder options. ("fields.embedding.distance.metric", "l2"), + ("ivf-flat.metric", "l2"), ("fields.embedding.index-type", "diskann"), ]; if let Some(metric) = metric { diff --git a/crates/paimon/src/table/vector_read.rs b/crates/paimon/src/table/vector_read.rs index a89338c7f..f9a43d6e1 100644 --- a/crates/paimon/src/table/vector_read.rs +++ b/crates/paimon/src/table/vector_read.rs @@ -88,16 +88,10 @@ impl BatchVectorRead { ) -> crate::Result { let core = CoreOptions::new(table.schema().options()); let reader = if targets_primary_key_column(table, column) { - let pk_col = core.primary_key_vector_index_column()?; - let params = if context.vector_index_type.is_some() { - PkVectorSearchParams::resolve_for_type( - table, options, filter, &pk_col, queries, limit, None, false, - )? - } else { - PkVectorSearchParams::resolve(table, options, filter, &pk_col, queries, limit)? - }; + let pk_col = core.single_vector_index_column()?; + PkVectorSearchParams::validate_inputs(table, filter, &pk_col, queries, limit)?; VectorReadKind::PrimaryKey(Box::new( - PkVectorRead::new(table, options, filter, &pk_col, queries, limit, params) + PkVectorRead::new(table, options, filter, &pk_col, queries, limit) .with_vector_index_type(context.vector_index_type.clone()), )) } else { diff --git a/crates/paimon/src/table/vector_scan.rs b/crates/paimon/src/table/vector_scan.rs index a7fb4f7a6..ab7079f86 100644 --- a/crates/paimon/src/table/vector_scan.rs +++ b/crates/paimon/src/table/vector_scan.rs @@ -267,7 +267,7 @@ impl VectorScan { let context = PlanContext::new(table, column, filter, include_row_ids, prepared)?; let core = CoreOptions::new(table.schema().options()); let scan = if targets_primary_key_column(table, column) { - let column = core.primary_key_vector_index_column()?; + let column = core.single_vector_index_column()?; let field_id = find_field_id_by_name(table.schema().fields(), &column).ok_or_else(|| { crate::Error::DataInvalid { @@ -278,8 +278,6 @@ impl VectorScan { VectorScanKind::PrimaryKey(Box::new(PkVectorScan::new( table, field_id, - super::pk_index_options::vector_index_type(table.schema().options(), &column)? - .to_string(), filter.cloned(), ))) } else { diff --git a/crates/paimon/src/table/vector_search_common.rs b/crates/paimon/src/table/vector_search_common.rs index e1fd9bc55..ffc66ff94 100644 --- a/crates/paimon/src/table/vector_search_common.rs +++ b/crates/paimon/src/table/vector_search_common.rs @@ -197,37 +197,54 @@ pub(super) fn validate_global_vector_index_types( Ok(()) } -pub(super) fn configured_vector_index_type( - table_options: &HashMap, - query_options: &HashMap, - column: &str, -) -> crate::Result> { - if let Some(kind) = super::global_index_options::vector_index_type(query_options, column)? { - return Ok(Some(kind.into())); - } - if let Some(kind) = super::global_index_options::vector_index_type(table_options, column)? { - return Ok(Some(kind.into())); - } - Ok(None) +/// Resolve query and table sources separately, preserving query-over-table +/// precedence while keeping the implementation and its parameters together. +pub(super) struct VectorSearchOptions { + pub table_metric: String, + pub resolved: crate::spec::VectorIndexOptions, + pub query_options: HashMap, } -/// Resolve only after discovering that no persisted vector implementation exists. -pub(super) fn raw_vector_index_type( +pub(super) fn resolve_vector_search_options( table_options: &HashMap, query_options: &HashMap, column: &str, explicit: Option<&str>, -) -> crate::Result> { - let Some(kind) = explicit else { - return configured_vector_index_type(table_options, query_options, column); - }; - let kind = kind.trim().to_ascii_lowercase(); - if !is_vindex_index_type(&kind) { - return Err(crate::Error::ConfigInvalid { - message: format!("Unsupported vector index type for {column}: {kind}"), - }); + default_metric: bool, +) -> crate::Result { + let mut query = + CoreOptions::new(query_options).vector_index_options(explicit, column, false)?; + let mut table = CoreOptions::new(table_options).vector_index_options( + query.index_type.as_deref().or(explicit), + column, + default_metric, + )?; + if query.index_type.is_none() && table.index_type.is_some() { + query = CoreOptions::new(query_options).vector_index_options( + table.index_type.as_deref(), + column, + false, + )?; } - Ok(Some(kind)) + let table_metric = table.metric.clone(); + if let Some(kind) = table.index_type.as_deref() { + if !is_vindex_index_type(kind) { + return Err(crate::Error::ConfigInvalid { + message: format!("Unsupported vector index type for {column}: {kind}"), + }); + } + if let Some(metric) = query.options.get(&format!("{kind}.metric")) { + table.metric = metric.clone(); + } + table.options = merge_vector_read_options(&table.options, &query.options, column, kind); + } else { + table.options.extend(query.options.clone()); + } + Ok(VectorSearchOptions { + table_metric, + resolved: table, + query_options: query.options, + }) } pub(super) fn current_tokio_runtime_handle() -> crate::Result { @@ -515,20 +532,34 @@ fn add_refine_prefixes(prefixes: &mut Vec, base: &str, index_type: &str) prefixes.push(base.to_string()); } +#[cfg(test)] pub(super) fn configured_refine_factor( search_options: &HashMap, table_options: &HashMap, field_name: &str, index_type: &str, ) -> crate::Result { - let query = - CoreOptions::new(search_options).vector_index_options(index_type, field_name, false)?; + let query = CoreOptions::new(search_options) + .vector_index_options(Some(index_type), field_name, false)? + .options; if let Some(value) = configured_refine_factor_from_options(&query, field_name, index_type) { return parse_refine_factor(&value); } - let resolved = - CoreOptions::new(table_options).vector_index_options(index_type, field_name, false)?; - if let Some(value) = configured_refine_factor_from_options(&resolved, field_name, index_type) { + let resolved = CoreOptions::new(table_options) + .vector_index_options(Some(index_type), field_name, false)? + .options; + resolved_refine_factor(&query, &resolved, field_name, index_type) +} + +pub(super) fn resolved_refine_factor( + query: &HashMap, + table: &HashMap, + field_name: &str, + index_type: &str, +) -> crate::Result { + if let Some(value) = configured_refine_factor_from_options(query, field_name, index_type) + .or_else(|| configured_refine_factor_from_options(table, field_name, index_type)) + { return parse_refine_factor(&value); } Ok(0) @@ -542,8 +573,21 @@ pub(super) fn vector_read_options( column: &str, kind: &str, ) -> crate::Result> { - let resolve = |source: &HashMap| -> crate::Result> { - let mut options = CoreOptions::new(source).vector_index_options(kind, column, false)?; + Ok( + resolve_vector_search_options(table, query, column, Some(kind), false)? + .resolved + .options, + ) +} + +pub(super) fn merge_vector_read_options( + table: &HashMap, + query: &HashMap, + column: &str, + kind: &str, +) -> HashMap { + let normalize = |source: &HashMap| { + let mut options = source.clone(); for (native, suffixes) in [ ("ivf.nprobe", &["nprobe", "ivf.nprobe"][..]), ("diskann.l_search", &["l_search", "diskann.l_search"][..]), @@ -559,11 +603,11 @@ pub(super) fn vector_read_options( options.insert(native.into(), value); } } - Ok(options) + options }; - let mut options = resolve(table)?; - options.extend(resolve(query)?); - Ok(options) + let mut options = normalize(table); + options.extend(normalize(query)); + options } fn configured_refine_factor_from_options( @@ -605,17 +649,41 @@ pub(super) fn targets_primary_key_column(table: &Table, column: &str) -> bool { let core = table.schema().core_options(); !table.schema().primary_keys().is_empty() && !core.data_evolution_enabled() - && core.primary_key_vector_index_enabled() - && core - .primary_key_vector_index_columns() - .ok() - .is_some_and(|columns| columns.iter().any(|c| c == column)) + && core.vector_index_enabled() + && core.vector_index_columns().iter().any(|c| c == column) } #[cfg(test)] mod tests { use super::*; use crate::io::FileIOBuilder; + + #[test] + fn shared_search_result_keeps_legacy_defaults_separate_from_query_metrics() { + let table = HashMap::from([ + ( + "global-index.vector.index-columns".into(), + "embedding".into(), + ), + ( + "fields.embedding.global-index.vector.index-type".into(), + "ivf-flat".into(), + ), + ("ivf-flat.metric".into(), "l2".into()), + ]); + let plain = + resolve_vector_search_options(&table, &HashMap::new(), "embedding", None, false) + .unwrap(); + assert_eq!(plain.resolved.metric, "inner_product"); + assert!(!plain.query_options.contains_key("ivf-flat.metric")); + // A query source can also carry a compatibility declaration. Only its + // explicit metric, not its compatibility build default, constrains reads. + let query = + resolve_vector_search_options(&table, &table, "embedding", None, false).unwrap(); + assert_eq!(query.table_metric, "inner_product"); + assert_eq!(query.resolved.metric, "l2"); + assert_eq!(query.query_options["ivf-flat.metric"], "l2"); + } use crate::spec::{GlobalIndexMeta, IndexFileMeta}; #[test] diff --git a/crates/paimon/src/vindex/pkvector/bucket.rs b/crates/paimon/src/vindex/pkvector/bucket.rs index e5deeeea2..2f98c46a5 100644 --- a/crates/paimon/src/vindex/pkvector/bucket.rs +++ b/crates/paimon/src/vindex/pkvector/bucket.rs @@ -55,6 +55,7 @@ pub(crate) type ExactFileSearchFuture<'a> = /// sources stay range-backed instead of holding the full index bytes. #[derive(Clone)] pub(crate) struct BucketAnnSegment { + pub index_type: String, pub source_meta: PkVectorSourceMeta, /// Resolved index-file path (globally unique; the key the loader reads by). pub path: String, @@ -70,6 +71,7 @@ impl BucketAnnSegment { /// `source_meta`-driven logic. pub(crate) fn for_test(source_meta: PkVectorSourceMeta) -> Self { Self { + index_type: "ivf-flat".to_string(), source_meta, path: "seg".to_string(), file_size: 0, @@ -1628,6 +1630,7 @@ mod tests { #[tokio::test] async fn test_rejects_duplicate_ann_segment_path() { let seg1 = BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: meta(&[("data-1", 2)]), path: "duplicate-path".to_string(), file_size: 100, @@ -1635,6 +1638,7 @@ mod tests { index_meta: vec![1, 2, 3], }; let seg2 = BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: meta(&[("data-2", 2)]), path: "duplicate-path".to_string(), file_size: 200, @@ -1670,6 +1674,7 @@ mod tests { #[tokio::test] async fn test_rejects_source_file_covered_by_multiple_segments() { let seg1 = BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: meta(&[("data-1", 2)]), path: "segment-1".to_string(), file_size: 100, @@ -1677,6 +1682,7 @@ mod tests { index_meta: vec![1, 2, 3], }; let seg2 = BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: meta(&[("data-1", 2), ("data-2", 2)]), path: "segment-2".to_string(), file_size: 200, @@ -2616,6 +2622,7 @@ mod tests { for i in 0..n { let src = format!("cov-{i}"); segments.push(BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: meta(&[(src.as_str(), 2)]), path: format!("seg-{i}"), file_size: 0, @@ -2780,6 +2787,7 @@ mod tests { let (mut segments, _active_files, probe, peak) = n_segment_bucket(2, None); // Break segment 1's source row count vs the active file (active says 2). segments[1] = BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: meta(&[("cov-1", 99)]), path: "seg-1".to_string(), file_size: 0, @@ -3264,6 +3272,7 @@ mod tests { loaded: loaded.clone(), }); let seg_a = BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: meta(&[("cov-a", 2)]), path: "seg-a".to_string(), file_size: 0, @@ -3271,6 +3280,7 @@ mod tests { index_meta: Vec::new(), }; let seg_b = BucketAnnSegment { + index_type: "ivf-flat".to_string(), source_meta: meta(&[("cov-b", 2)]), path: "seg-b".to_string(), file_size: 0, diff --git a/crates/paimon/tests/pk_compact/indexes.rs b/crates/paimon/tests/pk_compact/indexes.rs index 0f0ca87f3..9eb3e3413 100644 --- a/crates/paimon/tests/pk_compact/indexes.rs +++ b/crates/paimon/tests/pk_compact/indexes.rs @@ -842,8 +842,9 @@ async fn legacy_field_metric_aliases_override_pk_metric_consistently() { table .schema() .core_options() - .vector_index_options("ivf-flat", "embedding", true) - .unwrap()["ivf-flat.metric"], + .vector_index_options(Some("ivf-flat"), "embedding", true) + .unwrap() + .options["ivf-flat.metric"], "cosine" ); write_batch( diff --git a/crates/paimon/tests/pk_vector_baseline_test.rs b/crates/paimon/tests/pk_vector_baseline_test.rs index 39a004d20..8b0bd9422 100644 --- a/crates/paimon/tests/pk_vector_baseline_test.rs +++ b/crates/paimon/tests/pk_vector_baseline_test.rs @@ -573,6 +573,58 @@ fn fixture_discriminating() -> ([f32; DIM], Vec<[f32; DIM]>) { (query, vectors) } +#[cfg(not(windows))] +#[tokio::test] +async fn pk_vector_reads_captured_payload_after_algorithm_configuration_changes() { + let (query, vectors) = fixture_discriminating(); + let (_tmp, table) = build_table(&query, &vectors, 3).await; + let expected = analytic_topk(&query, &vectors, 3); + for declared in ["ivf-pq", "diskann"] { + let changed = table.copy_with_options(HashMap::from([ + ( + format!("fields.{VECTOR_COLUMN}.pk-vector.index.type"), + declared.into(), + ), + ( + format!("global-index.{declared}.index-columns"), + VECTOR_COLUMN.into(), + ), + (format!("{declared}.metric"), "invalid-unused-metric".into()), + // These parameters must resolve using the captured IVF-flat type. + ("ivf-flat.refine_factor".into(), "2".into()), + ])); + let mut builder = changed.new_vector_search_builder(); + builder + .with_vector_column(VECTOR_COLUMN) + .with_vector_index_type(VECTOR_COLUMN, "invalid-unused-hint") + .with_query_vector(query.to_vec()) + .with_limit(3); + let plan = builder.new_scan().unwrap().plan().await.unwrap(); + let result = builder.new_read().unwrap().read(plan).await.unwrap(); + assert_eq!( + result + .positions() + .unwrap() + .iter() + .map(|p| p.row_position as u64) + .collect::>(), + expected.iter().map(|(pos, _)| *pos).collect::>() + ); + assert_eq!( + result + .positions() + .unwrap() + .iter() + .map(|p| p.score) + .collect::>(), + expected + .iter() + .map(|(_, distance)| l2_score(*distance)) + .collect::>() + ); + } +} + // Gated off Windows: the fixture table location is a `file://` URL built from a // temp dir path, which `FileIO` cannot derive on Windows (see #397); the sibling // `rest_catalog_test` gates its identical `file://` tempdir tests the same way. @@ -1493,8 +1545,8 @@ async fn pk_vector_invalid_refine_factor_fails_loud_on_empty_table() { ); } - // Case 3: non-integer via a TABLE option (the fix moved table-option - // resolution ahead of planning too). + // Case 3: non-integer via a TABLE option is validated after planning, + // before the empty-plan early return. { let tmp = tempfile::tempdir().expect("create temp dir"); let location = format!("file://{}", tmp.path().display()); diff --git a/docs/src/compaction.md b/docs/src/compaction.md index 7cc253877..c19d55000 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -161,7 +161,12 @@ precedence; the hint selects the implementation for raw fallback when no vector index metadata is available. Captured scans, readers, hybrid routes and worker transport retain it. Ordinary parameters still use `with_options`, and readers leave an unspecified metric unset so global index metadata can supply it. PK -readers retain Java's table metric validation. +readers retain Java's table metric validation. PK scans collect supported vector +payloads for the column and discard stale source-file sets before readers select +the algorithm. All valid payloads, including completed empty segments, must use +one algorithm. With none left, readers use the configured declaration or Native +hint. Type, parameters and metric come from one captured resolver result, reused +for raw fallback, refinement and distributed index tasks. The option aliases `index_type`, `index-type`, `vector.index-type` and `fields..index-type` no longer select vector implementations. Use the From ea3a5c0c9c5b290c2e8efb76c4c5b59171b2eb3c Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 14:56:50 +0800 Subject: [PATCH 13/14] fix(native): accept raw index options without type setters --- .../python/pypaimon_rust/datafusion.pyi | 3 - bindings/python/src/hybrid_search.rs | 13 - bindings/python/src/vector_search.rs | 29 +- bindings/python/tests/test_vector_search.py | 92 +++- .../src/spec/core_options/index_options.rs | 19 + .../src/table/batch_vector_search_builder.rs | 17 +- crates/paimon/src/table/de_vector_read.rs | 18 +- .../paimon/src/table/de_vector_read/tests.rs | 16 +- .../src/table/full_text_search_builder.rs | 52 +++ .../table/full_text_search_builder/scan.rs | 1 - .../src/table/global_index_build_builder.rs | 29 +- .../src/table/global_index_build_common.rs | 4 +- .../paimon/src/table/global_index_options.rs | 41 ++ .../paimon/src/table/hybrid_search_builder.rs | 19 - crates/paimon/src/table/pk_vector_read.rs | 8 - .../paimon/src/table/pk_vector_read/tests.rs | 14 +- .../src/table/pk_vector_search_params.rs | 9 +- crates/paimon/src/table/search_scan_split.rs | 3 +- .../table_commit/auto_index_tests/search.rs | 117 +++++ crates/paimon/src/table/vector_read.rs | 12 +- crates/paimon/src/table/vector_scan.rs | 8 - .../paimon/src/table/vector_search_builder.rs | 24 +- ...e_type_tests.rs => index_options_tests.rs} | 432 ++++++++++-------- .../src/table/vindex_index_build_builder.rs | 85 ++-- crates/paimon/src/vector_search.rs | 3 - crates/paimon/src/vindex/mod.rs | 24 + crates/paimon/tests/pk_compact/indexes.rs | 1 - .../paimon/tests/pk_vector_baseline_test.rs | 2 - docs/src/compaction.md | 28 +- 29 files changed, 684 insertions(+), 439 deletions(-) rename crates/paimon/src/table/vector_search_builder/{native_type_tests.rs => index_options_tests.rs} (57%) diff --git a/bindings/python/python/pypaimon_rust/datafusion.pyi b/bindings/python/python/pypaimon_rust/datafusion.pyi index 6e0a8819a..0ede5a1fb 100644 --- a/bindings/python/python/pypaimon_rust/datafusion.pyi +++ b/bindings/python/python/pypaimon_rust/datafusion.pyi @@ -1139,7 +1139,6 @@ class SQLContext: class VectorSearchBuilder: - def with_vector_index_type(self, column: str, index_type: str) -> "VectorSearchBuilder": ... def with_vector_column(self, name: str) -> "VectorSearchBuilder": ... def with_limit(self, limit: int) -> "VectorSearchBuilder": ... def with_query_vector(self, vector: Sequence[float]) -> "VectorSearchBuilder": ... @@ -1154,7 +1153,6 @@ class VectorSearchBuilder: def execute(self) -> "GlobalIndexResult": ... class BatchVectorSearchBuilder: - def with_vector_index_type(self, column: str, index_type: str) -> "BatchVectorSearchBuilder": ... def with_vector_column(self, name: str) -> "BatchVectorSearchBuilder": ... def with_limit(self, limit: int) -> "BatchVectorSearchBuilder": ... def with_query_vectors(self, vectors: Sequence[Sequence[float]]) -> "BatchVectorSearchBuilder": ... @@ -1269,7 +1267,6 @@ class HybridSearchRouteResult: def result(self) -> GlobalIndexResult: ... class HybridSearchBuilder: - def with_vector_index_type(self, column: str, index_type: str) -> "HybridSearchBuilder": ... def with_limit(self, limit: int) -> HybridSearchBuilder: ... def with_snapshot(self, snapshot: Optional[Snapshot]) -> HybridSearchBuilder: """Pin the resolved snapshot object; None pins an empty read view.""" diff --git a/bindings/python/src/hybrid_search.rs b/bindings/python/src/hybrid_search.rs index cc52dce3d..6888f21cc 100644 --- a/bindings/python/src/hybrid_search.rs +++ b/bindings/python/src/hybrid_search.rs @@ -278,7 +278,6 @@ pub struct PyHybridSearchBuilder { limit: Option, ranker: HybridSearchRanker, pinned_snapshot: Option>, - vector_index_types: HashMap, } impl PyHybridSearchBuilder { @@ -290,15 +289,11 @@ impl PyHybridSearchBuilder { limit: None, ranker: HybridSearchRanker::Rrf, pinned_snapshot: None, - vector_index_types: HashMap::new(), } } fn builder(&self) -> paimon::table::HybridSearchBuilder<'_> { let mut builder = self.table.new_hybrid_search_builder(); - for (column, index_type) in &self.vector_index_types { - builder.with_vector_index_type(column, index_type); - } for route in &self.routes { builder.add_route(route.clone()); } @@ -321,14 +316,6 @@ impl PyHybridSearchBuilder { #[pymethods] impl PyHybridSearchBuilder { - fn with_vector_index_type( - mut slf: PyRefMut<'_, Self>, - column: String, - index_type: String, - ) -> PyRefMut<'_, Self> { - slf.vector_index_types.insert(column, index_type); - slf - } #[pyo3(signature = (field_name, vector, limit, weight=1.0, options=None))] fn add_vector_route( mut slf: PyRefMut<'_, Self>, diff --git a/bindings/python/src/vector_search.rs b/bindings/python/src/vector_search.rs index 2beffa261..45394422a 100644 --- a/bindings/python/src/vector_search.rs +++ b/bindings/python/src/vector_search.rs @@ -38,7 +38,6 @@ struct SearchConfig { column: Option, limit: Option, options: Vec>, - vector_index_types: HashMap, filters: Vec, partition_filters: Vec, } @@ -49,9 +48,6 @@ impl SearchConfig { table: &'a Table, ) -> paimon::Result> { let mut builder = table.new_batch_vector_search_builder(); - for (column, index_type) in &self.vector_index_types { - builder.with_vector_index_type(column, index_type); - } if let Some(column) = &self.column { builder.with_vector_column(column); } @@ -97,14 +93,6 @@ impl PyVectorSearchBuilder { slf.inner.with_vector_column(&name); slf } - fn with_vector_index_type( - mut slf: PyRefMut<'_, Self>, - column: String, - index_type: String, - ) -> PyRefMut<'_, Self> { - slf.inner.with_vector_index_type(&column, &index_type); - slf - } fn with_limit(mut slf: PyRefMut<'_, Self>, limit: usize) -> PyRefMut<'_, Self> { slf.inner.with_limit(limit); slf @@ -181,14 +169,6 @@ impl PyBatchVectorSearchBuilder { slf.config.column = Some(name); slf } - fn with_vector_index_type( - mut slf: PyRefMut<'_, Self>, - column: String, - index_type: String, - ) -> PyRefMut<'_, Self> { - slf.config.vector_index_types.insert(column, index_type); - slf - } fn with_limit(mut slf: PyRefMut<'_, Self>, limit: usize) -> PyRefMut<'_, Self> { slf.config.limit = Some(limit); slf @@ -683,22 +663,15 @@ for factory, expected in ( single = table.new_vector_search_builder().with_vector_column('embedding').with_query_vector([1., 0.]).with_limit(1) for factory in (table.new_vector_search_builder, table.new_batch_vector_search_builder, table.new_hybrid_search_builder): - configured = factory() - # This API must only store configuration: neither a query nor a snapshot - # is available, and a superseded invalid implementation is accepted here. - assert configured.with_vector_index_type(column='embedding', index_type='invalid') is configured - assert configured.with_vector_index_type('embedding', 'ivf-flat') is configured -assert single.with_vector_index_type('embedding', 'ivf-flat') is single + assert not hasattr(factory(), 'with_vector_index_type') assert isinstance(single.execute(), sdk.GlobalIndexResult) assert len(single.execute()) == 0 batch = table.new_batch_vector_search_builder().with_vector_column('embedding').with_query_vectors([[1., 0.], [0., 1.]]).with_limit(1) -assert batch.with_vector_index_type('embedding', 'diskann') is batch results = batch.execute() assert len(results) == 2 and all(isinstance(result, sdk.GlobalIndexResult) and len(result) == 0 for result in results) text = table.new_full_text_search_builder().with_query('body', 'needle').with_limit(1) assert isinstance(text.execute(), sdk.GlobalIndexResult) and len(text.execute()) == 0 hybrid = table.new_hybrid_search_builder().add_vector_route('embedding', [1., 0.], 1).add_full_text_route('body', 'needle', 1).with_limit(1) -assert hybrid.with_vector_index_type('embedding', 'ivf-flat') is hybrid assert isinstance(hybrid.execute(), sdk.GlobalIndexResult) and len(hybrid.execute()) == 0 try: batch.with_query_vectors([]).execute() diff --git a/bindings/python/tests/test_vector_search.py b/bindings/python/tests/test_vector_search.py index 881bbe86d..41d680b67 100644 --- a/bindings/python/tests/test_vector_search.py +++ b/bindings/python/tests/test_vector_search.py @@ -52,6 +52,67 @@ def _single(table): return table.new_vector_search_builder().with_vector_column('embedding').with_query_vector([1, 0]).with_limit(10) +def test_vector_build_accepts_raw_schema_options_and_field_json(table): + options = { + 'bucket': '-1', 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', + 'global-index.enabled': 'true', 'vector-index.search-mode': 'full', + 'global-index.ivf-flat.index-columns': 'embedding', + 'fields.embedding.pk-vector.index.options': 'invalid replaced JSON', + 'fields.embedding.index-options': '{"dimension":2,"nlist":1,"metric":"cosine"}', + 'ivf-flat.metric': 'l2', 'opaque.future.option': 'preserved', + } + configured = table.copy(options) + builder = (configured.new_global_index_build_builder().with_index_column('embedding') + .with_index_type('ivf-flat').with_options(options)) + assert builder.plan().shards() + assert builder.execute() > 0 + actual = _single(configured).execute().row_ids() + assert sorted(actual.values()) == pytest.approx([0, 1, 1]) + + +@pytest.mark.parametrize('shared_json', [False, True, 'aliases']) +def test_full_text_json_encoded_lists_keep_raw_and_indexed_semantics(tmp_path, shared_json): + import json + ctx = SQLContext() + ctx.register_catalog('paimon', {'warehouse': str(tmp_path)}) + ctx.sql('CREATE SCHEMA paimon.db') + ctx.sql("CREATE TABLE paimon.db.texts (id INT, body STRING) WITH (" + "'bucket'='-1','row-tracking.enabled'='true','data-evolution.enabled'='true'," + "'global-index.enabled'='true','write-only'='true','full-text-index.search-mode'='full')") + table = PaimonCatalog({'warehouse': str(tmp_path)}).get_table('db.texts') + write = table.new_batch_write_builder() + writer, commit = write.new_write(), write.new_commit() + try: + writer.write_arrow(pa.record_batch([[1, 2], ['paimon data', 'needle']], schema=pa.schema([ + ('id', pa.int32()), ('body', pa.string())]))) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + options = {'global-index.full-text.index-columns': 'body'} + if shared_json == 'aliases': + options.update({'full-text.stop-words': json.dumps(['paimon']), 'full-text.columns': json.dumps(['body'])}) + elif shared_json: + options['fields.body.index-options'] = json.dumps({'stop-words': ['paimon'], 'text-fields': ['body']}) + else: + options.update({'full-text.stop-words': json.dumps(['paimon']), + 'full-text.text-fields': json.dumps(['body'])}) + table = table.copy(options) + + def hits(query): + return (table.new_full_text_search_builder().with_query('body', json.dumps({'match': {'query': query}})) + .with_limit(3).execute().row_ids()) + + assert len(hits('paimon')) == 0 + before = hits('data') + assert len(before) == 1 + builder = (table.new_global_index_build_builder().with_index_column('body') + .with_index_type('full-text').with_options(options)) + assert builder.execute() == 1 + assert len(hits('paimon')) == 0 + assert hits('data') == before + + def test_filters_accumulate_and_partition_uses_table_schema(table): result = (_single(table).with_filter({'method': 'greaterThan', 'field': 'id', 'literals': [1]}) .with_filter({'method': 'lessThan', 'field': 'id', 'literals': [4]}) @@ -159,16 +220,19 @@ def test_single_vector_keyword_matches_stub(table): @pytest.mark.parametrize('batch', [False, True]) -def test_explicit_native_type_survives_scan_read_and_uses_raw_metric_options(table, batch): +def test_raw_options_survive_scan_read_and_use_raw_metric_options(table, batch): import pickle builder = (table.new_batch_vector_search_builder().with_query_vectors([[1, 0], [1, 0]]) if batch else table.new_vector_search_builder().with_query_vector([1, 0])) - assert builder.with_vector_index_type(column='embedding', index_type='ivf-flat') is builder + builder.with_options({'global-index.ivf-flat.index-columns': 'embedding', + 'fields.embedding.index-options': '{"dimension":2,"nlist":1}', + 'opaque.future.option': '{"value":[1,2]}'}) builder.with_vector_column('embedding').with_limit(3) plan = pickle.loads(pickle.dumps(builder.new_vector_search_scan().scan())) reader = builder.new_batch_vector_search_read() if batch else builder.new_vector_search_read() # Mutating the builder must not change captured reader configuration. - builder.with_vector_index_type('embedding', 'diskann').with_option('metric', 'l2') + builder.with_options({'global-index.ivf-flat.index-columns': 'other', + 'global-index.diskann.index-columns': 'embedding', 'metric': 'l2'}) results = reader.read_batch_plan(plan) if batch else [reader.read_plan(plan)] for result in results: assert sorted(result.row_ids().values()) == pytest.approx([0, 1, 1]) @@ -177,14 +241,14 @@ def test_explicit_native_type_survives_scan_read_and_uses_raw_metric_options(tab assert sorted(result.row_ids().values()) == pytest.approx([1 / 3, 1, 1]) -def test_native_type_keeps_index_metadata_and_ignores_removed_option_hints(table): +def test_options_keep_index_metadata_and_ignore_removed_option_hints(table): expected = _single(table).execute().row_ids() for key in ('index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type'): assert _single(table.copy({key: 'diskann'})).with_option(key, 'ivf-flat').execute().row_ids() == expected (table.new_global_index_build_builder().with_index_column('embedding') .with_index_type('ivf-flat').with_options({'dimension': '2', 'nlist': '1', 'metric': 'l2'}).execute()) expected = _single(table).execute().row_ids() - assert (_single(table).with_vector_index_type('embedding', 'invalid-unused-type') + assert (_single(table).with_options({'global-index.invalid-unused-type.index-columns': 'embedding'}) .execute().row_ids()) == expected # Table defaults are not explicit query metrics. Both JSON namespaces obey # that boundary when the persisted index uses a different metric. @@ -193,24 +257,30 @@ def test_native_type_keeps_index_metadata_and_ignores_removed_option_hints(table ('fields.embedding.index-options', '{"metric":"cosine"}'), ('fields.embedding.pk-vector.index.options', '{"metric":"cosine"}'), ): - assert (_single(table.copy({key: value})).with_vector_index_type('embedding', 'diskann') + assert (_single(table.copy({key: value, 'global-index.diskann.index-columns': 'embedding'})) .execute().row_ids()) == expected with pytest.raises(Exception, match='does not match index metric'): _single(table).with_option(key, value).execute() -def test_hybrid_native_type_is_captured_with_the_route_builder(table): +def test_hybrid_options_are_captured_with_the_route_builder(table): hybrid = table.new_hybrid_search_builder() - assert hybrid.with_vector_index_type('embedding', 'ivf-flat') is hybrid hybrid.add_vector_route('embedding', [1, 0], 3, options={ - 'ivf-flat.metric': 'l2', 'diskann.metric': 'inner_product'}).with_limit(3) + 'global-index.ivf-flat.index-columns': 'embedding', 'ivf-flat.metric': 'l2'}).with_limit(3) captured = hybrid.route_builders()[0].search_builder() - hybrid.with_vector_index_type('embedding', 'diskann') + hybrid.add_vector_route('embedding', [1, 0], 3, options={ + 'global-index.diskann.index-columns': 'embedding', 'diskann.metric': 'inner_product'}) assert sorted(captured.execute().row_ids().values()) == pytest.approx([1 / 3, 1, 1]) - updated = hybrid.route_builders()[0].search_builder() + updated = hybrid.route_builders()[1].search_builder() assert sorted(updated.execute().row_ids().values()) == pytest.approx([0, 1, 1]) +def test_builders_have_no_separate_type_setter(table): + for builder in (table.new_vector_search_builder(), table.new_batch_vector_search_builder(), + table.new_hybrid_search_builder()): + assert not hasattr(builder, 'with_vector_index_type') + + def test_external_result_uses_same_scan_type_without_search_context(table): found = _single(table).with_limit(1).execute() external = GlobalIndexResult.from_scored_row_ids(found.row_ids()) diff --git a/crates/paimon/src/spec/core_options/index_options.rs b/crates/paimon/src/spec/core_options/index_options.rs index b26d9185c..41660f5e1 100644 --- a/crates/paimon/src/spec/core_options/index_options.rs +++ b/crates/paimon/src/spec/core_options/index_options.rs @@ -200,6 +200,25 @@ impl CoreOptions<'_> { &legacy_key }; let mut resolved = self.options.clone(); + if kind == "full-text" { + for (_, value) in resolved + .iter_mut() + .filter(|(key, _)| key.starts_with("full-text.")) + { + if let Ok(values) = serde_json::from_str::>(value) { + *value = values + .into_iter() + .map(|value| { + value + .as_str() + .map(str::to_string) + .unwrap_or_else(|| value.to_string()) + }) + .collect::>() + .join(";"); + } + } + } if let Some(value) = self .options .get(&format!("{field}global-index.row-count-per-shard")) diff --git a/crates/paimon/src/table/batch_vector_search_builder.rs b/crates/paimon/src/table/batch_vector_search_builder.rs index 871c9b6c1..4853a27f3 100644 --- a/crates/paimon/src/table/batch_vector_search_builder.rs +++ b/crates/paimon/src/table/batch_vector_search_builder.rs @@ -34,7 +34,6 @@ pub struct BatchVectorSearchBuilder<'a> { query_vectors: Option>>, limit: Option, options: HashMap, - vector_index_types: HashMap, filter: Option, include_row_ids: Option>, prepared_filter: Option, @@ -48,7 +47,6 @@ impl<'a> BatchVectorSearchBuilder<'a> { query_vectors: None, limit: None, options: HashMap::new(), - vector_index_types: HashMap::new(), filter: None, include_row_ids: None, prepared_filter: None, @@ -60,13 +58,6 @@ impl<'a> BatchVectorSearchBuilder<'a> { self } - /// Store a raw-fallback implementation without planning or executing a query. - pub fn with_vector_index_type(&mut self, column: &str, index_type: &str) -> &mut Self { - self.vector_index_types - .insert(column.into(), index_type.into()); - self - } - pub fn with_query_vectors(&mut self, vectors: Vec>) -> &mut Self { self.query_vectors = Some(vectors); self @@ -131,23 +122,19 @@ impl<'a> BatchVectorSearchBuilder<'a> { self.include_row_ids.as_ref(), self.prepared_filter.as_ref(), ) - .map(|scan| { - scan.with_options(&self.options) - .with_vector_index_type(self.vector_index_types.get(column).cloned()) - }) + .map(|scan| scan.with_options(&self.options)) } /// Create an owned batch reader; result i belongs to input query i. pub fn new_read(&self) -> crate::Result { let column = self.column()?; - let mut context = PlanContext::new( + let context = PlanContext::new( self.table, column, self.filter.as_ref(), self.include_row_ids.as_ref(), self.prepared_filter.as_ref(), )?; - context.vector_index_type = self.vector_index_types.get(column).cloned(); let queries = self .query_vectors .as_deref() diff --git a/crates/paimon/src/table/de_vector_read.rs b/crates/paimon/src/table/de_vector_read.rs index 21cfade25..ada9b8a3d 100644 --- a/crates/paimon/src/table/de_vector_read.rs +++ b/crates/paimon/src/table/de_vector_read.rs @@ -73,13 +73,6 @@ pub(super) struct DeVectorRead { } impl DeVectorRead { - pub(super) fn with_vector_index_type(mut self, index_type: Option) -> Self { - for search in &mut self.vector_searches { - search.index_type = index_type.clone(); - } - self - } - /// Validate query parameters before planning, including for empty tables and /// filters. Snapshot-dependent row-ID filters are supplied by the plan. pub(super) fn new( @@ -759,8 +752,7 @@ async fn evaluate_batch_vector_search_with_executor( .find_map(|source| match source { crate::table::SearchScanSplit::Raw { index_type, .. } => index_type.as_deref(), _ => None, - }) - .or(vector_searches[0].index_type.as_deref()); + }); let configuration = resolve_vector_search_options( evaluation.table_options, &search_options, @@ -1701,7 +1693,7 @@ async fn resolve_raw_vector_metric( index_entries: &[IndexManifestEntry], field_id: i32, field_name: &str, - explicit_index_type: Option<&str>, + source_index_type: Option<&str>, ) -> crate::Result { let persisted_type = index_entries .iter() @@ -1719,7 +1711,7 @@ async fn resolve_raw_vector_metric( table_options, search_options, field_name, - persisted_type.or(explicit_index_type), + persisted_type.or(source_index_type), false, )?; resolve_raw_vector_metric_resolved( @@ -2664,8 +2656,6 @@ struct TaskVectorSearch { limit: usize, field: String, options: HashMap, - #[serde(default)] - index_type: Option, include: Option, } #[derive(Clone, Serialize, Deserialize)] @@ -2702,7 +2692,6 @@ impl TaskVectorQueries { limit: search.limit, field: search.field_name.clone(), options: search.options.clone(), - index_type: search.index_type.clone(), include, }) }) @@ -2733,7 +2722,6 @@ impl TaskVectorQueries { query.field.clone(), )? .with_options(query.options.clone()); - search.index_type = query.index_type.clone(); if let Some(index) = query.include { let shared = filters.get(index).ok_or_else(|| { crate::table::search_task::invalid( diff --git a/crates/paimon/src/table/de_vector_read/tests.rs b/crates/paimon/src/table/de_vector_read/tests.rs index 3df3f0c65..3c210847d 100644 --- a/crates/paimon/src/table/de_vector_read/tests.rs +++ b/crates/paimon/src/table/de_vector_read/tests.rs @@ -246,7 +246,13 @@ fn scheduled_vector_transport_restores_one_shared_raw_filter_per_bitmap() { .collect::>(); for search in &mut searches { search.set_shared_include_row_ids(Arc::clone(&shared)); - search.index_type = Some("diskann".into()); + search.options = HashMap::from([ + ( + "global-index.diskann.index-columns".into(), + "embedding".into(), + ), + ("opaque.parameter".into(), "unchanged".into()), + ]); } let wire = TaskVectorQueries::capture(&searches).unwrap(); assert_eq!( @@ -255,10 +261,10 @@ fn scheduled_vector_transport_restores_one_shared_raw_filter_per_bitmap() { "encode a shared bitmap once per query batch" ); let restored = wire.restore().unwrap(); - assert!(restored - .iter() - .all(|search| search.index_type.as_deref() == Some("diskann") - && !search.options.contains_key("index_type"))); + assert!(restored.iter().all( + |search| search.options["global-index.diskann.index-columns"] == "embedding" + && search.options["opaque.parameter"] == "unchanged" + )); let restored_shared = shared_batch_include_row_ids(&restored).unwrap(); assert_eq!(restored_shared.as_ref(), shared.as_ref()); let scoring = RawScoringPlan::new(&restored, RawVectorMetric::L2); diff --git a/crates/paimon/src/table/full_text_search_builder.rs b/crates/paimon/src/table/full_text_search_builder.rs index 60388875d..92e901e88 100644 --- a/crates/paimon/src/table/full_text_search_builder.rs +++ b/crates/paimon/src/table/full_text_search_builder.rs @@ -1331,6 +1331,58 @@ mod tests { assert_eq!(normalize_query_text(dsl, "body").unwrap(), dsl); } + #[test] + fn json_encoded_flat_analyzer_collections_match_field_json() { + for options in [ + HashMap::from([ + ( + "full-text.stop-words".into(), + r#"["paimon","index"]"#.into(), + ), + ("full-text.columns".into(), r#"["body","title"]"#.into()), + ]), + HashMap::from([ + ( + "full-text.stop-words".into(), + r#"["paimon","index"]"#.into(), + ), + ("full-text.text_fields".into(), r#"["body","title"]"#.into()), + ]), + HashMap::from([ + ( + "full-text.stop-words".into(), + r#"["paimon","index"]"#.into(), + ), + ("full-text.text-fields".into(), r#"["body","title"]"#.into()), + ]), + HashMap::from([( + "fields.body.index-options".into(), + r#"{"stop-words":["paimon","index"],"text-fields":["body","title"]}"#.into(), + )]), + ] { + let config = raw_full_text_config(&options, "body").unwrap(); + assert_eq!(config.text_fields, vec!["body", "title"]); + assert_eq!(config.tokenizer.stop_words, vec!["paimon", "index"]); + let defaults = super::super::global_index_options::field_index_options( + &options, + "body", + "full-text", + "full-text", + ) + .unwrap(); + let mut merged = options.clone(); + merged.extend(defaults); + let native = super::super::index_build::full_text::native_full_text_options(&merged); + for (key, value) in &native { + match key.replace('_', "-").as_str() { + "stop-words" => assert_eq!(value, "paimon;index"), + "text-fields" | "columns" => assert_eq!(value, "body;title"), + _ => (), + } + } + } + } + #[test] fn test_raw_full_text_config_preserves_native_text_fields() { let options = HashMap::from([( diff --git a/crates/paimon/src/table/full_text_search_builder/scan.rs b/crates/paimon/src/table/full_text_search_builder/scan.rs index f1102fb17..6d1fde5d5 100644 --- a/crates/paimon/src/table/full_text_search_builder/scan.rs +++ b/crates/paimon/src/table/full_text_search_builder/scan.rs @@ -246,7 +246,6 @@ impl FullTextScan { self.context.filter.as_ref(), &Default::default(), true, - None, ) .await? }; diff --git a/crates/paimon/src/table/global_index_build_builder.rs b/crates/paimon/src/table/global_index_build_builder.rs index 120f2de8b..d9968f081 100644 --- a/crates/paimon/src/table/global_index_build_builder.rs +++ b/crates/paimon/src/table/global_index_build_builder.rs @@ -110,13 +110,22 @@ impl<'a> GlobalIndexBuildBuilder<'a> { return Ok(self.table.clone()); } // Native vector builders keep declarations in their own option layer. - let defaults = super::global_index_options::declared_options( + let overrides = super::global_index_options::build_table_options( self.table.schema().options(), + &self.options, + columns[0], + ) + .into_iter() + .filter(|(key, value)| self.table.schema().options().get(key) != Some(value)) + .collect(); + let effective = self.table.copy_with_options(overrides); + let defaults = super::global_index_options::declared_options( + effective.schema().options(), columns[0], index_type, index_type, )?; - Ok(self.table.copy_with_options(defaults)) + Ok(effective.copy_with_options(defaults)) } /// Restrict the build to partitions. Repeated calls combine filters with AND. @@ -249,7 +258,15 @@ impl<'a> GlobalIndexBuildBuilder<'a> { builder .with_index_columns(&columns) .with_index_type(index_type) - .with_options(self.options.clone()); + .with_options(if columns.len() == 1 { + super::global_index_options::build_options( + &self.options, + columns[0], + index_type, + )? + } else { + self.options.clone() + }); if let Some(filter) = &self.partition_filter { builder.with_partition_filter(filter.clone())?; } @@ -258,9 +275,9 @@ impl<'a> GlobalIndexBuildBuilder<'a> { if index_type == FULL_TEXT_GLOBAL_INDEX_TYPE { let mut builder = FullTextIndexBuildBuilder::new(&table); builder.view = view; - builder - .with_index_column(columns[0]) - .with_options(self.options.clone()); + builder.with_index_column(columns[0]).with_options( + super::global_index_options::build_options(&self.options, columns[0], index_type)?, + ); builder.partition_filter = self.partition_filter.clone(); return Ok(builder.prepare().await?.into()); } diff --git a/crates/paimon/src/table/global_index_build_common.rs b/crates/paimon/src/table/global_index_build_common.rs index 1704d6072..099befa78 100644 --- a/crates/paimon/src/table/global_index_build_common.rs +++ b/crates/paimon/src/table/global_index_build_common.rs @@ -75,9 +75,7 @@ impl GlobalIndexBuildView { pub(super) async fn resolve(table: &Table, captured: Option<&Self>) -> Result { match captured { Some(view) => Ok(view.clone()), - None => { - Self::capture(table, table.snapshot_manager().get_latest_snapshot().await?).await - } + None => Self::capture(table, table.resolve_read_snapshot().await?).await, } } diff --git a/crates/paimon/src/table/global_index_options.rs b/crates/paimon/src/table/global_index_options.rs index 032bbf9c0..a6e9f72e2 100644 --- a/crates/paimon/src/table/global_index_options.rs +++ b/crates/paimon/src/table/global_index_options.rs @@ -176,6 +176,47 @@ pub(super) fn field_index_options( .collect()) } +/// A procedure JSON object replaces that field's schema object before parsing. +/// Flat procedure parameters keep their own precedence layer and cannot create +/// false conflicts with legacy JSON in the table source. +pub(super) fn build_table_options( + table: &HashMap, + user: &HashMap, + column: &str, +) -> HashMap { + let mut effective = table.clone(); + let prefix = format!("fields.{column}."); + effective.extend( + user.iter() + .filter(|(key, _)| { + key.starts_with(&prefix) + && (key.ends_with(".index-options") || key.ends_with(".index.options")) + }) + .map(|(key, value)| (key.clone(), value.clone())), + ); + effective +} + +/// Parse a procedure's original option map without treating table properties as +/// native vector parameters. Keep the original map in captured build plans. +pub(super) fn build_options( + options: &HashMap, + column: &str, + kind: &str, +) -> Result> { + let core = CoreOptions::new(options); + if crate::vindex::is_vindex_index_type(kind) { + let resolved = core.vector_index_options( + Some(kind), + column, + declares_column(options, column, "vector"), + )?; + Ok(resolved.options) + } else { + core.global_index_options(kind, column) + } +} + pub(super) fn declares_column( options: &HashMap, column: &str, diff --git a/crates/paimon/src/table/hybrid_search_builder.rs b/crates/paimon/src/table/hybrid_search_builder.rs index fcec26106..1eda3c33a 100644 --- a/crates/paimon/src/table/hybrid_search_builder.rs +++ b/crates/paimon/src/table/hybrid_search_builder.rs @@ -392,7 +392,6 @@ pub struct HybridSearchBuilder<'a> { ranker: HybridSearchRanker, filter: Option, pinned_snapshot: Option>, - vector_index_types: HashMap, } impl<'a> HybridSearchBuilder<'a> { @@ -404,7 +403,6 @@ impl<'a> HybridSearchBuilder<'a> { ranker: HybridSearchRanker::Rrf, filter: None, pinned_snapshot: None, - vector_index_types: HashMap::new(), } } @@ -413,13 +411,6 @@ impl<'a> HybridSearchBuilder<'a> { self } - /// Store each vector route's raw-fallback implementation, even before routes exist. - pub fn with_vector_index_type(&mut self, column: &str, index_type: &str) -> &mut Self { - self.vector_index_types - .insert(column.into(), index_type.into()); - self - } - pub fn add_vector_route( &mut self, field_name: &str, @@ -630,9 +621,6 @@ impl<'a> HybridSearchBuilder<'a> { super::VectorSearchBuilder::from_owned(table.clone()), route, self.filter.as_ref(), - self.vector_index_types - .get(&route.field_name) - .map(String::as_str), )) } HybridSearchRouteKind::FullText => { @@ -661,9 +649,6 @@ impl<'a> HybridSearchBuilder<'a> { table.new_vector_search_builder(), route, self.filter.as_ref(), - self.vector_index_types - .get(&route.field_name) - .map(String::as_str), ) } @@ -1249,16 +1234,12 @@ fn configure_vector_route_builder<'t>( mut builder: super::VectorSearchBuilder<'t>, route: &HybridSearchRoute, filter: Option<&Predicate>, - index_type: Option<&str>, ) -> super::VectorSearchBuilder<'t> { builder .with_vector_column(&route.field_name) .with_query_vector(route.vector.clone().expect("validated vector route")) .with_limit(route.limit) .with_options(route.options.clone()); - if let Some(index_type) = index_type { - builder.with_vector_index_type(&route.field_name, index_type); - } if let Some(filter) = filter { builder.with_filter(filter.clone()); } diff --git a/crates/paimon/src/table/pk_vector_read.rs b/crates/paimon/src/table/pk_vector_read.rs index ed74ecfce..aa3508b84 100644 --- a/crates/paimon/src/table/pk_vector_read.rs +++ b/crates/paimon/src/table/pk_vector_read.rs @@ -61,7 +61,6 @@ pub(super) struct PkVectorRead { vector_column: String, queries: Vec>, limit: usize, - raw_index_type: Option, } impl PkVectorRead { @@ -81,14 +80,8 @@ impl PkVectorRead { vector_column: vector_column.to_string(), queries: queries.iter().map(|query| query.to_vec()).collect(), limit, - raw_index_type: None, } } - - pub(super) fn with_vector_index_type(mut self, index_type: Option) -> Self { - self.raw_index_type = index_type; - self - } } impl Read for PkVectorRead { @@ -120,7 +113,6 @@ impl Read for PkVectorRead { &queries, self.limit, metadata_type, - self.raw_index_type.as_deref(), )?; let candidates = search_pk_candidates_batch_with_plan( &self.table, diff --git a/crates/paimon/src/table/pk_vector_read/tests.rs b/crates/paimon/src/table/pk_vector_read/tests.rs index e60e803fb..fffce91f2 100644 --- a/crates/paimon/src/table/pk_vector_read/tests.rs +++ b/crates/paimon/src/table/pk_vector_read/tests.rs @@ -947,17 +947,9 @@ async fn reader_uses_planned_splits_after_index_manifest_is_removed() { let table = build_committed_pk_vector_table(&[[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]).await; let options = HashMap::new(); let query = [0.0, 1.0]; - let params = PkVectorSearchParams::resolve( - &table, - &options, - None, - "embedding", - &[&query], - 2, - None, - None, - ) - .unwrap(); + let params = + PkVectorSearchParams::resolve(&table, &options, None, "embedding", &[&query], 2, None) + .unwrap(); let scan = crate::table::pk_vector_scan::PkVectorScan::new(&table, params.vector_field.id(), None); let read = PkVectorRead::new(&table, &options, None, "embedding", &[&query], 2); diff --git a/crates/paimon/src/table/pk_vector_search_params.rs b/crates/paimon/src/table/pk_vector_search_params.rs index c638b29fb..95daf9e72 100644 --- a/crates/paimon/src/table/pk_vector_search_params.rs +++ b/crates/paimon/src/table/pk_vector_search_params.rs @@ -96,7 +96,6 @@ impl PkVectorSearchParams { Ok(field) } - #[allow(clippy::too_many_arguments)] pub(super) fn resolve( table: &Table, query_options: &HashMap, @@ -105,7 +104,6 @@ impl PkVectorSearchParams { queries: &[&[f32]], limit: usize, metadata_type: Option<&str>, - raw_type: Option<&str>, ) -> crate::Result { let vector_field = Self::validate_inputs(table, filter, column, queries, limit)?; let core = CoreOptions::new(table.schema().options()); @@ -113,15 +111,12 @@ impl PkVectorSearchParams { table.schema().options(), query_options, column, - metadata_type.or(raw_type), + metadata_type, true, )?; let mut vector_options = configuration.resolved; // Like Java, persisted PK payloads are checked against the table metric. - // The Native hint affects only fallback when no valid payload exists. - if metadata_type.is_some() || raw_type.is_none() { - vector_options.metric = configuration.table_metric; - } + vector_options.metric = configuration.table_metric; let metric = VectorSearchMetric::parse(&vector_options.metric)?; let refine_factor = if let Some(kind) = vector_options.index_type.as_deref() { resolved_refine_factor( diff --git a/crates/paimon/src/table/search_scan_split.rs b/crates/paimon/src/table/search_scan_split.rs index 067f58dbe..26f1befc0 100644 --- a/crates/paimon/src/table/search_scan_split.rs +++ b/crates/paimon/src/table/search_scan_split.rs @@ -60,7 +60,6 @@ pub(super) async fn global_search_splits( filter: Option<&Predicate>, options: &HashMap, full_text: bool, - vector_index_type: Option<&str>, ) -> crate::Result> { let field_id = find_field_id_by_name(table.schema().fields(), column).ok_or_else(|| { crate::Error::ConfigInvalid { @@ -217,7 +216,7 @@ pub(super) async fn global_search_splits( table.schema().options(), options, column, - vector_index_type, + None, false, )? .resolved diff --git a/crates/paimon/src/table/table_commit/auto_index_tests/search.rs b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs index 17571856d..d3938264f 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests/search.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs @@ -781,3 +781,120 @@ async fn vector_algorithm_and_metric_changes_atomically_replace_incompatible_pay ); } } + +#[tokio::test] +async fn raw_native_build_maps_share_field_json_and_full_text_collections() { + let (_catalog, table) = search_table(&[ + ("write-only", "true"), + ("global-index.btree.index-columns", "id"), + ("btree-index.block-size", "invalid overridden default"), + ("fields.id.index-options", r#"{"block-size":128}"#), + ("global-index.full-text.index-columns", "text,folded_text"), + ("full-text.tokenizer", "invalid overridden tokenizer"), + ("full-text.stop-words", r#"["foo"]"#), + ( + "fields.text.index-options", + r#"{"tokenizer":"raw","lower-case":false}"#, + ), + ( + "fields.folded_text.index-options", + r#"{"tokenizer":"simple","lower-case":true}"#, + ), + ("full-text-index.search-mode", "full"), + ("opaque.future.parameter", "preserved"), + ]) + .await; + TableCommit::new(table.clone(), "raw-native-map".into()) + .commit(search_messages(&table, &[[1., 0.], [0., 1.]]).await) + .await + .unwrap(); + let snapshot = latest(&table).await; + let raw = table.copy_with_pinned_snapshot(Some(&snapshot)); + let ids = |table: Table, field: &'static str| async move { + table + .new_full_text_search_builder() + .with_query(field, r#"{"match":{"query":"FOO"}}"#) + .with_limit(10) + .execute() + .await + .unwrap() + .row_ids() + .unwrap() + .row_ids + .clone() + }; + assert_eq!(ids(raw.clone(), "text").await, vec![0]); + assert_eq!(ids(raw.clone(), "folded_text").await, Vec::::new()); + for (kind, field) in [("btree", "id"), ("full-text", "text")] { + let empty = table.copy_with_pinned_snapshot(None); + assert!(empty + .new_global_index_build_builder() + .with_index_column(field) + .with_index_type(kind) + .with_options(table.schema().options().clone()) + .build() + .await + .unwrap() + .is_empty()); + } + for (kind, field) in [ + ("btree", "id"), + ("full-text", "text"), + ("full-text", "folded_text"), + ] { + assert_eq!( + table + .new_global_index_build_builder() + .with_index_column(field) + .with_index_type(kind) + .with_options(table.schema().options().clone()) + .execute() + .await + .unwrap(), + 1 + ); + } + assert_eq!(ids(table.clone(), "text").await, vec![0]); + assert_eq!(ids(table.clone(), "folded_text").await, Vec::::new()); +} + +#[tokio::test] +async fn procedure_flat_parameters_override_legacy_json_without_source_conflicts() { + let (_catalog, table) = search_table(&[ + ("write-only", "true"), + ("fields.id.pk-btree.index.options", r#"{"block-size":128}"#), + ( + "fields.embedding.pk-vector.index.options", + r#"{"dimension":2,"nlist":1}"#, + ), + ]) + .await; + TableCommit::new(table.clone(), "procedure-overrides".into()) + .commit(search_messages(&table, &[[1., 0.], [0., 1.]]).await) + .await + .unwrap(); + let plan = table + .new_global_index_build_builder() + .with_index_column("embedding") + .with_index_type("ivf-flat") + .with_options(HashMap::from([("ivf-flat.nlist".into(), "2".into())])) + .plan() + .await + .unwrap(); + assert_eq!(plan.snapshot_id(), Some(1)); + assert_eq!(plan.shards().len(), 1); + assert_eq!( + table + .new_global_index_build_builder() + .with_index_column("id") + .with_index_type("btree") + .with_options(HashMap::from([( + "btree-index.block-size".into(), + "256".into() + )])) + .execute() + .await + .unwrap(), + 1 + ); +} diff --git a/crates/paimon/src/table/vector_read.rs b/crates/paimon/src/table/vector_read.rs index f9a43d6e1..a4a6fd837 100644 --- a/crates/paimon/src/table/vector_read.rs +++ b/crates/paimon/src/table/vector_read.rs @@ -90,15 +90,11 @@ impl BatchVectorRead { let reader = if targets_primary_key_column(table, column) { let pk_col = core.single_vector_index_column()?; PkVectorSearchParams::validate_inputs(table, filter, &pk_col, queries, limit)?; - VectorReadKind::PrimaryKey(Box::new( - PkVectorRead::new(table, options, filter, &pk_col, queries, limit) - .with_vector_index_type(context.vector_index_type.clone()), - )) + VectorReadKind::PrimaryKey(Box::new(PkVectorRead::new( + table, options, filter, &pk_col, queries, limit, + ))) } else { - VectorReadKind::DataEvolution( - DeVectorRead::new(column, queries, limit, options)? - .with_vector_index_type(context.vector_index_type.clone()), - ) + VectorReadKind::DataEvolution(DeVectorRead::new(column, queries, limit, options)?) }; Ok(Self { context, diff --git a/crates/paimon/src/table/vector_scan.rs b/crates/paimon/src/table/vector_scan.rs index ab7079f86..af66d0271 100644 --- a/crates/paimon/src/table/vector_scan.rs +++ b/crates/paimon/src/table/vector_scan.rs @@ -189,7 +189,6 @@ async fn primary_key_sources( #[derive(Clone)] pub(super) struct PlanContext { pub(super) column: String, - pub(super) vector_index_type: Option, pub(super) filter: Option, pub(super) include_row_ids: Option>, pub(super) prepared_snapshot: Option>, @@ -232,7 +231,6 @@ impl PlanContext { } Ok(Self { column: column.to_string(), - vector_index_type: None, filter: filter.cloned(), include_row_ids: prepared .map(PreparedVectorSearchFilter::include_row_ids) @@ -301,11 +299,6 @@ impl VectorScan { self } - pub(super) fn with_vector_index_type(mut self, index_type: Option) -> Self { - self.context.vector_index_type = index_type; - self - } - pub async fn plan(&self) -> crate::Result { match &self.scan { VectorScanKind::DataEvolution(scan) => { @@ -330,7 +323,6 @@ impl VectorScan { self.context.filter.as_ref(), &self.options, false, - self.context.vector_index_type.as_deref(), ) .await? }; diff --git a/crates/paimon/src/table/vector_search_builder.rs b/crates/paimon/src/table/vector_search_builder.rs index a9c6036e2..cd94a0766 100644 --- a/crates/paimon/src/table/vector_search_builder.rs +++ b/crates/paimon/src/table/vector_search_builder.rs @@ -43,7 +43,6 @@ struct VectorSearchConfig { query_vector: Option>, limit: Option, options: HashMap, - vector_index_types: HashMap, filter: Option, } @@ -75,17 +74,6 @@ impl<'a> VectorSearchBuilder<'a> { self } - /// Store the implementation used when no vector index metadata is available. - /// Does not plan or execute a query. Repeated calls replace the column's hint. - pub fn with_vector_index_type(&mut self, column: &str, index_type: &str) -> &mut Self { - self.config - .write() - .unwrap() - .vector_index_types - .insert(column.into(), index_type.into()); - self - } - pub fn with_query_vector(&mut self, vector: Vec) -> &mut Self { self.config.write().unwrap().query_vector = Some(vector); self @@ -138,10 +126,8 @@ impl<'a> VectorSearchBuilder<'a> { .ok_or_else(|| crate::Error::ConfigInvalid { message: "Vector column must be set via with_vector_column()".to_string(), })?; - VectorScan::new(&self.table, column, config.filter.as_ref(), None, None).map(|scan| { - scan.with_options(&config.options) - .with_vector_index_type(config.vector_index_types.get(column).cloned()) - }) + VectorScan::new(&self.table, column, config.filter.as_ref(), None, None) + .map(|scan| scan.with_options(&config.options)) } /// Create an owned reader; query errors are reported before planning. @@ -152,9 +138,7 @@ impl<'a> VectorSearchBuilder<'a> { fn read_with_config(&self, config: &VectorSearchConfig) -> crate::Result { let (column, query, limit) = self.query(config)?; - let mut context = - PlanContext::new(&self.table, column, config.filter.as_ref(), None, None)?; - context.vector_index_type = config.vector_index_types.get(column).cloned(); + let context = PlanContext::new(&self.table, column, config.filter.as_ref(), None, None)?; Ok(VectorRead { batch: BatchVectorRead::new( &self.table, @@ -210,4 +194,4 @@ impl<'a> VectorSearchBuilder<'a> { mod tests; #[cfg(test)] -mod native_type_tests; +mod index_options_tests; diff --git a/crates/paimon/src/table/vector_search_builder/native_type_tests.rs b/crates/paimon/src/table/vector_search_builder/index_options_tests.rs similarity index 57% rename from crates/paimon/src/table/vector_search_builder/native_type_tests.rs rename to crates/paimon/src/table/vector_search_builder/index_options_tests.rs index 2b368d98d..5608793bc 100644 --- a/crates/paimon/src/table/vector_search_builder/native_type_tests.rs +++ b/crates/paimon/src/table/vector_search_builder/index_options_tests.rs @@ -59,8 +59,10 @@ async fn two_vector_table(pk: bool, declaration: &str) -> Table { .primary_key(["id"]) .option("bucket", "1") .option("deletion-vectors.enabled", "true") - .option(declaration, "embedding") - .option("fields.embedding.pk-vector.index.type", "ivf-flat"); + .option(declaration, "embedding"); + if declaration == "pk-vector.index.columns" { + schema = schema.option("fields.embedding.pk-vector.index.type", "ivf-flat"); + } } else { schema = schema .option("row-tracking.enabled", "true") @@ -69,8 +71,8 @@ async fn two_vector_table(pk: bool, declaration: &str) -> Table { } let table = Table::new( FileIOBuilder::new("memory").build().unwrap(), - Identifier::new("default", "native_type"), - format!("memory:/native-type-{}", uuid::Uuid::new_v4()), + Identifier::new("default", "index_options"), + format!("memory:/index-options-{}", uuid::Uuid::new_v4()), TableSchema::new(0, &schema.build().unwrap()), None, ); @@ -98,7 +100,7 @@ async fn two_vector_table(pk: bool, declaration: &str) -> Table { Field::new("embedding", columns[1].data_type().clone(), true), Field::new("other", columns[2].data_type().clone(), true), ])); - let mut writer = TableWrite::new(&table, "native-type".into()).unwrap(); + let mut writer = TableWrite::new(&table, "index-options".into()).unwrap(); writer .write_arrow_batch(&RecordBatch::try_new(arrow, columns).unwrap()) .await @@ -114,7 +116,7 @@ async fn two_vector_table(pk: bool, declaration: &str) -> Table { file.file_source = Some(1); } } - TableCommit::new(table.clone(), "native-type".into()) + TableCommit::new(table.clone(), "index-options".into()) .commit(messages) .await .unwrap(); @@ -130,7 +132,7 @@ fn raw_type(plan: &VectorScanPlan) -> Option<&str> { struct Worker { table: Table, - expected_type: String, + options: HashMap, } #[async_trait::async_trait] @@ -142,9 +144,11 @@ impl SearchTaskExecutor for Worker { let task = SearchTask::deserialize_with_context(&frame, &context)?; let wire: serde_json::Value = serde_json::from_slice(&task.serialize()?).unwrap(); for query in wire["payload"]["queries"]["queries"].as_array().unwrap() { - assert_eq!(query["index_type"], self.expected_type); - assert!(query["options"].get("index_type").is_none()); - assert!(query["options"].get("metric").is_none()); + assert!(query.get("index_type").is_none()); + assert_eq!( + query["options"], + serde_json::to_value(&self.options).unwrap() + ); } results.push(SearchTaskResult::deserialize( &task.execute(&self.table).await?.serialize()?, @@ -155,30 +159,50 @@ impl SearchTaskExecutor for Worker { } } +fn declaration(kind: &str, column: &str) -> HashMap { + HashMap::from([(format!("global-index.{kind}.index-columns"), column.into())]) +} + #[tokio::test] -async fn native_type_is_captured_by_single_batch_plan_reader_and_worker() { +async fn configuration_is_captured_by_single_batch_plan_reader_and_worker() { let table = raw_table().await; + let mut options = declaration("ivf-flat", "embedding"); + options.insert( + "fields.embedding.index-options".into(), + r#"{"dimension":2,"nlist":1}"#.into(), + ); + options.insert( + "opaque.future.parameter".into(), + r#"{"value":[1,2]}"#.into(), + ); let mut builder = table.new_vector_search_builder(); - // Store before configuring a query; unrelated columns and superseded hints - // cannot influence the selected vector column. builder - .with_vector_index_type("other", "invalid") - .with_vector_index_type("embedding", "invalid") - .with_vector_index_type("embedding", "ivf-flat") + .with_options(options.clone()) .with_vector_column("embedding") .with_query_vector(vec![1.0, 0.0]) .with_limit(3); let scan = builder.new_scan().unwrap(); let reader = builder.new_read().unwrap(); - builder.with_vector_index_type("embedding", "diskann"); + let changed = HashMap::from([ + ("global-index.ivf-flat.index-columns".into(), "other".into()), + ( + "global-index.diskann.index-columns".into(), + "embedding".into(), + ), + ("ivf-flat.metric".into(), "l2".into()), + ("diskann.metric".into(), "cosine".into()), + ]); + builder.with_options(changed.clone()); let plan = scan.plan().await.unwrap(); assert_eq!(raw_type(&plan), Some("ivf-flat")); let restored: VectorScanPlan = serde_json::from_slice(&serde_json::to_vec(&plan).unwrap()).unwrap(); let local = reader.read(restored.clone()).await.unwrap(); assert_eq!(local.row_ids().unwrap().scores, vec![1.0, 1.0, 0.0]); + // Captured raw split metadata wins over a fresh reader's new declarations. let mut fresh = table.new_vector_search_builder(); fresh + .with_options(changed) .with_vector_column("embedding") .with_query_vector(vec![1.0, 0.0]) .with_limit(3); @@ -188,54 +212,30 @@ async fn native_type_is_captured_by_single_batch_plan_reader_and_worker() { .read(restored.clone()) .await .unwrap(); - assert_eq!( - result.row_ids().unwrap().scores, - local.row_ids().unwrap().scores - ); - // A transported raw source remains authoritative even if a fresh reader's - // fallback hint names a different parameter namespace. - fresh - .with_vector_index_type("embedding", "diskann") - .with_options(HashMap::from([ - ("ivf-flat.metric".into(), "l2".into()), - ("diskann.metric".into(), "cosine".into()), - ])); - let result = fresh - .new_read() - .unwrap() - .read(restored.clone()) - .await - .unwrap(); assert_eq!(result.row_ids().unwrap().scores, vec![1.0, 0.5, 1.0 / 3.0]); let worker = Worker { table: table.clone(), - expected_type: "ivf-flat".into(), + options: options.clone(), }; let remote = reader.read_with_executor(restored, &worker).await.unwrap(); - assert_eq!( - remote.row_ids().unwrap().row_ids, - local.row_ids().unwrap().row_ids - ); assert_eq!( remote.row_ids().unwrap().scores, local.row_ids().unwrap().scores ); - let mut batch = table.new_batch_vector_search_builder(); batch - .with_vector_index_type("embedding", "ivf-flat") + .with_options(options) .with_vector_column("embedding") .with_query_vectors(vec![vec![1.0, 0.0]; 2]) .with_limit(3); let scan = batch.new_scan().unwrap(); let reader = batch.new_read().unwrap(); - batch.with_vector_index_type("embedding", "diskann"); - let results = reader + batch.with_options(declaration("diskann", "embedding")); + for result in reader .read_with_executor(scan.plan().await.unwrap(), &worker) .await - .unwrap(); - assert_eq!(results.len(), 2); - for result in results { + .unwrap() + { assert_eq!( result.row_ids().unwrap().scores, local.row_ids().unwrap().scores @@ -244,7 +244,7 @@ async fn native_type_is_captured_by_single_batch_plan_reader_and_worker() { } #[tokio::test] -async fn native_type_raw_fallback_preserves_declarations_and_metric_options() { +async fn raw_fallback_preserves_unmodified_declarations_and_metric_options() { let raw = raw_table().await; for declaration in [ "global-index.ivf-flat.index-columns", @@ -279,12 +279,10 @@ async fn native_type_raw_fallback_preserves_declarations_and_metric_options() { builder.execute().await.unwrap().row_ids().unwrap().scores, vec![1.0, 1.0, 0.0] ); - builder - .with_vector_index_type("embedding", "diskann") - .with_options(HashMap::from([ - ("ivf-flat.metric".into(), "l2".into()), - ("diskann.metric".into(), "cosine".into()), - ])); + builder.with_options(HashMap::from([ + ("ivf-flat.metric".into(), "cosine".into()), + ("diskann.metric".into(), "l2".into()), + ])); let result = builder.execute().await.unwrap(); assert_eq!(result.row_ids().unwrap().row_ids, vec![0, 2, 1]); assert!((result.row_ids().unwrap().scores[1] - 1.0_f32 / 2.0_f32.sqrt()).abs() < 1e-6); @@ -319,39 +317,23 @@ async fn removed_type_options_do_not_change_raw_defaults() { } #[tokio::test] -async fn metadata_precedes_native_type_even_with_partial_raw_coverage() { - let table = de_vector_table_with_metric(Some("l2")) - .await - .copy_with_options(HashMap::from([ +async fn metadata_precedes_configuration_even_with_partial_raw_coverage() { + let original = de_vector_table_with_metric(Some("l2")).await; + for kind in ["diskann", "invalid-unused-type"] { + let table = original.copy_with_options(HashMap::from([ ("vector-index.search-mode".into(), "full".into()), - ("diskann.metric".into(), "cosine".into()), + ( + format!("global-index.{kind}.index-columns"), + "embedding".into(), + ), + (format!("{kind}.metric"), "invalid-unused-metric".into()), ])); - let mut builder = table.new_vector_search_builder(); - builder - .with_vector_column("embedding") - .with_query_vector(vec![1.0, 0.0]) - .with_limit(3) - .with_options(HashMap::from([("ivf.nprobe".into(), "2".into())])); - let expected = builder.execute().await.unwrap(); - assert_eq!( - expected.row_ids().unwrap().scores, - vec![1.0, 0.5, 1.0 / 3.0] - ); - for hint in ["diskann", "invalid-unused-type"] { - builder.with_vector_index_type("embedding", hint); + let mut builder = table.new_vector_search_builder(); + builder + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(3); let plan = builder.new_scan().unwrap().plan().await.unwrap(); - let result = builder - .new_read() - .unwrap() - .read(plan.clone()) - .await - .unwrap(); - assert_eq!( - result.row_ids().unwrap().scores, - expected.row_ids().unwrap().scores - ); - // Keep one authoritative indexed source and add raw coverage: the raw - // range must use persisted semantics too, without interpreting the hint. let mut splits = plan.splits().to_vec(); splits.push(SearchScanSplit::Raw { row_ranges: vec![crate::table::RowRange::new(0, 2)], @@ -364,106 +346,89 @@ async fn metadata_precedes_native_type_even_with_partial_raw_coverage() { .read(VectorScanPlan::new(plan.snapshot().cloned(), splits).unwrap()) .await .unwrap(); - assert_eq!( - result.row_ids().unwrap().row_ids, - expected.row_ids().unwrap().row_ids - ); - assert_eq!( - result.row_ids().unwrap().scores, - expected.row_ids().unwrap().scores - ); + assert_eq!(result.row_ids().unwrap().row_ids, vec![0, 2, 1]); + assert_eq!(result.row_ids().unwrap().scores, vec![1.0, 0.5, 1.0 / 3.0]); } } #[tokio::test] -async fn native_type_pk_raw_fallback_uses_its_dimension_metric_and_hybrid_route() { +async fn pk_raw_fallback_keeps_table_metric_in_single_batch_and_hybrid() { use futures::TryStreamExt; for declaration in [ "global-index.ivf-flat.index-columns", "global-index.ivf-flat.index-column", "pk-vector.index.columns", ] { - let table = two_vector_table(true, declaration).await; - for query in [ - vec![], - vec![1.0], - vec![f32::NAN, 0.0], - vec![0.0, f32::INFINITY], + let original = two_vector_table(true, declaration).await; + for table_options in [ + HashMap::new(), + HashMap::from([( + "fields.embedding.index-options".into(), + r#"{"dimension":2,"metric":"cosine"}"#.into(), + )]), ] { - let mut invalid = table.new_vector_search_builder(); - invalid - .with_vector_index_type("embedding", "diskann") - .with_vector_column("embedding") - .with_query_vector(query) - .with_limit(1); - assert!( - invalid.new_read().is_err(), - "fixed vector errors must precede planning" - ); - } - let options = HashMap::from([("diskann.metric".into(), "cosine".into())]); - let mut builder = table.new_vector_search_builder(); - builder - .with_vector_index_type("embedding", "diskann") - .with_vector_column("embedding") - .with_query_vector(vec![1.0, 0.0]) - .with_limit(1); - let plan = builder.new_scan().unwrap().plan().await.unwrap(); - assert!(!plan.splits().is_empty()); - assert!(plan.splits().iter().all(|source| matches!(source, SearchScanSplit::PrimaryKey { payloads, .. } if payloads.is_empty()))); - let result = builder - .new_read() - .unwrap() - .read(plan.clone()) - .await - .unwrap(); - assert_eq!(result.positions().unwrap().len(), 1); - assert_eq!(result.positions().unwrap()[0].score, 3.0); - builder.with_options(options.clone()); - let result = builder.new_read().unwrap().read(plan).await.unwrap(); - assert_eq!(result.positions().unwrap().len(), 1); - assert_eq!(result.positions().unwrap()[0].score, 1.0); - let mut batch = table.new_batch_vector_search_builder(); - batch - .with_vector_index_type("embedding", "diskann") - .with_vector_column("embedding") - .with_query_vectors(vec![vec![1.0, 0.0]; 2]) - .with_limit(1) - .with_options(options.clone()); - for result in batch.execute().await.unwrap() { - assert_eq!(result.positions().unwrap()[0].score, 1.0); + let cosine = !table_options.is_empty(); + let table = original.copy_with_options(table_options); + let expected = if cosine { 1.0 } else { 3.0 }; + // Java PK metrics come from the table result, including raw-only plans. + for query_options in [ + HashMap::from([("ivf-flat.metric".into(), "l2".into())]), + HashMap::from([( + "fields.embedding.index-options".into(), + r#"{"metric":"l2"}"#.into(), + )]), + ] { + let mut builder = table.new_vector_search_builder(); + builder + .with_options(query_options.clone()) + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(1); + assert_eq!( + builder.execute().await.unwrap().positions().unwrap()[0].score, + expected + ); + let mut batch = table.new_batch_vector_search_builder(); + batch + .with_options(query_options.clone()) + .with_vector_column("embedding") + .with_query_vectors(vec![vec![1.0, 0.0]; 2]) + .with_limit(1); + for result in batch.execute().await.unwrap() { + assert_eq!(result.positions().unwrap()[0].score, expected); + } + let mut hybrid = table.new_hybrid_search_builder(); + hybrid + .add_vector_route("embedding", vec![1.0, 0.0], 1, 1.0, query_options) + .unwrap() + .with_limit(1); + let batches = hybrid + .execute_read() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 1 + ); + assert_eq!( + batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 1 + ); + } } - let mut hybrid = table.new_hybrid_search_builder(); - hybrid.with_vector_index_type("embedding", "diskann"); - hybrid - .add_vector_route("embedding", vec![1.0, 0.0], 1, 1.0, options) - .unwrap() - .with_limit(1); - let batches = hybrid - .execute_read() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - assert_eq!( - batches.iter().map(|batch| batch.num_rows()).sum::(), - 1 - ); - assert_eq!( - batches[0] - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .value(0), - 1 - ); } } #[test] -fn native_type_does_not_defer_array_query_emptiness_or_finiteness() { +fn raw_configuration_keeps_eager_query_validation() { let table = crate::table::vector_search_test_utils::pk_vector_table(&[ ("pk-vector.index.columns", "embedding"), ("fields.embedding.pk-vector.index.type", "ivf-flat"), @@ -471,7 +436,7 @@ fn native_type_does_not_defer_array_query_emptiness_or_finiteness() { for query in [vec![], vec![f32::NAN, 0.0], vec![0.0, f32::INFINITY]] { let mut builder = table.new_vector_search_builder(); builder - .with_vector_index_type("embedding", "diskann") + .with_options(declaration("diskann", "embedding")) .with_vector_column("embedding") .with_query_vector(query) .with_limit(1); @@ -480,30 +445,38 @@ fn native_type_does_not_defer_array_query_emptiness_or_finiteness() { } #[tokio::test] -async fn native_type_hybrid_is_per_column_and_captured_before_builder_changes() { - let table = two_vector_table(false, "unused").await; - let options = HashMap::from([ - ("ivf-flat.metric".into(), "l2".into()), - ("diskann.metric".into(), "cosine".into()), - ]); +async fn hybrid_configuration_is_per_column_and_captured_with_each_route() { + let table = two_vector_table(false, "unused") + .await + .copy_with_options(HashMap::from([ + ( + "global-index.ivf-flat.index-columns".into(), + "embedding".into(), + ), + ("global-index.diskann.index-column".into(), "other".into()), + ( + "fields.embedding.index-options".into(), + r#"{"dimension":2,"metric":"l2"}"#.into(), + ), + ( + "fields.other.index-options".into(), + r#"{"dimension":2,"metric":"cosine"}"#.into(), + ), + ])); let mut hybrid = table.new_hybrid_search_builder(); - hybrid - .with_vector_index_type("embedding", "ivf-flat") - .with_vector_index_type("other", "diskann") - .with_vector_index_type("embedding", "diskann"); - hybrid - .add_vector_route("other", vec![1.0, 0.0], 1, 1.0, options.clone()) - .unwrap(); - hybrid - .add_vector_route("embedding", vec![1.0, 0.0], 1, 1.0, options) - .unwrap() - .with_limit(2); - // Give embedding a different implementation, independent of route order. - hybrid.with_vector_index_type("embedding", "ivf-flat"); + for field in ["other", "embedding"] { + hybrid + .add_vector_route( + field, + vec![1.0, 0.0], + 1, + 1.0, + HashMap::from([("opaque.route.parameter".into(), format!("value-{field}"))]), + ) + .unwrap(); + } + hybrid.with_limit(2); let routes = hybrid.route_builders().await.unwrap(); - hybrid - .with_vector_index_type("embedding", "diskann") - .with_vector_index_type("other", "ivf-flat"); for route in routes { let builder = route.vector_search_builder().unwrap(); let plan = builder.new_scan().unwrap().plan().await.unwrap(); @@ -513,11 +486,13 @@ async fn native_type_hybrid_is_per_column_and_captured_before_builder_changes() ("ivf-flat", 1) }; assert_eq!(raw_type(&plan), Some(kind)); - let result = route.execute().await.unwrap(); - assert_eq!(result.row_ids().unwrap().row_ids, vec![row_id]); + assert_eq!( + route.execute().await.unwrap().row_ids().unwrap().row_ids, + vec![row_id] + ); let worker = Worker { table: table.clone(), - expected_type: kind.into(), + options: route.route().options().clone(), }; let remote = builder .new_read() @@ -528,3 +503,72 @@ async fn native_type_hybrid_is_per_column_and_captured_before_builder_changes() assert_eq!(remote.row_ids().unwrap().row_ids, vec![row_id]); } } + +#[tokio::test] +async fn raw_build_map_preserves_json_precedence_and_serialized_shards() { + let table = raw_table().await; + let mut options = table.schema().options().clone(); + options.extend(declaration("ivf-flat", "embedding")); + options.extend(HashMap::from([ + ("ivf-flat.metric".into(), "l2".into()), + ( + "fields.embedding.index-options".into(), + r#"{"dimension":2,"nlist":1,"metric":"cosine"}"#.into(), + ), + ("opaque.future.option".into(), "value".into()), + ])); + let mut builder = table.new_global_index_build_builder(); + builder + .with_index_column("embedding") + .with_index_type("ivf-flat") + .with_options(options); + let plan = builder.plan().await.unwrap(); + assert_eq!(plan.snapshot_id(), Some(1)); + assert!(!plan.shards().is_empty()); + let worker = table.copy_with_options(HashMap::from([( + "fields.embedding.index-options".into(), + "invalid worker JSON".into(), + )])); + for shard in plan.shards() { + let shard: crate::table::GlobalIndexBuildShard = + serde_json::from_slice(&serde_json::to_vec(&shard).unwrap()).unwrap(); + let messages = worker + .new_global_index_build_builder() + .build_shard(&shard) + .await + .unwrap(); + assert!(!messages.is_empty()); + for file in messages.iter().flat_map(|message| &message.new_index_files) { + let meta: HashMap = serde_json::from_slice( + file.global_index_meta + .as_ref() + .unwrap() + .index_meta + .as_ref() + .unwrap(), + ) + .unwrap(); + assert_eq!(meta["dimension"], "2"); + assert_eq!(meta["nlist"], "1"); + assert_eq!(meta["metric"], "cosine"); + } + } + // Qualified typos still fail at the Rust parameter boundary. + builder.with_options(HashMap::from([("ivf-flat.nlsit".into(), "1".into())])); + assert!(builder + .plan() + .await + .unwrap_err() + .to_string() + .contains("ivf-flat.nlsit")); + let empty = table.copy_with_pinned_snapshot(None); + let plan = empty + .new_global_index_build_builder() + .with_index_column("embedding") + .with_index_type("ivf-flat") + .plan() + .await + .unwrap(); + assert_eq!(plan.snapshot_id(), None); + assert!(plan.shards().is_empty()); +} diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs b/crates/paimon/src/table/vindex_index_build_builder.rs index 3f243edc5..501a575cf 100644 --- a/crates/paimon/src/table/vindex_index_build_builder.rs +++ b/crates/paimon/src/table/vindex_index_build_builder.rs @@ -64,6 +64,37 @@ impl<'a> VindexIndexBuildBuilder<'a> { self } + fn configured_options( + &self, + column: &str, + field: &crate::spec::DataField, + ) -> Result<(HashMap, VindexVectorIndexOptions)> { + let mut effective = super::global_index_options::build_table_options( + self.table.schema().options(), + &self.options, + column, + ); + let defaults = super::global_index_options::declared_options( + &effective, + column, + "vector", + &self.index_type, + )?; + let user = + super::global_index_options::build_options(&self.options, column, &self.index_type)?; + let native_user = crate::vindex::build_parameters(&user, &self.index_type, column); + let native = VindexVectorIndexOptions::new_with_defaults( + &effective, + &defaults, + &native_user, + &self.index_type, + field, + )?; + effective.extend(defaults); + effective.extend(user); + Ok((effective, native)) + } + pub(super) async fn plan(&self) -> Result { // Building the index scans the table's rows. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; @@ -85,29 +116,13 @@ impl<'a> VindexIndexBuildBuilder<'a> { source: None, })?; - let defaults = super::global_index_options::declared_options( - self.table.schema().options(), - index_column, - "vector", - &self.index_type, - )?; - let mut merged_options = self.table.schema().options().clone(); - merged_options.extend(defaults.clone()); - merged_options.extend(self.options.clone()); - let core_options = CoreOptions::new(&merged_options); super::global_index_build_common::validate_build_table(self.table)?; - let rows_per_shard = core_options.global_index_row_count_per_shard()?; - core_options.global_index_build_parallelism()?; - let index_field = find_index_field(self.table, index_column)?; validate_vector_field(index_field)?; - let options = VindexVectorIndexOptions::new_with_defaults( - self.table.schema().options(), - &defaults, - &self.options, - &self.index_type, - index_field, - )?; + let (merged_options, options) = self.configured_options(index_column, index_field)?; + let core_options = CoreOptions::new(&merged_options); + let rows_per_shard = core_options.global_index_row_count_per_shard()?; + core_options.global_index_build_parallelism()?; checked_i32( options.dimension() as u64, "vindex dimension is too large for Rust builder", @@ -184,19 +199,7 @@ impl<'a> VindexIndexBuildBuilder<'a> { })?; let index_field = find_index_field(self.table, index_column)?; validate_vector_field(index_field)?; - let defaults = super::global_index_options::declared_options( - self.table.schema().options(), - index_column, - "vector", - &self.index_type, - )?; - let options = VindexVectorIndexOptions::new_with_defaults( - self.table.schema().options(), - &defaults, - &self.options, - &self.index_type, - index_field, - )?; + let (_, options) = self.configured_options(index_column, index_field)?; let dimension = checked_i32( options.dimension() as u64, "vindex dimension is too large for Rust builder", @@ -231,16 +234,12 @@ impl<'a> VindexIndexBuildBuilder<'a> { Vec, )> { let plan = self.plan().await?; - let mut options = self.table.schema().options().clone(); - if let Some(column) = &self.index_column { - options.extend(super::global_index_options::declared_options( - self.table.schema().options(), - column, - "vector", - &self.index_type, - )?); - } - options.extend(self.options.clone()); + let column = self + .index_column + .as_deref() + .expect("plan validates index column"); + let (options, _) = + self.configured_options(column, find_index_field(self.table, column)?)?; let parallelism = CoreOptions::new(&options).global_index_build_parallelism()?; let prepared = super::global_index_build_common::preparation::prepare_shards( self.table, diff --git a/crates/paimon/src/vector_search.rs b/crates/paimon/src/vector_search.rs index 7432499d1..dc5327857 100644 --- a/crates/paimon/src/vector_search.rs +++ b/crates/paimon/src/vector_search.rs @@ -27,8 +27,6 @@ pub struct VectorSearch { pub limit: usize, pub field_name: String, pub options: HashMap, - /// Implementation hint for raw fallback; persisted index metadata takes precedence. - pub(crate) index_type: Option, pub include_row_ids: Option, pub(crate) shared_include_row_ids: Option>, } @@ -58,7 +56,6 @@ impl VectorSearch { limit, field_name, options: HashMap::new(), - index_type: None, include_row_ids: None, shared_include_row_ids: None, }) diff --git a/crates/paimon/src/vindex/mod.rs b/crates/paimon/src/vindex/mod.rs index b81e45339..592b7feeb 100644 --- a/crates/paimon/src/vindex/mod.rs +++ b/crates/paimon/src/vindex/mod.rs @@ -266,6 +266,30 @@ fn compile_training_config( }) } +/// The generic build API accepts a complete table option map. Native parameters +/// remain scoped to this implementation/column; qualified typos still reach the +/// strict validator, while unrelated table properties stay in the captured plan. +pub(crate) fn build_parameters( + options: &HashMap, + kind: &str, + column: &str, +) -> HashMap { + let prefix = format!("{kind}."); + let field = format!("fields.{column}."); + options + .iter() + .filter(|(key, _)| { + key.starts_with(&prefix) + || native_option_key(key).is_some() + || matches!(key.as_str(), "index.type" | "vindex.build.granule.enabled") + || key.strip_prefix(&field).is_some_and(|suffix| { + native_option_key(suffix).is_some() || suffix == "train.sample-ratio" + }) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect() +} + fn validate_user_option_keys( options: &HashMap, index_type: &str, diff --git a/crates/paimon/tests/pk_compact/indexes.rs b/crates/paimon/tests/pk_compact/indexes.rs index 9eb3e3413..de167879a 100644 --- a/crates/paimon/tests/pk_compact/indexes.rs +++ b/crates/paimon/tests/pk_compact/indexes.rs @@ -242,7 +242,6 @@ async fn empty_vector_level_is_persisted_reused_and_searchable() { commit.commit_compact(first).await.unwrap(); let result = table .new_vector_search_builder() - .with_vector_index_type("embedding", "invalid-unused-type") .with_vector_column("embedding") .with_query_vector(vec![0., 0.]) .with_limit(1) diff --git a/crates/paimon/tests/pk_vector_baseline_test.rs b/crates/paimon/tests/pk_vector_baseline_test.rs index 8b0bd9422..b2b5c1fbf 100644 --- a/crates/paimon/tests/pk_vector_baseline_test.rs +++ b/crates/paimon/tests/pk_vector_baseline_test.rs @@ -596,7 +596,6 @@ async fn pk_vector_reads_captured_payload_after_algorithm_configuration_changes( let mut builder = changed.new_vector_search_builder(); builder .with_vector_column(VECTOR_COLUMN) - .with_vector_index_type(VECTOR_COLUMN, "invalid-unused-hint") .with_query_vector(query.to_vec()) .with_limit(3); let plan = builder.new_scan().unwrap().plan().await.unwrap(); @@ -641,7 +640,6 @@ async fn pk_vector_end_to_end_returns_expected_row_ids_and_scores() { // Search-only PK results preserve local positions and metric scores. let scored = table .new_vector_search_builder() - .with_vector_index_type(VECTOR_COLUMN, "invalid-unused-type") .with_vector_column(VECTOR_COLUMN) .with_query_vector(query.to_vec()) .with_limit(3) diff --git a/docs/src/compaction.md b/docs/src/compaction.md index c19d55000..40ae99c1d 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -154,19 +154,21 @@ overrides table implementation defaults. Vector metrics default to `inner_product`, and accepted metric aliases resolve consistently for builds, indexed reads, raw scoring and refinement. -Query builders expose `with_vector_index_type(column, index_type)` in Rust and -Python for single, batch and hybrid vector searches. This stores a per-column -implementation hint without planning a query. Existing index metadata takes -precedence; the hint selects the implementation for raw fallback when no vector -index metadata is available. Captured scans, readers, hybrid routes and worker -transport retain it. Ordinary parameters still use `with_options`, and readers -leave an unspecified metric unset so global index metadata can supply it. PK -readers retain Java's table metric validation. PK scans collect supported vector -payloads for the column and discard stale source-file sets before readers select -the algorithm. All valid payloads, including completed empty segments, must use -one algorithm. With none left, readers use the configured declaration or Native -hint. Type, parameters and metric come from one captured resolver result, reused -for raw fallback, refinement and distributed index tasks. +Native reads, builds and searches use the shared Rust resolver directly. Python +forwards the original schema options and explicit query/route options, including +field JSON and opaque parameters. Single, batch and hybrid builders use ordinary +options for declarations and parameters. Full-text analyzers remain scoped to +their schema columns, including JSON-encoded flat collection values. + +Existing valid index metadata determines the vector implementation. PK scans +collect supported payloads for the column and discard stale source-file sets +before readers select the algorithm. All valid payloads, including completed +empty segments, must use one algorithm. With none left, readers resolve the +configured declaration for raw fallback. Type, parameters and metric come from +one captured result, reused for raw fallback, refinement and distributed tasks. +Captured raw splits retain their typed `index_type` metadata. Readers leave an +unspecified query metric unset so global index metadata can supply it; PK readers +retain Java's table metric validation. The option aliases `index_type`, `index-type`, `vector.index-type` and `fields..index-type` no longer select vector implementations. Use the From da426cb01f4c44dcb14689c03dfc29b4c1d1ebe1 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 16:36:30 +0800 Subject: [PATCH 14/14] fix(index): preserve field metrics and native build option sources --- bindings/python/tests/test_vector_search.py | 54 +++++++- crates/paimon/src/table/de_vector_read.rs | 14 ++- .../paimon/src/table/global_index_options.rs | 70 +++++++++-- .../index_options_tests.rs | 92 ++++++++++++++ .../src/table/vindex_index_build_builder.rs | 8 +- .../table/vindex_index_build_builder/tests.rs | 118 ++++++++++++++++++ docs/src/compaction.md | 4 + 7 files changed, 342 insertions(+), 18 deletions(-) diff --git a/bindings/python/tests/test_vector_search.py b/bindings/python/tests/test_vector_search.py index 41d680b67..1ae98462d 100644 --- a/bindings/python/tests/test_vector_search.py +++ b/bindings/python/tests/test_vector_search.py @@ -52,6 +52,30 @@ def _single(table): return table.new_vector_search_builder().with_vector_column('embedding').with_query_vector([1, 0]).with_limit(10) +@pytest.mark.parametrize('mode', ['full', 'detail']) +@pytest.mark.parametrize('batch', [False, True]) +@pytest.mark.parametrize('source', ['table', 'query', 'both']) +def test_untyped_raw_field_json_metric_and_query_precedence(table, mode, batch, source): + import pickle + key = 'fields.embedding.index-options' + table_options = {'vector-index.search-mode': mode} + if source in ('table', 'both'): + table_options[key] = '{"metric":"cosine"}' + configured = table.copy(table_options) + builder = (configured.new_batch_vector_search_builder().with_query_vectors([[1, 0], [1, 0]]) + if batch else configured.new_vector_search_builder().with_query_vector([1, 0])) + query_options = {'opaque.query.option': '{"values":[1,2]}'} + if source in ('query', 'both'): + query_options[key] = '{"metric":"l2"}' if source == 'both' else '{"metric":"cosine"}' + builder.with_options(query_options).with_vector_column('embedding').with_limit(3) + plan = pickle.loads(pickle.dumps(builder.new_vector_search_scan().scan())) + reader = builder.new_batch_vector_search_read() if batch else builder.new_vector_search_read() + results = reader.read_batch_plan(plan) if batch else [reader.read_plan(plan)] + expected = [1 / 3, 1, 1] if source == 'both' else [0, 1, 1] + for result in results: + assert sorted(result.row_ids().values()) == pytest.approx(expected) + + def test_vector_build_accepts_raw_schema_options_and_field_json(table): options = { 'bucket': '-1', 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', @@ -70,8 +94,29 @@ def test_vector_build_accepts_raw_schema_options_and_field_json(table): assert sorted(actual.values()) == pytest.approx([0, 1, 1]) +@pytest.mark.parametrize('kind', ['ivf-flat', 'diskann']) +def test_vector_build_echoed_schema_retains_source_semantics(table, kind): + import pickle + configured = table.copy({ + f'{kind}.dimension': '2', f'{kind}.metric': 'cosine', + 'nlist': 'ignored table property', 'vindex.build.granule.enabled': 'true', + }) + options = configured.schema().options() + builder = (configured.new_global_index_build_builder().with_index_column('embedding') + .with_index_type(kind)) + assert builder.plan().shards() + builder.with_options(options) + shards = pickle.loads(pickle.dumps(builder.plan())).shards() + worker = configured.copy({f'{kind}.dimension': '999'}) + for shard in shards: + assert worker.new_global_index_build_builder().build_shard(shard) + assert builder.execute() > 0 + assert sorted(_single(configured).execute().row_ids().values()) == pytest.approx([0, 1, 1]) + + @pytest.mark.parametrize('shared_json', [False, True, 'aliases']) -def test_full_text_json_encoded_lists_keep_raw_and_indexed_semantics(tmp_path, shared_json): +@pytest.mark.parametrize('declared', [False, True]) +def test_full_text_json_encoded_lists_keep_raw_and_indexed_semantics(tmp_path, shared_json, declared): import json ctx = SQLContext() ctx.register_catalog('paimon', {'warehouse': str(tmp_path)}) @@ -89,7 +134,7 @@ def test_full_text_json_encoded_lists_keep_raw_and_indexed_semantics(tmp_path, s finally: writer.close() commit.close() - options = {'global-index.full-text.index-columns': 'body'} + options = {'global-index.full-text.index-columns': 'body'} if declared else {} if shared_json == 'aliases': options.update({'full-text.stop-words': json.dumps(['paimon']), 'full-text.columns': json.dumps(['body'])}) elif shared_json: @@ -98,16 +143,17 @@ def test_full_text_json_encoded_lists_keep_raw_and_indexed_semantics(tmp_path, s options.update({'full-text.stop-words': json.dumps(['paimon']), 'full-text.text-fields': json.dumps(['body'])}) table = table.copy(options) + search_table = table.copy({'global-index.full-text.index-columns': 'body'}) def hits(query): - return (table.new_full_text_search_builder().with_query('body', json.dumps({'match': {'query': query}})) + return (search_table.new_full_text_search_builder().with_query('body', json.dumps({'match': {'query': query}})) .with_limit(3).execute().row_ids()) assert len(hits('paimon')) == 0 before = hits('data') assert len(before) == 1 builder = (table.new_global_index_build_builder().with_index_column('body') - .with_index_type('full-text').with_options(options)) + .with_index_type('full-text').with_options(table.schema().options())) assert builder.execute() == 1 assert len(hits('paimon')) == 0 assert hits('data') == before diff --git a/crates/paimon/src/table/de_vector_read.rs b/crates/paimon/src/table/de_vector_read.rs index ada9b8a3d..6848ab013 100644 --- a/crates/paimon/src/table/de_vector_read.rs +++ b/crates/paimon/src/table/de_vector_read.rs @@ -1770,7 +1770,7 @@ async fn resolve_raw_vector_metric_resolved( if index_type.is_some() { return RawVectorMetric::parse(&configuration.resolved.metric); } - let table_metric = configured_raw_vector_metric_override(table_options, field_name)?; + let table_metric = resolved_vector_metric_override(table_options, field_name, None)?; if let Some(metric) = table_metric { return Ok(metric); } @@ -1788,6 +1788,18 @@ fn resolved_vector_metric_override( if let Some(kind) = index_type { direct_raw_vector_metric_override(options, field_name, Some(kind)) } else { + if options.contains_key(&format!("fields.{field_name}.index-options")) { + // Like Java, use the logical family only to parse explicit field + // parameters. It does not select a native backend or default metric. + let resolved = CoreOptions::new(options).vector_index_options( + Some("vector"), + field_name, + false, + )?; + if let Some(metric) = resolved.options.get("vector.metric") { + return RawVectorMetric::parse(metric).map(Some); + } + } configured_raw_vector_metric_override(options, field_name) } } diff --git a/crates/paimon/src/table/global_index_options.rs b/crates/paimon/src/table/global_index_options.rs index a6e9f72e2..daa9f4aea 100644 --- a/crates/paimon/src/table/global_index_options.rs +++ b/crates/paimon/src/table/global_index_options.rs @@ -21,6 +21,7 @@ use crate::spec::CoreOptions; use crate::table::global_index_types::normalize_global_index_type; use crate::table::{GlobalIndexBuildBuilder, Table}; use crate::Result; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; pub(super) struct Definition { @@ -176,7 +177,31 @@ pub(super) fn field_index_options( .collect()) } -/// A procedure JSON object replaces that field's schema object before parsing. +/// Native frontends can echo the entire effective schema in build options. +/// Unchanged schema entries keep table precedence and validation semantics; +/// a standalone procedure map retains the native parameter aliases. +pub(super) fn build_procedure_options<'a>( + table: &HashMap, + user: &'a HashMap, +) -> Cow<'a, HashMap> { + // Pinned worker tables add read selectors that are absent from the original + // frontend map. These selectors do not identify a build option source. + let mut schema_keys = table + .keys() + .filter(|key| !key.starts_with("scan.") && !key.starts_with("incremental-")) + .peekable(); + if schema_keys.peek().is_none() || !schema_keys.all(|key| user.contains_key(key)) { + return Cow::Borrowed(user); + } + Cow::Owned( + user.iter() + .filter(|(key, value)| table.get(*key) != Some(*value)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + ) +} + +/// A procedure's declarations and JSON replace the corresponding schema source. /// Flat procedure parameters keep their own precedence layer and cannot create /// false conflicts with legacy JSON in the table source. pub(super) fn build_table_options( @@ -185,18 +210,30 @@ pub(super) fn build_table_options( column: &str, ) -> HashMap { let mut effective = table.clone(); - let prefix = format!("fields.{column}."); effective.extend( user.iter() - .filter(|(key, _)| { - key.starts_with(&prefix) - && (key.ends_with(".index-options") || key.ends_with(".index.options")) - }) + .filter(|(key, _)| build_definition_option(key, column)) .map(|(key, value)| (key.clone(), value.clone())), ); effective } +fn build_definition_option(key: &str, column: &str) -> bool { + key.strip_prefix(&format!("fields.{column}.")) + .is_some_and(|suffix| { + suffix == "index-options" + || suffix.ends_with(".index-options") + || suffix.ends_with(".index.options") + || matches!( + suffix, + "pk-vector.index.type" | "global-index.vector.index-type" + ) + }) + || (key.starts_with("global-index.") + && (key.ends_with(".index-columns") || key.ends_with(".index-column"))) + || (key.starts_with("pk-") && key.ends_with(".index.columns")) +} + /// Parse a procedure's original option map without treating table properties as /// native vector parameters. Keep the original map in captured build plans. pub(super) fn build_options( @@ -206,11 +243,22 @@ pub(super) fn build_options( ) -> Result> { let core = CoreOptions::new(options); if crate::vindex::is_vindex_index_type(kind) { - let resolved = core.vector_index_options( - Some(kind), - column, - declares_column(options, column, "vector"), - )?; + // Declarations establish defaults in the schema layer. A procedure + // without a metric must not replace an existing table metric with IP. + let mut resolved = core.vector_index_options(Some(kind), column, false)?; + // The shared result also retains the former Rust family's metric + // priority. Apply it to explicit metric parameters without injecting + // an unspecified procedure metric. + for key in [ + format!("{kind}.metric"), + format!("{kind}.distance.metric"), + format!("fields.{column}.metric"), + format!("fields.{column}.distance.metric"), + ] { + if let Some(metric) = resolved.options.get_mut(&key) { + *metric = resolved.metric.clone(); + } + } Ok(resolved.options) } else { core.global_index_options(kind, column) diff --git a/crates/paimon/src/table/vector_search_builder/index_options_tests.rs b/crates/paimon/src/table/vector_search_builder/index_options_tests.rs index 5608793bc..99e2efa0e 100644 --- a/crates/paimon/src/table/vector_search_builder/index_options_tests.rs +++ b/crates/paimon/src/table/vector_search_builder/index_options_tests.rs @@ -163,6 +163,98 @@ fn declaration(kind: &str, column: &str) -> HashMap { HashMap::from([(format!("global-index.{kind}.index-columns"), column.into())]) } +#[tokio::test] +async fn untyped_raw_json_metrics_preserve_source_priority_and_worker_results() { + let original = raw_table().await; + let key = "fields.embedding.index-options"; + for mode in ["full", "detail"] { + for (table_json, query_json, expected) in [ + (None, None, vec![1.0, 0.5, 1.0 / 3.0]), + ( + Some(r#"{"metric":"cosine"}"#), + None, + vec![1.0, 1.0 / 2.0_f32.sqrt(), 0.0], + ), + ( + None, + Some(r#"{"distance.metric":"cosine"}"#), + vec![1.0, 1.0 / 2.0_f32.sqrt(), 0.0], + ), + ( + Some(r#"{"metric":"cosine"}"#), + Some(r#"{"metric":"l2"}"#), + vec![1.0, 0.5, 1.0 / 3.0], + ), + // An explicit query metric need not parse an unused invalid table JSON. + ( + Some("invalid unused table JSON"), + Some(r#"{"metric":"inner_product"}"#), + vec![1.0, 1.0, 0.0], + ), + // Other implementation namespaces in JSON cannot invent a metric/type. + ( + None, + Some(r#"{"diskann.metric":"cosine","index_type":"diskann"}"#), + vec![1.0, 0.5, 1.0 / 3.0], + ), + ] { + let table = original.copy_with_options( + table_json + .map(|json| HashMap::from([(key.into(), json.into())])) + .unwrap_or_default(), + ); + let mut options = HashMap::from([ + ("vector-index.search-mode".into(), mode.into()), + ("opaque.query.option".into(), r#"{"values":[1,2]}"#.into()), + ]); + if let Some(json) = query_json { + options.insert(key.into(), json.into()); + } + let mut builder = table.new_vector_search_builder(); + builder + .with_vector_column("embedding") + .with_query_vector(vec![1.0, 0.0]) + .with_limit(3) + .with_options(options.clone()); + let plan = builder.new_scan().unwrap().plan().await.unwrap(); + assert_eq!(raw_type(&plan), None); + let plan: VectorScanPlan = + serde_json::from_slice(&serde_json::to_vec(&plan).unwrap()).unwrap(); + let reader = builder.new_read().unwrap(); + let local = reader.read(plan.clone()).await.unwrap(); + assert_eq!(local.row_ids().unwrap().scores, expected); + let worker = Worker { + table: table.clone(), + options: options.clone(), + }; + let remote = reader + .read_with_executor(plan.clone(), &worker) + .await + .unwrap(); + assert_eq!( + remote.row_ids().unwrap().row_ids, + local.row_ids().unwrap().row_ids + ); + assert_eq!(remote.row_ids().unwrap().scores, expected); + let mut batch = table.new_batch_vector_search_builder(); + batch + .with_vector_column("embedding") + .with_query_vectors(vec![vec![1.0, 0.0]; 2]) + .with_limit(3) + .with_options(options); + for result in batch + .new_read() + .unwrap() + .read_with_executor(plan, &worker) + .await + .unwrap() + { + assert_eq!(result.row_ids().unwrap().scores, expected); + } + } + } +} + #[tokio::test] async fn configuration_is_captured_by_single_batch_plan_reader_and_worker() { let table = raw_table().await; diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs b/crates/paimon/src/table/vindex_index_build_builder.rs index 501a575cf..ad61040dc 100644 --- a/crates/paimon/src/table/vindex_index_build_builder.rs +++ b/crates/paimon/src/table/vindex_index_build_builder.rs @@ -69,9 +69,13 @@ impl<'a> VindexIndexBuildBuilder<'a> { column: &str, field: &crate::spec::DataField, ) -> Result<(HashMap, VindexVectorIndexOptions)> { - let mut effective = super::global_index_options::build_table_options( + let procedure = super::global_index_options::build_procedure_options( self.table.schema().options(), &self.options, + ); + let mut effective = super::global_index_options::build_table_options( + self.table.schema().options(), + &procedure, column, ); let defaults = super::global_index_options::declared_options( @@ -81,7 +85,7 @@ impl<'a> VindexIndexBuildBuilder<'a> { &self.index_type, )?; let user = - super::global_index_options::build_options(&self.options, column, &self.index_type)?; + super::global_index_options::build_options(&procedure, column, &self.index_type)?; let native_user = crate::vindex::build_parameters(&user, &self.index_type, column); let native = VindexVectorIndexOptions::new_with_defaults( &effective, diff --git a/crates/paimon/src/table/vindex_index_build_builder/tests.rs b/crates/paimon/src/table/vindex_index_build_builder/tests.rs index 0acaad6a4..137de863a 100644 --- a/crates/paimon/src/table/vindex_index_build_builder/tests.rs +++ b/crates/paimon/src/table/vindex_index_build_builder/tests.rs @@ -107,6 +107,124 @@ fn test_table(options: HashMap) -> Table { ) } +#[test] +fn echoed_schema_keeps_table_semantics_and_procedure_overrides() { + for kind in ["ivf-flat", "diskann"] { + let mut options = table_options("100"); + options.extend(HashMap::from([ + (format!("{kind}.dimension"), "2".into()), + (format!("{kind}.metric"), "cosine".into()), + ("nlist".into(), "ignored table property".into()), + ("index.type".into(), "ignored table property".into()), + ("vindex.build.granule.enabled".into(), "true".into()), + ])); + let table = test_table(options.clone()); + let field = find_index_field(&table, "embedding").unwrap(); + let mut builder = super::VindexIndexBuildBuilder::new(&table, kind); + let (_, expected) = builder.configured_options("embedding", field).unwrap(); + builder.with_options(options.clone()); + let (_, actual) = builder.configured_options("embedding", field).unwrap(); + assert_eq!(actual.native_options, expected.native_options); + assert!(!actual.native_options.contains_key("nlist")); + assert_eq!(actual.native_options["metric"], "cosine"); + // A canonical declaration alone cannot inject a procedure metric default. + options.insert( + format!("global-index.{kind}.index-columns"), + "embedding".into(), + ); + builder.with_options(options.clone()); + assert_eq!( + builder + .configured_options("embedding", field) + .unwrap() + .1 + .native_options["metric"], + "cosine" + ); + options.insert(format!("{kind}.metric"), "l2".into()); + builder.with_options(options); + let (_, actual) = builder.configured_options("embedding", field).unwrap(); + assert_eq!(actual.native_options["metric"], "l2"); + assert!(!actual.native_options.contains_key("nlist")); + // Direct SDK calls keep native aliases, including values equal to schema properties. + builder.with_options(HashMap::from([ + ("metric".into(), "l2".into()), + ("dimension".into(), "2".into()), + ])); + assert_eq!( + builder + .configured_options("embedding", field) + .unwrap() + .1 + .native_options["metric"], + "l2" + ); + if kind == "ivf-flat" { + builder.with_options(HashMap::from([( + "nlist".into(), + "ignored table property".into(), + )])); + assert_eq!( + builder + .configured_options("embedding", field) + .unwrap() + .1 + .native_options["nlist"], + "ignored table property" + ); + } + } +} + +#[test] +fn procedure_legacy_family_keeps_its_metric_default_and_flat_override() { + let table = test_table(HashMap::from([("ivf-flat.metric".into(), "cosine".into())])); + let field = find_index_field(&table, "embedding").unwrap(); + let mut builder = super::VindexIndexBuildBuilder::new(&table, "ivf-flat"); + let mut options = HashMap::from([ + ( + "global-index.vector.index-columns".into(), + "embedding".into(), + ), + ( + "fields.embedding.global-index.vector.index-type".into(), + "ivf-flat".into(), + ), + ( + "fields.embedding.global-index.vector.index-options".into(), + r#"{"metric":"cosine"}"#.into(), + ), + ]); + builder.with_options(options.clone()); + assert_eq!( + builder + .configured_options("embedding", field) + .unwrap() + .1 + .native_options["metric"], + "inner_product" + ); + options.insert("ivf-flat.metric".into(), "l2".into()); + builder.with_options(options); + assert_eq!( + builder + .configured_options("embedding", field) + .unwrap() + .1 + .native_options["metric"], + "inner_product" + ); + builder.with_options(HashMap::from([("ivf-flat.metric".into(), "l2".into())])); + assert_eq!( + builder + .configured_options("embedding", field) + .unwrap() + .1 + .native_options["metric"], + "l2" + ); +} + fn plan(entries: Vec, rows_per_shard: i64) -> Result> { plan_with_indexed(entries, rows_per_shard, &[]) } diff --git a/docs/src/compaction.md b/docs/src/compaction.md index 40ae99c1d..e594530ee 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -211,6 +211,10 @@ clients still depend on the legacy PK namespace. Existing stored options do not need migration for the updated resolver. Explicit single-column builds also use these declarations as defaults; explicit builder options take precedence over them. +Native vector builds can pass the complete schema option map. Unchanged entries +retain table precedence; changed entries remain procedure overrides. Standalone builder +option maps still accept native parameter aliases. Captured shards retain the +original schema and option map for worker execution. FULL/DETAIL raw fallback uses the same per-column parameters as automatic builds, including vector metric defaults and full-text tokenization. A declared full-text column can use raw fallback before its first index is published.