diff --git a/Cargo.lock b/Cargo.lock index a2fb5f151..01748f3ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4654,6 +4654,7 @@ dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", + "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index b655d2607..ea7329160 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -127,6 +127,7 @@ bzip2 = "0.6" liblzma = { version = "0.4", default-features = false, features = ["static"] } lzokay-native = { version = "0.1", default-features = false, features = ["decompress"] } arrow-array = { workspace = true } +arrow-ipc = "58.0" arrow-arith = { workspace = true } arrow-buffer = { workspace = true } arrow-cast = { workspace = true } diff --git a/crates/paimon/src/btree/sst_file.rs b/crates/paimon/src/btree/sst_file.rs index 5695a21a6..2b6a0308a 100644 --- a/crates/paimon/src/btree/sst_file.rs +++ b/crates/paimon/src/btree/sst_file.rs @@ -92,6 +92,18 @@ impl SstFileWriter { writer } + pub(crate) fn retained_bytes(&self) -> usize { + self.data_block_writer + .memory() + .saturating_add(self.index_block_writer.memory()) + .saturating_add( + self.bloom_hashes + .as_ref() + .map_or(0, |hashes| hashes.capacity() * 4), + ) + .saturating_add(self.last_key.as_ref().map_or(0, Vec::capacity)) + } + /// Current write position in the output. pub fn position(&self) -> u64 { self.bytes_written diff --git a/crates/paimon/src/btree/writer.rs b/crates/paimon/src/btree/writer.rs index 47708ef7a..c55d82d7c 100644 --- a/crates/paimon/src/btree/writer.rs +++ b/crates/paimon/src/btree/writer.rs @@ -159,6 +159,11 @@ impl crate::Result> BTreeIndexWriter { Ok(self) } + pub(crate) fn retained_bytes(&self) -> usize { + self.sst_writer + .retained_bytes() + .saturating_add(self.current_row_ids.capacity() * 8) + } /// Write a key and its associated row id. /// If key is None, the row id is added to the null bitmap. /// Keys must be written in sorted order; entries with the same key are combined. diff --git a/crates/paimon/src/deletion_vector/core.rs b/crates/paimon/src/deletion_vector/core.rs index e274e3017..9d0b242db 100644 --- a/crates/paimon/src/deletion_vector/core.rs +++ b/crates/paimon/src/deletion_vector/core.rs @@ -65,6 +65,13 @@ impl DeletionVector { (*self.bitmap).clone() } + pub(crate) fn retained_bytes(&self) -> usize { + self.bitmap + .serialized_size() + .saturating_mul(2) + .saturating_add(64) + } + /// Number of deleted positions in this vector. pub fn cardinality(&self) -> u64 { self.bitmap.len() diff --git a/crates/paimon/src/io/cache/generated.rs b/crates/paimon/src/io/cache/generated.rs new file mode 100644 index 000000000..a22006640 --- /dev/null +++ b/crates/paimon/src/io/cache/generated.rs @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Disposable generated files in the same backend and budget as cached reads. +//! Callers publish a descriptor only after closing every immutable file. A +//! missing block is an error, never EOF: the caller must rebuild the artifact. + +use super::LocalCache; +use crate::io::{FileRead, FileWrite}; +use crate::{Error, Result}; +use bytes::{Bytes, BytesMut}; +use std::ops::Range; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +impl LocalCache { + // Generated artifacts deliberately bypass the remote-file whitelist. Their + // paths include an artifact format and immutable build ID, and participate + // in the ordinary cache's namespace, accounting, CRCs and regular LRU. + pub(crate) async fn generated_get( + &self, + path: &str, + index: u64, + length: usize, + ) -> Option { + let token = self.read_token(path); + let block = self + .get_block(&self.block_key(path, index), length, &token) + .await?; + (block.len() == length).then_some(block) + } + + pub(crate) async fn generated_put(&self, path: &str, index: u64, value: Bytes) -> Result<()> { + let length = value.len(); + let token = self.read_token(path); + let key = self.block_key(path, index); + self.put_block(&key, value.clone(), &token).await; + if self.get_block(&key, length, &token).await.as_ref() != Some(&value) { + return Err(Error::ResourceExhausted { + message: "Local cache could not retain a Lookup File block; check local-cache.max-size and cache storage".into(), + }); + } + Ok(()) + } + + pub(crate) fn generated_block_size(&self) -> u64 { + self.block_size + } +} + +pub(crate) struct GeneratedFileRead { + cache: Arc, + path: String, + size: u64, +} + +impl GeneratedFileRead { + pub(crate) fn new(cache: Arc, path: String, size: u64) -> Self { + Self { cache, path, size } + } +} + +#[async_trait::async_trait] +impl FileRead for GeneratedFileRead { + async fn read(&self, range: Range) -> Result { + if range.start > range.end || range.end > self.size { + return Err(Error::DataInvalid { + message: "Generated cache file read exceeds its length".into(), + source: None, + }); + } + let mut output = + BytesMut::with_capacity(usize::try_from(range.end - range.start).map_err(|_| { + Error::DataInvalid { + message: "Generated cache range is too large".into(), + source: None, + } + })?); + let block_size = self.cache.block_size; + let mut offset = range.start; + while offset < range.end { + let index = offset / block_size; + let start = index * block_size; + let end = start.saturating_add(block_size).min(self.size); + let length = usize::try_from(end - start).map_err(|_| Error::DataInvalid { + message: "Generated cache block is too large".into(), + source: None, + })?; + let block = self.cache.generated_get(&self.path, index, length).await.ok_or_else(|| Error::DataInvalid { + message: "Lookup cache block was evicted or corrupt; rebuild the Lookup File (check local-cache.max-size if this repeats)".into(), source: None, + })?; + let take_end = range.end.min(end); + output + .extend_from_slice(&block[(offset - start) as usize..(take_end - start) as usize]); + offset = take_end; + } + Ok(output.freeze()) + } +} + +pub(crate) struct GeneratedFileWrite { + cache: Arc, + path: String, + buffer: BytesMut, + index: u64, + position: Arc, +} + +impl GeneratedFileWrite { + pub(crate) fn new(cache: Arc, path: String) -> (Self, Arc) { + let position = Arc::new(AtomicU64::new(0)); + ( + Self { + cache, + path, + buffer: BytesMut::new(), + index: 0, + position: position.clone(), + }, + position, + ) + } + + async fn flush(&mut self) -> Result<()> { + if !self.buffer.is_empty() { + self.cache + .generated_put(&self.path, self.index, self.buffer.split().freeze()) + .await?; + self.index += 1; + } + Ok(()) + } +} + +#[async_trait::async_trait] +impl FileWrite for GeneratedFileWrite { + async fn write(&mut self, mut bytes: Bytes) -> Result<()> { + let total = bytes.len() as u64; + let block_size = + usize::try_from(self.cache.block_size).map_err(|_| Error::ConfigInvalid { + message: "local-cache.block-size exceeds the address space".into(), + })?; + while !bytes.is_empty() { + let take = (block_size - self.buffer.len()).min(bytes.len()); + self.buffer.extend_from_slice(&bytes.split_to(take)); + if self.buffer.len() == block_size { + self.flush().await?; + } + } + self.position.fetch_add(total, Ordering::Relaxed); + Ok(()) + } + + async fn close(&mut self) -> Result<()> { + self.flush().await + } +} diff --git a/crates/paimon/src/io/cache/mod.rs b/crates/paimon/src/io/cache/mod.rs index 4fa399534..da7d35308 100644 --- a/crates/paimon/src/io/cache/mod.rs +++ b/crates/paimon/src/io/cache/mod.rs @@ -17,6 +17,7 @@ mod disk; mod file_type; +pub(crate) mod generated; mod memory; mod priority_lru; mod reader; @@ -37,7 +38,7 @@ use std::sync::Arc; use disk::shared::SharedDiskCache; use disk::DiskCache; -pub(super) use reader::CachedFileReader; +pub(crate) use reader::CachedFileReader; const CACHE_DIRECTORY_NAME: &str = "paimon-local-cache-v2"; const DEFAULT_FILE_SIZE_CAPACITY: usize = 65_536; diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 79779c3a4..dec7b603a 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -584,6 +584,10 @@ impl FileIO { Ok(self.cache_namespace.to_string()) } + pub(crate) fn local_cache(&self) -> Option> { + self.cache.clone() + } + #[cfg(test)] pub(crate) fn has_local_cache(&self) -> bool { self.cache.is_some() diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index f85edca0b..76129c0b4 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -2299,7 +2299,7 @@ fn parse_memory_size(value: &str) -> Option { /// does. Returns `None` for an empty string, a missing or non-numeric number, /// an unrecognized unit, or a value that would overflow `u64`, matching the /// inputs on which Java throws. -fn parse_duration_millis(value: &str) -> Option { +pub(crate) fn parse_duration_millis(value: &str) -> Option { let value = value.trim(); if value.is_empty() { return None; diff --git a/crates/paimon/src/table/compact.rs b/crates/paimon/src/table/compact.rs index 46f553064..6a9b1692d 100644 --- a/crates/paimon/src/table/compact.rs +++ b/crates/paimon/src/table/compact.rs @@ -19,8 +19,9 @@ mod append; pub(super) mod data_evolution; -mod primary_key; +pub(super) mod primary_key; +pub use primary_key::lookup_store::LookupCacheMetrics; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; @@ -114,6 +115,18 @@ impl<'a> CompactBuilder<'a> { Ok(CompactExecutor { table: self.table.clone(), resources: self.resources.clone(), + lookup_cache: if !self.table.schema().primary_keys().is_empty() + && primary_key::lookup_store::needed(self.table)? + { + Some(primary_key::lookup_store::LookupCache::open( + self.table.file_io().local_cache().ok_or_else(|| Error::ConfigInvalid { + message: "Lookup compaction requires local-cache.enabled=true on the catalog/FileIO".into(), + })?, + self.table.schema().options(), + )?) + } else { + None + }, }) } @@ -194,6 +207,8 @@ pub struct CompactTask { bucket: i32, total_buckets: i32, files: Vec, + #[serde(default)] + primary_key: Option, } impl CompactTask { @@ -213,6 +228,12 @@ impl CompactTask { self.bucket } + pub(super) fn primary_key_files(&self) -> impl Iterator { + self.primary_key + .iter() + .flat_map(|selection| selection.lookup_files.iter()) + } + pub fn files(&self) -> &[DataFileMeta] { &self.files } @@ -266,6 +287,7 @@ impl CompactPlanner { pub struct CompactExecutor { table: Table, resources: Option, + lookup_cache: Option>, } /// Prepared files owned by the caller until submitted to `TableCommit::commit_compact`. @@ -274,6 +296,10 @@ pub struct CompactExecutor { pub struct CompactResult { task: CompactTask, files: Vec, + #[serde(default)] + changelog_files: Vec, + #[serde(default)] + lookup_deletions: Vec, } impl CompactResult { @@ -281,12 +307,27 @@ impl CompactResult { &self.files } + pub fn changelog_files(&self) -> &[DataFileMeta] { + &self.changelog_files + } + pub fn metrics(&self) -> CompactMetrics { CompactMetrics { files_removed: self.task.files.len(), files_added: self.files.len(), rows_written: self.files.iter().map(|file| file.row_count as u64).sum(), bytes_written: self.files.iter().map(|file| file.file_size as u64).sum(), + changelog_files_added: self.changelog_files.len(), + changelog_rows_written: self + .changelog_files + .iter() + .map(|file| file.row_count as u64) + .sum(), + changelog_bytes_written: self + .changelog_files + .iter() + .map(|file| file.file_size as u64) + .sum(), } } } @@ -298,9 +339,21 @@ pub struct CompactMetrics { pub files_added: usize, pub rows_written: u64, pub bytes_written: u64, + #[serde(default)] + pub changelog_files_added: usize, + #[serde(default)] + pub changelog_rows_written: u64, + #[serde(default)] + pub changelog_bytes_written: u64, } impl CompactExecutor { + pub fn lookup_cache_metrics(&self) -> LookupCacheMetrics { + self.lookup_cache + .as_ref() + .map_or_else(LookupCacheMetrics::default, |cache| cache.metrics()) + } + pub async fn execute(&self, task: &CompactTask) -> Result { ensure_supported(&self.table)?; task.validate_target(&self.table)?; @@ -351,13 +404,24 @@ impl CompactExecutor { return Ok(CompactResult { task: task.clone(), files, + changelog_files: vec![], + lookup_deletions: vec![], }); } if task.is_primary_key() { - let files = primary_key::execute(&table, task, &split, self.resources.clone()).await?; + let (files, changelog_files, lookup_deletions) = primary_key::execute( + &table, + task, + &split, + self.resources.clone(), + self.lookup_cache.clone(), + ) + .await?; return Ok(CompactResult { task: task.clone(), files, + changelog_files, + lookup_deletions, }); } let mut read = table.new_read_builder().new_read()?.with_ordered(true); @@ -408,6 +472,8 @@ impl CompactExecutor { Ok(files) => Ok(CompactResult { task: task.clone(), files, + changelog_files: vec![], + lookup_deletions: vec![], }), Err(error) => { writer.abort().await; @@ -427,19 +493,12 @@ impl CompactTask { } fn output_level(&self) -> i32 { - if self.is_primary_key() { - self.files - .iter() - .map(|file| file.level) - .max() - .unwrap_or(0) - .max(1) - } else { - 0 - } + self.primary_key + .as_ref() + .map_or(0, |task| task.output_level) } - fn validate_inputs(&self, active: &[ManifestEntry]) -> Result<()> { + pub(super) fn validate_inputs(&self, active: &[ManifestEntry]) -> Result<()> { if self.is_data_evolution() { data_evolution::validate_inputs(&self.partition, self.bucket, &self.files, active)?; } @@ -452,10 +511,18 @@ impl CompactTask { }) .map(|entry| (entry.file().file_name.as_str(), entry)) .collect::>(); - if self.is_primary_key() && bucket_files.len() != self.files.len() { - return Err(invalid( - "PK compaction must include the complete captured bucket", - )); + if self.is_primary_key() { + let files = bucket_files + .values() + .map(|entry| entry.file().clone()) + .collect(); + let selected = primary_key::planner::select(files, self.context.schema.options())? + .ok_or_else(|| invalid("No incremental PK compaction is eligible"))?; + if selected.0 != self.files || Some(&selected.1) != self.primary_key.as_ref() { + return Err(invalid( + "PK compaction inputs do not match incremental run selection", + )); + } } for file in &self.files { if !bucket_files @@ -481,6 +548,16 @@ impl CompactTask { )); } ensure_snapshot_supported(table, &self.context.snapshot)?; + if self.is_primary_key() + && self + .primary_key + .as_ref() + .is_none_or(|task| task.output_level <= 0) + { + return Err(invalid( + "PK compaction requires an incremental run selection", + )); + } let minimum_files = if self.is_primary_key() || self.is_data_evolution() { 1 } else { @@ -513,11 +590,14 @@ impl CompactTask { /// Validated inputs for the existing atomic snapshot publication machinery. pub(super) struct PreparedCompact { pub entries: Vec, + pub changelog_entries: Vec, pub schema_id: i64, pub snapshot_id: i64, pub read_snapshot: Option>, pub commit_suffix: String, pub metrics: CompactMetrics, + pub pk_tasks: Vec, + pub lookup_deletions: Vec, } pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Result { @@ -529,7 +609,10 @@ pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Resul let mut inputs = HashSet::new(); let mut outputs = HashSet::new(); let mut entries = Vec::new(); + let mut changelog_entries = Vec::new(); let mut metrics = CompactMetrics::default(); + let mut pk_tasks = Vec::new(); + let mut lookup_deletions = Vec::new(); for result in results { let task = &result.task; task.validate_target(table)?; @@ -570,6 +653,13 @@ pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Resul if file.schema_id != task.context.schema.id() || file.file_source != Some(1) || file.level != task.output_level() + || (task.is_primary_key() + && task + .context + .schema + .core_options() + .deletion_vectors_enabled() + && file.delete_row_count != Some(0)) || !outputs.insert(file.file_name.clone()) || task.files.iter().any(|old| old.file_name == file.file_name) { @@ -599,17 +689,86 @@ pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Resul ); if task.is_primary_key() && (output_rows > input_rows - || (input_rows > 0 && output_rows == 0 && !ignores_retracts)) + || (input_rows > 0 + && output_rows == 0 + && !ignores_retracts + && !options.deletion_vectors_enabled() + && options.merge_engine()? != MergeEngine::FirstRow)) { return Err(invalid( - "PK compaction must retain each key's merged state unless retracts are ignored", + "PK compaction must retain each key's merged state unless retracts are ignored or deletion vectors are enabled", )); } + let lookup = options.try_changelog_producer()? == ChangelogProducer::Lookup; + if !lookup && !result.changelog_files.is_empty() { + return Err(invalid( + "Only lookup compaction may produce changelog files", + )); + } + let mut changelog_rows = 0u64; + for file in &result.changelog_files { + validate_file(file, true)?; + if file.schema_id != task.context.schema.id() + || file.file_source != Some(1) + || file.level != task.output_level() + || file + .delete_row_count + .is_none_or(|count| count > file.row_count) + || !outputs.insert(file.file_name.clone()) + || task.files.iter().any(|old| old.file_name == file.file_name) + { + return Err(invalid("Invalid compaction changelog file")); + } + changelog_rows += file.row_count as u64; + changelog_entries.push(ManifestEntry::new( + FileKind::Add, + task.partition.clone(), + task.bucket, + task.total_buckets, + file.clone(), + 2, + )); + } + let l0_rows = task + .files + .iter() + .filter(|file| file.level == 0) + .map(|file| file.row_count as u64) + .sum::(); + if changelog_rows > l0_rows.saturating_mul(2) { + return Err(invalid( + "Lookup changelogs must be generated by level-0 records", + )); + } + if task.is_primary_key() { + pk_tasks.push(task.clone()); + } + for deletion in &result.lookup_deletions { + let bitmap = roaring::RoaringBitmap::deserialize_from(deletion.bitmap.as_slice()) + .map_err(|error| invalid(&error.to_string()))?; + if !options.deletion_vectors_enabled() + || deletion.partition != task.partition + || deletion.bucket != task.bucket + || bitmap.is_empty() + || !task.primary_key.as_ref().is_some_and(|selection| { + selection.lookup_files.iter().any(|file| { + file.file_name == deletion.file_name + && u64::from(bitmap.max().unwrap()) < file.row_count as u64 + }) + }) + { + return Err(invalid("Invalid lookup deletion dependency")); + } + lookup_deletions.push(deletion.clone()); + } let part = result.metrics(); metrics.files_removed += part.files_removed; metrics.files_added += part.files_added; metrics.rows_written += part.rows_written; metrics.bytes_written += part.bytes_written; + metrics.changelog_files_added += part.changelog_files_added; + metrics.changelog_rows_written += part.changelog_rows_written; + metrics.changelog_bytes_written += part.changelog_bytes_written; } ids.sort_unstable(); if ids.windows(2).any(|pair| pair[0] == pair[1]) { @@ -623,6 +782,7 @@ pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Resul } Ok(PreparedCompact { entries, + changelog_entries, schema_id: first.task.context.schema.id(), snapshot_id: first.task.snapshot_id(), read_snapshot: first @@ -631,6 +791,8 @@ pub(super) fn prepare_results(table: &Table, results: &[CompactResult]) -> Resul .then(|| Box::new(first.task.context.snapshot.clone())), commit_suffix: hex::encode(digest.finalize()), metrics, + pk_tasks, + lookup_deletions, }) } @@ -672,7 +834,7 @@ pub(super) fn ensure_supported(table: &Table) -> Result<()> { if table.is_format_table() || options.row_tracking_enabled() || options.data_evolution_enabled() - || options.deletion_vectors_enabled() + || (options.deletion_vectors_enabled() && table.schema().primary_keys().is_empty()) || options.query_auth_enabled() || !options.blob_fields().is_empty() || table @@ -685,28 +847,35 @@ pub(super) fn ensure_supported(table: &Table) -> Result<()> { || options.bucket() < -1 { return Err(Error::Unsupported { - message: "Compaction does not support row tracking, data evolution, deletion vectors, query authorization, rowkind fields, postpone buckets or dedicated blob/vector storage".into(), + message: "Compaction does not support row tracking, data evolution, append deletion vectors, query authorization, rowkind fields, postpone buckets or dedicated blob/vector storage".into(), }); } if !table.schema().primary_keys().is_empty() && (options.bucket() == 0 || !matches!( options.try_changelog_producer()?, - ChangelogProducer::None | ChangelogProducer::Input + ChangelogProducer::None | ChangelogProducer::Input | ChangelogProducer::Lookup ) || (options.merge_engine()? == MergeEngine::FirstRow - && options.try_changelog_producer()? != ChangelogProducer::None)) + && !matches!( + options.try_changelog_producer()?, + ChangelogProducer::None | ChangelogProducer::Lookup + ))) { return Err(Error::Unsupported { - message: "PK compaction requires fixed or dynamic buckets and changelog-producer=none or input; first-row requires changelog-producer=none".into(), + message: "PK compaction requires fixed or dynamic buckets and changelog-producer=none, input or lookup; first-row requires changelog-producer=none or lookup".into(), }); } + if options.try_changelog_producer()? == ChangelogProducer::Lookup { + primary_key::lookup::validate_options(properties)?; + } options.validate_data_file_path_directory() } fn ensure_snapshot_supported(table: &Table, snapshot: &Snapshot) -> Result<()> { if snapshot.index_manifest().is_some() && !table.schema().core_options().data_evolution_enabled() + && !table.schema().core_options().deletion_vectors_enabled() && (table.schema().primary_keys().is_empty() || table.schema().core_options().bucket() != -1) { @@ -718,7 +887,8 @@ fn ensure_snapshot_supported(table: &Table, snapshot: &Snapshot) -> Result<()> { } /// HASH indexes contain key-to-bucket assignments, independent of data files. -/// Preserve their manifest verbatim, including hashes of deleted keys. Other +/// Preserve their entries, including hashes of deleted keys. PK deletion-vector +/// ranges are retired atomically when their data files are rewritten. Other /// index types may address rewritten files or row positions and cannot be reused. pub(super) async fn validate_snapshot_indexes( table: &Table, @@ -734,13 +904,23 @@ pub(super) async fn validate_snapshot_indexes( data_evolution::validate_indexes(&entries)?; return Ok(Some(name.to_string())); } + let options = table.schema().core_options(); if entries.iter().any(|entry| { - entry.index_file.index_type != "HASH" - || entry.index_file.deletion_vectors_ranges.is_some() - || entry.index_file.global_index_meta.is_some() + let file = &entry.index_file; + let hash = options.bucket() == -1 + && file.index_type == "HASH" + && file.deletion_vectors_ranges.is_none(); + let deletion_vector = options.deletion_vectors_enabled() + && file.index_type == "DELETION_VECTORS" + && file.deletion_vectors_ranges.as_ref().is_some_and(|ranges| { + ranges.values().all(|meta| { + meta.offset >= 0 && meta.length > 0 && meta.cardinality.is_none_or(|n| n >= 0) + }) + }); + file.global_index_meta.is_some() || !(hash || deletion_vector) }) { return Err(Error::Unsupported { - message: "Compaction only supports dynamic-bucket HASH index manifests".into(), + message: "PK compaction only supports dynamic-bucket HASH indexes and enabled deletion vectors".into(), }); } Ok(Some(name.to_string())) diff --git a/crates/paimon/src/table/compact/append.rs b/crates/paimon/src/table/compact/append.rs index 58fb92f36..b83c7b32c 100644 --- a/crates/paimon/src/table/compact/append.rs +++ b/crates/paimon/src/table/compact/append.rs @@ -60,6 +60,7 @@ impl AppendCompactPlanner { bucket, total_buckets, files, + primary_key: None, }); } } diff --git a/crates/paimon/src/table/compact/data_evolution.rs b/crates/paimon/src/table/compact/data_evolution.rs index 4e4b6f56c..2f16a2a36 100644 --- a/crates/paimon/src/table/compact/data_evolution.rs +++ b/crates/paimon/src/table/compact/data_evolution.rs @@ -193,6 +193,7 @@ impl DataEvolutionCompactPlanner { bucket, total_buckets, files, + primary_key: None, }); } } diff --git a/crates/paimon/src/table/compact/primary_key.rs b/crates/paimon/src/table/compact/primary_key.rs index 454c751fd..e500e8526 100644 --- a/crates/paimon/src/table/compact/primary_key.rs +++ b/crates/paimon/src/table/compact/primary_key.rs @@ -15,6 +15,11 @@ // specific language governing permissions and limitations // under the License. +pub(in crate::table) mod deletion_vectors; +pub(super) mod lookup; +pub(in crate::table) mod lookup_store; +pub(in crate::table) mod planner; + use std::sync::Arc; use arrow_schema::SchemaRef; @@ -23,8 +28,8 @@ use futures::TryStreamExt; use super::{split_files_by_bucket, validate_file, CompactContext, CompactTask}; use crate::resource::ResourceContext; use crate::spec::{ - BigIntType, DataField, DataFileMeta, DataType, MergeEngine, RowKind, TinyIntType, - SEQUENCE_NUMBER_FIELD_ID, SEQUENCE_NUMBER_FIELD_NAME, VALUE_KIND_FIELD_ID, + BigIntType, ChangelogProducer, DataField, DataFileMeta, DataType, MergeEngine, RowKind, + TinyIntType, SEQUENCE_NUMBER_FIELD_ID, SEQUENCE_NUMBER_FIELD_NAME, VALUE_KIND_FIELD_ID, VALUE_KIND_FIELD_NAME, }; use crate::table::data_file_index_writer::FileIndexOptions; @@ -33,6 +38,7 @@ use crate::table::kv_data_file_writer::{KeyValueFileOptions, KeyValueRollingWrit use crate::table::kv_file_reader::{KeyValueFileReader, KeyValueReadConfig}; use crate::table::sort_merge::{ compare_sequence_order, BufferedBatch, MergeFunction, MergeResult, MergeRow, + CHANGELOG_BATCH_METADATA_KEY, }; use crate::table::table_read::{ configured_mosaic_prefetch, configured_parquet_read_budget, MAX_MERGE_INPUT_STREAMS, @@ -40,9 +46,8 @@ use crate::table::table_read::{ use crate::table::{DataSplit, Plan, Table}; use crate::Result; -/// Select complete captured buckets that need an L0 upgrade or level merge. +/// Select incremental sorted runs using Java's universal size/run policy. pub(super) struct PrimaryKeyCompactPlanner; - impl PrimaryKeyCompactPlanner { pub(super) fn plan_tasks( &self, @@ -54,13 +59,7 @@ impl PrimaryKeyCompactPlanner { for file in &files { validate_file(file, true)?; } - // One positive level is already a non-overlapping sorted run, - // including when the previous compaction rolled several files. - let first_level = files.first().map(|file| file.level); - if files - .iter() - .any(|file| file.level == 0 || Some(file.level) != first_level) - { + if let Some((files, primary_key)) = planner::select(files, context.schema.options())? { tasks.push(CompactTask { context: context.clone(), task_id: tasks.len(), @@ -68,6 +67,7 @@ impl PrimaryKeyCompactPlanner { bucket, total_buckets, files, + primary_key: Some(primary_key), }); } } @@ -76,8 +76,8 @@ impl PrimaryKeyCompactPlanner { } /// Keep the same winner as a deduplicate query, including DELETE/UPDATE_BEFORE. -/// A later write may carry a lower user sequence, so even full compaction -/// must retain a winning retract rather than resurrect an older value. +/// A later write may carry a lower user sequence, so compaction must retain +/// a winning retract rather than resurrect an older value. struct CompactDeduplicate; impl MergeFunction for CompactDeduplicate { @@ -152,14 +152,46 @@ impl MergeFunction for CompactReducer { } } +struct CompactFirstRow(Box); +impl MergeFunction for CompactFirstRow { + fn merge( + &self, + rows: &[MergeRow], + batches: &[BufferedBatch], + columns: &[usize], + schema: &SchemaRef, + ) -> Result { + let result = self.0.merge(rows, batches, columns, schema)?; + if rows.iter().any(|row| match &batches[row.batch_idx] { + BufferedBatch::Source(batch) => batch + .schema() + .metadata() + .contains_key("paimon.compact.lookup-base"), + _ => false, + }) { + Ok(MergeResult::Omit) + } else { + Ok(result) + } + } +} + fn new_merge_function( config: &KeyValueReadConfig, fields: &[DataField], ) -> Result> { + if crate::spec::CoreOptions::new(&config.table_options).deletion_vectors_enabled() { + // Like Java's DV lookup rewriter, materialize live values and drop + // retracts. History comes from selected runs or local point lookup; + // both apply the captured physical deletion vectors before merging. + return Ok(Box::new(crate::table::sort_merge::LookupMergeFunction( + KeyValueFileReader::new_unwrapped_merge_function(config, fields)?, + ))); + } if config.merge_engine == MergeEngine::Deduplicate { return Ok(Box::new(CompactDeduplicate)); } - let inner = KeyValueFileReader::new_merge_function(config, fields)?; + let inner = KeyValueFileReader::new_unwrapped_merge_function(config, fields)?; let aggregate_functions = match config.merge_engine { MergeEngine::PartialUpdate => crate::spec::PartialUpdateConfig::new(&config.table_options) .validated_aggregate_functions(&config.table_fields, &config.primary_keys)? @@ -190,7 +222,7 @@ fn new_merge_function( }; Ok(if config.merge_engine == MergeEngine::FirstRow { // First-row must still reject retracts unless ignore-delete is enabled. - inner + Box::new(CompactFirstRow(inner)) as Box } else { Box::new(CompactReducer { inner, @@ -213,16 +245,8 @@ fn new_merge_function( }) } -pub(super) async fn execute( - table: &Table, - task: &CompactTask, - split: &DataSplit, - resources: Option, -) -> Result> { - let schema = table.schema(); - let options = schema.core_options(); - let merge_engine = options.merge_engine()?; - let mut read_type = vec![ +fn read_type(table: &Table) -> Vec { + let mut fields = vec![ DataField::new( SEQUENCE_NUMBER_FIELD_ID, SEQUENCE_NUMBER_FIELD_NAME.into(), @@ -234,12 +258,51 @@ pub(super) async fn execute( DataType::TinyInt(TinyIntType::new()), ), ]; - read_type.extend_from_slice(schema.fields()); + fields.extend_from_slice(table.schema().fields()); + fields +} + +pub(super) async fn execute( + table: &Table, + task: &CompactTask, + split: &DataSplit, + resources: Option, + cache: Option>, +) -> Result<( + Vec, + Vec, + Vec, +)> { + let schema = table.schema(); + let options = schema.core_options(); + let merge_engine = options.merge_engine()?; + let lookup_changelog = options.try_changelog_producer()? == ChangelogProducer::Lookup; + let read_type = read_type(table); let budget = configured_parquet_read_budget(table)?; let budget = match &resources { Some(resources) => Arc::new(budget.with_resources(resources.clone()).without_prefetch()), None => Arc::new(budget.without_prefetch()), }; + let (provider, deletions) = if let Some(cache) = cache { + let selection = task + .primary_key + .as_ref() + .ok_or_else(|| super::invalid("Missing incremental PK selection"))?; + let (provider, deletions) = lookup_store::provider( + table, + split, + &selection.lookup_files, + cache, + resources.clone(), + ) + .await?; + (Some(provider), deletions) + } else { + ( + None, + Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), + ) + }; let reader = KeyValueFileReader::new( table.file_io().clone(), KeyValueReadConfig { @@ -265,8 +328,21 @@ pub(super) async fn execute( mosaic_prefetch: configured_mosaic_prefetch(table)?, }, ); + let reader = reader.with_lookup(provider); + let deletion_split = if options.deletion_vectors_enabled() { + Some(deletion_vectors::with_deletion_files(table, split, &task.context.snapshot).await?) + } else { + None + }; + let split = deletion_split.as_ref().unwrap_or(split); let mut batches = - reader.read_with_merge_function(std::slice::from_ref(split), new_merge_function)?; + reader.read_with_merge_function(std::slice::from_ref(split), move |config, fields| { + if lookup_changelog { + lookup::new_merge_function(config, fields) + } else { + new_merge_function(config, fields) + } + })?; let keys = schema.trimmed_primary_keys(); let primary_key_indices = keys .iter() @@ -301,39 +377,94 @@ pub(super) async fn execute( schema.options(), )?; let file_format = options.file_format(); + let file_options = KeyValueFileOptions { + file_io: table.file_io(), + paths: &paths, + table_options: schema.options(), + value_fields: schema.fields(), + primary_key_indices: &primary_key_indices, + primary_key_types: &primary_key_types, + schema_id: schema.id(), + file_format: &file_format, + file_prefix: options.data_file_prefix(), + file_compression: options.file_compression(), + file_compression_zstd_level: options.file_compression_zstd_level(), + write_buffer_size: options.write_buffer_size()? as i64, + level: task.output_level(), + file_source: 1, + is_changelog: false, + index_options: index.as_ref(), + managed_blob_enabled: false, + resources: resources.as_ref(), + }; let mut writer = KeyValueRollingWriter::new( + file_options, + options.target_file_size(), + options.target_file_row_num()?, + ); + let changelog_format = options.changelog_file_format(); + let mut changelog_writer = KeyValueRollingWriter::new( KeyValueFileOptions { - file_io: table.file_io(), - paths: &paths, - table_options: schema.options(), - value_fields: schema.fields(), - primary_key_indices: &primary_key_indices, - primary_key_types: &primary_key_types, - schema_id: schema.id(), - file_format: &file_format, - file_prefix: options.data_file_prefix(), - file_compression: options.file_compression(), - file_compression_zstd_level: options.file_compression_zstd_level(), - write_buffer_size: options.write_buffer_size()? as i64, - level: task.output_level(), - file_source: 1, - is_changelog: false, - index_options: index.as_ref(), - managed_blob_enabled: false, - resources: resources.as_ref(), + file_format: &changelog_format, + file_prefix: options.changelog_file_prefix(), + file_compression: options.changelog_file_compression(), + is_changelog: true, + index_options: None, + ..file_options }, options.target_file_size(), options.target_file_row_num()?, ); + let result = async { while let Some(batch) = batches.try_next().await? { - writer.write(&batch).await?; + if batch + .schema() + .metadata() + .contains_key(CHANGELOG_BATCH_METADATA_KEY) + { + let mut metadata = batch.schema().metadata().clone(); + metadata.remove(CHANGELOG_BATCH_METADATA_KEY); + let batch = arrow_array::RecordBatch::try_new( + Arc::new(batch.schema().as_ref().clone().with_metadata(metadata)), + batch.columns().to_vec(), + ) + .map_err(|error| { + super::invalid(&format!("Invalid changelog output batch: {error}")) + })?; + changelog_writer.write(&batch).await?; + } else { + writer.write(&batch).await?; + } } - writer.prepare_commit().await + // Keep both prepared file lists owned by their writers until both have + // closed successfully, so either failure can clean up the whole task. + writer.finish().await?; + changelog_writer.finish().await?; + Ok(( + writer.prepare_commit().await?, + changelog_writer.prepare_commit().await?, + std::mem::take(&mut *deletions.lock().unwrap()) + .into_iter() + .map(|(file_name, bitmap)| { + let mut bytes = Vec::new(); + bitmap + .serialize_into(&mut bytes) + .map_err(|error| super::invalid(&error.to_string()))?; + Ok(lookup_store::LookupDeletion { + partition: task.partition.clone(), + bucket: task.bucket, + file_name, + bitmap: bytes, + }) + }) + .collect::>>()?, + )) } .await; if result.is_err() { writer.abort().await; + changelog_writer.abort().await; } result } diff --git a/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs b/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs new file mode 100644 index 000000000..81fe23634 --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/deletion_vectors.rs @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Snapshot-bound physical deletion filtering and commit-time index retirement. + +use std::collections::{HashMap, HashSet}; + +use crate::spec::{ + DeletionVectorMeta, FileKind, IndexManifest, IndexManifestEntry, ManifestEntry, Snapshot, +}; +use crate::table::index_file_path::IndexFileLocation; +use crate::table::{DataSplit, Table}; +use crate::{DeletionFile, Result}; + +async fn read_indexes(table: &Table, snapshot: &Snapshot) -> Result> { + match snapshot.index_manifest() { + Some(name) => { + IndexManifest::read( + table.file_io(), + &table.snapshot_manager().manifest_path(name), + ) + .await + } + None => Ok(vec![]), + } +} + +/// Attach the captured vectors to physical files before the KV reader merges +/// keys. Materialized high-level rows must never be accumulated twice. +pub(super) async fn with_deletion_files( + table: &Table, + split: &DataSplit, + snapshot: &Snapshot, +) -> Result { + let location = IndexFileLocation::BucketLocal { + table_path: table.location().trim_end_matches('/'), + bucket_path: split.bucket_path(), + index_file_in_data_file_dir: table.schema().core_options().index_file_in_data_file_dir(), + }; + let partition = split.partition().to_serialized_bytes(); + let mut by_name = HashMap::new(); + for entry in read_indexes(table, snapshot).await? { + if entry.kind != FileKind::Add + || entry.index_file.index_type != "DELETION_VECTORS" + || entry.partition != partition + || entry.bucket != split.bucket() + { + continue; + } + if let Some(ranges) = &entry.index_file.deletion_vectors_ranges { + let path = location.resolve( + &entry.index_file.file_name, + entry.index_file.external_path.as_deref(), + ); + for (name, meta) in ranges { + by_name.insert( + name.clone(), + DeletionFile::new( + path.clone(), + i64::from(meta.offset), + i64::from(meta.length), + meta.cardinality, + ), + ); + } + } + } + let deletion_files = split + .data_files() + .iter() + .map(|file| by_name.remove(&file.file_name)) + .collect(); + // Preserve the selected physical files, including L0, irrespective of the + // query reader's deletion-vectors.merge-on-read option. + DataSplit::builder() + .with_snapshot(split.snapshot_id()) + .with_partition(split.partition().clone()) + .with_bucket(split.bucket()) + .with_total_buckets(split.total_buckets()) + .with_bucket_path(split.bucket_path().to_string()) + .with_data_files(split.data_files().to_vec()) + .with_data_deletion_files(deletion_files) + .build() +} + +type FileId = (Vec, i32, String); +type VectorReference = (String, Option, DeletionVectorMeta); + +fn selected_vectors( + indexes: &[IndexManifestEntry], + inputs: &HashSet, +) -> HashMap { + let mut selected = HashMap::new(); + for entry in indexes { + if entry.kind != FileKind::Add || entry.index_file.index_type != "DELETION_VECTORS" { + continue; + } + if let Some(ranges) = &entry.index_file.deletion_vectors_ranges { + for (name, meta) in ranges { + let id = (entry.partition.clone(), entry.bucket, name.clone()); + if inputs.contains(&id) { + selected.insert( + id, + ( + entry.index_file.file_name.clone(), + entry.index_file.external_path.clone(), + meta.clone(), + ), + ); + } + } + } + } + selected +} + +/// Cache bytes are immutable physical rows. Snapshot vectors must still match +/// every rewrite/lookup dependency before publishing any result. +pub(in crate::table) async fn validate_dependencies( + table: &Table, + captured: &Snapshot, + latest: &Snapshot, + tasks: &[super::super::CompactTask], +) -> Result<()> { + let mut inputs = HashSet::new(); + for task in tasks { + for file in task.files.iter().chain( + task.primary_key + .iter() + .flat_map(|selection| &selection.lookup_files), + ) { + inputs.insert((task.partition.clone(), task.bucket, file.file_name.clone())); + } + } + if selected_vectors(&read_indexes(table, captured).await?, &inputs) + != selected_vectors(&read_indexes(table, latest).await?, &inputs) + { + return Err(super::super::invalid( + "Compaction input or lookup dependency deletion vectors changed; create a new plan", + )); + } + Ok(()) +} + +/// Apply new historical row deletions and retire rewritten inputs atomically. +/// Untouched ranges sharing an index file and concurrent independent indexes +/// are retained. Called on every CAS attempt against that attempt's snapshot. +pub(in crate::table) async fn rewrite_indexes( + table: &Table, + latest: &Snapshot, + delta: &[ManifestEntry], + deletions: &[super::lookup_store::LookupDeletion], + created_files: &mut Vec, +) -> Result>> { + use crate::table::data_evolution_writer::DataEvolutionDeleteWriter; + use crate::table::index_file_path::committed_index_file_path; + use indexmap::IndexMap; + use roaring::RoaringBitmap; + use std::collections::BTreeMap; + let mut indexes = read_indexes(table, latest).await?; + let mut buckets = + BTreeMap::<(Vec, i32), (HashSet, IndexMap)>::new(); + for entry in delta + .iter() + .filter(|entry| *entry.kind() == FileKind::Delete) + { + buckets + .entry((entry.partition().to_vec(), entry.bucket())) + .or_default() + .0 + .insert(entry.file().file_name.clone()); + } + for deletion in deletions { + let bitmap = RoaringBitmap::deserialize_from(deletion.bitmap.as_slice()) + .map_err(|error| super::super::invalid(&error.to_string()))?; + *buckets + .entry((deletion.partition.clone(), deletion.bucket)) + .or_default() + .1 + .entry(deletion.file_name.clone()) + .or_default() |= bitmap; + } + let writer = DataEvolutionDeleteWriter::for_primary_key_compaction(table); + let mut changed = false; + for ((partition, bucket), (rewritten, additions)) in buckets { + let targets = rewritten + .iter() + .cloned() + .chain(additions.keys().cloned()) + .collect::>(); + let (mut vectors, removed) = writer + .read_existing_bucket_deletion_vectors(&partition, bucket, latest, Some(&targets)) + .await?; + let mut retired = false; + for name in &rewritten { + retired |= vectors.shift_remove(name).is_some(); + } + if !retired && additions.is_empty() { + continue; + } + for (name, bitmap) in additions { + *vectors.entry(name).or_default() |= bitmap; + } + indexes.retain(|entry| { + !removed.iter().any(|file| { + entry.partition == partition && entry.bucket == bucket && entry.index_file == *file + }) + }); + if !vectors.values().all(RoaringBitmap::is_empty) { + let file = writer + .write_deletion_vector_index_file(&partition, bucket, vectors) + .await?; + let options = table.schema().core_options(); + let computer = crate::spec::PartitionComputer::new( + table.schema().partition_keys(), + table.schema().fields(), + options.partition_default_name(), + options.legacy_partition_name(), + )?; + let bucket_path = crate::spec::bucket_path( + &table.data_file_location(), + Some(&computer), + &crate::spec::BinaryRow::from_serialized_bytes(&partition)?, + bucket, + )?; + created_files.push(committed_index_file_path( + table.location(), + &bucket_path, + options.index_file_in_data_file_dir(), + &file, + )); + indexes.push(IndexManifestEntry { + version: 1, + kind: FileKind::Add, + partition, + bucket, + index_file: file, + }); + } + changed = true; + } + Ok(changed.then_some(indexes)) +} diff --git a/crates/paimon/src/table/compact/primary_key/lookup.rs b/crates/paimon/src/table/compact/primary_key/lookup.rs new file mode 100644 index 000000000..0b21a150a --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/lookup.rs @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Per-key lookup changelog generation, matching Java's lookup merge wrapper. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use arrow_array::{Int8Array, RecordBatch, UInt64Array}; +use arrow_schema::SchemaRef; + +use crate::spec::{DataField, MergeEngine, RowKind, VALUE_KIND_FIELD_NAME}; +use crate::table::kv_file_reader::KeyValueReadConfig; +use crate::table::sort_merge::{lookup_rows, BufferedBatch, MergeFunction, MergeResult, MergeRow}; +use crate::{Error, Result}; + +struct ChangelogOptions { + deduplicate: bool, + ignore_update_before: bool, + ignore_delete: bool, + ignored_fields: HashSet, +} + +impl ChangelogOptions { + fn parse(options: &HashMap) -> Result { + let boolean = |key: &str| match options.get(key).map(String::as_str) { + None => Ok(false), + Some(value) if value.eq_ignore_ascii_case("true") => Ok(true), + Some(value) if value.eq_ignore_ascii_case("false") => Ok(false), + Some(value) => Err(Error::ConfigInvalid { + message: format!("{key} must be true or false, got {value}"), + }), + }; + Ok(Self { + deduplicate: boolean("changelog-producer.row-deduplicate")?, + ignore_update_before: boolean("changelog-producer.ignore-update-before")?, + ignore_delete: boolean("changelog-producer.ignore-delete")?, + ignored_fields: options + .get("changelog-producer.row-deduplicate-ignore-fields") + .map(|fields| { + fields + .split(',') + .map(str::trim) + .filter(|field| !field.is_empty()) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(), + }) + } +} + +pub(in crate::table::compact) fn validate_options(options: &HashMap) -> Result<()> { + ChangelogOptions::parse(options).map(|_| ()) +} + +struct LookupChangelogMergeFunction { + inner: Box, + first_row: bool, + options: ChangelogOptions, + value_columns: Vec, +} + +pub(super) fn new_merge_function( + config: &KeyValueReadConfig, + fields: &[DataField], +) -> Result> { + let options = ChangelogOptions::parse(&config.table_options)?; + Ok(Box::new(LookupChangelogMergeFunction { + inner: super::new_merge_function(config, fields)?, + first_row: config.merge_engine == MergeEngine::FirstRow, + value_columns: fields + .iter() + .enumerate() + .filter(|(_, field)| { + config + .table_fields + .iter() + .any(|value| value.id() == field.id()) + && !options.ignored_fields.contains(field.name()) + }) + .map(|(index, _)| index) + .collect(), + options, + })) +} + +fn source_row( + batch_idx: usize, + row_idx: usize, + batches: &[BufferedBatch], + columns: &[usize], + schema: &SchemaRef, +) -> Result { + let indices = UInt64Array::from(vec![row_idx as u64]); + let values = (0..schema.fields().len()) + .map(|index| { + arrow_select::take::take( + batches[batch_idx].column_for_output(index, columns), + &indices, + None, + ) + .map_err(|error| invalid(format!("Failed to materialize lookup value: {error}"))) + }) + .collect::>>()?; + RecordBatch::try_new(schema.clone(), values) + .map_err(|error| invalid(format!("Invalid lookup value: {error}"))) +} + +fn result_row( + result: &MergeResult, + batches: &[BufferedBatch], + columns: &[usize], + schema: &SchemaRef, +) -> Result> { + match result { + MergeResult::SourceRow { batch_idx, row_idx } => { + source_row(*batch_idx, *row_idx, batches, columns, schema).map(Some) + } + MergeResult::MaterializedRow(batch) | MergeResult::MaterializedDeleteRow(batch) => { + Ok(Some(batch.clone())) + } + MergeResult::Omit => Ok(None), + } +} + +fn live_row(batch: Option) -> Result> { + match batch { + Some(batch) => { + let kinds = batch + .column_by_name(VALUE_KIND_FIELD_NAME) + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| invalid("Lookup value must carry its row kind"))?; + Ok(RowKind::from_value(kinds.value(0))? + .is_add() + .then_some(batch)) + } + None => Ok(None), + } +} + +fn with_kind(batch: &RecordBatch, kind: RowKind) -> Result { + let mut columns = batch.columns().to_vec(); + let index = batch + .schema() + .index_of(VALUE_KIND_FIELD_NAME) + .map_err(|error| invalid(error.to_string()))?; + columns[index] = Arc::new(Int8Array::from(vec![kind as i8])); + RecordBatch::try_new(batch.schema(), columns).map_err(|error| invalid(error.to_string())) +} + +impl MergeFunction for LookupChangelogMergeFunction { + fn merge( + &self, + rows: &[MergeRow], + batches: &[BufferedBatch], + columns: &[usize], + schema: &SchemaRef, + ) -> Result { + self.merge_with_changelog(rows, batches, columns, schema) + .map(|(result, _)| result) + } + + fn merge_with_changelog( + &self, + rows: &[MergeRow], + batches: &[BufferedBatch], + columns: &[usize], + schema: &SchemaRef, + ) -> Result<(MergeResult, Option)> { + let (base, has_l0, selected) = lookup_rows(rows, batches)?; + let result = self.inner.merge(&selected, batches, columns, schema)?; + if !has_l0 { + return Ok((result, None)); + } + let before = live_row( + base.as_ref() + .map(|row| source_row(row.batch_idx, row.row_idx, batches, columns, schema)) + .transpose()?, + )?; + let after = live_row(result_row(&result, batches, columns, schema)?)?; + let mut changes = Vec::with_capacity(2); + match (&before, &after) { + (None, Some(after)) => changes.push(with_kind(after, RowKind::Insert)?), + (Some(before), None) if !self.first_row && !self.options.ignore_delete => { + changes.push(with_kind(before, RowKind::Delete)?) + } + (Some(before), Some(after)) if !self.first_row => { + let equal = self.options.deduplicate + && self.value_columns.iter().all(|index| { + before.column(*index).to_data() == after.column(*index).to_data() + }); + if !equal { + if !self.options.ignore_update_before { + changes.push(with_kind(before, RowKind::UpdateBefore)?); + } + changes.push(with_kind(after, RowKind::UpdateAfter)?); + } + } + _ => {} + } + let changelog = if changes.is_empty() { + None + } else { + Some( + arrow_select::concat::concat_batches(schema, &changes) + .map_err(|error| invalid(error.to_string()))?, + ) + }; + Ok((result, changelog)) + } +} + +fn invalid(message: impl Into) -> Error { + Error::DataInvalid { + message: message.into(), + source: None, + } +} diff --git a/crates/paimon/src/table/compact/primary_key/lookup_store/cache.rs b/crates/paimon/src/table/compact/primary_key/lookup_store/cache.rs new file mode 100644 index 000000000..2b4834f03 --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/lookup_store/cache.rs @@ -0,0 +1,935 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Lookup artifacts stored in FileIO local-cache. SST keys are Arrow sort keys; values are +//! physical row positions. IPC payload blocks retain the entire KV state. No +//! table files or manifests depend on these disposable, versioned local files. + +use crate::btree::{ + BTreeDataBlockCache, BTreeFileFooter, BTreeIndexMeta, BTreeIndexReader, BTreeIndexWriter, + BlockCompressionType, +}; +use crate::common::options::parse_memory_size; +use crate::io::cache::generated::{GeneratedFileRead, GeneratedFileWrite}; +use crate::io::cache::LocalCache; +use crate::io::{FileRead, FileWrite}; +use crate::table::ArrowRecordBatchStream; +use crate::{Error, Result}; +use arrow_array::{RecordBatch, UInt64Array}; +use arrow_ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow_row::{RowConverter, SortField}; +use bytes::Bytes; +use futures::TryStreamExt; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::io::Cursor; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::time::{Duration, SystemTime}; + +pub(super) fn invalid(message: impl ToString) -> Error { + Error::DataInvalid { + message: message.to_string(), + source: None, + } +} +fn exhausted(message: impl Into) -> Error { + Error::ResourceExhausted { + message: message.into(), + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LookupCacheMetrics { + pub file_hits: u64, + pub file_builds: u64, + pub file_evictions: u64, +} + +#[derive(Serialize, Deserialize)] +struct PayloadBlock { + first: u64, + rows: u64, + offset: u64, + length: u64, + crc: u32, +} +#[derive(Serialize, Deserialize)] +struct Manifest { + version: u32, + identity: String, + generation: String, + created_millis: u64, + index_size: u64, + data_size: u64, + first_key: Option>, + last_key: Option>, + blocks: Vec, +} +type Index = BTreeIndexReader Result>; +pub(super) struct LookupFile { + index: Index, + data: GeneratedFileRead, + manifest: Manifest, + memory: usize, + identity: String, + payloads: Arc>, +} +struct PayloadCache { + entries: lru::LruCache<(String, u64), Arc>, + bytes: usize, + limit: usize, +} +impl PayloadCache { + fn insert(&mut self, key: (String, u64), batch: Arc) -> Result<()> { + let bytes = batch.get_array_memory_size(); + if bytes > self.limit { + return Err(exhausted( + "Decoded lookup block exceeds lookup.cache-max-memory-size", + )); + } + while self.bytes.saturating_add(bytes) > self.limit { + if let Some((_, old)) = self.entries.pop_lru() { + self.bytes = self.bytes.saturating_sub(old.get_array_memory_size()); + } else { + break; + } + } + if let Some(old) = self.entries.put(key, batch) { + self.bytes = self.bytes.saturating_sub(old.get_array_memory_size()); + } + self.bytes += bytes; + Ok(()) + } +} +impl LookupFile { + pub(super) async fn lookup(&self, key: &[u8]) -> Result> { + let rows = self.index.query_equal(key).await.map_err(invalid)?; + if rows.is_empty() { + return Ok(None); + } + if rows.len() != 1 { + return Err(invalid("Lookup history contains duplicate primary keys")); + } + let position = rows.iter().next().unwrap(); + let block_index = self + .manifest + .blocks + .partition_point(|block| block.first <= position); + let block = block_index + .checked_sub(1) + .and_then(|index| self.manifest.blocks.get(index)) + .filter(|block| position < block.first + block.rows) + .ok_or_else(|| invalid("Lookup position is outside payload blocks"))?; + let payload_key = (self.identity.clone(), block.offset); + let cached = self + .payloads + .lock() + .unwrap() + .entries + .get(&payload_key) + .cloned(); + let batch = if let Some(batch) = cached { + batch + } else { + let bytes = self + .data + .read(block.offset..block.offset + block.length) + .await?; + if crc32fast::hash(&bytes) != block.crc { + return Err(invalid("Lookup payload checksum mismatch")); + } + let mut reader = StreamReader::try_new(Cursor::new(bytes), None).map_err(invalid)?; + let batch = Arc::new( + reader + .next() + .ok_or_else(|| invalid("Empty lookup payload block"))? + .map_err(invalid)?, + ); + if batch.num_rows() as u64 != block.rows { + return Err(invalid("Lookup payload row count changed")); + } + self.payloads + .lock() + .unwrap() + .insert(payload_key, batch.clone())?; + batch + }; + let indices = UInt64Array::from(vec![position - block.first]); + let row = + arrow_select::take::take_record_batch(batch.as_ref(), &indices).map_err(invalid)?; + Ok(Some((row, position))) + } +} + +struct Entry { + used: SystemTime, + loaded: Arc, +} +struct State { + entries: HashMap, + locks: HashMap>>, + invalid: HashSet, +} + +pub(in crate::table) struct LookupCache { + local: Arc, + max_memory: usize, + retention: Duration, + compression: BlockCompressionType, + blocks: Arc, + payloads: Arc>, + state: Mutex, + hits: AtomicU64, + builds: AtomicU64, + evictions: AtomicU64, +} +type Registry = HashMap<(usize, usize, u64, u8), Weak>; +static CACHES: OnceLock> = OnceLock::new(); + +fn artifact_path(identity: &str, generation: &str, name: &str) -> String { + format!("__paimon_generated__/lookup-v2/{identity}/{generation}/{name}") +} +fn descriptor_path(identity: &str) -> String { + format!("__paimon_generated__/lookup-v2/{identity}/head") +} +fn now_millis() -> u64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} +impl LookupCache { + pub(in crate::table) fn open( + local: Arc, + options: &HashMap, + ) -> Result> { + if options.contains_key("lookup.cache-max-disk-size") { + return Err(Error::ConfigInvalid { + message: + "Lookup Files share local-cache.max-size; remove lookup.cache-max-disk-size" + .into(), + }); + } + let max_memory = options + .get("lookup.cache-max-memory-size") + .map(|value| { + let value = parse_memory_size(value) + .map_err(|_| invalid("Invalid lookup.cache-max-memory-size"))?; + usize::try_from(value) + .ok() + .filter(|value| *value > 0) + .ok_or_else(|| invalid("lookup.cache-max-memory-size must be positive")) + }) + .transpose()? + .unwrap_or(256 * 1024 * 1024); + let retention = options + .get("lookup.cache-file-retention") + .map(|value| { + crate::spec::parse_duration_millis(value) + .ok_or_else(|| invalid("Invalid lookup.cache-file-retention")) + }) + .transpose()? + .unwrap_or(3_600_000); + let compression = match options + .get("lookup.cache-spill-compression") + .map(String::as_str) + .unwrap_or("zstd") + { + "zstd" => BlockCompressionType::Zstd, + "none" => BlockCompressionType::None, + "lz4" => BlockCompressionType::Lz4, + "lzo" => BlockCompressionType::Lzo, + value => { + return Err(invalid(format!( + "Unsupported lookup.cache-spill-compression: {value}" + ))) + } + }; + let key = ( + Arc::as_ptr(&local) as usize, + max_memory, + retention, + compression as u8, + ); + let mut caches = CACHES.get_or_init(Default::default).lock().unwrap(); + if let Some(cache) = caches.get(&key).and_then(Weak::upgrade) { + return Ok(cache); + } + caches.retain(|_, cache| cache.strong_count() > 0); + let cache = Arc::new(Self { + local, + max_memory, + retention: Duration::from_millis(retention), + compression, + blocks: Arc::new(BTreeDataBlockCache::new(max_memory / 4)), + payloads: Arc::new(Mutex::new(PayloadCache { + entries: lru::LruCache::unbounded(), + bytes: 0, + limit: max_memory / 2, + })), + state: Mutex::new(State { + entries: HashMap::new(), + locks: HashMap::new(), + invalid: HashSet::new(), + }), + hits: AtomicU64::new(0), + builds: AtomicU64::new(0), + evictions: AtomicU64::new(0), + }); + caches.insert(key, Arc::downgrade(&cache)); + Ok(cache) + } + pub(super) fn memory_limit(&self) -> usize { + self.max_memory + } + pub(in crate::table) fn metrics(&self) -> LookupCacheMetrics { + LookupCacheMetrics { + file_hits: self.hits.load(Ordering::Relaxed), + file_builds: self.builds.load(Ordering::Relaxed), + file_evictions: self.evictions.load(Ordering::Relaxed), + } + } + pub(super) fn invalidate(&self, id: &str) { + let mut state = self.state.lock().unwrap(); + state.entries.remove(id); + state.invalid.insert(id.into()); + self.evictions.fetch_add(1, Ordering::Relaxed); + // Do not invalidate FileIO paths: a shared-cache storage mutation fence + // is unnecessary for immutable artifacts. The rebuild uses new block + // paths and publishes its descriptor last. Old blocks age out by LRU. + } + async fn descriptor(&self, identity: &str) -> Result> { + let Some(head) = self + .local + .generated_get(&descriptor_path(identity), 0, 24) + .await + else { + return Ok(None); + }; + let generation = uuid::Uuid::from_slice(&head[..16]) + .map_err(invalid)? + .to_string(); + let length = u64::from_le_bytes(head[16..24].try_into().unwrap()); + if length > self.max_memory as u64 / 4 { + return Err(exhausted( + "Lookup descriptor exceeds lookup.cache-max-memory-size", + )); + } + let reader = GeneratedFileRead::new( + self.local.clone(), + artifact_path(identity, &generation, "descriptor"), + length, + ); + let bytes = reader.read(0..length).await?; + let manifest: Manifest = serde_json::from_slice(&bytes).map_err(invalid)?; + if manifest.identity != identity + || manifest.generation != generation + || manifest.version != 2 + { + return Err(invalid("Lookup cache identity/version mismatch")); + } + if Duration::from_millis(now_millis().saturating_sub(manifest.created_millis)) + >= self.retention + { + return Ok(None); + } + Ok(Some((manifest, bytes.len()))) + } + pub(super) async fn get( + &self, + identity: &str, + input: F, + keys: &[usize], + ) -> Result> + where + F: FnOnce() -> Result + Send, + { + let lock = { + let mut state = self.state.lock().unwrap(); + if state.locks.len() > 1024 { + state.locks.retain(|_, lock| lock.strong_count() > 0); + } + let lock = state + .locks + .get(identity) + .and_then(Weak::upgrade) + .unwrap_or_else(|| { + let lock = Arc::new(tokio::sync::Mutex::new(())); + state.locks.insert(identity.into(), Arc::downgrade(&lock)); + lock + }); + lock + }; + let _guard = lock.lock().await; + let force = { + let mut state = self.state.lock().unwrap(); + if let Some(entry) = state.entries.get_mut(identity) { + if entry.used.elapsed().unwrap_or_default() < self.retention + || Arc::strong_count(&entry.loaded) > 1 + { + entry.used = SystemTime::now(); + self.hits.fetch_add(1, Ordering::Relaxed); + return Ok(entry.loaded.clone()); + } + state.entries.remove(identity); + state.invalid.insert(identity.into()); + self.evictions.fetch_add(1, Ordering::Relaxed); + } + state.invalid.remove(identity) + }; + let recovered = if force { + None + } else { + self.descriptor(identity).await? + }; + let built = recovered.is_none(); + let (manifest, manifest_size) = match recovered { + Some(value) => value, + None => self.build(identity, input()?, keys).await?, + }; + let mut end = 0u64; + let mut first = 0u64; + for block in &manifest.blocks { + if block.offset != end + || block.first != first + || block.rows == 0 + || block.length > self.max_memory as u64 / 2 + { + return Err(invalid("Invalid lookup payload block metadata")); + } + end = end + .checked_add(block.length) + .filter(|end| *end <= manifest.data_size) + .ok_or_else(|| invalid("Lookup payload exceeds file length"))?; + first = first + .checked_add(block.rows) + .ok_or_else(|| invalid("Lookup row count overflow"))?; + } + if end != manifest.data_size { + return Err(invalid("Lookup payload coverage is incomplete")); + } + let reader = GeneratedFileRead::new( + self.local.clone(), + artifact_path(identity, &manifest.generation, "index"), + manifest.index_size, + ); + let footer = BTreeFileFooter::read_footer( + &reader + .read( + manifest + .index_size + .checked_sub(52) + .ok_or_else(|| invalid("Truncated lookup index"))? + ..manifest.index_size, + ) + .await?, + ) + .map_err(invalid)?; + let memory = manifest_size + .saturating_mul(3) + .saturating_add(footer.index_block_handle.size as usize * 2) + .saturating_add( + footer + .bloom_filter_handle + .as_ref() + .map_or(0, |filter| filter.size as usize), + ) + .saturating_add(4096); + if memory > self.max_memory / 4 { + return Err(exhausted( + "Lookup file metadata exceeds lookup.cache-max-memory-size", + )); + } + let meta = + BTreeIndexMeta::new(manifest.first_key.clone(), manifest.last_key.clone(), false); + let generation_key = format!("{identity}/{}", manifest.generation); + let index = BTreeIndexReader::open_with_data_block_cache( + Box::new(reader), + manifest.index_size, + &meta, + (|a: &[u8], b: &[u8]| Ok(a.cmp(b))) as fn(&[u8], &[u8]) -> Result, + self.blocks.clone(), + Arc::from(generation_key.as_str()), + ) + .await + .map_err(invalid)?; + let data = GeneratedFileRead::new( + self.local.clone(), + artifact_path(identity, &manifest.generation, "rows"), + manifest.data_size, + ); + let file = Arc::new(LookupFile { + index, + data, + manifest, + memory, + identity: generation_key, + payloads: self.payloads.clone(), + }); + let mut state = self.state.lock().unwrap(); + let mut retained = state + .entries + .values() + .map(|entry| entry.loaded.memory) + .sum::(); + if retained.saturating_add(memory) > self.max_memory / 4 { + state.entries.retain(|_, entry| { + if Arc::strong_count(&entry.loaded) == 1 { + retained = retained.saturating_sub(entry.loaded.memory); + false + } else { + true + } + }); + } + if retained.saturating_add(memory) > self.max_memory / 4 { + return Err(exhausted( + "Lookup memory cache is full and readers are pinned", + )); + } + state.entries.insert( + identity.into(), + Entry { + used: SystemTime::now(), + loaded: file.clone(), + }, + ); + if !built { + self.hits.fetch_add(1, Ordering::Relaxed); + } + Ok(file) + } + async fn build( + &self, + identity: &str, + mut input: ArrowRecordBatchStream, + keys: &[usize], + ) -> Result<(Manifest, usize)> { + if self.local.generated_block_size() > self.max_memory as u64 / 4 { + return Err(exhausted( + "local-cache.block-size exceeds the Lookup File working memory budget", + )); + } + let generation = uuid::Uuid::new_v4(); + let generation_id = generation.to_string(); + let (index_writer, index_size) = GeneratedFileWrite::new( + self.local.clone(), + artifact_path(identity, &generation_id, "index"), + ); + let mut index = BTreeIndexWriter::with_comparator_and_options( + Box::new(index_writer), + 32 * 1024, + self.compression, + 1, + true, + (|a: &[u8], b: &[u8]| Ok(a.cmp(b))) as fn(&[u8], &[u8]) -> Result, + ); + let (mut data, _) = GeneratedFileWrite::new( + self.local.clone(), + artifact_path(identity, &generation_id, "rows"), + ); + let mut blocks = Vec::new(); + let mut offset = 0u64; + let mut position = 0u64; + let mut converter = None; + let mut previous = None::>; + while let Some(batch) = input.try_next().await? { + if batch.num_rows() == 0 { + continue; + } + if converter.is_none() { + converter = Some( + RowConverter::new( + keys.iter() + .map(|key| { + SortField::new(batch.schema().field(*key).data_type().clone()) + }) + .collect(), + ) + .map_err(invalid)?, + ); + } + let columns = keys + .iter() + .map(|index| batch.column(*index).clone()) + .collect::>(); + let rows = converter + .as_mut() + .unwrap() + .convert_columns(&columns) + .map_err(invalid)?; + for row in 0..batch.num_rows() { + let key = rows.row(row); + if previous + .as_deref() + .is_some_and(|previous| previous > key.as_ref()) + { + return Err(invalid("Lookup source is not primary-key ordered")); + } + previous = Some(key.as_ref().to_vec()); + index + .write( + Some(key.as_ref()), + i64::try_from(position + row as u64).map_err(invalid)?, + ) + .await + .map_err(invalid)?; + } + let mut bytes = Vec::new(); + { + let mut writer = + StreamWriter::try_new(&mut bytes, batch.schema().as_ref()).map_err(invalid)?; + writer.write(&batch).map_err(invalid)?; + writer.finish().map_err(invalid)?; + } + if bytes.len() > self.max_memory / 2 { + return Err(exhausted("Lookup payload block exceeds lookup.cache-max-memory-size; reduce read.batch-size")); + } + if index + .retained_bytes() + .saturating_add(blocks.capacity() * std::mem::size_of::()) + > self.max_memory / 4 + { + return Err(exhausted( + "Lookup index construction exceeds lookup.cache-max-memory-size", + )); + } + let length = bytes.len() as u64; + let crc = crc32fast::hash(&bytes); + data.write(Bytes::from(bytes)).await?; + blocks.push(PayloadBlock { + first: position, + rows: batch.num_rows() as u64, + offset, + length, + crc, + }); + offset += length; + position += batch.num_rows() as u64; + } + data.close().await?; + let result = index.finish().await.map_err(invalid)?; + let index_size = index_size.load(Ordering::Relaxed); + let manifest = Manifest { + version: 2, + generation: generation_id.clone(), + created_millis: now_millis(), + identity: identity.into(), + index_size, + data_size: offset, + first_key: result.meta.first_key, + last_key: result.meta.last_key, + blocks, + }; + let bytes = serde_json::to_vec(&manifest).map_err(invalid)?; + let length = bytes.len(); + if length > self.max_memory / 4 { + return Err(exhausted( + "Lookup descriptor exceeds lookup.cache-max-memory-size", + )); + } + let (mut descriptor, _) = GeneratedFileWrite::new( + self.local.clone(), + artifact_path(identity, &generation_id, "descriptor"), + ); + descriptor.write(Bytes::from(bytes)).await?; + descriptor.close().await?; + // The fixed-size head is the only mutable pointer. A failed/cancelled + // build never publishes it; partial blocks stay in the bounded LRU. + let mut head = generation.as_bytes().to_vec(); + head.extend_from_slice(&(length as u64).to_le_bytes()); + self.local + .generated_put(&descriptor_path(identity), 0, Bytes::from(head)) + .await?; + self.builds.fetch_add(1, Ordering::Relaxed); + Ok((manifest, length)) + } +} +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{ArrayRef, Int32Array, Int64Array, Int8Array, StringArray}; + use futures::StreamExt; + + fn batch() -> RecordBatch { + RecordBatch::try_from_iter([ + ( + "_SEQUENCE_NUMBER", + Arc::new(Int64Array::from_iter_values(0..100)) as ArrayRef, + ), + ( + "_VALUE_KIND", + Arc::new(Int8Array::from(vec![0; 100])) as ArrayRef, + ), + ( + "id", + Arc::new(Int32Array::from_iter_values(0..100)) as ArrayRef, + ), + ( + "text", + Arc::new(StringArray::from(vec!["x".repeat(256); 100])) as ArrayRef, + ), + ]) + .unwrap() + } + fn input(count: Arc) -> ArrowRecordBatchStream { + futures::stream::once(async move { + count.fetch_add(1, Ordering::Relaxed); + Ok(batch()) + }) + .boxed() + } + fn key(value: i32) -> Vec { + let converter = + RowConverter::new(vec![SortField::new(arrow_schema::DataType::Int32)]).unwrap(); + converter + .convert_columns(&[Arc::new(Int32Array::from(vec![value]))]) + .unwrap() + .row(0) + .as_ref() + .to_vec() + } + + fn local(directory: Option<&std::path::Path>, shared: bool, capacity: &str) -> Arc { + let mut options = crate::Options::new(); + options.set("local-cache.enabled", "true"); + options.set("local-cache.max-size", capacity); + options.set("local-cache.block-size", "8kb"); + if let Some(directory) = directory { + options.set("local-cache.dir", directory.to_str().unwrap()); + } + if shared { + options.set("local-cache.shared", "true"); + } + crate::io::cache::create_local_cache(&options) + .unwrap() + .unwrap() + } + + #[tokio::test] + async fn concurrent_misses_share_one_build_on_memory_disk_and_shared_backends() { + for backend in ["memory", "disk", "shared"] { + let directory = tempfile::tempdir().unwrap(); + let local = local( + (backend != "memory").then_some(directory.path()), + backend == "shared", + "4mb", + ); + let cache = LookupCache::open(local, &HashMap::new()).unwrap(); + let count = Arc::new(AtomicU64::new(0)); + let id = "a".repeat(64); + let (left, right) = tokio::join!( + cache.get(&id, || Ok(input(count.clone())), &[2]), + cache.get(&id, || Ok(input(count.clone())), &[2]) + ); + let left = left.unwrap(); + let right = right.unwrap(); + assert!(Arc::ptr_eq(&left, &right)); + let (row, position) = left.lookup(&key(42)).await.unwrap().unwrap(); + assert_eq!(position, 42); + assert_eq!( + row.column(3) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "x".repeat(256) + ); + assert!(right.lookup(&key(999)).await.unwrap().is_none()); + let warm = cache + .get( + &id, + || panic!("cache hits must not construct a source reader"), + &[2], + ) + .await + .unwrap(); + assert!(Arc::ptr_eq(&left, &warm)); + assert_eq!(count.load(Ordering::Relaxed), 1); + assert_eq!(cache.metrics().file_builds, 1); + } + } + + #[tokio::test] + async fn lookup_artifacts_compete_with_cached_reads_in_the_same_capacity() { + struct Source; + #[async_trait::async_trait] + impl FileRead for Source { + async fn read(&self, range: std::ops::Range) -> Result { + Ok(Bytes::from(vec![1; (range.end - range.start) as usize])) + } + } + for disk in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let local = local(disk.then_some(directory.path()), false, "128kb"); + let id = "b".repeat(64); + let count = Arc::new(AtomicU64::new(0)); + let cache = LookupCache::open(local.clone(), &HashMap::new()).unwrap(); + drop( + cache + .get(&id, || Ok(input(count.clone())), &[2]) + .await + .unwrap(), + ); + drop(cache); + // More ordinary data blocks than the entire cache can retain must + // evict generated blocks too; there is no separate lookup quota. + let reader = crate::io::cache::CachedFileReader::new( + Arc::new(Source), + "s3://bucket/data.parquet", + 256 * 1024, + local.clone(), + ); + reader.read(0..256 * 1024).await.unwrap(); + assert!(local + .generated_get(&descriptor_path(&id), 0, 24) + .await + .is_none()); + let reopened = LookupCache::open(local, &HashMap::new()).unwrap(); + assert!(reopened + .get(&id, || Ok(input(count.clone())), &[2]) + .await + .unwrap() + .lookup(&key(42)) + .await + .unwrap() + .is_some()); + assert_eq!(count.load(Ordering::Relaxed), 2); + } + } + + #[tokio::test] + async fn completed_artifacts_reopen_after_the_disk_cache_manager_is_recreated() { + let directory = tempfile::tempdir().unwrap(); + let id = "c".repeat(64); + let count = Arc::new(AtomicU64::new(0)); + { + let cache = + LookupCache::open(local(Some(directory.path()), false, "1mb"), &HashMap::new()) + .unwrap(); + drop( + cache + .get(&id, || Ok(input(count.clone())), &[2]) + .await + .unwrap(), + ); + } + let reopened = + LookupCache::open(local(Some(directory.path()), false, "1mb"), &HashMap::new()) + .unwrap(); + let file = reopened + .get( + &id, + || panic!("complete disk artifacts must survive manager recreation"), + &[2], + ) + .await + .unwrap(); + assert_eq!(file.lookup(&key(42)).await.unwrap().unwrap().1, 42); + assert_eq!(reopened.metrics().file_builds, 0); + assert_eq!(count.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn cancellation_and_capacity_failure_never_publish_a_descriptor() { + let id = "d".repeat(64); + let tiny = LookupCache::open(local(None, false, "1b"), &HashMap::new()).unwrap(); + assert!(matches!( + tiny.get(&id, || Ok(input(Arc::new(AtomicU64::new(0)))), &[2]) + .await + .err() + .unwrap(), + Error::ResourceExhausted { .. } + )); + assert!(tiny + .local + .generated_get(&descriptor_path(&id), 0, 24) + .await + .is_none()); + let cache = LookupCache::open(local(None, false, "128kb"), &HashMap::new()).unwrap(); + let (started, wait) = tokio::sync::oneshot::channel(); + let stream = futures::stream::once(async { Ok(batch()) }) + .chain(futures::stream::once(async move { + let _ = started.send(()); + futures::future::pending::>().await + })) + .boxed(); + let cloned = cache.clone(); + let task_id = id.clone(); + let task = tokio::spawn(async move { cloned.get(&task_id, || Ok(stream), &[2]).await }); + wait.await.unwrap(); + task.abort(); + assert!(task.await.err().unwrap().is_cancelled()); + assert!(cache + .local + .generated_get(&descriptor_path(&id), 0, 24) + .await + .is_none()); + assert!(cache + .get(&id, || Ok(input(Arc::new(AtomicU64::new(0)))), &[2]) + .await + .unwrap() + .lookup(&key(42)) + .await + .unwrap() + .is_some()); + assert_eq!(cache.metrics().file_builds, 1); + } + + #[tokio::test] + async fn decoded_memory_and_expired_pinned_reader_limits_are_enforced() { + let local = local(None, false, "1mb"); + let id = "e".repeat(64); + let tiny = LookupCache::open( + local.clone(), + &HashMap::from([("lookup.cache-max-memory-size".into(), "1b".into())]), + ) + .unwrap(); + assert!(matches!( + tiny.get(&id, || Ok(input(Arc::new(AtomicU64::new(0)))), &[2]) + .await + .err() + .unwrap(), + Error::ResourceExhausted { .. } + )); + let cache = LookupCache::open( + local, + &HashMap::from([("lookup.cache-file-retention".into(), "0ms".into())]), + ) + .unwrap(); + let count = Arc::new(AtomicU64::new(0)); + let pinned = cache + .get(&id, || Ok(input(count.clone())), &[2]) + .await + .unwrap(); + let reused = cache + .get(&id, || panic!("expired pinned reader stays usable"), &[2]) + .await + .unwrap(); + assert!(Arc::ptr_eq(&pinned, &reused)); + drop(reused); + drop(pinned); + drop( + cache + .get(&id, || Ok(input(count.clone())), &[2]) + .await + .unwrap(), + ); + assert_eq!(count.load(Ordering::Relaxed), 2); + } +} diff --git a/crates/paimon/src/table/compact/primary_key/lookup_store/mod.rs b/crates/paimon/src/table/compact/primary_key/lookup_store/mod.rs new file mode 100644 index 000000000..1bf03f22f --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/lookup_store/mod.rs @@ -0,0 +1,420 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Snapshot-bound level lookup with disposable local SST/IPC files. +mod cache; +pub(in crate::table) use cache::LookupCache; +pub use cache::LookupCacheMetrics; + +use crate::deletion_vector::{DeletionVector, DeletionVectorFactory}; +use crate::io::FileRead; +use crate::resource::ResourceContext; +use crate::spec::{ChangelogProducer, DataFileMeta, MergeEngine}; +use crate::table::data_file_reader::DataFileReader; +use crate::table::kv_file_reader::key_value_fields_extractor; +use crate::table::merge_tree_split_generator::KeyComparator; +use crate::table::sort_merge::{LookupProvider, FILE_LEVEL_METADATA_KEY}; +use crate::table::{DataSplit, Table}; +use crate::{Error, Result}; +use arrow_array::RecordBatch; +use arrow_row::{RowConverter, SortField}; +use cache::invalid; +pub(super) type LookupDeletions = Arc>>; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(in crate::table) struct LookupDeletion { + pub partition: Vec, + pub bucket: i32, + pub file_name: String, + #[serde(with = "serde_bytes")] + pub bitmap: Vec, +} + +pub(in crate::table) fn needed(table: &Table) -> Result { + let options = table.schema().core_options(); + Ok(options.deletion_vectors_enabled() + || options.try_changelog_producer()? == ChangelogProducer::Lookup + || options.merge_engine()? == MergeEngine::FirstRow) +} +struct HistoryFile { + meta: DataFileMeta, + identity: tokio::sync::OnceCell, + vector: Option, +} +struct LevelLookup { + table: Table, + split: DataSplit, + levels: BTreeMap>, + cache: Arc, + keys: Vec, + converter: Mutex, + key_fields: Vec, + comparator: Option, + resources: Option, + vectors: tokio::sync::Mutex, + schema_identity: Vec, + deletions: LookupDeletions, +} + +struct VectorCache { + entries: lru::LruCache< + String, + ( + Arc, + Option, + usize, + ), + >, + bytes: usize, + limit: usize, +} + +pub(super) async fn provider( + table: &Table, + split: &DataSplit, + files: &[DataFileMeta], + cache: Arc, + resources: Option, +) -> Result<(Arc, LookupDeletions)> { + let fields = super::read_type(table); + let arrow_schema = crate::arrow::build_read_arrow_schema(&fields)?; + let keys = table + .schema() + .trimmed_primary_keys() + .iter() + .map(|name| { + fields + .iter() + .position(|field| field.name() == name) + .unwrap() + }) + .collect::>(); + let key_fields = keys + .iter() + .map(|key| fields[*key].clone()) + .collect::>(); + let converter = RowConverter::new( + keys.iter() + .map(|key| SortField::new(arrow_schema.field(*key).data_type().clone())) + .collect(), + ) + .map_err(invalid)?; + let comparator = KeyComparator::from_table_schema(table.schema()); + let all_split = DataSplit::builder() + .with_snapshot(split.snapshot_id()) + .with_partition(split.partition().clone()) + .with_bucket(split.bucket()) + .with_total_buckets(split.total_buckets()) + .with_bucket_path(split.bucket_path().into()) + .with_data_files(files.to_vec()) + .build()?; + let all_split = super::deletion_vectors::with_deletion_files( + table, + &all_split, + table + .travel_snapshot() + .ok_or_else(|| invalid("Lookup needs its captured snapshot"))?, + ) + .await?; + let properties = table + .schema() + .options() + .iter() + .filter(|(key, _)| { + !key.starts_with("scan.") && key.as_str() != "incremental-between-timestamp" + }) + .collect::>(); + let schema_identity = serde_json::to_vec(&( + table.schema().id(), + table.schema().fields(), + table.schema().primary_keys(), + table.schema().partition_keys(), + properties, + )) + .map_err(invalid)?; + let mut levels = BTreeMap::>::new(); + for (index, file) in files.iter().enumerate() { + levels.entry(file.level).or_default().push(HistoryFile { + meta: file.clone(), + identity: tokio::sync::OnceCell::new(), + vector: all_split + .data_deletion_files() + .and_then(|files| files.get(index)) + .cloned() + .flatten(), + }); + } + for files in levels.values_mut() { + files.sort_by(|a, b| { + comparator + .as_ref() + .and_then(|cmp| cmp.compare(&a.meta.min_key, &b.meta.min_key)) + .unwrap_or(std::cmp::Ordering::Equal) + }); + for pair in files.windows(2) { + if comparator + .as_ref() + .and_then(|cmp| cmp.compare(&pair[0].meta.max_key, &pair[1].meta.min_key)) + != Some(std::cmp::Ordering::Less) + { + return Err(invalid( + "Lookup level has overlapping or unverifiable key ranges", + )); + } + } + } + let deletions = Arc::new(Mutex::new(HashMap::new())); + let vector_limit = cache.memory_limit() / 4; + Ok(( + Arc::new(LevelLookup { + table: table.clone(), + split: all_split, + levels, + cache, + keys, + converter: Mutex::new(converter), + key_fields, + comparator, + resources, + vectors: tokio::sync::Mutex::new(VectorCache { + entries: lru::LruCache::unbounded(), + bytes: 0, + limit: vector_limit, + }), + schema_identity, + deletions: deletions.clone(), + }), + deletions, + )) +} + +impl LevelLookup { + fn reader(&self) -> Result { + let schema = self.table.schema(); + let budget = crate::table::table_read::configured_parquet_read_budget(&self.table)?; + let budget = match &self.resources { + Some(resources) => budget.with_resources(resources.clone()), + None => budget.without_prefetch(), + }; + Ok(DataFileReader::new( + self.table.file_io().clone(), + self.table.schema_manager().clone(), + schema.id(), + schema.fields().to_vec(), + key_value_fields_extractor(schema.fields(), &schema.trimmed_primary_keys()), + super::read_type(&self.table), + vec![], + ) + .with_batch_size(Some(schema.core_options().read_batch_size()?)) + .with_table_options(schema.options().clone()) + .with_parquet_read_budget(Some(Arc::new(budget.without_prefetch())))) + } +} +#[async_trait::async_trait] +impl LookupProvider for LevelLookup { + async fn lookup(&self, key: &[u8]) -> Result> { + let binary_key = { + let converter = self.converter.lock().unwrap(); + let parser = converter.parser(); + let columns = converter + .convert_rows(std::iter::once(parser.parse(key))) + .map_err(invalid)?; + let schema = crate::arrow::build_read_arrow_schema(&self.key_fields)?; + let batch = RecordBatch::try_new(schema, columns).map_err(invalid)?; + let mut row = crate::spec::BinaryRowBuilder::new(self.key_fields.len() as i32); + for (index, field) in self.key_fields.iter().enumerate() { + if let Some(datum) = + crate::spec::extract_datum_from_arrow(&batch, 0, index, field.data_type())? + { + row.write_datum(index, &datum, field.data_type()); + } else { + row.set_null_at(index); + } + } + row.build_serialized() + }; + for files in self.levels.values() { + let index = files.partition_point(|file| { + self.comparator + .as_ref() + .and_then(|cmp| cmp.compare(&file.meta.max_key, &binary_key)) + == Some(std::cmp::Ordering::Less) + }); + let Some(file) = files.get(index) else { + continue; + }; + if self + .comparator + .as_ref() + .and_then(|cmp| cmp.compare(&file.meta.min_key, &binary_key)) + == Some(std::cmp::Ordering::Greater) + { + continue; + } + let identity = file + .identity + .get_or_try_init(|| async { + let reader = self + .table + .file_io() + .new_input(&self.split.data_file_path(&file.meta))? + .reader_with_size(file.meta.file_size as u64) + .await?; + let namespace = reader + .cache_key() + .map(str::to_string) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let mut physical = file.meta.clone(); + physical.level = 0; + let mut digest = Sha256::new(); + digest.update(b"paimon-lookup-v1"); + digest.update(namespace.as_bytes()); + digest.update(serde_json::to_vec(&physical).map_err(invalid)?); + digest.update(&self.schema_identity); + Ok::<_, Error>(hex::encode(digest.finalize())) + }) + .await?; + let mut result = None; + for attempt in 0..2 { + let cached = self + .cache + .get( + identity, + || { + self.reader()?.read_single_file_stream( + &self.split, + file.meta.clone(), + None, + None, + None, + ) + }, + &self.keys, + ) + .await; + match cached { + Ok(cached) => match cached.lookup(key).await { + Ok(row) => { + result = row; + break; + } + Err(error @ Error::ResourceExhausted { .. }) => return Err(error), + Err(error) => { + drop(cached); + self.cache.invalidate(identity); + if attempt == 1 { + return Err(error); + } + } + }, + Err(error @ Error::ResourceExhausted { .. }) => return Err(error), + Err(error) => { + self.cache.invalidate(identity); + if attempt == 1 { + return Err(error); + } + } + } + } + if let Some((batch, position)) = result { + if let Some(deletion_file) = &file.vector { + let mut vectors = self.vectors.lock().await; + if !vectors.entries.contains(&file.meta.file_name) { + if usize::try_from(deletion_file.length()) + .map_or(true, |length| length > vectors.limit) + { + return Err(Error::ResourceExhausted { + message: "Lookup deletion vector exceeds its cache memory budget" + .into(), + }); + } + let vector = Arc::new( + DeletionVectorFactory::read(self.table.file_io(), deletion_file) + .await?, + ); + let size = vector.retained_bytes(); + if size > vectors.limit { + return Err(Error::ResourceExhausted { + message: "Lookup deletion vector exceeds its cache memory budget" + .into(), + }); + } + while vectors.bytes.saturating_add(size) > vectors.limit { + if let Some((_, (_, reservation, bytes))) = vectors.entries.pop_lru() { + vectors.bytes = vectors.bytes.saturating_sub(bytes); + drop(reservation); + } else { + break; + } + } + let reservation = self + .resources + .as_ref() + .map(|resources| -> Result<_> { + let mut reservation = resources.reservation(); + reservation.try_grow(size)?; + Ok(reservation) + }) + .transpose()?; + vectors + .entries + .put(file.meta.file_name.clone(), (vector, reservation, size)); + vectors.bytes += size; + } + if vectors + .entries + .get(&file.meta.file_name) + .unwrap() + .0 + .is_deleted(position) + { + continue; + } + } + if self + .table + .schema() + .core_options() + .deletion_vectors_enabled() + { + let position = u32::try_from(position) + .map_err(|_| invalid("PK deletion vector position exceeds u32::MAX"))?; + self.deletions + .lock() + .unwrap() + .entry(file.meta.file_name.clone()) + .or_default() + .insert(position); + } + let mut metadata = batch.schema().metadata().clone(); + metadata.insert(FILE_LEVEL_METADATA_KEY.into(), file.meta.level.to_string()); + metadata.insert("paimon.compact.lookup-base".into(), "true".into()); + return RecordBatch::try_new( + Arc::new(batch.schema().as_ref().clone().with_metadata(metadata)), + batch.columns().to_vec(), + ) + .map(Some) + .map_err(invalid); + } + } + Ok(None) + } +} diff --git a/crates/paimon/src/table/compact/primary_key/planner.rs b/crates/paimon/src/table/compact/primary_key/planner.rs new file mode 100644 index 000000000..ae0fc6d0d --- /dev/null +++ b/crates/paimon/src/table/compact/primary_key/planner.rs @@ -0,0 +1,277 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Java UniversalCompaction run selection. Explicit compact drains L0 using +//! forcePickL0; positive-level maintenance uses size/run triggers, never a full +//! bucket override. + +use crate::spec::DataFileMeta; +use crate::{Error, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(in crate::table) struct PrimaryKeyTask { + pub output_level: i32, + pub lookup_files: Vec, +} + +fn number( + options: &std::collections::HashMap, + key: &str, + default: i32, + minimum: i32, +) -> Result { + let value = options + .get(key) + .map(|value| value.parse::()) + .transpose() + .map_err(|_| Error::ConfigInvalid { + message: format!("Invalid {key}"), + })? + .unwrap_or(default); + if value < minimum { + return Err(Error::ConfigInvalid { + message: format!("{key} must be at least {minimum}"), + }); + } + Ok(value) +} + +pub(in crate::table::compact) fn select( + files: Vec, + properties: &std::collections::HashMap, +) -> Result, PrimaryKeyTask)>> { + let trigger = number(properties, "num-sorted-run.compaction-trigger", 5, 2)?; + let configured_levels = number(properties, "num-levels", trigger.saturating_add(1), 2)?; + let levels = configured_levels.max( + files + .iter() + .map(|file| file.level.saturating_add(1)) + .max() + .unwrap_or(2), + ); + let ratio = number(properties, "compaction.size-ratio", 1, 0)?; + let amplification = number( + properties, + "compaction.max-size-amplification-percent", + 200, + 0, + )?; + let mut l0 = files + .iter() + .filter(|file| file.level == 0) + .cloned() + .collect::>(); + l0.sort_by(|a, b| { + b.max_sequence_number + .cmp(&a.max_sequence_number) + .then_with(|| a.file_name.cmp(&b.file_name)) + }); + let mut positive = BTreeMap::>::new(); + for file in files.into_iter().filter(|file| file.level > 0) { + positive.entry(file.level).or_default().push(file); + } + let l0_count = l0.len(); + let mut runs = l0 + .into_iter() + .map(|file| (0, vec![file])) + .collect::>(); + for (level, mut files) in positive { + files.sort_by(|a, b| a.file_name.cmp(&b.file_name)); + runs.push((level, files)); + } + if runs.is_empty() { + return Ok(None); + } + let sizes = runs + .iter() + .map(|(_, files)| { + files + .iter() + .map(|file| file.file_size as u128) + .sum::() + }) + .collect::>(); + let mut count; + if l0_count > 0 { + // Java prepare-commit forces L0 progress for lookup; the explicit Rust + // compact API requests the same bounded incremental work for all modes. + count = l0_count; + } else { + if runs.len() < trigger as usize { + return Ok(None); + } + let additional = sizes[..sizes.len() - 1].iter().sum::(); + if additional * 100 > amplification as u128 * sizes[sizes.len() - 1] { + count = runs.len(); + } else { + count = 1; + } + } + let mut size = sizes[..count].iter().sum::(); + while count < runs.len() { + let next = &runs[count]; + if size * (100 + ratio as u128) < sizes[count] * 100 + && (count <= 1 && l0_count == 0 || next.0 > 1) + { + break; + } + size += sizes[count]; + count += 1; + } + if l0_count == 0 && count == 1 { + if runs.len() <= trigger as usize { + return Ok(None); + } + count = runs.len() - trigger as usize + 1; + size = sizes[..count].iter().sum(); + while count < runs.len() + && (size * (100 + ratio as u128) >= sizes[count] * 100 || runs[count].0 <= 1) + { + size += sizes[count]; + count += 1; + } + } + let output_level = if count == runs.len() { + levels - 1 + } else { + (runs[count].0 - 1).max(1) + }; + let selected = runs[..count] + .iter() + .flat_map(|(_, files)| files.clone()) + .collect::>(); + if selected.len() == 1 && selected[0].level == output_level { + return Ok(None); + } + let lookup_files = runs[count..] + .iter() + .flat_map(|(_, files)| files.clone()) + .collect(); + Ok(Some(( + selected, + PrimaryKeyTask { + output_level, + lookup_files, + }, + ))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::stats::BinaryTableStats; + use std::collections::HashMap; + fn file(name: &str, level: i32, size: i64) -> DataFileMeta { + DataFileMeta { + file_name: name.into(), + file_size: size, + row_count: 1, + min_key: vec![], + max_key: vec![], + key_stats: BinaryTableStats::empty(), + value_stats: BinaryTableStats::empty(), + min_sequence_number: 0, + max_sequence_number: 1, + schema_id: 0, + level, + extra_files: vec![], + creation_time: None, + delete_row_count: Some(0), + embedded_index: None, + file_source: Some(0), + value_stats_cols: None, + external_path: None, + first_row_id: None, + write_cols: None, + column_max_sequence_numbers: None, + } + } + #[test] + fn java_force_l0_stops_before_a_large_higher_level_and_outputs_below_it() { + let (selected, task) = select( + vec![file("new", 0, 1), file("anchor", 5, 1000)], + &HashMap::new(), + ) + .unwrap() + .unwrap(); + assert_eq!( + selected + .iter() + .map(|file| file.file_name.as_str()) + .collect::>(), + vec!["new"] + ); + assert_eq!(task.output_level, 4); + assert_eq!(task.lookup_files[0].file_name, "anchor"); + let (selected, task) = select( + vec![ + file("new", 0, 1), + file("l1", 1, 100), + file("anchor", 5, 10000), + ], + &HashMap::new(), + ) + .unwrap() + .unwrap(); + assert_eq!( + selected.len(), + 2, + "Java always consumes L1 rather than outputting L0" + ); + assert_eq!(task.output_level, 4); + } + #[test] + fn positive_run_count_and_size_amplification_trigger_incremental_maintenance() { + let options = HashMap::from([ + ("num-sorted-run.compaction-trigger".into(), "2".into()), + ("num-levels".into(), "6".into()), + ]); + assert!( + select(vec![file("small", 3, 1), file("anchor", 5, 1000)], &options) + .unwrap() + .is_none() + ); + let (selected, task) = select( + vec![ + file("small", 3, 1), + file("medium", 4, 10), + file("anchor", 5, 1000), + ], + &options, + ) + .unwrap() + .unwrap(); + assert_eq!(selected.len(), 2); + assert_eq!(task.output_level, 4); + assert_eq!(task.lookup_files.len(), 1); + let (selected, task) = select( + vec![ + file("large", 1, 1000), + file("other", 3, 1000), + file("old", 5, 100), + ], + &options, + ) + .unwrap() + .unwrap(); + assert_eq!(selected.len(), 3); + assert_eq!(task.output_level, 5); + assert!(task.lookup_files.is_empty()); + } +} diff --git a/crates/paimon/src/table/data_evolution_writer.rs b/crates/paimon/src/table/data_evolution_writer.rs index e14e17816..15864cbf1 100644 --- a/crates/paimon/src/table/data_evolution_writer.rs +++ b/crates/paimon/src/table/data_evolution_writer.rs @@ -542,6 +542,15 @@ pub struct DataEvolutionDeleteWriter { } impl DataEvolutionDeleteWriter { + /// Shared bucket-local index I/O used by incremental PK compaction. Row-ID + /// DELETE operations still enter through the validated public constructor. + pub(in crate::table) fn for_primary_key_compaction(table: &Table) -> Self { + Self { + table: table.clone(), + row_ids: vec![], + } + } + pub fn new(table: &Table) -> Result { let schema = table.schema(); let core_options = CoreOptions::new(schema.options()); diff --git a/crates/paimon/src/table/incremental_scan.rs b/crates/paimon/src/table/incremental_scan.rs index d06b4c03b..48aa38002 100644 --- a/crates/paimon/src/table/incremental_scan.rs +++ b/crates/paimon/src/table/incremental_scan.rs @@ -63,8 +63,8 @@ pub enum IncrementalScanMode { /// Read existing changelog manifest files in the range. /// /// Skips [`OVERWRITE`](crate::spec::CommitKind::OVERWRITE) snapshots and - /// snapshots without a `changelog_manifest_list`. Does not generate - /// changelogs (no compact/lookup producer path). + /// snapshots without a `changelog_manifest_list`. Reads persisted input or + /// compaction-generated changelogs; it does not generate events during reads. Changelog, /// Resolve to [`Delta`](Self::Delta) when `changelog-producer=none`, /// otherwise to [`Changelog`](Self::Changelog). diff --git a/crates/paimon/src/table/kv_data_file_writer.rs b/crates/paimon/src/table/kv_data_file_writer.rs index 0943e3a7a..b96ebed5f 100644 --- a/crates/paimon/src/table/kv_data_file_writer.rs +++ b/crates/paimon/src/table/kv_data_file_writer.rs @@ -507,8 +507,12 @@ impl<'a> KeyValueRollingWriter<'a> { Ok(()) } + pub(super) async fn finish(&mut self) -> Result<()> { + self.close_current().await + } + pub(super) async fn prepare_commit(&mut self) -> Result> { - self.close_current().await?; + self.finish().await?; Ok(std::mem::take(&mut self.files)) } diff --git a/crates/paimon/src/table/kv_file_reader.rs b/crates/paimon/src/table/kv_file_reader.rs index e97d3a1bf..ea6440cf9 100644 --- a/crates/paimon/src/table/kv_file_reader.rs +++ b/crates/paimon/src/table/kv_file_reader.rs @@ -27,8 +27,9 @@ use super::data_file_reader::{file_index_selection_to_local_ranges, DataFileReader}; use super::sort_merge::{ - AggregateMergeFunction, DeduplicateMergeFunction, FirstRowMergeFunction, MergeFunction, - PartialUpdateMergeFunction, SortMergeReaderBuilder, + AggregateMergeFunction, DeduplicateMergeFunction, FirstRowMergeFunction, LookupMergeFunction, + MergeFunction, PartialUpdateMergeFunction, SortMergeReaderBuilder, + CHANGELOG_BATCH_METADATA_KEY, FILE_LEVEL_METADATA_KEY, }; use crate::arrow::format::MosaicPrefetchOptions; use crate::arrow::shredding::variant::assemble_variant_extraction_array; @@ -62,6 +63,8 @@ pub(crate) struct KeyValueFileReader { /// of a key survives); they are enforced by the post-merge residual /// filter using the full `config.predicates` instead. pushdown_predicates: Vec, + include_file_levels: bool, + lookup: Option>, #[cfg(test)] input_batch_sizes: Option>>>, } @@ -350,15 +353,32 @@ impl KeyValueFileReader { &config.table_fields, &config.primary_keys, ); + let include_file_levels = config + .table_options + .get("changelog-producer") + .is_some_and(|value| value.eq_ignore_ascii_case("lookup")); Self { file_io, config, + include_file_levels, + lookup: None, pushdown_predicates, #[cfg(test)] input_batch_sizes: None, } } + pub(super) fn with_lookup( + mut self, + lookup: Option>, + ) -> Self { + if lookup.is_some() { + self.include_file_levels = true; + } + self.lookup = lookup; + self + } + #[cfg(test)] fn with_input_batch_sizes( mut self, @@ -371,6 +391,25 @@ impl KeyValueFileReader { pub(super) fn new_merge_function( config: &KeyValueReadConfig, merge_output_fields: &[DataField], + ) -> crate::Result> { + let inner = Self::new_unwrapped_merge_function(config, merge_output_fields)?; + Ok( + if config.merge_engine != MergeEngine::FirstRow + && config + .table_options + .get("changelog-producer") + .is_some_and(|value| value.eq_ignore_ascii_case("lookup")) + { + Box::new(LookupMergeFunction(inner)) + } else { + inner + }, + ) + } + + pub(super) fn new_unwrapped_merge_function( + config: &KeyValueReadConfig, + merge_output_fields: &[DataField], ) -> crate::Result> { match config.merge_engine { MergeEngine::Deduplicate => Ok(Box::new(DeduplicateMergeFunction)), @@ -651,6 +690,8 @@ impl KeyValueFileReader { data_splits.into_iter().map(|split| vec![split]).collect() }; let file_io = self.file_io; + let include_file_levels = self.include_file_levels; + let lookup = self.lookup; let config = self.config; let table_schema_id = config.table_schema_id; let pushdown_predicates = self.pushdown_predicates; @@ -786,6 +827,7 @@ impl KeyValueFileReader { )), None => None, }; + let file_level = file_meta.level; let mut file_stream = reader.read_single_file_stream_with_schema_and_local_ranges( split.as_ref(), file_meta, @@ -795,7 +837,14 @@ impl KeyValueFileReader { local_row_ranges, )?; while let Some(batch) = file_stream.next().await { - yield batch?; + let batch = batch?; + if include_file_levels { + let mut metadata = batch.schema().metadata().clone(); + metadata.insert(FILE_LEVEL_METADATA_KEY.into(), file_level.to_string()); + yield RecordBatch::try_new( + Arc::new(batch.schema().as_ref().clone().with_metadata(metadata)), batch.columns().to_vec(), + ).map_err(|error| Error::DataInvalid { message: format!("Invalid lookup input batch: {error}"), source: Some(Box::new(error)) })?; + } else { yield batch; } } } }); @@ -828,6 +877,7 @@ impl KeyValueFileReader { merge_output_schema.clone(), merge_function(&config, &merge_output_fields)?, ) + .with_lookup(lookup.clone()) .with_user_sequence_descending( !CoreOptions::new(&config.table_options) .sequence_field_sort_order_is_ascending(), @@ -892,7 +942,11 @@ impl KeyValueFileReader { let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); let reordered = RecordBatch::try_new_with_options( - output_schema.clone(), + if batch.schema().metadata().contains_key(CHANGELOG_BATCH_METADATA_KEY) { + let mut metadata = output_schema.metadata().clone(); + metadata.insert(CHANGELOG_BATCH_METADATA_KEY.into(), "true".into()); + Arc::new(output_schema.as_ref().clone().with_metadata(metadata)) + } else { output_schema.clone() }, columns, &options, ) diff --git a/crates/paimon/src/table/merge_tree_split_generator.rs b/crates/paimon/src/table/merge_tree_split_generator.rs index 6eee84028..acd472623 100644 --- a/crates/paimon/src/table/merge_tree_split_generator.rs +++ b/crates/paimon/src/table/merge_tree_split_generator.rs @@ -72,6 +72,10 @@ impl KeyComparator { Some(Self::new(key_types)) } + pub(crate) fn compare(&self, left: &[u8], right: &[u8]) -> Option { + Some(compare_decoded(&self.decode(left)?, &self.decode(right)?)) + } + /// Decode a serialized min/max key. Returns `None` when the key is empty, /// malformed, or contains NaN, letting callers degrade to the safe "treat /// everything as overlapping" path instead of failing the scan. NaN must diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index a56f6e022..211c17ec4 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -208,7 +208,7 @@ pub use commit_callback::{CommitCallback, CommitCallbackContext}; pub use commit_message::{CommitMessage, COMMIT_MESSAGE_SERIALIZER_VERSION}; pub use compact::{ CompactBuilder, CompactExecutor, CompactMetrics, CompactPlan, CompactPlanner, CompactResult, - CompactTask, + CompactTask, LookupCacheMetrics, }; pub use consumer_manager::ConsumerManager; pub use cow_writer::{CopyOnWriteMergeWriter, FileInfo}; diff --git a/crates/paimon/src/table/sort_merge.rs b/crates/paimon/src/table/sort_merge.rs index b0cdec9fa..462dae52d 100644 --- a/crates/paimon/src/table/sort_merge.rs +++ b/crates/paimon/src/table/sort_merge.rs @@ -40,7 +40,7 @@ use futures::StreamExt; use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; // --------------------------------------------------------------------------- // MergeFunction @@ -57,7 +57,7 @@ pub(crate) enum BufferedBatch { } impl BufferedBatch { - fn column_for_output<'a>( + pub(super) fn column_for_output<'a>( &'a self, output_col_idx: usize, source_output_col_indices: &[usize], @@ -72,6 +72,7 @@ impl BufferedBatch { } /// A row reference as an index into the batch buffer. +#[derive(Clone)] pub(crate) struct MergeRow { /// Index into the shared batch buffer. pub batch_idx: usize, @@ -150,6 +151,87 @@ pub(crate) trait MergeFunction: Send + Sync { source_output_col_indices: &[usize], output_schema: &SchemaRef, ) -> crate::Result; + + /// Emit at most one key's changelog alongside its merged state. The reader + /// drains it before advancing to the next key, keeping memory bounded even + /// when all data rows are deleted. + fn merge_with_changelog( + &self, + rows: &[MergeRow], + batches: &[BufferedBatch], + columns: &[usize], + schema: &SchemaRef, + ) -> crate::Result<(MergeResult, Option)> { + Ok((self.merge(rows, batches, columns, schema)?, None)) + } +} + +#[async_trait::async_trait] +pub(crate) trait LookupProvider: Send + Sync { + /// Return a single physical KV row from an unselected higher-level file. + async fn lookup(&self, key: &[u8]) -> crate::Result>; +} + +/// Internal batch metadata used only by lookup compaction. Source levels +/// survive physical reads; changelog batches are routed to a separate writer. +pub(super) const FILE_LEVEL_METADATA_KEY: &str = "paimon.compact.file-level"; +pub(super) const CHANGELOG_BATCH_METADATA_KEY: &str = "paimon.compact.changelog"; + +/// Lookup states at positive levels are already materialized. The lowest +/// positive level is the latest state; older levels must not be reduced again. +/// Mirrors Java LookupMergeFunction#pickHighLevel/getResult. +pub(super) fn lookup_rows( + rows: &[MergeRow], + batches: &[BufferedBatch], +) -> crate::Result<(Option, bool, Vec)> { + let mut base = None; + let mut base_level = i32::MAX; + let mut levels = Vec::with_capacity(rows.len()); + for (index, row) in rows.iter().enumerate() { + let BufferedBatch::Source(batch) = &batches[row.batch_idx] else { + return Err(Error::UnexpectedError { + message: "Lookup input must be a source batch".into(), + source: None, + }); + }; + let level = batch + .schema() + .metadata() + .get(FILE_LEVEL_METADATA_KEY) + .and_then(|level| level.parse::().ok()) + .ok_or_else(|| Error::DataInvalid { + message: "Lookup input is missing its file level".into(), + source: None, + })?; + if level > 0 && (base.is_none() || level < base_level) { + base = Some(index); + base_level = level; + } + levels.push(level); + } + let has_l0 = levels.contains(&0); + let selected = rows + .iter() + .enumerate() + .filter(|(index, _)| levels[*index] <= 0 || base == Some(*index)) + .map(|(_, row)| row.clone()) + .collect(); + Ok((base.map(|index| rows[index].clone()), has_l0, selected)) +} + +pub(super) struct LookupMergeFunction(pub Box); + +impl MergeFunction for LookupMergeFunction { + fn merge( + &self, + rows: &[MergeRow], + batches: &[BufferedBatch], + columns: &[usize], + schema: &SchemaRef, + ) -> crate::Result { + let (_, _, selected) = lookup_rows(rows, batches)?; + self.0.merge(&selected, batches, columns, schema) + } } /// Deduplicate merge: keeps the row with the highest sequence. @@ -1222,6 +1304,7 @@ pub(crate) struct SortMergeReaderBuilder { output_schema: SchemaRef, merge_function: Box, batch_size: usize, + lookup: Option>, } impl SortMergeReaderBuilder { @@ -1249,9 +1332,15 @@ impl SortMergeReaderBuilder { output_schema, merge_function, batch_size: 1024, + lookup: None, } } + pub(crate) fn with_lookup(mut self, lookup: Option>) -> Self { + self.lookup = lookup; + self + } + pub(crate) fn with_batch_size(mut self, batch_size: usize) -> Self { self.batch_size = batch_size; self @@ -1309,6 +1398,7 @@ impl SortMergeReaderBuilder { self.output_schema, self.merge_function, self.batch_size, + self.lookup, ) } } @@ -1382,6 +1472,7 @@ fn sort_merge_stream( output_schema: SchemaRef, merge_function: Box, batch_size: usize, + lookup: Option>, ) -> crate::Result { let num_streams = streams.len(); if num_streams == 0 { @@ -1518,12 +1609,30 @@ fn sort_merge_stream( tree.update(|a, b| compare_cursors(&cursors, a, b).then_with(|| a.cmp(&b)).is_gt()); } - match merge_function.merge( - &same_key_rows, - &batch_buffer, - &source_output_col_indices, - &output_schema, - )? { + if let Some(lookup) = &lookup { + let has_base = same_key_rows.iter().any(|row| match &batch_buffer[row.batch_idx] { + BufferedBatch::Source(batch) => batch.schema().metadata().get(FILE_LEVEL_METADATA_KEY) + .and_then(|level| level.parse::().ok()).is_some_and(|level| level > 0), + _ => false, + }); + if !has_base { + if let Some(batch) = lookup.lookup(&winner_key).await? { + if batch.num_rows() != 1 { Err(Error::DataInvalid { message: "Lookup must return one KV row".into(), source: None })?; } + let keys = convert_batch_keys(&batch, &key_indices, &mut row_converter)?; + if keys.row(0).as_ref() != winner_key.as_slice() { Err(Error::DataInvalid { message: "Lookup returned a different primary key".into(), source: None })?; } + let user = convert_batch_user_sequences(&batch, &user_sequence_indices, user_sequence_converter.as_mut())?; + let sequence_number = batch.column(seq_index).as_any().downcast_ref::().ok_or_else(|| Error::DataInvalid { message: "Invalid lookup sequence".into(), source: None })?.value(0); + let value_kind = batch.column(value_kind_index).as_any().downcast_ref::().ok_or_else(|| Error::DataInvalid { message: "Invalid lookup row kind".into(), source: None })?.value(0); + let batch_idx = batch_buffer.len(); batch_buffer.push(BufferedBatch::Source(batch)); + same_key_rows.insert(0, MergeRow { batch_idx, row_idx: 0, sequence_number, value_kind, user_sequence: user.map(|rows| rows.row(0).owned()) }); + } + } + } + + let (merged, changelog) = merge_function.merge_with_changelog( + &same_key_rows, &batch_buffer, &source_output_col_indices, &output_schema, + )?; + match merged { MergeResult::SourceRow { batch_idx, row_idx } => { output_indices.push((batch_idx, row_idx)); } @@ -1550,6 +1659,20 @@ fn sort_merge_stream( MergeResult::Omit | MergeResult::MaterializedDeleteRow(_) => {} } + if let Some(changelog) = changelog { + if changelog.schema().as_ref() != output_schema.as_ref() { + Err(Error::UnexpectedError { message: "Changelog merge result schema does not match merge output schema".into(), source: None })?; + } + let mut metadata = output_schema.metadata().clone(); + metadata.insert(CHANGELOG_BATCH_METADATA_KEY.into(), "true".into()); + yield RecordBatch::try_new( + Arc::new(output_schema.as_ref().clone().with_metadata(metadata)), changelog.columns().to_vec(), + ).map_err(|error| Error::DataInvalid { message: format!("Invalid changelog batch: {error}"), source: Some(Box::new(error)) })?; + } + if output_indices.is_empty() { + compact_batch_buffer(&mut batch_buffer, &mut stream_batch_idx, &cursors); + } + // Yield a batch when we've accumulated enough rows. if output_indices.len() >= batch_size { let batch = build_output_interleave( diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index 6769098b5..ebd0817bc 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -442,6 +442,9 @@ impl TableCommit { .try_commit( CommitEntriesPlan::Compact { entries: prepared.entries, + changelog_entries: prepared.changelog_entries, + pk_tasks: prepared.pk_tasks, + lookup_deletions: prepared.lookup_deletions, schema_id: prepared.schema_id, read_snapshot: prepared.read_snapshot, planned_snapshot_id: prepared.snapshot_id, @@ -1252,8 +1255,14 @@ impl TableCommit { new_index_entries, .. } => (new_entries.as_slice(), &[][..], new_index_entries), - CommitEntriesPlan::Compact { entries, .. } => { - return self.check_recovery_entries(entries, &[], &[]).await; + CommitEntriesPlan::Compact { + entries, + changelog_entries, + .. + } => { + return self + .check_recovery_entries(entries, changelog_entries, &[]) + .await; } }; self.check_recovery_entries(entries, changelog_entries, index_entries) @@ -1382,6 +1391,27 @@ impl TableCommit { } } + if !self.data_evolution_enabled + && self.deletion_vectors_enabled + && resolved.kind == CommitKind::COMPACT + { + if let Some(snapshot) = latest_snapshot { + if let Some(indexes) = + super::compact::primary_key::deletion_vectors::rewrite_indexes( + &self.table, + snapshot, + &resolved.entries, + &resolved.lookup_deletions, + created_files, + ) + .await? + { + resolved.index_manifest_changed = true; + resolved.new_index_manifest_entries = Some(indexes); + } + } + } + // Row tracking let mut next_row_id: Option = None; if self.row_tracking_enabled { @@ -1900,9 +1930,12 @@ impl TableCommit { match plan { CommitEntriesPlan::Compact { entries, + changelog_entries, schema_id, read_snapshot, planned_snapshot_id, + pk_tasks, + lookup_deletions, } => { if latest_snapshot .as_ref() @@ -1928,6 +1961,7 @@ impl TableCommit { None => None, }; validate_file_entries(entries.iter())?; + validate_file_entries(changelog_entries.iter())?; // This runs after duplicate-commit filtering: an uncertain commit // can be retried even after the captured read snapshot expires. if let Some(captured) = read_snapshot { @@ -1944,6 +1978,9 @@ impl TableCommit { None, ) .await?; + for task in pk_tasks.iter() { + task.validate_inputs(&captured_files)?; + } let deletes = entries .iter() .filter(|e| *e.kind() == FileKind::Delete) @@ -1974,8 +2011,7 @@ impl TableCommit { .iter() .filter(|e| { buckets.contains(&(e.partition(), e.bucket())) - && (!self.data_evolution_enabled - || deleted_ids.contains(&commit_file_identifier(e))) + && deleted_ids.contains(&commit_file_identifier(e)) }) .map(|entry| (commit_file_identifier(entry), entry)) .collect::>(); @@ -1993,7 +2029,7 @@ impl TableCommit { message: if self.data_evolution_enabled { "Data-evolution compaction inputs must match the captured row-ID groups" } else { - "PK compaction must include the complete captured bucket" + "PK compaction must match its captured incremental inputs" }.into(), source: None, }); @@ -2042,14 +2078,66 @@ impl TableCommit { base_data_files.as_deref().unwrap_or_default(), entries, )?; + // Pin the whole positive-level lookup view, including negative + // lookups. Concurrent L0 appends do not change historical state. + if super::compact::primary_key::lookup_store::needed(&self.table)? { + for task in pk_tasks.iter() { + let expected = task + .files() + .iter() + .chain(task.primary_key_files()) + .filter(|file| file.level > 0) + .collect::>(); + let current = base_data_files + .as_deref() + .unwrap_or_default() + .iter() + .filter(|entry| { + entry.partition() == task.partition() + && entry.bucket() == task.bucket() + && entry.file().level > 0 + }) + .collect::>(); + if expected.len() != current.len() + || expected + .iter() + .any(|file| !current.iter().any(|entry| entry.file() == *file)) + { + return Err(crate::Error::DataInvalid { + message: "Lookup compaction history changed; create a new plan" + .into(), + source: None, + }); + } + } + } + if self.deletion_vectors_enabled && !self.data_evolution_enabled { + let captured = + read_snapshot + .as_deref() + .ok_or_else(|| crate::Error::DataInvalid { + message: + "PK deletion-vector compaction requires its captured snapshot" + .into(), + source: None, + })?; + super::compact::primary_key::deletion_vectors::validate_dependencies( + &self.table, + captured, + latest_snapshot.as_ref().expect("validated latest snapshot"), + pk_tasks, + ) + .await?; + } Ok(ResolvedCommit { entries: entries.clone(), - changelog_entries: vec![], + changelog_entries: changelog_entries.clone(), kind: CommitKind::COMPACT, index_manifest_name, index_manifest_changed: false, new_index_manifest_entries: None, base_data_files, + lookup_deletions: lookup_deletions.clone(), }) } CommitEntriesPlan::Direct { @@ -2123,6 +2211,7 @@ impl TableCommit { index_manifest_changed, new_index_manifest_entries, base_data_files, + lookup_deletions: vec![], }) } CommitEntriesPlan::Overwrite { .. } => { @@ -2195,6 +2284,7 @@ impl TableCommit { index_manifest_changed, new_index_manifest_entries, base_data_files, + lookup_deletions: vec![], }) } } @@ -3868,12 +3958,15 @@ fn build_partition_stats_row(datums: &[Option], data_types: &[DataType]) /// Plan for resolving commit entries. enum CommitEntriesPlan { - /// Replace unchanged data files without publishing logical row changes. + /// Replace unchanged data files and atomically publish generated changelogs. Compact { entries: Vec, + changelog_entries: Vec, schema_id: i64, read_snapshot: Option>, planned_snapshot_id: i64, + pk_tasks: Vec, + lookup_deletions: Vec, }, /// Caller-provided entries. May contain `FileKind::Delete` entries from CoW /// rewrites, in which case `resolve_commit` auto-promotes to `CommitKind::OVERWRITE`. @@ -3929,6 +4022,7 @@ fn direct_commit_kind( /// Fully resolved commit ready for writing. struct ResolvedCommit { + lookup_deletions: Vec, entries: Vec, changelog_entries: Vec, kind: CommitKind, diff --git a/crates/paimon/tests/pk_compact/deletion_vectors.rs b/crates/paimon/tests/pk_compact/deletion_vectors.rs new file mode 100644 index 000000000..5bf263dc5 --- /dev/null +++ b/crates/paimon/tests/pk_compact/deletion_vectors.rs @@ -0,0 +1,511 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use bytes::Bytes; +use indexmap::IndexMap; +use paimon::spec::{DeletionVectorMeta, IndexFileMeta, IndexManifest}; +use paimon::table::CommitMessage; + +async fn dv_table(options: &[(&str, &str)]) -> Table { + let mut configured = vec![("deletion-vectors.enabled", "true")]; + configured.extend_from_slice(options); + table(&configured).await +} + +async fn compact(table: &Table) -> paimon::table::CompactMetrics { + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + builder + .new_commit() + .unwrap() + .commit_compact(execute(table, &plan).await) + .await + .unwrap() +} + +async fn indexes(table: &Table) -> Vec { + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + match snapshot.index_manifest() { + Some(name) => IndexManifest::read( + table.file_io(), + &table.snapshot_manager().manifest_path(name), + ) + .await + .unwrap(), + None => vec![], + } +} + +// Write the same v1 index envelope as Java's BitmapDeletionVector. The selected +// file can be L0 or already compacted; positions are local physical row offsets. +pub(super) async fn delete_positions(table: &Table, file_name: &str, positions: &[u32]) { + let plan = table + .new_read_builder() + .new_scan() + .with_scan_all_files() + .plan() + .await + .unwrap(); + let split = plan + .splits() + .iter() + .find(|split| { + split + .data_files() + .iter() + .any(|file| file.file_name == file_name) + }) + .unwrap(); + let mut message = CommitMessage::new( + split.partition().to_serialized_bytes(), + split.bucket(), + vec![], + ); + message + .new_index_files + .push(write_vector_index(table, split, &[(file_name, positions)]).await); + for entry in indexes(table).await { + if entry.partition == message.partition + && entry.bucket == message.bucket + && entry + .index_file + .deletion_vectors_ranges + .as_ref() + .is_some_and(|ranges| ranges.contains_key(file_name)) + { + message.deleted_index_files.push(entry.index_file); + } + } + table + .new_write_builder() + .new_commit() + .commit(vec![message]) + .await + .unwrap(); +} + +async fn write_vector_index( + table: &Table, + split: &paimon::table::DataSplit, + deletions: &[(&str, &[u32])], +) -> IndexFileMeta { + let mut bytes = vec![1]; + let mut ranges = IndexMap::new(); + for (file_name, positions) in deletions { + let bitmap: roaring::RoaringBitmap = positions.iter().copied().collect(); + let mut payload = 1581511376u32.to_be_bytes().to_vec(); + bitmap.serialize_into(&mut payload).unwrap(); + let meta = DeletionVectorMeta { + offset: bytes.len() as i32, + length: payload.len() as i32, + cardinality: Some(positions.len() as i64), + }; + bytes.extend_from_slice(&meta.length.to_be_bytes()); + bytes.extend_from_slice(&payload); + bytes.extend_from_slice(&crc32fast::hash(&payload).to_be_bytes()); + ranges.insert(file_name.to_string(), meta); + } + let name = format!("index-{}-1", uuid::Uuid::new_v4()); + let path = if table.schema().core_options().index_file_in_data_file_dir() { + format!("{}/{}", split.bucket_path(), name) + } else { + format!("{}/index/{}", table.location(), name) + }; + table + .file_io() + .mkdirs(path.rsplit_once('/').unwrap().0) + .await + .unwrap(); + let size = bytes.len() as i64; + table + .file_io() + .new_output(&path) + .unwrap() + .write(Bytes::from(bytes)) + .await + .unwrap(); + IndexFileMeta { + index_type: "DELETION_VECTORS".into(), + file_name: name, + file_size: size, + row_count: ranges.len() as i64, + deletion_vectors_ranges: Some(ranges), + external_path: None, + global_index_meta: None, + } +} + +#[tokio::test] +async fn dv_compact_materializes_all_engines_and_continues_merging() { + for engine in ["deduplicate", "partial-update", "aggregation"] { + for bucket in ["1", "-1"] { + let mut options = vec![ + ("merge-engine", engine), + ("bucket", bucket), + ("read.batch-size", "1"), + ("target-file-row-num", "1"), + ]; + if engine == "aggregation" { + options.push(("fields.value.aggregate-function", "sum")); + } + let table = dv_table(&options).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + write_batch(&table, &make_batch(vec![1, 3], vec![5, 30])).await; + assert!(rows(&table).await.is_empty()); + let metrics = compact(&table).await; + assert_eq!(metrics.rows_written, 3, "{engine}/{bucket}"); + let value = match engine { + "aggregation" => 15, + _ => 5, + }; + assert_eq!(rows(&table).await, vec![(1, value), (2, 20), (3, 30)]); + assert!(table + .new_compact_builder() + .new_planner() + .unwrap() + .plan() + .await + .unwrap() + .tasks() + .is_empty()); + write_batch(&table, &make_batch(vec![1], vec![7])).await; + compact(&table).await; + let value = match engine { + "aggregation" => 22, + _ => 7, + }; + assert_eq!(rows(&table).await, vec![(1, value), (2, 20), (3, 30)]); + } + } +} + +#[tokio::test] +async fn dv_compact_drops_retracts_and_allows_an_empty_bucket() { + let table = dv_table(&[("changelog-producer", "input")]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + write_batch( + &table, + &make_batch_with_kinds(vec![1, 2, 3], vec![10, 20, 30], vec![3, 1, 3]), + ) + .await; + let before = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + let metrics = compact(&table).await; + assert_eq!(metrics.rows_written, 0); + assert_eq!(metrics.files_added, 0); + assert!(rows(&table).await.is_empty()); + let after = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + assert_eq!(after.commit_kind(), &CommitKind::COMPACT); + assert_eq!(before.changelog_record_count(), Some(3)); + assert_eq!(after.changelog_record_count(), None); +} + +#[tokio::test] +async fn dv_compact_filters_physical_positions_before_aggregation_and_preserves_history() { + for in_bucket in ["true", "false"] { + let table = dv_table(&[ + ("merge-engine", "aggregation"), + ("fields.value.aggregate-function", "sum"), + ("index-file-in-data-file-dir", in_bucket), + ("data-file.path-directory", "custom"), + ]) + .await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let plan = table + .new_read_builder() + .new_scan() + .with_scan_all_files() + .plan() + .await + .unwrap(); + let old_name = plan.splits()[0].data_files()[0].file_name.clone(); + delete_positions(&table, &old_name, &[0]).await; + write_batch(&table, &make_batch(vec![1], vec![15])).await; + let before = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + compact(&table).await; + assert_eq!(rows(&table).await, vec![(1, 15), (2, 20)]); + assert!(indexes(&table).await.is_empty()); + let history = table + .copy_with_pinned_snapshot(Some(&before)) + .copy_with_options(std::collections::HashMap::from([( + "deletion-vectors.merge-on-read".into(), + "true".into(), + )])); + assert_eq!(rows(&history).await, vec![(1, 15), (2, 20)]); + } +} + +#[tokio::test] +async fn dv_compact_reclaims_a_positive_level_run_when_selected_by_incremental_compaction() { + let table = dv_table(&[]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + compact(&table).await; + let plan = table.new_read_builder().new_scan().plan().await.unwrap(); + delete_positions(&table, &plan.splits()[0].data_files()[0].file_name, &[0, 1]).await; + let unchanged = indexes(&table).await; + let plan = table + .new_compact_builder() + .new_planner() + .unwrap() + .plan() + .await + .unwrap(); + assert!( + plan.tasks().is_empty(), + "no full-compaction path for one positive-level run" + ); + assert_eq!(indexes(&table).await, unchanged); + write_batch(&table, &make_batch(vec![3], vec![30])).await; + let metrics = compact(&table).await; + assert_eq!(metrics.rows_written, 1); + assert!(indexes(&table).await.is_empty()); + assert_eq!(rows(&table).await, vec![(3, 30)]); +} + +#[tokio::test] +async fn dv_compact_rejects_concurrent_deletions_on_inputs_but_accepts_l0_appends() { + let table = dv_table(&[]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + delete_positions(&table, &plan.tasks()[0].files()[0].file_name, &[0]).await; + let error = builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap_err(); + assert!(error.to_string().contains("deletion vector"), "{error}"); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + write_batch(&table, &make_batch(vec![1], vec![99])).await; + let committer = builder.new_commit().unwrap(); + let metrics = committer.commit_compact(results.clone()).await.unwrap(); + assert_eq!(metrics.rows_written, 1); + assert_eq!(rows(&table).await, vec![(2, 20)]); + assert_eq!(committer.commit_compact(results).await.unwrap(), metrics); + compact(&table).await; + assert_eq!(rows(&table).await, vec![(1, 99), (2, 20)]); +} + +#[tokio::test] +async fn dv_compact_obeys_user_sequence_and_output_format() { + for format in ["parquet", "orc", "avro", "row"] { + let table = dv_table(&[ + ("file.format", format), + ("file.compression", "none"), + ("sequence.field", "value"), + ("read.batch-size", "1"), + ("write-buffer-size", "1b"), + ]) + .await; + write_batch(&table, &make_batch(vec![1, 2, 3], vec![100, 20, 30])).await; + write_batch( + &table, + &make_batch_with_kinds(vec![1, 2, 3], vec![10, 21, 29], vec![0, 3, 3]), + ) + .await; + compact(&table).await; + assert_eq!(rows(&table).await, vec![(1, 100), (3, 30)], "{format}"); + let plan = table.new_read_builder().new_scan().plan().await.unwrap(); + assert!(plan + .splits() + .iter() + .flat_map(|split| split.data_files()) + .all(|file| file.level > 0 && file.delete_row_count == Some(0))); + } +} + +#[tokio::test] +async fn dv_compact_subsets_keep_other_vectors_and_latest_dynamic_hash_assignments() { + let table = dv_table(&[("bucket", "-1"), ("dynamic-bucket.target-row-num", "1")]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let builder = table.new_compact_builder(); + let initial = builder.new_planner().unwrap().plan().await.unwrap(); + assert_eq!(initial.tasks().len(), 2); + for task in initial.tasks() { + delete_positions(&table, &task.files()[0].file_name, &[0]).await; + } + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + let other_bucket = plan.tasks()[1].bucket(); + let other_dv = indexes(&table) + .await + .into_iter() + .find(|entry| { + entry.bucket == other_bucket && entry.index_file.index_type == "DELETION_VECTORS" + }) + .unwrap(); + write_batch(&table, &make_batch(vec![3], vec![30])).await; + let expected_hashes = indexes(&table) + .await + .into_iter() + .filter(|entry| entry.index_file.index_type == "HASH") + .collect::>(); + let committer = builder.new_commit().unwrap(); + committer + .commit_compact(vec![results[0].clone()]) + .await + .unwrap(); + let remaining = indexes(&table).await; + assert_eq!( + remaining + .iter() + .filter(|entry| entry.index_file.index_type == "HASH") + .cloned() + .collect::>(), + expected_hashes + ); + assert_eq!( + remaining + .iter() + .filter(|entry| entry.index_file.index_type == "DELETION_VECTORS") + .collect::>(), + vec![&other_dv] + ); + committer + .commit_compact(vec![results[1].clone()]) + .await + .unwrap(); + assert!(rows(&table).await.is_empty()); + compact(&table).await; + assert_eq!(rows(&table).await, vec![(3, 30)]); +} + +#[tokio::test] +async fn dv_compact_keeps_concurrent_file_ranges_in_a_shared_immutable_index() { + let table = dv_table(&[]).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let scan = table + .new_read_builder() + .new_scan() + .with_scan_all_files() + .plan() + .await + .unwrap(); + let split = &scan.splits()[0]; + let old_file = &split.data_files()[0].file_name; + let write = table.new_write_builder(); + let mut writer = write.new_write().unwrap(); + writer + .write_arrow_batch(&make_batch(vec![3], vec![30])) + .await + .unwrap(); + let pending = writer.prepare_commit().await.unwrap(); + let new_file = pending[0].new_files[0].file_name.clone(); + let shared = write_vector_index(&table, split, &[(old_file, &[0]), (&new_file, &[0])]).await; + let mut original = shared.clone(); + original + .deletion_vectors_ranges + .as_mut() + .unwrap() + .shift_remove(&new_file); + original.row_count = 1; + let mut deletion = CommitMessage::new( + split.partition().to_serialized_bytes(), + split.bucket(), + vec![], + ); + deletion.new_index_files.push(original.clone()); + write + .new_commit() + .commit(vec![deletion.clone()]) + .await + .unwrap(); + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + // Add a range without changing the captured file's vector or its immutable + // bytes. The appended file's vector must survive the planned replacement. + write.new_commit().commit(pending).await.unwrap(); + writer.close().await; + deletion.new_index_files = vec![shared.clone()]; + deletion.deleted_index_files = vec![original]; + write.new_commit().commit(vec![deletion]).await.unwrap(); + builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); + let remaining = indexes(&table).await; + assert_eq!(remaining.len(), 1); + let index = &remaining[0].index_file; + assert_eq!(index.row_count, 1); + let ranges = index.deletion_vectors_ranges.as_ref().unwrap(); + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[&new_file].cardinality, Some(1)); + let visible = table.copy_with_options(std::collections::HashMap::from([( + "deletion-vectors.merge-on-read".into(), + "true".into(), + )])); + assert_eq!(rows(&visible).await, vec![(2, 20)]); + assert_eq!(rows(&table).await, vec![(2, 20)]); + compact(&table).await; + assert_eq!(rows(&table).await, vec![(2, 20)]); + assert!(indexes(&table).await.is_empty()); +} + +#[tokio::test] +async fn dv_compact_rejects_serialized_outputs_with_unknown_or_nonzero_delete_counts() { + let table = dv_table(&[]).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + for count in [serde_json::Value::Null, serde_json::json!(1)] { + let mut invalid = serde_json::to_value(&results).unwrap(); + *invalid[0]["files"][0].get_mut("_DELETE_ROW_COUNT").unwrap() = count; + let invalid = serde_json::from_value(invalid).unwrap(); + assert!(builder + .new_commit() + .unwrap() + .commit_compact(invalid) + .await + .unwrap_err() + .to_string() + .contains("output file")); + } + builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); + assert_eq!(rows(&table).await, vec![(1, 10)]); +} diff --git a/crates/paimon/tests/pk_compact/incremental_lookup.rs b/crates/paimon/tests/pk_compact/incremental_lookup.rs new file mode 100644 index 000000000..1006d0e0d --- /dev/null +++ b/crates/paimon/tests/pk_compact/incremental_lookup.rs @@ -0,0 +1,384 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; + +#[tokio::test] +async fn lookup_compaction_requires_an_enabled_catalog_cache() { + for options in [ + vec![("deletion-vectors.enabled", "true")], + vec![("changelog-producer", "lookup")], + vec![("merge-engine", "first-row")], + ] { + let mut configured = options; + // A table option cannot fabricate a cache on an uncached FileIO. + configured.push(("local-cache.enabled", "true")); + let table = + table_with_cache(pk_schema(&configured), &[("local-cache.enabled", "false")]).await; + let error = table.new_compact_builder().new_executor().err().unwrap(); + assert!(matches!(error, paimon::Error::ConfigInvalid { .. })); + assert!(error.to_string().contains("local-cache.enabled=true")); + assert_eq!( + table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(), + None + ); + } + let ordinary = table_with_cache(pk_schema(&[]), &[("local-cache.enabled", "false")]).await; + assert!(ordinary.new_compact_builder().new_executor().is_ok()); +} + +async fn incremental_table(options: &[(&str, &str)]) -> Table { + incremental_table_with_cache(options, &[]).await +} + +async fn incremental_table_with_cache( + options: &[(&str, &str)], + cache_options: &[(&str, &str)], +) -> Table { + let mut configured = vec![ + ("bucket", "1"), + ("num-levels", "6"), + ("num-sorted-run.compaction-trigger", "5"), + ("deletion-vectors.enabled", "true"), + ("changelog-producer", "lookup"), + ]; + configured.extend_from_slice(options); + table_with_cache(pk_schema(&configured), cache_options).await +} + +async fn compact( + table: &Table, + executor: &paimon::table::CompactExecutor, +) -> (CompactPlan, Vec) { + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + let results: Vec = + serde_json::from_value(serde_json::to_value(results).unwrap()).unwrap(); + builder + .new_commit() + .unwrap() + .commit_compact(results.clone()) + .await + .unwrap(); + (plan, results) +} + +#[tokio::test] +async fn incremental_lookup_reuses_local_files_and_keeps_the_large_history_file() { + for engine in ["deduplicate", "partial-update", "aggregation"] { + for dv in ["true", "false"] { + let mut options = vec![ + ("merge-engine", engine), + ("deletion-vectors.enabled", dv), + ("read.batch-size", "32"), + ]; + if engine == "aggregation" { + options.push(("fields.value.aggregate-function", "sum")); + } + let table = incremental_table(&options).await; + let builder = table.new_compact_builder(); + let executor = builder.new_executor().unwrap(); + write_batch( + &table, + &make_batch((0..1000).collect(), (0..1000).collect()), + ) + .await; + let (_, first) = compact(&table, &executor).await; + assert_eq!(first[0].files().len(), 1); + let anchor = first[0].files()[0].clone(); + assert_eq!(anchor.level, 5); + for key in [42, 43] { + write_batch(&table, &make_batch(vec![key], vec![10000 + key])).await; + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + assert!(plan + .tasks() + .iter() + .flat_map(|task| task.files()) + .all(|file| file.file_name != anchor.file_name)); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); + let scan = table + .new_read_builder() + .new_scan() + .with_scan_all_files() + .plan() + .await + .unwrap(); + assert!(scan + .splits() + .iter() + .flat_map(|split| split.data_files()) + .any(|file| file == &anchor)); + } + let actual = rows(&table).await; + assert_eq!(actual.len(), 1000, "{engine}/{dv}"); + for key in [42, 43] { + assert_eq!( + actual[key as usize], + ( + key, + if engine == "aggregation" { + 10000 + 2 * key + } else { + 10000 + key + } + ) + ); + } + let metrics = executor.lookup_cache_metrics(); + assert_eq!( + metrics.file_builds, 1, + "{engine}/{dv}: only the anchor was built" + ); + assert!( + metrics.file_hits >= 1, + "{engine}/{dv}: subsequent lookups must use local files" + ); + let earliest = table.snapshot_manager().get_snapshot(2).await.unwrap(); + assert_eq!( + rows(&table.copy_with_pinned_snapshot(Some(&earliest))).await, + (0..1000).map(|key| (key, key)).collect::>() + ); + } + } +} + +#[tokio::test] +async fn incremental_lookup_pins_historical_vectors_and_reuses_cache_after_vector_changes() { + let table = incremental_table(&[ + ("merge-engine", "aggregation"), + ("fields.value.aggregate-function", "sum"), + ]) + .await; + let builder = table.new_compact_builder(); + let executor = builder.new_executor().unwrap(); + write_batch( + &table, + &make_batch((0..1000).collect(), (0..1000).collect()), + ) + .await; + let (_, first) = compact(&table, &executor).await; + let anchor = &first[0].files()[0]; + let start = table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap() + .unwrap(); + write_batch(&table, &make_batch(vec![42], vec![5])).await; + let stale = builder.new_planner().unwrap().plan().await.unwrap(); + assert!(stale.tasks()[0] + .files() + .iter() + .all(|file| file.file_name != anchor.file_name)); + let output = executor.execute(&stale.tasks()[0]).await.unwrap(); + super::deletion_vector_tests::delete_positions(&table, &anchor.file_name, &[42]).await; + assert!(builder + .new_commit() + .unwrap() + .commit_compact(vec![output]) + .await + .unwrap_err() + .to_string() + .contains("deletion vectors changed")); + + // The cached physical value remains 42, but the current snapshot's vector + // hides it. The retry must emit an INSERT of 5 rather than sum it to 47. + compact(&table, &executor).await; + assert_eq!(rows(&table).await[42], (42, 5)); + for (key, value) in [(43, 7), (42, 9)] { + write_batch(&table, &make_batch(vec![key], vec![value])).await; + compact(&table, &executor).await; + } + let actual = rows(&table).await; + assert_eq!(actual.len(), 1000); + assert_eq!(actual[42], (42, 14)); + assert_eq!(actual[43], (43, 50)); + assert_eq!(executor.lookup_cache_metrics().file_builds, 1); + let end = table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap() + .unwrap(); + let mut expected = vec![ + ("+I".to_string(), 42, 5), + ("-U".to_string(), 43, 43), + ("+U".to_string(), 43, 50), + ("-U".to_string(), 42, 5), + ("+U".to_string(), 42, 14), + ]; + expected.sort(); + assert_eq!( + super::lookup_changelog_tests::events(&table, start, end).await, + expected + ); +} + +#[tokio::test] +async fn incremental_first_row_does_not_duplicate_a_key_in_an_untouched_higher_level() { + let table = incremental_table(&[ + ("merge-engine", "first-row"), + ("deletion-vectors.enabled", "false"), + ]) + .await; + let executor = table.new_compact_builder().new_executor().unwrap(); + write_batch( + &table, + &make_batch((0..1000).collect(), (0..1000).collect()), + ) + .await; + compact(&table, &executor).await; + write_batch(&table, &make_batch(vec![42], vec![9999])).await; + let (_, results) = compact(&table, &executor).await; + assert!(results + .iter() + .all(|result| result.files().is_empty() && result.changelog_files().is_empty())); + let actual = rows(&table).await; + assert_eq!(actual.len(), 1000); + assert_eq!(actual[42], (42, 42)); +} + +#[tokio::test] +async fn incremental_plain_compaction_leaves_history_for_query_merging() { + for engine in ["deduplicate", "aggregation"] { + let mut options = vec![ + ("merge-engine", engine), + ("deletion-vectors.enabled", "false"), + ("changelog-producer", "none"), + ]; + if engine == "aggregation" { + options.push(("fields.value.aggregate-function", "sum")); + } + let table = incremental_table(&options).await; + let executor = table.new_compact_builder().new_executor().unwrap(); + write_batch( + &table, + &make_batch((0..1000).collect(), (0..1000).collect()), + ) + .await; + let (_, outputs) = compact(&table, &executor).await; + let anchor = outputs[0].files()[0].file_name.clone(); + write_batch(&table, &make_batch(vec![42], vec![10000])).await; + let (plan, _) = compact(&table, &executor).await; + assert!(plan.tasks()[0] + .files() + .iter() + .all(|file| file.file_name != anchor)); + let actual = rows(&table).await; + assert_eq!(actual.len(), 1000); + assert_eq!( + actual[42], + ( + 42, + if engine == "aggregation" { + 10042 + } else { + 10000 + } + ) + ); + assert_eq!(executor.lookup_cache_metrics().file_builds, 0); + } +} + +fn local_files(path: &std::path::Path) -> Vec { + let mut files = Vec::new(); + for entry in std::fs::read_dir(path).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + files.extend(local_files(&entry.path())); + } else { + files.push(entry.path()); + } + } + files +} + +#[tokio::test] +async fn incremental_lookup_reopens_completed_cache_files_and_rebuilds_corrupt_payloads() { + use std::io::Write; + for corrupt in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let table = incremental_table_with_cache( + &[], + &[("local-cache.dir", directory.path().to_str().unwrap())], + ) + .await; + write_batch( + &table, + &make_batch((0..1000).collect(), (0..1000).collect()), + ) + .await; + { + let executor = table.new_compact_builder().new_executor().unwrap(); + compact(&table, &executor).await; + write_batch(&table, &make_batch(vec![42], vec![10042])).await; + compact(&table, &executor).await; + assert_eq!(executor.lookup_cache_metrics().file_builds, 1); + } + let files = local_files(directory.path()); + assert!(files.iter().any(|file| file + .components() + .any(|component| component.as_os_str() == "paimon-local-cache-v2"))); + if corrupt { + let payload = files + .iter() + .find(|file| { + let content = std::fs::read(file).unwrap(); + content + .windows(b"/rows".len()) + .any(|window| window == b"/rows") + }) + .unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(payload) + .unwrap() + .write_all(b"broken") + .unwrap(); + } + let executor = table.new_compact_builder().new_executor().unwrap(); + write_batch(&table, &make_batch(vec![43], vec![10043])).await; + compact(&table, &executor).await; + assert_eq!( + executor.lookup_cache_metrics().file_builds, + u64::from(corrupt) + ); + assert!(executor.lookup_cache_metrics().file_hits > 0); + assert_eq!(rows(&table).await[43], (43, 10043)); + assert!(local_files(directory.path()).iter().all(|file| !file + .components() + .any(|part| part.as_os_str().to_string_lossy().starts_with(".building-")))); + } +} diff --git a/crates/paimon/tests/pk_compact/lookup_changelog.rs b/crates/paimon/tests/pk_compact/lookup_changelog.rs new file mode 100644 index 000000000..96d1eface --- /dev/null +++ b/crates/paimon/tests/pk_compact/lookup_changelog.rs @@ -0,0 +1,764 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; + +async fn lookup_table(options: &[(&str, &str)]) -> Table { + let mut configured = vec![("changelog-producer", "lookup")]; + configured.extend_from_slice(options); + table(&configured).await +} + +async fn compact(table: &Table) -> i64 { + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + builder + .new_commit() + .unwrap() + .commit_compact(execute(table, &plan).await) + .await + .unwrap(); + table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap() + .unwrap() +} + +pub(super) async fn events(table: &Table, start: i64, end: i64) -> Vec<(String, i32, i32)> { + let plan = table + .new_read_builder() + .new_incremental_scan(IncrementalScanMode::Auto, start, end) + .plan_combined() + .await + .unwrap(); + let batches: Vec = table + .new_read_builder() + .new_read() + .unwrap() + .to_arrow_with_row_kind(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut events = Vec::new(); + for batch in batches { + let kinds = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let ids = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + events.extend((0..batch.num_rows()).map(|row| { + ( + kinds.value(row).to_string(), + ids.value(row), + values.value(row), + ) + })); + } + events.sort(); + events +} + +fn expected(events: &[(&str, i32, i32)]) -> Vec<(String, i32, i32)> { + let mut events = events + .iter() + .map(|(kind, key, value)| (kind.to_string(), *key, *value)) + .collect::>(); + events.sort(); + events +} + +#[tokio::test] +async fn lookup_compaction_publishes_before_after_images_for_fixed_dynamic_and_dv_tables() { + for dv in ["false", "true"] { + for bucket in ["1", "-1"] { + let table = lookup_table(&[ + ("deletion-vectors.enabled", dv), + ("bucket", bucket), + ("read.batch-size", "1"), + ]) + .await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + assert!(events(&table, 0, 1).await.is_empty()); + let first = compact(&table).await; + assert_eq!( + events(&table, 0, first).await, + expected(&[("+I", 1, 10), ("+I", 2, 20)]) + ); + write_batch( + &table, + &make_batch_with_kinds(vec![1, 2, 3], vec![11, 999, 30], vec![0, 3, 0]), + ) + .await; + let second = compact(&table).await; + assert_eq!( + events(&table, first, second).await, + expected(&[("-U", 1, 10), ("+U", 1, 11), ("-D", 2, 20), ("+I", 3, 30)]) + ); + assert_eq!(rows(&table).await, vec![(1, 11), (3, 30)]); + assert_eq!( + compact(&table).await, + second, + "an already compacted bucket emits nothing" + ); + assert_eq!(events(&table, first, second).await.len(), 4); + } + } +} + +#[tokio::test] +async fn lookup_compaction_respects_java_changelog_flags_and_default_duplicate_updates() { + for (options, expected_events) in [ + ( + vec![], + expected(&[("-U", 1, 10), ("+U", 1, 10), ("-D", 2, 20)]), + ), + ( + vec![("changelog-producer.row-deduplicate", "true")], + expected(&[("-D", 2, 20)]), + ), + ( + vec![("changelog-producer.ignore-update-before", "true")], + expected(&[("+U", 1, 10), ("-D", 2, 20)]), + ), + ( + vec![("changelog-producer.ignore-delete", "true")], + expected(&[("-U", 1, 10), ("+U", 1, 10)]), + ), + ( + vec![ + ("changelog-producer.row-deduplicate", "true"), + ("changelog-producer.row-deduplicate-ignore-fields", "value"), + ("changelog-producer.ignore-delete", "true"), + ], + vec![], + ), + ] { + let table = lookup_table(&options).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let first = compact(&table).await; + let updated_value = if options + .iter() + .any(|(key, _)| *key == "changelog-producer.row-deduplicate-ignore-fields") + { + 11 + } else { + 10 + }; + write_batch( + &table, + &make_batch_with_kinds(vec![1, 2], vec![updated_value, 0], vec![0, 3]), + ) + .await; + let second = compact(&table).await; + assert_eq!( + events(&table, first, second).await, + expected_events, + "{options:?}" + ); + assert_eq!(rows(&table).await, vec![(1, updated_value)]); + } +} + +#[tokio::test] +async fn lookup_compaction_first_row_only_emits_new_keys() { + let table = lookup_table(&[("merge-engine", "first-row")]).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + let first = compact(&table).await; + assert_eq!(events(&table, 0, first).await, expected(&[("+I", 1, 10)])); + write_batch(&table, &make_batch(vec![1, 2], vec![99, 20])).await; + let second = compact(&table).await; + assert_eq!( + events(&table, first, second).await, + expected(&[("+I", 2, 20)]) + ); + assert_eq!(rows(&table).await, vec![(1, 10), (2, 20)]); +} + +#[tokio::test] +async fn lookup_compaction_materializes_aggregation_history_once_and_ignores_high_level_only_keys() +{ + for dv in ["false", "true"] { + let table = lookup_table(&[ + ("deletion-vectors.enabled", dv), + ("merge-engine", "aggregation"), + ("fields.value.aggregate-function", "sum"), + ]) + .await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let first = compact(&table).await; + write_batch(&table, &make_batch(vec![1, 3], vec![5, 30])).await; + let second = compact(&table).await; + assert_eq!( + events(&table, first, second).await, + expected(&[("-U", 1, 10), ("+U", 1, 15), ("+I", 3, 30)]) + ); + write_batch(&table, &make_batch(vec![1], vec![7])).await; + let third = compact(&table).await; + assert_eq!( + events(&table, second, third).await, + expected(&[("-U", 1, 15), ("+U", 1, 22)]) + ); + assert_eq!(rows(&table).await, vec![(1, 22), (2, 20), (3, 30)]); + } +} + +async fn append_at_level(table: &Table, value: i32, level: i32) { + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer + .write_arrow_batch(&make_batch(vec![1], vec![value])) + .await + .unwrap(); + let mut messages = writer.prepare_commit().await.unwrap(); + for file in messages + .iter_mut() + .flat_map(|message| &mut message.new_files) + { + file.level = level; + } + builder.new_commit().commit(messages).await.unwrap(); + writer.close().await; +} + +#[tokio::test] +async fn lookup_compaction_selects_the_lowest_positive_level_as_materialized_history() { + let table = lookup_table(&[ + ("merge-engine", "aggregation"), + ("fields.value.aggregate-function", "sum"), + ]) + .await; + // Simulate Java lookup states: L2 already includes L3. Reducing both would + // produce 30 rather than 20 when the L0 delta is applied. + append_at_level(&table, 10, 3).await; + append_at_level(&table, 15, 2).await; + let history = table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap() + .unwrap(); + write_batch(&table, &make_batch(vec![1], vec![5])).await; + assert_eq!(rows(&table).await, vec![(1, 20)]); + let end = compact(&table).await; + assert_eq!( + events(&table, history, end).await, + expected(&[("-U", 1, 15), ("+U", 1, 20)]) + ); + assert_eq!(rows(&table).await, vec![(1, 20)]); +} + +#[tokio::test] +async fn lookup_compaction_uses_user_sequence_and_tombstones_for_changelog_images() { + for dv in ["false", "true"] { + let table = lookup_table(&[ + ("sequence.field", "value"), + ("deletion-vectors.enabled", dv), + ("changelog-producer.row-deduplicate", "true"), + ]) + .await; + write_batch(&table, &make_batch(vec![1], vec![100])).await; + let first = compact(&table).await; + write_batch( + &table, + &make_batch_with_kinds(vec![1, 9], vec![99, 99], vec![3, 3]), + ) + .await; + let second = compact(&table).await; + assert!(events(&table, first, second).await.is_empty()); + assert_eq!(rows(&table).await, vec![(1, 100)]); + write_batch(&table, &make_batch_with_kinds(vec![1], vec![101], vec![3])).await; + let third = compact(&table).await; + assert_eq!( + events(&table, second, third).await, + expected(&[("-D", 1, 100)]) + ); + assert!(rows(&table).await.is_empty()); + write_batch(&table, &make_batch(vec![1], vec![102])).await; + let fourth = compact(&table).await; + assert_eq!( + events(&table, third, fourth).await, + expected(&[("+I", 1, 102)]) + ); + } +} + +#[tokio::test] +async fn lookup_compaction_subsets_and_retries_publish_each_changelog_once_with_concurrent_l0() { + for dv in ["false", "true"] { + let table = lookup_table(&[ + ("bucket", "-1"), + ("dynamic-bucket.target-row-num", "1"), + ("deletion-vectors.enabled", dv), + ]) + .await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let first = compact(&table).await; + write_batch(&table, &make_batch(vec![1, 2], vec![11, 21])).await; + let builder = table + .new_compact_builder() + .with_commit_user("lookup-retry") + .unwrap(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + assert_eq!(plan.tasks().len(), 2); + let results = execute(&table, &plan).await; + assert_eq!( + results + .iter() + .map(|result| result.metrics().changelog_rows_written) + .sum::(), + 4 + ); + write_batch(&table, &make_batch(vec![1], vec![12])).await; + let commit = builder.new_commit().unwrap(); + for result in results { + let metrics = commit.commit_compact(vec![result.clone()]).await.unwrap(); + let snapshot = table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(); + assert_eq!(commit.commit_compact(vec![result]).await.unwrap(), metrics); + assert_eq!( + table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(), + snapshot + ); + } + let second = table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap() + .unwrap(); + assert_eq!( + events(&table, first, second).await, + expected(&[("-U", 1, 10), ("+U", 1, 11), ("-U", 2, 20), ("+U", 2, 21)]) + ); + let third = compact(&table).await; + assert_eq!( + events(&table, second, third).await, + expected(&[("-U", 1, 11), ("+U", 1, 12)]) + ); + assert_eq!(rows(&table).await, vec![(1, 12), (2, 21)]); + } +} + +#[tokio::test] +async fn lookup_compaction_rejects_concurrent_positive_level_history_at_another_level() { + let table = lookup_table(&[]).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + compact(&table).await; + write_batch(&table, &make_batch(vec![1], vec![11])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + append_at_level(&table, 12, 3).await; + let snapshot = table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(); + assert!(builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap_err() + .to_string() + .contains("history changed")); + assert_eq!( + table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(), + snapshot + ); +} + +#[tokio::test] +async fn lookup_compaction_aborts_data_and_changelog_files_on_a_late_read_failure() { + let table = lookup_table(&[ + ("read.batch-size", "1"), + ("write-buffer-size", "1b"), + ("target-file-row-num", "1"), + ]) + .await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + write_batch(&table, &make_batch(vec![2], vec![20])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let scan = table.new_read_builder().new_scan().plan().await.unwrap(); + let second = plan.tasks()[0] + .files() + .iter() + .find(|file| file.max_sequence_number == 1) + .unwrap(); + table + .file_io() + .delete_file(&format!( + "{}/{}", + scan.splits()[0].bucket_path(), + second.file_name + )) + .await + .unwrap(); + let paths = || async { + table + .file_io() + .list_status_recursive(table.location()) + .await + .unwrap() + .into_iter() + .map(|file| file.path) + .collect::>() + }; + let before = paths().await; + assert!(builder + .new_executor() + .unwrap() + .execute(&plan.tasks()[0]) + .await + .is_err()); + assert_eq!(paths().await, before); + assert_eq!( + table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(), + Some(2) + ); +} + +#[tokio::test] +async fn lookup_compaction_writes_rolled_changelogs_in_each_supported_format() { + for format in ["parquet", "orc", "avro", "row"] { + let table = lookup_table(&[ + ("file.format", "parquet"), + ("changelog-file.format", format), + ("changelog-file.compression", "none"), + ("changelog-file.prefix", "changes-"), + ("target-file-row-num", "1"), + ("write-buffer-size", "1b"), + ]) + .await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + let logs = results[0].changelog_files(); + assert_eq!(logs.len(), 2, "{format}"); + assert!(logs + .iter() + .all(|file| file.file_name.starts_with("changes-") + && file.file_name.ends_with(format) + && file.row_count == 1 + && file.embedded_index.is_none())); + let metrics = builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); + assert_eq!(metrics.changelog_rows_written, 2); + assert_eq!(metrics.changelog_files_added, 2); + let end = table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap() + .unwrap(); + assert_eq!( + events(&table, 0, end).await, + expected(&[("+I", 1, 10), ("+I", 2, 20)]) + ); + } +} + +#[tokio::test] +async fn lookup_compaction_preserves_sparse_updates_and_nullable_before_after_images() { + for dv in ["false", "true"] { + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .column("text", DataType::VarChar(VarCharType::string_type())) + .primary_key(["id"]) + .option("bucket", "1") + .option("merge-engine", "partial-update") + .option("changelog-producer", "lookup") + .option("changelog-producer.row-deduplicate", "true") + .option("deletion-vectors.enabled", dv) + .build() + .unwrap(); + let table = table_with_schema(TableSchema::new(0, &schema)).await; + let sparse = |value, text| { + RecordBatch::try_from_iter([ + ( + "id", + std::sync::Arc::new(Int32Array::from(vec![1])) as arrow_array::ArrayRef, + ), + ( + "value", + std::sync::Arc::new(Int32Array::from(vec![value])) as arrow_array::ArrayRef, + ), + ( + "text", + std::sync::Arc::new(StringArray::from(vec![text])) as arrow_array::ArrayRef, + ), + ]) + .unwrap() + }; + write_batch(&table, &sparse(None, Some("old"))).await; + let first = compact(&table).await; + write_batch(&table, &sparse(Some(0), None)).await; + let second = compact(&table).await; + let plan = table + .new_read_builder() + .new_incremental_scan(IncrementalScanMode::Changelog, first, second) + .plan_combined() + .await + .unwrap(); + let batches: Vec = table + .new_read_builder() + .new_read() + .unwrap() + .to_arrow_with_row_kind(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut changes = Vec::new(); + for batch in batches { + let kinds = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let texts = batch + .column_by_name("text") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + changes.push(( + kinds.value(row).to_string(), + values.is_valid(row).then(|| values.value(row)), + texts.value(row).to_string(), + )); + } + } + changes.sort(); + assert_eq!( + changes, + vec![ + ("+U".into(), Some(0), "old".into()), + ("-U".into(), None, "old".into()) + ] + ); + let batches = read(&table, None).await; + assert_eq!( + batches[0] + .column_by_name("text") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "old" + ); + } +} + +#[tokio::test] +async fn lookup_compaction_changelogs_are_available_to_streaming_follow_up_without_replaying_l0() { + let table = lookup_table(&[]).await; + let mut scan = table.new_stream_scan().unwrap(); + assert!(scan.plan().await.unwrap().is_none()); + write_batch(&table, &make_batch(vec![1], vec![10])).await; + // Initial lookup state contains only materialized positive-level files. + let initial = scan.plan().await.unwrap().unwrap(); + assert!(initial.splits().is_empty()); + compact(&table).await; + let changes = scan.plan().await.unwrap().unwrap(); + let batches: Vec = table + .new_read_builder() + .new_read() + .unwrap() + .to_arrow_with_row_kind(changes.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); + assert_eq!( + batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "+I" + ); + assert!(scan.plan().await.unwrap().is_none()); + write_batch(&table, &make_batch(vec![1], vec![11])).await; + assert!(scan.plan().await.unwrap().is_none()); + compact(&table).await; + let changes = scan.plan().await.unwrap().unwrap(); + let batches: Vec = table + .new_read_builder() + .new_read() + .unwrap() + .to_arrow_with_row_kind(changes.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 2); + assert!(scan.plan().await.unwrap().is_none()); +} + +#[tokio::test] +async fn lookup_compaction_validates_changelog_flags_before_execution() { + for key in [ + "changelog-producer.row-deduplicate", + "changelog-producer.ignore-update-before", + "changelog-producer.ignore-delete", + ] { + let table = lookup_table(&[(key, "invalid")]).await; + assert!(table + .new_compact_builder() + .new_planner() + .err() + .unwrap() + .to_string() + .contains(key)); + } +} + +#[tokio::test] +async fn lookup_compaction_rejects_stale_dvs_then_emits_insert_for_the_deleted_history_key() { + let table = lookup_table(&[ + ("deletion-vectors.enabled", "true"), + ("merge-engine", "aggregation"), + ("fields.value.aggregate-function", "sum"), + ]) + .await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let first = compact(&table).await; + write_batch(&table, &make_batch(vec![1], vec![5])).await; + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + let results = execute(&table, &plan).await; + let history = &plan.tasks()[0] + .files() + .iter() + .find(|file| file.level > 0) + .unwrap() + .file_name; + super::deletion_vector_tests::delete_positions(&table, history, &[0]).await; + let snapshot = table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(); + assert!(builder + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap_err() + .to_string() + .contains("deletion vectors changed")); + assert_eq!( + table + .snapshot_manager() + .get_latest_snapshot_id() + .await + .unwrap(), + snapshot + ); + let end = compact(&table).await; + assert_eq!(events(&table, first, end).await, expected(&[("+I", 1, 5)])); + assert_eq!(rows(&table).await, vec![(1, 5), (2, 20)]); +} + +#[tokio::test] +async fn lookup_compaction_preserves_before_after_sequence_metadata_and_pair_order() { + let table = lookup_table(&[]).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + let first = compact(&table).await; + write_batch(&table, &make_batch(vec![1], vec![11])).await; + let second = compact(&table).await; + let mut read = table.new_read_builder(); + read.with_projection(&["id", "_SEQUENCE_NUMBER", "value"]) + .unwrap(); + let plan = read + .new_incremental_scan(IncrementalScanMode::Changelog, first, second) + .plan_combined() + .await + .unwrap(); + let batches: Vec = read + .new_read() + .unwrap() + .to_arrow_with_row_kind(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut metadata = Vec::new(); + for batch in batches { + let kinds = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let seqs = batch + .column_by_name("_SEQUENCE_NUMBER") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + metadata.push((kinds.value(row).to_string(), seqs.value(row))); + } + } + assert_eq!(metadata, vec![("-U".into(), 0), ("+U".into(), 1)]); +} diff --git a/crates/paimon/tests/pk_compact_test.rs b/crates/paimon/tests/pk_compact_test.rs index 64541bcca..3e30a1e4f 100644 --- a/crates/paimon/tests/pk_compact_test.rs +++ b/crates/paimon/tests/pk_compact_test.rs @@ -20,13 +20,22 @@ mod common; #[path = "pk_compact/merge_engines.rs"] mod engine_tests; +#[path = "pk_compact/incremental_lookup.rs"] +mod incremental_lookup_tests; + +#[path = "pk_compact/lookup_changelog.rs"] +mod lookup_changelog_tests; + +#[path = "pk_compact/deletion_vectors.rs"] +mod deletion_vector_tests; + #[path = "pk_compact/dynamic_bucket.rs"] mod dynamic_tests; use arrow_array::{Array, Int32Array, Int64Array, RecordBatch, StringArray}; use common::incremental_helpers::{ - make_batch, make_batch_with_kinds, make_partitioned_batch, memory_table, persist_table_schema, - pk_schema, setup_dirs, write_batch, + make_batch, make_batch_with_kinds, make_partitioned_batch, persist_table_schema, pk_schema, + setup_dirs, write_batch, }; use futures::TryStreamExt; use paimon::spec::{ @@ -35,15 +44,38 @@ use paimon::spec::{ use paimon::table::{CompactPlan, CompactResult, CompactTask, IncrementalScanMode, Table}; async fn table_with_schema(schema: TableSchema) -> Table { + table_with_cache(schema, &[]).await +} + +async fn table_with_cache(schema: TableSchema, cache_options: &[(&str, &str)]) -> Table { let path = format!("memory:/pk_compact/{}", uuid::Uuid::new_v4()); - let (io, table) = memory_table(&path, schema); + let mut options = paimon::Options::new(); + options.set("warehouse", &path); + options.set("local-cache.enabled", "true"); + for (name, value) in cache_options { + options.set(*name, *value); + } + let catalog = paimon::catalog::FileSystemCatalog::new(options).unwrap(); + let io = catalog.file_io().clone(); + let table = Table::new( + io.clone(), + paimon::catalog::Identifier::new("default", "pk_compact"), + path.clone(), + schema, + None, + ); setup_dirs(&io, &path).await; persist_table_schema(&io, &path, table.schema()).await; table } async fn table(options: &[(&str, &str)]) -> Table { - table_with_schema(pk_schema(options)).await + let mut configured = vec![ + ("num-levels", "2"), + ("num-sorted-run.compaction-trigger", "2"), + ]; + configured.extend_from_slice(options); + table_with_schema(pk_schema(&configured)).await } async fn read(table: &Table, projection: Option<&[&str]>) -> Vec { @@ -391,7 +423,7 @@ async fn pk_compaction_keeps_tombstones_against_later_out_of_order_user_sequence } #[tokio::test] -async fn pk_compaction_includes_large_files_and_whole_buckets_and_upgrades_single_l0() { +async fn pk_compaction_selects_l0_runs_including_large_files_and_upgrades_single_l0() { let table = table(&[("target-file-size", "1b")]).await; write_batch(&table, &make_batch(vec![1], vec![10])).await; let builder = table.new_compact_builder(); @@ -530,12 +562,8 @@ async fn pk_compaction_rolls_ordered_files_and_rebuilds_indexes_with_level_stats } #[tokio::test] -async fn pk_compaction_refuses_partial_buckets_and_unsupported_layouts() { +async fn pk_compaction_refuses_tampered_incremental_inputs_and_unsupported_layouts() { for options in [ - vec![ - ("merge-engine", "first-row"), - ("changelog-producer", "lookup"), - ], vec![("changelog-producer", "full-compaction")], vec![ ("merge-engine", "first-row"), @@ -565,7 +593,7 @@ async fn pk_compaction_refuses_partial_buckets_and_unsupported_layouts() { .await .unwrap_err() .to_string() - .contains("complete captured bucket")); + .contains("incremental run selection")); let results = execute(&table, &plan).await; let mut encoded = serde_json::to_value(&results).unwrap(); encoded[0]["task"]["files"].as_array_mut().unwrap().pop(); diff --git a/docs/src/compaction.md b/docs/src/compaction.md index 9e76a5a31..73852659a 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -20,7 +20,7 @@ under the License. # Table Compaction Rust supports small-file compaction for ordinary append tables, normal-file -compaction for Data Evolution tables, and full-bucket compaction for primary-key +compaction for Data Evolution tables, and incremental compaction for primary-key tables with fixed or dynamic buckets. `CompactBuilder` creates a `CompactPlanner`, a `CompactExecutor`, and a `TableCommit`. Planning and execution do not publish snapshots; `commit_compact` atomically publishes the @@ -114,16 +114,28 @@ snapshot expiration and storage reclamation are separate maintenance work. The same API compacts primary-key tables with `deduplicate`, `first-row`, `partial-update` or `aggregation` merge engines. Changelog producers -`none` and `input` are supported; `first-row` requires `none`. -The planner includes every active file of each selected partition/bucket, -including large files. Append size and count selection options do not split PK -buckets. A single level-0 file is also eligible. A bucket whose files already form one sorted -run at a single positive level needs no rewrite, including when the previous -compaction rolled multiple output files. Output is key ordered at the highest input level, or level 1 -when all inputs are level 0. - -Compaction uses each engine's existing merge function and persists its merge -state, including original sequence metadata and synthesized DELETE records: +`none`, `input` and `lookup` are supported; `first-row` supports `none` or `lookup`. +The planner follows Java's `UniversalCompaction` sorted-run selection. Each L0 +file is a run; all files at the same positive level form one run. An explicit +compact drains the captured L0 files, including a single L0 file, and extends +the selection through adjacent runs while their sizes are comparable. L1 is +included when necessary to avoid writing L0 output. Large higher-level runs +remain untouched and supply point-lookup history when required. + +Without L0 input, sorted-run count, size ratio and size amplification trigger +maintenance. The Java-compatible defaults are `num-sorted-run.compaction-trigger=5`, +`compaction.size-ratio=1` (percent), `compaction.max-size-amplification-percent=200`, +and `num-levels=compaction-trigger + 1`. Output is key ordered at the level below +the first untouched run, or the highest configured level when every run was +selected. The configured level count expands to accommodate existing files. +A bucket containing just one positive-level run needs no rewrite, even if that +run spans multiple files or carries deletion vectors. Append small-file packing +options do not control PK selection. There is no separate full-compact path; +the incremental policy may naturally select every run based on their sizes. + +Without deletion vectors, compaction uses each engine's existing merge function +and persists its merge state, including original sequence metadata and synthesized +DELETE records: - `deduplicate` retains the winning record, including retracts that must still suppress later out-of-order values. @@ -142,8 +154,9 @@ whole-record deletion also apply during compaction. If all records are legitimately ignored, compaction can remove the selected files without creating an output file. Key bounds, sequence bounds, delete counts and file indexes are regenerated, using metadata statistics options for the output level. -Compound `partial-update` DELETE states with field aggregators are rejected. -For both reducer engines, compound groups containing retracts are also rejected +Without deletion vectors, compound `partial-update` DELETE states with field +aggregators are rejected. In that mode, both reducer engines also reject +compound groups containing retracts when they use `first_value`, `first_non_null_value`, `collect`, `listagg`, `nested_update` or `nested_partial_update`. These states may carry initialization flags or raw DELETE payloads that cannot be reconstructed from one output row, @@ -175,6 +188,158 @@ publishing a snapshot. The captured snapshot must remain available until each task subset is successfully committed; retries of an already committed subset remain idempotent after its read snapshot expires. +## Primary-key deletion vectors + +Primary-key tables with `deletion-vectors.enabled=true` use the same incremental +compaction API with `deduplicate`, `partial-update` or `aggregation` and +`changelog-producer=none`, `input` or `lookup`. Schema validation excludes `first-row` +with deletion vectors. + +Planning includes L0 files even when normal batch scans hide them. Execution +reads the captured snapshot's vectors and filters physical +row positions before merging keys. This prevents a previously materialized +aggregate from being counted together with the older row it replaced. + +The output contains only live merged values, with a known zero delete-row count. +Winning retracts and fully deleted keys produce no output; a task may remove +all its input files without adding a data file. This follows Java's deletion-vector +lookup compaction semantics. When a key has no history in the selected runs, +point lookup searches the untouched levels and materializes the previous value +with its updates. The previous physical position is marked deleted in that +historical file. With +`changelog-producer=lookup`, it also writes the logical changes described below. + +Commit atomically removes vector ranges belonging to rewritten files and unions +new position bitmaps into untouched files' vectors. If an affected index also +contains untouched ranges, those vectors are retained when writing its replacement. +Unrelated indexes and the latest dynamic-bucket HASH entries are retained. +Historical snapshots retain their original data and vector metadata. + +A concurrent change to an input or lookup dependency's deletion-vector reference rejects the +commit, including a new vector on a previously undeleted file. Plan again from +the latest snapshot to avoid resurrecting deleted rows. Independent L0 appends, +new vectors on those appends, and changes in other buckets can commit alongside +the rewrite. Serialized results, task subsets and idempotent retries follow the +same rules as ordinary PK compaction. + +## Lookup changelog producer + +Set `changelog-producer=lookup` and run the same compaction API to publish data +and changelog files atomically in a `COMPACT` snapshot. Ordinary writes produce +L0 data files; changelogs become available when those files are compacted. +Batch incremental `Changelog` / `Auto` scans and streaming follow-up scans read +the resulting changelog manifest. `Delta` scans continue to skip compaction. + +For each key, the lowest positive level is its latest materialized history. +Older positive levels already contributed to that state and are not reduced +again. Compaction merges this history with the captured L0 records and emits +changes only for keys with L0 input: + +- No live history and a live result: `INSERT`. +- Live history and a deleted result: `DELETE` carrying the history's values. +- Live history and a live result: `UPDATE_BEFORE` followed by `UPDATE_AFTER`. +- No live history and no live result: no changelog. + +Before images retain the historical sequence metadata; after images carry the +merged result's metadata. `first-row` emits only inserts for previously unseen +keys. `partial-update` and `aggregation` use their existing reducers and state +preservation restrictions. Deletion vectors are applied before history selection +and merging, with deleted keys omitted from DV data output. + +Java-compatible options have these defaults: + +| Option | Default | Behavior | +| --- | --- | --- | +| `changelog-producer.row-deduplicate` | `false` | When true, skip update pairs whose user values are equal. | +| `changelog-producer.row-deduplicate-ignore-fields` | unset | Comma-separated fields excluded from that comparison. | +| `changelog-producer.ignore-update-before` | `false` | Omit `UPDATE_BEFORE` events. | +| `changelog-producer.ignore-delete` | `false` | Omit `DELETE` events. | + +With default options, even an unchanged value produces an update pair when its +key has L0 input. Ignoring changelog events does not change compacted table data. +`changelog-file.format`, `changelog-file.compression`, `changelog-file.prefix` +and changelog statistics options apply independently of data-file settings. +Changelog files roll at the ordinary size and row limits. `CompactResult` exposes +`changelog_files()`; `CompactMetrics` reports their file, row and byte counts in +`changelog_files_added`, `changelog_rows_written` and `changelog_bytes_written`. +The existing data metrics continue to count data files and rows only. + +The executor streams both outputs and aborts both writers on execution failure. +Prepared results include both file lists for serialization, subset commits and +idempotent retries. A concurrent positive-level addition in a selected bucket +rejects the commit because it may change the changelog's history. Concurrent L0 +appends remain eligible for a later compaction and produce no premature events. + +### Local Lookup File cache + +DV compaction, lookup changelog production and `first-row` require +`local-cache.enabled=true` on the catalog's FileIO. Setting this flag only in +table properties does not attach a cache and is rejected when creating an +executor. Ordinary PK compaction without these features does not require a cache. +Lookup visits positive levels from lowest to highest, uses +file key bounds to select a candidate, and stops at the first matching physical +key, including a tombstone. A missing key continues to the next level. Rows +deleted by the captured snapshot's DV are skipped. A historical key rejected by +`first-row` stays in its untouched file; no duplicate data row is written. + +The first lookup into an uncached file streams that immutable file once into a +local SST key/position index and Arrow IPC value blocks. Later lookups use the +local index and decoded-block LRU; they do not scan the source again. Bloom +filters avoid unnecessary local reads. Complete KV state, sequence metadata and +physical row positions are retained. Snapshot DVs are applied after lookup, so +the same physical cache can be reused across snapshots with different vectors. +Cache identities include storage namespace, physical file metadata and schema; +a level-only change does not invalidate the bytes. + +```rust +let mut options = Options::new(); +options.set(CatalogOptions::WAREHOUSE, "s3://bucket/warehouse"); +options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); +options.set(CatalogOptions::LOCAL_CACHE_DIR, "/local/paimon-cache"); +options.set(CatalogOptions::LOCAL_CACHE_MAX_SIZE, "20 GiB"); +let catalog = CatalogFactory::create(options).await?; +let table = catalog.get_table(&identifier).await?; +let executor = table.new_compact_builder().new_executor()?; +let cache_metrics = executor.lookup_cache_metrics(); +``` + +Generated SST and value blocks use the existing `local-cache` backend and share +its directory, byte accounting, CRC validation and LRU with ordinary cached +reads. `local-cache.max-size` bounds their combined encoded bytes on disk, or +payload bytes for the memory backend. There is no independent Lookup File +directory or disk quota; `lookup.cache-max-disk-size` is rejected. Configure +`local-cache.dir` to use disk; without it, enabled local-cache uses memory. +The remote-read whitelist does not disable generated Lookup File blocks. +They use the regular LRU, preserving the existing protection for metadata and +global-index reads. `local-cache.shared=true` also shares generated blocks +through the existing shared disk backend. + +Completed files survive executor recreation, and the disk backend can reopen +them after its cache manager restarts. Concurrent misses in one lookup manager +build a file once. Each build uses immutable versioned block names and publishes +its descriptor last. Incomplete builds have no published descriptor; +partial blocks remain subject to the same bounded LRU. Evicted, corrupt or +incompatible blocks cause a rebuild from the source file, with one retry per +lookup. Table data and snapshots never depend on disposable cache blocks. + +| Option | Default | Behavior | +| --- | --- | --- | +| `lookup.cache-max-memory-size` | `256mb` | Shared budget for local index metadata, SST blocks and decoded value blocks. | +| `lookup.cache-file-retention` | `1h` | Idle retention for loaded lookup readers; recovered descriptors older than this are rebuilt. | +| `lookup.cache-spill-compression` | `zstd` | SST compression: `zstd`, `lz4`, `lzo` or `none`. | + +Loaded readers are retained while in use; backend blocks can still be evicted +by other cache users. A block that cannot be retained causes `ResourceExhausted` +without publishing compact output. Repeated misses also fail the task; increase +`local-cache.max-size` to retain the needed Lookup Files. Increase the decoded +memory budget or reduce `read.batch-size` for large value blocks. Source +reads and lookup DV state also use the task's resource context. Metrics expose +`file_builds`, `file_hits` and `file_evictions` (expired or invalidated lookup +readers, rather than all backend block evictions). The cache is disposable and uses +a Rust-private versioned format; table snapshots never depend on its paths. +Java's remote prebuilt `.lookup` files and a background compaction scheduler are +not implemented. + ## Dynamic buckets Dynamic bucket compaction (`bucket=-1`) rewrites each existing partition/bucket @@ -241,15 +406,17 @@ deletions into reassigned row IDs are not provided. Setting ## Current scope -Compaction-generated changelogs (`full-compaction` / `lookup`) are unsupported. -Outside Data Evolution mode, row tracking, deletion vectors and dedicated -blob/vector storage are unsupported. Query authorization, rowkind fields and -postpone buckets are unsupported in all modes. +The `full-compaction` changelog producer and full compact are unsupported; +`lookup` is supported through incremental primary-key compaction. +Outside Data Evolution mode, row tracking and dedicated blob/vector storage +are unsupported. Deletion vectors are supported for primary-key compaction; +ordinary append deletion-vector compaction is unsupported. Query authorization, +rowkind fields and postpone buckets are unsupported in all modes. Ordered bucketed append, append incremental clustering, primary-key clustering override (`pk-clustering-override`) and record-level expiration (`record-level.expire-time`) are also unsupported. -Snapshot index manifests may contain dynamic-bucket HASH indexes, or deletion -vectors and global indexes in Data Evolution mode. +Snapshot index manifests may contain dynamic-bucket HASH indexes and primary-key +deletion vectors, or deletion vectors and global indexes in Data Evolution mode. Embedded and sidecar file indexes are regenerated according to the captured table options. Language bindings and background compaction scheduling are not yet provided. diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 5bcce1a87..284f2af1e 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -450,6 +450,12 @@ Use `local-cache.whitelist=meta,global-index,structured-data` to cache data blocks without dedicated BLOB, video, or vector-store files. `data` and `*` retain their broader behavior. +Primary-key compaction with deletion vectors, lookup changelog production or +`first-row` requires an enabled catalog cache. Its generated Lookup File blocks +share this backend's total capacity and regular LRU with cached reads; they do +not require an entry in the remote-read whitelist. Set `local-cache.dir` for disk +storage. See [Local Lookup File cache](compaction.md#local-lookup-file-cache). + The protected and regular LRUs share one total budget; either can use idle capacity. Catalogs or processes sharing a disk cache must use the same ratio. diff --git a/docs/src/sql.md b/docs/src/sql.md index 8fff70bdb..2026a13a0 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2820,8 +2820,9 @@ Set via `WITH ('key' = 'value')` at table creation time, or dynamically via `SET | `'merge-engine' = 'partial-update'` | Basic partial-update engine for PK tables | | `'merge-engine' = 'aggregation'` | Basic aggregation engine for PK tables | -For deletion-vector-enabled primary-key tables using the default `deduplicate` -engine, batch scans hide uncompacted level-0 files by default. Set +For deletion-vector-enabled primary-key tables using `deduplicate`, +`partial-update` or `aggregation`, batch scans hide uncompacted level-0 files by +default. Set `'deletion-vectors.merge-on-read' = 'true'` to include those files and merge their key versions on read. Existing deletion vectors are applied before the key merge. This option affects batch snapshot reads only; it does not change @@ -2871,12 +2872,12 @@ lower-precedence group sequence arrives, order-sensitive functions use Java compatible reversed aggregation. Unknown functions and incompatible function/type combinations fail closed. -Rust can read fully materialized compacted files from deletion-vector-enabled -partial-update and aggregation tables. Every split must be raw-convertible, -every file must have a known zero delete-row count, and -`deletion-vectors.merge-on-read` must remain disabled. Writing these table -combinations and reading plans that still require per-key merging are not -supported. +Rust supports writing deletion-vector-enabled partial-update and aggregation +tables and merging their uncompacted files with +`deletion-vectors.merge-on-read=true`. Use the Rust table compaction API to +materialize live rows for ordinary batch reads; see +[Primary-key deletion vectors](compaction.md#primary-key-deletion-vectors). +Compaction applies existing vectors before merging and drops deleted keys. Rust currently supports `merge-engine=aggregation` in basic mode only. It works with fixed buckets and ordinary dynamic buckets (`'bucket' = '-1'`) when the @@ -2890,11 +2891,14 @@ Sequence fields are always merged with `last_value`. Defining `fields..aggregate-function` is rejected, matching Java schema validation. -This is not full Java feature parity. Aggregation tables do not support retract -rows (`DELETE` / `UPDATE_BEFORE`), deletion-vector writes or merge-on-read -plans, cross-partition dynamic bucket writes, or advanced aggregation options -such as `ignore-retract`, `distinct`, `nested-key`, `count-limit`, and sequence -groups. +Incremental primary-key compaction supports `changelog-producer=lookup` with or without +deletion vectors and reuses local Lookup Files for untouched historical levels. +These compact modes require `local-cache.enabled=true` on the catalog, using +the same `local-cache.dir` and `local-cache.max-size` as other cached reads. +It publishes INSERT, UPDATE_BEFORE, UPDATE_AFTER and DELETE +changelogs when L0 files are compacted. Incremental and streaming scans then read +the persisted changes; see [Lookup changelog producer](compaction.md#lookup-changelog-producer). +`first-row` emits inserts for new keys only. ### Global Index Options @@ -2972,8 +2976,9 @@ the normal physical format without wrapping the writer. | `'manifest.sidecar.enabled' = 'true'` | Write and read manifest block sidecars for partition, row-id, and bucket pruning; when unset, inherits `manifest-sort.enabled` | | `'global-index.enabled' = 'true'` | Enable global index metadata and reads | | `'deletion-vectors.enabled' = 'true'` | Enable deletion vectors | -| `'deletion-vectors.merge-on-read' = 'true'` | Include and key-merge uncompacted level-0 files in DV-enabled deduplicate batch reads | -| `'changelog-producer' = 'input'` | Changelog producer; primary-key tables support reads and writes in this mode | +| `'deletion-vectors.merge-on-read' = 'true'` | Include and key-merge uncompacted level-0 files in DV-enabled primary-key batch reads | +| `'changelog-producer' = 'input'` | Persist original write events for primary-key tables | +| `'changelog-producer' = 'lookup'` | Generate logical before/after events through primary-key compaction | Manifest sidecars use the Java-compatible `.avro.sidecar` format and are published through the manifest metadata's `_EXTRA_FILES` list.