diff --git a/Cargo.lock b/Cargo.lock index 75851af..fdcfe23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,9 +610,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", ] @@ -1763,7 +1763,7 @@ dependencies = [ [[package]] name = "omnect-cli" -version = "0.28.5" +version = "0.28.6" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index c7db19b..8e90f87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0" name = "omnect-cli" readme = "README.md" repository = "https://github.com/omnect/omnect-cli" -version = "0.28.5" +version = "0.28.6" [dependencies] actix-web = { version = "4.17", default-features = false, features = ["macros"] } @@ -31,7 +31,7 @@ hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } -libfs = { version = "0.9", default-features = false } +libfs = { version = "0.9", default-features = false, features = ["use_linux"] } log = { version = "0.4", default-features = false } num_cpus = { version = "1.17", default-features = false } oauth2 = { version = "5.0", default-features = false, features = [ @@ -81,5 +81,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "bmap-tools, e2fsprogs, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/Dockerfile b/Dockerfile index add2cbf..03389a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ bmap-tools \ ca-certificates \ + e2fsprogs \ e2tools \ fdisk \ keychain \ @@ -37,7 +38,6 @@ RUN < Result> { + let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; + image + .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) + .context("read_ext4_journal: cannot read superblock")?; + let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); + let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); + + if le16(EXT4_S_MAGIC) != EXT4_MAGIC + || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 + { + return Ok(None); + } + + // an external journal has no inode in this filesystem + let inode = le32(EXT4_S_JOURNAL_INUM); + if inode == 0 { + return Ok(None); + } + + let log_block_size = le32(EXT4_S_LOG_BLOCK_SIZE); + anyhow::ensure!( + log_block_size <= EXT4_MAX_LOG_BLOCK_SIZE, + "read_ext4_journal: invalid block size" + ); + let block_size = EXT4_MIN_BLOCK_SIZE << log_block_size; + + Ok(Some(Ext4Journal { inode, block_size })) +} + +fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { + let debugfs_cmd = |program| { + let mut cmd = Command::new(program); + cmd.arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")); + cmd + }; + let mut debugfs = debugfs_cmd(DEBUGFS); + let output = match debugfs.output() { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + debugfs = debugfs_cmd(DEBUGFS_SBIN); + debugfs.output() + } + output => output, + } + .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + anyhow::ensure!( + output.status.success(), + "ext4_journal_blocks: {debugfs:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let blocks = String::from_utf8(output.stdout) + .context("ext4_journal_blocks: invalid debugfs output")? + .split_whitespace() + .map(|b| { + b.parse::() + .context(format!("ext4_journal_blocks: invalid block number {b}")) + }) + .collect::>>()?; + anyhow::ensure!( + !blocks.is_empty(), + "ext4_journal_blocks: no journal blocks at offset {offset}" + ); + + Ok(blocks) +} + +/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that +/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash +/// overwrites the journal an earlier installation left on the device. +fn allocate_ext4_journals(image_file: &str) -> Result<()> { + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + let image_len = image + .metadata() + .context("allocate_ext4_journals: cannot get image size")? + .len(); + + for partition in get_partitions(image_file)? { + let offset = partition.start * SECTOR_SIZE; + if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { + continue; + } + let Some(journal) = read_ext4_journal(&image, offset)? else { + continue; + }; + let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; + debug!( + "allocate_ext4_journals: partition {}: {} journal blocks", + partition.num, + blocks.len() + ); + + let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; + for block in blocks { + let pos = offset + block * journal.block_size; + image + .read_exact_at(&mut buf, pos) + .context("allocate_ext4_journals: cannot read journal block")?; + image + .write_all_at(&buf, pos) + .context("allocate_ext4_journals: cannot write journal block")?; + } + } + + image + .sync_all() + .context("allocate_ext4_journals: cannot sync image") +} + +pub fn generate_bmap_file(image_file: &str) -> Result<()> { + allocate_ext4_journals(image_file)?; + + let mut bmaptool = Command::new("bmaptool"); + bmaptool + .arg("create") + .arg("-o") + .arg(format!("{image_file}.bmap")) + .arg(image_file); + let status = bmaptool + .status() + .context(format!("generate_bmap_file: cannot run {bmaptool:?}"))?; + anyhow::ensure!(status.success(), "generate_bmap_file: {bmaptool:?} failed"); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::bmap::*; + use std::path::Path; + + const PART_START_LBA: u32 = 2048; + const PART_SECTORS: u32 = 16384; + const FS_BLOCK_SIZE: u64 = 1024; + const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; + const MBR_TYPE_LINUX: u8 = 0x83; + const BMAP_BLOCK_SIZE: u64 = 4096; + // content an earlier installation left on the device + const OLD_CONTENT_BYTE: u8 = 0xa5; + const JOURNAL_INODE: u32 = 8; + const INVALID_LOG_BLOCK_SIZE: u32 = 40; + + fn run(cmd: &mut Command) { + let status = cmd.status().expect("run command"); + assert!(status.success(), "{cmd:?} failed"); + } + + fn create_image_with_mbr(image: &Path, len: u64) { + let mut f = fs::File::create(image).expect("create image"); + f.set_len(len).expect("set image size"); + let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) + .expect("create mbr"); + mbr[1] = mbrman::MBRPartitionEntry { + boot: mbrman::BOOT_INACTIVE, + first_chs: mbrman::CHS::empty(), + sys: MBR_TYPE_LINUX, + last_chs: mbrman::CHS::empty(), + starting_lba: PART_START_LBA, + sectors: PART_SECTORS, + }; + mbr.write_into(&mut f).expect("write mbr"); + } + + fn create_image_with_ext4_journal(image: &Path) { + let raw = image.with_extension("raw"); + create_image_with_mbr(&raw, u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; + run(Command::new("mkfs.ext4") + .arg("-q") + .arg("-F") + .arg("-b") + .arg(FS_BLOCK_SIZE.to_string()) + .arg("-J") + .arg("size=1") + .arg("-E") + .arg(format!("offset={offset}")) + .arg(&raw) + .arg(fs_blocks.to_string())); + // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy + // turns the zeros into holes, as a sparse decompress does + run(Command::new("cp") + .arg("--sparse=always") + .arg(&raw) + .arg(image)); + } + + fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { + bmap.lines() + .filter_map(|l| { + let range = l.split_once("')?.1; + let range = range.split_once(" usize { + let ranges = bmap_ranges(bmap); + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let file = fs::File::open(image).expect("open image"); + let journal = read_ext4_journal(&file, offset) + .expect("read superblock") + .expect("ext4 with journal"); + ext4_journal_blocks(image, offset, journal.inode) + .expect("journal blocks") + .iter() + .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) + .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) + .count() + } + + /// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap + /// flash does. + fn bmap_flash(image: &str, bmap: &str, device: &Path) { + let image = fs::File::open(image).expect("open image"); + let len = image.metadata().expect("image metadata").len(); + fs::write( + device, + vec![OLD_CONTENT_BYTE; usize::try_from(len).expect("image size")], + ) + .expect("write old content"); + let device = fs::OpenOptions::new() + .write(true) + .open(device) + .expect("open device"); + + let mut buf = vec![0u8; usize::try_from(BMAP_BLOCK_SIZE).expect("block size")]; + for (first, last) in bmap_ranges(bmap) { + for block in first..=last { + image + .read_exact_at(&mut buf, block * BMAP_BLOCK_SIZE) + .expect("read image block"); + device + .write_all_at(&buf, block * BMAP_BLOCK_SIZE) + .expect("write device block"); + } + } + } + + fn journal_on_device_matches_image(image: &str, device: &Path) -> bool { + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let image_file = fs::File::open(image).expect("open image"); + let device = fs::File::open(device).expect("open device"); + let journal = read_ext4_journal(&image_file, offset) + .expect("read superblock") + .expect("ext4 with journal"); + + let block_size = usize::try_from(journal.block_size).expect("block size"); + let (mut a, mut b) = (vec![0u8; block_size], vec![0u8; block_size]); + ext4_journal_blocks(image, offset, journal.inode) + .expect("journal blocks") + .iter() + .all(|block| { + let pos = offset + block * journal.block_size; + image_file.read_exact_at(&mut a, pos).expect("read image"); + device.read_exact_at(&mut b, pos).expect("read device"); + a == b + }) + } + + #[test] + fn generate_bmap_file_maps_ext4_journal() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_path = dir.path().join("image.wic"); + create_image_with_ext4_journal(&image_path); + let image = image_path.to_str().expect("image path"); + let bmap_path = format!("{image}.bmap"); + + run(Command::new("bmaptool") + .arg("create") + .arg("-o") + .arg(&bmap_path) + .arg(image)); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert!(unmapped_journal_blocks(image, &bmap) > 0); + let device = dir.path().join("device"); + bmap_flash(image, &bmap, &device); + assert!(!journal_on_device_matches_image(image, &device)); + + generate_bmap_file(image).expect("generate bmap"); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert_eq!(unmapped_journal_blocks(image, &bmap), 0); + bmap_flash(image, &bmap, &device); + assert!(journal_on_device_matches_image(image, &device)); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + run(Command::new("e2fsck") + .arg("-fn") + .arg(format!("{image}?offset={offset}"))); + } + + #[test] + fn read_ext4_journal_rejects_invalid_block_size() { + let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; + let mut set = |o: usize, v: &[u8]| sb[o..o + v.len()].copy_from_slice(v); + set(EXT4_S_MAGIC, &EXT4_MAGIC.to_le_bytes()); + set( + EXT4_S_FEATURE_COMPAT, + &EXT4_FEATURE_COMPAT_HAS_JOURNAL.to_le_bytes(), + ); + set(EXT4_S_JOURNAL_INUM, &JOURNAL_INODE.to_le_bytes()); + set(EXT4_S_LOG_BLOCK_SIZE, &INVALID_LOG_BLOCK_SIZE.to_le_bytes()); + + let image = tempfile::tempfile().expect("create image"); + image + .write_all_at(&sb, EXT4_SUPERBLOCK_OFFSET) + .expect("write superblock"); + assert!(read_ext4_journal(&image, 0).is_err()); + } + + #[test] + fn allocate_ext4_journals_skips_partition_past_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + create_image_with_mbr(&image, u64::from(PART_START_LBA) * SECTOR_SIZE); + + allocate_ext4_journals(image.to_str().expect("image path")).expect("allocate journals"); + } +} diff --git a/src/file/compression.rs b/src/file/compression.rs index 16f1862..fca98d4 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -3,11 +3,14 @@ use filemagic::Magic; use log::debug; use std::env; use std::fs::File; +use std::io::{Seek, SeekFrom, Write}; use std::path::PathBuf; use std::str::FromStr; use strum::IntoEnumIterator; use strum_macros::EnumIter; +const SPARSE_BLOCK_SIZE: u64 = 4096; + #[derive(Clone, Debug, EnumIter)] #[allow(non_camel_case_types)] pub enum Compression { @@ -76,15 +79,21 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut dec: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(destination)), - Compression::gzip => Box::new(flate2::write::GzDecoder::new(destination)), - Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(destination)), + let sparse = SparseWriter(destination); + let mut dec: Box = match &self { + Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(sparse)), + Compression::gzip => Box::new(flate2::write::GzDecoder::new(sparse)), + Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(sparse)), }; let bytes_written = std::io::copy(source, &mut dec)?; dec.write_all(&[])?; dec.flush()?; + drop(dec); + + // zero blocks at the end were only seeked over + let len = destination.stream_position()?; + destination.set_len(len)?; Ok(bytes_written) } @@ -126,6 +135,27 @@ impl Compression { } } +/// Seeks over zero blocks instead of writing them, so they stay holes in a new file. +struct SparseWriter<'a>(&'a mut File); + +impl Write for SparseWriter<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let pos = self.0.stream_position()?; + let to_block_end = SPARSE_BLOCK_SIZE - pos % SPARSE_BLOCK_SIZE; + let chunk = &buf[..buf.len().min(to_block_end as usize)]; + if chunk.iter().all(|b| *b == 0) { + self.0.seek(SeekFrom::Current(chunk.len() as i64))?; + Ok(chunk.len()) + } else { + self.0.write(chunk) + } + } + + fn flush(&mut self) -> std::io::Result<()> { + self.0.flush() + } +} + pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Result { let mut new_image_file = PathBuf::from(image_file_name); @@ -157,3 +187,62 @@ pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result< debug!("image::compress: copied {} bytes.", bytes_written); Ok(new_image_file) } + +#[cfg(test)] +mod tests { + use crate::file::compression::*; + use std::io::Read; + use std::os::unix::fs::MetadataExt; + + // unit of `st_blocks` + const STAT_BLOCK_SIZE: u64 = 512; + const DATA_OFFSET: usize = 1024 * 1024 + 100; + const IMAGE_LEN: usize = 3 * 1024 * 1024 + 123; + + fn roundtrip(compression: Compression, image: &[u8]) -> File { + let mut source = tempfile::tempfile().expect("create source"); + source.write_all(image).expect("write source"); + source.rewind().expect("rewind source"); + + let mut compressed = tempfile::tempfile().expect("create compressed"); + compression + .compress(&mut source, &mut compressed) + .expect("compress"); + compressed.rewind().expect("rewind compressed"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + decompressed + } + + #[test] + fn decompress_keeps_zero_blocks_as_holes() { + let mut image = vec![0u8; IMAGE_LEN]; + image[DATA_OFFSET..DATA_OFFSET + 3].copy_from_slice(b"abc"); + + for compression in [ + Compression::bzip2, + Compression::gzip, + Compression::xz { + compression_level: 1, + }, + ] { + let mut decompressed = roundtrip(compression.clone(), &image); + + let mut content = Vec::new(); + decompressed.rewind().expect("rewind decompressed"); + decompressed + .read_to_end(&mut content) + .expect("read decompressed"); + assert!(content == image, "{compression:?}: content differs"); + + let meta = decompressed.metadata().expect("metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "{compression:?}: no holes" + ); + } + } +} diff --git a/src/file/functions.rs b/src/file/functions.rs index 4b8cf95..d6591e9 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,14 +1,19 @@ +use crate::file::partition::SECTOR_SIZE; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; use std::fmt::{self, Display}; use std::fs; +use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::process::Command; use std::str::FromStr; use stdext::function_name; use uuid::Uuid; +const COMPARE_BLOCK_SIZE: usize = 4096; +const COPY_CHUNK_SIZE: usize = 1024 * 1024; + #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] #[allow(non_camel_case_types)] @@ -415,7 +420,7 @@ fn read_partition( let mut dd = Command::new("dd"); dd.arg(format!("if={image_file}")) .arg(format!("of={partition_file}")) - .arg("bs=512") + .arg(format!("bs={SECTOR_SIZE}")) .arg(format!("skip={}", partition_info.start)) .arg(format!("count={}", partition_info.count)) .arg("conv=sparse") @@ -433,34 +438,49 @@ fn write_partition( partition_file: &str, partition_info: &PartitionInfo, ) -> Result<()> { - let mut dd = Command::new("dd"); - dd.arg(format!("if={partition_file}")) - .arg(format!("of={image_file}")) - .arg("bs=512") - .arg(format!("seek={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=notrunc,sparse") - .arg("status=none"); - exec_cmd!(dd); - - let mut fallocate = Command::new("fallocate"); - fallocate.arg("-d").arg(image_file); - exec_cmd!(fallocate); - - let mut sync = Command::new("sync"); - exec_cmd!(sync); - - Ok(()) -} - -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - let mut bmaptool = Command::new("bmaptool"); - bmaptool - .arg("create") - .arg("-o") - .arg(format!("{image_file}.bmap")) - .arg(image_file); - exec_cmd!(bmaptool); + let partition = fs::File::open(partition_file) + .context(format!("write_partition: cannot open {partition_file}"))?; + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("write_partition: cannot open {image_file}"))?; + + let image_offset = partition_info.start * SECTOR_SIZE; + let len = partition + .metadata() + .context("write_partition: cannot get partition file size")? + .len() + .min(partition_info.count * SECTOR_SIZE); + + // Only write blocks that differ: unchanged blocks keep their allocation, so a bmap + // created from the result still maps every block the source image mapped. + let mut src = vec![0u8; COPY_CHUNK_SIZE]; + let mut dst = vec![0u8; COPY_CHUNK_SIZE]; + let mut pos = 0; + while pos < len { + let n = usize::try_from((len - pos).min(COPY_CHUNK_SIZE as u64))?; + partition + .read_exact_at(&mut src[..n], pos) + .context("write_partition: cannot read partition file")?; + image + .read_exact_at(&mut dst[..n], image_offset + pos) + .context("write_partition: cannot read image")?; + + let blocks = src[..n] + .chunks(COMPARE_BLOCK_SIZE) + .zip(dst[..n].chunks(COMPARE_BLOCK_SIZE)); + for (i, (s, d)) in blocks.enumerate() { + if s != d { + image + .write_all_at(s, image_offset + pos + (i * COMPARE_BLOCK_SIZE) as u64) + .context("write_partition: cannot write image")?; + } + } + pos += n as u64; + } - Ok(()) + image + .sync_all() + .context("write_partition: cannot sync image") } diff --git a/src/file/mod.rs b/src/file/mod.rs index 6f9db48..e9ccd43 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -1,3 +1,4 @@ +pub mod bmap; pub mod compression; pub mod functions; mod partition; diff --git a/src/file/partition.rs b/src/file/partition.rs index 04d2fc6..12cbc5e 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -4,6 +4,8 @@ use std::io::Seek; use std::io::SeekFrom; use std::path::Path; +pub const SECTOR_SIZE: u64 = 512; + pub struct PartitionData { pub num: u32, pub start: u64, @@ -41,7 +43,7 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result // Try MBR — iter() includes logical partitions (5, 6, ...) file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, 512) + let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; for (i, p) in mbr.iter() { if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { @@ -55,7 +57,62 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result anyhow::bail!("partition {partition_num} not found in image") } +pub fn get_partitions>(path: P) -> Result> { + let path = path.as_ref(); + let mut file = + File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; + + let gpt_err = match gptman::GPT::find_from(&mut file) { + Ok(gpt) => { + return Ok(gpt + .iter() + .filter(|(_, e)| e.is_used() && e.ending_lba >= e.starting_lba) + .map(|(num, e)| PartitionData { + num, + start: e.starting_lba, + count: e.ending_lba - e.starting_lba + 1, + }) + .collect()); + } + Err(e) => e, + }; + + file.seek(SeekFrom::Start(0)) + .context("failed to seek to start of image")?; + let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) + .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; + mbr.iter() + .filter(|(_, p)| p.is_used() && !p.is_extended()) + .map(|(num, p)| { + Ok(PartitionData { + num: u32::try_from(num).context("MBR partition number out of range")?, + start: p.starting_lba as u64, + count: p.sectors as u64, + }) + }) + .collect() +} + pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } + +#[cfg(test)] +mod tests { + use crate::file::partition::*; + + const TEST_IMAGE: &str = "testfiles/image.wic"; + const MBR_EXTENDED_PARTITION: u32 = 4; + + #[test] + fn get_partitions_skips_mbr_extended_entry() { + let nums: Vec = get_partitions(TEST_IMAGE) + .expect("read partitions") + .iter() + .map(|p| p.num) + .collect(); + assert!(!nums.contains(&MBR_EXTENDED_PARTITION)); + assert!(nums.contains(&(MBR_EXTENDED_PARTITION + 1))); + } +} diff --git a/src/lib.rs b/src/lib.rs index 343f912..c487ced 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,7 +22,7 @@ use cli::{ }; use file::{compression::Compression, functions::FileCopyToParams}; use log::error; -use std::{fs, path::Path, path::PathBuf}; +use std::{fs, io::Read, path::Path, path::PathBuf}; use tokio::fs::remove_dir_all; use uuid::Uuid; @@ -93,9 +93,8 @@ where tmp_image_file = compression::decompress(&tmp_image_file, &source_compression)?; dest_image_file.set_extension(""); } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&image_file, &tmp_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", + copy_image(&image_file, &tmp_image_file).context(format!( + "error: copy_image({:?}, {:?})", image_file, tmp_image_file ))?; } @@ -115,7 +114,7 @@ where .to_str() .context("cannot get image file path")? )); - file::functions::generate_bmap_file( + file::bmap::generate_bmap_file( tmp_image_file .to_str() .context("cannot get image file path")?, @@ -140,9 +139,8 @@ where tmp_image_file, dest_image_file ))?; } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&tmp_image_file, &dest_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", + copy_image(&tmp_image_file, &dest_image_file).context(format!( + "error: copy_image({:?}, {:?})", tmp_image_file, dest_image_file ))?; } @@ -150,6 +148,35 @@ where Ok(()) } +/// Copies `from` to `to` and keeps the holes of a sparse `from`. +fn copy_image(from: &Path, to: &Path) -> Result<()> { + let src = fs::File::open(from).context("copy_image: cannot open source")?; + let len = src + .metadata() + .context("copy_image: cannot get source size")? + .len(); + let dst = fs::File::create(to).context("copy_image: cannot create destination")?; + dst.set_len(len) + .context("copy_image: cannot set destination size")?; + + // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a segment is done + let sparse = libfs::probably_sparse(&src)?; + let mut pos = 0; + while pos < len { + let (data, hole) = if sparse { + libfs::next_sparse_segments(&src, &dst, pos)? + } else { + (pos, len) + }; + let copied = std::io::copy(&mut (&src).take(hole - data), &mut &dst) + .context("copy_image: cannot copy data")?; + anyhow::ensure!(copied == hole - data, "copy_image: source ended early"); + pos = hole; + } + + Ok(()) +} + struct CertInfo { cert_path: PathBuf, key_path: PathBuf, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index ac3e716..c807429 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -5,7 +5,7 @@ use common::Testrunner; use httpmock::prelude::*; use omnect_cli::ssh; use predicates::prelude::*; -use std::{fs::create_dir_all, path::PathBuf}; +use std::{fs::create_dir_all, os::unix::fs::MetadataExt, path::PathBuf}; use stdext::function_name; #[test] @@ -895,6 +895,92 @@ fn check_file_copy(tr: Testrunner, partition: &str) { assert!(file_diff::diff(in_file4, out_file4)); } +// unit of `st_blocks` +const STAT_BLOCK_SIZE: u64 = 512; + +fn bmap_value(bmap: &str, tag: &str) -> u64 { + // the header comment of a bmap file mentions the tags too; only the closing tag is unique + let open_tag = format!("<{tag}>"); + let end = bmap + .find(&format!("")) + .expect("closing tag not found in bmap"); + let start = bmap[..end] + .rfind(&open_tag) + .expect("opening tag not found in bmap") + + open_tag.len(); + bmap[start..end] + .trim() + .parse() + .expect("tag value is no number") +} + +#[test] +fn check_file_copy_keeps_image_layout() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + let meta = image_path.metadata().expect("image metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE >= meta.len(), + "test image must be fully allocated" + ); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .success(); + + // zero blocks of the input must stay mapped, e.g. zeroed ext4 journals + let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); + assert_eq!( + bmap_value(&bmap, "MappedBlocksCount"), + bmap_value(&bmap, "BlocksCount") + ); +} + +#[test] +fn check_file_copy_keeps_holes() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + let status = std::process::Command::new("cp") + .arg("--sparse=always") + .arg(&image_path) + .arg(&sparse_path) + .status() + .expect("run cp"); + assert!(status.success(), "sparse copy of the test image failed"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("-b") + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); + let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); + assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); @@ -965,6 +1051,9 @@ fn check_bmap_generation_wic_xz() { .assert(); assert.success(); + let bmap = std::fs::read_to_string(&image_path_bmap).expect("read bmap"); + assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); + // use bmaptool to verify that the checksum of the bmap file and the image // still match after the copy operations let assert = Command::new("bmaptool")