mirror of
https://github.com/imjasonh/oci2squashfs
synced 2026-07-07 00:33:35 +00:00
initial implementation
Signed-off-by: Steven Noonan <steven@edera.dev>
This commit is contained in:
parent
61d3f40cc1
commit
8ffd6583d4
18 changed files with 2448 additions and 0 deletions
7
.github/dependabot.yml
vendored
Normal file
7
.github/dependabot.yml
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "cargo"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
|
||||
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
/target
|
||||
1002
Cargo.lock
generated
Normal file
1002
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
3
Cargo.toml
Normal file
3
Cargo.toml
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
[workspace]
|
||||
members = ["oci_squashfs", "oci_squashfs_cli"]
|
||||
resolver = "2"
|
||||
20
oci_squashfs/Cargo.toml
Normal file
20
oci_squashfs/Cargo.toml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
[package]
|
||||
name = "oci_squashfs"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tar = "0.4"
|
||||
flate2 = { version = "1", features = ["zlib-rs"], default-features = false }
|
||||
zstd = "0.13"
|
||||
bzip2 = "0.6"
|
||||
xz2 = "0.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
sha2 = "0.10"
|
||||
tempfile = "3"
|
||||
anyhow = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
94
oci_squashfs/src/canonical.rs
Normal file
94
oci_squashfs/src/canonical.rs
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
//! CanonicalTarHeader: a tar Header paired with its PAX extensions.
|
||||
//! Ported from production vfs.rs — lets the `tar` crate own PAX serialization.
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use std::borrow::Cow;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tar::{Builder, EntryType, Header};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CanonicalTarHeader {
|
||||
pub header: Header,
|
||||
pub pax_extensions: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl CanonicalTarHeader {
|
||||
pub fn from_entry<R: Read>(entry: &mut tar::Entry<'_, R>) -> Result<Self> {
|
||||
let header = entry.header().clone();
|
||||
let pax_extensions = match entry.pax_extensions() {
|
||||
Err(e) => return Err(anyhow!("failed to read PAX extensions: {e}")),
|
||||
Ok(None) => vec![],
|
||||
Ok(Some(exts)) => {
|
||||
let mut pairs = Vec::new();
|
||||
for ext in exts {
|
||||
let ext = ext.map_err(|e| anyhow!("invalid PAX extension: {e}"))?;
|
||||
let key = ext
|
||||
.key()
|
||||
.map_err(|e| anyhow!("invalid PAX key: {e}"))?
|
||||
.to_string();
|
||||
let val = ext
|
||||
.value()
|
||||
.map_err(|e| anyhow!("invalid PAX value: {e}"))?
|
||||
.to_string();
|
||||
pairs.push((key, val));
|
||||
}
|
||||
pairs
|
||||
}
|
||||
};
|
||||
Ok(Self {
|
||||
header,
|
||||
pax_extensions,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn path(&self) -> Result<Cow<'_, Path>> {
|
||||
self.header
|
||||
.path()
|
||||
.map_err(|e| anyhow!("reading path from header: {e}"))
|
||||
}
|
||||
|
||||
pub fn entry_type(&self) -> EntryType {
|
||||
self.header.entry_type()
|
||||
}
|
||||
|
||||
/// Return the link target path, preferring the PAX `linkpath` extension
|
||||
/// over the USTAR header field. This is necessary because `Header::link_name()`
|
||||
/// only reads the raw 100-byte USTAR field and will return a truncated path
|
||||
/// for targets longer than 100 bytes, whereas the PAX extension carries the
|
||||
/// full value.
|
||||
pub fn link_name(&self) -> Result<Option<PathBuf>> {
|
||||
// Check PAX extensions first.
|
||||
if let Some((_, v)) = self.pax_extensions.iter().find(|(k, _)| k == "linkpath") {
|
||||
return Ok(Some(PathBuf::from(v)));
|
||||
}
|
||||
// Fall back to the USTAR field.
|
||||
Ok(self
|
||||
.header
|
||||
.link_name()
|
||||
.map_err(|e| anyhow!("reading link_name from header: {e}"))?
|
||||
.map(|p| p.into_owned()))
|
||||
}
|
||||
|
||||
pub fn write_to_tar<W: Write, R: Read>(
|
||||
&self,
|
||||
path: &Path,
|
||||
data: R,
|
||||
builder: &mut Builder<W>,
|
||||
) -> Result<()> {
|
||||
if !self.pax_extensions.is_empty() {
|
||||
builder
|
||||
.append_pax_extensions(
|
||||
self.pax_extensions
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str(), v.as_bytes())),
|
||||
)
|
||||
.map_err(|e| anyhow!("failed to append PAX extensions: {e}"))?;
|
||||
}
|
||||
let mut header = self.header.clone();
|
||||
builder
|
||||
.append_data(&mut header, path, data)
|
||||
.map_err(|e| anyhow!("failed to append entry: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
117
oci_squashfs/src/image.rs
Normal file
117
oci_squashfs/src/image.rs
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
//! Parse OCI index.json + manifest, resolve layer blobs.
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use serde::Deserialize;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct OciIndex {
|
||||
pub manifests: Vec<OciDescriptor>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct OciDescriptor {
|
||||
pub digest: String,
|
||||
#[serde(rename = "mediaType", default)]
|
||||
pub media_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct OciManifest {
|
||||
pub layers: Vec<OciDescriptor>,
|
||||
}
|
||||
|
||||
/// One layer blob ready for processing.
|
||||
#[derive(Debug)]
|
||||
pub struct LayerBlob {
|
||||
pub path: PathBuf,
|
||||
pub media_type: String,
|
||||
pub index: usize,
|
||||
}
|
||||
|
||||
/// Load the OCI manifest from index.json → manifest blob.
|
||||
pub fn load_manifest(image_dir: &Path) -> Result<OciManifest> {
|
||||
// Try index.json first (OCI layout), fall back to manifest.json (Docker save).
|
||||
let index_path = image_dir.join("index.json");
|
||||
if index_path.exists() {
|
||||
let data = std::fs::read_to_string(&index_path)
|
||||
.with_context(|| format!("reading {}", index_path.display()))?;
|
||||
let index: OciIndex = serde_json::from_str(&data).context("parsing index.json")?;
|
||||
let desc = index
|
||||
.manifests
|
||||
.into_iter()
|
||||
.next()
|
||||
.context("index.json has no manifests")?;
|
||||
let digest = strip_digest_prefix(&desc.digest)?;
|
||||
let manifest_path = image_dir.join("blobs").join("sha256").join(digest);
|
||||
let mdata = std::fs::read_to_string(&manifest_path)
|
||||
.with_context(|| format!("reading manifest blob {}", manifest_path.display()))?;
|
||||
let manifest: OciManifest =
|
||||
serde_json::from_str(&mdata).context("parsing manifest blob")?;
|
||||
return Ok(manifest);
|
||||
}
|
||||
|
||||
// Docker save manifest.json
|
||||
let manifest_path = image_dir.join("manifest.json");
|
||||
if manifest_path.exists() {
|
||||
#[derive(Deserialize)]
|
||||
struct DockerManifest {
|
||||
#[serde(rename = "Layers")]
|
||||
layers: Vec<String>,
|
||||
}
|
||||
let data = std::fs::read_to_string(&manifest_path).context("reading manifest.json")?;
|
||||
let manifests: Vec<DockerManifest> =
|
||||
serde_json::from_str(&data).context("parsing manifest.json")?;
|
||||
let dm = manifests
|
||||
.into_iter()
|
||||
.next()
|
||||
.context("manifest.json is empty")?;
|
||||
let layers = dm
|
||||
.layers
|
||||
.into_iter()
|
||||
.map(|l| OciDescriptor {
|
||||
digest: l,
|
||||
media_type: "application/vnd.docker.image.rootfs.diff.tar.gzip".into(),
|
||||
})
|
||||
.collect();
|
||||
return Ok(OciManifest { layers });
|
||||
}
|
||||
|
||||
bail!(
|
||||
"no index.json or manifest.json found in {}",
|
||||
image_dir.display()
|
||||
);
|
||||
}
|
||||
|
||||
/// Resolve layer descriptors to actual file paths.
|
||||
pub fn resolve_layers(image_dir: &Path, manifest: &OciManifest) -> Result<Vec<LayerBlob>> {
|
||||
manifest
|
||||
.layers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, desc)| {
|
||||
let path = if desc.digest.contains(':') {
|
||||
// OCI digest: sha256:<hex>
|
||||
let hex = strip_digest_prefix(&desc.digest)?;
|
||||
image_dir.join("blobs").join("sha256").join(hex)
|
||||
} else {
|
||||
// Docker save: relative path like "abc123.../layer.tar"
|
||||
image_dir.join(&desc.digest)
|
||||
};
|
||||
if !path.exists() {
|
||||
bail!("layer blob not found: {}", path.display());
|
||||
}
|
||||
Ok(LayerBlob {
|
||||
path,
|
||||
media_type: desc.media_type.clone(),
|
||||
index: i,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn strip_digest_prefix(digest: &str) -> Result<&str> {
|
||||
digest
|
||||
.strip_prefix("sha256:")
|
||||
.with_context(|| format!("unsupported digest algorithm in: {digest}"))
|
||||
}
|
||||
40
oci_squashfs/src/layers.rs
Normal file
40
oci_squashfs/src/layers.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
//! Open a layer blob and return a decompressed tar::Archive.
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{self, BufReader, Read},
|
||||
path::Path,
|
||||
};
|
||||
use tar::Archive;
|
||||
|
||||
/// A type-erased decompressed tar reader.
|
||||
pub type DynArchive = Archive<Box<dyn Read + Send + 'static>>;
|
||||
|
||||
pub fn open_layer(path: &Path, media_type: &str) -> Result<DynArchive> {
|
||||
let file = File::open(path)?;
|
||||
let buf = BufReader::new(file);
|
||||
let reader: Box<dyn Read + Send + 'static> = match media_type {
|
||||
t if t.ends_with("+gzip") || t == "application/vnd.docker.image.rootfs.diff.tar.gzip" => {
|
||||
Box::new(flate2::read::GzDecoder::new(buf))
|
||||
}
|
||||
t if t.ends_with("+zstd") => Box::new(zstd::stream::read::Decoder::new(buf)?),
|
||||
t if t.ends_with("+bzip2") => Box::new(bzip2::read::BzDecoder::new(buf)),
|
||||
t if t.ends_with("+xz") || t.ends_with("+lzma") => Box::new(xz2::read::XzDecoder::new(buf)),
|
||||
"application/vnd.oci.image.layer.v1.tar" => Box::new(buf),
|
||||
other => bail!("unsupported layer media type: {other}"),
|
||||
};
|
||||
let mut archive = Archive::new(reader);
|
||||
archive.set_preserve_permissions(true);
|
||||
archive.set_preserve_mtime(true);
|
||||
archive.set_unpack_xattrs(true);
|
||||
Ok(archive)
|
||||
}
|
||||
|
||||
/// Read all raw bytes of a layer entry into memory.
|
||||
/// Returns (header_bytes, data_bytes) — header_bytes is the raw 512-byte block(s).
|
||||
pub fn read_entry_data(entry: &mut tar::Entry<impl Read>) -> Result<Vec<u8>> {
|
||||
let mut buf = Vec::new();
|
||||
io::copy(entry, &mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
26
oci_squashfs/src/lib.rs
Normal file
26
oci_squashfs/src/lib.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
pub mod canonical;
|
||||
pub mod image;
|
||||
pub mod layers;
|
||||
pub mod overlay;
|
||||
pub mod squashfs;
|
||||
pub mod tracker;
|
||||
pub mod verify;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::Path;
|
||||
|
||||
/// Convert an extracted OCI image directory into a squashfs file.
|
||||
///
|
||||
/// Layer content is streamed directly into mksquashfs's stdin — the merged
|
||||
/// tar is never fully materialised in memory.
|
||||
pub async fn convert(image_dir: &Path, output_squashfs: &Path) -> Result<()> {
|
||||
let image_dir = image_dir.to_path_buf();
|
||||
let output_squashfs = output_squashfs.to_path_buf();
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let manifest = image::load_manifest(&image_dir)?;
|
||||
let layers = image::resolve_layers(&image_dir, &manifest)?;
|
||||
squashfs::write_squashfs(layers, &output_squashfs)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
128
oci_squashfs/src/overlay.rs
Normal file
128
oci_squashfs/src/overlay.rs
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
//! Core algorithm: merge OCI layers into a single flat tar stream.
|
||||
//!
|
||||
//! File content is never buffered in full — entries are streamed directly
|
||||
//! into the provided `Write` sink (typically mksquashfs's stdin pipe).
|
||||
//! The only in-memory state is the tracker data structures and the small
|
||||
//! hard-link metadata structs deferred to the end.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::{
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use tar::{Builder, EntryType};
|
||||
|
||||
use crate::{
|
||||
canonical::CanonicalTarHeader,
|
||||
image::LayerBlob,
|
||||
layers::open_layer,
|
||||
tracker::{EmittedPathTracker, HardLinkTracker, WhiteoutTracker},
|
||||
};
|
||||
|
||||
/// Merge layers, streaming the resulting tar into `sink`.
|
||||
///
|
||||
/// `sink` is typically the stdin pipe of a `mksquashfs` subprocess.
|
||||
/// File data flows directly from the layer blobs into `sink` without
|
||||
/// being accumulated in memory.
|
||||
pub fn merge_layers_into<W: Write>(mut layers: Vec<LayerBlob>, sink: W) -> Result<()> {
|
||||
// Process in reverse (newest first).
|
||||
layers.sort_by_key(|l| std::cmp::Reverse(l.index));
|
||||
|
||||
let mut whiteout = WhiteoutTracker::default();
|
||||
let mut emitted = EmittedPathTracker::default();
|
||||
let mut hardlinks = HardLinkTracker::default();
|
||||
|
||||
let mut output = Builder::new(sink);
|
||||
output.mode(tar::HeaderMode::Complete);
|
||||
|
||||
for blob in &layers {
|
||||
let mut archive = open_layer(&blob.path, &blob.media_type)
|
||||
.with_context(|| format!("opening layer {}", blob.path.display()))?;
|
||||
|
||||
let entries = archive.entries().context("reading tar entries")?;
|
||||
for entry_result in entries {
|
||||
let mut entry = entry_result.context("reading tar entry")?;
|
||||
let raw_path = entry.path().context("entry path")?.into_owned();
|
||||
let path = normalize_path(&raw_path);
|
||||
|
||||
// 1. Check whiteout suppression.
|
||||
if whiteout.is_suppressed(&path, blob.index) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Check already-emitted.
|
||||
if emitted.contains(&path) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3. Handle whiteout entries.
|
||||
let file_name = path
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
if file_name == ".wh..wh..opq" {
|
||||
let parent = path.parent().unwrap_or(Path::new(""));
|
||||
whiteout.insert_opaque(parent, blob.index);
|
||||
continue;
|
||||
}
|
||||
if let Some(real_name) = file_name.strip_prefix(".wh.") {
|
||||
let parent = path.parent().unwrap_or(Path::new(""));
|
||||
whiteout.insert_simple(&parent.join(real_name), blob.index);
|
||||
continue;
|
||||
}
|
||||
|
||||
// We capture the header + PAX extensions first (cheap), then
|
||||
// stream the entry body straight into the Builder without
|
||||
// buffering it in a Vec.
|
||||
let canonical =
|
||||
CanonicalTarHeader::from_entry(&mut entry).context("capturing entry header")?;
|
||||
|
||||
// 4. Handle hard links — defer, emitting only metadata structs.
|
||||
if canonical.entry_type() == EntryType::Link {
|
||||
let link_target = canonical
|
||||
.link_name()
|
||||
.context("reading hard link target")?
|
||||
.context("hard link has no target")?;
|
||||
let target_path = normalize_path(&link_target);
|
||||
if !whiteout.is_suppressed(&target_path, blob.index) {
|
||||
hardlinks.record(path, target_path, blob.index, canonical);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 5. Stream entry directly into the output tar.
|
||||
//
|
||||
canonical
|
||||
.write_to_tar(&path, &mut entry, &mut output)
|
||||
.with_context(|| format!("emitting {}", path.display()))?;
|
||||
emitted.insert(&path);
|
||||
}
|
||||
}
|
||||
|
||||
// Emit deferred hard links (oldest-layer first).
|
||||
// These are pure metadata — no file content to stream.
|
||||
for hl in hardlinks.drain_sorted() {
|
||||
if !emitted.contains(&hl.target_path) {
|
||||
continue;
|
||||
}
|
||||
hl.canonical
|
||||
.write_to_tar(&hl.link_path, &[] as &[u8], &mut output)
|
||||
.with_context(|| format!("emitting hard link {}", hl.link_path.display()))?;
|
||||
emitted.insert(&hl.link_path);
|
||||
}
|
||||
|
||||
output.finish()?;
|
||||
// Flush and drop the Builder, closing the write end of the pipe so
|
||||
// mksquashfs sees EOF and knows the tar stream is complete.
|
||||
let mut sink = output.into_inner()?;
|
||||
sink.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Strip leading `./` or `/` from paths.
|
||||
pub fn normalize_path(p: &Path) -> PathBuf {
|
||||
let s = p.to_string_lossy();
|
||||
let s = s.trim_start_matches("./").trim_start_matches('/');
|
||||
PathBuf::from(s)
|
||||
}
|
||||
58
oci_squashfs/src/squashfs.rs
Normal file
58
oci_squashfs/src/squashfs.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
//! Spawn mksquashfs and stream the merged tar directly into its stdin.
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use std::{
|
||||
path::Path,
|
||||
process::{Command, Stdio},
|
||||
};
|
||||
|
||||
use crate::{image::LayerBlob, overlay::merge_layers_into};
|
||||
|
||||
/// Convert `layers` into a squashfs image at `output` by streaming a merged
|
||||
/// tar directly into mksquashfs's stdin. No full tar buffer is held in memory.
|
||||
pub fn write_squashfs(layers: Vec<LayerBlob>, output: &Path) -> Result<()> {
|
||||
if output.exists() {
|
||||
std::fs::remove_file(output)
|
||||
.with_context(|| format!("removing existing {}", output.display()))?;
|
||||
}
|
||||
|
||||
let mut child = Command::new("mksquashfs")
|
||||
.args([
|
||||
"-",
|
||||
output.to_str().context("output path is not UTF-8")?,
|
||||
"-tar",
|
||||
"-noappend",
|
||||
"-no-fragments",
|
||||
"-comp",
|
||||
"zstd",
|
||||
"-Xcompression-level",
|
||||
"2",
|
||||
"-quiet",
|
||||
])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.context("spawning mksquashfs — is it installed?")?;
|
||||
|
||||
let stdin = child.stdin.take().context("child stdin")?;
|
||||
|
||||
// Drive the merge on the current thread, writing directly into the pipe.
|
||||
// mksquashfs reads and compresses concurrently in its own process, so
|
||||
// the pipe provides natural backpressure without a helper thread.
|
||||
let merge_result = merge_layers_into(layers, stdin);
|
||||
|
||||
// Wait for mksquashfs regardless of whether the merge succeeded, so we
|
||||
// don't leave a zombie process behind.
|
||||
let exit = child.wait_with_output().context("waiting for mksquashfs")?;
|
||||
|
||||
// Surface merge errors before subprocess errors — they're more actionable.
|
||||
merge_result.context("merging layers into mksquashfs stdin")?;
|
||||
|
||||
if !exit.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&exit.stderr);
|
||||
bail!("mksquashfs failed:\n{stderr}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
139
oci_squashfs/src/tracker.rs
Normal file
139
oci_squashfs/src/tracker.rs
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
//! WhiteoutTracker, EmittedPathTracker, HardLinkTracker.
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
path::{Component, Path, PathBuf},
|
||||
};
|
||||
|
||||
use crate::canonical::CanonicalTarHeader;
|
||||
|
||||
// ─── WhiteoutTracker ────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum WhiteoutState {
|
||||
Simple { layer_index: usize },
|
||||
Opaque { layer_index: usize },
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct WhiteoutNode {
|
||||
state: Option<WhiteoutState>,
|
||||
children: HashMap<String, WhiteoutNode>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct WhiteoutTracker {
|
||||
root: WhiteoutNode,
|
||||
}
|
||||
|
||||
impl WhiteoutTracker {
|
||||
/// Mark a specific path as suppressed (simple `.wh.<name>` whiteout).
|
||||
pub fn insert_simple(&mut self, path: &Path, layer_index: usize) {
|
||||
let node = self.walk_or_create(path);
|
||||
node.state = Some(WhiteoutState::Simple { layer_index });
|
||||
}
|
||||
|
||||
/// Mark a directory path as opaque (`.wh..wh..opq`).
|
||||
pub fn insert_opaque(&mut self, dir_path: &Path, layer_index: usize) {
|
||||
let node = self.walk_or_create(dir_path);
|
||||
node.state = Some(WhiteoutState::Opaque { layer_index });
|
||||
}
|
||||
|
||||
/// Returns true if `path` from `current_layer` should be suppressed.
|
||||
pub fn is_suppressed(&self, path: &Path, current_layer: usize) -> bool {
|
||||
let components = normal_components(path);
|
||||
let mut node = &self.root;
|
||||
for (i, comp) in components.iter().enumerate() {
|
||||
if let Some(state) = &node.state {
|
||||
if let WhiteoutState::Opaque { layer_index } = state {
|
||||
if current_layer < *layer_index {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
match node.children.get(comp.as_str()) {
|
||||
Some(child) => node = child,
|
||||
None => return false,
|
||||
}
|
||||
// At terminal component: check Simple or Opaque.
|
||||
if i == components.len() - 1 {
|
||||
if let Some(state) = &node.state {
|
||||
match state {
|
||||
WhiteoutState::Simple { layer_index }
|
||||
| WhiteoutState::Opaque { layer_index } => {
|
||||
return current_layer < *layer_index;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn walk_or_create(&mut self, path: &Path) -> &mut WhiteoutNode {
|
||||
let components = normal_components(path);
|
||||
let mut node = &mut self.root;
|
||||
for comp in &components {
|
||||
node = node.children.entry(comp.clone()).or_default();
|
||||
}
|
||||
node
|
||||
}
|
||||
}
|
||||
|
||||
fn normal_components(path: &Path) -> Vec<String> {
|
||||
path.components()
|
||||
.filter_map(|c| match c {
|
||||
Component::Normal(s) => Some(s.to_string_lossy().into_owned()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ─── EmittedPathTracker ─────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct EmittedPathTracker(HashSet<PathBuf>);
|
||||
|
||||
impl EmittedPathTracker {
|
||||
pub fn insert(&mut self, path: &Path) {
|
||||
self.0.insert(path.to_path_buf());
|
||||
}
|
||||
pub fn contains(&self, path: &Path) -> bool {
|
||||
self.0.contains(path)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── HardLinkTracker ────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct HardLinkEntry {
|
||||
pub link_path: PathBuf,
|
||||
pub target_path: PathBuf,
|
||||
pub layer_index: usize,
|
||||
pub canonical: CanonicalTarHeader,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct HardLinkTracker(pub Vec<HardLinkEntry>);
|
||||
|
||||
impl HardLinkTracker {
|
||||
pub fn record(
|
||||
&mut self,
|
||||
link_path: PathBuf,
|
||||
target_path: PathBuf,
|
||||
layer_index: usize,
|
||||
canonical: CanonicalTarHeader,
|
||||
) {
|
||||
self.0.push(HardLinkEntry {
|
||||
link_path,
|
||||
target_path,
|
||||
layer_index,
|
||||
canonical,
|
||||
});
|
||||
}
|
||||
/// Return deferred links sorted by ascending layer index.
|
||||
pub fn drain_sorted(mut self) -> Vec<HardLinkEntry> {
|
||||
self.0.sort_by_key(|e| e.layer_index);
|
||||
self.0
|
||||
}
|
||||
}
|
||||
239
oci_squashfs/src/verify.rs
Normal file
239
oci_squashfs/src/verify.rs
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
//! Verification helper: mount squashfs and diff against a reference directory.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs,
|
||||
io::Read,
|
||||
os::unix::fs::{MetadataExt, PermissionsExt},
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
// ─── RAII mount guard ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Mounts a squashfs via squashfuse and unmounts it on drop, even if an error
|
||||
/// occurs during the walk. The underlying `TempDir` is kept alive for the
|
||||
/// lifetime of this guard so the mountpoint isn't deleted while still mounted.
|
||||
struct SquashMount {
|
||||
mountpoint: TempDir,
|
||||
}
|
||||
|
||||
impl SquashMount {
|
||||
fn new(squashfs: &Path) -> Result<Self> {
|
||||
let mountpoint = TempDir::new().context("creating temp mount dir")?;
|
||||
let status = Command::new("squashfuse")
|
||||
.arg(squashfs)
|
||||
.arg(mountpoint.path())
|
||||
.status()
|
||||
.context("spawning squashfuse — is it installed?")?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("squashfuse failed with status {status}");
|
||||
}
|
||||
Ok(Self { mountpoint })
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
self.mountpoint.path()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SquashMount {
|
||||
fn drop(&mut self) {
|
||||
// Try fusermount first (Linux), fall back to umount (macOS/BSD).
|
||||
let ok = Command::new("fusermount")
|
||||
.args(["-u", self.mountpoint.path().to_str().unwrap_or("")])
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false);
|
||||
|
||||
if !ok {
|
||||
let _ = Command::new("umount").arg(self.mountpoint.path()).status();
|
||||
}
|
||||
// TempDir::drop runs after this and removes the now-unmounted directory.
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Public API ───────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct VerifyReport {
|
||||
pub only_in_squashfs: Vec<PathBuf>,
|
||||
pub only_in_reference: Vec<PathBuf>,
|
||||
pub differences: Vec<FileDiff>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct FileDiff {
|
||||
pub path: PathBuf,
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
pub fn verify(squashfs: &Path, reference: &Path) -> Result<VerifyReport> {
|
||||
let mount = SquashMount::new(squashfs)?;
|
||||
|
||||
let squashfs_tree = walk_tree(mount.path()).context("walking squashfs mount")?;
|
||||
let reference_tree = walk_tree(reference).context("walking reference directory")?;
|
||||
|
||||
let mut report = VerifyReport {
|
||||
only_in_squashfs: Vec::new(),
|
||||
only_in_reference: Vec::new(),
|
||||
differences: Vec::new(),
|
||||
};
|
||||
|
||||
for (rel, sq_info) in &squashfs_tree {
|
||||
match reference_tree.get(rel) {
|
||||
None => report.only_in_squashfs.push(rel.clone()),
|
||||
Some(ref_info) => report
|
||||
.differences
|
||||
.extend(compare_entries(rel, sq_info, ref_info)),
|
||||
}
|
||||
}
|
||||
for rel in reference_tree.keys() {
|
||||
if !squashfs_tree.contains_key(rel) {
|
||||
report.only_in_reference.push(rel.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
// ─── Tree walking ─────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug)]
|
||||
struct EntryInfo {
|
||||
kind: EntryKind,
|
||||
mode: u32,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
size: u64,
|
||||
symlink_target: Option<PathBuf>,
|
||||
sha256: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
enum EntryKind {
|
||||
File,
|
||||
Dir,
|
||||
Symlink,
|
||||
Other,
|
||||
}
|
||||
|
||||
fn walk_tree(root: &Path) -> Result<HashMap<PathBuf, EntryInfo>> {
|
||||
let mut map = HashMap::new();
|
||||
walk_dir(root, root, &mut map)?;
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
fn walk_dir(root: &Path, current: &Path, map: &mut HashMap<PathBuf, EntryInfo>) -> Result<()> {
|
||||
for entry in
|
||||
fs::read_dir(current).with_context(|| format!("reading dir {}", current.display()))?
|
||||
{
|
||||
let entry = entry?;
|
||||
let abs = entry.path();
|
||||
let rel = abs
|
||||
.strip_prefix(root)
|
||||
.context("strip prefix")?
|
||||
.to_path_buf();
|
||||
let meta = fs::symlink_metadata(&abs)
|
||||
.with_context(|| format!("metadata for {}", abs.display()))?;
|
||||
let ft = meta.file_type();
|
||||
|
||||
let (kind, symlink_target, sha256) = if ft.is_symlink() {
|
||||
(EntryKind::Symlink, Some(fs::read_link(&abs)?), None)
|
||||
} else if ft.is_file() {
|
||||
(EntryKind::File, None, Some(hash_file(&abs)?))
|
||||
} else if ft.is_dir() {
|
||||
(EntryKind::Dir, None, None)
|
||||
} else {
|
||||
(EntryKind::Other, None, None)
|
||||
};
|
||||
|
||||
// Mask to permission bits (rwxrwxrwx + setuid/setgid/sticky).
|
||||
// File type bits are captured separately in `kind`.
|
||||
const PERMISSION_BITS: u32 = 0o7777;
|
||||
|
||||
map.insert(
|
||||
rel,
|
||||
EntryInfo {
|
||||
kind,
|
||||
mode: meta.permissions().mode() & PERMISSION_BITS,
|
||||
uid: meta.uid(),
|
||||
gid: meta.gid(),
|
||||
size: meta.len(),
|
||||
symlink_target,
|
||||
sha256,
|
||||
},
|
||||
);
|
||||
|
||||
if ft.is_dir() {
|
||||
walk_dir(root, &abs, map)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn hash_file(path: &Path) -> Result<String> {
|
||||
let mut file = fs::File::open(path)?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buf = [0u8; 8192];
|
||||
loop {
|
||||
let n = file.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
}
|
||||
|
||||
// ─── Comparison ───────────────────────────────────────────────────────────────
|
||||
|
||||
fn compare_entries(rel: &Path, sq: &EntryInfo, rf: &EntryInfo) -> Vec<FileDiff> {
|
||||
let mut diffs = Vec::new();
|
||||
macro_rules! diff {
|
||||
($msg:expr) => {
|
||||
diffs.push(FileDiff {
|
||||
path: rel.to_path_buf(),
|
||||
detail: $msg,
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
if sq.kind != rf.kind {
|
||||
diff!(format!(
|
||||
"type mismatch: squashfs={:?} ref={:?}",
|
||||
sq.kind, rf.kind
|
||||
));
|
||||
return diffs;
|
||||
}
|
||||
if sq.mode != rf.mode {
|
||||
diff!(format!(
|
||||
"mode: squashfs={:04o} ref={:04o}",
|
||||
sq.mode, rf.mode
|
||||
));
|
||||
}
|
||||
if sq.uid != rf.uid {
|
||||
diff!(format!("uid: squashfs={} ref={}", sq.uid, rf.uid));
|
||||
}
|
||||
if sq.gid != rf.gid {
|
||||
diff!(format!("gid: squashfs={} ref={}", sq.gid, rf.gid));
|
||||
}
|
||||
if sq.symlink_target != rf.symlink_target {
|
||||
diff!(format!(
|
||||
"symlink target: squashfs={:?} ref={:?}",
|
||||
sq.symlink_target, rf.symlink_target
|
||||
));
|
||||
}
|
||||
if sq.kind == EntryKind::File {
|
||||
if sq.size != rf.size {
|
||||
diff!(format!("size: squashfs={} ref={}", sq.size, rf.size));
|
||||
}
|
||||
if sq.sha256 != rf.sha256 {
|
||||
diff!(format!("sha256 mismatch"));
|
||||
}
|
||||
}
|
||||
diffs
|
||||
}
|
||||
212
oci_squashfs/tests/helpers/mod.rs
Normal file
212
oci_squashfs/tests/helpers/mod.rs
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
//! Shared test helpers for integration and regression tests.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use oci_squashfs::canonical::CanonicalTarHeader;
|
||||
use oci_squashfs::image::LayerBlob;
|
||||
use std::io::{Cursor, Write};
|
||||
use tar::{Archive, Builder, EntryType, Header};
|
||||
|
||||
// ─── LayerBuilder ────────────────────────────────────────────────────────────
|
||||
|
||||
pub struct LayerBuilder {
|
||||
inner: Builder<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl LayerBuilder {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
inner: Builder::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_file(mut self, path: &str, data: &[u8], mode: u32) -> Self {
|
||||
if path.len() > 99 {
|
||||
self.inner
|
||||
.append_pax_extensions([("path", path.as_bytes())])
|
||||
.unwrap();
|
||||
}
|
||||
let mut hdr = Header::new_ustar();
|
||||
hdr.set_path(truncate(path, 99)).unwrap();
|
||||
hdr.set_size(data.len() as u64);
|
||||
hdr.set_mode(mode);
|
||||
hdr.set_mtime(0);
|
||||
hdr.set_uid(0);
|
||||
hdr.set_gid(0);
|
||||
hdr.set_cksum();
|
||||
self.inner.append(&hdr, Cursor::new(data)).unwrap();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_dir(mut self, path: &str) -> Self {
|
||||
if path.len() > 99 {
|
||||
self.inner
|
||||
.append_pax_extensions([("path", path.as_bytes())])
|
||||
.unwrap();
|
||||
}
|
||||
let mut hdr = Header::new_ustar();
|
||||
hdr.set_path(truncate(path, 99)).unwrap();
|
||||
hdr.set_entry_type(EntryType::Directory);
|
||||
hdr.set_size(0);
|
||||
hdr.set_mode(0o755);
|
||||
hdr.set_mtime(0);
|
||||
hdr.set_uid(0);
|
||||
hdr.set_gid(0);
|
||||
hdr.set_cksum();
|
||||
self.inner.append(&hdr, Cursor::new(b"" as &[u8])).unwrap();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_symlink(mut self, path: &str, target: &str) -> Self {
|
||||
let mut pax: Vec<(&str, &[u8])> = Vec::new();
|
||||
if path.len() > 99 {
|
||||
pax.push(("path", path.as_bytes()));
|
||||
}
|
||||
if target.len() > 99 {
|
||||
pax.push(("linkpath", target.as_bytes()));
|
||||
}
|
||||
if !pax.is_empty() {
|
||||
self.inner.append_pax_extensions(pax).unwrap();
|
||||
}
|
||||
let mut hdr = Header::new_ustar();
|
||||
hdr.set_path(truncate(path, 99)).unwrap();
|
||||
hdr.set_entry_type(EntryType::Symlink);
|
||||
hdr.set_link_name(truncate(target, 99)).ok();
|
||||
hdr.set_size(0);
|
||||
hdr.set_mode(0o777);
|
||||
hdr.set_mtime(0);
|
||||
hdr.set_uid(0);
|
||||
hdr.set_gid(0);
|
||||
hdr.set_cksum();
|
||||
self.inner.append(&hdr, Cursor::new(b"" as &[u8])).unwrap();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_hardlink(mut self, path: &str, target: &str) -> Self {
|
||||
let mut pax: Vec<(&str, &[u8])> = Vec::new();
|
||||
if path.len() > 99 {
|
||||
pax.push(("path", path.as_bytes()));
|
||||
}
|
||||
if target.len() > 99 {
|
||||
pax.push(("linkpath", target.as_bytes()));
|
||||
}
|
||||
if !pax.is_empty() {
|
||||
self.inner.append_pax_extensions(pax).unwrap();
|
||||
}
|
||||
let mut hdr = Header::new_ustar();
|
||||
hdr.set_path(truncate(path, 99)).unwrap();
|
||||
hdr.set_entry_type(EntryType::Link);
|
||||
hdr.set_link_name(truncate(target, 99)).ok();
|
||||
hdr.set_size(0);
|
||||
hdr.set_mode(0o644);
|
||||
hdr.set_mtime(0);
|
||||
hdr.set_uid(0);
|
||||
hdr.set_gid(0);
|
||||
hdr.set_cksum();
|
||||
self.inner.append(&hdr, Cursor::new(b"" as &[u8])).unwrap();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn add_whiteout(self, dir: &str, name: &str) -> Self {
|
||||
let path = if dir.is_empty() {
|
||||
format!(".wh.{name}")
|
||||
} else {
|
||||
format!("{dir}/.wh.{name}")
|
||||
};
|
||||
self.add_file(&path, b"", 0o644)
|
||||
}
|
||||
|
||||
pub fn add_opaque_whiteout(self, dir: &str) -> Self {
|
||||
self.add_file(&format!("{dir}/.wh..wh..opq"), b"", 0o644)
|
||||
}
|
||||
|
||||
pub fn finish(mut self) -> Vec<u8> {
|
||||
self.inner.finish().unwrap();
|
||||
self.inner.into_inner().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
/// Truncate a string to at most `max_chars` characters.
|
||||
fn truncate(s: &str, max_chars: usize) -> &str {
|
||||
match s.char_indices().nth(max_chars) {
|
||||
Some((idx, _)) => &s[..idx],
|
||||
None => s,
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Blob / merge helpers ─────────────────────────────────────────────────────
|
||||
|
||||
pub fn blob(bytes: Vec<u8>, index: usize) -> LayerBlob {
|
||||
use tempfile::NamedTempFile;
|
||||
let mut f = NamedTempFile::new().unwrap();
|
||||
f.write_all(&bytes).unwrap();
|
||||
let (_, path) = f.keep().unwrap();
|
||||
LayerBlob {
|
||||
path,
|
||||
media_type: "application/vnd.oci.image.layer.v1.tar".into(),
|
||||
index,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge(layers: Vec<LayerBlob>) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
oci_squashfs::overlay::merge_layers_into(layers, &mut out).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
// ─── Tar inspection helpers ───────────────────────────────────────────────────
|
||||
|
||||
pub fn paths_in_tar(tar_bytes: &[u8]) -> Vec<String> {
|
||||
let mut archive = Archive::new(Cursor::new(tar_bytes));
|
||||
archive
|
||||
.entries()
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
!matches!(
|
||||
e.header().entry_type(),
|
||||
EntryType::XHeader | EntryType::XGlobalHeader
|
||||
)
|
||||
})
|
||||
.map(|e| e.path().unwrap().to_string_lossy().into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Read the symlink target for `link_path`, preferring PAX `linkpath`.
|
||||
pub fn symlink_target_in_tar(tar_bytes: &[u8], link_path: &str) -> Option<String> {
|
||||
let mut archive = Archive::new(Cursor::new(tar_bytes));
|
||||
for mut entry in archive.entries().unwrap().flatten() {
|
||||
let canonical = CanonicalTarHeader::from_entry(&mut entry).ok()?;
|
||||
if canonical.entry_type() != EntryType::Symlink {
|
||||
continue;
|
||||
}
|
||||
if canonical.path().unwrap().to_string_lossy() != link_path {
|
||||
continue;
|
||||
}
|
||||
return canonical
|
||||
.link_name()
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|p| p.to_string_lossy().into_owned());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Read the hard link target for `link_path`, preferring PAX `linkpath`.
|
||||
pub fn hardlink_target_in_tar(tar_bytes: &[u8], link_path: &str) -> Option<String> {
|
||||
let mut archive = Archive::new(Cursor::new(tar_bytes));
|
||||
for mut entry in archive.entries().unwrap().flatten() {
|
||||
let canonical = CanonicalTarHeader::from_entry(&mut entry).ok()?;
|
||||
if canonical.entry_type() != EntryType::Link {
|
||||
continue;
|
||||
}
|
||||
if canonical.path().unwrap().to_string_lossy() != link_path {
|
||||
continue;
|
||||
}
|
||||
return canonical
|
||||
.link_name()
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|p| p.to_string_lossy().into_owned());
|
||||
}
|
||||
None
|
||||
}
|
||||
110
oci_squashfs/tests/integration.rs
Normal file
110
oci_squashfs/tests/integration.rs
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
//! Synthetic integration tests for the OCI → squashfs merge pipeline.
|
||||
//! These tests exercise `overlay::merge_layers_into` directly on in-memory
|
||||
//! blobs and inspect the resulting merged tar without invoking mksquashfs.
|
||||
|
||||
#[path = "helpers/mod.rs"]
|
||||
mod helpers;
|
||||
use helpers::{
|
||||
blob, hardlink_target_in_tar, merge, paths_in_tar, symlink_target_in_tar, LayerBuilder,
|
||||
};
|
||||
|
||||
// ─── Tests ───────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_long_symlink_pax_preserved() {
|
||||
let long_target: String = "a".repeat(200);
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_symlink("link_to_long", &long_target)
|
||||
.finish();
|
||||
let merged = merge(vec![blob(layer0, 0)]);
|
||||
assert_eq!(
|
||||
symlink_target_in_tar(&merged, "link_to_long").as_deref(),
|
||||
Some(long_target.as_str()),
|
||||
"long symlink target must round-trip via PAX"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_long_hardlink_pax_preserved() {
|
||||
let long_target: String = "a".repeat(200);
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_file(&long_target, b"hello", 0o644)
|
||||
.add_hardlink("link_to_long", &long_target)
|
||||
.finish();
|
||||
let merged = merge(vec![blob(layer0, 0)]);
|
||||
assert_eq!(
|
||||
hardlink_target_in_tar(&merged, "link_to_long").as_deref(),
|
||||
Some(long_target.as_str()),
|
||||
"long hardlink target must round-trip via PAX"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hardlink_across_layers() {
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_file("original.txt", b"hello", 0o644)
|
||||
.finish();
|
||||
let layer1 = LayerBuilder::new()
|
||||
.add_hardlink("link.txt", "original.txt")
|
||||
.finish();
|
||||
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
|
||||
let paths = paths_in_tar(&merged);
|
||||
assert!(paths.contains(&"original.txt".to_string()));
|
||||
assert!(paths.contains(&"link.txt".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_simple_whiteout() {
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_file("secret.txt", b"private", 0o644)
|
||||
.finish();
|
||||
let layer1 = LayerBuilder::new().add_whiteout("", "secret.txt").finish();
|
||||
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
|
||||
assert!(!paths_in_tar(&merged).iter().any(|p| p == "secret.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_opaque_whiteout() {
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_dir("mydir")
|
||||
.add_file("mydir/old.txt", b"old", 0o644)
|
||||
.finish();
|
||||
let layer1 = LayerBuilder::new()
|
||||
.add_dir("mydir")
|
||||
.add_opaque_whiteout("mydir")
|
||||
.finish();
|
||||
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
|
||||
assert!(!paths_in_tar(&merged).iter().any(|p| p == "mydir/old.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_opaque_whiteout_with_repopulation() {
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_dir("dir")
|
||||
.add_file("dir/old.txt", b"old", 0o644)
|
||||
.finish();
|
||||
let layer1 = LayerBuilder::new()
|
||||
.add_dir("dir")
|
||||
.add_opaque_whiteout("dir")
|
||||
.add_file("dir/new.txt", b"new", 0o644)
|
||||
.finish();
|
||||
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
|
||||
let paths = paths_in_tar(&merged);
|
||||
assert!(paths.iter().any(|p| p == "dir/new.txt"));
|
||||
assert!(!paths.iter().any(|p| p == "dir/old.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hardlink_to_whiteout_target_dropped() {
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_file("gone.txt", b"data", 0o644)
|
||||
.finish();
|
||||
let layer1 = LayerBuilder::new()
|
||||
.add_whiteout("", "gone.txt")
|
||||
.add_hardlink("link.txt", "gone.txt")
|
||||
.finish();
|
||||
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
|
||||
let paths = paths_in_tar(&merged);
|
||||
assert!(!paths.iter().any(|p| p == "gone.txt"));
|
||||
assert!(!paths.iter().any(|p| p == "link.txt"));
|
||||
}
|
||||
165
oci_squashfs/tests/regression.rs
Normal file
165
oci_squashfs/tests/regression.rs
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
//! Regression tests, one per production bug found during verify runs.
|
||||
//! Each test is named after the bug it guards against and has a comment
|
||||
//! explaining the root cause and how it was discovered.
|
||||
|
||||
use std::io::Cursor;
|
||||
use tar::{Builder, EntryType, Header};
|
||||
|
||||
#[path = "helpers/mod.rs"]
|
||||
mod helpers;
|
||||
use helpers::{blob, merge, paths_in_tar, LayerBuilder};
|
||||
// regression.rs also uses:
|
||||
use helpers::hardlink_target_in_tar;
|
||||
|
||||
// ─── Regression tests ────────────────────────────────────────────────────────
|
||||
|
||||
/// Bug: whiteout suppression condition was inverted (`current_layer > layer_index`
|
||||
/// instead of `current_layer < layer_index`), causing whiteouts to suppress
|
||||
/// entries from *newer* layers rather than older ones. Files removed via
|
||||
/// `.wh.<name>` in a later layer were still appearing in the output.
|
||||
///
|
||||
/// Discovered via: `usr/share/vulkan/icd.d/lvp_icd.json` appearing in squashfs
|
||||
/// output despite being removed by a dpkg-divert whiteout in a later layer.
|
||||
#[test]
|
||||
fn regress_whiteout_suppression_direction() {
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_file("usr/share/vulkan/icd.d/lvp_icd.json", b"data", 0o644)
|
||||
.finish();
|
||||
// Layer 1 whiteouts the file from layer 0.
|
||||
let layer1 = LayerBuilder::new()
|
||||
.add_whiteout("usr/share/vulkan/icd.d", "lvp_icd.json")
|
||||
.finish();
|
||||
|
||||
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
|
||||
let paths = paths_in_tar(&merged);
|
||||
assert!(
|
||||
!paths
|
||||
.iter()
|
||||
.any(|p| p == "usr/share/vulkan/icd.d/lvp_icd.json"),
|
||||
"whited-out file must not appear: got paths {paths:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Bug: hard link targets longer than 100 bytes were being silently truncated
|
||||
/// because we read them via `entry.header().link_name()`, which only reads the
|
||||
/// raw 100-byte USTAR `linkname` field. The PAX `linkpath` extension carrying
|
||||
/// the full target was ignored. The truncated target path was then not found in
|
||||
/// the emitted-path tracker, so the link was silently dropped.
|
||||
///
|
||||
/// Discovered via: PostgreSQL/hammerdb timezone hard-link aliases all missing
|
||||
/// from squashfs output (e.g. `timezone/Jamaica -> .../America/...` dropped
|
||||
/// because the full target path exceeded 100 bytes and was truncated to
|
||||
/// `America` or similar).
|
||||
#[test]
|
||||
fn regress_long_hardlink_target_truncated() {
|
||||
// Construct a target path that exceeds 100 bytes.
|
||||
let long_dir = "a".repeat(95);
|
||||
let target = format!("{long_dir}/canonical_file");
|
||||
assert!(
|
||||
target.len() > 100,
|
||||
"test setup: target must exceed 100 bytes"
|
||||
);
|
||||
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_dir(&long_dir)
|
||||
.add_file(&target, b"content", 0o644)
|
||||
.add_hardlink("alias_file", &target)
|
||||
.finish();
|
||||
|
||||
let merged = merge(vec![blob(layer0, 0)]);
|
||||
let paths = paths_in_tar(&merged);
|
||||
|
||||
assert!(
|
||||
paths.iter().any(|p| p == &target),
|
||||
"canonical target must be present"
|
||||
);
|
||||
assert!(
|
||||
paths.iter().any(|p| p == "alias_file"),
|
||||
"hard link alias must be present; was it silently dropped due to truncated target?"
|
||||
);
|
||||
|
||||
let resolved = hardlink_target_in_tar(&merged, "alias_file");
|
||||
assert_eq!(
|
||||
resolved.as_deref(),
|
||||
Some(target.as_str()),
|
||||
"hard link target must be the full untruncated path"
|
||||
);
|
||||
}
|
||||
|
||||
/// Variant of the above: hard link and target in different layers, with a long
|
||||
/// target path. Guards against the combination of cross-layer deferral and PAX
|
||||
/// target resolution both being required simultaneously.
|
||||
#[test]
|
||||
fn regress_long_hardlink_target_cross_layer() {
|
||||
let long_dir = "b".repeat(60);
|
||||
let target = format!("{long_dir}/canonical_file");
|
||||
|
||||
let layer0 = LayerBuilder::new()
|
||||
.add_dir(&long_dir)
|
||||
.add_file(&target, b"content", 0o644)
|
||||
.finish();
|
||||
let layer1 = LayerBuilder::new()
|
||||
.add_hardlink("alias_file", &target)
|
||||
.finish();
|
||||
|
||||
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
|
||||
let paths = paths_in_tar(&merged);
|
||||
|
||||
assert!(
|
||||
paths.iter().any(|p| p == &target),
|
||||
"canonical target must be present"
|
||||
);
|
||||
assert!(
|
||||
paths.iter().any(|p| p == "alias_file"),
|
||||
"cross-layer hard link with long target must not be silently dropped"
|
||||
);
|
||||
}
|
||||
|
||||
/// Bug: normalize_path only stripped leading `./` but not leading `/`. Hard
|
||||
/// link targets stored as absolute paths in the tar (e.g. `/usr/share/foo`)
|
||||
/// would not match the normalized emitted path (`usr/share/foo`), causing the
|
||||
/// link to be silently dropped.
|
||||
#[test]
|
||||
fn regress_absolute_hardlink_target_normalized() {
|
||||
// Manually construct a layer with an absolute-path hard link target,
|
||||
// which some tar producers emit.
|
||||
let mut builder = Builder::new(Vec::new());
|
||||
|
||||
let mut file_hdr = Header::new_ustar();
|
||||
file_hdr.set_path("usr/share/foo").unwrap();
|
||||
file_hdr.set_size(4);
|
||||
file_hdr.set_mode(0o644);
|
||||
file_hdr.set_mtime(0);
|
||||
file_hdr.set_uid(0);
|
||||
file_hdr.set_gid(0);
|
||||
file_hdr.set_cksum();
|
||||
builder.append(&file_hdr, Cursor::new(b"data")).unwrap();
|
||||
|
||||
// Use a PAX linkpath with a leading slash — the absolute form.
|
||||
builder
|
||||
.append_pax_extensions([("linkpath", b"/usr/share/foo" as &[u8])])
|
||||
.unwrap();
|
||||
let mut link_hdr = Header::new_ustar();
|
||||
link_hdr.set_path("usr/share/bar").unwrap();
|
||||
link_hdr.set_entry_type(EntryType::Link);
|
||||
link_hdr.set_link_name("usr/share/foo").ok(); // truncated USTAR field (no leading slash here)
|
||||
link_hdr.set_size(0);
|
||||
link_hdr.set_mode(0o644);
|
||||
link_hdr.set_mtime(0);
|
||||
link_hdr.set_uid(0);
|
||||
link_hdr.set_gid(0);
|
||||
link_hdr.set_cksum();
|
||||
builder
|
||||
.append(&link_hdr, Cursor::new(b"" as &[u8]))
|
||||
.unwrap();
|
||||
|
||||
builder.finish().unwrap();
|
||||
let layer0 = builder.into_inner().unwrap();
|
||||
|
||||
let merged = merge(vec![blob(layer0, 0)]);
|
||||
let paths = paths_in_tar(&merged);
|
||||
assert!(
|
||||
paths.iter().any(|p| p == "usr/share/bar"),
|
||||
"hard link with absolute PAX linkpath must not be dropped after normalization"
|
||||
);
|
||||
}
|
||||
14
oci_squashfs_cli/Cargo.toml
Normal file
14
oci_squashfs_cli/Cargo.toml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
[package]
|
||||
name = "oci_squashfs_cli"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "oci2squashfs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
oci_squashfs = { path = "../oci_squashfs" }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
anyhow = "1"
|
||||
73
oci_squashfs_cli/src/main.rs
Normal file
73
oci_squashfs_cli/src/main.rs
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "oci2squashfs", about = "Convert an OCI image to squashfs")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Convert an OCI image directory to a squashfs image.
|
||||
Convert {
|
||||
/// Path to the extracted OCI image directory.
|
||||
#[arg(short, long)]
|
||||
image: PathBuf,
|
||||
/// Output squashfs file path.
|
||||
#[arg(short, long)]
|
||||
output: PathBuf,
|
||||
},
|
||||
/// Verify a squashfs image against a reference directory.
|
||||
Verify {
|
||||
/// Path to the .squashfs file.
|
||||
#[arg(short, long)]
|
||||
squashfs: PathBuf,
|
||||
/// Path to the reference directory (e.g. containerd-unpacked rootfs).
|
||||
#[arg(short, long)]
|
||||
reference: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
match cli.command {
|
||||
Commands::Convert { image, output } => {
|
||||
println!("Converting {} → {}", image.display(), output.display());
|
||||
oci_squashfs::convert(&image, &output).await?;
|
||||
println!("Done: {}", output.display());
|
||||
}
|
||||
Commands::Verify {
|
||||
squashfs,
|
||||
reference,
|
||||
} => {
|
||||
let report = tokio::task::spawn_blocking(move || {
|
||||
oci_squashfs::verify::verify(&squashfs, &reference)
|
||||
})
|
||||
.await??;
|
||||
|
||||
if report.only_in_squashfs.is_empty()
|
||||
&& report.only_in_reference.is_empty()
|
||||
&& report.differences.is_empty()
|
||||
{
|
||||
println!("✓ No differences found.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for p in &report.only_in_squashfs {
|
||||
println!("+ squashfs only: {}", p.display());
|
||||
}
|
||||
for p in &report.only_in_reference {
|
||||
println!("- reference only: {}", p.display());
|
||||
}
|
||||
for d in &report.differences {
|
||||
println!("~ {}: {}", d.path.display(), d.detail);
|
||||
}
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue