1
0
Fork 0
mirror of https://github.com/imjasonh/oci2squashfs synced 2026-07-07 00:33:35 +00:00

initial implementation

Signed-off-by: Steven Noonan <steven@edera.dev>
This commit is contained in:
Steven Noonan 2026-03-08 04:21:01 -07:00
parent 61d3f40cc1
commit 8ffd6583d4
No known key found for this signature in database
GPG key ID: 408EEB508ED0CD4D
18 changed files with 2448 additions and 0 deletions

7
.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,7 @@
version: 2
updates:
- package-ecosystem: "cargo"
directory: "/"
schedule:
interval: "weekly"

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/target

1002
Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

3
Cargo.toml Normal file
View file

@ -0,0 +1,3 @@
[workspace]
members = ["oci_squashfs", "oci_squashfs_cli"]
resolver = "2"

20
oci_squashfs/Cargo.toml Normal file
View file

@ -0,0 +1,20 @@
[package]
name = "oci_squashfs"
version = "0.1.0"
edition = "2021"
[dependencies]
tokio = { version = "1", features = ["full"] }
tar = "0.4"
flate2 = { version = "1", features = ["zlib-rs"], default-features = false }
zstd = "0.13"
bzip2 = "0.6"
xz2 = "0.1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "0.10"
tempfile = "3"
anyhow = "1"
[dev-dependencies]
tokio = { version = "1", features = ["full"] }

View file

@ -0,0 +1,94 @@
//! CanonicalTarHeader: a tar Header paired with its PAX extensions.
//! Ported from production vfs.rs — lets the `tar` crate own PAX serialization.
use anyhow::{anyhow, Result};
use std::borrow::Cow;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use tar::{Builder, EntryType, Header};
#[derive(Clone, Debug)]
pub struct CanonicalTarHeader {
pub header: Header,
pub pax_extensions: Vec<(String, String)>,
}
impl CanonicalTarHeader {
pub fn from_entry<R: Read>(entry: &mut tar::Entry<'_, R>) -> Result<Self> {
let header = entry.header().clone();
let pax_extensions = match entry.pax_extensions() {
Err(e) => return Err(anyhow!("failed to read PAX extensions: {e}")),
Ok(None) => vec![],
Ok(Some(exts)) => {
let mut pairs = Vec::new();
for ext in exts {
let ext = ext.map_err(|e| anyhow!("invalid PAX extension: {e}"))?;
let key = ext
.key()
.map_err(|e| anyhow!("invalid PAX key: {e}"))?
.to_string();
let val = ext
.value()
.map_err(|e| anyhow!("invalid PAX value: {e}"))?
.to_string();
pairs.push((key, val));
}
pairs
}
};
Ok(Self {
header,
pax_extensions,
})
}
pub fn path(&self) -> Result<Cow<'_, Path>> {
self.header
.path()
.map_err(|e| anyhow!("reading path from header: {e}"))
}
pub fn entry_type(&self) -> EntryType {
self.header.entry_type()
}
/// Return the link target path, preferring the PAX `linkpath` extension
/// over the USTAR header field. This is necessary because `Header::link_name()`
/// only reads the raw 100-byte USTAR field and will return a truncated path
/// for targets longer than 100 bytes, whereas the PAX extension carries the
/// full value.
pub fn link_name(&self) -> Result<Option<PathBuf>> {
// Check PAX extensions first.
if let Some((_, v)) = self.pax_extensions.iter().find(|(k, _)| k == "linkpath") {
return Ok(Some(PathBuf::from(v)));
}
// Fall back to the USTAR field.
Ok(self
.header
.link_name()
.map_err(|e| anyhow!("reading link_name from header: {e}"))?
.map(|p| p.into_owned()))
}
pub fn write_to_tar<W: Write, R: Read>(
&self,
path: &Path,
data: R,
builder: &mut Builder<W>,
) -> Result<()> {
if !self.pax_extensions.is_empty() {
builder
.append_pax_extensions(
self.pax_extensions
.iter()
.map(|(k, v)| (k.as_str(), v.as_bytes())),
)
.map_err(|e| anyhow!("failed to append PAX extensions: {e}"))?;
}
let mut header = self.header.clone();
builder
.append_data(&mut header, path, data)
.map_err(|e| anyhow!("failed to append entry: {e}"))?;
Ok(())
}
}

117
oci_squashfs/src/image.rs Normal file
View file

@ -0,0 +1,117 @@
//! Parse OCI index.json + manifest, resolve layer blobs.
use anyhow::{bail, Context, Result};
use serde::Deserialize;
use std::path::{Path, PathBuf};
#[derive(Debug, Deserialize)]
pub struct OciIndex {
pub manifests: Vec<OciDescriptor>,
}
#[derive(Debug, Deserialize)]
pub struct OciDescriptor {
pub digest: String,
#[serde(rename = "mediaType", default)]
pub media_type: String,
}
#[derive(Debug, Deserialize)]
pub struct OciManifest {
pub layers: Vec<OciDescriptor>,
}
/// One layer blob ready for processing.
#[derive(Debug)]
pub struct LayerBlob {
pub path: PathBuf,
pub media_type: String,
pub index: usize,
}
/// Load the OCI manifest from index.json → manifest blob.
pub fn load_manifest(image_dir: &Path) -> Result<OciManifest> {
// Try index.json first (OCI layout), fall back to manifest.json (Docker save).
let index_path = image_dir.join("index.json");
if index_path.exists() {
let data = std::fs::read_to_string(&index_path)
.with_context(|| format!("reading {}", index_path.display()))?;
let index: OciIndex = serde_json::from_str(&data).context("parsing index.json")?;
let desc = index
.manifests
.into_iter()
.next()
.context("index.json has no manifests")?;
let digest = strip_digest_prefix(&desc.digest)?;
let manifest_path = image_dir.join("blobs").join("sha256").join(digest);
let mdata = std::fs::read_to_string(&manifest_path)
.with_context(|| format!("reading manifest blob {}", manifest_path.display()))?;
let manifest: OciManifest =
serde_json::from_str(&mdata).context("parsing manifest blob")?;
return Ok(manifest);
}
// Docker save manifest.json
let manifest_path = image_dir.join("manifest.json");
if manifest_path.exists() {
#[derive(Deserialize)]
struct DockerManifest {
#[serde(rename = "Layers")]
layers: Vec<String>,
}
let data = std::fs::read_to_string(&manifest_path).context("reading manifest.json")?;
let manifests: Vec<DockerManifest> =
serde_json::from_str(&data).context("parsing manifest.json")?;
let dm = manifests
.into_iter()
.next()
.context("manifest.json is empty")?;
let layers = dm
.layers
.into_iter()
.map(|l| OciDescriptor {
digest: l,
media_type: "application/vnd.docker.image.rootfs.diff.tar.gzip".into(),
})
.collect();
return Ok(OciManifest { layers });
}
bail!(
"no index.json or manifest.json found in {}",
image_dir.display()
);
}
/// Resolve layer descriptors to actual file paths.
pub fn resolve_layers(image_dir: &Path, manifest: &OciManifest) -> Result<Vec<LayerBlob>> {
manifest
.layers
.iter()
.enumerate()
.map(|(i, desc)| {
let path = if desc.digest.contains(':') {
// OCI digest: sha256:<hex>
let hex = strip_digest_prefix(&desc.digest)?;
image_dir.join("blobs").join("sha256").join(hex)
} else {
// Docker save: relative path like "abc123.../layer.tar"
image_dir.join(&desc.digest)
};
if !path.exists() {
bail!("layer blob not found: {}", path.display());
}
Ok(LayerBlob {
path,
media_type: desc.media_type.clone(),
index: i,
})
})
.collect()
}
pub fn strip_digest_prefix(digest: &str) -> Result<&str> {
digest
.strip_prefix("sha256:")
.with_context(|| format!("unsupported digest algorithm in: {digest}"))
}

View file

@ -0,0 +1,40 @@
//! Open a layer blob and return a decompressed tar::Archive.
use anyhow::{bail, Result};
use std::{
fs::File,
io::{self, BufReader, Read},
path::Path,
};
use tar::Archive;
/// A type-erased decompressed tar reader.
pub type DynArchive = Archive<Box<dyn Read + Send + 'static>>;
pub fn open_layer(path: &Path, media_type: &str) -> Result<DynArchive> {
let file = File::open(path)?;
let buf = BufReader::new(file);
let reader: Box<dyn Read + Send + 'static> = match media_type {
t if t.ends_with("+gzip") || t == "application/vnd.docker.image.rootfs.diff.tar.gzip" => {
Box::new(flate2::read::GzDecoder::new(buf))
}
t if t.ends_with("+zstd") => Box::new(zstd::stream::read::Decoder::new(buf)?),
t if t.ends_with("+bzip2") => Box::new(bzip2::read::BzDecoder::new(buf)),
t if t.ends_with("+xz") || t.ends_with("+lzma") => Box::new(xz2::read::XzDecoder::new(buf)),
"application/vnd.oci.image.layer.v1.tar" => Box::new(buf),
other => bail!("unsupported layer media type: {other}"),
};
let mut archive = Archive::new(reader);
archive.set_preserve_permissions(true);
archive.set_preserve_mtime(true);
archive.set_unpack_xattrs(true);
Ok(archive)
}
/// Read all raw bytes of a layer entry into memory.
/// Returns (header_bytes, data_bytes) — header_bytes is the raw 512-byte block(s).
pub fn read_entry_data(entry: &mut tar::Entry<impl Read>) -> Result<Vec<u8>> {
let mut buf = Vec::new();
io::copy(entry, &mut buf)?;
Ok(buf)
}

26
oci_squashfs/src/lib.rs Normal file
View file

@ -0,0 +1,26 @@
pub mod canonical;
pub mod image;
pub mod layers;
pub mod overlay;
pub mod squashfs;
pub mod tracker;
pub mod verify;
use anyhow::Result;
use std::path::Path;
/// Convert an extracted OCI image directory into a squashfs file.
///
/// Layer content is streamed directly into mksquashfs's stdin — the merged
/// tar is never fully materialised in memory.
pub async fn convert(image_dir: &Path, output_squashfs: &Path) -> Result<()> {
let image_dir = image_dir.to_path_buf();
let output_squashfs = output_squashfs.to_path_buf();
tokio::task::spawn_blocking(move || {
let manifest = image::load_manifest(&image_dir)?;
let layers = image::resolve_layers(&image_dir, &manifest)?;
squashfs::write_squashfs(layers, &output_squashfs)
})
.await?
}

128
oci_squashfs/src/overlay.rs Normal file
View file

@ -0,0 +1,128 @@
//! Core algorithm: merge OCI layers into a single flat tar stream.
//!
//! File content is never buffered in full — entries are streamed directly
//! into the provided `Write` sink (typically mksquashfs's stdin pipe).
//! The only in-memory state is the tracker data structures and the small
//! hard-link metadata structs deferred to the end.
use anyhow::{Context, Result};
use std::{
io::Write,
path::{Path, PathBuf},
};
use tar::{Builder, EntryType};
use crate::{
canonical::CanonicalTarHeader,
image::LayerBlob,
layers::open_layer,
tracker::{EmittedPathTracker, HardLinkTracker, WhiteoutTracker},
};
/// Merge layers, streaming the resulting tar into `sink`.
///
/// `sink` is typically the stdin pipe of a `mksquashfs` subprocess.
/// File data flows directly from the layer blobs into `sink` without
/// being accumulated in memory.
pub fn merge_layers_into<W: Write>(mut layers: Vec<LayerBlob>, sink: W) -> Result<()> {
// Process in reverse (newest first).
layers.sort_by_key(|l| std::cmp::Reverse(l.index));
let mut whiteout = WhiteoutTracker::default();
let mut emitted = EmittedPathTracker::default();
let mut hardlinks = HardLinkTracker::default();
let mut output = Builder::new(sink);
output.mode(tar::HeaderMode::Complete);
for blob in &layers {
let mut archive = open_layer(&blob.path, &blob.media_type)
.with_context(|| format!("opening layer {}", blob.path.display()))?;
let entries = archive.entries().context("reading tar entries")?;
for entry_result in entries {
let mut entry = entry_result.context("reading tar entry")?;
let raw_path = entry.path().context("entry path")?.into_owned();
let path = normalize_path(&raw_path);
// 1. Check whiteout suppression.
if whiteout.is_suppressed(&path, blob.index) {
continue;
}
// 2. Check already-emitted.
if emitted.contains(&path) {
continue;
}
// 3. Handle whiteout entries.
let file_name = path
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
if file_name == ".wh..wh..opq" {
let parent = path.parent().unwrap_or(Path::new(""));
whiteout.insert_opaque(parent, blob.index);
continue;
}
if let Some(real_name) = file_name.strip_prefix(".wh.") {
let parent = path.parent().unwrap_or(Path::new(""));
whiteout.insert_simple(&parent.join(real_name), blob.index);
continue;
}
// We capture the header + PAX extensions first (cheap), then
// stream the entry body straight into the Builder without
// buffering it in a Vec.
let canonical =
CanonicalTarHeader::from_entry(&mut entry).context("capturing entry header")?;
// 4. Handle hard links — defer, emitting only metadata structs.
if canonical.entry_type() == EntryType::Link {
let link_target = canonical
.link_name()
.context("reading hard link target")?
.context("hard link has no target")?;
let target_path = normalize_path(&link_target);
if !whiteout.is_suppressed(&target_path, blob.index) {
hardlinks.record(path, target_path, blob.index, canonical);
}
continue;
}
// 5. Stream entry directly into the output tar.
//
canonical
.write_to_tar(&path, &mut entry, &mut output)
.with_context(|| format!("emitting {}", path.display()))?;
emitted.insert(&path);
}
}
// Emit deferred hard links (oldest-layer first).
// These are pure metadata — no file content to stream.
for hl in hardlinks.drain_sorted() {
if !emitted.contains(&hl.target_path) {
continue;
}
hl.canonical
.write_to_tar(&hl.link_path, &[] as &[u8], &mut output)
.with_context(|| format!("emitting hard link {}", hl.link_path.display()))?;
emitted.insert(&hl.link_path);
}
output.finish()?;
// Flush and drop the Builder, closing the write end of the pipe so
// mksquashfs sees EOF and knows the tar stream is complete.
let mut sink = output.into_inner()?;
sink.flush()?;
Ok(())
}
/// Strip leading `./` or `/` from paths.
pub fn normalize_path(p: &Path) -> PathBuf {
let s = p.to_string_lossy();
let s = s.trim_start_matches("./").trim_start_matches('/');
PathBuf::from(s)
}

View file

@ -0,0 +1,58 @@
//! Spawn mksquashfs and stream the merged tar directly into its stdin.
use anyhow::{bail, Context, Result};
use std::{
path::Path,
process::{Command, Stdio},
};
use crate::{image::LayerBlob, overlay::merge_layers_into};
/// Convert `layers` into a squashfs image at `output` by streaming a merged
/// tar directly into mksquashfs's stdin. No full tar buffer is held in memory.
pub fn write_squashfs(layers: Vec<LayerBlob>, output: &Path) -> Result<()> {
if output.exists() {
std::fs::remove_file(output)
.with_context(|| format!("removing existing {}", output.display()))?;
}
let mut child = Command::new("mksquashfs")
.args([
"-",
output.to_str().context("output path is not UTF-8")?,
"-tar",
"-noappend",
"-no-fragments",
"-comp",
"zstd",
"-Xcompression-level",
"2",
"-quiet",
])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("spawning mksquashfs — is it installed?")?;
let stdin = child.stdin.take().context("child stdin")?;
// Drive the merge on the current thread, writing directly into the pipe.
// mksquashfs reads and compresses concurrently in its own process, so
// the pipe provides natural backpressure without a helper thread.
let merge_result = merge_layers_into(layers, stdin);
// Wait for mksquashfs regardless of whether the merge succeeded, so we
// don't leave a zombie process behind.
let exit = child.wait_with_output().context("waiting for mksquashfs")?;
// Surface merge errors before subprocess errors — they're more actionable.
merge_result.context("merging layers into mksquashfs stdin")?;
if !exit.status.success() {
let stderr = String::from_utf8_lossy(&exit.stderr);
bail!("mksquashfs failed:\n{stderr}");
}
Ok(())
}

139
oci_squashfs/src/tracker.rs Normal file
View file

@ -0,0 +1,139 @@
//! WhiteoutTracker, EmittedPathTracker, HardLinkTracker.
use std::{
collections::{HashMap, HashSet},
path::{Component, Path, PathBuf},
};
use crate::canonical::CanonicalTarHeader;
// ─── WhiteoutTracker ────────────────────────────────────────────────────────
#[derive(Debug, Clone)]
enum WhiteoutState {
Simple { layer_index: usize },
Opaque { layer_index: usize },
}
#[derive(Debug, Default)]
struct WhiteoutNode {
state: Option<WhiteoutState>,
children: HashMap<String, WhiteoutNode>,
}
#[derive(Debug, Default)]
pub struct WhiteoutTracker {
root: WhiteoutNode,
}
impl WhiteoutTracker {
/// Mark a specific path as suppressed (simple `.wh.<name>` whiteout).
pub fn insert_simple(&mut self, path: &Path, layer_index: usize) {
let node = self.walk_or_create(path);
node.state = Some(WhiteoutState::Simple { layer_index });
}
/// Mark a directory path as opaque (`.wh..wh..opq`).
pub fn insert_opaque(&mut self, dir_path: &Path, layer_index: usize) {
let node = self.walk_or_create(dir_path);
node.state = Some(WhiteoutState::Opaque { layer_index });
}
/// Returns true if `path` from `current_layer` should be suppressed.
pub fn is_suppressed(&self, path: &Path, current_layer: usize) -> bool {
let components = normal_components(path);
let mut node = &self.root;
for (i, comp) in components.iter().enumerate() {
if let Some(state) = &node.state {
if let WhiteoutState::Opaque { layer_index } = state {
if current_layer < *layer_index {
return true;
}
}
}
match node.children.get(comp.as_str()) {
Some(child) => node = child,
None => return false,
}
// At terminal component: check Simple or Opaque.
if i == components.len() - 1 {
if let Some(state) = &node.state {
match state {
WhiteoutState::Simple { layer_index }
| WhiteoutState::Opaque { layer_index } => {
return current_layer < *layer_index;
}
}
}
}
}
false
}
fn walk_or_create(&mut self, path: &Path) -> &mut WhiteoutNode {
let components = normal_components(path);
let mut node = &mut self.root;
for comp in &components {
node = node.children.entry(comp.clone()).or_default();
}
node
}
}
fn normal_components(path: &Path) -> Vec<String> {
path.components()
.filter_map(|c| match c {
Component::Normal(s) => Some(s.to_string_lossy().into_owned()),
_ => None,
})
.collect()
}
// ─── EmittedPathTracker ─────────────────────────────────────────────────────
#[derive(Debug, Default)]
pub struct EmittedPathTracker(HashSet<PathBuf>);
impl EmittedPathTracker {
pub fn insert(&mut self, path: &Path) {
self.0.insert(path.to_path_buf());
}
pub fn contains(&self, path: &Path) -> bool {
self.0.contains(path)
}
}
// ─── HardLinkTracker ────────────────────────────────────────────────────────
#[derive(Debug)]
pub struct HardLinkEntry {
pub link_path: PathBuf,
pub target_path: PathBuf,
pub layer_index: usize,
pub canonical: CanonicalTarHeader,
}
#[derive(Debug, Default)]
pub struct HardLinkTracker(pub Vec<HardLinkEntry>);
impl HardLinkTracker {
pub fn record(
&mut self,
link_path: PathBuf,
target_path: PathBuf,
layer_index: usize,
canonical: CanonicalTarHeader,
) {
self.0.push(HardLinkEntry {
link_path,
target_path,
layer_index,
canonical,
});
}
/// Return deferred links sorted by ascending layer index.
pub fn drain_sorted(mut self) -> Vec<HardLinkEntry> {
self.0.sort_by_key(|e| e.layer_index);
self.0
}
}

239
oci_squashfs/src/verify.rs Normal file
View file

@ -0,0 +1,239 @@
//! Verification helper: mount squashfs and diff against a reference directory.
use anyhow::{Context, Result};
use sha2::{Digest, Sha256};
use std::{
collections::HashMap,
fs,
io::Read,
os::unix::fs::{MetadataExt, PermissionsExt},
path::{Path, PathBuf},
process::Command,
};
use tempfile::TempDir;
// ─── RAII mount guard ─────────────────────────────────────────────────────────
/// Mounts a squashfs via squashfuse and unmounts it on drop, even if an error
/// occurs during the walk. The underlying `TempDir` is kept alive for the
/// lifetime of this guard so the mountpoint isn't deleted while still mounted.
struct SquashMount {
mountpoint: TempDir,
}
impl SquashMount {
fn new(squashfs: &Path) -> Result<Self> {
let mountpoint = TempDir::new().context("creating temp mount dir")?;
let status = Command::new("squashfuse")
.arg(squashfs)
.arg(mountpoint.path())
.status()
.context("spawning squashfuse — is it installed?")?;
if !status.success() {
anyhow::bail!("squashfuse failed with status {status}");
}
Ok(Self { mountpoint })
}
fn path(&self) -> &Path {
self.mountpoint.path()
}
}
impl Drop for SquashMount {
fn drop(&mut self) {
// Try fusermount first (Linux), fall back to umount (macOS/BSD).
let ok = Command::new("fusermount")
.args(["-u", self.mountpoint.path().to_str().unwrap_or("")])
.status()
.map(|s| s.success())
.unwrap_or(false);
if !ok {
let _ = Command::new("umount").arg(self.mountpoint.path()).status();
}
// TempDir::drop runs after this and removes the now-unmounted directory.
}
}
// ─── Public API ───────────────────────────────────────────────────────────────
#[derive(Debug)]
pub struct VerifyReport {
pub only_in_squashfs: Vec<PathBuf>,
pub only_in_reference: Vec<PathBuf>,
pub differences: Vec<FileDiff>,
}
#[derive(Debug)]
pub struct FileDiff {
pub path: PathBuf,
pub detail: String,
}
pub fn verify(squashfs: &Path, reference: &Path) -> Result<VerifyReport> {
let mount = SquashMount::new(squashfs)?;
let squashfs_tree = walk_tree(mount.path()).context("walking squashfs mount")?;
let reference_tree = walk_tree(reference).context("walking reference directory")?;
let mut report = VerifyReport {
only_in_squashfs: Vec::new(),
only_in_reference: Vec::new(),
differences: Vec::new(),
};
for (rel, sq_info) in &squashfs_tree {
match reference_tree.get(rel) {
None => report.only_in_squashfs.push(rel.clone()),
Some(ref_info) => report
.differences
.extend(compare_entries(rel, sq_info, ref_info)),
}
}
for rel in reference_tree.keys() {
if !squashfs_tree.contains_key(rel) {
report.only_in_reference.push(rel.clone());
}
}
Ok(report)
}
// ─── Tree walking ─────────────────────────────────────────────────────────────
#[derive(Debug)]
struct EntryInfo {
kind: EntryKind,
mode: u32,
uid: u32,
gid: u32,
size: u64,
symlink_target: Option<PathBuf>,
sha256: Option<String>,
}
#[derive(Debug, PartialEq)]
enum EntryKind {
File,
Dir,
Symlink,
Other,
}
fn walk_tree(root: &Path) -> Result<HashMap<PathBuf, EntryInfo>> {
let mut map = HashMap::new();
walk_dir(root, root, &mut map)?;
Ok(map)
}
fn walk_dir(root: &Path, current: &Path, map: &mut HashMap<PathBuf, EntryInfo>) -> Result<()> {
for entry in
fs::read_dir(current).with_context(|| format!("reading dir {}", current.display()))?
{
let entry = entry?;
let abs = entry.path();
let rel = abs
.strip_prefix(root)
.context("strip prefix")?
.to_path_buf();
let meta = fs::symlink_metadata(&abs)
.with_context(|| format!("metadata for {}", abs.display()))?;
let ft = meta.file_type();
let (kind, symlink_target, sha256) = if ft.is_symlink() {
(EntryKind::Symlink, Some(fs::read_link(&abs)?), None)
} else if ft.is_file() {
(EntryKind::File, None, Some(hash_file(&abs)?))
} else if ft.is_dir() {
(EntryKind::Dir, None, None)
} else {
(EntryKind::Other, None, None)
};
// Mask to permission bits (rwxrwxrwx + setuid/setgid/sticky).
// File type bits are captured separately in `kind`.
const PERMISSION_BITS: u32 = 0o7777;
map.insert(
rel,
EntryInfo {
kind,
mode: meta.permissions().mode() & PERMISSION_BITS,
uid: meta.uid(),
gid: meta.gid(),
size: meta.len(),
symlink_target,
sha256,
},
);
if ft.is_dir() {
walk_dir(root, &abs, map)?;
}
}
Ok(())
}
fn hash_file(path: &Path) -> Result<String> {
let mut file = fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buf = [0u8; 8192];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
Ok(format!("{:x}", hasher.finalize()))
}
// ─── Comparison ───────────────────────────────────────────────────────────────
fn compare_entries(rel: &Path, sq: &EntryInfo, rf: &EntryInfo) -> Vec<FileDiff> {
let mut diffs = Vec::new();
macro_rules! diff {
($msg:expr) => {
diffs.push(FileDiff {
path: rel.to_path_buf(),
detail: $msg,
})
};
}
if sq.kind != rf.kind {
diff!(format!(
"type mismatch: squashfs={:?} ref={:?}",
sq.kind, rf.kind
));
return diffs;
}
if sq.mode != rf.mode {
diff!(format!(
"mode: squashfs={:04o} ref={:04o}",
sq.mode, rf.mode
));
}
if sq.uid != rf.uid {
diff!(format!("uid: squashfs={} ref={}", sq.uid, rf.uid));
}
if sq.gid != rf.gid {
diff!(format!("gid: squashfs={} ref={}", sq.gid, rf.gid));
}
if sq.symlink_target != rf.symlink_target {
diff!(format!(
"symlink target: squashfs={:?} ref={:?}",
sq.symlink_target, rf.symlink_target
));
}
if sq.kind == EntryKind::File {
if sq.size != rf.size {
diff!(format!("size: squashfs={} ref={}", sq.size, rf.size));
}
if sq.sha256 != rf.sha256 {
diff!(format!("sha256 mismatch"));
}
}
diffs
}

View file

@ -0,0 +1,212 @@
//! Shared test helpers for integration and regression tests.
#![allow(dead_code)]
use oci_squashfs::canonical::CanonicalTarHeader;
use oci_squashfs::image::LayerBlob;
use std::io::{Cursor, Write};
use tar::{Archive, Builder, EntryType, Header};
// ─── LayerBuilder ────────────────────────────────────────────────────────────
pub struct LayerBuilder {
inner: Builder<Vec<u8>>,
}
impl LayerBuilder {
pub fn new() -> Self {
Self {
inner: Builder::new(Vec::new()),
}
}
pub fn add_file(mut self, path: &str, data: &[u8], mode: u32) -> Self {
if path.len() > 99 {
self.inner
.append_pax_extensions([("path", path.as_bytes())])
.unwrap();
}
let mut hdr = Header::new_ustar();
hdr.set_path(truncate(path, 99)).unwrap();
hdr.set_size(data.len() as u64);
hdr.set_mode(mode);
hdr.set_mtime(0);
hdr.set_uid(0);
hdr.set_gid(0);
hdr.set_cksum();
self.inner.append(&hdr, Cursor::new(data)).unwrap();
self
}
pub fn add_dir(mut self, path: &str) -> Self {
if path.len() > 99 {
self.inner
.append_pax_extensions([("path", path.as_bytes())])
.unwrap();
}
let mut hdr = Header::new_ustar();
hdr.set_path(truncate(path, 99)).unwrap();
hdr.set_entry_type(EntryType::Directory);
hdr.set_size(0);
hdr.set_mode(0o755);
hdr.set_mtime(0);
hdr.set_uid(0);
hdr.set_gid(0);
hdr.set_cksum();
self.inner.append(&hdr, Cursor::new(b"" as &[u8])).unwrap();
self
}
pub fn add_symlink(mut self, path: &str, target: &str) -> Self {
let mut pax: Vec<(&str, &[u8])> = Vec::new();
if path.len() > 99 {
pax.push(("path", path.as_bytes()));
}
if target.len() > 99 {
pax.push(("linkpath", target.as_bytes()));
}
if !pax.is_empty() {
self.inner.append_pax_extensions(pax).unwrap();
}
let mut hdr = Header::new_ustar();
hdr.set_path(truncate(path, 99)).unwrap();
hdr.set_entry_type(EntryType::Symlink);
hdr.set_link_name(truncate(target, 99)).ok();
hdr.set_size(0);
hdr.set_mode(0o777);
hdr.set_mtime(0);
hdr.set_uid(0);
hdr.set_gid(0);
hdr.set_cksum();
self.inner.append(&hdr, Cursor::new(b"" as &[u8])).unwrap();
self
}
pub fn add_hardlink(mut self, path: &str, target: &str) -> Self {
let mut pax: Vec<(&str, &[u8])> = Vec::new();
if path.len() > 99 {
pax.push(("path", path.as_bytes()));
}
if target.len() > 99 {
pax.push(("linkpath", target.as_bytes()));
}
if !pax.is_empty() {
self.inner.append_pax_extensions(pax).unwrap();
}
let mut hdr = Header::new_ustar();
hdr.set_path(truncate(path, 99)).unwrap();
hdr.set_entry_type(EntryType::Link);
hdr.set_link_name(truncate(target, 99)).ok();
hdr.set_size(0);
hdr.set_mode(0o644);
hdr.set_mtime(0);
hdr.set_uid(0);
hdr.set_gid(0);
hdr.set_cksum();
self.inner.append(&hdr, Cursor::new(b"" as &[u8])).unwrap();
self
}
pub fn add_whiteout(self, dir: &str, name: &str) -> Self {
let path = if dir.is_empty() {
format!(".wh.{name}")
} else {
format!("{dir}/.wh.{name}")
};
self.add_file(&path, b"", 0o644)
}
pub fn add_opaque_whiteout(self, dir: &str) -> Self {
self.add_file(&format!("{dir}/.wh..wh..opq"), b"", 0o644)
}
pub fn finish(mut self) -> Vec<u8> {
self.inner.finish().unwrap();
self.inner.into_inner().unwrap()
}
}
/// Truncate a string to at most `max_chars` characters.
fn truncate(s: &str, max_chars: usize) -> &str {
match s.char_indices().nth(max_chars) {
Some((idx, _)) => &s[..idx],
None => s,
}
}
// ─── Blob / merge helpers ─────────────────────────────────────────────────────
pub fn blob(bytes: Vec<u8>, index: usize) -> LayerBlob {
use tempfile::NamedTempFile;
let mut f = NamedTempFile::new().unwrap();
f.write_all(&bytes).unwrap();
let (_, path) = f.keep().unwrap();
LayerBlob {
path,
media_type: "application/vnd.oci.image.layer.v1.tar".into(),
index,
}
}
pub fn merge(layers: Vec<LayerBlob>) -> Vec<u8> {
let mut out = Vec::new();
oci_squashfs::overlay::merge_layers_into(layers, &mut out).unwrap();
out
}
// ─── Tar inspection helpers ───────────────────────────────────────────────────
pub fn paths_in_tar(tar_bytes: &[u8]) -> Vec<String> {
let mut archive = Archive::new(Cursor::new(tar_bytes));
archive
.entries()
.unwrap()
.filter_map(|e| e.ok())
.filter(|e| {
!matches!(
e.header().entry_type(),
EntryType::XHeader | EntryType::XGlobalHeader
)
})
.map(|e| e.path().unwrap().to_string_lossy().into_owned())
.collect()
}
/// Read the symlink target for `link_path`, preferring PAX `linkpath`.
pub fn symlink_target_in_tar(tar_bytes: &[u8], link_path: &str) -> Option<String> {
let mut archive = Archive::new(Cursor::new(tar_bytes));
for mut entry in archive.entries().unwrap().flatten() {
let canonical = CanonicalTarHeader::from_entry(&mut entry).ok()?;
if canonical.entry_type() != EntryType::Symlink {
continue;
}
if canonical.path().unwrap().to_string_lossy() != link_path {
continue;
}
return canonical
.link_name()
.ok()
.flatten()
.map(|p| p.to_string_lossy().into_owned());
}
None
}
/// Read the hard link target for `link_path`, preferring PAX `linkpath`.
pub fn hardlink_target_in_tar(tar_bytes: &[u8], link_path: &str) -> Option<String> {
let mut archive = Archive::new(Cursor::new(tar_bytes));
for mut entry in archive.entries().unwrap().flatten() {
let canonical = CanonicalTarHeader::from_entry(&mut entry).ok()?;
if canonical.entry_type() != EntryType::Link {
continue;
}
if canonical.path().unwrap().to_string_lossy() != link_path {
continue;
}
return canonical
.link_name()
.ok()
.flatten()
.map(|p| p.to_string_lossy().into_owned());
}
None
}

View file

@ -0,0 +1,110 @@
//! Synthetic integration tests for the OCI → squashfs merge pipeline.
//! These tests exercise `overlay::merge_layers_into` directly on in-memory
//! blobs and inspect the resulting merged tar without invoking mksquashfs.
#[path = "helpers/mod.rs"]
mod helpers;
use helpers::{
blob, hardlink_target_in_tar, merge, paths_in_tar, symlink_target_in_tar, LayerBuilder,
};
// ─── Tests ───────────────────────────────────────────────────────────────────
#[test]
fn test_long_symlink_pax_preserved() {
let long_target: String = "a".repeat(200);
let layer0 = LayerBuilder::new()
.add_symlink("link_to_long", &long_target)
.finish();
let merged = merge(vec![blob(layer0, 0)]);
assert_eq!(
symlink_target_in_tar(&merged, "link_to_long").as_deref(),
Some(long_target.as_str()),
"long symlink target must round-trip via PAX"
);
}
#[test]
fn test_long_hardlink_pax_preserved() {
let long_target: String = "a".repeat(200);
let layer0 = LayerBuilder::new()
.add_file(&long_target, b"hello", 0o644)
.add_hardlink("link_to_long", &long_target)
.finish();
let merged = merge(vec![blob(layer0, 0)]);
assert_eq!(
hardlink_target_in_tar(&merged, "link_to_long").as_deref(),
Some(long_target.as_str()),
"long hardlink target must round-trip via PAX"
);
}
#[test]
fn test_hardlink_across_layers() {
let layer0 = LayerBuilder::new()
.add_file("original.txt", b"hello", 0o644)
.finish();
let layer1 = LayerBuilder::new()
.add_hardlink("link.txt", "original.txt")
.finish();
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
let paths = paths_in_tar(&merged);
assert!(paths.contains(&"original.txt".to_string()));
assert!(paths.contains(&"link.txt".to_string()));
}
#[test]
fn test_simple_whiteout() {
let layer0 = LayerBuilder::new()
.add_file("secret.txt", b"private", 0o644)
.finish();
let layer1 = LayerBuilder::new().add_whiteout("", "secret.txt").finish();
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
assert!(!paths_in_tar(&merged).iter().any(|p| p == "secret.txt"));
}
#[test]
fn test_opaque_whiteout() {
let layer0 = LayerBuilder::new()
.add_dir("mydir")
.add_file("mydir/old.txt", b"old", 0o644)
.finish();
let layer1 = LayerBuilder::new()
.add_dir("mydir")
.add_opaque_whiteout("mydir")
.finish();
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
assert!(!paths_in_tar(&merged).iter().any(|p| p == "mydir/old.txt"));
}
#[test]
fn test_opaque_whiteout_with_repopulation() {
let layer0 = LayerBuilder::new()
.add_dir("dir")
.add_file("dir/old.txt", b"old", 0o644)
.finish();
let layer1 = LayerBuilder::new()
.add_dir("dir")
.add_opaque_whiteout("dir")
.add_file("dir/new.txt", b"new", 0o644)
.finish();
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
let paths = paths_in_tar(&merged);
assert!(paths.iter().any(|p| p == "dir/new.txt"));
assert!(!paths.iter().any(|p| p == "dir/old.txt"));
}
#[test]
fn test_hardlink_to_whiteout_target_dropped() {
let layer0 = LayerBuilder::new()
.add_file("gone.txt", b"data", 0o644)
.finish();
let layer1 = LayerBuilder::new()
.add_whiteout("", "gone.txt")
.add_hardlink("link.txt", "gone.txt")
.finish();
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
let paths = paths_in_tar(&merged);
assert!(!paths.iter().any(|p| p == "gone.txt"));
assert!(!paths.iter().any(|p| p == "link.txt"));
}

View file

@ -0,0 +1,165 @@
//! Regression tests, one per production bug found during verify runs.
//! Each test is named after the bug it guards against and has a comment
//! explaining the root cause and how it was discovered.
use std::io::Cursor;
use tar::{Builder, EntryType, Header};
#[path = "helpers/mod.rs"]
mod helpers;
use helpers::{blob, merge, paths_in_tar, LayerBuilder};
// regression.rs also uses:
use helpers::hardlink_target_in_tar;
// ─── Regression tests ────────────────────────────────────────────────────────
/// Bug: whiteout suppression condition was inverted (`current_layer > layer_index`
/// instead of `current_layer < layer_index`), causing whiteouts to suppress
/// entries from *newer* layers rather than older ones. Files removed via
/// `.wh.<name>` in a later layer were still appearing in the output.
///
/// Discovered via: `usr/share/vulkan/icd.d/lvp_icd.json` appearing in squashfs
/// output despite being removed by a dpkg-divert whiteout in a later layer.
#[test]
fn regress_whiteout_suppression_direction() {
let layer0 = LayerBuilder::new()
.add_file("usr/share/vulkan/icd.d/lvp_icd.json", b"data", 0o644)
.finish();
// Layer 1 whiteouts the file from layer 0.
let layer1 = LayerBuilder::new()
.add_whiteout("usr/share/vulkan/icd.d", "lvp_icd.json")
.finish();
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
let paths = paths_in_tar(&merged);
assert!(
!paths
.iter()
.any(|p| p == "usr/share/vulkan/icd.d/lvp_icd.json"),
"whited-out file must not appear: got paths {paths:?}"
);
}
/// Bug: hard link targets longer than 100 bytes were being silently truncated
/// because we read them via `entry.header().link_name()`, which only reads the
/// raw 100-byte USTAR `linkname` field. The PAX `linkpath` extension carrying
/// the full target was ignored. The truncated target path was then not found in
/// the emitted-path tracker, so the link was silently dropped.
///
/// Discovered via: PostgreSQL/hammerdb timezone hard-link aliases all missing
/// from squashfs output (e.g. `timezone/Jamaica -> .../America/...` dropped
/// because the full target path exceeded 100 bytes and was truncated to
/// `America` or similar).
#[test]
fn regress_long_hardlink_target_truncated() {
// Construct a target path that exceeds 100 bytes.
let long_dir = "a".repeat(95);
let target = format!("{long_dir}/canonical_file");
assert!(
target.len() > 100,
"test setup: target must exceed 100 bytes"
);
let layer0 = LayerBuilder::new()
.add_dir(&long_dir)
.add_file(&target, b"content", 0o644)
.add_hardlink("alias_file", &target)
.finish();
let merged = merge(vec![blob(layer0, 0)]);
let paths = paths_in_tar(&merged);
assert!(
paths.iter().any(|p| p == &target),
"canonical target must be present"
);
assert!(
paths.iter().any(|p| p == "alias_file"),
"hard link alias must be present; was it silently dropped due to truncated target?"
);
let resolved = hardlink_target_in_tar(&merged, "alias_file");
assert_eq!(
resolved.as_deref(),
Some(target.as_str()),
"hard link target must be the full untruncated path"
);
}
/// Variant of the above: hard link and target in different layers, with a long
/// target path. Guards against the combination of cross-layer deferral and PAX
/// target resolution both being required simultaneously.
#[test]
fn regress_long_hardlink_target_cross_layer() {
let long_dir = "b".repeat(60);
let target = format!("{long_dir}/canonical_file");
let layer0 = LayerBuilder::new()
.add_dir(&long_dir)
.add_file(&target, b"content", 0o644)
.finish();
let layer1 = LayerBuilder::new()
.add_hardlink("alias_file", &target)
.finish();
let merged = merge(vec![blob(layer0, 0), blob(layer1, 1)]);
let paths = paths_in_tar(&merged);
assert!(
paths.iter().any(|p| p == &target),
"canonical target must be present"
);
assert!(
paths.iter().any(|p| p == "alias_file"),
"cross-layer hard link with long target must not be silently dropped"
);
}
/// Bug: normalize_path only stripped leading `./` but not leading `/`. Hard
/// link targets stored as absolute paths in the tar (e.g. `/usr/share/foo`)
/// would not match the normalized emitted path (`usr/share/foo`), causing the
/// link to be silently dropped.
#[test]
fn regress_absolute_hardlink_target_normalized() {
// Manually construct a layer with an absolute-path hard link target,
// which some tar producers emit.
let mut builder = Builder::new(Vec::new());
let mut file_hdr = Header::new_ustar();
file_hdr.set_path("usr/share/foo").unwrap();
file_hdr.set_size(4);
file_hdr.set_mode(0o644);
file_hdr.set_mtime(0);
file_hdr.set_uid(0);
file_hdr.set_gid(0);
file_hdr.set_cksum();
builder.append(&file_hdr, Cursor::new(b"data")).unwrap();
// Use a PAX linkpath with a leading slash — the absolute form.
builder
.append_pax_extensions([("linkpath", b"/usr/share/foo" as &[u8])])
.unwrap();
let mut link_hdr = Header::new_ustar();
link_hdr.set_path("usr/share/bar").unwrap();
link_hdr.set_entry_type(EntryType::Link);
link_hdr.set_link_name("usr/share/foo").ok(); // truncated USTAR field (no leading slash here)
link_hdr.set_size(0);
link_hdr.set_mode(0o644);
link_hdr.set_mtime(0);
link_hdr.set_uid(0);
link_hdr.set_gid(0);
link_hdr.set_cksum();
builder
.append(&link_hdr, Cursor::new(b"" as &[u8]))
.unwrap();
builder.finish().unwrap();
let layer0 = builder.into_inner().unwrap();
let merged = merge(vec![blob(layer0, 0)]);
let paths = paths_in_tar(&merged);
assert!(
paths.iter().any(|p| p == "usr/share/bar"),
"hard link with absolute PAX linkpath must not be dropped after normalization"
);
}

View file

@ -0,0 +1,14 @@
[package]
name = "oci_squashfs_cli"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "oci2squashfs"
path = "src/main.rs"
[dependencies]
oci_squashfs = { path = "../oci_squashfs" }
tokio = { version = "1", features = ["full"] }
clap = { version = "4", features = ["derive"] }
anyhow = "1"

View file

@ -0,0 +1,73 @@
use anyhow::Result;
use clap::{Parser, Subcommand};
use std::path::PathBuf;
#[derive(Parser)]
#[command(name = "oci2squashfs", about = "Convert an OCI image to squashfs")]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Convert an OCI image directory to a squashfs image.
Convert {
/// Path to the extracted OCI image directory.
#[arg(short, long)]
image: PathBuf,
/// Output squashfs file path.
#[arg(short, long)]
output: PathBuf,
},
/// Verify a squashfs image against a reference directory.
Verify {
/// Path to the .squashfs file.
#[arg(short, long)]
squashfs: PathBuf,
/// Path to the reference directory (e.g. containerd-unpacked rootfs).
#[arg(short, long)]
reference: PathBuf,
},
}
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Convert { image, output } => {
println!("Converting {}{}", image.display(), output.display());
oci_squashfs::convert(&image, &output).await?;
println!("Done: {}", output.display());
}
Commands::Verify {
squashfs,
reference,
} => {
let report = tokio::task::spawn_blocking(move || {
oci_squashfs::verify::verify(&squashfs, &reference)
})
.await??;
if report.only_in_squashfs.is_empty()
&& report.only_in_reference.is_empty()
&& report.differences.is_empty()
{
println!("✓ No differences found.");
return Ok(());
}
for p in &report.only_in_squashfs {
println!("+ squashfs only: {}", p.display());
}
for p in &report.only_in_reference {
println!("- reference only: {}", p.display());
}
for d in &report.differences {
println!("~ {}: {}", d.path.display(), d.detail);
}
std::process::exit(1);
}
}
Ok(())
}