diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fec58ab..7829e76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: rustfmt name: rustfmt - entry: make check-fmt + entry: make fmt language: system types: [rust] pass_filenames: false diff --git a/CLAUDE.md b/CLAUDE.md index b155b6a..dd9e624 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,6 +57,34 @@ krust pushes images by default (use `--no-push` to skip) because: - Then push manifest referencing those blobs - Manifest URL contains the final digest +### Google Artifact Registry (GAR) Blob Uploads + +GAR has special handling for blob uploads that differs from the standard OCI spec: + +1. **Location Header Format**: + - POST to `/v2/.../blobs/uploads/` returns `location: /artifacts-uploads/...` + - The location is a relative path starting with `/artifacts-uploads/`, NOT `/v2/` + - Must build absolute URL as `https://{registry}{location}` for ANY path starting with `/` + +2. **Upload Flow** (Resumable): + - **Monolithic upload not supported**: PUT with body to `location?digest=` returns `301 Moved Permanently` + - Must use resumable upload flow instead: + 1. POST to `/v2/.../blobs/uploads/` → get upload location + 2. PATCH to upload location with blob data → returns `202 Accepted` + 3. PUT to finalize location with `?digest=` and empty body → returns `201 Created` + +3. **Critical Implementation Details**: + - GAR returns `301` redirect on monolithic PUT attempts (not `307`) + - HTTP spec says `301` means don't resend body, so automatic redirect following fails + - Must explicitly handle `301` as a signal to switch to resumable upload + - PATCH response may include a new `location` header for the finalize PUT + - Use `reqwest` with `redirect::Policy::none()` to handle redirects manually + +4. **Why reqwest over hyper**: + - Initially used raw `hyper` but it doesn't auto-follow redirects with request bodies + - Switched to `reqwest` for cleaner API and better redirect handling + - Disabled automatic redirects to manually handle GAR's upload flow + ### Cross-Compilation on macOS For Linux targets from macOS, you need: @@ -100,7 +128,7 @@ src/ Key crates chosen: - `clap` - CLI parsing with derive macros - `tokio` - Async runtime for registry operations -- `oci-distribution` - OCI registry client +- `reqwest` - HTTP client with automatic redirect handling - `tar` + `flate2` - Layer creation - `sha256` - Digest calculation - `tracing` - Structured logging @@ -124,26 +152,34 @@ The iterative development process: ## Future Improvements Potential enhancements identified: -1. Registry authentication support +1. ~~Registry authentication support~~ ✓ Implemented (supports Docker credential helpers) 2. Multi-platform image manifests 3. Build caching 4. Image layer optimization 5. Support for custom Dockerfile-like configs 6. SBOM (Software Bill of Materials) generation +7. Optimize blob uploads (check if blob exists before uploading) ## Useful Commands ```bash -# Test the full workflow +# Test with anonymous registry (ttl.sh) export KRUST_REPO=ttl.sh/test docker run $(krust build example/hello-krust) +# Test with Google Artifact Registry +export KRUST_REPO=us-central1-docker.pkg.dev/project-id/repo-name +krust build example/hello-krust + # Debug output krust build -v 2>&1 | less # Check static linking file target/x86_64-unknown-linux-musl/release/binary ldd target/x86_64-unknown-linux-musl/release/binary # should say "not a dynamic executable" + +# Verify pushed image +crane manifest $(krust build --no-push example/hello-krust) ``` ## Resources diff --git a/Cargo.toml b/Cargo.toml index 171fa25..9d0ef24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,10 +19,7 @@ sha256 = "1.5" chrono = "0.4" tar = "0.4" flate2 = "1.0" -hyper = { version = "1.0", features = ["full"] } -hyper-util = { version = "0.1", features = ["full"] } -http-body-util = "0.1" -hyper-tls = "0.6" +reqwest = { version = "0.12", features = ["json", "stream"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } dirs = "6.0" @@ -36,7 +33,3 @@ tempfile = "3.9" assert_cmd = "2.0" predicates = "3.0" testscript-rs = "0.2" - -[[example]] -name = "auto_auth_demo" -path = "examples/auto_auth_demo.rs" diff --git a/Makefile b/Makefile index f959099..5055459 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test test-unit test-e2e build run clean fmt lint check-fmt check test-verbose setup-cross-compile verify-cross-compile +.PHONY: test test-unit test-e2e test-testscript build run clean fmt lint check-fmt check install setup-cross-compile verify-cross-compile # Test flags - run single-threaded to avoid env var races TEST_FLAGS := -- --test-threads=1 @@ -60,6 +60,19 @@ test-unit: test-e2e: cargo test --verbose --test '*' $(TEST_FLAGS) +# Run testscript tests only +test-testscript: + cargo test --verbose --test testscript_test + +# Install krust to ~/.local/bin +install: + @echo "Building krust in release mode..." + @cargo build --release + @mkdir -p ~/.local/bin + @cp target/release/krust ~/.local/bin/krust + @echo "Installed krust to ~/.local/bin/krust" + @echo "Make sure ~/.local/bin is in your PATH" + # Clean build artifacts clean: cargo clean diff --git a/examples/auto_auth_demo.rs b/examples/auto_auth_demo.rs deleted file mode 100644 index e64c441..0000000 --- a/examples/auto_auth_demo.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Example demonstrating automatic authentication with oci-distribution -//! -//! This example shows how to use the automatic auth methods that resolve -//! credentials from Docker config files and credential helpers. - -use anyhow::Result; -use oci_distribution::{client::Client, Reference}; - -#[tokio::main] -async fn main() -> Result<()> { - // Initialize logging - tracing_subscriber::fmt::init(); - - // Create a client - let client = Client::default(); - - // Example 1: Pull a public image (anonymous auth) - println!("Pulling public image alpine:latest..."); - let reference: Reference = "docker.io/library/alpine:latest".parse()?; - - match client.pull_manifest_auto(&reference).await { - Ok((manifest, digest)) => { - println!("✓ Successfully pulled manifest"); - println!(" Digest: {}", digest); - match manifest { - oci_distribution::manifest::OciManifest::Image(img) => { - println!(" Type: Single platform image"); - println!(" Config: {}", img.config.digest); - } - oci_distribution::manifest::OciManifest::ImageIndex(idx) => { - println!(" Type: Multi-platform image"); - println!(" Platforms: {}", idx.manifests.len()); - } - } - } - Err(e) => { - println!("✗ Failed to pull: {}", e); - } - } - - // Example 2: Get platforms for an image - println!("\nDetecting platforms for rust:alpine..."); - let rust_ref: Reference = "docker.io/library/rust:alpine".parse()?; - - match client.get_image_platforms_auto(&rust_ref).await { - Ok(platforms) => { - println!("✓ Found {} platforms:", platforms.len()); - for (os, arch) in platforms { - println!(" - {}/{}", os, arch); - } - } - Err(e) => { - println!("✗ Failed to get platforms: {}", e); - } - } - - // Example 3: Private registry (requires auth) - println!("\nNote: To test with a private registry, ensure you have credentials in:"); - println!(" - ~/.docker/config.json"); - println!(" - Or DOCKER_CONFIG environment variable"); - println!(" - Or using a credential helper"); - - // Uncomment to test with a private registry: - // let private_ref: Reference = "ghcr.io/your-username/your-image:latest".parse()?; - // match client.pull_manifest_auto(&private_ref).await { - // Ok((_, digest)) => println!("✓ Successfully authenticated and pulled private image: {}", digest), - // Err(e) => println!("✗ Failed to pull private image: {}", e), - // } - - Ok(()) -} diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 2534f3f..28fe2f2 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -21,10 +21,6 @@ pub enum Commands { #[arg(value_name = "DIRECTORY")] path: Option, - /// Target image reference (overrides KRUST_REPO) - #[arg(short, long, env = "KRUST_IMAGE")] - image: Option, - /// Target platforms (e.g., linux/amd64, linux/arm64) /// Can be specified multiple times or as a comma-separated list #[arg(long, value_delimiter = ',')] @@ -34,12 +30,13 @@ pub enum Commands { #[arg(long)] no_push: bool, - /// Tag to apply to the manifest list (e.g., latest, v1.0.0) + /// Tag to apply to the image (e.g., latest, v1.0.0) + /// If not specified, only pushes by digest #[arg(long)] tag: Option, /// Repository prefix (e.g., ghcr.io/username) - #[arg(long, env = "KRUST_REPO")] + #[arg(env = "KRUST_REPO")] repo: Option, /// Additional cargo build arguments diff --git a/src/main.rs b/src/main.rs index 1cacb67..4871980 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,7 +31,6 @@ async fn main() -> Result<()> { match cli.command { Commands::Build { path, - image, platform, no_push, tag, @@ -49,20 +48,10 @@ async fn main() -> Result<()> { .base_image .unwrap_or(config.base_image.clone()); - // Determine the base repository name (without any tag) - let target_repo = if let Some(image) = image { - // Use explicit image if provided, strip any tag/digest - if let Some(pos) = image.rfind([':', '@']) { - image[..pos].to_string() - } else { - image - } - } else { - // Build repository name from repo and project name - let repo = repo.context("Either --image or KRUST_REPO must be set")?; - let project_name = get_project_name(&project_path)?; - format!("{}/{}", repo, project_name) - }; + // Build repository name from KRUST_REPO and project name + let repo = repo.context("KRUST_REPO must be set")?; + let project_name = get_project_name(&project_path)?; + let target_repo = format!("{}/{}", repo, project_name); // Initialize registry client let mut registry_client = RegistryClient::new()?; @@ -142,7 +131,8 @@ async fn main() -> Result<()> { "application/vnd.docker.image.rootfs.diff.tar.gzip".to_string() }); - // Push layered image (copy base layers if needed + push app layer + manifest) + // Push layered image by digest only (no tag) + // This will be referenced by digest in the final manifest list let (digest_ref, manifest_size) = registry_client .push_layered_image( &target_repo, @@ -192,20 +182,25 @@ async fn main() -> Result<()> { info!("Creating and pushing manifest list..."); // Determine the target for the manifest list - let manifest_target = if let Some(tag_name) = tag { + let has_tag = tag.is_some(); + let manifest_target = if let Some(tag_name) = &tag { // If --tag is specified, push to that tag format!("{}:{}", target_repo, tag_name) } else { - // If no tag specified, push digest-only by using a temporary tag - // We'll use a temporary tag and return the digest reference - format!("{}:temp-{}", target_repo, std::process::id()) + // If no tag specified, push digest-only (no tag) + target_repo.clone() }; // Get auth for the final image push let final_auth = resolve_auth(&manifest_target)?; let manifest_list_ref = registry_client - .push_manifest_list(&manifest_target, manifest_descriptors, &final_auth) + .push_manifest_list( + &manifest_target, + manifest_descriptors, + &final_auth, + has_tag, + ) .await?; // Output the manifest list reference (always by digest) diff --git a/src/registry/mod.rs b/src/registry/mod.rs index eabc6ac..f49e037 100644 --- a/src/registry/mod.rs +++ b/src/registry/mod.rs @@ -1,11 +1,6 @@ use anyhow::{Context, Result}; use base64::Engine; -use http_body_util::{BodyExt, Full}; -use hyper::body::Bytes; -use hyper::{Method, Request, StatusCode}; -use hyper_tls::HttpsConnector; -use hyper_util::client::legacy::{connect::HttpConnector, Client}; -use hyper_util::rt::TokioExecutor; +use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use tracing::{debug, info}; @@ -188,21 +183,77 @@ impl ImageReference { } pub struct RegistryClient { - client: Client, Full>, + client: reqwest::Client, #[allow(dead_code)] auth_cache: HashMap, // registry -> token } impl RegistryClient { pub fn new() -> Result { - let https = HttpsConnector::new(); - let client = Client::builder(TokioExecutor::new()).build(https); + // Create two clients - one with redirects, one without + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build()?; Ok(Self { client, auth_cache: HashMap::new(), }) } + /// Check if a blob exists in the registry using HEAD request + async fn blob_exists( + &mut self, + registry: &str, + repository: &str, + digest: &str, + auth: &RegistryAuth, + ) -> Result { + let url = format!("https://{}/v2/{}/blobs/{}", registry, repository, digest); + + let token = self.authenticate(registry, repository, auth).await?; + + let mut req = self.client.head(&url); + + if let Some(token) = token { + req = req.header("Authorization", format!("Bearer {}", token)); + } + + let response = req.send().await?; + + Ok(response.status().is_success()) + } + + /// Check if a manifest exists in the registry using HEAD request + async fn manifest_exists( + &mut self, + registry: &str, + repository: &str, + digest: &str, + auth: &RegistryAuth, + ) -> Result { + let url = format!( + "https://{}/v2/{}/manifests/{}", + registry, repository, digest + ); + + let token = self.authenticate(registry, repository, auth).await?; + + let mut req = self.client + .head(&url) + .header( + "Accept", + "application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json", + ); + + if let Some(token) = token { + req = req.header("Authorization", format!("Bearer {}", token)); + } + + let response = req.send().await?; + + Ok(response.status().is_success()) + } + // Authenticate with registry and get bearer token if needed async fn authenticate( &mut self, @@ -216,9 +267,17 @@ impl RegistryClient { self.get_anonymous_token(registry, repository).await } RegistryAuth::Basic { username, password } => { - // Use basic auth directly or get token - self.get_token_with_basic_auth(registry, repository, username, password) - .await + // Check if this is actually an OAuth token disguised as basic auth + // GCR/GAR credential helpers return username like "_dcgcloud_token" or "oauth2accesstoken" + // with the password being an OAuth token + if username.starts_with("_") || username == "oauth2accesstoken" { + // Treat the password as a bearer token + Ok(Some(password.clone())) + } else { + // Use basic auth directly or get token + self.get_token_with_basic_auth(registry, repository, username, password) + .await + } } RegistryAuth::Bearer { token } => Ok(Some(token.clone())), } @@ -231,12 +290,7 @@ impl RegistryClient { ) -> Result> { // First check API support let check_url = format!("https://{}/v2/", registry); - let req = Request::builder() - .method(Method::GET) - .uri(&check_url) - .body(Full::new(Bytes::new()))?; - - let response = self.client.request(req).await?; + let response = self.client.get(&check_url).send().await?; if response.status() == StatusCode::UNAUTHORIZED { if let Some(www_auth) = response.headers().get("www-authenticate") { @@ -262,13 +316,12 @@ impl RegistryClient { let auth_header = format!("{}:{}", username, password); let encoded_auth = base64::engine::general_purpose::STANDARD.encode(auth_header.as_bytes()); - let req = Request::builder() - .method(Method::GET) - .uri(&check_url) + let response = self + .client + .get(&check_url) .header("Authorization", format!("Basic {}", encoded_auth)) - .body(Full::new(Bytes::new()))?; - - let response = self.client.request(req).await?; + .send() + .await?; if response.status() == StatusCode::UNAUTHORIZED { if let Some(www_auth) = response.headers().get("www-authenticate") { @@ -336,16 +389,10 @@ impl RegistryClient { challenge.realm, challenge.service, scope ); - let req = Request::builder() - .method(Method::GET) - .uri(&token_url) - .body(Full::new(Bytes::new()))?; - - let response = self.client.request(req).await?; + let response = self.client.get(&token_url).send().await?; if response.status().is_success() { - let body = response.collect().await?.to_bytes(); - let token_response: TokenResponse = serde_json::from_slice(&body)?; + let token_response: TokenResponse = response.json().await?; let token = if !token_response.token.is_empty() { token_response.token } else { @@ -377,17 +424,15 @@ impl RegistryClient { let auth_header = format!("{}:{}", username, password); let encoded_auth = base64::engine::general_purpose::STANDARD.encode(auth_header.as_bytes()); - let req = Request::builder() - .method(Method::GET) - .uri(&token_url) + let response = self + .client + .get(&token_url) .header("Authorization", format!("Basic {}", encoded_auth)) - .body(Full::new(Bytes::new()))?; - - let response = self.client.request(req).await?; + .send() + .await?; if response.status().is_success() { - let body = response.collect().await?.to_bytes(); - let token_response: TokenResponse = serde_json::from_slice(&body)?; + let token_response: TokenResponse = response.json().await?; let token = if !token_response.token.is_empty() { token_response.token } else { @@ -428,17 +473,15 @@ impl RegistryClient { debug!("Pulling manifest from URL: {}", url); - let mut req_builder = Request::builder() - .method(Method::GET) - .uri(&url) + let mut req = self.client + .get(&url) .header("Accept", "application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json"); if let Some(token) = token { - req_builder = req_builder.header("Authorization", format!("Bearer {}", token)); + req = req.header("Authorization", format!("Bearer {}", token)); } - let req = req_builder.body(Full::new(Bytes::new()))?; - let response = self.client.request(req).await?; + let response = req.send().await?; if !response.status().is_success() { anyhow::bail!("Failed to pull manifest: {}", response.status()); @@ -451,7 +494,7 @@ impl RegistryClient { .unwrap_or("") .to_string(); - let body = response.collect().await?.to_bytes(); + let body = response.bytes().await?; debug!("Manifest response body: {}", String::from_utf8_lossy(&body)); // Try to parse as either image manifest or image index @@ -472,9 +515,8 @@ impl RegistryClient { debug!("Pulling platform-specific manifest from URL: {}", url); - let mut req_builder = Request::builder() - .method(Method::GET) - .uri(&url) + let mut req = self.client + .get(&url) .header("Accept", "application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json"); // Re-authenticate for the platform-specific request @@ -482,17 +524,16 @@ impl RegistryClient { .authenticate(&reference.registry, &reference.repository, auth) .await?; if let Some(token) = platform_token { - req_builder = req_builder.header("Authorization", format!("Bearer {}", token)); + req = req.header("Authorization", format!("Bearer {}", token)); } - let req = req_builder.body(Full::new(Bytes::new()))?; - let response = self.client.request(req).await?; + let response = req.send().await?; if !response.status().is_success() { anyhow::bail!("Failed to pull platform manifest: {}", response.status()); } - let platform_body = response.collect().await?.to_bytes(); + let platform_body = response.bytes().await?; debug!( "Platform manifest response body: {}", String::from_utf8_lossy(&platform_body) @@ -526,45 +567,30 @@ impl RegistryClient { reference.registry, reference.repository, descriptor.digest ); - let mut req_builder = Request::builder().method(Method::GET).uri(&url); + let mut req = self.client.get(&url); if let Some(token) = token { - req_builder = req_builder.header("Authorization", format!("Bearer {}", token)); + req = req.header("Authorization", format!("Bearer {}", token)); } - let req = req_builder.body(Full::new(Bytes::new()))?; - let response = self.client.request(req).await?; + let response = req.send().await?; - // Handle redirect responses (common for blob downloads) - if response.status() == StatusCode::TEMPORARY_REDIRECT - || response.status() == StatusCode::MOVED_PERMANENTLY - { + // Handle redirects manually (since we disabled automatic redirects) + if response.status().is_redirection() { if let Some(location) = response.headers().get("location") { let redirect_url = location.to_str()?; - debug!("Following redirect to: {}", redirect_url); - - let redirect_req = Request::builder() - .method(Method::GET) - .uri(redirect_url) - .body(Full::new(Bytes::new()))?; - - let redirect_response = self.client.request(redirect_req).await?; - + debug!("Following blob download redirect to: {}", redirect_url); + // Don't include auth header for redirects (might be to CDN/GCS) + let redirect_response = self.client.get(redirect_url).send().await?; if !redirect_response.status().is_success() { anyhow::bail!( - "Failed to pull blob {} from redirect URL: {}", + "Failed to pull blob {} from redirect: {}", descriptor.digest, redirect_response.status() ); } - - let body = redirect_response.collect().await?.to_bytes(); + let body = redirect_response.bytes().await?; return Ok(body.to_vec()); - } else { - anyhow::bail!( - "Received redirect for blob {} but no location header", - descriptor.digest - ); } } @@ -576,7 +602,7 @@ impl RegistryClient { ); } - let body = response.collect().await?.to_bytes(); + let body = response.bytes().await?; Ok(body.to_vec()) } @@ -588,8 +614,18 @@ impl RegistryClient { digest: &str, auth: &RegistryAuth, ) -> Result<()> { - info!("Starting blob push for digest: {} to {}", digest, image_ref); let reference = ImageReference::parse(image_ref)?; + + // Check if blob already exists + if self + .blob_exists(&reference.registry, &reference.repository, digest, auth) + .await? + { + debug!("Blob {} already exists, skipping push", digest); + return Ok(()); + } + + info!("Pushing blob: {} to {}", digest, image_ref); let token = self .authenticate(&reference.registry, &reference.repository, auth) .await?; @@ -600,17 +636,13 @@ impl RegistryClient { reference.registry, reference.repository ); - let mut req_builder = Request::builder() - .method(Method::POST) - .uri(&upload_url) - .header("Content-Length", "0"); + let mut req = self.client.post(&upload_url).header("Content-Length", "0"); if let Some(token) = &token { - req_builder = req_builder.header("Authorization", format!("Bearer {}", token)); + req = req.header("Authorization", format!("Bearer {}", token)); } - let req = req_builder.body(Full::new(Bytes::new()))?; - let response = self.client.request(req).await?; + let response = req.send().await?; if !response.status().is_success() { anyhow::bail!("Failed to start blob upload: {}", response.status()); @@ -624,16 +656,15 @@ impl RegistryClient { debug!("Upload location header: {}", location); - // Complete upload with PUT + // Try monolithic upload (PUT with body and ?digest=) + // If GAR redirects, it means it wants resumable upload instead let put_url = if location.starts_with("http") { - // Check if location already has query parameters if location.contains('?') { format!("{}&digest={}", location, digest) } else { format!("{}?digest={}", location, digest) } } else if location.starts_with("/v2/") { - // Relative URL starting with /v2/ if location.contains('?') { format!( "https://{}{}&digest={}", @@ -646,33 +677,133 @@ impl RegistryClient { ) } } else { - // Assume it's just a UUID or path segment format!( "https://{}/v2/{}/blobs/uploads/{}?digest={}", reference.registry, reference.repository, location, digest ) }; - debug!("Uploading blob to URL: {}", put_url); + debug!("Uploading blob to: {}", &put_url[..100.min(put_url.len())]); - let mut req_builder = Request::builder() - .method(Method::PUT) - .uri(&put_url) + // Try monolithic upload first + let mut monolithic_req = self + .client + .put(&put_url) .header("Content-Type", "application/octet-stream") - .header("Content-Length", data.len().to_string()); + .body(data.to_vec()); - if let Some(token) = token { - req_builder = req_builder.header("Authorization", format!("Bearer {}", token)); + if let Some(ref token_str) = token { + monolithic_req = + monolithic_req.header("Authorization", format!("Bearer {}", token_str)); } - let req = req_builder.body(Full::new(Bytes::copy_from_slice(data)))?; - let response = self.client.request(req).await?; + let monolithic_response = monolithic_req.send().await?; + let monolithic_status = monolithic_response.status(); - if !response.status().is_success() { - anyhow::bail!("Failed to upload blob: {}", response.status()); + // If monolithic upload succeeds, we're done + if monolithic_status.is_success() { + return Ok(()); } - Ok(()) + // If we get a redirect, GAR wants resumable upload + // Don't follow the redirect - just use resumable flow + if monolithic_status.is_redirection() { + // Build upload location without digest for PATCH + let upload_location = if location.starts_with("http") { + location.to_string() + } else if location.starts_with("/") { + // Relative URL starting with / (handles /v2/... and /artifacts-uploads/...) + format!("https://{}{}", reference.registry, location) + } else { + // Just a UUID + format!( + "https://{}/v2/{}/blobs/uploads/{}", + reference.registry, reference.repository, location + ) + }; + + // PATCH to upload data (don't follow redirects manually) + let mut patch_req = self + .client + .patch(&upload_location) + .header("Content-Type", "application/octet-stream") + .body(data.to_vec()); + + if let Some(ref token_str) = token { + patch_req = patch_req.header("Authorization", format!("Bearer {}", token_str)); + } + + let patch_response = patch_req.send().await?; + let patch_status = patch_response.status(); + let patch_headers = patch_response.headers().clone(); + + // PATCH might also return 301 redirect - treat as success if so + let finalize_location = if patch_status.is_redirection() { + patch_headers + .get("location") + .and_then(|h| h.to_str().ok()) + .unwrap_or(location) + } else if patch_status.is_success() { + // Get location from successful PATCH response + patch_headers + .get("location") + .and_then(|h| h.to_str().ok()) + .unwrap_or(location) + } else { + let body = patch_response.text().await.unwrap_or_default(); + anyhow::bail!("Failed to PATCH blob: {} - {}", patch_status, body); + }; + + // Build finalize URL with digest + let finalize_url = if finalize_location.starts_with("http") { + if finalize_location.contains('?') { + format!("{}&digest={}", finalize_location, digest) + } else { + format!("{}?digest={}", finalize_location, digest) + } + } else if finalize_location.starts_with("/") { + // Relative URL starting with / (handles /v2/... and /artifacts-uploads/...) + if finalize_location.contains('?') { + format!( + "https://{}{}&digest={}", + reference.registry, finalize_location, digest + ) + } else { + format!( + "https://{}{}?digest={}", + reference.registry, finalize_location, digest + ) + } + } else { + // Just a UUID + format!( + "https://{}/v2/{}/blobs/uploads/{}?digest={}", + reference.registry, reference.repository, finalize_location, digest + ) + }; + + // PUT to finalize + let mut finalize_req = self.client.put(&finalize_url).header("Content-Length", "0"); + + if let Some(ref token_str) = token { + finalize_req = + finalize_req.header("Authorization", format!("Bearer {}", token_str)); + } + + let finalize_response = finalize_req.send().await?; + let finalize_status = finalize_response.status(); + + if !finalize_status.is_success() { + let body = finalize_response.text().await.unwrap_or_default(); + anyhow::bail!("Failed to finalize: {} - {}", finalize_status, body); + } + + return Ok(()); + } + + // If not success or redirect, fail + let body = monolithic_response.text().await.unwrap_or_default(); + anyhow::bail!("Failed to upload blob: {} - {}", monolithic_status, body) } // Push a manifest to the registry @@ -683,44 +814,68 @@ impl RegistryClient { auth: &RegistryAuth, ) -> Result<(String, String)> { let reference = ImageReference::parse(image_ref)?; + let manifest_json = serde_json::to_vec_pretty(manifest)?; + let manifest_digest = format!("sha256:{}", sha256::digest(&manifest_json)); + + // Check if manifest already exists + if self + .manifest_exists( + &reference.registry, + &reference.repository, + &manifest_digest, + auth, + ) + .await? + { + debug!("Manifest {} already exists, skipping push", manifest_digest); + let digest_ref = format!( + "{}/{}@{}", + reference.registry, reference.repository, manifest_digest + ); + return Ok((digest_ref, manifest_digest)); + } + + info!("Pushing manifest with digest: {}", manifest_digest); + let token = self .authenticate(&reference.registry, &reference.repository, auth) .await?; - let manifest_ref = reference.tag.as_deref().unwrap_or("latest"); + // Use tag if provided, otherwise push by digest + let manifest_ref = reference.tag.as_deref().unwrap_or(&manifest_digest); let url = format!( "https://{}/v2/{}/manifests/{}", reference.registry, reference.repository, manifest_ref ); - let manifest_json = serde_json::to_vec_pretty(manifest)?; + info!("Pushing manifest to: {}", url); - let mut req_builder = Request::builder() - .method(Method::PUT) - .uri(&url) + let mut req = self + .client + .put(&url) .header("Content-Type", &manifest.media_type) - .header("Content-Length", manifest_json.len().to_string()); + .body(manifest_json.clone()); - if let Some(token) = token { - req_builder = req_builder.header("Authorization", format!("Bearer {}", token)); + if let Some(token) = &token { + req = req.header("Authorization", format!("Bearer {}", token)); } - let req = req_builder.body(Full::new(Bytes::copy_from_slice(&manifest_json)))?; - let response = self.client.request(req).await?; + let response = req.send().await?; + let status = response.status(); + let headers = response.headers().clone(); - if !response.status().is_success() { - anyhow::bail!("Failed to push manifest: {}", response.status()); + if !status.is_success() { + let body_str = response.text().await.unwrap_or_default(); + anyhow::bail!("Failed to push manifest: {} - {}", status, body_str); } - let digest = response - .headers() + let digest = headers .get("docker-content-digest") .and_then(|h| h.to_str().ok()) .unwrap_or("") .to_string(); - let location = response - .headers() + let location = headers .get("location") .and_then(|h| h.to_str().ok()) .unwrap_or(&url) @@ -737,12 +892,10 @@ impl RegistryClient { layers: Vec<(Vec, String)>, auth: &RegistryAuth, ) -> Result<(String, usize)> { - let image_ref = format!("{}:temp", repository); - // Push config blob let config_digest = format!("sha256:{}", sha256::digest(&config_data)); debug!("Pushing config blob: {}", config_digest); - self.push_blob(&image_ref, &config_data, &config_digest, auth) + self.push_blob(repository, &config_data, &config_digest, auth) .await?; // Push layers and build manifest @@ -750,7 +903,7 @@ impl RegistryClient { for (layer_data, media_type) in layers { let digest = format!("sha256:{}", sha256::digest(&layer_data)); debug!("Pushing layer: {}", digest); - self.push_blob(&image_ref, &layer_data, &digest, auth) + self.push_blob(repository, &layer_data, &digest, auth) .await?; manifest_layers.push(OciDescriptor { @@ -777,8 +930,8 @@ impl RegistryClient { annotations: None, }; - let (_, digest) = self.push_manifest(&image_ref, &manifest, auth).await?; - let reference = ImageReference::parse(&image_ref)?; + let (_, digest) = self.push_manifest(repository, &manifest, auth).await?; + let reference = ImageReference::parse(repository)?; let digest_ref = format!("{}/{}@{}", reference.registry, reference.repository, digest); let manifest_size = serde_json::to_vec(&manifest)?.len(); @@ -825,17 +978,14 @@ impl RegistryClient { base_image_ref: &str, base_auth: &RegistryAuth, ) -> Result<(String, usize)> { - let image_ref = format!("{}:temp", repository); - // Push config blob let config_digest = format!("sha256:{}", sha256::digest(&config_data)); - debug!("Pushing config blob: {}", config_digest); - self.push_blob(&image_ref, &config_data, &config_digest, auth) + self.push_blob(repository, &config_data, &config_digest, auth) .await?; // Copy base image layers if they don't exist in target registry let base_reference = ImageReference::parse(base_image_ref)?; - let target_reference = ImageReference::parse(&image_ref)?; + let target_reference = ImageReference::parse(repository)?; // Check if we need to copy base layers (cross-registry scenario) let need_copy_layers = base_reference.registry != target_reference.registry; @@ -868,7 +1018,7 @@ impl RegistryClient { .await?; // Push the layer to target registry - self.push_blob(&image_ref, &layer_data, &layer.digest, auth) + self.push_blob(repository, &layer_data, &layer.digest, auth) .await?; } } @@ -876,7 +1026,7 @@ impl RegistryClient { // Push the new application layer let new_layer_digest = format!("sha256:{}", sha256::digest(&new_layer_data)); debug!("Pushing new application layer: {}", new_layer_digest); - self.push_blob(&image_ref, &new_layer_data, &new_layer_digest, auth) + self.push_blob(repository, &new_layer_data, &new_layer_digest, auth) .await?; // Create manifest with all layers (base + new) @@ -906,7 +1056,7 @@ impl RegistryClient { annotations: None, }; - let (_, digest) = self.push_manifest(&image_ref, &oci_manifest, auth).await?; + let (_, digest) = self.push_manifest(repository, &oci_manifest, auth).await?; let digest_ref = format!( "{}/{}@{}", target_reference.registry, target_reference.repository, digest @@ -928,6 +1078,7 @@ impl RegistryClient { image_ref: &str, manifest_descriptors: Vec, auth: &RegistryAuth, + push_tag: bool, ) -> Result { let reference = ImageReference::parse(image_ref)?; @@ -969,9 +1120,17 @@ impl RegistryClient { ); } - // Serialize and push as manifest + // Serialize and calculate digest let manifest_json = serde_json::to_vec_pretty(&oci_index)?; - let manifest_ref = reference.tag.as_deref().unwrap_or("latest"); + let manifest_digest = format!("sha256:{}", sha256::digest(&manifest_json)); + + // Push by digest or tag based on push_tag flag + let manifest_ref = if push_tag { + reference.tag.as_deref().unwrap_or("latest") + } else { + &manifest_digest + }; + let url = format!( "https://{}/v2/{}/manifests/{}", reference.registry, reference.repository, manifest_ref @@ -981,30 +1140,31 @@ impl RegistryClient { .authenticate(&reference.registry, &reference.repository, auth) .await?; - let mut req_builder = Request::builder() - .method(Method::PUT) - .uri(&url) + let mut req = self + .client + .put(&url) .header("Content-Type", "application/vnd.oci.image.index.v1+json") - .header("Content-Length", manifest_json.len().to_string()); + .body(manifest_json.clone()); if let Some(token) = token { - req_builder = req_builder.header("Authorization", format!("Bearer {}", token)); + req = req.header("Authorization", format!("Bearer {}", token)); } - let req = req_builder.body(Full::new(Bytes::copy_from_slice(&manifest_json)))?; - let response = self.client.request(req).await?; + let response = req.send().await?; if !response.status().is_success() { anyhow::bail!("Failed to push manifest list: {}", response.status()); } + // Get digest from response or use the calculated one let digest = response .headers() .get("docker-content-digest") .and_then(|h| h.to_str().ok()) - .unwrap_or("") + .unwrap_or(&manifest_digest) .to_string(); + // Always return by digest let image_ref = format!("{}/{}@{}", reference.registry, reference.repository, digest); Ok(image_ref) diff --git a/tests/testdata/alpine_platforms.txt b/tests/testdata/alpine_platforms.txt index 4933136..83c9a18 100644 --- a/tests/testdata/alpine_platforms.txt +++ b/tests/testdata/alpine_platforms.txt @@ -17,6 +17,6 @@ fn main() { # Build with Alpine base - should detect many platforms env RUST_LOG=info -exec ./krust build --no-push --image test.local/alpine:latest . +exec ./krust build --no-push . stderr 'Detecting available platforms from base image: alpine:latest' stderr 'platforms' diff --git a/tests/testdata/auth_base64.txt b/tests/testdata/auth_base64.txt index becbb68..54021da 100644 --- a/tests/testdata/auth_base64.txt +++ b/tests/testdata/auth_base64.txt @@ -29,6 +29,6 @@ env HOME=$WORK env RUST_LOG=debug # Try to build (will fail at registry, but auth should be resolved) -! exec ./krust build --platform linux/amd64 --image ghcr.io/user/app:latest . +! exec ./krust build --platform linux/amd64 . stderr 'Resolving auth' stderr 'ghcr.io' diff --git a/tests/testdata/auth_credential_helper.txt b/tests/testdata/auth_credential_helper.txt index b63622a..04399b1 100644 --- a/tests/testdata/auth_credential_helper.txt +++ b/tests/testdata/auth_credential_helper.txt @@ -57,6 +57,6 @@ env RUST_LOG=debug # Try to push to mock.registry.io (will fail at network level, but should call helper) # Using --no-push to avoid actual network call -! exec ./krust build --platform linux/amd64 --image mock.registry.io/test:latest . +! exec ./krust build --platform linux/amd64 . # Should see the helper being attempted stderr 'credential helper' diff --git a/tests/testdata/auth_docker_config_basic.txt b/tests/testdata/auth_docker_config_basic.txt index aaccc3b..55d87e1 100644 --- a/tests/testdata/auth_docker_config_basic.txt +++ b/tests/testdata/auth_docker_config_basic.txt @@ -31,6 +31,6 @@ env RUST_LOG=debug # Try to build and push (will fail at registry level, but auth should be resolved) # We can verify auth was read from config in debug logs -! exec ./krust build --platform linux/amd64 --image test.example.com/app:latest . +! exec ./krust build --platform linux/amd64 . stderr 'Resolving auth' stderr 'test.example.com' diff --git a/tests/testdata/build_requires_repo.txt b/tests/testdata/build_requires_repo.txt index 9f81b64..d2e8b2a 100644 --- a/tests/testdata/build_requires_repo.txt +++ b/tests/testdata/build_requires_repo.txt @@ -12,6 +12,6 @@ fn main() { println!("Hello, world!"); } -# Build without KRUST_REPO or --image should fail +# Build without KRUST_REPO should fail ! exec ./krust build --no-push . -stderr 'Either --image or KRUST_REPO must be set' +stderr 'KRUST_REPO must be set' diff --git a/tests/testdata/build_with_env.txt b/tests/testdata/build_with_env.txt index 42a1289..be62f4e 100644 --- a/tests/testdata/build_with_env.txt +++ b/tests/testdata/build_with_env.txt @@ -13,7 +13,6 @@ fn main() { } # Build with KRUST_REPO should succeed -env KRUST_REPO=test.local [linux] exec ./krust build --no-push --platform linux/amd64 . [darwin] exec ./krust build --no-push --platform linux/amd64 . stderr 'Building Rust project' diff --git a/tests/testdata/build_with_image_flag.txt b/tests/testdata/build_with_image_flag.txt deleted file mode 100644 index 13e37c3..0000000 --- a/tests/testdata/build_with_image_flag.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Test build with --image flag - --- Cargo.toml -- -[package] -name = "test-app" -version = "0.1.0" -edition = "2021" - -[dependencies] --- src/main.rs -- -fn main() { - println!("Hello, world!"); -} - -# Build with --image flag should succeed -[linux] exec ./krust build --no-push --platform linux/amd64 --image test.local/myapp:v1 . -[darwin] exec ./krust build --no-push --platform linux/amd64 --image test.local/myapp:v1 . -stderr 'Building Rust project' -stderr 'Successfully built image' diff --git a/tests/testdata/clean_output.txt b/tests/testdata/clean_output.txt index c695642..9535488 100644 --- a/tests/testdata/clean_output.txt +++ b/tests/testdata/clean_output.txt @@ -13,8 +13,8 @@ fn main() { } # With --no-push, stdout should be empty -[linux] exec ./krust build --no-push --platform linux/amd64 --image test.local/clean:latest . -[darwin] exec ./krust build --no-push --platform linux/amd64 --image test.local/clean:latest . +[linux] exec ./krust build --no-push --platform linux/amd64 . +[darwin] exec ./krust build --no-push --platform linux/amd64 . ! stdout . stderr 'Building Rust project' stderr 'Successfully built image' diff --git a/tests/testdata/custom_base_image.txt b/tests/testdata/custom_base_image.txt index 5308c9c..b114e51 100644 --- a/tests/testdata/custom_base_image.txt +++ b/tests/testdata/custom_base_image.txt @@ -16,8 +16,12 @@ fn main() { } # Build with custom base image -[linux] exec ./krust build --no-push --platform linux/amd64 --image test.local/alpine-app:latest . -[darwin] exec ./krust build --no-push --platform linux/amd64 --image test.local/alpine-app:latest . -stderr 'Building Rust project' -stderr 'base-image.*alpine' -stderr 'Successfully built image' +[linux] exec ./krust build --no-push --platform linux/amd64 . +[linux] stderr 'Building Rust project' +[linux] stderr 'base-image.*alpine' +[linux] stderr 'Successfully built image' + +[darwin] exec ./krust build --no-push --platform linux/amd64 . +[darwin] stderr 'Building Rust project' +[darwin] stderr 'base-image.*alpine' +[darwin] stderr 'Successfully built image' diff --git a/tests/testdata/multi_platform.txt b/tests/testdata/multi_platform.txt index 3ac6713..52132c8 100644 --- a/tests/testdata/multi_platform.txt +++ b/tests/testdata/multi_platform.txt @@ -14,10 +14,10 @@ fn main() { # Build for multiple platforms # Note: This may fail if toolchains aren't installed, which is expected -[linux] exec ./krust build --no-push --platform linux/amd64,linux/arm64 --image test.local/multi:latest . +[linux] exec ./krust build --no-push --platform linux/amd64,linux/arm64 . [linux] stderr 'Building for platform: linux/amd64' [linux] stderr 'Building for platform: linux/arm64' -[darwin] exec ./krust build --no-push --platform linux/amd64,linux/arm64 --image test.local/multi:latest . +[darwin] exec ./krust build --no-push --platform linux/amd64,linux/arm64 . [darwin] stderr 'Building for platform: linux/amd64' [darwin] stderr 'Building for platform: linux/arm64' diff --git a/tests/testdata/platform_detection.txt b/tests/testdata/platform_detection.txt index d6fc399..d9b39a6 100644 --- a/tests/testdata/platform_detection.txt +++ b/tests/testdata/platform_detection.txt @@ -14,6 +14,6 @@ fn main() { # Build without explicit platform - should detect from base image env RUST_LOG=info -exec ./krust build --no-push --image test.local/detection:latest . +exec ./krust build --no-push . stderr 'Detecting available platforms' stderr 'Detected platforms' diff --git a/tests/testdata/platform_override.txt b/tests/testdata/platform_override.txt index 7792161..00635f3 100644 --- a/tests/testdata/platform_override.txt +++ b/tests/testdata/platform_override.txt @@ -14,7 +14,7 @@ fn main() { # When platform is explicit, no detection should happen env RUST_LOG=info -[linux] exec ./krust build --no-push --platform linux/amd64 --image test.local/explicit:latest . -[darwin] exec ./krust build --no-push --platform linux/amd64 --image test.local/explicit:latest . +[linux] exec ./krust build --no-push --platform linux/amd64 . +[darwin] exec ./krust build --no-push --platform linux/amd64 . ! stderr 'Detecting available platforms' stderr 'Building for platform: linux/amd64' diff --git a/tests/testdata/single_platform.txt b/tests/testdata/single_platform.txt index 1ddce5c..e1ba6c1 100644 --- a/tests/testdata/single_platform.txt +++ b/tests/testdata/single_platform.txt @@ -13,10 +13,10 @@ fn main() { } # Build for single platform -[linux] exec ./krust build --no-push --platform linux/amd64 --image test.local/single:latest . +[linux] exec ./krust build --no-push --platform linux/amd64 . [linux] stderr 'Building for platform: linux/amd64' [linux] stderr 'Successfully built image for 1 platform' -[darwin] exec ./krust build --no-push --platform linux/amd64 --image test.local/single:latest . +[darwin] exec ./krust build --no-push --platform linux/amd64 . [darwin] stderr 'Building for platform: linux/amd64' [darwin] stderr 'Successfully built image for 1 platform' diff --git a/tests/testdata/verbose_logging.txt b/tests/testdata/verbose_logging.txt index 23827a7..f0737ce 100644 --- a/tests/testdata/verbose_logging.txt +++ b/tests/testdata/verbose_logging.txt @@ -13,6 +13,6 @@ fn main() { } # Build with --verbose should show DEBUG logs -[linux] exec ./krust --verbose build --no-push --platform linux/amd64 --image test.local/myapp:latest . -[darwin] exec ./krust --verbose build --no-push --platform linux/amd64 --image test.local/myapp:latest . +[linux] exec ./krust --verbose build --no-push --platform linux/amd64 . +[darwin] exec ./krust --verbose build --no-push --platform linux/amd64 . stderr 'DEBUG' diff --git a/tests/testdata/workspace_support.txt b/tests/testdata/workspace_support.txt index 2927879..230338e 100644 --- a/tests/testdata/workspace_support.txt +++ b/tests/testdata/workspace_support.txt @@ -17,7 +17,7 @@ fn main() { } # Build workspace member by specifying its directory -[linux] exec ./krust build --no-push --platform linux/amd64 --image test.local/workspace:latest app -[darwin] exec ./krust build --no-push --platform linux/amd64 --image test.local/workspace:latest app +[linux] exec ./krust build --no-push --platform linux/amd64 app +[darwin] exec ./krust build --no-push --platform linux/amd64 app stderr 'Building Rust project' stderr 'Successfully built image'