diff --git a/pymage/DESIGN.md b/pymage/DESIGN.md index 47e03cc..dda8f45 100644 --- a/pymage/DESIGN.md +++ b/pymage/DESIGN.md @@ -1,7 +1,8 @@ # `pymage` — a docker-less, layer-aware Python image builder -> Status: **Design / plan** (no code yet). This document proposes the architecture -> and an incremental implementation plan. Comments and pushback welcome. +> Status: **Implemented.** This document captures the architecture and rationale; +> the builder, tests, and CI now live alongside it in this directory. Some items +> noted as future work (e.g. a network wheel fetcher) remain unimplemented. ## 1. Goal diff --git a/pymage/README.md b/pymage/README.md index 7ff4c0e..8670eba 100644 --- a/pymage/README.md +++ b/pymage/README.md @@ -23,11 +23,15 @@ See [`DESIGN.md`](./DESIGN.md) for the full rationale. ## Usage ``` +# --lock is a pinned + hashed requirements file +# (pip-compile / uv pip compile --generate-hashes) +# --find-links is a directory of the resolved .whl files +# --source is the application source (optional) pymage build \ --base cgr.dev/chainguard/python:latest \ - --lock requirements.txt \ # pinned + hashed (pip-compile / uv pip compile --generate-hashes) - --find-links ./wheelhouse \ # directory of resolved .whl files - --source ./ \ # application source (optional) + --lock requirements.txt \ + --find-links ./wheelhouse \ + --source ./ \ --entrypoint python --entrypoint -m --entrypoint myapp \ -t registry.example.com/me/myapp:latest ``` @@ -45,8 +49,9 @@ is intentionally local so builds are hermetic and reproducible. | `--print-digest` | Print only the resulting image digest (no push). | | `--sbom PATH` | Write a CycloneDX SBOM of the resolved wheels. | | `--layer-strategy` | `per-wheel` (default) or `single-deps-layer`. | -| `--platform` | Platform for base resolution, e.g. `linux/amd64`. | -| `--python` | site-packages interpreter dir (default `python3.12`). | +| `--platform` | Target platform; selects compatible wheels and the base, e.g. `linux/amd64`. | +| `--python` | Interpreter version / site-packages dir (default `python3.12`); also selects compatible wheels. | +| `--cache-dir` | Content-addressed layer cache; reuses compressed layers across rebuilds. | | `--prefix` | install prefix / venv root (default `/app/.venv`). | | `--workdir` | image working dir and source destination (default `/app`). | | `--user` | image user, e.g. `65532`. | diff --git a/pymage/internal/build/build.go b/pymage/internal/build/build.go index 997fb5e..cc9068d 100644 --- a/pymage/internal/build/build.go +++ b/pymage/internal/build/build.go @@ -13,18 +13,25 @@ import ( "os" "path" "path/filepath" + "runtime" "sort" "strings" + "sync" "time" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/imjasonh/terraform-playground/pymage/internal/cache" "github.com/imjasonh/terraform-playground/pymage/internal/ptar" "github.com/imjasonh/terraform-playground/pymage/internal/wheel" "github.com/imjasonh/terraform-playground/pymage/internal/wheelhouse" ) +// layerFormatVersion is part of the cache key so a change in how we lay out +// wheels invalidates previously cached layers. +const layerFormatVersion = "1" + // LayerStrategy controls how dependency layers are partitioned. type LayerStrategy string @@ -65,12 +72,21 @@ type Options struct { User string // Labels are added to the image config. Labels map[string]string + + // Cache, if non-nil, stores/reuses built dependency layers across builds. + Cache *cache.Cache } var defaultIgnore = []string{ ".git", ".git/**", "**/__pycache__/**", "**/*.pyc", "**/*.pyo", ".venv/**", "**/.pytest_cache/**", "**/.mypy_cache/**", + // Avoid baking common secret material into the image. + "**/.env", "**/.env.*", + "**/*.pem", "**/*.key", "**/*.pfx", "**/*.p12", + "**/id_rsa", "**/id_dsa", "**/id_ecdsa", "**/id_ed25519", + "**/.netrc", + ".ssh", ".ssh/**", ".aws", ".aws/**", } // Build assembles and returns the image. It performs no network I/O; callers @@ -134,20 +150,16 @@ func dependencyAddendums(opts Options) ([]mutate.Addendum, error) { return []mutate.Addendum{{Layer: layer, History: history("python dependencies")}}, nil case PerWheel: - adds := make([]mutate.Addendum, 0, len(opts.Wheels)) - for _, rw := range opts.Wheels { - files, err := wheelFiles(rw, opts.Layout) - if err != nil { - return nil, err - } - layer, err := ptar.Layer(files) - if err != nil { - return nil, err - } - adds = append(adds, mutate.Addendum{ - Layer: layer, + layers, err := perWheelLayers(opts) + if err != nil { + return nil, err + } + adds := make([]mutate.Addendum, len(opts.Wheels)) + for i, rw := range opts.Wheels { + adds[i] = mutate.Addendum{ + Layer: layers[i], History: history(fmt.Sprintf("wheel %s==%s", rw.Name, rw.Version)), - }) + } } return adds, nil @@ -156,6 +168,68 @@ func dependencyAddendums(opts Options) ([]mutate.Addendum, error) { } } +// perWheelLayers builds one layer per wheel concurrently (bounded by CPU +// count), preserving input order. Layer construction is independent and +// CPU-bound (read + gzip), so parallelism is a straightforward win. +func perWheelLayers(opts Options) ([]v1.Layer, error) { + layers := make([]v1.Layer, len(opts.Wheels)) + errs := make([]error, len(opts.Wheels)) + + workers := runtime.NumCPU() + if workers < 1 { + workers = 1 + } + sem := make(chan struct{}, workers) + var wg sync.WaitGroup + for i := range opts.Wheels { + wg.Add(1) + go func(i int) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + layers[i], errs[i] = buildWheelLayer(opts.Wheels[i], opts.Layout, opts.Cache) + }(i) + } + wg.Wait() + + for _, err := range errs { + if err != nil { + return nil, err + } + } + return layers, nil +} + +// buildWheelLayer returns the layer for a wheel, using the cache when present. +func buildWheelLayer(rw wheelhouse.ResolvedWheel, layout wheel.Layout, c *cache.Cache) (v1.Layer, error) { + key := wheelCacheKey(rw, layout) + if c != nil { + if l, ok := c.Get(key); ok { + return l, nil + } + } + files, err := wheelFiles(rw, layout) + if err != nil { + return nil, err + } + layer, err := ptar.Layer(files) + if err != nil { + return nil, err + } + if c != nil { + // A cache write failure must not fail the build; we just lose the + // speedup for this layer. + _ = c.Put(key, layer) + } + return layer, nil +} + +func wheelCacheKey(rw wheelhouse.ResolvedWheel, layout wheel.Layout) string { + return strings.Join([]string{ + "wheel", layerFormatVersion, rw.SHA256, layout.Prefix, layout.PythonTag, + }, "|") +} + func wheelFiles(rw wheelhouse.ResolvedWheel, layout wheel.Layout) ([]ptar.File, error) { w, err := wheel.Open(rw.Path) if err != nil { @@ -217,7 +291,7 @@ func sourceLayer(srcDir, workingDir string, extraIgnore []string) (v1.Layer, err if len(files) == 0 { return nil, nil } - sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) + sort.SliceStable(files, func(i, j int) bool { return files[i].Path < files[j].Path }) return ptar.Layer(files) } diff --git a/pymage/internal/build/build_test.go b/pymage/internal/build/build_test.go index e6e73ad..90cf766 100644 --- a/pymage/internal/build/build_test.go +++ b/pymage/internal/build/build_test.go @@ -11,6 +11,7 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/imjasonh/terraform-playground/pymage/internal/cache" "github.com/imjasonh/terraform-playground/pymage/internal/testwheel" "github.com/imjasonh/terraform-playground/pymage/internal/wheel" "github.com/imjasonh/terraform-playground/pymage/internal/wheelhouse" @@ -193,6 +194,53 @@ func TestPerWheelOnlyNewDepChangesLayers(t *testing.T) { } } +func TestBuildWithCacheIsConsistentAndPopulates(t *testing.T) { + dir := t.TempDir() + wheels := []wheelhouse.ResolvedWheel{mkWheel(t, dir, "alpha", "1.0"), mkWheel(t, dir, "beta", "2.0")} + + c, err := cache.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + + opts := baseOpts(wheels) + opts.Cache = c + + // Cold cache build. + img1, err := Build(opts) + if err != nil { + t.Fatal(err) + } + d1, err := img1.Digest() + if err != nil { + t.Fatal(err) + } + + // Warm cache build must produce an identical image. + img2, err := Build(opts) + if err != nil { + t.Fatal(err) + } + d2, err := img2.Digest() + if err != nil { + t.Fatal(err) + } + if d1 != d2 { + t.Fatalf("cached build digest differs: %s != %s", d1, d2) + } + + // And it must match a build with no cache at all (cache changes nothing + // about the output). + img3, err := Build(baseOpts(wheels)) + if err != nil { + t.Fatal(err) + } + d3, _ := img3.Digest() + if d3 != d1 { + t.Fatalf("cached vs uncached digest differ: %s != %s", d1, d3) + } +} + func TestSourceLayerIgnore(t *testing.T) { src := t.TempDir() must := func(p, content string) { diff --git a/pymage/internal/cache/cache.go b/pymage/internal/cache/cache.go new file mode 100644 index 0000000..69f8478 --- /dev/null +++ b/pymage/internal/cache/cache.go @@ -0,0 +1,77 @@ +// Package cache is a content-addressed, on-disk store for built OCI layers. +// +// Building a layer from a wheel means reading every member and gzip-compressing +// the tar — pure CPU that is wasted on every rebuild for dependencies that have +// not changed. Keyed by the wheel's sha256 (plus the install layout), the cache +// lets an unchanged dependency reuse its previously compressed blob verbatim, +// so a rebuild does no decompression/recompression for those layers. +package cache + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// Cache stores compressed layer blobs under a directory. +type Cache struct { + dir string +} + +// New returns a cache rooted at dir, creating it if necessary. +func New(dir string) (*Cache, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("cache: create %q: %w", dir, err) + } + return &Cache{dir: dir}, nil +} + +func (c *Cache) path(key string) string { + sum := sha256.Sum256([]byte(key)) + return filepath.Join(c.dir, hex.EncodeToString(sum[:])+".tar.gz") +} + +// Get returns the cached layer for key, or (nil, false) on a miss. +func (c *Cache) Get(key string) (v1.Layer, bool) { + p := c.path(key) + if _, err := os.Stat(p); err != nil { + return nil, false + } + l, err := tarball.LayerFromFile(p) + if err != nil { + return nil, false + } + return l, true +} + +// Put stores the layer's compressed blob under key. Writes are atomic (temp +// file + rename) so concurrent builders and interrupted writes are safe. +func (c *Cache) Put(key string, l v1.Layer) error { + rc, err := l.Compressed() + if err != nil { + return err + } + defer func() { _ = rc.Close() }() + + tmp, err := os.CreateTemp(c.dir, ".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer func() { _ = os.Remove(tmpName) }() + + if _, err := io.Copy(tmp, rc); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpName, c.path(key)) +} diff --git a/pymage/internal/cache/cache_test.go b/pymage/internal/cache/cache_test.go new file mode 100644 index 0000000..16cbe21 --- /dev/null +++ b/pymage/internal/cache/cache_test.go @@ -0,0 +1,43 @@ +package cache + +import ( + "testing" + + "github.com/imjasonh/terraform-playground/pymage/internal/ptar" +) + +func TestPutGetRoundTrip(t *testing.T) { + c, err := New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + + layer, err := ptar.Layer([]ptar.File{{Path: "site/foo.py", Data: []byte("X=1\n")}}) + if err != nil { + t.Fatal(err) + } + want, err := layer.Digest() + if err != nil { + t.Fatal(err) + } + + const key = "wheel|1|abc|/app/.venv|python3.12" + if _, ok := c.Get(key); ok { + t.Fatal("expected a miss before Put") + } + if err := c.Put(key, layer); err != nil { + t.Fatal(err) + } + + cached, ok := c.Get(key) + if !ok { + t.Fatal("expected a hit after Put") + } + got, err := cached.Digest() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("cached layer digest %s != original %s", got, want) + } +} diff --git a/pymage/internal/cli/cli.go b/pymage/internal/cli/cli.go index 8fdd54a..71f4455 100644 --- a/pymage/internal/cli/cli.go +++ b/pymage/internal/cli/cli.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "os" + "strconv" "strings" "github.com/google/go-containerregistry/pkg/authn" @@ -16,6 +17,7 @@ import ( "github.com/spf13/cobra" "github.com/imjasonh/terraform-playground/pymage/internal/build" + "github.com/imjasonh/terraform-playground/pymage/internal/cache" "github.com/imjasonh/terraform-playground/pymage/internal/lock" "github.com/imjasonh/terraform-playground/pymage/internal/sbom" "github.com/imjasonh/terraform-playground/pymage/internal/wheel" @@ -58,6 +60,7 @@ type buildFlags struct { sbomOut string requireHash bool insecure bool + cacheDir string } func buildCmd() *cobra.Command { @@ -92,6 +95,7 @@ func buildCmd() *cobra.Command { fs.StringVar(&f.sbomOut, "sbom", "", "write a CycloneDX SBOM to this path") fs.BoolVar(&f.requireHash, "require-hashes", true, "require every requirement to carry --hash entries") fs.BoolVar(&f.insecure, "insecure", false, "use plain HTTP for the registry") + fs.StringVar(&f.cacheDir, "cache-dir", "", "directory for the content-addressed layer cache (speeds up rebuilds)") return cmd } @@ -106,6 +110,14 @@ func runBuild(ctx context.Context, f *buildFlags) error { return fmt.Errorf("--find-links is required (a directory of wheels)") } + if _, err := keyValues(f.env); err != nil { + return fmt.Errorf("--env: %w", err) + } + labels, err := keyValues(f.labels) + if err != nil { + return fmt.Errorf("--label: %w", err) + } + reqs, err := lock.ParseFile(f.lockFile) if err != nil { return err @@ -113,10 +125,6 @@ func runBuild(ctx context.Context, f *buildFlags) error { if err := lock.Validate(reqs, f.requireHash); err != nil { return err } - wheels, err := wheelhouse.Resolve(reqs, f.findLinks) - if err != nil { - return err - } var platform *v1.Platform if f.platform != "" { @@ -126,6 +134,15 @@ func runBuild(ctx context.Context, f *buildFlags) error { } } + target, err := resolveTarget(platform, f.pythonTag) + if err != nil { + return err + } + wheels, err := wheelhouse.Resolve(reqs, f.findLinks, target) + if err != nil { + return err + } + var nameOpts []name.Option if f.insecure { nameOpts = append(nameOpts, name.Insecure) @@ -136,9 +153,12 @@ func runBuild(ctx context.Context, f *buildFlags) error { return err } - labels, err := keyValues(f.labels) - if err != nil { - return fmt.Errorf("--label: %w", err) + var layerCache *cache.Cache + if f.cacheDir != "" { + layerCache, err = cache.New(f.cacheDir) + if err != nil { + return err + } } img, err := build.Build(build.Options{ @@ -154,6 +174,7 @@ func runBuild(ctx context.Context, f *buildFlags) error { Env: f.env, User: f.user, Labels: labels, + Cache: layerCache, }) if err != nil { return err @@ -214,6 +235,44 @@ func runBuild(ctx context.Context, f *buildFlags) error { return nil } +// resolveTarget derives the wheel-selection target (platform + interpreter) +// from the --platform and --python flags, defaulting to linux/amd64. +func resolveTarget(platform *v1.Platform, pythonTag string) (wheel.Target, error) { + t := wheel.Target{OS: "linux", Arch: "amd64"} + if platform != nil { + if platform.OS != "" { + t.OS = platform.OS + } + if platform.Architecture != "" { + t.Arch = platform.Architecture + } + } + maj, min, err := parsePythonTag(pythonTag) + if err != nil { + return wheel.Target{}, err + } + t.PyMajor, t.PyMinor = maj, min + return t, nil +} + +// parsePythonTag parses "python3.12" into (3, 12). +func parsePythonTag(tag string) (int, int, error) { + s := strings.TrimPrefix(tag, "python") + majStr, minStr, ok := strings.Cut(s, ".") + if !ok { + return 0, 0, fmt.Errorf("--python %q must look like pythonX.Y", tag) + } + maj, err := strconv.Atoi(majStr) + if err != nil { + return 0, 0, fmt.Errorf("--python %q: bad major version", tag) + } + min, err := strconv.Atoi(minStr) + if err != nil { + return 0, 0, fmt.Errorf("--python %q: bad minor version", tag) + } + return maj, min, nil +} + func resolveBase(ctx context.Context, ref string, platform *v1.Platform, nameOpts ...name.Option) (v1.Image, error) { if ref == "scratch" { return empty.Image, nil diff --git a/pymage/internal/cli/cli_test.go b/pymage/internal/cli/cli_test.go index 8cd569d..2f7e438 100644 --- a/pymage/internal/cli/cli_test.go +++ b/pymage/internal/cli/cli_test.go @@ -17,6 +17,35 @@ import ( "github.com/imjasonh/terraform-playground/pymage/internal/testwheel" ) +func TestParsePythonTag(t *testing.T) { + maj, min, err := parsePythonTag("python3.12") + if err != nil || maj != 3 || min != 12 { + t.Fatalf("parsePythonTag = %d.%d, %v", maj, min, err) + } + if _, _, err := parsePythonTag("python3"); err == nil { + t.Error("expected error for missing minor version") + } +} + +func TestResolveTargetDefaults(t *testing.T) { + tg, err := resolveTarget(nil, "python3.11") + if err != nil { + t.Fatal(err) + } + if tg.OS != "linux" || tg.Arch != "amd64" || tg.PyMajor != 3 || tg.PyMinor != 11 { + t.Fatalf("unexpected target %+v", tg) + } +} + +func TestEnvValidation(t *testing.T) { + if _, err := keyValues([]string{"GOOD=1"}); err != nil { + t.Errorf("unexpected error: %v", err) + } + if _, err := keyValues([]string{"NOEQUALS"}); err == nil { + t.Error("expected error for env entry without '='") + } +} + // TestBuildCommandEndToEnd drives the cobra `build` command exactly as a user // would: a hashed requirements file + a wheelhouse + source, pushed to a local // registry, then verifies the pushed image is pullable and reproducible.