build/cli: layer cache + parallel wheel layers, secret deny-list, env validation, docs

- add internal/cache content-addressed layer store; build per-wheel layers in
  parallel and reuse cached compressed blobs across rebuilds (--cache-dir)
- thread platform/python target into wheel resolution; validate --env
- add common secret patterns to the default source ignore list
- README: copy/paste-safe usage example + new flags; DESIGN status updated

Co-authored-by: Jason Hall <imjasonh@users.noreply.github.com>

pymage/DESIGN.md

@@ -1,7 +1,8 @@
 # `pymage` — a docker-less, layer-aware Python image builder
 
-> Status: **Design / plan** (no code yet). This document proposes the architecture
-> and an incremental implementation plan. Comments and pushback welcome.
+> Status: **Implemented.** This document captures the architecture and rationale;
+> the builder, tests, and CI now live alongside it in this directory. Some items
+> noted as future work (e.g. a network wheel fetcher) remain unimplemented.
 
 ## 1. Goal
 

pymage/README.md

@@ -23,11 +23,15 @@ See [`DESIGN.md`](./DESIGN.md) for the full rationale.
 ## Usage
 
 ```
+# --lock is a pinned + hashed requirements file
+#   (pip-compile / uv pip compile --generate-hashes)
+# --find-links is a directory of the resolved .whl files
+# --source is the application source (optional)
 pymage build \
   --base cgr.dev/chainguard/python:latest \
-  --lock requirements.txt \      # pinned + hashed (pip-compile / uv pip compile --generate-hashes)
-  --find-links ./wheelhouse \    # directory of resolved .whl files
-  --source ./ \                  # application source (optional)
+  --lock requirements.txt \
+  --find-links ./wheelhouse \
+  --source ./ \
   --entrypoint python --entrypoint -m --entrypoint myapp \
   -t registry.example.com/me/myapp:latest
 ```
@@ -45,8 +49,9 @@ is intentionally local so builds are hermetic and reproducible.
 | `--print-digest` | Print only the resulting image digest (no push). |
 | `--sbom PATH` | Write a CycloneDX SBOM of the resolved wheels. |
 | `--layer-strategy` | `per-wheel` (default) or `single-deps-layer`. |
-| `--platform` | Platform for base resolution, e.g. `linux/amd64`. |
-| `--python` | site-packages interpreter dir (default `python3.12`). |
+| `--platform` | Target platform; selects compatible wheels and the base, e.g. `linux/amd64`. |
+| `--python` | Interpreter version / site-packages dir (default `python3.12`); also selects compatible wheels. |
+| `--cache-dir` | Content-addressed layer cache; reuses compressed layers across rebuilds. |
 | `--prefix` | install prefix / venv root (default `/app/.venv`). |
 | `--workdir` | image working dir and source destination (default `/app`). |
 | `--user` | image user, e.g. `65532`. |

pymage/internal/build/build.go

@@ -13,18 +13,25 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 
+	"github.com/imjasonh/terraform-playground/pymage/internal/cache"
 	"github.com/imjasonh/terraform-playground/pymage/internal/ptar"
 	"github.com/imjasonh/terraform-playground/pymage/internal/wheel"
 	"github.com/imjasonh/terraform-playground/pymage/internal/wheelhouse"
 )
 
+// layerFormatVersion is part of the cache key so a change in how we lay out
+// wheels invalidates previously cached layers.
+const layerFormatVersion = "1"
+
 // LayerStrategy controls how dependency layers are partitioned.
 type LayerStrategy string
 
@@ -65,12 +72,21 @@ type Options struct {
 	User string
 	// Labels are added to the image config.
 	Labels map[string]string
+
+	// Cache, if non-nil, stores/reuses built dependency layers across builds.
+	Cache *cache.Cache
 }
 
 var defaultIgnore = []string{
 	".git", ".git/**",
 	"**/__pycache__/**", "**/*.pyc", "**/*.pyo",
 	".venv/**", "**/.pytest_cache/**", "**/.mypy_cache/**",
+	// Avoid baking common secret material into the image.
+	"**/.env", "**/.env.*",
+	"**/*.pem", "**/*.key", "**/*.pfx", "**/*.p12",
+	"**/id_rsa", "**/id_dsa", "**/id_ecdsa", "**/id_ed25519",
+	"**/.netrc",
+	".ssh", ".ssh/**", ".aws", ".aws/**",
 }
 
 // Build assembles and returns the image. It performs no network I/O; callers
@@ -134,20 +150,16 @@ func dependencyAddendums(opts Options) ([]mutate.Addendum, error) {
 		return []mutate.Addendum{{Layer: layer, History: history("python dependencies")}}, nil
 
 	case PerWheel:
-		adds := make([]mutate.Addendum, 0, len(opts.Wheels))
-		for _, rw := range opts.Wheels {
-			files, err := wheelFiles(rw, opts.Layout)
-			if err != nil {
-				return nil, err
-			}
-			layer, err := ptar.Layer(files)
-			if err != nil {
-				return nil, err
-			}
-			adds = append(adds, mutate.Addendum{
-				Layer:   layer,
+		layers, err := perWheelLayers(opts)
+		if err != nil {
+			return nil, err
+		}
+		adds := make([]mutate.Addendum, len(opts.Wheels))
+		for i, rw := range opts.Wheels {
+			adds[i] = mutate.Addendum{
+				Layer:   layers[i],
 				History: history(fmt.Sprintf("wheel %s==%s", rw.Name, rw.Version)),
-			})
+			}
 		}
 		return adds, nil
 
@@ -156,6 +168,68 @@ func dependencyAddendums(opts Options) ([]mutate.Addendum, error) {
 	}
 }
 
+// perWheelLayers builds one layer per wheel concurrently (bounded by CPU
+// count), preserving input order. Layer construction is independent and
+// CPU-bound (read + gzip), so parallelism is a straightforward win.
+func perWheelLayers(opts Options) ([]v1.Layer, error) {
+	layers := make([]v1.Layer, len(opts.Wheels))
+	errs := make([]error, len(opts.Wheels))
+
+	workers := runtime.NumCPU()
+	if workers < 1 {
+		workers = 1
+	}
+	sem := make(chan struct{}, workers)
+	var wg sync.WaitGroup
+	for i := range opts.Wheels {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
+			layers[i], errs[i] = buildWheelLayer(opts.Wheels[i], opts.Layout, opts.Cache)
+		}(i)
+	}
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			return nil, err
+		}
+	}
+	return layers, nil
+}
+
+// buildWheelLayer returns the layer for a wheel, using the cache when present.
+func buildWheelLayer(rw wheelhouse.ResolvedWheel, layout wheel.Layout, c *cache.Cache) (v1.Layer, error) {
+	key := wheelCacheKey(rw, layout)
+	if c != nil {
+		if l, ok := c.Get(key); ok {
+			return l, nil
+		}
+	}
+	files, err := wheelFiles(rw, layout)
+	if err != nil {
+		return nil, err
+	}
+	layer, err := ptar.Layer(files)
+	if err != nil {
+		return nil, err
+	}
+	if c != nil {
+		// A cache write failure must not fail the build; we just lose the
+		// speedup for this layer.
+		_ = c.Put(key, layer)
+	}
+	return layer, nil
+}
+
+func wheelCacheKey(rw wheelhouse.ResolvedWheel, layout wheel.Layout) string {
+	return strings.Join([]string{
+		"wheel", layerFormatVersion, rw.SHA256, layout.Prefix, layout.PythonTag,
+	}, "|")
+}
+
 func wheelFiles(rw wheelhouse.ResolvedWheel, layout wheel.Layout) ([]ptar.File, error) {
 	w, err := wheel.Open(rw.Path)
 	if err != nil {
@@ -217,7 +291,7 @@ func sourceLayer(srcDir, workingDir string, extraIgnore []string) (v1.Layer, err
 	if len(files) == 0 {
 		return nil, nil
 	}
-	sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path })
+	sort.SliceStable(files, func(i, j int) bool { return files[i].Path < files[j].Path })
 	return ptar.Layer(files)
 }
 

pymage/internal/build/build_test.go

@@ -11,6 +11,7 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 
+	"github.com/imjasonh/terraform-playground/pymage/internal/cache"
 	"github.com/imjasonh/terraform-playground/pymage/internal/testwheel"
 	"github.com/imjasonh/terraform-playground/pymage/internal/wheel"
 	"github.com/imjasonh/terraform-playground/pymage/internal/wheelhouse"
@@ -193,6 +194,53 @@ func TestPerWheelOnlyNewDepChangesLayers(t *testing.T) {
 	}
 }
 
+func TestBuildWithCacheIsConsistentAndPopulates(t *testing.T) {
+	dir := t.TempDir()
+	wheels := []wheelhouse.ResolvedWheel{mkWheel(t, dir, "alpha", "1.0"), mkWheel(t, dir, "beta", "2.0")}
+
+	c, err := cache.New(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	opts := baseOpts(wheels)
+	opts.Cache = c
+
+	// Cold cache build.
+	img1, err := Build(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d1, err := img1.Digest()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Warm cache build must produce an identical image.
+	img2, err := Build(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d2, err := img2.Digest()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if d1 != d2 {
+		t.Fatalf("cached build digest differs: %s != %s", d1, d2)
+	}
+
+	// And it must match a build with no cache at all (cache changes nothing
+	// about the output).
+	img3, err := Build(baseOpts(wheels))
+	if err != nil {
+		t.Fatal(err)
+	}
+	d3, _ := img3.Digest()
+	if d3 != d1 {
+		t.Fatalf("cached vs uncached digest differ: %s != %s", d1, d3)
+	}
+}
+
 func TestSourceLayerIgnore(t *testing.T) {
 	src := t.TempDir()
 	must := func(p, content string) {

pymage/internal/cache/cache.go

@@ -0,0 +1,77 @@
+// Package cache is a content-addressed, on-disk store for built OCI layers.
+//
+// Building a layer from a wheel means reading every member and gzip-compressing
+// the tar — pure CPU that is wasted on every rebuild for dependencies that have
+// not changed. Keyed by the wheel's sha256 (plus the install layout), the cache
+// lets an unchanged dependency reuse its previously compressed blob verbatim,
+// so a rebuild does no decompression/recompression for those layers.
+package cache
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+)
+
+// Cache stores compressed layer blobs under a directory.
+type Cache struct {
+	dir string
+}
+
+// New returns a cache rooted at dir, creating it if necessary.
+func New(dir string) (*Cache, error) {
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return nil, fmt.Errorf("cache: create %q: %w", dir, err)
+	}
+	return &Cache{dir: dir}, nil
+}
+
+func (c *Cache) path(key string) string {
+	sum := sha256.Sum256([]byte(key))
+	return filepath.Join(c.dir, hex.EncodeToString(sum[:])+".tar.gz")
+}
+
+// Get returns the cached layer for key, or (nil, false) on a miss.
+func (c *Cache) Get(key string) (v1.Layer, bool) {
+	p := c.path(key)
+	if _, err := os.Stat(p); err != nil {
+		return nil, false
+	}
+	l, err := tarball.LayerFromFile(p)
+	if err != nil {
+		return nil, false
+	}
+	return l, true
+}
+
+// Put stores the layer's compressed blob under key. Writes are atomic (temp
+// file + rename) so concurrent builders and interrupted writes are safe.
+func (c *Cache) Put(key string, l v1.Layer) error {
+	rc, err := l.Compressed()
+	if err != nil {
+		return err
+	}
+	defer func() { _ = rc.Close() }()
+
+	tmp, err := os.CreateTemp(c.dir, ".tmp-*")
+	if err != nil {
+		return err
+	}
+	tmpName := tmp.Name()
+	defer func() { _ = os.Remove(tmpName) }()
+
+	if _, err := io.Copy(tmp, rc); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpName, c.path(key))
+}

pymage/internal/cache/cache_test.go

@@ -0,0 +1,43 @@
+package cache
+
+import (
+	"testing"
+
+	"github.com/imjasonh/terraform-playground/pymage/internal/ptar"
+)
+
+func TestPutGetRoundTrip(t *testing.T) {
+	c, err := New(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer, err := ptar.Layer([]ptar.File{{Path: "site/foo.py", Data: []byte("X=1\n")}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want, err := layer.Digest()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const key = "wheel|1|abc|/app/.venv|python3.12"
+	if _, ok := c.Get(key); ok {
+		t.Fatal("expected a miss before Put")
+	}
+	if err := c.Put(key, layer); err != nil {
+		t.Fatal(err)
+	}
+
+	cached, ok := c.Get(key)
+	if !ok {
+		t.Fatal("expected a hit after Put")
+	}
+	got, err := cached.Digest()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Fatalf("cached layer digest %s != original %s", got, want)
+	}
+}

pymage/internal/cli/cli.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -16,6 +17,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/imjasonh/terraform-playground/pymage/internal/build"
+	"github.com/imjasonh/terraform-playground/pymage/internal/cache"
 	"github.com/imjasonh/terraform-playground/pymage/internal/lock"
 	"github.com/imjasonh/terraform-playground/pymage/internal/sbom"
 	"github.com/imjasonh/terraform-playground/pymage/internal/wheel"
@@ -58,6 +60,7 @@ type buildFlags struct {
 	sbomOut     string
 	requireHash bool
 	insecure    bool
+	cacheDir    string
 }
 
 func buildCmd() *cobra.Command {
@@ -92,6 +95,7 @@ func buildCmd() *cobra.Command {
 	fs.StringVar(&f.sbomOut, "sbom", "", "write a CycloneDX SBOM to this path")
 	fs.BoolVar(&f.requireHash, "require-hashes", true, "require every requirement to carry --hash entries")
 	fs.BoolVar(&f.insecure, "insecure", false, "use plain HTTP for the registry")
+	fs.StringVar(&f.cacheDir, "cache-dir", "", "directory for the content-addressed layer cache (speeds up rebuilds)")
 	return cmd
 }
 
@@ -106,6 +110,14 @@ func runBuild(ctx context.Context, f *buildFlags) error {
 		return fmt.Errorf("--find-links is required (a directory of wheels)")
 	}
 
+	if _, err := keyValues(f.env); err != nil {
+		return fmt.Errorf("--env: %w", err)
+	}
+	labels, err := keyValues(f.labels)
+	if err != nil {
+		return fmt.Errorf("--label: %w", err)
+	}
+
 	reqs, err := lock.ParseFile(f.lockFile)
 	if err != nil {
 		return err
@@ -113,10 +125,6 @@ func runBuild(ctx context.Context, f *buildFlags) error {
 	if err := lock.Validate(reqs, f.requireHash); err != nil {
 		return err
 	}
-	wheels, err := wheelhouse.Resolve(reqs, f.findLinks)
-	if err != nil {
-		return err
-	}
 
 	var platform *v1.Platform
 	if f.platform != "" {
@@ -126,6 +134,15 @@ func runBuild(ctx context.Context, f *buildFlags) error {
 		}
 	}
 
+	target, err := resolveTarget(platform, f.pythonTag)
+	if err != nil {
+		return err
+	}
+	wheels, err := wheelhouse.Resolve(reqs, f.findLinks, target)
+	if err != nil {
+		return err
+	}
+
 	var nameOpts []name.Option
 	if f.insecure {
 		nameOpts = append(nameOpts, name.Insecure)
@@ -136,9 +153,12 @@ func runBuild(ctx context.Context, f *buildFlags) error {
 		return err
 	}
 
-	labels, err := keyValues(f.labels)
-	if err != nil {
-		return fmt.Errorf("--label: %w", err)
+	var layerCache *cache.Cache
+	if f.cacheDir != "" {
+		layerCache, err = cache.New(f.cacheDir)
+		if err != nil {
+			return err
+		}
 	}
 
 	img, err := build.Build(build.Options{
@@ -154,6 +174,7 @@ func runBuild(ctx context.Context, f *buildFlags) error {
 		Env:        f.env,
 		User:       f.user,
 		Labels:     labels,
+		Cache:      layerCache,
 	})
 	if err != nil {
 		return err
@@ -214,6 +235,44 @@ func runBuild(ctx context.Context, f *buildFlags) error {
 	return nil
 }
 
+// resolveTarget derives the wheel-selection target (platform + interpreter)
+// from the --platform and --python flags, defaulting to linux/amd64.
+func resolveTarget(platform *v1.Platform, pythonTag string) (wheel.Target, error) {
+	t := wheel.Target{OS: "linux", Arch: "amd64"}
+	if platform != nil {
+		if platform.OS != "" {
+			t.OS = platform.OS
+		}
+		if platform.Architecture != "" {
+			t.Arch = platform.Architecture
+		}
+	}
+	maj, min, err := parsePythonTag(pythonTag)
+	if err != nil {
+		return wheel.Target{}, err
+	}
+	t.PyMajor, t.PyMinor = maj, min
+	return t, nil
+}
+
+// parsePythonTag parses "python3.12" into (3, 12).
+func parsePythonTag(tag string) (int, int, error) {
+	s := strings.TrimPrefix(tag, "python")
+	majStr, minStr, ok := strings.Cut(s, ".")
+	if !ok {
+		return 0, 0, fmt.Errorf("--python %q must look like pythonX.Y", tag)
+	}
+	maj, err := strconv.Atoi(majStr)
+	if err != nil {
+		return 0, 0, fmt.Errorf("--python %q: bad major version", tag)
+	}
+	min, err := strconv.Atoi(minStr)
+	if err != nil {
+		return 0, 0, fmt.Errorf("--python %q: bad minor version", tag)
+	}
+	return maj, min, nil
+}
+
 func resolveBase(ctx context.Context, ref string, platform *v1.Platform, nameOpts ...name.Option) (v1.Image, error) {
 	if ref == "scratch" {
 		return empty.Image, nil

pymage/internal/cli/cli_test.go

@@ -17,6 +17,35 @@ import (
 	"github.com/imjasonh/terraform-playground/pymage/internal/testwheel"
 )
 
+func TestParsePythonTag(t *testing.T) {
+	maj, min, err := parsePythonTag("python3.12")
+	if err != nil || maj != 3 || min != 12 {
+		t.Fatalf("parsePythonTag = %d.%d, %v", maj, min, err)
+	}
+	if _, _, err := parsePythonTag("python3"); err == nil {
+		t.Error("expected error for missing minor version")
+	}
+}
+
+func TestResolveTargetDefaults(t *testing.T) {
+	tg, err := resolveTarget(nil, "python3.11")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tg.OS != "linux" || tg.Arch != "amd64" || tg.PyMajor != 3 || tg.PyMinor != 11 {
+		t.Fatalf("unexpected target %+v", tg)
+	}
+}
+
+func TestEnvValidation(t *testing.T) {
+	if _, err := keyValues([]string{"GOOD=1"}); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if _, err := keyValues([]string{"NOEQUALS"}); err == nil {
+		t.Error("expected error for env entry without '='")
+	}
+}
+
 // TestBuildCommandEndToEnd drives the cobra `build` command exactly as a user
 // would: a hashed requirements file + a wheelhouse + source, pushed to a local
 // registry, then verifies the pushed image is pullable and reproducible.