diff --git a/pymage/DESIGN.md b/pymage/DESIGN.md index 5fd6ce4..efb0c2b 100644 --- a/pymage/DESIGN.md +++ b/pymage/DESIGN.md @@ -112,13 +112,18 @@ across all arches. Per-wheel layering maximizes reuse but a large dependency tree can exceed practical limits (registries/runtimes tolerate many layers, but ~100+ tiny blobs slow pulls -and some tooling caps near 127). Strategies, selectable via flag: +and some tooling caps near 127). Strategies, selectable via flag/config: -- `per-wheel` (default): best reuse, one layer per dist. -- `hybrid`: large/heavily-shared wheels get their own layer; the long tail of tiny - wheels is **bin-packed by a stable partition** (e.g. hash of name into K buckets) - so adding a dep usually perturbs only one bucket layer. Bounded layer count, - slightly weaker "only-new-dep" guarantee for bucketed deps. +- `auto` (**default, implemented**): one layer per wheel while the total image + layer count fits a budget — `max-layers` (default **127**, counting base + + deps + app), or `max-wheel-layers` to cap dependency layers directly. When the + wheel count exceeds the budget, wheels are **bin-packed by hashing each + distribution's normalized name into K = budget buckets**. Because a wheel's + bucket depends only on its name, adding/removing/version-bumping one dependency + changes only that one bucket's layer; every other layer keeps its digest. This + is reuse-optimal for a fixed budget (size-balanced packing was rejected because + it reshuffles on insert). +- `per-wheel`: best reuse, one layer per dist, no cap. - `single-deps-layer`: all deps in one layer + app layer. Simplest, weakest reuse (any dep change rebuilds the whole deps layer). Useful for tiny apps. diff --git a/pymage/README.md b/pymage/README.md index 82e3dc1..d987d42 100644 --- a/pymage/README.md +++ b/pymage/README.md @@ -68,7 +68,9 @@ the config value, which overrides the built-in default. | `tags` | `-t`/`--tag` (repeatable) | `["latest"]` | | `base` | `--base` | `cgr.dev/chainguard/python:latest` | | `platforms` | `--platform` | the platforms the **base image** supports | -| `layer-strategy` | `--layer-strategy` | `per-wheel` | +| `layer-strategy` | `--layer-strategy` | `auto` | +| `max-layers` | `--max-layers` | `127` | +| `max-wheel-layers` | `--max-wheel-layers` | *(derived from `max-layers`)* | | `python` | `--python` | auto-detected from the base | | `prefix` | `--prefix` | `/app/.venv` | | `workdir` | `--workdir` | `/app` | @@ -110,6 +112,21 @@ index. Because no Docker daemon is involved, this works from any host OS — Lin macOS, or Windows. Each platform selects its own compatible wheels from `uv.lock` (pure-python wheels are shared across arches). +### Layer budget (`auto`) + +By default (`layer-strategy = "auto"`) pymage keeps **one layer per wheel** for +maximum reuse, as long as the total image stays within a layer budget — `127` +layers by default (`max-layers`, counting the base image's layers, the +dependency layers, and the app source layer). Set `max-wheel-layers` to cap the +dependency layers directly. + +When there are more wheels than the budget allows, pymage **bin-packs** them by +hashing each distribution's (normalized) name to a stable bucket. Because a +wheel's bucket depends only on its name, adding, removing, or version-bumping a +single dependency only changes **that one bucket's layer** — every other layer +keeps its digest and is reused. (`per-wheel` forces one layer per wheel with no +cap; `single-deps-layer` puts everything in one layer.) + ### Hashed requirements.txt (pip-compile / uv pip compile) The original lock format is still supported (flags work in place of, or on top @@ -154,7 +171,9 @@ base that advertises its version. | `--oci-layout DIR` | Also write the image to an OCI layout directory. | | `--print-digest` | Print only the resulting image digest (no push). | | `--sbom PATH` | Write a CycloneDX SBOM of the resolved wheels. | -| `--layer-strategy` | `per-wheel` (default) or `single-deps-layer`. | +| `--layer-strategy` | `auto` (default), `per-wheel`, or `single-deps-layer`. | +| `--max-layers` | Cap on total image layers (base + deps + app) for `auto` (default 127). | +| `--max-wheel-layers` | Cap the dependency layer count directly (overrides `--max-layers`). | | `--platform` | Target platform(s); selects compatible wheels and base. Repeatable / comma-separated (e.g. `linux/amd64,linux/arm64`) builds a multi-arch image index. Defaults to the platforms the base image supports. | | `--python` | Interpreter version, e.g. `python3.12`. Optional — **auto-detected from the base** when omitted; if set, must match the base. Drives wheel selection and the site-packages layout. | | `--cache-dir` | Content-addressed layer cache; reuses compressed layers and downloaded wheels across rebuilds. | diff --git a/pymage/internal/cli/cli.go b/pymage/internal/cli/cli.go index bdd40be..13a8cf4 100644 --- a/pymage/internal/cli/cli.go +++ b/pymage/internal/cli/cli.go @@ -57,6 +57,9 @@ type buildFlags struct { ignore []string strategy string + maxLayers int + maxWheelLayers int + push bool ociLayout string printDigest bool @@ -97,7 +100,9 @@ func buildCmd() *cobra.Command { fs.StringVar(&f.user, "user", "", "image user, e.g. 65532") fs.StringArrayVar(&f.labels, "label", nil, "image label KEY=VALUE (repeatable)") fs.StringArrayVar(&f.ignore, "ignore", nil, "extra source ignore glob (repeatable)") - fs.StringVar(&f.strategy, "layer-strategy", string(build.PerWheel), "per-wheel | single-deps-layer") + fs.StringVar(&f.strategy, "layer-strategy", string(build.Auto), "auto | per-wheel | single-deps-layer") + fs.IntVar(&f.maxLayers, "max-layers", build.DefaultMaxLayers, "max total image layers (base + deps + app) for the auto strategy") + fs.IntVar(&f.maxWheelLayers, "max-wheel-layers", 0, "cap the number of dependency layers directly (overrides --max-layers when > 0)") fs.BoolVar(&f.push, "push", true, "push the image to --repo (by digest)") fs.StringVar(&f.ociLayout, "oci-layout", "", "also write the image to this OCI layout directory") fs.BoolVar(&f.printDigest, "print-digest", false, "print only the resulting image digest") @@ -238,19 +243,21 @@ func buildOne(ctx context.Context, f *buildFlags, reqs []lock.Requirement, platf return nil, nil, err } img, err := build.Build(build.Options{ - Base: base, - Wheels: wheels, - Layout: wheel.Layout{Prefix: f.prefix, PythonTag: pyTag}, - Strategy: build.LayerStrategy(f.strategy), - SourceDir: f.source, - Ignore: f.ignore, - WorkingDir: f.workingDir, - Entrypoint: f.entrypoint, - Cmd: f.cmd, - Env: f.env, - User: f.user, - Labels: labels, - Cache: layerCache, + Base: base, + Wheels: wheels, + Layout: wheel.Layout{Prefix: f.prefix, PythonTag: pyTag}, + Strategy: build.LayerStrategy(f.strategy), + MaxLayers: f.maxLayers, + MaxWheelLayers: f.maxWheelLayers, + SourceDir: f.source, + Ignore: f.ignore, + WorkingDir: f.workingDir, + Entrypoint: f.entrypoint, + Cmd: f.cmd, + Env: f.env, + User: f.user, + Labels: labels, + Cache: layerCache, }) if err != nil { return nil, nil, err diff --git a/pymage/internal/cli/defaults.go b/pymage/internal/cli/defaults.go index 214099a..b7a33f5 100644 --- a/pymage/internal/cli/defaults.go +++ b/pymage/internal/cli/defaults.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" + "github.com/imjasonh/terraform-playground/pymage/internal/build" "github.com/imjasonh/terraform-playground/pymage/internal/project" ) @@ -53,6 +54,12 @@ func applyDefaults(cmd *cobra.Command, f *buildFlags) error { if !changed("layer-strategy") && cfg.LayerStrategy != "" { f.strategy = cfg.LayerStrategy } + if !changed("max-layers") && cfg.MaxLayers != 0 { + f.maxLayers = cfg.MaxLayers + } + if !changed("max-wheel-layers") && cfg.MaxWheelLayers != 0 { + f.maxWheelLayers = cfg.MaxWheelLayers + } if !changed("python") && cfg.Python != "" { f.pythonTag = cfg.Python } @@ -114,6 +121,19 @@ func validateBuildFlags(f *buildFlags) error { if f.lockFile == "" { return fmt.Errorf("no lock file found (expected uv.lock in the source directory, or pass --lock)") } + // "" is the zero value meaning the default (Auto); a 0 budget likewise means + // "use the default", so only reject genuinely invalid values. + switch f.strategy { + case "", string(build.Auto), string(build.PerWheel), string(build.SingleDepsLayer): + default: + return fmt.Errorf("invalid --layer-strategy %q (want auto, per-wheel, or single-deps-layer)", f.strategy) + } + if f.maxLayers < 0 { + return fmt.Errorf("--max-layers must be >= 1, got %d", f.maxLayers) + } + if f.maxWheelLayers < 0 { + return fmt.Errorf("--max-wheel-layers must be >= 0, got %d", f.maxWheelLayers) + } if len(f.entrypoint) == 0 { return fmt.Errorf("no entrypoint: set [project.scripts] in pyproject.toml, add entrypoint to [tool.pymage], or pass --entrypoint") } diff --git a/pymage/internal/project/project.go b/pymage/internal/project/project.go index 05ad368..1e6c064 100644 --- a/pymage/internal/project/project.go +++ b/pymage/internal/project/project.go @@ -25,20 +25,22 @@ type Info struct { // a build flag of the same name; flags passed on the command line take // precedence over these values, which take precedence over built-in defaults. type Config struct { - Repo string `toml:"repo"` - Tags []string `toml:"tags"` - Base string `toml:"base"` - Platforms []string `toml:"platforms"` - LayerStrategy string `toml:"layer-strategy"` - Python string `toml:"python"` - Prefix string `toml:"prefix"` - Workdir string `toml:"workdir"` - User string `toml:"user"` - Entrypoint []string `toml:"entrypoint"` - Cmd []string `toml:"cmd"` - Env []string `toml:"env"` - Labels map[string]string `toml:"labels"` - FindLinks []string `toml:"find-links"` + Repo string `toml:"repo"` + Tags []string `toml:"tags"` + Base string `toml:"base"` + Platforms []string `toml:"platforms"` + LayerStrategy string `toml:"layer-strategy"` + MaxLayers int `toml:"max-layers"` + MaxWheelLayers int `toml:"max-wheel-layers"` + Python string `toml:"python"` + Prefix string `toml:"prefix"` + Workdir string `toml:"workdir"` + User string `toml:"user"` + Entrypoint []string `toml:"entrypoint"` + Cmd []string `toml:"cmd"` + Env []string `toml:"env"` + Labels map[string]string `toml:"labels"` + FindLinks []string `toml:"find-links"` } const defaultBase = "cgr.dev/chainguard/python:latest" diff --git a/pymage/internal/project/project_test.go b/pymage/internal/project/project_test.go index a066135..a4b8113 100644 --- a/pymage/internal/project/project_test.go +++ b/pymage/internal/project/project_test.go @@ -98,6 +98,28 @@ func TestScriptEntrypoint(t *testing.T) { } } +func TestConfigLayerBudget(t *testing.T) { + dir := t.TempDir() + pyproject := "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n\n" + + "[tool.pymage]\nmax-layers = 50\nmax-wheel-layers = 8\nlayer-strategy = \"auto\"\n" + mustWrite(t, filepath.Join(dir, "pyproject.toml"), pyproject) + mustWrite(t, filepath.Join(dir, "requirements.txt"), "alpha==1.0\n") + + info, err := Discover(dir) + if err != nil { + t.Fatal(err) + } + if info.Config.MaxLayers != 50 { + t.Errorf("max-layers = %d, want 50", info.Config.MaxLayers) + } + if info.Config.MaxWheelLayers != 8 { + t.Errorf("max-wheel-layers = %d, want 8", info.Config.MaxWheelLayers) + } + if info.Config.LayerStrategy != "auto" { + t.Errorf("layer-strategy = %q, want auto", info.Config.LayerStrategy) + } +} + func mustWrite(t *testing.T, path, contents string) { t.Helper() if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {