diff --git a/.github/workflows/pymage.yaml b/.github/workflows/pymage.yaml index 2af836e..4e0f5ff 100644 --- a/.github/workflows/pymage.yaml +++ b/.github/workflows/pymage.yaml @@ -21,8 +21,8 @@ jobs: run: shell: bash env: + # Python version is auto-detected from the base image (no --python flag). BASE: python:3.12-slim - PYTHON: python3.12 REGISTRY: localhost:1338 steps: - uses: actions/checkout@v6.0.2 @@ -104,7 +104,6 @@ jobs: --lock demo/requirements.txt \ --find-links demo/wheelhouse \ --source demo/src \ - --python "$PYTHON" \ --entrypoint python --entrypoint /app/app.py \ --insecure \ -t "$REGISTRY/demo:multi" @@ -129,7 +128,7 @@ jobs: run: | set -euo pipefail common=(--base "$BASE" --platform linux/amd64,linux/arm64 --lock demo/requirements.txt \ - --find-links demo/wheelhouse --source demo/src --python "$PYTHON" \ + --find-links demo/wheelhouse --source demo/src \ --entrypoint python --entrypoint /app/app.py --insecure --push=false) d1=$(go run . build "${common[@]}" --print-digest) d2=$(go run . build "${common[@]}" --print-digest) @@ -158,7 +157,7 @@ jobs: run: | set -euo pipefail common=(--base "$BASE" --platform linux/amd64 --lock demo/requirements.txt \ - --find-links demo/wheelhouse --source demo/src --python "$PYTHON" \ + --find-links demo/wheelhouse --source demo/src \ --entrypoint python --entrypoint /app/app.py --insecure) go run . build "${common[@]}" -t "$REGISTRY/demo:v1" diff --git a/pymage/README.md b/pymage/README.md index f243acf..c8b5585 100644 --- a/pymage/README.md +++ b/pymage/README.md @@ -73,12 +73,14 @@ A floating tag such as `cgr.dev/chainguard/python:latest` works, but be aware: keep working (they're matched by `py3` and found via `PYTHONPATH`), but version-specific compiled wheels (`cp312`…) break when the interpreter moves. -To prevent silent breakage, pymage detects the base's Python version and -**fails the build** if it doesn't match `--python`, telling you which version to -target. It looks at the `PYTHON_VERSION` env var (official python images) and, -when that's absent, the `python-X.Y` package in `/etc/apko.json` from the top -layer (Chainguard/Wolfi images). Bases that expose neither can't be validated — -another reason to pin. +pymage detects the base's Python version and uses it automatically, so +`--python` is usually unnecessary. Detection looks at the `PYTHON_VERSION` env +var (official python images) and, when that's absent, the `python-X.Y` package +in `/etc/apko.json` from the top layer (Chainguard/Wolfi images). If you do pass +`--python`, it **must match** the detected base version or the build fails fast +(catching a floating tag that slid to a different Python). Bases that expose +neither signal can't be auto-detected — pass `--python` explicitly, or pin a +base that advertises its version. ### Useful flags @@ -90,7 +92,7 @@ another reason to pin. | `--sbom PATH` | Write a CycloneDX SBOM of the resolved wheels. | | `--layer-strategy` | `per-wheel` (default) or `single-deps-layer`. | | `--platform` | Target platform(s); selects compatible wheels and base. Repeatable / comma-separated (e.g. `linux/amd64,linux/arm64`) builds a multi-arch image index. | -| `--python` | Interpreter version / site-packages dir (default `python3.12`); also selects compatible wheels. | +| `--python` | Interpreter version, e.g. `python3.12`. Optional — **auto-detected from the base** when omitted; if set, must match the base. Drives wheel selection and the site-packages layout. | | `--cache-dir` | Content-addressed layer cache; reuses compressed layers across rebuilds. | | `--prefix` | install prefix / venv root (default `/app/.venv`). | | `--workdir` | image working dir and source destination (default `/app`). | diff --git a/pymage/internal/cli/cli.go b/pymage/internal/cli/cli.go index 8e393ba..0b1b71b 100644 --- a/pymage/internal/cli/cli.go +++ b/pymage/internal/cli/cli.go @@ -80,7 +80,7 @@ func buildCmd() *cobra.Command { fs.StringVar(&f.source, "source", "", "application source directory (optional)") fs.StringVarP(&f.tag, "tag", "t", "", "image tag to push, e.g. registry/repo:tag") fs.StringSliceVar(&f.platforms, "platform", nil, "target platform(s); repeatable or comma-separated, e.g. linux/amd64,linux/arm64 (multiple builds a multi-arch index)") - fs.StringVar(&f.pythonTag, "python", "python3.12", "python lib directory name for site-packages") + fs.StringVar(&f.pythonTag, "python", "", "interpreter version, e.g. python3.12 (default: auto-detected from the base image; if set, must match the base)") fs.StringVar(&f.prefix, "prefix", "/app/.venv", "install prefix (venv-like root)") fs.StringVar(&f.workingDir, "workdir", "/app", "image working directory and source destination") fs.StringArrayVar(&f.entrypoint, "entrypoint", nil, "entrypoint argv (repeatable)") @@ -196,30 +196,29 @@ func runBuild(ctx context.Context, f *buildFlags) error { // buildOne resolves the base and wheels for a single platform and builds the // image, returning the resolved wheels for SBOM aggregation. func buildOne(ctx context.Context, f *buildFlags, reqs []lock.Requirement, platform *v1.Platform, labels map[string]string, layerCache *cache.Cache, nameOpts []name.Option) (v1.Image, []wheelhouse.ResolvedWheel, error) { - target, err := resolveTarget(platform, f.pythonTag) - if err != nil { - return nil, nil, err - } - wheels, err := wheelhouse.Resolve(reqs, f.findLinks, target) - if err != nil { - return nil, nil, err - } base, err := resolveBase(ctx, f.base, platform, nameOpts...) if err != nil { return nil, nil, err } - // If the base advertises its interpreter version, make sure it matches the - // requested --python. This catches a floating base tag (e.g. ":latest") - // sliding to a Python version that the selected wheels/layout don't target, - // which would otherwise silently produce a broken image. - if maj, min, ok := build.InterpreterVersion(base); ok && (maj != target.PyMajor || min != target.PyMinor) { - return nil, nil, fmt.Errorf("base %q provides Python %d.%d but --python is python%d.%d; pass --python python%d.%d (or pin a matching base)", - f.base, maj, min, target.PyMajor, target.PyMinor, maj, min) + + // Determine the interpreter: honor --python (validated against the base) or + // auto-detect it from the base image. + major, minor, pyTag, err := resolveInterpreter(f, base) + if err != nil { + return nil, nil, err + } + + target := platformTarget(platform) + target.PyMajor, target.PyMinor = major, minor + + wheels, err := wheelhouse.Resolve(reqs, f.findLinks, target) + if err != nil { + return nil, nil, err } img, err := build.Build(build.Options{ Base: base, Wheels: wheels, - Layout: wheel.Layout{Prefix: f.prefix, PythonTag: f.pythonTag}, + Layout: wheel.Layout{Prefix: f.prefix, PythonTag: pyTag}, Strategy: build.LayerStrategy(f.strategy), SourceDir: f.source, Ignore: f.ignore, @@ -237,6 +236,31 @@ func buildOne(ctx context.Context, f *buildFlags, reqs []lock.Requirement, platf return img, wheels, nil } +// resolveInterpreter decides the target Python version and the site-packages +// directory tag. If --python is set it is authoritative but must match the +// base when the base advertises a version; otherwise the version is detected +// from the base image. +func resolveInterpreter(f *buildFlags, base v1.Image) (major, minor int, pyTag string, err error) { + baseMaj, baseMin, baseOK := build.InterpreterVersion(base) + + if f.pythonTag != "" { + maj, min, err := parsePythonTag(f.pythonTag) + if err != nil { + return 0, 0, "", err + } + if baseOK && (baseMaj != maj || baseMin != min) { + return 0, 0, "", fmt.Errorf("base %q provides Python %d.%d but --python is python%d.%d; pass --python python%d.%d (or pin a matching base)", + f.base, baseMaj, baseMin, maj, min, baseMaj, baseMin) + } + return maj, min, f.pythonTag, nil + } + + if !baseOK { + return 0, 0, "", fmt.Errorf("could not determine the base image's Python version (no PYTHON_VERSION env or python package in /etc/apko.json); pass --python pythonX.Y") + } + return baseMaj, baseMin, fmt.Sprintf("python%d.%d", baseMaj, baseMin), nil +} + // output writes/pushes either a single image or a multi-arch index (exactly one // of img/idx is non-nil). func output(ctx context.Context, f *buildFlags, nameOpts []name.Option, img v1.Image, idx v1.ImageIndex) error { @@ -334,9 +358,10 @@ func dedupeWheels(in []wheelhouse.ResolvedWheel) []wheelhouse.ResolvedWheel { return out } -// resolveTarget derives the wheel-selection target (platform + interpreter) -// from the --platform and --python flags, defaulting to linux/amd64. -func resolveTarget(platform *v1.Platform, pythonTag string) (wheel.Target, error) { +// platformTarget derives the OS/arch part of the wheel-selection target from +// the --platform value, defaulting to linux/amd64. The interpreter fields are +// filled in by the caller. +func platformTarget(platform *v1.Platform) wheel.Target { t := wheel.Target{OS: "linux", Arch: "amd64"} if platform != nil { if platform.OS != "" { @@ -346,12 +371,7 @@ func resolveTarget(platform *v1.Platform, pythonTag string) (wheel.Target, error t.Arch = platform.Architecture } } - maj, min, err := parsePythonTag(pythonTag) - if err != nil { - return wheel.Target{}, err - } - t.PyMajor, t.PyMinor = maj, min - return t, nil + return t } // parsePythonTag parses "python3.12" into (3, 12). diff --git a/pymage/internal/cli/cli_test.go b/pymage/internal/cli/cli_test.go index b27d3b1..2d19596 100644 --- a/pymage/internal/cli/cli_test.go +++ b/pymage/internal/cli/cli_test.go @@ -112,7 +112,7 @@ func writeMultiArchBase(t *testing.T, ref string, arches []string) { img, err := mutate.ConfigFile(empty.Image, &v1.ConfigFile{ OS: "linux", Architecture: arch, - Config: v1.Config{Env: []string{"PATH=/usr/local/bin:/usr/bin:/bin"}}, + Config: v1.Config{Env: []string{"PYTHON_VERSION=3.12.7", "PATH=/usr/local/bin:/usr/bin:/bin"}}, }) if err != nil { t.Fatal(err) @@ -186,13 +186,51 @@ func TestParsePythonTag(t *testing.T) { } } -func TestResolveTargetDefaults(t *testing.T) { - tg, err := resolveTarget(nil, "python3.11") - if err != nil { - t.Fatal(err) +func TestPlatformTargetDefaults(t *testing.T) { + if tg := platformTarget(nil); tg.OS != "linux" || tg.Arch != "amd64" { + t.Fatalf("unexpected default target %+v", tg) } - if tg.OS != "linux" || tg.Arch != "amd64" || tg.PyMajor != 3 || tg.PyMinor != 11 { - t.Fatalf("unexpected target %+v", tg) + tg := platformTarget(&v1.Platform{OS: "linux", Architecture: "arm64"}) + if tg.OS != "linux" || tg.Arch != "arm64" { + t.Fatalf("expected linux/arm64, got %+v", tg) + } +} + +func TestResolveInterpreter(t *testing.T) { + withVersion := func(env []string) v1.Image { + img, err := mutate.ConfigFile(empty.Image, &v1.ConfigFile{Config: v1.Config{Env: env}}) + if err != nil { + t.Fatal(err) + } + return img + } + base313 := withVersion([]string{"PYTHON_VERSION=3.13.1"}) + bareBase := withVersion(nil) + + // Auto-detect when --python is omitted. + maj, min, tag, err := resolveInterpreter(&buildFlags{}, base313) + if err != nil || maj != 3 || min != 13 || tag != "python3.13" { + t.Fatalf("auto-detect: %d.%d %q err=%v", maj, min, tag, err) + } + + // Explicit --python that matches is accepted. + if _, _, tag, err := resolveInterpreter(&buildFlags{pythonTag: "python3.13"}, base313); err != nil || tag != "python3.13" { + t.Fatalf("matching --python: %q err=%v", tag, err) + } + + // Explicit --python that mismatches the base is rejected. + if _, _, _, err := resolveInterpreter(&buildFlags{pythonTag: "python3.12"}, base313); err == nil { + t.Fatal("expected mismatch error") + } + + // No --python and an undetectable base is an error. + if _, _, _, err := resolveInterpreter(&buildFlags{}, bareBase); err == nil { + t.Fatal("expected error when base version is unknown and --python is unset") + } + + // Explicit --python works even when the base can't be validated. + if _, _, tag, err := resolveInterpreter(&buildFlags{pythonTag: "python3.10"}, bareBase); err != nil || tag != "python3.10" { + t.Fatalf("explicit on bare base: %q err=%v", tag, err) } } @@ -216,13 +254,21 @@ func TestBuildCommandEndToEnd(t *testing.T) { t.Cleanup(s.Close) host := strings.TrimPrefix(s.URL, "http://") // 127.0.0.1:port -> ggcr uses http - // Seed a base image. + // Seed a base image that advertises its Python version, so the build can + // auto-detect the interpreter (no --python flag below). baseRef := host + "/base:latest" + base, err := mutate.ConfigFile(empty.Image, &v1.ConfigFile{ + OS: "linux", Architecture: "amd64", + Config: v1.Config{Env: []string{"PYTHON_VERSION=3.12.7", "PATH=/usr/bin"}}, + }) + if err != nil { + t.Fatal(err) + } br, err := name.ParseReference(baseRef) if err != nil { t.Fatal(err) } - if err := remote.Write(br, empty.Image, remote.WithContext(context.Background())); err != nil { + if err := remote.Write(br, base, remote.WithContext(context.Background())); err != nil { t.Fatal(err) } @@ -280,6 +326,16 @@ func TestBuildCommandEndToEnd(t *testing.T) { if len(layers) != 3 { t.Fatalf("got %d layers, want 3", len(layers)) } + + // The interpreter was auto-detected (3.12) from the base and used for the + // site-packages layout. + cf, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(cf.Config.Env, "\n"), "lib/python3.12/site-packages") { + t.Errorf("expected auto-detected python3.12 site-packages in env, got %v", cf.Config.Env) + } pushedDigest, _ := img.Digest() // Running the command again is reproducible: same digest.