diff --git a/pymage/README.md b/pymage/README.md index 8670eba..db657df 100644 --- a/pymage/README.md +++ b/pymage/README.md @@ -40,6 +40,25 @@ The wheels referenced by the lock must be present in a `--find-links` directory (e.g. produced by `pip download -r requirements.txt -d ./wheelhouse`). Resolution is intentionally local so builds are hermetic and reproducible. +### Multi-arch + +Pass multiple platforms to build a multi-arch image **index** (one image per +platform, assembled into an OCI index). Because no Docker daemon is involved, +this works from any host OS — Linux, macOS, or Windows: + +``` +pymage build \ + --base python:3.12-slim \ + --platform linux/amd64,linux/arm64 \ + --lock requirements.txt --find-links ./wheelhouse --source ./ \ + --entrypoint python --entrypoint -m --entrypoint myapp \ + -t registry.example.com/me/myapp:latest +``` + +Each platform selects its own compatible wheels (pure-python wheels are shared), +so the wheelhouse must contain a compatible wheel per platform for any compiled +dependency. + ### Useful flags | Flag | Description | @@ -49,7 +68,7 @@ is intentionally local so builds are hermetic and reproducible. | `--print-digest` | Print only the resulting image digest (no push). | | `--sbom PATH` | Write a CycloneDX SBOM of the resolved wheels. | | `--layer-strategy` | `per-wheel` (default) or `single-deps-layer`. | -| `--platform` | Target platform; selects compatible wheels and the base, e.g. `linux/amd64`. | +| `--platform` | Target platform(s); selects compatible wheels and base. Repeatable / comma-separated (e.g. `linux/amd64,linux/arm64`) builds a multi-arch image index. | | `--python` | Interpreter version / site-packages dir (default `python3.12`); also selects compatible wheels. | | `--cache-dir` | Content-addressed layer cache; reuses compressed layers across rebuilds. | | `--prefix` | install prefix / venv root (default `/app/.venv`). | diff --git a/pymage/internal/cli/cli.go b/pymage/internal/cli/cli.go index c639b7c..4db5dfc 100644 --- a/pymage/internal/cli/cli.go +++ b/pymage/internal/cli/cli.go @@ -13,6 +13,7 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/cobra" @@ -42,7 +43,7 @@ type buildFlags struct { findLinks []string source string tag string - platform string + platforms []string pythonTag string prefix string workingDir string @@ -78,7 +79,7 @@ func buildCmd() *cobra.Command { fs.StringArrayVar(&f.findLinks, "find-links", nil, "directory of wheels to resolve against (repeatable, required)") fs.StringVar(&f.source, "source", "", "application source directory (optional)") fs.StringVarP(&f.tag, "tag", "t", "", "image tag to push, e.g. registry/repo:tag") - fs.StringVar(&f.platform, "platform", "", "platform for base resolution, e.g. linux/amd64") + fs.StringSliceVar(&f.platforms, "platform", nil, "target platform(s); repeatable or comma-separated, e.g. linux/amd64,linux/arm64 (multiple builds a multi-arch index)") fs.StringVar(&f.pythonTag, "python", "python3.12", "python lib directory name for site-packages") fs.StringVar(&f.prefix, "prefix", "/app/.venv", "install prefix (venv-like root)") fs.StringVar(&f.workingDir, "workdir", "/app", "image working directory and source destination") @@ -126,19 +127,7 @@ func runBuild(ctx context.Context, f *buildFlags) error { return err } - var platform *v1.Platform - if f.platform != "" { - platform, err = v1.ParsePlatform(f.platform) - if err != nil { - return err - } - } - - target, err := resolveTarget(platform, f.pythonTag) - if err != nil { - return err - } - wheels, err := wheelhouse.Resolve(reqs, f.findLinks, target) + platforms, err := parsePlatforms(f.platforms) if err != nil { return err } @@ -148,11 +137,6 @@ func runBuild(ctx context.Context, f *buildFlags) error { nameOpts = append(nameOpts, name.Insecure) } - base, err := resolveBase(ctx, f.base, platform, nameOpts...) - if err != nil { - return err - } - var layerCache *cache.Cache if f.cacheDir != "" { layerCache, err = cache.New(f.cacheDir) @@ -161,6 +145,69 @@ func runBuild(ctx context.Context, f *buildFlags) error { } } + // Build one image per target platform. With more than one platform we + // assemble them into a multi-arch OCI image index; with zero or one we push + // a plain image manifest. + var ( + images []v1.Image + adds []mutate.IndexAddendum + allDeps []wheelhouse.ResolvedWheel + ) + targets := platforms + if len(targets) == 0 { + targets = []v1.Platform{{}} // single build using registry defaults + } + for i := range targets { + p := targets[i] + var pp *v1.Platform + if p.OS != "" || p.Architecture != "" { + pp = &p + } + img, deps, err := buildOne(ctx, f, reqs, pp, labels, layerCache, nameOpts) + if err != nil { + return err + } + images = append(images, img) + allDeps = append(allDeps, deps...) + desc := v1.Descriptor{} + if pp != nil { + desc.Platform = pp + } + adds = append(adds, mutate.IndexAddendum{Add: img, Descriptor: desc}) + } + + if f.sbomOut != "" { + data, err := sbom.Generate(dedupeWheels(allDeps)) + if err != nil { + return err + } + if err := os.WriteFile(f.sbomOut, data, 0o644); err != nil { + return err + } + } + + if len(platforms) > 1 { + idx := mutate.AppendManifests(empty.Index, adds...) + return output(ctx, f, nameOpts, nil, idx) + } + return output(ctx, f, nameOpts, images[0], nil) +} + +// buildOne resolves the base and wheels for a single platform and builds the +// image, returning the resolved wheels for SBOM aggregation. +func buildOne(ctx context.Context, f *buildFlags, reqs []lock.Requirement, platform *v1.Platform, labels map[string]string, layerCache *cache.Cache, nameOpts []name.Option) (v1.Image, []wheelhouse.ResolvedWheel, error) { + target, err := resolveTarget(platform, f.pythonTag) + if err != nil { + return nil, nil, err + } + wheels, err := wheelhouse.Resolve(reqs, f.findLinks, target) + if err != nil { + return nil, nil, err + } + base, err := resolveBase(ctx, f.base, platform, nameOpts...) + if err != nil { + return nil, nil, err + } img, err := build.Build(build.Options{ Base: base, Wheels: wheels, @@ -177,33 +224,38 @@ func runBuild(ctx context.Context, f *buildFlags) error { Cache: layerCache, }) if err != nil { - return err + return nil, nil, err } + return img, wheels, nil +} - if f.sbomOut != "" { - data, err := sbom.Generate(wheels) - if err != nil { - return err - } - if err := os.WriteFile(f.sbomOut, data, 0o644); err != nil { - return err - } +// output writes/pushes either a single image or a multi-arch index (exactly one +// of img/idx is non-nil). +func output(ctx context.Context, f *buildFlags, nameOpts []name.Option, img v1.Image, idx v1.ImageIndex) error { + var ( + digest v1.Hash + err error + ) + if idx != nil { + digest, err = idx.Digest() + } else { + digest, err = img.Digest() } - - digest, err := img.Digest() if err != nil { return err } if f.ociLayout != "" { - if _, err := layout.Write(f.ociLayout, empty.Index); err != nil { - return err - } - p, err := layout.FromPath(f.ociLayout) + p, err := layout.Write(f.ociLayout, empty.Index) if err != nil { return err } - if err := p.AppendImage(img); err != nil { + if idx != nil { + err = p.AppendIndex(idx) + } else { + err = p.AppendImage(img) + } + if err != nil { return err } } @@ -221,10 +273,16 @@ func runBuild(ctx context.Context, f *buildFlags) error { if err != nil { return err } - if err := remote.Write(ref, img, + opts := []remote.Option{ remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain), - ); err != nil { + } + if idx != nil { + err = remote.WriteIndex(ref, idx, opts...) + } else { + err = remote.Write(ref, img, opts...) + } + if err != nil { return fmt.Errorf("push: %w", err) } fmt.Printf("pushed %s@%s\n", ref.Context().Name(), digest) @@ -235,6 +293,39 @@ func runBuild(ctx context.Context, f *buildFlags) error { return nil } +// parsePlatforms parses the (comma-split, repeatable) --platform values. +func parsePlatforms(specs []string) ([]v1.Platform, error) { + var out []v1.Platform + for _, s := range specs { + s = strings.TrimSpace(s) + if s == "" { + continue + } + p, err := v1.ParsePlatform(s) + if err != nil { + return nil, err + } + out = append(out, *p) + } + return out, nil +} + +// dedupeWheels removes duplicate wheels (same name+version+sha) so a multi-arch +// SBOM doesn't list shared pure-python wheels multiple times. +func dedupeWheels(in []wheelhouse.ResolvedWheel) []wheelhouse.ResolvedWheel { + seen := map[string]bool{} + var out []wheelhouse.ResolvedWheel + for _, w := range in { + k := w.Name + "\x00" + w.Version + "\x00" + w.SHA256 + if seen[k] { + continue + } + seen[k] = true + out = append(out, w) + } + return out +} + // resolveTarget derives the wheel-selection target (platform + interpreter) // from the --platform and --python flags, defaulting to linux/amd64. func resolveTarget(platform *v1.Platform, pythonTag string) (wheel.Target, error) { diff --git a/pymage/internal/cli/cli_test.go b/pymage/internal/cli/cli_test.go index d2621c8..ca41c06 100644 --- a/pymage/internal/cli/cli_test.go +++ b/pymage/internal/cli/cli_test.go @@ -11,12 +11,124 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/imjasonh/terraform-playground/pymage/internal/testwheel" ) +// TestBuildMultiArchIndex drives the CLI to build a linux/amd64 + linux/arm64 +// image index from a single host, and verifies the pushed artifact is an index +// covering both platforms and is reproducible. +func TestBuildMultiArchIndex(t *testing.T) { + s := httptest.NewServer(registry.New()) + t.Cleanup(s.Close) + host := strings.TrimPrefix(s.URL, "http://") + + // A multi-arch base (linux/amd64 + linux/arm64), as a real registry would + // serve python:3.12-slim. + baseRef := host + "/base:latest" + writeMultiArchBase(t, baseRef, []string{"amd64", "arm64"}) + + // Pure-python wheels are compatible with every platform. + wh := t.TempDir() + _, shaA := testwheel.Write(t, wh, testwheel.Spec{Name: "alpha", Version: "1.0", Modules: map[string]string{"alpha/__init__.py": "V='1.0'\n"}}) + reqDir := t.TempDir() + reqFile := filepath.Join(reqDir, "requirements.txt") + if err := os.WriteFile(reqFile, []byte(fmt.Sprintf("alpha==1.0 --hash=sha256:%s\n", shaA)), 0o644); err != nil { + t.Fatal(err) + } + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "app.py"), []byte("print('hi')\n"), 0o644); err != nil { + t.Fatal(err) + } + + build := func(tag string) error { + cmd := Root() + cmd.SetArgs([]string{ + "build", + "--base", baseRef, + "--lock", reqFile, + "--find-links", wh, + "--source", src, + "--platform", "linux/amd64,linux/arm64", + "--entrypoint", "python", "--entrypoint", "/app/app.py", + "-t", tag, + }) + cmd.SetOut(os.Stderr) + return cmd.Execute() + } + + if err := build(host + "/multi:v1"); err != nil { + t.Fatalf("multi-arch build failed: %v", err) + } + + ref, _ := name.ParseReference(host + "/multi:v1") + idx, err := remote.Index(ref, remote.WithContext(context.Background())) + if err != nil { + t.Fatalf("pulled artifact is not an index: %v", err) + } + im, err := idx.IndexManifest() + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, m := range im.Manifests { + if m.Platform != nil { + got[m.Platform.OS+"/"+m.Platform.Architecture] = true + } + } + for _, want := range []string{"linux/amd64", "linux/arm64"} { + if !got[want] { + t.Errorf("index missing platform %s; got %v", want, got) + } + } + if len(im.Manifests) != 2 { + t.Fatalf("expected 2 manifests, got %d", len(im.Manifests)) + } + d1, _ := idx.Digest() + + // Reproducible: a second build yields the same index digest. + if err := build(host + "/multi:v2"); err != nil { + t.Fatal(err) + } + ref2, _ := name.ParseReference(host + "/multi:v2") + idx2, err := remote.Index(ref2, remote.WithContext(context.Background())) + if err != nil { + t.Fatal(err) + } + d2, _ := idx2.Digest() + if d1 != d2 { + t.Fatalf("multi-arch index not reproducible: %s != %s", d1, d2) + } +} + +func writeMultiArchBase(t *testing.T, ref string, arches []string) { + t.Helper() + var adds []mutate.IndexAddendum + for _, arch := range arches { + img, err := mutate.ConfigFile(empty.Image, &v1.ConfigFile{ + OS: "linux", + Architecture: arch, + Config: v1.Config{Env: []string{"PATH=/usr/local/bin:/usr/bin:/bin"}}, + }) + if err != nil { + t.Fatal(err) + } + adds = append(adds, mutate.IndexAddendum{ + Add: img, + Descriptor: v1.Descriptor{Platform: &v1.Platform{OS: "linux", Architecture: arch}}, + }) + } + idx := mutate.AppendManifests(empty.Index, adds...) + r, _ := name.ParseReference(ref) + if err := remote.WriteIndex(r, idx, remote.WithContext(context.Background())); err != nil { + t.Fatalf("write base index: %v", err) + } +} + func TestParsePythonTag(t *testing.T) { maj, min, err := parsePythonTag("python3.12") if err != nil || maj != 3 || min != 12 {