diff --git a/py-image-builder/go.mod b/py-image-builder/go.mod new file mode 100644 index 0000000..6559a01 --- /dev/null +++ b/py-image-builder/go.mod @@ -0,0 +1,27 @@ +module github.com/imjasonh/terraform-playground/py-image-builder + +go 1.22.2 + +require ( + github.com/google/go-containerregistry v0.19.2 + github.com/spf13/cobra v1.8.1 +) + +require ( + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect + github.com/docker/cli v24.0.0+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/docker v24.0.0+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/vbatts/tar-split v0.11.3 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.15.0 // indirect +) diff --git a/py-image-builder/go.sum b/py-image-builder/go.sum new file mode 100644 index 0000000..46dc335 --- /dev/null +++ b/py-image-builder/go.sum @@ -0,0 +1,66 @@ +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v24.0.0+incompatible h1:0+1VshNwBQzQAx9lOl+OYCTCEAD8fKs/qeXMx3O0wqM= +github.com/docker/cli v24.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.0+incompatible h1:z4bf8HvONXX9Tde5lGBMQ7yCJgNahmJumdrStZAbeY4= +github.com/docker/docker v24.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.19.2 h1:TannFKE1QSajsP6hPWb5oJNgKe1IKjHukIKDUmvsV6w= +github.com/google/go-containerregistry v0.19.2/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.1 h1:Ou41VVR3nMWWmTiEUnj0OlsgOSCUFgsPAOl6jRIcVtQ= +github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/py-image-builder/internal/build/build.go b/py-image-builder/internal/build/build.go new file mode 100644 index 0000000..3fdc9cb --- /dev/null +++ b/py-image-builder/internal/build/build.go @@ -0,0 +1,371 @@ +// Package build assembles an OCI image from a base image, a set of resolved +// wheels (each its own deterministic layer by default), and an application +// source layer on top. +// +// The output is reproducible: identical inputs (same base digest, same wheels, +// same source) yield an identical image digest. This is what lets a rebuild +// transfer no dependency bytes — every unchanged layer keeps its digest and is +// already present in the registry. +package build + +import ( + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/ptar" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheel" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheelhouse" +) + +// LayerStrategy controls how dependency layers are partitioned. +type LayerStrategy string + +const ( + // PerWheel gives each wheel its own layer. Adding a dependency creates + // exactly one new layer and leaves the rest byte-identical. (default) + PerWheel LayerStrategy = "per-wheel" + // SingleDepsLayer collapses all wheels into one layer. Simplest, but any + // dependency change rebuilds the whole layer. + SingleDepsLayer LayerStrategy = "single-deps-layer" +) + +// Options configures an image build. +type Options struct { + // Base is the resolved base image (use Base to resolve a reference). + Base v1.Image + // Wheels are the resolved dependency wheels, already in deterministic order. + Wheels []wheelhouse.ResolvedWheel + // Layout describes where wheels are installed in the image. + Layout wheel.Layout + // Strategy selects the dependency layering strategy. + Strategy LayerStrategy + + // SourceDir is the application source directory (optional). + SourceDir string + // Ignore holds glob patterns (relative, slash-separated) to exclude from + // the source layer, in addition to a built-in default set. + Ignore []string + + // WorkingDir is the image working directory and app source destination. + WorkingDir string + // Entrypoint and Cmd configure the image. + Entrypoint []string + Cmd []string + // Env holds extra KEY=VALUE entries to set (in addition to the venv vars). + Env []string + // User sets the image user (e.g. "65532"). + User string + // Labels are added to the image config. + Labels map[string]string +} + +var defaultIgnore = []string{ + ".git", ".git/**", + "**/__pycache__/**", "**/*.pyc", "**/*.pyo", + ".venv/**", "**/.pytest_cache/**", "**/.mypy_cache/**", +} + +// Build assembles and returns the image. It performs no network I/O; callers +// push the result with remote.Write (see cmd) or write it locally. +func Build(opts Options) (v1.Image, error) { + if opts.Base == nil { + return nil, fmt.Errorf("build: Base image is required") + } + if opts.WorkingDir == "" { + opts.WorkingDir = "/app" + } + if opts.Strategy == "" { + opts.Strategy = PerWheel + } + + adds, err := dependencyAddendums(opts) + if err != nil { + return nil, err + } + + if opts.SourceDir != "" { + appLayer, err := sourceLayer(opts.SourceDir, opts.WorkingDir, opts.Ignore) + if err != nil { + return nil, err + } + if appLayer != nil { + adds = append(adds, mutate.Addendum{ + Layer: appLayer, + History: history("application source"), + }) + } + } + + img, err := mutate.Append(opts.Base, adds...) + if err != nil { + return nil, fmt.Errorf("build: append layers: %w", err) + } + + return applyConfig(img, opts) +} + +// dependencyAddendums builds the per-strategy dependency layer addendums. +func dependencyAddendums(opts Options) ([]mutate.Addendum, error) { + switch opts.Strategy { + case SingleDepsLayer: + var all []ptar.File + for _, rw := range opts.Wheels { + files, err := wheelFiles(rw, opts.Layout) + if err != nil { + return nil, err + } + all = append(all, files...) + } + if len(all) == 0 { + return nil, nil + } + layer, err := ptar.Layer(all) + if err != nil { + return nil, err + } + return []mutate.Addendum{{Layer: layer, History: history("python dependencies")}}, nil + + case PerWheel: + adds := make([]mutate.Addendum, 0, len(opts.Wheels)) + for _, rw := range opts.Wheels { + files, err := wheelFiles(rw, opts.Layout) + if err != nil { + return nil, err + } + layer, err := ptar.Layer(files) + if err != nil { + return nil, err + } + adds = append(adds, mutate.Addendum{ + Layer: layer, + History: history(fmt.Sprintf("wheel %s==%s", rw.Name, rw.Version)), + }) + } + return adds, nil + + default: + return nil, fmt.Errorf("build: unknown layer strategy %q", opts.Strategy) + } +} + +func wheelFiles(rw wheelhouse.ResolvedWheel, layout wheel.Layout) ([]ptar.File, error) { + w, err := wheel.Open(rw.Path) + if err != nil { + return nil, err + } + defer w.Close() + return w.Files(layout) +} + +// sourceLayer builds a deterministic layer from the application source dir, +// placing files under workingDir. Returns nil if there is nothing to include. +func sourceLayer(srcDir, workingDir string, extraIgnore []string) (v1.Layer, error) { + patterns := append(append([]string(nil), defaultIgnore...), extraIgnore...) + dest := strings.TrimPrefix(path.Clean(workingDir), "/") + + var files []ptar.File + err := filepath.WalkDir(srcDir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcDir, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if rel == "." { + return nil + } + if matchAny(patterns, rel) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() { + return nil + } + if !d.Type().IsRegular() { + return nil // skip symlinks/sockets/etc for determinism + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + info, err := d.Info() + if err != nil { + return err + } + files = append(files, ptar.File{ + Path: path.Join(dest, rel), + Data: data, + Executable: info.Mode().Perm()&0o111 != 0, + }) + return nil + }) + if err != nil { + return nil, fmt.Errorf("build: scan source: %w", err) + } + if len(files) == 0 { + return nil, nil + } + sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) + return ptar.Layer(files) +} + +// matchAny reports whether rel matches any of the glob patterns. A pattern +// ending in "/**" matches the directory and everything under it. +func matchAny(patterns []string, rel string) bool { + for _, pat := range patterns { + if strings.HasSuffix(pat, "/**") { + base := strings.TrimSuffix(pat, "/**") + if rel == base || strings.HasPrefix(rel, base+"/") || pathMatch(base, rel) { + return true + } + // also allow the prefix itself to be a glob like "**/__pycache__" + if matchTail(base, rel) { + return true + } + continue + } + if pathMatch(pat, rel) { + return true + } + if matchTail(pat, rel) { + return true + } + } + return false +} + +// pathMatch matches a single path against a glob (no '/' crossing for '*'). +func pathMatch(pattern, name string) bool { + ok, err := path.Match(pattern, name) + return err == nil && ok +} + +// matchTail handles "**/x" patterns by matching x against any path suffix. +func matchTail(pattern, rel string) bool { + if !strings.HasPrefix(pattern, "**/") { + return false + } + tail := strings.TrimPrefix(pattern, "**/") + if pathMatch(tail, rel) { + return true + } + parts := strings.Split(rel, "/") + for i := range parts { + if pathMatch(tail, strings.Join(parts[i:], "/")) { + return true + } + if pathMatch(tail, parts[i]) { + return true + } + } + return false +} + +// applyConfig rewrites the image config deterministically: venv env vars, +// entrypoint/cmd, user, labels, and zeroed timestamps. +func applyConfig(img v1.Image, opts Options) (v1.Image, error) { + cf, err := img.ConfigFile() + if err != nil { + return nil, err + } + cfg := cf.DeepCopy() + + cfg.Created = v1.Time{Time: epoch()} + cfg.Author = "" + for i := range cfg.History { + cfg.History[i].Created = v1.Time{Time: epoch()} + cfg.History[i].Author = "" + } + + c := &cfg.Config + c.Env = mergeEnv(c.Env, opts.Layout, opts.Env) + c.WorkingDir = opts.WorkingDir + if len(opts.Entrypoint) > 0 { + c.Entrypoint = opts.Entrypoint + c.Cmd = opts.Cmd + } else if len(opts.Cmd) > 0 { + c.Cmd = opts.Cmd + } + if opts.User != "" { + c.User = opts.User + } + if len(opts.Labels) > 0 { + if c.Labels == nil { + c.Labels = map[string]string{} + } + for k, v := range opts.Labels { + c.Labels[k] = v + } + } + + return mutate.ConfigFile(img, cfg) +} + +// mergeEnv layers the venv environment variables on top of the base env in a +// deterministic order. PATH gets the venv bin prepended; VIRTUAL_ENV and +// PYTHONPATH are set so imports and console scripts work regardless of the base +// interpreter location. +func mergeEnv(base []string, layout wheel.Layout, extra []string) []string { + prefix := abs(layout.Prefix) + bin := abs(layout.BinDir()) + site := abs(layout.SitePackages()) + + env := map[string]string{} + var order []string + set := func(k, v string) { + if _, ok := env[k]; !ok { + order = append(order, k) + } + env[k] = v + } + for _, kv := range base { + k, v, _ := strings.Cut(kv, "=") + set(k, v) + } + + if existing, ok := env["PATH"]; ok && existing != "" { + set("PATH", bin+":"+existing) + } else { + set("PATH", bin+":/usr/local/bin:/usr/bin:/bin") + } + set("VIRTUAL_ENV", prefix) + if existing, ok := env["PYTHONPATH"]; ok && existing != "" { + set("PYTHONPATH", site+":"+existing) + } else { + set("PYTHONPATH", site) + } + + for _, kv := range extra { + k, v, _ := strings.Cut(kv, "=") + set(k, v) + } + + out := make([]string, 0, len(order)) + for _, k := range order { + out = append(out, k+"="+env[k]) + } + return out +} + +func history(createdBy string) v1.History { + return v1.History{ + Created: v1.Time{Time: epoch()}, + CreatedBy: "py-image-builder: " + createdBy, + } +} + +func epoch() time.Time { return time.Unix(ptar.Epoch, 0).UTC() } + +func abs(p string) string { return "/" + strings.TrimPrefix(path.Clean(p), "/") } diff --git a/py-image-builder/internal/build/build_test.go b/py-image-builder/internal/build/build_test.go new file mode 100644 index 0000000..30ef1be --- /dev/null +++ b/py-image-builder/internal/build/build_test.go @@ -0,0 +1,229 @@ +package build + +import ( + "archive/tar" + "io" + "os" + "path/filepath" + "strings" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/testwheel" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheel" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheelhouse" +) + +func layerPaths(t *testing.T, l v1.Layer) map[string]bool { + t.Helper() + rc, err := l.Uncompressed() + if err != nil { + t.Fatal(err) + } + defer rc.Close() + names := map[string]bool{} + tr := tar.NewReader(rc) + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if h.Typeflag == tar.TypeReg { + names[h.Name] = true + } + } + return names +} + +var testLayout = wheel.Layout{Prefix: "/app/.venv", PythonTag: "python3.12"} + +func mkWheel(t *testing.T, dir, name, version string) wheelhouse.ResolvedWheel { + t.Helper() + path, sha := testwheel.Write(t, dir, testwheel.Spec{ + Name: name, + Version: version, + Modules: map[string]string{name + "/__init__.py": "V='" + version + "'\n"}, + }) + return wheelhouse.ResolvedWheel{Name: name, Version: version, Path: path, SHA256: sha} +} + +func baseOpts(wheels []wheelhouse.ResolvedWheel) Options { + return Options{ + Base: empty.Image, + Wheels: wheels, + Layout: testLayout, + Strategy: PerWheel, + WorkingDir: "/app", + Entrypoint: []string{"python", "-m", "app"}, + } +} + +func TestBuildReproducible(t *testing.T) { + dir := t.TempDir() + wheels := []wheelhouse.ResolvedWheel{mkWheel(t, dir, "alpha", "1.0"), mkWheel(t, dir, "beta", "2.0")} + + img1, err := Build(baseOpts(wheels)) + if err != nil { + t.Fatal(err) + } + img2, err := Build(baseOpts(wheels)) + if err != nil { + t.Fatal(err) + } + d1, err := img1.Digest() + if err != nil { + t.Fatal(err) + } + d2, err := img2.Digest() + if err != nil { + t.Fatal(err) + } + if d1 != d2 { + t.Fatalf("image not reproducible: %s vs %s", d1, d2) + } +} + +func TestConfigAndLayerCount(t *testing.T) { + dir := t.TempDir() + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "main.py"), []byte("print('hi')\n"), 0o644); err != nil { + t.Fatal(err) + } + wheels := []wheelhouse.ResolvedWheel{mkWheel(t, dir, "alpha", "1.0"), mkWheel(t, dir, "beta", "2.0")} + opts := baseOpts(wheels) + opts.SourceDir = src + opts.User = "65532" + opts.Labels = map[string]string{"org.test": "yes"} + + img, err := Build(opts) + if err != nil { + t.Fatal(err) + } + + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + if len(layers) != 3 { // 2 wheels + 1 source + t.Fatalf("got %d layers, want 3", len(layers)) + } + + cf, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + c := cf.Config + env := strings.Join(c.Env, "\n") + for _, want := range []string{ + "VIRTUAL_ENV=/app/.venv", + "PYTHONPATH=/app/.venv/lib/python3.12/site-packages", + "PATH=/app/.venv/bin:", + } { + if !strings.Contains(env, want) { + t.Errorf("env missing %q; got:\n%s", want, env) + } + } + if c.WorkingDir != "/app" { + t.Errorf("workdir = %q", c.WorkingDir) + } + if c.User != "65532" { + t.Errorf("user = %q", c.User) + } + if strings.Join(c.Entrypoint, " ") != "python -m app" { + t.Errorf("entrypoint = %v", c.Entrypoint) + } + if c.Labels["org.test"] != "yes" { + t.Errorf("label not set: %v", c.Labels) + } + if !cf.Created.Time.Equal(epoch()) { + t.Errorf("config Created = %v, want epoch", cf.Created) + } +} + +// TestPerWheelOnlyNewDepChangesLayers is the central guarantee: adding a new +// dependency adds exactly one new layer and leaves every existing dependency +// layer byte-identical (same digest). +func TestPerWheelOnlyNewDepChangesLayers(t *testing.T) { + dir := t.TempDir() + a := mkWheel(t, dir, "alpha", "1.0") + b := mkWheel(t, dir, "beta", "2.0") + c := mkWheel(t, dir, "gamma", "3.0") + + digestsOf := func(wheels []wheelhouse.ResolvedWheel) []string { + img, err := Build(baseOpts(wheels)) + if err != nil { + t.Fatal(err) + } + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + var out []string + for _, l := range layers { + d, err := l.Digest() + if err != nil { + t.Fatal(err) + } + out = append(out, d.String()) + } + return out + } + + before := digestsOf([]wheelhouse.ResolvedWheel{a, b}) + after := digestsOf([]wheelhouse.ResolvedWheel{a, b, c}) + + if len(before) != 2 || len(after) != 3 { + t.Fatalf("layer counts: before=%d after=%d", len(before), len(after)) + } + // alpha and beta layers must be unchanged. + beforeSet := map[string]bool{before[0]: true, before[1]: true} + matches := 0 + for _, d := range after { + if beforeSet[d] { + matches++ + } + } + if matches != 2 { + t.Fatalf("expected 2 unchanged dependency layers, got %d (before=%v after=%v)", matches, before, after) + } +} + +func TestSourceLayerIgnore(t *testing.T) { + src := t.TempDir() + must := func(p, content string) { + full := filepath.Join(src, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + must("main.py", "x") + must("pkg/__pycache__/main.cpython-312.pyc", "junk") + must("pkg/mod.py", "y") + must(".git/config", "secret") + + layer, err := sourceLayer(src, "/app", nil) + if err != nil { + t.Fatal(err) + } + names := layerPaths(t, layer) + + want := map[string]bool{"app/main.py": true, "app/pkg/mod.py": true} + for n := range want { + if !names[n] { + t.Errorf("expected %q in source layer", n) + } + } + for n := range names { + if strings.Contains(n, "__pycache__") || strings.Contains(n, ".git") || strings.HasSuffix(n, ".pyc") { + t.Errorf("ignored path leaked into layer: %q", n) + } + } +} diff --git a/py-image-builder/internal/build/resolve.go b/py-image-builder/internal/build/resolve.go new file mode 100644 index 0000000..ffb6677 --- /dev/null +++ b/py-image-builder/internal/build/resolve.go @@ -0,0 +1,36 @@ +package build + +import ( + "context" + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// Base resolves a base image reference to a v1.Image for the given platform. +// Only the base manifest and config are fetched; its layer blobs are referenced +// by digest and never downloaded. +func Base(ctx context.Context, ref string, platform *v1.Platform, kc authn.Keychain) (v1.Image, error) { + r, err := name.ParseReference(ref) + if err != nil { + return nil, fmt.Errorf("build: parse base ref %q: %w", ref, err) + } + if kc == nil { + kc = authn.DefaultKeychain + } + opts := []remote.Option{ + remote.WithContext(ctx), + remote.WithAuthFromKeychain(kc), + } + if platform != nil { + opts = append(opts, remote.WithPlatform(*platform)) + } + img, err := remote.Image(r, opts...) + if err != nil { + return nil, fmt.Errorf("build: fetch base %q: %w", ref, err) + } + return img, nil +} diff --git a/py-image-builder/internal/cli/cli.go b/py-image-builder/internal/cli/cli.go new file mode 100644 index 0000000..81ca3a5 --- /dev/null +++ b/py-image-builder/internal/cli/cli.go @@ -0,0 +1,230 @@ +// Package cli implements the py-image-builder command line. +package cli + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/build" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/lock" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/sbom" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheel" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheelhouse" +) + +// Root returns the root cobra command. +func Root() *cobra.Command { + root := &cobra.Command{ + Use: "py-image-builder", + Short: "Docker-less, layer-aware Python container image builder", + SilenceUsage: true, + SilenceErrors: true, + } + root.AddCommand(buildCmd()) + return root +} + +type buildFlags struct { + base string + lockFile string + findLinks []string + source string + tag string + platform string + pythonTag string + prefix string + workingDir string + entrypoint []string + cmd []string + env []string + user string + labels []string + ignore []string + strategy string + + push bool + ociLayout string + printDigest bool + sbomOut string + requireHash bool +} + +func buildCmd() *cobra.Command { + f := &buildFlags{} + cmd := &cobra.Command{ + Use: "build", + Short: "Build a Python application image and push it", + RunE: func(cmd *cobra.Command, _ []string) error { + return runBuild(cmd.Context(), f) + }, + } + fs := cmd.Flags() + fs.StringVar(&f.base, "base", "", "base image reference (required)") + fs.StringVar(&f.lockFile, "lock", "", "hashed requirements file (required)") + fs.StringArrayVar(&f.findLinks, "find-links", nil, "directory of wheels to resolve against (repeatable, required)") + fs.StringVar(&f.source, "source", "", "application source directory (optional)") + fs.StringVarP(&f.tag, "tag", "t", "", "image tag to push, e.g. registry/repo:tag") + fs.StringVar(&f.platform, "platform", "", "platform for base resolution, e.g. linux/amd64") + fs.StringVar(&f.pythonTag, "python", "python3.12", "python lib directory name for site-packages") + fs.StringVar(&f.prefix, "prefix", "/app/.venv", "install prefix (venv-like root)") + fs.StringVar(&f.workingDir, "workdir", "/app", "image working directory and source destination") + fs.StringArrayVar(&f.entrypoint, "entrypoint", nil, "entrypoint argv (repeatable)") + fs.StringArrayVar(&f.cmd, "cmd", nil, "cmd argv (repeatable)") + fs.StringArrayVar(&f.env, "env", nil, "extra KEY=VALUE env (repeatable)") + fs.StringVar(&f.user, "user", "", "image user, e.g. 65532") + fs.StringArrayVar(&f.labels, "label", nil, "image label KEY=VALUE (repeatable)") + fs.StringArrayVar(&f.ignore, "ignore", nil, "extra source ignore glob (repeatable)") + fs.StringVar(&f.strategy, "layer-strategy", string(build.PerWheel), "per-wheel | single-deps-layer") + fs.BoolVar(&f.push, "push", true, "push the image to --tag") + fs.StringVar(&f.ociLayout, "oci-layout", "", "also write the image to this OCI layout directory") + fs.BoolVar(&f.printDigest, "print-digest", false, "print only the resulting image digest") + fs.StringVar(&f.sbomOut, "sbom", "", "write a CycloneDX SBOM to this path") + fs.BoolVar(&f.requireHash, "require-hashes", true, "require every requirement to carry --hash entries") + return cmd +} + +func runBuild(ctx context.Context, f *buildFlags) error { + if f.base == "" { + return fmt.Errorf("--base is required") + } + if f.lockFile == "" { + return fmt.Errorf("--lock is required") + } + if len(f.findLinks) == 0 { + return fmt.Errorf("--find-links is required (a directory of wheels)") + } + + reqs, err := lock.ParseFile(f.lockFile) + if err != nil { + return err + } + if err := lock.Validate(reqs, f.requireHash); err != nil { + return err + } + wheels, err := wheelhouse.Resolve(reqs, f.findLinks) + if err != nil { + return err + } + + var platform *v1.Platform + if f.platform != "" { + platform, err = v1.ParsePlatform(f.platform) + if err != nil { + return err + } + } + + base, err := resolveBase(ctx, f.base, platform) + if err != nil { + return err + } + + labels, err := keyValues(f.labels) + if err != nil { + return fmt.Errorf("--label: %w", err) + } + + img, err := build.Build(build.Options{ + Base: base, + Wheels: wheels, + Layout: wheel.Layout{Prefix: f.prefix, PythonTag: f.pythonTag}, + Strategy: build.LayerStrategy(f.strategy), + SourceDir: f.source, + Ignore: f.ignore, + WorkingDir: f.workingDir, + Entrypoint: f.entrypoint, + Cmd: f.cmd, + Env: f.env, + User: f.user, + Labels: labels, + }) + if err != nil { + return err + } + + if f.sbomOut != "" { + data, err := sbom.Generate(wheels) + if err != nil { + return err + } + if err := os.WriteFile(f.sbomOut, data, 0o644); err != nil { + return err + } + } + + digest, err := img.Digest() + if err != nil { + return err + } + + if f.ociLayout != "" { + if _, err := layout.Write(f.ociLayout, empty.Index); err != nil { + return err + } + p, err := layout.FromPath(f.ociLayout) + if err != nil { + return err + } + if err := p.AppendImage(img); err != nil { + return err + } + } + + if f.printDigest { + fmt.Println(digest.String()) + return nil + } + + if f.push { + if f.tag == "" { + return fmt.Errorf("--tag is required to push (or use --push=false)") + } + ref, err := name.ParseReference(f.tag) + if err != nil { + return err + } + if err := remote.Write(ref, img, + remote.WithContext(ctx), + remote.WithAuthFromKeychain(authn.DefaultKeychain), + ); err != nil { + return fmt.Errorf("push: %w", err) + } + fmt.Printf("pushed %s@%s\n", ref.Context().Name(), digest) + return nil + } + + fmt.Println(digest.String()) + return nil +} + +func resolveBase(ctx context.Context, ref string, platform *v1.Platform) (v1.Image, error) { + if ref == "scratch" { + return empty.Image, nil + } + return build.Base(ctx, ref, platform, authn.DefaultKeychain) +} + +func keyValues(in []string) (map[string]string, error) { + if len(in) == 0 { + return nil, nil + } + out := map[string]string{} + for _, kv := range in { + k, v, ok := strings.Cut(kv, "=") + if !ok { + return nil, fmt.Errorf("expected KEY=VALUE, got %q", kv) + } + out[k] = v + } + return out, nil +} diff --git a/py-image-builder/internal/lock/lock.go b/py-image-builder/internal/lock/lock.go new file mode 100644 index 0000000..a2eb194 --- /dev/null +++ b/py-image-builder/internal/lock/lock.go @@ -0,0 +1,138 @@ +// Package lock parses fully-pinned, hashed requirements files (the output of +// `pip-compile` / `uv pip compile --generate-hashes`). +// +// We deliberately do not resolve dependencies ourselves; we consume a lock that +// already pins every distribution to an exact version with one or more sha256 +// hashes. The hashes are what make a build reproducible and let us key the +// layer cache by wheel content. +package lock + +import ( + "bufio" + "fmt" + "io" + "os" + "regexp" + "strings" +) + +// Requirement is a single pinned distribution. +type Requirement struct { + // Name is the project name as written (normalize with NormalizeName for + // comparisons). + Name string + // Version is the exact pinned version (from `==`). + Version string + // Hashes are the allowed sha256 hex digests (without the "sha256:" prefix). + Hashes []string +} + +var ( + // pinRE matches "name==version" optionally followed by extras/markers we + // ignore for matching purposes. + pinRE = regexp.MustCompile(`^\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*==\s*([^\s;]+)`) + hashRE = regexp.MustCompile(`--hash=sha256:([0-9a-fA-F]{64})`) + normRE = regexp.MustCompile(`[-_.]+`) +) + +// NormalizeName implements PEP 503 name normalization. +func NormalizeName(name string) string { + return strings.ToLower(normRE.ReplaceAllString(name, "-")) +} + +// ParseFile reads and parses a requirements file from disk. +func ParseFile(path string) ([]Requirement, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + return Parse(f) +} + +// Parse reads pinned requirements from r. Logical lines may span multiple +// physical lines using a trailing backslash, which is how `--hash` entries are +// typically formatted. +func Parse(r io.Reader) ([]Requirement, error) { + var reqs []Requirement + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) + + var logical strings.Builder + flush := func() error { + line := logical.String() + logical.Reset() + stripped := stripComment(line) + if strings.TrimSpace(stripped) == "" { + return nil + } + // Skip pip options like -r/-c/--index-url that aren't pins. + if isOption(stripped) { + return nil + } + m := pinRE.FindStringSubmatch(stripped) + if m == nil { + // Not a pinned requirement (e.g. a bare option line); ignore. + return nil + } + req := Requirement{Name: m[1], Version: m[2]} + for _, h := range hashRE.FindAllStringSubmatch(stripped, -1) { + req.Hashes = append(req.Hashes, strings.ToLower(h[1])) + } + reqs = append(reqs, req) + return nil + } + + for sc.Scan() { + line := sc.Text() + trimmed := strings.TrimRight(line, " \t") + if strings.HasSuffix(trimmed, "\\") { + logical.WriteString(strings.TrimSuffix(trimmed, "\\")) + logical.WriteString(" ") + continue + } + logical.WriteString(line) + if err := flush(); err != nil { + return nil, err + } + } + if err := sc.Err(); err != nil { + return nil, err + } + if logical.Len() > 0 { + if err := flush(); err != nil { + return nil, err + } + } + return reqs, nil +} + +func stripComment(s string) string { + // A '#' starts a comment unless escaped; requirements files don't escape so + // we treat the first unquoted '#' as a comment start. + if i := strings.Index(s, " #"); i >= 0 { + return s[:i] + } + if strings.HasPrefix(strings.TrimSpace(s), "#") { + return "" + } + return s +} + +func isOption(s string) bool { + t := strings.TrimSpace(s) + return strings.HasPrefix(t, "-") +} + +// Validate ensures every requirement is pinned and (optionally) hashed. +func Validate(reqs []Requirement, requireHashes bool) error { + for _, r := range reqs { + if r.Version == "" { + return fmt.Errorf("lock: requirement %q is not pinned with ==", r.Name) + } + if requireHashes && len(r.Hashes) == 0 { + return fmt.Errorf("lock: requirement %q has no --hash entries", r.Name) + } + } + return nil +} diff --git a/py-image-builder/internal/lock/lock_test.go b/py-image-builder/internal/lock/lock_test.go new file mode 100644 index 0000000..ca9bbf9 --- /dev/null +++ b/py-image-builder/internal/lock/lock_test.go @@ -0,0 +1,74 @@ +package lock + +import ( + "strings" + "testing" +) + +const sample = ` +# This file is autogenerated by pip-compile +--index-url https://pypi.org/simple + +flask==2.3.3 \ + --hash=sha256:09c347a92aa7ff4a8e7f3206795f30d826654baf38b873d0744cd571ca609efc \ + --hash=sha256:f69fcd559dc907ed196ab9df0e48471709175e696d6e698dd4dbe940f96ce66b + # via -r requirements.in +Werkzeug==2.3.7 \ + --hash=sha256:2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8 +click == 8.1.7 ; python_version >= "3.8" \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 +` + +func TestParse(t *testing.T) { + reqs, err := Parse(strings.NewReader(sample)) + if err != nil { + t.Fatal(err) + } + if len(reqs) != 3 { + t.Fatalf("got %d reqs, want 3: %+v", len(reqs), reqs) + } + + if reqs[0].Name != "flask" || reqs[0].Version != "2.3.3" { + t.Errorf("req0 = %+v", reqs[0]) + } + if len(reqs[0].Hashes) != 2 { + t.Errorf("flask hashes = %v, want 2", reqs[0].Hashes) + } + if reqs[2].Name != "click" || reqs[2].Version != "8.1.7" { + t.Errorf("req2 = %+v (markers should be stripped)", reqs[2]) + } + if len(reqs[2].Hashes) != 1 { + t.Errorf("click hashes = %v, want 1", reqs[2].Hashes) + } +} + +func TestValidate(t *testing.T) { + ok := []Requirement{{Name: "a", Version: "1", Hashes: []string{"abc"}}} + if err := Validate(ok, true); err != nil { + t.Errorf("unexpected error: %v", err) + } + noHash := []Requirement{{Name: "a", Version: "1"}} + if err := Validate(noHash, true); err == nil { + t.Error("expected error for missing hash") + } + if err := Validate(noHash, false); err != nil { + t.Errorf("unexpected error when hashes not required: %v", err) + } + noPin := []Requirement{{Name: "a"}} + if err := Validate(noPin, false); err == nil { + t.Error("expected error for unpinned requirement") + } +} + +func TestNormalizeName(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"Flask", "flask"}, + {"typing_extensions", "typing-extensions"}, + {"ruamel.yaml", "ruamel-yaml"}, + {"a--b__c", "a-b-c"}, + } { + if got := NormalizeName(tc.in); got != tc.want { + t.Errorf("NormalizeName(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/py-image-builder/internal/ptar/ptar.go b/py-image-builder/internal/ptar/ptar.go new file mode 100644 index 0000000..1ece255 --- /dev/null +++ b/py-image-builder/internal/ptar/ptar.go @@ -0,0 +1,151 @@ +// Package ptar builds deterministic (reproducible) tar archives and turns them +// into OCI layers. +// +// Reproducibility is the load-bearing property of py-image-builder: identical +// inputs must yield byte-identical layers so that their content-addressed +// digests match and registries can dedupe/skip/mount them. To guarantee that, +// every tar we emit: +// +// - sorts entries by path, +// - synthesizes parent directory entries deterministically, +// - zeroes mtime (a fixed epoch), uid/gid, and uname/gname, +// - uses a fixed permission scheme (0644 files, 0755 dirs, 0755 for +// executables), +// - never emits device, xattr, or PAX time records. +package ptar + +import ( + "archive/tar" + "bytes" + "io" + "path" + "sort" + "strings" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// Epoch is the fixed modification time stamped on every tar entry. We use the +// Unix epoch rather than time.Now so builds are reproducible. +const Epoch = 0 + +func unixEpoch() time.Time { return time.Unix(Epoch, 0).UTC() } + +// File is a single regular file to include in a tar archive. Directories are +// synthesized automatically from the set of file paths. +type File struct { + // Path is the slash-separated path within the image, without a leading + // slash (e.g. "app/.venv/lib/python3.12/site-packages/foo/__init__.py"). + Path string + // Data is the file contents. + Data []byte + // Executable marks the file mode as 0755 instead of 0644. + Executable bool +} + +const ( + fileMode = 0o644 + execMode = 0o755 + dirMode = 0o755 +) + +// WriteTar writes a deterministic, uncompressed tar stream of files to w. +// +// The same files (regardless of input order) always produce byte-identical +// output. +func WriteTar(w io.Writer, files []File) error { + tw := tar.NewWriter(w) + + // Collect the set of directories implied by the files, plus the files + // themselves, then emit everything in a single sorted pass so the order is + // stable and parents always precede children. + dirs := map[string]struct{}{} + for _, f := range files { + clean := path.Clean(strings.TrimPrefix(f.Path, "/")) + dir := path.Dir(clean) + for dir != "." && dir != "/" && dir != "" { + dirs[dir] = struct{}{} + dir = path.Dir(dir) + } + } + + type entry struct { + name string + dir bool + file File + } + entries := make([]entry, 0, len(files)+len(dirs)) + for d := range dirs { + entries = append(entries, entry{name: d + "/", dir: true}) + } + seen := map[string]bool{} + for _, f := range files { + clean := path.Clean(strings.TrimPrefix(f.Path, "/")) + if seen[clean] { + continue + } + seen[clean] = true + entries = append(entries, entry{name: clean, file: f}) + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].name < entries[j].name }) + + for _, e := range entries { + if e.dir { + if err := tw.WriteHeader(&tar.Header{ + Name: e.name, + Typeflag: tar.TypeDir, + Mode: dirMode, + ModTime: unixEpoch(), + Format: tar.FormatPAX, + }); err != nil { + return err + } + continue + } + mode := int64(fileMode) + if e.file.Executable { + mode = execMode + } + if err := tw.WriteHeader(&tar.Header{ + Name: e.name, + Typeflag: tar.TypeReg, + Mode: mode, + Size: int64(len(e.file.Data)), + ModTime: unixEpoch(), + Format: tar.FormatPAX, + }); err != nil { + return err + } + if _, err := tw.Write(e.file.Data); err != nil { + return err + } + } + return tw.Close() +} + +// TarBytes returns the deterministic tar stream for files. +func TarBytes(files []File) ([]byte, error) { + var buf bytes.Buffer + if err := WriteTar(&buf, files); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Layer builds a reproducible OCI layer from files. +// +// The returned layer compresses deterministically (go-containerregistry uses +// stdlib gzip at a fixed level with a zeroed header), so its digest is stable +// across builds. +func Layer(files []File) (v1.Layer, error) { + raw, err := TarBytes(files) + if err != nil { + return nil, err + } + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(raw)), nil + }) +} diff --git a/py-image-builder/internal/ptar/ptar_test.go b/py-image-builder/internal/ptar/ptar_test.go new file mode 100644 index 0000000..2a877d0 --- /dev/null +++ b/py-image-builder/internal/ptar/ptar_test.go @@ -0,0 +1,108 @@ +package ptar + +import ( + "archive/tar" + "bytes" + "io" + "testing" +) + +func TestWriteTarDeterministicAcrossOrder(t *testing.T) { + a := []File{ + {Path: "b/y.txt", Data: []byte("y")}, + {Path: "a/x.txt", Data: []byte("x")}, + {Path: "a/z.txt", Data: []byte("z")}, + } + b := []File{ + {Path: "a/z.txt", Data: []byte("z")}, + {Path: "a/x.txt", Data: []byte("x")}, + {Path: "b/y.txt", Data: []byte("y")}, + } + ba, err := TarBytes(a) + if err != nil { + t.Fatal(err) + } + bb, err := TarBytes(b) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(ba, bb) { + t.Fatal("tar bytes differ for the same files in different input order") + } +} + +func TestTarSynthesizesParentDirsAndSorts(t *testing.T) { + raw, err := TarBytes([]File{ + {Path: "app/pkg/mod.py", Data: []byte("x")}, + {Path: "app/main.py", Data: []byte("y"), Executable: true}, + }) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(bytes.NewReader(raw)) + var names []string + modes := map[string]int64{} + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + names = append(names, h.Name) + modes[h.Name] = h.Mode + if !h.ModTime.Equal(unixEpoch()) { + t.Errorf("entry %q has non-epoch mtime %v", h.Name, h.ModTime) + } + if h.Uid != 0 || h.Gid != 0 { + t.Errorf("entry %q has non-zero uid/gid", h.Name) + } + } + want := []string{"app/", "app/main.py", "app/pkg/", "app/pkg/mod.py"} + if len(names) != len(want) { + t.Fatalf("entries = %v, want %v", names, want) + } + for i := range want { + if names[i] != want[i] { + t.Fatalf("entries = %v, want %v", names, want) + } + } + if modes["app/main.py"] != execMode { + t.Errorf("executable mode = %o, want %o", modes["app/main.py"], execMode) + } + if modes["app/pkg/mod.py"] != fileMode { + t.Errorf("file mode = %o, want %o", modes["app/pkg/mod.py"], fileMode) + } +} + +func TestLayerStableDigest(t *testing.T) { + files := []File{ + {Path: "site-packages/foo/__init__.py", Data: []byte("print('hi')\n")}, + {Path: "site-packages/foo/util.py", Data: []byte("X = 1\n")}, + } + l1, err := Layer(files) + if err != nil { + t.Fatal(err) + } + l2, err := Layer(files) + if err != nil { + t.Fatal(err) + } + d1, err := l1.Digest() + if err != nil { + t.Fatal(err) + } + d2, err := l2.Digest() + if err != nil { + t.Fatal(err) + } + if d1 != d2 { + t.Fatalf("layer digests differ across builds: %s vs %s", d1, d2) + } + diff1, _ := l1.DiffID() + diff2, _ := l2.DiffID() + if diff1 != diff2 { + t.Fatalf("layer diffIDs differ across builds: %s vs %s", diff1, diff2) + } +} diff --git a/py-image-builder/internal/sbom/sbom.go b/py-image-builder/internal/sbom/sbom.go new file mode 100644 index 0000000..371b3ad --- /dev/null +++ b/py-image-builder/internal/sbom/sbom.go @@ -0,0 +1,55 @@ +// Package sbom emits a minimal, deterministic CycloneDX-style SBOM listing the +// resolved wheels that went into an image. Because the builder already holds +// every wheel's name, version, and sha256, producing an SBOM is essentially +// free. +package sbom + +import ( + "encoding/json" + "sort" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheelhouse" +) + +type doc struct { + BOMFormat string `json:"bomFormat"` + SpecVersion string `json:"specVersion"` + Version int `json:"version"` + Components []component `json:"components"` +} + +type component struct { + Type string `json:"type"` + Name string `json:"name"` + Version string `json:"version"` + PURL string `json:"purl"` + Hashes []hash `json:"hashes,omitempty"` +} + +type hash struct { + Alg string `json:"alg"` + Content string `json:"content"` +} + +// Generate returns a deterministic CycloneDX JSON document for the wheels. +func Generate(wheels []wheelhouse.ResolvedWheel) ([]byte, error) { + sorted := append([]wheelhouse.ResolvedWheel(nil), wheels...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Name != sorted[j].Name { + return sorted[i].Name < sorted[j].Name + } + return sorted[i].Version < sorted[j].Version + }) + + d := doc{BOMFormat: "CycloneDX", SpecVersion: "1.5", Version: 1} + for _, w := range sorted { + d.Components = append(d.Components, component{ + Type: "library", + Name: w.Name, + Version: w.Version, + PURL: "pkg:pypi/" + w.Name + "@" + w.Version, + Hashes: []hash{{Alg: "SHA-256", Content: w.SHA256}}, + }) + } + return json.MarshalIndent(d, "", " ") +} diff --git a/py-image-builder/internal/sbom/sbom_test.go b/py-image-builder/internal/sbom/sbom_test.go new file mode 100644 index 0000000..5de99b1 --- /dev/null +++ b/py-image-builder/internal/sbom/sbom_test.go @@ -0,0 +1,34 @@ +package sbom + +import ( + "bytes" + "strings" + "testing" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheelhouse" +) + +func TestGenerateDeterministicAndSorted(t *testing.T) { + wheels := []wheelhouse.ResolvedWheel{ + {Name: "beta", Version: "2.0", SHA256: "bbbb"}, + {Name: "alpha", Version: "1.0", SHA256: "aaaa"}, + } + a, err := Generate(wheels) + if err != nil { + t.Fatal(err) + } + b, err := Generate(wheels) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(a, b) { + t.Fatal("SBOM not deterministic") + } + s := string(a) + if !strings.Contains(s, "pkg:pypi/alpha@1.0") || !strings.Contains(s, "pkg:pypi/beta@2.0") { + t.Errorf("missing purls:\n%s", s) + } + if strings.Index(s, "alpha") > strings.Index(s, "beta") { + t.Error("components not sorted (alpha should precede beta)") + } +} diff --git a/py-image-builder/internal/testwheel/testwheel.go b/py-image-builder/internal/testwheel/testwheel.go new file mode 100644 index 0000000..3cadacc --- /dev/null +++ b/py-image-builder/internal/testwheel/testwheel.go @@ -0,0 +1,83 @@ +// Package testwheel synthesizes minimal but valid Python wheels on disk for use +// in tests, so tests stay hermetic (no pip, no network). +package testwheel + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "testing" + "time" +) + +// Spec describes a wheel to synthesize. +type Spec struct { + // Name is the distribution name (use underscores, no dashes). + Name string + // Version is the version (no dashes). + Version string + // Modules maps top-level module file paths (e.g. "foo/__init__.py") to + // their contents, placed in the wheel root (i.e. into site-packages). + Modules map[string]string + // ConsoleScripts maps script name -> "module:attr" entry points. + ConsoleScripts map[string]string +} + +// Write creates the wheel file in dir and returns its path and hex sha256. +func Write(t *testing.T, dir string, s Spec) (path, sha string) { + t.Helper() + data := Bytes(t, s) + name := fmt.Sprintf("%s-%s-py3-none-any.whl", s.Name, s.Version) + path = filepath.Join(dir, name) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write wheel: %v", err) + } + sum := sha256.Sum256(data) + return path, hex.EncodeToString(sum[:]) +} + +// Bytes returns the wheel zip bytes for the spec, deterministically. +func Bytes(t *testing.T, s Spec) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + add := func(name, content string) { + w, err := zw.CreateHeader(&zip.FileHeader{ + Name: name, + Method: zip.Store, // stored: stable bytes + Modified: time.Unix(0, 0).UTC(), + }) + if err != nil { + t.Fatalf("zip header %s: %v", name, err) + } + if _, err := w.Write([]byte(content)); err != nil { + t.Fatalf("zip write %s: %v", name, err) + } + } + + for p, c := range s.Modules { + add(p, c) + } + + distInfo := fmt.Sprintf("%s-%s.dist-info", s.Name, s.Version) + add(distInfo+"/METADATA", fmt.Sprintf("Metadata-Version: 2.1\nName: %s\nVersion: %s\n", s.Name, s.Version)) + add(distInfo+"/WHEEL", "Wheel-Version: 1.0\nGenerator: testwheel\nRoot-Is-Purelib: true\nTag: py3-none-any\n") + if len(s.ConsoleScripts) > 0 { + ep := "[console_scripts]\n" + for name, target := range s.ConsoleScripts { + ep += fmt.Sprintf("%s = %s\n", name, target) + } + add(distInfo+"/entry_points.txt", ep) + } + add(distInfo+"/RECORD", "") + + if err := zw.Close(); err != nil { + t.Fatalf("zip close: %v", err) + } + return buf.Bytes() +} diff --git a/py-image-builder/internal/wheel/wheel.go b/py-image-builder/internal/wheel/wheel.go new file mode 100644 index 0000000..ab1cd1e --- /dev/null +++ b/py-image-builder/internal/wheel/wheel.go @@ -0,0 +1,261 @@ +// Package wheel parses Python wheel files (PEP 427) and turns each wheel into a +// deterministic set of installed files suitable for a single OCI layer. +// +// "Installing" a wheel is deliberately simple and byte-stable: a wheel is a zip +// archive whose members are laid down relative to a target prefix. Most members +// go straight into site-packages; members under the "-.data" +// directory are redirected (scripts -> bin, purelib/platlib -> site-packages, +// data -> prefix root). We then synthesize console-script entry points. +// +// We intentionally do NOT byte-compile (no __pycache__/*.pyc) because compiled +// output is not reliably reproducible across interpreter patch releases; the +// interpreter compiles on first import at runtime instead. +package wheel + +import ( + "archive/zip" + "bufio" + "fmt" + "io" + "path" + "sort" + "strings" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/ptar" +) + +// Layout describes where files are installed inside the image filesystem. +type Layout struct { + // Prefix is the virtualenv-like root, e.g. "/app/.venv". + Prefix string + // PythonTag is the interpreter dir under lib, e.g. "python3.12". + PythonTag string +} + +// SitePackages returns the slash path (no leading slash) of site-packages. +func (l Layout) SitePackages() string { + return path.Join(trimSlash(l.Prefix), "lib", l.PythonTag, "site-packages") +} + +// BinDir returns the slash path (no leading slash) of the scripts/bin dir. +func (l Layout) BinDir() string { + return path.Join(trimSlash(l.Prefix), "bin") +} + +// Wheel is a parsed wheel file. +type Wheel struct { + // Name is the distribution name as it appears in the filename (with + // underscores), e.g. "Flask". + Name string + // Version is the distribution version. + Version string + // distInfo is the "-.dist-info" directory name. + distInfo string + // dataDir is the "-.data" directory name. + dataDir string + + z *zip.ReadCloser +} + +// ParseFilename extracts the distribution name and version from a wheel +// filename per PEP 427: name-version(-build)?-pytag-abitag-plattag.whl. +func ParseFilename(filename string) (name, version string, err error) { + base := path.Base(filename) + parts := strings.Split(strings.TrimSuffix(base, ".whl"), "-") + if len(parts) < 5 { + return "", "", fmt.Errorf("wheel: malformed filename %q", base) + } + // The trailing 3 fields are pytag, abitag, plattag. An optional build tag + // may sit before them; name/version are everything before that. + // name is parts[0], version is parts[1]; build/tag fields follow. + return parts[0], parts[1], nil +} + +// Open reads and parses a wheel file from disk. +func Open(filename string) (*Wheel, error) { + z, err := zip.OpenReader(filename) + if err != nil { + return nil, fmt.Errorf("wheel: open %q: %w", filename, err) + } + w := &Wheel{z: z} + for _, f := range z.File { + top := strings.SplitN(f.Name, "/", 2)[0] + switch { + case strings.HasSuffix(top, ".dist-info") && w.distInfo == "": + w.distInfo = top + case strings.HasSuffix(top, ".data") && w.dataDir == "": + w.dataDir = top + } + } + if w.distInfo == "" { + z.Close() + return nil, fmt.Errorf("wheel: %q has no .dist-info directory", filename) + } + base := strings.TrimSuffix(w.distInfo, ".dist-info") + if i := strings.LastIndex(base, "-"); i >= 0 { + w.Name, w.Version = base[:i], base[i+1:] + } else { + w.Name = base + } + return w, nil +} + +// Close releases the underlying zip reader. +func (w *Wheel) Close() error { return w.z.Close() } + +// Files returns the deterministic set of installed files for this wheel under +// the given layout. +func (w *Wheel) Files(layout Layout) ([]ptar.File, error) { + site := layout.SitePackages() + bin := layout.BinDir() + prefix := trimSlash(layout.Prefix) + + var out []ptar.File + for _, f := range w.z.File { + if f.FileInfo().IsDir() { + continue + } + dst, exec, skip := w.dest(f.Name, site, bin, prefix) + if skip { + continue + } + data, err := readZip(f) + if err != nil { + return nil, err + } + out = append(out, ptar.File{Path: dst, Data: data, Executable: exec || isExec(f)}) + } + + scripts, err := w.consoleScripts(bin) + if err != nil { + return nil, err + } + out = append(out, scripts...) + + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out, nil +} + +// dest maps a wheel member name to its install destination. It handles the +// special "-.data/{scripts,purelib,platlib,data,headers}" tree. +func (w *Wheel) dest(name, site, bin, prefix string) (dst string, exec, skip bool) { + if w.dataDir != "" && strings.HasPrefix(name, w.dataDir+"/") { + rest := strings.TrimPrefix(name, w.dataDir+"/") + category, sub, ok := strings.Cut(rest, "/") + if !ok { + return "", false, true + } + switch category { + case "scripts": + return path.Join(bin, sub), true, false + case "purelib", "platlib": + return path.Join(site, sub), false, false + case "data": + return path.Join(prefix, sub), false, false + default: // headers and anything else: skip for simplicity + return "", false, true + } + } + return path.Join(site, name), false, false +} + +// consoleScripts synthesizes launcher scripts for [console_scripts] entry +// points declared in entry_points.txt. +func (w *Wheel) consoleScripts(bin string) ([]ptar.File, error) { + f := w.fileByName(w.distInfo + "/entry_points.txt") + if f == nil { + return nil, nil + } + data, err := readZip(f) + if err != nil { + return nil, err + } + eps := parseConsoleScripts(string(data)) + names := make([]string, 0, len(eps)) + for k := range eps { + names = append(names, k) + } + sort.Strings(names) + + var out []ptar.File + for _, n := range names { + module, attr := eps[n].module, eps[n].attr + script := fmt.Sprintf(`#!/usr/bin/env python +# Generated by py-image-builder. +import sys +from %s import %s +if __name__ == "__main__": + sys.exit(%s()) +`, module, rootAttr(attr), attr) + out = append(out, ptar.File{Path: path.Join(bin, n), Data: []byte(script), Executable: true}) + } + return out, nil +} + +type entryPoint struct{ module, attr string } + +// parseConsoleScripts parses the [console_scripts] section of an +// entry_points.txt file into name -> {module, attr}. +func parseConsoleScripts(s string) map[string]entryPoint { + out := map[string]entryPoint{} + section := "" + sc := bufio.NewScanner(strings.NewReader(s)) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + section = strings.TrimSuffix(strings.TrimPrefix(line, "["), "]") + continue + } + if section != "console_scripts" { + continue + } + name, target, ok := strings.Cut(line, "=") + if !ok { + continue + } + name = strings.TrimSpace(name) + target = strings.TrimSpace(target) + module, attr, _ := strings.Cut(target, ":") + if attr == "" { + attr = "main" + } + out[name] = entryPoint{module: strings.TrimSpace(module), attr: strings.TrimSpace(attr)} + } + return out +} + +// rootAttr returns the top-level attribute name (before any dotted access) so +// the generated `from module import ` is valid. +func rootAttr(attr string) string { + if i := strings.Index(attr, "."); i >= 0 { + return attr[:i] + } + return attr +} + +func (w *Wheel) fileByName(name string) *zip.File { + for _, f := range w.z.File { + if f.Name == name { + return f + } + } + return nil +} + +func readZip(f *zip.File) ([]byte, error) { + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() + return io.ReadAll(rc) +} + +func isExec(f *zip.File) bool { + return f.FileInfo().Mode().Perm()&0o111 != 0 +} + +func trimSlash(s string) string { return strings.TrimPrefix(path.Clean(s), "/") } diff --git a/py-image-builder/internal/wheel/wheel_test.go b/py-image-builder/internal/wheel/wheel_test.go new file mode 100644 index 0000000..e273e6f --- /dev/null +++ b/py-image-builder/internal/wheel/wheel_test.go @@ -0,0 +1,113 @@ +package wheel + +import ( + "strings" + "testing" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/ptar" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/testwheel" +) + +var layout = Layout{Prefix: "/app/.venv", PythonTag: "python3.12"} + +func writeFixture(t *testing.T) string { + t.Helper() + dir := t.TempDir() + path, _ := testwheel.Write(t, dir, testwheel.Spec{ + Name: "examplepkg", + Version: "1.2.3", + Modules: map[string]string{ + "examplepkg/__init__.py": "VERSION = '1.2.3'\n", + "examplepkg/cli.py": "def main():\n print('hi')\n", + }, + ConsoleScripts: map[string]string{"example": "examplepkg.cli:main"}, + }) + return path +} + +func TestParseFilename(t *testing.T) { + name, version, err := ParseFilename("examplepkg-1.2.3-py3-none-any.whl") + if err != nil { + t.Fatal(err) + } + if name != "examplepkg" || version != "1.2.3" { + t.Fatalf("got %q %q", name, version) + } + if _, _, err := ParseFilename("not-a-wheel.txt"); err == nil { + t.Error("expected error for malformed filename") + } +} + +func TestFilesLayout(t *testing.T) { + w, err := Open(writeFixture(t)) + if err != nil { + t.Fatal(err) + } + defer w.Close() + + if w.Name != "examplepkg" || w.Version != "1.2.3" { + t.Fatalf("parsed name/version = %q %q", w.Name, w.Version) + } + + files, err := w.Files(layout) + if err != nil { + t.Fatal(err) + } + + byPath := map[string]ptar.File{} + for _, f := range files { + byPath[f.Path] = f + } + + site := "app/.venv/lib/python3.12/site-packages" + for _, want := range []string{ + site + "/examplepkg/__init__.py", + site + "/examplepkg/cli.py", + site + "/examplepkg-1.2.3.dist-info/METADATA", + } { + if _, ok := byPath[want]; !ok { + t.Errorf("missing installed file %q", want) + } + } + + script, ok := byPath["app/.venv/bin/example"] + if !ok { + t.Fatal("console script not generated") + } + if !script.Executable { + t.Error("console script should be executable") + } + if !strings.Contains(string(script.Data), "from examplepkg.cli import main") { + t.Errorf("script body unexpected:\n%s", script.Data) + } + if !strings.HasPrefix(string(script.Data), "#!") { + t.Error("script should start with a shebang") + } +} + +func TestFilesDeterministic(t *testing.T) { + path := writeFixture(t) + digest := func() string { + w, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer w.Close() + files, err := w.Files(layout) + if err != nil { + t.Fatal(err) + } + l, err := ptar.Layer(files) + if err != nil { + t.Fatal(err) + } + d, err := l.Digest() + if err != nil { + t.Fatal(err) + } + return d.String() + } + if digest() != digest() { + t.Fatal("wheel layer digest is not deterministic") + } +} diff --git a/py-image-builder/internal/wheelhouse/wheelhouse.go b/py-image-builder/internal/wheelhouse/wheelhouse.go new file mode 100644 index 0000000..dfb2aa5 --- /dev/null +++ b/py-image-builder/internal/wheelhouse/wheelhouse.go @@ -0,0 +1,124 @@ +// Package wheelhouse resolves pinned requirements to concrete wheel files held +// in one or more local directories ("find-links" style), verifying that each +// file's sha256 matches the lock. +// +// Keeping resolution local keeps builds hermetic and reproducible: the lock + +// the wheelhouse fully determine the output. (A network fetcher that downloads +// wheels by URL/sha256 is a straightforward future addition that feeds the same +// ResolvedWheel path.) +package wheelhouse + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/lock" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/wheel" +) + +// ResolvedWheel pairs a pinned requirement with the wheel file that satisfies +// it and that file's content hash. +type ResolvedWheel struct { + Name string + Version string + Path string + SHA256 string // hex +} + +// candidate is a wheel file discovered in the wheelhouse. +type candidate struct { + path string + name string // normalized + version string +} + +// Resolve matches each requirement to a wheel file found in dirs. It returns +// resolved wheels sorted by (normalized name, version) for deterministic layer +// ordering. +func Resolve(reqs []lock.Requirement, dirs []string) ([]ResolvedWheel, error) { + cands, err := scan(dirs) + if err != nil { + return nil, err + } + + out := make([]ResolvedWheel, 0, len(reqs)) + for _, req := range reqs { + want := lock.NormalizeName(req.Name) + var match *candidate + for i := range cands { + c := &cands[i] + if c.name == want && c.version == req.Version { + match = c + break + } + } + if match == nil { + return nil, fmt.Errorf("wheelhouse: no wheel found for %s==%s in %v", req.Name, req.Version, dirs) + } + sum, err := hashFile(match.path) + if err != nil { + return nil, err + } + if len(req.Hashes) > 0 && !containsHash(req.Hashes, sum) { + return nil, fmt.Errorf("wheelhouse: %s==%s hash mismatch: file %s has sha256:%s, not in lock", req.Name, req.Version, match.path, sum) + } + out = append(out, ResolvedWheel{Name: req.Name, Version: req.Version, Path: match.path, SHA256: sum}) + } + + sort.Slice(out, func(i, j int) bool { + ni, nj := lock.NormalizeName(out[i].Name), lock.NormalizeName(out[j].Name) + if ni != nj { + return ni < nj + } + return out[i].Version < out[j].Version + }) + return out, nil +} + +func scan(dirs []string) ([]candidate, error) { + var cands []candidate + for _, dir := range dirs { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("wheelhouse: read dir %q: %w", dir, err) + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".whl") { + continue + } + name, version, err := wheel.ParseFilename(e.Name()) + if err != nil { + continue + } + cands = append(cands, candidate{ + path: filepath.Join(dir, e.Name()), + name: lock.NormalizeName(name), + version: version, + }) + } + } + return cands, nil +} + +func hashFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +func containsHash(hashes []string, sum string) bool { + for _, h := range hashes { + if strings.EqualFold(h, sum) { + return true + } + } + return false +} diff --git a/py-image-builder/internal/wheelhouse/wheelhouse_test.go b/py-image-builder/internal/wheelhouse/wheelhouse_test.go new file mode 100644 index 0000000..7ef7388 --- /dev/null +++ b/py-image-builder/internal/wheelhouse/wheelhouse_test.go @@ -0,0 +1,49 @@ +package wheelhouse + +import ( + "testing" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/lock" + "github.com/imjasonh/terraform-playground/py-image-builder/internal/testwheel" +) + +func TestResolveSortedAndVerified(t *testing.T) { + dir := t.TempDir() + _, shaB := testwheel.Write(t, dir, testwheel.Spec{Name: "beta", Version: "2.0", Modules: map[string]string{"beta.py": "x=1\n"}}) + _, shaA := testwheel.Write(t, dir, testwheel.Spec{Name: "alpha", Version: "1.0", Modules: map[string]string{"alpha.py": "x=1\n"}}) + + reqs := []lock.Requirement{ + {Name: "Beta", Version: "2.0", Hashes: []string{shaB}}, + {Name: "alpha", Version: "1.0", Hashes: []string{shaA}}, + } + got, err := Resolve(reqs, []string{dir}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("got %d wheels", len(got)) + } + if got[0].Name != "alpha" || got[1].Name != "Beta" { + t.Fatalf("not sorted by normalized name: %+v", got) + } + if got[0].SHA256 != shaA { + t.Errorf("alpha sha mismatch") + } +} + +func TestResolveHashMismatch(t *testing.T) { + dir := t.TempDir() + testwheel.Write(t, dir, testwheel.Spec{Name: "alpha", Version: "1.0", Modules: map[string]string{"alpha.py": "x=1\n"}}) + reqs := []lock.Requirement{{Name: "alpha", Version: "1.0", Hashes: []string{"deadbeef"}}} + if _, err := Resolve(reqs, []string{dir}); err == nil { + t.Fatal("expected hash mismatch error") + } +} + +func TestResolveMissing(t *testing.T) { + dir := t.TempDir() + reqs := []lock.Requirement{{Name: "ghost", Version: "9.9"}} + if _, err := Resolve(reqs, []string{dir}); err == nil { + t.Fatal("expected missing-wheel error") + } +} diff --git a/py-image-builder/main.go b/py-image-builder/main.go new file mode 100644 index 0000000..f652755 --- /dev/null +++ b/py-image-builder/main.go @@ -0,0 +1,21 @@ +// Command py-image-builder builds and pushes OCI images for Python applications +// without a Docker daemon, in the spirit of ko. +// +// It installs each resolved wheel into its own deterministic, content-addressed +// layer (so adding a dependency creates exactly one new layer and unchanged +// dependencies are never re-uploaded) and lays the application source on top. +package main + +import ( + "fmt" + "os" + + "github.com/imjasonh/terraform-playground/py-image-builder/internal/cli" +) + +func main() { + if err := cli.Root().Execute(); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +}