From d9a25a373d4641c331d6008254616106834c584d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 11 Jun 2026 01:30:46 +0000 Subject: [PATCH] wheelhouse: build wheels from sdists when no compatible wheel is available Co-authored-by: Jason Hall --- pymage/internal/wheelhouse/sdist.go | 220 +++++++++++++++++++++++ pymage/internal/wheelhouse/wheelhouse.go | 17 +- 2 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 pymage/internal/wheelhouse/sdist.go diff --git a/pymage/internal/wheelhouse/sdist.go b/pymage/internal/wheelhouse/sdist.go new file mode 100644 index 0000000..090d603 --- /dev/null +++ b/pymage/internal/wheelhouse/sdist.go @@ -0,0 +1,220 @@ +package wheelhouse + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/imjasonh/terraform-playground/pymage/internal/lock" + "github.com/imjasonh/terraform-playground/pymage/internal/wheel" +) + +// buildFromSdist builds a wheel from a requirement's source distribution and +// returns it as a ResolvedWheel. The built wheel is cached (keyed by the sdist +// hash and target) so it isn't rebuilt on subsequent builds. +// +// Building runs the host's `pip wheel`, so a Python toolchain must be present. +// A pure-python sdist yields a `py3-none-any` wheel usable on any target; a +// compiled sdist is built for the host platform only, so cross-arch builds of +// such packages will fail the compatibility check with a clear error. +func buildFromSdist(ctx context.Context, req lock.Requirement, target wheel.Target, cache *wheelCache) (ResolvedWheel, error) { + if cache == nil { + return ResolvedWheel{}, fmt.Errorf("wheelhouse: building %s==%s from sdist requires a cache dir (or use --no-cache)", req.Name, req.Version) + } + key := req.Sdist.SHA256 + "-" + targetKey(target) + if p, ok := cache.getBuilt(key); ok { + sum, err := fileSHA256(p) + if err != nil { + return ResolvedWheel{}, err + } + return ResolvedWheel{Name: req.Name, Version: req.Version, Path: p, SHA256: sum}, nil + } + + work, err := os.MkdirTemp("", "pymage-sdist-") + if err != nil { + return ResolvedWheel{}, err + } + defer func() { _ = os.RemoveAll(work) }() + + sdistPath := filepath.Join(work, req.Sdist.Filename) + if err := downloadVerify(ctx, req.Sdist.URL, req.Sdist.SHA256, sdistPath); err != nil { + return ResolvedWheel{}, fmt.Errorf("wheelhouse: %s==%s sdist: %w", req.Name, req.Version, err) + } + + out := filepath.Join(work, "out") + if err := os.MkdirAll(out, 0o755); err != nil { + return ResolvedWheel{}, err + } + if err := buildWheel(ctx, sdistPath, out); err != nil { + return ResolvedWheel{}, fmt.Errorf("wheelhouse: build %s==%s from sdist: %w", req.Name, req.Version, err) + } + + built, err := selectBuiltWheel(out, req, target) + if err != nil { + return ResolvedWheel{}, err + } + + dst := cache.builtPath(key) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return ResolvedWheel{}, err + } + if err := os.Rename(built, dst); err != nil { + if err := copyFile(built, dst); err != nil { + return ResolvedWheel{}, err + } + } + sum, err := fileSHA256(dst) + if err != nil { + return ResolvedWheel{}, err + } + return ResolvedWheel{Name: req.Name, Version: req.Version, Path: dst, SHA256: sum}, nil +} + +func targetKey(t wheel.Target) string { + return fmt.Sprintf("%s-%s-cp%d%d", t.OS, t.Arch, t.PyMajor, t.PyMinor) +} + +func (c *wheelCache) builtPath(key string) string { + return filepath.Join(c.dir, "built", key+".whl") +} + +func (c *wheelCache) getBuilt(key string) (string, bool) { + if c == nil { + return "", false + } + if _, err := os.Stat(c.builtPath(key)); err == nil { + return c.builtPath(key), true + } + return "", false +} + +// buildWheel builds a wheel from an sdist into outDir using `pip wheel`. +func buildWheel(ctx context.Context, sdist, outDir string) error { + py := findPython() + if py == "" { + return fmt.Errorf("no python3/python on PATH to build the sdist (install Python+pip, supply a pre-built wheel via --find-links, or use a lock with wheels)") + } + cmd := exec.CommandContext(ctx, py, "-m", "pip", "wheel", "--no-deps", "--no-cache-dir", "--wheel-dir", outDir, sdist) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("pip wheel: %w\n%s", err, tail(string(out), 20)) + } + return nil +} + +func findPython() string { + for _, name := range []string{"python3", "python"} { + if p, err := exec.LookPath(name); err == nil { + return p + } + } + return "" +} + +// selectBuiltWheel returns the built wheel in dir matching the requirement and +// compatible with the target. +func selectBuiltWheel(dir string, req lock.Requirement, target wheel.Target) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", err + } + var compatible []candidate + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".whl") { + continue + } + tags, err := wheel.ParseTags(e.Name()) + if err != nil { + continue + } + if lock.NormalizeName(tags.Name) != lock.NormalizeName(req.Name) || tags.Version != req.Version { + continue + } + if !tags.CompatibleWith(target) { + continue + } + compatible = append(compatible, candidate{path: filepath.Join(dir, e.Name()), tags: tags}) + } + if len(compatible) == 0 { + return "", fmt.Errorf("wheelhouse: built wheel for %s==%s is not compatible with %s/%s python%d.%d (a compiled package can only be built for the host platform)", + req.Name, req.Version, target.OS, target.Arch, target.PyMajor, target.PyMinor) + } + return pickBest(compatible).path, nil +} + +func downloadVerify(ctx context.Context, url, wantSum, dest string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + client := &http.Client{Timeout: 5 * time.Minute} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("download %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download %s: HTTP %s", url, resp.Status) + } + f, err := os.Create(dest) + if err != nil { + return err + } + h := sha256.New() + if _, err := io.Copy(io.MultiWriter(f, h), resp.Body); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + if got := hex.EncodeToString(h.Sum(nil)); !strings.EqualFold(got, wantSum) { + return fmt.Errorf("sdist hash %s != expected %s", got, wantSum) + } + return nil +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.Create(dst) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + return out.Close() +} + +func tail(s string, n int) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.Join(lines, "\n") +} diff --git a/pymage/internal/wheelhouse/wheelhouse.go b/pymage/internal/wheelhouse/wheelhouse.go index bcdcaa3..20461d6 100644 --- a/pymage/internal/wheelhouse/wheelhouse.go +++ b/pymage/internal/wheelhouse/wheelhouse.go @@ -118,20 +118,31 @@ func resolveOne(ctx context.Context, req lock.Requirement, index map[string][]ca } return ResolvedWheel{Name: req.Name, Version: req.Version, Path: match.path, SHA256: sum}, nil } - if len(req.Wheels) == 0 { + if len(req.Wheels) == 0 && req.Sdist == nil { return ResolvedWheel{}, fmt.Errorf("wheelhouse: no wheel for %s==%s is compatible with %s/%s python%d.%d (found %d incompatible candidate(s))", req.Name, req.Version, target.OS, target.Arch, target.PyMajor, target.PyMinor, len(cands)) } } } - if len(req.Wheels) == 0 { + if len(req.Wheels) == 0 && req.Sdist == nil { return ResolvedWheel{}, fmt.Errorf("wheelhouse: no wheel found for %s==%s (pass --find-links or use a uv.lock with wheel URLs)", req.Name, req.Version) } if cache == nil { return ResolvedWheel{}, fmt.Errorf("wheelhouse: %s==%s not in local wheelhouse and no wheel cache dir configured", req.Name, req.Version) } - return fetchRequirement(ctx, req, target, cache) + + // Prefer a pre-built lock wheel; fall back to building from the sdist. + if len(req.Wheels) > 0 { + rw, err := fetchRequirement(ctx, req, target, cache) + if err == nil { + return rw, nil + } + if req.Sdist == nil { + return ResolvedWheel{}, err + } + } + return buildFromSdist(ctx, req, target, cache) } // pickBest chooses the most specific compatible wheel: platform-specific and