From a3c96855496ec7c12c26a118087c9cc37895230c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 11 Jun 2026 02:45:52 +0000 Subject: [PATCH] lock: require uv for uv.lock resolution; fail fast instead of falling back Removes the built-in closure walker (resolveUV/runtimeClosure/ParseUVLockFile/ ParseAny) entirely so there is no second, potentially divergent resolver. uv.lock resolution now goes only through `uv export`; if uv is not on PATH the build fails with an actionable error pointing at requirements.txt + --find-links. uv.lock parsing is kept solely to attach wheel/sdist URLs. Co-authored-by: Jason Hall --- pymage/DESIGN.md | 11 +- pymage/README.md | 7 +- pymage/internal/lock/lock.go | 36 ++-- pymage/internal/lock/uvexport.go | 12 -- pymage/internal/lock/uvexport_test.go | 8 + pymage/internal/lock/uvlock.go | 201 +------------------ pymage/internal/lock/uvlock_test.go | 272 -------------------------- 7 files changed, 38 insertions(+), 509 deletions(-) delete mode 100644 pymage/internal/lock/uvlock_test.go diff --git a/pymage/DESIGN.md b/pymage/DESIGN.md index 3808a3d..e987608 100644 --- a/pymage/DESIGN.md +++ b/pymage/DESIGN.md @@ -32,11 +32,12 @@ because it exploits content-addressed layering: pre-build a wheel out-of-band and supply it via `--find-links`. - Replacing dependency *resolution*. We delegate resolution to existing, correct tools and consume their output: for uv projects we shell out to - `uv export --frozen --no-dev` (when uv is available) to get the exact - per-target requirement set — closure, extras, groups, and workspace selection - are uv's responsibility, not ours. pymage only evaluates the markers uv emits - and maps each pin to a wheel. A built-in lock walker remains as a fallback for - environments without uv. Likewise, install layout is validated against a real + `uv export --frozen --no-dev` to get the exact per-target requirement set — + closure, extras, groups, and workspace selection are uv's responsibility, not + ours. pymage only evaluates the markers uv emits and maps each pin to a wheel. + uv is therefore **required** to build from a uv.lock; we fail fast if it's + absent rather than fall back to a second, divergent resolver. Likewise, install + layout is validated against a real `uv pip install` in a conformance test (installing a wheel runs no code, so there is no RCE risk in using it as an oracle). - A general-purpose Dockerfile interpreter. This is a focused Python app builder diff --git a/pymage/README.md b/pymage/README.md index 55b35d6..59c6818 100644 --- a/pymage/README.md +++ b/pymage/README.md @@ -164,8 +164,11 @@ installed and a `pyproject.toml` is present, resolution is delegated to `uv export --frozen --no-dev` — uv is the source of truth for the closure, extras, groups, and workspace selection, so pymage doesn't reimplement its resolver. pymage then evaluates the environment markers uv emits for the target -and attaches wheel URLs from the lock. (Without uv, a built-in closure walker is -used as a fallback.) The selectors below map onto `uv export` flags: +and attaches wheel URLs from the lock. **`uv` is therefore required to build +from a `uv.lock`** — if it isn't installed the build fails (rather than falling +back to a second, potentially divergent resolver); as an alternative, export a +hashed `requirements.txt` and build with `--lock requirements.txt --find-links`. +The selectors below map onto `uv export` flags: - `--extra ` enables one of the project's own `[project.optional-dependencies]` groups (repeatable). Extras requested *by* diff --git a/pymage/internal/lock/lock.go b/pymage/internal/lock/lock.go index 7becb38..a101671 100644 --- a/pymage/internal/lock/lock.go +++ b/pymage/internal/lock/lock.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "regexp" "strings" @@ -80,21 +81,6 @@ func ParseFile(path string) ([]Requirement, error) { return Parse(f) } -// ParseAny reads a lock file, choosing the parser from the filename: -// uv.lock is parsed as TOML; everything else is treated as requirements.txt. -func ParseAny(path string) ([]Requirement, error) { - if strings.HasSuffix(strings.ToLower(filepath.Base(path)), ".lock") && - !strings.HasSuffix(strings.ToLower(path), "requirements.lock") { - // uv.lock / poetry.lock-style names ending in .lock (not requirements.lock - // which is pip-compile output in requirements format). - if strings.EqualFold(filepath.Base(path), "uv.lock") { - return ParseUVLockFile(path) - } - } - // requirements.lock and requirements.txt both use the pip format. - return ParseFile(path) -} - // Lock is a parsed lock file (parsed once) that can be resolved per target. type Lock struct { isUV bool @@ -133,22 +119,22 @@ func Load(path string) (*Lock, error) { // Resolve returns the pinned requirements for the given target. // // For a uv.lock, resolution (the runtime closure, extras, workspace package, -// and group selection) is delegated to `uv export` when uv and a pyproject.toml -// are available — uv is the source of truth, so we don't reimplement its -// resolver. We then evaluate the per-requirement environment markers uv emits -// for the target and attach wheel/sdist URLs from the lock. When uv isn't -// available (e.g. air-gapped, or a bare uv.lock with no project), we fall back -// to a built-in closure. For requirements-format locks the list is already flat -// and pinned; we only evaluate any inline markers. +// and group selection) is delegated to `uv export` — uv is the source of truth, +// so we do not reimplement its resolver. We then evaluate the per-requirement +// environment markers uv emits for the target and attach wheel/sdist URLs from +// the lock. uv is therefore **required** to build from a uv.lock: if it isn't +// available we fail rather than fall back to a divergent resolver. For +// requirements-format locks the list is already flat and pinned; we only +// evaluate any inline markers. func (l *Lock) Resolve(o Options) ([]Requirement, error) { env := o.markerEnv() if !l.isUV { return filterByMarker(l.reqs, env), nil } - if l.path != "" && uvExportUsable(l.path) { - return uvExportResolve(l.path, l.uv, o, env) + if _, err := exec.LookPath("uv"); err != nil { + return nil, fmt.Errorf("resolving %s requires uv on PATH (pymage delegates uv.lock resolution to `uv export`); install uv (https://docs.astral.sh/uv/), or export a hashed requirements.txt and build with --lock requirements.txt --find-links", filepath.Base(l.path)) } - return resolveUV(l.uv, UVOptions{Package: o.Package, Extras: o.Extras, Env: env}) + return uvExportResolve(l.path, l.uv, o, env) } // markerEnv builds the marker environment for the target, or nil when no target diff --git a/pymage/internal/lock/uvexport.go b/pymage/internal/lock/uvexport.go index a9771da..63dd0a3 100644 --- a/pymage/internal/lock/uvexport.go +++ b/pymage/internal/lock/uvexport.go @@ -3,23 +3,11 @@ package lock import ( "bytes" "fmt" - "os" "os/exec" "path/filepath" "strings" ) -// uvExportUsable reports whether we should resolve via `uv export`: uv must be -// on PATH and the lock must sit next to a pyproject.toml (a real uv project, as -// `uv export` requires). Bare/synthetic locks fall back to the built-in walker. -func uvExportUsable(lockPath string) bool { - if _, err := exec.LookPath("uv"); err != nil { - return false - } - _, err := os.Stat(filepath.Join(filepath.Dir(lockPath), "pyproject.toml")) - return err == nil -} - // uvExportResolve delegates resolution to `uv export`, which produces the // project's universal runtime requirement set (closure, extras, groups, and // workspace selection all handled by uv) with per-requirement markers and diff --git a/pymage/internal/lock/uvexport_test.go b/pymage/internal/lock/uvexport_test.go index 981c782..597454b 100644 --- a/pymage/internal/lock/uvexport_test.go +++ b/pymage/internal/lock/uvexport_test.go @@ -107,6 +107,14 @@ build-backend = "hatchling.build" } } +func names(reqs []Requirement) map[string]bool { + m := map[string]bool{} + for _, r := range reqs { + m[NormalizeName(r.Name)] = true + } + return m +} + func writeFile(t *testing.T, path, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { diff --git a/pymage/internal/lock/uvlock.go b/pymage/internal/lock/uvlock.go index cc2894c..3748efd 100644 --- a/pymage/internal/lock/uvlock.go +++ b/pymage/internal/lock/uvlock.go @@ -3,34 +3,24 @@ package lock import ( "fmt" "os" - "path/filepath" "strings" "github.com/pelletier/go-toml/v2" ) +// uvLockFile is the subset of uv.lock we read: package name/version plus the +// wheel and sdist artifact references. Resolution (which packages to install +// for a target) is delegated to `uv export` — see uvexport.go — so we do not +// model the dependency graph, extras, groups, or markers here. type uvLockFile struct { Package []uvPackage `toml:"package"` } type uvPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` - Source map[string]any `toml:"source"` - Sdist uvArtifact `toml:"sdist"` - Wheels []uvArtifact `toml:"wheels"` - Dependencies []uvDep `toml:"dependencies"` - OptionalDeps map[string][]uvDep `toml:"optional-dependencies"` - DevDeps map[string][]uvDep `toml:"dev-dependencies"` -} - -// uvDep is a dependency edge in uv.lock. Extras pull a package's -// optional-dependencies; dev-dependencies groups are never followed. A marker -// gates the edge on the target environment. -type uvDep struct { - Name string `toml:"name"` - Extras []string `toml:"extra"` - Marker string `toml:"marker"` + Name string `toml:"name"` + Version string `toml:"version"` + Sdist uvArtifact `toml:"sdist"` + Wheels []uvArtifact `toml:"wheels"` } type uvArtifact struct { @@ -38,18 +28,6 @@ type uvArtifact struct { Hash string `toml:"hash"` } -// UVOptions selects which part of a uv.lock to install. -type UVOptions struct { - // Package roots the closure at a single workspace member (by name). Empty - // unions the closures of all local (project/workspace) packages. - Package string - // Extras enables the root project's own optional-dependency groups. - Extras []string - // Env, when non-nil, evaluates dependency markers for the target so - // platform/python-gated deps are included only when they apply. - Env MarkerEnv -} - // parseUVLock reads and unmarshals a uv.lock file. func parseUVLock(path string) (*uvLockFile, error) { data, err := os.ReadFile(path) @@ -63,169 +41,6 @@ func parseUVLock(path string) (*uvLockFile, error) { return &lf, nil } -// ParseUVLockFile reads a uv.lock and returns its full runtime closure with -// default options (all local roots, no extras, no marker filtering). Prefer -// Load + Resolve to pass a package/extras/target. -func ParseUVLockFile(path string) ([]Requirement, error) { - lf, err := parseUVLock(path) - if err != nil { - return nil, err - } - return resolveUV(lf, UVOptions{}) -} - -// resolveUV returns the project's runtime dependency closure as pinned -// packages with wheel URLs/hashes (and an sdist fallback when present). -// -// Only packages reachable from the local (project/workspace) packages' runtime -// dependencies — expanding requested extras via optional-dependencies and -// honoring markers — are included. Dev-dependency groups are excluded (like -// `uv sync --no-dev`). Local packages are skipped; app code comes from source. -func resolveUV(lf *uvLockFile, opts UVOptions) ([]Requirement, error) { - include, err := runtimeClosure(lf.Package, opts) - if err != nil { - return nil, err - } - - var reqs []Requirement - for _, pkg := range lf.Package { - if isLocalSource(pkg.Source) { - continue - } - if include != nil && !include[NormalizeName(pkg.Name)] { - continue - } - - req := Requirement{Name: pkg.Name, Version: pkg.Version} - seen := map[string]bool{} - for _, w := range pkg.Wheels { - sum, err := parseUVHash(w.Hash) - if err != nil { - return nil, fmt.Errorf("uv.lock: %s==%s: %w", pkg.Name, pkg.Version, err) - } - if w.URL == "" { - return nil, fmt.Errorf("uv.lock: %s==%s: wheel missing url", pkg.Name, pkg.Version) - } - if seen[sum] { - continue - } - seen[sum] = true - req.Hashes = append(req.Hashes, sum) - req.Wheels = append(req.Wheels, WheelRef{URL: w.URL, SHA256: sum, Filename: filepath.Base(w.URL)}) - } - - // An sdist provides a wheel-build fallback when no wheel is compatible. - if pkg.Sdist.URL != "" { - sum, err := parseUVHash(pkg.Sdist.Hash) - if err != nil { - return nil, fmt.Errorf("uv.lock: %s==%s sdist: %w", pkg.Name, pkg.Version, err) - } - req.Sdist = &SdistRef{URL: pkg.Sdist.URL, SHA256: sum, Filename: filepath.Base(pkg.Sdist.URL)} - if !seen[sum] { - req.Hashes = append(req.Hashes, sum) // counts as "hashed" - } - } - - if len(req.Wheels) == 0 && req.Sdist == nil { - return nil, fmt.Errorf("uv.lock: %s==%s has neither wheels nor an sdist", pkg.Name, pkg.Version) - } - reqs = append(reqs, req) - } - - if len(reqs) == 0 { - return nil, fmt.Errorf("uv.lock: no installable packages (closure empty)") - } - return reqs, nil -} - -// runtimeClosure returns the set of (normalized) package names reachable from -// the selected local packages' runtime dependencies, expanding extras and -// honoring markers. It returns nil when the lock has no local/project package -// (e.g. a bare requirements lock), signaling "install everything". -func runtimeClosure(pkgs []uvPackage, opts UVOptions) (map[string]bool, error) { - byName := make(map[string]*uvPackage, len(pkgs)) - var localRoots []*uvPackage - for i := range pkgs { - p := &pkgs[i] - byName[NormalizeName(p.Name)] = p - if isLocalSource(p.Source) { - localRoots = append(localRoots, p) - } - } - - var roots []*uvPackage - if opts.Package != "" { - p := byName[NormalizeName(opts.Package)] - if p == nil { - return nil, fmt.Errorf("uv.lock: --package %q not found in lock", opts.Package) - } - roots = []*uvPackage{p} - } else { - roots = localRoots - } - if len(roots) == 0 { - return nil, nil - } - - type item struct { - name string - extras []string - } - var queue []item - enqueue := func(deps []uvDep) { - for _, d := range deps { - if opts.Env != nil && !EvalMarker(d.Marker, opts.Env) { - continue - } - queue = append(queue, item{NormalizeName(d.Name), d.Extras}) - } - } - for _, r := range roots { - enqueue(r.Dependencies) - // Enable the root project's own optional-dependency groups (--extra). - for _, ex := range opts.Extras { - enqueue(r.OptionalDeps[ex]) - } - } - - include := map[string]bool{} - visited := map[string]bool{} - for len(queue) > 0 { - it := queue[0] - queue = queue[1:] - key := it.name + "|" + strings.Join(it.extras, ",") - if visited[key] { - continue - } - visited[key] = true - - p := byName[it.name] - if p == nil { - continue - } - if !isLocalSource(p.Source) { - include[it.name] = true - } - enqueue(p.Dependencies) - for _, ex := range it.extras { - enqueue(p.OptionalDeps[ex]) - } - } - return include, nil -} - -func isLocalSource(src map[string]any) bool { - if src == nil { - return false - } - for _, k := range []string{"virtual", "editable", "workspace", "directory", "git"} { - if _, ok := src[k]; ok { - return true - } - } - return false -} - func parseUVHash(h string) (string, error) { h = strings.TrimSpace(h) if h == "" { diff --git a/pymage/internal/lock/uvlock_test.go b/pymage/internal/lock/uvlock_test.go deleted file mode 100644 index 862f22e..0000000 --- a/pymage/internal/lock/uvlock_test.go +++ /dev/null @@ -1,272 +0,0 @@ -package lock - -import ( - "os" - "path/filepath" - "testing" -) - -// uvLockFixture exercises: a virtual + an editable local package (both -// skipped), the runtime closure of the editable project (alpha, its transitive -// beta, and extradep pulled by the requested alpha[x] extra), a multi-wheel -// package (native), and a dev-only package (linter) that must be excluded. -const uvLockFixture = `version = 1 -requires-python = ">=3.14" - -[[package]] -name = "virtual-pkg" -version = "0.1.0" -source = { virtual = "." } - -[[package]] -name = "app" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "alpha", extra = ["x"] }, - { name = "native" }, -] - -[package.dev-dependencies] -dev = [ - { name = "linter" }, -] - -[[package]] -name = "alpha" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beta" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a/alpha-1.0.0-py3-none-any.whl", hash = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, -] - -[package.optional-dependencies] -x = [ - { name = "extradep" }, -] - -[[package]] -name = "beta" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://example.com/beta-1.0.0-py3-none-any.whl", hash = "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" }, -] - -[[package]] -name = "extradep" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://example.com/extradep-1.0.0-py3-none-any.whl", hash = "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, -] - -[[package]] -name = "native" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://example.com/native-2.0.0-py3-none-any.whl", hash = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, - { url = "https://example.com/native-2.0.0-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" }, -] - -[[package]] -name = "linter" -version = "9.9.9" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://example.com/linter-9.9.9-py3-none-any.whl", hash = "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" }, -] -` - -func TestParseUVLockFile(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "uv.lock") - if err := os.WriteFile(path, []byte(uvLockFixture), 0o644); err != nil { - t.Fatal(err) - } - reqs, err := ParseUVLockFile(path) - if err != nil { - t.Fatal(err) - } - - got := map[string]Requirement{} - for _, r := range reqs { - got[r.Name] = r - } - // Runtime closure: alpha, its transitive beta, extradep (alpha[x]), native. - for _, want := range []string{"alpha", "beta", "extradep", "native"} { - if _, ok := got[want]; !ok { - t.Errorf("missing runtime dep %q (got %v)", want, keys(got)) - } - } - // Local packages skipped; dev-only package excluded. - for _, bad := range []string{"app", "virtual-pkg", "linter"} { - if _, ok := got[bad]; ok { - t.Errorf("%q should not be installed", bad) - } - } - if len(reqs) != 4 { - t.Fatalf("got %d reqs, want 4: %v", len(reqs), keys(got)) - } - if len(got["native"].Wheels) != 2 { - t.Errorf("native should keep both wheels, got %d", len(got["native"].Wheels)) - } - if got["alpha"].Wheels[0].Filename != "alpha-1.0.0-py3-none-any.whl" { - t.Errorf("alpha filename = %q", got["alpha"].Wheels[0].Filename) - } -} - -func TestParseAnyPrefersUVLock(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "uv.lock"), []byte(uvLockFixture), 0o644); err != nil { - t.Fatal(err) - } - reqs, err := ParseAny(filepath.Join(dir, "uv.lock")) - if err != nil { - t.Fatal(err) - } - if len(reqs) != 4 { - t.Fatalf("got %d, want 4", len(reqs)) - } -} - -func names(reqs []Requirement) map[string]bool { - m := map[string]bool{} - for _, r := range reqs { - m[NormalizeName(r.Name)] = true - } - return m -} - -// closureLock exercises extras, workspace --package selection, and markers. -const closureLock = `version = 1 - -[[package]] -name = "svc" -version = "0.1.0" -source = { editable = "members/svc" } -dependencies = [ - { name = "core" }, - { name = "winonly", marker = "sys_platform == 'win32'" }, -] - -[package.optional-dependencies] -gpu = [ - { name = "accel" }, -] - -[[package]] -name = "worker" -version = "0.1.0" -source = { editable = "members/worker" } -dependencies = [ - { name = "worktool" }, -] - -[[package]] -name = "core" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ { url = "https://e/core-1.0.0-py3-none-any.whl", hash = "sha256:1111111111111111111111111111111111111111111111111111111111111111" } ] - -[[package]] -name = "accel" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ { url = "https://e/accel-1.0.0-py3-none-any.whl", hash = "sha256:2222222222222222222222222222222222222222222222222222222222222222" } ] - -[[package]] -name = "winonly" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ { url = "https://e/winonly-1.0.0-py3-none-any.whl", hash = "sha256:3333333333333333333333333333333333333333333333333333333333333333" } ] - -[[package]] -name = "worktool" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ { url = "https://e/worktool-1.0.0-py3-none-any.whl", hash = "sha256:4444444444444444444444444444444444444444444444444444444444444444" } ] -` - -func loadClosure(t *testing.T) *Lock { - t.Helper() - dir := t.TempDir() - p := filepath.Join(dir, "uv.lock") - if err := os.WriteFile(p, []byte(closureLock), 0o644); err != nil { - t.Fatal(err) - } - lk, err := Load(p) - if err != nil { - t.Fatal(err) - } - return lk -} - -func TestUVClosureMarkers(t *testing.T) { - lk := loadClosure(t) - // Linux target: winonly (sys_platform == win32) is excluded. - reqs, err := lk.Resolve(Options{OS: "linux", Arch: "amd64", PyMajor: 3, PyMinor: 12}) - if err != nil { - t.Fatal(err) - } - got := names(reqs) - if !got["core"] || !got["worktool"] { - t.Fatalf("missing runtime deps: %v", got) - } - if got["winonly"] { - t.Error("winonly should be excluded on linux (marker)") - } - if got["accel"] { - t.Error("accel should be excluded without --extra gpu") - } -} - -func TestUVClosureExtras(t *testing.T) { - lk := loadClosure(t) - reqs, err := lk.Resolve(Options{Extras: []string{"gpu"}, OS: "linux", Arch: "amd64", PyMajor: 3, PyMinor: 12}) - if err != nil { - t.Fatal(err) - } - if !names(reqs)["accel"] { - t.Errorf("--extra gpu should include accel: %v", names(reqs)) - } -} - -func TestUVClosurePackage(t *testing.T) { - lk := loadClosure(t) - // --package svc: only svc's closure (core), not worker's worktool. - reqs, err := lk.Resolve(Options{Package: "svc", OS: "linux", Arch: "amd64", PyMajor: 3, PyMinor: 12}) - if err != nil { - t.Fatal(err) - } - got := names(reqs) - if !got["core"] || got["worktool"] { - t.Errorf("--package svc closure = %v, want {core} (no worktool)", got) - } - - // --package worker: only worktool. - reqs, err = lk.Resolve(Options{Package: "worker", OS: "linux", Arch: "amd64", PyMajor: 3, PyMinor: 12}) - if err != nil { - t.Fatal(err) - } - got = names(reqs) - if !got["worktool"] || got["core"] { - t.Errorf("--package worker closure = %v, want {worktool}", got) - } - - if _, err := lk.Resolve(Options{Package: "nope"}); err == nil { - t.Error("expected error for unknown --package") - } -} - -func keys(m map[string]Requirement) []string { - out := make([]string, 0, len(m)) - for k := range m { - out = append(out, k) - } - return out -}