lock: delegate uv.lock resolution to uv export (closure/extras/groups/workspace), evaluate emitted markers per target, attach URLs from lock; built-in walker kept as fallback when uv/pyproject absent

Co-authored-by: Jason Hall <imjasonh@users.noreply.github.com>

pymage/internal/lock/lock.go

@@ -46,6 +46,9 @@ type Requirement struct {
 	// Sdist, when set, is the source distribution to build a wheel from if no
 	// compatible wheel is available.
 	Sdist *SdistRef
+	// Marker is the PEP 508 environment marker for this requirement (the text
+	// after ";"), if any. Evaluated against the target at resolution time.
+	Marker string
 }
 
 var (
@@ -95,6 +98,7 @@ func ParseAny(path string) ([]Requirement, error) {
 // Lock is a parsed lock file (parsed once) that can be resolved per target.
 type Lock struct {
 	isUV bool
+	path string        // path to the lock file on disk
 	uv   *uvLockFile   // uv.lock
 	reqs []Requirement // requirements format
 }
@@ -117,29 +121,58 @@ func Load(path string) (*Lock, error) {
 		if err != nil {
 			return nil, err
 		}
-		return &Lock{isUV: true, uv: lf}, nil
+		return &Lock{isUV: true, path: path, uv: lf}, nil
 	}
 	reqs, err := ParseFile(path)
 	if err != nil {
 		return nil, err
 	}
-	return &Lock{reqs: reqs}, nil
+	return &Lock{path: path, reqs: reqs}, nil
 }
 
-// Resolve returns the pinned requirements for the given target. For uv.lock
-// this is the runtime closure (extras/package/markers applied); for a
-// requirements file it is the flat pinned list.
+// Resolve returns the pinned requirements for the given target.
+//
+// For a uv.lock, resolution (the runtime closure, extras, workspace package,
+// and group selection) is delegated to `uv export` when uv and a pyproject.toml
+// are available — uv is the source of truth, so we don't reimplement its
+// resolver. We then evaluate the per-requirement environment markers uv emits
+// for the target and attach wheel/sdist URLs from the lock. When uv isn't
+// available (e.g. air-gapped, or a bare uv.lock with no project), we fall back
+// to a built-in closure. For requirements-format locks the list is already flat
+// and pinned; we only evaluate any inline markers.
 func (l *Lock) Resolve(o Options) ([]Requirement, error) {
+	env := o.markerEnv()
 	if !l.isUV {
-		return append([]Requirement(nil), l.reqs...), nil
+		return filterByMarker(l.reqs, env), nil
 	}
-	var env MarkerEnv
-	if o.PyMajor != 0 {
-		env = NewMarkerEnv(o.OS, o.Arch, o.PyMajor, o.PyMinor)
+	if l.path != "" && uvExportUsable(l.path) {
+		return uvExportResolve(l.path, l.uv, o, env)
 	}
 	return resolveUV(l.uv, UVOptions{Package: o.Package, Extras: o.Extras, Env: env})
 }
 
+// markerEnv builds the marker environment for the target, or nil when no target
+// is given (meaning: don't filter on markers).
+func (o Options) markerEnv() MarkerEnv {
+	if o.PyMajor == 0 {
+		return nil
+	}
+	return NewMarkerEnv(o.OS, o.Arch, o.PyMajor, o.PyMinor)
+}
+
+// filterByMarker drops requirements whose marker is false for env. A nil env
+// keeps everything.
+func filterByMarker(reqs []Requirement, env MarkerEnv) []Requirement {
+	out := make([]Requirement, 0, len(reqs))
+	for _, r := range reqs {
+		if env != nil && !EvalMarker(r.Marker, env) {
+			continue
+		}
+		out = append(out, r)
+	}
+	return out
+}
+
 // DiscoverLock returns the path to a lock file in dir, preferring uv.lock.
 func DiscoverLock(dir string) (string, error) {
 	for _, name := range []string{"uv.lock", "requirements.lock", "requirements.txt"} {
@@ -181,7 +214,7 @@ func Parse(r io.Reader) ([]Requirement, error) {
 			// quietly omit a dependency from the image.
 			return fmt.Errorf("lock: unsupported requirement line %q (expected name==version)", strings.TrimSpace(stripped))
 		}
-		req := Requirement{Name: m[1], Version: m[2]}
+		req := Requirement{Name: m[1], Version: m[2], Marker: extractMarker(stripped)}
 		for _, h := range hashRE.FindAllStringSubmatch(stripped, -1) {
 			req.Hashes = append(req.Hashes, strings.ToLower(h[1]))
 		}
@@ -213,6 +246,20 @@ func Parse(r io.Reader) ([]Requirement, error) {
 	return reqs, nil
 }
 
+// extractMarker returns the PEP 508 marker on a (comment-stripped) requirement
+// line: the text after the first ";" and before any "--hash" entries.
+func extractMarker(line string) string {
+	i := strings.Index(line, ";")
+	if i < 0 {
+		return ""
+	}
+	rest := line[i+1:]
+	if h := strings.Index(rest, "--hash"); h >= 0 {
+		rest = rest[:h]
+	}
+	return strings.TrimSpace(rest)
+}
+
 func stripComment(s string) string {
 	// A '#' starts a comment. Requirements files don't support escaping '#',
 	// and no valid pinned/hashed line contains one, so we strip from the first

pymage/internal/lock/marker.go

@@ -202,7 +202,7 @@ func (p *markerParser) parseAtom() (bool, error) {
 }
 
 func (p *markerParser) parseComparison() (bool, error) {
-	l, lver, err := p.parseValue()
+	l, lver, lextra, err := p.parseValue()
 	if err != nil {
 		return false, err
 	}
@@ -210,10 +210,16 @@ func (p *markerParser) parseComparison() (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	r, rver, err := p.parseValue()
+	r, rver, rextra, err := p.parseValue()
 	if err != nil {
 		return false, err
 	}
+	// `extra` is resolved upstream (we ask uv/pip for the exact extras), so a
+	// clause referencing it is neutral here: treat it as satisfied rather than
+	// re-deciding extra membership.
+	if lextra || rextra {
+		return true, nil
+	}
 	return compareMarker(l, lver, op, r, rver), nil
 }
 
@@ -242,27 +248,31 @@ func (p *markerParser) parseOp() (string, error) {
 	return "", fmt.Errorf("expected operator, got %q", t.val)
 }
 
-// parseValue returns the value and whether it is a version-valued variable
-// (python_version / python_full_version / implementation_version).
-func (p *markerParser) parseValue() (string, bool, error) {
+// parseValue returns the value, whether it is a version-valued variable
+// (python_version / python_full_version / implementation_version), and whether
+// it is the `extra` variable (handled specially by the caller).
+func (p *markerParser) parseValue() (val string, isVersion, isExtra bool, err error) {
 	t, ok := p.peek()
 	if !ok {
-		return "", false, fmt.Errorf("expected value")
+		return "", false, false, fmt.Errorf("expected value")
 	}
 	p.pos++
 	switch t.kind {
 	case mtokString:
-		return t.val, false, nil
+		return t.val, false, false, nil
 	case mtokIdent:
+		if t.val == "extra" {
+			return "", false, true, nil
+		}
 		v, known := p.env[t.val]
 		if !known {
 			// Unknown variable: treat as empty, non-version.
-			return "", false, nil
+			return "", false, false, nil
 		}
 		isVer := t.val == "python_version" || t.val == "python_full_version" || t.val == "implementation_version"
-		return v, isVer, nil
+		return v, isVer, false, nil
 	default:
-		return "", false, fmt.Errorf("unexpected token %q", t.val)
+		return "", false, false, fmt.Errorf("unexpected token %q", t.val)
 	}
 }
 

pymage/internal/lock/marker_test.go

@@ -22,7 +22,9 @@ func TestEvalMarker(t *testing.T) {
 		{"python_version >= '3.10' and python_version < '3.13'", true},
 		{"python_version < '3.10' or sys_platform == 'linux'", true},
 		{"python_version < '3.10' or sys_platform == 'win32'", false},
-		{"extra == 'foo'", false}, // unknown var -> empty
+		{"extra == 'foo'", true},                              // extra is resolved upstream -> neutral
+		{"extra == 'foo' and sys_platform == 'win32'", false}, // extra neutral; platform decides
+		{"extra == 'foo' and sys_platform == 'linux'", true},
 		{"(sys_platform == 'win32') or (os_name == 'posix')", true},
 		{"platform_machine in 'x86_64 aarch64'", true}, // substring membership
 		{"'arm' not in platform_machine", true},

pymage/internal/lock/uvexport.go

@@ -0,0 +1,108 @@
+package lock
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// uvExportUsable reports whether we should resolve via `uv export`: uv must be
+// on PATH and the lock must sit next to a pyproject.toml (a real uv project, as
+// `uv export` requires). Bare/synthetic locks fall back to the built-in walker.
+func uvExportUsable(lockPath string) bool {
+	if _, err := exec.LookPath("uv"); err != nil {
+		return false
+	}
+	_, err := os.Stat(filepath.Join(filepath.Dir(lockPath), "pyproject.toml"))
+	return err == nil
+}
+
+// uvExportResolve delegates resolution to `uv export`, which produces the
+// project's universal runtime requirement set (closure, extras, groups, and
+// workspace selection all handled by uv) with per-requirement markers and
+// hashes. We then evaluate the markers for the target and attach wheel/sdist
+// URLs from the parsed uv.lock so the downloader can fetch them.
+//
+// `uv export --frozen` reads the existing lock; it does not re-resolve, run
+// build code, or hit the network.
+func uvExportResolve(lockPath string, lf *uvLockFile, o Options, env MarkerEnv) ([]Requirement, error) {
+	args := []string{
+		"export", "--frozen", "--no-dev",
+		"--no-emit-workspace", // omit the project/workspace members themselves
+		"--no-annotate", "--format", "requirements-txt",
+	}
+	if o.Package != "" {
+		args = append(args, "--package", o.Package)
+	}
+	for _, e := range o.Extras {
+		args = append(args, "--extra", e)
+	}
+
+	cmd := exec.Command("uv", args...)
+	cmd.Dir = filepath.Dir(lockPath)
+	var out, errb bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &out, &errb
+	if err := cmd.Run(); err != nil {
+		return nil, fmt.Errorf("uv export: %w: %s", err, strings.TrimSpace(errb.String()))
+	}
+
+	pins, err := Parse(&out)
+	if err != nil {
+		return nil, fmt.Errorf("parse uv export output: %w", err)
+	}
+	return mergeExportPins(pins, lf, env)
+}
+
+// mergeExportPins filters the exported pins by their markers and attaches
+// wheel/sdist artifacts from the lock. Split out from uvExportResolve so it can
+// be tested without invoking uv.
+func mergeExportPins(pins []Requirement, lf *uvLockFile, env MarkerEnv) ([]Requirement, error) {
+	byName := make(map[string]*uvPackage, len(lf.Package))
+	for i := range lf.Package {
+		byName[NormalizeName(lf.Package[i].Name)] = &lf.Package[i]
+	}
+
+	out := make([]Requirement, 0, len(pins))
+	for _, r := range pins {
+		if env != nil && !EvalMarker(r.Marker, env) {
+			continue
+		}
+		if p := byName[NormalizeName(r.Name)]; p != nil {
+			if err := attachArtifacts(&r, p); err != nil {
+				return nil, err
+			}
+		}
+		// If the package isn't in the lock (shouldn't happen with --frozen) we
+		// keep the pin with its export hashes; the wheelhouse will report a
+		// clear "no wheel" error rather than silently dropping it.
+		out = append(out, r)
+	}
+	return out, nil
+}
+
+// attachArtifacts copies the wheel URLs and sdist reference from a uv.lock
+// package onto a requirement resolved via uv export. Hashes already come from
+// the export output.
+func attachArtifacts(r *Requirement, p *uvPackage) error {
+	for _, w := range p.Wheels {
+		if w.URL == "" {
+			continue
+		}
+		sum, err := parseUVHash(w.Hash)
+		if err != nil {
+			return fmt.Errorf("uv.lock: %s==%s: %w", p.Name, p.Version, err)
+		}
+		r.Wheels = append(r.Wheels, WheelRef{URL: w.URL, SHA256: sum, Filename: filepath.Base(w.URL)})
+	}
+	if p.Sdist.URL != "" {
+		sum, err := parseUVHash(p.Sdist.Hash)
+		if err != nil {
+			return fmt.Errorf("uv.lock: %s==%s sdist: %w", p.Name, p.Version, err)
+		}
+		r.Sdist = &SdistRef{URL: p.Sdist.URL, SHA256: sum, Filename: filepath.Base(p.Sdist.URL)}
+	}
+	return nil
+}

pymage/internal/lock/uvexport_test.go

@@ -0,0 +1,118 @@
+package lock
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// hermetic: exercise the export post-processing (marker filtering + artifact
+// attach + extra neutralization) without invoking uv.
+func TestMergeExportPins(t *testing.T) {
+	// uv export-style output: universal pins with markers + hashes.
+	export := strings.Join([]string{
+		"core==1.0.0 \\",
+		"    --hash=sha256:1111111111111111111111111111111111111111111111111111111111111111",
+		"winonly==1.0.0 ; sys_platform == 'win32' \\",
+		"    --hash=sha256:3333333333333333333333333333333333333333333333333333333333333333",
+		"linonly==1.0.0 ; sys_platform == 'linux' \\",
+		"    --hash=sha256:4444444444444444444444444444444444444444444444444444444444444444",
+		"withextra==1.0.0 ; extra == 'gpu' and sys_platform == 'linux' \\",
+		"    --hash=sha256:5555555555555555555555555555555555555555555555555555555555555555",
+	}, "\n") + "\n"
+
+	pins, err := Parse(strings.NewReader(export))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lf := &uvLockFile{Package: []uvPackage{
+		{Name: "core", Version: "1.0.0", Wheels: []uvArtifact{{URL: "https://e/core-1.0.0-py3-none-any.whl", Hash: "sha256:1111111111111111111111111111111111111111111111111111111111111111"}}},
+		{Name: "linonly", Version: "1.0.0", Wheels: []uvArtifact{{URL: "https://e/linonly-1.0.0-py3-none-any.whl", Hash: "sha256:4444444444444444444444444444444444444444444444444444444444444444"}}},
+		{Name: "withextra", Version: "1.0.0", Wheels: []uvArtifact{{URL: "https://e/withextra-1.0.0-py3-none-any.whl", Hash: "sha256:5555555555555555555555555555555555555555555555555555555555555555"}}},
+	}}
+
+	env := NewMarkerEnv("linux", "amd64", 3, 12)
+	got, err := mergeExportPins(pins, lf, env)
+	if err != nil {
+		t.Fatal(err)
+	}
+	names := names(got)
+	if !names["core"] || !names["linonly"] {
+		t.Fatalf("missing expected runtime deps: %v", names)
+	}
+	if names["winonly"] {
+		t.Error("winonly should be filtered out on linux (sys_platform marker)")
+	}
+	if !names["withextra"] {
+		t.Error("withextra should be kept: extra clause is neutral, linux satisfies the rest")
+	}
+	// URLs attached from the lock.
+	for _, r := range got {
+		if NormalizeName(r.Name) == "core" && (len(r.Wheels) != 1 || r.Wheels[0].Filename != "core-1.0.0-py3-none-any.whl") {
+			t.Errorf("core wheels not attached from lock: %+v", r.Wheels)
+		}
+	}
+}
+
+// integration: when uv is available, drive a real `uv lock` + Resolve via
+// `uv export`. Skips without uv or network.
+func TestResolveViaUVExport(t *testing.T) {
+	if _, err := exec.LookPath("uv"); err != nil {
+		t.Skip("uv not installed")
+	}
+	dir := t.TempDir()
+	writeFile(t, filepath.Join(dir, "pyproject.toml"), `[project]
+name = "demo"
+version = "0.1.0"
+requires-python = ">=3.12,<3.13"
+dependencies = ["idna==3.10"]
+
+[dependency-groups]
+dev = ["iniconfig==2.0.0"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+`)
+	writeFile(t, filepath.Join(dir, "demo", "__init__.py"), "")
+	cmd := exec.Command("uv", "lock")
+	cmd.Dir = dir
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Skipf("uv lock failed (likely offline): %v\n%s", err, out)
+	}
+
+	lk, err := Load(filepath.Join(dir, "uv.lock"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	reqs, err := lk.Resolve(Options{OS: "linux", Arch: "amd64", PyMajor: 3, PyMinor: 12})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := names(reqs)
+	if !got["idna"] {
+		t.Errorf("runtime dep idna missing: %v", got)
+	}
+	if got["iniconfig"] {
+		t.Error("dev-group dep iniconfig should be excluded by --no-dev")
+	}
+	// uv.lock wheel URL should be attached for the downloader.
+	for _, r := range reqs {
+		if NormalizeName(r.Name) == "idna" && len(r.Wheels) == 0 {
+			t.Error("idna wheel URL not attached from uv.lock")
+		}
+	}
+}
+
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}