diff --git a/pymage/internal/lock/lock.go b/pymage/internal/lock/lock.go index f662306..7becb38 100644 --- a/pymage/internal/lock/lock.go +++ b/pymage/internal/lock/lock.go @@ -46,6 +46,9 @@ type Requirement struct { // Sdist, when set, is the source distribution to build a wheel from if no // compatible wheel is available. Sdist *SdistRef + // Marker is the PEP 508 environment marker for this requirement (the text + // after ";"), if any. Evaluated against the target at resolution time. + Marker string } var ( @@ -95,6 +98,7 @@ func ParseAny(path string) ([]Requirement, error) { // Lock is a parsed lock file (parsed once) that can be resolved per target. type Lock struct { isUV bool + path string // path to the lock file on disk uv *uvLockFile // uv.lock reqs []Requirement // requirements format } @@ -117,29 +121,58 @@ func Load(path string) (*Lock, error) { if err != nil { return nil, err } - return &Lock{isUV: true, uv: lf}, nil + return &Lock{isUV: true, path: path, uv: lf}, nil } reqs, err := ParseFile(path) if err != nil { return nil, err } - return &Lock{reqs: reqs}, nil + return &Lock{path: path, reqs: reqs}, nil } -// Resolve returns the pinned requirements for the given target. For uv.lock -// this is the runtime closure (extras/package/markers applied); for a -// requirements file it is the flat pinned list. +// Resolve returns the pinned requirements for the given target. +// +// For a uv.lock, resolution (the runtime closure, extras, workspace package, +// and group selection) is delegated to `uv export` when uv and a pyproject.toml +// are available — uv is the source of truth, so we don't reimplement its +// resolver. We then evaluate the per-requirement environment markers uv emits +// for the target and attach wheel/sdist URLs from the lock. When uv isn't +// available (e.g. air-gapped, or a bare uv.lock with no project), we fall back +// to a built-in closure. For requirements-format locks the list is already flat +// and pinned; we only evaluate any inline markers. func (l *Lock) Resolve(o Options) ([]Requirement, error) { + env := o.markerEnv() if !l.isUV { - return append([]Requirement(nil), l.reqs...), nil + return filterByMarker(l.reqs, env), nil } - var env MarkerEnv - if o.PyMajor != 0 { - env = NewMarkerEnv(o.OS, o.Arch, o.PyMajor, o.PyMinor) + if l.path != "" && uvExportUsable(l.path) { + return uvExportResolve(l.path, l.uv, o, env) } return resolveUV(l.uv, UVOptions{Package: o.Package, Extras: o.Extras, Env: env}) } +// markerEnv builds the marker environment for the target, or nil when no target +// is given (meaning: don't filter on markers). +func (o Options) markerEnv() MarkerEnv { + if o.PyMajor == 0 { + return nil + } + return NewMarkerEnv(o.OS, o.Arch, o.PyMajor, o.PyMinor) +} + +// filterByMarker drops requirements whose marker is false for env. A nil env +// keeps everything. +func filterByMarker(reqs []Requirement, env MarkerEnv) []Requirement { + out := make([]Requirement, 0, len(reqs)) + for _, r := range reqs { + if env != nil && !EvalMarker(r.Marker, env) { + continue + } + out = append(out, r) + } + return out +} + // DiscoverLock returns the path to a lock file in dir, preferring uv.lock. func DiscoverLock(dir string) (string, error) { for _, name := range []string{"uv.lock", "requirements.lock", "requirements.txt"} { @@ -181,7 +214,7 @@ func Parse(r io.Reader) ([]Requirement, error) { // quietly omit a dependency from the image. return fmt.Errorf("lock: unsupported requirement line %q (expected name==version)", strings.TrimSpace(stripped)) } - req := Requirement{Name: m[1], Version: m[2]} + req := Requirement{Name: m[1], Version: m[2], Marker: extractMarker(stripped)} for _, h := range hashRE.FindAllStringSubmatch(stripped, -1) { req.Hashes = append(req.Hashes, strings.ToLower(h[1])) } @@ -213,6 +246,20 @@ func Parse(r io.Reader) ([]Requirement, error) { return reqs, nil } +// extractMarker returns the PEP 508 marker on a (comment-stripped) requirement +// line: the text after the first ";" and before any "--hash" entries. +func extractMarker(line string) string { + i := strings.Index(line, ";") + if i < 0 { + return "" + } + rest := line[i+1:] + if h := strings.Index(rest, "--hash"); h >= 0 { + rest = rest[:h] + } + return strings.TrimSpace(rest) +} + func stripComment(s string) string { // A '#' starts a comment. Requirements files don't support escaping '#', // and no valid pinned/hashed line contains one, so we strip from the first diff --git a/pymage/internal/lock/marker.go b/pymage/internal/lock/marker.go index 6208106..8943b90 100644 --- a/pymage/internal/lock/marker.go +++ b/pymage/internal/lock/marker.go @@ -202,7 +202,7 @@ func (p *markerParser) parseAtom() (bool, error) { } func (p *markerParser) parseComparison() (bool, error) { - l, lver, err := p.parseValue() + l, lver, lextra, err := p.parseValue() if err != nil { return false, err } @@ -210,10 +210,16 @@ func (p *markerParser) parseComparison() (bool, error) { if err != nil { return false, err } - r, rver, err := p.parseValue() + r, rver, rextra, err := p.parseValue() if err != nil { return false, err } + // `extra` is resolved upstream (we ask uv/pip for the exact extras), so a + // clause referencing it is neutral here: treat it as satisfied rather than + // re-deciding extra membership. + if lextra || rextra { + return true, nil + } return compareMarker(l, lver, op, r, rver), nil } @@ -242,27 +248,31 @@ func (p *markerParser) parseOp() (string, error) { return "", fmt.Errorf("expected operator, got %q", t.val) } -// parseValue returns the value and whether it is a version-valued variable -// (python_version / python_full_version / implementation_version). -func (p *markerParser) parseValue() (string, bool, error) { +// parseValue returns the value, whether it is a version-valued variable +// (python_version / python_full_version / implementation_version), and whether +// it is the `extra` variable (handled specially by the caller). +func (p *markerParser) parseValue() (val string, isVersion, isExtra bool, err error) { t, ok := p.peek() if !ok { - return "", false, fmt.Errorf("expected value") + return "", false, false, fmt.Errorf("expected value") } p.pos++ switch t.kind { case mtokString: - return t.val, false, nil + return t.val, false, false, nil case mtokIdent: + if t.val == "extra" { + return "", false, true, nil + } v, known := p.env[t.val] if !known { // Unknown variable: treat as empty, non-version. - return "", false, nil + return "", false, false, nil } isVer := t.val == "python_version" || t.val == "python_full_version" || t.val == "implementation_version" - return v, isVer, nil + return v, isVer, false, nil default: - return "", false, fmt.Errorf("unexpected token %q", t.val) + return "", false, false, fmt.Errorf("unexpected token %q", t.val) } } diff --git a/pymage/internal/lock/marker_test.go b/pymage/internal/lock/marker_test.go index 78d521b..23bb391 100644 --- a/pymage/internal/lock/marker_test.go +++ b/pymage/internal/lock/marker_test.go @@ -22,7 +22,9 @@ func TestEvalMarker(t *testing.T) { {"python_version >= '3.10' and python_version < '3.13'", true}, {"python_version < '3.10' or sys_platform == 'linux'", true}, {"python_version < '3.10' or sys_platform == 'win32'", false}, - {"extra == 'foo'", false}, // unknown var -> empty + {"extra == 'foo'", true}, // extra is resolved upstream -> neutral + {"extra == 'foo' and sys_platform == 'win32'", false}, // extra neutral; platform decides + {"extra == 'foo' and sys_platform == 'linux'", true}, {"(sys_platform == 'win32') or (os_name == 'posix')", true}, {"platform_machine in 'x86_64 aarch64'", true}, // substring membership {"'arm' not in platform_machine", true}, diff --git a/pymage/internal/lock/uvexport.go b/pymage/internal/lock/uvexport.go new file mode 100644 index 0000000..a9771da --- /dev/null +++ b/pymage/internal/lock/uvexport.go @@ -0,0 +1,108 @@ +package lock + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// uvExportUsable reports whether we should resolve via `uv export`: uv must be +// on PATH and the lock must sit next to a pyproject.toml (a real uv project, as +// `uv export` requires). Bare/synthetic locks fall back to the built-in walker. +func uvExportUsable(lockPath string) bool { + if _, err := exec.LookPath("uv"); err != nil { + return false + } + _, err := os.Stat(filepath.Join(filepath.Dir(lockPath), "pyproject.toml")) + return err == nil +} + +// uvExportResolve delegates resolution to `uv export`, which produces the +// project's universal runtime requirement set (closure, extras, groups, and +// workspace selection all handled by uv) with per-requirement markers and +// hashes. We then evaluate the markers for the target and attach wheel/sdist +// URLs from the parsed uv.lock so the downloader can fetch them. +// +// `uv export --frozen` reads the existing lock; it does not re-resolve, run +// build code, or hit the network. +func uvExportResolve(lockPath string, lf *uvLockFile, o Options, env MarkerEnv) ([]Requirement, error) { + args := []string{ + "export", "--frozen", "--no-dev", + "--no-emit-workspace", // omit the project/workspace members themselves + "--no-annotate", "--format", "requirements-txt", + } + if o.Package != "" { + args = append(args, "--package", o.Package) + } + for _, e := range o.Extras { + args = append(args, "--extra", e) + } + + cmd := exec.Command("uv", args...) + cmd.Dir = filepath.Dir(lockPath) + var out, errb bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errb + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("uv export: %w: %s", err, strings.TrimSpace(errb.String())) + } + + pins, err := Parse(&out) + if err != nil { + return nil, fmt.Errorf("parse uv export output: %w", err) + } + return mergeExportPins(pins, lf, env) +} + +// mergeExportPins filters the exported pins by their markers and attaches +// wheel/sdist artifacts from the lock. Split out from uvExportResolve so it can +// be tested without invoking uv. +func mergeExportPins(pins []Requirement, lf *uvLockFile, env MarkerEnv) ([]Requirement, error) { + byName := make(map[string]*uvPackage, len(lf.Package)) + for i := range lf.Package { + byName[NormalizeName(lf.Package[i].Name)] = &lf.Package[i] + } + + out := make([]Requirement, 0, len(pins)) + for _, r := range pins { + if env != nil && !EvalMarker(r.Marker, env) { + continue + } + if p := byName[NormalizeName(r.Name)]; p != nil { + if err := attachArtifacts(&r, p); err != nil { + return nil, err + } + } + // If the package isn't in the lock (shouldn't happen with --frozen) we + // keep the pin with its export hashes; the wheelhouse will report a + // clear "no wheel" error rather than silently dropping it. + out = append(out, r) + } + return out, nil +} + +// attachArtifacts copies the wheel URLs and sdist reference from a uv.lock +// package onto a requirement resolved via uv export. Hashes already come from +// the export output. +func attachArtifacts(r *Requirement, p *uvPackage) error { + for _, w := range p.Wheels { + if w.URL == "" { + continue + } + sum, err := parseUVHash(w.Hash) + if err != nil { + return fmt.Errorf("uv.lock: %s==%s: %w", p.Name, p.Version, err) + } + r.Wheels = append(r.Wheels, WheelRef{URL: w.URL, SHA256: sum, Filename: filepath.Base(w.URL)}) + } + if p.Sdist.URL != "" { + sum, err := parseUVHash(p.Sdist.Hash) + if err != nil { + return fmt.Errorf("uv.lock: %s==%s sdist: %w", p.Name, p.Version, err) + } + r.Sdist = &SdistRef{URL: p.Sdist.URL, SHA256: sum, Filename: filepath.Base(p.Sdist.URL)} + } + return nil +} diff --git a/pymage/internal/lock/uvexport_test.go b/pymage/internal/lock/uvexport_test.go new file mode 100644 index 0000000..981c782 --- /dev/null +++ b/pymage/internal/lock/uvexport_test.go @@ -0,0 +1,118 @@ +package lock + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// hermetic: exercise the export post-processing (marker filtering + artifact +// attach + extra neutralization) without invoking uv. +func TestMergeExportPins(t *testing.T) { + // uv export-style output: universal pins with markers + hashes. + export := strings.Join([]string{ + "core==1.0.0 \\", + " --hash=sha256:1111111111111111111111111111111111111111111111111111111111111111", + "winonly==1.0.0 ; sys_platform == 'win32' \\", + " --hash=sha256:3333333333333333333333333333333333333333333333333333333333333333", + "linonly==1.0.0 ; sys_platform == 'linux' \\", + " --hash=sha256:4444444444444444444444444444444444444444444444444444444444444444", + "withextra==1.0.0 ; extra == 'gpu' and sys_platform == 'linux' \\", + " --hash=sha256:5555555555555555555555555555555555555555555555555555555555555555", + }, "\n") + "\n" + + pins, err := Parse(strings.NewReader(export)) + if err != nil { + t.Fatal(err) + } + + lf := &uvLockFile{Package: []uvPackage{ + {Name: "core", Version: "1.0.0", Wheels: []uvArtifact{{URL: "https://e/core-1.0.0-py3-none-any.whl", Hash: "sha256:1111111111111111111111111111111111111111111111111111111111111111"}}}, + {Name: "linonly", Version: "1.0.0", Wheels: []uvArtifact{{URL: "https://e/linonly-1.0.0-py3-none-any.whl", Hash: "sha256:4444444444444444444444444444444444444444444444444444444444444444"}}}, + {Name: "withextra", Version: "1.0.0", Wheels: []uvArtifact{{URL: "https://e/withextra-1.0.0-py3-none-any.whl", Hash: "sha256:5555555555555555555555555555555555555555555555555555555555555555"}}}, + }} + + env := NewMarkerEnv("linux", "amd64", 3, 12) + got, err := mergeExportPins(pins, lf, env) + if err != nil { + t.Fatal(err) + } + names := names(got) + if !names["core"] || !names["linonly"] { + t.Fatalf("missing expected runtime deps: %v", names) + } + if names["winonly"] { + t.Error("winonly should be filtered out on linux (sys_platform marker)") + } + if !names["withextra"] { + t.Error("withextra should be kept: extra clause is neutral, linux satisfies the rest") + } + // URLs attached from the lock. + for _, r := range got { + if NormalizeName(r.Name) == "core" && (len(r.Wheels) != 1 || r.Wheels[0].Filename != "core-1.0.0-py3-none-any.whl") { + t.Errorf("core wheels not attached from lock: %+v", r.Wheels) + } + } +} + +// integration: when uv is available, drive a real `uv lock` + Resolve via +// `uv export`. Skips without uv or network. +func TestResolveViaUVExport(t *testing.T) { + if _, err := exec.LookPath("uv"); err != nil { + t.Skip("uv not installed") + } + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "pyproject.toml"), `[project] +name = "demo" +version = "0.1.0" +requires-python = ">=3.12,<3.13" +dependencies = ["idna==3.10"] + +[dependency-groups] +dev = ["iniconfig==2.0.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" +`) + writeFile(t, filepath.Join(dir, "demo", "__init__.py"), "") + cmd := exec.Command("uv", "lock") + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Skipf("uv lock failed (likely offline): %v\n%s", err, out) + } + + lk, err := Load(filepath.Join(dir, "uv.lock")) + if err != nil { + t.Fatal(err) + } + reqs, err := lk.Resolve(Options{OS: "linux", Arch: "amd64", PyMajor: 3, PyMinor: 12}) + if err != nil { + t.Fatal(err) + } + got := names(reqs) + if !got["idna"] { + t.Errorf("runtime dep idna missing: %v", got) + } + if got["iniconfig"] { + t.Error("dev-group dep iniconfig should be excluded by --no-dev") + } + // uv.lock wheel URL should be attached for the downloader. + for _, r := range reqs { + if NormalizeName(r.Name) == "idna" && len(r.Wheels) == 0 { + t.Error("idna wheel URL not attached from uv.lock") + } + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +}