From 1df0e423c5b55c51afe6b423d303464b2089a0d8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 10 Jun 2026 23:55:43 +0000 Subject: [PATCH] push: set explicit, tunable concurrent layer uploads go-containerregistry already uploads layers (and index children) concurrently, but only at its default of 4. Set remote.WithJobs explicitly with an auto default that scales with CPU count (floor 4), plus a --push-concurrency flag and [tool.pymage] push-concurrency config to tune it. Co-authored-by: Jason Hall --- pymage/README.md | 2 ++ pymage/internal/cli/cli.go | 19 +++++++++++++++++++ pymage/internal/cli/cli_test.go | 9 +++++++++ pymage/internal/cli/defaults.go | 6 ++++++ pymage/internal/project/project.go | 1 + 5 files changed, 37 insertions(+) diff --git a/pymage/README.md b/pymage/README.md index d987d42..18d0c1a 100644 --- a/pymage/README.md +++ b/pymage/README.md @@ -71,6 +71,7 @@ the config value, which overrides the built-in default. | `layer-strategy` | `--layer-strategy` | `auto` | | `max-layers` | `--max-layers` | `127` | | `max-wheel-layers` | `--max-wheel-layers` | *(derived from `max-layers`)* | +| `push-concurrency` | `--push-concurrency` | auto (≥ 4, scales with CPUs) | | `python` | `--python` | auto-detected from the base | | `prefix` | `--prefix` | `/app/.venv` | | `workdir` | `--workdir` | `/app` | @@ -174,6 +175,7 @@ base that advertises its version. | `--layer-strategy` | `auto` (default), `per-wheel`, or `single-deps-layer`. | | `--max-layers` | Cap on total image layers (base + deps + app) for `auto` (default 127). | | `--max-wheel-layers` | Cap the dependency layer count directly (overrides `--max-layers`). | +| `--push-concurrency` | Max concurrent layer uploads when pushing (0 = auto). | | `--platform` | Target platform(s); selects compatible wheels and base. Repeatable / comma-separated (e.g. `linux/amd64,linux/arm64`) builds a multi-arch image index. Defaults to the platforms the base image supports. | | `--python` | Interpreter version, e.g. `python3.12`. Optional — **auto-detected from the base** when omitted; if set, must match the base. Drives wheel selection and the site-packages layout. | | `--cache-dir` | Content-addressed layer cache; reuses compressed layers and downloaded wheels across rebuilds. | diff --git a/pymage/internal/cli/cli.go b/pymage/internal/cli/cli.go index 13a8cf4..94a3b3f 100644 --- a/pymage/internal/cli/cli.go +++ b/pymage/internal/cli/cli.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "os" + "runtime" "strconv" "strings" @@ -59,6 +60,7 @@ type buildFlags struct { maxLayers int maxWheelLayers int + pushJobs int push bool ociLayout string @@ -103,6 +105,7 @@ func buildCmd() *cobra.Command { fs.StringVar(&f.strategy, "layer-strategy", string(build.Auto), "auto | per-wheel | single-deps-layer") fs.IntVar(&f.maxLayers, "max-layers", build.DefaultMaxLayers, "max total image layers (base + deps + app) for the auto strategy") fs.IntVar(&f.maxWheelLayers, "max-wheel-layers", 0, "cap the number of dependency layers directly (overrides --max-layers when > 0)") + fs.IntVar(&f.pushJobs, "push-concurrency", 0, "max concurrent layer uploads when pushing (0 = auto)") fs.BoolVar(&f.push, "push", true, "push the image to --repo (by digest)") fs.StringVar(&f.ociLayout, "oci-layout", "", "also write the image to this OCI layout directory") fs.BoolVar(&f.printDigest, "print-digest", false, "print only the resulting image digest") @@ -337,6 +340,8 @@ func output(cmd *cobra.Command, f *buildFlags, nameOpts []name.Option, img v1.Im opts := []remote.Option{ remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain), + // Upload layers (and, for an index, child manifests) concurrently. + remote.WithJobs(pushConcurrency(f.pushJobs)), } // The artifact is pushed by content; each tag is just another pointer to // the same digest. @@ -388,6 +393,20 @@ func defaultPlatformsFromBase(ctx context.Context, f *buildFlags, nameOpts []nam return build.BasePlatforms(ctx, f.base, authn.DefaultKeychain, nameOpts...) } +// pushConcurrency resolves the number of concurrent layer uploads. 0 means +// auto: scale with CPU count but never below go-containerregistry's default of +// 4 (layer uploads are network-bound, so some parallelism always helps). +func pushConcurrency(requested int) int { + if requested > 0 { + return requested + } + n := runtime.NumCPU() + if n < 4 { + n = 4 + } + return n +} + // parsePlatforms parses the (comma-split, repeatable) --platform values. func parsePlatforms(specs []string) ([]v1.Platform, error) { var out []v1.Platform diff --git a/pymage/internal/cli/cli_test.go b/pymage/internal/cli/cli_test.go index 1115eb6..ba69ec7 100644 --- a/pymage/internal/cli/cli_test.go +++ b/pymage/internal/cli/cli_test.go @@ -423,6 +423,15 @@ func TestBuildStdoutIsImageRef(t *testing.T) { } } +func TestPushConcurrency(t *testing.T) { + if got := pushConcurrency(7); got != 7 { + t.Errorf("explicit value not honored: got %d, want 7", got) + } + if got := pushConcurrency(0); got < 4 { + t.Errorf("auto concurrency = %d, want >= 4", got) + } +} + func TestParsePythonTag(t *testing.T) { maj, min, err := parsePythonTag("python3.12") if err != nil || maj != 3 || min != 12 { diff --git a/pymage/internal/cli/defaults.go b/pymage/internal/cli/defaults.go index b7a33f5..064b63e 100644 --- a/pymage/internal/cli/defaults.go +++ b/pymage/internal/cli/defaults.go @@ -60,6 +60,9 @@ func applyDefaults(cmd *cobra.Command, f *buildFlags) error { if !changed("max-wheel-layers") && cfg.MaxWheelLayers != 0 { f.maxWheelLayers = cfg.MaxWheelLayers } + if !changed("push-concurrency") && cfg.PushJobs != 0 { + f.pushJobs = cfg.PushJobs + } if !changed("python") && cfg.Python != "" { f.pythonTag = cfg.Python } @@ -134,6 +137,9 @@ func validateBuildFlags(f *buildFlags) error { if f.maxWheelLayers < 0 { return fmt.Errorf("--max-wheel-layers must be >= 0, got %d", f.maxWheelLayers) } + if f.pushJobs < 0 { + return fmt.Errorf("--push-concurrency must be >= 0, got %d", f.pushJobs) + } if len(f.entrypoint) == 0 { return fmt.Errorf("no entrypoint: set [project.scripts] in pyproject.toml, add entrypoint to [tool.pymage], or pass --entrypoint") } diff --git a/pymage/internal/project/project.go b/pymage/internal/project/project.go index 1e6c064..65e4c29 100644 --- a/pymage/internal/project/project.go +++ b/pymage/internal/project/project.go @@ -32,6 +32,7 @@ type Config struct { LayerStrategy string `toml:"layer-strategy"` MaxLayers int `toml:"max-layers"` MaxWheelLayers int `toml:"max-wheel-layers"` + PushJobs int `toml:"push-concurrency"` Python string `toml:"python"` Prefix string `toml:"prefix"` Workdir string `toml:"workdir"`