push: set explicit, tunable concurrent layer uploads

go-containerregistry already uploads layers (and index children) concurrently,
but only at its default of 4. Set remote.WithJobs explicitly with an auto
default that scales with CPU count (floor 4), plus a --push-concurrency flag and
[tool.pymage] push-concurrency config to tune it.

Co-authored-by: Jason Hall <imjasonh@users.noreply.github.com>

pymage/README.md

@@ -71,6 +71,7 @@ the config value, which overrides the built-in default.
 | `layer-strategy` | `--layer-strategy` | `auto` |
 | `max-layers` | `--max-layers` | `127` |
 | `max-wheel-layers` | `--max-wheel-layers` | *(derived from `max-layers`)* |
+| `push-concurrency` | `--push-concurrency` | auto (≥ 4, scales with CPUs) |
 | `python` | `--python` | auto-detected from the base |
 | `prefix` | `--prefix` | `/app/.venv` |
 | `workdir` | `--workdir` | `/app` |
@@ -174,6 +175,7 @@ base that advertises its version.
 | `--layer-strategy` | `auto` (default), `per-wheel`, or `single-deps-layer`. |
 | `--max-layers` | Cap on total image layers (base + deps + app) for `auto` (default 127). |
 | `--max-wheel-layers` | Cap the dependency layer count directly (overrides `--max-layers`). |
+| `--push-concurrency` | Max concurrent layer uploads when pushing (0 = auto). |
 | `--platform` | Target platform(s); selects compatible wheels and base. Repeatable / comma-separated (e.g. `linux/amd64,linux/arm64`) builds a multi-arch image index. Defaults to the platforms the base image supports. |
 | `--python` | Interpreter version, e.g. `python3.12`. Optional — **auto-detected from the base** when omitted; if set, must match the base. Drives wheel selection and the site-packages layout. |
 | `--cache-dir` | Content-addressed layer cache; reuses compressed layers and downloaded wheels across rebuilds. |

pymage/internal/cli/cli.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -59,6 +60,7 @@ type buildFlags struct {
 
 	maxLayers      int
 	maxWheelLayers int
+	pushJobs       int
 
 	push        bool
 	ociLayout   string
@@ -103,6 +105,7 @@ func buildCmd() *cobra.Command {
 	fs.StringVar(&f.strategy, "layer-strategy", string(build.Auto), "auto | per-wheel | single-deps-layer")
 	fs.IntVar(&f.maxLayers, "max-layers", build.DefaultMaxLayers, "max total image layers (base + deps + app) for the auto strategy")
 	fs.IntVar(&f.maxWheelLayers, "max-wheel-layers", 0, "cap the number of dependency layers directly (overrides --max-layers when > 0)")
+	fs.IntVar(&f.pushJobs, "push-concurrency", 0, "max concurrent layer uploads when pushing (0 = auto)")
 	fs.BoolVar(&f.push, "push", true, "push the image to --repo (by digest)")
 	fs.StringVar(&f.ociLayout, "oci-layout", "", "also write the image to this OCI layout directory")
 	fs.BoolVar(&f.printDigest, "print-digest", false, "print only the resulting image digest")
@@ -337,6 +340,8 @@ func output(cmd *cobra.Command, f *buildFlags, nameOpts []name.Option, img v1.Im
 	opts := []remote.Option{
 		remote.WithContext(ctx),
 		remote.WithAuthFromKeychain(authn.DefaultKeychain),
+		// Upload layers (and, for an index, child manifests) concurrently.
+		remote.WithJobs(pushConcurrency(f.pushJobs)),
 	}
 	// The artifact is pushed by content; each tag is just another pointer to
 	// the same digest.
@@ -388,6 +393,20 @@ func defaultPlatformsFromBase(ctx context.Context, f *buildFlags, nameOpts []nam
 	return build.BasePlatforms(ctx, f.base, authn.DefaultKeychain, nameOpts...)
 }
 
+// pushConcurrency resolves the number of concurrent layer uploads. 0 means
+// auto: scale with CPU count but never below go-containerregistry's default of
+// 4 (layer uploads are network-bound, so some parallelism always helps).
+func pushConcurrency(requested int) int {
+	if requested > 0 {
+		return requested
+	}
+	n := runtime.NumCPU()
+	if n < 4 {
+		n = 4
+	}
+	return n
+}
+
 // parsePlatforms parses the (comma-split, repeatable) --platform values.
 func parsePlatforms(specs []string) ([]v1.Platform, error) {
 	var out []v1.Platform

pymage/internal/cli/cli_test.go

@@ -423,6 +423,15 @@ func TestBuildStdoutIsImageRef(t *testing.T) {
 	}
 }
 
+func TestPushConcurrency(t *testing.T) {
+	if got := pushConcurrency(7); got != 7 {
+		t.Errorf("explicit value not honored: got %d, want 7", got)
+	}
+	if got := pushConcurrency(0); got < 4 {
+		t.Errorf("auto concurrency = %d, want >= 4", got)
+	}
+}
+
 func TestParsePythonTag(t *testing.T) {
 	maj, min, err := parsePythonTag("python3.12")
 	if err != nil || maj != 3 || min != 12 {

pymage/internal/cli/defaults.go

@@ -60,6 +60,9 @@ func applyDefaults(cmd *cobra.Command, f *buildFlags) error {
 	if !changed("max-wheel-layers") && cfg.MaxWheelLayers != 0 {
 		f.maxWheelLayers = cfg.MaxWheelLayers
 	}
+	if !changed("push-concurrency") && cfg.PushJobs != 0 {
+		f.pushJobs = cfg.PushJobs
+	}
 	if !changed("python") && cfg.Python != "" {
 		f.pythonTag = cfg.Python
 	}
@@ -134,6 +137,9 @@ func validateBuildFlags(f *buildFlags) error {
 	if f.maxWheelLayers < 0 {
 		return fmt.Errorf("--max-wheel-layers must be >= 0, got %d", f.maxWheelLayers)
 	}
+	if f.pushJobs < 0 {
+		return fmt.Errorf("--push-concurrency must be >= 0, got %d", f.pushJobs)
+	}
 	if len(f.entrypoint) == 0 {
 		return fmt.Errorf("no entrypoint: set [project.scripts] in pyproject.toml, add entrypoint to [tool.pymage], or pass --entrypoint")
 	}

pymage/internal/project/project.go

@@ -32,6 +32,7 @@ type Config struct {
 	LayerStrategy  string            `toml:"layer-strategy"`
 	MaxLayers      int               `toml:"max-layers"`
 	MaxWheelLayers int               `toml:"max-wheel-layers"`
+	PushJobs       int               `toml:"push-concurrency"`
 	Python         string            `toml:"python"`
 	Prefix         string            `toml:"prefix"`
 	Workdir        string            `toml:"workdir"`