imjasonh/kontain.me
1
0
Fork 0
mirror of https://github.com/imjasonh/kontain.me synced 2026-07-16 20:34:22 +00:00
Code Activity

git: stream worktree into the layer instead of unpacking to disk

Browse source 
git.kontain.me used to clone+checkout a repo onto Cloud Run's tmpfs (memory)
and then tar the result. Now it fetches the packfile into an in-memory git dir,
walks the requested commit's tree, and streams each blob straight into the
layer tar. Only the packfile bytes (plus go-git's delta cache) are buffered, so
memory no longer scales with the size of the checked-out worktree.

The shipped .git directory is preserved (with core.bare unset and a generated
index) so the git CLI still works inside the checkout. Object SHAs change since
the tar is rebuilt differently, but the contents are the same.

https://claude.ai/code/session_01Jfhuuwb5haNaasc9hs53jw
This commit is contained in:
Claude 2026-05-12 14:54:11 +00:00
parent 0b1057459f
commit 61e2ace4bc
No known key found for this signature in database
3 changed files with 393 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

165
cmd/git/build_test.go Normal file
View file

@ -0,0 +1,165 @@
package main
import (
"archive/tar"
"context"
"crypto/sha256"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strings"
"testing"
"github.com/go-git/go-git/v5"
"github.com/google/go-containerregistry/pkg/v1/validate"
)
const (
testRepoURL = "https://github.com/octocat/Hello-World"
testRepoName = "Hello-World"
testRef = "master"
)
// layerFiles builds the git layer for the test repo and returns a map of
// "<perm>:<sha256>" (or "symlink:<target>") keyed by tar entry name.
func layerFiles(t *testing.T, prefix string) map[string]string {
t.Helper()
ctx := context.Background()
hash, refName, err := resolveRef(ctx, testRepoURL, testRef)
if err != nil {
t.Fatalf("resolveRef: %v", err)
}
layer, err := gitLayer(ctx, testRepoURL, refName, hash, prefix)
if err != nil {
t.Fatalf("gitLayer: %v", err)
}
if err := validate.Layer(layer); err != nil {
t.Fatalf("validate.Layer: %v", err)
}
rc, err := layer.Uncompressed()
if err != nil {
t.Fatalf("Uncompressed: %v", err)
}
defer rc.Close()
files := map[string]string{}
tr := tar.NewReader(rc)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("tar.Next: %v", err)
}
switch hdr.Typeflag {
case tar.TypeSymlink:
files[hdr.Name] = "symlink:" + hdr.Linkname
case tar.TypeReg:
h := sha256.New()
if _, err := io.Copy(h, tr); err != nil {
t.Fatalf("read %s: %v", hdr.Name, err)
}
files[hdr.Name] = fmt.Sprintf("%o:%x", hdr.Mode&0777, h.Sum(nil))
}
}
return files
}
// clonedWorktreeFiles does a real shallow clone and returns the same kind of
// map for the worktree files (everything outside .git).
func clonedWorktreeFiles(t *testing.T, prefix string) map[string]string {
t.Helper()
checkout := filepath.Join(t.TempDir(), testRepoName)
if _, err := git.PlainCloneContext(context.Background(), checkout, false, &git.CloneOptions{
URL: testRepoURL,
ReferenceName: "refs/heads/" + testRef,
SingleBranch: true,
Depth: 1,
Tags: git.NoTags,
}); err != nil {
t.Fatalf("clone: %v", err)
}
files := map[string]string{}
if err := filepath.Walk(checkout, func(p string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(checkout, p)
if err != nil {
return err
}
if rel == "." || rel == ".git" || strings.HasPrefix(rel, ".git"+string(os.PathSeparator)) {
return nil
}
name := path.Join(prefix, filepath.ToSlash(rel))
switch {
case fi.IsDir():
case fi.Mode()&os.ModeSymlink != 0:
target, err := os.Readlink(p)
if err != nil {
return err
}
files[name] = "symlink:" + target
case fi.Mode().IsRegular():
b, err := os.ReadFile(p)
if err != nil {
return err
}
files[name] = fmt.Sprintf("%o:%x", fi.Mode().Perm(), sha256.Sum256(b))
}
return nil
}); err != nil {
t.Fatalf("walk: %v", err)
}
return files
}
func TestGitLayer(t *testing.T) {
if testing.Short() {
t.Skip("clones from github")
}
prefix := path.Join("git", testRepoName)
got := layerFiles(t, prefix)
// The .git directory is shipped so the git CLI works inside the checkout.
for _, want := range []string{"HEAD", "config", "index"} {
if _, ok := got[path.Join(prefix, ".git", want)]; !ok {
t.Errorf("layer missing %s", path.Join(prefix, ".git", want))
}
}
hasPack := false
for name := range got {
if strings.HasPrefix(name, path.Join(prefix, ".git", "objects", "pack")+"/") && strings.HasSuffix(name, ".pack") {
hasPack = true
}
}
if !hasPack {
t.Errorf("layer has no packfile under %s/.git/objects/pack/", prefix)
}
// Worktree files must match a real checkout exactly.
gotWT := map[string]string{}
for name, v := range got {
if !strings.HasPrefix(name, path.Join(prefix, ".git")+"/") {
gotWT[name] = v
}
}
want := clonedWorktreeFiles(t, prefix)
if len(gotWT) == 0 || len(want) == 0 {
t.Fatalf("no worktree files: layer=%d clone=%d", len(gotWT), len(want))
}
for name, w := range want {
if g, ok := gotWT[name]; !ok {
t.Errorf("missing %s in layer", name)
} else if g != w {
t.Errorf("%s: layer=%s clone=%s", name, g, w)
}
}
for name := range gotWT {
if _, ok := want[name]; !ok {
t.Errorf("unexpected %s in layer", name)
}
}
}

317
cmd/git/main.go
View file

@ -17,9 +17,17 @@ import (
"time"
"github.com/chainguard-dev/clog/gcp"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
billyutil "github.com/go-git/go-billy/v5/util"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/format/index"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/storage/filesystem"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
@ -123,15 +131,7 @@ func (s *server) serveGitManifest(w http.ResponseWriter, r *http.Request) {
return
}
dir, err := os.MkdirTemp("", "git-*")
if err != nil {
slog.ErrorContext(ctx, "MkdirTemp", "err", err)
serve.Error(w, err)
return
}
defer os.RemoveAll(dir)
img, err := build(ctx, dir, cloneURL, repoName, refName, hash)
img, err := build(ctx, cloneURL, repoName, refName, hash)
if err != nil {
slog.ErrorContext(ctx, "build", "url", cloneURL, "ref", ref, "err", err)
serve.Error(w, err)
@ -196,47 +196,13 @@ func resolveRef(ctx context.Context, cloneURL, ref string) (plumbing.Hash, plumb
// baseRef contains git and busybox
var baseRef = name.MustParseReference("cgr.dev/chainguard/git:latest-dev")
// build clones cloneURL into dir/repoName and produces a single-layer image
// containing that checkout at /git/<repoName>.
func build(ctx context.Context, dir, cloneURL, repoName string, refName plumbing.ReferenceName, hash plumbing.Hash) (v1.Image, error) {
checkout := filepath.Join(dir, repoName)
if refName == "" {
// A bare commit SHA: full clone, then check the commit out.
repo, err := git.PlainCloneContext(ctx, checkout, false, &git.CloneOptions{
URL: cloneURL,
Tags: git.NoTags,
})
if err != nil {
return nil, fmt.Errorf("clone %q: %w", cloneURL, err)
}
wt, err := repo.Worktree()
if err != nil {
return nil, err
}
if err := wt.Checkout(&git.CheckoutOptions{Hash: hash}); err != nil {
return nil, fmt.Errorf("checkout %s: %w", hash, err)
}
} else {
// A branch or tag: shallow, single-branch clone.
refn := refName
if refn == plumbing.HEAD {
refn = "" // let go-git pick the default branch
}
if _, err := git.PlainCloneContext(ctx, checkout, false, &git.CloneOptions{
URL: cloneURL,
ReferenceName: refn,
SingleBranch: true,
Depth: 1,
Tags: git.NoTags,
}); err != nil {
return nil, fmt.Errorf("clone %q at %q: %w", cloneURL, refName, err)
}
}
layer, err := tarball.LayerFromOpener(tarDirOpener(checkout, path.Join("git", repoName)))
// build produces a single-layer image containing a checkout of the requested
// commit at /git/<repoName>, alongside the (shallow) .git directory, on top of
// the git base image.
func build(ctx context.Context, cloneURL, repoName string, refName plumbing.ReferenceName, hash plumbing.Hash) (v1.Image, error) {
layer, err := gitLayer(ctx, cloneURL, refName, hash, path.Join("git", repoName))
if err != nil {
return nil, fmt.Errorf("LayerFromOpener: %w", err)
return nil, err
}
cfgLayer, err := gitConfigLayer()
if err != nil {
@ -263,6 +229,79 @@ func build(ctx context.Context, dir, cloneURL, repoName string, refName plumbing
return mutate.ConfigFile(img, cf)
}
// gitLayer clones cloneURL into an in-memory git directory and returns a layer
// containing a checkout of the requested commit (every entry rooted under
// prefix, e.g. "git/<repoName>") alongside the (shallow) .git directory.
//
// Only the fetched packfile is buffered in memory (so offset deltas can be
// resolved); worktree file contents are streamed straight into the layer tar
// as the commit's tree is walked, rather than being unpacked to disk first.
func gitLayer(ctx context.Context, cloneURL string, refName plumbing.ReferenceName, hash plumbing.Hash, prefix string) (v1.Layer, error) {
dotgit := memfs.New()
store := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault())
var co git.CloneOptions
if refName == "" {
// A bare commit SHA: full clone, then resolve that commit.
co = git.CloneOptions{URL: cloneURL, Tags: git.NoTags, NoCheckout: true}
} else {
// A branch or tag: shallow, single-branch clone.
refn := refName
if refn == plumbing.HEAD {
refn = "" // let go-git pick the default branch
}
co = git.CloneOptions{
URL: cloneURL,
ReferenceName: refn,
SingleBranch: true,
Depth: 1,
Tags: git.NoTags,
NoCheckout: true,
}
}
repo, err := git.CloneContext(ctx, store, nil, &co)
if err != nil {
return nil, fmt.Errorf("clone %q: %w", cloneURL, err)
}
commitHash := hash
if refName != "" {
head, err := repo.Head()
if err != nil {
return nil, fmt.Errorf("resolving HEAD of %q: %w", cloneURL, err)
}
commitHash = head.Hash()
}
commit, err := resolveCommit(repo, commitHash)
if err != nil {
return nil, err
}
tree, err := commit.Tree()
if err != nil {
return nil, fmt.Errorf("tree for %s: %w", commit.Hash, err)
}
// The image ships a worktree, so the .git directory we copy out isn't bare;
// also record a .git/index for the checkout so `git status` looks clean.
if cfg, err := repo.Config(); err == nil {
cfg.Core.IsBare = false
_ = repo.Storer.SetConfig(cfg)
}
if err := writeIndex(repo, tree); err != nil {
return nil, fmt.Errorf("writing index: %w", err)
}
layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) {
pr, pw := io.Pipe()
go func() { pw.CloseWithError(writeRepoTar(pw, dotgit, tree, prefix)) }()
return pr, nil
})
if err != nil {
return nil, fmt.Errorf("LayerFromOpener: %w", err)
}
return layer, nil
}
var epoch = time.Unix(0, 0)
// gitConfigLayer is a layer adding /etc/gitconfig that marks every directory as
@ -294,64 +333,162 @@ func gitConfigLayer() (v1.Layer, error) {
})
}
// tarDirOpener returns an opener that produces a tar stream of dir, with every
// entry rooted under prefix.
func tarDirOpener(dir, prefix string) tarball.Opener {
return func() (io.ReadCloser, error) {
pr, pw := io.Pipe()
go func() { pw.CloseWithError(writeTar(pw, dir, prefix)) }()
return pr, nil
// resolveCommit returns the commit identified by h, dereferencing annotated tag
// objects as needed.
func resolveCommit(repo *git.Repository, h plumbing.Hash) (*object.Commit, error) {
for {
if c, err := repo.CommitObject(h); err == nil {
return c, nil
}
t, err := repo.TagObject(h)
if err != nil {
return nil, fmt.Errorf("resolving %s to a commit: %w", h, err)
}
h = t.Target
}
}
func writeTar(w io.Writer, dir, prefix string) error {
// writeIndex records a .git/index describing the contents of tree, so the
// checked-out worktree looks clean to git even though it's never materialized
// on disk during the build.
func writeIndex(repo *git.Repository, tree *object.Tree) error {
idx := &index.Index{Version: 2}
if err := tree.Files().ForEach(func(f *object.File) error {
idx.Entries = append(idx.Entries, &index.Entry{
Name: f.Name,
Hash: f.Hash,
Mode: f.Mode,
Size: uint32(f.Size),
})
return nil
}); err != nil {
return err
}
return repo.Storer.SetIndex(idx)
}
// writeRepoTar streams a tar of the checked-out worktree (every entry rooted
// under prefix, e.g. "git/<repoName>") followed by the repository's .git
// directory. Worktree file contents are read straight from the in-memory
// packfile via tree; only .git itself lives in dotgit.
func writeRepoTar(w io.Writer, dotgit billy.Filesystem, tree *object.Tree, prefix string) error {
tw := tar.NewWriter(w)
if err := filepath.Walk(dir, func(p string, fi os.FileInfo, err error) error {
walker := object.NewTreeWalker(tree, true, nil)
for {
name, entry, err := walker.Next()
if err == io.EOF {
break
}
if err != nil {
walker.Close()
tw.Close()
return err
}
if err := writeWorktreeEntry(tw, tree, path.Join(prefix, name), entry); err != nil {
walker.Close()
tw.Close()
return err
}
}
walker.Close()
gitDir := path.Join(prefix, ".git")
if err := billyutil.Walk(dotgit, "/", func(p string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(dir, p)
if err != nil {
return err
}
if rel == "." {
rel := strings.Trim(filepath.ToSlash(p), "/")
if rel == "" {
return nil
}
var link string
if fi.Mode()&os.ModeSymlink != 0 {
if link, err = os.Readlink(p); err != nil {
return err
}
name := path.Join(gitDir, rel)
if fi.IsDir() {
return tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeDir,
Name: name + "/",
Mode: 0755,
ModTime: epoch,
})
}
hdr, err := tar.FileInfoHeader(fi, link)
if err := tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeReg,
Name: name,
Mode: 0644,
Size: fi.Size(),
ModTime: epoch,
}); err != nil {
return err
}
f, err := dotgit.Open(p)
if err != nil {
return err
}
hdr.Name = path.Join(prefix, filepath.ToSlash(rel))
if fi.IsDir() {
hdr.Name += "/"
_, err = io.Copy(tw, f)
if cerr := f.Close(); err == nil {
err = cerr
}
hdr.ModTime, hdr.AccessTime, hdr.ChangeTime = epoch, epoch, epoch
hdr.Uid, hdr.Gid, hdr.Uname, hdr.Gname = 0, 0, "", ""
if err := tw.WriteHeader(hdr); err != nil {
return err
}
if fi.Mode().IsRegular() {
f, err := os.Open(p)
if err != nil {
return err
}
_, err = io.Copy(tw, f)
f.Close()
if err != nil {
return err
}
}
return nil
return err
}); err != nil {
tw.Close()
return err
}
return tw.Close()
}
func writeWorktreeEntry(tw *tar.Writer, tree *object.Tree, name string, entry object.TreeEntry) error {
switch entry.Mode {
case filemode.Dir:
return tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeDir,
Name: name + "/",
Mode: 0755,
ModTime: epoch,
})
case filemode.Submodule:
// Submodules have no file content of their own.
return nil
}
f, err := tree.TreeEntryFile(&entry)
if err != nil {
return err
}
if entry.Mode == filemode.Symlink {
target, err := f.Contents()
if err != nil {
return err
}
return tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeSymlink,
Name: name,
Linkname: target,
Mode: 0777,
ModTime: epoch,
})
}
mode := int64(0644)
if entry.Mode == filemode.Executable {
mode = 0755
}
if err := tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeReg,
Name: name,
Mode: mode,
Size: f.Size,
ModTime: epoch,
}); err != nil {
return err
}
rc, err := f.Reader()
if err != nil {
return err
}
_, err = io.Copy(tw, rc)
if cerr := rc.Close(); err == nil {
err = cerr
}
return err
}