diff --git a/README.md b/README.md index b23a12f..7df4665 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,20 @@ These include: [`ko`](https://github.com/google/ko). * [`apko.kontain.me`](./cmd/apko), which builds a minimal base image containing APK packages, using [`apko`](https://apko.dev). +* [`git.kontain.me`](./cmd/git), which serves an image containing a shallow + clone of a Git repo at a given branch, tag, or commit, using + [`go-git`](https://github.com/go-git/go-git). * [`wait.kontain.me`](./cmd/wait), which enqueues a background task to serve a random image after some amount of time. This repo also serves [`viz.kontain.me`](./cmd/viz), which visualizes shared image layers using [Graphviz](https://graphviz.org/). +There's also two other services in this random grab-bag: + +* [`infinite-git.kontain.me`](https://github.com/imjasonh/infinite-git) generates a new Git commit every time the repo is pulled +* `infinite-go.kontain.me` (also in the `infinite-git` repo) generates new Go module version every time it's fetched + # Caveats * The registry does not accept pushes. diff --git a/apps.tf b/apps.tf index 25dfccf..825d349 100644 --- a/apps.tf +++ b/apps.tf @@ -14,6 +14,13 @@ locals { timeout_seconds = 120 # 2m base_image = "cgr.dev/chainguard/static:latest-glibc" } + "git" : { + cpu = 1 + ram = "2Gi" + container_concurrency = 5 + timeout_seconds = 900 # 15m + base_image = "cgr.dev/chainguard/static:latest-glibc" + } "ko" : { cpu = 2 ram = "4Gi" @@ -47,10 +54,18 @@ locals { cpu = 1 ram = "1Gi" container_concurrency = 1000 - importpath = "github.com/imjasonh/infinite-git" + importpath = "github.com/imjasonh/infinite-git/cmd/infinite-git" base_image = "cgr.dev/chainguard/git:latest" timeout_seconds = 60 # 1m } + # "infinite-go" : { + # cpu = 1 + # ram = "1Gi" + # container_concurrency = 1000 + # importpath = "github.com/imjasonh/infinite-git/cmd/infinite-go" + # base_image = "cgr.dev/chainguard/static:latest-glibc" + # timeout_seconds = 60 # 1m + # } } } diff --git a/cmd/git/README.md b/cmd/git/README.md new file mode 100644 index 0000000..6302f88 --- /dev/null +++ b/cmd/git/README.md @@ -0,0 +1,49 @@ +# `git.kontain.me` + +`docker pull git.kontain.me/[repo]:[ref]` serves an image containing a shallow +clone of `[repo]` (fetched over HTTPS) checked out at `/git/[name]`, including +its `.git` directory, on top of [`cgr.dev/chainguard/git:latest-dev`]. + +The image tag `[ref]` is a branch, a tag, or a full commit SHA. It defaults to +`latest`, which means the remote's default branch unless a branch or tag named +`latest` exists. A trailing `.git` on `[repo]` is optional. + +[`cgr.dev/chainguard/git:latest-dev`]: https://images.chainguard.dev/directory/image/git/overview + +## Examples + +Get an image with this repo's default branch checked out at `/git/kontain.me`: + +``` +docker pull git.kontain.me/github.com/imjasonh/kontain.me +``` + +```dockerfile +FROM git.kontain.me/github.com/imjasonh/kontain.me AS src +FROM alpine +COPY --from=src /git /git +``` + +A specific branch, tag, or commit: + +``` +docker pull git.kontain.me/github.com/google/go-containerregistry:main +docker pull git.kontain.me/github.com/google/go-containerregistry:v0.20.0 +docker pull git.kontain.me/github.com/google/go-containerregistry:8a28419ab6b35f06f7148f3b5fc9e75b663f7e2c +``` + +Since the base image is `cgr.dev/chainguard/git`, the `git` CLI works inside the +checkout: + +``` +docker run --rm git.kontain.me/github.com/imjasonh/kontain.me git log -1 +``` + +## Caveats + +* Branch and tag clones are shallow (`--depth=1`), so history isn't available. + Pulling by commit SHA does a full clone first. +* Refs with `/` in them (e.g. `feature/foo`) can't be expressed as an OCI tag; + pull by commit SHA instead. +* The image ships an `/etc/gitconfig` with `safe.directory = *` so `git` works + in the checkout regardless of which user the container runs as. diff --git a/cmd/git/main.go b/cmd/git/main.go new file mode 100644 index 0000000..1d06f69 --- /dev/null +++ b/cmd/git/main.go @@ -0,0 +1,357 @@ +package main + +import ( + "archive/tar" + "bytes" + "context" + "crypto/md5" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/chainguard-dev/clog/gcp" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/imjasonh/kontain.me/pkg/serve" +) + +func main() { + ctx := context.Background() + st, err := serve.NewStorage(ctx) + if err != nil { + slog.ErrorContext(ctx, "serve.NewStorage", "err", err) + os.Exit(1) + } + http.Handle("/v2/", gcp.WithCloudTraceContext(&server{storage: st})) + http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/git", http.StatusSeeOther)) + + port := os.Getenv("PORT") + if port == "" { + port = "8080" + slog.InfoContext(ctx, "Defaulting port", "port", port) + } + slog.InfoContext(ctx, "Listening...", "port", port) + slog.ErrorContext(ctx, "ListenAndServe", "err", http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) +} + +type server struct{ storage *serve.Storage } + +func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.String(), "/v2/") + + switch { + case path == "": + // API Version check. + w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") + return + case strings.Contains(path, "/blobs/"), + strings.Contains(path, "/manifests/sha256:"): + // Extract requested blob digest and redirect to serve it from GCS. + // If it doesn't exist, this will return 404. + parts := strings.Split(r.URL.Path, "/") + digest := parts[len(parts)-1] + serve.Blob(w, r, digest) + case strings.Contains(path, "/manifests/"): + s.serveGitManifest(w, r) + default: + serve.Error(w, serve.ErrNotFound) + } +} + +func cacheKey(cloneURL string, hash plumbing.Hash) string { + return fmt.Sprintf("git-%x", md5.Sum([]byte(cloneURL+"@"+hash.String()))) +} + +var fullSHA = regexp.MustCompile(`^[0-9a-f]{40}$`) + +// git.kontain.me/github.com/imjasonh/kontain.me:main +// +// -> shallow-clone https://github.com/imjasonh/kontain.me at "main", +// serve an image with it checked out at /git/kontain.me +// +// The image tag is the ref: a branch, a tag, or a full commit SHA. The default +// tag "latest" resolves to the remote's default branch unless a branch or tag +// named "latest" exists. A trailing ".git" on the repo path is tolerated. +func (s *server) serveGitManifest(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + upath := strings.TrimPrefix(r.URL.Path, "/v2/") + parts := strings.Split(upath, "/") + if len(parts) < 3 { + serve.Error(w, serve.ErrNotFound) + return + } + // Path is "//manifests/"; the ref is the image tag. + ref := parts[len(parts)-1] + repoPath := strings.Join(parts[:len(parts)-2], "/") + for strings.HasPrefix(repoPath, "git.kontain.me/") { + repoPath = strings.TrimPrefix(repoPath, "git.kontain.me/") + } + if !strings.Contains(repoPath, "/") { + serve.Error(w, fmt.Errorf("invalid repo path %q: want /", repoPath)) + return + } + cloneURL := "https://" + repoPath + repoName := strings.TrimSuffix(path.Base(repoPath), ".git") + + // Resolve the requested ref to a commit so we can check the cache before + // cloning. + hash, refName, err := resolveRef(ctx, cloneURL, ref) + if err != nil { + slog.ErrorContext(ctx, "resolveRef", "url", cloneURL, "ref", ref, "err", err) + serve.Error(w, err) + return + } + + ck := cacheKey(cloneURL, hash) + if _, err := s.storage.BlobExists(ctx, ck); err == nil { + slog.InfoContext(ctx, "serving cached manifest", "ck", ck) + serve.Blob(w, r, ck) + return + } + + dir, err := os.MkdirTemp("", "git-*") + if err != nil { + slog.ErrorContext(ctx, "MkdirTemp", "err", err) + serve.Error(w, err) + return + } + defer os.RemoveAll(dir) + + img, err := build(ctx, dir, cloneURL, repoName, refName, hash) + if err != nil { + slog.ErrorContext(ctx, "build", "url", cloneURL, "ref", ref, "err", err) + serve.Error(w, err) + return + } + + if err := s.storage.ServeManifest(w, r, img, ck); err != nil { + slog.ErrorContext(ctx, "storage.ServeManifest", "err", err) + serve.Error(w, err) + } +} + +// resolveRef resolves ref (an OCI image tag) against the remote and returns the +// commit it points to plus the reference name to clone. ref is matched first as +// a branch, then as a tag; the default tag "latest" otherwise resolves to the +// remote's default branch. A full commit SHA matching no branch or tag is used +// as-is, with an empty refName so the caller checks it out from a full clone. +func resolveRef(ctx context.Context, cloneURL, ref string) (plumbing.Hash, plumbing.ReferenceName, error) { + rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: "origin", + URLs: []string{cloneURL}, + }) + refs, err := rem.ListContext(ctx, &git.ListOptions{}) + if err != nil { + return plumbing.ZeroHash, "", err + } + byName := make(map[plumbing.ReferenceName]*plumbing.Reference, len(refs)) + var head *plumbing.Reference + for _, r := range refs { + byName[r.Name()] = r + if r.Name() == plumbing.HEAD { + head = r + } + } + + if r, ok := byName[plumbing.NewBranchReferenceName(ref)]; ok { + return r.Hash(), r.Name(), nil + } + if r, ok := byName[plumbing.NewTagReferenceName(ref)]; ok { + return r.Hash(), r.Name(), nil + } + if ref == "latest" || ref == "" { + // No branch or tag named "latest": use the remote's default branch. + if head == nil { + return plumbing.ZeroHash, "", fmt.Errorf("remote %q advertises no HEAD", cloneURL) + } + if head.Type() == plumbing.SymbolicReference { + t, ok := byName[head.Target()] + if !ok { + return plumbing.ZeroHash, "", fmt.Errorf("remote %q HEAD points to unknown ref %q", cloneURL, head.Target()) + } + return t.Hash(), t.Name(), nil + } + return head.Hash(), plumbing.HEAD, nil + } + if fullSHA.MatchString(ref) { + return plumbing.NewHash(ref), "", nil + } + return plumbing.ZeroHash, "", fmt.Errorf("ref %q not found in %q: not a branch, tag, or full commit SHA", ref, cloneURL) +} + +// baseRef contains git and busybox +var baseRef = name.MustParseReference("cgr.dev/chainguard/git:latest-dev") + +// build clones cloneURL into dir/repoName and produces a single-layer image +// containing that checkout at /git/. +func build(ctx context.Context, dir, cloneURL, repoName string, refName plumbing.ReferenceName, hash plumbing.Hash) (v1.Image, error) { + checkout := filepath.Join(dir, repoName) + + if refName == "" { + // A bare commit SHA: full clone, then check the commit out. + repo, err := git.PlainCloneContext(ctx, checkout, false, &git.CloneOptions{ + URL: cloneURL, + Tags: git.NoTags, + }) + if err != nil { + return nil, fmt.Errorf("clone %q: %w", cloneURL, err) + } + wt, err := repo.Worktree() + if err != nil { + return nil, err + } + if err := wt.Checkout(&git.CheckoutOptions{Hash: hash}); err != nil { + return nil, fmt.Errorf("checkout %s: %w", hash, err) + } + } else { + // A branch or tag: shallow, single-branch clone. + refn := refName + if refn == plumbing.HEAD { + refn = "" // let go-git pick the default branch + } + if _, err := git.PlainCloneContext(ctx, checkout, false, &git.CloneOptions{ + URL: cloneURL, + ReferenceName: refn, + SingleBranch: true, + Depth: 1, + Tags: git.NoTags, + }); err != nil { + return nil, fmt.Errorf("clone %q at %q: %w", cloneURL, refName, err) + } + } + + layer, err := tarball.LayerFromOpener(tarDirOpener(checkout, path.Join("git", repoName))) + if err != nil { + return nil, fmt.Errorf("LayerFromOpener: %w", err) + } + cfgLayer, err := gitConfigLayer() + if err != nil { + return nil, fmt.Errorf("gitConfigLayer: %w", err) + } + base, err := remote.Image(baseRef) + if err != nil { + return nil, fmt.Errorf("remote.Image(%q): %w", baseRef, err) + } + img, err := mutate.AppendLayers(base, layer, cfgLayer) + if err != nil { + return nil, fmt.Errorf("AppendLayers: %w", err) + } + cf, err := img.ConfigFile() + if err != nil { + return nil, err + } + cf = cf.DeepCopy() + cf.Author = "git.kontain.me" + cf.OS = "linux" + cf.Architecture = "amd64" + cf.Config.WorkingDir = path.Join("/git", repoName) + cf.Config.Entrypoint = []string{"sh"} + return mutate.ConfigFile(img, cf) +} + +var epoch = time.Unix(0, 0) + +// gitConfigLayer is a layer adding /etc/gitconfig that marks every directory as +// safe, so the bundled git won't refuse to operate on the checkout when the +// image runs as a different user than wrote the files (the "dubious ownership" +// error). +func gitConfigLayer() (v1.Layer, error) { + const cfg = "[safe]\n\tdirectory = *\n" + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + if err := tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: "etc/gitconfig", + Mode: 0644, + Size: int64(len(cfg)), + ModTime: epoch, + }); err != nil { + return nil, err + } + if _, err := io.WriteString(tw, cfg); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + b := buf.Bytes() + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(b)), nil + }) +} + +// tarDirOpener returns an opener that produces a tar stream of dir, with every +// entry rooted under prefix. +func tarDirOpener(dir, prefix string) tarball.Opener { + return func() (io.ReadCloser, error) { + pr, pw := io.Pipe() + go func() { pw.CloseWithError(writeTar(pw, dir, prefix)) }() + return pr, nil + } +} + +func writeTar(w io.Writer, dir, prefix string) error { + tw := tar.NewWriter(w) + if err := filepath.Walk(dir, func(p string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + if rel == "." { + return nil + } + + var link string + if fi.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(p); err != nil { + return err + } + } + hdr, err := tar.FileInfoHeader(fi, link) + if err != nil { + return err + } + hdr.Name = path.Join(prefix, filepath.ToSlash(rel)) + if fi.IsDir() { + hdr.Name += "/" + } + hdr.ModTime, hdr.AccessTime, hdr.ChangeTime = epoch, epoch, epoch + hdr.Uid, hdr.Gid, hdr.Uname, hdr.Gname = 0, 0, "", "" + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if fi.Mode().IsRegular() { + f, err := os.Open(p) + if err != nil { + return err + } + _, err = io.Copy(tw, f) + f.Close() + if err != nil { + return err + } + } + return nil + }); err != nil { + tw.Close() + return err + } + return tw.Close() +} diff --git a/cmd/git/test.sh b/cmd/git/test.sh new file mode 100755 index 0000000..1fbc9a9 --- /dev/null +++ b/cmd/git/test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -eux + +time crane validate --remote=git.kontain.me/github.com/imjasonh/kontain.me:main +# Default tag resolves to the default branch (main): same commit, cache hit. +time crane validate --remote=git.kontain.me/github.com/imjasonh/kontain.me diff --git a/go.mod b/go.mod index 1b6cc0c..9581a1d 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/chainguard-dev/clog v1.8.0 github.com/chainguard-dev/terraform-infra-common v1.0.2 github.com/dustin/go-humanize v1.0.1 + github.com/go-git/go-git/v5 v5.17.2 github.com/google/go-containerregistry v0.21.3 github.com/google/ko v0.18.1 github.com/imjasonh/delay v0.0.0-20210102151318-8339250e8458 @@ -70,7 +71,6 @@ require ( github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.8.0 // indirect - github.com/go-git/go-git/v5 v5.17.2 // indirect github.com/go-jose/go-jose/v3 v3.0.5 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logfmt/logfmt v0.6.1 // indirect @@ -113,7 +113,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.8 // indirect - github.com/imjasonh/infinite-git v0.0.0-20260402080002-16db43a3afa4 // indirect + github.com/imjasonh/infinite-git v0.0.0-20260403121828-4ff9f1cae02b // indirect github.com/in-toto/attestation v1.2.0 // indirect github.com/in-toto/in-toto-golang v0.10.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index ebd2498..d35efa7 100644 --- a/go.sum +++ b/go.sum @@ -483,6 +483,8 @@ github.com/imjasonh/delay v0.0.0-20210102151318-8339250e8458 h1:KSkVhq4WEv9rlP6G github.com/imjasonh/delay v0.0.0-20210102151318-8339250e8458/go.mod h1:Fa7ozpRszH4I+tsBQpsszFbSKnzIXPCvv2RorJijAMg= github.com/imjasonh/infinite-git v0.0.0-20260402080002-16db43a3afa4 h1:Ijnd9NANURkfdzmPsVzrZPbkhTdAVZxWnI5ZbRavKsw= github.com/imjasonh/infinite-git v0.0.0-20260402080002-16db43a3afa4/go.mod h1:ZNlXx8+BPRvWzqE+O7ubCmLK0TxECtmsCX+1cWFmBXk= +github.com/imjasonh/infinite-git v0.0.0-20260403121828-4ff9f1cae02b h1:z3xqP8egwlUyC0mseituP4gNgabRQ5+8UZwclCz4aXk= +github.com/imjasonh/infinite-git v0.0.0-20260403121828-4ff9f1cae02b/go.mod h1:ZNlXx8+BPRvWzqE+O7ubCmLK0TxECtmsCX+1cWFmBXk= github.com/in-toto/attestation v1.2.0 h1:aPRUZ3azbqD7yEBD5fP3TD8Dszf+YHo284SOcpahjQk= github.com/in-toto/attestation v1.2.0/go.mod h1:r79G45gOmzPismgObLSL+rZTFxUgZLOQJI6LofTZgXk= github.com/in-toto/in-toto-golang v0.10.0 h1:+s2eZQSK3WmWfYV85qXVSBfqgawi/5L02MaqA4o/tpM=