Add cnb.kontain.me, refactor and share a lot

cmd/buildpack/main.go

@@ -0,0 +1,187 @@
+package main
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/compute/metadata"
+	"cloud.google.com/go/logging"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/imjasonh/kontain.me/pkg"
+	"golang.org/x/oauth2/google"
+)
+
+var projectID = ""
+
+func init() {
+	p, err := metadata.ProjectID()
+	if err != nil {
+		log.Fatalf("metadata.ProjectID: %v", err)
+	}
+	projectID = p
+}
+
+const (
+	source = "https://github.com/buildpack/sample-java-app/archive/master.tar.gz"
+	base   = "packs/run:v3alpha2"
+)
+
+func main() {
+	ctx := context.Background()
+	client, err := logging.NewClient(ctx, projectID)
+	if err != nil {
+		log.Fatalf("logging.NewClient: %v", err)
+	}
+	lg := client.Logger("server")
+
+	http.Handle("/v2/", &server{
+		info:  lg.StandardLogger(logging.Info),
+		error: lg.StandardLogger(logging.Error),
+	})
+	http.Handle("/", http.FileServer(http.Dir("/var/run/ko")))
+
+	log.Println("Starting...")
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "8080"
+		log.Printf("Defaulting to port %s", port)
+	}
+	log.Printf("Listening on port %s", port)
+	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
+}
+
+type server struct {
+	info, error *log.Logger
+}
+
+func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.info.Println("handler:", r.Method, r.URL)
+	path := strings.TrimPrefix(r.URL.String(), "/v2/")
+
+	switch {
+	case path == "": // API Version check.
+		w.Header().Set("Docker-Distribution-API-Version", "registry/2.0")
+	case strings.HasPrefix(path, "buildpack/") && strings.Contains(path, "/manifests/"):
+		s.serveBuildpackManifest(w, r)
+	case strings.HasPrefix(path, "buildpack/blobs/") && strings.Contains(path, "/blobs/"):
+		pkg.ServeBlob(w, r)
+	default:
+		http.Error(w, "not found", http.StatusNotFound)
+	}
+}
+
+func (s *server) serveBuildpackManifest(w http.ResponseWriter, r *http.Request) {
+	// Prepare workspace.
+	src, layers, err := s.prepareWorkspace()
+	if err != nil {
+		s.error.Println("ERROR:", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	// Clean up workspace.
+	defer func() {
+		for _, path := range []string{
+			src, layers, os.Getenv("HOME"),
+		} {
+			if err := os.RemoveAll(path); err != nil {
+				s.error.Printf("RemoveAll(%q): %v", path, err)
+			}
+		}
+		os.Setenv("HOME", "/home/")
+	}()
+
+	// Fetch, detect and build source.
+	image, err := s.fetchAndBuild(src, layers)
+	if err != nil {
+		s.error.Println("ERROR:", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Serve new image manifest.
+	img, err := s.getImage(image)
+	if err != nil {
+		s.error.Println("ERROR:", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	pkg.ServeManifest(w, img)
+}
+
+func (s *server) prepareWorkspace() (string, string, error) {
+	// Write Docker config.
+	tok, err := google.ComputeTokenSource("").Token()
+	if err != nil {
+		return "", "", fmt.Errorf("getting access token from metadata: %v", err)
+	}
+	auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("oauth2accesstoken:%s", tok.AccessToken)))
+	configJSON := fmt.Sprintf(`{
+	"auths": {
+		"https://gcr.io": {
+			"auth": %q
+		}
+	}
+}`, auth)
+	if err := pkg.Run(s.info.Writer(), "mkdir -p ~/.docker/ && cat << EOF > ~/.docker/config.json\n"+string(configJSON)+"\nEOF"); err != nil {
+		return "", "", err
+	}
+
+	// Create tempdir to store app source.
+	src, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", "", err
+	}
+
+	// Create layers dir.
+	layers, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", "", err
+	}
+
+	// Create and set $HOME.
+	home, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", "", err
+	}
+	os.Setenv("HOME", home)
+
+	return src, layers, nil
+}
+
+func (s *server) fetchAndBuild(src, layers string) (string, error) {
+	image := fmt.Sprintf("gcr.io/%s/built-%d", projectID, time.Now().Unix)
+
+	for _, cmd := range []string{
+		fmt.Sprintf("chown -R %d:%d %s", os.Geteuid(), os.Getgid(), src),
+		fmt.Sprintf("chown -R %d:%d %s", os.Geteuid(), os.Getgid(), layers),
+		fmt.Sprintf("wget -qO- %s | tar xvz --strip-components=1 -C %s", source, src),
+		fmt.Sprintf("ls -R %s", src),
+		fmt.Sprintf("/lifecycle/detector -app=%s -group=%s/group.toml -plan=%s/plan.toml", src, layers, layers),
+		fmt.Sprintf("/lifecycle/analyzer -layers=%s -helpers=true -group=%s/group.toml %s", layers, layers, image),
+		fmt.Sprintf("/lifecycle/builder -layers=%s -app=%s -group=%s/group.toml -plan=%s/plan.toml", layers, src, layers, layers),
+		fmt.Sprintf("/lifecycle/exporter -layers=%s -helpers=true -app=%s -image=%s -group=%s/group.toml %s", layers, src, base, layers, image),
+	} {
+		if err := pkg.Run(s.info.Writer(), cmd); err != nil {
+			return "", fmt.Errorf("Running %q: %v", cmd, err)
+		}
+	}
+	return image, nil
+}
+
+func (s *server) getImage(image string) (v1.Image, error) {
+	ref, err := name.NewTag(image, name.WeakValidation)
+	if err != nil {
+		return nil, err
+	}
+	return remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+}

cmd/ko/kodata/index.html

@@ -0,0 +1,37 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Kontain.me</title>
+<style>
+body { font-family: Arial; line-height: 1.5; }
+code { background-color: lightgrey; padding: 3px;}
+a { color: blue; }
+</style>
+</head>
+<body>
+<p><b>kontain.me</b> serves Docker container images generated on-demand at the
+time they are requested.</p>
+
+<p><code>docker pull kontain.me/random:latest</code> serves an image containing
+random data. By default the image contains one layer containing 10 MB of random
+bytes.  You can request a specific size and shape of random image. For example,
+<code>kontain.me/random:4x100</code> generates a random image of 4 layers of
+100 random bytes each.
+
+<p><code>docker pull kontain.me/ko/[import path]</code> serves an image
+containing a Go binary fetched using <code>go get</code> and built into a
+container image using <a
+href="https://github.com/google/ko"><code>ko</code></a>. For example,
+<code>docker pull kontain.me/ko/github.com/google/ko/cmd/ko</code> will fetch,
+build and (eventually) serve a Docker image containing <code>ko</code> itself.
+<i>Koception!</i></p>
+
+<p>The registry does not accept pushes and does not handle requests for images
+by digest. This is a silly hack and probably isn't stable. Don't rely on it for
+anything serious. It could probably do a lot of smart things to be a lot
+faster.</p>
+
+<p>Source is available on
+<a href="https://github.com/imjasonh/kontain.me">GitHub</a>.</p>
+</body>
+</html>

cmd/ko/main.go

@@ -0,0 +1,170 @@
+// Copyright 2018 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/compute/metadata"
+	"cloud.google.com/go/logging"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/random"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/ko/pkg/build"
+	"github.com/imjasonh/kontain.me/pkg"
+)
+
+func main() {
+	ctx := context.Background()
+	projectID, err := metadata.ProjectID()
+	if err != nil {
+		log.Fatalf("metadata.ProjectID: %v", err)
+	}
+	client, err := logging.NewClient(ctx, projectID)
+	if err != nil {
+		log.Fatalf("logging.NewClient: %v", err)
+	}
+	lg := client.Logger("server")
+
+	http.Handle("/v2/", &server{
+		info:  lg.StandardLogger(logging.Info),
+		error: lg.StandardLogger(logging.Error),
+	})
+	http.Handle("/", http.FileServer(http.Dir("/var/run/ko")))
+
+	log.Println("Starting...")
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "8080"
+		log.Printf("Defaulting to port %s", port)
+	}
+	log.Printf("Listening on port %s", port)
+	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
+}
+
+type server struct {
+	info, error *log.Logger
+}
+
+func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.info.Println("handler:", r.Method, r.URL)
+	path := strings.TrimPrefix(r.URL.String(), "/v2/")
+
+	switch {
+	case path == "":
+		// API Version check.
+		w.Header().Set("Docker-Distribution-API-Version", "registry/2.0")
+		return
+	case strings.HasPrefix(path, "random/manifests/"):
+		s.serveRandomManifest(w, r)
+	case strings.HasPrefix(path, "ko/") && strings.Contains(path, "/manifests/"):
+		s.serveKoManifest(w, r)
+	case strings.HasPrefix(path, "random/blobs/"),
+		strings.HasPrefix(path, "ko/") && strings.Contains(path, "/blobs/"):
+		pkg.ServeBlob(w, r)
+	default:
+		http.Error(w, "not found", http.StatusNotFound)
+	}
+}
+
+func getDefaultBaseImage(string) (v1.Image, error) {
+	// TODO: memoize
+	ref, err := name.ParseReference("gcr.io/distroless/base", name.WeakValidation)
+	if err != nil {
+		return nil, err
+	}
+	return remote.Image(ref)
+}
+
+// konta.in/ko/github.com/knative/build/cmd/controller -> ko build and serve
+func (s *server) serveKoManifest(w http.ResponseWriter, r *http.Request) {
+	path := strings.TrimPrefix(r.URL.Path, "/v2/ko/")
+	parts := strings.Split(path, "/")
+	ip := strings.Join(parts[:len(parts)-2], "/")
+
+	tag := parts[len(parts)-1]
+	s.info.Printf("requested image tag :%s", tag)
+
+	// go get the package.
+	s.info.Printf("go get %s...", ip)
+	if err := pkg.Run(s.info.Writer(), fmt.Sprintf("go get %s", ip)); err != nil {
+		s.error.Printf("ERROR (go get): %s", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	// TODO: Check image tag for version, resolve branches -> commits and redirect to img:<commit>
+	// TODO: For requests for commit SHAs, check if it's already built and serve that instead.
+	// TODO: Look for $GOPATH/$importPath/.ko.yaml and up, to base image config.
+
+	// ko build the package.
+	g, err := build.NewGo(
+		build.WithBaseImages(getDefaultBaseImage),
+		build.WithCreationTime(v1.Time{time.Unix(0, 0)}),
+	)
+	if err != nil {
+		s.error.Printf("ERROR (build.NewGo): %s", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if !g.IsSupportedReference(ip) {
+		s.error.Printf("ERROR (IsSupportedReference): %s", err)
+		http.Error(w, fmt.Sprintf("%q is not a supported reference", ip), http.StatusBadRequest)
+		return
+	}
+	s.info.Printf("ko build %s...", ip)
+	img, err := g.Build(ip)
+	if err != nil {
+		s.error.Printf("ERROR (ko build): %s", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	pkg.ServeManifest(w, img)
+}
+
+// Capture up to 99 layers of up to 99.9MB each.
+var randomTagRE = regexp.MustCompile("([0-9]{1,2})x([0-9]{1,8})")
+
+// konta.in/random:3x10mb
+// konta.in/random(:latest) -> 1x10mb
+func (s *server) serveRandomManifest(w http.ResponseWriter, r *http.Request) {
+	tag := strings.TrimPrefix(r.URL.Path, "/v2/random/manifests/")
+	var num, size int64 = 1, 10000000 // 10MB
+
+	// Captured requested num + size from tag.
+	all := randomTagRE.FindStringSubmatch(tag)
+	if len(all) >= 3 {
+		num, _ = strconv.ParseInt(all[1], 10, 64)
+		size, _ = strconv.ParseInt(all[2], 10, 64)
+	}
+	s.info.Printf("generating random image with %d layers of %d bytes", num, size)
+
+	// Generate a random image.
+	img, err := random.Image(size, num)
+	if err != nil {
+		s.error.Printf("ERROR (random.Image): %s", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	pkg.ServeManifest(w, img)
+}