diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index d4f2726..f6793eb 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,42 +2,61 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "4.62.1" + version = "5.6.0" hashes = [ - "h1:qyh6IbIxKmDqvJlaIrKznYyLdP3k14pSA6UagmkdBwI=", - "zh:15cb2755054d99ec0d7919f52f1a8a08c018d3f076a46251c5b0382f94337cdf", - "zh:2286d2d182dd3df835665e8bb591ab72ed75af83a822ed91e44ed02d02e399d1", - "zh:2507695cd914fe08cccb2c6fd7ff0a1f566647fd733b89ba83396d8bbac1d4b7", - "zh:256a120ba34df742d328af2d8d2152a3f709eb8de571acec412b814d12b83c80", - "zh:45f19aced5d9f597d8f7d63b7ca980aa3d0f58c342bc56aec8cbe0321955cc06", - "zh:6eef7ba36ad5cf011f1d8b6f16dc84ea93f3f0b19df209f3bd1c694414529e04", - "zh:739a0ab7647153c9b555d3e68b56f746bef17d822969124108775473ca375bfe", - "zh:c5eb04297d298d75592d7671d4c707fdb3a7367aa1c2720c042b6f018268c0e1", - "zh:d227ca244766b76913f70639c612d1e6b7a996484ad4177f41a5316fddf26594", - "zh:dca333c358afe417d3f0610c2b89fc7e3741260a82f9626bcd1006e9016e3eff", - "zh:ebfe6d76660341ea11c120dc2d1780fffed415cfb9b514421cb12963d6f8ac69", + "h1:qhj3soOCQLWWtz4PRZp68q0sJ1F7+JSCBhGualLOEjk=", + "zh:102b6a2672fade82114eb14ed46923fb1b74be2aaca3a50b4f35f7057a9a94b9", + "zh:1a56b63175068c67efbe7d130986ba2839a938f5ffc96a14fd450153174dbfa3", + "zh:1ba1c5e0c86e8aaa8037406390846e78c89b63faf9e527c7874641f35d436e1b", + "zh:3f7161b9288b47cbe89d2f9675f78d83b58ad5880c793b01f50a71ee2583844b", + "zh:66912d6e4180dac37185d17424b345a9d4e3c3c791d45e0737b35e32c9536b35", + "zh:6f06f56e9fac2e55b50e74ffac42d9522bb379394e51dca1eddd4c3b7a68545c", + "zh:8741861ebfa13bb1ed74ea7f4865388a0725ca3a781b6d873ce45e6a4630fe41", + "zh:ae89a9c538665fbc30bb83aa3b13acb18d8380e551ccf242e1c0ab4d626089ab", + "zh:c510f8321c7599aa601b1870fdc0c76cbad3054ed5cc70fe8e37a13a8046a71f", + "zh:cf143a53d5a25c6216d09a9c0b115bb473ffcebd5c4c62b2b2594b1ebc13e662", + "zh:de05b957e5dfdbaf92db47cd9b3ef46a0f8d94599eea6d472928f33058856add", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } -provider "registry.terraform.io/ko-build/ko" { - version = "0.0.7" +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" hashes = [ - "h1:I/n4PtPmWeg4j1QzcKRW5FjEc6ra8g7wIRzv35A8aVc=", - "zh:03063ce498831d2a482ef395df642b93f6151d56c92070065f0b479bddb6bee9", - "zh:099c507db5502970a63be45f997d051e8de3c3b228c4e6db91e46d948d74c3b6", - "zh:38b4803bbc4dde60586fa05c54306ce5eddf10c5ce90e878f8771709145335d4", - "zh:4e4b13d2241a1eb5958ae2b53968921f20da790b63fb34a8c8d225b76db5e0ad", - "zh:6b3d9100e03963c8e87a119acb76920f3cafb9a469915145719e178315ba352b", - "zh:7f9727423ec07821a502003ea4abf264e02a07379c20bc7f1b83621d94705a56", - "zh:8105d33bb68c7cb947aae111f2f4958c3aa6a1fde351068a604c1b6cb0f35543", - "zh:988353256e5f3817af8c308a01579d8db0db6016a8a4095674a669a05f7dfa82", - "zh:a2bb9e2e3bd35c5c38ad18f817edcddace0b617eb4f52e2f8664dc2941fd04ff", - "zh:b1c9037f4c7b11f0c900c4348e3899ccc0671a31763d6b9c865586f681610924", - "zh:d6bd79f087518af69b542603cbeb0f30aa5c0d97c806aac40057a1579e7b7d62", - "zh:dd952f7d54e8de7246ce9a448b7f4e74a4ab107c771691c791ae0b5b33cb3771", - "zh:f2a17668eef15939c49eed218832d1d85e82d63e85f2d8102faff13c1f0f0480", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:fd0b9f8f8d8a1536d565c9c4ebbb4abc02286919d33d1c21379c390ab23c2aa8", + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + ] +} + +provider "registry.terraform.io/ko-build/ko" { + version = "0.0.9" + hashes = [ + "h1:je0tK6MmWJhDdWORsV6KNqeYdIEV8yaI2m34etnH5vQ=", + "zh:08628b4bc5a3209b1a43eb077d04ff2230f9239a1c2898ea6a185d449f61d6db", + "zh:286899a2b02140eea247607d46599255a945d2b287793d5470c3a07cd2c3daf4", + "zh:2d32bfa8f5f04e9c495c2a44c4800b17e76d2945d5f000009e82e2d5301b0c34", + "zh:3994fd3a327083533259d8a61807594236fcca8cc5985aac8306842b5132037a", + "zh:3ad7eb0c463bbc05b12fce6476f4811bbe097f19f9eab92a6170f53d9470f853", + "zh:4446583e4013b72d321ece6cf6dd003f7fc24606515adb2768764dee13e20f07", + "zh:5b5371c9b16437e8e96c8b7a74ff53aef3f2ab199e290a76c1594eff7fc3c787", + "zh:660b3c93d51bff28a39ad0bf2894be5ca6b39df20c911ef8d5916a186f176256", + "zh:690ce99b85d958f8bdfc121f7cc3681601409fa23727ea4df56884c06c20d474", + "zh:81ef6014f9e84fe8a6173448bbc584644125d7e8afab76f365599eab57946398", + "zh:838aaccc77cddca84114a021b626034298cf90af32a9e0e9826ec51e660d8b48", + "zh:b48bbcdd82a158cd71ee294c44f288711587be49f1c399ccad133bd0ea7bb9b0", + "zh:bb6c41cef589899e740a691939161604213cb869afce9764960d6573abc34f51", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fca1b2c17fea7883c04c31c9c43f633f9214aaefeabfa781c2d51b7631103d7e", ] } diff --git a/README.md b/README.md index dd24abf..b23a12f 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,10 @@ These include: * [`mirror.kontain.me`](./cmd/mirror), which pulls and caches images from other registries. * [`flatten.kontain.me`](./cmd/flatten), which pulls and flattens images from other registries, so they contain only one layer. -* [`kaniko.kontain.me`](./cmd/kaniko), which builds a GitHub repo using - [Kaniko](https://github.com/GoogleContainerTools/kaniko). * [`ko.kontain.me`](./cmd/ko), which builds a Go binary into a container image using [`ko`](https://github.com/google/ko). * [`apko.kontain.me`](./cmd/apko), which builds a minimal base image containing APK packages, using [`apko`](https://apko.dev). -* [`buildpack.kontain.me`](./cmd/buildpack), which builds a GitHub repo using [CNCF - Buildpacks](https://buildpacks.io). * [`wait.kontain.me`](./cmd/wait), which enqueues a background task to serve a random image after some amount of time. diff --git a/apps.tf b/apps.tf new file mode 100644 index 0000000..8e4e91b --- /dev/null +++ b/apps.tf @@ -0,0 +1,69 @@ +locals { + apps = { + "apko" : { + cpu = 1 + ram = "512Mi" + container_concurrency = 1 + timeout_seconds = 900 # 15m + base_image = "cgr.dev/chainguard/static:latest-glibc" + } + "flatten" : { + cpu = 1 + ram = "1Gi" + container_concurrency = 80 + timeout_seconds = 120 # 2m + base_image = "cgr.dev/chainguard/static:latest-glibc" + } + "ko" : { + cpu = 2 + ram = "4Gi" + container_concurrency = 1 + timeout_seconds = 900 # 15m + base_image = "cgr.dev/chainguard/go:latest-dev" + } + "mirror" : { + cpu = 2 + ram = "4Gi" + container_concurrency = 1 + timeout_seconds = 900 # 15m + base_image = "cgr.dev/chainguard/static:latest-glibc" + } + "random" : { + cpu = 1 + ram = "256Mi" + container_concurrency = 1000 + timeout_seconds = 60 # 1m + base_image = "cgr.dev/chainguard/static:latest-glibc" + } + wait : { + cpu = 1 + ram = "1Gi" + container_concurrency = 80 + timeout_seconds = 60 # 1m + base_image = "cgr.dev/chainguard/static:latest-glibc" + } + } +} + +module "app" { + for_each = local.apps + depends_on = [google_project_service.run-api] + source = "./module" + name = each.key + + project_id = var.project_id + location = var.location + domain = var.domain + dns_zone = local.dns_zone + bucket = google_storage_bucket.bucket.name + service_account_name = google_service_account.service_account.email + + cpu = each.value.cpu + ram = each.value.ram + container_concurrency = each.value.container_concurrency + timeout_seconds = each.value.timeout_seconds + base_image = each.value.base_image +} + +output "cloudrun_url" { value = { for k, _ in local.apps : k => module.app[k].cloudrun_url } } +output "vanity_url" { value = { for k, _ in local.apps : k => module.app[k].vanity_url } } diff --git a/cmd/apko/main.go b/cmd/apko/main.go index ea67bed..bd5587a 100644 --- a/cmd/apko/main.go +++ b/cmd/apko/main.go @@ -4,7 +4,7 @@ import ( "context" "crypto/md5" "fmt" - "log" + "log/slog" "net/http" "os" "sort" @@ -17,6 +17,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" v1tar "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/imjasonh/gcpslog" "github.com/imjasonh/kontain.me/pkg/serve" "gopkg.in/yaml.v2" ) @@ -25,32 +26,26 @@ func main() { ctx := context.Background() st, err := serve.NewStorage(ctx) if err != nil { - log.Fatalf("serve.NewStorage: %v", err) + slog.Error("serve.NewStorage", "err", err) + os.Exit(1) } - http.Handle("/v2/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - storage: st, - }) + http.Handle("/v2/", gcpslog.WithCloudTraceContext(&server{storage: st})) http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/apko", http.StatusSeeOther)) - log.Println("Starting...") + slog.Info("Starting...") port := os.Getenv("PORT") if port == "" { port = "8080" - log.Printf("Defaulting to port %s", port) + slog.Info("Defaulting port", "port", port) } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) + slog.Info("Listening", "port", port) + slog.Error("ListenAndServe", "err", http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) } -type server struct { - info, error *log.Logger - storage *serve.Storage -} +type server struct{ storage *serve.Storage } func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) + slog.Info("handler", "method", r.Method, "url", r.URL) path := strings.TrimPrefix(r.URL.String(), "/v2/") switch { @@ -85,7 +80,7 @@ func (s *server) serveApkoManifest(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(tagOrDigest, "sha256:") { desc, err := s.storage.BlobExists(ctx, tagOrDigest) if err != nil { - s.error.Printf("ERROR (storage.BlobExists): %s", err) + slog.Error("storage.BlobExists", "err", err) serve.Error(w, serve.ErrNotFound) return } @@ -104,14 +99,14 @@ func (s *server) serveApkoManifest(w http.ResponseWriter, r *http.Request) { if packages[0] == "url" { resp, err := http.Get("https://" + strings.Join(packages[1:], "/")) if err != nil { - s.error.Printf("ERROR (fetch): %s", err) + slog.Error("http.Get", "err", err) serve.Error(w, err) return } defer resp.Body.Close() if err := yaml.NewDecoder(resp.Body).Decode(&ic); err != nil { - s.error.Printf("ERROR (parse fetched IC): %s", err) + slog.Error("yaml.Decode", "err", err) serve.Error(w, err) return } @@ -127,7 +122,7 @@ contents: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: [%s] `, strings.Join(packages, ",")))).Decode(&ic); err != nil { - s.error.Printf("ERROR (parse generated IC): %s", err) + slog.Error("yaml.Decode", "err", err) serve.Error(w, err) return } @@ -136,7 +131,7 @@ contents: // Check if we've already got a manifest for this set of packages. if _, err := s.storage.BlobExists(ctx, ck); err == nil { - s.info.Println("serving cached manifest:", ck) + slog.Info("serving cached manifest", "ck", ck) serve.Blob(w, r, ck) return } @@ -144,13 +139,13 @@ contents: // Build the image. img, err := s.build(ctx, ic) if err != nil { - s.error.Printf("ERROR (build): %s", err) + slog.Error("build", "err", err) serve.Error(w, err) return } if err := s.storage.ServeManifest(w, r, img, ck); err != nil { - s.error.Printf("ERROR (storage.ServeIndex): %v", err) + slog.Error("storage.ServeManifest", "err", err) serve.Error(w, err) } } diff --git a/cmd/buildpack/README.md b/cmd/buildpack/README.md deleted file mode 100644 index 7ae344c..0000000 --- a/cmd/buildpack/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# `buildpack.kontain.me` - -`docker pull buildpack.kontain.me/[ghuser]/[ghrepo][/path/to/app]:[revision]` -serves an image fetched from source on GitHub and built using [CNCF -Buildpacks](https://buildpacks.io) - -## Examples - -Build the latest revision of a repo: - -``` -docker pull buildpack.kontain.me/googlecloudplatform/buildpack-samples/sample-go -``` - -Build a specific commit: - -``` -docker pull buildpack.kontain.me/googlecloudplatform/buildpack-samples/sample-go:ed393d -``` diff --git a/cmd/buildpack/main.go b/cmd/buildpack/main.go deleted file mode 100644 index e1976eb..0000000 --- a/cmd/buildpack/main.go +++ /dev/null @@ -1,313 +0,0 @@ -package main - -import ( - "context" - "crypto/md5" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "regexp" - "strings" - "time" - - "cloud.google.com/go/compute/metadata" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - gauthn "github.com/google/go-containerregistry/pkg/v1/google" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-github/v32/github" - "github.com/imjasonh/kontain.me/pkg/run" - "github.com/imjasonh/kontain.me/pkg/serve" - "golang.org/x/oauth2/google" -) - -var projectID = "" - -func init() { - var err error - projectID, err = metadata.ProjectID() - if err != nil { - log.Fatalf("metadata.ProjectID: %v", err) - } -} - -var commitRE = regexp.MustCompile("[a-f0-9]{40}") - -const base = "gcr.io/buildpacks/gcp/run:v1" - -func main() { - ctx := context.Background() - st, err := serve.NewStorage(ctx) - if err != nil { - log.Fatalf("serve.NewStorage: %v", err) - } - http.Handle("/v2/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - storage: st, - }) - http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/buildpack", http.StatusSeeOther)) - - log.Println("Starting...") - port := os.Getenv("PORT") - if port == "" { - port = "8080" - log.Printf("Defaulting to port %s", port) - } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) -} - -type server struct { - info, error *log.Logger - storage *serve.Storage -} - -func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) - path := strings.TrimPrefix(r.URL.String(), "/v2/") - - switch { - case path == "": // API Version check. - w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") - case strings.Contains(path, "/blobs/"), - strings.Contains(path, "/manifests/sha256:"): - // Extract requested blob digest and redirect to serve it from GCS. - // If it doesn't exist, this will return 404. - parts := strings.Split(r.URL.Path, "/") - digest := parts[len(parts)-1] - serve.Blob(w, r, digest) - case strings.Contains(path, "/manifests/"): - s.serveBuildpackManifest(w, r) - default: - serve.Error(w, serve.ErrNotFound) - } -} - -func (s *server) serveBuildpackManifest(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - // Prepare workspace. - src, layers, err := s.prepareWorkspace() - if err != nil { - s.error.Println("ERROR:", err) - serve.Error(w, err) - return - } - // Clean up workspace. - defer func() { - for _, path := range []string{ - src, layers, os.Getenv("HOME"), - } { - if err := os.RemoveAll(path); err != nil { - s.error.Printf("RemoveAll(%q): %v", path, err) - } - } - os.Setenv("HOME", "/home/") - }() - - // Determine source repo and revision. - parts := strings.Split(r.URL.Path, "/") - if len(parts) < 4 { - serve.Error(w, errors.New("Must specify GitHub repository")) - return - } - ghOwner, ghRepo := parts[2], parts[3] - path := strings.Join(parts[4:len(parts)-2], "/") - tagOrDigest := parts[len(parts)-1] - - // If request is for image by digest, try to serve it from GCS. - if strings.HasPrefix(tagOrDigest, "sha256:") { - desc, err := s.storage.BlobExists(ctx, tagOrDigest) - if err != nil { - s.error.Printf("ERROR (storage.BlobExists): %s", err) - serve.Error(w, serve.ErrNotFound) - return - } - if r.Method == http.MethodHead { - w.Header().Set("Docker-Content-Digest", tagOrDigest) - w.Header().Set("Content-Type", string(desc.MediaType)) - w.Header().Set("Content-Length", fmt.Sprintf("%d", desc.Size)) - return - } - serve.Blob(w, r, tagOrDigest) - return - } - - // If the image tag looks like a commit SHA, see if we already have a - // manifest cached for that revision and serve it directly. Otherwise, - // resolve the branch/tag/whatever to a SHA and redirect to that SHA - // image tag. - revision := tagOrDigest - if commitRE.MatchString(revision) { - ck := cacheKey(path, revision) - if _, err := s.storage.BlobExists(ctx, ck); err == nil { - s.info.Println("serving cached manifest:", ck) - serve.Blob(w, r, ck) - return - } - } else { - revision, err = s.resolveCommit(ghOwner, ghRepo, revision) - if err != nil { - s.error.Println("ERROR(resolveCommit):", err) - serve.Error(w, err) - return - } - path := r.URL.Path[:strings.LastIndex(r.URL.Path, "/")+1] + revision - http.Redirect(w, r, path, http.StatusSeeOther) - return - } - - // Fetch, detect and build source. - image, err := s.fetchAndBuild(src, layers, ghOwner, ghRepo, revision, path) - if err != nil { - s.error.Println("ERROR:", err) - serve.Error(w, err) - return - } - - // Serve new image manifest. - img, err := s.getImage(ctx, image) - if err != nil { - s.error.Println("ERROR:", err) - serve.Error(w, err) - return - } - - // Serve the manifest. - ck := cacheKey(path, revision) - if err := s.storage.ServeManifest(w, r, img, ck); err != nil { - s.error.Printf("ERROR (storage.ServeManifest): %v", err) - serve.Error(w, err) - return - } -} - -func cacheKey(path, revision string) string { - ck := []byte(fmt.Sprintf("%s-%s", strings.ReplaceAll(path, "/", "_"), revision)) - return fmt.Sprintf("buildpack-%x", md5.Sum(ck)) -} - -// Resolves a ref (branch, tag, PR, commit) into its SHA. -// If the image tag is "latest", use the repo's default branch. -// If the image tag is "latest-release", look up the repo's latest release tag. -// In any case, resolve the branch/tag/whatever to a commit SHA. -func (s *server) resolveCommit(owner, repo, ref string) (string, error) { - ctx := context.Background() - client := github.NewClient(nil) - - if ref == "latest" { - repo, _, err := client.Repositories.Get(ctx, owner, repo) - if err != nil { - return "", err - } - ref = repo.GetDefaultBranch() - } - if ref == "latest-release" { - release, resp, err := client.Repositories.GetLatestRelease(ctx, owner, repo) - if resp.StatusCode == http.StatusNotFound { - return "", errors.New("Repository has no releases") - } - if err != nil { - return "", err - } - ref = release.GetTagName() - } - - commit, _, err := client.Repositories.GetCommit(ctx, owner, repo, ref) - if err != nil { - return "", err - } - s.info.Printf("Resolved %q -> %q\n", ref, commit.GetSHA()) - return commit.GetSHA(), nil -} - -func (s *server) prepareWorkspace() (string, string, error) { - // Create tempdir to store app source. - src, err := ioutil.TempDir("", "") - if err != nil { - return "", "", err - } - - // Create layers dir. - layers, err := ioutil.TempDir("", "") - if err != nil { - return "", "", err - } - - // Create and set $HOME, which is otherwise not writable. - home, err := ioutil.TempDir("", "") - if err != nil { - return "", "", err - } - os.Setenv("HOME", home) - - return src, layers, nil -} - -func (s *server) fetchAndBuild(src, layers, ghOwner, ghRepo, revision, path string) (string, error) { - image := fmt.Sprintf("gcr.io/%s/built-%d", projectID, time.Now().Unix()) - source := fmt.Sprintf("https://github.com/%s/%s/archive/%s.tar.gz", ghOwner, ghRepo, revision) - - if resp, err := http.Head(source); err != nil { - return "", err - } else if resp.StatusCode == http.StatusNotFound { - return "", serve.ErrNotFound - } else if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("HEAD %s (%d): %s", source, resp.StatusCode, resp.Status) - } - - srcpath := filepath.Join(src, path) - - ts, err := google.DefaultTokenSource(context.Background(), "https://www.googleapis.com/auth/cloud-platform") - if err != nil { - return "", fmt.Errorf("google.DefaultTokenSource: %v", err) - } - tok, err := ts.Token() - if err != nil { - return "", fmt.Errorf("credentials.Token: %v", err) - } - - for _, cmd := range []string{ - fmt.Sprintf("chown -R %d:%d %s", os.Geteuid(), os.Getgid(), src), - fmt.Sprintf("chown -R %d:%d %s", os.Geteuid(), os.Getgid(), layers), - fmt.Sprintf("curl -fsSL %s | tar xz --strip-components=1 -C %s", source, src), - fmt.Sprintf("cd %s", srcpath), - fmt.Sprintf(` -mkdir -p ~/.docker/ && cat > ~/.docker/config.json << EOF -{ - "auths": { - "gcr.io": { - "username": "oauth2accesstoken", - "password": "%s" - } - } -} -EOF`, tok.AccessToken), - fmt.Sprintf("/lifecycle/detector -app=%s -group=%s/group.toml -plan=%s/plan.toml", srcpath, layers, layers), - fmt.Sprintf("/lifecycle/analyzer -layers=%s -group=%s/group.toml %s", layers, layers, image), - fmt.Sprintf("/lifecycle/builder -layers=%s -app=%s -group=%s/group.toml -plan=%s/plan.toml", layers, srcpath, layers, layers), - fmt.Sprintf("/lifecycle/exporter -layers=%s -app=%s -image=%s -group=%s/group.toml %s", layers, srcpath, base, layers, image), - } { - if err := run.Do(s.info.Writer(), cmd); err != nil { - return "", fmt.Errorf("Running %q: %v", cmd, err) - } - } - return image, nil -} - -func (s *server) getImage(ctx context.Context, image string) (v1.Image, error) { - ref, err := name.NewTag(image, name.WeakValidation) - if err != nil { - return nil, err - } - authn, err := gauthn.NewEnvAuthenticator() - if err != nil { - return nil, err - } - return remote.Image(ref, remote.WithAuth(authn), remote.WithContext(ctx)) -} diff --git a/cmd/buildpack/test.sh b/cmd/buildpack/test.sh deleted file mode 100755 index f7c92f7..0000000 --- a/cmd/buildpack/test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -time crane validate --remote=buildpack.kontain.me/googlecloudplatform/buildpack-samples/sample-go:ed393d -time crane validate --remote=buildpack.kontain.me/googlecloudplatform/buildpack-samples/sample-go:ed393d diff --git a/cmd/flatten/main.go b/cmd/flatten/main.go index afb465c..1648870 100644 --- a/cmd/flatten/main.go +++ b/cmd/flatten/main.go @@ -4,7 +4,7 @@ import ( "context" "fmt" "io" - "log" + "log/slog" "net/http" "os" "strings" @@ -16,6 +16,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/imjasonh/gcpslog" "github.com/imjasonh/kontain.me/pkg/serve" "golang.org/x/sync/errgroup" ) @@ -24,32 +25,26 @@ func main() { ctx := context.Background() st, err := serve.NewStorage(ctx) if err != nil { - log.Fatalf("serve.NewStorage: %v", err) + slog.Error("serve.NewStorage", "err", err) + os.Exit(1) } - http.Handle("/v2/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - storage: st, - }) + http.Handle("/v2/", gcpslog.WithCloudTraceContext(&server{storage: st})) http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/flatten", http.StatusSeeOther)) - log.Println("Starting...") + slog.Info("Starting...") port := os.Getenv("PORT") if port == "" { port = "8080" - log.Printf("Defaulting to port %s", port) + slog.Info("Defaulting port", "port", port) } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) + slog.Info("Listening", "port", port) + slog.Error("ListenAndServe", "err", http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) } -type server struct { - info, error *log.Logger - storage *serve.Storage -} +type server struct{ storage *serve.Storage } func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) + slog.Info("handler", "method", r.Method, "url", r.URL) path := strings.TrimPrefix(r.URL.String(), "/v2/") switch { @@ -99,7 +94,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { ref, err := name.ParseReference(refstr) if err != nil { - s.error.Printf("ERROR (ParseReference(%q)): %v", refstr, err) + slog.Error("name.ParseReference", "ref", refstr, "err", err) serve.Error(w, err) return } @@ -111,15 +106,15 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { // Determine whether the ref is for an image or index. d, err := remote.Head(ref, remote.WithContext(ctx)) if err != nil { - s.error.Printf("ERROR (remote.Head(%q)): %v", ref, err) + slog.Error("remote.Head", "ref", refstr, "err", err) var h v1.Hash // HEAD failed, let's figure out if it was an index or image by doing GETs. idx, err = remote.Index(ref, remote.WithContext(ctx)) if err != nil { - s.error.Printf("ERROR (remote.Index): %v", err) + slog.Error("remote.Index", "ref", refstr, "err", err) img, err = remote.Image(ref, remote.WithContext(ctx)) if err != nil { - s.error.Printf("ERROR (remote.Image): %v", err) + slog.Error("remote.Image", "ref", refstr, "err", err) serve.Error(w, err) return } @@ -131,7 +126,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { h, err = img.Digest() } if err != nil { - s.error.Printf("ERROR (Digest): %v", err) + slog.Error("Digest()", "ref", refstr, "err", err) serve.Error(w, err) return } @@ -140,14 +135,14 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { // before), and if so serve it directly. ck = cacheKey(h.String()) if _, err := s.storage.BlobExists(ctx, ck); err == nil { - s.info.Println("serving cached manifest:", ck) + slog.Info("serving cached manifest", "ck", ck) serve.Blob(w, r, ck) return } } else { if !acceptableMediaTypes[d.MediaType] { err = fmt.Errorf("unknown media type: %s", d.MediaType) - s.error.Printf("ERROR (serveFlattenManifest): %v", err) + slog.Error("unknown media type", "ref", refstr, "err", err) serve.Error(w, err) return } @@ -156,7 +151,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { // directly. ck = cacheKey(d.Digest.String()) if _, err := s.storage.BlobExists(ctx, ck); err == nil { - s.info.Println("serving cached manifest:", ck) + slog.Info("serving cached manifest", "ck", ck) serve.Blob(w, r, ck) return } @@ -166,7 +161,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { idx, err = remote.Index(ref, remote.WithContext(ctx)) if err != nil { err = fmt.Errorf("remote.Index: %v", err) - s.error.Printf("ERROR (serveFlattenManifest): %v", err) + slog.Error("remote.Index", "ref", refstr, "err", err) serve.Error(w, err) return } @@ -174,7 +169,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { img, err = remote.Image(ref, remote.WithContext(ctx)) if err != nil { err = fmt.Errorf("remote.Image: %v", err) - s.error.Printf("ERROR (serveFlattenManifest): %v", err) + slog.Error("remote.Image", "ref", refstr, "err", err) serve.Error(w, err) return } @@ -189,7 +184,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { } if err := s.storage.ServeIndex(w, r, fidx, ck); err != nil { - s.error.Printf("ERROR (storage.ServeIndex): %v", err) + slog.Error("storage.ServeIndex", "err", err) serve.Error(w, err) return } @@ -204,7 +199,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { } if err := s.storage.ServeManifest(w, r, fimg, ck); err != nil { - s.error.Printf("ERROR (storage.ServeManifest): %v", err) + slog.Error("storage.ServeManifest", "err", err) serve.Error(w, err) return } @@ -216,7 +211,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) { func (s *server) flattenIndex(idx v1.ImageIndex) (v1.ImageIndex, error) { im, err := idx.IndexManifest() if err != nil { - s.error.Printf("ERROR (index.IndexManifest): %v", err) + slog.Error("idx.IndexManifest", "err", err) return nil, err } // Flatten each image in the manifest. @@ -227,7 +222,7 @@ func (s *server) flattenIndex(idx v1.ImageIndex) (v1.ImageIndex, error) { g.Go(func() error { img, err := idx.Image(m.Digest) if err != nil { - s.error.Printf("ERROR (idx.Image): %v", err) + slog.Error("idx.Image", "err", err) return err } fimg, err := s.flatten(img) @@ -236,7 +231,7 @@ func (s *server) flattenIndex(idx v1.ImageIndex) (v1.ImageIndex, error) { } m.Digest, err = fimg.Digest() if err != nil { - s.error.Printf("ERROR (fimg.Digest): %v", err) + slog.Error("fimg.Digest", "err", err) return err } adds[i] = mutate.IndexAddendum{ @@ -247,7 +242,7 @@ func (s *server) flattenIndex(idx v1.ImageIndex) (v1.ImageIndex, error) { }) } if err := g.Wait(); err != nil { - s.error.Printf("ERROR (flattenIndex): %v", err) + slog.Error("g.Wait", "err", err) return nil, err } return mutate.AppendManifests(empty.Index, adds...), nil @@ -256,24 +251,24 @@ func (s *server) flattenIndex(idx v1.ImageIndex) (v1.ImageIndex, error) { func (s *server) flatten(img v1.Image) (v1.Image, error) { l, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return mutate.Extract(img), nil }) if err != nil { - s.error.Printf("ERROR (tarball.LayerFromOpener): %v", err) + slog.Error("tarball.LayerFromOpener", "err", err) return nil, err } fimg, err := mutate.AppendLayers(empty.Image, l) if err != nil { - s.error.Printf("ERROR (mutate.AppendLayers): %v", err) + slog.Error("mutate.AppendLayers", "err", err) return nil, err } // Copy over basic information from original config file. ocf, err := img.ConfigFile() if err != nil { - s.error.Printf("ERROR (img.ConfigFile): %v", err) + slog.Error("img.ConfigFile", "err", err) return nil, err } ncf, err := fimg.ConfigFile() if err != nil { - s.error.Printf("ERROR (empty.Image.ConfigFile): %v", err) + slog.Error("fimg.ConfigFile", "err", err) return nil, err } cf := ncf.DeepCopy() diff --git a/cmd/kaniko/README.md b/cmd/kaniko/README.md deleted file mode 100644 index d711445..0000000 --- a/cmd/kaniko/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# `kaniko.kontain.me` - -`docker pull kaniko.kontain.me/[ghuser]/[ghrepo][/path/to/app]:[revision]` -serves an image fetched from source on GitHub and built using -[Kaniko](https://github.com/GoogleContainerTools/kaniko). - -## Caveats - -* The `Dockerfile` must be named `Dockerfile`, and must be at the root of the - path specified in the image name. This path also describes the root of the - source context. - -## Examples - -Build the latest revision of a repo: - -``` -docker pull kaniko.kontain.me/dockersamples/node-bulletin-board/bulletin-board-app -``` - -Build a specific commit: - -``` -docker pull kaniko.kontain.me/dockersamples/node-bulletin-board/bulletin-board-app:3f08afd -``` diff --git a/cmd/kaniko/main.go b/cmd/kaniko/main.go deleted file mode 100644 index 2dd4323..0000000 --- a/cmd/kaniko/main.go +++ /dev/null @@ -1,273 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "regexp" - "strings" - "time" - - "cloud.google.com/go/compute/metadata" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - gauthn "github.com/google/go-containerregistry/pkg/v1/google" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-github/v32/github" - "github.com/imjasonh/kontain.me/pkg/run" - "github.com/imjasonh/kontain.me/pkg/serve" -) - -var projectID = "" - -func init() { - var err error - projectID, err = metadata.ProjectID() - if err != nil { - log.Fatalf("metadata.ProjectID: %v", err) - } -} - -var commitRE = regexp.MustCompile("[a-f0-9]{40}") - -const base = "packs/run:v3alpha2" - -func main() { - ctx := context.Background() - st, err := serve.NewStorage(ctx) - if err != nil { - log.Fatalf("serve.NewStorage: %v", err) - } - http.Handle("/v2/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - storage: st, - }) - http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/kaniko", http.StatusSeeOther)) - - log.Println("Starting...") - port := os.Getenv("PORT") - if port == "" { - port = "8080" - log.Printf("Defaulting to port %s", port) - } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) -} - -type server struct { - info, error *log.Logger - storage *serve.Storage -} - -func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) - path := strings.TrimPrefix(r.URL.String(), "/v2/") - - switch { - case path == "": // API Version check. - w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") - case strings.Contains(path, "/blobs/"), - strings.Contains(path, "/manifests/sha256:"): - // Extract requested blob digest and redirect to serve it from GCS. - // If it doesn't exist, this will return 404. - parts := strings.Split(r.URL.Path, "/") - digest := parts[len(parts)-1] - serve.Blob(w, r, digest) - case strings.Contains(path, "/manifests/"): - s.serveKanikoManifest(w, r) - default: - serve.Error(w, serve.ErrNotFound) - } -} - -func (s *server) serveKanikoManifest(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - // Prepare workspace. - if err := s.prepareWorkspace(); err != nil { - s.error.Println("ERROR:", err) - serve.Error(w, err) - return - } - // Clean up workspace. - defer func() { - if err := os.RemoveAll("/tmp"); err != nil { - s.error.Printf("RemoveAll(/tmp): %v", err) - } - os.Setenv("HOME", "/home/") - }() - - // Determine source repo and revision. - parts := strings.Split(r.URL.Path, "/") - if len(parts) < 4 { - serve.Error(w, errors.New("Must specify GitHub repository")) - return - } - ghOwner, ghRepo := parts[2], parts[3] - path := strings.Join(parts[4:len(parts)-2], "/") - tagOrDigest := parts[len(parts)-1] - - // If request is for image by digest, try to serve it from GCS. - if strings.HasPrefix(tagOrDigest, "sha256:") { - desc, err := s.storage.BlobExists(ctx, tagOrDigest) - if err != nil { - s.error.Printf("ERROR (storage.BlobExists): %s", err) - serve.Error(w, serve.ErrNotFound) - return - } - if r.Method == http.MethodHead { - w.Header().Set("Docker-Content-Digest", tagOrDigest) - w.Header().Set("Content-Type", string(desc.MediaType)) - w.Header().Set("Content-Length", fmt.Sprintf("%d", desc.Size)) - return - } - serve.Blob(w, r, tagOrDigest) - return - } - - // If the image tag looks like a commit SHA, see if we already have a - // manifest cached for that revision and serve it directly. Otherwise, - // resolve the branch/tag/whatever to a SHA and redirect to that SHA - // image tag. - revision := tagOrDigest - if commitRE.MatchString(revision) { - ck := cacheKey(path, revision) - if _, err := s.storage.BlobExists(ctx, ck); err == nil { - s.info.Println("serving cached manifest:", ck) - serve.Blob(w, r, ck) - return - } - } else { - revision, err := s.resolveCommit(ghOwner, ghRepo, revision) - if err != nil { - s.error.Println("ERROR(resolveCommit):", err) - serve.Error(w, err) - return - } - path := r.URL.Path[:strings.LastIndex(r.URL.Path, "/")+1] + revision - http.Redirect(w, r, path, http.StatusSeeOther) - return - } - - // Fetch, detect and build source. - image, err := s.fetchAndBuild(ghOwner, ghRepo, revision, path) - if err != nil { - s.error.Println("ERROR:", err) - serve.Error(w, err) - return - } - - // Serve new image manifest. - img, err := s.getImage(ctx, image) - if err != nil { - s.error.Println("ERROR:", err) - serve.Error(w, err) - return - } - - // Serve the manifest. - ck := cacheKey(path, revision) - if err := s.storage.ServeManifest(w, r, img, ck); err != nil { - s.error.Printf("ERROR (storage.ServeManifest): %v", err) - serve.Error(w, err) - return - } -} - -func cacheKey(path, revision string) string { - ck := fmt.Sprintf("%s-%s", strings.ReplaceAll(path, "/", "_"), revision) - return fmt.Sprintf("kaniko-%x", ck) -} - -// Resolves a ref (branch, tag, PR, commit) into its SHA. -// If the image tag is "latest", use the repo's default branch. -// If the image tag is "latest-release", look up the repo's latest release tag. -// In any case, resolve the branch/tag/whatever to a commit SHA. -func (s *server) resolveCommit(owner, repo, ref string) (string, error) { - ctx := context.Background() - client := github.NewClient(nil) - - if ref == "latest" { - repo, _, err := client.Repositories.Get(ctx, owner, repo) - if err != nil { - return "", err - } - ref = repo.GetDefaultBranch() - } - if ref == "latest-release" { - release, resp, err := client.Repositories.GetLatestRelease(ctx, owner, repo) - if resp.StatusCode == http.StatusNotFound { - return "", errors.New("Repository has no releases") - } - if err != nil { - return "", err - } - ref = release.GetTagName() - } - - commit, _, err := client.Repositories.GetCommit(ctx, owner, repo, ref) - if err != nil { - return "", err - } - s.info.Printf("Resolved %q -> %q\n", ref, commit.GetSHA()) - return commit.GetSHA(), nil -} - -func (s *server) prepareWorkspace() error { - // Create tempdir to store app source. - src, err := ioutil.TempDir("", "") - if err != nil { - return err - } - - // cd into the temp dir. - if err := os.Chdir(src); err != nil { - return err - } - - // Create and set $HOME, which is otherwise not writable. - home, err := ioutil.TempDir("", "") - if err != nil { - return err - } - os.Setenv("HOME", home) - - return nil -} - -func (s *server) fetchAndBuild(ghOwner, ghRepo, revision, path string) (string, error) { - image := fmt.Sprintf("gcr.io/%s/kaniko:built-at-%d", projectID, time.Now().Unix()) - source := fmt.Sprintf("https://github.com/%s/%s/archive/%s.tar.gz", ghOwner, ghRepo, revision) - - for _, cmd := range []string{ - fmt.Sprintf("wget -qO- %s | tar xz --strip-components=1 -C .", source), - fmt.Sprintf(` -/kaniko/executor --force \ - --dockerfile=./%s/Dockerfile \ - --context=./%s \ - --destination=%s \ - --cache-repo=gcr.io/%s`, path, path, image, projectID), - } { - if err := run.Do(s.info.Writer(), cmd); err != nil { - return "", fmt.Errorf("Running %q: %v", cmd, err) - } - } - return image, nil -} - -func (s *server) getImage(ctx context.Context, image string) (v1.Image, error) { - ref, err := name.NewTag(image, name.WeakValidation) - if err != nil { - return nil, err - } - authn, err := gauthn.NewEnvAuthenticator() - if err != nil { - return nil, err - } - return remote.Image(ref, remote.WithAuth(authn), remote.WithContext(ctx)) -} diff --git a/cmd/kaniko/test.sh b/cmd/kaniko/test.sh deleted file mode 100755 index 1ae07b0..0000000 --- a/cmd/kaniko/test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -time crane validate --remote=kaniko.kontain.me/dockersamples/node-bulletin-board/bulletin-board-app:3f08afd -time crane validate --remote=kaniko.kontain.me/dockersamples/node-bulletin-board/bulletin-board-app:3f08afd - diff --git a/cmd/ko/main.go b/cmd/ko/main.go index 412ff3f..350aa43 100644 --- a/cmd/ko/main.go +++ b/cmd/ko/main.go @@ -8,7 +8,7 @@ import ( "fmt" "io" "io/ioutil" - "log" + "log/slog" "net/http" "os" "strings" @@ -19,6 +19,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/ko/pkg/build" + "github.com/imjasonh/gcpslog" "github.com/imjasonh/kontain.me/pkg/serve" "golang.org/x/mod/module" "golang.org/x/mod/zip" @@ -29,32 +30,26 @@ func main() { ctx := context.Background() st, err := serve.NewStorage(ctx) if err != nil { - log.Fatalf("serve.NewStorage: %v", err) + slog.Error("serve.NewStorage", "err", err) + os.Exit(1) } - http.Handle("/v2/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - storage: st, - }) + http.Handle("/v2/", gcpslog.WithCloudTraceContext(&server{storage: st})) http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/ko", http.StatusSeeOther)) - log.Println("Starting...") + slog.Info("Starting...") port := os.Getenv("PORT") if port == "" { port = "8080" - log.Printf("Defaulting to port %s", port) + slog.Info("Defaulting port", "port", port) } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) + slog.Info("Listening", "port", port) + slog.Error("ListenAndServe", "err", http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) } -type server struct { - info, error *log.Logger - storage *serve.Storage -} +type server struct{ storage *serve.Storage } func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) + slog.Info("handler", "method", r.Method, "url", r.URL) path := strings.TrimPrefix(r.URL.String(), "/v2/") switch { @@ -91,7 +86,7 @@ func (s *server) serveKoManifest(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(tagOrDigest, "sha256:") { desc, err := s.storage.BlobExists(ctx, tagOrDigest) if err != nil { - s.error.Printf("ERROR (storage.BlobExists): %s", err) + slog.Error("storage.BlobExists", "err", err) serve.Error(w, serve.ErrNotFound) return } @@ -109,7 +104,7 @@ func (s *server) serveKoManifest(w http.ResponseWriter, r *http.Request) { // whether the path is a module path that returns a version. module, version, err := walkUp(ctx, ip, tagOrDigest) if err != nil { - s.error.Printf("ERROR (walkUp): %s", err) + slog.Error("walkUp", "err", err) serve.Error(w, err) return } @@ -117,7 +112,7 @@ func (s *server) serveKoManifest(w http.ResponseWriter, r *http.Request) { // Check if we've already got a manifest for this importpath + resolved version. ck := cacheKey(ip, version) if _, err := s.storage.BlobExists(ctx, ck); err == nil { - s.info.Println("serving cached manifest:", ck) + slog.Info("serving cached manifest", "ck", ck) serve.Blob(w, r, ck) return } @@ -126,21 +121,21 @@ func (s *server) serveKoManifest(w http.ResponseWriter, r *http.Request) { // Pull the module source from the module proxy and build it. br, err := s.fetchAndBuild(ctx, module, version, filepath) if err != nil { - s.error.Printf("ERROR (fetchAndBuild): %s", err) + slog.Error("fetchAndBuild", "err", err) serve.Error(w, err) return } if idx, ok := br.(v1.ImageIndex); ok { if err := s.storage.ServeIndex(w, r, idx, ck); err != nil { - s.error.Printf("ERROR (storage.ServeIndex): %v", err) + slog.Error("storage.ServeIndex", "err", err) serve.Error(w, err) } return } if img, ok := br.(v1.Image); ok { if err := s.storage.ServeManifest(w, r, img, ck); err != nil { - s.error.Printf("ERROR (storage.ServeManifest): %v", err) + slog.Error("storage.ServeManifest", "err", err) serve.Error(w, err) } return @@ -162,7 +157,7 @@ func (s *server) getBaseImage(ctx context.Context, ip string) (name.Reference, b f, err := os.Open(".ko.yaml") if err == nil { defer f.Close() - s.info.Printf("Found .ko.yaml") + slog.Info("Found .ko.yaml") var y struct { DefaultBaseImage string `yaml:"defaultBaseImage"` BaseImageOverrides map[string]string `yaml:"baseImageOverrides"` @@ -177,7 +172,7 @@ func (s *server) getBaseImage(ctx context.Context, ip string) (name.Reference, b base = bio } } - s.info.Printf("Using base image %q for %s", base, ip) + slog.Info("Using base image", "base", base, "ip", ip) ref, err := name.ParseReference(base) if err != nil { @@ -189,11 +184,11 @@ func (s *server) getBaseImage(ctx context.Context, ip string) (name.Reference, b } switch d.MediaType { case types.DockerManifestList, types.OCIImageIndex: - s.info.Printf("Base image %q is manifest list", base) + slog.Info("Base image is index", "base", base) idx, err := remote.Index(ref) return ref, idx, err case types.DockerManifestSchema2, types.OCIManifestSchema1: - s.info.Printf("Base image %q is image", base) + slog.Info("Base image is image", "base", base) img, err := remote.Image(ref) return ref, img, err default: @@ -306,6 +301,6 @@ func (s *server) fetchAndBuild(ctx context.Context, mod, version, filepath strin if err := g.IsSupportedReference(ip); err != nil { return nil, err } - s.info.Printf("ko build %s...", ip) + slog.Info("ko build", "ip", ip) return g.Build(ctx, ip) } diff --git a/cmd/mirror/main.go b/cmd/mirror/main.go index d7d3ad7..a3519d3 100644 --- a/cmd/mirror/main.go +++ b/cmd/mirror/main.go @@ -3,7 +3,7 @@ package main import ( "context" "fmt" - "log" + "log/slog" "net/http" "os" "strings" @@ -12,6 +12,7 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/imjasonh/gcpslog" "github.com/imjasonh/kontain.me/pkg/serve" ) @@ -19,32 +20,26 @@ func main() { ctx := context.Background() st, err := serve.NewStorage(ctx) if err != nil { - log.Fatalf("serve.NewStorage: %v", err) + slog.Error("serve.NewStorage", "err", err) + os.Exit(1) } - http.Handle("/v2/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - storage: st, - }) + http.Handle("/v2/", gcpslog.WithCloudTraceContext(&server{storage: st})) http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/mirror", http.StatusSeeOther)) - log.Println("Starting...") + slog.Info("Starting...") port := os.Getenv("PORT") if port == "" { port = "8080" - log.Printf("Defaulting to port %s", port) + slog.Info("Defaulting port", "port", port) } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) + slog.Info("Listening", "port", port) + slog.Error("ListenAndServe", "err", http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) } -type server struct { - info, error *log.Logger - storage *serve.Storage -} +type server struct{ storage *serve.Storage } func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) + slog.Info("handler", "method", r.Method, "url", r.URL) path := strings.TrimPrefix(r.URL.String(), "/v2/") w.Header().Set("Access-Control-Allow-Origin", "*") // Allow CORS requests from any domain. @@ -94,7 +89,7 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) { ref, err := name.ParseReference(refstr) if err != nil { - s.error.Printf("ERROR (ParseReference(%q)): %v", refstr, err) + slog.Error("name.ParseReference", "ref", refstr, "err", err) serve.Error(w, err) return } @@ -121,7 +116,7 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) { // blob. d, err := remote.Head(ref, remote.WithContext(ctx)) if err != nil { - s.error.Printf("ERROR (remote.Head(%q)): %v", ref, err) + slog.Error("remote.Head", "ref", ref, "err", err) var desci interface { Digest() (v1.Hash, error) Size() (int64, error) @@ -130,10 +125,10 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) { // HEAD failed, let's figure out if it was an index or image by doing GETs. idx, err = remote.Index(ref, remote.WithContext(ctx)) if err != nil { - s.error.Printf("ERROR (remote.Index): %v", err) + slog.Error("remote.Index", "ref", ref, "err", err) img, err = remote.Image(ref, remote.WithContext(ctx)) if err != nil { - s.error.Printf("ERROR (remote.Image): %v", err) + slog.Error("remote.Image", "ref", ref, "err", err) serve.Error(w, err) return } @@ -144,19 +139,19 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) { h, err := desci.Digest() if err != nil { - s.error.Printf("ERROR (desc.Digest): %v", err) + slog.Error("Digest()", "ref", ref, "err", err) serve.Error(w, err) return } sz, err := desci.Size() if err != nil { - s.error.Printf("ERROR (desc.Size): %v", err) + slog.Error("Size()", "ref", ref, "err", err) serve.Error(w, err) return } mt, err := desci.MediaType() if err != nil { - s.error.Printf("ERROR (desc.MediaType): %v", err) + slog.Error("MediaType()", "ref", ref, "err", err) serve.Error(w, err) return } @@ -176,7 +171,7 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) { serve.Blob(w, r, d.Digest.String()) return } else { - s.info.Printf("INFO (serve.BlobExists(%q)): %v", d.Digest, err) + slog.Info("BlobExists", "digest", d.Digest.String(), "err", err) } // Blob doesn't exist yet. Try to get the image manifest+layers @@ -188,13 +183,13 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) { // the image index. idx, err = remote.Index(ref, remote.WithContext(ctx)) if err != nil { - s.error.Printf("ERROR (remote.Index): %v", err) + slog.Error("remote.Index", "ref", ref, "err", err) serve.Error(w, err) return } } if err := s.storage.ServeIndex(w, r, idx); err != nil { - s.error.Printf("ERROR (storage.ServeIndex): %v", err) + slog.Error("storage.ServeIndex", "err", err) serve.Error(w, err) return } @@ -204,19 +199,19 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) { // manifest. img, err = remote.Image(ref, remote.WithContext(ctx)) if err != nil { - s.error.Printf("ERROR (remote.Image): %v", err) + slog.Error("remote.Image", "ref", ref, "err", err) serve.Error(w, err) return } } if err := s.storage.ServeManifest(w, r, img); err != nil { - s.error.Printf("ERROR (storage.ServeManifest): %v", err) + slog.Error("storage.ServeManifest", "err", err) serve.Error(w, err) return } default: err := fmt.Errorf("unknown media type: %s", d.MediaType) - s.error.Printf("ERROR (serveMirrorManifest): %v", err) + slog.Error("unknown media type", "ref", refstr, "err", err) serve.Error(w, err) } } diff --git a/cmd/random/main.go b/cmd/random/main.go index a0f7305..006491f 100644 --- a/cmd/random/main.go +++ b/cmd/random/main.go @@ -3,7 +3,7 @@ package main import ( "context" "fmt" - "log" + "log/slog" "net/http" "os" "regexp" @@ -11,6 +11,7 @@ import ( "strings" "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/imjasonh/gcpslog" "github.com/imjasonh/kontain.me/pkg/serve" ) @@ -18,32 +19,26 @@ func main() { ctx := context.Background() st, err := serve.NewStorage(ctx) if err != nil { - log.Fatalf("serve.NewStorage: %v", err) + slog.Error("serve.NewStorage", "err", err) + os.Exit(1) } - http.Handle("/v2/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - storage: st, - }) + http.Handle("/v2/", gcpslog.WithCloudTraceContext(&server{storage: st})) http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/random", http.StatusSeeOther)) - log.Println("Starting...") + slog.Info("Starting...") port := os.Getenv("PORT") if port == "" { port = "8080" - log.Printf("Defaulting to port %s", port) + slog.Info("Defaulting port", "port", port) } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) + slog.Info("Listening", "port", port) + slog.Error("ListenAndServe", "err", http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) } -type server struct { - info, error *log.Logger - storage *serve.Storage -} +type server struct{ storage *serve.Storage } func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) + slog.Info("handler", "method", r.Method, "url", r.URL) path := strings.TrimPrefix(r.URL.String(), "/v2/") switch { @@ -79,7 +74,7 @@ func (s *server) serveRandomManifest(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(tagOrDigest, "sha256:") { desc, err := s.storage.BlobExists(ctx, tagOrDigest) if err != nil { - s.error.Printf("ERROR (storage.BlobExists): %s", err) + slog.Error("storage.BlobExists", "err", err) serve.Error(w, serve.ErrNotFound) return } @@ -99,17 +94,17 @@ func (s *server) serveRandomManifest(w http.ResponseWriter, r *http.Request) { num, _ = strconv.ParseInt(all[1], 10, 64) size, _ = strconv.ParseInt(all[2], 10, 64) } - s.info.Printf("generating random image with %d layers of %d bytes", num, size) + slog.Info("generating random image", "layers", num, "size", size) // Generate a random image. img, err := random.Image(size, num) if err != nil { - s.error.Printf("ERROR (random.Image): %s", err) + slog.Error("random.Image", "err", err) serve.Error(w, err) return } if err := s.storage.ServeManifest(w, r, img); err != nil { - s.error.Printf("ERROR (storage.ServeManifest): %v", err) + slog.Error("storage.ServeManifest", "err", err) serve.Error(w, err) return } diff --git a/cmd/viz/main.go b/cmd/viz/main.go index dc92e12..a91a5a7 100644 --- a/cmd/viz/main.go +++ b/cmd/viz/main.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log" + "log/slog" "net/http" "os" "os/exec" @@ -14,32 +15,28 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/imjasonh/gcpslog" "github.com/tmc/dot" ) func main() { - http.Handle("/", http.FileServer(http.Dir("/var/run/ko"))) - http.Handle("/viz", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - }) - log.Println("Starting...") + http.Handle("/v2/", gcpslog.WithCloudTraceContext(&server{})) + + slog.Info("Starting...") port := os.Getenv("PORT") if port == "" { port = "8080" - log.Printf("Defaulting to port %s", port) + slog.Info("Defaulting port", "port", port) } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) + slog.Info("Listening", "port", port) + slog.Error("ListenAndServe", "err", http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) } -type server struct { - info, error *log.Logger -} +type server struct{} func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - s.info.Println("handler:", r.Method, r.URL) + slog.Info("handler", "method", r.Method, "url", r.URL) if r.Method != http.MethodPost { http.Error(w, "must be post", http.StatusMethodNotAllowed) return diff --git a/cmd/wait/main.go b/cmd/wait/main.go index 6e8e57c..f2ba2cb 100644 --- a/cmd/wait/main.go +++ b/cmd/wait/main.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "fmt" "log" + "log/slog" "net/http" "os" "strings" @@ -12,42 +13,36 @@ import ( "github.com/google/go-containerregistry/pkg/v1/random" "github.com/imjasonh/delay/pkg/delay" + "github.com/imjasonh/gcpslog" "github.com/imjasonh/kontain.me/pkg/serve" ) const queueName = "wait-queue" func main() { - delay.Init() ctx := context.Background() st, err := serve.NewStorage(ctx) if err != nil { - log.Fatalf("serve.NewStorage: %v", err) + slog.Error("serve.NewStorage", "err", err) + os.Exit(1) } - http.Handle("/v2/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - storage: st, - }) - http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/wait", http.StatusSeeOther)) + http.Handle("/v2/", gcpslog.WithCloudTraceContext(&server{storage: st})) + http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/main/cmd/random", http.StatusSeeOther)) - log.Println("Starting...") + slog.Info("Starting...") port := os.Getenv("PORT") if port == "" { port = "8080" - log.Printf("Defaulting to port %s", port) + slog.Info("Defaulting port", "port", port) } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) + slog.Info("Listening", "port", port) + slog.Error("ListenAndServe", "err", http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) } -type server struct { - info, error *log.Logger - storage *serve.Storage -} +type server struct{ storage *serve.Storage } func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) + slog.Info("handler", "method", r.Method, "url", r.URL) path := strings.TrimPrefix(r.URL.String(), "/v2/") switch { @@ -89,7 +84,7 @@ func (s *server) serveWaitManifest(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(tagOrDigest, "sha256:") { desc, err := s.storage.BlobExists(ctx, tagOrDigest) if err != nil { - s.error.Printf("ERROR (storage.BlobExists): %s", err) + slog.Error("storage.BlobExists", "err", err) serve.Error(w, serve.ErrNotFound) return } @@ -106,7 +101,7 @@ func (s *server) serveWaitManifest(w http.ResponseWriter, r *http.Request) { // The image has already been built; serve it. ck := cacheKey(name) if _, err := s.storage.BlobExists(ctx, ck); err == nil { - s.info.Printf("blob %q exists, serving", ck) + slog.Info("blob exists", "ck", ck) serve.Blob(w, r, ck) return } @@ -115,7 +110,7 @@ func (s *server) serveWaitManifest(w http.ResponseWriter, r *http.Request) { // contents. phn := fmt.Sprintf("placeholder-%s", ck) if _, err := s.storage.BlobExists(ctx, phn); err == nil { - s.info.Printf("placeholder %q exists", phn) + slog.Info("placeholder exists", "phn", phn) serve.Error(w, fmt.Errorf("waiting for image...")) return } @@ -127,30 +122,30 @@ func (s *server) serveWaitManifest(w http.ResponseWriter, r *http.Request) { } dur, err := time.ParseDuration(tag) if err != nil { - s.error.Println(err) + slog.Error("time.ParseDuration", "tag", tag, "err", err) serve.Error(w, err) return } if dur > time.Hour { err := fmt.Errorf("duration > 1h (%s)", dur) - s.error.Println(err) + slog.Error("duration > 1h", "tag", tag, "err", err) serve.Error(w, err) return } - s.info.Printf("generating random image %q in %s", ck, dur) + slog.Info("generating random image", "ck", ck, "dur", dur) // Enqueue the task for later. if err := laterFunc.Call(ctx, r, queueName, delay.WithArgs(ck), delay.WithDelay(dur)); err != nil { - s.error.Printf("ERROR (laterFunc.Call): %v", err) + slog.Error("laterFunc.Call", "err", err) serve.Error(w, err) return } // Write the placeholder object. if err := s.storage.WriteObject(ctx, phn, fmt.Sprintf("serving image at %s", time.Now().Add(dur))); err != nil { - s.error.Printf("ERROR (storage.WriteObject): %v", err) + slog.Error("storage.WriteObject", "err", err) serve.Error(w, err) return } diff --git a/cmd/wait/test.sh b/cmd/wait/test.sh index d611290..1cf2b01 100755 --- a/cmd/wait/test.sh +++ b/cmd/wait/test.sh @@ -2,6 +2,10 @@ set -euxo pipefail +exit 0 # TODO + + + uid=image-$(date +%s) crane manifest wait.kontain.me/${uid} || true sleep 11 # for good measure diff --git a/cors.json b/cors.json deleted file mode 100644 index c196139..0000000 --- a/cors.json +++ /dev/null @@ -1,6 +0,0 @@ -[{ - "origin": ["*"], - "responseHeader": ["*"], - "method": ["GET","HEAD"], - "maxAgeSeconds": 3600 -}] diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index fea58c6..0000000 --- a/deploy.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -find cmd -name 'deploy.sh' -print0 | - while IFS= read -r -d '' line; do - bash -c "$line" - done - -gsutil cors set cors.json gs://kontaindotme diff --git a/go.mod b/go.mod index af96cc7..36edfa1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/imjasonh/kontain.me -go 1.20 +go 1.21.0 require ( chainguard.dev/apko v0.7.3 @@ -11,6 +11,7 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/google/ko v0.13.0 github.com/imjasonh/delay v0.0.0-20210102151318-8339250e8458 + github.com/imjasonh/gcpslog v0.0.0-20231112043542-653be1bb0599 github.com/tmc/dot v0.0.0-20210901225022-f9bc17da75c0 golang.org/x/mod v0.10.0 golang.org/x/oauth2 v0.7.0 diff --git a/go.sum b/go.sum index 75cac04..ff0f83c 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,7 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1 cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -71,12 +72,14 @@ github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:W github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/vfs v1.0.0 h1:AUZUgulCDzbaNjTRWEP45X7m/J10brAptZpSRKRZBZc= github.com/blang/vfs v1.0.0/go.mod h1:jjuNUc/IKcRNNWC9NUCvz4fR9PZLPIKxEygtPs/4tSI= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -260,6 +263,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -297,6 +301,8 @@ github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/imjasonh/delay v0.0.0-20210102151318-8339250e8458 h1:KSkVhq4WEv9rlP6GvAbJFRMb0WtPSUOYiFyk52SoqMU= github.com/imjasonh/delay v0.0.0-20210102151318-8339250e8458/go.mod h1:Fa7ozpRszH4I+tsBQpsszFbSKnzIXPCvv2RorJijAMg= +github.com/imjasonh/gcpslog v0.0.0-20231112043542-653be1bb0599 h1:c6Rv/SECa61p+kUmHDdRndQkfYkDjXec/9xu9BiIt9g= +github.com/imjasonh/gcpslog v0.0.0-20231112043542-653be1bb0599/go.mod h1:hfhJTxc/iAHDaUCOYnKEn6l4m9AjNSyQi8CFgJB/BHk= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -306,6 +312,7 @@ github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= +github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -341,6 +348,7 @@ github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kN github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -369,10 +377,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e h1:51xcRlSMBU5rhM9KahnJGfEsBPVPz3182TgFRowA8yY= github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e/go.mod h1:tcaRap0jS3eifrEEllL6ZMd9dg8IlDpi2S1oARrQ+NI= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -382,6 +394,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= @@ -420,10 +433,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/theupdateframework/go-tuf v0.5.2 h1:habfDzTmpbzBLIFGWa2ZpVhYvFBoK0C1onC3a4zuPRA= github.com/theupdateframework/go-tuf v0.5.2/go.mod h1:SyMV5kg5n4uEclsyxXJZI2UxPFJNDc4Y+r7wv+MlvTA= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/tmc/dot v0.0.0-20210901225022-f9bc17da75c0 h1:hwIpbdjckSFqmZ6hod7WZgGR7tVVrSUzZrBfNZl7AOg= @@ -462,7 +477,9 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= +go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= +go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/build v0.0.0-20230424202922-5a08e2b03d59 h1:SzDmQk385MC0IxV6v7pcDQ3hI8AT2dFMzHmwZaaPGfo= @@ -839,6 +856,7 @@ gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= +gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.tf b/main.tf index d29eac6..2cc4cc2 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,7 @@ terraform { backend "gcs" { - bucket = "kontaindotme-tfstate" - prefix = "terraform/state" + bucket = "kontaindotme-tfstate" + prefix = "terraform/state" } } @@ -72,15 +72,14 @@ resource "google_dns_managed_zone" "zone" { name = "kontainme-zone" dns_name = "${var.domain}." - depends_on = [ - google_project_service.dns-api, - ] + depends_on = [google_project_service.dns-api] } // Enable Cloud DNS API. resource "google_project_service" "dns-api" { - project = var.project_id - service = "dns.googleapis.com" + project = var.project_id + service = "dns.googleapis.com" + disable_on_destroy = false } variable "dns_zone" { @@ -121,99 +120,12 @@ resource "google_dns_record_set" "root-aaaa-record" { ] } -locals { - // If var.dns_zone is set, use it. Otherwise, use the managed zone. - dns_zone = var.dns_zone != "" ? var.dns_zone : google_dns_managed_zone.zone[0].name +// If var.dns_zone is set, use it. Otherwise, use the managed zone. +locals { dns_zone = var.dns_zone != "" ? var.dns_zone : google_dns_managed_zone.zone[0].name } - apps = { - "apko" : { - cpu = 1 - ram = "512Mi" - container_concurrency = 1 - timeout_seconds = 900 # 15m - base_image = "cgr.dev/chainguard/static:latest-glibc" - }, - "buildpack" : { - cpu = 2 - ram = "4Gi" - container_concurrency = 1 - timeout_seconds = 900 # 15m - base_image = "gcr.io/buildpacks/builder" - }, - "flatten" : { - cpu = 1 - ram = "1Gi" - container_concurrency = 80 - timeout_seconds = 120 # 2m - base_image = "cgr.dev/chainguard/static:latest-glibc" - }, - "kaniko" : { - cpu = 2 - ram = "4Gi" - container_concurrency = 1 - timeout_seconds = 900 # 15m - base_image = "gcr.io/kaniko-project/executor:debug" - }, - "ko" : { - cpu = 2 - ram = "4Gi" - container_concurrency = 1 - timeout_seconds = 900 # 15m - base_image = "golang" - # After https://github.com/chainguard-images/images/pull/511 - #base_image = "cgr.dev/chainguard/go:latest-dev" - }, - "mirror" : { - cpu = 2 - ram = "4Gi" - container_concurrency = 1 - timeout_seconds = 900 # 15m - base_image = "cgr.dev/chainguard/static:latest-glibc" - }, - "random" : { - cpu = 1 - ram = "256Mi" - container_concurrency = 1000 - timeout_seconds = 60 # 1m - base_image = "cgr.dev/chainguard/static:latest-glibc" - }, - "wait" : { - cpu = 1 - ram = "1Gi" - container_concurrency = 80 - timeout_seconds = 60 # 1m - base_image = "cgr.dev/chainguard/static:latest-glibc" - }, - } -} - -module "app" { - for_each = local.apps - source = "./module" - - project_id = var.project_id - location = var.location - domain = var.domain - dns_zone = local.dns_zone - bucket = google_storage_bucket.bucket.name - service_account_name = google_service_account.service_account.email - - name = each.key - base_image = each.value.base_image - cpu = each.value.cpu - ram = each.value.ram - container_concurrency = each.value.container_concurrency - timeout_seconds = each.value.timeout_seconds -} - -output "cloudrun_url" { - value = { - for k, v in local.apps : k => module.app[k].cloudrun_url - } -} - -output "vanity_url" { - value = { - for k, v in local.apps : k => module.app[k].vanity_url - } +// Enable Cloud Run API. +resource "google_project_service" "run-api" { + project = var.project_id + service = "run.googleapis.com" + disable_on_destroy = false } diff --git a/module/main.tf b/module/main.tf index e541df5..a88ad8c 100644 --- a/module/main.tf +++ b/module/main.tf @@ -1,11 +1,7 @@ terraform { required_providers { - ko = { - source = "ko-build/ko" - } - google = { - source = "hashicorp/google" - } + ko = { source = "ko-build/ko" } + google = { source = "hashicorp/google" } } } @@ -43,24 +39,12 @@ resource "google_cloud_run_service" "service" { } } } - - depends_on = [ - google_project_service.run-api, - ] -} - -// Enable Cloud Run API. -resource "google_project_service" "run-api" { - project = var.project_id - service = "run.googleapis.com" } data "google_iam_policy" "noauth" { binding { - role = "roles/run.invoker" - members = [ - "allUsers", - ] + role = "roles/run.invoker" + members = ["allUsers"] } } @@ -93,3 +77,11 @@ resource "google_dns_record_set" "dns-record" { rrdatas = [google_cloud_run_domain_mapping.mapping.status[0].resource_records[0].rrdata] } + +locals { test_script = file("${path.module}/../cmd/${var.name}/test.sh") } + +resource "null_resource" "test" { + count = fileexists(local.test_script) != "" ? 1 : 0 + depends_on = [google_dns_record_set.dns-record] + provisioner "local-exec" { command = local.test_script } +} diff --git a/module/variables.tf b/module/variables.tf index 38025ca..1c942f2 100644 --- a/module/variables.tf +++ b/module/variables.tf @@ -26,6 +26,7 @@ variable "dns_zone" { variable "location" { type = string description = "The location of the Cloud Run service." + default = "us-east4" } variable "service_account_name" { diff --git a/pkg/run/run.go b/pkg/run/run.go index fb9b7a2..633f3ae 100644 --- a/pkg/run/run.go +++ b/pkg/run/run.go @@ -4,17 +4,17 @@ import ( "bytes" "fmt" "io" + "os" "os/exec" ) -func Do(stdout io.Writer, command string) error { +func Do(command string) error { var out bytes.Buffer cmd := exec.Command("sh", "-c", command) - cmd.Stdout = io.MultiWriter(stdout, &out) - cmd.Stderr = io.MultiWriter(stdout, &out) + cmd.Stdout = io.MultiWriter(os.Stdout, &out) + cmd.Stderr = io.MultiWriter(os.Stderr, &out) if err := cmd.Run(); err != nil { return fmt.Errorf("Error running %q: %s\n=====Command output=====\n%s", command, err, string(out.Bytes())) - return err } return nil } diff --git a/test.sh b/test.sh deleted file mode 100755 index 9b81acb..0000000 --- a/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -find cmd -name 'test.sh' -print0 | - while IFS= read -r -d '' line; do - bash -c "$line" - done