diff --git a/cmd/api/README.md b/cmd/api/README.md deleted file mode 100644 index 07cb719..0000000 --- a/cmd/api/README.md +++ /dev/null @@ -1,90 +0,0 @@ -## What is this? - -An experimental GCB API-compatible buildpack service running on Cloud Run. A GCB -user can use any recently released `gcloud` along with this service to upload -and build buildpack-compatible source and produce a container image. - -Any requested `steps` are ignored and a buildpack build is executed on the -source instead. The request must specify exactly one image to build in `images`, -and must specify a `storageSource`. - -This is **an experiment** and should absolutely not be used for anything serious. - -## How do I use it? - -First, get into a local directory containing buildpack-detectable source: - -``` -$ git clone git@github.com:buildpack/sample-java-app.git -$ cd sample-java-app -``` - -Then, by using `gcloud` and overriding the address where API requests are sent, -you can create Build requests that execute buildpacks builds: - -``` -$ CLOUDSDK_API_ENDPOINT_OVERRIDES_CLOUDBUILD=https://api-an3qnndwmq-uc.a.run.app/ gcloud builds submit --tag=gcr.io/my-project/built -Creating temporary tarball archive of 15 file(s) totalling 91.8 KiB before compression. -Some files were not included in the source upload. - -Check the gcloud log [/Users/jasonhall/.config/gcloud/logs/2019.05.16/00.35.06.407646.log] to see which files and the contents of the -default gcloudignore file used (see `$ gcloud topic gcloudignore` to learn -more). - -Uploading tarball of [.] to [gs://my-project_cloudbuild/source/1557981306.47-9ee5987ef42e4dc988d7dcd4a4dc0bdc.tgz] -Created [https://api-an3qnndwmq-uc.a.run.app/v1/projects/my-project/builds/a33da1cc-8e3c-4579-92cd-d7bab749ba22]. - -... snip ... -ID CREATE_TIME DURATION SOURCE IMAGES STATUS -a33da1cc-8e3c-4579-92cd-d7bab749ba22 2019-05-16T04:35:07+00:00 1M6S gs://my-project_cloudbuild/source/1557981306.47-9ee5987ef42e4dc988d7dcd4a4dc0bdc.tgz gcr.io/my-project/built SUCCESS -``` - -You can also get build details: - -``` -$ CLOUDSDK_API_ENDPOINT_OVERRIDES_CLOUDBUILD=https://api-an3qnndwmq-uc.a.run.app/ gcloud builds describe a33da1cc-8e3c-4579-92cd-d7bab749ba22 -createTime: '2019-05-16T04:35:07.426525401Z' -finishTime: '2019-05-16T04:36:13.70673836Z' -id: a33da1cc-8e3c-4579-92cd-d7bab749ba22 -images: -- gcr.io/my-project/built -logsBucket: my-project_cloudbuild -projectId: my-project -results: - images: - - digest: sha256:de35ebf2e6e39bc7e2047bc261095435dd6b710ff09af38edcb059e640e8c35e - name: gcr.io/my-project/built -source: - storageSource: - bucket: my-project_cloudbuild - generation: '1557981307124794' - object: source/1557981306.47-9ee5987ef42e4dc988d7dcd4a4dc0bdc.tgz -startTime: '2019-05-16T04:35:07.426525401Z' -status: SUCCESS -statusDetail: '' -``` - -## Known differences - -- Builds are performed entirely in the context of the `projects.builds.create` - request, not by polling a long-running operation. The `--async` flag has no -effect. -- Build operations (source pulls and image pushes) are authorized using the - end-user credentials, not the project's builder service account. -- Build logs are written to Cloud Storage in one shot, at the end of the build. - When the build request completes, `gcloud` will show all build logs at once -without streaming. -- Build logs are written to the source upload bucket, and not a separate logs - bucket as is the default in GCB. -- Builds cannot be cancelled. The client doesn't know the build ID until it's - completed. - -## Not yet implemented - -- [ ] `timing` is not collected or reported. -- [ ] `timeout` is not configurable. If Cloud Run request times out, client - gets a 502. -- [ ] `sourceProvenance` is not yet collected or reported, or uploaded to a - Grafeas instance. -- [ ] `projects.builds.list` is not yet implemented. -- [ ] `operations.get` and `operations.list` are not yet implemented. diff --git a/cmd/api/main.go b/cmd/api/main.go deleted file mode 100644 index cc851ca..0000000 --- a/cmd/api/main.go +++ /dev/null @@ -1,321 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "regexp" - "strings" - "time" - - "cloud.google.com/go/compute/metadata" - "cloud.google.com/go/datastore" - "cloud.google.com/go/storage" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/uuid" - "github.com/imjasonh/kontain.me/pkg/run" - "golang.org/x/oauth2" - gcb "google.golang.org/api/cloudbuild/v1" - "google.golang.org/api/option" -) - -const base = "gcr.io/buildpacks/gcp/run:v1" - -var ( - projectRE = regexp.MustCompile("/v1/projects/([a-z0-9-]+)/") - buildRE = regexp.MustCompile("/v1/projects/[a-z0-9-]+/builds/([a-z0-9-]+)") -) - -func main() { - ctx := context.Background() - projectID, err := metadata.ProjectID() - if err != nil { - log.Fatalf("metadata.ProjectID: %v", err) - } - ds, err := datastore.NewClient(ctx, projectID) - if err != nil { - log.Fatalf("datastore.NewClient: %v", err) - } - - http.Handle("/v1/projects/", &server{ - info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile), - error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile), - ds: ds, - }) - http.Handle("/", http.RedirectHandler("https://kontain.me", http.StatusMovedPermanently)) - - log.Println("Starting...") - port := os.Getenv("PORT") - if port == "" { - port = "8080" - log.Printf("Defaulting to port %s", port) - } - log.Printf("Listening on port %s", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil)) -} - -type server struct { - info, error *log.Logger - ds *datastore.Client -} - -func extractPath(path string, re *regexp.Regexp) string { - found := re.FindStringSubmatch(path) - if len(found) < 2 { - return "" - } - return found[1] -} - -func extractToken(r *http.Request) string { - hdr := r.Header.Get("Authorization") - if strings.HasPrefix(hdr, "Bearer ") { - return strings.TrimPrefix(hdr, "Bearer ") - } - return r.URL.Query().Get("access_token") -} - -func (s *server) logWriter(req *gcb.Build, tok string) io.WriteCloser { - ctx := context.Background() // TODO - gcs, err := storage.NewClient(ctx, option.WithTokenSource(oauth2.StaticTokenSource(&oauth2.Token{AccessToken: tok}))) - if err != nil { - log.Fatalf("storage.NewClient: %v", err) - } - return gcs.Bucket(req.LogsBucket).Object(fmt.Sprintf("log-%s.txt", req.Id)).NewWriter(ctx) -} - -func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.info.Println("handler:", r.Method, r.URL) - projectID := extractPath(r.URL.Path, projectRE) - if projectID == "" { - http.Error(w, "missing project", http.StatusBadRequest) - return - } - buildID := extractPath(r.URL.Path, buildRE) - - switch { - case r.Method == http.MethodGet && buildID != "": - s.getBuild(w, r, buildID) - case r.Method == http.MethodPost && buildID == "": - s.createBuild(w, r, projectID) - default: - http.Error(w, "not found", http.StatusNotFound) - } -} - -func (s *server) getBuild(w http.ResponseWriter, r *http.Request, buildID string) { - io.Copy(w, bytes.NewReader(s.get(buildID))) -} - -func (s *server) createBuild(w http.ResponseWriter, r *http.Request, projectID string) { - start := time.Now() - tok := extractToken(r) - if tok == "" { - http.Error(w, "bad auth", http.StatusUnauthorized) - return - } - - defer r.Body.Close() - var req gcb.Build - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - s.error.Printf("json.Decode: %v", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - req.Id = uuid.New().String() - req.ProjectId = projectID - req.CreateTime = start.Format(time.RFC3339Nano) - req.StartTime = req.CreateTime - req.LogsBucket = req.Source.StorageSource.Bucket // TODO: actually write logs somewhere. - - // Do the build... - if err := s.buildImage(&req, tok); err != nil { - req.Status = "FAILURE" - req.StatusDetail = err.Error() - } else { - req.Status = "SUCCESS" - } - req.FinishTime = time.Now().Format(time.RFC3339Nano) - - bomd, err := json.Marshal(&gcb.BuildOperationMetadata{Build: &req}) - if err != nil { - s.error.Printf("json.Encode: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if err := json.NewEncoder(w).Encode(&gcb.Operation{ - Name: base64.StdEncoding.EncodeToString([]byte(req.Id)), - Done: true, - Metadata: bomd, - }); err != nil { - s.error.Printf("Encode: %v", err) - } - s.put(req) -} - -type e struct { - Bytes []byte `datastore:",noindex"` -} - -func (s *server) put(req gcb.Build) { - ctx := context.Background() // TODO - k := datastore.NameKey("Builds", req.Id, nil) - b, err := json.Marshal(req) - if err != nil { - s.error.Printf("json.Marshal: %v", err) - return - } - if _, err := s.ds.Put(ctx, k, &e{b}); err != nil { - s.error.Printf("datastore.Put: %v", err) - } -} - -func (s *server) get(id string) []byte { - ctx := context.Background() // TODO - k := datastore.NameKey("Builds", id, nil) - var e e - if err := s.ds.Get(ctx, k, &e); err != nil { - s.error.Printf("datastore.Get: %v", err) - } - return e.Bytes -} - -func (s *server) buildImage(req *gcb.Build, tok string) error { - // Validate request. - if err := s.validate(req); err != nil { - return err - } - - // Prepare workspace. - src, err := s.prepareWorkspace(tok) - if err != nil { - return err - } - // Clean up workspace. - defer func() { - for _, path := range []string{ - src, os.Getenv("HOME"), - } { - if err := os.RemoveAll(path); err != nil { - s.error.Printf("RemoveAll(%q): %v", path, err) - } - } - os.Setenv("HOME", "/home/") - }() - - // Fetch, detect and build image. - if err := s.fetchAndBuild(src, tok, req); err != nil { - return err - } - - // Get the digest of the image we just pushed. - if img, err := s.getImage(req.Images[0]); err != nil { - return err - } else { - d, err := img.Digest() - if err != nil { - return err - } - req.Results = &gcb.Results{ - Images: []*gcb.BuiltImage{{ - Name: req.Images[0], - Digest: d.String(), - }}, - } - } - return nil -} - -func (s *server) validate(req *gcb.Build) error { - if len(req.Images) != 1 { - return errors.New("must request exactly one image") - } - if req.Source.StorageSource.Bucket == "" || - req.Source.StorageSource.Object == "" { - return errors.New("must request bucket and object") - } - return nil -} - -func (s *server) prepareWorkspace(tok string) (string, error) { - // Create and set $HOME. - home, err := ioutil.TempDir("", "") - if err != nil { - return "", err - } - os.Setenv("HOME", home) - - // Write Docker config with user credentials. - auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("oauth2accesstoken:%s", tok))) - configJSON := fmt.Sprintf(`{ - "auths": { - "https://gcr.io": { - "auth": %q - } - } -}`, auth) - if err := run.Do(s.info.Writer(), "mkdir -p $HOME/.docker/ && cat << EOF > $HOME/.docker/config.json\n"+string(configJSON)+"\nEOF"); err != nil { - return "", err - } - - // Create tempdir to store app source. - return ioutil.TempDir("", "") -} - -func (s *server) fetchAndBuild(src, tok string, req *gcb.Build) error { - image := req.Images[0] - source := fmt.Sprintf("https://storage.googleapis.com/%s/%s?access_token=%s", req.Source.StorageSource.Bucket, req.Source.StorageSource.Object, tok) - w := s.logWriter(req, tok) - defer func() { - if err := w.Close(); err != nil { - s.error.Printf("Closing GCS log: %v", err) - } - }() - - for _, cmd := range []string{ - fmt.Sprintf("mkdir -p /tmp/layers"), - fmt.Sprintf("chown -R %d:%d %s", os.Geteuid(), os.Getgid(), src), - fmt.Sprintf("chown -R %d:%d /tmp/layers", os.Geteuid(), os.Getgid()), - fmt.Sprintf("curl -fsSL %s | tar xz --strip-components=1 -C %s", source, src), - fmt.Sprintf(` -mkdir -p ~/.docker/ && cat > ~/.docker/config.json << EOF -{ - "auths": { - "gcr.io": { - "username": "oauth2accesstoken", - "password": "%s" - } - } -} -EOF`, tok), - fmt.Sprintf("/lifecycle/detector -app=%s -group=/tmp/layers/group.toml -plan=/tmp/layers/plan.toml", src), - fmt.Sprintf("/lifecycle/analyzer -layers=/tmp/layers -group=/tmp/layers/group.toml %s", image), - fmt.Sprintf("/lifecycle/builder -layers=/tmp/layers -app=%s -group=/tmp/layers/group.toml -plan=/tmp/layers/plan.toml", src), - fmt.Sprintf("/lifecycle/exporter -layers=/tmp/layers -app=%s -image=%s -group=/tmp/layers/group.toml %s", src, base, image), - } { - if err := run.Do(io.MultiWriter(s.info.Writer(), w), cmd); err != nil { - return fmt.Errorf("Running %q: %v", cmd, err) - } - } - return nil -} - -func (s *server) getImage(image string) (v1.Image, error) { - ref, err := name.NewTag(image) - if err != nil { - return nil, err - } - return remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) -} diff --git a/go.mod b/go.mod index 608718c..c79328b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.14 require ( cloud.google.com/go v0.60.0 - cloud.google.com/go/datastore v1.1.0 cloud.google.com/go/storage v1.10.0 github.com/docker/docker v1.13.1 // indirect github.com/dustin/go-humanize v1.0.0 @@ -12,7 +11,6 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/google/ko v0.6.0 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible // indirect - github.com/google/uuid v1.1.1 github.com/opencontainers/runc v0.1.1 // indirect github.com/tmc/dot v0.0.0-20180926222610-6d252d5ff882 go.opencensus.io v0.22.4 // indirect