imjasonh/kontain.me
1
0
Fork 0
mirror of https://github.com/imjasonh/kontain.me synced 2026-07-16 20:34:22 +00:00
Code Activity

resolve TODOs in pkg/serve

Browse source
This commit is contained in:
Jason Hall 2020-12-31 07:44:38 -05:00
parent d0af4eaa9b
commit ce2d2cc45c
7 changed files with 122 additions and 108 deletions
Download patch file Download diff file Expand all files Collapse all files

19
cmd/buildpack/main.go
View file

@ -39,9 +39,15 @@ var commitRE = regexp.MustCompile("[a-f0-9]{40}")
const base = "packs/run:v3alpha2"
func main() {
ctx := context.Background()
st, err := serve.NewStorage(ctx)
if err != nil {
log.Fatalf("serve.NewStorage: %v", err)
}
http.Handle("/v2/", &server{
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
storage: st,
})
http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/master/cmd/buildpack", http.StatusSeeOther))
@ -57,6 +63,7 @@ func main() {
type server struct {
info, error *log.Logger
storage *serve.Storage
}
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -81,6 +88,8 @@ func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
func (s *server) serveBuildpackManifest(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
// Prepare workspace.
src, layers, err := s.prepareWorkspace()
if err != nil {
@ -116,7 +125,7 @@ func (s *server) serveBuildpackManifest(w http.ResponseWriter, r *http.Request)
// image tag.
if commitRE.MatchString(revision) {
ck := cacheKey(path, revision)
if err := serve.BlobExists(ck); err == nil {
if err := s.storage.BlobExists(ctx, ck); err == nil {
s.info.Println("serving cached manifest:", ck)
serve.Blob(w, r, ck)
return
@ -151,8 +160,8 @@ func (s *server) serveBuildpackManifest(w http.ResponseWriter, r *http.Request)
// Serve the manifest.
ck := cacheKey(path, revision)
if err := serve.Manifest(w, r, img, ck); err != nil {
s.error.Printf("ERROR (serve.Manifest): %v", err)
if err := s.storage.ServeManifest(w, r, img, ck); err != nil {
s.error.Printf("ERROR (storage.ServeManifest): %v", err)
serve.Error(w, err)
return
}

37
cmd/flatten/main.go
View file

@ -1,20 +1,7 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"io"
"log"
@ -34,9 +21,15 @@ import (
)
func main() {
ctx := context.Background()
st, err := serve.NewStorage(ctx)
if err != nil {
log.Fatalf("serve.NewStorage: %v", err)
}
http.Handle("/v2/", &server{
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
storage: st,
})
http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/master/cmd/flatten", http.StatusSeeOther))
@ -52,6 +45,7 @@ func main() {
type server struct {
info, error *log.Logger
storage *serve.Storage
}
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -83,6 +77,7 @@ func cacheKey(orig string) string {
// flatten.kontain.me/ubuntu -> flatten ubuntu and serve
func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
path := strings.TrimPrefix(r.URL.Path, "/v2/")
parts := strings.Split(path, "/")
@ -115,7 +110,7 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) {
// Check if we have a flattened manifest cached, and if so serve it
// directly.
ck := cacheKey(d.Digest.String())
if err := serve.BlobExists(ck); err == nil {
if err := s.storage.BlobExists(ctx, ck); err == nil {
s.info.Println("serving cached manifest:", ck)
serve.Blob(w, r, ck)
return
@ -167,8 +162,8 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) {
return
}
fidx := mutate.AppendManifests(empty.Index, adds...)
if err := serve.Index(w, r, fidx, ck); err != nil {
s.error.Printf("ERROR (serve.Index): %v", err)
if err := s.storage.ServeIndex(w, r, fidx, ck); err != nil {
s.error.Printf("ERROR (storage.ServeIndex): %v", err)
serve.Error(w, err)
return
}
@ -187,8 +182,8 @@ func (s *server) serveFlattenManifest(w http.ResponseWriter, r *http.Request) {
return
}
if err := serve.Manifest(w, r, fimg, ck); err != nil {
s.error.Printf("ERROR (serve.Manifest): %v", err)
if err := s.storage.ServeManifest(w, r, fimg, ck); err != nil {
s.error.Printf("ERROR (storage.ServeManifest): %v", err)
serve.Error(w, err)
return
}

19
cmd/kaniko/main.go
View file

@ -38,9 +38,15 @@ var commitRE = regexp.MustCompile("[a-f0-9]{40}")
const base = "packs/run:v3alpha2"
func main() {
ctx := context.Background()
st, err := serve.NewStorage(ctx)
if err != nil {
log.Fatalf("serve.NewStorage: %v", err)
}
http.Handle("/v2/", &server{
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
storage: st,
})
http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/master/cmd/kaniko", http.StatusSeeOther))
@ -56,6 +62,7 @@ func main() {
type server struct {
info, error *log.Logger
storage *serve.Storage
}
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -80,6 +87,8 @@ func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
func (s *server) serveKanikoManifest(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
// Prepare workspace.
if err := s.prepareWorkspace(); err != nil {
s.error.Println("ERROR:", err)
@ -110,7 +119,7 @@ func (s *server) serveKanikoManifest(w http.ResponseWriter, r *http.Request) {
// image tag.
if commitRE.MatchString(revision) {
ck := cacheKey(path, revision)
if err := serve.BlobExists(ck); err == nil {
if err := s.storage.BlobExists(ctx, ck); err == nil {
s.info.Println("serving cached manifest:", ck)
serve.Blob(w, r, ck)
return
@ -145,8 +154,8 @@ func (s *server) serveKanikoManifest(w http.ResponseWriter, r *http.Request) {
// Serve the manifest.
ck := cacheKey(path, revision)
if err := serve.Manifest(w, r, img, ck); err != nil {
s.error.Printf("ERROR (serve.Manifest): %v", err)
if err := s.storage.ServeManifest(w, r, img, ck); err != nil {
s.error.Printf("ERROR (storage.ServeManifest): %v", err)
serve.Error(w, err)
return
}

33
cmd/ko/main.go
View file

@ -1,17 +1,3 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
@ -37,9 +23,15 @@ import (
)
func main() {
ctx := context.Background()
st, err := serve.NewStorage(ctx)
if err != nil {
log.Fatalf("serve.NewStorage: %v", err)
}
http.Handle("/v2/", &server{
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
storage: st,
})
http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/master/cmd/ko", http.StatusSeeOther))
@ -55,6 +47,7 @@ func main() {
type server struct {
info, error *log.Logger
storage *serve.Storage
}
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -125,15 +118,15 @@ func (s *server) serveKoManifest(w http.ResponseWriter, r *http.Request) {
return
}
if idx, ok := br.(v1.ImageIndex); ok {
if err := serve.Index(w, r, idx); err != nil {
s.error.Printf("ERROR (serve.Index): %v", err)
if err := s.storage.ServeIndex(w, r, idx); err != nil {
s.error.Printf("ERROR (storage.ServeIndex): %v", err)
serve.Error(w, err)
}
return
}
if img, ok := br.(v1.Image); ok {
if err := serve.Manifest(w, r, img); err != nil {
s.error.Printf("ERROR (serve.Manifest): %v", err)
if err := s.storage.ServeManifest(w, r, img); err != nil {
s.error.Printf("ERROR (storage.ServeManifest): %v", err)
serve.Error(w, err)
}
return

37
cmd/mirror/main.go
View file

@ -1,20 +1,7 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"log"
"net/http"
@ -28,9 +15,15 @@ import (
)
func main() {
ctx := context.Background()
st, err := serve.NewStorage(ctx)
if err != nil {
log.Fatalf("serve.NewStorage: %v", err)
}
http.Handle("/v2/", &server{
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
storage: st,
})
http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/master/cmd/mirror", http.StatusSeeOther))
@ -46,6 +39,7 @@ func main() {
type server struct {
info, error *log.Logger
storage *serve.Storage
}
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -73,6 +67,7 @@ func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// mirror.kontain.me/ubuntu -> mirror ubuntu and serve
func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
path := strings.TrimPrefix(r.URL.Path, "/v2/")
parts := strings.Split(path, "/")
@ -102,7 +97,7 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) {
serve.Error(w, err)
return
}
if err := serve.BlobExists(d.Digest.String()); err == nil {
if err := s.storage.BlobExists(ctx, d.Digest.String()); err == nil {
serve.Blob(w, r, d.Digest.String())
return
} else {
@ -121,8 +116,8 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) {
serve.Error(w, err)
return
}
if err := serve.Index(w, r, idx); err != nil {
s.error.Printf("ERROR (serve.Index): %v", err)
if err := s.storage.ServeIndex(w, r, idx); err != nil {
s.error.Printf("ERROR (storage.ServeIndex): %v", err)
serve.Error(w, err)
return
}
@ -135,8 +130,8 @@ func (s *server) serveMirrorManifest(w http.ResponseWriter, r *http.Request) {
serve.Error(w, err)
return
}
if err := serve.Manifest(w, r, img); err != nil {
s.error.Printf("ERROR (serve.Manifest): %v", err)
if err := s.storage.ServeManifest(w, r, img); err != nil {
s.error.Printf("ERROR (storage.ServeManifest): %v", err)
serve.Error(w, err)
return
}

14
cmd/random/main.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"log"
"net/http"
@ -14,9 +15,15 @@ import (
)
func main() {
ctx := context.Background()
st, err := serve.NewStorage(ctx)
if err != nil {
log.Fatalf("serve.NewStorage: %v", err)
}
http.Handle("/v2/", &server{
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
info: log.New(os.Stdout, "I ", log.Ldate|log.Ltime|log.Lshortfile),
error: log.New(os.Stderr, "E ", log.Ldate|log.Ltime|log.Lshortfile),
storage: st,
})
http.Handle("/", http.RedirectHandler("https://github.com/imjasonh/kontain.me/blob/master/cmd/random", http.StatusSeeOther))
@ -32,6 +39,7 @@ func main() {
type server struct {
info, error *log.Logger
storage *serve.Storage
}
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -83,7 +91,7 @@ func (s *server) serveRandomManifest(w http.ResponseWriter, r *http.Request) {
serve.Error(w, err)
return
}
if err := serve.Manifest(w, r, img); err != nil {
if err := s.storage.ServeManifest(w, r, img); err != nil {
s.error.Printf("ERROR (serve.Manifest): %v", err)
serve.Error(w, err)
return

71
pkg/serve/serve.go
View file

@ -9,6 +9,7 @@ import (
"log"
"net/http"
"os"
"time"
"cloud.google.com/go/storage"
v1 "github.com/google/go-containerregistry/pkg/v1"
@ -23,26 +24,31 @@ func Blob(w http.ResponseWriter, r *http.Request, name string) {
http.Redirect(w, r, url, http.StatusSeeOther)
}
func BlobExists(name string) error {
ctx := context.Background() // TODO
type Storage struct {
client *storage.Client
}
func NewStorage(ctx context.Context) (*Storage, error) {
client, err := storage.NewClient(ctx)
if err != nil {
return fmt.Errorf("NewClient: %v", err)
return nil, fmt.Errorf("NewClient: %v", err)
}
_, err = client.Bucket(bucket).Object(fmt.Sprintf("blobs/%s", name)).Attrs(ctx)
return &Storage{client}, nil
}
func (s *Storage) BlobExists(ctx context.Context, name string) error {
_, err := s.client.Bucket(bucket).Object(fmt.Sprintf("blobs/%s", name)).Attrs(ctx)
return err
}
func writeBlob(name string, h v1.Hash, rc io.ReadCloser, contentType string) error {
ctx := context.Background() // TODO
client, err := storage.NewClient(ctx)
if err != nil {
return fmt.Errorf("NewClient: %v", err)
}
func (s *Storage) writeBlob(ctx context.Context, name string, h v1.Hash, rc io.ReadCloser, contentType string) error {
start := time.Now()
defer func() { log.Printf("writeBlob(%q) took %s", name, time.Since(start)) }()
// The DoesNotExist precondition can be hit when writing or flushing
// data, which can happen any of three places. Anywhere it happens,
// just ignore the error since that means the blob already exists.
w := client.Bucket(bucket).Object(fmt.Sprintf("blobs/%s", name)).
w := s.client.Bucket(bucket).Object(fmt.Sprintf("blobs/%s", name)).
If(storage.Conditions{DoesNotExist: true}).
NewWriter(ctx)
w.ObjectAttrs.ContentType = contentType
@ -65,14 +71,14 @@ func writeBlob(name string, h v1.Hash, rc io.ReadCloser, contentType string) err
}
return fmt.Errorf("w.Close: %v", err)
}
log.Printf("wrote blob %s", name)
return nil
}
// Index writes manifest, config and layer blobs for each image in the index,
// then writes and redirects to the index manifest contents pointing to those
// blobs.
func Index(w http.ResponseWriter, r *http.Request, idx v1.ImageIndex, also ...string) error {
// ServeIndex writes manifest, config and layer blobs for each image in the
// index, then writes and redirects to the index manifest contents pointing to
// those blobs.
func (s *Storage) ServeIndex(w http.ResponseWriter, r *http.Request, idx v1.ImageIndex, also ...string) error {
ctx := r.Context()
im, err := idx.IndexManifest()
if err != nil {
return err
@ -85,7 +91,7 @@ func Index(w http.ResponseWriter, r *http.Request, idx v1.ImageIndex, also ...st
if err != nil {
return err
}
return writeImage(img)
return s.writeImage(ctx, img)
})
}
if err := g.Wait(); err != nil {
@ -105,16 +111,14 @@ func Index(w http.ResponseWriter, r *http.Request, idx v1.ImageIndex, also ...st
if err != nil {
return err
}
log.Printf("writing index manifest %q", digest)
if err := writeBlob(digest.String(), digest, ioutil.NopCloser(bytes.NewReader(b)), string(mt)); err != nil {
if err := s.writeBlob(ctx, digest.String(), digest, ioutil.NopCloser(bytes.NewReader(b)), string(mt)); err != nil {
return err
}
for _, a := range also {
a := a
g.Go(func() error {
log.Printf("writing index manifest %q also to %q", digest, a)
return writeBlob(a, digest, ioutil.NopCloser(bytes.NewReader(b)), string(mt))
return s.writeBlob(ctx, a, digest, ioutil.NopCloser(bytes.NewReader(b)), string(mt))
})
}
if err := g.Wait(); err != nil {
@ -127,7 +131,7 @@ func Index(w http.ResponseWriter, r *http.Request, idx v1.ImageIndex, also ...st
}
// writeImage writes the layer blobs, config blob and manifest.
func writeImage(img v1.Image, also ...string) error {
func (s *Storage) writeImage(ctx context.Context, img v1.Image, also ...string) error {
// Write config blob for later serving.
ch, err := img.ConfigName()
if err != nil {
@ -137,8 +141,7 @@ func writeImage(img v1.Image, also ...string) error {
if err != nil {
return err
}
log.Printf("writing config blob %q", ch)
if err := writeBlob(ch.String(), ch, ioutil.NopCloser(bytes.NewReader(cb)), ""); err != nil {
if err := s.writeBlob(ctx, ch.String(), ch, ioutil.NopCloser(bytes.NewReader(cb)), "application/json"); err != nil {
return err
}
@ -159,8 +162,11 @@ func writeImage(img v1.Image, also ...string) error {
if err != nil {
return err
}
log.Printf("writing layer blob %q", lh)
return writeBlob(lh.String(), lh, rc, "")
mt, err := l.MediaType()
if err != nil {
return err
}
return s.writeBlob(ctx, lh.String(), lh, rc, string(mt))
})
}
if err := g.Wait(); err != nil {
@ -180,25 +186,24 @@ func writeImage(img v1.Image, also ...string) error {
if err != nil {
return err
}
log.Printf("writing image manifest %q", digest)
if err := writeBlob(digest.String(), digest, ioutil.NopCloser(bytes.NewReader(b)), string(mt)); err != nil {
if err := s.writeBlob(ctx, digest.String(), digest, ioutil.NopCloser(bytes.NewReader(b)), string(mt)); err != nil {
return err
}
for _, a := range also {
a := a
g.Go(func() error {
log.Printf("writing image manifest %q also to %q", digest, a)
return writeBlob(a, digest, ioutil.NopCloser(bytes.NewReader(b)), string(mt))
return s.writeBlob(ctx, a, digest, ioutil.NopCloser(bytes.NewReader(b)), string(mt))
})
}
return g.Wait()
return nil
}
// Manifest writes config and layer blobs for the image, then writes and
// ServeManifest writes config and layer blobs for the image, then writes and
// redirects to the image manifest contents pointing to those blobs.
func Manifest(w http.ResponseWriter, r *http.Request, img v1.Image, also ...string) error {
if err := writeImage(img, also...); err != nil {
func (s *Storage) ServeManifest(w http.ResponseWriter, r *http.Request, img v1.Image, also ...string) error {
ctx := r.Context()
if err := s.writeImage(ctx, img, also...); err != nil {
return err
}