imjasonh/gcsbuf
1
0
Fork 0
mirror of https://github.com/imjasonh/gcsbuf synced 2026-07-19 07:08:12 +00:00
Code Activity

add test binary, observability

Browse source 
Signed-off-by: Jason Hall <imjasonh@gmail.com>
This commit is contained in:
Jason Hall 2026-03-24 14:31:48 -04:00
parent eeb0713f5a
commit f6fd234e2d
6 changed files with 235 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

78
writer.go
View file

@ -30,6 +30,8 @@ type WriterOptions struct {
FlushInterval time.Duration
// FlushSize is the buffer size threshold that triggers a synchronous flush on Write. Default 4MB.
FlushSize int
// OnOp, if set, is called after each GCS API operation completes.
OnOp OnOp
}
type Writer struct {
@ -46,7 +48,10 @@ type Writer struct {
created bool // whether the GCS object exists yet
closed bool
err error
gen int64 // expected generation of the GCS object
gen int64 // expected generation of the GCS object
componentCount int64 // tracked locally from compose results
onOp OnOp
stopCh chan struct{}
doneCh chan struct{}
@ -69,6 +74,7 @@ func NewWriter(ctx context.Context, opts WriterOptions) *Writer {
flushInterval: interval,
flushSize: size,
buf: bytes.NewBuffer(nil),
onOp: opts.OnOp,
stopCh: make(chan struct{}),
doneCh: make(chan struct{}),
}
@ -139,16 +145,21 @@ func (w *Writer) flushLocked() {
if !w.created {
// First write: create the object directly.
// DoesNotExist ensures we don't overwrite an object created by someone else.
gw := objh.If(storage.Conditions{DoesNotExist: true}).NewWriter(w.ctx)
if _, err := gw.Write(data); err != nil {
attrs, err := retry(w.ctx, w.onOp, "upload", func() (*storage.ObjectAttrs, error) {
gw := objh.If(storage.Conditions{DoesNotExist: true}).NewWriter(w.ctx)
if _, err := gw.Write(data); err != nil {
return nil, err
}
if err := gw.Close(); err != nil {
return nil, err
}
return gw.Attrs(), nil
})
if err != nil {
w.err = fmt.Errorf("writing initial object: %w", err)
return
}
if err := gw.Close(); err != nil {
w.err = fmt.Errorf("closing initial object writer: %w", err)
return
}
w.gen = gw.Attrs().Generation
w.gen = attrs.Generation
w.created = true
w.buf.Reset()
return
@ -158,24 +169,19 @@ func (w *Writer) flushLocked() {
tmpname := fmt.Sprintf("%s-temp-%d", w.object, now().UnixNano())
tmpobj := bh.Object(tmpname)
gw := tmpobj.If(storage.Conditions{DoesNotExist: true}).NewWriter(w.ctx)
if _, err := gw.Write(data); err != nil {
if err := retryVoid(w.ctx, w.onOp, "upload", func() error {
gw := tmpobj.If(storage.Conditions{DoesNotExist: true}).NewWriter(w.ctx)
if _, err := gw.Write(data); err != nil {
return err
}
return gw.Close()
}); err != nil {
w.err = fmt.Errorf("writing temp object: %w", err)
return
}
if err := gw.Close(); err != nil {
w.err = fmt.Errorf("closing temp object writer: %w", err)
return
}
// Check component count before composing.
attrs, err := objh.If(storage.Conditions{GenerationMatch: w.gen}).Attrs(w.ctx)
if err != nil {
w.err = fmt.Errorf("getting object attrs: %w", err)
return
}
if attrs.ComponentCount >= maxComponentCount-1 {
// Reset component count: copy to temp2, copy back, delete temp2.
if w.componentCount >= maxComponentCount-1 {
if err := w.resetComponentCount(bh, objh); err != nil {
w.err = fmt.Errorf("resetting component count: %w", err)
return
@ -183,16 +189,20 @@ func (w *Writer) flushLocked() {
}
// Compose existing + temp into existing.
composer := objh.If(storage.Conditions{GenerationMatch: w.gen}).ComposerFrom(objh, tmpobj)
composeAttrs, err := composer.Run(w.ctx)
composeAttrs, err := retry(w.ctx, w.onOp, "compose", func() (*storage.ObjectAttrs, error) {
return objh.If(storage.Conditions{GenerationMatch: w.gen}).ComposerFrom(objh, tmpobj).Run(w.ctx)
})
if err != nil {
w.err = fmt.Errorf("composing objects: %w", err)
return
}
w.gen = composeAttrs.Generation
w.componentCount = composeAttrs.ComponentCount
// Delete temp object.
if err := tmpobj.Delete(w.ctx); err != nil {
if err := retryVoid(w.ctx, w.onOp, "delete", func() error {
return tmpobj.Delete(w.ctx)
}); err != nil {
w.err = fmt.Errorf("deleting temp object: %w", err)
return
}
@ -207,20 +217,27 @@ func (w *Writer) resetComponentCount(bh *storage.BucketHandle, objh *storage.Obj
// Copy to temp (this creates a non-composite object).
// Source precondition ensures we're copying the generation we expect.
if _, err := tmpobj.CopierFrom(objh.If(storage.Conditions{GenerationMatch: w.gen})).Run(w.ctx); err != nil {
if err := retryVoid(w.ctx, w.onOp, "rewrite", func() error {
_, err := tmpobj.CopierFrom(objh.If(storage.Conditions{GenerationMatch: w.gen})).Run(w.ctx)
return err
}); err != nil {
return fmt.Errorf("copying to reset temp: %w", err)
}
// Copy back.
// Destination precondition ensures nobody modified the main object while we copied.
attrs, err := objh.If(storage.Conditions{GenerationMatch: w.gen}).CopierFrom(tmpobj).Run(w.ctx)
attrs, err := retry(w.ctx, w.onOp, "rewrite", func() (*storage.ObjectAttrs, error) {
return objh.If(storage.Conditions{GenerationMatch: w.gen}).CopierFrom(tmpobj).Run(w.ctx)
})
if err != nil {
return fmt.Errorf("copying back from reset temp: %w", err)
}
w.gen = attrs.Generation
// Delete the reset temp.
if err := tmpobj.Delete(w.ctx); err != nil {
if err := retryVoid(w.ctx, w.onOp, "delete", func() error {
return tmpobj.Delete(w.ctx)
}); err != nil {
return fmt.Errorf("deleting reset temp: %w", err)
}
@ -254,8 +271,11 @@ func (w *Writer) Close() error {
// Set done metadata. Only if the object was created.
if w.created {
objh := w.client.Bucket(w.bucket).Object(w.object)
if _, err := objh.If(storage.Conditions{GenerationMatch: w.gen}).Update(w.ctx, storage.ObjectAttrsToUpdate{
Metadata: map[string]string{"done": "true"},
if err := retryVoid(w.ctx, w.onOp, "patch", func() error {
_, err := objh.If(storage.Conditions{GenerationMatch: w.gen}).Update(w.ctx, storage.ObjectAttrsToUpdate{
Metadata: map[string]string{"done": "true"},
})
return err
}); err != nil {
return fmt.Errorf("setting done metadata: %w", err)
}