Add ring buffer overflow detection and metrics

Implement comprehensive ring buffer overflow handling:

- Add dropped_events BPF map to track overflow in kernel space
- Create submit_event() helper that checks bpf_ringbuf_output() return
  value and atomically increments drop counter on failure
- Add Drops() method to read drop counter from eBPF map
- Add snoop_events_dropped_total Prometheus metric
- Track drop deltas and log warnings when overflow occurs
- Include drop count in JSON reports

Ring buffer overflows can occur under high load when events are
generated faster than userspace can consume them. This implementation
provides full observability into overflow events without data loss
of the drop count itself.

Signed-off-by: Jason Hall <jason@chainguard.dev>

cmd/snoop/main.go

@@ -127,26 +127,45 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time
 	startedAt := time.Now()
 	log.Infof("Writing reports to: %s (interval: %s)", reportPath, reportInterval)
 
+	// Track last seen drops count for computing delta
+	var lastDrops uint64
+
 	// Start periodic report writer
 	reportTicker := time.NewTicker(reportInterval)
 	defer reportTicker.Stop()
 
 	writeReport := func() {
 		stats := proc.Stats()
+		drops, err := probe.Drops()
+		if err != nil {
+			log.Warnf("Failed to read drops counter: %v", err)
+			drops = 0
+		}
+
+		// Update the drops counter metric with the delta
+		if drops > lastDrops {
+			delta := drops - lastDrops
+			m.EventsDropped.Add(float64(delta))
+			if delta > 0 {
+				log.Warnf("Ring buffer overflow: %d events dropped since last report", delta)
+			}
+			lastDrops = drops
+		}
+
 		report := &reporter.Report{
 			ContainerID:   containerID,
 			ImageRef:      imageRef,
 			StartedAt:     startedAt,
 			Files:         proc.Files(),
 			TotalEvents:   stats.EventsReceived,
-			DroppedEvents: 0, // TODO: track ring buffer drops
+			DroppedEvents: drops,
 		}
 		if err := rep.Update(ctx, report); err != nil {
 			log.Errorf("Error writing report: %v", err)
 			m.ReportWriteErrors.Inc()
 		} else {
-			log.Infof("Report written: %d unique files, %d events processed",
-				stats.UniqueFiles, stats.EventsProcessed)
+			log.Infof("Report written: %d unique files, %d events processed, %d dropped",
+				stats.UniqueFiles, stats.EventsProcessed, drops)
 			m.ReportWrites.Inc()
 		}
 		// Update gauge for unique files count

pkg/ebpf/bpf/snoop.c

@@ -37,6 +37,14 @@ struct {
     __type(value, u8);     // dummy value (presence = traced)
 } traced_cgroups SEC(".maps");
 
+// Counter for tracking dropped events due to ring buffer overflow
+struct {
+    __uint(type, BPF_MAP_TYPE_ARRAY);
+    __uint(max_entries, 1);
+    __type(key, u32);
+    __type(value, u64);
+} dropped_events SEC(".maps");
+
 // Helper to check if current task's cgroup should be traced
 static __always_inline bool should_trace() {
     struct task_struct *task = (struct task_struct *)bpf_get_current_task();
@@ -47,6 +55,19 @@ static __always_inline bool should_trace() {
     return val != NULL;
 }
 
+// Helper to submit event to ring buffer and track drops
+static __always_inline void submit_event(struct event *e) {
+    int ret = bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    if (ret != 0) {
+        // Ring buffer is full, increment drop counter
+        u32 key = 0;
+        u64 *drop_count = bpf_map_lookup_elem(&dropped_events, &key);
+        if (drop_count) {
+            __sync_fetch_and_add(drop_count, 1);
+        }
+    }
+}
+
 // Tracepoint for openat syscall
 SEC("tracepoint/syscalls/sys_enter_openat")
 int trace_openat(struct trace_event_raw_sys_enter *ctx) {
@@ -75,7 +96,7 @@ int trace_openat(struct trace_event_raw_sys_enter *ctx) {
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
     // Submit event to ring buffer
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }
@@ -102,7 +123,7 @@ int trace_execve(struct trace_event_raw_sys_enter *ctx) {
     const char *pathname = (const char *)ctx->args[0];
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }
@@ -129,7 +150,7 @@ int trace_execveat(struct trace_event_raw_sys_enter *ctx) {
     const char *pathname = (const char *)ctx->args[1];
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }
@@ -156,7 +177,7 @@ int trace_openat2(struct trace_event_raw_sys_enter *ctx) {
     const char *pathname = (const char *)ctx->args[1];
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }
@@ -183,7 +204,7 @@ int trace_statx(struct trace_event_raw_sys_enter *ctx) {
     const char *pathname = (const char *)ctx->args[1];
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }
@@ -210,7 +231,7 @@ int trace_newfstatat(struct trace_event_raw_sys_enter *ctx) {
     const char *pathname = (const char *)ctx->args[1];
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }
@@ -237,7 +258,7 @@ int trace_faccessat(struct trace_event_raw_sys_enter *ctx) {
     const char *pathname = (const char *)ctx->args[1];
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }
@@ -264,7 +285,7 @@ int trace_faccessat2(struct trace_event_raw_sys_enter *ctx) {
     const char *pathname = (const char *)ctx->args[1];
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }
@@ -291,7 +312,7 @@ int trace_readlinkat(struct trace_event_raw_sys_enter *ctx) {
     const char *pathname = (const char *)ctx->args[1];
     bpf_probe_read_user_str(&e->path, MAX_PATH_LEN, pathname);
     
-    bpf_ringbuf_output(&events, e, sizeof(*e), 0);
+    submit_event(e);
     
     return 0;
 }

pkg/ebpf/probe.go

@@ -195,6 +195,17 @@ func (p *Probe) ReadEvent(ctx context.Context) (*Event, error) {
 	return event, nil
 }
 
+// Drops returns the total number of events dropped due to ring buffer overflow
+// This reads the counter from the eBPF dropped_events map
+func (p *Probe) Drops() (uint64, error) {
+	var key uint32 = 0
+	var drops uint64
+	if err := p.objs.DroppedEvents.Lookup(&key, &drops); err != nil {
+		return 0, fmt.Errorf("reading dropped events counter: %w", err)
+	}
+	return drops, nil
+}
+
 // Close cleans up all resources
 func (p *Probe) Close() error {
 	var errs []error

pkg/metrics/metrics.go

@@ -14,6 +14,7 @@ type Metrics struct {
 	EventsProcessed prometheus.Counter
 	EventsExcluded  prometheus.Counter
 	EventsDuplicate prometheus.Counter
+	EventsDropped   prometheus.Counter
 	UniqueFiles     prometheus.Gauge
 
 	ReportWrites      prometheus.Counter
@@ -43,6 +44,10 @@ func New() *Metrics {
 			Name: "snoop_events_duplicate_total",
 			Help: "Total number of events for already-seen file paths.",
 		}),
+		EventsDropped: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "snoop_events_dropped_total",
+			Help: "Total number of events dropped due to ring buffer overflow.",
+		}),
 		UniqueFiles: prometheus.NewGauge(prometheus.GaugeOpts{
 			Name: "snoop_unique_files",
 			Help: "Current number of unique files recorded.",
@@ -64,6 +69,7 @@ func New() *Metrics {
 		m.EventsProcessed,
 		m.EventsExcluded,
 		m.EventsDuplicate,
+		m.EventsDropped,
 		m.UniqueFiles,
 		m.ReportWrites,
 		m.ReportWriteErrors,

plan.md

@@ -7,8 +7,9 @@
 Milestone 3 progress:
 - ✅ Prometheus metrics endpoint (`/metrics` on port 9090)
 - ✅ Structured logging with clog
+- ✅ Ring buffer overflow handling and metrics
 
-**Next**: Ring buffer overflow handling and metrics
+**Next**: Memory-bounded deduplication
 
 See [Milestone 3](#milestone-3-production-hardening) for remaining tasks.
 
@@ -362,7 +363,7 @@ Volume mounts:
 **Deliverables**:
 - [x] Prometheus metrics endpoint
 - [x] Structured logging with levels (clog)
-- [ ] Ring buffer overflow handling and metrics
+- [x] Ring buffer overflow handling and metrics
 - [ ] Memory-bounded deduplication (LRU or bloom filter for extreme cases)
 - [ ] Health check endpoint
 - [ ] Configuration validation