Add health check endpoint for production readiness

Implements /healthz endpoint on port 9090 alongside /metrics.

The health checker monitors:
- eBPF program load status
- Event reception (with 5min grace period)
- Report writes (with 2min grace period)

Returns 200 OK when healthy, 503 when unhealthy, with JSON
response containing detailed status including uptime, timestamps,
and diagnostic messages.

Part of Milestone 3: Production Hardening.

Signed-off-by: Jason Hall <jason@chainguard.dev>

cmd/snoop/main.go

@@ -13,8 +13,10 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/chainguard-dev/clog"
 	"github.com/imjasonh/snoop/pkg/cgroup"
 	"github.com/imjasonh/snoop/pkg/ebpf"
+	"github.com/imjasonh/snoop/pkg/health"
 	"github.com/imjasonh/snoop/pkg/metrics"
 	"github.com/imjasonh/snoop/pkg/processor"
 	"github.com/imjasonh/snoop/pkg/reporter"
@@ -66,19 +68,21 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time
 		cancel()
 	}()
 
-	// Initialize metrics
+	// Initialize metrics and health checker
 	m := metrics.New()
+	healthChecker := health.New()
 
-	// Start metrics server if address is provided
+	// Start metrics and health server if address is provided
 	if metricsAddr != "" {
 		mux := http.NewServeMux()
 		mux.Handle("/metrics", m.Handler())
+		mux.Handle("/healthz", healthChecker.Handler())
 		server := &http.Server{
 			Addr:    metricsAddr,
 			Handler: mux,
 		}
 		go func() {
-			log.Infof("Starting metrics server on %s", metricsAddr)
+			log.Infof("Starting metrics and health server on %s", metricsAddr)
 			if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
 				log.Errorf("Metrics server error: %v", err)
 			}
@@ -99,6 +103,7 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time
 	defer probe.Close()
 
 	log.Info("eBPF program loaded successfully")
+	healthChecker.SetEBPFLoaded()
 
 	// Add cgroup to trace
 	if cgroupPath == "" {
@@ -180,6 +185,7 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time
 			log.Infof("Report written: %d unique files, %d events processed, %d dropped, %d evicted",
 				stats.UniqueFiles, stats.EventsProcessed, drops, stats.EventsEvicted)
 			m.ReportWrites.Inc()
+			healthChecker.RecordReportWritten()
 		}
 		// Update gauge for unique files count
 		m.UniqueFiles.Set(float64(stats.UniqueFiles))
@@ -221,6 +227,7 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time
 
 			// Update received counter
 			m.EventsReceived.Inc()
+			healthChecker.RecordEventReceived()
 
 			path, result := proc.Process(procEvent)
 			switch result {

pkg/health/health.go

@@ -0,0 +1,141 @@
+// Package health provides health checking functionality for snoop.
+package health
+
+import (
+	"encoding/json"
+	"net/http"
+	"sync"
+	"time"
+)
+
+// Checker tracks the health status of various snoop components.
+type Checker struct {
+	mu                sync.RWMutex
+	ebpfLoaded        bool
+	lastEventReceived time.Time
+	lastReportWritten time.Time
+	startTime         time.Time
+}
+
+// New creates a new health checker.
+func New() *Checker {
+	return &Checker{
+		startTime: time.Now(),
+	}
+}
+
+// SetEBPFLoaded marks the eBPF program as successfully loaded.
+func (c *Checker) SetEBPFLoaded() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.ebpfLoaded = true
+}
+
+// RecordEventReceived updates the timestamp of the last event received.
+func (c *Checker) RecordEventReceived() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.lastEventReceived = time.Now()
+}
+
+// RecordReportWritten updates the timestamp of the last successful report write.
+func (c *Checker) RecordReportWritten() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.lastReportWritten = time.Now()
+}
+
+// Status represents the current health status.
+type Status struct {
+	Healthy            bool    `json:"healthy"`
+	Uptime             string  `json:"uptime"`
+	EBPFLoaded         bool    `json:"ebpf_loaded"`
+	LastEventReceived  string  `json:"last_event_received,omitempty"`
+	LastReportWritten  string  `json:"last_report_written,omitempty"`
+	SecondsSinceEvent  float64 `json:"seconds_since_event,omitempty"`
+	SecondsSinceReport float64 `json:"seconds_since_report,omitempty"`
+	Message            string  `json:"message,omitempty"`
+}
+
+// Check returns the current health status.
+// It considers the service healthy if:
+// - eBPF program is loaded
+// - Events have been received (or it's been less than 5 minutes since start)
+// - Reports have been written (or it's been less than 5 minutes since start)
+func (c *Checker) Check() Status {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	now := time.Now()
+	uptime := now.Sub(c.startTime)
+
+	status := Status{
+		Healthy:    true,
+		Uptime:     uptime.Round(time.Second).String(),
+		EBPFLoaded: c.ebpfLoaded,
+	}
+
+	// Check eBPF loaded
+	if !c.ebpfLoaded {
+		status.Healthy = false
+		status.Message = "eBPF program not loaded"
+		return status
+	}
+
+	// Check event reception (but allow grace period after startup)
+	if !c.lastEventReceived.IsZero() {
+		timeSinceEvent := now.Sub(c.lastEventReceived)
+		status.SecondsSinceEvent = timeSinceEvent.Seconds()
+		status.LastEventReceived = c.lastEventReceived.Format(time.RFC3339)
+
+		// Alert if no events in 5 minutes (might indicate cgroup filter issue)
+		if timeSinceEvent > 5*time.Minute {
+			status.Message = "no events received recently (check cgroup filter)"
+		}
+	} else if uptime > 5*time.Minute {
+		// No events at all after 5 minutes of uptime
+		status.Message = "no events received yet (check cgroup filter)"
+	}
+
+	// Check report writes
+	if !c.lastReportWritten.IsZero() {
+		timeSinceReport := now.Sub(c.lastReportWritten)
+		status.SecondsSinceReport = timeSinceReport.Seconds()
+		status.LastReportWritten = c.lastReportWritten.Format(time.RFC3339)
+
+		// Alert if no report written in 2 minutes (should write every 30s by default)
+		if timeSinceReport > 2*time.Minute {
+			status.Healthy = false
+			if status.Message != "" {
+				status.Message += "; "
+			}
+			status.Message += "report write stalled"
+		}
+	} else if uptime > 2*time.Minute {
+		// No reports at all after 2 minutes of uptime
+		status.Healthy = false
+		if status.Message != "" {
+			status.Message += "; "
+		}
+		status.Message += "no reports written yet"
+	}
+
+	return status
+}
+
+// Handler returns an HTTP handler for the /healthz endpoint.
+// Returns 200 OK if healthy, 503 Service Unavailable if unhealthy.
+func (c *Checker) Handler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		status := c.Check()
+
+		w.Header().Set("Content-Type", "application/json")
+		if !status.Healthy {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+
+		json.NewEncoder(w).Encode(status)
+	}
+}

pkg/health/health_test.go

@@ -0,0 +1,181 @@
+package health
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestHealthChecker(t *testing.T) {
+	for _, tt := range []struct {
+		desc        string
+		setup       func(*Checker)
+		wantHealthy bool
+		wantMessage string
+	}{
+		{
+			desc: "newly created checker is unhealthy (eBPF not loaded)",
+			setup: func(c *Checker) {
+				// No setup - checker is in initial state
+			},
+			wantHealthy: false,
+			wantMessage: "eBPF program not loaded",
+		},
+		{
+			desc: "healthy with eBPF loaded and recent activity",
+			setup: func(c *Checker) {
+				c.SetEBPFLoaded()
+				c.RecordEventReceived()
+				c.RecordReportWritten()
+			},
+			wantHealthy: true,
+			wantMessage: "",
+		},
+		{
+			desc: "healthy with eBPF loaded but no events yet (within grace period)",
+			setup: func(c *Checker) {
+				c.SetEBPFLoaded()
+				c.RecordReportWritten()
+			},
+			wantHealthy: true,
+			wantMessage: "",
+		},
+		{
+			desc: "unhealthy when report write stalled",
+			setup: func(c *Checker) {
+				c.SetEBPFLoaded()
+				c.RecordEventReceived()
+				// Set last report to 3 minutes ago
+				c.mu.Lock()
+				c.lastReportWritten = time.Now().Add(-3 * time.Minute)
+				c.mu.Unlock()
+			},
+			wantHealthy: false,
+			wantMessage: "report write stalled",
+		},
+		{
+			desc: "warning when no recent events but reports working",
+			setup: func(c *Checker) {
+				c.SetEBPFLoaded()
+				c.RecordReportWritten()
+				// Set last event to 6 minutes ago
+				c.mu.Lock()
+				c.lastEventReceived = time.Now().Add(-6 * time.Minute)
+				c.mu.Unlock()
+			},
+			wantHealthy: true, // Still healthy, just a warning
+			wantMessage: "no events received recently (check cgroup filter)",
+		},
+	} {
+		t.Run(tt.desc, func(t *testing.T) {
+			c := New()
+			tt.setup(c)
+
+			status := c.Check()
+
+			if status.Healthy != tt.wantHealthy {
+				t.Errorf("Healthy: got %v, want %v", status.Healthy, tt.wantHealthy)
+			}
+
+			if status.Message != tt.wantMessage {
+				t.Errorf("Message: got %q, want %q", status.Message, tt.wantMessage)
+			}
+
+			if !status.EBPFLoaded && tt.wantHealthy {
+				t.Error("Cannot be healthy without eBPF loaded")
+			}
+		})
+	}
+}
+
+func TestHealthHandler(t *testing.T) {
+	for _, tt := range []struct {
+		desc           string
+		setup          func(*Checker)
+		wantStatusCode int
+	}{
+		{
+			desc: "returns 503 when unhealthy",
+			setup: func(c *Checker) {
+				// No setup - checker is unhealthy by default
+			},
+			wantStatusCode: http.StatusServiceUnavailable,
+		},
+		{
+			desc: "returns 200 when healthy",
+			setup: func(c *Checker) {
+				c.SetEBPFLoaded()
+				c.RecordEventReceived()
+				c.RecordReportWritten()
+			},
+			wantStatusCode: http.StatusOK,
+		},
+	} {
+		t.Run(tt.desc, func(t *testing.T) {
+			c := New()
+			tt.setup(c)
+
+			req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
+			rec := httptest.NewRecorder()
+
+			c.Handler()(rec, req)
+
+			if rec.Code != tt.wantStatusCode {
+				t.Errorf("Status code: got %d, want %d", rec.Code, tt.wantStatusCode)
+			}
+
+			// Verify JSON response
+			var status Status
+			if err := json.NewDecoder(rec.Body).Decode(&status); err != nil {
+				t.Fatalf("Failed to decode response: %v", err)
+			}
+
+			// Verify Content-Type
+			if got := rec.Header().Get("Content-Type"); got != "application/json" {
+				t.Errorf("Content-Type: got %q, want %q", got, "application/json")
+			}
+
+			// Verify response matches health status
+			expectedHealthy := tt.wantStatusCode == http.StatusOK
+			if status.Healthy != expectedHealthy {
+				t.Errorf("Response healthy field: got %v, want %v", status.Healthy, expectedHealthy)
+			}
+		})
+	}
+}
+
+func TestHealthStatus(t *testing.T) {
+	c := New()
+	c.SetEBPFLoaded()
+	c.RecordEventReceived()
+	c.RecordReportWritten()
+
+	status := c.Check()
+
+	// Verify all expected fields are present
+	if status.Uptime == "" {
+		t.Error("Expected uptime to be set")
+	}
+
+	if !status.EBPFLoaded {
+		t.Error("Expected EBPFLoaded to be true")
+	}
+
+	if status.LastEventReceived == "" {
+		t.Error("Expected LastEventReceived to be set")
+	}
+
+	if status.LastReportWritten == "" {
+		t.Error("Expected LastReportWritten to be set")
+	}
+
+	if status.SecondsSinceEvent < 0 {
+		t.Errorf("Expected non-negative SecondsSinceEvent, got %f", status.SecondsSinceEvent)
+	}
+
+	if status.SecondsSinceReport < 0 {
+		t.Errorf("Expected non-negative SecondsSinceReport, got %f", status.SecondsSinceReport)
+	}
+}

plan.md

@@ -9,8 +9,9 @@ Milestone 3 progress:
 - ✅ Structured logging with clog
 - ✅ Ring buffer overflow handling and metrics
 - ✅ Memory-bounded deduplication with LRU cache
+- ✅ Health check endpoint (`/healthz` on port 9090)
 
-**Next**: Health check endpoint
+**Next**: Configuration validation
 
 See [Milestone 3](#milestone-3-production-hardening) for remaining tasks.
 
@@ -366,7 +367,7 @@ Volume mounts:
 - [x] Structured logging with levels (clog)
 - [x] Ring buffer overflow handling and metrics
 - [x] Memory-bounded deduplication (LRU cache with configurable max size)
-- [ ] Health check endpoint
+- [x] Health check endpoint
 - [ ] Configuration validation
 - [ ] Resource limit recommendations documented