diff --git a/cmd/snoop/main.go b/cmd/snoop/main.go index 99497da..37c73c5 100644 --- a/cmd/snoop/main.go +++ b/cmd/snoop/main.go @@ -13,8 +13,10 @@ import ( "syscall" "time" + "github.com/chainguard-dev/clog" "github.com/imjasonh/snoop/pkg/cgroup" "github.com/imjasonh/snoop/pkg/ebpf" + "github.com/imjasonh/snoop/pkg/health" "github.com/imjasonh/snoop/pkg/metrics" "github.com/imjasonh/snoop/pkg/processor" "github.com/imjasonh/snoop/pkg/reporter" @@ -66,19 +68,21 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time cancel() }() - // Initialize metrics + // Initialize metrics and health checker m := metrics.New() + healthChecker := health.New() - // Start metrics server if address is provided + // Start metrics and health server if address is provided if metricsAddr != "" { mux := http.NewServeMux() mux.Handle("/metrics", m.Handler()) + mux.Handle("/healthz", healthChecker.Handler()) server := &http.Server{ Addr: metricsAddr, Handler: mux, } go func() { - log.Infof("Starting metrics server on %s", metricsAddr) + log.Infof("Starting metrics and health server on %s", metricsAddr) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Errorf("Metrics server error: %v", err) } @@ -99,6 +103,7 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time defer probe.Close() log.Info("eBPF program loaded successfully") + healthChecker.SetEBPFLoaded() // Add cgroup to trace if cgroupPath == "" { @@ -180,6 +185,7 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time log.Infof("Report written: %d unique files, %d events processed, %d dropped, %d evicted", stats.UniqueFiles, stats.EventsProcessed, drops, stats.EventsEvicted) m.ReportWrites.Inc() + healthChecker.RecordReportWritten() } // Update gauge for unique files count m.UniqueFiles.Set(float64(stats.UniqueFiles)) @@ -221,6 +227,7 @@ func run(ctx context.Context, cgroupPath, reportPath string, reportInterval time // Update received counter m.EventsReceived.Inc() + healthChecker.RecordEventReceived() path, result := proc.Process(procEvent) switch result { diff --git a/pkg/health/health.go b/pkg/health/health.go new file mode 100644 index 0000000..ef01a2d --- /dev/null +++ b/pkg/health/health.go @@ -0,0 +1,141 @@ +// Package health provides health checking functionality for snoop. +package health + +import ( + "encoding/json" + "net/http" + "sync" + "time" +) + +// Checker tracks the health status of various snoop components. +type Checker struct { + mu sync.RWMutex + ebpfLoaded bool + lastEventReceived time.Time + lastReportWritten time.Time + startTime time.Time +} + +// New creates a new health checker. +func New() *Checker { + return &Checker{ + startTime: time.Now(), + } +} + +// SetEBPFLoaded marks the eBPF program as successfully loaded. +func (c *Checker) SetEBPFLoaded() { + c.mu.Lock() + defer c.mu.Unlock() + c.ebpfLoaded = true +} + +// RecordEventReceived updates the timestamp of the last event received. +func (c *Checker) RecordEventReceived() { + c.mu.Lock() + defer c.mu.Unlock() + c.lastEventReceived = time.Now() +} + +// RecordReportWritten updates the timestamp of the last successful report write. +func (c *Checker) RecordReportWritten() { + c.mu.Lock() + defer c.mu.Unlock() + c.lastReportWritten = time.Now() +} + +// Status represents the current health status. +type Status struct { + Healthy bool `json:"healthy"` + Uptime string `json:"uptime"` + EBPFLoaded bool `json:"ebpf_loaded"` + LastEventReceived string `json:"last_event_received,omitempty"` + LastReportWritten string `json:"last_report_written,omitempty"` + SecondsSinceEvent float64 `json:"seconds_since_event,omitempty"` + SecondsSinceReport float64 `json:"seconds_since_report,omitempty"` + Message string `json:"message,omitempty"` +} + +// Check returns the current health status. +// It considers the service healthy if: +// - eBPF program is loaded +// - Events have been received (or it's been less than 5 minutes since start) +// - Reports have been written (or it's been less than 5 minutes since start) +func (c *Checker) Check() Status { + c.mu.RLock() + defer c.mu.RUnlock() + + now := time.Now() + uptime := now.Sub(c.startTime) + + status := Status{ + Healthy: true, + Uptime: uptime.Round(time.Second).String(), + EBPFLoaded: c.ebpfLoaded, + } + + // Check eBPF loaded + if !c.ebpfLoaded { + status.Healthy = false + status.Message = "eBPF program not loaded" + return status + } + + // Check event reception (but allow grace period after startup) + if !c.lastEventReceived.IsZero() { + timeSinceEvent := now.Sub(c.lastEventReceived) + status.SecondsSinceEvent = timeSinceEvent.Seconds() + status.LastEventReceived = c.lastEventReceived.Format(time.RFC3339) + + // Alert if no events in 5 minutes (might indicate cgroup filter issue) + if timeSinceEvent > 5*time.Minute { + status.Message = "no events received recently (check cgroup filter)" + } + } else if uptime > 5*time.Minute { + // No events at all after 5 minutes of uptime + status.Message = "no events received yet (check cgroup filter)" + } + + // Check report writes + if !c.lastReportWritten.IsZero() { + timeSinceReport := now.Sub(c.lastReportWritten) + status.SecondsSinceReport = timeSinceReport.Seconds() + status.LastReportWritten = c.lastReportWritten.Format(time.RFC3339) + + // Alert if no report written in 2 minutes (should write every 30s by default) + if timeSinceReport > 2*time.Minute { + status.Healthy = false + if status.Message != "" { + status.Message += "; " + } + status.Message += "report write stalled" + } + } else if uptime > 2*time.Minute { + // No reports at all after 2 minutes of uptime + status.Healthy = false + if status.Message != "" { + status.Message += "; " + } + status.Message += "no reports written yet" + } + + return status +} + +// Handler returns an HTTP handler for the /healthz endpoint. +// Returns 200 OK if healthy, 503 Service Unavailable if unhealthy. +func (c *Checker) Handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + status := c.Check() + + w.Header().Set("Content-Type", "application/json") + if !status.Healthy { + w.WriteHeader(http.StatusServiceUnavailable) + } else { + w.WriteHeader(http.StatusOK) + } + + json.NewEncoder(w).Encode(status) + } +} diff --git a/pkg/health/health_test.go b/pkg/health/health_test.go new file mode 100644 index 0000000..3c709c3 --- /dev/null +++ b/pkg/health/health_test.go @@ -0,0 +1,181 @@ +package health + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestHealthChecker(t *testing.T) { + for _, tt := range []struct { + desc string + setup func(*Checker) + wantHealthy bool + wantMessage string + }{ + { + desc: "newly created checker is unhealthy (eBPF not loaded)", + setup: func(c *Checker) { + // No setup - checker is in initial state + }, + wantHealthy: false, + wantMessage: "eBPF program not loaded", + }, + { + desc: "healthy with eBPF loaded and recent activity", + setup: func(c *Checker) { + c.SetEBPFLoaded() + c.RecordEventReceived() + c.RecordReportWritten() + }, + wantHealthy: true, + wantMessage: "", + }, + { + desc: "healthy with eBPF loaded but no events yet (within grace period)", + setup: func(c *Checker) { + c.SetEBPFLoaded() + c.RecordReportWritten() + }, + wantHealthy: true, + wantMessage: "", + }, + { + desc: "unhealthy when report write stalled", + setup: func(c *Checker) { + c.SetEBPFLoaded() + c.RecordEventReceived() + // Set last report to 3 minutes ago + c.mu.Lock() + c.lastReportWritten = time.Now().Add(-3 * time.Minute) + c.mu.Unlock() + }, + wantHealthy: false, + wantMessage: "report write stalled", + }, + { + desc: "warning when no recent events but reports working", + setup: func(c *Checker) { + c.SetEBPFLoaded() + c.RecordReportWritten() + // Set last event to 6 minutes ago + c.mu.Lock() + c.lastEventReceived = time.Now().Add(-6 * time.Minute) + c.mu.Unlock() + }, + wantHealthy: true, // Still healthy, just a warning + wantMessage: "no events received recently (check cgroup filter)", + }, + } { + t.Run(tt.desc, func(t *testing.T) { + c := New() + tt.setup(c) + + status := c.Check() + + if status.Healthy != tt.wantHealthy { + t.Errorf("Healthy: got %v, want %v", status.Healthy, tt.wantHealthy) + } + + if status.Message != tt.wantMessage { + t.Errorf("Message: got %q, want %q", status.Message, tt.wantMessage) + } + + if !status.EBPFLoaded && tt.wantHealthy { + t.Error("Cannot be healthy without eBPF loaded") + } + }) + } +} + +func TestHealthHandler(t *testing.T) { + for _, tt := range []struct { + desc string + setup func(*Checker) + wantStatusCode int + }{ + { + desc: "returns 503 when unhealthy", + setup: func(c *Checker) { + // No setup - checker is unhealthy by default + }, + wantStatusCode: http.StatusServiceUnavailable, + }, + { + desc: "returns 200 when healthy", + setup: func(c *Checker) { + c.SetEBPFLoaded() + c.RecordEventReceived() + c.RecordReportWritten() + }, + wantStatusCode: http.StatusOK, + }, + } { + t.Run(tt.desc, func(t *testing.T) { + c := New() + tt.setup(c) + + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + rec := httptest.NewRecorder() + + c.Handler()(rec, req) + + if rec.Code != tt.wantStatusCode { + t.Errorf("Status code: got %d, want %d", rec.Code, tt.wantStatusCode) + } + + // Verify JSON response + var status Status + if err := json.NewDecoder(rec.Body).Decode(&status); err != nil { + t.Fatalf("Failed to decode response: %v", err) + } + + // Verify Content-Type + if got := rec.Header().Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type: got %q, want %q", got, "application/json") + } + + // Verify response matches health status + expectedHealthy := tt.wantStatusCode == http.StatusOK + if status.Healthy != expectedHealthy { + t.Errorf("Response healthy field: got %v, want %v", status.Healthy, expectedHealthy) + } + }) + } +} + +func TestHealthStatus(t *testing.T) { + c := New() + c.SetEBPFLoaded() + c.RecordEventReceived() + c.RecordReportWritten() + + status := c.Check() + + // Verify all expected fields are present + if status.Uptime == "" { + t.Error("Expected uptime to be set") + } + + if !status.EBPFLoaded { + t.Error("Expected EBPFLoaded to be true") + } + + if status.LastEventReceived == "" { + t.Error("Expected LastEventReceived to be set") + } + + if status.LastReportWritten == "" { + t.Error("Expected LastReportWritten to be set") + } + + if status.SecondsSinceEvent < 0 { + t.Errorf("Expected non-negative SecondsSinceEvent, got %f", status.SecondsSinceEvent) + } + + if status.SecondsSinceReport < 0 { + t.Errorf("Expected non-negative SecondsSinceReport, got %f", status.SecondsSinceReport) + } +} diff --git a/plan.md b/plan.md index 0f26a57..7afba01 100644 --- a/plan.md +++ b/plan.md @@ -9,8 +9,9 @@ Milestone 3 progress: - ✅ Structured logging with clog - ✅ Ring buffer overflow handling and metrics - ✅ Memory-bounded deduplication with LRU cache +- ✅ Health check endpoint (`/healthz` on port 9090) -**Next**: Health check endpoint +**Next**: Configuration validation See [Milestone 3](#milestone-3-production-hardening) for remaining tasks. @@ -366,7 +367,7 @@ Volume mounts: - [x] Structured logging with levels (clog) - [x] Ring buffer overflow handling and metrics - [x] Memory-bounded deduplication (LRU cache with configurable max size) -- [ ] Health check endpoint +- [x] Health check endpoint - [ ] Configuration validation - [ ] Resource limit recommendations documented