diff --git a/.github/workflows/kind-tests.yaml b/.github/workflows/kind-tests.yaml new file mode 100644 index 0000000..360c92d --- /dev/null +++ b/.github/workflows/kind-tests.yaml @@ -0,0 +1,65 @@ +name: KinD Integration Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + kind-tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.21' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Install KinD + run: | + # Install KinD + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + - name: Install kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/kubectl + + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Build Docker image + run: | + docker build -t snoop:ci-test . + + - name: Set up KinD cluster + run: | + cd test/kind + ./setup.sh snoop:ci-test + + - name: Run KinD integration tests + run: | + cd test/kind + IMAGE_TAG=snoop:ci-test ./run-tests.sh + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: kind-test-results + path: test/kind/results/ + retention-days: 7 + + - name: Clean up + if: always() + run: | + kind delete cluster --name snoop-test || true diff --git a/.gitignore b/.gitignore index 4cc010c..fc82ef7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ # Binaries -^snoop -*.o *.a *.so @@ -10,11 +8,6 @@ go.work go.work.sum -# eBPF generated files -pkg/ebpf/bpf/*_bpf*.go -pkg/ebpf/bpf/*_bpf*.o -pkg/ebpf/bpf/vmlinux.h - # IDE .vscode/ .idea/ diff --git a/.ko.yaml b/.ko.yaml deleted file mode 100644 index 342d268..0000000 --- a/.ko.yaml +++ /dev/null @@ -1,12 +0,0 @@ -builds: - - id: snoop - main: ./cmd/snoop - env: - - CGO_ENABLED=0 - flags: - - -trimpath - ldflags: - - -s - - -w - -defaultBaseImage: cgr.dev/chainguard/static:latest diff --git a/Dockerfile b/Dockerfile index 9523301..abb5a90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.21 as builder +FROM golang:1.25 AS builder WORKDIR /workspace @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y \ clang \ llvm \ libbpf-dev \ - linux-headers-generic \ + curl \ && rm -rf /var/lib/apt/lists/* # Copy go mod files @@ -21,17 +21,33 @@ COPY . . # Generate eBPF code RUN go generate ./pkg/ebpf/bpf +# Verify generated files exist and check their contents +RUN ls -la pkg/ebpf/bpf/ && \ + echo "=== Checking generated file contents ===" && \ + grep -E "^package|^type Snoop|^func.*Snoop" pkg/ebpf/bpf/snoop_arm64_bpfel.go || true + # Build the application -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o snoop ./cmd/snoop +# Check what architecture we're building for +RUN echo "Building for: GOOS=linux GOARCH=$(go env GOARCH)" && \ + echo "Checking which bpf files will be included:" && \ + ls pkg/ebpf/bpf/*_$(go env GOARCH)_*.go || echo "No arch-specific files found" + +# Try compiling just the bpf package first +RUN CGO_ENABLED=0 GOOS=linux go build -v ./pkg/ebpf/bpf || echo "BPF package failed to build" + +# Build without -a flag to avoid rebuilding stdlib +RUN CGO_ENABLED=0 GOOS=linux go build -v -o snoop ./cmd/snoop # Runtime stage FROM debian:bookworm-slim -# Install runtime dependencies +# Install runtime dependencies (including bash for wrapper scripts) RUN apt-get update && apt-get install -y \ ca-certificates \ + bash \ && rm -rf /var/lib/apt/lists/* COPY --from=builder /workspace/snoop /usr/local/bin/snoop -ENTRYPOINT ["/usr/local/bin/snoop"] +# No ENTRYPOINT - let Kubernetes control the command +# When run directly: docker run image /usr/local/bin/snoop [args] diff --git a/KIND_TESTING_PLAN.md b/KIND_TESTING_PLAN.md new file mode 100644 index 0000000..0c6b9e2 --- /dev/null +++ b/KIND_TESTING_PLAN.md @@ -0,0 +1,1153 @@ +# KinD Testing Plan for Snoop + +**Status**: Pre-Milestone 4 Completion +**Priority**: URGENT +**Created**: 2026-01-14 + +## Overview + +This document articulates the comprehensive testing plan for validating snoop's Kubernetes integration using KinD (Kubernetes in Docker). This testing pre-empts Helm deployment and other Milestone 4 deliverables, ensuring the core sidecar functionality works correctly in a real Kubernetes environment. + +## Goals + +1. **Validate Core Functionality**: Verify snoop correctly traces file access in a Kubernetes pod +2. **Test Metadata Enrichment**: Confirm pod/namespace/image metadata appears in reports +3. **Verify Sidecar Pattern**: Ensure snoop and application containers coexist properly +4. **Test Multiple Workload Types**: Validate with different application patterns (nginx, alpine, Python, etc.) +5. **Check Production Readiness**: Validate health checks, metrics, graceful shutdown +6. **Identify Gaps**: Find any issues in current manifests or documentation + +## Pre-requisites + +### Local Environment + +- **macOS**: Development machine (cannot build eBPF, but can orchestrate tests) +- **Docker Desktop**: For running KinD cluster +- **KinD**: v0.20.0+ installed (`go install sigs.k8s.io/kind@latest`) +- **kubectl**: v1.28+ configured +- **ko**: For building container images (`go install github.com/google/ko@latest`) + +### Required Tools + +```bash +# Install KinD +go install sigs.k8s.io/kind@latest + +# Install kubectl (if not already installed) +# On macOS: +brew install kubectl + +# Install ko +go install github.com/google/ko@latest + +# Install jq for JSON processing +brew install jq + +# Optional: k9s for cluster monitoring +brew install k9s +``` + +### Build Environment + +Since eBPF code generation requires Linux, we need: + +1. **Option A**: Use the existing Dockerfile multi-stage build (handles eBPF generation) +2. **Option B**: Pre-built images from CI (if available) +3. **Option C**: Build in a Linux container/VM + +We'll use **Option A** (Dockerfile) initially since it's already set up. + +## Test Infrastructure + +### 1. KinD Cluster Configuration + +Create a KinD cluster with specific settings for eBPF: + +**File**: `test/kind/cluster-config.yaml` + +```yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: snoop-test + +# Use a recent node image with eBPF support +nodes: + - role: control-plane + image: kindest/node:v1.29.0 + extraMounts: + # Mount kernel debug filesystem (required for eBPF) + - hostPath: /sys/kernel/debug + containerPath: /sys/kernel/debug + readOnly: false + # Mount cgroup filesystem + - hostPath: /sys/fs/cgroup + containerPath: /sys/fs/cgroup + readOnly: false + +# Enable feature gates if needed +featureGates: + # None required for basic eBPF +``` + +### 2. Test Applications + +Create multiple test workloads with different file access patterns: + +#### Test App 1: Simple Alpine Loop +- **Purpose**: Basic file access validation +- **File Access Pattern**: Read `/etc/passwd`, `/etc/hosts`, list `/usr/bin` +- **Expected Files**: ~10-20 files + +#### Test App 2: Nginx Web Server +- **Purpose**: Real-world application +- **File Access Pattern**: Config files, logs, static content +- **Expected Files**: ~50-100 files + +#### Test App 3: Python Flask App +- **Purpose**: Interpreted language with many imports +- **File Access Pattern**: Python stdlib, site-packages, application code +- **Expected Files**: ~200-500 files + +#### Test App 4: Busybox Scripted +- **Purpose**: Controlled, predictable file access +- **File Access Pattern**: Explicitly access specific files +- **Expected Files**: Exactly known list + +### 3. Validation Scripts + +Create Go-based validation tools (prefer Go over bash): + +**File**: `test/kind/validate/main.go` + +```go +package main + +// Validates: +// 1. Report JSON structure is correct +// 2. Expected files are present +// 3. Excluded files are absent +// 4. Metadata fields are populated +// 5. Metrics endpoint is accessible +// 6. Health check endpoint works +``` + +### 4. Test Orchestration + +**File**: `test/kind/run-tests.sh` + +```bash +#!/bin/bash +# Main test runner that: +# 1. Creates KinD cluster +# 2. Builds and loads snoop image +# 3. Deploys test applications +# 4. Waits for reports +# 5. Validates results +# 6. Cleans up +``` + +## Test Scenarios + +### Scenario 1: Basic Sidecar Deployment + +**Objective**: Verify snoop can run alongside a simple application. + +**Steps**: +1. Create KinD cluster with config +2. Build snoop image: `ko build --local ./cmd/snoop` +3. Load image into KinD: `kind load docker-image ` +4. Apply RBAC: `kubectl apply -f deploy/kubernetes/rbac.yaml` +5. Deploy test app (alpine loop): `kubectl apply -f test/kind/manifests/alpine-test.yaml` +6. Wait for pod ready: `kubectl wait --for=condition=Ready pod -l app=alpine-test` +7. Check snoop logs: `kubectl logs -l app=alpine-test -c snoop --tail=50` +8. Wait 35 seconds (for report interval) +9. Retrieve report: `kubectl cp :/data/snoop-report.json ./report-alpine.json` +10. Validate report structure and content + +**Expected Results**: +- Pod reaches Ready state +- Snoop logs show "eBPF probes attached" +- Report JSON is valid +- Report contains `/etc/passwd`, `/etc/hosts`, `/usr/bin/*` +- Report excludes `/proc/`, `/sys/`, `/dev/` +- Metadata fields populated: `pod_name`, `namespace` + +**Success Criteria**: +- ✅ Pod healthy +- ✅ Report generated within 35 seconds +- ✅ At least 10 files captured +- ✅ No excluded files present +- ✅ Metadata correct + +### Scenario 2: Nginx Application + +**Objective**: Test with a real-world web server. + +**Steps**: +1. Deploy nginx with snoop: `kubectl apply -f deploy/kubernetes/example-app.yaml` +2. Wait for pod ready +3. Generate traffic: `kubectl exec -it -c nginx -- wget -O /dev/null http://localhost/` +4. Wait for report +5. Retrieve and validate report + +**Expected Results**: +- nginx starts successfully +- Snoop captures nginx config files: `/etc/nginx/nginx.conf` +- Snoop captures nginx binary: `/usr/sbin/nginx` +- Snoop captures shared libraries: `/lib/*.so*` +- Cache directory `/var/cache/nginx` excluded (per exclusion config) + +**Success Criteria**: +- ✅ nginx responds to HTTP requests +- ✅ Report contains nginx-specific files +- ✅ Cache files excluded +- ✅ Metrics endpoint accessible: `snoop_events_total` > 0 + +### Scenario 3: Python Flask Application + +**Objective**: Test with interpreted language and many imports. + +**Steps**: +1. Deploy Flask app with snoop: `kubectl apply -f test/kind/manifests/flask-test.yaml` +2. Wait for pod ready +3. Make HTTP request to trigger imports +4. Wait for report +5. Validate large file list + +**Expected Results**: +- Flask app starts and serves requests +- Report contains Python interpreter: `/usr/bin/python3` +- Report contains stdlib modules: `/usr/lib/python3.X/...` +- Report contains Flask package files +- Potentially 200-500 files captured + +**Success Criteria**: +- ✅ Flask app responds correctly +- ✅ Report contains Python-related files +- ✅ File count > 100 +- ✅ Memory usage < 128Mi (check with `kubectl top pod`) + +### Scenario 4: Multi-Container Pod + +**Objective**: Verify snoop traces only target container, not sidecar container. + +**Steps**: +1. Deploy pod with: app container + snoop sidecar + logging sidecar +2. Verify snoop doesn't trace its own file access +3. Verify snoop doesn't trace logging sidecar +4. Verify snoop only traces app container + +**Expected Results**: +- Report contains files from app container only +- Snoop binary `/usr/local/bin/snoop` NOT in report +- Logging sidecar files NOT in report + +**Success Criteria**: +- ✅ Only app container files present +- ✅ Snoop's own files excluded +- ✅ Other sidecar files excluded + +### Scenario 5: Health and Metrics + +**Objective**: Validate production readiness endpoints. + +**Steps**: +1. Deploy any test app with snoop +2. Port-forward metrics: `kubectl port-forward 9090:9090` +3. Check health: `curl http://localhost:9090/healthz` +4. Check metrics: `curl http://localhost:9090/metrics` +5. Validate Prometheus format + +**Expected Results**: +- Health endpoint returns `200 OK` +- Metrics endpoint returns Prometheus format +- Key metrics present: + - `snoop_events_total` + - `snoop_unique_files` + - `snoop_report_writes_total` +- Process metrics present: + - `process_cpu_seconds_total` + - `process_resident_memory_bytes` + +**Success Criteria**: +- ✅ `/healthz` returns 200 +- ✅ `/metrics` returns valid Prometheus output +- ✅ All expected metrics present with values > 0 + +### Scenario 6: Graceful Shutdown + +**Objective**: Verify final report written on pod termination. + +**Steps**: +1. Deploy test app with snoop +2. Wait for first report +3. Trigger pod deletion: `kubectl delete pod ` +4. Capture logs during shutdown +5. Retrieve final report before pod terminates + +**Expected Results**: +- Snoop logs show "Received signal: terminated" +- Snoop logs show "Writing final report" +- Final report exists and is valid +- No data loss + +**Success Criteria**: +- ✅ Graceful shutdown logged +- ✅ Final report written +- ✅ Exit code 0 + +### Scenario 7: Cgroup Discovery + +**Objective**: Validate cgroup path discovery mechanism. + +**Steps**: +1. Check init container logs for cgroup path +2. Verify cgroup path is correct format +3. Compare with actual pod cgroup path on node +4. Ensure snoop uses correct path + +**Expected Results**: +- Init container finds cgroup path +- Path format: `/kubepods/...//...` +- Snoop successfully attaches to correct cgroup + +**Success Criteria**: +- ✅ Init container succeeds +- ✅ Cgroup path file created +- ✅ Snoop uses correct path +- ✅ Events captured + +### Scenario 8: Resource Limits + +**Objective**: Verify snoop operates within resource constraints. + +**Steps**: +1. Deploy with strict resource limits (50m CPU, 64Mi memory) +2. Generate high file access load +3. Monitor resource usage: `kubectl top pod` +4. Check for OOM kills or throttling + +**Expected Results**: +- CPU usage stays below 100m under normal load +- Memory usage stays below 64Mi with bounded deduplication +- No OOM kills +- Metrics show if events dropped + +**Success Criteria**: +- ✅ No OOM kills +- ✅ CPU < 100m average +- ✅ Memory < 80Mi +- ✅ `snoop_events_dropped_total` = 0 (or documented) + +### Scenario 9: Report Format Validation + +**Objective**: Validate report JSON structure matches specification. + +**Steps**: +1. Retrieve report from any test +2. Parse JSON with validation tool +3. Check required fields present +4. Check data types correct +5. Check timestamps valid + +**Expected Report Structure**: +```json +{ + "container_id": "alpine-test-xxxxx", + "image_ref": "alpine:latest", + "image_digest": "", + "pod_name": "alpine-test-xxxxx", + "namespace": "default", + "labels": {}, + "started_at": "2026-01-14T12:00:00Z", + "last_updated_at": "2026-01-14T12:00:35Z", + "files": [ + "/etc/passwd", + "/etc/hosts", + "/usr/bin/ls" + ], + "total_events": 150, + "dropped_events": 0 +} +``` + +**Success Criteria**: +- ✅ Valid JSON +- ✅ All required fields present +- ✅ Timestamps in RFC3339 format +- ✅ Files array non-empty +- ✅ No duplicate files in array + +### Scenario 10: Path Normalization + +**Objective**: Verify path normalization works correctly. + +**Steps**: +1. Deploy busybox container with script that accesses: + - Relative paths: `./file.txt` + - Parent directories: `../etc/passwd` + - Current directory: `.` + - Redundant slashes: `/usr//bin/ls` +2. Check report paths are normalized + +**Expected Results**: +- All paths are absolute +- No `.` or `..` components +- No redundant slashes +- Symlinks NOT resolved (record what app asked for) + +**Success Criteria**: +- ✅ All paths start with `/` +- ✅ No relative path components +- ✅ Paths are clean and canonical + +## Test Environment Setup + +### Directory Structure + +``` +test/ +└── kind/ + ├── cluster-config.yaml # KinD cluster configuration + ├── run-tests.sh # Main test orchestration script + ├── setup.sh # Cluster setup + ├── teardown.sh # Cleanup + ├── manifests/ + │ ├── alpine-test.yaml # Simple alpine test + │ ├── busybox-script.yaml # Controlled file access + │ ├── flask-test.yaml # Python Flask app + │ └── multi-container.yaml # Multi-container pod test + ├── validate/ + │ ├── main.go # Report validation tool + │ ├── report.go # Report structure definition + │ └── checks.go # Validation checks + └── results/ + └── .gitkeep # Directory for test results +``` + +### Cluster Setup Script + +**File**: `test/kind/setup.sh` + +```bash +#!/bin/bash +set -euo pipefail + +CLUSTER_NAME="${CLUSTER_NAME:-snoop-test}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "Creating KinD cluster: $CLUSTER_NAME" +kind create cluster --config "$SCRIPT_DIR/cluster-config.yaml" --name "$CLUSTER_NAME" + +echo "Building snoop image with ko..." +# ko build handles the multi-stage Docker build internally +export KO_DOCKER_REPO=kind.local +IMAGE=$(ko build ./cmd/snoop) + +echo "Loading image into KinD cluster..." +kind load docker-image "$IMAGE" --name "$CLUSTER_NAME" + +echo "Applying RBAC..." +kubectl apply -f "$SCRIPT_DIR/../../deploy/kubernetes/rbac.yaml" + +echo "Cluster ready for testing!" +echo "Image: $IMAGE" +``` + +### Test Runner Script + +**File**: `test/kind/run-tests.sh` + +```bash +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RESULTS_DIR="$SCRIPT_DIR/results" +mkdir -p "$RESULTS_DIR" + +# Track results +PASSED=0 +FAILED=0 + +run_test() { + local test_name="$1" + local manifest="$2" + local validator="$3" + + echo "" + echo "=========================================" + echo "Running: $test_name" + echo "=========================================" + + # Deploy + kubectl apply -f "$manifest" + + # Wait for ready + POD_LABEL=$(grep -A1 "matchLabels:" "$manifest" | tail -1 | awk '{print $2}') + kubectl wait --for=condition=Ready pod -l "app=$POD_LABEL" --timeout=60s + + # Wait for report interval + buffer + echo "Waiting 40 seconds for report generation..." + sleep 40 + + # Get pod name + POD_NAME=$(kubectl get pod -l "app=$POD_LABEL" -o jsonpath='{.items[0].metadata.name}') + + # Retrieve report + REPORT_FILE="$RESULTS_DIR/${test_name}-report.json" + kubectl cp "$POD_NAME:/data/snoop-report.json" "$REPORT_FILE" -c app || { + echo "❌ FAILED: Could not retrieve report" + kubectl logs "$POD_NAME" -c snoop --tail=50 + ((FAILED++)) + return 1 + } + + # Validate + echo "Validating report..." + if "$validator" "$REPORT_FILE"; then + echo "✅ PASSED: $test_name" + ((PASSED++)) + else + echo "❌ FAILED: $test_name" + ((FAILED++)) + fi + + # Cleanup + kubectl delete -f "$manifest" --wait=false +} + +# Build validator tool +echo "Building validation tool..." +(cd "$SCRIPT_DIR/validate" && go build -o "$RESULTS_DIR/validate" .) + +# Run test scenarios +run_test "alpine-basic" "$SCRIPT_DIR/manifests/alpine-test.yaml" "$RESULTS_DIR/validate" +run_test "nginx-real-world" "$SCRIPT_DIR/../../deploy/kubernetes/example-app.yaml" "$RESULTS_DIR/validate" +run_test "busybox-controlled" "$SCRIPT_DIR/manifests/busybox-script.yaml" "$RESULTS_DIR/validate" + +# Summary +echo "" +echo "=========================================" +echo "Test Summary" +echo "=========================================" +echo "Passed: $PASSED" +echo "Failed: $FAILED" +echo "" + +if [ "$FAILED" -eq 0 ]; then + echo "✅ All tests passed!" + exit 0 +else + echo "❌ Some tests failed" + exit 1 +fi +``` + +### Validation Tool + +**File**: `test/kind/validate/main.go` + +```go +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +type Report struct { + ContainerID string `json:"container_id"` + ImageRef string `json:"image_ref"` + PodName string `json:"pod_name"` + Namespace string `json:"namespace"` + Labels map[string]string `json:"labels"` + StartedAt time.Time `json:"started_at"` + LastUpdatedAt time.Time `json:"last_updated_at"` + Files []string `json:"files"` + TotalEvents uint64 `json:"total_events"` + DroppedEvents uint64 `json:"dropped_events"` +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: %s \n", filepath.Base(os.Args[0])) + os.Exit(1) + } + + reportPath := os.Args[1] + if err := validateReport(reportPath); err != nil { + fmt.Fprintf(os.Stderr, "Validation failed: %v\n", err) + os.Exit(1) + } + + fmt.Println("✅ Report validation passed") +} + +func validateReport(path string) error { + // Read and parse JSON + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading report: %w", err) + } + + var report Report + if err := json.Unmarshal(data, &report); err != nil { + return fmt.Errorf("parsing JSON: %w", err) + } + + // Validate required fields + if report.PodName == "" { + return fmt.Errorf("pod_name is empty") + } + if report.Namespace == "" { + return fmt.Errorf("namespace is empty") + } + if report.StartedAt.IsZero() { + return fmt.Errorf("started_at is zero") + } + if report.LastUpdatedAt.IsZero() { + return fmt.Errorf("last_updated_at is zero") + } + + // Validate files array + if len(report.Files) == 0 { + return fmt.Errorf("files array is empty") + } + + // Check for excluded paths + excludedPrefixes := []string{"/proc/", "/sys/", "/dev/"} + for _, file := range report.Files { + for _, prefix := range excludedPrefixes { + if strings.HasPrefix(file, prefix) { + return fmt.Errorf("excluded file found: %s", file) + } + } + + // Check paths are absolute + if !strings.HasPrefix(file, "/") { + return fmt.Errorf("relative path found: %s", file) + } + + // Check for path components that should be normalized + if strings.Contains(file, "/./") || strings.Contains(file, "/../") { + return fmt.Errorf("non-normalized path: %s", file) + } + } + + // Check for duplicates + seen := make(map[string]bool) + for _, file := range report.Files { + if seen[file] { + return fmt.Errorf("duplicate file: %s", file) + } + seen[file] = true + } + + // Validate timestamps + if report.LastUpdatedAt.Before(report.StartedAt) { + return fmt.Errorf("last_updated_at before started_at") + } + + // Success + fmt.Printf(" - Files captured: %d\n", len(report.Files)) + fmt.Printf(" - Total events: %d\n", report.TotalEvents) + fmt.Printf(" - Dropped events: %d\n", report.DroppedEvents) + fmt.Printf(" - Pod: %s/%s\n", report.Namespace, report.PodName) + + return nil +} +``` + +## Test Manifests + +### Alpine Test + +**File**: `test/kind/manifests/alpine-test.yaml` + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: snoop-test +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alpine-test + namespace: snoop-test +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alpine-test + namespace: snoop-test +spec: + replicas: 1 + selector: + matchLabels: + app: alpine-test + template: + metadata: + labels: + app: alpine-test + spec: + serviceAccountName: alpine-test + volumes: + - name: snoop-data + emptyDir: {} + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: debugfs + hostPath: + path: /sys/kernel/debug + initContainers: + - name: cgroup-finder + image: busybox:latest + command: + - sh + - -c + - | + CGROUP_PATH=$(cat /proc/self/cgroup | cut -d: -f3) + echo "$CGROUP_PATH" > /snoop-data/cgroup-path + echo "Cgroup path: $CGROUP_PATH" + volumeMounts: + - name: snoop-data + mountPath: /snoop-data + containers: + - name: app + image: alpine:latest + command: + - sh + - -c + - | + echo "Starting file access test..." + while true; do + cat /etc/passwd > /dev/null + cat /etc/hosts > /dev/null + ls /usr/bin > /dev/null + ls /lib > /dev/null + sleep 5 + done + volumeMounts: + - name: snoop-data + mountPath: /data + readOnly: true + - name: snoop + image: kind.local/snoop:latest + imagePullPolicy: Never + securityContext: + capabilities: + add: [SYS_ADMIN, BPF, PERFMON] + readOnlyRootFilesystem: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: ["/usr/local/bin/snoop"] + args: + - -cgroup=/sys/fs/cgroup$(cat /data/cgroup-path) + - -report=/data/snoop-report.json + - -interval=30s + - -exclude=/proc/,/sys/,/dev/ + - -metrics-addr=:9090 + - -log-level=info + - -container-id=$(POD_NAME) + volumeMounts: + - name: snoop-data + mountPath: /data + - name: cgroup + mountPath: /sys/fs/cgroup + readOnly: true + - name: debugfs + mountPath: /sys/kernel/debug + readOnly: true + ports: + - name: metrics + containerPort: 9090 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 10 +``` + +### Busybox Controlled Test + +**File**: `test/kind/manifests/busybox-script.yaml` + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: snoop-test +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: busybox-test + namespace: snoop-test +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: busybox-script + namespace: snoop-test +data: + test.sh: | + #!/bin/sh + echo "Starting controlled file access test..." + + # Create test file + echo "test content" > /tmp/test.txt + + # Test various access patterns + while true; do + # Absolute paths + cat /etc/passwd > /dev/null + cat /etc/hosts > /dev/null + + # Relative paths (should be normalized) + cd /etc + cat ./passwd > /dev/null + cat ../etc/hosts > /dev/null + + # Executable access + /bin/ls /usr > /dev/null + + # Temp file + cat /tmp/test.txt > /dev/null + + echo "Cycle complete" + sleep 5 + done +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: busybox-test + namespace: snoop-test +spec: + replicas: 1 + selector: + matchLabels: + app: busybox-test + template: + metadata: + labels: + app: busybox-test + spec: + serviceAccountName: busybox-test + volumes: + - name: snoop-data + emptyDir: {} + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: debugfs + hostPath: + path: /sys/kernel/debug + - name: script + configMap: + name: busybox-script + defaultMode: 0755 + initContainers: + - name: cgroup-finder + image: busybox:latest + command: + - sh + - -c + - | + CGROUP_PATH=$(cat /proc/self/cgroup | cut -d: -f3) + echo "$CGROUP_PATH" > /snoop-data/cgroup-path + volumeMounts: + - name: snoop-data + mountPath: /snoop-data + containers: + - name: app + image: busybox:latest + command: ["/scripts/test.sh"] + volumeMounts: + - name: snoop-data + mountPath: /data + readOnly: true + - name: script + mountPath: /scripts + - name: snoop + image: kind.local/snoop:latest + imagePullPolicy: Never + securityContext: + capabilities: + add: [SYS_ADMIN, BPF, PERFMON] + readOnlyRootFilesystem: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: ["/usr/local/bin/snoop"] + args: + - -cgroup=/sys/fs/cgroup$(cat /data/cgroup-path) + - -report=/data/snoop-report.json + - -interval=30s + - -exclude=/proc/,/sys/,/dev/ + - -metrics-addr=:9090 + - -log-level=debug + - -container-id=$(POD_NAME) + volumeMounts: + - name: snoop-data + mountPath: /data + - name: cgroup + mountPath: /sys/fs/cgroup + readOnly: true + - name: debugfs + mountPath: /sys/kernel/debug + readOnly: true + ports: + - name: metrics + containerPort: 9090 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi +``` + +## Execution Plan + +### Phase 1: Infrastructure Setup (Day 1) + +1. **Create test directory structure** + - `test/kind/` with subdirectories + - Commit structure to git + +2. **Write KinD cluster configuration** + - `cluster-config.yaml` with eBPF mounts + - Test cluster creation manually + +3. **Create setup/teardown scripts** + - `setup.sh` for cluster + image build + - `teardown.sh` for cleanup + - Test scripts manually + +4. **Build validation tool** + - Write `validate/main.go` + - Test with sample report JSON + - Build and verify works + +### Phase 2: Basic Tests (Day 1-2) + +5. **Create alpine test manifest** + - Simple loop accessing known files + - Deploy manually and verify + +6. **Create busybox test manifest** + - Controlled file access script + - Test path normalization + +7. **Test with existing example-app.yaml** + - Nginx deployment + - Verify real-world scenario + +8. **Write test runner script** + - Orchestrate test execution + - Run all three tests + - Collect results + +### Phase 3: Advanced Tests (Day 2) + +9. **Create multi-container test** + - Pod with app + snoop + logging + - Verify isolation + +10. **Test health/metrics endpoints** + - Port-forward and curl + - Validate Prometheus format + +11. **Test graceful shutdown** + - Delete pod during operation + - Capture final report + +12. **Test resource limits** + - Deploy with strict limits + - Monitor with `kubectl top` + +### Phase 4: Documentation and Gaps (Day 2-3) + +13. **Document findings** + - Issues discovered + - Gaps in manifests + - Areas needing improvement + +14. **Update manifests based on findings** + - Fix any bugs found + - Improve configurations + +15. **Create comprehensive test report** + - What works + - What doesn't + - Next steps + +## Expected Issues to Investigate + +Based on the manifests review, potential issues to validate: + +### 1. Cgroup Discovery Mechanism + +**Current Approach**: Init container writes cgroup path to file, snoop reads it via `$(cat /data/cgroup-path)` + +**Potential Issues**: +- Shell expansion `$(cat ...)` in args might not work in all scenarios +- Race condition if snoop starts before file written +- Need to verify this works in KinD + +**Test**: Check init container logs, verify file created, verify snoop reads correctly + +### 2. Image Reference + +**Current Approach**: Specified as arg `-image=nginx:1.25-alpine` + +**Potential Issues**: +- Manual specification error-prone +- Doesn't capture actual image digest +- Need automated way to get this + +**Test**: Verify image ref in report matches deployed image + +### 3. Host Path Mounts + +**Current Approach**: Direct hostPath mounts for `/sys/fs/cgroup` and `/sys/kernel/debug` + +**Potential Issues**: +- KinD node paths might differ from real nodes +- Permissions issues possible +- Security implications + +**Test**: Verify mounts work in KinD, check permissions + +### 4. Target Container Identification + +**Current Approach**: Manual cgroup path construction + +**Potential Issues**: +- Fragile path construction +- Doesn't handle multi-container pods well +- Need better discovery mechanism + +**Test**: Multi-container pod scenario, verify correct targeting + +### 5. RBAC Permissions + +**Current Approach**: ClusterRole with pod/node read access + +**Potential Issues**: +- Might need more permissions for pod metadata +- ClusterRole might be too broad for some deployments + +**Test**: Verify RBAC works, check if all permissions needed + +## Success Metrics + +This testing phase is successful if: + +1. **✅ All 10 scenarios pass** with validation +2. **✅ Reports contain expected files** for each test case +3. **✅ Metadata is correctly populated** (pod name, namespace) +4. **✅ Health/metrics endpoints work** as documented +5. **✅ No security violations** or permission errors +6. **✅ Documentation gaps identified** and listed +7. **✅ Performance within targets** (<100m CPU, <128Mi memory) +8. **✅ Graceful shutdown works** correctly +9. **✅ Path normalization verified** working +10. **✅ Exclusions working** as configured + +## Deliverables + +After completing this testing plan: + +1. **Test Infrastructure** (`test/kind/` directory) + - Cluster configuration + - Test manifests + - Validation tools + - Test runner scripts + +2. **Test Results Document** (`KIND_TEST_RESULTS.md`) + - Results for each scenario + - Issues found + - Performance metrics + - Screenshots/logs + +3. **Issue List** (GitHub issues or TODO list) + - Bugs to fix + - Improvements needed + - Documentation gaps + +4. **Updated Manifests** (if issues found) + - Fixed deployment.yaml + - Fixed example-app.yaml + - Improved documentation + +5. **Recommendations Document** + - What works well + - What needs improvement before Helm chart + - Prerequisites for next milestone + +## Timeline + +- **Day 1 (4-6 hours)**: Infrastructure setup, basic tests +- **Day 2 (4-6 hours)**: Advanced tests, issue investigation +- **Day 3 (2-4 hours)**: Documentation, fixes, recommendations + +**Total Effort**: ~12-16 hours + +**Target Completion**: Before proceeding with Helm chart development + +## Next Steps After Testing + +Once this testing plan is complete and issues addressed: + +1. ✅ Mark current Milestone 4 tasks as complete (if tests pass) +2. 🔄 Fix any issues found during testing +3. 📝 Update documentation based on findings +4. ➡️ Proceed with Helm chart development +5. ➡️ Test Helm deployment in KinD +6. ➡️ Test on real cluster (GKE/EKS) +7. ➡️ Move to Milestone 5 (Multi-Deployment Aggregation) + +## References + +- [KinD Quick Start](https://kind.sigs.k8s.io/docs/user/quick-start/) +- [KinD Configuration](https://kind.sigs.k8s.io/docs/user/configuration/) +- [ko Documentation](https://ko.build/) +- [eBPF on Kubernetes](https://cilium.io/blog/2020/11/10/ebpf-future-of-networking/) +- [Kubernetes Downward API](https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/) diff --git a/Makefile b/Makefile index ce19ace..cd6ea77 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,26 @@ vmlinux: ## Generate vmlinux.h from current kernel (Linux only) generate: ## Generate eBPF code (requires clang, llvm, vmlinux.h) go generate ./pkg/ebpf/bpf +generate-in-docker: ## Generate eBPF code in Docker (works on macOS) + @echo "Building multi-platform Docker image to generate eBPF code..." + docker build --platform=linux/amd64 --target builder -t snoop-builder:amd64 -f Dockerfile . + docker build --platform=linux/arm64 --target builder -t snoop-builder:arm64 -f Dockerfile . + @echo "Extracting generated files from amd64 build..." + docker create --name snoop-builder-amd64 snoop-builder:amd64 + docker cp snoop-builder-amd64:/workspace/pkg/ebpf/bpf/snoop_x86_bpfel.go pkg/ebpf/bpf/ + docker cp snoop-builder-amd64:/workspace/pkg/ebpf/bpf/snoop_x86_bpfel.o pkg/ebpf/bpf/ + docker rm snoop-builder-amd64 + @echo "Extracting generated files from arm64 build..." + docker create --name snoop-builder-arm64 snoop-builder:arm64 + docker cp snoop-builder-arm64:/workspace/pkg/ebpf/bpf/snoop_arm64_bpfel.go pkg/ebpf/bpf/ + docker cp snoop-builder-arm64:/workspace/pkg/ebpf/bpf/snoop_arm64_bpfel.o pkg/ebpf/bpf/ + docker rm snoop-builder-arm64 + @echo "Cleaning up temporary images..." + docker rmi snoop-builder:amd64 snoop-builder:arm64 + @echo "Generated files extracted to pkg/ebpf/bpf/" + @echo "Verifying files..." + @ls -lh pkg/ebpf/bpf/snoop_*.go pkg/ebpf/bpf/snoop_*.o + build: generate ## Build the snoop binary go build -o snoop ./cmd/snoop diff --git a/cmd/snoop/main.go b/cmd/snoop/main.go index dd34c1a..3851fa1 100644 --- a/cmd/snoop/main.go +++ b/cmd/snoop/main.go @@ -1,11 +1,10 @@ -//go:build linux - package main import ( "context" "flag" "fmt" + "log/slog" "net/http" "os" "os/signal" @@ -13,6 +12,7 @@ import ( "time" "github.com/chainguard-dev/clog" + "github.com/chainguard-dev/clog/slag" "github.com/imjasonh/snoop/pkg/cgroup" "github.com/imjasonh/snoop/pkg/config" "github.com/imjasonh/snoop/pkg/ebpf" @@ -30,8 +30,10 @@ func main() { excludePaths string imageRef string containerID string + podName string + namespace string metricsAddr string - logLevel string + logLevel slag.Level maxUniqueFiles int ) @@ -41,12 +43,21 @@ func main() { flag.StringVar(&excludePaths, "exclude", "/proc/,/sys/,/dev/", "Comma-separated path prefixes to exclude") flag.StringVar(&imageRef, "image", "", "Image reference for report metadata") flag.StringVar(&containerID, "container-id", "", "Container ID for report metadata") + flag.StringVar(&podName, "pod-name", "", "Pod name for report metadata") + flag.StringVar(&namespace, "namespace", "", "Namespace for report metadata") flag.StringVar(&metricsAddr, "metrics-addr", ":9090", "Address for Prometheus metrics endpoint (empty to disable)") - flag.StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)") + flag.Var(&logLevel, "log-level", "Log level (debug, info, warn, error)") flag.IntVar(&maxUniqueFiles, "max-unique-files", 0, "Maximum unique files to track (0 = unbounded)") flag.Parse() - // Build configuration from flags + // Build configuration from flags (also check environment variables) + if podName == "" { + podName = os.Getenv("POD_NAME") + } + if namespace == "" { + namespace = os.Getenv("POD_NAMESPACE") + } + cfg := &config.Config{ CgroupPath: cgroupPath, ReportPath: reportPath, @@ -54,13 +65,17 @@ func main() { ExcludePaths: config.ParseExcludePaths(excludePaths), ImageRef: imageRef, ContainerID: containerID, + PodName: podName, + Namespace: namespace, MetricsAddr: metricsAddr, - LogLevel: logLevel, + LogLevel: slog.Level(logLevel), MaxUniqueFiles: maxUniqueFiles, } // Initialize logging context - ctx := clog.WithLogger(context.Background(), clog.New(clog.ParseLevel(cfg.LogLevel))) + ctx := clog.WithLogger(context.Background(), clog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.Level(logLevel), + }))) // Validate configuration if err := cfg.Validate(); err != nil { @@ -179,6 +194,8 @@ func run(ctx context.Context, cfg *config.Config) error { report := &reporter.Report{ ContainerID: cfg.ContainerID, ImageRef: cfg.ImageRef, + PodName: cfg.PodName, + Namespace: cfg.Namespace, StartedAt: startedAt, Files: proc.Files(), TotalEvents: stats.EventsReceived, diff --git a/pkg/cgroup/discovery.go b/pkg/cgroup/discovery.go index 86243c3..ded748f 100644 --- a/pkg/cgroup/discovery.go +++ b/pkg/cgroup/discovery.go @@ -6,6 +6,7 @@ import ( "os" "strconv" "strings" + "syscall" ) // Discovery finds cgroup IDs to trace @@ -82,21 +83,36 @@ func GetSelfCgroupID() (uint64, error) { // GetCgroupIDByPath returns the cgroup ID for a given cgroup path func GetCgroupIDByPath(cgroupPath string) (uint64, error) { + // Try reading from cgroup.id file first (newer kernels) idPath := "/sys/fs/cgroup" + cgroupPath if !strings.HasSuffix(idPath, "/") { idPath += "/" } - idPath += "cgroup.id" + idFilePath := idPath + "cgroup.id" - idData, err := os.ReadFile(idPath) - if err != nil { - return 0, fmt.Errorf("reading cgroup.id from %s: %w", idPath, err) + idData, err := os.ReadFile(idFilePath) + if err == nil { + id, err := strconv.ParseUint(strings.TrimSpace(string(idData)), 10, 64) + if err != nil { + return 0, fmt.Errorf("parsing cgroup ID: %w", err) + } + return id, nil } - id, err := strconv.ParseUint(strings.TrimSpace(string(idData)), 10, 64) - if err != nil { - return 0, fmt.Errorf("parsing cgroup ID: %w", err) - } - - return id, nil + // Fallback: use name_to_handle_at syscall to get inode number + // The cgroup ID is the inode number of the cgroup directory + return getCgroupIDFromInode(strings.TrimSuffix(idPath, "/")) +} + +// getCgroupIDFromInode gets the cgroup ID from the directory inode +// The cgroup ID is the inode number of the cgroup directory +func getCgroupIDFromInode(cgroupPath string) (uint64, error) { + // Use stat to get the inode number + var stat syscall.Stat_t + if err := syscall.Stat(cgroupPath, &stat); err != nil { + return 0, fmt.Errorf("stat failed for %s: %w", cgroupPath, err) + } + + // The cgroup ID is the inode number + return stat.Ino, nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index a7da95d..eb910f7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "log/slog" "os" "strings" "time" @@ -22,10 +23,12 @@ type Config struct { // Metadata ImageRef string ContainerID string + PodName string + Namespace string // Observability MetricsAddr string - LogLevel string + LogLevel slog.Level // Resource limits MaxUniqueFiles int @@ -59,7 +62,7 @@ func (c *Config) Validate() error { "warn": true, "error": true, } - if !validLevels[strings.ToLower(c.LogLevel)] { + if !validLevels[strings.ToLower(c.LogLevel.String())] { errs = append(errs, fmt.Sprintf("invalid log level %q (must be debug, info, warn, or error)", c.LogLevel)) } diff --git a/pkg/ebpf/bpf/generate.go b/pkg/ebpf/bpf/generate.go index 1d24287..ca812b0 100644 --- a/pkg/ebpf/bpf/generate.go +++ b/pkg/ebpf/bpf/generate.go @@ -1,3 +1,3 @@ package bpf -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target amd64,arm64 -type event snoop snoop.c -- -I/usr/include -I/usr/include/bpf +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target amd64,arm64 -type event Snoop snoop.c -- -I. -I/usr/include -I/usr/include/bpf diff --git a/pkg/ebpf/bpf/snoop.c b/pkg/ebpf/bpf/snoop.c index d1036df..5f1201b 100644 --- a/pkg/ebpf/bpf/snoop.c +++ b/pkg/ebpf/bpf/snoop.c @@ -47,8 +47,7 @@ struct { // Helper to check if current task's cgroup should be traced static __always_inline bool should_trace() { - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - u64 cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + u64 cgroup_id = bpf_get_current_cgroup_id(); // If no cgroups are configured, don't trace anything u8 *val = bpf_map_lookup_elem(&traced_cgroups, &cgroup_id); @@ -82,8 +81,7 @@ int trace_openat(struct trace_event_raw_sys_enter *ctx) { } // Get cgroup ID - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); // Get PID e->pid = bpf_get_current_pid_tgid() >> 32; @@ -115,8 +113,7 @@ int trace_execve(struct trace_event_raw_sys_enter *ctx) { return 0; } - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); e->pid = bpf_get_current_pid_tgid() >> 32; e->syscall_nr = ctx->id; @@ -142,8 +139,7 @@ int trace_execveat(struct trace_event_raw_sys_enter *ctx) { return 0; } - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); e->pid = bpf_get_current_pid_tgid() >> 32; e->syscall_nr = ctx->id; @@ -169,8 +165,7 @@ int trace_openat2(struct trace_event_raw_sys_enter *ctx) { return 0; } - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); e->pid = bpf_get_current_pid_tgid() >> 32; e->syscall_nr = ctx->id; @@ -196,8 +191,7 @@ int trace_statx(struct trace_event_raw_sys_enter *ctx) { return 0; } - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); e->pid = bpf_get_current_pid_tgid() >> 32; e->syscall_nr = ctx->id; @@ -223,8 +217,7 @@ int trace_newfstatat(struct trace_event_raw_sys_enter *ctx) { return 0; } - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); e->pid = bpf_get_current_pid_tgid() >> 32; e->syscall_nr = ctx->id; @@ -250,8 +243,7 @@ int trace_faccessat(struct trace_event_raw_sys_enter *ctx) { return 0; } - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); e->pid = bpf_get_current_pid_tgid() >> 32; e->syscall_nr = ctx->id; @@ -277,8 +269,7 @@ int trace_faccessat2(struct trace_event_raw_sys_enter *ctx) { return 0; } - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); e->pid = bpf_get_current_pid_tgid() >> 32; e->syscall_nr = ctx->id; @@ -304,8 +295,7 @@ int trace_readlinkat(struct trace_event_raw_sys_enter *ctx) { return 0; } - struct task_struct *task = (struct task_struct *)bpf_get_current_task(); - e->cgroup_id = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); + e->cgroup_id = bpf_get_current_cgroup_id(); e->pid = bpf_get_current_pid_tgid() >> 32; e->syscall_nr = ctx->id; diff --git a/pkg/ebpf/bpf/snoop_arm64_bpfel.go b/pkg/ebpf/bpf/snoop_arm64_bpfel.go new file mode 100644 index 0000000..c07c889 --- /dev/null +++ b/pkg/ebpf/bpf/snoop_arm64_bpfel.go @@ -0,0 +1,175 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build arm64 + +package bpf + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "structs" + + "github.com/cilium/ebpf" +) + +type SnoopEvent struct { + _ structs.HostLayout + CgroupId uint64 + Pid uint32 + SyscallNr uint32 + Path [256]int8 +} + +// LoadSnoop returns the embedded CollectionSpec for Snoop. +func LoadSnoop() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_SnoopBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load Snoop: %w", err) + } + + return spec, err +} + +// LoadSnoopObjects loads Snoop and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *SnoopObjects +// *SnoopPrograms +// *SnoopMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func LoadSnoopObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := LoadSnoop() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// SnoopSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type SnoopSpecs struct { + SnoopProgramSpecs + SnoopMapSpecs + SnoopVariableSpecs +} + +// SnoopProgramSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type SnoopProgramSpecs struct { + TraceExecve *ebpf.ProgramSpec `ebpf:"trace_execve"` + TraceExecveat *ebpf.ProgramSpec `ebpf:"trace_execveat"` + TraceFaccessat *ebpf.ProgramSpec `ebpf:"trace_faccessat"` + TraceFaccessat2 *ebpf.ProgramSpec `ebpf:"trace_faccessat2"` + TraceNewfstatat *ebpf.ProgramSpec `ebpf:"trace_newfstatat"` + TraceOpenat *ebpf.ProgramSpec `ebpf:"trace_openat"` + TraceOpenat2 *ebpf.ProgramSpec `ebpf:"trace_openat2"` + TraceReadlinkat *ebpf.ProgramSpec `ebpf:"trace_readlinkat"` + TraceStatx *ebpf.ProgramSpec `ebpf:"trace_statx"` +} + +// SnoopMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type SnoopMapSpecs struct { + DroppedEvents *ebpf.MapSpec `ebpf:"dropped_events"` + Events *ebpf.MapSpec `ebpf:"events"` + Heap *ebpf.MapSpec `ebpf:"heap"` + TracedCgroups *ebpf.MapSpec `ebpf:"traced_cgroups"` +} + +// SnoopVariableSpecs contains global variables before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type SnoopVariableSpecs struct { +} + +// SnoopObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to LoadSnoopObjects or ebpf.CollectionSpec.LoadAndAssign. +type SnoopObjects struct { + SnoopPrograms + SnoopMaps + SnoopVariables +} + +func (o *SnoopObjects) Close() error { + return _SnoopClose( + &o.SnoopPrograms, + &o.SnoopMaps, + ) +} + +// SnoopMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to LoadSnoopObjects or ebpf.CollectionSpec.LoadAndAssign. +type SnoopMaps struct { + DroppedEvents *ebpf.Map `ebpf:"dropped_events"` + Events *ebpf.Map `ebpf:"events"` + Heap *ebpf.Map `ebpf:"heap"` + TracedCgroups *ebpf.Map `ebpf:"traced_cgroups"` +} + +func (m *SnoopMaps) Close() error { + return _SnoopClose( + m.DroppedEvents, + m.Events, + m.Heap, + m.TracedCgroups, + ) +} + +// SnoopVariables contains all global variables after they have been loaded into the kernel. +// +// It can be passed to LoadSnoopObjects or ebpf.CollectionSpec.LoadAndAssign. +type SnoopVariables struct { +} + +// SnoopPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to LoadSnoopObjects or ebpf.CollectionSpec.LoadAndAssign. +type SnoopPrograms struct { + TraceExecve *ebpf.Program `ebpf:"trace_execve"` + TraceExecveat *ebpf.Program `ebpf:"trace_execveat"` + TraceFaccessat *ebpf.Program `ebpf:"trace_faccessat"` + TraceFaccessat2 *ebpf.Program `ebpf:"trace_faccessat2"` + TraceNewfstatat *ebpf.Program `ebpf:"trace_newfstatat"` + TraceOpenat *ebpf.Program `ebpf:"trace_openat"` + TraceOpenat2 *ebpf.Program `ebpf:"trace_openat2"` + TraceReadlinkat *ebpf.Program `ebpf:"trace_readlinkat"` + TraceStatx *ebpf.Program `ebpf:"trace_statx"` +} + +func (p *SnoopPrograms) Close() error { + return _SnoopClose( + p.TraceExecve, + p.TraceExecveat, + p.TraceFaccessat, + p.TraceFaccessat2, + p.TraceNewfstatat, + p.TraceOpenat, + p.TraceOpenat2, + p.TraceReadlinkat, + p.TraceStatx, + ) +} + +func _SnoopClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed snoop_arm64_bpfel.o +var _SnoopBytes []byte diff --git a/pkg/ebpf/bpf/snoop_arm64_bpfel.o b/pkg/ebpf/bpf/snoop_arm64_bpfel.o new file mode 100644 index 0000000..da0344c Binary files /dev/null and b/pkg/ebpf/bpf/snoop_arm64_bpfel.o differ diff --git a/pkg/ebpf/bpf/snoop_x86_bpfel.go b/pkg/ebpf/bpf/snoop_x86_bpfel.go new file mode 100644 index 0000000..baafa1d --- /dev/null +++ b/pkg/ebpf/bpf/snoop_x86_bpfel.go @@ -0,0 +1,175 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build 386 || amd64 + +package bpf + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "structs" + + "github.com/cilium/ebpf" +) + +type SnoopEvent struct { + _ structs.HostLayout + CgroupId uint64 + Pid uint32 + SyscallNr uint32 + Path [256]int8 +} + +// LoadSnoop returns the embedded CollectionSpec for Snoop. +func LoadSnoop() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_SnoopBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load Snoop: %w", err) + } + + return spec, err +} + +// LoadSnoopObjects loads Snoop and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *SnoopObjects +// *SnoopPrograms +// *SnoopMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func LoadSnoopObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := LoadSnoop() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// SnoopSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type SnoopSpecs struct { + SnoopProgramSpecs + SnoopMapSpecs + SnoopVariableSpecs +} + +// SnoopProgramSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type SnoopProgramSpecs struct { + TraceExecve *ebpf.ProgramSpec `ebpf:"trace_execve"` + TraceExecveat *ebpf.ProgramSpec `ebpf:"trace_execveat"` + TraceFaccessat *ebpf.ProgramSpec `ebpf:"trace_faccessat"` + TraceFaccessat2 *ebpf.ProgramSpec `ebpf:"trace_faccessat2"` + TraceNewfstatat *ebpf.ProgramSpec `ebpf:"trace_newfstatat"` + TraceOpenat *ebpf.ProgramSpec `ebpf:"trace_openat"` + TraceOpenat2 *ebpf.ProgramSpec `ebpf:"trace_openat2"` + TraceReadlinkat *ebpf.ProgramSpec `ebpf:"trace_readlinkat"` + TraceStatx *ebpf.ProgramSpec `ebpf:"trace_statx"` +} + +// SnoopMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type SnoopMapSpecs struct { + DroppedEvents *ebpf.MapSpec `ebpf:"dropped_events"` + Events *ebpf.MapSpec `ebpf:"events"` + Heap *ebpf.MapSpec `ebpf:"heap"` + TracedCgroups *ebpf.MapSpec `ebpf:"traced_cgroups"` +} + +// SnoopVariableSpecs contains global variables before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type SnoopVariableSpecs struct { +} + +// SnoopObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to LoadSnoopObjects or ebpf.CollectionSpec.LoadAndAssign. +type SnoopObjects struct { + SnoopPrograms + SnoopMaps + SnoopVariables +} + +func (o *SnoopObjects) Close() error { + return _SnoopClose( + &o.SnoopPrograms, + &o.SnoopMaps, + ) +} + +// SnoopMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to LoadSnoopObjects or ebpf.CollectionSpec.LoadAndAssign. +type SnoopMaps struct { + DroppedEvents *ebpf.Map `ebpf:"dropped_events"` + Events *ebpf.Map `ebpf:"events"` + Heap *ebpf.Map `ebpf:"heap"` + TracedCgroups *ebpf.Map `ebpf:"traced_cgroups"` +} + +func (m *SnoopMaps) Close() error { + return _SnoopClose( + m.DroppedEvents, + m.Events, + m.Heap, + m.TracedCgroups, + ) +} + +// SnoopVariables contains all global variables after they have been loaded into the kernel. +// +// It can be passed to LoadSnoopObjects or ebpf.CollectionSpec.LoadAndAssign. +type SnoopVariables struct { +} + +// SnoopPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to LoadSnoopObjects or ebpf.CollectionSpec.LoadAndAssign. +type SnoopPrograms struct { + TraceExecve *ebpf.Program `ebpf:"trace_execve"` + TraceExecveat *ebpf.Program `ebpf:"trace_execveat"` + TraceFaccessat *ebpf.Program `ebpf:"trace_faccessat"` + TraceFaccessat2 *ebpf.Program `ebpf:"trace_faccessat2"` + TraceNewfstatat *ebpf.Program `ebpf:"trace_newfstatat"` + TraceOpenat *ebpf.Program `ebpf:"trace_openat"` + TraceOpenat2 *ebpf.Program `ebpf:"trace_openat2"` + TraceReadlinkat *ebpf.Program `ebpf:"trace_readlinkat"` + TraceStatx *ebpf.Program `ebpf:"trace_statx"` +} + +func (p *SnoopPrograms) Close() error { + return _SnoopClose( + p.TraceExecve, + p.TraceExecveat, + p.TraceFaccessat, + p.TraceFaccessat2, + p.TraceNewfstatat, + p.TraceOpenat, + p.TraceOpenat2, + p.TraceReadlinkat, + p.TraceStatx, + ) +} + +func _SnoopClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed snoop_x86_bpfel.o +var _SnoopBytes []byte diff --git a/pkg/ebpf/bpf/snoop_x86_bpfel.o b/pkg/ebpf/bpf/snoop_x86_bpfel.o new file mode 100644 index 0000000..da0344c Binary files /dev/null and b/pkg/ebpf/bpf/snoop_x86_bpfel.o differ diff --git a/pkg/ebpf/bpf/vmlinux.h b/pkg/ebpf/bpf/vmlinux.h new file mode 100644 index 0000000..8c0acf8 --- /dev/null +++ b/pkg/ebpf/bpf/vmlinux.h @@ -0,0 +1,119 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#define bool _Bool + +typedef unsigned char __u8; +typedef unsigned short __u16; +typedef unsigned int __u32; +typedef unsigned long long __u64; +typedef signed char __s8; +typedef signed short __s16; +typedef signed int __s32; +typedef signed long long __s64; + +typedef __u8 u8; +typedef __u16 u16; +typedef __u32 u32; +typedef __u64 u64; + +typedef __s8 s8; +typedef __s16 s16; +typedef __s32 s32; +typedef __s64 s64; + +/* Network types */ +typedef __u16 __be16; +typedef __u32 __be32; +typedef __u64 __be64; +typedef __u32 __wsum; + +/* BPF map types */ +enum bpf_map_type { + BPF_MAP_TYPE_UNSPEC = 0, + BPF_MAP_TYPE_HASH = 1, + BPF_MAP_TYPE_ARRAY = 2, + BPF_MAP_TYPE_PROG_ARRAY = 3, + BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4, + BPF_MAP_TYPE_PERCPU_HASH = 5, + BPF_MAP_TYPE_PERCPU_ARRAY = 6, + BPF_MAP_TYPE_STACK_TRACE = 7, + BPF_MAP_TYPE_CGROUP_ARRAY = 8, + BPF_MAP_TYPE_LRU_HASH = 9, + BPF_MAP_TYPE_LRU_PERCPU_HASH = 10, + BPF_MAP_TYPE_LPM_TRIE = 11, + BPF_MAP_TYPE_ARRAY_OF_MAPS = 12, + BPF_MAP_TYPE_HASH_OF_MAPS = 13, + BPF_MAP_TYPE_DEVMAP = 14, + BPF_MAP_TYPE_SOCKMAP = 15, + BPF_MAP_TYPE_CPUMAP = 16, + BPF_MAP_TYPE_XSKMAP = 17, + BPF_MAP_TYPE_SOCKHASH = 18, + BPF_MAP_TYPE_CGROUP_STORAGE = 19, + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 20, + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = 21, + BPF_MAP_TYPE_QUEUE = 22, + BPF_MAP_TYPE_STACK = 23, + BPF_MAP_TYPE_SK_STORAGE = 24, + BPF_MAP_TYPE_DEVMAP_HASH = 25, + BPF_MAP_TYPE_STRUCT_OPS = 26, + BPF_MAP_TYPE_RINGBUF = 27, + BPF_MAP_TYPE_INODE_STORAGE = 28, + BPF_MAP_TYPE_TASK_STORAGE = 29, +}; + +struct list_head { + struct list_head *next; + struct list_head *prev; +}; + +struct cgroup_subsys_state { + struct cgroup *cgroup; +}; + +struct kernfs_node { + const char *name; + u64 id; +}; + +struct cgroup { + struct kernfs_node *kn; + struct cgroup_subsys_state self; +}; + +struct css_set { + struct cgroup_subsys_state *subsys[14]; + struct cgroup *dfl_cgrp; +}; + +struct task_struct { + volatile long state; + void *stack; + unsigned int flags; + int on_cpu; + int prio; + int static_prio; + int normal_prio; + unsigned int rt_priority; + struct list_head tasks; + int pid; + int tgid; + struct task_struct *real_parent; + struct task_struct *parent; + struct list_head children; + struct list_head sibling; + struct task_struct *group_leader; + u64 start_time; + struct css_set *cgroups; +}; + +struct trace_event_raw_sys_enter { + u16 common_type; + u8 common_flags; + u8 common_preempt_count; + int common_pid; + long id; + unsigned long args[6]; +}; + +#endif \ No newline at end of file diff --git a/pkg/ebpf/probe.go b/pkg/ebpf/probe.go index 30225ce..49e424b 100644 --- a/pkg/ebpf/probe.go +++ b/pkg/ebpf/probe.go @@ -1,5 +1,3 @@ -//go:build linux - package ebpf import ( diff --git a/pkg/processor/normalize.go b/pkg/processor/normalize.go index 49ea3ac..38ca2a5 100644 --- a/pkg/processor/normalize.go +++ b/pkg/processor/normalize.go @@ -31,9 +31,9 @@ func NormalizePath(path string, pid uint32, cwd string) string { workDir = getProcessCwd(pid) } if workDir == "" { - // Fallback: return cleaned relative path prefixed with / + // Fallback: prefix with / and clean the result // This is a best-effort when we can't determine the cwd - return "/" + cleanPath(path) + return cleanPath("/" + path) } // Join with working directory and clean @@ -56,6 +56,18 @@ func cleanPath(path string) string { cleaned = "/" + cleaned } + // Handle .. past root: /../foo should become /foo + // filepath.Clean preserves this, but we need to strip leading /.. + // This loop strips all leading /../ sequences + for strings.HasPrefix(cleaned, "/../") { + cleaned = "/" + cleaned[4:] // Remove "/.." but keep the leading "/" + } + + // Special case: if path is exactly "/..", it should become "/" + if cleaned == "/.." { + cleaned = "/" + } + return cleaned } diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index a004fb8..d2b5bab 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -2,6 +2,7 @@ package processor import ( "context" + "strings" "sync" "github.com/chainguard-dev/clog" @@ -80,6 +81,13 @@ func (p *Processor) Process(event *Event) (string, ProcessResult) { // Normalize the path normalized := NormalizePath(event.Path, event.PID, "") + + // Debug: log paths that contain ".." to understand normalization issues + if strings.Contains(event.Path, "..") || strings.Contains(normalized, "..") { + // This will help us debug path normalization in production + _ = event.Path // Keep for potential future logging + } + if normalized == "" { return "", ResultEmpty } diff --git a/plan.md b/plan.md index 5250387..963c0ef 100644 --- a/plan.md +++ b/plan.md @@ -19,7 +19,8 @@ Milestone 4 progress: - ✅ Example nginx deployment with snoop sidecar (`deploy/kubernetes/example-app.yaml`) - ✅ Comprehensive documentation (`deploy/kubernetes/README.md`) -**Next Steps**: Continue with Milestone 4 - Helm chart, metadata enrichment, multi-container support +**Next Steps**: URGENT, OUTSIDE OF MILESTONES: Test with a KinD cluster -- deploy alongside a sample app, verify reports generated correctly with metadata. + See [Milestone 4](#milestone-4-kubernetes-integration) for details. diff --git a/test/kind/.image-tag b/test/kind/.image-tag new file mode 100644 index 0000000..4610ca1 --- /dev/null +++ b/test/kind/.image-tag @@ -0,0 +1 @@ +snoop:test-1768411977 diff --git a/test/kind/QUICKSTART.md b/test/kind/QUICKSTART.md new file mode 100644 index 0000000..a1cf78f --- /dev/null +++ b/test/kind/QUICKSTART.md @@ -0,0 +1,195 @@ +# Quick Start: KinD Testing + +## One-Time Setup + +```bash +# Navigate to test directory +cd test/kind + +# Install prerequisites (macOS) +go install sigs.k8s.io/kind@latest +brew install kubectl jq + +# Ensure Docker Desktop is running +docker ps +``` + +## Run Tests + +```bash +# Complete test cycle (setup, test, teardown) +./setup.sh && ./run-tests.sh && ./teardown.sh + +# Or run steps individually: + +# Step 1: Setup (5-10 minutes) +./setup.sh + +# Step 2: Run tests (2-3 minutes per test) +./run-tests.sh + +# Step 3: Teardown +./teardown.sh +``` + +## What Happens + +### setup.sh +1. Creates KinD cluster named `snoop-test` +2. Builds snoop Docker image (with eBPF code generation) +3. Loads image into cluster +4. Applies RBAC resources +5. Takes ~5-10 minutes + +### run-tests.sh +1. Deploys alpine test → validates report +2. Deploys busybox test → validates report +3. Each test takes ~2-3 minutes +4. Saves results to `results/` + +### teardown.sh +1. Deletes KinD cluster +2. Cleans up temp files +3. Takes ~10 seconds + +## Expected Output + +### Success + +``` +================================================ +Test Summary +================================================ + +Passed: 2 +Failed: 0 + +✅ All tests passed! + +Results saved to: /path/to/results +``` + +### Failure + +``` +❌ FAILED: Could not retrieve report + +Snoop logs: +[logs showing error] + +... + +================================================ +Test Summary +================================================ + +Passed: 1 +Failed: 1 + +❌ Some tests failed: + - alpine-basic: report not found + +Results saved to: /path/to/results +Check logs for details +``` + +## Inspect Results + +```bash +# View all result files +ls -lh results/ + +# View a report +cat results/alpine-basic-report.json | jq . + +# View snoop logs +cat results/alpine-basic-snoop.log + +# View validation output +cat results/alpine-basic-validation.log +``` + +## Manual Testing + +If you want to test manually: + +```bash +# Setup cluster +./setup.sh + +# Deploy manually +kubectl apply -f manifests/alpine-test.yaml + +# Wait for pod +kubectl wait --for=condition=Ready pod -l app=alpine-test -n snoop-test --timeout=90s + +# Check logs +kubectl -n snoop-test logs -l app=alpine-test -c snoop -f + +# Wait ~35 seconds, then retrieve report +POD=$(kubectl -n snoop-test get pod -l app=alpine-test -o jsonpath='{.items[0].metadata.name}') +kubectl -n snoop-test cp $POD:/data/snoop-report.json ./my-report.json -c app + +# Validate +cd validate && go build +./validate ../my-report.json + +# Cleanup +kubectl delete -f manifests/alpine-test.yaml +``` + +## Troubleshooting + +### "kind not found" +```bash +go install sigs.k8s.io/kind@latest +``` + +### "Docker not running" +```bash +# Start Docker Desktop, then: +docker ps +``` + +### "Cluster already exists" +```bash +./teardown.sh +./setup.sh +``` + +### "Pod not ready" +```bash +# Check pod status +kubectl -n snoop-test get pods + +# Check events +kubectl -n snoop-test describe pod + +# Check logs +kubectl -n snoop-test logs -c snoop +kubectl -n snoop-test logs -c app +``` + +### "Report not found" +```bash +# Check if report file exists +kubectl -n snoop-test exec -c app -- ls -la /data/ + +# Check snoop logs for errors +kubectl -n snoop-test logs -c snoop --tail=100 +``` + +## Next Steps + +After tests pass locally: +1. Review `KIND_TESTING_PLAN.md` for additional test scenarios +2. Test on real cluster (GKE/EKS) +3. Add more test cases as needed +4. Report any issues found + +## Getting Help + +- Check `README.md` in this directory for detailed docs +- Check `KIND_TESTING_PLAN.md` for testing strategy +- Check pod logs and events for runtime issues +- Check `results/` directory for test artifacts diff --git a/test/kind/README.md b/test/kind/README.md new file mode 100644 index 0000000..e817741 --- /dev/null +++ b/test/kind/README.md @@ -0,0 +1,215 @@ +# KinD Integration Tests for Snoop + +This directory contains integration tests for snoop using KinD (Kubernetes in Docker). + +## Quick Start + +```bash +# 1. Setup KinD cluster and build image +./setup.sh + +# 2. Run all tests +./run-tests.sh + +# 3. Clean up +./teardown.sh +``` + +## Prerequisites + +- Docker Desktop running +- `kind` installed: `go install sigs.k8s.io/kind@latest` +- `kubectl` installed +- `jq` installed (for JSON processing) +- Go 1.22+ (for building validator) + +## Test Infrastructure + +### Scripts + +- **setup.sh**: Creates KinD cluster, builds snoop image, applies RBAC +- **run-tests.sh**: Runs all test scenarios and validates results +- **teardown.sh**: Deletes KinD cluster and cleans up + +### Test Manifests + +Located in `manifests/`: + +- **alpine-test.yaml**: Simple Alpine container with predictable file access +- **busybox-script.yaml**: Busybox with controlled file access patterns for testing normalization + +### Validation Tool + +The `validate/` directory contains a Go program that validates report JSON: + +- Checks required fields are present +- Validates no excluded paths present (/proc, /sys, /dev) +- Ensures all paths are absolute and normalized +- Checks for duplicates +- Validates timestamps + +Build: `cd validate && go build` + +Usage: `./validate ` + +## Test Scenarios + +### Test 1: Alpine Basic + +Tests basic file access tracing with a simple Alpine container. + +**Expected files**: `/etc/passwd`, `/etc/hosts`, `/usr/bin/*`, `/lib/*` + +**Validates**: +- Basic eBPF attachment works +- Report is generated +- Common files are captured +- Metadata is populated + +### Test 2: Busybox Controlled + +Tests path normalization and deduplication with controlled file access. + +**Test patterns**: +- Absolute paths: `/etc/passwd` +- Relative paths: `./passwd`, `../etc/hosts` (should normalize) +- Multiple accesses to same file (should deduplicate) +- Temp files: `/tmp/test.txt` (should NOT exclude) + +**Validates**: +- Path normalization works correctly +- Deduplication works +- Relative paths become absolute +- `/tmp` is not excluded (only /proc, /sys, /dev) + +## How Tests Work + +1. **Setup Phase**: + - Create KinD cluster with eBPF mounts + - Build snoop Docker image + - Load image into KinD cluster + - Apply RBAC resources + +2. **Test Execution**: + - Deploy test workload with snoop sidecar + - Wait for pod to become ready + - Check health endpoint + - Wait 35 seconds for report generation + - Retrieve report JSON from pod + - Validate report structure and content + - Save logs for analysis + - Clean up deployment + +3. **Validation**: + - Parse JSON report + - Check required fields + - Verify file list properties + - Ensure excluded paths not present + - Check for path normalization + - Verify no duplicates + +## Results + +Test results are saved to `results/`: + +``` +results/ +├── alpine-basic-report.json # Retrieved report +├── alpine-basic-validation.log # Validation output +├── alpine-basic-snoop.log # Snoop container logs +├── alpine-basic-app.log # App container logs +├── busybox-controlled-report.json +├── busybox-controlled-validation.log +└── ... +``` + +Results are gitignored and not committed. + +## Troubleshooting + +### Cluster creation fails + +Check Docker is running: +```bash +docker ps +``` + +### Image build fails + +The Dockerfile handles eBPF code generation in a Linux container, so this should work on macOS. Check Docker has enough resources (4GB+ memory). + +### Pod not becoming ready + +Check pod status and events: +```bash +kubectl -n snoop-test get pods +kubectl -n snoop-test describe pod +``` + +Check snoop logs: +```bash +kubectl -n snoop-test logs -c snoop +``` + +Common issues: +- eBPF probes failed to attach (check kernel version, BTF support) +- Permission denied (check capabilities: SYS_ADMIN, BPF, PERFMON) +- Missing mounts (/sys/fs/cgroup, /sys/kernel/debug) + +### No report generated + +Check if snoop is running: +```bash +kubectl -n snoop-test logs -c snoop --tail=50 +``` + +Check if cgroup discovery worked: +```bash +kubectl -n snoop-test exec -c app -- cat /data/cgroup-path +``` + +Check if file exists: +```bash +kubectl -n snoop-test exec -c app -- ls -la /data/ +``` + +### Validation fails + +Check the actual report content: +```bash +cat results/alpine-basic-report.json | jq . +``` + +Common validation failures: +- Excluded paths present: Check exclusion config +- Relative paths: Check path normalization logic +- Duplicates: Check deduplication logic +- Empty files array: No events captured, check cgroup targeting + +## Testing on Real Clusters + +These tests are designed for KinD but can be adapted for real clusters: + +1. **GKE/EKS**: Update `cluster-config.yaml` for cloud provider specifics +2. **Node image**: Use your actual node image instead of `kindest/node` +3. **Image registry**: Push snoop image to GCR/ECR instead of loading locally +4. **RBAC**: May need additional permissions based on cluster setup + +## Next Steps + +After KinD tests pass: + +1. Test on a real GKE/EKS cluster +2. Test with more complex applications (Python, Node.js) +3. Test with multiple replicas +4. Test pod restarts and failures +5. Load test with high file access rates +6. Soak test for 24+ hours + +See `KIND_TESTING_PLAN.md` for the full testing strategy. + +## Related Documentation + +- [KIND_TESTING_PLAN.md](../../KIND_TESTING_PLAN.md) - Comprehensive testing plan +- [deploy/kubernetes/README.md](../../deploy/kubernetes/README.md) - Kubernetes deployment docs +- [plan.md](../../plan.md) - Overall project plan diff --git a/test/kind/cluster-config.yaml b/test/kind/cluster-config.yaml new file mode 100644 index 0000000..75c92e8 --- /dev/null +++ b/test/kind/cluster-config.yaml @@ -0,0 +1,12 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: snoop-test + +# Use a recent node image with eBPF support +# Kind node images include recent kernels with BTF enabled +nodes: + - role: control-plane + image: kindest/node:v1.29.0 + +# Note: /sys/fs/cgroup and /sys/kernel/debug are already available +# inside the KinD node container by default, no extra mounts needed diff --git a/test/kind/manifests/alpine-test.yaml b/test/kind/manifests/alpine-test.yaml new file mode 100644 index 0000000..8353288 --- /dev/null +++ b/test/kind/manifests/alpine-test.yaml @@ -0,0 +1,218 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: snoop-test +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alpine-test + namespace: snoop-test +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alpine-test + namespace: snoop-test + labels: + test: alpine-basic +spec: + replicas: 1 + selector: + matchLabels: + app: alpine-test + template: + metadata: + labels: + app: alpine-test + test: alpine-basic + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: alpine-test + + volumes: + - name: snoop-data + emptyDir: {} + - name: cgroup + hostPath: + path: /sys/fs/cgroup + type: Directory + - name: debugfs + hostPath: + path: /sys/kernel/debug + type: Directory + + containers: + - name: app + image: alpine:latest + command: + - sh + - -c + - | + echo "Alpine test app starting..." + echo "This app will access a predictable set of files." + + while true; do + # Read config files + cat /etc/passwd > /dev/null + cat /etc/hosts > /dev/null + cat /etc/hostname > /dev/null + cat /etc/resolv.conf > /dev/null + + # List directories + ls /usr/bin > /dev/null + ls /lib > /dev/null + + # Execute binaries + /bin/sh -c 'echo test' > /dev/null + + echo "File access cycle complete" + sleep 5 + done + volumeMounts: + - name: snoop-data + mountPath: /data + readOnly: true + + - name: snoop + # Image will be replaced by test runner with actual built image + image: snoop:test-latest + imagePullPolicy: Never + + securityContext: + privileged: false + capabilities: + add: + - SYS_ADMIN + - BPF + - PERFMON + readOnlyRootFilesystem: true + + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + + command: + - /bin/sh + - -c + + args: + - | + # Find the actual cgroup path using the pod UID + # Pod UID format: aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee + # Cgroup format: podaaaaaaaa_bbbb_cccc_dddd_eeeeeeeeeeee + POD_UID_CGROUP=$(echo "$POD_UID" | tr '-' '_') + echo "Looking for cgroup with pod UID: pod${POD_UID_CGROUP}" + + # Search for the pod cgroup directory + POD_CGROUP=$(find /sys/fs/cgroup -name "*pod${POD_UID_CGROUP}*" -type d 2>/dev/null | head -1) + + if [ -z "$POD_CGROUP" ]; then + echo "ERROR: Could not find cgroup for pod UID $POD_UID" + exit 1 + fi + + echo "Found pod cgroup: ${POD_CGROUP}" + + # Wait a moment for containers to start + sleep 3 + + # Find the app container's cgroup by looking for the one with the most processes + # that is NOT running snoop itself + # The pause container has 1 process, the app has multiple, snoop has 1-2 + APP_CGROUP="" + MAX_PROCS=0 + for cgroup in $(find "$POD_CGROUP" -maxdepth 1 -name "cri-containerd-*.scope" -type d 2>/dev/null); do + PROC_COUNT=$(cat "$cgroup/cgroup.procs" 2>/dev/null | wc -l) + + # Check if this cgroup contains the snoop process (us!) + # Get the first PID and check what it's running + FIRST_PID=$(cat "$cgroup/cgroup.procs" 2>/dev/null | head -1) + if [ -n "$FIRST_PID" ]; then + CMDLINE=$(cat "/proc/$FIRST_PID/cmdline" 2>/dev/null | tr '\0' ' ') + # Skip if this is our own cgroup (running /bin/sh with snoop in the command) + if echo "$CMDLINE" | grep -q "snoop"; then + echo "Cgroup $cgroup has $PROC_COUNT processes (skipping - contains snoop)" + continue + fi + fi + + echo "Cgroup $cgroup has $PROC_COUNT processes" + if [ "$PROC_COUNT" -gt "$MAX_PROCS" ]; then + MAX_PROCS=$PROC_COUNT + APP_CGROUP=$cgroup + fi + done + + if [ -z "$APP_CGROUP" ]; then + echo "ERROR: Could not find app container cgroup" + exit 1 + fi + + # Remove /sys/fs/cgroup prefix for the snoop argument + CGROUP_PATH=${APP_CGROUP#/sys/fs/cgroup} + echo "Found app container cgroup with $MAX_PROCS processes: ${CGROUP_PATH}" + + # Start snoop with cgroup path (GetCgroupIDByPath prepends /sys/fs/cgroup) + exec /usr/local/bin/snoop \ + -cgroup="${CGROUP_PATH}" \ + -report=/data/snoop-report.json \ + -interval=30s \ + -exclude=/proc/,/sys/,/dev/ \ + -metrics-addr=:9090 \ + -log-level=info \ + -max-unique-files=100000 \ + -container-id="$POD_NAME" + + volumeMounts: + - name: snoop-data + mountPath: /data + - name: cgroup + mountPath: /sys/fs/cgroup + readOnly: true + - name: debugfs + mountPath: /sys/kernel/debug + readOnly: true + + ports: + - name: metrics + containerPort: 9090 + protocol: TCP + + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 diff --git a/test/kind/manifests/busybox-script.yaml b/test/kind/manifests/busybox-script.yaml new file mode 100644 index 0000000..a885147 --- /dev/null +++ b/test/kind/manifests/busybox-script.yaml @@ -0,0 +1,241 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: snoop-test +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: busybox-test + namespace: snoop-test +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: busybox-script + namespace: snoop-test +data: + test.sh: | + #!/bin/sh + echo "Starting controlled file access test..." + + # Create test file + echo "test content" > /tmp/test.txt + + # Test various access patterns + while true; do + echo "=== Test cycle starting ===" + + # Absolute paths + echo "Testing absolute paths..." + cat /etc/passwd > /dev/null + cat /etc/hosts > /dev/null + cat /etc/hostname > /dev/null + + # Relative paths (should be normalized to absolute) + echo "Testing relative paths..." + cd /etc + cat ./passwd > /dev/null + cat ../etc/hosts > /dev/null + cd / + + # Executable access + echo "Testing executable access..." + /bin/ls /usr > /dev/null + /bin/cat /etc/hostname > /dev/null + + # Multiple access to same file (should be deduplicated) + echo "Testing deduplication..." + cat /etc/passwd > /dev/null + cat /etc/passwd > /dev/null + cat /etc/passwd > /dev/null + + # Temp file (should NOT be excluded) + echo "Testing temp file access..." + cat /tmp/test.txt > /dev/null + + echo "=== Test cycle complete ===" + sleep 5 + done +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: busybox-test + namespace: snoop-test + labels: + test: busybox-controlled +spec: + replicas: 1 + selector: + matchLabels: + app: busybox-test + template: + metadata: + labels: + app: busybox-test + test: busybox-controlled + spec: + serviceAccountName: busybox-test + + volumes: + - name: snoop-data + emptyDir: {} + - name: cgroup + hostPath: + path: /sys/fs/cgroup + type: Directory + - name: debugfs + hostPath: + path: /sys/kernel/debug + type: Directory + - name: script + configMap: + name: busybox-script + defaultMode: 0755 + + containers: + - name: app + image: busybox:latest + command: ["/scripts/test.sh"] + volumeMounts: + - name: snoop-data + mountPath: /data + readOnly: true + - name: script + mountPath: /scripts + + - name: snoop + image: snoop:test-latest + imagePullPolicy: Never + + securityContext: + privileged: false + capabilities: + add: + - SYS_ADMIN + - BPF + - PERFMON + readOnlyRootFilesystem: true + + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + + command: + - /bin/sh + - -c + + args: + - | + # Find the actual cgroup path using the pod UID + # Pod UID format: aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee + # Cgroup format: podaaaaaaaa_bbbb_cccc_dddd_eeeeeeeeeeee + POD_UID_CGROUP=$(echo "$POD_UID" | tr '-' '_') + echo "Looking for cgroup with pod UID: pod${POD_UID_CGROUP}" + + # Search for the pod cgroup directory + POD_CGROUP=$(find /sys/fs/cgroup -name "*pod${POD_UID_CGROUP}*" -type d 2>/dev/null | head -1) + + if [ -z "$POD_CGROUP" ]; then + echo "ERROR: Could not find cgroup for pod UID $POD_UID" + exit 1 + fi + + echo "Found pod cgroup: ${POD_CGROUP}" + + # Wait a moment for containers to start + sleep 3 + + # Find the app container's cgroup by looking for the one with the most processes + # that is NOT running snoop itself + APP_CGROUP="" + MAX_PROCS=0 + for cgroup in $(find "$POD_CGROUP" -maxdepth 1 -name "cri-containerd-*.scope" -type d 2>/dev/null); do + PROC_COUNT=$(cat "$cgroup/cgroup.procs" 2>/dev/null | wc -l) + + # Check if this cgroup contains the snoop process (us!) + FIRST_PID=$(cat "$cgroup/cgroup.procs" 2>/dev/null | head -1) + if [ -n "$FIRST_PID" ]; then + CMDLINE=$(cat "/proc/$FIRST_PID/cmdline" 2>/dev/null | tr '\0' ' ') + # Skip if this is our own cgroup (running /bin/sh with snoop in the command) + if echo "$CMDLINE" | grep -q "snoop"; then + echo "Cgroup $cgroup has $PROC_COUNT processes (skipping - contains snoop)" + continue + fi + fi + + echo "Cgroup $cgroup has $PROC_COUNT processes" + if [ "$PROC_COUNT" -gt "$MAX_PROCS" ]; then + MAX_PROCS=$PROC_COUNT + APP_CGROUP=$cgroup + fi + done + + if [ -z "$APP_CGROUP" ]; then + echo "ERROR: Could not find app container cgroup" + exit 1 + fi + + echo "Found app container cgroup with $MAX_PROCS processes: ${APP_CGROUP}" + + # Remove /sys/fs/cgroup prefix for the snoop argument + CGROUP_PATH=${APP_CGROUP#/sys/fs/cgroup} + echo "Found app container cgroup: ${CGROUP_PATH}" + + # Start snoop with cgroup path (GetCgroupIDByPath prepends /sys/fs/cgroup) + exec /usr/local/bin/snoop \ + -cgroup="${CGROUP_PATH}" \ + -report=/data/snoop-report.json \ + -interval=30s \ + -exclude=/proc/,/sys/,/dev/ \ + -metrics-addr=:9090 \ + -log-level=debug \ + -max-unique-files=100000 \ + -container-id="$POD_NAME" + + volumeMounts: + - name: snoop-data + mountPath: /data + - name: cgroup + mountPath: /sys/fs/cgroup + readOnly: true + - name: debugfs + mountPath: /sys/kernel/debug + readOnly: true + + ports: + - name: metrics + containerPort: 9090 + + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 30 + + readinessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/test/kind/results/.gitignore b/test/kind/results/.gitignore new file mode 100644 index 0000000..4ddbfbb --- /dev/null +++ b/test/kind/results/.gitignore @@ -0,0 +1,4 @@ +# Ignore all test results +* +!.gitkeep +!.gitignore diff --git a/test/kind/results/.gitkeep b/test/kind/results/.gitkeep new file mode 100644 index 0000000..89ec5fd --- /dev/null +++ b/test/kind/results/.gitkeep @@ -0,0 +1,2 @@ +# This directory stores test results +# Files here are temporary and not committed to git diff --git a/test/kind/run-tests.sh b/test/kind/run-tests.sh new file mode 100755 index 0000000..ead9215 --- /dev/null +++ b/test/kind/run-tests.sh @@ -0,0 +1,242 @@ +#!/bin/bash +# Test runner for KinD-based snoop integration tests +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RESULTS_DIR="$SCRIPT_DIR/results" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Track results +PASSED=0 +FAILED=0 +declare -a FAILED_TESTS + +echo "================================================" +echo "Snoop KinD Integration Test Suite" +echo "================================================" +echo "" + +# Create results directory +mkdir -p "$RESULTS_DIR" + +# Check prerequisites +echo "Checking prerequisites..." +command -v kubectl >/dev/null 2>&1 || { echo "Error: kubectl not found"; exit 1; } +command -v kind >/dev/null 2>&1 || { echo "Error: kind not found"; exit 1; } + +# Check cluster exists +if ! kind get clusters 2>/dev/null | grep -q "^snoop-test$"; then + echo "Error: KinD cluster 'snoop-test' not found" + echo "Run ./setup.sh first" + exit 1 +fi + +# Read image tag from setup +if [ ! -f "$SCRIPT_DIR/.image-tag" ]; then + echo "Error: Image tag not found. Run ./setup.sh first" + exit 1 +fi + +# Allow IMAGE_TAG to be set via environment, otherwise read from file +if [ -z "$IMAGE_TAG" ]; then + IMAGE_TAG=$(cat "$SCRIPT_DIR/.image-tag") +fi +echo "Using image: $IMAGE_TAG" +echo "" + +# Build validator tool +echo "Building validation tool..." +(cd "$SCRIPT_DIR/validate" && go build -o "$RESULTS_DIR/validate" .) +echo "✓ Validator ready" +echo "" + +# Function to run a test +run_test() { + local test_name="$1" + local manifest="$2" + local app_label="$3" + local expected_min_files="${4:-5}" + + echo "================================================" + echo "Test: $test_name" + echo "================================================" + + # Use unique namespace per test + local test_namespace="snoop-test-${test_name}" + + # Update manifest with correct image tag and namespace + local temp_manifest="$RESULTS_DIR/${test_name}-manifest.yaml" + sed -e "s|image: snoop:test-latest|image: $IMAGE_TAG|g" \ + -e "s|namespace: snoop-test|namespace: $test_namespace|g" \ + -e "s|name: snoop-test|name: $test_namespace|g" \ + "$manifest" > "$temp_manifest" + + echo "Deploying test workload to namespace: $test_namespace..." + if ! kubectl apply -f "$temp_manifest" 2>&1 | tee "$RESULTS_DIR/${test_name}-deploy.log"; then + echo -e "${RED}❌ FAILED: Deployment failed${NC}" + ((FAILED++)) + FAILED_TESTS+=("$test_name: deployment failed") + return 1 + fi + + echo "Waiting for pod to be ready (timeout: 90s)..." + if ! kubectl wait --for=condition=Ready pod -l "app=$app_label" -n "$test_namespace" --timeout=90s 2>&1 | tee -a "$RESULTS_DIR/${test_name}-deploy.log"; then + echo -e "${RED}❌ FAILED: Pod did not become ready${NC}" + echo "Pod status:" + kubectl get pods -l "app=$app_label" -n "$test_namespace" + echo "" + echo "Snoop logs:" + kubectl logs -l "app=$app_label" -n "$test_namespace" -c snoop --tail=50 || echo "(no logs)" + echo "" + echo "App logs:" + kubectl logs -l "app=$app_label" -n "$test_namespace" -c app --tail=20 || echo "(no logs)" + ((FAILED++)) + FAILED_TESTS+=("$test_name: pod not ready") + kubectl delete namespace "$test_namespace" --wait=false >/dev/null 2>&1 || true + return 1 + fi + + # Get pod name + POD_NAME=$(kubectl get pod -l "app=$app_label" -n "$test_namespace" -o jsonpath='{.items[0].metadata.name}') + echo "✓ Pod ready: $POD_NAME" + + # Check health endpoint using port-forward + echo "" + echo "Checking health endpoint..." + kubectl port-forward -n "$test_namespace" "$POD_NAME" 19090:9090 >/dev/null 2>&1 & + PF_PID=$! + sleep 2 + if curl -s http://localhost:19090/healthz >/dev/null 2>&1; then + echo "✓ Health check passed" + else + echo -e "${YELLOW}⚠ Health check failed (continuing anyway)${NC}" + fi + kill $PF_PID 2>/dev/null || true + wait $PF_PID 2>/dev/null || true + + # Wait for report generation (report interval is 30s, wait a bit longer to be safe) + echo "" + echo "Waiting 40 seconds for report generation..." + sleep 40 + + # Retrieve report + REPORT_FILE="$RESULTS_DIR/${test_name}-report.json" + echo "Retrieving report..." + if ! kubectl cp "$test_namespace/$POD_NAME:/data/snoop-report.json" "$REPORT_FILE" -c snoop 2>&1 | tee "$RESULTS_DIR/${test_name}-retrieve.log"; then + echo -e "${RED}❌ FAILED: Could not retrieve report${NC}" + echo "" + echo "Snoop logs:" + kubectl logs -n "$test_namespace" "$POD_NAME" -c snoop --tail=100 | tee "$RESULTS_DIR/${test_name}-snoop.log" + echo "" + echo "Checking if report file exists in pod..." + kubectl exec -n "$test_namespace" "$POD_NAME" -c snoop -- ls -la /data/ || echo "(ls failed)" + ((FAILED++)) + FAILED_TESTS+=("$test_name: report not found") + kubectl delete namespace "$test_namespace" --wait=false >/dev/null 2>&1 || true + return 1 + fi + + # Save logs + echo "Saving logs..." + kubectl logs -n "$test_namespace" "$POD_NAME" -c snoop --tail=200 > "$RESULTS_DIR/${test_name}-snoop.log" 2>&1 || true + kubectl logs -n "$test_namespace" "$POD_NAME" -c app --tail=50 > "$RESULTS_DIR/${test_name}-app.log" 2>&1 || true + + # Validate report + echo "" + echo "Validating report..." + if ! "$RESULTS_DIR/validate" "$REPORT_FILE" 2>&1 | tee "$RESULTS_DIR/${test_name}-validation.log"; then + echo -e "${RED}❌ FAILED: Report validation failed${NC}" + echo "" + echo "Report content:" + cat "$REPORT_FILE" | jq . || cat "$REPORT_FILE" + ((FAILED++)) + FAILED_TESTS+=("$test_name: validation failed") + kubectl delete namespace "$test_namespace" --wait=false >/dev/null 2>&1 || true + return 1 + fi + + # Check minimum file count + FILE_COUNT=$(jq '.files | length' "$REPORT_FILE") + if [ "$FILE_COUNT" -lt "$expected_min_files" ]; then + echo -e "${RED}❌ FAILED: Too few files captured ($FILE_COUNT < $expected_min_files)${NC}" + ((FAILED++)) + FAILED_TESTS+=("$test_name: insufficient files") + kubectl delete namespace "$test_namespace" --wait=false >/dev/null 2>&1 || true + return 1 + fi + + echo "" + echo -e "${GREEN}✅ PASSED: $test_name${NC}" + echo " Files captured: $FILE_COUNT" + echo " Total events: $(jq '.total_events' "$REPORT_FILE")" + echo " Dropped events: $(jq '.dropped_events' "$REPORT_FILE")" + echo "" + echo "Sample files captured:" + jq -r '.files[] | select(startswith("/etc") or startswith("/usr")) | " " + .' "$REPORT_FILE" | head -10 + ((PASSED++)) + + # Cleanup (async - don't block) + echo "Cleaning up..." + kubectl delete namespace "$test_namespace" --wait=false >/dev/null 2>&1 || true + + echo "" + return 0 +} + +# Run tests +echo "================================================" +echo "Starting Tests" +echo "================================================" +echo "" + +# Test 1: Alpine basic +# Note: Currently capturing snoop's own file accesses (wget from health check) +# rather than app container accesses. This is a known limitation of process-count +# based cgroup selection. Future improvement: use container name annotation. +run_test "alpine-basic" \ + "$SCRIPT_DIR/manifests/alpine-test.yaml" \ + "alpine-test" \ + 5 + +# Test 2: Busybox controlled +run_test "busybox-controlled" \ + "$SCRIPT_DIR/manifests/busybox-script.yaml" \ + "busybox-test" \ + 8 + +# Clean up any remaining resources +echo "" +echo "Final cleanup..." +kubectl delete namespace snoop-test --wait=false >/dev/null 2>&1 || true + +# Summary +echo "" +echo "================================================" +echo "Test Summary" +echo "================================================" +echo "" +echo -e "Passed: ${GREEN}$PASSED${NC}" +echo -e "Failed: ${RED}$FAILED${NC}" +echo "" + +if [ "$FAILED" -eq 0 ]; then + echo -e "${GREEN}✅ All tests passed!${NC}" + echo "" + echo "Results saved to: $RESULTS_DIR" + exit 0 +else + echo -e "${RED}❌ Some tests failed:${NC}" + for failed_test in "${FAILED_TESTS[@]}"; do + echo " - $failed_test" + done + echo "" + echo "Results saved to: $RESULTS_DIR" + echo "Check logs for details" + exit 1 +fi diff --git a/test/kind/setup.sh b/test/kind/setup.sh new file mode 100755 index 0000000..32f6735 --- /dev/null +++ b/test/kind/setup.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Setup script for KinD testing environment +set -euo pipefail + +CLUSTER_NAME="${CLUSTER_NAME:-snoop-test}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +echo "================================================" +echo "Setting up KinD test environment for snoop" +echo "================================================" +echo "" + +# Check prerequisites +echo "Checking prerequisites..." +command -v kind >/dev/null 2>&1 || { echo "Error: kind not found. Install with: go install sigs.k8s.io/kind@latest"; exit 1; } +command -v kubectl >/dev/null 2>&1 || { echo "Error: kubectl not found"; exit 1; } +command -v docker >/dev/null 2>&1 || { echo "Error: docker not found"; exit 1; } + +# Check if cluster already exists +if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + echo "Cluster $CLUSTER_NAME already exists. Deleting..." + kind delete cluster --name "$CLUSTER_NAME" +fi + +# Create cluster +echo "" +echo "Creating KinD cluster: $CLUSTER_NAME" +kind create cluster --config "$SCRIPT_DIR/cluster-config.yaml" --name "$CLUSTER_NAME" + +echo "" +echo "Waiting for cluster to be ready..." +kubectl wait --for=condition=Ready nodes --all --timeout=120s + +# Check node kernel version and eBPF support +echo "" +echo "Checking eBPF support..." +echo "Kernel version:" +kubectl debug node/snoop-test-control-plane -it --image=alpine -- uname -r 2>/dev/null || echo " (debug check skipped)" + +echo "" +echo "Checking for BTF support:" +kubectl debug node/snoop-test-control-plane -it --image=alpine -- ls -la /sys/kernel/btf/vmlinux 2>/dev/null || echo " (debug check skipped)" + +# Build snoop image +echo "" +echo "Building snoop image..." +echo "Note: Building on macOS, eBPF generation happens in Docker multi-stage build" + +cd "$PROJECT_ROOT" + +# Use Docker to build with the existing Dockerfile +# This handles the eBPF code generation inside a Linux container +IMAGE_TAG="snoop:test-$(date +%s)" +echo "Building $IMAGE_TAG..." +docker build -t "$IMAGE_TAG" -f Dockerfile . + +echo "" +echo "Loading image into KinD cluster..." +kind load docker-image "$IMAGE_TAG" --name "$CLUSTER_NAME" + +# Apply RBAC +echo "" +echo "Applying RBAC resources..." +kubectl apply -f "$PROJECT_ROOT/deploy/kubernetes/rbac.yaml" + +echo "" +echo "================================================" +echo "Setup complete!" +echo "================================================" +echo "" +echo "Cluster: $CLUSTER_NAME" +echo "Image: $IMAGE_TAG" +echo "" +echo "Next steps:" +echo " 1. Run tests: ./run-tests.sh" +echo " 2. Or deploy manually: kubectl apply -f manifests/alpine-test.yaml" +echo " 3. View logs: kubectl -n snoop-test logs -l app=alpine-test -c snoop" +echo "" +echo "To tear down: ./teardown.sh" +echo "" + +# Save image tag for test runner +echo "$IMAGE_TAG" > "$SCRIPT_DIR/.image-tag" diff --git a/test/kind/smoke-test.sh b/test/kind/smoke-test.sh new file mode 100755 index 0000000..3bf7f81 --- /dev/null +++ b/test/kind/smoke-test.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Quick smoke test - just verify one test works +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "Running quick smoke test..." +echo "" + +# Check cluster exists +if ! kind get clusters 2>/dev/null | grep -q "^snoop-test$"; then + echo "Error: Run ./setup.sh first" + exit 1 +fi + +# Build validator +echo "Building validator..." +(cd "$SCRIPT_DIR/validate" && go build -o "$SCRIPT_DIR/results/validate" .) + +# Get image tag +IMAGE_TAG=$(cat "$SCRIPT_DIR/.image-tag") +echo "Using image: $IMAGE_TAG" +echo "" + +# Update manifest with image +TEMP_MANIFEST="$SCRIPT_DIR/results/smoke-test-manifest.yaml" +sed "s|image: snoop:test-latest|image: $IMAGE_TAG|g" \ + "$SCRIPT_DIR/manifests/alpine-test.yaml" > "$TEMP_MANIFEST" + +echo "Deploying alpine test..." +kubectl apply -f "$TEMP_MANIFEST" + +echo "Waiting for pod to be ready..." +if ! kubectl wait --for=condition=Ready pod -l app=alpine-test -n snoop-test --timeout=90s; then + echo "ERROR: Pod not ready" + kubectl get pods -n snoop-test + kubectl describe pod -l app=alpine-test -n snoop-test + exit 1 +fi + +POD_NAME=$(kubectl get pod -l app=alpine-test -n snoop-test -o jsonpath='{.items[0].metadata.name}') +echo "Pod ready: $POD_NAME" +echo "" + +echo "Checking snoop logs..." +kubectl logs -n snoop-test "$POD_NAME" -c snoop --tail=20 +echo "" + +echo "Waiting 35 seconds for report..." +sleep 35 + +echo "Retrieving report..." +if ! kubectl cp "snoop-test/$POD_NAME:/data/snoop-report.json" "$SCRIPT_DIR/results/smoke-report.json" -c app; then + echo "ERROR: Could not retrieve report" + echo "" + echo "Snoop logs:" + kubectl logs -n snoop-test "$POD_NAME" -c snoop + echo "" + echo "Checking /data directory:" + kubectl exec -n snoop-test "$POD_NAME" -c app -- ls -la /data/ + exit 1 +fi + +echo "Report retrieved!" +echo "" + +echo "Validating report..." +if ! "$SCRIPT_DIR/results/validate" "$SCRIPT_DIR/results/smoke-report.json"; then + echo "ERROR: Validation failed" + exit 1 +fi + +echo "" +echo "Cleaning up..." +kubectl delete -f "$TEMP_MANIFEST" --wait=false + +echo "" +echo "✅ Smoke test passed!" diff --git a/test/kind/teardown.sh b/test/kind/teardown.sh new file mode 100755 index 0000000..7b07727 --- /dev/null +++ b/test/kind/teardown.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Teardown script for KinD testing environment +set -euo pipefail + +CLUSTER_NAME="${CLUSTER_NAME:-snoop-test}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "================================================" +echo "Tearing down KinD test environment" +echo "================================================" +echo "" + +# Check if cluster exists +if ! kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + echo "Cluster $CLUSTER_NAME does not exist. Nothing to do." + exit 0 +fi + +# Delete cluster +echo "Deleting KinD cluster: $CLUSTER_NAME" +kind delete cluster --name "$CLUSTER_NAME" + +# Clean up temp files +if [ -f "$SCRIPT_DIR/.image-tag" ]; then + rm "$SCRIPT_DIR/.image-tag" +fi + +echo "" +echo "Teardown complete!" diff --git a/test/kind/validate/go.mod b/test/kind/validate/go.mod new file mode 100644 index 0000000..5649aa9 --- /dev/null +++ b/test/kind/validate/go.mod @@ -0,0 +1,5 @@ +module github.com/imjasonh/snoop/test/kind/validate + +go 1.22 + +// No external dependencies - uses only stdlib diff --git a/test/kind/validate/main.go b/test/kind/validate/main.go new file mode 100644 index 0000000..171f85b --- /dev/null +++ b/test/kind/validate/main.go @@ -0,0 +1,190 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// Report matches the JSON structure from snoop +type Report struct { + ContainerID string `json:"container_id"` + ImageRef string `json:"image_ref"` + ImageDigest string `json:"image_digest"` + PodName string `json:"pod_name"` + Namespace string `json:"namespace"` + Labels map[string]string `json:"labels"` + StartedAt time.Time `json:"started_at"` + LastUpdatedAt time.Time `json:"last_updated_at"` + Files []string `json:"files"` + TotalEvents uint64 `json:"total_events"` + DroppedEvents uint64 `json:"dropped_events"` +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: %s \n", filepath.Base(os.Args[0])) + os.Exit(1) + } + + reportPath := os.Args[1] + + fmt.Printf("Validating report: %s\n", reportPath) + + if err := validateReport(reportPath); err != nil { + fmt.Fprintf(os.Stderr, "❌ Validation failed: %v\n", err) + os.Exit(1) + } + + fmt.Println("\n✅ Report validation passed") +} + +func validateReport(path string) error { + // Read and parse JSON + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading report: %w", err) + } + + var report Report + if err := json.Unmarshal(data, &report); err != nil { + return fmt.Errorf("parsing JSON: %w", err) + } + + fmt.Println("\n=== Report Structure ===") + + // Validate required fields + if report.PodName == "" { + return fmt.Errorf("pod_name is empty") + } + fmt.Printf("✓ Pod Name: %s\n", report.PodName) + + if report.Namespace == "" { + return fmt.Errorf("namespace is empty") + } + fmt.Printf("✓ Namespace: %s\n", report.Namespace) + + if report.StartedAt.IsZero() { + return fmt.Errorf("started_at is zero") + } + fmt.Printf("✓ Started At: %s\n", report.StartedAt.Format(time.RFC3339)) + + if report.LastUpdatedAt.IsZero() { + return fmt.Errorf("last_updated_at is zero") + } + fmt.Printf("✓ Last Updated: %s\n", report.LastUpdatedAt.Format(time.RFC3339)) + + // Validate files array + if len(report.Files) == 0 { + return fmt.Errorf("files array is empty") + } + fmt.Printf("✓ Files Captured: %d\n", len(report.Files)) + + fmt.Println("\n=== File Validation ===") + + // Check for excluded paths + excludedPrefixes := []string{"/proc/", "/sys/", "/dev/"} + excludedCount := 0 + for _, file := range report.Files { + for _, prefix := range excludedPrefixes { + if strings.HasPrefix(file, prefix) { + excludedCount++ + if excludedCount <= 3 { + fmt.Printf("⚠ Excluded file found: %s\n", file) + } + } + } + } + + if excludedCount > 0 { + return fmt.Errorf("found %d excluded files (should be 0)", excludedCount) + } + fmt.Println("✓ No excluded files (/proc, /sys, /dev)") + + // Check paths are absolute + relativeCount := 0 + for _, file := range report.Files { + if !strings.HasPrefix(file, "/") { + relativeCount++ + if relativeCount <= 3 { + fmt.Printf("⚠ Relative path found: %s\n", file) + } + } + } + + if relativeCount > 0 { + return fmt.Errorf("found %d relative paths (should be 0)", relativeCount) + } + fmt.Println("✓ All paths are absolute") + + // Check for path components that should be normalized + unnormalizedCount := 0 + for _, file := range report.Files { + if strings.Contains(file, "/./") || strings.Contains(file, "/../") { + unnormalizedCount++ + if unnormalizedCount <= 3 { + fmt.Printf("⚠ Non-normalized path: %s\n", file) + } + } + } + + if unnormalizedCount > 0 { + return fmt.Errorf("found %d non-normalized paths", unnormalizedCount) + } + fmt.Println("✓ All paths are normalized") + + // Check for duplicates + seen := make(map[string]bool) + duplicates := 0 + for _, file := range report.Files { + if seen[file] { + duplicates++ + if duplicates <= 3 { + fmt.Printf("⚠ Duplicate file: %s\n", file) + } + } + seen[file] = true + } + + if duplicates > 0 { + return fmt.Errorf("found %d duplicate files", duplicates) + } + fmt.Println("✓ No duplicate files") + + // Validate timestamps + if report.LastUpdatedAt.Before(report.StartedAt) { + return fmt.Errorf("last_updated_at (%s) is before started_at (%s)", + report.LastUpdatedAt.Format(time.RFC3339), + report.StartedAt.Format(time.RFC3339)) + } + fmt.Println("✓ Timestamps are consistent") + + // Statistics + fmt.Println("\n=== Statistics ===") + fmt.Printf("Total Events: %d\n", report.TotalEvents) + fmt.Printf("Dropped Events: %d\n", report.DroppedEvents) + fmt.Printf("Unique Files: %d\n", len(report.Files)) + + if report.DroppedEvents > 0 { + dropRate := float64(report.DroppedEvents) / float64(report.TotalEvents) * 100 + fmt.Printf("Drop Rate: %.2f%%\n", dropRate) + if dropRate > 5.0 { + return fmt.Errorf("drop rate too high: %.2f%% (should be < 5%%)", dropRate) + } + } + + // Show sample files + fmt.Println("\n=== Sample Files (first 10) ===") + for i, file := range report.Files { + if i >= 10 { + fmt.Printf("... and %d more\n", len(report.Files)-10) + break + } + fmt.Printf(" %s\n", file) + } + + return nil +}