Merge pull request #16 from imjasonh/copilot/add-pr-risk-scorer

Add PR risk scorer with LLM-based assessment and evaluation framework
2026-07-06 23:12:22 +00:00 · 2026-02-18 21:56:31 -05:00 · 2026-02-18 21:56:31 -05:00 · 42fd405534
commit 42fd405534
parent a0b010d627 91d03c0c0b
15 changed files with 1496 additions and 75 deletions
--- a/cf-worker-container/go.mod
+++ b/cf-worker-container/go.mod
@ -3,27 +3,6 @@ module server
 go 1.24.3

 require (
-	github.com/aws/aws-sdk-go-v2 v1.39.5
-	github.com/aws/aws-sdk-go-v2/config v1.31.16
-	github.com/aws/aws-sdk-go-v2/credentials v1.18.20
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1
 	github.com/chainguard-dev/clog v1.7.0
 	github.com/sethvargo/go-envconfig v1.3.0
 )
-
-require (
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
-	github.com/aws/smithy-go v1.23.1 // indirect
-)
--- a/cf-worker-container/go.sum
+++ b/cf-worker-container/go.sum
@ -1,39 +1,3 @@
-github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w=
-github.com/aws/aws-sdk-go-v2 v1.39.5/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
-github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc=
-github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
-github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
-github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
-github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/chainguard-dev/clog v1.7.0 h1:guPznsK8vLHvzz1QJe2yU6MFeYaiSOFOQBYw4OXu+g8=
 github.com/chainguard-dev/clog v1.7.0/go.mod h1:4+WFhRMsGH79etYXY3plYdp+tCz/KCkU8fAr0HoaPvs=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
--- a/driftlessaf/cmd/pr-reconciler/main.go
+++ b/driftlessaf/cmd/pr-reconciler/main.go
@ -53,12 +53,12 @@ func (d ReconcilerDetails) Markdown() string {
 	// Auto-labeler section
 	if d.FilesAnalyzed > 0 {
 		sb.WriteString("## Auto-Labeler\n\n")
-		sb.WriteString(fmt.Sprintf("Analyzed **%d files** with **%d lines** changed.\n\n", d.FilesAnalyzed, d.TotalChanges))
+		fmt.Fprintf(&sb, "Analyzed **%d files** with **%d lines** changed.\n\n", d.FilesAnalyzed, d.TotalChanges)

 		if len(d.LabelsApplied) > 0 {
 			sb.WriteString("### Labels Applied\n\n")
 			for _, label := range d.LabelsApplied {
-				sb.WriteString(fmt.Sprintf("- `%s`\n", label))
+				fmt.Fprintf(&sb, "- `%s`\n", label)
 			}
 			sb.WriteString("\n")
 		} else {
@ -73,13 +73,13 @@ func (d ReconcilerDetails) Markdown() string {
 		if d.CIFixPending {
 			// CI is still running, waiting to evaluate
 			sb.WriteString("⏳ **Waiting for CI to complete**\n\n")
-			sb.WriteString(fmt.Sprintf("%s\n\n", d.CIFixReasoning))
+			fmt.Fprintf(&sb, "%s\n\n", d.CIFixReasoning)
 		} else if d.CIFixNeedsHuman {
 			// Agent determined human intervention is needed
 			sb.WriteString("⚠️ **Human intervention requested**\n\n")
-			sb.WriteString(fmt.Sprintf("Reason: %s\n\n", d.CIFixReasoning))
+			fmt.Fprintf(&sb, "Reason: %s\n\n", d.CIFixReasoning)
 			if d.CIFixTurns > 0 {
-				sb.WriteString(fmt.Sprintf("Turns attempted: %d\n\n", d.CIFixTurns))
+				fmt.Fprintf(&sb, "Turns attempted: %d\n\n", d.CIFixTurns)
 			}
 		} else if d.CIFixSuccess {
 			// CI is passing
@ -89,14 +89,14 @@ func (d ReconcilerDetails) Markdown() string {
 				if len(d.CIFixFiles) > 0 {
 					sb.WriteString("### Files Modified\n\n")
 					for _, f := range d.CIFixFiles {
-						sb.WriteString(fmt.Sprintf("- `%s`\n", f))
+						fmt.Fprintf(&sb, "- `%s`\n", f)
 					}
 					sb.WriteString("\n")
 				}
 				if d.CIFixCommit != "" {
-					sb.WriteString(fmt.Sprintf("Commit: `%s`\n\n", d.CIFixCommit))
+					fmt.Fprintf(&sb, "Commit: `%s`\n\n", d.CIFixCommit)
 				}
-				sb.WriteString(fmt.Sprintf("Turns used: %d\n\n", d.CIFixTurns))
+				fmt.Fprintf(&sb, "Turns used: %d\n\n", d.CIFixTurns)
 			} else {
 				// CI was already passing, no fixes needed
 				sb.WriteString("✅ **CI is passing**\n\n")
@ -105,9 +105,9 @@ func (d ReconcilerDetails) Markdown() string {
 		} else if d.CIFixAttempted {
 			// We attempted fixes but CI still failing
 			sb.WriteString("❌ **Could not fix CI failure**\n\n")
-			sb.WriteString(fmt.Sprintf("Reason: %s\n\n", d.CIFixReasoning))
+			fmt.Fprintf(&sb, "Reason: %s\n\n", d.CIFixReasoning)
 			if d.CIFixTurns > 0 {
-				sb.WriteString(fmt.Sprintf("Turns attempted: %d\n\n", d.CIFixTurns))
+				fmt.Fprintf(&sb, "Turns attempted: %d\n\n", d.CIFixTurns)
 			}
 		}
 	}
@ -734,8 +734,8 @@ func (r *PRReconciler) getCheckLogs(ctx context.Context, gh *github.Client, owne

 	var sb strings.Builder
 	for _, a := range annotations {
-		sb.WriteString(fmt.Sprintf("%s:%d: %s - %s\n",
-			a.GetPath(), a.GetStartLine(), a.GetAnnotationLevel(), a.GetMessage()))
+		fmt.Fprintf(&sb, "%s:%d: %s - %s\n",
+			a.GetPath(), a.GetStartLine(), a.GetAnnotationLevel(), a.GetMessage())
 	}

 	// Also try to get the check run output
--- a/driftlessaf/cmd/pr-risk-scorer/main.go
+++ b/driftlessaf/cmd/pr-risk-scorer/main.go
@ -0,0 +1,418 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+
+	"chainguard.dev/driftlessaf/agents/executor/claudeexecutor"
+	"chainguard.dev/driftlessaf/reconcilers/githubreconciler"
+	"chainguard.dev/driftlessaf/reconcilers/githubreconciler/statusmanager"
+	"chainguard.dev/driftlessaf/workqueue"
+	"github.com/bradleyfalzon/ghinstallation/v2"
+	"github.com/chainguard-dev/clog"
+	_ "github.com/chainguard-dev/clog/gcp/init"
+	"github.com/chainguard-dev/terraform-infra-common/pkg/httpmetrics"
+	"github.com/google/go-github/v75/github"
+	"github.com/sethvargo/go-envconfig"
+	"google.golang.org/grpc"
+
+	"github.com/imjasonh/terraform-playground/driftlessaf/internal/risk"
+)
+
+const reconcilerIdentity = "pr-risk-scorer"
+
+// ReconcilerDetails contains the state tracked by the status manager.
+type ReconcilerDetails struct {
+	RiskLevel   string   `json:"riskLevel,omitempty"`
+	Reasoning   string   `json:"reasoning,omitempty"`
+	RiskyFiles  []string `json:"riskyFiles,omitempty"`
+	RiskFactors []string `json:"riskFactors,omitempty"`
+	Confidence  float64  `json:"confidence,omitempty"`
+}
+
+// Markdown renders the details for display in the GitHub Check Run.
+func (d ReconcilerDetails) Markdown() string {
+	var sb strings.Builder
+
+	sb.WriteString("## Risk Assessment\n\n")
+
+	// Display risk level with emoji
+	var riskEmoji string
+	switch d.RiskLevel {
+	case "high":
+		riskEmoji = "🚨"
+	case "medium":
+		riskEmoji = "⚠️"
+	default:
+		riskEmoji = "✅"
+	}
+
+	fmt.Fprintf(&sb, "%s **Risk Level: %s**\n\n", riskEmoji, strings.ToUpper(d.RiskLevel))
+
+	if d.Confidence > 0 {
+		fmt.Fprintf(&sb, "*Confidence: %.0f%%*\n\n", d.Confidence*100)
+	}
+
+	if d.Reasoning != "" {
+		sb.WriteString("### Assessment\n\n")
+		sb.WriteString(d.Reasoning + "\n\n")
+	}
+
+	if len(d.RiskFactors) > 0 {
+		sb.WriteString("### Risk Factors\n\n")
+		for _, factor := range d.RiskFactors {
+			fmt.Fprintf(&sb, "- %s\n", factor)
+		}
+		sb.WriteString("\n")
+	}
+
+	if len(d.RiskyFiles) > 0 {
+		sb.WriteString("### High-Risk Files\n\n")
+		for _, file := range d.RiskyFiles {
+			fmt.Fprintf(&sb, "- `%s`\n", file)
+		}
+		sb.WriteString("\n")
+	}
+
+	return sb.String()
+}
+
+type envConfig struct {
+	Port             int    `env:"PORT,default=8080"`
+	GithubAppID      int64  `env:"GITHUB_APP_ID,required"`
+	GithubPrivateKey string `env:"GITHUB_PRIVATE_KEY,required"`
+	GCPProjectID     string `env:"GCP_PROJECT_ID"`
+	GCPRegion        string `env:"GCP_REGION,default=us-east5"`
+	ClaudeModel      string `env:"CLAUDE_MODEL,default=claude-sonnet-4@20250514"`
+}
+
+func main() {
+	ctx := context.Background()
+	log := clog.FromContext(ctx)
+
+	var cfg envConfig
+	envconfig.MustProcess(ctx, &cfg)
+
+	privateKey := []byte(cfg.GithubPrivateKey)
+
+	// Create an App-level transport for looking up installations
+	appTransport, err := ghinstallation.NewAppsTransport(
+		httpmetrics.WrapTransport(http.DefaultTransport),
+		cfg.GithubAppID,
+		privateKey,
+	)
+	if err != nil {
+		log.Fatalf("failed to create GitHub app transport: %v", err)
+	}
+	appClient := github.NewClient(&http.Client{Transport: appTransport})
+
+	// Create status manager for tracking reconciliation state
+	statusMgr, err := statusmanager.NewStatusManager[ReconcilerDetails](ctx, reconcilerIdentity)
+	if err != nil {
+		log.Fatalf("failed to create status manager: %v", err)
+	}
+
+	// Create the risk assessment agent
+	agent, err := risk.NewAgent(ctx, risk.AgentConfig{
+		GCPProjectID: cfg.GCPProjectID,
+		GCPRegion:    cfg.GCPRegion,
+		Model:        cfg.ClaudeModel,
+	})
+	if err != nil {
+		log.Fatalf("failed to create risk assessment agent: %v", err)
+	}
+
+	reconciler := &PRReconciler{
+		appID:      cfg.GithubAppID,
+		privateKey: privateKey,
+		appClient:  appClient,
+		clients:    make(map[int64]*github.Client),
+		transports: make(map[int64]*ghinstallation.Transport),
+		statusMgr:  statusMgr,
+		agent:      agent,
+	}
+
+	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.Port))
+	if err != nil {
+		log.Fatalf("failed to listen: %v", err)
+	}
+
+	grpcServer := grpc.NewServer()
+	workqueue.RegisterWorkqueueServiceServer(grpcServer, reconciler)
+
+	// ServeMetrics must run in a goroutine - it blocks!
+	go httpmetrics.ServeMetrics()
+	defer httpmetrics.SetupTracer(ctx)()
+
+	log.Infof("starting gRPC server on port %d with model %s", cfg.Port, cfg.ClaudeModel)
+	if err := grpcServer.Serve(lis); err != nil {
+		log.Fatalf("failed to serve: %v", err)
+	}
+}
+
+type PRReconciler struct {
+	workqueue.UnimplementedWorkqueueServiceServer
+
+	appID      int64
+	privateKey []byte
+	appClient  *github.Client
+	statusMgr  *statusmanager.StatusManager[ReconcilerDetails]
+	agent      claudeexecutor.Interface[*risk.PRContext, *risk.RiskAssessment]
+
+	mu         sync.RWMutex
+	clients    map[int64]*github.Client
+	transports map[int64]*ghinstallation.Transport
+}
+
+// getClientForRepo returns a GitHub client authenticated for the installation
+// that has access to the given owner/repo.
+func (r *PRReconciler) getClientForRepo(ctx context.Context, owner, repo string) (*github.Client, error) {
+	log := clog.FromContext(ctx)
+
+	// Look up which installation has access to this repo
+	installation, _, err := r.appClient.Apps.FindRepositoryInstallation(ctx, owner, repo)
+	if err != nil {
+		return nil, fmt.Errorf("finding installation for %s/%s: %w", owner, repo, err)
+	}
+
+	installationID := installation.GetID()
+
+	// Check cache first
+	r.mu.RLock()
+	client, ok := r.clients[installationID]
+	r.mu.RUnlock()
+	if ok {
+		return client, nil
+	}
+
+	// Create new client for this installation
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// Double-check after acquiring write lock
+	if client, ok := r.clients[installationID]; ok {
+		return client, nil
+	}
+
+	itr, err := ghinstallation.New(
+		httpmetrics.WrapTransport(http.DefaultTransport),
+		r.appID,
+		installationID,
+		r.privateKey,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("creating installation transport: %w", err)
+	}
+
+	client = github.NewClient(&http.Client{Transport: itr})
+	r.clients[installationID] = client
+	r.transports[installationID] = itr
+
+	log.Infof("created new client for installation %d", installationID)
+	return client, nil
+}
+
+// prURLRegex matches GitHub PR URLs like https://github.com/owner/repo/pull/123
+var prURLRegex = regexp.MustCompile(`^https://github\.com/([^/]+)/([^/]+)/pull/(\d+)$`)
+
+func parsePRURL(url string) (owner, repo string, number int, err error) {
+	matches := prURLRegex.FindStringSubmatch(url)
+	if matches == nil {
+		return "", "", 0, fmt.Errorf("invalid PR URL: %s", url)
+	}
+	number, err = strconv.Atoi(matches[3])
+	if err != nil {
+		return "", "", 0, fmt.Errorf("invalid PR number in URL %s: %w", url, err)
+	}
+	return matches[1], matches[2], number, nil
+}
+
+func (r *PRReconciler) Process(ctx context.Context, req *workqueue.ProcessRequest) (*workqueue.ProcessResponse, error) {
+	log := clog.FromContext(ctx).With("key", req.Key)
+
+	owner, repo, number, err := parsePRURL(req.Key)
+	if err != nil {
+		log.Warnf("skipping unknown key format: %v", err)
+		return &workqueue.ProcessResponse{}, nil
+	}
+
+	log = log.With("owner", owner, "repo", repo, "number", number)
+	ctx = clog.WithLogger(ctx, log)
+
+	gh, err := r.getClientForRepo(ctx, owner, repo)
+	if err != nil {
+		log.Errorf("failed to get GitHub client: %v", err)
+		return nil, err
+	}
+
+	pr, _, err := gh.PullRequests.Get(ctx, owner, repo, number)
+	if err != nil {
+		log.Errorf("failed to fetch PR: %v", err)
+		return nil, err
+	}
+
+	// Add SHA to logger context for Cloud Logging filtering
+	sha := pr.GetHead().GetSHA()
+	log = log.With("sha", sha)
+	ctx = clog.WithLogger(ctx, log)
+
+	// Skip closed PRs
+	if pr.GetState() != "open" {
+		log.Infof("skipping closed PR")
+		return &workqueue.ProcessResponse{}, nil
+	}
+
+	log.Infof("processing PR: title=%q state=%s author=%s",
+		pr.GetTitle(), pr.GetState(), pr.GetUser().GetLogin())
+
+	// Create status manager session for this PR
+	resource := &githubreconciler.Resource{
+		Type:  githubreconciler.ResourceTypePullRequest,
+		Owner: owner,
+		Repo:  repo,
+		URL:   req.Key,
+		Ref:   pr.GetBase().GetRef(),
+	}
+	session := r.statusMgr.NewSession(gh, resource, sha)
+
+	// Check existing state for this SHA
+	existingStatus, err := session.ObservedState(ctx)
+	if err != nil {
+		log.Warnf("failed to get observed state: %v", err)
+		// Continue anyway - we'll create a new check run
+	}
+
+	// Skip if we've already processed this SHA
+	if existingStatus != nil && existingStatus.ObservedGeneration == sha {
+		log.Infof("already processed SHA %s, skipping", sha[:8])
+		return &workqueue.ProcessResponse{}, nil
+	}
+
+	// Set in-progress status
+	if err := session.SetActualState(ctx, "Assessing risk...", &statusmanager.Status[ReconcilerDetails]{
+		Status: "in_progress",
+	}); err != nil {
+		log.Warnf("failed to set in-progress status: %v", err)
+	}
+
+	// Get changed files
+	files, _, err := gh.PullRequests.ListFiles(ctx, owner, repo, number, &github.ListOptions{PerPage: 100})
+	if err != nil {
+		log.Errorf("failed to list files: %v", err)
+		return nil, fmt.Errorf("listing files: %w", err)
+	}
+
+	// Build file list and calculate totals
+	filesChanged := make([]string, 0, len(files))
+	var additions, deletions int
+	for _, f := range files {
+		filesChanged = append(filesChanged, f.GetFilename())
+		additions += f.GetAdditions()
+		deletions += f.GetDeletions()
+	}
+
+	// Build PR context for the agent
+	prContext := &risk.PRContext{
+		Owner:        owner,
+		Repo:         repo,
+		PRNumber:     number,
+		Title:        pr.GetTitle(),
+		Description:  pr.GetBody(),
+		Author:       pr.GetUser().GetLogin(),
+		FilesChanged: filesChanged,
+		Additions:    additions,
+		Deletions:    deletions,
+	}
+
+	// Execute the risk assessment agent
+	log.Infof("running risk assessment agent on %d files (%d additions, %d deletions)",
+		len(filesChanged), additions, deletions)
+
+	assessment, err := r.agent.Execute(ctx, prContext, nil)
+	if err != nil {
+		log.Errorf("risk assessment failed: %v", err)
+		return nil, fmt.Errorf("risk assessment: %w", err)
+	}
+
+	log.Infof("risk assessment complete: level=%s confidence=%.2f",
+		assessment.RiskLevel, assessment.Confidence)
+
+	details := ReconcilerDetails{
+		RiskLevel:   assessment.RiskLevel,
+		Reasoning:   assessment.Reasoning,
+		RiskyFiles:  assessment.RiskyFiles,
+		RiskFactors: assessment.RiskFactors,
+		Confidence:  assessment.Confidence,
+	}
+
+	// Apply risk label
+	riskLabel := fmt.Sprintf("risk/%s", assessment.RiskLevel)
+	if !hasLabel(pr.Labels, riskLabel) {
+		log.Infof("adding risk label: %s", riskLabel)
+		_, _, err = gh.Issues.AddLabelsToIssue(ctx, owner, repo, number, []string{riskLabel})
+		if err != nil {
+			log.Warnf("failed to add label: %v", err)
+		}
+	}
+
+	// Remove old risk labels
+	for _, label := range pr.Labels {
+		name := label.GetName()
+		if strings.HasPrefix(name, "risk/") && name != riskLabel {
+			log.Infof("removing old risk label: %s", name)
+			_, err = gh.Issues.RemoveLabelForIssue(ctx, owner, repo, number, name)
+			if err != nil {
+				log.Warnf("failed to remove label %s: %v", name, err)
+			}
+		}
+	}
+
+	// Determine conclusion and status based on risk level
+	status := "completed"
+	var conclusion, summary string
+
+	switch assessment.RiskLevel {
+	case "high":
+		conclusion = "failure"
+		summary = "⚠️ High risk changes detected - careful review required"
+	case "medium":
+		conclusion = "neutral"
+		summary = "Medium risk changes - review recommended"
+	default:
+		conclusion = "success"
+		summary = "Low risk changes"
+	}
+
+	// Set final status
+	if err := session.SetActualState(ctx, summary, &statusmanager.Status[ReconcilerDetails]{
+		ObservedGeneration: sha,
+		Status:             status,
+		Conclusion:         conclusion,
+		Details:            details,
+	}); err != nil {
+		log.Warnf("failed to set final status: %v", err)
+	}
+
+	log.Infof("risk assessment complete: level=%s conclusion=%s", assessment.RiskLevel, conclusion)
+
+	return &workqueue.ProcessResponse{}, nil
+}
+
+func hasLabel(labels []*github.Label, name string) bool {
+	for _, l := range labels {
+		if l.GetName() == name {
+			return true
+		}
+	}
+	return false
+}
+
+func (r *PRReconciler) GetKeyState(ctx context.Context, req *workqueue.GetKeyStateRequest) (*workqueue.KeyState, error) {
+	return &workqueue.KeyState{}, nil
+}
--- a/driftlessaf/cmd/pr-risk-scorer/main_test.go
+++ b/driftlessaf/cmd/pr-risk-scorer/main_test.go
@ -0,0 +1,118 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestParsePRURL(t *testing.T) {
+	for _, tt := range []struct {
+		desc       string
+		url        string
+		wantOwner  string
+		wantRepo   string
+		wantNumber int
+		wantErr    bool
+	}{{
+		desc:       "valid PR URL",
+		url:        "https://github.com/owner/repo/pull/123",
+		wantOwner:  "owner",
+		wantRepo:   "repo",
+		wantNumber: 123,
+		wantErr:    false,
+	}, {
+		desc:       "another valid PR URL",
+		url:        "https://github.com/imjasonh/terraform-playground/pull/456",
+		wantOwner:  "imjasonh",
+		wantRepo:   "terraform-playground",
+		wantNumber: 456,
+		wantErr:    false,
+	}, {
+		desc:    "invalid URL - not a PR",
+		url:     "https://github.com/owner/repo/issues/123",
+		wantErr: true,
+	}} {
+		t.Run(tt.desc, func(t *testing.T) {
+			owner, repo, number, err := parsePRURL(tt.url)
+
+			if tt.wantErr {
+				if err == nil {
+					t.Errorf("parsePRURL(%q) expected error, got nil", tt.url)
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("parsePRURL(%q) unexpected error: %v", tt.url, err)
+			}
+
+			if owner != tt.wantOwner {
+				t.Errorf("owner = %q, want %q", owner, tt.wantOwner)
+			}
+			if repo != tt.wantRepo {
+				t.Errorf("repo = %q, want %q", repo, tt.wantRepo)
+			}
+			if number != tt.wantNumber {
+				t.Errorf("number = %d, want %d", number, tt.wantNumber)
+			}
+		})
+	}
+}
+
+func TestReconcilerDetails_Markdown(t *testing.T) {
+	tests := []struct {
+		desc    string
+		details ReconcilerDetails
+		want    []string // Substrings that should be present
+	}{{
+		desc: "high risk",
+		details: ReconcilerDetails{
+			RiskLevel:   "high",
+			Reasoning:   "Infrastructure changes detected",
+			RiskyFiles:  []string{"main.tf", "Dockerfile"},
+			RiskFactors: []string{"Terraform changes", "Docker config"},
+			Confidence:  0.95,
+		},
+		want: []string{"Risk Assessment", "HIGH", "Infrastructure changes", "main.tf"},
+	}, {
+		desc: "medium risk",
+		details: ReconcilerDetails{
+			RiskLevel:  "medium",
+			Reasoning:  "API changes require review",
+			Confidence: 0.85,
+		},
+		want: []string{"Risk Assessment", "MEDIUM", "API changes"},
+	}, {
+		desc: "low risk",
+		details: ReconcilerDetails{
+			RiskLevel:  "low",
+			Reasoning:  "Documentation only",
+			Confidence: 0.99,
+		},
+		want: []string{"Risk Assessment", "LOW", "Documentation"},
+	}}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			md := tt.details.Markdown()
+
+			for _, substr := range tt.want {
+				if !contains(md, substr) {
+					t.Errorf("Markdown() missing expected substring %q\nGot:\n%s", substr, md)
+				}
+			}
+		})
+	}
+}
+
+func contains(s, substr string) bool {
+	return indexOf(s, substr) >= 0
+}
+
+func indexOf(s, substr string) int {
+	for i := 0; i <= len(s)-len(substr); i++ {
+		if s[i:i+len(substr)] == substr {
+			return i
+		}
+	}
+	return -1
+}
--- a/driftlessaf/internal/cifixer/eval_test.go
+++ b/driftlessaf/internal/cifixer/eval_test.go
@ -212,15 +212,15 @@ func normalizeWhitespaceForEval(s string) string {
 func formatAgentResult(fs *MockFileSystem, result *CIFixResult) string {
 	var sb strings.Builder
 	sb.WriteString("Agent Result:\n")
-	sb.WriteString(fmt.Sprintf("Success: %v\n", result.Success))
-	sb.WriteString(fmt.Sprintf("Files Changed: %v\n", result.FilesChanged))
-	sb.WriteString(fmt.Sprintf("Commit Message: %s\n", result.CommitMessage))
-	sb.WriteString(fmt.Sprintf("Reasoning: %s\n", result.Reasoning))
+	fmt.Fprintf(&sb, "Success: %v\n", result.Success)
+	fmt.Fprintf(&sb, "Files Changed: %v\n", result.FilesChanged)
+	fmt.Fprintf(&sb, "Commit Message: %s\n", result.CommitMessage)
+	fmt.Fprintf(&sb, "Reasoning: %s\n", result.Reasoning)
 	sb.WriteString("\nFile Contents:\n")

 	for _, path := range result.FilesChanged {
 		content, _ := fs.ReadFile(path)
-		sb.WriteString(fmt.Sprintf("--- %s ---\n%s\n", path, content))
+		fmt.Fprintf(&sb, "--- %s ---\n%s\n", path, content)
 	}

 	return sb.String()
@ -231,7 +231,7 @@ func formatExpectedResult(expected map[string]string) string {
 	sb.WriteString("Expected Result:\n")

 	for path, content := range expected {
-		sb.WriteString(fmt.Sprintf("--- %s ---\n%s\n", path, content))
+		fmt.Fprintf(&sb, "--- %s ---\n%s\n", path, content)
 	}

 	return sb.String()
--- a/driftlessaf/internal/risk/agent.go
+++ b/driftlessaf/internal/risk/agent.go
@ -0,0 +1,170 @@
+package risk
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"chainguard.dev/driftlessaf/agents/executor/claudeexecutor"
+	"chainguard.dev/driftlessaf/agents/submitresult"
+	"cloud.google.com/go/compute/metadata"
+	"github.com/anthropics/anthropic-sdk-go"
+	"github.com/anthropics/anthropic-sdk-go/vertex"
+)
+
+// AgentConfig holds the configuration for creating a risk assessment agent.
+type AgentConfig struct {
+	// GCPProjectID is the Google Cloud project ID for Vertex AI.
+	GCPProjectID string
+
+	// GCPRegion is the Google Cloud region for Vertex AI (e.g., "us-central1").
+	GCPRegion string
+
+	// Model is the Claude model to use (e.g., "claude-sonnet-4@20250514").
+	Model string
+
+	// MaxTokens is the maximum number of tokens in the response.
+	MaxTokens int64
+
+	// Temperature controls randomness (0.0-1.0). Lower is more deterministic.
+	Temperature float64
+}
+
+// DefaultConfig returns a default agent configuration.
+func DefaultConfig() AgentConfig {
+	return AgentConfig{
+		Model:       "claude-sonnet-4@20250514", // Vertex AI format uses @ instead of -
+		MaxTokens:   8192,
+		Temperature: 0.1, // Low temperature for consistent risk assessment
+	}
+}
+
+// detectGCPProjectID attempts to detect the GCP project ID from:
+// 1. GCE metadata service (if running on GCP)
+// 2. gcloud CLI configuration
+func detectGCPProjectID() string {
+	// Try metadata service first (fast on GCP, quick timeout elsewhere)
+	if metadata.OnGCE() {
+		if project, err := metadata.ProjectIDWithContext(context.Background()); err == nil && project != "" {
+			return project
+		}
+	}
+
+	// Try gcloud CLI
+	cmd := exec.Command("gcloud", "config", "get-value", "project")
+	out, err := cmd.Output()
+	if err == nil {
+		project := strings.TrimSpace(string(out))
+		if project != "" && project != "(unset)" {
+			return project
+		}
+	}
+
+	return ""
+}
+
+// detectGCPRegion attempts to detect the GCP region from:
+// 1. GCE metadata service (if running on GCP)
+// 2. gcloud CLI configuration
+// Falls back to "us-east5" if nothing is found.
+func detectGCPRegion() string {
+	// Try metadata service first
+	if metadata.OnGCE() {
+		// Zone is like "us-central1-a", we want "us-central1"
+		if zone, err := metadata.ZoneWithContext(context.Background()); err == nil && zone != "" {
+			parts := strings.Split(zone, "-")
+			if len(parts) >= 2 {
+				return strings.Join(parts[:len(parts)-1], "-")
+			}
+		}
+	}
+
+	// Try gcloud CLI for compute region
+	cmd := exec.Command("gcloud", "config", "get-value", "compute/region")
+	out, err := cmd.Output()
+	if err == nil {
+		region := strings.TrimSpace(string(out))
+		if region != "" && region != "(unset)" {
+			return region
+		}
+	}
+
+	// Default to us-east5 which typically has Claude models available
+	return "us-east5"
+}
+
+// NewAgent creates a new risk assessment agent.
+// If GCPProjectID or GCPRegion are not provided, they will be detected from:
+// 1. GCE metadata service (if running on GCP)
+// 2. gcloud CLI configuration
+// 3. Default to "us-east5" for region (project ID has no default)
+func NewAgent(ctx context.Context, cfg AgentConfig) (claudeexecutor.Interface[*PRContext, *RiskAssessment], error) {
+	// Detect GCP project ID if not provided
+	if cfg.GCPProjectID == "" {
+		cfg.GCPProjectID = detectGCPProjectID()
+		if cfg.GCPProjectID == "" {
+			return nil, fmt.Errorf("GCPProjectID is required (could not detect from metadata or gcloud)")
+		}
+	}
+
+	// Detect GCP region if not provided
+	if cfg.GCPRegion == "" {
+		cfg.GCPRegion = detectGCPRegion()
+	}
+
+	// Create Anthropic client via Vertex AI
+	client := anthropic.NewClient(
+		vertex.WithGoogleAuth(ctx, cfg.GCPRegion, cfg.GCPProjectID),
+	)
+
+	// Apply defaults
+	if cfg.Model == "" {
+		cfg.Model = DefaultConfig().Model
+	}
+	if cfg.MaxTokens == 0 {
+		cfg.MaxTokens = DefaultConfig().MaxTokens
+	}
+	if cfg.Temperature == 0 {
+		cfg.Temperature = DefaultConfig().Temperature
+	}
+
+	return claudeexecutor.New[*PRContext, *RiskAssessment](
+		client,
+		UserPrompt,
+		claudeexecutor.WithModel[*PRContext, *RiskAssessment](cfg.Model),
+		claudeexecutor.WithMaxTokens[*PRContext, *RiskAssessment](cfg.MaxTokens),
+		claudeexecutor.WithTemperature[*PRContext, *RiskAssessment](cfg.Temperature),
+		claudeexecutor.WithSystemInstructions[*PRContext, *RiskAssessment](SystemInstructions),
+		claudeexecutor.WithSubmitResultProvider[*PRContext, *RiskAssessment](
+			submitresult.ClaudeToolForResponse[*RiskAssessment],
+		),
+	)
+}
+
+// NewAgentWithClient creates a new risk assessment agent with a provided Anthropic client.
+// This is useful for testing or when using a non-Vertex AI client.
+func NewAgentWithClient(client anthropic.Client, cfg AgentConfig) (claudeexecutor.Interface[*PRContext, *RiskAssessment], error) {
+	// Apply defaults
+	if cfg.Model == "" {
+		cfg.Model = DefaultConfig().Model
+	}
+	if cfg.MaxTokens == 0 {
+		cfg.MaxTokens = DefaultConfig().MaxTokens
+	}
+	if cfg.Temperature == 0 {
+		cfg.Temperature = DefaultConfig().Temperature
+	}
+
+	return claudeexecutor.New[*PRContext, *RiskAssessment](
+		client,
+		UserPrompt,
+		claudeexecutor.WithModel[*PRContext, *RiskAssessment](cfg.Model),
+		claudeexecutor.WithMaxTokens[*PRContext, *RiskAssessment](cfg.MaxTokens),
+		claudeexecutor.WithTemperature[*PRContext, *RiskAssessment](cfg.Temperature),
+		claudeexecutor.WithSystemInstructions[*PRContext, *RiskAssessment](SystemInstructions),
+		claudeexecutor.WithSubmitResultProvider[*PRContext, *RiskAssessment](
+			submitresult.ClaudeToolForResponse[*RiskAssessment],
+		),
+	)
+}
--- a/driftlessaf/internal/risk/eval_test.go
+++ b/driftlessaf/internal/risk/eval_test.go
@ -0,0 +1,252 @@
+package risk
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"chainguard.dev/driftlessaf/agents/executor/claudeexecutor"
+	"chainguard.dev/driftlessaf/agents/judge"
+	"github.com/chainguard-dev/clog"
+)
+
+var (
+	runEvals     = flag.Bool("run-evals", false, "Run LLM-based evaluations (requires credentials)")
+	gcpProjectID = flag.String("gcp-project", os.Getenv("GCP_PROJECT_ID"), "GCP project ID for Vertex AI")
+	gcpRegion    = flag.String("gcp-region", os.Getenv("GCP_REGION"), "GCP region for Vertex AI")
+	evalModel    = flag.String("eval-model", "claude-sonnet-4@20250514", "Model to use for evaluation (Vertex AI format)")
+	judgeModel   = flag.String("judge-model", "claude-sonnet-4@20250514", "Model to use for judging (Vertex AI format)")
+	verbose      = flag.Bool("verbose", false, "Print detailed output")
+)
+
+// TestRiskAssessmentEvals runs the risk assessment agent against test cases and evaluates results.
+// This test is skipped by default and only runs when -run-evals flag is set.
+func TestRiskAssessmentEvals(t *testing.T) {
+	if !*runEvals {
+		t.Skip("Skipping LLM-based evals (use -run-evals to enable)")
+	}
+
+	ctx := context.Background()
+	log := clog.FromContext(ctx)
+
+	// Create the risk assessment agent (will auto-detect project/region if not provided)
+	agent, err := NewAgent(ctx, AgentConfig{
+		GCPProjectID: *gcpProjectID,
+		GCPRegion:    *gcpRegion,
+		Model:        *evalModel,
+	})
+	if err != nil {
+		t.Fatalf("creating agent: %v", err)
+	}
+
+	// Resolve project/region for judge (use same detection logic)
+	projectID := *gcpProjectID
+	if projectID == "" {
+		projectID = detectGCPProjectID()
+		if projectID == "" {
+			t.Fatal("could not detect GCP project ID from metadata or gcloud")
+		}
+	}
+	region := *gcpRegion
+	if region == "" {
+		region = detectGCPRegion()
+	}
+
+	log.Infof("Using GCP project=%s region=%s", projectID, region)
+
+	// Create the judge for evaluating results
+	judgeInstance, err := judge.NewVertex(ctx, projectID, region, *judgeModel)
+	if err != nil {
+		t.Fatalf("creating judge: %v", err)
+	}
+
+	// Track results for summary
+	var results []evalResult
+
+	for _, tc := range EvalTestCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			result := runSingleEval(t, ctx, agent, judgeInstance, tc)
+			results = append(results, result)
+
+			if *verbose {
+				log.Infof("Test %s: score=%.2f, success=%v", tc.Name, result.Score, result.Success)
+				if result.Reasoning != "" {
+					log.Infof("  Reasoning: %s", result.Reasoning)
+				}
+			}
+		})
+	}
+
+	// Print summary
+	printEvalSummary(t, results)
+}
+
+type evalResult struct {
+	Name            string
+	Success         bool
+	Score           float64
+	Reasoning       string
+	LevelMatch      bool
+	ActualLevel     string
+	ExpectedLevel   string
+	AgentAssessment *RiskAssessment
+	Error           error
+}
+
+func runSingleEval(t *testing.T, ctx context.Context, agent claudeexecutor.Interface[*PRContext, *RiskAssessment], judgeInstance judge.Interface, tc EvalTestCase) evalResult {
+	result := evalResult{
+		Name:          tc.Name,
+		ExpectedLevel: string(tc.ExpectedLevel),
+	}
+
+	// Execute agent - no tools needed for risk assessment
+	assessment, err := agent.Execute(ctx, tc.PRContext, nil)
+	if err != nil {
+		result.Error = err
+		t.Errorf("agent execution failed: %v", err)
+		return result
+	}
+
+	result.AgentAssessment = assessment
+	result.ActualLevel = assessment.RiskLevel
+
+	// Check if risk level matches expected
+	result.LevelMatch = strings.EqualFold(assessment.RiskLevel, string(tc.ExpectedLevel))
+
+	if !result.LevelMatch && *verbose {
+		t.Logf("Level mismatch: got %s, want %s", assessment.RiskLevel, tc.ExpectedLevel)
+		t.Logf("Reasoning: %s", assessment.Reasoning)
+	}
+
+	// Use judge for quality evaluation
+	actualAnswer := formatAgentAssessment(assessment)
+
+	judgement, err := judgeInstance.Judge(ctx, &judge.Request{
+		Mode:         judge.StandaloneMode,
+		ActualAnswer: actualAnswer,
+		Criterion:    tc.Criterion,
+	})
+
+	if err != nil {
+		result.Error = fmt.Errorf("judge error: %w", err)
+		t.Errorf("judge error: %v", err)
+		return result
+	}
+
+	result.Score = judgement.Score
+	result.Reasoning = judgement.Reasoning
+	result.Success = judgement.Score >= 0.7
+
+	if !result.Success {
+		t.Errorf("judge score %.2f below threshold 0.7: %s", judgement.Score, judgement.Reasoning)
+	}
+
+	return result
+}
+
+func formatAgentAssessment(assessment *RiskAssessment) string {
+	var sb strings.Builder
+	fmt.Fprintf(&sb, "Risk Level: %s\n", assessment.RiskLevel)
+	fmt.Fprintf(&sb, "Confidence: %.2f\n", assessment.Confidence)
+	fmt.Fprintf(&sb, "\nReasoning:\n%s\n", assessment.Reasoning)
+
+	if len(assessment.RiskFactors) > 0 {
+		sb.WriteString("\nRisk Factors:\n")
+		for _, factor := range assessment.RiskFactors {
+			fmt.Fprintf(&sb, "- %s\n", factor)
+		}
+	}
+
+	if len(assessment.RiskyFiles) > 0 {
+		sb.WriteString("\nRisky Files:\n")
+		for _, file := range assessment.RiskyFiles {
+			fmt.Fprintf(&sb, "- %s\n", file)
+		}
+	}
+
+	return sb.String()
+}
+
+// formatExpectedAssessment formats an expected assessment for comparison.
+// Kept for potential future use in golden mode testing.
+var _ = formatExpectedAssessment
+
+func formatExpectedAssessment(tc EvalTestCase) string {
+	return fmt.Sprintf("Expected Risk Level: %s\n\nDescription: %s", tc.ExpectedLevel, tc.Description)
+}
+
+func printEvalSummary(t *testing.T, results []evalResult) {
+	t.Helper()
+
+	totalTests := len(results)
+	passed := 0
+	levelMatches := 0
+
+	var totalScore float64
+
+	t.Log("\n" + strings.Repeat("=", 80))
+	t.Log("EVALUATION SUMMARY")
+	t.Log(strings.Repeat("=", 80))
+
+	for _, r := range results {
+		if r.Success {
+			passed++
+		}
+		if r.LevelMatch {
+			levelMatches++
+		}
+		totalScore += r.Score
+
+		status := "❌ FAIL"
+		if r.Success {
+			status = "✅ PASS"
+		}
+
+		levelStatus := "❌"
+		if r.LevelMatch {
+			levelStatus = "✅"
+		}
+
+		t.Logf("%s | Score: %.2f | %s Level: %s (expected %s) | %s",
+			status, r.Score, levelStatus, r.ActualLevel, r.ExpectedLevel, r.Name)
+
+		if r.Error != nil {
+			t.Logf("  Error: %v", r.Error)
+		}
+	}
+
+	avgScore := totalScore / float64(totalTests)
+	passRate := float64(passed) / float64(totalTests) * 100
+	levelMatchRate := float64(levelMatches) / float64(totalTests) * 100
+
+	t.Log(strings.Repeat("=", 80))
+	t.Logf("Tests Passed: %d/%d (%.1f%%)", passed, totalTests, passRate)
+	t.Logf("Level Matches: %d/%d (%.1f%%)", levelMatches, totalTests, levelMatchRate)
+	t.Logf("Average Score: %.2f", avgScore)
+	t.Log(strings.Repeat("=", 80))
+
+	// Write results to file for tracking
+	if err := writeResultsToFile(results); err != nil {
+		t.Logf("Warning: failed to write results to file: %v", err)
+	}
+}
+
+func writeResultsToFile(results []evalResult) error {
+	f, err := os.Create("eval_results.log")
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if closeErr := f.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+	}()
+
+	encoder := json.NewEncoder(f)
+	encoder.SetIndent("", "  ")
+	return encoder.Encode(results)
+}
--- a/driftlessaf/internal/risk/prompts.go
+++ b/driftlessaf/internal/risk/prompts.go
@ -0,0 +1,87 @@
+package risk
+
+import (
+	"chainguard.dev/driftlessaf/agents/promptbuilder"
+)
+
+// SystemInstructions provides the system prompt for the PR risk assessment agent.
+var SystemInstructions = promptbuilder.MustNewPrompt(`ROLE: You are a PR risk assessment agent. Your job is to analyze pull requests and assess their risk level based on the changes made.
+
+WORKFLOW:
+1. Review the PR context including title, description, author, and files changed
+2. Analyze the file diffs to understand the nature of the changes
+3. Consider multiple risk dimensions:
+   - **Stability Risk**: Could these changes break production? Crash systems? Cause outages?
+   - **Cost Risk**: Could these changes significantly increase cloud costs or resource usage?
+   - **Security Risk**: Do these changes introduce security vulnerabilities or weaken security posture?
+   - **API Breaking Risk**: Do these changes break backwards compatibility or public APIs?
+   - **Data Risk**: Could these changes cause data loss, corruption, or integrity issues?
+   - **Complexity Risk**: Are the changes overly complex or hard to review/test?
+4. Assign an appropriate risk level (low, medium, or high)
+5. Provide clear reasoning and identify specific risk factors
+
+RISK LEVEL GUIDELINES:
+
+**HIGH RISK** - Assign when changes involve:
+- Infrastructure as Code (Terraform, CloudFormation) that manages production resources
+- Database schema migrations or data transformations
+- Authentication, authorization, or security-critical code
+- Core CI/CD pipeline or deployment configurations
+- Changes to payment processing or financial systems
+- Dependency upgrades with known breaking changes
+- Large refactors (>500 lines changed) in critical systems
+- Changes to disaster recovery or backup systems
+- Kubernetes/Docker configurations for production services
+- Direct manipulation of production databases
+
+**MEDIUM RISK** - Assign when changes involve:
+- Backend application code (Go, Python, Java, etc.)
+- Frontend application code (TypeScript, JavaScript, React, etc.)
+- API endpoints or service interfaces
+- Configuration files that affect application behavior
+- Non-critical dependency updates
+- Test infrastructure or CI configurations for non-production
+- Moderate-sized changes (200-500 lines)
+- Logging, monitoring, or alerting configurations
+
+**LOW RISK** - Assign when changes involve:
+- Documentation updates (README, docs, comments)
+- Code formatting or style fixes
+- Test-only changes that don't affect production code
+- Minor dependency updates (patch versions)
+- Small bug fixes with clear scope (<50 lines)
+- Development tooling or local setup scripts
+- Very small changes (<200 lines) that are well-tested
+
+CRITICAL RULES - YOU MUST FOLLOW THESE:
+- ALWAYS provide a clear, specific reasoning for your assessment
+- IDENTIFY specific files that contribute most to the risk
+- LIST concrete risk factors based on the actual changes
+- BE CONSERVATIVE: When in doubt, err on the side of higher risk
+- CONSIDER the cumulative risk of multiple small changes
+- LOOK for subtle issues like race conditions, error handling gaps, or edge cases
+- ASSESS both the direct impact and potential cascading failures
+- Your confidence score should reflect uncertainty in the assessment
+
+PROVIDING REASONING:
+- Start with the most significant risk factor
+- Reference specific files and changes
+- Explain potential failure modes or impacts
+- Be concise but thorough
+- Use concrete examples from the PR
+
+Remember: Your assessment helps teams prioritize reviews and testing. Be thoughtful and thorough.`)
+
+// UserPrompt is the user-facing prompt template for the risk assessment agent.
+var UserPrompt = promptbuilder.MustNewPrompt(`Please assess the risk level of this pull request.
+
+{{pr_context}}
+
+Analyze the changes and provide:
+1. Risk level (low, medium, or high)
+2. Detailed reasoning for your assessment
+3. Specific files that contribute most to the risk
+4. List of concrete risk factors
+5. Your confidence in this assessment (0.0-1.0)
+
+Be thorough and conservative in your assessment.`)
--- a/driftlessaf/internal/risk/risk_test.go
+++ b/driftlessaf/internal/risk/risk_test.go
@ -0,0 +1,49 @@
+package risk
+
+import (
+	"testing"
+)
+
+func TestDefaultConfig(t *testing.T) {
+	cfg := DefaultConfig()
+
+	if cfg.Model == "" {
+		t.Error("default model should not be empty")
+	}
+
+	if cfg.MaxTokens == 0 {
+		t.Error("default max tokens should not be zero")
+	}
+
+	if cfg.Temperature < 0 || cfg.Temperature > 1 {
+		t.Errorf("default temperature should be between 0 and 1, got %f", cfg.Temperature)
+	}
+}
+
+func TestRiskLevels(t *testing.T) {
+	levels := []Level{LevelLow, LevelMedium, LevelHigh}
+	expected := []string{"low", "medium", "high"}
+
+	for i, level := range levels {
+		if string(level) != expected[i] {
+			t.Errorf("level %d: got %s, want %s", i, level, expected[i])
+		}
+	}
+}
+
+func TestPRContextBind(t *testing.T) {
+	ctx := &PRContext{
+		Owner:        "test",
+		Repo:         "repo",
+		PRNumber:     1,
+		Title:        "Test PR",
+		FilesChanged: []string{"file1.go", "file2.go"},
+		Additions:    10,
+		Deletions:    5,
+	}
+
+	// Just verify it doesn't panic
+	if ctx.Owner != "test" {
+		t.Error("context not initialized properly")
+	}
+}
--- a/driftlessaf/internal/risk/testdata.go
+++ b/driftlessaf/internal/risk/testdata.go
@ -0,0 +1,251 @@
+package risk
+
+// EvalTestCase represents a test case for evaluating the risk assessment agent.
+type EvalTestCase struct {
+	Name          string
+	Description   string
+	PRContext     *PRContext
+	ExpectedLevel Level
+	Criterion     string
+}
+
+// EvalTestCases contains test cases for evaluating the risk assessment agent.
+var EvalTestCases = []EvalTestCase{
+	{
+		Name:        "terraform-infrastructure-change",
+		Description: "Terraform changes to production infrastructure should be high risk",
+		PRContext: &PRContext{
+			Owner:       "test",
+			Repo:        "test-repo",
+			PRNumber:    1,
+			Title:       "Update production database instance size",
+			Description: "Increasing RDS instance from t3.medium to t3.large for better performance",
+			Author:      "devops-team",
+			FilesChanged: []string{
+				"terraform/prod/main.tf",
+				"terraform/prod/variables.tf",
+			},
+			Additions: 15,
+			Deletions: 10,
+			FileDiffs: map[string]string{
+				"terraform/prod/main.tf": `@@ -10,7 +10,7 @@ resource "aws_db_instance" "main" {
+   identifier           = "production-db"
+   allocated_storage    = 100
+-  instance_class       = "db.t3.medium"
+  instance_class       = "db.t3.large"
+   engine               = "postgres"
+   engine_version       = "14.7"`,
+			},
+		},
+		ExpectedLevel: LevelHigh,
+		Criterion:     "Assessment should identify this as high risk because it modifies production infrastructure (database sizing) which could impact cost and stability",
+	},
+	{
+		Name:        "database-migration",
+		Description: "Database schema migrations should be high risk",
+		PRContext: &PRContext{
+			Owner:       "test",
+			Repo:        "app",
+			PRNumber:    42,
+			Title:       "Add user authentication table",
+			Description: "Creating new users table with authentication fields",
+			Author:      "backend-dev",
+			FilesChanged: []string{
+				"db/migrations/20250218_add_users_table.sql",
+				"models/user.go",
+			},
+			Additions: 85,
+			Deletions: 2,
+			FileDiffs: map[string]string{
+				"db/migrations/20250218_add_users_table.sql": `CREATE TABLE users (
+    id SERIAL PRIMARY KEY,
+    email VARCHAR(255) UNIQUE NOT NULL,
+    password_hash VARCHAR(255) NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX idx_users_email ON users(email);`,
+			},
+		},
+		ExpectedLevel: LevelHigh,
+		Criterion:     "Assessment should identify this as high risk because database migrations can affect data integrity and require careful review, especially for authentication-related tables",
+	},
+	{
+		Name:        "backend-api-change",
+		Description: "Backend API changes should be medium risk",
+		PRContext: &PRContext{
+			Owner:       "test",
+			Repo:        "api-server",
+			PRNumber:    15,
+			Title:       "Add pagination to user list endpoint",
+			Description: "Implementing cursor-based pagination for /api/users endpoint",
+			Author:      "api-team",
+			FilesChanged: []string{
+				"handlers/users.go",
+				"handlers/users_test.go",
+			},
+			Additions: 120,
+			Deletions: 35,
+			FileDiffs: map[string]string{
+				"handlers/users.go": `@@ -20,10 +20,18 @@ func (h *Handler) ListUsers(w http.ResponseWriter, r *http.Request) {
+    // Parse pagination parameters
+    limit := 50
+    if l := r.URL.Query().Get("limit"); l != "" {
+        limit, _ = strconv.Atoi(l)
+    }
+    cursor := r.URL.Query().Get("cursor")
+
+-    users, err := h.db.GetAllUsers(ctx)
+    users, nextCursor, err := h.db.GetUsersPaginated(ctx, limit, cursor)`,
+			},
+		},
+		ExpectedLevel: LevelMedium,
+		Criterion:     "Assessment should identify this as medium risk because it modifies an API endpoint which could affect clients, but pagination is a common pattern with manageable risk",
+	},
+	{
+		Name:        "documentation-update",
+		Description: "Documentation-only changes should be low risk",
+		PRContext: &PRContext{
+			Owner:       "test",
+			Repo:        "docs",
+			PRNumber:    99,
+			Title:       "Update installation instructions",
+			Description: "Adding troubleshooting section to README",
+			Author:      "docs-contributor",
+			FilesChanged: []string{
+				"README.md",
+				"docs/installation.md",
+			},
+			Additions: 45,
+			Deletions: 8,
+			FileDiffs: map[string]string{
+				"README.md": `@@ -50,6 +50,15 @@ npm install
+ npm start
+'''
+
+## Troubleshooting
+
+If you encounter issues during installation:
+
+1. Clear your npm cache: 'npm cache clean --force'
+2. Delete node_modules and package-lock.json
+3. Run 'npm install' again`,
+			},
+		},
+		ExpectedLevel: LevelLow,
+		Criterion:     "Assessment should identify this as low risk because it only changes documentation which cannot break production systems",
+	},
+	{
+		Name:        "docker-config-change",
+		Description: "Docker configuration changes should be high risk",
+		PRContext: &PRContext{
+			Owner:       "test",
+			Repo:        "services",
+			PRNumber:    23,
+			Title:       "Update production Dockerfile",
+			Description: "Switching base image to Alpine for smaller size",
+			Author:      "devops",
+			FilesChanged: []string{
+				"Dockerfile",
+				"docker-compose.yml",
+			},
+			Additions: 12,
+			Deletions: 8,
+			FileDiffs: map[string]string{
+				"Dockerfile": `@@ -1,4 +1,4 @@
+-FROM node:18
+FROM node:18-alpine
+ 
+ WORKDIR /app
+ COPY package*.json ./`,
+			},
+		},
+		ExpectedLevel: LevelHigh,
+		Criterion:     "Assessment should identify this as high risk because changing base Docker images can introduce compatibility issues and affect production deployments",
+	},
+	{
+		Name:        "ci-workflow-change",
+		Description: "CI/CD workflow changes should be high risk",
+		PRContext: &PRContext{
+			Owner:       "test",
+			Repo:        "main-app",
+			PRNumber:    67,
+			Title:       "Update deployment workflow",
+			Description: "Adding production deployment stage to GitHub Actions",
+			Author:      "ci-team",
+			FilesChanged: []string{
+				".github/workflows/deploy.yml",
+			},
+			Additions: 28,
+			Deletions: 3,
+			FileDiffs: map[string]string{
+				".github/workflows/deploy.yml": "@@ -30,6 +30,18 @@ jobs:\n" +
+					"      - name: Run tests\n" +
+					"        run: npm test\n" +
+					"\n" +
+					"+  deploy-production:\n" +
+					"+    needs: test\n" +
+					"+    runs-on: ubuntu-latest\n" +
+					"+    if: github.ref == 'refs/heads/main'\n" +
+					"+    steps:\n" +
+					"+      - name: Deploy to production\n" +
+					"+        run: |\n" +
+					"+          kubectl apply -f k8s/prod/",
+			},
+		},
+		ExpectedLevel: LevelHigh,
+		Criterion:     "Assessment should identify this as high risk because CI/CD pipeline changes affect deployment processes and could cause production outages if misconfigured",
+	},
+	{
+		Name:        "frontend-component-change",
+		Description: "Frontend component changes should be medium risk",
+		PRContext: &PRContext{
+			Owner:       "test",
+			Repo:        "web-app",
+			PRNumber:    88,
+			Title:       "Add loading spinner to user profile",
+			Description: "Improving UX by showing loading state",
+			Author:      "frontend-dev",
+			FilesChanged: []string{
+				"src/components/UserProfile.tsx",
+				"src/components/LoadingSpinner.tsx",
+			},
+			Additions: 45,
+			Deletions: 5,
+			FileDiffs: map[string]string{
+				"src/components/UserProfile.tsx": `@@ -10,6 +10,8 @@ export function UserProfile({ userId }: Props) {
+   const { data: user, isLoading } = useUser(userId);
+ 
+  if (isLoading) return <LoadingSpinner />;
+
+   if (!user) return <div>User not found</div>;`,
+			},
+		},
+		ExpectedLevel: LevelMedium,
+		Criterion:     "Assessment should identify this as medium risk because frontend changes affect user experience but have limited impact on backend systems",
+	},
+	{
+		Name:        "large-refactor",
+		Description: "Large refactors should be high risk due to size and complexity",
+		PRContext: &PRContext{
+			Owner:       "test",
+			Repo:        "backend",
+			PRNumber:    134,
+			Title:       "Refactor authentication system",
+			Description: "Moving from JWT to session-based auth",
+			Author:      "senior-dev",
+			FilesChanged: []string{
+				"auth/jwt.go",
+				"auth/session.go",
+				"auth/middleware.go",
+				"handlers/login.go",
+				"handlers/logout.go",
+				"models/session.go",
+			},
+			Additions: 450,
+			Deletions: 380,
+		},
+		ExpectedLevel: LevelHigh,
+		Criterion:     "Assessment should identify this as high risk because it's a large refactor (450+ additions) of critical authentication code which affects security and stability",
+	},
+}
--- a/driftlessaf/internal/risk/types.go
+++ b/driftlessaf/internal/risk/types.go
@ -0,0 +1,45 @@
+// Package risk provides AI-powered risk assessment for GitHub pull requests.
+package risk
+
+import (
+	"chainguard.dev/driftlessaf/agents/promptbuilder"
+)
+
+// Level represents the risk level of a PR.
+type Level string
+
+const (
+	LevelLow    Level = "low"
+	LevelMedium Level = "medium"
+	LevelHigh   Level = "high"
+)
+
+// PRContext contains the context for a PR risk assessment request.
+type PRContext struct {
+	Owner        string   `xml:"owner"`
+	Repo         string   `xml:"repo"`
+	PRNumber     int      `xml:"pr_number"`
+	Title        string   `xml:"title"`
+	Description  string   `xml:"description"`
+	Author       string   `xml:"author"`
+	FilesChanged []string `xml:"files_changed>file"`
+	Additions    int      `xml:"additions"`
+	Deletions    int      `xml:"deletions"`
+	// FileContents maps filename to file diff/content for analysis
+	FileDiffs map[string]string `xml:"-"`
+}
+
+// Bind implements promptbuilder.Bindable.
+func (c *PRContext) Bind(prompt *promptbuilder.Prompt) (*promptbuilder.Prompt, error) {
+	// Bind the full context as XML
+	return prompt.BindXML("pr_context", c)
+}
+
+// RiskAssessment is the structured result from the risk assessment agent.
+type RiskAssessment struct {
+	RiskLevel   string   `json:"risk_level" jsonschema:"description=Risk level: low|medium|high,enum=low,enum=medium,enum=high,required"`
+	Reasoning   string   `json:"reasoning" jsonschema:"description=Detailed explanation of the risk assessment,required"`
+	RiskyFiles  []string `json:"risky_files" jsonschema:"description=List of files that contribute most to the risk"`
+	RiskFactors []string `json:"risk_factors" jsonschema:"description=Specific risk factors identified"`
+	Confidence  float64  `json:"confidence" jsonschema:"description=Confidence in the assessment (0.0-1.0),minimum=0,maximum=1"`
+}
--- a/driftlessaf/risk-scorer.tf
+++ b/driftlessaf/risk-scorer.tf
@ -0,0 +1,52 @@
+resource "google_service_account" "risk_scorer" {
+  project    = var.project_id
+  account_id = "driftlessaf-pr-risk-scorer"
+}
+
+# Grant Vertex AI permissions for risk assessment (Claude API access)
+resource "google_project_iam_member" "risk_scorer_vertex_ai" {
+  project = var.project_id
+  role    = "roles/aiplatform.user"
+  member  = "serviceAccount:${google_service_account.risk_scorer.email}"
+}
+
+module "pr-risk-scorer" {
+  source = "chainguard-dev/common/infra//modules/regional-go-reconciler"
+
+  project_id      = var.project_id
+  name            = "pr-risk-scorer"
+  regions         = module.networking.regional-networks
+  service_account = google_service_account.risk_scorer.email
+
+  # Allow direct internet access for GitHub API calls
+  egress = "PRIVATE_RANGES_ONLY"
+
+  containers = {
+    "risk-scorer" = {
+      source = {
+        working_dir = path.module
+        importpath  = "./cmd/pr-risk-scorer"
+      }
+      ports = [{ container_port = 8080 }]
+      env = [
+        { name = "GITHUB_APP_ID", value = var.github_app_id },
+        {
+          name = "GITHUB_PRIVATE_KEY"
+          value_source = {
+            secret_key_ref = {
+              secret  = google_secret_manager_secret.github_app_key.secret_id
+              version = "latest"
+            }
+          }
+        },
+        { name = "GCP_PROJECT_ID", value = var.project_id },
+        { name = "GCP_REGION", value = var.risk_scorer_region },
+        { name = "CLAUDE_MODEL", value = var.risk_scorer_model },
+      ]
+    }
+  }
+
+  team                  = var.team
+  notification_channels = []
+  deletion_protection   = var.deletion_protection
+}
--- a/driftlessaf/trigger.tf
+++ b/driftlessaf/trigger.tf
@ -20,3 +20,26 @@ module "pr-workqueue" {
  notification_channels = []
  deletion_protection   = var.deletion_protection
 }
+
+module "pr-risk-scorer-workqueue" {
+  source = "chainguard-dev/common/infra//modules/cloudevents-workqueue"
+
+  project_id = var.project_id
+  name       = "pr-risk-scorer-events"
+  regions    = module.networking.regional-networks
+  broker     = module.cloudevent-broker.broker
+
+  # Empty filter matches all events that have pullrequesturl extension
+  # This includes pull_request, issue_comment on PRs, pull_request_review, etc.
+  filters = [{}]
+
+  extension_key = "pullrequesturl"
+
+  workqueue = {
+    name = module.pr-risk-scorer.receiver.name
+  }
+
+  team                  = var.team
+  notification_channels = []
+  deletion_protection   = var.deletion_protection
+}
--- a/driftlessaf/variables.tf
+++ b/driftlessaf/variables.tf
@ -60,3 +60,16 @@ variable "ci_fixer_region" {
  description = "GCP region for Vertex AI Claude API calls."
  default     = "us-east5"
 }
+
+# Risk Scorer configuration
+variable "risk_scorer_model" {
+  type        = string
+  description = "The Claude model to use for risk assessment."
+  default     = "claude-sonnet-4@20250514"
+}
+
+variable "risk_scorer_region" {
+  type        = string
+  description = "GCP region for Vertex AI Claude API calls."
+  default     = "us-east5"
+}