package cifixer // EvalTestCase represents a test case for evaluating the CI fixer agent. type EvalTestCase struct { Name string Description string Files map[string]string CILogs string ExpectedFix map[string]string Criterion string Deterministic bool } // EvalTestCases contains test cases for evaluating the CI fixer agent. // These are used for offline evaluation before integrating with GitHub. var EvalTestCases = []EvalTestCase{ { Name: "go-missing-return-type", Description: "Go function missing return type annotation", Files: map[string]string{ "main.go": `package main func add(a, b int) { return a + b } func main() { result := add(1, 2) println(result) } `, }, CILogs: `main.go:4:2: too many return values have (int) want ()`, ExpectedFix: map[string]string{ "main.go": `package main func add(a, b int) int { return a + b } func main() { result := add(1, 2) println(result) } `, }, Criterion: "Fix should add the missing 'int' return type to the function signature without changing any other code", Deterministic: true, }, { Name: "go-missing-import", Description: "Go file using fmt without importing it", Files: map[string]string{ "main.go": `package main func main() { fmt.Println("hello world") } `, }, CILogs: `main.go:4:2: undefined: fmt`, ExpectedFix: map[string]string{ "main.go": `package main import "fmt" func main() { fmt.Println("hello world") } `, }, Criterion: "Fix should add the missing 'fmt' import without changing any other code", Deterministic: true, }, { Name: "go-type-mismatch", Description: "Go type mismatch in assignment", Files: map[string]string{ "main.go": `package main func main() { var count int count = "5" println(count) } `, }, CILogs: `main.go:5:9: cannot use "5" (untyped string constant) as int value in assignment`, ExpectedFix: map[string]string{ "main.go": `package main func main() { var count int count = 5 println(count) } `, }, Criterion: "Fix should change the string '5' to integer 5 without changing any other code", Deterministic: true, }, { Name: "go-unused-import", Description: "Go file with unused import", Files: map[string]string{ "main.go": `package main import ( "fmt" "os" ) func main() { fmt.Println("hello") } `, }, CILogs: `main.go:5:2: "os" imported and not used`, ExpectedFix: map[string]string{ "main.go": `package main import ( "fmt" ) func main() { fmt.Println("hello") } `, }, Criterion: "Fix should remove the unused 'os' import without changing any other code", Deterministic: true, }, { Name: "go-missing-error-check", Description: "Go function ignoring error return value", Files: map[string]string{ "main.go": `package main import ( "os" ) func main() { f, _ := os.Open("test.txt") defer f.Close() } `, }, CILogs: `main.go:8:5: Error return value of 'os.Open' is not checked (errcheck)`, ExpectedFix: map[string]string{ "main.go": `package main import ( "log" "os" ) func main() { f, err := os.Open("test.txt") if err != nil { log.Fatal(err) } defer f.Close() } `, }, Criterion: "Fix should properly handle the error from os.Open with appropriate error handling", Deterministic: false, }, { Name: "typescript-missing-type", Description: "TypeScript function parameter without type annotation", Files: map[string]string{ "src/utils.ts": `export function greet(name) { return "Hello, " + name; } `, }, CILogs: `src/utils.ts:1:24 - error TS7006: Parameter 'name' implicitly has an 'any' type.`, ExpectedFix: map[string]string{ "src/utils.ts": `export function greet(name: string) { return "Hello, " + name; } `, }, Criterion: "Fix should add the string type annotation to the name parameter", Deterministic: true, }, { Name: "python-indentation-error", Description: "Python file with indentation error", Files: map[string]string{ "app.py": `def hello(): print("hello") print("world") # wrong indentation `, }, CILogs: ` File "app.py", line 3 print("world") ^ IndentationError: unindent does not match any outer indentation level`, ExpectedFix: map[string]string{ "app.py": `def hello(): print("hello") print("world") `, }, Criterion: "Fix should correct the indentation of line 3 to match line 2", Deterministic: true, }, { Name: "go-test-failure", Description: "Go test failing due to incorrect implementation", Files: map[string]string{ "math.go": `package math func Add(a, b int) int { return a - b // bug: should be + } `, "math_test.go": `package math import "testing" func TestAdd(t *testing.T) { result := Add(2, 3) if result != 5 { t.Errorf("Add(2, 3) = %d; want 5", result) } } `, }, CILogs: `--- FAIL: TestAdd (0.00s) math_test.go:8: Add(2, 3) = -1; want 5 FAIL`, ExpectedFix: map[string]string{ "math.go": `package math func Add(a, b int) int { return a + b } `, }, Criterion: "Fix should change the subtraction operator to addition in math.go, not modify the test", Deterministic: true, }, { Name: "go-multiple-errors", Description: "Go file with multiple type errors", Files: map[string]string{ "main.go": `package main func process(data string) int { length := len(data) return length } func main() { result := process(42) fmt.Println(result) } `, }, CILogs: `main.go:9:19: cannot use 42 (untyped int constant) as string value in argument to process main.go:10:2: undefined: fmt`, ExpectedFix: map[string]string{ "main.go": `package main import "fmt" func process(data string) int { length := len(data) return length } func main() { result := process("42") fmt.Println(result) } `, }, Criterion: "Fix should add the fmt import and change 42 to \"42\" to fix both errors", Deterministic: true, }, { Name: "go-lint-exported-function", Description: "Go exported function without documentation comment", Files: map[string]string{ "api.go": `package api func ProcessRequest(req string) string { return "processed: " + req } `, }, CILogs: `api.go:3:1: exported function ProcessRequest should have comment or be unexported (golint)`, ExpectedFix: map[string]string{ "api.go": `package api // ProcessRequest processes the given request string. func ProcessRequest(req string) string { return "processed: " + req } `, }, Criterion: "Fix should add a documentation comment for the exported function", Deterministic: false, }, }