package agent import ( "bytes" "context" "encoding/json" "errors" "image" "image/png" "io" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" "strings" "sync" "testing" "time" "github.com/johannSo/codey/internal/llm" "github.com/johannSo/codey/internal/metrics" "github.com/johannSo/codey/internal/skills" "github.com/johannSo/codey/internal/tools" ) // fakeAPI serves a scripted sequence of responses and records every request // body it received, so tests can assert on what the loop actually sent. type fakeAPI struct { t *testing.T responses []string requests []llm.Request summary string srv *httptest.Server } // compactions counts requests that were summarization calls, so they do not // consume scripted responses. func (f *fakeAPI) compactions() int { var n int for _, req := range f.requests { if len(req.Messages) > 0 { last := req.Messages[len(req.Messages)-1] for _, b := range last.Content { if strings.Contains(b.Text, "Summarize the conversation above") { n++ } } } } return n } func newFakeAPI(t *testing.T, responses ...string) *fakeAPI { f := &fakeAPI{t: t, responses: responses, summary: "Compacted: edited loop.go, the config loads from X not Y."} f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) var req llm.Request if err := json.Unmarshal(body, &req); err != nil { t.Errorf("codey sent a body the API types cannot parse: %v\n%s", err, body) } f.requests = append(f.requests, req) if bytes.Contains(body, []byte("Summarize the conversation above")) { io.WriteString(w, msg("end_turn", textBlock(f.summary))) return } i := len(f.requests) - 1 - f.compactions() if i >= len(f.responses) { t.Errorf("codey made %d requests, script only has %d", i+1, len(f.responses)) w.WriteHeader(http.StatusInternalServerError) return } io.WriteString(w, f.responses[i]) })) t.Cleanup(f.srv.Close) return f } // rawRequest returns the n-th request body re-encoded, for byte-level assertions. func (f *fakeAPI) rawRequest(n int) string { b, err := json.Marshal(f.requests[n]) if err != nil { f.t.Fatal(err) } return string(b) } func msg(stopReason string, blocks ...string) string { return `{"id":"msg","type":"message","role":"assistant","model":"claude-opus-5","content":[` + strings.Join(blocks, ",") + `],"stop_reason":"` + stopReason + `",` + `"usage":{"input_tokens":100,"output_tokens":20,"cache_read_input_tokens":40}}` } func textBlock(s string) string { return `{"type":"text","text":"` + s + `"}` } // msgWithTokens is msg with a controllable input_tokens, for driving compaction. func msgWithTokens(stopReason string, inputTokens int, blocks ...string) string { return `{"id":"msg","type":"message","role":"assistant","model":"claude-opus-5","content":[` + strings.Join(blocks, ",") + `],"stop_reason":"` + stopReason + `",` + `"usage":{"input_tokens":` + strconv.Itoa(inputTokens) + `,"output_tokens":20}}` } func toolUseBlock(id, name, input string) string { return `{"type":"tool_use","id":"` + id + `","name":"` + name + `","input":` + input + `}` } // newTestAgent wires an agent against the fake API, with read_file rooted at // a temp dir plus a couple of tools that make failure modes easy to trigger. func newTestAgent(t *testing.T, f *fakeAPI, root string, logBuf *bytes.Buffer) *Agent { t.Helper() return newTestAgentOnModel(t, f, root, logBuf, llm.DefaultModel) } // newTestAgentOnModel is newTestAgent pinned to a model, for the parts of a // request whose shape depends on which model it is going to. func newTestAgentOnModel(t *testing.T, f *fakeAPI, root string, logBuf *bytes.Buffer, model string) *Agent { t.Helper() log := metrics.NewWriter(logBuf, "test") state := NewState() reg := tools.NewRegistry(log).Register( tools.NewRead(root, state), tools.NewEdit(root, state), tools.NewWrite(root, state), tools.New("boom", "Always fails", func(context.Context, struct{}) (string, error) { return "", errors.New("detonated") }), ) client := llm.New(llm.WithBaseURL(f.srv.URL), llm.WithAPIKey("k"), llm.WithModel(model)) return New(client, reg, state, log, Options{}) } func TestRunReadsAFileAndAnswers(t *testing.T) { root := t.TempDir() os.WriteFile(filepath.Join(root, "main.go"), []byte("package main\n"), 0o644) f := newFakeAPI(t, msg("tool_use", textBlock("Let me look."), toolUseBlock("tu_1", "read_file", `{"path":"main.go"}`)), msg("end_turn", textBlock("It declares package main.")), ) a := newTestAgent(t, f, root, &bytes.Buffer{}) got, err := a.Run(context.Background(), "what does main.go do?") if err != nil { t.Fatalf("Run: %v", err) } if got != "It declares package main." { t.Errorf("answer = %q", got) } if len(f.requests) != 2 { t.Fatalf("made %d requests, want 2", len(f.requests)) } // Second request: user prompt, assistant turn, then the tool results. second := f.requests[1].Messages if len(second) != 3 { t.Fatalf("second request had %d messages, want 3:\n%s", len(second), f.rawRequest(1)) } if second[2].Role != llm.RoleUser { t.Errorf("tool results must come back under the user role, got %q", second[2].Role) } if n := len(second[2].Content); n != 1 { t.Fatalf("want 1 tool_result block, got %d", n) } res := second[2].Content[0] if res.Type != llm.BlockToolResult || res.ToolUseID != "tu_1" || res.IsError { t.Errorf("tool_result = %+v, want a successful result for tu_1", res) } if !strings.Contains(res.Content, "package main") { t.Errorf("tool_result content = %q, want the file contents", res.Content) } } // A screenshot read with read_file reaches the model as an image block inside // the tool_result, and the transcript sees the caption. func TestImageReadReachesTheModelAsAnImage(t *testing.T) { root := t.TempDir() // The smallest valid PNG: 1×1, from image/png's own encoder. var buf bytes.Buffer png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 1, 1))) os.WriteFile(filepath.Join(root, "shot.png"), buf.Bytes(), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"shot.png"}`)), msg("end_turn", textBlock("looks fine")), ) a := newTestAgent(t, f, root, &bytes.Buffer{}) var seen []Event a.Observe = func(e Event) { if e.Kind == EventToolResult { seen = append(seen, e) } } if _, err := a.Run(context.Background(), "look at the screenshot"); err != nil { t.Fatalf("Run: %v", err) } raw := f.rawRequest(1) if !strings.Contains(raw, `"type":"image"`) || !strings.Contains(raw, `"media_type":"image/png"`) { t.Errorf("second request carries no image block:\n%s", raw) } res := f.requests[1].Messages[2].Content[0] if len(res.Blocks) != 2 || res.Blocks[1].Type != llm.BlockImage { t.Errorf("tool_result blocks = %+v, want caption and image", res.Blocks) } if len(seen) != 1 || !strings.Contains(seen[0].Result, "PNG image, 1×1") { t.Errorf("the transcript should see the caption, got %+v", seen) } } // Superseding a read of an image drops the picture too: the model must not // keep reasoning from a screenshot of a file that has since changed. func TestSupersedeDropsImageBlocks(t *testing.T) { root := t.TempDir() var buf bytes.Buffer png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 1, 1))) os.WriteFile(filepath.Join(root, "shot.png"), buf.Bytes(), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"shot.png"}`)), msg("tool_use", toolUseBlock("tu_2", "read_file", `{"path":"shot.png"}`)), msg("tool_use", toolUseBlock("tu_3", "write_file", `{"path":"shot.png","content":"not a png any more"}`)), msg("end_turn", textBlock("done")), ) a := newTestAgent(t, f, root, &bytes.Buffer{}) if _, err := a.Run(context.Background(), "redo the screenshot"); err != nil { t.Fatalf("Run: %v", err) } var images, superseded int for _, m := range a.State().History() { for _, b := range m.Content { if b.Type != llm.BlockToolResult { continue } if strings.HasPrefix(b.Content, "[superseded:") { superseded++ if len(b.Blocks) != 0 { t.Errorf("a superseded result still carries %d blocks", len(b.Blocks)) } } for _, inner := range b.Blocks { if inner.Type == llm.BlockImage { images++ } } } } if superseded != 1 || images != 1 { t.Errorf("superseded=%d images=%d, want 1 and 1 (the later read keeps its picture)", superseded, images) } } func TestParallelToolCallsReturnInOneUserMessage(t *testing.T) { root := t.TempDir() os.WriteFile(filepath.Join(root, "a.go"), []byte("A\n"), 0o644) os.WriteFile(filepath.Join(root, "b.go"), []byte("B\n"), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"a.go"}`), toolUseBlock("tu_2", "read_file", `{"path":"b.go"}`)), msg("end_turn", textBlock("done")), ) a := newTestAgent(t, f, root, &bytes.Buffer{}) if _, err := a.Run(context.Background(), "read both"); err != nil { t.Fatalf("Run: %v", err) } last := f.requests[1].Messages[len(f.requests[1].Messages)-1] if len(last.Content) != 2 { t.Fatalf("want both results in ONE message, got %d messages ending with %d blocks", len(f.requests[1].Messages), len(last.Content)) } if last.Content[0].ToolUseID != "tu_1" || last.Content[1].ToolUseID != "tu_2" { t.Errorf("results out of order: %s, %s", last.Content[0].ToolUseID, last.Content[1].ToolUseID) } } func TestFailedToolStillProducesAResult(t *testing.T) { tests := []struct { name string call string wantText string }{ {name: "tool returns an error", call: toolUseBlock("tu_1", "boom", `{}`), wantText: "detonated"}, {name: "unknown tool", call: toolUseBlock("tu_1", "telepathy", `{}`), wantText: `unknown tool "telepathy"`}, {name: "malformed arguments", call: toolUseBlock("tu_1", "read_file", `{"path":42}`), wantText: "invalid arguments"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := newFakeAPI(t, msg("tool_use", tc.call), msg("end_turn", textBlock("ok"))) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("a failing tool is a normal event, not a Run error: %v", err) } last := f.requests[1].Messages[len(f.requests[1].Messages)-1] if len(last.Content) != 1 { t.Fatalf("every tool_use id needs exactly one result, got %d", len(last.Content)) } res := last.Content[0] if !res.IsError { t.Error("is_error must be set so the model knows the call failed") } if !strings.Contains(res.Content, tc.wantText) { t.Errorf("result = %q, want it to contain %q", res.Content, tc.wantText) } }) } } // Thinking signatures must be echoed back byte-identical or the next request // is rejected. func TestThinkingBlocksAreEchoedBackUnchanged(t *testing.T) { f := newFakeAPI(t, msg("tool_use", `{"type":"thinking","thinking":"","signature":"sig-xyz"}`, toolUseBlock("tu_1", "boom", `{}`)), msg("end_turn", textBlock("fine")), ) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } body := f.rawRequest(1) if !strings.Contains(body, `"signature":"sig-xyz"`) { t.Errorf("thinking signature was dropped from the resent history:\n%s", body) } if !strings.Contains(body, `"thinking":""`) { t.Errorf("empty thinking text was dropped:\n%s", body) } } func TestRequestShape(t *testing.T) { f := newFakeAPI(t, msg("end_turn", textBlock("hi"))) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) if _, err := a.Run(context.Background(), "hello"); err != nil { t.Fatalf("Run: %v", err) } req := f.requests[0] if len(req.System) != 1 || req.System[0].Text == "" { t.Error("system prompt missing") } if strings.ContainsAny(req.System[0].Text, "0123456789") { t.Error("system prompt should hold no digits: timestamps or ids here would break the cached prefix") } // The agent runs on llm.DefaultModel, which takes adaptive thinking: a // model that supports it must still be asked for it. if req.Thinking == nil || req.Thinking.Type != "adaptive" { t.Errorf("thinking = %+v, want adaptive (budget_tokens is a 400 on current models)", req.Thinking) } var names []string for _, tl := range req.Tools { names = append(names, tl.Name) } if want := "boom, edit_file, read_file, write_file"; strings.Join(names, ", ") != want { t.Errorf("tools = %v, want them sorted for a byte-stable prefix (%s)", names, want) } for _, tl := range req.Tools { if len(tl.InputSchema) == 0 { t.Errorf("tool %q has no input_schema", tl.Name) } } } // A cap only applies when one was asked for, and what the model had managed to // say by then comes back with the error rather than being thrown away. func TestMaxTurnsStopsTheLoopWhenOneWasAskedFor(t *testing.T) { var script []string for i := 0; i < 5; i++ { script = append(script, msg("tool_use", textBlock("still working on it"), toolUseBlock("tu", "boom", `{}`))) } f := newFakeAPI(t, script...) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) a.opts.MaxTurns = 3 text, err := a.Run(context.Background(), "loop forever") if !errors.Is(err, ErrMaxTurns) { t.Fatalf("error = %v, want ErrMaxTurns", err) } if !strings.Contains(err.Error(), "3 turns") { t.Errorf("error should say how many turns were spent: %v", err) } if text != "still working on it" { t.Errorf("text = %q, want what the model had got to by the cap", text) } if len(f.requests) != 3 { t.Errorf("made %d requests, want exactly the cap of 3", len(f.requests)) } } // Nothing caps a session that did not ask to be capped. Thirty turns is a // short afternoon's work, and a run that stops there has thrown away a task // that was going fine; what bounds a session is the window, the bill and the // person who can interrupt it. func TestNoTurnLimitByDefault(t *testing.T) { const turns = 80 var script []string for i := 0; i < turns; i++ { script = append(script, msg("tool_use", toolUseBlock("tu", "boom", `{}`))) } script = append(script, msg("end_turn", textBlock("finally done"))) f := newFakeAPI(t, script...) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) if a.opts.MaxTurns > NoTurnLimit { t.Fatalf("MaxTurns = %d, want a session with no cap of its own", a.opts.MaxTurns) } text, err := a.Run(context.Background(), "a long job") if err != nil { t.Fatalf("Run: %v", err) } if text != "finally done" { t.Errorf("text = %q, want the answer after %d turns", text, turns) } if len(f.requests) != turns+1 { t.Errorf("made %d requests, want all %d", len(f.requests), turns+1) } } func TestRefusalIsReportedWithItsCategory(t *testing.T) { f := newFakeAPI(t, `{"id":"m","type":"message","role":"assistant","model":"claude-opus-5", "content":[],"stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber","explanation":"nope"}, "usage":{"input_tokens":5,"output_tokens":0}}`) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) _, err := a.Run(context.Background(), "do something") if err == nil { t.Fatal("want an error") } if !strings.Contains(err.Error(), "cyber") || !strings.Contains(err.Error(), "nope") { t.Errorf("error = %v, want the category and explanation", err) } } func TestPauseTurnResends(t *testing.T) { f := newFakeAPI(t, msg("pause_turn", textBlock("thinking")), msg("end_turn", textBlock("done")), ) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) got, err := a.Run(context.Background(), "go") if err != nil { t.Fatalf("Run: %v", err) } if got != "done" || len(f.requests) != 2 { t.Errorf("got %q after %d requests, want the loop to resend after pause_turn", got, len(f.requests)) } } func serverToolUseBlock(id, query string) string { return `{"type":"server_tool_use","id":"` + id + `","name":"web_search","input":{"query":"` + query + `"}}` } func webSearchResultBlock(toolUseID, content string) string { return `{"type":"web_search_tool_result","tool_use_id":"` + toolUseID + `","content":` + content + `}` } // A hosted search is run by the API, which also supplies its result. The loop // has two jobs: show it to the user, who otherwise sees nothing while the // model searches, and keep its hands off — no registry call, no tool_result. func TestHostedWebSearchIsVisibleButNotDispatched(t *testing.T) { f := newFakeAPI(t, msg("pause_turn", textBlock("Let me check."), serverToolUseBlock("srvtoolu_1", "go 1.23 release notes"), webSearchResultBlock("srvtoolu_1", `[{"type":"web_search_result","title":"Go 1.23 Release Notes","url":"https://go.dev/doc/go1.23","encrypted_content":"x"},{"type":"web_search_result","title":"Go Blog","url":"https://go.dev/blog/go1.23","encrypted_content":"y"}]`), ), msg("end_turn", textBlock("done")), ) // The registry has no web_search: the hosted tool is not a local one. a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) a.opts.ServerTools = []llm.ToolDef{llm.WebSearchTool(5)} var events []Event a.Observe = func(ev Event) { events = append(events, ev) } got, err := a.Run(context.Background(), "what's new in go 1.23") if err != nil { t.Fatalf("Run: %v", err) } if got != "done" || len(f.requests) != 2 { t.Fatalf("got %q after %d requests, want the pause_turn resend to finish", got, len(f.requests)) } var start, result *Event for i := range events { switch events[i].Kind { case EventToolStart: start = &events[i] case EventToolResult: result = &events[i] } } if start == nil || start.Tool != llm.WebSearchToolName || !strings.Contains(string(start.Input), "go 1.23 release notes") { t.Errorf("tool_start = %+v, want the hosted search with its query", start) } if result == nil || result.Tool != llm.WebSearchToolName || result.IsError { t.Fatalf("tool_result = %+v, want a successful web_search result", result) } if want := "Go 1.23 Release Notes — https://go.dev/doc/go1.23\nGo Blog — https://go.dev/blog/go1.23"; result.Result != want { t.Errorf("result summary = %q, want %q", result.Result, want) } if strings.Contains(result.Result, "encrypted_content") { t.Error("the summary must not dump the encrypted payload on the user") } // The resend carries the assistant turn unchanged and nothing else: the // API supplied the web_search_tool_result, so a local tool_result for the // hosted call would be an invariant violation, not a courtesy. resend := f.requests[1] if len(resend.Messages) != 2 { t.Fatalf("resend has %d messages, want user + assistant only", len(resend.Messages)) } for _, m := range resend.Messages { for _, b := range m.Content { if b.Type == llm.BlockToolResult { t.Errorf("a tool_result was sent for the hosted call: %+v", b) } } } raw := f.rawRequest(1) if !strings.Contains(raw, `"query":"go 1.23 release notes"`) { t.Errorf("resend lost the server_tool_use query:\n%s", raw) } if !strings.Contains(raw, `"web_search_tool_result"`) || !strings.Contains(raw, `"encrypted_content":"x"`) { t.Errorf("resend must echo the web_search_tool_result verbatim:\n%s", raw) } last := resend.Tools[len(resend.Tools)-1] if last.Type != llm.WebSearchToolType || last.Name != llm.WebSearchToolName || last.MaxUses != 5 { t.Errorf("last tool = %+v, want the hosted web search definition after the local tools", last) } } // A failed hosted search is reported as such, and the loop still gets on with // the resend rather than treating an API-side error as a local one. func TestHostedWebSearchErrorIsReported(t *testing.T) { f := newFakeAPI(t, msg("pause_turn", serverToolUseBlock("srvtoolu_1", "anything"), webSearchResultBlock("srvtoolu_1", `{"type":"web_search_tool_result_error","error_code":"max_uses_exceeded"}`), ), msg("end_turn", textBlock("done")), ) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) var result *Event a.Observe = func(ev Event) { if ev.Kind == EventToolResult { result = &ev } } if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } if result == nil || !result.IsError || !strings.Contains(result.Result, "max_uses_exceeded") { t.Errorf("tool_result = %+v, want an error naming max_uses_exceeded", result) } } // The hosted definitions sit after the sorted local ones: the local list is // sorted for a byte-stable cached prefix, and appending a constant tail to a // stable list keeps it stable. func TestServerToolsFollowTheLocalTools(t *testing.T) { f := newFakeAPI(t, msg("end_turn", textBlock("hi"))) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) a.opts.ServerTools = []llm.ToolDef{llm.WebSearchTool(3)} if _, err := a.Run(context.Background(), "hello"); err != nil { t.Fatalf("Run: %v", err) } var names []string for _, tl := range f.requests[0].Tools { names = append(names, tl.Name) } if want := "boom, edit_file, read_file, write_file, web_search"; strings.Join(names, ", ") != want { t.Errorf("tools = %v, want %s", names, want) } hosted := f.requests[0].Tools[len(names)-1] if hosted.Type != llm.WebSearchToolType || len(hosted.InputSchema) != 0 || hosted.Description != "" { t.Errorf("hosted tool = %+v, want type set and no local schema or description", hosted) } } func TestSummarizeWebSearchResult(t *testing.T) { tests := []struct { name string block string want string wantErr bool }{ { name: "one line per result", block: webSearchResultBlock("s", `[{"type":"web_search_result","title":"A","url":"https://a"},{"type":"web_search_result","title":"B","url":"https://b"}]`), want: "A — https://a\nB — https://b", }, { name: "missing title falls back to the url alone", block: webSearchResultBlock("s", `[{"type":"web_search_result","url":"https://a"}]`), want: "https://a", }, { name: "no results says so rather than going silent", block: webSearchResultBlock("s", `[]`), want: "(no results)", }, { name: "error object", block: webSearchResultBlock("s", `{"type":"web_search_tool_result_error","error_code":"too_many_requests"}`), want: "web search failed: too_many_requests", wantErr: true, }, { name: "unparseable content is reported, not hidden", block: webSearchResultBlock("s", `"what"`), want: "web search returned an unrecognised result", wantErr: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var b llm.ContentBlock if err := json.Unmarshal([]byte(tc.block), &b); err != nil { t.Fatalf("unmarshal: %v", err) } got, isErr := summarizeWebSearchResult(b) if got != tc.want || isErr != tc.wantErr { t.Errorf("got (%q, %v), want (%q, %v)", got, isErr, tc.want, tc.wantErr) } }) } } func TestUsageAndEventsAreRecorded(t *testing.T) { root := t.TempDir() os.WriteFile(filepath.Join(root, "x.go"), []byte("x\n"), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"x.go"}`)), msg("end_turn", textBlock("done")), ) var logBuf bytes.Buffer a := newTestAgent(t, f, root, &logBuf) var kinds []EventKind a.Observe = func(ev Event) { kinds = append(kinds, ev.Kind) } if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } usage, turns := a.State().Usage() if turns != 2 || usage.InputTokens != 200 || usage.CacheReadInputTokens != 80 { t.Errorf("usage = %+v over %d turns, want both turns accumulated", usage, turns) } if !containsKind(kinds, EventToolStart) || !containsKind(kinds, EventToolResult) || !containsKind(kinds, EventText) { t.Errorf("observer saw %v, want tool and text events", kinds) } log := logBuf.String() if n := strings.Count(log, `"type":"turn"`); n != 2 { t.Errorf("logged %d turn events, want 2:\n%s", n, log) } if n := strings.Count(log, `"type":"tool_call"`); n != 1 { t.Errorf("logged %d tool_call events, want 1:\n%s", n, log) } if !strings.Contains(log, `"cache_read_input_tokens":40`) { t.Errorf("cache reads must be logged; that is the Phase 4 lever:\n%s", log) } } func TestReadFileMarksStateSoEditCanCheckLater(t *testing.T) { root := t.TempDir() path := filepath.Join(root, "tracked.go") os.WriteFile(path, []byte("x\n"), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"tracked.go"}`)), msg("end_turn", textBlock("done")), ) a := newTestAgent(t, f, root, &bytes.Buffer{}) if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } if _, ok := a.State().LastRead(path); !ok { t.Error("read_file must record the read in State for Phase 2's edit_file") } } func TestRunPropagatesAPIErrors(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) io.WriteString(w, `{"error":{"type":"invalid_request_error","message":"bad model"}}`) })) defer srv.Close() state := NewState() a := New(llm.New(llm.WithBaseURL(srv.URL), llm.WithAPIKey("k")), tools.NewRegistry(nil), state, nil, Options{}) if _, err := a.Run(context.Background(), "go"); err == nil || !strings.Contains(err.Error(), "bad model") { t.Errorf("error = %v, want the API message surfaced", err) } } func containsKind(kinds []EventKind, want EventKind) bool { for _, k := range kinds { if k == want { return true } } return false } // streamingAPI serves one scripted SSE response. func streamingAPI(t *testing.T, events ...string) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { for _, e := range events { io.WriteString(w, "event: x\ndata: "+e+"\n\n") } })) t.Cleanup(srv.Close) return srv } func TestStreamingRunEmitsDeltas(t *testing.T) { srv := streamingAPI(t, `{"type":"message_start","message":{"id":"m","model":"claude-opus-5","usage":{"input_tokens":50}}}`, `{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"All "}}`, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"done."}}`, `{"type":"content_block_stop","index":0}`, `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}`, ) state := NewState() a := New(llm.New(llm.WithBaseURL(srv.URL), llm.WithAPIKey("k")), tools.NewRegistry(nil), state, nil, Options{Stream: true}) var deltas []string var finalText int a.Observe = func(ev Event) { switch ev.Kind { case EventTextDelta: deltas = append(deltas, ev.Text) case EventText: finalText++ } } got, err := a.Run(context.Background(), "go") if err != nil { t.Fatalf("Run: %v", err) } if got != "All done." { t.Errorf("answer = %q", got) } if strings.Join(deltas, "") != "All done." { t.Errorf("deltas = %v", deltas) } if finalText != 0 { t.Error("the whole text must not be emitted again after streaming it") } } // denyAll is an Approver that refuses everything it is asked about. type denyAll struct{ asked []string } func (d *denyAll) Approve(_ context.Context, tool string, _ json.RawMessage) error { d.asked = append(d.asked, tool) return errors.New("the user declined to run " + tool) } func TestDeniedToolBecomesAToolError(t *testing.T) { root := t.TempDir() os.WriteFile(filepath.Join(root, "a.go"), []byte("package a\n"), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"a.go"}`)), msg("end_turn", textBlock("understood")), ) var logBuf bytes.Buffer a := newTestAgent(t, f, root, &logBuf) gate := &denyAll{} a.Approve = gate got, err := a.Run(context.Background(), "read it") if err != nil { t.Fatalf("a denial is not a failure of ours: %v", err) } if got != "understood" { t.Errorf("answer = %q", got) } if len(gate.asked) != 1 || gate.asked[0] != "read_file" { t.Errorf("approver saw %v", gate.asked) } last := f.requests[1].Messages[len(f.requests[1].Messages)-1] res := last.Content[0] if !res.IsError || !strings.Contains(res.Content, "declined") { t.Errorf("tool_result = %+v, want the denial passed to the model", res) } if !strings.Contains(logBuf.String(), `"type":"tool_denied"`) { t.Error("denials should be logged: they explain gaps in the eval numbers") } // The tool must not have run. if strings.Contains(res.Content, "package a") { t.Error("the file was read despite the denial") } } func TestSystemPromptListsSkills(t *testing.T) { a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{ Skills: []skills.Skill{{Name: "commit-helper", Description: "Use when writing commit messages"}}, }) sys := a.opts.System if !strings.Contains(sys, "commit-helper: Use when writing commit messages") { t.Errorf("system prompt does not list the skill:\n%s", sys) } if !strings.Contains(sys, "/") { t.Error("system prompt does not explain what a leading slash means") } if !strings.HasPrefix(sys, defaultSystem) { t.Error("the listing replaced the default prompt instead of being appended to it") } } func TestSystemPromptUnchangedWithoutSkills(t *testing.T) { a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{}) if a.opts.System != defaultSystem { t.Errorf("system prompt differs from the default with no skills:\n%s", a.opts.System) } } func TestSystemPromptIsStableAcrossRequests(t *testing.T) { // The listing sits in the cached prefix. If it varied per request, every // turn would miss the cache. a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{ Skills: []skills.Skill{{Name: "a", Description: "one"}, {Name: "b", Description: "two"}}, }) a.state.Append(llm.Message{Role: llm.RoleUser, Content: []llm.ContentBlock{llm.TextBlock("hi")}}) first := a.request().System[0].Text second := a.request().System[0].Text if first != second { t.Error("the system prompt changed between two requests in one session") } } func TestSystemPrefixBecomesTheFirstBlock(t *testing.T) { a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{SystemPrefix: ClaudeCodeIdentity}) blocks := a.systemBlocks() if len(blocks) != 2 { t.Fatalf("got %d system blocks, want 2", len(blocks)) } if blocks[0].Text != ClaudeCodeIdentity { t.Errorf("first block = %q, want the identity string", blocks[0].Text) } if !strings.Contains(blocks[1].Text, "coding agent") { t.Errorf("second block should be Codey's own system prompt, got %q", blocks[1].Text) } } // The cache breakpoint has to sit on the last block, or only the identity // string gets cached and the prompt behind it is re-read every turn. func TestCacheBreakpointIsOnTheLastSystemBlock(t *testing.T) { a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{SystemPrefix: ClaudeCodeIdentity}) blocks := a.systemBlocks() if blocks[0].CacheControl != nil { t.Error("the first block must not carry the breakpoint") } if blocks[len(blocks)-1].CacheControl == nil { t.Error("the last block must carry the breakpoint") } } // Without OAuth nothing changes, including byte-for-byte. func TestNoPrefixMeansOneBlockAsBefore(t *testing.T) { a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{}) blocks := a.systemBlocks() if len(blocks) != 1 { t.Fatalf("got %d system blocks, want 1", len(blocks)) } if blocks[0].Text != defaultSystem { t.Error("the prompt must be unchanged when no prefix is set") } } // Effort is a knob the TUI turns mid-session, so a request carries whatever // was last set rather than whatever the process started with. func TestSetEffortAppliesToTheNextRequest(t *testing.T) { f := newFakeAPI(t, msg("end_turn", textBlock("one")), msg("end_turn", textBlock("two"))) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) if _, err := a.Run(context.Background(), "first"); err != nil { t.Fatalf("Run: %v", err) } if oc := f.requests[0].OutputConfig; oc != nil { t.Errorf("output_config = %+v with no effort set, want it omitted so the API picks", oc) } a.SetEffort("low") if got := a.Effort(); got != "low" { t.Errorf("Effort() = %q, want low", got) } if _, err := a.Run(context.Background(), "second"); err != nil { t.Fatalf("Run: %v", err) } oc := f.requests[1].OutputConfig if oc == nil || oc.Effort != "low" { t.Errorf("output_config = %+v, want effort low", oc) } } // The agent chooses which model each subagent runs on, so it has to be told // what this endpoint actually serves — and which of them is the newest, which // is the one to reach for unless there is a reason not to. func TestModelListingNamesWhatTheEndpointServes(t *testing.T) { models := []llm.Model{ {ID: "claude-sonnet-5", DisplayName: "Claude Sonnet 5", CreatedAt: time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)}, {ID: "claude-fable-5-1", DisplayName: "Claude Fable 5.1", CreatedAt: time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)}, {ID: "claude-haiku-4-5", DisplayName: "Claude Haiku 4.5", CreatedAt: time.Date(2025, 10, 1, 0, 0, 0, 0, time.UTC)}, } got := modelListing(models) for _, m := range models { if !strings.Contains(got, m.ID) { t.Errorf("the listing never names %q:\n%s", m.ID, got) } } // Newest first, so "the newest" is not a claim the agent has to work out. fable, haiku := strings.Index(got, "claude-fable-5-1"), strings.Index(got, "claude-haiku-4-5") if fable > haiku { t.Errorf("the listing is not newest first:\n%s", got) } for _, want := range []string{"newest", "this session is currently using", "the user asked"} { if !strings.Contains(got, want) { t.Errorf("the listing does not say %q:\n%s", want, got) } } } // Nothing to list — an endpoint that does not answer GET /v1/models — must add // nothing at all rather than a paragraph about an empty set. func TestModelListingIsEmptyWithoutModels(t *testing.T) { if got := modelListing(nil); got != "" { t.Errorf("modelListing(nil) = %q", got) } } // The listing is part of the cached prefix, so it must be identical between // two requests in one session. func TestModelListingIsStableAcrossRequests(t *testing.T) { models := []llm.Model{{ID: "claude-opus-5"}, {ID: "claude-haiku-4-5"}} a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{Models: models}) a.state.Append(llm.Message{Role: llm.RoleUser, Content: []llm.ContentBlock{llm.TextBlock("hi")}}) if first, second := a.request().System[0].Text, a.request().System[0].Text; first != second { t.Error("the model listing changed between two requests in one session") } if !strings.Contains(a.request().System[0].Text, "claude-haiku-4-5") { t.Error("the listing never reached the system prompt") } } // SetTools is called by the front end when /worktree moves the work, which can // land while a turn is reading the registry. Both sides go through the mutex; // this is the test that says so. func TestSetToolsIsSafeAgainstAReadingTurn(t *testing.T) { a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{}) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() for i := 0; i < 200; i++ { a.SetTools(tools.NewRegistry(nil)) } }() go func() { defer wg.Done() for i := 0; i < 200; i++ { _ = a.registry().List() } }() wg.Wait() // A nil set is refused rather than leaving the agent with no tools at all. before := a.registry() a.SetTools(nil) if a.registry() != before { t.Error("SetTools(nil) replaced the tool set") } } func TestToolProgressReachesTheObserver(t *testing.T) { f := newFakeAPI(t, msg("tool_use", toolUseBlock("call-1", "noisy", `{}`)), msg("end_turn", textBlock("done")), ) var logBuf bytes.Buffer log := metrics.NewWriter(&logBuf, "test") state := NewState() reg := tools.NewRegistry(log).Register( tools.New("noisy", "Reports as it goes", func(ctx context.Context, _ struct{}) (string, error) { if p := tools.ProgressFrom(ctx); p != nil { p("half ") p("way") } return "all of it", nil }), ) client := llm.New(llm.WithBaseURL(f.srv.URL), llm.WithAPIKey("k")) a := New(client, reg, state, log, Options{}) var mu sync.Mutex var deltas []Event a.Observe = func(ev Event) { if ev.Kind == EventToolDelta { mu.Lock() deltas = append(deltas, ev) mu.Unlock() } } if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } mu.Lock() defer mu.Unlock() if len(deltas) != 2 { t.Fatalf("got %d deltas, want 2", len(deltas)) } var text strings.Builder for _, d := range deltas { if d.Tool != "noisy" || d.ID != "call-1" { t.Errorf("a delta could not be matched to its call: %+v", d) } text.WriteString(d.Text) } if text.String() != "half way" { t.Errorf("deltas reassembled to %q", text.String()) } } func TestNoObserverMeansNoProgressSink(t *testing.T) { // A tool asks whether anyone is watching before it does the extra work of // streaming. With no observer the answer must be no, so the eval runner // and the -p path stay on exactly the code they were on before. f := newFakeAPI(t, msg("tool_use", toolUseBlock("call-1", "checker", `{}`)), msg("end_turn", textBlock("done")), ) var logBuf bytes.Buffer log := metrics.NewWriter(&logBuf, "test") state := NewState() watched := true reg := tools.NewRegistry(log).Register( tools.New("checker", "Reports whether anyone is watching", func(ctx context.Context, _ struct{}) (string, error) { watched = tools.ProgressFrom(ctx) != nil return "ok", nil }), ) a := New(llm.New(llm.WithBaseURL(f.srv.URL), llm.WithAPIKey("k")), reg, state, log, Options{}) if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } if watched { t.Error("a sink was installed with no observer to receive it") } } func thinkingBlock(s string) string { return `{"type":"thinking","thinking":"` + s + `","signature":"sig"}` } func TestReasoningReachesTheObserver(t *testing.T) { f := newFakeAPI(t, msg("end_turn", thinkingBlock("the parser is the suspect"), textBlock("it is the parser"))) var logBuf bytes.Buffer a := newTestAgent(t, f, t.TempDir(), &logBuf) var thoughts, text strings.Builder a.Observe = func(ev Event) { switch ev.Kind { case EventThinking: thoughts.WriteString(ev.Text) case EventText: text.WriteString(ev.Text) } } if _, err := a.Run(context.Background(), "why does it fail?"); err != nil { t.Fatalf("Run: %v", err) } if got := thoughts.String(); got != "the parser is the suspect" { t.Errorf("thinking = %q", got) } if got := text.String(); got != "it is the parser" { t.Errorf("text = %q", got) } } func TestRedactedReasoningIsNotReported(t *testing.T) { // Encrypted bytes that go back to the API untouched. There is nothing in // them to show, and an empty thought chunk is a blank row in a UI. f := newFakeAPI(t, msg("end_turn", `{"type":"redacted_thinking","data":"AAAA"}`, textBlock("done"))) var logBuf bytes.Buffer a := newTestAgent(t, f, t.TempDir(), &logBuf) var thinking int a.Observe = func(ev Event) { if ev.Kind == EventThinking { thinking++ } } if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } if thinking != 0 { t.Errorf("%d thinking events for a redacted block", thinking) } // And it must still be in the history: the API rejects a follow-up request // whose redacted block was dropped or altered. var kept bool for _, m := range a.State().History() { for _, b := range m.Content { if b.Type == llm.BlockRedactedThinking { kept = true } } } if !kept { t.Error("the redacted block was dropped from the conversation") } }