package agent
import (
"bytes"
"context"
"encoding/json"
"errors"
"image"
"image/png"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/johannSo/codey/internal/llm"
"github.com/johannSo/codey/internal/metrics"
"github.com/johannSo/codey/internal/skills"
"github.com/johannSo/codey/internal/tools"
)
// fakeAPI serves a scripted sequence of responses and records every request
// body it received, so tests can assert on what the loop actually sent.
type fakeAPI struct {
t *testing.T
responses []string
requests []llm.Request
summary string
srv *httptest.Server
}
// compactions counts requests that were summarization calls, so they do not
// consume scripted responses.
func (f *fakeAPI) compactions() int {
var n int
for _, req := range f.requests {
if len(req.Messages) > 0 {
last := req.Messages[len(req.Messages)-1]
for _, b := range last.Content {
if strings.Contains(b.Text, "Summarize the conversation above") {
n++
}
}
}
}
return n
}
func newFakeAPI(t *testing.T, responses ...string) *fakeAPI {
f := &fakeAPI{t: t, responses: responses, summary: "Compacted: edited loop.go, the config loads from X not Y."}
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
var req llm.Request
if err := json.Unmarshal(body, &req); err != nil {
t.Errorf("codey sent a body the API types cannot parse: %v\n%s", err, body)
}
f.requests = append(f.requests, req)
if bytes.Contains(body, []byte("Summarize the conversation above")) {
io.WriteString(w, msg("end_turn", textBlock(f.summary)))
return
}
i := len(f.requests) - 1 - f.compactions()
if i >= len(f.responses) {
t.Errorf("codey made %d requests, script only has %d", i+1, len(f.responses))
w.WriteHeader(http.StatusInternalServerError)
return
}
io.WriteString(w, f.responses[i])
}))
t.Cleanup(f.srv.Close)
return f
}
// rawRequest returns the n-th request body re-encoded, for byte-level assertions.
func (f *fakeAPI) rawRequest(n int) string {
b, err := json.Marshal(f.requests[n])
if err != nil {
f.t.Fatal(err)
}
return string(b)
}
func msg(stopReason string, blocks ...string) string {
return `{"id":"msg","type":"message","role":"assistant","model":"claude-opus-5","content":[` +
strings.Join(blocks, ",") + `],"stop_reason":"` + stopReason + `",` +
`"usage":{"input_tokens":100,"output_tokens":20,"cache_read_input_tokens":40}}`
}
func textBlock(s string) string { return `{"type":"text","text":"` + s + `"}` }
// msgWithTokens is msg with a controllable input_tokens, for driving compaction.
func msgWithTokens(stopReason string, inputTokens int, blocks ...string) string {
return `{"id":"msg","type":"message","role":"assistant","model":"claude-opus-5","content":[` +
strings.Join(blocks, ",") + `],"stop_reason":"` + stopReason + `",` +
`"usage":{"input_tokens":` + strconv.Itoa(inputTokens) + `,"output_tokens":20}}`
}
func toolUseBlock(id, name, input string) string {
return `{"type":"tool_use","id":"` + id + `","name":"` + name + `","input":` + input + `}`
}
// newTestAgent wires an agent against the fake API, with read_file rooted at
// a temp dir plus a couple of tools that make failure modes easy to trigger.
func newTestAgent(t *testing.T, f *fakeAPI, root string, logBuf *bytes.Buffer) *Agent {
t.Helper()
return newTestAgentOnModel(t, f, root, logBuf, llm.DefaultModel)
}
// newTestAgentOnModel is newTestAgent pinned to a model, for the parts of a
// request whose shape depends on which model it is going to.
func newTestAgentOnModel(t *testing.T, f *fakeAPI, root string, logBuf *bytes.Buffer, model string) *Agent {
t.Helper()
log := metrics.NewWriter(logBuf, "test")
state := NewState()
reg := tools.NewRegistry(log).Register(
tools.NewRead(root, state),
tools.NewEdit(root, state),
tools.NewWrite(root, state),
tools.New("boom", "Always fails", func(context.Context, struct{}) (string, error) {
return "", errors.New("detonated")
}),
)
client := llm.New(llm.WithBaseURL(f.srv.URL), llm.WithAPIKey("k"), llm.WithModel(model))
return New(client, reg, state, log, Options{})
}
func TestRunReadsAFileAndAnswers(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "main.go"), []byte("package main\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use", textBlock("Let me look."), toolUseBlock("tu_1", "read_file", `{"path":"main.go"}`)),
msg("end_turn", textBlock("It declares package main.")),
)
a := newTestAgent(t, f, root, &bytes.Buffer{})
got, err := a.Run(context.Background(), "what does main.go do?")
if err != nil {
t.Fatalf("Run: %v", err)
}
if got != "It declares package main." {
t.Errorf("answer = %q", got)
}
if len(f.requests) != 2 {
t.Fatalf("made %d requests, want 2", len(f.requests))
}
// Second request: user prompt, assistant turn, then the tool results.
second := f.requests[1].Messages
if len(second) != 3 {
t.Fatalf("second request had %d messages, want 3:\n%s", len(second), f.rawRequest(1))
}
if second[2].Role != llm.RoleUser {
t.Errorf("tool results must come back under the user role, got %q", second[2].Role)
}
if n := len(second[2].Content); n != 1 {
t.Fatalf("want 1 tool_result block, got %d", n)
}
res := second[2].Content[0]
if res.Type != llm.BlockToolResult || res.ToolUseID != "tu_1" || res.IsError {
t.Errorf("tool_result = %+v, want a successful result for tu_1", res)
}
if !strings.Contains(res.Content, "package main") {
t.Errorf("tool_result content = %q, want the file contents", res.Content)
}
}
// A screenshot read with read_file reaches the model as an image block inside
// the tool_result, and the transcript sees the caption.
func TestImageReadReachesTheModelAsAnImage(t *testing.T) {
root := t.TempDir()
// The smallest valid PNG: 1×1, from image/png's own encoder.
var buf bytes.Buffer
png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 1, 1)))
os.WriteFile(filepath.Join(root, "shot.png"), buf.Bytes(), 0o644)
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"shot.png"}`)),
msg("end_turn", textBlock("looks fine")),
)
a := newTestAgent(t, f, root, &bytes.Buffer{})
var seen []Event
a.Observe = func(e Event) {
if e.Kind == EventToolResult {
seen = append(seen, e)
}
}
if _, err := a.Run(context.Background(), "look at the screenshot"); err != nil {
t.Fatalf("Run: %v", err)
}
raw := f.rawRequest(1)
if !strings.Contains(raw, `"type":"image"`) || !strings.Contains(raw, `"media_type":"image/png"`) {
t.Errorf("second request carries no image block:\n%s", raw)
}
res := f.requests[1].Messages[2].Content[0]
if len(res.Blocks) != 2 || res.Blocks[1].Type != llm.BlockImage {
t.Errorf("tool_result blocks = %+v, want caption and image", res.Blocks)
}
if len(seen) != 1 || !strings.Contains(seen[0].Result, "PNG image, 1×1") {
t.Errorf("the transcript should see the caption, got %+v", seen)
}
}
// Superseding a read of an image drops the picture too: the model must not
// keep reasoning from a screenshot of a file that has since changed.
func TestSupersedeDropsImageBlocks(t *testing.T) {
root := t.TempDir()
var buf bytes.Buffer
png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 1, 1)))
os.WriteFile(filepath.Join(root, "shot.png"), buf.Bytes(), 0o644)
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"shot.png"}`)),
msg("tool_use", toolUseBlock("tu_2", "read_file", `{"path":"shot.png"}`)),
msg("tool_use", toolUseBlock("tu_3", "write_file", `{"path":"shot.png","content":"not a png any more"}`)),
msg("end_turn", textBlock("done")),
)
a := newTestAgent(t, f, root, &bytes.Buffer{})
if _, err := a.Run(context.Background(), "redo the screenshot"); err != nil {
t.Fatalf("Run: %v", err)
}
var images, superseded int
for _, m := range a.State().History() {
for _, b := range m.Content {
if b.Type != llm.BlockToolResult {
continue
}
if strings.HasPrefix(b.Content, "[superseded:") {
superseded++
if len(b.Blocks) != 0 {
t.Errorf("a superseded result still carries %d blocks", len(b.Blocks))
}
}
for _, inner := range b.Blocks {
if inner.Type == llm.BlockImage {
images++
}
}
}
}
if superseded != 1 || images != 1 {
t.Errorf("superseded=%d images=%d, want 1 and 1 (the later read keeps its picture)", superseded, images)
}
}
func TestParallelToolCallsReturnInOneUserMessage(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "a.go"), []byte("A\n"), 0o644)
os.WriteFile(filepath.Join(root, "b.go"), []byte("B\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use",
toolUseBlock("tu_1", "read_file", `{"path":"a.go"}`),
toolUseBlock("tu_2", "read_file", `{"path":"b.go"}`)),
msg("end_turn", textBlock("done")),
)
a := newTestAgent(t, f, root, &bytes.Buffer{})
if _, err := a.Run(context.Background(), "read both"); err != nil {
t.Fatalf("Run: %v", err)
}
last := f.requests[1].Messages[len(f.requests[1].Messages)-1]
if len(last.Content) != 2 {
t.Fatalf("want both results in ONE message, got %d messages ending with %d blocks",
len(f.requests[1].Messages), len(last.Content))
}
if last.Content[0].ToolUseID != "tu_1" || last.Content[1].ToolUseID != "tu_2" {
t.Errorf("results out of order: %s, %s", last.Content[0].ToolUseID, last.Content[1].ToolUseID)
}
}
func TestFailedToolStillProducesAResult(t *testing.T) {
tests := []struct {
name string
call string
wantText string
}{
{name: "tool returns an error", call: toolUseBlock("tu_1", "boom", `{}`), wantText: "detonated"},
{name: "unknown tool", call: toolUseBlock("tu_1", "telepathy", `{}`), wantText: `unknown tool "telepathy"`},
{name: "malformed arguments", call: toolUseBlock("tu_1", "read_file", `{"path":42}`), wantText: "invalid arguments"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
f := newFakeAPI(t, msg("tool_use", tc.call), msg("end_turn", textBlock("ok")))
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("a failing tool is a normal event, not a Run error: %v", err)
}
last := f.requests[1].Messages[len(f.requests[1].Messages)-1]
if len(last.Content) != 1 {
t.Fatalf("every tool_use id needs exactly one result, got %d", len(last.Content))
}
res := last.Content[0]
if !res.IsError {
t.Error("is_error must be set so the model knows the call failed")
}
if !strings.Contains(res.Content, tc.wantText) {
t.Errorf("result = %q, want it to contain %q", res.Content, tc.wantText)
}
})
}
}
// Thinking signatures must be echoed back byte-identical or the next request
// is rejected.
func TestThinkingBlocksAreEchoedBackUnchanged(t *testing.T) {
f := newFakeAPI(t,
msg("tool_use",
`{"type":"thinking","thinking":"","signature":"sig-xyz"}`,
toolUseBlock("tu_1", "boom", `{}`)),
msg("end_turn", textBlock("fine")),
)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
body := f.rawRequest(1)
if !strings.Contains(body, `"signature":"sig-xyz"`) {
t.Errorf("thinking signature was dropped from the resent history:\n%s", body)
}
if !strings.Contains(body, `"thinking":""`) {
t.Errorf("empty thinking text was dropped:\n%s", body)
}
}
func TestRequestShape(t *testing.T) {
f := newFakeAPI(t, msg("end_turn", textBlock("hi")))
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
if _, err := a.Run(context.Background(), "hello"); err != nil {
t.Fatalf("Run: %v", err)
}
req := f.requests[0]
if len(req.System) != 1 || req.System[0].Text == "" {
t.Error("system prompt missing")
}
if strings.ContainsAny(req.System[0].Text, "0123456789") {
t.Error("system prompt should hold no digits: timestamps or ids here would break the cached prefix")
}
// The agent runs on llm.DefaultModel, which takes adaptive thinking: a
// model that supports it must still be asked for it.
if req.Thinking == nil || req.Thinking.Type != "adaptive" {
t.Errorf("thinking = %+v, want adaptive (budget_tokens is a 400 on current models)", req.Thinking)
}
var names []string
for _, tl := range req.Tools {
names = append(names, tl.Name)
}
if want := "boom, edit_file, read_file, write_file"; strings.Join(names, ", ") != want {
t.Errorf("tools = %v, want them sorted for a byte-stable prefix (%s)", names, want)
}
for _, tl := range req.Tools {
if len(tl.InputSchema) == 0 {
t.Errorf("tool %q has no input_schema", tl.Name)
}
}
}
// A cap only applies when one was asked for, and what the model had managed to
// say by then comes back with the error rather than being thrown away.
func TestMaxTurnsStopsTheLoopWhenOneWasAskedFor(t *testing.T) {
var script []string
for i := 0; i < 5; i++ {
script = append(script,
msg("tool_use", textBlock("still working on it"), toolUseBlock("tu", "boom", `{}`)))
}
f := newFakeAPI(t, script...)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
a.opts.MaxTurns = 3
text, err := a.Run(context.Background(), "loop forever")
if !errors.Is(err, ErrMaxTurns) {
t.Fatalf("error = %v, want ErrMaxTurns", err)
}
if !strings.Contains(err.Error(), "3 turns") {
t.Errorf("error should say how many turns were spent: %v", err)
}
if text != "still working on it" {
t.Errorf("text = %q, want what the model had got to by the cap", text)
}
if len(f.requests) != 3 {
t.Errorf("made %d requests, want exactly the cap of 3", len(f.requests))
}
}
// Nothing caps a session that did not ask to be capped. Thirty turns is a
// short afternoon's work, and a run that stops there has thrown away a task
// that was going fine; what bounds a session is the window, the bill and the
// person who can interrupt it.
func TestNoTurnLimitByDefault(t *testing.T) {
const turns = 80
var script []string
for i := 0; i < turns; i++ {
script = append(script, msg("tool_use", toolUseBlock("tu", "boom", `{}`)))
}
script = append(script, msg("end_turn", textBlock("finally done")))
f := newFakeAPI(t, script...)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
if a.opts.MaxTurns > NoTurnLimit {
t.Fatalf("MaxTurns = %d, want a session with no cap of its own", a.opts.MaxTurns)
}
text, err := a.Run(context.Background(), "a long job")
if err != nil {
t.Fatalf("Run: %v", err)
}
if text != "finally done" {
t.Errorf("text = %q, want the answer after %d turns", text, turns)
}
if len(f.requests) != turns+1 {
t.Errorf("made %d requests, want all %d", len(f.requests), turns+1)
}
}
func TestRefusalIsReportedWithItsCategory(t *testing.T) {
f := newFakeAPI(t, `{"id":"m","type":"message","role":"assistant","model":"claude-opus-5",
"content":[],"stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber","explanation":"nope"},
"usage":{"input_tokens":5,"output_tokens":0}}`)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
_, err := a.Run(context.Background(), "do something")
if err == nil {
t.Fatal("want an error")
}
if !strings.Contains(err.Error(), "cyber") || !strings.Contains(err.Error(), "nope") {
t.Errorf("error = %v, want the category and explanation", err)
}
}
func TestPauseTurnResends(t *testing.T) {
f := newFakeAPI(t,
msg("pause_turn", textBlock("thinking")),
msg("end_turn", textBlock("done")),
)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
got, err := a.Run(context.Background(), "go")
if err != nil {
t.Fatalf("Run: %v", err)
}
if got != "done" || len(f.requests) != 2 {
t.Errorf("got %q after %d requests, want the loop to resend after pause_turn", got, len(f.requests))
}
}
func serverToolUseBlock(id, query string) string {
return `{"type":"server_tool_use","id":"` + id + `","name":"web_search","input":{"query":"` + query + `"}}`
}
func webSearchResultBlock(toolUseID, content string) string {
return `{"type":"web_search_tool_result","tool_use_id":"` + toolUseID + `","content":` + content + `}`
}
// A hosted search is run by the API, which also supplies its result. The loop
// has two jobs: show it to the user, who otherwise sees nothing while the
// model searches, and keep its hands off — no registry call, no tool_result.
func TestHostedWebSearchIsVisibleButNotDispatched(t *testing.T) {
f := newFakeAPI(t,
msg("pause_turn",
textBlock("Let me check."),
serverToolUseBlock("srvtoolu_1", "go 1.23 release notes"),
webSearchResultBlock("srvtoolu_1", `[{"type":"web_search_result","title":"Go 1.23 Release Notes","url":"https://go.dev/doc/go1.23","encrypted_content":"x"},{"type":"web_search_result","title":"Go Blog","url":"https://go.dev/blog/go1.23","encrypted_content":"y"}]`),
),
msg("end_turn", textBlock("done")),
)
// The registry has no web_search: the hosted tool is not a local one.
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
a.opts.ServerTools = []llm.ToolDef{llm.WebSearchTool(5)}
var events []Event
a.Observe = func(ev Event) { events = append(events, ev) }
got, err := a.Run(context.Background(), "what's new in go 1.23")
if err != nil {
t.Fatalf("Run: %v", err)
}
if got != "done" || len(f.requests) != 2 {
t.Fatalf("got %q after %d requests, want the pause_turn resend to finish", got, len(f.requests))
}
var start, result *Event
for i := range events {
switch events[i].Kind {
case EventToolStart:
start = &events[i]
case EventToolResult:
result = &events[i]
}
}
if start == nil || start.Tool != llm.WebSearchToolName || !strings.Contains(string(start.Input), "go 1.23 release notes") {
t.Errorf("tool_start = %+v, want the hosted search with its query", start)
}
if result == nil || result.Tool != llm.WebSearchToolName || result.IsError {
t.Fatalf("tool_result = %+v, want a successful web_search result", result)
}
if want := "Go 1.23 Release Notes — https://go.dev/doc/go1.23\nGo Blog — https://go.dev/blog/go1.23"; result.Result != want {
t.Errorf("result summary = %q, want %q", result.Result, want)
}
if strings.Contains(result.Result, "encrypted_content") {
t.Error("the summary must not dump the encrypted payload on the user")
}
// The resend carries the assistant turn unchanged and nothing else: the
// API supplied the web_search_tool_result, so a local tool_result for the
// hosted call would be an invariant violation, not a courtesy.
resend := f.requests[1]
if len(resend.Messages) != 2 {
t.Fatalf("resend has %d messages, want user + assistant only", len(resend.Messages))
}
for _, m := range resend.Messages {
for _, b := range m.Content {
if b.Type == llm.BlockToolResult {
t.Errorf("a tool_result was sent for the hosted call: %+v", b)
}
}
}
raw := f.rawRequest(1)
if !strings.Contains(raw, `"query":"go 1.23 release notes"`) {
t.Errorf("resend lost the server_tool_use query:\n%s", raw)
}
if !strings.Contains(raw, `"web_search_tool_result"`) || !strings.Contains(raw, `"encrypted_content":"x"`) {
t.Errorf("resend must echo the web_search_tool_result verbatim:\n%s", raw)
}
last := resend.Tools[len(resend.Tools)-1]
if last.Type != llm.WebSearchToolType || last.Name != llm.WebSearchToolName || last.MaxUses != 5 {
t.Errorf("last tool = %+v, want the hosted web search definition after the local tools", last)
}
}
// A failed hosted search is reported as such, and the loop still gets on with
// the resend rather than treating an API-side error as a local one.
func TestHostedWebSearchErrorIsReported(t *testing.T) {
f := newFakeAPI(t,
msg("pause_turn",
serverToolUseBlock("srvtoolu_1", "anything"),
webSearchResultBlock("srvtoolu_1", `{"type":"web_search_tool_result_error","error_code":"max_uses_exceeded"}`),
),
msg("end_turn", textBlock("done")),
)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
var result *Event
a.Observe = func(ev Event) {
if ev.Kind == EventToolResult {
result = &ev
}
}
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
if result == nil || !result.IsError || !strings.Contains(result.Result, "max_uses_exceeded") {
t.Errorf("tool_result = %+v, want an error naming max_uses_exceeded", result)
}
}
// The hosted definitions sit after the sorted local ones: the local list is
// sorted for a byte-stable cached prefix, and appending a constant tail to a
// stable list keeps it stable.
func TestServerToolsFollowTheLocalTools(t *testing.T) {
f := newFakeAPI(t, msg("end_turn", textBlock("hi")))
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
a.opts.ServerTools = []llm.ToolDef{llm.WebSearchTool(3)}
if _, err := a.Run(context.Background(), "hello"); err != nil {
t.Fatalf("Run: %v", err)
}
var names []string
for _, tl := range f.requests[0].Tools {
names = append(names, tl.Name)
}
if want := "boom, edit_file, read_file, write_file, web_search"; strings.Join(names, ", ") != want {
t.Errorf("tools = %v, want %s", names, want)
}
hosted := f.requests[0].Tools[len(names)-1]
if hosted.Type != llm.WebSearchToolType || len(hosted.InputSchema) != 0 || hosted.Description != "" {
t.Errorf("hosted tool = %+v, want type set and no local schema or description", hosted)
}
}
func TestSummarizeWebSearchResult(t *testing.T) {
tests := []struct {
name string
block string
want string
wantErr bool
}{
{
name: "one line per result",
block: webSearchResultBlock("s", `[{"type":"web_search_result","title":"A","url":"https://a"},{"type":"web_search_result","title":"B","url":"https://b"}]`),
want: "A — https://a\nB — https://b",
},
{
name: "missing title falls back to the url alone",
block: webSearchResultBlock("s", `[{"type":"web_search_result","url":"https://a"}]`),
want: "https://a",
},
{
name: "no results says so rather than going silent",
block: webSearchResultBlock("s", `[]`),
want: "(no results)",
},
{
name: "error object",
block: webSearchResultBlock("s", `{"type":"web_search_tool_result_error","error_code":"too_many_requests"}`),
want: "web search failed: too_many_requests",
wantErr: true,
},
{
name: "unparseable content is reported, not hidden",
block: webSearchResultBlock("s", `"what"`),
want: "web search returned an unrecognised result",
wantErr: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var b llm.ContentBlock
if err := json.Unmarshal([]byte(tc.block), &b); err != nil {
t.Fatalf("unmarshal: %v", err)
}
got, isErr := summarizeWebSearchResult(b)
if got != tc.want || isErr != tc.wantErr {
t.Errorf("got (%q, %v), want (%q, %v)", got, isErr, tc.want, tc.wantErr)
}
})
}
}
func TestUsageAndEventsAreRecorded(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "x.go"), []byte("x\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"x.go"}`)),
msg("end_turn", textBlock("done")),
)
var logBuf bytes.Buffer
a := newTestAgent(t, f, root, &logBuf)
var kinds []EventKind
a.Observe = func(ev Event) { kinds = append(kinds, ev.Kind) }
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
usage, turns := a.State().Usage()
if turns != 2 || usage.InputTokens != 200 || usage.CacheReadInputTokens != 80 {
t.Errorf("usage = %+v over %d turns, want both turns accumulated", usage, turns)
}
if !containsKind(kinds, EventToolStart) || !containsKind(kinds, EventToolResult) || !containsKind(kinds, EventText) {
t.Errorf("observer saw %v, want tool and text events", kinds)
}
log := logBuf.String()
if n := strings.Count(log, `"type":"turn"`); n != 2 {
t.Errorf("logged %d turn events, want 2:\n%s", n, log)
}
if n := strings.Count(log, `"type":"tool_call"`); n != 1 {
t.Errorf("logged %d tool_call events, want 1:\n%s", n, log)
}
if !strings.Contains(log, `"cache_read_input_tokens":40`) {
t.Errorf("cache reads must be logged; that is the Phase 4 lever:\n%s", log)
}
}
func TestReadFileMarksStateSoEditCanCheckLater(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "tracked.go")
os.WriteFile(path, []byte("x\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"tracked.go"}`)),
msg("end_turn", textBlock("done")),
)
a := newTestAgent(t, f, root, &bytes.Buffer{})
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
if _, ok := a.State().LastRead(path); !ok {
t.Error("read_file must record the read in State for Phase 2's edit_file")
}
}
func TestRunPropagatesAPIErrors(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
io.WriteString(w, `{"error":{"type":"invalid_request_error","message":"bad model"}}`)
}))
defer srv.Close()
state := NewState()
a := New(llm.New(llm.WithBaseURL(srv.URL), llm.WithAPIKey("k")),
tools.NewRegistry(nil), state, nil, Options{})
if _, err := a.Run(context.Background(), "go"); err == nil || !strings.Contains(err.Error(), "bad model") {
t.Errorf("error = %v, want the API message surfaced", err)
}
}
func containsKind(kinds []EventKind, want EventKind) bool {
for _, k := range kinds {
if k == want {
return true
}
}
return false
}
// streamingAPI serves one scripted SSE response.
func streamingAPI(t *testing.T, events ...string) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for _, e := range events {
io.WriteString(w, "event: x\ndata: "+e+"\n\n")
}
}))
t.Cleanup(srv.Close)
return srv
}
func TestStreamingRunEmitsDeltas(t *testing.T) {
srv := streamingAPI(t,
`{"type":"message_start","message":{"id":"m","model":"claude-opus-5","usage":{"input_tokens":50}}}`,
`{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
`{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"All "}}`,
`{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"done."}}`,
`{"type":"content_block_stop","index":0}`,
`{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}`,
)
state := NewState()
a := New(llm.New(llm.WithBaseURL(srv.URL), llm.WithAPIKey("k")),
tools.NewRegistry(nil), state, nil, Options{Stream: true})
var deltas []string
var finalText int
a.Observe = func(ev Event) {
switch ev.Kind {
case EventTextDelta:
deltas = append(deltas, ev.Text)
case EventText:
finalText++
}
}
got, err := a.Run(context.Background(), "go")
if err != nil {
t.Fatalf("Run: %v", err)
}
if got != "All done." {
t.Errorf("answer = %q", got)
}
if strings.Join(deltas, "") != "All done." {
t.Errorf("deltas = %v", deltas)
}
if finalText != 0 {
t.Error("the whole text must not be emitted again after streaming it")
}
}
// denyAll is an Approver that refuses everything it is asked about.
type denyAll struct{ asked []string }
func (d *denyAll) Approve(_ context.Context, tool string, _ json.RawMessage) error {
d.asked = append(d.asked, tool)
return errors.New("the user declined to run " + tool)
}
func TestDeniedToolBecomesAToolError(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "a.go"), []byte("package a\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"a.go"}`)),
msg("end_turn", textBlock("understood")),
)
var logBuf bytes.Buffer
a := newTestAgent(t, f, root, &logBuf)
gate := &denyAll{}
a.Approve = gate
got, err := a.Run(context.Background(), "read it")
if err != nil {
t.Fatalf("a denial is not a failure of ours: %v", err)
}
if got != "understood" {
t.Errorf("answer = %q", got)
}
if len(gate.asked) != 1 || gate.asked[0] != "read_file" {
t.Errorf("approver saw %v", gate.asked)
}
last := f.requests[1].Messages[len(f.requests[1].Messages)-1]
res := last.Content[0]
if !res.IsError || !strings.Contains(res.Content, "declined") {
t.Errorf("tool_result = %+v, want the denial passed to the model", res)
}
if !strings.Contains(logBuf.String(), `"type":"tool_denied"`) {
t.Error("denials should be logged: they explain gaps in the eval numbers")
}
// The tool must not have run.
if strings.Contains(res.Content, "package a") {
t.Error("the file was read despite the denial")
}
}
func TestSystemPromptListsSkills(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{
Skills: []skills.Skill{{Name: "commit-helper", Description: "Use when writing commit messages"}},
})
sys := a.opts.System
if !strings.Contains(sys, "commit-helper: Use when writing commit messages") {
t.Errorf("system prompt does not list the skill:\n%s", sys)
}
if !strings.Contains(sys, "/<name>") {
t.Error("system prompt does not explain what a leading slash means")
}
if !strings.HasPrefix(sys, defaultSystem) {
t.Error("the listing replaced the default prompt instead of being appended to it")
}
}
func TestSystemPromptUnchangedWithoutSkills(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{})
if a.opts.System != defaultSystem {
t.Errorf("system prompt differs from the default with no skills:\n%s", a.opts.System)
}
}
func TestSystemPromptIsStableAcrossRequests(t *testing.T) {
// The listing sits in the cached prefix. If it varied per request, every
// turn would miss the cache.
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{
Skills: []skills.Skill{{Name: "a", Description: "one"}, {Name: "b", Description: "two"}},
})
a.state.Append(llm.Message{Role: llm.RoleUser, Content: []llm.ContentBlock{llm.TextBlock("hi")}})
first := a.request().System[0].Text
second := a.request().System[0].Text
if first != second {
t.Error("the system prompt changed between two requests in one session")
}
}
func TestSystemPrefixBecomesTheFirstBlock(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{SystemPrefix: ClaudeCodeIdentity})
blocks := a.systemBlocks()
if len(blocks) != 2 {
t.Fatalf("got %d system blocks, want 2", len(blocks))
}
if blocks[0].Text != ClaudeCodeIdentity {
t.Errorf("first block = %q, want the identity string", blocks[0].Text)
}
if !strings.Contains(blocks[1].Text, "coding agent") {
t.Errorf("second block should be Codey's own system prompt, got %q", blocks[1].Text)
}
}
// The cache breakpoint has to sit on the last block, or only the identity
// string gets cached and the prompt behind it is re-read every turn.
func TestCacheBreakpointIsOnTheLastSystemBlock(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{SystemPrefix: ClaudeCodeIdentity})
blocks := a.systemBlocks()
if blocks[0].CacheControl != nil {
t.Error("the first block must not carry the breakpoint")
}
if blocks[len(blocks)-1].CacheControl == nil {
t.Error("the last block must carry the breakpoint")
}
}
// Without OAuth nothing changes, including byte-for-byte.
func TestNoPrefixMeansOneBlockAsBefore(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{})
blocks := a.systemBlocks()
if len(blocks) != 1 {
t.Fatalf("got %d system blocks, want 1", len(blocks))
}
if blocks[0].Text != defaultSystem {
t.Error("the prompt must be unchanged when no prefix is set")
}
}
// Effort is a knob the TUI turns mid-session, so a request carries whatever
// was last set rather than whatever the process started with.
func TestSetEffortAppliesToTheNextRequest(t *testing.T) {
f := newFakeAPI(t, msg("end_turn", textBlock("one")), msg("end_turn", textBlock("two")))
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
if _, err := a.Run(context.Background(), "first"); err != nil {
t.Fatalf("Run: %v", err)
}
if oc := f.requests[0].OutputConfig; oc != nil {
t.Errorf("output_config = %+v with no effort set, want it omitted so the API picks", oc)
}
a.SetEffort("low")
if got := a.Effort(); got != "low" {
t.Errorf("Effort() = %q, want low", got)
}
if _, err := a.Run(context.Background(), "second"); err != nil {
t.Fatalf("Run: %v", err)
}
oc := f.requests[1].OutputConfig
if oc == nil || oc.Effort != "low" {
t.Errorf("output_config = %+v, want effort low", oc)
}
}
// The agent chooses which model each subagent runs on, so it has to be told
// what this endpoint actually serves — and which of them is the newest, which
// is the one to reach for unless there is a reason not to.
func TestModelListingNamesWhatTheEndpointServes(t *testing.T) {
models := []llm.Model{
{ID: "claude-sonnet-5", DisplayName: "Claude Sonnet 5", CreatedAt: time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)},
{ID: "claude-fable-5-1", DisplayName: "Claude Fable 5.1", CreatedAt: time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)},
{ID: "claude-haiku-4-5", DisplayName: "Claude Haiku 4.5", CreatedAt: time.Date(2025, 10, 1, 0, 0, 0, 0, time.UTC)},
}
got := modelListing(models)
for _, m := range models {
if !strings.Contains(got, m.ID) {
t.Errorf("the listing never names %q:\n%s", m.ID, got)
}
}
// Newest first, so "the newest" is not a claim the agent has to work out.
fable, haiku := strings.Index(got, "claude-fable-5-1"), strings.Index(got, "claude-haiku-4-5")
if fable > haiku {
t.Errorf("the listing is not newest first:\n%s", got)
}
for _, want := range []string{"newest", "this session is currently using", "the user asked"} {
if !strings.Contains(got, want) {
t.Errorf("the listing does not say %q:\n%s", want, got)
}
}
}
// Nothing to list — an endpoint that does not answer GET /v1/models — must add
// nothing at all rather than a paragraph about an empty set.
func TestModelListingIsEmptyWithoutModels(t *testing.T) {
if got := modelListing(nil); got != "" {
t.Errorf("modelListing(nil) = %q", got)
}
}
// The listing is part of the cached prefix, so it must be identical between
// two requests in one session.
func TestModelListingIsStableAcrossRequests(t *testing.T) {
models := []llm.Model{{ID: "claude-opus-5"}, {ID: "claude-haiku-4-5"}}
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{Models: models})
a.state.Append(llm.Message{Role: llm.RoleUser, Content: []llm.ContentBlock{llm.TextBlock("hi")}})
if first, second := a.request().System[0].Text, a.request().System[0].Text; first != second {
t.Error("the model listing changed between two requests in one session")
}
if !strings.Contains(a.request().System[0].Text, "claude-haiku-4-5") {
t.Error("the listing never reached the system prompt")
}
}
// SetTools is called by the front end when /worktree moves the work, which can
// land while a turn is reading the registry. Both sides go through the mutex;
// this is the test that says so.
func TestSetToolsIsSafeAgainstAReadingTurn(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{})
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for i := 0; i < 200; i++ {
a.SetTools(tools.NewRegistry(nil))
}
}()
go func() {
defer wg.Done()
for i := 0; i < 200; i++ {
_ = a.registry().List()
}
}()
wg.Wait()
// A nil set is refused rather than leaving the agent with no tools at all.
before := a.registry()
a.SetTools(nil)
if a.registry() != before {
t.Error("SetTools(nil) replaced the tool set")
}
}
func TestToolProgressReachesTheObserver(t *testing.T) {
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("call-1", "noisy", `{}`)),
msg("end_turn", textBlock("done")),
)
var logBuf bytes.Buffer
log := metrics.NewWriter(&logBuf, "test")
state := NewState()
reg := tools.NewRegistry(log).Register(
tools.New("noisy", "Reports as it goes", func(ctx context.Context, _ struct{}) (string, error) {
if p := tools.ProgressFrom(ctx); p != nil {
p("half ")
p("way")
}
return "all of it", nil
}),
)
client := llm.New(llm.WithBaseURL(f.srv.URL), llm.WithAPIKey("k"))
a := New(client, reg, state, log, Options{})
var mu sync.Mutex
var deltas []Event
a.Observe = func(ev Event) {
if ev.Kind == EventToolDelta {
mu.Lock()
deltas = append(deltas, ev)
mu.Unlock()
}
}
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
mu.Lock()
defer mu.Unlock()
if len(deltas) != 2 {
t.Fatalf("got %d deltas, want 2", len(deltas))
}
var text strings.Builder
for _, d := range deltas {
if d.Tool != "noisy" || d.ID != "call-1" {
t.Errorf("a delta could not be matched to its call: %+v", d)
}
text.WriteString(d.Text)
}
if text.String() != "half way" {
t.Errorf("deltas reassembled to %q", text.String())
}
}
func TestNoObserverMeansNoProgressSink(t *testing.T) {
// A tool asks whether anyone is watching before it does the extra work of
// streaming. With no observer the answer must be no, so the eval runner
// and the -p path stay on exactly the code they were on before.
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("call-1", "checker", `{}`)),
msg("end_turn", textBlock("done")),
)
var logBuf bytes.Buffer
log := metrics.NewWriter(&logBuf, "test")
state := NewState()
watched := true
reg := tools.NewRegistry(log).Register(
tools.New("checker", "Reports whether anyone is watching", func(ctx context.Context, _ struct{}) (string, error) {
watched = tools.ProgressFrom(ctx) != nil
return "ok", nil
}),
)
a := New(llm.New(llm.WithBaseURL(f.srv.URL), llm.WithAPIKey("k")), reg, state, log, Options{})
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
if watched {
t.Error("a sink was installed with no observer to receive it")
}
}
func thinkingBlock(s string) string {
return `{"type":"thinking","thinking":"` + s + `","signature":"sig"}`
}
func TestReasoningReachesTheObserver(t *testing.T) {
f := newFakeAPI(t, msg("end_turn", thinkingBlock("the parser is the suspect"), textBlock("it is the parser")))
var logBuf bytes.Buffer
a := newTestAgent(t, f, t.TempDir(), &logBuf)
var thoughts, text strings.Builder
a.Observe = func(ev Event) {
switch ev.Kind {
case EventThinking:
thoughts.WriteString(ev.Text)
case EventText:
text.WriteString(ev.Text)
}
}
if _, err := a.Run(context.Background(), "why does it fail?"); err != nil {
t.Fatalf("Run: %v", err)
}
if got := thoughts.String(); got != "the parser is the suspect" {
t.Errorf("thinking = %q", got)
}
if got := text.String(); got != "it is the parser" {
t.Errorf("text = %q", got)
}
}
func TestRedactedReasoningIsNotReported(t *testing.T) {
// Encrypted bytes that go back to the API untouched. There is nothing in
// them to show, and an empty thought chunk is a blank row in a UI.
f := newFakeAPI(t, msg("end_turn",
`{"type":"redacted_thinking","data":"AAAA"}`, textBlock("done")))
var logBuf bytes.Buffer
a := newTestAgent(t, f, t.TempDir(), &logBuf)
var thinking int
a.Observe = func(ev Event) {
if ev.Kind == EventThinking {
thinking++
}
}
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
if thinking != 0 {
t.Errorf("%d thinking events for a redacted block", thinking)
}
// And it must still be in the history: the API rejects a follow-up request
// whose redacted block was dropped or altered.
var kept bool
for _, m := range a.State().History() {
for _, b := range m.Content {
if b.Type == llm.BlockRedactedThinking {
kept = true
}
}
}
if !kept {
t.Error("the redacted block was dropped from the conversation")
}
}