package agent
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"harness/internal/llm"
"harness/internal/metrics"
"harness/internal/skills"
"harness/internal/tools"
)
// fakeAPI serves a scripted sequence of responses and records every request
// body it received, so tests can assert on what the loop actually sent.
type fakeAPI struct {
t *testing.T
responses []string
requests []llm.Request
summary string
srv *httptest.Server
}
// compactions counts requests that were summarization calls, so they do not
// consume scripted responses.
func (f *fakeAPI) compactions() int {
var n int
for _, req := range f.requests {
if len(req.Messages) > 0 {
last := req.Messages[len(req.Messages)-1]
for _, b := range last.Content {
if strings.Contains(b.Text, "Summarize the conversation above") {
n++
}
}
}
}
return n
}
func newFakeAPI(t *testing.T, responses ...string) *fakeAPI {
f := &fakeAPI{t: t, responses: responses, summary: "Compacted: edited loop.go, the config loads from X not Y."}
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
var req llm.Request
if err := json.Unmarshal(body, &req); err != nil {
t.Errorf("harness sent a body the API types cannot parse: %v\n%s", err, body)
}
f.requests = append(f.requests, req)
if bytes.Contains(body, []byte("Summarize the conversation above")) {
io.WriteString(w, msg("end_turn", textBlock(f.summary)))
return
}
i := len(f.requests) - 1 - f.compactions()
if i >= len(f.responses) {
t.Errorf("harness made %d requests, script only has %d", i+1, len(f.responses))
w.WriteHeader(http.StatusInternalServerError)
return
}
io.WriteString(w, f.responses[i])
}))
t.Cleanup(f.srv.Close)
return f
}
// rawRequest returns the n-th request body re-encoded, for byte-level assertions.
func (f *fakeAPI) rawRequest(n int) string {
b, err := json.Marshal(f.requests[n])
if err != nil {
f.t.Fatal(err)
}
return string(b)
}
func msg(stopReason string, blocks ...string) string {
return `{"id":"msg","type":"message","role":"assistant","model":"claude-opus-5","content":[` +
strings.Join(blocks, ",") + `],"stop_reason":"` + stopReason + `",` +
`"usage":{"input_tokens":100,"output_tokens":20,"cache_read_input_tokens":40}}`
}
func textBlock(s string) string { return `{"type":"text","text":"` + s + `"}` }
// msgWithTokens is msg with a controllable input_tokens, for driving compaction.
func msgWithTokens(stopReason string, inputTokens int, blocks ...string) string {
return `{"id":"msg","type":"message","role":"assistant","model":"claude-opus-5","content":[` +
strings.Join(blocks, ",") + `],"stop_reason":"` + stopReason + `",` +
`"usage":{"input_tokens":` + strconv.Itoa(inputTokens) + `,"output_tokens":20}}`
}
func toolUseBlock(id, name, input string) string {
return `{"type":"tool_use","id":"` + id + `","name":"` + name + `","input":` + input + `}`
}
// newTestAgent wires an agent against the fake API, with read_file rooted at
// a temp dir plus a couple of tools that make failure modes easy to trigger.
func newTestAgent(t *testing.T, f *fakeAPI, root string, logBuf *bytes.Buffer) *Agent {
t.Helper()
log := metrics.NewWriter(logBuf, "test")
state := NewState()
reg := tools.NewRegistry(log).Register(
tools.NewRead(root, state),
tools.NewEdit(root, state),
tools.NewWrite(root, state),
tools.New("boom", "Always fails", func(context.Context, struct{}) (string, error) {
return "", errors.New("detonated")
}),
)
client := llm.New(llm.WithBaseURL(f.srv.URL), llm.WithAPIKey("k"))
return New(client, reg, state, log, Options{})
}
func TestRunReadsAFileAndAnswers(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "main.go"), []byte("package main\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use", textBlock("Let me look."), toolUseBlock("tu_1", "read_file", `{"path":"main.go"}`)),
msg("end_turn", textBlock("It declares package main.")),
)
a := newTestAgent(t, f, root, &bytes.Buffer{})
got, err := a.Run(context.Background(), "what does main.go do?")
if err != nil {
t.Fatalf("Run: %v", err)
}
if got != "It declares package main." {
t.Errorf("answer = %q", got)
}
if len(f.requests) != 2 {
t.Fatalf("made %d requests, want 2", len(f.requests))
}
// Second request: user prompt, assistant turn, then the tool results.
second := f.requests[1].Messages
if len(second) != 3 {
t.Fatalf("second request had %d messages, want 3:\n%s", len(second), f.rawRequest(1))
}
if second[2].Role != llm.RoleUser {
t.Errorf("tool results must come back under the user role, got %q", second[2].Role)
}
if n := len(second[2].Content); n != 1 {
t.Fatalf("want 1 tool_result block, got %d", n)
}
res := second[2].Content[0]
if res.Type != llm.BlockToolResult || res.ToolUseID != "tu_1" || res.IsError {
t.Errorf("tool_result = %+v, want a successful result for tu_1", res)
}
if !strings.Contains(res.Content, "package main") {
t.Errorf("tool_result content = %q, want the file contents", res.Content)
}
}
func TestParallelToolCallsReturnInOneUserMessage(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "a.go"), []byte("A\n"), 0o644)
os.WriteFile(filepath.Join(root, "b.go"), []byte("B\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use",
toolUseBlock("tu_1", "read_file", `{"path":"a.go"}`),
toolUseBlock("tu_2", "read_file", `{"path":"b.go"}`)),
msg("end_turn", textBlock("done")),
)
a := newTestAgent(t, f, root, &bytes.Buffer{})
if _, err := a.Run(context.Background(), "read both"); err != nil {
t.Fatalf("Run: %v", err)
}
last := f.requests[1].Messages[len(f.requests[1].Messages)-1]
if len(last.Content) != 2 {
t.Fatalf("want both results in ONE message, got %d messages ending with %d blocks",
len(f.requests[1].Messages), len(last.Content))
}
if last.Content[0].ToolUseID != "tu_1" || last.Content[1].ToolUseID != "tu_2" {
t.Errorf("results out of order: %s, %s", last.Content[0].ToolUseID, last.Content[1].ToolUseID)
}
}
func TestFailedToolStillProducesAResult(t *testing.T) {
tests := []struct {
name string
call string
wantText string
}{
{name: "tool returns an error", call: toolUseBlock("tu_1", "boom", `{}`), wantText: "detonated"},
{name: "unknown tool", call: toolUseBlock("tu_1", "telepathy", `{}`), wantText: `unknown tool "telepathy"`},
{name: "malformed arguments", call: toolUseBlock("tu_1", "read_file", `{"path":42}`), wantText: "invalid arguments"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
f := newFakeAPI(t, msg("tool_use", tc.call), msg("end_turn", textBlock("ok")))
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("a failing tool is a normal event, not a Run error: %v", err)
}
last := f.requests[1].Messages[len(f.requests[1].Messages)-1]
if len(last.Content) != 1 {
t.Fatalf("every tool_use id needs exactly one result, got %d", len(last.Content))
}
res := last.Content[0]
if !res.IsError {
t.Error("is_error must be set so the model knows the call failed")
}
if !strings.Contains(res.Content, tc.wantText) {
t.Errorf("result = %q, want it to contain %q", res.Content, tc.wantText)
}
})
}
}
// Thinking signatures must be echoed back byte-identical or the next request
// is rejected.
func TestThinkingBlocksAreEchoedBackUnchanged(t *testing.T) {
f := newFakeAPI(t,
msg("tool_use",
`{"type":"thinking","thinking":"","signature":"sig-xyz"}`,
toolUseBlock("tu_1", "boom", `{}`)),
msg("end_turn", textBlock("fine")),
)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
body := f.rawRequest(1)
if !strings.Contains(body, `"signature":"sig-xyz"`) {
t.Errorf("thinking signature was dropped from the resent history:\n%s", body)
}
if !strings.Contains(body, `"thinking":""`) {
t.Errorf("empty thinking text was dropped:\n%s", body)
}
}
func TestRequestShape(t *testing.T) {
f := newFakeAPI(t, msg("end_turn", textBlock("hi")))
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
if _, err := a.Run(context.Background(), "hello"); err != nil {
t.Fatalf("Run: %v", err)
}
req := f.requests[0]
if len(req.System) != 1 || req.System[0].Text == "" {
t.Error("system prompt missing")
}
if strings.ContainsAny(req.System[0].Text, "0123456789") {
t.Error("system prompt should hold no digits: timestamps or ids here would break the cached prefix")
}
if req.Thinking == nil || req.Thinking.Type != "adaptive" {
t.Errorf("thinking = %+v, want adaptive (budget_tokens is a 400 on current models)", req.Thinking)
}
var names []string
for _, tl := range req.Tools {
names = append(names, tl.Name)
}
if want := "boom, edit_file, read_file, write_file"; strings.Join(names, ", ") != want {
t.Errorf("tools = %v, want them sorted for a byte-stable prefix (%s)", names, want)
}
for _, tl := range req.Tools {
if len(tl.InputSchema) == 0 {
t.Errorf("tool %q has no input_schema", tl.Name)
}
}
}
func TestMaxTurnsStopsTheLoop(t *testing.T) {
var script []string
for i := 0; i < 5; i++ {
script = append(script, msg("tool_use", toolUseBlock("tu", "boom", `{}`)))
}
f := newFakeAPI(t, script...)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
a.opts.MaxTurns = 3
_, err := a.Run(context.Background(), "loop forever")
if !errors.Is(err, ErrMaxTurns) {
t.Fatalf("error = %v, want ErrMaxTurns", err)
}
if !strings.Contains(err.Error(), "3 turns") {
t.Errorf("error should say how many turns were spent: %v", err)
}
if len(f.requests) != 3 {
t.Errorf("made %d requests, want exactly the cap of 3", len(f.requests))
}
}
func TestRefusalIsReportedWithItsCategory(t *testing.T) {
f := newFakeAPI(t, `{"id":"m","type":"message","role":"assistant","model":"claude-opus-5",
"content":[],"stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber","explanation":"nope"},
"usage":{"input_tokens":5,"output_tokens":0}}`)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
_, err := a.Run(context.Background(), "do something")
if err == nil {
t.Fatal("want an error")
}
if !strings.Contains(err.Error(), "cyber") || !strings.Contains(err.Error(), "nope") {
t.Errorf("error = %v, want the category and explanation", err)
}
}
func TestPauseTurnResends(t *testing.T) {
f := newFakeAPI(t,
msg("pause_turn", textBlock("thinking")),
msg("end_turn", textBlock("done")),
)
a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{})
got, err := a.Run(context.Background(), "go")
if err != nil {
t.Fatalf("Run: %v", err)
}
if got != "done" || len(f.requests) != 2 {
t.Errorf("got %q after %d requests, want the loop to resend after pause_turn", got, len(f.requests))
}
}
func TestUsageAndEventsAreRecorded(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "x.go"), []byte("x\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"x.go"}`)),
msg("end_turn", textBlock("done")),
)
var logBuf bytes.Buffer
a := newTestAgent(t, f, root, &logBuf)
var kinds []EventKind
a.Observe = func(ev Event) { kinds = append(kinds, ev.Kind) }
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
usage, turns := a.State().Usage()
if turns != 2 || usage.InputTokens != 200 || usage.CacheReadInputTokens != 80 {
t.Errorf("usage = %+v over %d turns, want both turns accumulated", usage, turns)
}
if !containsKind(kinds, EventToolStart) || !containsKind(kinds, EventToolResult) || !containsKind(kinds, EventText) {
t.Errorf("observer saw %v, want tool and text events", kinds)
}
log := logBuf.String()
if n := strings.Count(log, `"type":"turn"`); n != 2 {
t.Errorf("logged %d turn events, want 2:\n%s", n, log)
}
if n := strings.Count(log, `"type":"tool_call"`); n != 1 {
t.Errorf("logged %d tool_call events, want 1:\n%s", n, log)
}
if !strings.Contains(log, `"cache_read_input_tokens":40`) {
t.Errorf("cache reads must be logged; that is the Phase 4 lever:\n%s", log)
}
}
func TestReadFileMarksStateSoEditCanCheckLater(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "tracked.go")
os.WriteFile(path, []byte("x\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"tracked.go"}`)),
msg("end_turn", textBlock("done")),
)
a := newTestAgent(t, f, root, &bytes.Buffer{})
if _, err := a.Run(context.Background(), "go"); err != nil {
t.Fatalf("Run: %v", err)
}
if _, ok := a.State().LastRead(path); !ok {
t.Error("read_file must record the read in State for Phase 2's edit_file")
}
}
func TestRunPropagatesAPIErrors(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
io.WriteString(w, `{"error":{"type":"invalid_request_error","message":"bad model"}}`)
}))
defer srv.Close()
state := NewState()
a := New(llm.New(llm.WithBaseURL(srv.URL), llm.WithAPIKey("k")),
tools.NewRegistry(nil), state, nil, Options{})
if _, err := a.Run(context.Background(), "go"); err == nil || !strings.Contains(err.Error(), "bad model") {
t.Errorf("error = %v, want the API message surfaced", err)
}
}
func containsKind(kinds []EventKind, want EventKind) bool {
for _, k := range kinds {
if k == want {
return true
}
}
return false
}
// streamingAPI serves one scripted SSE response.
func streamingAPI(t *testing.T, events ...string) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for _, e := range events {
io.WriteString(w, "event: x\ndata: "+e+"\n\n")
}
}))
t.Cleanup(srv.Close)
return srv
}
func TestStreamingRunEmitsDeltas(t *testing.T) {
srv := streamingAPI(t,
`{"type":"message_start","message":{"id":"m","model":"claude-opus-5","usage":{"input_tokens":50}}}`,
`{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
`{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"All "}}`,
`{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"done."}}`,
`{"type":"content_block_stop","index":0}`,
`{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}`,
)
state := NewState()
a := New(llm.New(llm.WithBaseURL(srv.URL), llm.WithAPIKey("k")),
tools.NewRegistry(nil), state, nil, Options{Stream: true})
var deltas []string
var finalText int
a.Observe = func(ev Event) {
switch ev.Kind {
case EventTextDelta:
deltas = append(deltas, ev.Text)
case EventText:
finalText++
}
}
got, err := a.Run(context.Background(), "go")
if err != nil {
t.Fatalf("Run: %v", err)
}
if got != "All done." {
t.Errorf("answer = %q", got)
}
if strings.Join(deltas, "") != "All done." {
t.Errorf("deltas = %v", deltas)
}
if finalText != 0 {
t.Error("the whole text must not be emitted again after streaming it")
}
}
// denyAll is an Approver that refuses everything it is asked about.
type denyAll struct{ asked []string }
func (d *denyAll) Approve(_ context.Context, tool string, _ json.RawMessage) error {
d.asked = append(d.asked, tool)
return errors.New("the user declined to run " + tool)
}
func TestDeniedToolBecomesAToolError(t *testing.T) {
root := t.TempDir()
os.WriteFile(filepath.Join(root, "a.go"), []byte("package a\n"), 0o644)
f := newFakeAPI(t,
msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"a.go"}`)),
msg("end_turn", textBlock("understood")),
)
var logBuf bytes.Buffer
a := newTestAgent(t, f, root, &logBuf)
gate := &denyAll{}
a.Approve = gate
got, err := a.Run(context.Background(), "read it")
if err != nil {
t.Fatalf("a denial is not a harness failure: %v", err)
}
if got != "understood" {
t.Errorf("answer = %q", got)
}
if len(gate.asked) != 1 || gate.asked[0] != "read_file" {
t.Errorf("approver saw %v", gate.asked)
}
last := f.requests[1].Messages[len(f.requests[1].Messages)-1]
res := last.Content[0]
if !res.IsError || !strings.Contains(res.Content, "declined") {
t.Errorf("tool_result = %+v, want the denial passed to the model", res)
}
if !strings.Contains(logBuf.String(), `"type":"tool_denied"`) {
t.Error("denials should be logged: they explain gaps in the eval numbers")
}
// The tool must not have run.
if strings.Contains(res.Content, "package a") {
t.Error("the file was read despite the denial")
}
}
func TestSystemPromptListsSkills(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{
Skills: []skills.Skill{{Name: "commit-helper", Description: "Use when writing commit messages"}},
})
sys := a.opts.System
if !strings.Contains(sys, "commit-helper: Use when writing commit messages") {
t.Errorf("system prompt does not list the skill:\n%s", sys)
}
if !strings.Contains(sys, "/<name>") {
t.Error("system prompt does not explain what a leading slash means")
}
if !strings.HasPrefix(sys, defaultSystem) {
t.Error("the listing replaced the default prompt instead of being appended to it")
}
}
func TestSystemPromptUnchangedWithoutSkills(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{})
if a.opts.System != defaultSystem {
t.Errorf("system prompt differs from the default with no skills:\n%s", a.opts.System)
}
}
func TestSystemPromptIsStableAcrossRequests(t *testing.T) {
// The listing sits in the cached prefix. If it varied per request, every
// turn would miss the cache.
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{
Skills: []skills.Skill{{Name: "a", Description: "one"}, {Name: "b", Description: "two"}},
})
a.state.Append(llm.Message{Role: llm.RoleUser, Content: []llm.ContentBlock{llm.TextBlock("hi")}})
first := a.request().System[0].Text
second := a.request().System[0].Text
if first != second {
t.Error("the system prompt changed between two requests in one session")
}
}
func TestSystemPrefixBecomesTheFirstBlock(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{SystemPrefix: ClaudeCodeIdentity})
blocks := a.systemBlocks()
if len(blocks) != 2 {
t.Fatalf("got %d system blocks, want 2", len(blocks))
}
if blocks[0].Text != ClaudeCodeIdentity {
t.Errorf("first block = %q, want the identity string", blocks[0].Text)
}
if !strings.Contains(blocks[1].Text, "coding agent") {
t.Errorf("second block should be the harness prompt, got %q", blocks[1].Text)
}
}
// The cache breakpoint has to sit on the last block, or only the identity
// string gets cached and the prompt behind it is re-read every turn.
func TestCacheBreakpointIsOnTheLastSystemBlock(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{SystemPrefix: ClaudeCodeIdentity})
blocks := a.systemBlocks()
if blocks[0].CacheControl != nil {
t.Error("the first block must not carry the breakpoint")
}
if blocks[len(blocks)-1].CacheControl == nil {
t.Error("the last block must carry the breakpoint")
}
}
// Without OAuth nothing changes, including byte-for-byte.
func TestNoPrefixMeansOneBlockAsBefore(t *testing.T) {
a := New(nil, tools.NewRegistry(nil), NewState(), nil, Options{})
blocks := a.systemBlocks()
if len(blocks) != 1 {
t.Fatalf("got %d system blocks, want 1", len(blocks))
}
if blocks[0].Text != defaultSystem {
t.Error("the prompt must be unchanged when no prefix is set")
}
}