package agent import ( "bytes" "context" "encoding/json" "errors" "io" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" "strings" "testing" "harness/internal/llm" "harness/internal/metrics" "harness/internal/tools" ) // fakeAPI serves a scripted sequence of responses and records every request // body it received, so tests can assert on what the loop actually sent. type fakeAPI struct { t *testing.T responses []string requests []llm.Request summary string srv *httptest.Server } // compactions counts requests that were summarization calls, so they do not // consume scripted responses. func (f *fakeAPI) compactions() int { var n int for _, req := range f.requests { if len(req.Messages) > 0 { last := req.Messages[len(req.Messages)-1] for _, b := range last.Content { if strings.Contains(b.Text, "Summarize the conversation above") { n++ } } } } return n } func newFakeAPI(t *testing.T, responses ...string) *fakeAPI { f := &fakeAPI{t: t, responses: responses, summary: "Compacted: edited loop.go, the config loads from X not Y."} f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) var req llm.Request if err := json.Unmarshal(body, &req); err != nil { t.Errorf("harness sent a body the API types cannot parse: %v\n%s", err, body) } f.requests = append(f.requests, req) if bytes.Contains(body, []byte("Summarize the conversation above")) { io.WriteString(w, msg("end_turn", textBlock(f.summary))) return } i := len(f.requests) - 1 - f.compactions() if i >= len(f.responses) { t.Errorf("harness made %d requests, script only has %d", i+1, len(f.responses)) w.WriteHeader(http.StatusInternalServerError) return } io.WriteString(w, f.responses[i]) })) t.Cleanup(f.srv.Close) return f } // rawRequest returns the n-th request body re-encoded, for byte-level assertions. func (f *fakeAPI) rawRequest(n int) string { b, err := json.Marshal(f.requests[n]) if err != nil { f.t.Fatal(err) } return string(b) } func msg(stopReason string, blocks ...string) string { return `{"id":"msg","type":"message","role":"assistant","model":"claude-opus-5","content":[` + strings.Join(blocks, ",") + `],"stop_reason":"` + stopReason + `",` + `"usage":{"input_tokens":100,"output_tokens":20,"cache_read_input_tokens":40}}` } func textBlock(s string) string { return `{"type":"text","text":"` + s + `"}` } // msgWithTokens is msg with a controllable input_tokens, for driving compaction. func msgWithTokens(stopReason string, inputTokens int, blocks ...string) string { return `{"id":"msg","type":"message","role":"assistant","model":"claude-opus-5","content":[` + strings.Join(blocks, ",") + `],"stop_reason":"` + stopReason + `",` + `"usage":{"input_tokens":` + strconv.Itoa(inputTokens) + `,"output_tokens":20}}` } func toolUseBlock(id, name, input string) string { return `{"type":"tool_use","id":"` + id + `","name":"` + name + `","input":` + input + `}` } // newTestAgent wires an agent against the fake API, with read_file rooted at // a temp dir plus a couple of tools that make failure modes easy to trigger. func newTestAgent(t *testing.T, f *fakeAPI, root string, logBuf *bytes.Buffer) *Agent { t.Helper() log := metrics.NewWriter(logBuf, "test") state := NewState() reg := tools.NewRegistry(log).Register( tools.NewRead(root, state), tools.NewEdit(root, state), tools.NewWrite(root, state), tools.New("boom", "Always fails", func(context.Context, struct{}) (string, error) { return "", errors.New("detonated") }), ) client := llm.New(llm.WithBaseURL(f.srv.URL), llm.WithAPIKey("k")) return New(client, reg, state, log, Options{}) } func TestRunReadsAFileAndAnswers(t *testing.T) { root := t.TempDir() os.WriteFile(filepath.Join(root, "main.go"), []byte("package main\n"), 0o644) f := newFakeAPI(t, msg("tool_use", textBlock("Let me look."), toolUseBlock("tu_1", "read_file", `{"path":"main.go"}`)), msg("end_turn", textBlock("It declares package main.")), ) a := newTestAgent(t, f, root, &bytes.Buffer{}) got, err := a.Run(context.Background(), "what does main.go do?") if err != nil { t.Fatalf("Run: %v", err) } if got != "It declares package main." { t.Errorf("answer = %q", got) } if len(f.requests) != 2 { t.Fatalf("made %d requests, want 2", len(f.requests)) } // Second request: user prompt, assistant turn, then the tool results. second := f.requests[1].Messages if len(second) != 3 { t.Fatalf("second request had %d messages, want 3:\n%s", len(second), f.rawRequest(1)) } if second[2].Role != llm.RoleUser { t.Errorf("tool results must come back under the user role, got %q", second[2].Role) } if n := len(second[2].Content); n != 1 { t.Fatalf("want 1 tool_result block, got %d", n) } res := second[2].Content[0] if res.Type != llm.BlockToolResult || res.ToolUseID != "tu_1" || res.IsError { t.Errorf("tool_result = %+v, want a successful result for tu_1", res) } if !strings.Contains(res.Content, "package main") { t.Errorf("tool_result content = %q, want the file contents", res.Content) } } func TestParallelToolCallsReturnInOneUserMessage(t *testing.T) { root := t.TempDir() os.WriteFile(filepath.Join(root, "a.go"), []byte("A\n"), 0o644) os.WriteFile(filepath.Join(root, "b.go"), []byte("B\n"), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"a.go"}`), toolUseBlock("tu_2", "read_file", `{"path":"b.go"}`)), msg("end_turn", textBlock("done")), ) a := newTestAgent(t, f, root, &bytes.Buffer{}) if _, err := a.Run(context.Background(), "read both"); err != nil { t.Fatalf("Run: %v", err) } last := f.requests[1].Messages[len(f.requests[1].Messages)-1] if len(last.Content) != 2 { t.Fatalf("want both results in ONE message, got %d messages ending with %d blocks", len(f.requests[1].Messages), len(last.Content)) } if last.Content[0].ToolUseID != "tu_1" || last.Content[1].ToolUseID != "tu_2" { t.Errorf("results out of order: %s, %s", last.Content[0].ToolUseID, last.Content[1].ToolUseID) } } func TestFailedToolStillProducesAResult(t *testing.T) { tests := []struct { name string call string wantText string }{ {name: "tool returns an error", call: toolUseBlock("tu_1", "boom", `{}`), wantText: "detonated"}, {name: "unknown tool", call: toolUseBlock("tu_1", "telepathy", `{}`), wantText: `unknown tool "telepathy"`}, {name: "malformed arguments", call: toolUseBlock("tu_1", "read_file", `{"path":42}`), wantText: "invalid arguments"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := newFakeAPI(t, msg("tool_use", tc.call), msg("end_turn", textBlock("ok"))) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("a failing tool is a normal event, not a Run error: %v", err) } last := f.requests[1].Messages[len(f.requests[1].Messages)-1] if len(last.Content) != 1 { t.Fatalf("every tool_use id needs exactly one result, got %d", len(last.Content)) } res := last.Content[0] if !res.IsError { t.Error("is_error must be set so the model knows the call failed") } if !strings.Contains(res.Content, tc.wantText) { t.Errorf("result = %q, want it to contain %q", res.Content, tc.wantText) } }) } } // Thinking signatures must be echoed back byte-identical or the next request // is rejected. func TestThinkingBlocksAreEchoedBackUnchanged(t *testing.T) { f := newFakeAPI(t, msg("tool_use", `{"type":"thinking","thinking":"","signature":"sig-xyz"}`, toolUseBlock("tu_1", "boom", `{}`)), msg("end_turn", textBlock("fine")), ) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } body := f.rawRequest(1) if !strings.Contains(body, `"signature":"sig-xyz"`) { t.Errorf("thinking signature was dropped from the resent history:\n%s", body) } if !strings.Contains(body, `"thinking":""`) { t.Errorf("empty thinking text was dropped:\n%s", body) } } func TestRequestShape(t *testing.T) { f := newFakeAPI(t, msg("end_turn", textBlock("hi"))) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) if _, err := a.Run(context.Background(), "hello"); err != nil { t.Fatalf("Run: %v", err) } req := f.requests[0] if len(req.System) != 1 || req.System[0].Text == "" { t.Error("system prompt missing") } if strings.ContainsAny(req.System[0].Text, "0123456789") { t.Error("system prompt should hold no digits: timestamps or ids here would break the cached prefix") } if req.Thinking == nil || req.Thinking.Type != "adaptive" { t.Errorf("thinking = %+v, want adaptive (budget_tokens is a 400 on current models)", req.Thinking) } var names []string for _, tl := range req.Tools { names = append(names, tl.Name) } if want := "boom, edit_file, read_file, write_file"; strings.Join(names, ", ") != want { t.Errorf("tools = %v, want them sorted for a byte-stable prefix (%s)", names, want) } for _, tl := range req.Tools { if len(tl.InputSchema) == 0 { t.Errorf("tool %q has no input_schema", tl.Name) } } } func TestMaxTurnsStopsTheLoop(t *testing.T) { var script []string for i := 0; i < 5; i++ { script = append(script, msg("tool_use", toolUseBlock("tu", "boom", `{}`))) } f := newFakeAPI(t, script...) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) a.opts.MaxTurns = 3 _, err := a.Run(context.Background(), "loop forever") if !errors.Is(err, ErrMaxTurns) { t.Fatalf("error = %v, want ErrMaxTurns", err) } if !strings.Contains(err.Error(), "3 turns") { t.Errorf("error should say how many turns were spent: %v", err) } if len(f.requests) != 3 { t.Errorf("made %d requests, want exactly the cap of 3", len(f.requests)) } } func TestRefusalIsReportedWithItsCategory(t *testing.T) { f := newFakeAPI(t, `{"id":"m","type":"message","role":"assistant","model":"claude-opus-5", "content":[],"stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber","explanation":"nope"}, "usage":{"input_tokens":5,"output_tokens":0}}`) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) _, err := a.Run(context.Background(), "do something") if err == nil { t.Fatal("want an error") } if !strings.Contains(err.Error(), "cyber") || !strings.Contains(err.Error(), "nope") { t.Errorf("error = %v, want the category and explanation", err) } } func TestPauseTurnResends(t *testing.T) { f := newFakeAPI(t, msg("pause_turn", textBlock("thinking")), msg("end_turn", textBlock("done")), ) a := newTestAgent(t, f, t.TempDir(), &bytes.Buffer{}) got, err := a.Run(context.Background(), "go") if err != nil { t.Fatalf("Run: %v", err) } if got != "done" || len(f.requests) != 2 { t.Errorf("got %q after %d requests, want the loop to resend after pause_turn", got, len(f.requests)) } } func TestUsageAndEventsAreRecorded(t *testing.T) { root := t.TempDir() os.WriteFile(filepath.Join(root, "x.go"), []byte("x\n"), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"x.go"}`)), msg("end_turn", textBlock("done")), ) var logBuf bytes.Buffer a := newTestAgent(t, f, root, &logBuf) var kinds []EventKind a.Observe = func(ev Event) { kinds = append(kinds, ev.Kind) } if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } usage, turns := a.State().Usage() if turns != 2 || usage.InputTokens != 200 || usage.CacheReadInputTokens != 80 { t.Errorf("usage = %+v over %d turns, want both turns accumulated", usage, turns) } if !containsKind(kinds, EventToolStart) || !containsKind(kinds, EventToolResult) || !containsKind(kinds, EventText) { t.Errorf("observer saw %v, want tool and text events", kinds) } log := logBuf.String() if n := strings.Count(log, `"type":"turn"`); n != 2 { t.Errorf("logged %d turn events, want 2:\n%s", n, log) } if n := strings.Count(log, `"type":"tool_call"`); n != 1 { t.Errorf("logged %d tool_call events, want 1:\n%s", n, log) } if !strings.Contains(log, `"cache_read_input_tokens":40`) { t.Errorf("cache reads must be logged; that is the Phase 4 lever:\n%s", log) } } func TestReadFileMarksStateSoEditCanCheckLater(t *testing.T) { root := t.TempDir() path := filepath.Join(root, "tracked.go") os.WriteFile(path, []byte("x\n"), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"tracked.go"}`)), msg("end_turn", textBlock("done")), ) a := newTestAgent(t, f, root, &bytes.Buffer{}) if _, err := a.Run(context.Background(), "go"); err != nil { t.Fatalf("Run: %v", err) } if _, ok := a.State().LastRead(path); !ok { t.Error("read_file must record the read in State for Phase 2's edit_file") } } func TestRunPropagatesAPIErrors(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) io.WriteString(w, `{"error":{"type":"invalid_request_error","message":"bad model"}}`) })) defer srv.Close() state := NewState() a := New(llm.New(llm.WithBaseURL(srv.URL), llm.WithAPIKey("k")), tools.NewRegistry(nil), state, nil, Options{}) if _, err := a.Run(context.Background(), "go"); err == nil || !strings.Contains(err.Error(), "bad model") { t.Errorf("error = %v, want the API message surfaced", err) } } func containsKind(kinds []EventKind, want EventKind) bool { for _, k := range kinds { if k == want { return true } } return false } // streamingAPI serves one scripted SSE response. func streamingAPI(t *testing.T, events ...string) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { for _, e := range events { io.WriteString(w, "event: x\ndata: "+e+"\n\n") } })) t.Cleanup(srv.Close) return srv } func TestStreamingRunEmitsDeltas(t *testing.T) { srv := streamingAPI(t, `{"type":"message_start","message":{"id":"m","model":"claude-opus-5","usage":{"input_tokens":50}}}`, `{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"All "}}`, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"done."}}`, `{"type":"content_block_stop","index":0}`, `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}`, ) state := NewState() a := New(llm.New(llm.WithBaseURL(srv.URL), llm.WithAPIKey("k")), tools.NewRegistry(nil), state, nil, Options{Stream: true}) var deltas []string var finalText int a.Observe = func(ev Event) { switch ev.Kind { case EventTextDelta: deltas = append(deltas, ev.Text) case EventText: finalText++ } } got, err := a.Run(context.Background(), "go") if err != nil { t.Fatalf("Run: %v", err) } if got != "All done." { t.Errorf("answer = %q", got) } if strings.Join(deltas, "") != "All done." { t.Errorf("deltas = %v", deltas) } if finalText != 0 { t.Error("the whole text must not be emitted again after streaming it") } } // denyAll is an Approver that refuses everything it is asked about. type denyAll struct{ asked []string } func (d *denyAll) Approve(_ context.Context, tool string, _ json.RawMessage) error { d.asked = append(d.asked, tool) return errors.New("the user declined to run " + tool) } func TestDeniedToolBecomesAToolError(t *testing.T) { root := t.TempDir() os.WriteFile(filepath.Join(root, "a.go"), []byte("package a\n"), 0o644) f := newFakeAPI(t, msg("tool_use", toolUseBlock("tu_1", "read_file", `{"path":"a.go"}`)), msg("end_turn", textBlock("understood")), ) var logBuf bytes.Buffer a := newTestAgent(t, f, root, &logBuf) gate := &denyAll{} a.Approve = gate got, err := a.Run(context.Background(), "read it") if err != nil { t.Fatalf("a denial is not a harness failure: %v", err) } if got != "understood" { t.Errorf("answer = %q", got) } if len(gate.asked) != 1 || gate.asked[0] != "read_file" { t.Errorf("approver saw %v", gate.asked) } last := f.requests[1].Messages[len(f.requests[1].Messages)-1] res := last.Content[0] if !res.IsError || !strings.Contains(res.Content, "declined") { t.Errorf("tool_result = %+v, want the denial passed to the model", res) } if !strings.Contains(logBuf.String(), `"type":"tool_denied"`) { t.Error("denials should be logged: they explain gaps in the eval numbers") } // The tool must not have run. if strings.Contains(res.Content, "package a") { t.Error("the file was read despite the denial") } }