package llm import ( "bufio" "bytes" "context" "encoding/json" "fmt" "io" "net/http" "strings" ) // StreamEventKind is what a streaming callback is being told about. type StreamEventKind string const ( StreamText StreamEventKind = "text" StreamThinking StreamEventKind = "thinking" StreamToolStart StreamEventKind = "tool_start" StreamToolInput StreamEventKind = "tool_input" StreamBlockStop StreamEventKind = "block_stop" ) // StreamEvent is one incremental update. type StreamEvent struct { Kind StreamEventKind Index int Text string // the delta, not the accumulated value ToolID string ToolName string } // StreamMessage sends a streaming request, calls onEvent for each delta, and // returns the assembled response. The callback may be nil. func (c *Client) StreamMessage(ctx context.Context, req Request, onEvent func(StreamEvent)) (*Response, error) { if !c.HasCredentials() { return nil, fmt.Errorf("llm: no credentials: set ANTHROPIC_API_KEY, or run `ant auth login`") } if req.Model == "" { req.Model = c.Model } if req.MaxTokens == 0 { req.MaxTokens = c.MaxTokens } req.Stream = true body, err := json.Marshal(req) if err != nil { return nil, fmt.Errorf("llm: encoding request: %w", err) } var lastErr error for attempt := 0; attempt <= c.MaxRetries; attempt++ { if attempt > 0 { if err := sleep(ctx, c.backoff(attempt, lastErr)); err != nil { return nil, err } } resp, err := c.streamAttempt(ctx, body, onEvent) if err == nil { return resp, nil } lastErr = err if ctx.Err() != nil { return nil, ctx.Err() } var apiErr *APIError if ok := asAPIError(err, &apiErr); ok && !apiErr.Retryable() { return nil, err } // A stream that broke partway is not safely retryable either: the // caller may already have seen deltas. if _, isAPI := err.(*APIError); !isAPI { return nil, err } } return nil, lastErr } func (c *Client) streamAttempt(ctx context.Context, body []byte, onEvent func(StreamEvent)) (*Response, error) { httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/v1/messages", bytes.NewReader(body)) if err != nil { return nil, err } httpReq.Header.Set("content-type", "application/json") httpReq.Header.Set("accept", "text/event-stream") httpReq.Header.Set("anthropic-version", apiVersion) if c.apiKey != "" { httpReq.Header.Set("x-api-key", c.apiKey) } else { httpReq.Header.Set("authorization", "Bearer "+c.authToken) httpReq.Header.Set("anthropic-beta", oauthBeta) } httpResp, err := c.HTTP.Do(httpReq) if err != nil { return nil, fmt.Errorf("llm: %w", err) } defer httpResp.Body.Close() if httpResp.StatusCode != http.StatusOK { raw, _ := io.ReadAll(httpResp.Body) return nil, parseAPIError(httpResp.StatusCode, httpResp.Header.Get("retry-after"), raw) } return readStream(httpResp.Body, onEvent) } // blockBuilder accumulates one content block across its deltas. Tool input // arrives as a series of JSON fragments that are only valid once concatenated, // which is the fiddly part of streaming. type blockBuilder struct { block ContentBlock json strings.Builder } func readStream(r io.Reader, onEvent func(StreamEvent)) (*Response, error) { emit := func(ev StreamEvent) { if onEvent != nil { onEvent(ev) } } resp := &Response{Role: RoleAssistant} blocks := map[int]*blockBuilder{} var order []int sc := bufio.NewScanner(r) sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) for sc.Scan() { line := sc.Text() if !strings.HasPrefix(line, "data:") { continue // "event:" lines are redundant; the payload names its type } payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) if payload == "" || payload == "[DONE]" { continue } var ev struct { Type string `json:"type"` Index int `json:"index"` Message *struct { ID string `json:"id"` Model string `json:"model"` Usage Usage `json:"usage"` } `json:"message"` ContentBlock *ContentBlock `json:"content_block"` Delta *struct { Type string `json:"type"` Text string `json:"text"` PartialJSON string `json:"partial_json"` Thinking string `json:"thinking"` Signature string `json:"signature"` StopReason string `json:"stop_reason"` StopDetails *StopDetails `json:"stop_details"` } `json:"delta"` Usage *Usage `json:"usage"` Error *struct { Type string `json:"type"` Message string `json:"message"` } `json:"error"` } if err := json.Unmarshal([]byte(payload), &ev); err != nil { return nil, fmt.Errorf("llm: malformed stream event: %w", err) } switch ev.Type { case "error": if ev.Error != nil { return nil, &APIError{StatusCode: 200, Type: ev.Error.Type, Message: ev.Error.Message} } return nil, fmt.Errorf("llm: stream error") case "message_start": if ev.Message != nil { resp.ID, resp.Model, resp.Usage = ev.Message.ID, ev.Message.Model, ev.Message.Usage } case "content_block_start": if ev.ContentBlock == nil { continue } blocks[ev.Index] = &blockBuilder{block: *ev.ContentBlock} order = append(order, ev.Index) if ev.ContentBlock.Type == BlockToolUse { emit(StreamEvent{Kind: StreamToolStart, Index: ev.Index, ToolID: ev.ContentBlock.ID, ToolName: ev.ContentBlock.Name}) } case "content_block_delta": b, ok := blocks[ev.Index] if !ok || ev.Delta == nil { continue } switch ev.Delta.Type { case "text_delta": b.block.Text += ev.Delta.Text emit(StreamEvent{Kind: StreamText, Index: ev.Index, Text: ev.Delta.Text}) case "thinking_delta": b.block.Thinking += ev.Delta.Thinking emit(StreamEvent{Kind: StreamThinking, Index: ev.Index, Text: ev.Delta.Thinking}) case "signature_delta": b.block.Signature += ev.Delta.Signature case "input_json_delta": // Fragments are not individually parseable; buffer and parse // the whole thing at content_block_stop. b.json.WriteString(ev.Delta.PartialJSON) emit(StreamEvent{Kind: StreamToolInput, Index: ev.Index, Text: ev.Delta.PartialJSON}) } case "content_block_stop": b, ok := blocks[ev.Index] if !ok { continue } if s := b.json.String(); s != "" { if !json.Valid([]byte(s)) { return nil, fmt.Errorf("llm: tool %q streamed invalid JSON arguments: %s", b.block.Name, s) } b.block.Input = json.RawMessage(s) } emit(StreamEvent{Kind: StreamBlockStop, Index: ev.Index}) case "message_delta": if ev.Delta != nil { resp.StopReason = ev.Delta.StopReason if ev.Delta.StopDetails != nil { resp.StopDetails = ev.Delta.StopDetails } } if ev.Usage != nil { // message_delta carries the final output count; the input // counts arrived with message_start. resp.Usage.OutputTokens = ev.Usage.OutputTokens if ev.Usage.InputTokens > 0 { resp.Usage.InputTokens = ev.Usage.InputTokens } } } } if err := sc.Err(); err != nil { return nil, fmt.Errorf("llm: reading stream: %w", err) } for _, i := range order { if b := blocks[i]; b != nil { // A tool_use with no input at all still needs a valid object. if b.block.Type == BlockToolUse && len(b.block.Input) == 0 { b.block.Input = json.RawMessage("{}") } resp.Content = append(resp.Content, b.block) } } if resp.StopReason == "" { return nil, fmt.Errorf("llm: stream ended without a stop_reason") } return resp, nil } // parseAPIError builds an APIError from a non-2xx body. func parseAPIError(status int, retryAfterHeader string, raw []byte) *APIError { apiErr := &APIError{StatusCode: status, Body: string(raw)} var envelope struct { Error struct { Type string `json:"type"` Message string `json:"message"` } `json:"error"` } if json.Unmarshal(raw, &envelope) == nil { apiErr.Type = envelope.Error.Type apiErr.Message = envelope.Error.Message } if retryAfterHeader != "" { apiErr.Message += " (retry-after " + retryAfterHeader + "s)" } return apiErr }