Vectrum
Sign In
193 lines · 6.8 KB · Go
package llm

import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"testing"
)

func TestListModelsReadsEveryPage(t *testing.T) {
	var paths []string
	var headers http.Header
	c := testClient(t, func(w http.ResponseWriter, r *http.Request) {
		headers = r.Header.Clone()
		paths = append(paths, r.URL.RequestURI())
		if after := r.URL.Query().Get("after_id"); after == "" {
			fmt.Fprint(w, `{"data":[
				{"type":"model","id":"claude-opus-5","display_name":"Claude Opus 5","created_at":"2026-02-01T00:00:00Z"},
				{"type":"model","id":"claude-sonnet-5","display_name":"Claude Sonnet 5","created_at":"2025-09-29T00:00:00Z"}],
				"has_more":true,"last_id":"claude-sonnet-5"}`)
			return
		}
		fmt.Fprint(w, `{"data":[
			{"type":"model","id":"claude-haiku-4-5-20251001","display_name":"Claude Haiku 4.5","created_at":"2025-10-01T00:00:00Z"}],
			"has_more":false}`)
	})

	got, err := c.ListModels(context.Background())
	if err != nil {
		t.Fatalf("ListModels: %v", err)
	}

	var ids []string
	for _, m := range got {
		ids = append(ids, m.ID)
	}
	if want := "claude-opus-5 claude-sonnet-5 claude-haiku-4-5-20251001"; strings.Join(ids, " ") != want {
		t.Errorf("ids = %v, want %s in the order the API gave them", ids, want)
	}
	if got[0].DisplayName != "Claude Opus 5" {
		t.Errorf("display name = %q", got[0].DisplayName)
	}
	if len(paths) != 2 {
		t.Fatalf("requests = %v, want a second one for the second page", paths)
	}
	if !strings.HasPrefix(paths[0], "/v1/models?") {
		t.Errorf("path = %q, want /v1/models", paths[0])
	}
	if !strings.Contains(paths[1], "after_id=claude-sonnet-5") {
		t.Errorf("second request = %q, want it to carry the last id", paths[1])
	}
	if headers.Get("x-api-key") != "test-key" || headers.Get("anthropic-version") == "" {
		t.Errorf("credentials or version missing: %v", headers)
	}
}

// Not every endpoint serves this route. The error has to name what went wrong
// so the picker can say it out loud.
func TestListModelsReportsAnEndpointThatWillNotAnswer(t *testing.T) {
	c := testClient(t, func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, `{"type":"error","error":{"type":"not_found_error","message":"unknown route"}}`, http.StatusNotFound)
	})

	_, err := c.ListModels(context.Background())
	if err == nil {
		t.Fatal("a 404 produced no error")
	}
	if !strings.Contains(err.Error(), "404") && !strings.Contains(err.Error(), "not_found") {
		t.Errorf("error = %q, want it to say what the endpoint answered", err)
	}
}

// A bearer credential is what OAuth tokens and most gateways use, and the
// listing has to authenticate the same way the messages call does.
func TestListModelsSendsABearerCredential(t *testing.T) {
	var headers http.Header
	c := testClient(t, func(w http.ResponseWriter, r *http.Request) {
		headers = r.Header.Clone()
		fmt.Fprint(w, `{"data":[],"has_more":false}`)
	})
	WithAuthToken("oauth-token")(c)

	if _, err := c.ListModels(context.Background()); err != nil {
		t.Fatalf("ListModels: %v", err)
	}
	if got := headers.Get("authorization"); got != "Bearer oauth-token" {
		t.Errorf("authorization = %q", got)
	}
	if headers.Get("anthropic-beta") == "" {
		t.Error("the oauth beta flag is missing")
	}
}

func TestListModelsRefusesWithoutCredentials(t *testing.T) {
	c := New(WithBaseURL("http://127.0.0.1:1"))
	c.apiKey, c.authToken = "", ""
	if _, err := c.ListModels(context.Background()); err == nil {
		t.Fatal("listing without credentials should not even be attempted")
	}
}

// output_config.effort is not a universal parameter: Haiku 4.5 rejects the
// request outright, and xhigh only exists from Opus 4.7 on. Sending it anyway
// is a 400 that reads like a bug in the harness, because it is one.
func TestSupportsEffortKnowsWhichModelsTakeIt(t *testing.T) {
	cases := []struct {
		model, level string
		want         bool
	}{
		{"claude-opus-5", "high", true},
		{"claude-opus-5", "max", true},
		{"claude-sonnet-5", "xhigh", true},
		{"claude-fable-5-1", "low", true},
		{"claude-opus-4-8", "xhigh", true},

		// The one that bit us: a subagent on Haiku with the session's effort.
		{"claude-haiku-4-5", "high", false},
		{"claude-haiku-4-5", "low", false},
		{"claude-haiku-4-5-20251001", "high", false},
		{"claude-sonnet-4-5", "high", false},

		// xhigh arrived with Opus 4.7; 4.6 takes the other four.
		{"claude-opus-4-6", "high", true},
		{"claude-opus-4-6", "max", true},
		{"claude-opus-4-6", "xhigh", false},
		{"claude-sonnet-4-6", "xhigh", false},

		// A bracketed context variant is the same model.
		{"claude-opus-5[1m]", "max", true},

		// An empty level means "the API's own default", which is always fine.
		// A model this build has never heard of is treated as taking none —
		// see TestAnUnknownModelIsNotSentAnEffortLevel for why that way round.
		{"claude-haiku-4-5", "", true},
		{"some-gateway/llama-4", "high", false},
	}
	for _, c := range cases {
		if got := SupportsEffort(c.model, c.level); got != c.want {
			t.Errorf("SupportsEffort(%q, %q) = %v, want %v", c.model, c.level, got, c.want)
		}
	}
}

// A gateway that serves an id this build has never seen is the case the
// default has to be chosen for. The two mistakes are not symmetric: sending
// effort to something that does not take it is a 400 that ends the run, where
// omitting it costs the level and says so in the status line. The recoverable
// failure is the one to default to — the same reading adaptive thinking makes.
func TestAnUnknownModelIsNotSentAnEffortLevel(t *testing.T) {
	for _, model := range []string{"some-gateway/llama-4", "gpt-4o", "mistral-large", ""} {
		if SupportsEffort(model, "high") {
			t.Errorf("SupportsEffort(%q, high) = true; a 400 from an unknown model ends the run", model)
		}
	}
	// And the level nobody set is never a problem.
	if !SupportsEffort("some-gateway/llama-4", "") {
		t.Error("an unset level was treated as unsupported")
	}
}

// Gateways decorate ids: a vendor prefix, a Bedrock namespace, a Vertex
// version suffix. They name the same model, and a capability lookup that does
// not see past the decoration silently downgrades every request through a
// proxy.
func TestModelIdsAreNormalisedPastGatewayDecoration(t *testing.T) {
	cases := []string{
		"claude-opus-5",
		"anthropic/claude-opus-5",
		"anthropic.claude-opus-5",
		"claude-opus-5@20260401",
		"openrouter/anthropic/claude-opus-5",
		"claude-opus-5-20260401",
		"claude-opus-5[1m]",
	}
	for _, id := range cases {
		if !SupportsEffort(id, "max") {
			t.Errorf("SupportsEffort(%q, max) = false, want the same answer as the bare id", id)
		}
		if !SupportsAdaptiveThinking(id) {
			t.Errorf("SupportsAdaptiveThinking(%q) = false, want the same answer as the bare id", id)
		}
		if _, known := PriceOf(id); !known {
			t.Errorf("PriceOf(%q) does not recognise the model", id)
		}
	}
	// Haiku stays itself through the same decoration.
	for _, id := range []string{"anthropic/claude-haiku-4-5", "anthropic.claude-haiku-4-5"} {
		if SupportsEffort(id, "high") {
			t.Errorf("SupportsEffort(%q, high) = true", id)
		}
	}
}