Vectrum
Sign In
201 lines · 6.9 KB · Go
package ask

import (
	"context"
	"encoding/json"
	"errors"
	"strings"
	"testing"
)

func twoOptions() Question {
	return Question{
		Header:   "Storage",
		Question: "How should sessions be stored?",
		Options:  []Option{{Label: "Postgres"}, {Label: "SQLite"}},
	}
}

// Every rejection is read by the model, which gets to send another call. A
// message that does not say what to send instead wastes that turn.
func TestValidateRejectsWhatCannotBeDrawn(t *testing.T) {
	tooMany := Request{}
	for i := 0; i < MaxQuestions+1; i++ {
		tooMany.Questions = append(tooMany.Questions, twoOptions())
	}
	oneOption := twoOptions()
	oneOption.Options = oneOption.Options[:1]
	fiveOptions := twoOptions()
	for i := 0; i < 4; i++ {
		fiveOptions.Options = append(fiveOptions.Options, Option{Label: string(rune('a' + i))})
	}
	noHeader := twoOptions()
	noHeader.Header = "  "
	noText := twoOptions()
	noText.Question = ""
	dup := twoOptions()
	dup.Options[1].Label = "postgres"
	blank := twoOptions()
	blank.Options[1].Label = ""

	cases := map[string]struct {
		req  Request
		want string
	}{
		"nothing asked":   {Request{}, "1 and 4"},
		"too many":        {tooMany, "at most 4"},
		"one option":      {Request{Questions: []Question{oneOption}}, "at least 2"},
		"five options":    {Request{Questions: []Question{fiveOptions}}, "at most 4"},
		"no header":       {Request{Questions: []Question{noHeader}}, "header"},
		"no question":     {Request{Questions: []Question{noText}}, "no text"},
		"same label":      {Request{Questions: []Question{dup}}, "twice"},
		"blank label":     {Request{Questions: []Question{blank}}, "no label"},
		"a good question": {Request{Questions: []Question{twoOptions()}}, ""},
	}
	for name, c := range cases {
		err := Validate(c.req)
		switch {
		case c.want == "" && err != nil:
			t.Errorf("%s: rejected a valid request: %v", name, err)
		case c.want == "":
		case err == nil:
			t.Errorf("%s: accepted", name)
		case !strings.Contains(err.Error(), c.want):
			t.Errorf("%s: %q does not mention %q", name, err, c.want)
		}
	}
}

// The model reads the result many turns later, by which time a bare label has
// stopped meaning anything. The question has to travel with the answer.
func TestFormatRepeatsTheQuestion(t *testing.T) {
	got := Response{Answers: []Answer{
		{Header: "Storage", Question: "How should sessions be stored?", Chosen: []string{"Postgres"}},
		{Header: "Scope", Question: "What should it cover?", Chosen: []string{"the CLI", "the TUI"}},
		{Header: "Naming", Question: "What should it be called?", Other: "call it sessiondir"},
	}}.Format()

	for _, want := range []string{
		"How should sessions be stored?", "Postgres",
		"the CLI, the TUI",
		"in their own words: call it sessiondir",
	} {
		if !strings.Contains(got, want) {
			t.Errorf("missing %q from:\n%s", want, got)
		}
	}
}

func TestAskerForwardsAndCanBeReplaced(t *testing.T) {
	a := NewAsker(nil)
	if _, err := a.Ask(context.Background(), Request{}); err == nil {
		t.Error("an asker with nothing installed should say so rather than hang")
	}

	a.Use(func(context.Context, Request) (Response, error) {
		return Response{Answers: []Answer{{Chosen: []string{"first"}}}}, nil
	})
	got, err := a.Ask(context.Background(), Request{})
	if err != nil || got.Answers[0].Chosen[0] != "first" {
		t.Fatalf("Ask = %+v, %v", got, err)
	}

	// The TUI installs itself over the line front end at startup; the tool is
	// already built by then and must follow the swap.
	a.Use(func(context.Context, Request) (Response, error) {
		return Response{Answers: []Answer{{Chosen: []string{"second"}}}}, nil
	})
	if got, _ := a.Ask(context.Background(), Request{}); got.Answers[0].Chosen[0] != "second" {
		t.Errorf("Ask = %+v, want the prompt installed last", got)
	}
}

func TestToolAnswersTheModel(t *testing.T) {
	var seen Request
	tool := Tool(func(_ context.Context, r Request) (Response, error) {
		seen = r
		return Response{Answers: []Answer{{Header: "Storage", Question: "Where?", Chosen: []string{"Postgres"}}}}, nil
	})

	out, err := tool.Execute(context.Background(), json.RawMessage(`{"questions":[
		{"header":"Storage","question":"Where?","options":[
			{"label":"Postgres","description":"already running"},
			{"label":"SQLite"}]}]}`))
	if err != nil {
		t.Fatalf("Execute: %v", err)
	}
	if !strings.Contains(out, "Postgres") {
		t.Errorf("result does not carry the answer: %q", out)
	}
	if len(seen.Questions) != 1 || seen.Questions[0].Options[0].Description != "already running" {
		t.Errorf("the prompt saw %+v", seen)
	}
}

// A dismissal is not an answer, and reporting it as one would have the model
// act on a decision nobody made.
func TestToolReportsDismissalAsAnError(t *testing.T) {
	tool := Tool(func(context.Context, Request) (Response, error) {
		return Response{Dismissed: true}, nil
	})
	out, err := tool.Execute(context.Background(), json.RawMessage(`{"questions":[
		{"header":"Storage","question":"Where?","options":[{"label":"a"},{"label":"b"}]}]}`))
	if err == nil {
		t.Fatalf("a dismissed question returned %q and no error", out)
	}
	if !strings.Contains(err.Error(), "pick the option you think best") {
		t.Errorf("the error should say what to do instead, got %q", err)
	}
}

// A call that cannot be drawn never reaches the user: the model is told to fix
// it while the screen is still quiet.
func TestToolValidatesBeforeAsking(t *testing.T) {
	called := false
	tool := Tool(func(context.Context, Request) (Response, error) {
		called = true
		return Response{}, nil
	})
	if _, err := tool.Execute(context.Background(), json.RawMessage(`{"questions":[
		{"header":"Storage","question":"Where?","options":[{"label":"only one"}]}]}`)); err == nil {
		t.Fatal("a one-option question was accepted")
	}
	if called {
		t.Error("the user was asked a question that could not be drawn")
	}
}

func TestToolWithoutAPromptSaysSoRatherThanPanicking(t *testing.T) {
	_, err := Tool(nil).Execute(context.Background(), json.RawMessage(`{"questions":[
		{"header":"Storage","question":"Where?","options":[{"label":"a"},{"label":"b"}]}]}`))
	if err == nil {
		t.Fatal("expected an error")
	}
}

func TestToolPassesTheErrorFromACancelledTurn(t *testing.T) {
	want := errors.New("turn cancelled")
	_, err := Tool(func(context.Context, Request) (Response, error) {
		return Response{}, want
	}).Execute(context.Background(), json.RawMessage(`{"questions":[
		{"header":"Storage","question":"Where?","options":[{"label":"a"},{"label":"b"}]}]}`))
	if !errors.Is(err, want) {
		t.Fatalf("err = %v, want the prompt's own error", err)
	}
}

// The schema is the only thing the model sees before it calls. A field
// description that stops at the first comma is worse than none: it reads as a
// complete instruction and is half of one.
func TestSchemaKeepsWholeDescriptions(t *testing.T) {
	schema := string(Tool(nil).InputSchema())
	for _, want := range []string{
		"1 to 4 of them",
		"2 to 4 of them",
		"the consequence, not the label again",
	} {
		if !strings.Contains(schema, want) {
			t.Errorf("schema lost %q:\n%s", want, schema)
		}
	}
}