// Package websearch is everything in Codey that reaches the open internet.
//
// It is one package rather than part of internal/tools so that the network
// lives behind a seam: the tools call Search and Fetch, and the rules about
// what may be dialled, how long it may take and how much may be read are in
// here where they can be read in one sitting. It imports no other internal
// package.
package websearch
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os/exec"
"strconv"
"strings"
"time"
)
// DefaultCommand is the search this package runs when nothing else is
// configured: the DuckDuckGo CLI, `pipx install ddgs`.
//
// It is the default because it needs no account and no API key. A search tool
// whose first instruction is "go and register for an API key" is a search tool
// most people never turn on, and the ones who do have a key can say so.
//
// Note what is not in it: `-o json`. That flag does not print JSON, it writes
// a file — `text_my_query_20260918_205106.json`, into whatever directory the
// agent happens to be working in. A default that returns nothing at all and
// litters the user's repository once per search is worse than having no
// default, so the plain stdout format is asked for and parsed instead. `-nc`
// turns off the colour codes that would otherwise land inside every title.
const DefaultCommand = "ddgs text -q {{query}} -m {{count}} -nc"
// Placeholders the command line may contain. They are substituted into
// individual argv elements after the line has been split, never into the line
// before it — the query is written by the model, and a query interpolated into
// a string that later meets a shell is a command-injection hole with extra
// steps.
const (
queryPlaceholder = "{{query}}"
countPlaceholder = "{{count}}"
)
// commandTimeout is how long a search may take. A search is something the user
// is waiting through, so it is a fraction of the bash tool's minute: a provider
// that has not answered in half a minute is a provider to give up on and say so.
const commandTimeout = 30 * time.Second
// Result is one search hit, reduced to the three things a coding agent can use.
type Result struct {
Title string
URL string
Snippet string
}
// Searcher is a source of search results. The interface exists so the tool can
// be tested against a stub and so a second provider is a new file rather than a
// change to the tool.
type Searcher interface {
// Name is what the provider is called, for the line the model reads under
// its results.
Name() string
Search(ctx context.Context, query string, count int) ([]Result, error)
}
// command is a Searcher that runs an external program and reads JSON off its
// stdout.
type command struct {
argv []string
name string
}
// NewCommand builds a Searcher from a command line containing {{query}} and,
// optionally, {{count}}. An empty line means DefaultCommand.
//
// The line is split here, once, so a template that cannot be parsed is an error
// at startup — where the user is looking — rather than on the first search,
// where only the model would see it.
func NewCommand(template string) (Searcher, error) {
line := strings.TrimSpace(template)
if line == "" {
line = DefaultCommand
}
argv, err := splitArgs(line)
if err != nil {
return nil, fmt.Errorf("search command %q: %w", line, err)
}
if len(argv) == 0 {
return nil, fmt.Errorf("search command is empty")
}
if !strings.Contains(line, queryPlaceholder) {
// Without it the command would run the same search every time, which
// looks like a broken provider rather than a broken config.
return nil, fmt.Errorf("search command %q does not contain %s", line, queryPlaceholder)
}
return &command{argv: argv, name: baseName(argv[0])}, nil
}
// Available reports whether the command's program is on PATH. The caller uses
// it to decide whether to offer the tool at all: an agent told it can search,
// which then discovers on its first attempt that nothing is installed, has
// spent a turn to learn what startup already knew.
func Available(template string) error {
s, err := NewCommand(template)
if err != nil {
return err
}
bin := s.(*command).argv[0]
if _, err := exec.LookPath(bin); err != nil {
if bin == "ddgs" {
return fmt.Errorf("%s is not on PATH — install it with `pipx install ddgs`, or set another search command", bin)
}
return fmt.Errorf("%s is not on PATH", bin)
}
return nil
}
func (c *command) Name() string { return c.name }
func (c *command) Search(ctx context.Context, query string, count int) ([]Result, error) {
query = strings.TrimSpace(query)
if query == "" {
return nil, errors.New("query is required")
}
if count <= 0 {
count = 5
}
argv := make([]string, len(c.argv))
for i, a := range c.argv {
a = strings.ReplaceAll(a, queryPlaceholder, query)
argv[i] = strings.ReplaceAll(a, countPlaceholder, strconv.Itoa(count))
}
ctx, cancel := context.WithTimeout(ctx, commandTimeout)
defer cancel()
// No shell. argv goes to execve as it stands, so a query containing a
// semicolon, a backtick or $(…) is a query and not a second command.
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
var stdout, stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
runErr := cmd.Run()
if runErr != nil {
if errors.Is(runErr, exec.ErrNotFound) {
return nil, notInstalled(argv[0])
}
if ctx.Err() == context.DeadlineExceeded {
return nil, fmt.Errorf("%s did not answer within %s", c.name, commandTimeout)
}
// Exit 127 is what a shell wrapper reports for "not found", which is
// what the user actually needs to hear.
var exitErr *exec.ExitError
if errors.As(runErr, &exitErr) && exitErr.ExitCode() == 127 {
return nil, notInstalled(argv[0])
}
return nil, fmt.Errorf("%s failed (%v): %s", c.name, runErr, preview(stderr.Bytes(), stdout.Bytes()))
}
results, err := parseResults(stdout.Bytes())
if err != nil {
return nil, fmt.Errorf("could not read %s output: %v — it printed: %s",
c.name, err, preview(stdout.Bytes(), stderr.Bytes()))
}
if len(results) > count {
results = results[:count]
}
return results, nil
}
func notInstalled(bin string) error {
if bin == "ddgs" {
return fmt.Errorf("ddgs is not installed — `pipx install ddgs`, or configure another search command")
}
return fmt.Errorf("%s is not installed or not on PATH", bin)
}
// preview is the first part of whatever the command actually printed. It goes
// into the error because "could not parse the output" without the output is a
// message that sends someone to run the command by hand to learn what this
// process already knew.
func preview(streams ...[]byte) string {
const max = 200
for _, s := range streams {
t := strings.TrimSpace(string(s))
if t == "" {
continue
}
t = strings.Join(strings.Fields(t), " ")
if len(t) > max {
t = t[:max] + "…"
}
return t
}
return "(nothing)"
}
// parseResults reads the shapes search tools print: a JSON array, a JSON
// object wrapping one, newline-delimited JSON objects, and ddgs's own aligned
// key/value text. Which one you get depends on the tool and, for ddgs, on its
// version and flags — so all of them are read rather than pinning users to one.
func parseResults(out []byte) ([]Result, error) {
trimmed := bytes.TrimSpace(out)
if len(trimmed) == 0 {
return nil, errors.New("no output")
}
var rows []map[string]any
switch trimmed[0] {
case '[':
if err := json.Unmarshal(trimmed, &rows); err != nil {
return nil, err
}
case '{':
// Either one wrapper object with an array inside, or the first line of
// newline-delimited JSON.
var obj map[string]json.RawMessage
if err := json.Unmarshal(trimmed, &obj); err == nil {
for _, key := range []string{"results", "data", "items", "hits"} {
if raw, ok := obj[key]; ok {
if err := json.Unmarshal(raw, &rows); err != nil {
return nil, err
}
return rowsToResults(rows), nil
}
}
}
var err error
rows, err = parseNDJSON(trimmed)
if err != nil {
return nil, err
}
default:
// Not JSON at all. Before giving up, try the aligned text format —
// this is the branch the shipped default lands in.
if results := parseKeyValueText(trimmed); len(results) > 0 {
return results, nil
}
return nil, fmt.Errorf("not JSON, and not a result listing either")
}
return rowsToResults(rows), nil
}
// parseKeyValueText reads what ddgs prints when it is not asked for a file:
//
// 1.
// title errors package - Go Packages
// href https://pkg.go.dev/errors
// body The error formats as the concatenation
// of the strings obtained by calling…
// 2.
// …
//
// A record opens with a number and a dot. A field is a known key followed by
// two or more spaces; anything else indented is a continuation of the field
// before it, because the tool wraps long values to the terminal width. Only
// the three keys this package has a use for are recognised — an unknown key
// ends the field it follows rather than being folded into it.
func parseKeyValueText(out []byte) []Result {
var (
results []Result
cur Result
open bool
field *string
)
flush := func() {
if open && cur.URL != "" {
if cur.Title == "" {
cur.Title = cur.URL
}
results = append(results, cur)
}
cur, open, field = Result{}, false, nil
}
for _, raw := range strings.Split(string(out), "\n") {
line := strings.TrimRight(raw, " \t\r")
if strings.TrimSpace(line) == "" {
continue
}
// "12." on its own opens the next record.
if t := strings.TrimSpace(line); strings.HasSuffix(t, ".") && isDigits(t[:len(t)-1]) {
flush()
open = true
continue
}
// An indented line with no key of its own continues the last field.
if strings.HasPrefix(line, " ") && field != nil {
if key, _, isField := splitField(strings.TrimSpace(line)); !isField || fieldFor(&cur, key) == nil {
*field = strings.TrimSpace(*field + " " + strings.TrimSpace(line))
continue
}
}
key, value, isField := splitField(strings.TrimSpace(line))
if !isField {
field = nil
continue
}
target := fieldFor(&cur, key)
if target == nil {
field = nil
continue
}
open = true
*target = value
field = target
}
flush()
return results
}
// splitField cuts "title some value" at the run of spaces that aligns
// the column.
func splitField(line string) (key, value string, ok bool) {
i := strings.Index(line, " ")
if i <= 0 {
return "", "", false
}
return strings.ToLower(line[:i]), strings.TrimSpace(line[i:]), true
}
// fieldFor maps one of ddgs's keys onto the Result field it fills, or nil for
// a key this package does not use.
func fieldFor(r *Result, key string) *string {
switch key {
case "title":
return &r.Title
case "href", "url", "link":
return &r.URL
case "body", "snippet", "description":
return &r.Snippet
}
return nil
}
func isDigits(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if r < '0' || r > '9' {
return false
}
}
return true
}
func parseNDJSON(out []byte) ([]map[string]any, error) {
var rows []map[string]any
for _, line := range bytes.Split(out, []byte("\n")) {
line = bytes.TrimSpace(line)
if len(line) == 0 {
continue
}
var row map[string]any
if err := json.Unmarshal(line, &row); err != nil {
return nil, err
}
rows = append(rows, row)
}
if len(rows) == 0 {
return nil, errors.New("no JSON objects")
}
return rows, nil
}
// rowsToResults maps whatever the tool called its fields onto the three this
// package has a use for. The alternatives are not a wish list: ddgs says
// title/href/body, most APIs say title/url/snippet, and a few say link and
// description.
func rowsToResults(rows []map[string]any) []Result {
out := make([]Result, 0, len(rows))
for _, row := range rows {
lower := make(map[string]any, len(row))
for k, v := range row {
lower[strings.ToLower(k)] = v
}
r := Result{
Title: firstString(lower, "title", "heading", "name"),
URL: firstString(lower, "href", "url", "link"),
Snippet: firstString(lower, "body", "snippet", "description", "abstract", "content", "excerpt"),
}
// A hit with no URL is not a search result; it is a row of something
// else that happened to be on stdout.
if r.URL == "" {
continue
}
if r.Title == "" {
r.Title = r.URL
}
out = append(out, r)
}
return out
}
func firstString(row map[string]any, keys ...string) string {
for _, k := range keys {
if v, ok := row[k].(string); ok && strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
// splitArgs splits a command line into argv the way a shell would for the one
// case that matters here — quoted arguments — and in no other way. There is no
// expansion, no globbing, no substitution and no operators: this is a list of
// arguments written in a config file, not a program.
func splitArgs(line string) ([]string, error) {
var (
args []string
cur strings.Builder
open bool
quot rune
)
flush := func() {
if open || cur.Len() > 0 {
args = append(args, cur.String())
cur.Reset()
}
}
for _, r := range line {
switch {
case quot != 0:
if r == quot {
quot = 0
continue
}
cur.WriteRune(r)
case r == '\'' || r == '"':
quot = r
open = true
case r == ' ' || r == '\t':
flush()
open = false
default:
cur.WriteRune(r)
}
}
if quot != 0 {
return nil, errors.New("unbalanced quote")
}
flush()
return args, nil
}
// baseName is the program's name without its directory, for the line under the
// results. A user who configured /opt/homebrew/bin/ddgs wants to read "ddgs".
func baseName(p string) string {
if i := strings.LastIndexAny(p, `/\`); i >= 0 && i+1 < len(p) {
return p[i+1:]
}
return p
}