Vectrum
Sign In
282 lines · 9.5 KB · Go
// Package attach holds the pictures a prompt carries.
//
// A terminal cannot paste an image. Ctrl-V over a running full-screen program
// delivers whatever text form the clipboard has — for a screenshot, usually
// nothing at all, sometimes a file path the window manager put there. So the
// image is not taken from the keystroke: the keystroke is the signal to go and
// ask the system clipboard directly, with the same helper a person would run
// by hand (wl-paste, xclip, pngpaste, PowerShell).
//
// What lands in the prompt box is a marker, [image 1]. It is text like any
// other text: it can be moved, retyped, or deleted, and deleting it is how a
// paste is taken back — only markers still standing when enter is pressed turn
// into image blocks, in the place where they stand.
package attach

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"codey/internal/llm"
)

// MaxBytes caps one image. The API refuses anything over 5MB, and it counts
// the encoded payload, so the raw ceiling sits below that by the 4/3 base64
// grows by.
const MaxBytes = 3_750_000

// ErrNoImage means the clipboard held no picture. It is the ordinary answer to
// ctrl-V with text on the clipboard, not a fault, and the caller should fall
// through to pasting text rather than complain.
var ErrNoImage = errors.New("no image on the clipboard")

// ErrTooBig is an image found and refused. Unlike a format that is not an
// image at all, this one is worth interrupting someone over: they pasted a
// picture and it is not going to arrive.
var ErrTooBig = errors.New("image too large")

// Image is one picture waiting for the next prompt. Data is the raw file, not
// base64: encoding happens once, on the way out.
type Image struct {
	Media  string
	Data   []byte
	Origin string // "clipboard", or the path it was read from — for the status line
}

// Block renders the image as an API content block.
func (im Image) Block() llm.ContentBlock {
	return llm.ImageBlock(im.Media, base64.StdEncoding.EncodeToString(im.Data))
}

// Sniff identifies an image by its leading bytes. Extensions lie — a .png that
// is really a JPEG is common enough, and the API checks the bytes — so the
// magic number is what decides, and a format the API does not take is not an
// image as far as this package is concerned.
func Sniff(b []byte) (string, bool) {
	switch {
	case bytes.HasPrefix(b, []byte("\x89PNG\r\n\x1a\n")):
		return "image/png", true
	case bytes.HasPrefix(b, []byte{0xff, 0xd8, 0xff}):
		return "image/jpeg", true
	case bytes.HasPrefix(b, []byte("GIF87a")), bytes.HasPrefix(b, []byte("GIF89a")):
		return "image/gif", true
	case len(b) >= 12 && bytes.HasPrefix(b, []byte("RIFF")) && bytes.Equal(b[8:12], []byte("WEBP")):
		return "image/webp", true
	}
	return "", false
}

// New wraps bytes that are already in hand, checking them the way anything
// else that becomes an Image is checked.
func New(data []byte, origin string) (Image, error) {
	if len(data) == 0 {
		return Image{}, ErrNoImage
	}
	if len(data) > MaxBytes {
		return Image{}, fmt.Errorf("%w: it is %s, over the %s the API takes", ErrTooBig, size(len(data)), size(MaxBytes))
	}
	media, ok := Sniff(data)
	if !ok {
		return Image{}, errors.New("not a PNG, JPEG, GIF or WebP")
	}
	return Image{Media: media, Data: data, Origin: origin}, nil
}

// FromFile reads an image off disk. This is the path a drag-and-drop takes:
// most terminals answer a dropped file by pasting its name.
func FromFile(path string) (Image, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return Image{}, err
	}
	im, err := New(data, filepath.Base(path))
	if err != nil {
		return Image{}, fmt.Errorf("%s: %w", filepath.Base(path), err)
	}
	return im, nil
}

func size(n int) string {
	if n < 1<<20 {
		return fmt.Sprintf("%dKB", n>>10)
	}
	return fmt.Sprintf("%.1fMB", float64(n)/(1<<20))
}

// imageExts are the endings worth trying to open. A pasted path is only read
// from disk when it looks like an image; every other paste is text, and text
// that happens to name a file should stay text.
var imageExts = map[string]bool{
	".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".webp": true,
}

// Path pulls a local image path out of pasted text, and reports whether the
// paste was one. Terminals hand over a dropped file in several dresses: bare,
// quoted, percent-escaped behind file://, or with the spaces backslashed the
// way a shell would want them.
func Path(s string) (string, bool) {
	s = strings.TrimSpace(s)
	if s == "" || strings.ContainsAny(s, "\n\r") {
		return "", false // more than one line is prose, not a file name
	}
	s = strings.Trim(s, "'\"")
	if rest, ok := strings.CutPrefix(s, "file://"); ok {
		if i := strings.Index(rest, "/"); i > 0 {
			rest = rest[i:] // drop a host, including the empty one in file:///
		}
		s = unescapeURL(rest)
	} else if !strings.Contains(s, `\\`) {
		// A dropped name arrives shell-quoted: "my holiday.png" as
		// my\ holiday.png. Undoing that is only safe where the backslash is
		// not itself a path separator, so Windows paths are left alone.
		s = strings.ReplaceAll(s, `\ `, " ")
	}
	if !imageExts[strings.ToLower(filepath.Ext(s))] {
		return "", false
	}
	if strings.HasPrefix(s, "~/") {
		if home, err := os.UserHomeDir(); err == nil {
			s = filepath.Join(home, s[2:])
		}
	}
	if fi, err := os.Stat(s); err != nil || fi.IsDir() {
		return "", false
	}
	return s, true
}

func unescapeURL(s string) string {
	var b strings.Builder
	for i := 0; i < len(s); i++ {
		if s[i] == '%' && i+2 < len(s) {
			if n, err := strconv.ParseUint(s[i+1:i+3], 16, 8); err == nil {
				b.WriteByte(byte(n))
				i += 2
				continue
			}
		}
		b.WriteByte(s[i])
	}
	return b.String()
}

// ---- markers ------------------------------------------------------------

var markerRE = regexp.MustCompile(`\[image (\d+)\]`)

// Marker is what stands in the prompt box for image n, counting from one.
func Marker(n int) string { return "[image " + strconv.Itoa(n) + "]" }

// Set is the images pasted into the prompt being typed. It is emptied when
// that prompt is sent.
type Set struct {
	imgs []Image
}

// ExpandPaths turns image paths left in a prompt into attached images and
// returns the text with each one replaced by its marker.
//
// This is the backstop under the paste key, and it is what actually carries
// most terminals. Catching a paste as it happens needs the terminal to bracket
// it — to wrap it in an escape sequence meaning "this was pasted" — and plenty
// of them do not. Text that arrives as ordinary keystrokes cannot be told from
// typing, so it is not caught on the way in; it is caught here, on the way
// out, where how it arrived has stopped mattering.
//
// Only a path that really is a readable image on disk is taken. A filename in
// a sentence, a path to source, a file that is not there: all left exactly as
// written, because a prompt that talks about a file is the ordinary case and
// has to survive untouched.
func (s *Set) ExpandPaths(text string) string {
	// A line that is nothing but one path is tried whole first, so a name with
	// spaces in it survives the splitting below.
	if p, ok := Path(text); ok {
		if im, err := FromFile(p); err == nil {
			return Marker(s.Add(im))
		}
		return text
	}

	// Replacements are gathered before any is made: rewriting the text while
	// walking it would shift everything after the first hit.
	type swap struct{ from, to string }
	var swaps []swap
	seen := map[string]bool{}
	for _, f := range strings.Fields(text) {
		if seen[f] {
			continue // the same path twice is one picture, mentioned twice
		}
		p, ok := Path(f)
		if !ok {
			continue
		}
		im, err := FromFile(p)
		if err != nil {
			continue
		}
		seen[f] = true
		swaps = append(swaps, swap{from: f, to: Marker(s.Add(im))})
	}
	for _, sw := range swaps {
		text = strings.ReplaceAll(text, sw.from, sw.to)
	}
	return text
}

// Add takes an image and returns its number, which is what the marker says.
func (s *Set) Add(im Image) int {
	s.imgs = append(s.imgs, im)
	return len(s.imgs)
}

func (s *Set) Len() int { return len(s.imgs) }

func (s *Set) Reset() { s.imgs = nil }

// Blocks turns a typed prompt into the content of one user message, with each
// surviving marker replaced in place by the picture it names.
//
// A marker whose number names no image is left alone: it is then literally
// what someone typed, and typing "[image 7]" at the model is allowed to mean
// what it says. An image whose marker was deleted is dropped, which is what
// makes deleting the marker an undo.
func (s *Set) Blocks(text string) []llm.ContentBlock {
	var out []llm.ContentBlock
	words := false
	// The marker stood inside a sentence with spaces around it. Those spaces
	// held it apart from the words and have nothing to do once it is gone, so
	// each run of text is trimmed at the edge where a picture was lifted out.
	addText := func(t string) {
		if t = strings.TrimSpace(t); t != "" {
			out = append(out, llm.TextBlock(t))
			words = true
		}
	}

	last := 0
	for _, loc := range markerRE.FindAllStringSubmatchIndex(text, -1) {
		n, err := strconv.Atoi(text[loc[2]:loc[3]])
		if err != nil || n < 1 || n > len(s.imgs) {
			continue // not one of ours; it stays in the text around it
		}
		addText(text[last:loc[0]])
		out = append(out, s.imgs[n-1].Block())
		last = loc[1]
	}
	addText(text[last:])

	if !words {
		// Nothing but markers. The message still needs words, because a user
		// turn of images alone reads as a question with no question in it —
		// and an empty content list is an API error besides.
		out = append(out, llm.TextBlock("(see the image)"))
	}
	return out
}