Vectrum
Sign In
codePublic
Like1 likeFork0 forks

Browser panel the agent and the user share, on Vercel's agent-browser

Each thread gets its own agent-browser session (vectrum-<thread id>). The
agent drives it with the CLI: its process gets AGENT_BROWSER_SESSION and a
short --append-system-prompt (Claude and Pi) when agent-browser is
installed. The panel (Ctrl+Shift+E, or the globe in the top bar) takes the
project panel's place: back/forward/reload, a URL field, an "Agent" light
while the agent runs commands, and the live viewport from the daemon's
WebSocket stream, fit to the panel, taking clicks, scrolling, typing and
pastes. A light on the globe shows a running browser while the panel is
closed. Deleting a thread closes its browser.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Browse Files
JjohannSocommitted 1 parent 0781cf9commitf5941cc

Showing 11 changed files with 1700 additions and 10 deletions.

+35−0Cargo.lockView File
@@ -1313,6 +1313,12 @@ dependencies = [
13131313 "linktime-proc-macro",
13141314 ]
13151315
1316+ [[package]]
1317+ name = "data-encoding"
1318+ version = "2.11.1"
1319+ source = "registry+https://github.com/rust-lang/crates.io-index"
1320+ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
1321+
13161322 [[package]]
13171323 name = "data-url"
13181324 version = "0.3.2"
@@ -6082,6 +6088,17 @@ dependencies = [
60826088 "serde",
60836089 ]
60846090
6091+ [[package]]
6092+ name = "sha1"
6093+ version = "0.11.0"
6094+ source = "registry+https://github.com/rust-lang/crates.io-index"
6095+ checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
6096+ dependencies = [
6097+ "cfg-if",
6098+ "cpufeatures 0.3.1",
6099+ "digest 0.11.3",
6100+ ]
6101+
60856102 [[package]]
60866103 name = "sha1_smol"
60876104 version = "1.0.1"
@@ -7025,6 +7042,22 @@ dependencies = [
70257042 "core_maths",
70267043 ]
70277044
7045+ [[package]]
7046+ name = "tungstenite"
7047+ version = "0.30.0"
7048+ source = "registry+https://github.com/rust-lang/crates.io-index"
7049+ checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22"
7050+ dependencies = [
7051+ "bytes",
7052+ "data-encoding",
7053+ "http",
7054+ "httparse",
7055+ "log",
7056+ "rand 0.10.3",
7057+ "sha1",
7058+ "thiserror 2.0.21",
7059+ ]
7060+
70287061 [[package]]
70297062 name = "typeid"
70307063 version = "1.0.3"
@@ -7299,10 +7332,12 @@ dependencies = [
72997332 "base64 0.22.1",
73007333 "chrono",
73017334 "gpui-kit",
7335+ "image",
73027336 "libc",
73037337 "serde",
73047338 "serde_json",
73057339 "smol",
7340+ "tungstenite",
73067341 "ureq",
73077342 ]
73087343
+2−0Cargo.tomlView File
@@ -14,6 +14,8 @@ smol = "2"
1414 base64 = "0.22"
1515 ureq = { version = "2", default-features = false, features = ["tls"] }
1616 chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
17+ tungstenite = { version = "0.30", default-features = false, features = ["handshake"] }
18+ image = { version = "0.25", default-features = false, features = ["jpeg"] }
1719
1820 [profile.dev]
1921 debug = "line-tables-only"
+7−0README.mdView File
@@ -53,6 +53,11 @@ follow the desktop.
5353 Sublime Text, Kate or the file manager, whichever are installed), the branch with a
5454 switcher, **Commit** (asks the agent to commit, or commit and push), the size of the
5555 uncommitted changes, and the thread's cost.
56+ - A browser the agent and you share (Ctrl+Shift+E, or the globe in the top bar), on
57+ Vercel's [agent-browser](https://github.com/vercel-labs/agent-browser): each thread has
58+ its own headless Chrome session, which its agent drives with the `agent-browser` CLI
59+ while the panel shows it live. Click, scroll and type in the page yourself, or go to a
60+ URL. When the agent opens a page while the panel is closed, a light on the globe says so.
5661 - Interrupt with Esc or the stop button. Messages sent while the agent works are queued.
5762 - Crash recovery: if a process dies, the next message restarts it and resumes the session.
5863 - A settings view (Ctrl+, or **Settings** in the sidebar), laid out like T3 Code's:
@@ -71,6 +76,8 @@ follow the desktop.
7176 - The `claude` CLI, logged in (`claude` once interactively)
7277 - Optional: [Pi](https://github.com/earendil-works/pi) for Pi threads
7378 (`npm install -g @earendil-works/pi-coding-agent`, then log in to a provider with `pi`)
79+ - Optional: agent-browser for the browser panel
80+ (`npm install -g agent-browser && agent-browser install`)
7481 - GPUI's Linux system libraries: Vulkan, Wayland/X11, xkbcommon, fontconfig, freetype
7582 (on Arch/CachyOS: `vulkan-icd-loader` plus your GPU's Vulkan driver, `wayland`,
7683 `libxkbcommon-x11`, `libxcb`, `fontconfig`, `freetype2`)
+926−0src/browser.rsaddedView File
@@ -0,0 +1,926 @@
1+ //! The browser a thread shares with its agent: a headless Chrome run by
2+ //! Vercel's `agent-browser` CLI, one session per thread. The agent drives it
3+ //! with CLI commands. The app runs the same commands for the panel's buttons,
4+ //! watches the daemon's viewport stream (JPEG frames over a WebSocket on
5+ //! localhost) and sends the user's mouse and keyboard input back over it.
6+
7+ use std::{
8+ collections::HashSet,
9+ io::ErrorKind,
10+ net::{SocketAddr, TcpStream},
11+ path::{Path, PathBuf},
12+ process::{Command, Stdio},
13+ sync::{
14+ Arc, Mutex,
15+ atomic::{AtomicBool, Ordering},
16+ mpsc,
17+ },
18+ thread,
19+ time::{Duration, Instant},
20+ };
21+
22+ use anyhow::{Context as _, Result, anyhow};
23+ use base64::Engine as _;
24+ use gpui_kit::RenderImage;
25+ use serde::{Deserialize, Serialize};
26+ use serde_json::Value;
27+ use tungstenite::{Message, WebSocket};
28+
29+ use crate::claude::is_executable;
30+
31+ pub const INSTALL_COMMAND: &str = "npm install -g agent-browser && agent-browser install";
32+ const BINARY: &str = "agent-browser";
33+ /// The stream sends at most this many frames a second.
34+ const MAX_FPS: u32 = 30;
35+ /// How long a socket read waits before the stream thread sends queued input.
36+ const POLL: Duration = Duration::from_millis(10);
37+ /// How often the stream thread looks for a daemon while none runs.
38+ const WAIT: Duration = Duration::from_millis(500);
39+ /// A command the panel ran shows up in the stream within this long, if the
40+ /// stream was connected when it started; a later one is the agent's.
41+ const OWN_COMMAND: Duration = Duration::from_secs(3);
42+
43+ /// The agent-browser session of a thread.
44+ pub fn session_name(thread_id: &str) -> String {
45+ format!("vectrum-{thread_id}")
46+ }
47+
48+ /// Where `agent-browser` is: PATH, then the usual npm, bun and Homebrew bin folders.
49+ pub fn find_binary() -> Option<PathBuf> {
50+ let on_path = std::env::var_os("PATH")
51+ .into_iter()
52+ .flat_map(|paths| std::env::split_paths(&paths).collect::<Vec<_>>())
53+ .map(|dir| dir.join(BINARY));
54+ // Desktop launchers often start apps without the user's shell PATH.
55+ let home = std::env::var_os("HOME").map(PathBuf::from).unwrap_or_default();
56+ let mut fallbacks: Vec<PathBuf> = [".npm-global/bin", ".local/bin", ".bun/bin", ".volta/bin", ".local/share/pnpm"]
57+ .iter()
58+ .map(|dir| home.join(dir).join(BINARY))
59+ .collect();
60+ fallbacks.extend(
61+ ["/home/linuxbrew/.linuxbrew/bin", "/opt/homebrew/bin", "/usr/local/bin", "/usr/bin"]
62+ .iter()
63+ .map(|dir| Path::new(dir).join(BINARY)),
64+ );
65+ // nvm keeps one bin directory per Node version.
66+ if let Ok(versions) = std::fs::read_dir(home.join(".nvm/versions/node")) {
67+ fallbacks.extend(versions.flatten().map(|version| version.path().join("bin").join(BINARY)));
68+ }
69+ on_path.chain(fallbacks).find(|path| is_executable(path))
70+ }
71+
72+ /// The folder agent-browser keeps its daemons' sockets, pid and port files in.
73+ fn run_dir() -> PathBuf {
74+ let var = |name: &str| std::env::var_os(name).filter(|value| !value.is_empty());
75+ let base = var("AGENT_BROWSER_SOCKET_DIR").map(PathBuf::from).unwrap_or_else(|| match var("XDG_RUNTIME_DIR") {
76+ Some(dir) => PathBuf::from(dir).join("agent-browser"),
77+ None => var("HOME").map(PathBuf::from).unwrap_or_else(std::env::temp_dir).join(".agent-browser"),
78+ });
79+ match std::env::var("AGENT_BROWSER_NAMESPACE") {
80+ Ok(namespace) if !namespace.is_empty() => base.join("namespaces").join(namespace).join("run"),
81+ _ => base,
82+ }
83+ }
84+
85+ fn pid_alive(pid: i32) -> bool {
86+ pid > 0 && unsafe { libc::kill(pid, 0) } == 0
87+ }
88+
89+ /// Whether a session's daemon runs, from its pid file.
90+ pub fn is_running(session: &str) -> bool {
91+ std::fs::read_to_string(run_dir().join(format!("{session}.pid")))
92+ .ok()
93+ .and_then(|pid| pid.trim().parse().ok())
94+ .is_some_and(pid_alive)
95+ }
96+
97+ /// The sessions whose daemons run.
98+ pub fn running_sessions() -> HashSet<String> {
99+ let Ok(entries) = std::fs::read_dir(run_dir()) else {
100+ return HashSet::new();
101+ };
102+ entries
103+ .flatten()
104+ .filter_map(|entry| {
105+ let name = entry.file_name().into_string().ok()?;
106+ let session = name.strip_suffix(".pid")?;
107+ let pid = std::fs::read_to_string(entry.path()).ok()?.trim().parse().ok()?;
108+ pid_alive(pid).then(|| session.to_string())
109+ })
110+ .collect()
111+ }
112+
113+ /// The localhost port a running daemon streams its viewport on. Every daemon
114+ /// opens one; the daemon writes it next to its socket.
115+ fn stream_port(session: &str) -> Option<u16> {
116+ if !is_running(session) {
117+ return None;
118+ }
119+ std::fs::read_to_string(run_dir().join(format!("{session}.stream"))).ok()?.trim().parse().ok()
120+ }
121+
122+ /// What the agent is told about the panel.
123+ pub fn agent_prompt(session: &str) -> String {
124+ format!(
125+ "The user's app has a browser panel showing a shared headless browser that you drive with the \
126+ `agent-browser` CLI. Its session is `{session}`, already set in AGENT_BROWSER_SESSION, so plain \
127+ `agent-browser <command>` uses it; don't pass another --session. The user watches the browser \
128+ live and may click and type in it too. Core commands: `agent-browser open <url>`, \
129+ `agent-browser snapshot -i` (interactive elements with @refs), `agent-browser click @e2`, \
130+ `agent-browser fill @e3 \"text\"`, `agent-browser screenshot [path]`, `agent-browser get url`, \
131+ `agent-browser back|forward|reload`. `agent-browser skills get core` has the full guide. Use it \
132+ to check web pages you build or need to see, and leave the page open when you're done so the \
133+ user can look at it."
134+ )
135+ }
136+
137+ /// Sets a thread's browser session in an agent's environment, with
138+ /// `agent-browser`'s folder on PATH, and returns the prompt that tells the
139+ /// agent about it. `None` when agent-browser isn't installed.
140+ pub fn prepare_agent(command: &mut Command, session: &str) -> Option<String> {
141+ let binary = find_binary()?;
142+ command.env("AGENT_BROWSER_SESSION", session);
143+ if let Some(dir) = binary.parent() {
144+ let current = command
145+ .get_envs()
146+ .find(|(key, _)| *key == "PATH")
147+ .and_then(|(_, value)| value.map(ToOwned::to_owned))
148+ .or_else(|| std::env::var_os("PATH"))
149+ .unwrap_or_default();
150+ let mut paths: Vec<PathBuf> = std::env::split_paths(&current).collect();
151+ if !paths.iter().any(|path| path == dir) {
152+ paths.push(dir.to_path_buf());
153+ if let Ok(joined) = std::env::join_paths(paths) {
154+ command.env("PATH", joined);
155+ }
156+ }
157+ }
158+ Some(agent_prompt(session))
159+ }
160+
161+ /// Closes a session's browser and daemon, if it runs.
162+ pub fn stop(session: &str) {
163+ if is_running(session)
164+ && let Some(binary) = find_binary()
165+ {
166+ let _ = Command::new(binary)
167+ .args(["--session", session, "close"])
168+ .stdin(Stdio::null())
169+ .stdout(Stdio::null())
170+ .stderr(Stdio::null())
171+ .status();
172+ }
173+ }
174+
175+ /// What the user typed into the address field, as a URL: `https://` is added
176+ /// when there's no scheme, `http://` for local servers.
177+ pub fn normalize_url(input: &str) -> Option<String> {
178+ let input = input.trim();
179+ if input.is_empty() {
180+ return None;
181+ }
182+ if input.contains("://") || ["about:", "data:", "file:", "chrome:"].iter().any(|scheme| input.starts_with(scheme)) {
183+ return Some(input.to_string());
184+ }
185+ let host = input.split(['/', '?', '#']).next().unwrap_or_default();
186+ let host = host
187+ .rsplit_once(':')
188+ .filter(|(_, port)| port.chars().all(|c| c.is_ascii_digit()))
189+ .map_or(host, |(host, _)| host);
190+ let local =
191+ matches!(host, "localhost" | "0.0.0.0" | "[::1]") || host.starts_with("127.") || host.ends_with(".localhost");
192+ Some(format!("{}://{input}", if local { "http" } else { "https" }))
193+ }
194+
195+ // ---- the stream protocol --------------------------------------------------
196+
197+ /// A message the daemon's stream sends.
198+ #[derive(Debug, Deserialize)]
199+ #[serde(tag = "type", rename_all = "snake_case")]
200+ pub enum StreamMessage {
201+ /// A screencast frame: a base64 JPEG of the viewport.
202+ Frame { data: String, metadata: FrameMetadata },
203+ /// The active tab navigated.
204+ Url { url: String },
205+ /// Sent on connect and when the browser or screencast starts or stops.
206+ Status {
207+ /// Whether a browser runs; the daemon can run without one.
208+ #[serde(default)]
209+ connected: bool,
210+ },
211+ /// A command the daemon started running, from any client.
212+ Command { action: String },
213+ #[serde(other)]
214+ Other,
215+ }
216+
217+ #[derive(Debug, Deserialize)]
218+ #[serde(rename_all = "camelCase")]
219+ pub struct FrameMetadata {
220+ /// The viewport in CSS pixels, the space input coordinates are in.
221+ pub device_width: f32,
222+ pub device_height: f32,
223+ }
224+
225+ /// Input sent to the page over the stream; the daemon passes it on to
226+ /// Chrome's `Input.dispatchMouseEvent` and `Input.dispatchKeyEvent`.
227+ #[derive(Clone, Debug, Serialize)]
228+ #[serde(tag = "type")]
229+ pub enum Input {
230+ #[serde(rename = "input_mouse")]
231+ Mouse(MouseInput),
232+ #[serde(rename = "input_keyboard")]
233+ Key(KeyInput),
234+ }
235+
236+ /// CDP's modifier bits.
237+ pub mod modifiers {
238+ pub const ALT: u8 = 1;
239+ pub const CTRL: u8 = 2;
240+ pub const META: u8 = 4;
241+ pub const SHIFT: u8 = 8;
242+ }
243+
244+ #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
245+ pub enum MouseEventType {
246+ #[serde(rename = "mouseMoved")]
247+ Move,
248+ #[serde(rename = "mousePressed")]
249+ Down,
250+ #[serde(rename = "mouseReleased")]
251+ Up,
252+ #[serde(rename = "mouseWheel")]
253+ Wheel,
254+ }
255+
256+ #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
257+ #[serde(rename_all = "lowercase")]
258+ pub enum PointerButton {
259+ None,
260+ Left,
261+ Middle,
262+ Right,
263+ }
264+
265+ #[derive(Clone, Debug, Serialize)]
266+ #[serde(rename_all = "camelCase")]
267+ pub struct MouseInput {
268+ pub event_type: MouseEventType,
269+ pub x: f64,
270+ pub y: f64,
271+ pub button: PointerButton,
272+ pub click_count: u32,
273+ pub delta_x: f64,
274+ pub delta_y: f64,
275+ pub modifiers: u8,
276+ }
277+
278+ impl MouseInput {
279+ pub fn new(event_type: MouseEventType, (x, y): (f64, f64), modifiers: u8) -> Self {
280+ Self { event_type, x, y, button: PointerButton::None, click_count: 0, delta_x: 0., delta_y: 0., modifiers }
281+ }
282+ }
283+
284+ #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
285+ pub enum KeyEventType {
286+ /// A key press that types `text`.
287+ #[serde(rename = "keyDown")]
288+ Down,
289+ /// A key press that types nothing (arrows, shortcuts).
290+ #[serde(rename = "rawKeyDown")]
291+ RawDown,
292+ #[serde(rename = "keyUp")]
293+ Up,
294+ /// Text typed without a key, like a paste.
295+ #[serde(rename = "char")]
296+ Char,
297+ }
298+
299+ #[derive(Clone, Debug, PartialEq, Serialize)]
300+ #[serde(rename_all = "camelCase")]
301+ pub struct KeyInput {
302+ pub event_type: KeyEventType,
303+ #[serde(skip_serializing_if = "Option::is_none")]
304+ pub key: Option<String>,
305+ #[serde(skip_serializing_if = "Option::is_none")]
306+ pub code: Option<String>,
307+ #[serde(skip_serializing_if = "Option::is_none")]
308+ pub text: Option<String>,
309+ pub windows_virtual_key_code: u32,
310+ pub modifiers: u8,
311+ }
312+
313+ /// A GPUI key press or release as a DOM key event. `key` is GPUI's key name
314+ /// (`a`, `enter`, `left`), `typed` the character it types, if any.
315+ pub fn key_input(key: &str, typed: Option<&str>, modifiers: u8, down: bool) -> KeyInput {
316+ let named = |dom: &str, code: &str, key_code: u32| (dom.to_string(), Some(code.to_string()), key_code);
317+ let (dom_key, code, key_code) = match key {
318+ "enter" => named("Enter", "Enter", 13),
319+ "backspace" => named("Backspace", "Backspace", 8),
320+ "tab" => named("Tab", "Tab", 9),
321+ "escape" => named("Escape", "Escape", 27),
322+ "space" => named(" ", "Space", 32),
323+ "left" => named("ArrowLeft", "ArrowLeft", 37),
324+ "up" => named("ArrowUp", "ArrowUp", 38),
325+ "right" => named("ArrowRight", "ArrowRight", 39),
326+ "down" => named("ArrowDown", "ArrowDown", 40),
327+ "home" => named("Home", "Home", 36),
328+ "end" => named("End", "End", 35),
329+ "pageup" => named("PageUp", "PageUp", 33),
330+ "pagedown" => named("PageDown", "PageDown", 34),
331+ "delete" => named("Delete", "Delete", 46),
332+ "insert" => named("Insert", "Insert", 45),
333+ _ if key.len() > 1 && key.starts_with('f') && key[1..].parse::<u32>().is_ok_and(|n| (1..=12).contains(&n)) => {
334+ let n: u32 = key[1..].parse().unwrap_or(1);
335+ (format!("F{n}"), Some(format!("F{n}")), 111 + n)
336+ }
337+ _ => {
338+ let mut chars = key.chars();
339+ match (chars.next(), chars.next()) {
340+ (Some(c), None) if c.is_ascii_alphabetic() => {
341+ let upper = c.to_ascii_uppercase();
342+ (typed.unwrap_or(key).to_string(), Some(format!("Key{upper}")), upper as u32)
343+ }
344+ (Some(c), None) if c.is_ascii_digit() => {
345+ (typed.unwrap_or(key).to_string(), Some(format!("Digit{c}")), c as u32)
346+ }
347+ _ => (typed.unwrap_or(key).to_string(), None, 0),
348+ }
349+ }
350+ };
351+ let shortcut = modifiers & (modifiers::CTRL | modifiers::META | modifiers::ALT) != 0;
352+ // Enter types a carriage return, which is what submits forms.
353+ let text = match key {
354+ "enter" => Some("\r".to_string()),
355+ "space" => Some(" ".to_string()),
356+ _ => typed.filter(|text| !text.is_empty() && !text.chars().any(char::is_control)).map(str::to_string),
357+ }
358+ .filter(|_| !shortcut);
359+ let event_type = match (down, &text) {
360+ (false, _) => KeyEventType::Up,
361+ (true, Some(_)) => KeyEventType::Down,
362+ (true, None) => KeyEventType::RawDown,
363+ };
364+ KeyInput {
365+ event_type,
366+ key: Some(dom_key),
367+ code,
368+ text: if down { text } else { None },
369+ windows_virtual_key_code: key_code,
370+ modifiers,
371+ }
372+ }
373+
374+ /// Text typed at the caret without keys, as a paste does: one `char` event
375+ /// per character, which is all Chrome takes from one.
376+ pub fn text_input(text: &str) -> Vec<KeyInput> {
377+ text.chars()
378+ .filter(|c| *c != '\r')
379+ .map(|c| KeyInput {
380+ event_type: KeyEventType::Char,
381+ key: None,
382+ code: None,
383+ text: Some(if c == '\n' { "\r".to_string() } else { c.to_string() }),
384+ windows_virtual_key_code: 0,
385+ modifiers: 0,
386+ })
387+ .collect()
388+ }
389+
390+ /// Where a frame of `image` pixels is drawn when fit inside `area` keeping its
391+ /// aspect ratio and centered, as GPUI's `ObjectFit::Contain` draws it:
392+ /// `(x, y, width, height)` relative to the area.
393+ pub fn contain(area: (f32, f32), image: (f32, f32)) -> (f32, f32, f32, f32) {
394+ let (area_w, area_h) = area;
395+ let (image_w, image_h) = image;
396+ if image_w <= 0. || image_h <= 0. || area_w <= 0. || area_h <= 0. {
397+ return (0., 0., 0., 0.);
398+ }
399+ let (w, h) = if area_w / area_h > image_w / image_h {
400+ (image_w * area_h / image_h, area_h)
401+ } else {
402+ (area_w, image_h * area_w / image_w)
403+ };
404+ ((area_w - w) / 2., (area_h - h) / 2., w, h)
405+ }
406+
407+ /// Maps a point in the viewport area (relative to its top left) to page
408+ /// coordinates in CSS pixels. `None` outside the drawn frame.
409+ pub fn page_point(point: (f32, f32), area: (f32, f32), image: (f32, f32), viewport: (f32, f32)) -> Option<(f64, f64)> {
410+ let (x, y, w, h) = contain(area, image);
411+ if w <= 0. || h <= 0. {
412+ return None;
413+ }
414+ let (rel_x, rel_y) = ((point.0 - x) / w, (point.1 - y) / h);
415+ if !(0. ..=1.).contains(&rel_x) || !(0. ..=1.).contains(&rel_y) {
416+ return None;
417+ }
418+ let page_x = (rel_x * viewport.0).min(viewport.0 - 1.).max(0.);
419+ let page_y = (rel_y * viewport.1).min(viewport.1 - 1.).max(0.);
420+ Some((page_x as f64, page_y as f64))
421+ }
422+
423+ // ---- a session ------------------------------------------------------------
424+
425+ /// A decoded viewport frame.
426+ pub struct Frame {
427+ pub image: Arc<RenderImage>,
428+ /// The frame's size in pixels.
429+ pub size: (f32, f32),
430+ /// The page's viewport in CSS pixels.
431+ pub viewport: (f32, f32),
432+ }
433+
434+ /// What a session reports to the panel.
435+ #[derive(Debug, PartialEq)]
436+ pub enum BrowserEvent {
437+ /// `agent-browser` isn't installed.
438+ Missing,
439+ /// No daemon runs for the session.
440+ Waiting,
441+ /// Watching the daemon's stream; `browser` tells whether Chrome runs.
442+ Live {
443+ browser: bool,
444+ },
445+ /// A new frame waits in `BrowserSession::take_frame`.
446+ Frame,
447+ Url(String),
448+ /// The agent (or anyone but the panel) ran a command.
449+ Agent(String),
450+ /// One of the panel's commands failed.
451+ Failed(String),
452+ }
453+
454+ struct Shared {
455+ stop: AtomicBool,
456+ /// Whether the panel shows, so the stream is worth watching.
457+ watching: AtomicBool,
458+ frame: Mutex<Option<Frame>>,
459+ /// Commands the panel ran that the stream hasn't echoed yet.
460+ own: Mutex<Vec<(&'static str, Instant)>>,
461+ }
462+
463+ /// One thread's browser, as the panel drives and watches it.
464+ pub struct BrowserSession {
465+ name: String,
466+ binary: Option<PathBuf>,
467+ shared: Arc<Shared>,
468+ events: smol::channel::Sender<BrowserEvent>,
469+ input: mpsc::Sender<String>,
470+ }
471+
472+ impl BrowserSession {
473+ /// Starts watching a session. Nothing runs until `set_watching(true)`.
474+ pub fn new(name: String, events: smol::channel::Sender<BrowserEvent>) -> Self {
475+ let binary = find_binary();
476+ let shared = Arc::new(Shared {
477+ stop: AtomicBool::new(false),
478+ watching: AtomicBool::new(false),
479+ frame: Mutex::new(None),
480+ own: Mutex::new(Vec::new()),
481+ });
482+ let (input, input_rx) = mpsc::channel();
483+ if binary.is_some() {
484+ let (name, shared, events) = (name.clone(), shared.clone(), events.clone());
485+ let _ = thread::Builder::new()
486+ .name("browser-stream".into())
487+ .spawn(move || watch(&name, &shared, &events, &input_rx));
488+ } else {
489+ let _ = events.try_send(BrowserEvent::Missing);
490+ }
491+ Self { name, binary, shared, events, input }
492+ }
493+
494+ pub fn name(&self) -> &str {
495+ &self.name
496+ }
497+
498+ pub fn installed(&self) -> bool {
499+ self.binary.is_some()
500+ }
501+
502+ /// Connects to the stream while the panel shows, and lets go of it otherwise.
503+ pub fn set_watching(&self, watching: bool) {
504+ self.shared.watching.store(watching, Ordering::Relaxed);
505+ }
506+
507+ pub fn open(&self, url: &str) {
508+ self.run("navigate", &["open", url]);
509+ }
510+
511+ pub fn back(&self) {
512+ self.run("back", &["back"]);
513+ }
514+
515+ pub fn forward(&self) {
516+ self.run("forward", &["forward"]);
517+ }
518+
519+ pub fn reload(&self) {
520+ self.run("reload", &["reload"]);
521+ }
522+
523+ /// Sizes the page's viewport, in CSS pixels at `scale` device pixels each.
524+ pub fn set_viewport(&self, width: u32, height: u32, scale: f32) {
525+ let (width, height, scale) = (width.to_string(), height.to_string(), format!("{scale:.2}"));
526+ self.run("viewport", &["set", "viewport", &width, &height, &scale]);
527+ }
528+
529+ pub fn send(&self, input: Input) {
530+ if let Ok(json) = serde_json::to_string(&input) {
531+ let _ = self.input.send(json);
532+ }
533+ }
534+
535+ /// The newest frame not taken yet.
536+ pub fn take_frame(&self) -> Option<Frame> {
537+ self.shared.frame.lock().ok()?.take()
538+ }
539+
540+ /// Runs `agent-browser --session <name> <args> --json` in the background.
541+ fn run(&self, action: &'static str, args: &[&str]) {
542+ let Some(binary) = self.binary.clone() else {
543+ let _ = self.events.try_send(BrowserEvent::Missing);
544+ return;
545+ };
546+ if let Ok(mut own) = self.shared.own.lock() {
547+ own.retain(|(_, at)| at.elapsed() < OWN_COMMAND);
548+ own.push((action, Instant::now()));
549+ }
550+ let mut command = Command::new(binary);
551+ command.args(["--session", &self.name]).args(args).arg("--json");
552+ let events = self.events.clone();
553+ let _ = thread::Builder::new().name("browser-command".into()).spawn(move || {
554+ if let Err(err) = run_command(command) {
555+ let _ = events.send_blocking(BrowserEvent::Failed(format!("{err:#}")));
556+ }
557+ });
558+ }
559+ }
560+
561+ impl Drop for BrowserSession {
562+ fn drop(&mut self) {
563+ self.shared.stop.store(true, Ordering::Relaxed);
564+ }
565+ }
566+
567+ fn run_command(mut command: Command) -> Result<()> {
568+ let output = command
569+ .stdin(Stdio::null())
570+ .stdout(Stdio::piped())
571+ .stderr(Stdio::piped())
572+ .output()
573+ .context("failed to run agent-browser")?;
574+ let reply: Option<Value> = serde_json::from_slice(&output.stdout).ok();
575+ let success = reply.as_ref().and_then(|reply| reply["success"].as_bool()).unwrap_or(output.status.success());
576+ if success {
577+ return Ok(());
578+ }
579+ let message = reply
580+ .as_ref()
581+ .and_then(|reply| reply["error"].as_str().map(str::to_string))
582+ .or_else(|| {
583+ let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
584+ (!stderr.is_empty()).then_some(stderr)
585+ })
586+ .unwrap_or_else(|| format!("agent-browser exited with {}", output.status));
587+ Err(anyhow!(message))
588+ }
589+
590+ /// The stream thread: waits for the session's daemon, watches its stream
591+ /// while the panel shows, and reconnects when the daemon restarts.
592+ fn watch(name: &str, shared: &Shared, events: &smol::channel::Sender<BrowserEvent>, input: &mpsc::Receiver<String>) {
593+ let mut waiting = false;
594+ while !shared.stop.load(Ordering::Relaxed) && !events.is_closed() {
595+ if !shared.watching.load(Ordering::Relaxed) {
596+ while input.try_recv().is_ok() {}
597+ thread::sleep(WAIT / 2);
598+ continue;
599+ }
600+ let socket = stream_port(name).map(connect);
601+ match socket {
602+ Some(Ok(mut socket)) => {
603+ waiting = false;
604+ stream(&mut socket, shared, events, input);
605+ let _ = socket.close(None);
606+ let _ = socket.flush();
607+ }
608+ Some(Err(_)) => thread::sleep(WAIT),
609+ None => {
610+ if !waiting {
611+ waiting = true;
612+ let _ = events.send_blocking(BrowserEvent::Waiting);
613+ }
614+ while input.try_recv().is_ok() {}
615+ thread::sleep(WAIT);
616+ }
617+ }
618+ }
619+ }
620+
621+ fn connect(port: u16) -> Result<WebSocket<TcpStream>> {
622+ let address = SocketAddr::from(([127, 0, 0, 1], port));
623+ let tcp = TcpStream::connect_timeout(&address, Duration::from_secs(2))?;
624+ tcp.set_nodelay(true)?;
625+ // A daemon that just started or is shutting down may never answer.
626+ tcp.set_read_timeout(Some(Duration::from_secs(3)))?;
627+ let url = format!("ws://127.0.0.1:{port}/?maxFps={MAX_FPS}");
628+ let (socket, _) = tungstenite::client(url, tcp).map_err(|err| anyhow!("stream handshake failed: {err}"))?;
629+ socket.get_ref().set_read_timeout(Some(POLL))?;
630+ Ok(socket)
631+ }
632+
633+ /// Relays one connection: frames and page changes to the panel, the user's
634+ /// input to the page. Returns when the connection ends or isn't wanted.
635+ fn stream(
636+ socket: &mut WebSocket<TcpStream>,
637+ shared: &Shared,
638+ events: &smol::channel::Sender<BrowserEvent>,
639+ input: &mpsc::Receiver<String>,
640+ ) {
641+ loop {
642+ if shared.stop.load(Ordering::Relaxed) || !shared.watching.load(Ordering::Relaxed) {
643+ return;
644+ }
645+ while let Ok(message) = input.try_recv() {
646+ if socket.send(Message::text(message)).is_err() {
647+ return;
648+ }
649+ }
650+ let text = match socket.read() {
651+ Ok(Message::Text(text)) => text,
652+ Ok(Message::Close(_)) => return,
653+ Ok(_) => continue,
654+ Err(tungstenite::Error::Io(err)) if matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => {
655+ continue;
656+ }
657+ Err(_) => return,
658+ };
659+ let Ok(message) = serde_json::from_str::<StreamMessage>(text.as_str()) else {
660+ continue;
661+ };
662+ let event = match message {
663+ StreamMessage::Frame { data, metadata, .. } => {
664+ let Ok(frame) = decode_frame(&data, &metadata) else {
665+ continue;
666+ };
667+ let Ok(mut slot) = shared.frame.lock() else {
668+ return;
669+ };
670+ // The panel hasn't drawn the last one yet; this one replaces it.
671+ if slot.replace(frame).is_some() {
672+ continue;
673+ }
674+ BrowserEvent::Frame
675+ }
676+ StreamMessage::Url { url } => BrowserEvent::Url(url),
677+ StreamMessage::Status { connected } => BrowserEvent::Live { browser: connected },
678+ StreamMessage::Command { action } => {
679+ let own = shared.own.lock().ok().and_then(|mut own| {
680+ let ix = own.iter().position(|(own, at)| *own == action && at.elapsed() < OWN_COMMAND)?;
681+ Some(own.remove(ix))
682+ });
683+ if own.is_some() {
684+ continue;
685+ }
686+ BrowserEvent::Agent(action)
687+ }
688+ StreamMessage::Other => continue,
689+ };
690+ if events.send_blocking(event).is_err() {
691+ return;
692+ }
693+ }
694+ }
695+
696+ /// Decodes a frame's JPEG into the BGRA image GPUI draws.
697+ fn decode_frame(data: &str, metadata: &FrameMetadata) -> Result<Frame> {
698+ let jpeg = base64::engine::general_purpose::STANDARD.decode(data)?;
699+ let mut pixels = image::load_from_memory_with_format(&jpeg, image::ImageFormat::Jpeg)?.into_rgba8();
700+ let size = (pixels.width() as f32, pixels.height() as f32);
701+ for pixel in pixels.chunks_exact_mut(4) {
702+ pixel.swap(0, 2);
703+ }
704+ Ok(Frame {
705+ image: Arc::new(RenderImage::new(vec![image::Frame::new(pixels)])),
706+ size,
707+ viewport: (metadata.device_width, metadata.device_height),
708+ })
709+ }
710+
711+ #[cfg(test)]
712+ mod tests {
713+ use super::*;
714+ use serde_json::json;
715+
716+ #[::core::prelude::v1::test]
717+ fn parses_stream_messages() {
718+ let frame = json!({
719+ "type": "frame", "seq": 41, "data": "abc",
720+ "metadata": {"deviceWidth": 1280, "deviceHeight": 720, "pageScaleFactor": 1, "offsetTop": 0,
721+ "scrollOffsetX": 0, "scrollOffsetY": 0, "timestamp": 1785038682238u64},
722+ });
723+ match serde_json::from_value::<StreamMessage>(frame).unwrap() {
724+ StreamMessage::Frame { data, metadata } => {
725+ assert_eq!(data, "abc");
726+ assert_eq!((metadata.device_width, metadata.device_height), (1280., 720.));
727+ }
728+ other => panic!("unexpected {other:?}"),
729+ }
730+ let url = json!({"type": "url", "url": "https://example.com/", "timestamp": 1});
731+ assert!(
732+ matches!(serde_json::from_value(url).unwrap(), StreamMessage::Url { url } if url == "https://example.com/")
733+ );
734+ let status = json!({"type": "status", "connected": true, "screencasting": false, "viewportWidth": 1280,
735+ "viewportHeight": 720, "engine": "chrome", "recording": false});
736+ assert!(matches!(serde_json::from_value(status).unwrap(), StreamMessage::Status { connected: true }));
737+ let tabs = json!({"type": "tabs", "timestamp": 1, "tabs": [
738+ {"active": true, "label": null, "tabId": "t1", "title": "One", "type": "page", "url": "a"},
739+ ]});
740+ assert!(matches!(serde_json::from_value(tabs).unwrap(), StreamMessage::Other));
741+ let command = json!({"type": "command", "action": "navigate", "id": "r1", "params": {}, "timestamp": 1});
742+ assert!(
743+ matches!(serde_json::from_value(command).unwrap(), StreamMessage::Command { action } if action == "navigate")
744+ );
745+ let result = json!({"type": "result", "id": "r1", "action": "navigate", "success": true, "data": {}});
746+ assert!(matches!(serde_json::from_value(result).unwrap(), StreamMessage::Other));
747+ }
748+
749+ #[::core::prelude::v1::test]
750+ fn serializes_input() {
751+ let mut click = MouseInput::new(MouseEventType::Down, (100., 200.), modifiers::SHIFT);
752+ click.button = PointerButton::Left;
753+ click.click_count = 1;
754+ assert_eq!(
755+ serde_json::to_value(Input::Mouse(click)).unwrap(),
756+ json!({"type": "input_mouse", "eventType": "mousePressed", "x": 100.0, "y": 200.0, "button": "left",
757+ "clickCount": 1, "deltaX": 0.0, "deltaY": 0.0, "modifiers": 8}),
758+ );
759+ let mut wheel = MouseInput::new(MouseEventType::Wheel, (1., 2.), 0);
760+ wheel.delta_y = 120.;
761+ let wheel = serde_json::to_value(Input::Mouse(wheel)).unwrap();
762+ assert_eq!((wheel["eventType"].as_str(), wheel["deltaY"].as_f64()), (Some("mouseWheel"), Some(120.)));
763+ assert_eq!(
764+ serde_json::to_value(Input::Key(key_input("a", Some("a"), 0, true))).unwrap(),
765+ json!({"type": "input_keyboard", "eventType": "keyDown", "key": "a", "code": "KeyA", "text": "a",
766+ "windowsVirtualKeyCode": 65, "modifiers": 0}),
767+ );
768+ assert_eq!(
769+ serde_json::to_value(Input::Key(text_input("p").remove(0))).unwrap(),
770+ json!({"type": "input_keyboard", "eventType": "char", "text": "p", "windowsVirtualKeyCode": 0,
771+ "modifiers": 0}),
772+ );
773+ }
774+
775+ #[::core::prelude::v1::test]
776+ fn maps_keys() {
777+ let enter = key_input("enter", None, 0, true);
778+ assert_eq!(
779+ (enter.event_type, enter.key.as_deref(), enter.text.as_deref()),
780+ (KeyEventType::Down, Some("Enter"), Some("\r"))
781+ );
782+ assert_eq!(enter.windows_virtual_key_code, 13);
783+ let left = key_input("left", None, 0, true);
784+ assert_eq!(
785+ (left.event_type, left.key.as_deref(), left.windows_virtual_key_code),
786+ (KeyEventType::RawDown, Some("ArrowLeft"), 37)
787+ );
788+ let shifted = key_input("a", Some("A"), modifiers::SHIFT, true);
789+ assert_eq!(
790+ (shifted.key.as_deref(), shifted.code.as_deref(), shifted.text.as_deref()),
791+ (Some("A"), Some("KeyA"), Some("A"))
792+ );
793+ let shortcut = key_input("a", None, modifiers::CTRL, true);
794+ assert_eq!((shortcut.event_type, shortcut.text), (KeyEventType::RawDown, None));
795+ let up = key_input("a", Some("a"), 0, false);
796+ assert_eq!((up.event_type, up.text), (KeyEventType::Up, None));
797+ assert_eq!(key_input("f5", None, 0, true).windows_virtual_key_code, 116);
798+ assert_eq!(key_input("7", Some("7"), 0, true).code.as_deref(), Some("Digit7"));
799+ assert_eq!(key_input(",", Some(","), 0, true).code, None);
800+ }
801+
802+ #[::core::prelude::v1::test]
803+ fn maps_points_onto_the_page() {
804+ // A 1280×720 frame in a 640×600 area: drawn 640×360, centered vertically.
805+ assert_eq!(contain((640., 600.), (1280., 720.)), (0., 120., 640., 360.));
806+ assert_eq!(page_point((320., 300.), (640., 600.), (1280., 720.), (1280., 720.)), Some((640., 360.)));
807+ assert_eq!(page_point((10., 50.), (640., 600.), (1280., 720.), (1280., 720.)), None);
808+ // A wide area pillarboxes; the frame may be scaled from the CSS viewport.
809+ assert_eq!(contain((1000., 360.), (640., 360.)), (180., 0., 640., 360.));
810+ assert_eq!(page_point((180., 0.), (1000., 360.), (640., 360.), (1280., 720.)), Some((0., 0.)));
811+ assert_eq!(page_point((820., 360.), (1000., 360.), (640., 360.), (1280., 720.)), Some((1279., 719.)));
812+ assert_eq!(page_point((5., 5.), (0., 0.), (1280., 720.), (1280., 720.)), None);
813+ }
814+
815+ #[::core::prelude::v1::test]
816+ fn normalizes_urls() {
817+ assert_eq!(normalize_url("example.com").as_deref(), Some("https://example.com"));
818+ assert_eq!(normalize_url(" example.com/a?b ").as_deref(), Some("https://example.com/a?b"));
819+ assert_eq!(normalize_url("http://example.com").as_deref(), Some("http://example.com"));
820+ assert_eq!(normalize_url("localhost:3000/x").as_deref(), Some("http://localhost:3000/x"));
821+ assert_eq!(normalize_url("127.0.0.1:8080").as_deref(), Some("http://127.0.0.1:8080"));
822+ assert_eq!(normalize_url("app.localhost").as_deref(), Some("http://app.localhost"));
823+ assert_eq!(normalize_url("about:blank").as_deref(), Some("about:blank"));
824+ assert_eq!(normalize_url("file:///tmp/a.html").as_deref(), Some("file:///tmp/a.html"));
825+ assert_eq!(normalize_url(" "), None);
826+ }
827+
828+ /// Drives a real session: `cargo test live_session -- --ignored`. Needs
829+ /// agent-browser and its Chrome, and the network for example.com.
830+ #[::core::prelude::v1::test]
831+ #[ignore]
832+ fn live_session() {
833+ let binary = find_binary().expect("agent-browser is not installed");
834+ let name = session_name(&format!("test-{}", std::process::id()));
835+ let cli = |args: &[&str]| -> Value {
836+ let output = Command::new(&binary).args(["--session", &name]).args(args).arg("--json").output().unwrap();
837+ serde_json::from_slice(&output.stdout).unwrap()
838+ };
839+ let (tx, rx) = smol::channel::unbounded();
840+ let session = BrowserSession::new(name.clone(), tx);
841+ session.set_watching(true);
842+ let deadline = Instant::now() + Duration::from_secs(60);
843+ let agent_actions = std::cell::RefCell::new(Vec::new());
844+ let next = || {
845+ let left = deadline.saturating_duration_since(Instant::now());
846+ let event = smol::block_on(smol::future::or(async { rx.recv().await.ok() }, async {
847+ smol::Timer::after(left).await;
848+ None
849+ }))
850+ .expect("timed out waiting for the browser");
851+ match &event {
852+ BrowserEvent::Failed(err) => panic!("{err}"),
853+ BrowserEvent::Agent(action) => agent_actions.borrow_mut().push(action.clone()),
854+ _ => {}
855+ }
856+ event
857+ };
858+
859+ // No daemon yet; the panel's `open` starts one, and frames follow.
860+ assert_eq!(next(), BrowserEvent::Waiting);
861+ session.open("https://example.com");
862+ let frame = loop {
863+ if next() == BrowserEvent::Frame {
864+ break session.take_frame().expect("a frame");
865+ }
866+ };
867+ assert!(frame.size.0 > 0. && frame.size.1 > 0.);
868+ assert_eq!(frame.viewport, (1280., 720.));
869+
870+ // A page with a link and a field at known places.
871+ let page = std::env::temp_dir().join(format!("vectrum-browser-test-{}.html", std::process::id()));
872+ std::fs::write(
873+ &page,
874+ r##"<!doctype html><title>test</title><body style="margin:0">
875+ <a href="#clicked" style="position:absolute;left:100px;top:100px">link</a>
876+ <input id="field" style="position:absolute;left:100px;top:200px;width:300px"></body>"##,
877+ )
878+ .unwrap();
879+ session.open(&format!("file://{}", page.display()));
880+ while !matches!(next(), BrowserEvent::Url(url) if url.ends_with(".html")) {}
881+ let click = |(x, y): (f64, f64)| {
882+ session.send(Input::Mouse(MouseInput::new(MouseEventType::Move, (x, y), 0)));
883+ for event_type in [MouseEventType::Down, MouseEventType::Up] {
884+ let mut click = MouseInput::new(event_type, (x, y), 0);
885+ click.button = PointerButton::Left;
886+ click.click_count = 1;
887+ session.send(Input::Mouse(click));
888+ }
889+ };
890+ click((110., 108.));
891+ while !matches!(next(), BrowserEvent::Url(url) if url.ends_with("#clicked")) {}
892+
893+ // Typing and pasting into the field.
894+ click((150., 210.));
895+ for (key, typed) in [("h", "h"), ("i", "i")] {
896+ session.send(Input::Key(key_input(key, Some(typed), 0, true)));
897+ session.send(Input::Key(key_input(key, Some(typed), 0, false)));
898+ }
899+ for input in text_input(" there") {
900+ session.send(Input::Key(input));
901+ }
902+ thread::sleep(Duration::from_millis(500));
903+ assert_eq!(cli(&["get", "value", "#field"])["data"]["value"], "hi there");
904+
905+ // Commands from anyone but the panel show as the agent's.
906+ while !agent_actions.borrow().iter().any(|action| action == "inputvalue") {
907+ next();
908+ }
909+ assert!(!agent_actions.borrow().iter().any(|action| action == "navigate"), "{agent_actions:?}");
910+
911+ drop(session);
912+ let _ = std::fs::remove_file(page);
913+ stop(&name);
914+ let closing = Instant::now();
915+ while is_running(&name) && closing.elapsed() < Duration::from_secs(10) {
916+ thread::sleep(Duration::from_millis(100));
917+ }
918+ assert!(!is_running(&name));
919+ }
920+
921+ #[::core::prelude::v1::test]
922+ fn names_sessions() {
923+ assert_eq!(session_name("18f-2a-0"), "vectrum-18f-2a-0");
924+ assert!(agent_prompt("vectrum-x").contains("`vectrum-x`"));
925+ }
926+ }
+10−1src/claude.rsView File
@@ -14,6 +14,7 @@ use std::{
1414 use anyhow::{Context as _, Result, anyhow};
1515 use serde_json::{Value, json};
1616
17+ use crate::browser;
1718 use crate::models::Provider;
1819 use crate::settings::{ProviderInstance, expand_home};
1920
@@ -147,6 +148,8 @@ pub struct LaunchConfig {
147148 /// Session to resume (a Claude session id or a Pi session file), used to
148149 /// restart a thread whose process stopped.
149150 pub resume: Option<String>,
151+ /// The thread's agent-browser session, shared with the browser panel.
152+ pub browser_session: Option<String>,
150153 }
151154
152155 /// Everything the background reader threads report back to the UI.
@@ -164,10 +167,16 @@ pub struct AgentProcess {
164167
165168 impl AgentProcess {
166169 pub fn spawn(config: &LaunchConfig, events: smol::channel::Sender<ProcessEvent>) -> Result<Self> {
167- let (binary, command) = match config.instance.kind {
170+ let (binary, mut command) = match config.instance.kind {
168171 Provider::Claude => claude_command(config)?,
169172 Provider::Pi => pi_command(config)?,
170173 };
174+ // Both agents take extra system prompt with this flag.
175+ if let Some(session) = &config.browser_session
176+ && let Some(prompt) = browser::prepare_agent(&mut command, session)
177+ {
178+ command.args(["--append-system-prompt", &prompt]);
179+ }
171180 Self::spawn_command(&binary, command, events)
172181 }
173182
+4−0src/main.rsView File
@@ -3,6 +3,7 @@
33 // GPUI's derive macros expand to `gpui::` paths; route them through the Kit.
44 extern crate gpui_kit as gpui;
55
6+ mod browser;
67 mod claude;
78 mod clock;
89 mod git;
@@ -34,6 +35,7 @@ gpui_kit::actions!(
3435 Interrupt,
3536 ToggleSidebar,
3637 ToggleProjectPanel,
38+ ToggleBrowser,
3739 FocusSearch,
3840 OpenSettings,
3941 CloseSettings,
@@ -58,12 +60,14 @@ fn main() {
5860 metal::load_fonts(cx);
5961 metal::restore_colorway(cx);
6062 models::Catalog::init(cx);
63+ ui::browser::BrowserWatch::init(cx);
6164
6265 cx.bind_keys([
6366 KeyBinding::new("ctrl-n", NewThread, None),
6467 KeyBinding::new("ctrl-o", OpenProject, None),
6568 KeyBinding::new("ctrl-b", ToggleSidebar, None),
6669 KeyBinding::new("ctrl-shift-b", ToggleProjectPanel, None),
70+ KeyBinding::new("ctrl-shift-e", ToggleBrowser, None),
6771 KeyBinding::new("ctrl-k", FocusSearch, None),
6872 KeyBinding::new("ctrl-,", OpenSettings, None),
6973 KeyBinding::new("ctrl-q", Quit, None),
+2−0src/session.rsView File
@@ -14,6 +14,7 @@ use gpui_kit::*;
1414 use serde::{Deserialize, Serialize};
1515 use serde_json::{Value, json};
1616
17+ use crate::browser;
1718 use crate::claude::{AgentProcess, Effort, LaunchConfig, PermissionMode, ProcessEvent};
1819 use crate::git::{self, TurnChanges};
1920 use crate::models::{Catalog, Provider};
@@ -434,6 +435,7 @@ impl Session {
434435 permission_mode: self.permission_mode,
435436 effort: self.reasoning.effort(),
436437 resume,
438+ browser_session: Some(browser::session_name(&self.id)),
437439 };
438440 let process = match AgentProcess::spawn(&config, tx) {
439441 Ok(process) => process,
+619−0src/ui/browser.rsaddedView File
@@ -0,0 +1,619 @@
1+ //! The browser panel: the thread's shared browser, live, beside the
2+ //! conversation, like T3 Code's browser preview. The agent drives it with the
3+ //! `agent-browser` CLI; the header navigates, and the viewport passes the
4+ //! user's clicks, scrolling and typing on to the page.
5+
6+ use std::{
7+ collections::HashSet,
8+ time::{Duration, Instant},
9+ };
10+
11+ use gpui_kit::component::{
12+ Icon, IconName, h_flex,
13+ input::{Input, InputEvent, InputState},
14+ v_flex,
15+ };
16+ use gpui_kit::prelude::FluentBuilder as _;
17+ use gpui_kit::*;
18+
19+ use crate::browser::{
20+ self, BrowserEvent, BrowserSession, Frame, Input as PageInput, MouseEventType, MouseInput, PointerButton, modifiers,
21+ };
22+ use crate::ui::metal::{self, Led, MetalButton, MetalStyled as _};
23+
24+ /// The panel's share of the sheet until the user drags it.
25+ const SHARE: f32 = 0.45;
26+ const MIN_WIDTH: f32 = 420.;
27+ /// How long the panel shows the agent driving after its last command.
28+ const AGENT_LINGER: Duration = Duration::from_secs(4);
29+ /// How long the viewport has to keep its size before the page is fit to it.
30+ const FIT_DELAY: Duration = Duration::from_millis(400);
31+ /// Pixels a wheel line scrolls.
32+ const LINE: f32 = 40.;
33+
34+ pub enum BrowserPanelEvent {
35+ Close,
36+ }
37+
38+ /// The panel's width once the user drags its edge, shared by all threads.
39+ #[derive(Default)]
40+ struct BrowserWidth(Option<Pixels>);
41+
42+ impl Global for BrowserWidth {}
43+
44+ /// The edge the user drags to resize the panel.
45+ struct ResizeBrowser;
46+
47+ /// Which threads' browsers run, refreshed in the background, so the
48+ /// workspace can mark the browser button when the agent opens a page while
49+ /// the panel is closed.
50+ #[derive(Default)]
51+ pub struct BrowserWatch {
52+ live: HashSet<String>,
53+ }
54+
55+ impl Global for BrowserWatch {}
56+
57+ impl BrowserWatch {
58+ pub fn init(cx: &mut App) {
59+ cx.set_global(BrowserWatch::default());
60+ cx.spawn(async move |cx| {
61+ loop {
62+ cx.background_executor().timer(Duration::from_secs(2)).await;
63+ let live = cx.background_executor().spawn(async { browser::running_sessions() }).await;
64+ cx.update(|cx| {
65+ if cx.global::<BrowserWatch>().live != live {
66+ cx.global_mut::<BrowserWatch>().live = live;
67+ cx.refresh_windows();
68+ }
69+ });
70+ }
71+ })
72+ .detach();
73+ }
74+
75+ /// Whether a thread's browser runs.
76+ pub fn is_live(thread_id: &str, cx: &App) -> bool {
77+ cx.try_global::<BrowserWatch>().is_some_and(|watch| watch.live.contains(&browser::session_name(thread_id)))
78+ }
79+ }
80+
81+ #[derive(Clone, Copy, PartialEq, Eq)]
82+ enum PageState {
83+ /// `agent-browser` isn't installed.
84+ Missing,
85+ /// Looking for the session's daemon.
86+ Starting,
87+ /// No browser runs yet.
88+ Idle,
89+ Live,
90+ }
91+
92+ pub struct BrowserPanel {
93+ session: BrowserSession,
94+ url: Entity<InputState>,
95+ state: PageState,
96+ frame: Option<Frame>,
97+ /// When the agent last ran a command.
98+ agent_at: Option<Instant>,
99+ error: Option<SharedString>,
100+ visible: bool,
101+ focus_handle: FocusHandle,
102+ /// Where the viewport was last drawn, in window coordinates.
103+ area: Bounds<Pixels>,
104+ /// The page size last asked for, so the page is fit once per size.
105+ fitted: Option<(u32, u32)>,
106+ _fit: Task<()>,
107+ _agent_timer: Task<()>,
108+ _events: Task<()>,
109+ _subscriptions: Vec<Subscription>,
110+ }
111+
112+ impl EventEmitter<BrowserPanelEvent> for BrowserPanel {}
113+
114+ impl BrowserPanel {
115+ pub fn new(thread_id: &str, window: &mut Window, cx: &mut Context<Self>) -> Self {
116+ let (session, events) = Self::watch(browser::session_name(thread_id), window, cx);
117+ let url = cx.new(|cx| InputState::new(window, cx).placeholder("Type a URL"));
118+ let subscriptions = vec![
119+ cx.subscribe_in(&url, window, |this, url, event: &InputEvent, window, cx| {
120+ if matches!(event, InputEvent::PressEnter { .. }) {
121+ let text = url.read(cx).value().to_string();
122+ if let Some(target) = browser::normalize_url(&text) {
123+ url.update(cx, |url, cx| url.set_value(target.clone(), window, cx));
124+ this.error = None;
125+ this.session.open(&target);
126+ this.focus_handle.focus(window, cx);
127+ cx.notify();
128+ }
129+ }
130+ }),
131+ cx.on_release(|this, cx| {
132+ if let Some(frame) = this.frame.take() {
133+ cx.drop_image(frame.image, None);
134+ }
135+ }),
136+ ];
137+ let state = if session.installed() { PageState::Starting } else { PageState::Missing };
138+ Self {
139+ session,
140+ url,
141+ state,
142+ frame: None,
143+ agent_at: None,
144+ error: None,
145+ visible: false,
146+ focus_handle: cx.focus_handle(),
147+ area: Bounds::default(),
148+ fitted: None,
149+ _fit: Task::ready(()),
150+ _agent_timer: Task::ready(()),
151+ _events: events,
152+ _subscriptions: subscriptions,
153+ }
154+ }
155+
156+ /// A session and the task that applies what it reports.
157+ fn watch(name: String, window: &mut Window, cx: &mut Context<Self>) -> (BrowserSession, Task<()>) {
158+ let (tx, rx) = smol::channel::unbounded();
159+ let session = BrowserSession::new(name, tx);
160+ let events = cx.spawn_in(window, async move |this, cx| {
161+ while let Ok(event) = rx.recv().await {
162+ let mut batch = vec![event];
163+ while let Ok(event) = rx.try_recv() {
164+ batch.push(event);
165+ }
166+ let updated = this.update_in(cx, |this, window, cx| {
167+ for event in batch {
168+ this.handle_event(event, window, cx);
169+ }
170+ cx.notify();
171+ });
172+ if updated.is_err() {
173+ break;
174+ }
175+ }
176+ });
177+ (session, events)
178+ }
179+
180+ /// Looks for agent-browser again, after the user installed it.
181+ fn retry(&mut self, window: &mut Window, cx: &mut Context<Self>) {
182+ let (session, events) = Self::watch(self.session.name().to_string(), window, cx);
183+ self.session = session;
184+ self._events = events;
185+ self.session.set_watching(self.visible);
186+ self.state = if self.session.installed() { PageState::Starting } else { PageState::Missing };
187+ cx.notify();
188+ }
189+
190+ /// Watches the stream while the panel shows; a hidden panel lets go of it.
191+ pub fn set_visible(&mut self, visible: bool, cx: &mut Context<Self>) {
192+ if self.visible != visible {
193+ self.visible = visible;
194+ self.session.set_watching(visible);
195+ cx.notify();
196+ }
197+ }
198+
199+ fn handle_event(&mut self, event: BrowserEvent, window: &mut Window, cx: &mut Context<Self>) {
200+ match event {
201+ BrowserEvent::Missing => self.state = PageState::Missing,
202+ BrowserEvent::Waiting => {
203+ self.state = PageState::Idle;
204+ self.set_frame(None, window, cx);
205+ }
206+ BrowserEvent::Live { browser } => {
207+ self.state = if browser { PageState::Live } else { PageState::Idle };
208+ if !browser {
209+ self.set_frame(None, window, cx);
210+ }
211+ }
212+ BrowserEvent::Frame => {
213+ if let Some(frame) = self.session.take_frame() {
214+ self.state = PageState::Live;
215+ self.set_frame(Some(frame), window, cx);
216+ self.fit_page(window, cx);
217+ }
218+ }
219+ BrowserEvent::Url(url) => {
220+ if !self.url.read(cx).focus_handle(cx).is_focused(window) {
221+ self.url.update(cx, |input, cx| input.set_value(url, window, cx));
222+ }
223+ }
224+ BrowserEvent::Agent(_) => {
225+ self.agent_at = Some(Instant::now());
226+ // Repaint once the indicator should go out.
227+ self._agent_timer = cx.spawn(async move |this, cx| {
228+ cx.background_executor().timer(AGENT_LINGER).await;
229+ let _ = this.update(cx, |_, cx| cx.notify());
230+ });
231+ }
232+ BrowserEvent::Failed(err) => self.error = Some(err.into()),
233+ }
234+ }
235+
236+ /// Shows a new frame and frees the GPU texture of the one it replaces.
237+ fn set_frame(&mut self, frame: Option<Frame>, window: &mut Window, cx: &mut Context<Self>) {
238+ let browser_restarted = self.frame.is_none() && frame.is_some();
239+ if let Some(old) = std::mem::replace(&mut self.frame, frame) {
240+ cx.drop_image(old.image, Some(window));
241+ }
242+ if browser_restarted {
243+ self.fitted = None;
244+ }
245+ }
246+
247+ /// Sizes the page to the viewport, so it renders at the panel's size
248+ /// rather than scaled down. Once per size: a viewport the agent picks
249+ /// stays until the panel changes size.
250+ fn fit_page(&mut self, window: &Window, cx: &mut Context<Self>) {
251+ let size = self.area.size;
252+ let target = (f32::from(size.width).round() as u32, f32::from(size.height).round() as u32);
253+ if !self.visible || self.state != PageState::Live || target.0 < 200 || target.1 < 150 {
254+ return;
255+ }
256+ if self.fitted == Some(target) {
257+ return;
258+ }
259+ let scale = window.scale_factor();
260+ self._fit = cx.spawn(async move |this, cx| {
261+ cx.background_executor().timer(FIT_DELAY).await;
262+ let _ = this.update(cx, |this, _| {
263+ let size = this.area.size;
264+ let current = (f32::from(size.width).round() as u32, f32::from(size.height).round() as u32);
265+ if current == target && this.fitted != Some(target) {
266+ this.fitted = Some(target);
267+ this.session.set_viewport(target.0, target.1, scale);
268+ }
269+ });
270+ });
271+ }
272+
273+ /// A window position as page coordinates, if it's over the page.
274+ fn page_point(&self, position: Point<Pixels>) -> Option<(f64, f64)> {
275+ let frame = self.frame.as_ref()?;
276+ let origin = self.area.origin;
277+ browser::page_point(
278+ (f32::from(position.x - origin.x), f32::from(position.y - origin.y)),
279+ (f32::from(self.area.size.width), f32::from(self.area.size.height)),
280+ frame.size,
281+ frame.viewport,
282+ )
283+ }
284+
285+ /// Page pixels per window pixel.
286+ fn page_scale(&self) -> f32 {
287+ let Some(frame) = &self.frame else {
288+ return 1.;
289+ };
290+ let area = (f32::from(self.area.size.width), f32::from(self.area.size.height));
291+ let (_, _, width, _) = browser::contain(area, frame.size);
292+ if width > 0. { frame.viewport.0 / width } else { 1. }
293+ }
294+
295+ fn send_mouse(
296+ &self,
297+ event_type: MouseEventType,
298+ position: Point<Pixels>,
299+ button: Option<MouseButton>,
300+ click_count: usize,
301+ mods: &Modifiers,
302+ ) {
303+ let Some(point) = self.page_point(position) else {
304+ return;
305+ };
306+ let mut input = MouseInput::new(event_type, point, modifier_bits(mods));
307+ input.button = match button {
308+ Some(MouseButton::Left) => PointerButton::Left,
309+ Some(MouseButton::Middle) => PointerButton::Middle,
310+ Some(MouseButton::Right) => PointerButton::Right,
311+ _ => PointerButton::None,
312+ };
313+ input.click_count = click_count as u32;
314+ self.session.send(PageInput::Mouse(input));
315+ }
316+
317+ fn on_key_down(&mut self, event: &KeyDownEvent, _: &mut Window, cx: &mut Context<Self>) {
318+ let keystroke = &event.keystroke;
319+ let mods = modifier_bits(&keystroke.modifiers);
320+ // Chrome runs no clipboard shortcuts headless; paste the text instead.
321+ if keystroke.key == "v" && keystroke.modifiers.control && !keystroke.modifiers.alt {
322+ if let Some(text) = cx.read_from_clipboard().and_then(|item| item.text()) {
323+ for input in browser::text_input(&text) {
324+ self.session.send(PageInput::Key(input));
325+ }
326+ }
327+ cx.stop_propagation();
328+ return;
329+ }
330+ let input = browser::key_input(&keystroke.key, keystroke.key_char.as_deref(), mods, true);
331+ self.session.send(PageInput::Key(input));
332+ cx.stop_propagation();
333+ }
334+
335+ fn on_key_up(&mut self, event: &KeyUpEvent, _: &mut Window, cx: &mut Context<Self>) {
336+ let keystroke = &event.keystroke;
337+ let mods = modifier_bits(&keystroke.modifiers);
338+ let input = browser::key_input(&keystroke.key, keystroke.key_char.as_deref(), mods, false);
339+ self.session.send(PageInput::Key(input));
340+ cx.stop_propagation();
341+ }
342+
343+ fn render_header(&self, cx: &mut Context<Self>) -> impl IntoElement {
344+ let p = metal::palette(cx);
345+ let live = self.state == PageState::Live;
346+ let driving = self.agent_at.is_some_and(|at| at.elapsed() < AGENT_LINGER);
347+ h_flex()
348+ .h(px(48.))
349+ .flex_none()
350+ .px(px(8.))
351+ .gap(px(4.))
352+ .child(
353+ MetalButton::new("browser-back")
354+ .ghost()
355+ .mini()
356+ .icon(IconName::ArrowLeft)
357+ .tooltip("Back")
358+ .disabled(!live)
359+ .on_click(cx.listener(|this, _, _, _| this.session.back())),
360+ )
361+ .child(
362+ MetalButton::new("browser-forward")
363+ .ghost()
364+ .mini()
365+ .icon(IconName::ArrowRight)
366+ .tooltip("Forward")
367+ .disabled(!live)
368+ .on_click(cx.listener(|this, _, _, _| this.session.forward())),
369+ )
370+ .child(
371+ MetalButton::new("browser-reload")
372+ .ghost()
373+ .mini()
374+ .icon(IconName::RotateCw)
375+ .tooltip("Reload")
376+ .disabled(!live)
377+ .on_click(cx.listener(|this, _, _, _| this.session.reload())),
378+ )
379+ .child(
380+ h_flex()
381+ .flex_1()
382+ .min_w_0()
383+ .h(px(30.))
384+ .ml(px(4.))
385+ .pl(px(10.))
386+ .pr(px(8.))
387+ .gap(px(8.))
388+ .rounded(px(12.))
389+ .well(p)
390+ .text_size(px(12.5))
391+ .text_color(p.ink)
392+ .child(Icon::new(IconName::Globe).size(px(13.)).text_color(p.field_ink))
393+ .child(div().flex_1().min_w_0().child(Input::new(&self.url).appearance(false)))
394+ .when(driving, |this| {
395+ this.child(
396+ h_flex()
397+ .flex_none()
398+ .gap(px(5.))
399+ .child(metal::led("browser-agent", Led::Live, true, p))
400+ .child(metal::engraved("Agent", p)),
401+ )
402+ }),
403+ )
404+ .child(
405+ MetalButton::new("browser-close")
406+ .ghost()
407+ .mini()
408+ .icon(IconName::Close)
409+ .tooltip("Close browser (Ctrl+Shift+E)")
410+ .on_click(cx.listener(|_, _, _, cx| cx.emit(BrowserPanelEvent::Close))),
411+ )
412+ }
413+
414+ /// The live page, or what stands in for it.
415+ fn render_viewport(&self, cx: &mut Context<Self>) -> AnyElement {
416+ let entity = cx.entity().downgrade();
417+ let track_area = canvas(
418+ move |bounds, window, cx| {
419+ let _ = entity.update(cx, |this, cx| {
420+ if this.area != bounds {
421+ this.area = bounds;
422+ this.fit_page(window, cx);
423+ }
424+ });
425+ },
426+ |_, _, _, _| {},
427+ )
428+ .absolute()
429+ .size_full();
430+
431+ let Some(frame) = &self.frame else {
432+ return div()
433+ .relative()
434+ .size_full()
435+ .child(track_area)
436+ .child(self.render_placeholder(cx))
437+ .into_any_element();
438+ };
439+ div()
440+ .id("browser-viewport")
441+ .relative()
442+ .size_full()
443+ .key_context("BrowserViewport")
444+ .track_focus(&self.focus_handle)
445+ .cursor_default()
446+ .child(track_area)
447+ .child(img(ImageSource::Render(frame.image.clone())).size_full().object_fit(ObjectFit::Contain))
448+ .on_mouse_down(MouseButton::Left, cx.listener(Self::mouse_down))
449+ .on_mouse_down(MouseButton::Right, cx.listener(Self::mouse_down))
450+ .on_mouse_down(MouseButton::Middle, cx.listener(Self::mouse_down))
451+ .on_mouse_up(MouseButton::Left, cx.listener(Self::mouse_up))
452+ .on_mouse_up(MouseButton::Right, cx.listener(Self::mouse_up))
453+ .on_mouse_up(MouseButton::Middle, cx.listener(Self::mouse_up))
454+ .on_mouse_move(cx.listener(|this, event: &MouseMoveEvent, _, _| {
455+ this.send_mouse(MouseEventType::Move, event.position, event.pressed_button, 0, &event.modifiers);
456+ }))
457+ .on_scroll_wheel(cx.listener(|this, event: &ScrollWheelEvent, _, cx| {
458+ let Some(point) = this.page_point(event.position) else {
459+ return;
460+ };
461+ let delta = event.delta.pixel_delta(px(LINE));
462+ let scale = this.page_scale();
463+ let mut input = MouseInput::new(MouseEventType::Wheel, point, modifier_bits(&event.modifiers));
464+ // GPUI's deltas move the content; the page's move the viewport.
465+ input.delta_x = -(f32::from(delta.x) * scale) as f64;
466+ input.delta_y = -(f32::from(delta.y) * scale) as f64;
467+ this.session.send(PageInput::Mouse(input));
468+ cx.stop_propagation();
469+ }))
470+ .on_key_down(cx.listener(Self::on_key_down))
471+ .on_key_up(cx.listener(Self::on_key_up))
472+ .into_any_element()
473+ }
474+
475+ fn mouse_down(&mut self, event: &MouseDownEvent, window: &mut Window, cx: &mut Context<Self>) {
476+ self.focus_handle.focus(window, cx);
477+ self.send_mouse(MouseEventType::Move, event.position, None, 0, &event.modifiers);
478+ self.send_mouse(MouseEventType::Down, event.position, Some(event.button), event.click_count, &event.modifiers);
479+ }
480+
481+ fn mouse_up(&mut self, event: &MouseUpEvent, _: &mut Window, _: &mut Context<Self>) {
482+ self.send_mouse(MouseEventType::Up, event.position, Some(event.button), event.click_count, &event.modifiers);
483+ }
484+
485+ fn render_placeholder(&self, cx: &mut Context<Self>) -> impl IntoElement {
486+ let p = metal::palette(cx);
487+ let body = |text: &'static str| {
488+ div().max_w(px(320.)).text_center().text_size(px(13.)).line_height(px(19.)).text_color(p.ink3).child(text)
489+ };
490+ let content = v_flex().items_center().gap(px(12.)).px(px(24.));
491+ let content = match self.state {
492+ PageState::Missing => content
493+ .child(metal::engraved("agent-browser not installed", p))
494+ .child(body("The browser panel runs on Vercel's agent-browser. Install it and its Chrome:"))
495+ .child(
496+ h_flex()
497+ .max_w_full()
498+ .gap(px(6.))
499+ .pl(px(12.))
500+ .pr(px(4.))
501+ .py(px(5.))
502+ .rounded(px(12.))
503+ .well(p)
504+ .child(
505+ div()
506+ .min_w_0()
507+ .font_family(metal::MONO)
508+ .text_size(px(11.5))
509+ .text_color(p.ink)
510+ .child(browser::INSTALL_COMMAND),
511+ )
512+ .child(
513+ MetalButton::new("copy-install")
514+ .ghost()
515+ .mini()
516+ .icon(IconName::Copy)
517+ .tooltip("Copy")
518+ .on_click(|_, _, cx| {
519+ cx.write_to_clipboard(ClipboardItem::new_string(
520+ browser::INSTALL_COMMAND.to_string(),
521+ ))
522+ }),
523+ ),
524+ )
525+ .child(body("Threads started afterwards can use it."))
526+ .child(
527+ MetalButton::new("browser-retry")
528+ .compact()
529+ .label("Check again")
530+ .on_click(cx.listener(|this, _, window, cx| this.retry(window, cx))),
531+ ),
532+ PageState::Starting | PageState::Live => content
533+ .child(metal::led("browser-starting", Led::Waiting, true, p))
534+ .child(metal::engraved("Starting the browser", p)),
535+ PageState::Idle => content
536+ .child(Icon::new(IconName::Globe).size(px(22.)).text_color(p.ink3))
537+ .child(metal::engraved("No page yet", p))
538+ .child(body("The agent opens pages here, or type a URL.")),
539+ };
540+ div().size_full().flex().items_center().justify_center().child(content)
541+ }
542+ }
543+
544+ impl Render for BrowserPanel {
545+ fn render(&mut self, _: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
546+ let p = metal::palette(cx);
547+ let width = cx.default_global::<BrowserWidth>().0;
548+ let error = self.error.clone();
549+ v_flex()
550+ .relative()
551+ .h_full()
552+ .min_w(px(MIN_WIDTH))
553+ .map(|this| match width {
554+ Some(width) => this.w(width),
555+ None => this.flex_basis(relative(SHARE)),
556+ })
557+ .p(px(12.))
558+ .pl(px(0.))
559+ .on_drag_move(cx.listener(|_, event: &DragMoveEvent<ResizeBrowser>, _, cx| {
560+ // The panel's right edge stays put; its left edge follows the pointer.
561+ let right = event.bounds.right();
562+ let width = (right - event.event.position.x).max(px(MIN_WIDTH));
563+ cx.set_global(BrowserWidth(Some(width)));
564+ cx.notify();
565+ }))
566+ .child(
567+ div()
568+ .id("browser-resize")
569+ .absolute()
570+ .left(px(-6.))
571+ .top_0()
572+ .bottom_0()
573+ .w(px(8.))
574+ .cursor_col_resize()
575+ .on_drag(ResizeBrowser, |_, _, _, cx| cx.new(|_| EmptyView)),
576+ )
577+ .child(
578+ v_flex()
579+ .size_full()
580+ .rounded(px(18.))
581+ .raised(p)
582+ .child(self.render_header(cx))
583+ .when_some(error, |this, error| {
584+ this.child(
585+ div().px(px(14.)).pb(px(6.)).truncate().text_size(px(11.5)).text_color(p.red).child(error),
586+ )
587+ })
588+ .child(
589+ div()
590+ .flex_1()
591+ .min_h_0()
592+ .m(px(6.))
593+ .mt(px(0.))
594+ .rounded(px(12.))
595+ .overflow_hidden()
596+ .well(p)
597+ .child(self.render_viewport(cx)),
598+ ),
599+ )
600+ }
601+ }
602+
603+ /// GPUI's modifier keys as CDP's modifier bits.
604+ fn modifier_bits(mods: &Modifiers) -> u8 {
605+ let mut bits = 0;
606+ if mods.alt {
607+ bits |= modifiers::ALT;
608+ }
609+ if mods.control {
610+ bits |= modifiers::CTRL;
611+ }
612+ if mods.platform {
613+ bits |= modifiers::META;
614+ }
615+ if mods.shift {
616+ bits |= modifiers::SHIFT;
617+ }
618+ bits
619+ }
+1−0src/ui/mod.rsView File
@@ -1,3 +1,4 @@
1+ pub mod browser;
12 pub mod changes;
23 pub mod chat;
34 pub mod icons;
+1−0src/ui/settings.rsView File
@@ -1450,6 +1450,7 @@ impl SettingsView {
14501450 shortcut("Search threads", &["Ctrl", "K"]),
14511451 shortcut("Show or hide the sidebar", &["Ctrl", "B"]),
14521452 shortcut("Show or hide the project panel", &["Ctrl", "Shift", "B"]),
1453+ shortcut("Show or hide the browser", &["Ctrl", "Shift", "E"]),
14531454 shortcut("Open or close settings", &["Ctrl", ","]),
14541455 shortcut(if hold { "Quit (hold, or press twice)" } else { "Quit" }, &["Ctrl", "Q"]),
14551456 ],
+93−9src/ui/workspace.rsView File
@@ -25,6 +25,7 @@ use gpui_kit::component::{
2525 use gpui_kit::prelude::FluentBuilder as _;
2626 use gpui_kit::*;
2727
28+ use crate::browser;
2829 use crate::claude::PermissionMode;
2930 use crate::clock;
3031 use crate::git::{self, GitInfo};
@@ -32,13 +33,14 @@ use crate::models::{Catalog, ModelEntry, Provider, split_key};
3233 use crate::session::{NewSession, Reasoning, Session, SessionStatus, from_millis, to_millis};
3334 use crate::settings::{self, BackgroundActivity, ProjectOrder, WorkspaceMode};
3435 use crate::store::{self, AppState, SavedProject};
36+ use crate::ui::browser::{BrowserPanel, BrowserPanelEvent, BrowserWatch};
3537 use crate::ui::chat::{ChatEvent, ChatView, display_path};
3638 use crate::ui::icons::Lucide;
3739 use crate::ui::metal::{self, Led, MetalButton, MetalStyled as _, Palette};
3840 use crate::ui::pickers;
3941 use crate::ui::settings::{SettingsEvent, SettingsView};
4042 use crate::worktree::{self, WorktreePlan};
41- use crate::{FocusSearch, NewThread, OpenProject, OpenSettings, ToggleProjectPanel, ToggleSidebar};
43+ use crate::{FocusSearch, NewThread, OpenProject, OpenSettings, ToggleBrowser, ToggleProjectPanel, ToggleSidebar};
4244
4345 pub(crate) const SIDEBAR_WIDTH: f32 = 280.;
4446 const PANEL_WIDTH: f32 = 292.;
@@ -136,11 +138,17 @@ struct Activity {
136138 pub struct Workspace {
137139 projects: Vec<Project>,
138140 chats: HashMap<EntityId, Entity<ChatView>>,
141+ /// Each thread's browser panel, made the first time it's opened.
142+ browsers: HashMap<EntityId, Entity<BrowserPanel>>,
143+ /// The browser panel showing, which watches its stream.
144+ browser_shown: Option<EntityId>,
139145 active: Option<Entity<Session>>,
140146 active_project: usize,
141147 search: Entity<InputState>,
142148 sidebar_open: bool,
143149 panel_open: bool,
150+ /// The browser takes the project panel's place while open.
151+ browser_open: bool,
144152 settled_open: bool,
145153 snoozed_open: bool,
146154 working_open: bool,
@@ -225,11 +233,14 @@ impl Workspace {
225233 let mut workspace = Self {
226234 projects: Vec::new(),
227235 chats: HashMap::new(),
236+ browsers: HashMap::new(),
237+ browser_shown: None,
228238 active: None,
229239 active_project: 0,
230240 search,
231241 sidebar_open: true,
232242 panel_open: true,
243+ browser_open: false,
233244 settled_open: false,
234245 snoozed_open: false,
235246 working_open: false,
@@ -1020,6 +1031,7 @@ impl Workspace {
10201031 cx.background_executor()
10211032 .spawn(async move {
10221033 store::delete_thread(&id);
1034+ browser::stop(&browser::session_name(&id));
10231035 if let Some(worktree) = worktree {
10241036 let _ = worktree::remove(&worktree);
10251037 }
@@ -1040,6 +1052,7 @@ impl Workspace {
10401052 session.update(cx, |session, _| session.shutdown());
10411053 let id = session.entity_id();
10421054 self.chats.remove(&id);
1055+ self.browsers.remove(&id);
10431056 self.observers.remove(&id);
10441057 self.activity.remove(&id);
10451058 self.dirty.remove(&id);
@@ -1832,6 +1845,73 @@ impl Workspace {
18321845 .child(self.render_sidebar_footer(cx))
18331846 }
18341847
1848+ fn toggle_project_panel(&mut self, cx: &mut Context<Self>) {
1849+ // With the browser in its place, the project panel comes back.
1850+ if self.browser_open {
1851+ self.browser_open = false;
1852+ self.panel_open = true;
1853+ } else {
1854+ self.panel_open = !self.panel_open;
1855+ }
1856+ cx.notify();
1857+ }
1858+
1859+ /// The open thread's browser panel while it shows, made on first use.
1860+ /// The others let go of their streams.
1861+ fn shown_browser(&mut self, window: &mut Window, cx: &mut Context<Self>) -> Option<Entity<BrowserPanel>> {
1862+ let session = self.active.clone().filter(|_| self.browser_open && !self.settings_open);
1863+ let panel = session.map(|session| {
1864+ let id = session.entity_id();
1865+ if let Some(panel) = self.browsers.get(&id) {
1866+ return panel.clone();
1867+ }
1868+ let thread_id = session.read(cx).id.clone();
1869+ let panel = cx.new(|cx| BrowserPanel::new(&thread_id, window, cx));
1870+ let subscription = cx.subscribe(&panel, |this, _, event: &BrowserPanelEvent, cx| match event {
1871+ BrowserPanelEvent::Close => {
1872+ this.browser_open = false;
1873+ cx.notify();
1874+ }
1875+ });
1876+ self.observers.entry(id).or_default().push(subscription);
1877+ self.browsers.insert(id, panel.clone());
1878+ panel
1879+ });
1880+ let shown = panel.as_ref().map(|panel| panel.entity_id());
1881+ if shown != self.browser_shown {
1882+ self.browser_shown = shown;
1883+ for browser in self.browsers.values() {
1884+ let visible = Some(browser.entity_id()) == shown;
1885+ browser.update(cx, |browser, cx| browser.set_visible(visible, cx));
1886+ }
1887+ }
1888+ panel
1889+ }
1890+
1891+ /// The browser button, lit when the open thread's browser runs while
1892+ /// the panel is closed, as when the agent opens a page.
1893+ fn render_browser_toggle(&self, cx: &mut Context<Self>) -> impl IntoElement {
1894+ let p = metal::palette(cx);
1895+ let live = !self.browser_open
1896+ && self.active.as_ref().is_some_and(|session| BrowserWatch::is_live(&session.read(cx).id, cx));
1897+ div()
1898+ .relative()
1899+ .child(
1900+ MetalButton::new("toggle-browser")
1901+ .ghost()
1902+ .mini()
1903+ .icon(IconName::Globe)
1904+ .tooltip("Browser (Ctrl+Shift+E)")
1905+ .on_click(cx.listener(|this, _, _, cx| {
1906+ this.browser_open = !this.browser_open;
1907+ cx.notify();
1908+ })),
1909+ )
1910+ .when(live, |this| {
1911+ this.child(div().absolute().top(px(1.)).right(px(1.)).child(metal::led("browser-live", Led::Live, false, p)))
1912+ })
1913+ }
1914+
18351915 fn render_top_bar(&self, panel_visible: bool, cx: &mut Context<Self>) -> impl IntoElement {
18361916 let p = metal::palette(cx);
18371917 let project = self.active_project();
@@ -1881,16 +1961,14 @@ impl Workspace {
18811961 )
18821962 })
18831963 .child(div().flex_1())
1964+ .child(self.render_browser_toggle(cx))
18841965 .child(
18851966 MetalButton::new("toggle-panel")
18861967 .ghost()
18871968 .mini()
18881969 .icon(if panel_visible { Lucide::PanelRightClose } else { Lucide::PanelRightOpen })
18891970 .tooltip("Project panel (Ctrl+Shift+B)")
1890- .on_click(cx.listener(|this, _, _, cx| {
1891- this.panel_open = !this.panel_open;
1892- cx.notify();
1893- })),
1971+ .on_click(cx.listener(|this, _, _, cx| this.toggle_project_panel(cx))),
18941972 ),
18951973 )
18961974 .child(metal::rule(p))
@@ -2204,10 +2282,14 @@ impl Render for Workspace {
22042282 .collect(),
22052283 };
22062284 cx.set_global(directory);
2285+ let browser = self.shown_browser(window, cx);
22072286 let p = metal::palette(cx);
22082287 let panel_visible = self.panel_open
2288+ && browser.is_none()
22092289 && self.active_project().is_some()
22102290 && window.viewport_size().width >= px(PANEL_MIN_WINDOW);
2291+ // The browser gives way before the conversation gets too narrow.
2292+ let chat_min = if browser.is_some() { px(360.) } else { px(0.) };
22112293
22122294 h_flex()
22132295 .key_context("Workspace")
@@ -2230,8 +2312,9 @@ impl Render for Workspace {
22302312 this.sidebar_open = !this.sidebar_open;
22312313 cx.notify();
22322314 }))
2233- .on_action(cx.listener(|this, _: &ToggleProjectPanel, _, cx| {
2234- this.panel_open = !this.panel_open;
2315+ .on_action(cx.listener(|this, _: &ToggleProjectPanel, _, cx| this.toggle_project_panel(cx)))
2316+ .on_action(cx.listener(|this, _: &ToggleBrowser, _, cx| {
2317+ this.browser_open = !this.browser_open;
22352318 cx.notify();
22362319 }))
22372320 .on_action(cx.listener(|this, _: &FocusSearch, window, cx| {
@@ -2268,11 +2351,12 @@ impl Render for Workspace {
22682351 .flex_1()
22692352 .min_h_0()
22702353 .items_start()
2271- .child(div().flex_1().min_w_0().h_full().map(|this| match chat {
2354+ .child(div().flex_1().min_w(chat_min).h_full().map(|this| match chat {
22722355 Some(chat) => this.child(chat),
22732356 None => this.child(self.render_empty_sheet(cx)),
22742357 }))
2275- .when(panel_visible, |this| this.child(self.render_panel(cx))),
2358+ .when(panel_visible, |this| this.child(self.render_panel(cx)))
2359+ .when_some(browser, |this, browser| this.child(browser)),
22762360 ),
22772361 ),
22782362 ))