//! The browser panel: the thread's shared browser, live, beside the
//! conversation, like T3 Code's browser preview. The agent drives it with the
//! `agent-browser` CLI; the header navigates, and the viewport passes the
//! user's clicks, scrolling and typing on to the page.
use std::{
collections::HashSet,
time::{Duration, Instant},
};
use gpui_kit::component::{
Icon, IconName, h_flex,
input::{Input, InputEvent, InputState},
v_flex,
};
use gpui_kit::prelude::FluentBuilder as _;
use gpui_kit::*;
use crate::browser::{
self, BrowserEvent, BrowserSession, Frame, Input as PageInput, MouseEventType, MouseInput, PointerButton, modifiers,
};
use crate::ui::metal::{self, Led, MetalButton, MetalStyled as _};
/// The panel's share of the sheet until the user drags it.
const SHARE: f32 = 0.45;
const MIN_WIDTH: f32 = 420.;
/// How long the panel shows the agent driving after its last command.
const AGENT_LINGER: Duration = Duration::from_secs(4);
/// How long the viewport has to keep its size before the page is fit to it.
const FIT_DELAY: Duration = Duration::from_millis(400);
/// Pixels a wheel line scrolls.
const LINE: f32 = 40.;
pub enum BrowserPanelEvent {
Close,
}
/// The panel's width once the user drags its edge, shared by all threads.
#[derive(Default)]
struct BrowserWidth(Option<Pixels>);
impl Global for BrowserWidth {}
/// The edge the user drags to resize the panel.
struct ResizeBrowser;
/// Which threads' browsers run, refreshed in the background, so the
/// workspace can mark the browser button when the agent opens a page while
/// the panel is closed.
#[derive(Default)]
pub struct BrowserWatch {
live: HashSet<String>,
}
impl Global for BrowserWatch {}
impl BrowserWatch {
pub fn init(cx: &mut App) {
cx.set_global(BrowserWatch::default());
cx.spawn(async move |cx| {
loop {
cx.background_executor().timer(Duration::from_secs(2)).await;
let live = cx.background_executor().spawn(async { browser::running_sessions() }).await;
cx.update(|cx| {
if cx.global::<BrowserWatch>().live != live {
cx.global_mut::<BrowserWatch>().live = live;
cx.refresh_windows();
}
});
}
})
.detach();
}
/// Whether a thread's browser runs.
pub fn is_live(thread_id: &str, cx: &App) -> bool {
cx.try_global::<BrowserWatch>().is_some_and(|watch| watch.live.contains(&browser::session_name(thread_id)))
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum PageState {
/// `agent-browser` isn't installed.
Missing,
/// Looking for the session's daemon.
Starting,
/// No browser runs yet.
Idle,
Live,
}
pub struct BrowserPanel {
session: BrowserSession,
url: Entity<InputState>,
state: PageState,
frame: Option<Frame>,
/// When the agent last ran a command.
agent_at: Option<Instant>,
error: Option<SharedString>,
visible: bool,
focus_handle: FocusHandle,
/// Where the viewport was last drawn, in window coordinates.
area: Bounds<Pixels>,
/// The page size last asked for, so the page is fit once per size.
fitted: Option<(u32, u32)>,
_fit: Task<()>,
_agent_timer: Task<()>,
_events: Task<()>,
_subscriptions: Vec<Subscription>,
}
impl EventEmitter<BrowserPanelEvent> for BrowserPanel {}
impl BrowserPanel {
pub fn new(thread_id: &str, window: &mut Window, cx: &mut Context<Self>) -> Self {
let (session, events) = Self::watch(browser::session_name(thread_id), window, cx);
let url = cx.new(|cx| InputState::new(window, cx).placeholder("Type a URL"));
let subscriptions = vec![
cx.subscribe_in(&url, window, |this, url, event: &InputEvent, window, cx| {
if matches!(event, InputEvent::PressEnter { .. }) {
let text = url.read(cx).value().to_string();
if let Some(target) = browser::normalize_url(&text) {
url.update(cx, |url, cx| url.set_value(target.clone(), window, cx));
this.error = None;
this.session.open(&target);
this.focus_handle.focus(window, cx);
cx.notify();
}
}
}),
cx.on_release(|this, cx| {
if let Some(frame) = this.frame.take() {
cx.drop_image(frame.image, None);
}
}),
];
let state = if session.installed() { PageState::Starting } else { PageState::Missing };
Self {
session,
url,
state,
frame: None,
agent_at: None,
error: None,
visible: false,
focus_handle: cx.focus_handle(),
area: Bounds::default(),
fitted: None,
_fit: Task::ready(()),
_agent_timer: Task::ready(()),
_events: events,
_subscriptions: subscriptions,
}
}
/// A session and the task that applies what it reports.
fn watch(name: String, window: &mut Window, cx: &mut Context<Self>) -> (BrowserSession, Task<()>) {
let (tx, rx) = smol::channel::unbounded();
let session = BrowserSession::new(name, tx);
let events = cx.spawn_in(window, async move |this, cx| {
while let Ok(event) = rx.recv().await {
let mut batch = vec![event];
while let Ok(event) = rx.try_recv() {
batch.push(event);
}
let updated = this.update_in(cx, |this, window, cx| {
for event in batch {
this.handle_event(event, window, cx);
}
cx.notify();
});
if updated.is_err() {
break;
}
}
});
(session, events)
}
/// Looks for agent-browser again, after the user installed it.
fn retry(&mut self, window: &mut Window, cx: &mut Context<Self>) {
let (session, events) = Self::watch(self.session.name().to_string(), window, cx);
self.session = session;
self._events = events;
self.session.set_watching(self.visible);
self.state = if self.session.installed() { PageState::Starting } else { PageState::Missing };
cx.notify();
}
/// Watches the stream while the panel shows; a hidden panel lets go of it.
pub fn set_visible(&mut self, visible: bool, cx: &mut Context<Self>) {
if self.visible != visible {
self.visible = visible;
self.session.set_watching(visible);
cx.notify();
}
}
fn handle_event(&mut self, event: BrowserEvent, window: &mut Window, cx: &mut Context<Self>) {
match event {
BrowserEvent::Missing => self.state = PageState::Missing,
BrowserEvent::Waiting => {
self.state = PageState::Idle;
self.set_frame(None, window, cx);
}
BrowserEvent::Live { browser } => {
self.state = if browser { PageState::Live } else { PageState::Idle };
if !browser {
self.set_frame(None, window, cx);
}
}
BrowserEvent::Frame => {
if let Some(frame) = self.session.take_frame() {
self.state = PageState::Live;
self.set_frame(Some(frame), window, cx);
self.fit_page(window, cx);
}
}
BrowserEvent::Url(url) => {
if !self.url.read(cx).focus_handle(cx).is_focused(window) {
self.url.update(cx, |input, cx| input.set_value(url, window, cx));
}
}
BrowserEvent::Agent(_) => {
self.agent_at = Some(Instant::now());
// Repaint once the indicator should go out.
self._agent_timer = cx.spawn(async move |this, cx| {
cx.background_executor().timer(AGENT_LINGER).await;
let _ = this.update(cx, |_, cx| cx.notify());
});
}
BrowserEvent::Failed(err) => self.error = Some(err.into()),
}
}
/// Shows a new frame and frees the GPU texture of the one it replaces.
fn set_frame(&mut self, frame: Option<Frame>, window: &mut Window, cx: &mut Context<Self>) {
let browser_restarted = self.frame.is_none() && frame.is_some();
if let Some(old) = std::mem::replace(&mut self.frame, frame) {
cx.drop_image(old.image, Some(window));
}
if browser_restarted {
self.fitted = None;
}
}
/// Sizes the page to the viewport, so it renders at the panel's size
/// rather than scaled down. Once per size: a viewport the agent picks
/// stays until the panel changes size.
fn fit_page(&mut self, window: &Window, cx: &mut Context<Self>) {
let size = self.area.size;
let target = (f32::from(size.width).round() as u32, f32::from(size.height).round() as u32);
if !self.visible || self.state != PageState::Live || target.0 < 200 || target.1 < 150 {
return;
}
if self.fitted == Some(target) {
return;
}
let scale = window.scale_factor();
self._fit = cx.spawn(async move |this, cx| {
cx.background_executor().timer(FIT_DELAY).await;
let _ = this.update(cx, |this, _| {
let size = this.area.size;
let current = (f32::from(size.width).round() as u32, f32::from(size.height).round() as u32);
if current == target && this.fitted != Some(target) {
this.fitted = Some(target);
this.session.set_viewport(target.0, target.1, scale);
}
});
});
}
/// A window position as page coordinates, if it's over the page.
fn page_point(&self, position: Point<Pixels>) -> Option<(f64, f64)> {
let frame = self.frame.as_ref()?;
let origin = self.area.origin;
browser::page_point(
(f32::from(position.x - origin.x), f32::from(position.y - origin.y)),
(f32::from(self.area.size.width), f32::from(self.area.size.height)),
frame.size,
frame.viewport,
)
}
/// Page pixels per window pixel.
fn page_scale(&self) -> f32 {
let Some(frame) = &self.frame else {
return 1.;
};
let area = (f32::from(self.area.size.width), f32::from(self.area.size.height));
let (_, _, width, _) = browser::contain(area, frame.size);
if width > 0. { frame.viewport.0 / width } else { 1. }
}
fn send_mouse(
&self,
event_type: MouseEventType,
position: Point<Pixels>,
button: Option<MouseButton>,
click_count: usize,
mods: &Modifiers,
) {
let Some(point) = self.page_point(position) else {
return;
};
let mut input = MouseInput::new(event_type, point, modifier_bits(mods));
input.button = match button {
Some(MouseButton::Left) => PointerButton::Left,
Some(MouseButton::Middle) => PointerButton::Middle,
Some(MouseButton::Right) => PointerButton::Right,
_ => PointerButton::None,
};
input.click_count = click_count as u32;
self.session.send(PageInput::Mouse(input));
}
fn on_key_down(&mut self, event: &KeyDownEvent, _: &mut Window, cx: &mut Context<Self>) {
let keystroke = &event.keystroke;
let mods = modifier_bits(&keystroke.modifiers);
// Chrome runs no clipboard shortcuts headless; paste the text instead.
if keystroke.key == "v" && keystroke.modifiers.control && !keystroke.modifiers.alt {
if let Some(text) = cx.read_from_clipboard().and_then(|item| item.text()) {
for input in browser::text_input(&text) {
self.session.send(PageInput::Key(input));
}
}
cx.stop_propagation();
return;
}
let input = browser::key_input(&keystroke.key, keystroke.key_char.as_deref(), mods, true);
self.session.send(PageInput::Key(input));
cx.stop_propagation();
}
fn on_key_up(&mut self, event: &KeyUpEvent, _: &mut Window, cx: &mut Context<Self>) {
let keystroke = &event.keystroke;
let mods = modifier_bits(&keystroke.modifiers);
let input = browser::key_input(&keystroke.key, keystroke.key_char.as_deref(), mods, false);
self.session.send(PageInput::Key(input));
cx.stop_propagation();
}
fn render_header(&self, cx: &mut Context<Self>) -> impl IntoElement {
let p = metal::palette(cx);
let live = self.state == PageState::Live;
let driving = self.agent_at.is_some_and(|at| at.elapsed() < AGENT_LINGER);
h_flex()
.h(px(48.))
.flex_none()
.px(px(8.))
.gap(px(4.))
.child(
MetalButton::new("browser-back")
.ghost()
.mini()
.icon(IconName::ArrowLeft)
.tooltip("Back")
.disabled(!live)
.on_click(cx.listener(|this, _, _, _| this.session.back())),
)
.child(
MetalButton::new("browser-forward")
.ghost()
.mini()
.icon(IconName::ArrowRight)
.tooltip("Forward")
.disabled(!live)
.on_click(cx.listener(|this, _, _, _| this.session.forward())),
)
.child(
MetalButton::new("browser-reload")
.ghost()
.mini()
.icon(IconName::RotateCw)
.tooltip("Reload")
.disabled(!live)
.on_click(cx.listener(|this, _, _, _| this.session.reload())),
)
.child(
h_flex()
.flex_1()
.min_w_0()
.h(px(30.))
.ml(px(4.))
.pl(px(10.))
.pr(px(8.))
.gap(px(8.))
.rounded(px(12.))
.well(p)
.text_size(px(12.5))
.text_color(p.ink)
.child(Icon::new(IconName::Globe).size(px(13.)).text_color(p.field_ink))
.child(div().flex_1().min_w_0().child(Input::new(&self.url).appearance(false)))
.when(driving, |this| {
this.child(
h_flex()
.flex_none()
.gap(px(5.))
.child(metal::led("browser-agent", Led::Live, true, p))
.child(metal::engraved("Agent", p)),
)
}),
)
.child(
MetalButton::new("browser-close")
.ghost()
.mini()
.icon(IconName::Close)
.tooltip("Close browser (Ctrl+Shift+E)")
.on_click(cx.listener(|_, _, _, cx| cx.emit(BrowserPanelEvent::Close))),
)
}
/// The live page, or what stands in for it.
fn render_viewport(&self, cx: &mut Context<Self>) -> AnyElement {
let entity = cx.entity().downgrade();
let track_area = canvas(
move |bounds, window, cx| {
let _ = entity.update(cx, |this, cx| {
if this.area != bounds {
this.area = bounds;
this.fit_page(window, cx);
}
});
},
|_, _, _, _| {},
)
.absolute()
.size_full();
let Some(frame) = &self.frame else {
return div()
.relative()
.size_full()
.child(track_area)
.child(self.render_placeholder(cx))
.into_any_element();
};
div()
.id("browser-viewport")
.relative()
.size_full()
.key_context("BrowserViewport")
.track_focus(&self.focus_handle)
.cursor_default()
.child(track_area)
.child(img(ImageSource::Render(frame.image.clone())).size_full().object_fit(ObjectFit::Contain))
.on_mouse_down(MouseButton::Left, cx.listener(Self::mouse_down))
.on_mouse_down(MouseButton::Right, cx.listener(Self::mouse_down))
.on_mouse_down(MouseButton::Middle, cx.listener(Self::mouse_down))
.on_mouse_up(MouseButton::Left, cx.listener(Self::mouse_up))
.on_mouse_up(MouseButton::Right, cx.listener(Self::mouse_up))
.on_mouse_up(MouseButton::Middle, cx.listener(Self::mouse_up))
.on_mouse_move(cx.listener(|this, event: &MouseMoveEvent, _, _| {
this.send_mouse(MouseEventType::Move, event.position, event.pressed_button, 0, &event.modifiers);
}))
.on_scroll_wheel(cx.listener(|this, event: &ScrollWheelEvent, _, cx| {
let Some(point) = this.page_point(event.position) else {
return;
};
let delta = event.delta.pixel_delta(px(LINE));
let scale = this.page_scale();
let mut input = MouseInput::new(MouseEventType::Wheel, point, modifier_bits(&event.modifiers));
// GPUI's deltas move the content; the page's move the viewport.
input.delta_x = -(f32::from(delta.x) * scale) as f64;
input.delta_y = -(f32::from(delta.y) * scale) as f64;
this.session.send(PageInput::Mouse(input));
cx.stop_propagation();
}))
.on_key_down(cx.listener(Self::on_key_down))
.on_key_up(cx.listener(Self::on_key_up))
.into_any_element()
}
fn mouse_down(&mut self, event: &MouseDownEvent, window: &mut Window, cx: &mut Context<Self>) {
self.focus_handle.focus(window, cx);
self.send_mouse(MouseEventType::Move, event.position, None, 0, &event.modifiers);
self.send_mouse(MouseEventType::Down, event.position, Some(event.button), event.click_count, &event.modifiers);
}
fn mouse_up(&mut self, event: &MouseUpEvent, _: &mut Window, _: &mut Context<Self>) {
self.send_mouse(MouseEventType::Up, event.position, Some(event.button), event.click_count, &event.modifiers);
}
fn render_placeholder(&self, cx: &mut Context<Self>) -> impl IntoElement {
let p = metal::palette(cx);
let body = |text: &'static str| {
div().max_w(px(320.)).text_center().text_size(px(13.)).line_height(px(19.)).text_color(p.ink3).child(text)
};
let content = v_flex().items_center().gap(px(12.)).px(px(24.));
let content = match self.state {
PageState::Missing => content
.child(metal::engraved("agent-browser not installed", p))
.child(body("The browser panel runs on Vercel's agent-browser. Install it and its Chrome:"))
.child(
h_flex()
.max_w_full()
.gap(px(6.))
.pl(px(12.))
.pr(px(4.))
.py(px(5.))
.rounded(px(12.))
.well(p)
.child(
div()
.min_w_0()
.font_family(metal::MONO)
.text_size(px(11.5))
.text_color(p.ink)
.child(browser::INSTALL_COMMAND),
)
.child(
MetalButton::new("copy-install")
.ghost()
.mini()
.icon(IconName::Copy)
.tooltip("Copy")
.on_click(|_, _, cx| {
cx.write_to_clipboard(ClipboardItem::new_string(
browser::INSTALL_COMMAND.to_string(),
))
}),
),
)
.child(body("Threads started afterwards can use it."))
.child(
MetalButton::new("browser-retry")
.compact()
.label("Check again")
.on_click(cx.listener(|this, _, window, cx| this.retry(window, cx))),
),
PageState::Starting | PageState::Live => content
.child(metal::led("browser-starting", Led::Waiting, true, p))
.child(metal::engraved("Starting the browser", p)),
PageState::Idle => content
.child(Icon::new(IconName::Globe).size(px(22.)).text_color(p.ink3))
.child(metal::engraved("No page yet", p))
.child(body("The agent opens pages here, or type a URL.")),
};
div().size_full().flex().items_center().justify_center().child(content)
}
}
impl Render for BrowserPanel {
fn render(&mut self, _: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
let p = metal::palette(cx);
let width = cx.default_global::<BrowserWidth>().0;
let error = self.error.clone();
v_flex()
.relative()
.h_full()
.min_w(px(MIN_WIDTH))
.map(|this| match width {
Some(width) => this.w(width),
None => this.flex_basis(relative(SHARE)),
})
.p(px(12.))
.pl(px(0.))
.on_drag_move(cx.listener(|_, event: &DragMoveEvent<ResizeBrowser>, _, cx| {
// The panel's right edge stays put; its left edge follows the pointer.
let right = event.bounds.right();
let width = (right - event.event.position.x).max(px(MIN_WIDTH));
cx.set_global(BrowserWidth(Some(width)));
cx.notify();
}))
.child(
div()
.id("browser-resize")
.absolute()
.left(px(-6.))
.top_0()
.bottom_0()
.w(px(8.))
.cursor_col_resize()
.on_drag(ResizeBrowser, |_, _, _, cx| cx.new(|_| EmptyView)),
)
.child(
v_flex()
.size_full()
.rounded(px(18.))
.raised(p)
.child(self.render_header(cx))
.when_some(error, |this, error| {
this.child(
div().px(px(14.)).pb(px(6.)).truncate().text_size(px(11.5)).text_color(p.red).child(error),
)
})
.child(
div()
.flex_1()
.min_h_0()
.m(px(6.))
.mt(px(0.))
.rounded(px(12.))
.overflow_hidden()
.well(p)
.child(self.render_viewport(cx)),
),
)
}
}
/// GPUI's modifier keys as CDP's modifier bits.
fn modifier_bits(mods: &Modifiers) -> u8 {
let mut bits = 0;
if mods.alt {
bits |= modifiers::ALT;
}
if mods.control {
bits |= modifiers::CTRL;
}
if mods.platform {
bits |= modifiers::META;
}
if mods.shift {
bits |= modifiers::SHIFT;
}
bits
}