//! The workspace's side of devices: each thread's Device panel, the device
//! that floats over the chat, devices agents open and close, and the
//! `device_*` tools agents call.
use std::collections::HashMap;
use base64::Engine as _;
use gpui_kit::component::{Icon, h_flex, v_flex};
use gpui_kit::*;
use serde_json::{Value, json};
use super::Workspace;
use super::delegation::Answer;
use super::surfaces::Surface;
use crate::device::{self, DeviceSummary, Event, HostStatus, LOCAL_HOST, Platform};
use crate::settings;
use crate::ui::device::{DevicePanel, DevicePanelEvent, DeviceSetup, DeviceWatch};
use crate::ui::icons::Lucide;
use crate::ui::metal::{self, MetalButton, MetalStyled as _};
pub const DEVICE_LIST: &str = "device_list";
pub const DEVICE_OPEN: &str = "device_open";
pub const DEVICE_SCREENSHOT: &str = "device_screenshot";
pub const DEVICE_CLOSE: &str = "device_close";
const FLOAT_DEFAULT: Size<Pixels> = Size { width: px(300.), height: px(560.) };
const FLOAT_MIN: Size<Pixels> = Size { width: px(200.), height: px(260.) };
const FLOAT_MARGIN: f32 = 16.;
const FLOAT_HEADER: f32 = 34.;
/// Which part of the floating window a drag holds.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Grip {
Move,
Left,
Right,
Top,
Bottom,
TopLeft,
TopRight,
BottomLeft,
BottomRight,
}
struct FloatDrag {
grip: Grip,
start: Point<Pixels>,
rect: Bounds<Pixels>,
}
#[derive(Default)]
pub(super) struct Devices {
/// Each thread's Device panel, by thread id.
panels: HashMap<String, Entity<DevicePanel>>,
/// The floating window, in window coordinates, once placed.
float_rect: Option<Bounds<Pixels>>,
float_drag: Option<FloatDrag>,
/// The conversation's area, which the floating window stays inside.
chat_area: Bounds<Pixels>,
/// Whether the setup dialog was offered since the Device surface opened.
setup_offered: bool,
_events: Option<Task<()>>,
}
impl Workspace {
/// Follows devices agents open and close.
pub(super) fn listen_for_devices(&mut self, window: &mut Window, cx: &mut Context<Self>) {
let events = device::service().subscribe();
self.devices._events = Some(cx.spawn_in(window, async move |this, cx| {
while let Ok(event) = events.recv().await {
let handled = this.update_in(cx, |this, window, cx| match event {
Event::AgentOpened { thread, device } => this.agent_opened_device(&thread, device, window, cx),
Event::AgentClosed { thread, device } => {
if let Some(panel) = this.devices.panels.get(&thread).cloned() {
panel.update(cx, |panel, cx| panel.forget_device(device.as_deref(), cx));
}
}
Event::Changed => {}
});
if handled.is_err() {
break;
}
}
}));
}
/// A device an agent opened floats over the chat, or opens in the right
/// panel when Auto-show floating preview is off.
fn agent_opened_device(&mut self, thread: &str, device: DeviceSummary, window: &mut Window, cx: &mut Context<Self>) {
let float = settings::get(|settings| settings.auto_float_preview);
let panel = self.device_panel(thread, window, cx);
panel.update(cx, |panel, cx| panel.show_device(device, float, window, cx));
let active = self.active.as_ref().is_some_and(|session| session.read(cx).id == thread);
if active && !float {
self.open_surface(Surface::Device, window, cx);
}
cx.notify();
}
/// A thread's Device panel, made on first use.
pub(super) fn device_panel(&mut self, thread: &str, window: &mut Window, cx: &mut Context<Self>) -> Entity<DevicePanel> {
if let Some(panel) = self.devices.panels.get(thread) {
return panel.clone();
}
let panel = cx.new(|cx| DevicePanel::new(thread.to_string(), window, cx));
let thread_id = thread.to_string();
cx.subscribe_in(&panel, window, move |this, _, event: &DevicePanelEvent, window, cx| {
let active = this.active.as_ref().is_some_and(|session| session.read(cx).id == thread_id);
match event {
DevicePanelEvent::Close => {
if active && this.surface_open(Surface::Device) {
this.right_panel = None;
}
}
// Floating closes the panel, as the browser's floating preview does.
DevicePanelEvent::Floated => {
if active && this.surface_open(Surface::Device) {
this.right_panel = None;
}
}
DevicePanelEvent::Docked => {
if active {
this.open_surface(Surface::Device, window, cx);
}
}
}
cx.notify();
})
.detach();
self.devices.panels.insert(thread.to_string(), panel.clone());
panel
}
/// The open thread's Device panel while the Device surface shows. Tells
/// every panel what's on screen, so only that device streams.
pub(super) fn shown_device_panel(&mut self, window: &mut Window, cx: &mut Context<Self>) -> Option<Entity<DevicePanel>> {
let thread = self.active.as_ref().map(|session| session.read(cx).id.clone());
let surface = self.surface_open(Surface::Device) && !self.settings_open;
let shown = match (&thread, surface) {
(Some(thread), true) => Some(self.device_panel(thread, window, cx)),
_ => None,
};
if !surface {
self.devices.setup_offered = false;
} else if !self.devices.setup_offered {
self.devices.setup_offered = true;
let state = DeviceWatch::state(cx);
if state.host_status == HostStatus::Disabled || !state.onboarding_completed {
let workspace = cx.weak_entity();
cx.defer_in(window, move |_, window, cx| {
DeviceSetup::open_with(
window,
cx,
Some(move |_: &mut Window, cx: &mut App| {
let _ = workspace.update(cx, |this, cx| {
if this.surface_open(Surface::Device) {
this.right_panel = None;
cx.notify();
}
});
}),
);
});
}
}
let panels: Vec<(String, Entity<DevicePanel>)> =
self.devices.panels.iter().map(|(thread, panel)| (thread.clone(), panel.clone())).collect();
for (id, panel) in panels {
let current = thread.as_deref() == Some(id.as_str());
let panel_visible = current && surface;
let float_visible = current && !self.settings_open;
panel.update(cx, |panel, cx| panel.set_visible(panel_visible, float_visible, window, cx));
}
shown
}
/// Drops a deleted thread's panel and closes its devices.
pub(super) fn forget_thread_devices(&mut self, thread: &str, cx: &mut Context<Self>) {
if self.devices.panels.remove(thread).is_some() {
let thread = thread.to_string();
cx.background_executor().spawn(async move { device::service().close(&thread, None, false) }).detach();
}
}
// ---- the floating window ----
pub(super) fn track_chat_area(&mut self, bounds: Bounds<Pixels>) {
self.devices.chat_area = bounds;
}
/// The floating window's place: where the user put it, kept inside the conversation.
fn float_rect(&self) -> Bounds<Pixels> {
let area = self.devices.chat_area;
let margin = px(FLOAT_MARGIN);
let rect = self.devices.float_rect.unwrap_or_else(|| {
let size = Size {
width: FLOAT_DEFAULT.width.min((area.size.width - margin * 2.).max(FLOAT_MIN.width)),
height: FLOAT_DEFAULT.height.min((area.size.height - margin * 2.).max(FLOAT_MIN.height)),
};
Bounds { origin: point(area.right() - size.width - margin, area.bottom() - size.height - margin), size }
});
let width = rect.size.width.min(area.size.width).max(FLOAT_MIN.width);
let height = rect.size.height.min(area.size.height).max(FLOAT_MIN.height);
let x = rect.origin.x.min(area.right() - width).max(area.left());
let y = rect.origin.y.min(area.bottom() - height).max(area.top());
Bounds { origin: point(x, y), size: Size { width, height } }
}
pub(super) fn float_dragging(&self) -> bool {
self.devices.float_drag.is_some()
}
pub(super) fn drag_float(&mut self, event: &MouseMoveEvent, _: &mut Window, cx: &mut Context<Self>) {
let Some(drag) = &self.devices.float_drag else { return };
let delta = event.position - drag.start;
let mut rect = drag.rect;
let (left, right, top, bottom) = match drag.grip {
Grip::Move => {
rect.origin = rect.origin + delta;
self.devices.float_rect = Some(rect);
cx.notify();
return;
}
Grip::Left => (true, false, false, false),
Grip::Right => (false, true, false, false),
Grip::Top => (false, false, true, false),
Grip::Bottom => (false, false, false, true),
Grip::TopLeft => (true, false, true, false),
Grip::TopRight => (false, true, true, false),
Grip::BottomLeft => (true, false, false, true),
Grip::BottomRight => (false, true, false, true),
};
if left {
let width = (rect.size.width - delta.x).max(FLOAT_MIN.width);
rect.origin.x = rect.right() - width;
rect.size.width = width;
}
if right {
rect.size.width = (rect.size.width + delta.x).max(FLOAT_MIN.width);
}
if top {
let height = (rect.size.height - delta.y).max(FLOAT_MIN.height);
rect.origin.y = rect.bottom() - height;
rect.size.height = height;
}
if bottom {
rect.size.height = (rect.size.height + delta.y).max(FLOAT_MIN.height);
}
self.devices.float_rect = Some(rect);
cx.notify();
}
pub(super) fn end_float_drag(&mut self, cx: &mut Context<Self>) {
if self.devices.float_drag.take().is_some() {
self.devices.float_rect = Some(self.float_rect());
cx.notify();
}
}
fn start_float_drag(&mut self, grip: Grip, position: Point<Pixels>, cx: &mut Context<Self>) {
self.devices.float_drag = Some(FloatDrag { grip, start: position, rect: self.float_rect() });
cx.notify();
}
/// The open thread's floating device, over the chat.
pub(super) fn render_floating_device(&mut self, cx: &mut Context<Self>) -> Option<AnyElement> {
if self.settings_open {
return None;
}
let thread = self.active.as_ref()?.read(cx).id.clone();
let panel = self.devices.panels.get(&thread)?.clone();
let view = panel.read(cx).floating_view()?;
let p = metal::palette(cx);
let rect = self.float_rect();
let name = view.read(cx).device.name.clone();
let edge = |id: &'static str, grip: Grip| {
div().id(id).absolute().on_mouse_down(
MouseButton::Left,
cx.listener(move |this, event: &MouseDownEvent, _, cx| {
this.start_float_drag(grip, event.position, cx);
cx.stop_propagation();
}),
)
};
let dock_panel = panel.clone();
let close_panel = panel.clone();
let window = v_flex()
.id("device-float")
.absolute()
.left(rect.origin.x - self.devices.chat_area.origin.x)
.top(rect.origin.y - self.devices.chat_area.origin.y)
.w(rect.size.width)
.h(rect.size.height)
.rounded(px(16.))
.raised(p)
.occlude()
.child(
h_flex()
.id("device-float-handle")
.h(px(FLOAT_HEADER))
.flex_none()
.pl(px(8.))
.pr(px(4.))
.gap(px(6.))
.cursor_grab()
.on_mouse_down(
MouseButton::Left,
cx.listener(|this, event: &MouseDownEvent, _, cx| {
this.start_float_drag(Grip::Move, event.position, cx);
cx.stop_propagation();
}),
)
.child(Icon::new(Lucide::GripHorizontal).size(px(14.)).text_color(p.ink3))
.child(div().flex_1().min_w_0().truncate().text_size(px(12.)).font_weight(FontWeight::MEDIUM).text_color(p.ink2).child(name))
.child(
MetalButton::new("device-float-dock")
.ghost()
.mini()
.icon(Lucide::PanelRightOpen)
.tooltip("Open in right panel")
.on_click(move |_, _, cx| {
dock_panel.update(cx, |panel, cx| panel.dock(cx));
}),
)
.child(
MetalButton::new("device-float-close")
.ghost()
.mini()
.icon(Lucide::X)
.tooltip("Close floating device")
.on_click(move |_, _, cx| {
close_panel.update(cx, |panel, cx| panel.unfloat(cx));
}),
),
)
.child(div().flex_1().min_h_0().mx(px(6.)).mb(px(6.)).rounded(px(11.)).overflow_hidden().well(p).child(view))
.child(edge("device-float-left", Grip::Left).left(px(-3.)).top(px(12.)).bottom(px(12.)).w(px(6.)).cursor_col_resize())
.child(edge("device-float-right", Grip::Right).right(px(-3.)).top(px(12.)).bottom(px(12.)).w(px(6.)).cursor_col_resize())
.child(edge("device-float-top", Grip::Top).top(px(-3.)).left(px(12.)).right(px(12.)).h(px(6.)).cursor_row_resize())
.child(edge("device-float-bottom", Grip::Bottom).bottom(px(-3.)).left(px(12.)).right(px(12.)).h(px(6.)).cursor_row_resize())
.child(edge("device-float-tl", Grip::TopLeft).top(px(-3.)).left(px(-3.)).size(px(12.)).cursor_nwse_resize())
.child(edge("device-float-tr", Grip::TopRight).top(px(-3.)).right(px(-3.)).size(px(12.)).cursor_nesw_resize())
.child(edge("device-float-bl", Grip::BottomLeft).bottom(px(-3.)).left(px(-3.)).size(px(12.)).cursor_nesw_resize())
.child(edge("device-float-br", Grip::BottomRight).bottom(px(-3.)).right(px(-3.)).size(px(12.)).cursor_nwse_resize());
Some(window.into_any_element())
}
// ---- agents' device tools ----
/// Answers a `device_*` call. The work runs off the UI thread.
pub(super) fn answer_device_tool(&mut self, thread: &str, tool: &'static str, args: Value, answer: Answer, cx: &mut Context<Self>) {
let thread = thread.to_string();
cx.spawn(async move |_, cx| {
let outcome = cx.background_executor().spawn(async move { device_tool(&thread, tool, &args) }).await;
let _ = cx.update(|cx| answer.send(outcome, cx));
})
.detach();
}
}
/// Runs one device tool call for a thread's agent.
fn device_tool(thread: &str, tool: &str, args: &Value) -> Result<Value, String> {
let service = device::service();
if !settings::get(|settings| settings.enable_device_support && settings.enable_agent_device_access) {
return Err("Agent device access is turned off for this environment.".into());
}
let text = |key: &str| args.get(key).and_then(Value::as_str).map(str::trim).filter(|value| !value.is_empty()).map(str::to_string);
let host = text("hostId").unwrap_or_else(|| LOCAL_HOST.into());
if host != LOCAL_HOST {
return Err(format!("Device host {host} is unavailable: Unknown host."));
}
let disabled = "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.";
match tool {
DEVICE_LIST => {
let state = service.list();
if state.host_status == HostStatus::Disabled {
return Err(disabled.into());
}
let open: Vec<Value> = service.sessions_for(thread).iter().map(|session| json!({"hostId": session.host_id, "deviceId": session.device_id})).collect();
Ok(json!({
"hostStatuses": {LOCAL_HOST: {"status": state.host_status, "detail": state.host_detail}},
"hosts": state.hosts,
"devices": state.devices,
"open": open,
}))
}
DEVICE_OPEN => {
let state = service.list();
if state.host_status == HostStatus::Disabled {
return Err(disabled.into());
}
let platform = text("platform").and_then(|platform| Platform::from_key(&platform));
let target = pick_device(&state.devices, text("deviceId").as_deref(), platform)?;
// Agent access and connectivity come before booting or opening anything.
let agent_args = service.agent_target(thread, &target.id).map_err(|err| format!("{err:#}"))?;
let session = service.open(thread, &target.id, true).map_err(|err| format!("{err:#}"))?;
let device = service.state().device(&session.host_id, &session.device_id).cloned().unwrap_or(target);
let mut target_args = device::agent::target_args(&device);
target_args.extend(agent_args);
let command = service.agent_cli().map_err(|err| format!("Could not prepare the agent-device launcher: {err:#}"))?;
let command = command.display().to_string();
let quick_start = device::agent::quick_start(&device, &target_args, &command);
service.notify_agent_opened(thread, device.clone());
Ok(json!({
"device": device,
"agentDevice": {"command": command, "targetArgs": target_args},
"quickStart": quick_start,
}))
}
DEVICE_SCREENSHOT => {
let wanted = text("deviceId");
// Only devices this thread opened; another thread's isn't this agent's to watch.
let session = service
.sessions_for(thread)
.into_iter()
.rev()
.find(|session| wanted.as_deref().is_none_or(|id| session.device_id == id))
.ok_or_else(|| match &wanted {
None => "No device is open in this thread. Call device_open first.".to_string(),
Some(id) => format!("Device {id} on host {host} is not open in this thread. Call device_open first."),
})?;
let (device, png) = service.screenshot(&session.device_id).map_err(|err| format!("{err:#}"))?;
let (width, height) = device::png_size(&png);
let saved = std::env::temp_dir().join(format!("vectrum-device-{}.png", chrono::Local::now().timestamp_millis()));
let saved = std::fs::write(&saved, &png).ok().map(|_| saved.display().to_string());
Ok(json!({
"device": device,
"screenshot": {
"mimeType": "image/png",
"data": base64::engine::general_purpose::STANDARD.encode(&png),
"width": width,
"height": height,
},
"savedTo": saved,
}))
}
DEVICE_CLOSE => {
let device = text("deviceId");
let shutdown = args.get("shutdown").and_then(Value::as_bool).unwrap_or(false);
service.close(thread, device.as_deref(), shutdown).map_err(|err| format!("{err:#}"))?;
service.notify_agent_closed(thread, device);
Ok(json!({}))
}
_ => Err(format!("Vectrum Code has no tool named {tool}.")),
}
}
/// The device a `device_open` means: the one named, else the booted (or
/// first) device of the platform.
fn pick_device(devices: &[DeviceSummary], id: Option<&str>, platform: Option<Platform>) -> Result<DeviceSummary, String> {
if let Some(id) = id {
return devices
.iter()
.find(|device| device.id == id)
.cloned()
.ok_or_else(|| format!("No device {id} on host {LOCAL_HOST}. Call device_list for current ids."));
}
let candidates: Vec<&DeviceSummary> = devices.iter().filter(|device| platform.is_none_or(|platform| device.platform == platform)).collect();
if candidates.is_empty() {
return Err(match platform {
None => "No simulators or emulators were found. Call device_list to see why.".into(),
Some(platform) => format!("No {} devices were found on host {LOCAL_HOST}. Call device_list to see why.", platform.key()),
});
}
let first = candidates[0].platform;
if platform.is_none() && candidates.iter().any(|device| device.platform != first) {
return Err("Both iOS and Android devices are available; pass platform or deviceId.".into());
}
Ok(candidates.iter().find(|device| device.booted).copied().unwrap_or(candidates[0]).clone())
}
#[cfg(test)]
mod tests {
use super::pick_device;
use crate::device::{DeviceSummary, Platform};
fn device(id: &str, platform: Platform, booted: bool) -> DeviceSummary {
DeviceSummary {
host_id: "local".into(),
id: id.into(),
platform,
name: id.into(),
version: String::new(),
booted,
physical: false,
}
}
#[::core::prelude::v1::test]
fn picks_the_device_an_agent_means() {
let devices = vec![device("pixel", Platform::Android, false), device("emulator-5554", Platform::Android, true)];
assert_eq!(pick_device(&devices, None, None).unwrap().id, "emulator-5554");
assert_eq!(pick_device(&devices, Some("pixel"), None).unwrap().id, "pixel");
assert!(pick_device(&devices, Some("nope"), None).is_err());
assert!(pick_device(&devices, None, Some(Platform::Ios)).unwrap_err().contains("No ios devices"));
let mixed = vec![device("a", Platform::Android, false), device("b", Platform::Ios, false)];
assert!(pick_device(&mixed, None, None).unwrap_err().contains("pass platform"));
assert_eq!(pick_device(&mixed, None, Some(Platform::Ios)).unwrap().id, "b");
}
}