//! The workspace's side of devices: each thread's Device panel, the device //! that floats over the chat, devices agents open and close, and the //! `device_*` tools agents call. A thread on another device uses that //! device's devices, over the remote connection; this device serves its own //! to the devices connected to it. use std::{ collections::{HashMap, HashSet}, sync::Arc, }; use base64::Engine as _; use gpui_kit::component::{Icon, h_flex, v_flex}; use gpui_kit::*; use serde_json::{Value, json}; use super::Workspace; use super::delegation::Answer; use super::surfaces::Surface; use crate::device::{self, DeviceSummary, Event, HostStatus, LOCAL_HOST, Platform, State}; use crate::remote::{DeviceCall, ServerMessage}; use crate::session::Session; use crate::settings; use crate::ui::device::{Backend, DevicePanel, DevicePanelEvent, DeviceSetup, DeviceWatch, RemoteDevices, answer_device_call}; use crate::ui::icons::Lucide; use crate::ui::metal::{self, MetalButton, MetalStyled as _}; pub const DEVICE_LIST: &str = "device_list"; pub const DEVICE_OPEN: &str = "device_open"; pub const DEVICE_SCREENSHOT: &str = "device_screenshot"; pub const DEVICE_CLOSE: &str = "device_close"; const FLOAT_DEFAULT: Size = Size { width: px(300.), height: px(560.) }; const FLOAT_MIN: Size = Size { width: px(200.), height: px(260.) }; const FLOAT_MARGIN: f32 = 16.; const FLOAT_HEADER: f32 = 34.; /// Which part of the floating window a drag holds. #[derive(Clone, Copy, PartialEq, Eq)] enum Grip { Move, Left, Right, Top, Bottom, TopLeft, TopRight, BottomLeft, BottomRight, } struct FloatDrag { grip: Grip, start: Point, rect: Bounds, } #[derive(Default)] pub(super) struct Devices { /// Each thread's Device panel, by thread id. panels: HashMap>, /// Connected devices' devices, by the connection's id. links: HashMap>, /// Clients of this device that follow its devices, by connection. watchers: HashSet, /// The floating window, in window coordinates, once placed. float_rect: Option>, float_drag: Option, /// The conversation's area, which the floating window stays inside. chat_area: Bounds, /// Whether the setup dialog was offered since the Device surface opened. setup_offered: bool, _events: Option>, } impl Workspace { /// Follows devices agents open and close. pub(super) fn listen_for_devices(&mut self, window: &mut Window, cx: &mut Context) { let events = device::service().subscribe(); self.devices._events = Some(cx.spawn_in(window, async move |this, cx| { while let Ok(event) = events.recv().await { let handled = this.update_in(cx, |this, window, cx| match event { Event::AgentOpened { thread, device } => { // Connected devices show it too, for their view of the thread. if let Some(server) = this.remote_server() { server.broadcast(&ServerMessage::DeviceOpened { thread: thread.clone(), device: device.clone() }); } if this.local_thread(&thread, cx).is_some() { this.agent_opened_device(&thread, device, window, cx); } } Event::AgentClosed { thread, device } => { if let Some(server) = this.remote_server() { server.broadcast(&ServerMessage::DeviceClosed { thread: thread.clone(), device: device.clone() }); } this.agent_closed_device(&thread, device.as_deref(), cx); } Event::Changed => this.send_device_state(), }); if handled.is_err() { break; } } })); } /// A device an agent opened floats over the chat, or opens in the right /// panel when Auto-show floating preview is off. pub(super) fn agent_opened_device(&mut self, thread: &str, device: DeviceSummary, window: &mut Window, cx: &mut Context) { let float = settings::get(|settings| settings.auto_float_preview); let panel = self.device_panel(thread, window, cx); panel.update(cx, |panel, cx| panel.show_device(device, float, window, cx)); let active = self.active.as_ref().is_some_and(|session| session.read(cx).id == thread); if active && !float { self.open_surface(Surface::Device, window, cx); } else if let Some(session) = self.session_by_id(thread, cx) { let id = session.entity_id(); if !float { // It's there when you open the thread. self.set_thread_surface(id, Some(Surface::Device)); } else if self.thread_surface(id) == Some(Surface::Device) { // Floating closes the panel, as the browser's floating preview does. self.set_thread_surface(id, None); } } cx.notify(); } pub(super) fn agent_closed_device(&mut self, thread: &str, device: Option<&str>, cx: &mut Context) { if let Some(panel) = self.devices.panels.get(thread).cloned() { panel.update(cx, |panel, cx| panel.forget_device(device, cx)); } } /// Any thread, here or on a connected device, by its id here. fn session_by_id(&self, id: &str, cx: &App) -> Option> { self.projects.iter().flat_map(|project| project.sessions.iter()).find(|session| session.read(cx).id == id).cloned() } /// Where a thread's devices are, and its id there: a thread on another /// device uses that device's. fn device_target(&mut self, thread: &str, cx: &mut Context) -> (Backend, String) { let remote = self.session_by_id(thread, cx).and_then(|session| { let remote = session.read(cx).remote.as_ref()?; Some((remote.env.clone(), remote.thread.clone())) }); match remote { Some((env, host_thread)) => match self.devices_link(&env) { // A thread not created there yet gets an id of its own. Some(link) => (Backend::Remote(link), host_thread.unwrap_or_else(|| format!("pending-{thread}"))), None => (Backend::Local, thread.to_string()), }, None => (Backend::Local, thread.to_string()), } } /// A connected device's devices, made on first use. pub(super) fn devices_link(&mut self, env: &str) -> Option> { if let Some(link) = self.devices.links.get(env) { return Some(link.clone()); } let environment = self.remote.environments.iter().find(|candidate| candidate.host.id == env)?; let link = Arc::new(RemoteDevices::new( env.to_string(), environment.host.label(), environment.client.sender(), environment.route.as_ref().map_or(&environment.host.address, |route| &route.address), environment.host.key.clone(), )); link.watch(); self.devices.links.insert(env.to_string(), link.clone()); Some(link) } /// A connected device's connection moved to another route. pub(super) fn move_devices_link(&self, env: &str, address: &str) { if let Some(link) = self.devices.links.get(env) { link.set_address(address); } } /// A connected device went away: its devices with it. pub(super) fn forget_devices_link(&mut self, env: &str, cx: &mut Context) { self.devices.links.remove(env); if cx.global_mut::().remote.remove(env).is_some() { cx.notify(); } } /// A connected device's devices changed. pub(super) fn remote_device_state(&mut self, env: &str, mut state: State, cx: &mut Context) { // Its "This machine" is that device here. let name = self.environment_name(env); for host in &mut state.hosts { if host.id == LOCAL_HOST { host.label = name.clone(); } } cx.global_mut::().remote.insert(env.to_string(), state); cx.notify(); } // ---- serving this device's devices ---- /// Answers a connected device's call for this device's devices, off the UI thread. pub(super) fn answer_device_call(&mut self, conn: u64, request: u64, call: DeviceCall, cx: &mut Context) { let Some(server) = self.remote_server() else { return }; let sender = server.sender(conn); if call == DeviceCall::Watch { self.devices.watchers.insert(conn); sender.send(&ServerMessage::DeviceState { state: device::service().state() }); return; } self.devices.watchers.insert(conn); cx.background_executor() .spawn(async move { let (ok, error) = match answer_device_call(call) { Ok(value) => (Some(value), None), Err(error) => (None, Some(error)), }; sender.send(&ServerMessage::DeviceReply { request, ok, error }); }) .detach(); } /// Sends this device's devices to the clients following them. fn send_device_state(&self) { let Some(server) = self.remote_server() else { return }; if self.devices.watchers.is_empty() { return; } let message = ServerMessage::DeviceState { state: device::service().state() }; for conn in &self.devices.watchers { server.send(*conn, &message); } } pub(super) fn forget_device_watcher(&mut self, conn: u64) { self.devices.watchers.remove(&conn); } /// A thread's Device panel, made on first use. pub(super) fn device_panel(&mut self, thread: &str, window: &mut Window, cx: &mut Context) -> Entity { if let Some(panel) = self.devices.panels.get(thread) { return panel.clone(); } let (backend, target) = self.device_target(thread, cx); let panel = cx.new(|cx| DevicePanel::new(target, backend, window, cx)); let thread_id = thread.to_string(); cx.subscribe_in(&panel, window, move |this, _, event: &DevicePanelEvent, window, cx| { let active = this.active.as_ref().is_some_and(|session| session.read(cx).id == thread_id); let session = this.session_by_id(&thread_id, cx).map(|session| session.entity_id()); match event { // Floating closes the panel, as the browser's floating preview does. DevicePanelEvent::Close | DevicePanelEvent::Floated => { if let Some(session) = session.filter(|session| this.thread_surface(*session) == Some(Surface::Device)) { this.set_thread_surface(session, None); } } DevicePanelEvent::Docked => { if active { this.open_surface(Surface::Device, window, cx); } else if let Some(session) = session { this.set_thread_surface(session, Some(Surface::Device)); } } } cx.notify(); }) .detach(); self.devices.panels.insert(thread.to_string(), panel.clone()); panel } /// The open thread's Device panel while the Device surface shows. Tells /// every panel what's on screen, so only that device streams. pub(super) fn shown_device_panel(&mut self, window: &mut Window, cx: &mut Context) -> Option> { let thread = self.active.as_ref().map(|session| session.read(cx).id.clone()); let surface = self.surface_open(Surface::Device) && !self.settings_open; let shown = match (&thread, surface) { (Some(thread), true) => Some(self.device_panel(thread, window, cx)), _ => None, }; // A new thread on another device gets its id there once it's made. if let (Some(thread), Some(panel)) = (&thread, &shown) { let (_, target) = self.device_target(thread, cx); panel.update(cx, |panel, _| panel.set_thread_id(target)); } if !surface { self.devices.setup_offered = false; } else if let Some(panel) = shown.as_ref().filter(|_| !self.devices.setup_offered) { self.devices.setup_offered = true; let backend = panel.read(cx).backend().clone(); let state = backend.state(cx); if state.host_status == HostStatus::Disabled || !state.onboarding_completed { let workspace = cx.weak_entity(); cx.defer_in(window, move |_, window, cx| { DeviceSetup::open_with( backend, window, cx, Some(move |_: &mut Window, cx: &mut App| { let _ = workspace.update(cx, |this, cx| { if this.surface_open(Surface::Device) { this.set_right_panel(None); cx.notify(); } }); }), ); }); } } let panels: Vec<(String, Entity)> = self.devices.panels.iter().map(|(thread, panel)| (thread.clone(), panel.clone())).collect(); for (id, panel) in panels { let current = thread.as_deref() == Some(id.as_str()); let panel_visible = current && surface; let float_visible = current && !self.settings_open; panel.update(cx, |panel, cx| panel.set_visible(panel_visible, float_visible, window, cx)); } shown } /// Drops a deleted thread's panel and closes its devices. pub(super) fn forget_thread_devices(&mut self, thread: &str, cx: &mut Context) { if let Some(panel) = self.devices.panels.remove(thread) { let (backend, thread) = { let panel = panel.read(cx); (panel.backend().clone(), panel.thread_id().to_string()) }; cx.background_executor().spawn(async move { backend.close(&thread, None, false) }).detach(); } } // ---- the floating window ---- pub(super) fn track_chat_area(&mut self, bounds: Bounds) { self.devices.chat_area = bounds; } /// The floating window's place: where the user put it, kept inside the conversation. fn float_rect(&self) -> Bounds { let area = self.devices.chat_area; let margin = px(FLOAT_MARGIN); let rect = self.devices.float_rect.unwrap_or_else(|| { let size = Size { width: FLOAT_DEFAULT.width.min((area.size.width - margin * 2.).max(FLOAT_MIN.width)), height: FLOAT_DEFAULT.height.min((area.size.height - margin * 2.).max(FLOAT_MIN.height)), }; Bounds { origin: point(area.right() - size.width - margin, area.bottom() - size.height - margin), size } }); let width = rect.size.width.min(area.size.width).max(FLOAT_MIN.width); let height = rect.size.height.min(area.size.height).max(FLOAT_MIN.height); let x = rect.origin.x.min(area.right() - width).max(area.left()); let y = rect.origin.y.min(area.bottom() - height).max(area.top()); Bounds { origin: point(x, y), size: Size { width, height } } } pub(super) fn float_dragging(&self) -> bool { self.devices.float_drag.is_some() } pub(super) fn drag_float(&mut self, event: &MouseMoveEvent, _: &mut Window, cx: &mut Context) { let Some(drag) = &self.devices.float_drag else { return }; let delta = event.position - drag.start; let mut rect = drag.rect; let (left, right, top, bottom) = match drag.grip { Grip::Move => { rect.origin = rect.origin + delta; self.devices.float_rect = Some(rect); cx.notify(); return; } Grip::Left => (true, false, false, false), Grip::Right => (false, true, false, false), Grip::Top => (false, false, true, false), Grip::Bottom => (false, false, false, true), Grip::TopLeft => (true, false, true, false), Grip::TopRight => (false, true, true, false), Grip::BottomLeft => (true, false, false, true), Grip::BottomRight => (false, true, false, true), }; if left { let width = (rect.size.width - delta.x).max(FLOAT_MIN.width); rect.origin.x = rect.right() - width; rect.size.width = width; } if right { rect.size.width = (rect.size.width + delta.x).max(FLOAT_MIN.width); } if top { let height = (rect.size.height - delta.y).max(FLOAT_MIN.height); rect.origin.y = rect.bottom() - height; rect.size.height = height; } if bottom { rect.size.height = (rect.size.height + delta.y).max(FLOAT_MIN.height); } self.devices.float_rect = Some(rect); cx.notify(); } pub(super) fn end_float_drag(&mut self, cx: &mut Context) { if self.devices.float_drag.take().is_some() { self.devices.float_rect = Some(self.float_rect()); cx.notify(); } } fn start_float_drag(&mut self, grip: Grip, position: Point, cx: &mut Context) { self.devices.float_drag = Some(FloatDrag { grip, start: position, rect: self.float_rect() }); cx.notify(); } /// The open thread's floating device, over the chat. pub(super) fn render_floating_device(&mut self, cx: &mut Context) -> Option { if self.settings_open { return None; } let thread = self.active.as_ref()?.read(cx).id.clone(); let panel = self.devices.panels.get(&thread)?.clone(); let view = panel.read(cx).floating_view()?; let p = metal::palette(cx); let rect = self.float_rect(); let name = view.read(cx).device.name.clone(); let edge = |id: &'static str, grip: Grip| { div().id(id).absolute().on_mouse_down( MouseButton::Left, cx.listener(move |this, event: &MouseDownEvent, _, cx| { this.start_float_drag(grip, event.position, cx); cx.stop_propagation(); }), ) }; let dock_panel = panel.clone(); let close_panel = panel.clone(); let window = v_flex() .id("device-float") .absolute() .left(rect.origin.x - self.devices.chat_area.origin.x) .top(rect.origin.y - self.devices.chat_area.origin.y) .w(rect.size.width) .h(rect.size.height) .rounded(px(16.)) .raised(p) .occlude() .child( h_flex() .id("device-float-handle") .h(px(FLOAT_HEADER)) .flex_none() .pl(px(8.)) .pr(px(4.)) .gap(px(6.)) .cursor_grab() .on_mouse_down( MouseButton::Left, cx.listener(|this, event: &MouseDownEvent, _, cx| { this.start_float_drag(Grip::Move, event.position, cx); cx.stop_propagation(); }), ) .child(Icon::new(Lucide::GripHorizontal).size(px(14.)).text_color(p.ink3)) .child(div().flex_1().min_w_0().truncate().text_size(px(12.)).font_weight(FontWeight::MEDIUM).text_color(p.ink2).child(name)) .child( MetalButton::new("device-float-dock") .ghost() .mini() .icon(Lucide::PanelRightOpen) .tooltip("Open in right panel") .on_click(move |_, _, cx| { dock_panel.update(cx, |panel, cx| panel.dock(cx)); }), ) .child( MetalButton::new("device-float-close") .ghost() .mini() .icon(Lucide::X) .tooltip("Close floating device") .on_click(move |_, _, cx| { close_panel.update(cx, |panel, cx| panel.unfloat(cx)); }), ), ) .child(div().flex_1().min_h_0().mx(px(6.)).mb(px(6.)).rounded(px(11.)).overflow_hidden().well(p).child(view)) .child(edge("device-float-left", Grip::Left).left(px(-3.)).top(px(12.)).bottom(px(12.)).w(px(6.)).cursor_col_resize()) .child(edge("device-float-right", Grip::Right).right(px(-3.)).top(px(12.)).bottom(px(12.)).w(px(6.)).cursor_col_resize()) .child(edge("device-float-top", Grip::Top).top(px(-3.)).left(px(12.)).right(px(12.)).h(px(6.)).cursor_row_resize()) .child(edge("device-float-bottom", Grip::Bottom).bottom(px(-3.)).left(px(12.)).right(px(12.)).h(px(6.)).cursor_row_resize()) .child(edge("device-float-tl", Grip::TopLeft).top(px(-3.)).left(px(-3.)).size(px(12.)).cursor_nwse_resize()) .child(edge("device-float-tr", Grip::TopRight).top(px(-3.)).right(px(-3.)).size(px(12.)).cursor_nesw_resize()) .child(edge("device-float-bl", Grip::BottomLeft).bottom(px(-3.)).left(px(-3.)).size(px(12.)).cursor_nesw_resize()) .child(edge("device-float-br", Grip::BottomRight).bottom(px(-3.)).right(px(-3.)).size(px(12.)).cursor_nwse_resize()); Some(window.into_any_element()) } // ---- agents' device tools ---- /// Answers a `device_*` call. The work runs off the UI thread. pub(super) fn answer_device_tool(&mut self, thread: &str, tool: &'static str, args: Value, answer: Answer, cx: &mut Context) { let thread = thread.to_string(); cx.spawn(async move |_, cx| { let outcome = cx.background_executor().spawn(async move { device_tool(&thread, tool, &args) }).await; let _ = cx.update(|cx| answer.send(outcome, cx)); }) .detach(); } } /// Runs one device tool call for a thread's agent. fn device_tool(thread: &str, tool: &str, args: &Value) -> Result { let service = device::service(); if !settings::get(|settings| settings.enable_device_support && settings.enable_agent_device_access) { return Err("Agent device access is turned off for this environment.".into()); } let text = |key: &str| args.get(key).and_then(Value::as_str).map(str::trim).filter(|value| !value.is_empty()).map(str::to_string); let host = text("hostId").unwrap_or_else(|| LOCAL_HOST.into()); if host != LOCAL_HOST { return Err(format!("Device host {host} is unavailable: Unknown host.")); } let disabled = "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools."; match tool { DEVICE_LIST => { let state = service.list(); if state.host_status == HostStatus::Disabled { return Err(disabled.into()); } let open: Vec = service.sessions_for(thread).iter().map(|session| json!({"hostId": session.host_id, "deviceId": session.device_id})).collect(); Ok(json!({ "hostStatuses": {LOCAL_HOST: {"status": state.host_status, "detail": state.host_detail}}, "hosts": state.hosts, "devices": state.devices, "open": open, })) } DEVICE_OPEN => { let state = service.list(); if state.host_status == HostStatus::Disabled { return Err(disabled.into()); } let platform = text("platform").and_then(|platform| Platform::from_key(&platform)); let target = pick_device(&state.devices, text("deviceId").as_deref(), platform)?; // Agent access and connectivity come before booting or opening anything. let agent_args = service.agent_target(thread, &target.id).map_err(|err| format!("{err:#}"))?; let session = service.open(thread, &target.id, true).map_err(|err| format!("{err:#}"))?; let device = service.state().device(&session.host_id, &session.device_id).cloned().unwrap_or(target); let mut target_args = device::agent::target_args(&device); target_args.extend(agent_args); let command = service.agent_cli().map_err(|err| format!("Could not prepare the agent-device launcher: {err:#}"))?; let command = command.display().to_string(); let quick_start = device::agent::quick_start(&device, &target_args, &command); service.notify_agent_opened(thread, device.clone()); Ok(json!({ "device": device, "agentDevice": {"command": command, "targetArgs": target_args}, "quickStart": quick_start, })) } DEVICE_SCREENSHOT => { let wanted = text("deviceId"); // Only devices this thread opened; another thread's isn't this agent's to watch. let session = service .sessions_for(thread) .into_iter() .rev() .find(|session| wanted.as_deref().is_none_or(|id| session.device_id == id)) .ok_or_else(|| match &wanted { None => "No device is open in this thread. Call device_open first.".to_string(), Some(id) => format!("Device {id} on host {host} is not open in this thread. Call device_open first."), })?; let (device, png) = service.screenshot(&session.device_id).map_err(|err| format!("{err:#}"))?; let (width, height) = device::png_size(&png); let saved = std::env::temp_dir().join(format!("vectrum-device-{}.png", chrono::Local::now().timestamp_millis())); let saved = std::fs::write(&saved, &png).ok().map(|_| saved.display().to_string()); Ok(json!({ "device": device, "screenshot": { "mimeType": "image/png", "data": base64::engine::general_purpose::STANDARD.encode(&png), "width": width, "height": height, }, "savedTo": saved, })) } DEVICE_CLOSE => { let device = text("deviceId"); let shutdown = args.get("shutdown").and_then(Value::as_bool).unwrap_or(false); service.close(thread, device.as_deref(), shutdown).map_err(|err| format!("{err:#}"))?; service.notify_agent_closed(thread, device); Ok(json!({})) } _ => Err(format!("Vectrum Code has no tool named {tool}.")), } } /// The device a `device_open` means: the one named, else the booted (or /// first) device of the platform. fn pick_device(devices: &[DeviceSummary], id: Option<&str>, platform: Option) -> Result { if let Some(id) = id { return devices .iter() .find(|device| device.id == id) .cloned() .ok_or_else(|| format!("No device {id} on host {LOCAL_HOST}. Call device_list for current ids.")); } let candidates: Vec<&DeviceSummary> = devices.iter().filter(|device| platform.is_none_or(|platform| device.platform == platform)).collect(); if candidates.is_empty() { return Err(match platform { None => "No simulators or emulators were found. Call device_list to see why.".into(), Some(platform) => format!("No {} devices were found on host {LOCAL_HOST}. Call device_list to see why.", platform.key()), }); } let first = candidates[0].platform; if platform.is_none() && candidates.iter().any(|device| device.platform != first) { return Err("Both iOS and Android devices are available; pass platform or deviceId.".into()); } Ok(candidates.iter().find(|device| device.booted).copied().unwrap_or(candidates[0]).clone()) } #[cfg(test)] mod tests { use super::pick_device; use crate::device::{DeviceSummary, Platform}; fn device(id: &str, platform: Platform, booted: bool) -> DeviceSummary { DeviceSummary { host_id: "local".into(), id: id.into(), platform, name: id.into(), version: String::new(), booted, physical: false, } } #[::core::prelude::v1::test] fn picks_the_device_an_agent_means() { let devices = vec![device("pixel", Platform::Android, false), device("emulator-5554", Platform::Android, true)]; assert_eq!(pick_device(&devices, None, None).unwrap().id, "emulator-5554"); assert_eq!(pick_device(&devices, Some("pixel"), None).unwrap().id, "pixel"); assert!(pick_device(&devices, Some("nope"), None).is_err()); assert!(pick_device(&devices, None, Some(Platform::Ios)).unwrap_err().contains("No ios devices")); let mixed = vec![device("a", Platform::Android, false), device("b", Platform::Ios, false)]; assert!(pick_device(&mixed, None, None).unwrap_err().contains("pass platform")); assert_eq!(pick_device(&mixed, None, Some(Platform::Ios)).unwrap().id, "b"); } }