//! The live screen of one device, from the hub's per-device streams. The hub //! vendors two servers with different wire formats, as T3 Code's viewer //! knows them: //! //! - Android (serve-emu): one WebSocket at `ws?device=&frame-meta=1` //! carries H.264 access units behind a 16-byte "SEMU" header (magic, //! version, key flag, pts) and takes JSON gestures back. The app decodes //! the video with OpenH264. //! - iOS (serve-sim): the MJPEG endpoint for video, and input over //! `helper/ws?device=` as `[tag][json]` packets. //! //! A stream only runs while its panel shows; a hidden one costs nothing. use std::{ io::{ErrorKind, Read as _}, net::{SocketAddr, TcpStream}, sync::{ Arc, Mutex, atomic::{AtomicBool, Ordering}, mpsc, }, thread, time::{Duration, Instant}, }; use anyhow::{Result, anyhow}; use gpui_kit::RenderImage; use openh264::formats::YUVSource as _; use serde::Deserialize; use serde_json::json; use tungstenite::{Message, WebSocket}; use super::{Orientation, Platform}; const RETRY_DELAY: Duration = Duration::from_secs(1); const FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(15); /// How long a socket read waits before the stream thread sends queued input. const POLL: Duration = Duration::from_millis(8); const SEMU_MAGIC: u32 = 0x5345_4d55; const SEMU_HEADER: usize = 16; // serve-sim's input packet tags, app to helper. const IOS_TOUCH: u8 = 0x03; const IOS_BUTTON: u8 = 0x04; const IOS_KEY: u8 = 0x06; const IOS_ORIENTATION: u8 = 0x07; // And helper to app. const IOS_SCREEN_CONFIG: u8 = 0x82; const IOS_INPUT_ADMITTED: u8 = 0x83; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Status { Connecting, Streaming, Error, } /// The device's screen as the stream reports it. #[derive(Clone, Copy, Debug, PartialEq)] pub struct Screen { pub width: f32, pub height: f32, pub orientation: Orientation, } impl Screen { /// Width over height as the user sees the screen. pub fn aspect(&self) -> f32 { let (long, short) = (self.width.max(self.height), self.width.min(self.height).max(1.)); if self.orientation.is_landscape() { long / short } else { short / long } } } /// What a stream reports to its panel. #[derive(Debug)] pub enum StreamEvent { Status(Status, Option), /// A new frame waits in `DeviceStream::take_frame`. Frame, Screen(Screen), /// Whether touches and keys reach the device. Input(bool, Option), } pub struct Frame { pub image: Arc, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TouchPhase { Begin, Move, End, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Button { Home, Back, Recents, } /// A key the user pressed while the screen had focus. #[derive(Clone, Debug)] pub struct Key { /// GPUI's key name: `a`, `enter`, `backspace`, `up`. pub key: String, /// The text it types, if any. pub text: Option, pub shift: bool, /// Control, Alt or the platform key was held: no text. pub command: bool, } #[derive(Debug)] enum Input { /// Normalized 0..1 coordinates in the displayed frame. Touch(TouchPhase, f32, f32), Key(Key), Text(String), Button(Button), Rotate, } struct Shared { stop: AtomicBool, watching: AtomicBool, frame: Mutex>, screen: Mutex>, /// Bumped by `restart`, so the thread drops its connection and reconnects. generation: std::sync::atomic::AtomicU64, } /// Where a device hub answers: this machine's on loopback, or another /// device's through its remote server, which passes allowlisted stream /// routes on to its hub for anyone with its key. #[derive(Clone, Debug, PartialEq)] pub struct HubTarget { /// `host:port`. pub address: String, /// Put before every route: `/device-hub` through a remote server. pub prefix: String, /// The remote server's key, sent as a bearer token. pub key: Option, } impl HubTarget { /// This machine's hub at its loopback origin, `http://127.0.0.1:`. pub fn local(origin: &str) -> Self { Self { address: origin.trim_start_matches("http://").to_string(), prefix: String::new(), key: None } } fn connect(&self, timeout: Duration) -> Result { use std::net::ToSocketAddrs as _; let address: SocketAddr = self.address.to_socket_addrs()?.next().ok_or_else(|| anyhow!("{} can't be reached", self.address))?; let tcp = TcpStream::connect_timeout(&address, timeout)?; tcp.set_nodelay(true)?; Ok(tcp) } /// The request head of a GET, with the key when there is one. fn get_head(&self, path: &str, extra: &str) -> String { let auth = self.key.as_ref().map(|key| format!("Authorization: Bearer {key}\r\n")).unwrap_or_default(); format!("GET {}{path} HTTP/1.1\r\nHost: {}\r\n{auth}{extra}Connection: close\r\n\r\n", self.prefix, self.address) } } /// One device's live screen, as its panel watches and drives it. pub struct DeviceStream { shared: Arc, input: mpsc::Sender, } impl DeviceStream { /// Starts the stream's thread. Nothing connects until `set_watching(true)`. pub fn new(hub: HubTarget, platform: Platform, device: String, events: smol::channel::Sender) -> Self { let shared = Arc::new(Shared { stop: AtomicBool::new(false), watching: AtomicBool::new(false), frame: Mutex::new(None), screen: Mutex::new(None), generation: Default::default(), }); let (input, input_rx) = mpsc::channel(); let thread_shared = shared.clone(); let _ = thread::Builder::new().name("device-stream".into()).spawn(move || { let stream = Streamer { hub, device, shared: thread_shared, events, input: input_rx }; match platform { Platform::Android => stream.run_android(), Platform::Ios => stream.run_ios(), } }); Self { shared, input } } pub fn set_watching(&self, watching: bool) { self.shared.watching.store(watching, Ordering::Relaxed); } /// Drops the connection and connects again: the Reconnect button. pub fn restart(&self) { self.shared.generation.fetch_add(1, Ordering::Relaxed); } /// The newest frame not taken yet. pub fn take_frame(&self) -> Option { self.shared.frame.lock().ok()?.take() } pub fn touch(&self, phase: TouchPhase, x: f32, y: f32) { let _ = self.input.send(Input::Touch(phase, x.clamp(0., 1.), y.clamp(0., 1.))); } pub fn key(&self, key: Key) { let _ = self.input.send(Input::Key(key)); } /// Types text, as a paste does. pub fn text(&self, text: String) { let _ = self.input.send(Input::Text(text)); } pub fn button(&self, button: Button) { let _ = self.input.send(Input::Button(button)); } /// Turns an iOS Simulator to its next orientation. pub fn rotate(&self) { let _ = self.input.send(Input::Rotate); } } impl Drop for DeviceStream { fn drop(&mut self) { self.shared.stop.store(true, Ordering::Relaxed); } } struct Streamer { hub: HubTarget, device: String, shared: Arc, events: smol::channel::Sender, input: mpsc::Receiver, } /// Why a connection ended. enum End { /// Stopped, hidden or restarted: nothing to report. Quiet, /// It dropped; reconnect after a moment. Retry(Option), /// It can't work; say so and wait for Reconnect. Fail(String), } impl Streamer { fn stopped(&self) -> bool { self.shared.stop.load(Ordering::Relaxed) || self.events.is_closed() } fn wanted(&self, generation: u64) -> bool { !self.stopped() && self.shared.watching.load(Ordering::Relaxed) && self.shared.generation.load(Ordering::Relaxed) == generation } fn emit(&self, event: StreamEvent) { let _ = self.events.send_blocking(event); } /// Keeps a connection up while the panel shows: `connect` runs one /// connection and says how it ended. fn supervise(&self, mut connect: impl FnMut(&Self, u64) -> End) { let mut failed = false; let mut failed_generation = 0; while !self.stopped() { let generation = self.shared.generation.load(Ordering::Relaxed); if !self.shared.watching.load(Ordering::Relaxed) || (failed && generation == failed_generation) { while self.input.try_recv().is_ok() {} thread::sleep(Duration::from_millis(100)); continue; } failed = false; self.emit(StreamEvent::Status(Status::Connecting, None)); match connect(self, generation) { End::Quiet => {} End::Retry(detail) => { self.emit(StreamEvent::Input(false, detail.clone())); self.emit(StreamEvent::Status(Status::Connecting, detail)); thread::sleep(RETRY_DELAY); } End::Fail(detail) => { self.emit(StreamEvent::Input(false, Some(detail.clone()))); self.emit(StreamEvent::Status(Status::Error, Some(detail))); failed = true; failed_generation = generation; } } } } fn socket(&self, path: &str) -> Result> { use tungstenite::client::IntoClientRequest as _; let tcp = self.hub.connect(Duration::from_secs(4))?; tcp.set_read_timeout(Some(Duration::from_secs(8)))?; let mut request = format!("ws://{}{}{path}", self.hub.address, self.hub.prefix).into_client_request()?; if let Some(key) = &self.hub.key { request.headers_mut().insert("Authorization", format!("Bearer {key}").parse()?); } let (socket, _) = tungstenite::client(request, tcp).map_err(|err| match err { tungstenite::HandshakeError::Failure(tungstenite::Error::Http(response)) if response.status() == 401 => { anyhow!("The other device refused the key.") } tungstenite::HandshakeError::Failure(tungstenite::Error::Http(response)) if response.status() == 503 => { anyhow!("Device support isn't running on the other device.") } err => anyhow!("stream handshake failed: {err}"), })?; socket.get_ref().set_read_timeout(Some(POLL))?; Ok(socket) } /// Hands a frame to the panel; one it hasn't drawn yet is replaced. fn present(&self, frame: Frame) { let Ok(mut slot) = self.shared.frame.lock() else { return }; if slot.replace(frame).is_none() { drop(slot); self.emit(StreamEvent::Frame); } } fn set_screen(&self, screen: Screen) { let Ok(mut current) = self.shared.screen.lock() else { return }; if *current != Some(screen) { *current = Some(screen); drop(current); self.emit(StreamEvent::Screen(screen)); } } fn screen(&self) -> Option { *self.shared.screen.lock().ok()? } // ---- Android ---- fn run_android(self) { let device = urlencode(&self.device); self.supervise(|this, generation| this.android_connection(&device, generation)); } fn android_connection(&self, device: &str, generation: u64) -> End { let mut socket = match self.socket(&format!("/vendor/serve-emu/ws?device={device}&frame-meta=1")) { Ok(socket) => socket, Err(err) => return End::Retry(Some(err.to_string())), }; self.emit(StreamEvent::Input(true, None)); let mut decoder = match openh264::decoder::Decoder::new() { Ok(decoder) => decoder, Err(err) => return End::Fail(format!("Video decoder: {err}")), }; let mut awaiting_key = true; let mut first_frame = false; let mut last_keyframe_request: Option = None; let started = Instant::now(); let mut request_keyframe = |socket: &mut WebSocket| { if last_keyframe_request.is_none_or(|at| at.elapsed() > Duration::from_millis(500)) { last_keyframe_request = Some(Instant::now()); let _ = socket.send(Message::text(json!({"type": "reset-video", "ack": false}).to_string())); } }; request_keyframe(&mut socket); let end = loop { if !self.wanted(generation) { break End::Quiet; } if !first_frame && started.elapsed() > FIRST_FRAME_TIMEOUT { break End::Fail("No video received from the device. Reconnect to try again.".into()); } while let Ok(input) = self.input.try_recv() { for message in android_messages(input) { if socket.send(Message::text(message)).is_err() { return End::Retry(None); } } } let data = match socket.read() { Ok(Message::Binary(data)) => data, Ok(Message::Text(text)) => { // A new encoder session: its first frame must be a keyframe. if serde_json::from_str::(text.as_str()) .is_ok_and(|message| message["type"] == "video-session") { if let Ok(fresh) = openh264::decoder::Decoder::new() { decoder = fresh; } awaiting_key = true; request_keyframe(&mut socket); } continue; } Ok(Message::Close(frame)) => break End::Retry(frame.map(|frame| frame.reason.to_string()).filter(|r| !r.is_empty())), Ok(_) => continue, Err(tungstenite::Error::Io(err)) if matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => continue, Err(err) => break End::Retry(Some(err.to_string())), }; let (payload, key) = parse_semu(&data); let key = key.unwrap_or_else(|| has_idr(payload)); if awaiting_key { if !key { request_keyframe(&mut socket); continue; } awaiting_key = false; } match decoder.decode(payload) { Ok(Some(yuv)) => { let (width, height) = yuv.dimensions(); let mut pixels = vec![0u8; width * height * 4]; yuv.write_rgba8(&mut pixels); let Some(frame) = bgra_frame(pixels, width as u32, height as u32) else { continue }; self.set_screen(Screen { width: width as f32, height: height as f32, orientation: if width > height { Orientation::LandscapeLeft } else { Orientation::Portrait }, }); self.present(frame); if !first_frame { first_frame = true; self.emit(StreamEvent::Status(Status::Streaming, None)); } } Ok(None) => {} Err(_) => { // Start over from the next keyframe. if let Ok(fresh) = openh264::decoder::Decoder::new() { decoder = fresh; } awaiting_key = true; request_keyframe(&mut socket); } } }; let _ = socket.close(None); let _ = socket.flush(); end } // ---- iOS ---- fn run_ios(self) { let device = urlencode(&self.device); self.supervise(|this, generation| this.ios_connection(&device, generation)); } /// The MJPEG stream on a thread of its own, input on this one. fn ios_connection(&self, device: &str, generation: u64) -> End { let video_done = Arc::new(AtomicBool::new(false)); let (video_tx, video_rx) = mpsc::channel::(); { let (hub, device, done) = (self.hub.clone(), device.to_string(), video_done.clone()); thread::spawn(move || read_mjpeg(&hub, &device, &done, &video_tx)); } let mut socket: Option> = None; let mut admitted = false; let mut input_unavailable = false; let mut next_input_attempt = Instant::now(); let mut first_frame = false; let started = Instant::now(); let end = loop { if !self.wanted(generation) { break End::Quiet; } if !first_frame && started.elapsed() > FIRST_FRAME_TIMEOUT { break End::Fail("No video received from the device. Reconnect to try again.".into()); } // Frames. let mut drained = false; while let Ok(message) = video_rx.try_recv() { drained = true; match message { VideoMessage::Jpeg(jpeg) => { let screen = self.screen(); if let Some(frame) = decode_jpeg(&jpeg, screen) { self.present(frame); if !first_frame { first_frame = true; self.emit(StreamEvent::Status(Status::Streaming, None)); } } } VideoMessage::Ended(detail) => { video_done.store(true, Ordering::Relaxed); if let Some(socket) = socket.as_mut() { let _ = socket.close(None); } return End::Retry(detail); } } } // Input: the helper takes it once the video runs. if socket.is_none() && Instant::now() >= next_input_attempt { match self.socket(&format!("/vendor/serve-sim/helper/ws?device={device}")) { Ok(connected) => { socket = Some(connected); admitted = false; } Err(err) => { self.emit(StreamEvent::Input(false, Some(err.to_string()))); next_input_attempt = Instant::now() + RETRY_DELAY; } } } let Some(ws) = socket.as_mut() else { if !drained { thread::sleep(POLL); } continue; }; while let Ok(input) = self.input.try_recv() { if !admitted || input_unavailable { continue; } for packet in ios_packets(input, self.screen()) { let _ = ws.send(Message::binary(packet)); } } match ws.read() { Ok(Message::Binary(data)) if !data.is_empty() => match data[0] { IOS_INPUT_ADMITTED => { admitted = true; self.emit(StreamEvent::Input(!input_unavailable, None)); } IOS_SCREEN_CONFIG => { if let Ok(config) = serde_json::from_slice::(&data[1..]) { input_unavailable = config.input_unavailable.unwrap_or(false); self.emit(StreamEvent::Input( admitted && !input_unavailable, input_unavailable.then(|| "Simulator input is unavailable".to_string()), )); if let Some(orientation) = Orientation::from_key(&config.orientation) { self.set_screen(Screen { width: config.width, height: config.height, orientation }); } } } _ => {} }, Ok(Message::Close(_)) | Err(tungstenite::Error::ConnectionClosed | tungstenite::Error::AlreadyClosed) => { socket = None; admitted = false; self.emit(StreamEvent::Input(false, Some("input socket closed".into()))); next_input_attempt = Instant::now() + RETRY_DELAY; } Ok(_) => {} Err(tungstenite::Error::Io(err)) if matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => {} Err(err) => { socket = None; admitted = false; self.emit(StreamEvent::Input(false, Some(err.to_string()))); next_input_attempt = Instant::now() + RETRY_DELAY; } } }; video_done.store(true, Ordering::Relaxed); if let Some(mut socket) = socket { let _ = socket.close(None); let _ = socket.flush(); } end } } #[derive(Deserialize)] struct IosScreenConfig { width: f32, height: f32, orientation: String, #[serde(rename = "inputUnavailable")] input_unavailable: Option, } enum VideoMessage { Jpeg(Vec), Ended(Option), } /// Reads serve-sim's MJPEG body and passes each JPEG on. fn read_mjpeg(hub: &HubTarget, device: &str, done: &AtomicBool, frames: &mpsc::Sender) { let result = (|| -> Result<()> { let mut tcp = hub.connect(Duration::from_secs(4))?; tcp.set_read_timeout(Some(Duration::from_millis(500)))?; let request = hub.get_head(&format!("/vendor/serve-sim/helper/{device}/stream.mjpeg"), "Accept: multipart/x-mixed-replace\r\n"); std::io::Write::write_all(&mut tcp, request.as_bytes())?; let mut buffer: Vec = Vec::with_capacity(512 * 1024); let mut chunk = vec![0u8; 64 * 1024]; let mut headers_done = false; let mut idle = Instant::now(); while !done.load(Ordering::Relaxed) { match tcp.read(&mut chunk) { Ok(0) => return Err(anyhow!("video stream ended")), Ok(n) => { idle = Instant::now(); buffer.extend_from_slice(&chunk[..n]); } Err(err) if matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => { if idle.elapsed() > FIRST_FRAME_TIMEOUT { return Err(anyhow!("Device stream stopped receiving video. Reconnecting…")); } continue; } Err(err) => return Err(err.into()), } if !headers_done { let Some(end) = find(&buffer, b"\r\n\r\n") else { continue }; let head = String::from_utf8_lossy(&buffer[..end]).to_string(); let status = head.split_whitespace().nth(1).unwrap_or_default().to_string(); if status != "200" { return Err(anyhow!("stream {status}")); } buffer.drain(..end + 4); headers_done = true; } // Each part is one JPEG: cut from its start marker to its end marker. while let Some(start) = find(&buffer, &[0xff, 0xd8, 0xff]) { let Some(end) = find(&buffer[start + 2..], &[0xff, 0xd9]).map(|end| start + 2 + end + 2) else { buffer.drain(..start); break; }; let jpeg = buffer[start..end].to_vec(); buffer.drain(..end); if frames.send(VideoMessage::Jpeg(jpeg)).is_err() { return Ok(()); } } if buffer.len() > 32 << 20 { buffer.clear(); } } Ok(()) })(); if let Err(err) = result && !done.load(Ordering::Relaxed) { let _ = frames.send(VideoMessage::Ended(Some(err.to_string()))); } } fn find(haystack: &[u8], needle: &[u8]) -> Option { haystack.windows(needle.len()).position(|window| window == needle) } /// serve-sim streams the raw framebuffer: a device that reports landscape /// while its frames stay portrait is turned here, so the frame shows as the /// user holds the device. fn decode_jpeg(jpeg: &[u8], screen: Option) -> Option { let mut image = image::load_from_memory_with_format(jpeg, image::ImageFormat::Jpeg).ok()?; if let Some(screen) = screen.filter(|screen| screen.width <= screen.height) { image = match screen.orientation { Orientation::LandscapeLeft => image.rotate90(), Orientation::LandscapeRight => image.rotate270(), Orientation::PortraitUpsideDown => image.rotate180(), Orientation::Portrait => image, }; } let rgba = image.into_rgba8(); let (width, height) = rgba.dimensions(); bgra_frame(rgba.into_raw(), width, height) } /// RGBA pixels as the BGRA image GPUI draws. fn bgra_frame(mut pixels: Vec, width: u32, height: u32) -> Option { for pixel in pixels.chunks_exact_mut(4) { pixel.swap(0, 2); } let buffer = image::RgbaImage::from_raw(width, height, pixels)?; Some(Frame { image: Arc::new(RenderImage::new(vec![image::Frame::new(buffer)])) }) } /// The Annex-B payload of a serve-emu message, and its key flag when the /// SEMU header carries one. pub fn parse_semu(data: &[u8]) -> (&[u8], Option) { if data.len() > SEMU_HEADER && u32::from_be_bytes([data[0], data[1], data[2], data[3]]) == SEMU_MAGIC && data[4] == 1 { (&data[SEMU_HEADER..], Some(data[5] & 1 != 0)) } else { (data, None) } } /// Whether an Annex-B access unit holds an IDR slice. pub fn has_idr(data: &[u8]) -> bool { let mut i = 0; while i + 3 < data.len() { if data[i] == 0 && data[i + 1] == 0 { let code = if data[i + 2] == 1 { 3 } else if data[i + 2] == 0 && data[i + 3] == 1 { 4 } else { 0 }; if code > 0 { if data.get(i + code).is_some_and(|nal| nal & 0x1f == 5) { return true; } i += code + 1; continue; } } i += 1; } false } /// Android key codes for keys that type no text. fn android_keycode(key: &str) -> Option { Some(match key { "up" => 19, "down" => 20, "left" => 21, "right" => 22, "tab" => 61, "enter" => 66, "backspace" => 67, "delete" => 112, "home" => 122, "end" => 123, "pageup" => 92, "pagedown" => 93, _ => return None, }) } /// serve-emu's JSON messages for one input. fn android_messages(input: Input) -> Vec { match input { Input::Touch(phase, x, y) => { let action = match phase { TouchPhase::Begin => "down", TouchPhase::Move => "move", TouchPhase::End => "up", }; vec![json!({"type": "touch", "action": action, "x": x, "y": y}).to_string()] } Input::Key(key) => { if key.key == "escape" { return vec![json!({"type": "back"}).to_string()]; } if let Some(keycode) = android_keycode(&key.key) { return vec![json!({"type": "key", "keycode": keycode}).to_string()]; } match key.text.filter(|text| !text.is_empty() && !key.command) { Some(text) => vec![json!({"type": "text", "text": text}).to_string()], None => Vec::new(), } } Input::Text(text) => vec![json!({"type": "text", "text": text}).to_string()], Input::Button(button) => { let kind = match button { Button::Home => "home", Button::Back => "back", Button::Recents => "recents", }; vec![json!({"type": kind}).to_string()] } Input::Rotate => Vec::new(), } } fn tagged(tag: u8, payload: serde_json::Value) -> Vec { let mut packet = vec![tag]; packet.extend_from_slice(payload.to_string().as_bytes()); packet } /// A displayed point as a framebuffer point: frames stay portrait while the /// device reports landscape. fn raw_point(x: f32, y: f32, screen: Option) -> (f32, f32) { match screen.filter(|screen| screen.width <= screen.height).map(|screen| screen.orientation) { Some(Orientation::LandscapeLeft) => (y, 1. - x), Some(Orientation::LandscapeRight) => (1. - y, x), Some(Orientation::PortraitUpsideDown) => (1. - x, 1. - y), _ => (x, y), } } /// HID usages for GPUI key names, and whether Shift goes with them. fn hid_usage(key: &str) -> Option<(u8, bool)> { let named = match key { "enter" => 0x28, "escape" => 0x29, "backspace" => 0x2a, "tab" => 0x2b, "space" => 0x2c, "delete" => 0x4c, "right" => 0x4f, "left" => 0x50, "down" => 0x51, "up" => 0x52, _ => 0, }; if named != 0 { return Some((named, false)); } let mut chars = key.chars(); let (Some(c), None) = (chars.next(), chars.next()) else { return None }; hid_char(c) } /// The HID usage that types a character on a US layout. fn hid_char(c: char) -> Option<(u8, bool)> { Some(match c { 'a'..='z' => (0x04 + (c as u8 - b'a'), false), 'A'..='Z' => (0x04 + (c as u8 - b'A'), true), '1'..='9' => (0x1e + (c as u8 - b'1'), false), '0' => (0x27, false), ' ' => (0x2c, false), '\n' => (0x28, false), '\t' => (0x2b, false), _ => { const PLAIN: &str = "-=[]\\;'`,./"; const SHIFTED: &str = "_+{}|:\"~<>?"; const USAGES: [u8; 11] = [0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38]; const DIGITS: &str = "!@#$%^&*()"; if let Some(ix) = PLAIN.find(c) { (USAGES[ix], false) } else if let Some(ix) = SHIFTED.find(c) { (USAGES[ix], true) } else if let Some(ix) = DIGITS.find(c) { (if ix == 9 { 0x27 } else { 0x1e + ix as u8 }, true) } else { return None; } } }) } const HID_SHIFT: u8 = 0xe1; fn hid_press(usage: u8, shift: bool) -> Vec> { let key = |kind: &str, usage: u8| tagged(IOS_KEY, json!({"type": kind, "usage": usage})); let mut packets = Vec::new(); if shift { packets.push(key("down", HID_SHIFT)); } packets.push(key("down", usage)); packets.push(key("up", usage)); if shift { packets.push(key("up", HID_SHIFT)); } packets } /// serve-sim's packets for one input. fn ios_packets(input: Input, screen: Option) -> Vec> { match input { Input::Touch(phase, x, y) => { let kind = match phase { TouchPhase::Begin => "begin", TouchPhase::Move => "move", TouchPhase::End => "end", }; let (x, y) = raw_point(x, y, screen); vec![tagged(IOS_TOUCH, json!({"type": kind, "x": x, "y": y}))] } Input::Key(key) => { let typed = key.text.as_deref().and_then(|text| { let mut chars = text.chars(); match (chars.next(), chars.next()) { (Some(c), None) if !key.command => hid_char(c), _ => None, } }); match typed.or_else(|| hid_usage(&key.key).map(|(usage, shift)| (usage, shift || key.shift))) { Some((usage, shift)) => hid_press(usage, shift), None => Vec::new(), } } Input::Text(text) => text.chars().filter_map(hid_char).flat_map(|(usage, shift)| hid_press(usage, shift)).collect(), Input::Button(button) => { let name = match button { Button::Home => "home", Button::Recents => "app_switcher", Button::Back => return Vec::new(), }; vec![tagged(IOS_BUTTON, json!({"button": name}))] } Input::Rotate => { let current = screen.map(|screen| screen.orientation).unwrap_or(Orientation::Portrait); vec![tagged(IOS_ORIENTATION, json!({"orientation": current.next().key()}))] } } } /// Percent-encodes a device id for a query or path. pub fn urlencode(text: &str) -> String { text.bytes() .map(|byte| match byte { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (byte as char).to_string(), _ => format!("%{byte:02X}"), }) .collect() } #[cfg(test)] mod tests { use super::*; #[::core::prelude::v1::test] fn parses_semu_packets() { let mut packet = vec![0x53, 0x45, 0x4d, 0x55, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7]; packet.extend_from_slice(&[0, 0, 0, 1, 0x65, 0xaa]); let (payload, key) = parse_semu(&packet); assert_eq!(key, Some(true)); assert_eq!(payload, &[0, 0, 0, 1, 0x65, 0xaa]); assert!(has_idr(payload)); let (payload, key) = parse_semu(&[0, 0, 1, 0x41, 0x9a]); assert_eq!(key, None); assert!(!has_idr(payload)); } #[::core::prelude::v1::test] fn maps_points_for_rotated_frames() { let landscape = Some(Screen { width: 1170., height: 2532., orientation: Orientation::LandscapeLeft }); assert_eq!(raw_point(0.25, 0.75, landscape), (0.75, 0.75)); assert_eq!(raw_point(0.25, 0.75, None), (0.25, 0.75)); } #[::core::prelude::v1::test] fn turns_keys_into_device_input() { let key = |key: &str, text: Option<&str>| Key { key: key.into(), text: text.map(str::to_string), shift: false, command: false }; assert_eq!(android_messages(Input::Key(key("escape", None))), [r#"{"type":"back"}"#]); assert_eq!(android_messages(Input::Key(key("enter", None))), [r#"{"type":"key","keycode":66}"#]); assert_eq!(android_messages(Input::Key(key("a", Some("A")))), [r#"{"type":"text","text":"A"}"#]); assert_eq!(hid_char('A'), Some((0x04, true))); assert_eq!(hid_char('?'), Some((0x38, true))); assert_eq!(hid_char(')'), Some((0x27, true))); assert_eq!(ios_packets(Input::Key(key("b", Some("b"))), None).len(), 2); } /// Decodes a stream dumped from serve-emu, when one is at hand. #[::core::prelude::v1::test] fn decodes_a_dumped_android_stream() { let Ok(data) = std::fs::read("/tmp/devhub-test/stream.h264") else { return }; let mut decoder = openh264::decoder::Decoder::new().unwrap(); let mut frames = 0; for packet in openh264::nal_units(&data) { if let Ok(Some(yuv)) = decoder.decode(packet) { assert_eq!(yuv.dimensions(), (576, 1280)); frames += 1; } } assert!(frames > 0); } }