//! The live screen of one device, from the hub's per-device streams. The hub
//! vendors two servers with different wire formats, as T3 Code's viewer
//! knows them:
//!
//! - Android (serve-emu): one WebSocket at `ws?device=<serial>&frame-meta=1`
//! carries H.264 access units behind a 16-byte "SEMU" header (magic,
//! version, key flag, pts) and takes JSON gestures back. The app decodes
//! the video with OpenH264.
//! - iOS (serve-sim): the MJPEG endpoint for video, and input over
//! `helper/ws?device=<udid>` as `[tag][json]` packets.
//!
//! A stream only runs while its panel shows; a hidden one costs nothing.
use std::{
io::{ErrorKind, Read as _},
net::{SocketAddr, TcpStream},
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
mpsc,
},
thread,
time::{Duration, Instant},
};
use anyhow::{Result, anyhow};
use gpui_kit::RenderImage;
use openh264::formats::YUVSource as _;
use serde::Deserialize;
use serde_json::json;
use tungstenite::{Message, WebSocket};
use super::{Orientation, Platform};
const RETRY_DELAY: Duration = Duration::from_secs(1);
const FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(15);
/// How long a socket read waits before the stream thread sends queued input.
const POLL: Duration = Duration::from_millis(8);
const SEMU_MAGIC: u32 = 0x5345_4d55;
const SEMU_HEADER: usize = 16;
// serve-sim's input packet tags, app to helper.
const IOS_TOUCH: u8 = 0x03;
const IOS_BUTTON: u8 = 0x04;
const IOS_KEY: u8 = 0x06;
const IOS_ORIENTATION: u8 = 0x07;
// And helper to app.
const IOS_SCREEN_CONFIG: u8 = 0x82;
const IOS_INPUT_ADMITTED: u8 = 0x83;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Status {
Connecting,
Streaming,
Error,
}
/// The device's screen as the stream reports it.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Screen {
pub width: f32,
pub height: f32,
pub orientation: Orientation,
}
impl Screen {
/// Width over height as the user sees the screen.
pub fn aspect(&self) -> f32 {
let (long, short) = (self.width.max(self.height), self.width.min(self.height).max(1.));
if self.orientation.is_landscape() { long / short } else { short / long }
}
}
/// What a stream reports to its panel.
#[derive(Debug)]
pub enum StreamEvent {
Status(Status, Option<String>),
/// A new frame waits in `DeviceStream::take_frame`.
Frame,
Screen(Screen),
/// Whether touches and keys reach the device.
Input(bool, Option<String>),
}
pub struct Frame {
pub image: Arc<RenderImage>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TouchPhase {
Begin,
Move,
End,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Button {
Home,
Back,
Recents,
}
/// A key the user pressed while the screen had focus.
#[derive(Clone, Debug)]
pub struct Key {
/// GPUI's key name: `a`, `enter`, `backspace`, `up`.
pub key: String,
/// The text it types, if any.
pub text: Option<String>,
pub shift: bool,
/// Control, Alt or the platform key was held: no text.
pub command: bool,
}
#[derive(Debug)]
enum Input {
/// Normalized 0..1 coordinates in the displayed frame.
Touch(TouchPhase, f32, f32),
Key(Key),
Text(String),
Button(Button),
Rotate,
}
struct Shared {
stop: AtomicBool,
watching: AtomicBool,
frame: Mutex<Option<Frame>>,
screen: Mutex<Option<Screen>>,
/// Bumped by `restart`, so the thread drops its connection and reconnects.
generation: std::sync::atomic::AtomicU64,
}
/// Where a device hub answers: this machine's on loopback, or another
/// device's through its remote server, which passes allowlisted stream
/// routes on to its hub for anyone with its key.
#[derive(Clone, Debug, PartialEq)]
pub struct HubTarget {
/// `host:port`.
pub address: String,
/// Put before every route: `/device-hub` through a remote server.
pub prefix: String,
/// The remote server's key, sent as a bearer token.
pub key: Option<String>,
}
impl HubTarget {
/// This machine's hub at its loopback origin, `http://127.0.0.1:<port>`.
pub fn local(origin: &str) -> Self {
Self { address: origin.trim_start_matches("http://").to_string(), prefix: String::new(), key: None }
}
fn connect(&self, timeout: Duration) -> Result<TcpStream> {
use std::net::ToSocketAddrs as _;
let address: SocketAddr =
self.address.to_socket_addrs()?.next().ok_or_else(|| anyhow!("{} can't be reached", self.address))?;
let tcp = TcpStream::connect_timeout(&address, timeout)?;
tcp.set_nodelay(true)?;
Ok(tcp)
}
/// The request head of a GET, with the key when there is one.
fn get_head(&self, path: &str, extra: &str) -> String {
let auth = self.key.as_ref().map(|key| format!("Authorization: Bearer {key}\r\n")).unwrap_or_default();
format!("GET {}{path} HTTP/1.1\r\nHost: {}\r\n{auth}{extra}Connection: close\r\n\r\n", self.prefix, self.address)
}
}
/// One device's live screen, as its panel watches and drives it.
pub struct DeviceStream {
shared: Arc<Shared>,
input: mpsc::Sender<Input>,
}
impl DeviceStream {
/// Starts the stream's thread. Nothing connects until `set_watching(true)`.
pub fn new(hub: HubTarget, platform: Platform, device: String, events: smol::channel::Sender<StreamEvent>) -> Self {
let shared = Arc::new(Shared {
stop: AtomicBool::new(false),
watching: AtomicBool::new(false),
frame: Mutex::new(None),
screen: Mutex::new(None),
generation: Default::default(),
});
let (input, input_rx) = mpsc::channel();
let thread_shared = shared.clone();
let _ = thread::Builder::new().name("device-stream".into()).spawn(move || {
let stream = Streamer { hub, device, shared: thread_shared, events, input: input_rx };
match platform {
Platform::Android => stream.run_android(),
Platform::Ios => stream.run_ios(),
}
});
Self { shared, input }
}
pub fn set_watching(&self, watching: bool) {
self.shared.watching.store(watching, Ordering::Relaxed);
}
/// Drops the connection and connects again: the Reconnect button.
pub fn restart(&self) {
self.shared.generation.fetch_add(1, Ordering::Relaxed);
}
/// The newest frame not taken yet.
pub fn take_frame(&self) -> Option<Frame> {
self.shared.frame.lock().ok()?.take()
}
pub fn touch(&self, phase: TouchPhase, x: f32, y: f32) {
let _ = self.input.send(Input::Touch(phase, x.clamp(0., 1.), y.clamp(0., 1.)));
}
pub fn key(&self, key: Key) {
let _ = self.input.send(Input::Key(key));
}
/// Types text, as a paste does.
pub fn text(&self, text: String) {
let _ = self.input.send(Input::Text(text));
}
pub fn button(&self, button: Button) {
let _ = self.input.send(Input::Button(button));
}
/// Turns an iOS Simulator to its next orientation.
pub fn rotate(&self) {
let _ = self.input.send(Input::Rotate);
}
}
impl Drop for DeviceStream {
fn drop(&mut self) {
self.shared.stop.store(true, Ordering::Relaxed);
}
}
struct Streamer {
hub: HubTarget,
device: String,
shared: Arc<Shared>,
events: smol::channel::Sender<StreamEvent>,
input: mpsc::Receiver<Input>,
}
/// Why a connection ended.
enum End {
/// Stopped, hidden or restarted: nothing to report.
Quiet,
/// It dropped; reconnect after a moment.
Retry(Option<String>),
/// It can't work; say so and wait for Reconnect.
Fail(String),
}
impl Streamer {
fn stopped(&self) -> bool {
self.shared.stop.load(Ordering::Relaxed) || self.events.is_closed()
}
fn wanted(&self, generation: u64) -> bool {
!self.stopped()
&& self.shared.watching.load(Ordering::Relaxed)
&& self.shared.generation.load(Ordering::Relaxed) == generation
}
fn emit(&self, event: StreamEvent) {
let _ = self.events.send_blocking(event);
}
/// Keeps a connection up while the panel shows: `connect` runs one
/// connection and says how it ended.
fn supervise(&self, mut connect: impl FnMut(&Self, u64) -> End) {
let mut failed = false;
let mut failed_generation = 0;
while !self.stopped() {
let generation = self.shared.generation.load(Ordering::Relaxed);
if !self.shared.watching.load(Ordering::Relaxed) || (failed && generation == failed_generation) {
while self.input.try_recv().is_ok() {}
thread::sleep(Duration::from_millis(100));
continue;
}
failed = false;
self.emit(StreamEvent::Status(Status::Connecting, None));
match connect(self, generation) {
End::Quiet => {}
End::Retry(detail) => {
self.emit(StreamEvent::Input(false, detail.clone()));
self.emit(StreamEvent::Status(Status::Connecting, detail));
thread::sleep(RETRY_DELAY);
}
End::Fail(detail) => {
self.emit(StreamEvent::Input(false, Some(detail.clone())));
self.emit(StreamEvent::Status(Status::Error, Some(detail)));
failed = true;
failed_generation = generation;
}
}
}
}
fn socket(&self, path: &str) -> Result<WebSocket<TcpStream>> {
use tungstenite::client::IntoClientRequest as _;
let tcp = self.hub.connect(Duration::from_secs(4))?;
tcp.set_read_timeout(Some(Duration::from_secs(8)))?;
let mut request = format!("ws://{}{}{path}", self.hub.address, self.hub.prefix).into_client_request()?;
if let Some(key) = &self.hub.key {
request.headers_mut().insert("Authorization", format!("Bearer {key}").parse()?);
}
let (socket, _) = tungstenite::client(request, tcp).map_err(|err| match err {
tungstenite::HandshakeError::Failure(tungstenite::Error::Http(response)) if response.status() == 401 => {
anyhow!("The other device refused the key.")
}
tungstenite::HandshakeError::Failure(tungstenite::Error::Http(response)) if response.status() == 503 => {
anyhow!("Device support isn't running on the other device.")
}
err => anyhow!("stream handshake failed: {err}"),
})?;
socket.get_ref().set_read_timeout(Some(POLL))?;
Ok(socket)
}
/// Hands a frame to the panel; one it hasn't drawn yet is replaced.
fn present(&self, frame: Frame) {
let Ok(mut slot) = self.shared.frame.lock() else { return };
if slot.replace(frame).is_none() {
drop(slot);
self.emit(StreamEvent::Frame);
}
}
fn set_screen(&self, screen: Screen) {
let Ok(mut current) = self.shared.screen.lock() else { return };
if *current != Some(screen) {
*current = Some(screen);
drop(current);
self.emit(StreamEvent::Screen(screen));
}
}
fn screen(&self) -> Option<Screen> {
*self.shared.screen.lock().ok()?
}
// ---- Android ----
fn run_android(self) {
let device = urlencode(&self.device);
self.supervise(|this, generation| this.android_connection(&device, generation));
}
fn android_connection(&self, device: &str, generation: u64) -> End {
let mut socket = match self.socket(&format!("/vendor/serve-emu/ws?device={device}&frame-meta=1")) {
Ok(socket) => socket,
Err(err) => return End::Retry(Some(err.to_string())),
};
self.emit(StreamEvent::Input(true, None));
let mut decoder = match openh264::decoder::Decoder::new() {
Ok(decoder) => decoder,
Err(err) => return End::Fail(format!("Video decoder: {err}")),
};
let mut awaiting_key = true;
let mut first_frame = false;
let mut last_keyframe_request: Option<Instant> = None;
let started = Instant::now();
let mut request_keyframe = |socket: &mut WebSocket<TcpStream>| {
if last_keyframe_request.is_none_or(|at| at.elapsed() > Duration::from_millis(500)) {
last_keyframe_request = Some(Instant::now());
let _ = socket.send(Message::text(json!({"type": "reset-video", "ack": false}).to_string()));
}
};
request_keyframe(&mut socket);
let end = loop {
if !self.wanted(generation) {
break End::Quiet;
}
if !first_frame && started.elapsed() > FIRST_FRAME_TIMEOUT {
break End::Fail("No video received from the device. Reconnect to try again.".into());
}
while let Ok(input) = self.input.try_recv() {
for message in android_messages(input) {
if socket.send(Message::text(message)).is_err() {
return End::Retry(None);
}
}
}
let data = match socket.read() {
Ok(Message::Binary(data)) => data,
Ok(Message::Text(text)) => {
// A new encoder session: its first frame must be a keyframe.
if serde_json::from_str::<serde_json::Value>(text.as_str())
.is_ok_and(|message| message["type"] == "video-session")
{
if let Ok(fresh) = openh264::decoder::Decoder::new() {
decoder = fresh;
}
awaiting_key = true;
request_keyframe(&mut socket);
}
continue;
}
Ok(Message::Close(frame)) => break End::Retry(frame.map(|frame| frame.reason.to_string()).filter(|r| !r.is_empty())),
Ok(_) => continue,
Err(tungstenite::Error::Io(err)) if matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => continue,
Err(err) => break End::Retry(Some(err.to_string())),
};
let (payload, key) = parse_semu(&data);
let key = key.unwrap_or_else(|| has_idr(payload));
if awaiting_key {
if !key {
request_keyframe(&mut socket);
continue;
}
awaiting_key = false;
}
match decoder.decode(payload) {
Ok(Some(yuv)) => {
let (width, height) = yuv.dimensions();
let mut pixels = vec![0u8; width * height * 4];
yuv.write_rgba8(&mut pixels);
let Some(frame) = bgra_frame(pixels, width as u32, height as u32) else { continue };
self.set_screen(Screen {
width: width as f32,
height: height as f32,
orientation: if width > height { Orientation::LandscapeLeft } else { Orientation::Portrait },
});
self.present(frame);
if !first_frame {
first_frame = true;
self.emit(StreamEvent::Status(Status::Streaming, None));
}
}
Ok(None) => {}
Err(_) => {
// Start over from the next keyframe.
if let Ok(fresh) = openh264::decoder::Decoder::new() {
decoder = fresh;
}
awaiting_key = true;
request_keyframe(&mut socket);
}
}
};
let _ = socket.close(None);
let _ = socket.flush();
end
}
// ---- iOS ----
fn run_ios(self) {
let device = urlencode(&self.device);
self.supervise(|this, generation| this.ios_connection(&device, generation));
}
/// The MJPEG stream on a thread of its own, input on this one.
fn ios_connection(&self, device: &str, generation: u64) -> End {
let video_done = Arc::new(AtomicBool::new(false));
let (video_tx, video_rx) = mpsc::channel::<VideoMessage>();
{
let (hub, device, done) = (self.hub.clone(), device.to_string(), video_done.clone());
thread::spawn(move || read_mjpeg(&hub, &device, &done, &video_tx));
}
let mut socket: Option<WebSocket<TcpStream>> = None;
let mut admitted = false;
let mut input_unavailable = false;
let mut next_input_attempt = Instant::now();
let mut first_frame = false;
let started = Instant::now();
let end = loop {
if !self.wanted(generation) {
break End::Quiet;
}
if !first_frame && started.elapsed() > FIRST_FRAME_TIMEOUT {
break End::Fail("No video received from the device. Reconnect to try again.".into());
}
// Frames.
let mut drained = false;
while let Ok(message) = video_rx.try_recv() {
drained = true;
match message {
VideoMessage::Jpeg(jpeg) => {
let screen = self.screen();
if let Some(frame) = decode_jpeg(&jpeg, screen) {
self.present(frame);
if !first_frame {
first_frame = true;
self.emit(StreamEvent::Status(Status::Streaming, None));
}
}
}
VideoMessage::Ended(detail) => {
video_done.store(true, Ordering::Relaxed);
if let Some(socket) = socket.as_mut() {
let _ = socket.close(None);
}
return End::Retry(detail);
}
}
}
// Input: the helper takes it once the video runs.
if socket.is_none() && Instant::now() >= next_input_attempt {
match self.socket(&format!("/vendor/serve-sim/helper/ws?device={device}")) {
Ok(connected) => {
socket = Some(connected);
admitted = false;
}
Err(err) => {
self.emit(StreamEvent::Input(false, Some(err.to_string())));
next_input_attempt = Instant::now() + RETRY_DELAY;
}
}
}
let Some(ws) = socket.as_mut() else {
if !drained {
thread::sleep(POLL);
}
continue;
};
while let Ok(input) = self.input.try_recv() {
if !admitted || input_unavailable {
continue;
}
for packet in ios_packets(input, self.screen()) {
let _ = ws.send(Message::binary(packet));
}
}
match ws.read() {
Ok(Message::Binary(data)) if !data.is_empty() => match data[0] {
IOS_INPUT_ADMITTED => {
admitted = true;
self.emit(StreamEvent::Input(!input_unavailable, None));
}
IOS_SCREEN_CONFIG => {
if let Ok(config) = serde_json::from_slice::<IosScreenConfig>(&data[1..]) {
input_unavailable = config.input_unavailable.unwrap_or(false);
self.emit(StreamEvent::Input(
admitted && !input_unavailable,
input_unavailable.then(|| "Simulator input is unavailable".to_string()),
));
if let Some(orientation) = Orientation::from_key(&config.orientation) {
self.set_screen(Screen { width: config.width, height: config.height, orientation });
}
}
}
_ => {}
},
Ok(Message::Close(_)) | Err(tungstenite::Error::ConnectionClosed | tungstenite::Error::AlreadyClosed) => {
socket = None;
admitted = false;
self.emit(StreamEvent::Input(false, Some("input socket closed".into())));
next_input_attempt = Instant::now() + RETRY_DELAY;
}
Ok(_) => {}
Err(tungstenite::Error::Io(err)) if matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => {}
Err(err) => {
socket = None;
admitted = false;
self.emit(StreamEvent::Input(false, Some(err.to_string())));
next_input_attempt = Instant::now() + RETRY_DELAY;
}
}
};
video_done.store(true, Ordering::Relaxed);
if let Some(mut socket) = socket {
let _ = socket.close(None);
let _ = socket.flush();
}
end
}
}
#[derive(Deserialize)]
struct IosScreenConfig {
width: f32,
height: f32,
orientation: String,
#[serde(rename = "inputUnavailable")]
input_unavailable: Option<bool>,
}
enum VideoMessage {
Jpeg(Vec<u8>),
Ended(Option<String>),
}
/// Reads serve-sim's MJPEG body and passes each JPEG on.
fn read_mjpeg(hub: &HubTarget, device: &str, done: &AtomicBool, frames: &mpsc::Sender<VideoMessage>) {
let result = (|| -> Result<()> {
let mut tcp = hub.connect(Duration::from_secs(4))?;
tcp.set_read_timeout(Some(Duration::from_millis(500)))?;
let request = hub.get_head(&format!("/vendor/serve-sim/helper/{device}/stream.mjpeg"), "Accept: multipart/x-mixed-replace\r\n");
std::io::Write::write_all(&mut tcp, request.as_bytes())?;
let mut buffer: Vec<u8> = Vec::with_capacity(512 * 1024);
let mut chunk = vec![0u8; 64 * 1024];
let mut headers_done = false;
let mut idle = Instant::now();
while !done.load(Ordering::Relaxed) {
match tcp.read(&mut chunk) {
Ok(0) => return Err(anyhow!("video stream ended")),
Ok(n) => {
idle = Instant::now();
buffer.extend_from_slice(&chunk[..n]);
}
Err(err) if matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => {
if idle.elapsed() > FIRST_FRAME_TIMEOUT {
return Err(anyhow!("Device stream stopped receiving video. Reconnecting…"));
}
continue;
}
Err(err) => return Err(err.into()),
}
if !headers_done {
let Some(end) = find(&buffer, b"\r\n\r\n") else { continue };
let head = String::from_utf8_lossy(&buffer[..end]).to_string();
let status = head.split_whitespace().nth(1).unwrap_or_default().to_string();
if status != "200" {
return Err(anyhow!("stream {status}"));
}
buffer.drain(..end + 4);
headers_done = true;
}
// Each part is one JPEG: cut from its start marker to its end marker.
while let Some(start) = find(&buffer, &[0xff, 0xd8, 0xff]) {
let Some(end) = find(&buffer[start + 2..], &[0xff, 0xd9]).map(|end| start + 2 + end + 2) else {
buffer.drain(..start);
break;
};
let jpeg = buffer[start..end].to_vec();
buffer.drain(..end);
if frames.send(VideoMessage::Jpeg(jpeg)).is_err() {
return Ok(());
}
}
if buffer.len() > 32 << 20 {
buffer.clear();
}
}
Ok(())
})();
if let Err(err) = result
&& !done.load(Ordering::Relaxed)
{
let _ = frames.send(VideoMessage::Ended(Some(err.to_string())));
}
}
fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack.windows(needle.len()).position(|window| window == needle)
}
/// serve-sim streams the raw framebuffer: a device that reports landscape
/// while its frames stay portrait is turned here, so the frame shows as the
/// user holds the device.
fn decode_jpeg(jpeg: &[u8], screen: Option<Screen>) -> Option<Frame> {
let mut image = image::load_from_memory_with_format(jpeg, image::ImageFormat::Jpeg).ok()?;
if let Some(screen) = screen.filter(|screen| screen.width <= screen.height) {
image = match screen.orientation {
Orientation::LandscapeLeft => image.rotate90(),
Orientation::LandscapeRight => image.rotate270(),
Orientation::PortraitUpsideDown => image.rotate180(),
Orientation::Portrait => image,
};
}
let rgba = image.into_rgba8();
let (width, height) = rgba.dimensions();
bgra_frame(rgba.into_raw(), width, height)
}
/// RGBA pixels as the BGRA image GPUI draws.
fn bgra_frame(mut pixels: Vec<u8>, width: u32, height: u32) -> Option<Frame> {
for pixel in pixels.chunks_exact_mut(4) {
pixel.swap(0, 2);
}
let buffer = image::RgbaImage::from_raw(width, height, pixels)?;
Some(Frame { image: Arc::new(RenderImage::new(vec![image::Frame::new(buffer)])) })
}
/// The Annex-B payload of a serve-emu message, and its key flag when the
/// SEMU header carries one.
pub fn parse_semu(data: &[u8]) -> (&[u8], Option<bool>) {
if data.len() > SEMU_HEADER && u32::from_be_bytes([data[0], data[1], data[2], data[3]]) == SEMU_MAGIC && data[4] == 1 {
(&data[SEMU_HEADER..], Some(data[5] & 1 != 0))
} else {
(data, None)
}
}
/// Whether an Annex-B access unit holds an IDR slice.
pub fn has_idr(data: &[u8]) -> bool {
let mut i = 0;
while i + 3 < data.len() {
if data[i] == 0 && data[i + 1] == 0 {
let code = if data[i + 2] == 1 { 3 } else if data[i + 2] == 0 && data[i + 3] == 1 { 4 } else { 0 };
if code > 0 {
if data.get(i + code).is_some_and(|nal| nal & 0x1f == 5) {
return true;
}
i += code + 1;
continue;
}
}
i += 1;
}
false
}
/// Android key codes for keys that type no text.
fn android_keycode(key: &str) -> Option<u32> {
Some(match key {
"up" => 19,
"down" => 20,
"left" => 21,
"right" => 22,
"tab" => 61,
"enter" => 66,
"backspace" => 67,
"delete" => 112,
"home" => 122,
"end" => 123,
"pageup" => 92,
"pagedown" => 93,
_ => return None,
})
}
/// serve-emu's JSON messages for one input.
fn android_messages(input: Input) -> Vec<String> {
match input {
Input::Touch(phase, x, y) => {
let action = match phase {
TouchPhase::Begin => "down",
TouchPhase::Move => "move",
TouchPhase::End => "up",
};
vec![json!({"type": "touch", "action": action, "x": x, "y": y}).to_string()]
}
Input::Key(key) => {
if key.key == "escape" {
return vec![json!({"type": "back"}).to_string()];
}
if let Some(keycode) = android_keycode(&key.key) {
return vec![json!({"type": "key", "keycode": keycode}).to_string()];
}
match key.text.filter(|text| !text.is_empty() && !key.command) {
Some(text) => vec![json!({"type": "text", "text": text}).to_string()],
None => Vec::new(),
}
}
Input::Text(text) => vec![json!({"type": "text", "text": text}).to_string()],
Input::Button(button) => {
let kind = match button {
Button::Home => "home",
Button::Back => "back",
Button::Recents => "recents",
};
vec![json!({"type": kind}).to_string()]
}
Input::Rotate => Vec::new(),
}
}
fn tagged(tag: u8, payload: serde_json::Value) -> Vec<u8> {
let mut packet = vec![tag];
packet.extend_from_slice(payload.to_string().as_bytes());
packet
}
/// A displayed point as a framebuffer point: frames stay portrait while the
/// device reports landscape.
fn raw_point(x: f32, y: f32, screen: Option<Screen>) -> (f32, f32) {
match screen.filter(|screen| screen.width <= screen.height).map(|screen| screen.orientation) {
Some(Orientation::LandscapeLeft) => (y, 1. - x),
Some(Orientation::LandscapeRight) => (1. - y, x),
Some(Orientation::PortraitUpsideDown) => (1. - x, 1. - y),
_ => (x, y),
}
}
/// HID usages for GPUI key names, and whether Shift goes with them.
fn hid_usage(key: &str) -> Option<(u8, bool)> {
let named = match key {
"enter" => 0x28,
"escape" => 0x29,
"backspace" => 0x2a,
"tab" => 0x2b,
"space" => 0x2c,
"delete" => 0x4c,
"right" => 0x4f,
"left" => 0x50,
"down" => 0x51,
"up" => 0x52,
_ => 0,
};
if named != 0 {
return Some((named, false));
}
let mut chars = key.chars();
let (Some(c), None) = (chars.next(), chars.next()) else { return None };
hid_char(c)
}
/// The HID usage that types a character on a US layout.
fn hid_char(c: char) -> Option<(u8, bool)> {
Some(match c {
'a'..='z' => (0x04 + (c as u8 - b'a'), false),
'A'..='Z' => (0x04 + (c as u8 - b'A'), true),
'1'..='9' => (0x1e + (c as u8 - b'1'), false),
'0' => (0x27, false),
' ' => (0x2c, false),
'\n' => (0x28, false),
'\t' => (0x2b, false),
_ => {
const PLAIN: &str = "-=[]\\;'`,./";
const SHIFTED: &str = "_+{}|:\"~<>?";
const USAGES: [u8; 11] = [0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38];
const DIGITS: &str = "!@#$%^&*()";
if let Some(ix) = PLAIN.find(c) {
(USAGES[ix], false)
} else if let Some(ix) = SHIFTED.find(c) {
(USAGES[ix], true)
} else if let Some(ix) = DIGITS.find(c) {
(if ix == 9 { 0x27 } else { 0x1e + ix as u8 }, true)
} else {
return None;
}
}
})
}
const HID_SHIFT: u8 = 0xe1;
fn hid_press(usage: u8, shift: bool) -> Vec<Vec<u8>> {
let key = |kind: &str, usage: u8| tagged(IOS_KEY, json!({"type": kind, "usage": usage}));
let mut packets = Vec::new();
if shift {
packets.push(key("down", HID_SHIFT));
}
packets.push(key("down", usage));
packets.push(key("up", usage));
if shift {
packets.push(key("up", HID_SHIFT));
}
packets
}
/// serve-sim's packets for one input.
fn ios_packets(input: Input, screen: Option<Screen>) -> Vec<Vec<u8>> {
match input {
Input::Touch(phase, x, y) => {
let kind = match phase {
TouchPhase::Begin => "begin",
TouchPhase::Move => "move",
TouchPhase::End => "end",
};
let (x, y) = raw_point(x, y, screen);
vec![tagged(IOS_TOUCH, json!({"type": kind, "x": x, "y": y}))]
}
Input::Key(key) => {
let typed = key.text.as_deref().and_then(|text| {
let mut chars = text.chars();
match (chars.next(), chars.next()) {
(Some(c), None) if !key.command => hid_char(c),
_ => None,
}
});
match typed.or_else(|| hid_usage(&key.key).map(|(usage, shift)| (usage, shift || key.shift))) {
Some((usage, shift)) => hid_press(usage, shift),
None => Vec::new(),
}
}
Input::Text(text) => text.chars().filter_map(hid_char).flat_map(|(usage, shift)| hid_press(usage, shift)).collect(),
Input::Button(button) => {
let name = match button {
Button::Home => "home",
Button::Recents => "app_switcher",
Button::Back => return Vec::new(),
};
vec![tagged(IOS_BUTTON, json!({"button": name}))]
}
Input::Rotate => {
let current = screen.map(|screen| screen.orientation).unwrap_or(Orientation::Portrait);
vec![tagged(IOS_ORIENTATION, json!({"orientation": current.next().key()}))]
}
}
}
/// Percent-encodes a device id for a query or path.
pub fn urlencode(text: &str) -> String {
text.bytes()
.map(|byte| match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (byte as char).to_string(),
_ => format!("%{byte:02X}"),
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[::core::prelude::v1::test]
fn parses_semu_packets() {
let mut packet = vec![0x53, 0x45, 0x4d, 0x55, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7];
packet.extend_from_slice(&[0, 0, 0, 1, 0x65, 0xaa]);
let (payload, key) = parse_semu(&packet);
assert_eq!(key, Some(true));
assert_eq!(payload, &[0, 0, 0, 1, 0x65, 0xaa]);
assert!(has_idr(payload));
let (payload, key) = parse_semu(&[0, 0, 1, 0x41, 0x9a]);
assert_eq!(key, None);
assert!(!has_idr(payload));
}
#[::core::prelude::v1::test]
fn maps_points_for_rotated_frames() {
let landscape = Some(Screen { width: 1170., height: 2532., orientation: Orientation::LandscapeLeft });
assert_eq!(raw_point(0.25, 0.75, landscape), (0.75, 0.75));
assert_eq!(raw_point(0.25, 0.75, None), (0.25, 0.75));
}
#[::core::prelude::v1::test]
fn turns_keys_into_device_input() {
let key = |key: &str, text: Option<&str>| Key { key: key.into(), text: text.map(str::to_string), shift: false, command: false };
assert_eq!(android_messages(Input::Key(key("escape", None))), [r#"{"type":"back"}"#]);
assert_eq!(android_messages(Input::Key(key("enter", None))), [r#"{"type":"key","keycode":66}"#]);
assert_eq!(android_messages(Input::Key(key("a", Some("A")))), [r#"{"type":"text","text":"A"}"#]);
assert_eq!(hid_char('A'), Some((0x04, true)));
assert_eq!(hid_char('?'), Some((0x38, true)));
assert_eq!(hid_char(')'), Some((0x27, true)));
assert_eq!(ios_packets(Input::Key(key("b", Some("b"))), None).len(), 2);
}
/// Decodes a stream dumped from serve-emu, when one is at hand.
#[::core::prelude::v1::test]
fn decodes_a_dumped_android_stream() {
let Ok(data) = std::fs::read("/tmp/devhub-test/stream.h264") else { return };
let mut decoder = openh264::decoder::Decoder::new().unwrap();
let mut frames = 0;
for packet in openh264::nal_units(&data) {
if let Ok(Some(yuv)) = decoder.decode(packet) {
assert_eq!(yuv.dimensions(), (576, 1280));
frames += 1;
}
}
assert!(frames > 0);
}
}