Fix some issues causing input jank (#106)

This PR fixes two issues that can cause a degraded user experience under
certain conditions:

### Focus-stealing

When focus lands on the `document.body`, it's typically bad for
accessibility and a sign of improper input handling. This generally
occurs when the currently-focused element is removed from the DOM. In
our particular case, it also prevents the Bevy application from
correctly handling keyboard events.

Previously, the problem of `document.body` focus had been fixed by
checking the `document.activeElement` every frame, and redirecting focus
to the Bevy canvas whenever it landed on the `document.body`. However,
when running in an iframe embedded on https://fakeyou.com, our
application will report `document.body` as the `document.activeElement`
whenever the parent application has focus, effectively stealing focus
from the parent application and making it impossible for the user to,
e.g., interact with text fields in the parent app.

This PR removes that focus-stealing behavior. Instead, we should
properly implement the ARIA specifications for any elements that may
disappear from the DOM so that user focus is managed in a precise and
predictable way. That will be addressed in follow-up PRs.

### "Stuck" input states

In certain circumstances, the browser window hosting our application may
lose focus while certain keys/buttons are held down which are bound to
long-running actions in the Bevy application, like camera manipulation.
When that occurs, Bevy is unable to detect when the bound keys/buttons
are released, causing the input state to become "stuck" reading those
keys/buttons as being pressed.

As a workaround, this PR introduces a listener for "blur" events on the
`window`, which dispatches synthetic "keyup"/"pointerup" events to the
Bevy canvas for all affected keyboard keys and mouse buttons.

studio-web/src/lib/canvas-provider.element.ts

@@ -74,6 +74,13 @@ export class CanvasProviderElement
 	@on("keydown")
 	@on("keyup")
 	_pipeKeyboardEvents(event: KeyboardEvent): void {
+		if (event.repeat) return;
+		if (this.#canvas != null) {
+			// If we own the canvas, we're going to catch events emitting from the
+			// canvas and then repeat them, eventually causing a stack overflow
+			return;
+		}
+
 		// When the user interacts with one of the web UI elements, the Bevy
 		// canvas will lose focus, preventing it from receiving keyboard events.
 		// As a workaround, we can listen for any events that have bubbled up to
@@ -100,6 +107,52 @@ export class CanvasProviderElement
 			}));
 	}
 
+	@on("window:blur")
+	onWindowFocusLost(): void {
+		// If the browser window we're running in loses focus while certain keys
+		// or mouse buttons are held down, it can cause the input state to become
+		// "stuck," e.g. manipulating the camera, because Bevy is unable to detect
+		// when the bound key/mouse button is released.
+		//
+		// This is a hacky workaround -- we dispatch synthetic "keyup" and
+		// "pointerup" events to the canvas for each key and mouse button that may
+		// be bound to long-running input actions.
+
+		const canvas = document.getElementById(this.#canvasId);
+		if (!canvas) return;
+
+		const eventConfig = {
+			altKey: false,
+			ctrlKey: false,
+			metaKey: false,
+			shiftKey: false,
+			bubbles: true,
+			cancelable: true,
+			composed: true,
+		};
+
+		const platform = sniffPlatform();
+
+		for (let key of ["Alt", "Control", "Shift"] as const) {
+			const map = KEYCODE_MAPS[key];
+			for (let { code, keyCode } of map[platform]) {
+				canvas.dispatchEvent(new KeyboardEvent("keyup", {
+					key,
+					code,
+					keyCode,
+					...eventConfig,
+				}));
+			}
+		}
+
+		for (let button of [0, 1, 2, 3, 4]) {
+			canvas.dispatchEvent(new PointerEvent("pointerup", {
+				button,
+				...eventConfig,
+			}));
+		}
+	}
+
 	protected override render = () => html`
 		<slot></slot>
 		<slot name="canvas"></slot>
@@ -122,3 +175,164 @@ declare global {
 		"sts-canvas-provider": CanvasProviderElement;
 	}
 }
+
+
+enum Platform {
+	Windows,
+	Apple,
+	Linux,
+	Android,
+}
+
+interface KeyCodes {
+	code: string;
+	keyCode: number;
+}
+
+/**
+ * Map of `KeyboardEvent.key` identifiers to their platform-specific `keyCode`s.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/UI_Events/Keyboard_event_key_values
+ */
+const KEYCODE_MAPS: {
+	[K in "Alt"|"Control"|"Shift"]: {
+		[P in Platform]: KeyCodes[];
+	}
+ } = {
+	"Alt": {
+		[Platform.Windows]: [{
+			code: "AltLeft",
+			keyCode: 0x12,
+		}, {
+			code: "AltLeft",
+			keyCode: 0xA4,
+		}, {
+			code: "AltRight",
+			keyCode: 0xA5,
+		}],
+		[Platform.Apple]: [{
+			code: "AltLeft",
+			keyCode: 0x3A,
+		}, {
+			code: "AltRight",
+			keyCode: 0x3D,
+		}],
+		[Platform.Linux]: [{
+			code: "AltLeft",
+			keyCode: 0xFFE9,
+		}, {
+			code: "AltRight",
+			keyCode: 0xFFEA,
+		}, {
+			code: "AltLeft",
+			keyCode: 0x01000023,
+		}],
+		[Platform.Android]: [{
+			code: "AltLeft",
+			keyCode: 57,
+		}, {
+			code: "AltRight",
+			keyCode: 58,
+		}],
+	},
+	"Control": {
+		[Platform.Windows]: [{
+			code: "ControlLeft",
+			keyCode: 0x11,
+		}, {
+			code: "ControlLeft",
+			keyCode: 0xA2,
+		}, {
+			code: "ControlRight",
+			keyCode: 0xA3,
+		}],
+		[Platform.Apple]: [{
+			code: "ControlLeft",
+			keyCode: 0x3B,
+		}, {
+			code: "ControlRight",
+			keyCode: 0x3E,
+		}, {
+			code: "MetaLeft",
+			keyCode: 0x37,
+		}, {
+			code: "MetaRight",
+			keyCode: 0x36,
+		}],
+		[Platform.Linux]: [{
+			code: "ControlLeft",
+			keyCode: 0xFFE3,
+		}, {
+			code: "ControlRight",
+			keyCode: 0xFFE4,
+		}, {
+			code: "ControlLeft",
+			keyCode: 0x01000021,
+		}],
+		[Platform.Android]: [{
+			code: "ControlLeft",
+			keyCode: 113,
+		}, {
+			code: "ControlRight",
+			keyCode: 114,
+		}]
+	},
+	"Shift": {
+		[Platform.Windows]: [{
+			code: "ShiftLeft",
+			keyCode: 0x10,
+		}, {
+			code: "ShiftLeft",
+			keyCode: 0xA0,
+		}, {
+			code: "ShiftRight",
+			keyCode: 0xA1,
+		}],
+		[Platform.Apple]: [{
+			code: "ShiftLeft",
+			keyCode: 0x38,
+		}, {
+			code: "ShiftRight",
+			keyCode: 0x3C,
+		}],
+		[Platform.Linux]: [{
+			code: "ShiftLeft",
+			keyCode: 0xFFE1,
+		}, {
+			code: "ShiftRight",
+			keyCode: 0xFFE2,
+		}, {
+			code: "ShiftLeft",
+			keyCode: 0x01000020,
+		}],
+		[Platform.Android]: [{
+			code: "ShiftLeft",
+			keyCode: 59,
+		}, {
+			code: "ShiftRight",
+			keyCode: 60,
+		}],
+	},
+};
+
+/**
+ * Try to determine the current {@linkcode Platform} by sniffing the User Agent
+ * string.
+ *
+ * @returns `Platform.Windows` if the actual platform could not be determined.
+ */
+function sniffPlatform(): Platform {
+	const userAgent = navigator.userAgent;
+	const platformString = userAgent.match(/\((.+?)\)/)?.[1] ?? "Windows";
+
+	if (platformString.includes("Windows"))
+		return Platform.Windows;
+	if (platformString.includes("Android"))
+		return Platform.Android;
+	if (platformString.includes("Mac OS"))
+		return Platform.Apple;
+	if (platformString.includes("Linux"))
+		return Platform.Linux;
+
+	return Platform.Windows;
+}

studio-web/src/lib/studio.element.ts

@@ -28,7 +28,6 @@ export class StudioElement extends LitElement {
 
 	#canvas?: HTMLCanvasElement;
 	#resizeObserver?: ResizeObserver;
-	#frameRequest?: number;
 
 	override connectedCallback(): void {
 		super.connectedCallback();
@@ -41,14 +40,9 @@ export class StudioElement extends LitElement {
 
 		if ((this.#canvas = this._canvasProvider.takeCanvas()))
 			this.appendChild(this.#canvas);
-
-		this.#tick();
 	}
 
 	override disconnectedCallback(): void {
-		if (this.#frameRequest != null)
-			cancelAnimationFrame(this.#frameRequest);
-
 		if (this.#canvas)
 			this._canvasProvider?.returnCanvas(this.removeChild(this.#canvas));
 
@@ -63,6 +57,8 @@ export class StudioElement extends LitElement {
 		} else {
 			// TODO
 		}
+
+		super.updated(changes);
 	}
 
 	#onResize(): void {
@@ -77,17 +73,6 @@ export class StudioElement extends LitElement {
 		}));
 	}
 
-	#tick(): void {
-		// If user focus lands on the `document.body`, neither the frontend app
-		// nor the Bevy app will properly handle keyboard events. Thie function
-		// runs every tick to ensure that user focus never becomes "stranded" in
-		// this way.
-		if (document.activeElement === document.body)
-			this.#canvas?.focus();
-
-		this.#frameRequest = requestAnimationFrame(() => this.#tick());
-	}
-
 	#start(): void {
 		if (!this.#canvas) return;
 		if (this.mode == null) return;

studio/src/hierarchy.rs

@@ -35,9 +35,7 @@ pub(crate) fn nearest_common_ancestor(
     }
 
     // Find the lowest depth of all the entities
-    let Some(lowest_depth) = depths.values().copied().reduce(usize::min) else {
-        return None;
-    };
+    let lowest_depth = depths.values().copied().reduce(usize::min)?;
 
     // Traverse up the tree from each entity until we find each one's ancestor
     // at the level of lowest_depth

studio/src/input.rs

@@ -39,27 +39,88 @@ impl Plugin for StudioInputPlugin {
                 .after(bevy::input::InputSystem)
                 .after(read_mouse_delta),
         );
+
+        app.add_systems(
+            PostUpdate,
+            (apply_deferred, log_input_actions).after(StudioInputSystem),
+        );
     }
 }
 
-#[derive(Event, Clone, Copy)]
+/// Logs all `InputAction`s emitted, along with the input state at the time they
+/// were fired, with `debug` verbosity.
+fn log_input_actions(
+    ri_keyboard: Res<Input<KeyCode>>,
+    ri_mouse_buttons: Res<Input<MouseButton>>,
+    mut er_input_actions: EventReader<InputAction>,
+) {
+    if er_input_actions.is_empty() {
+        return;
+    }
+
+    let lmb = if ri_mouse_buttons.pressed(MouseButton::Left) {
+        'X'
+    } else {
+        '-'
+    };
+    let mmb = if ri_mouse_buttons.pressed(MouseButton::Middle) {
+        'X'
+    } else {
+        '-'
+    };
+    let rmb = if ri_mouse_buttons.pressed(MouseButton::Right) {
+        'X'
+    } else {
+        '-'
+    };
+
+    use KeyCode::*;
+    let mod_keys = [AltLeft, AltRight, ShiftLeft, ControlLeft, ControlRight]
+        .into_iter()
+        .collect::<HashSet<_>>();
+    let action_keys = [W, A, S, D, E, Q, Escape, Grave, Delete, F]
+        .into_iter()
+        .collect::<HashSet<_>>();
+
+    let keys_pressed = ri_keyboard.get_pressed().copied().collect::<HashSet<_>>();
+    let keys_display = keys_pressed
+        .intersection(&mod_keys)
+        .map(|key| format!("{key:?}"))
+        .chain(
+            keys_pressed
+                .intersection(&action_keys)
+                .map(|key| format!("{key:?}")),
+        )
+        .collect::<Vec<_>>()
+        .join("+");
+
+    for action in er_input_actions.read() {
+        debug!(
+            "Mouse: [{lmb}{mmb}{rmb}]  \
+            Keyboard: [{keys_display}]  \
+            Action: [{action:.2?}]"
+        );
+    }
+}
+
+#[derive(Event, Clone, Copy, Debug)]
 pub struct InputAction(pub Lifecycle, pub Action);
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug)]
 pub enum Lifecycle {
     Start,
     Ongoing,
     Stop,
 }
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug)]
 pub enum Action {
     Camera(CameraMotion),
     Selection(Selection),
     ToggleDebug,
 }
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug)]
 pub enum CameraMotion {
     Translate(Vec3),
     Pan(Vec2),
@@ -73,7 +134,7 @@ impl From<CameraMotion> for Action {
     }
 }
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug)]
 pub enum Selection {
     /// Clear the current selection
     Clear,
@@ -206,7 +267,7 @@ fn translate_dcc_camera_events(
             CameraMotion::Rotate(Vec2::ZERO).into(),
         ));
     }
-    // Alt Camera Rotation (designed for laptop)
+    // Alternative Camera Rotation (designed for laptop)
     else if keyboard.pressed(KeyCode::ShiftLeft) {
         let lifecycle = if keyboard.just_pressed(KeyCode::ShiftLeft) {
             Lifecycle::Start