Bootstrap Image Generator application (#53)

This PR begins scaffolding the Stable Diffusion Turbo application (title
TBD), which is a separate frontend for Storyteller Engine which focuses
on image generation.

### Design Overview

* The application renders a "split-screen" view, with Storyteller Studio
  on the left, and the Stable Diffusion Turbo output on the right
* When changes are made in the Studio editor:
  - A screenshot is taken from the Bevy application
  - Screenshot is piped into Stable Diffusion Turbo model
  - Model output is rendered to the screen
* This all happens in as close to real-time as possible

### Work completed in this PR

* The `studio-frontend` project has been (somewhat crudely) refactored
  to split most of its components into reusable libraries
  - `studio-ui` is a use-case-agnostic design system library for core UI
    components. These are "dumb" components which are controlled
    via `@property` inputs and send `CustomEvent`s in response to user
    input.
  - `studio-frontend` is a collection of reusable Storyteller Studio
    widgets. These are "smart" components which depend on
    `@storyteller/studio` directly, manage their own state, and require very
    little configuration or styling. They are designed to be drop-in
    building blocks that can be reused between various frontends.
* The new application has been bootstrapped as `image-generator-demo`
  - Currently, this simply grabs screenshots from the Bevy application and
    renders those screenshots on the right side of the split-screen view. A
    future PR will wire up to the Storyteller.ai backend to pipe these
    screenshots through Stable Diffusion Turbo, and render that model's
    output instead.

### Testing

* Install dependencies as described in the repo's README
* Run `npx nx serve image-generator-demo`
* Open [localhost:4200](http://localhost:4200) in your browser
  - Use the camera controller in the left side of the view to manipulate
    the Bevy scene
  - Verify the right side of the view updates with a fresh screenshot at a
    rate of ~5 fps
* Toggle the "Realtime Capture" checkbox
  - Verify the right side view no longer updates while the user is
    actively manipulating the editor
  - This should allow the app to run smoother on lower-spec hardware

---------

Co-authored-by: Danny McGee <dannymcgee@gmail.com>

studio-web/src/index.ts

@@ -2,4 +2,7 @@ export * from "./lib/bvh-viewer.element";
 export * from "./lib/canvas-provider.element";
 export * from "./lib/editor.element";
 export * from "./lib/mixamo-viewer.element";
+export * from "./lib/scene-hierarchy.element";
+export * from "./lib/transform-toolbar.element";
+export * from "./lib/url-params";
 export * from "./lib/viewer.element";

studio-web/src/lib/canvas-provider.element.ts

@@ -1,4 +1,4 @@
-import { Token, UniqueToken, provide } from "@storyteller/framework";
+import { Token, UniqueToken, on, provide } from "@storyteller/framework";
 import { LitElement, html, unsafeCSS } from "lit";
 import { customElement } from "lit/decorators.js";
 
@@ -71,6 +71,35 @@ export class CanvasProviderElement
 		super.connectedCallback();
 	}
 
+	@on("keydown")
+	@on("keyup")
+	_pipeKeyboardEvents(event: KeyboardEvent): void {
+		// When the user interacts with one of the web UI elements, the Bevy
+		// canvas will lose focus, preventing it from receiving keyboard events.
+		// As a workaround, we can listen for any events that have bubbled up to
+		// the app root and repeat them for the canvas.
+		//
+		// This does unfortunately burden the web front-end with some additional
+		// responsibility: when we handle a keyboard event and _don't_ want the
+		// Bevy app to receive it, we'll need to call `event.stopPropagation()` in
+		// the handler to prevent it from reaching this listener.
+		const {
+			type,
+			code, charCode, key, keyCode, location, repeat,
+			altKey, ctrlKey, metaKey, shiftKey,
+		} = event;
+
+		document
+			.getElementById(this.#canvasId)
+			?.dispatchEvent(new KeyboardEvent(type, {
+				code, charCode, key, keyCode, location, repeat,
+				altKey, ctrlKey, metaKey, shiftKey,
+				bubbles: true,
+				cancelable: true,
+				composed: true,
+			}));
+	}
+
 	protected override render = () => html`
 		<slot></slot>
 		<slot name="canvas"></slot>

studio-web/src/lib/scene-hierarchy.element.scss

@@ -0,0 +1,24 @@
+:host {
+	box-sizing: border-box;
+	padding: 0.5rem 1rem;
+	overflow: auto;
+
+	background: #0006;
+	backdrop-filter: blur(16px);
+	border: 8px solid transparent;
+	border-radius: 0.5rem;
+
+	&::-webkit-scrollbar {
+		width: 8px;
+		height: 8px;
+
+		&-thumb {
+			border-radius: 2px;
+			background: #FFF2;
+
+			&:hover {
+				background: #FFF3;
+			}
+		}
+	}
+}

studio-web/src/lib/scene-hierarchy.element.ts

@@ -0,0 +1,77 @@
+import * as studio from "@storyteller/studio";
+import {
+	type EntityDespawnEvent,
+	type EntityMultiSpawnEvent,
+	type EntitySpawnEvent,
+} from "@storyteller/studio";
+import { on } from "@storyteller/framework";
+import { LitElement, html, unsafeCSS } from "lit";
+import { customElement, state } from "lit/decorators.js";
+
+import styles from "./scene-hierarchy.element.scss?inline";
+
+import "@storyteller/studio-ui/tree";
+
+interface SceneObject extends Omit<studio.SceneObject, "children"> {
+	children: SceneObject[];
+}
+
+@customElement("sts-scene-hierarchy")
+export class SceneHierarchyElement extends LitElement {
+	static override styles = unsafeCSS(styles);
+
+	get tree() { return this._tree as readonly SceneObject[]; }
+	get entityMap() { return this.#entityMap as ReadonlyMap<string, SceneObject>; }
+
+	@state() private _tree: SceneObject[] = [];
+	#sceneObjectsFlat: studio.SceneObject[] = [];
+	#entityMap = new Map<string, SceneObject>();
+
+	@on("window:entity-spawn")
+	@on("window:entity-multi-spawn")
+	_onEntitiesSpawned({ detail: entities }: EntitySpawnEvent | EntityMultiSpawnEvent): void {
+		this.#sceneObjectsFlat = this.#sceneObjectsFlat.concat(entities);
+		this.#rebuildTree();
+	}
+
+	@on("window:entity-despawn")
+	_onEntityDespawn({ detail: entity }: EntityDespawnEvent): void {
+		this.#sceneObjectsFlat = this.#sceneObjectsFlat.filter(({ id }) => id !== entity);
+		this.#rebuildTree();
+	}
+
+	#rebuildTree(): void {
+		this.#entityMap.clear();
+
+		for (let obj of this.#sceneObjectsFlat)
+			this.#entityMap.set(obj.id, { ...obj, children: [] });
+
+		for (let obj of this.#entityMap.values()) {
+			if (obj.parent != null) {
+				const parent = this.#entityMap.get(obj.parent);
+				if (!parent)
+					throw new Error(`Missing parent: ${obj.parent}`);
+
+				parent.children.push(obj);
+			}
+		}
+
+		for (let obj of this.#entityMap.values())
+			obj.children.sort((a, b) => a.name.localeCompare(b.name));
+
+		this._tree = Array
+			.from(this.#entityMap.values())
+			.filter(obj => obj.parent == null)
+			.sort((a, b) => a.name.localeCompare(b.name))
+	}
+
+	protected override render = () => html`
+		${this.tree.map(obj => html`
+			<sts-tree
+				id=${obj.id}
+				.name=${obj.name}
+				.treeChildren=${obj.children}
+			></sts-tree>
+		`)}
+	`
+}

studio-web/src/lib/studio-base.element.scss

@@ -10,4 +10,5 @@
 	inset: 0;
 	width: 100%;
 	height: 100%;
+	outline: none !important;
 }

studio-web/src/lib/studio-base.element.ts

@@ -10,11 +10,14 @@ import styles from "./studio-base.element.scss?inline";
 export abstract class StudioBaseElement extends LitElement {
 	static override styles = unsafeCSS(styles);
 
+	get canvas() { return this.#canvas; }
+
 	@inject(CANVAS_PROVIDER)
 	_canvasProvider?: CanvasProvider;
 
 	#canvas?: HTMLCanvasElement;
 	#resizeObserver?: ResizeObserver;
+	#frameRequest?: number;
 
 	abstract onStart(viewportSize: ViewportSize): void;
 	abstract onChanges?(changes: PropertyValues<this>): void;
@@ -30,9 +33,14 @@ export abstract class StudioBaseElement extends LitElement {
 
 		if ((this.#canvas = this._canvasProvider.takeCanvas()))
 			this.appendChild(this.#canvas);
+
+		this.#tick();
 	}
 
 	override disconnectedCallback(): void {
+		if (this.#frameRequest != null)
+			cancelAnimationFrame(this.#frameRequest);
+
 		if (this.#canvas)
 			this._canvasProvider?.returnCanvas(this.removeChild(this.#canvas));
 
@@ -54,6 +62,17 @@ export abstract class StudioBaseElement extends LitElement {
 		studio.resize(new ViewportSize(width, height));
 	}
 
+	#tick(): void {
+		// If user focus lands on the `document.body`, neither the frontend app
+		// nor the Bevy app will properly handle keyboard events. Thie function
+		// runs every tick to ensure that user focus never becomes "stranded" in
+		// this way.
+		if (document.activeElement === document.body)
+			this.#canvas?.focus();
+
+		this.#frameRequest = requestAnimationFrame(() => this.#tick());
+	}
+
 	#start(): void {
 		if (!this.#canvas) return;
 

studio-web/src/lib/transform-toolbar.element.scss

@@ -0,0 +1,57 @@
+.btn-toggle {
+	display: inline-flex;
+	align-items: center;
+	justify-content: center;
+	position: relative;
+	font-size: 16px;
+	padding: 10px 12px;
+	pointer-events: auto;
+
+	// TODO: Copypasta
+	background: #0006;
+	color: #FFFC;
+	backdrop-filter: blur(16px);
+	transition: {
+		property: background, color;
+		duration: 0.1s;
+		timing-function: linear;
+	}
+
+	&:hover {
+		background: #0008;
+		color: #FFF;
+	}
+
+	&:has(:focus-visible) {
+		z-index: 1;
+		outline: 4px solid #2F83FFBF;
+	}
+
+	&:has(:checked) {
+		background: #7A2C34E3;
+		color: #FFF;
+
+		&:hover {
+			background: #90363F;
+		}
+	}
+
+	&:has(:disabled) {
+		background: #0003 !important;
+		color: #FFF6 !important;
+	}
+
+	input {
+		display: block;
+		appearance: none;
+		border: 0;
+		margin: 0;
+		padding: 0;
+		opacity: 0;
+		position: absolute;
+		inset: 0;
+		width: 100%;
+		height: 100%;
+		z-index: 1;
+	}
+}

studio-web/src/lib/transform-toolbar.element.ts

@@ -0,0 +1,103 @@
+import * as studio from "@storyteller/studio";
+import { TransformSpace, TransformType } from "@storyteller/studio";
+import { match } from "@storyteller/utility";
+import { LitElement, html, unsafeCSS } from "lit";
+import { customElement, state } from "lit/decorators.js";
+
+import "@storyteller/studio-ui/icon";
+import "@storyteller/studio-ui/toolbar";
+
+import styles from "./transform-toolbar.element.scss?inline";
+
+@customElement("sts-transform-toolbar")
+export class TransformToolbarElement extends LitElement {
+	static override styles = unsafeCSS(styles);
+
+	get space() { return this._space; }
+	@state() private _space = TransformSpace.Local;
+
+	get type() { return this._type; }
+	@state() private _type = TransformType.Translate;
+
+	#onSpaceChange(event: InputEvent): void {
+		const { value } = event.target as HTMLInputElement;
+		this._space = match (value, {
+			"local": () => TransformSpace.Local,
+			"world": () => TransformSpace.World,
+			_: () => {
+				throw new Error(`Unknown transform space: "${value}"`);
+			}
+		});
+
+		studio.setTransformSpace(this._space);
+	}
+
+	#onTypeChange(event: InputEvent): void {
+		const { name, checked } = event.target as HTMLInputElement;
+		const flag = match (name, {
+			"translate": () => TransformType.Translate,
+			"rotate": () => TransformType.Rotate,
+			"scale": () => TransformType.Scale,
+			_: () => {
+				throw new Error(`Unknown transform type: "${name}"`);
+			}
+		});
+
+		if (checked) this._type |= flag;
+		else this._type &= ~flag;
+
+		studio.setTransformType(this._type);
+	}
+
+	protected override render = () => html`
+		<sts-toolbar>
+			<sts-toolbar-group>
+				<label class="btn-toggle">
+					<sts-icon icon="cube">Local</sts-icon>
+					<input type="radio"
+						name="transform-space"
+						value="local"
+						?checked=${this._space === TransformSpace.Local}
+						@input=${this.#onSpaceChange}
+					/>
+				</label>
+				<label class="btn-toggle">
+					<sts-icon icon="globe">Global</sts-icon>
+					<input type="radio"
+						name="transform-space"
+						value="world"
+						?checked=${this._space === TransformSpace.World}
+						@input=${this.#onSpaceChange}
+					/>
+				</label>
+			</sts-toolbar-group>
+			<sts-toolbar-group>
+				<label class="btn-toggle">
+					<sts-icon icon="up-down-left-right">Translate</sts-icon>
+					<input type="checkbox"
+						name="translate"
+						?checked=${Boolean(this._type & TransformType.Translate)}
+						@input=${this.#onTypeChange}
+					/>
+				</label>
+				<label class="btn-toggle">
+					<sts-icon icon="rotate">Rotate</sts-icon>
+					<input type="checkbox"
+						name="rotate"
+						?checked=${Boolean(this._type & TransformType.Rotate)}
+						@input=${this.#onTypeChange}
+					/>
+				</label>
+				<label class="btn-toggle">
+					<sts-icon icon="scale">Scale</sts-icon>
+					<input type="checkbox"
+						name="scale"
+						disabled
+						?checked=${Boolean(this._type & TransformType.Scale)}
+						@input=${this.#onTypeChange}
+					/>
+				</label>
+			</sts-toolbar-group>
+		</sts-toolbar>
+	`;
+}

studio-web/src/lib/url-params.ts

@@ -0,0 +1,20 @@
+import { StudioMode } from "@storyteller/studio";
+import { match } from "@storyteller/utility";
+
+export class StudioURLParams {
+	#_nativeParams?: URLSearchParams;
+	get #nativeParams() {
+		return this.#_nativeParams ??= new URLSearchParams(document.location.search);
+	}
+
+	get bvh() { return this.#nativeParams.get("bvh"); }
+	get mixamo() { return this.#nativeParams.get("mixamo"); }
+	get scene() { return this.#nativeParams.get("scene"); }
+	get mode(): StudioMode {
+		return match(this.#nativeParams.get("mode"), {
+			"studio": () => StudioMode.Editor,
+			"viewer": () => StudioMode.Viewer,
+			_: () => StudioMode.Viewer,
+		});
+	}
+}