More Image Generator stuff (#59)

YOLO

image-generator-demo/src/app/app.element.scss

@@ -27,10 +27,7 @@
 	display: block;
 	width: 100%;
 	height: 100%;
-	padding: 1rem {
-		inline-end: 0.5rem;
-	}
-	border: none;
+	border-inline-end: none;
 	background: transparent;
 }
 .hierarchy-resize-handle {
@@ -73,8 +70,9 @@
 	border-radius: 0.5rem;
 }
 
-.config-prompt {
+.config-prompts {
 	display: flex;
+	flex-direction: column;
 	gap: 0.5rem;
 }
 
@@ -118,10 +116,6 @@
 		color: #FFFF;
 	}
 
-	&.prompt {
-		width: 512px;
-	}
-
 	&.seed {
 		width: 96px;
 		appearance: textfield;
@@ -134,8 +128,32 @@
 	}
 }
 
+.prompt {
+	flex: 1 0 0;
+}
+
 .config-field-group {
+	display: flex;
+	align-items: center;
+	gap: 0.5rem;
 	position: relative;
+
+	.pre-icon {
+		&.red {
+			color: rgb(220, 53, 69);
+			font-size: 1.333rem;
+		}
+		&.green {
+			color: rgb(25, 135, 84);
+			font-size: 1.15rem;
+		}
+	}
+}
+
+.config-row {
+	display: flex;
+	width: 768px;
+	gap: 0.5rem;
 }
 
 .seed-refresh {

image-generator-demo/src/app/app.element.ts

@@ -7,10 +7,13 @@ import {
 	StudioMode,
 } from "@storyteller/studio";
 import { match } from "@storyteller/utility";
-import { LitElement, html, nothing } from "lit";
+import { LitElement, type PropertyValues, html, nothing } from "lit";
 import { customElement, state } from "lit/decorators.js";
-import { styleMap } from "lit/directives/style-map.js";
 import { classMap } from "lit/directives/class-map.js";
+import { createRef, ref } from "lit/directives/ref.js";
+import { styleMap } from "lit/directives/style-map.js";
+
+import { ImageInfo, LcmClient } from "./lcm-client";
 
 import "@storyteller/studio-web/canvas-provider";
 import "@storyteller/studio-web/scene-hierarchy";
@@ -21,26 +24,81 @@ import "./app.element.scss"
 
 @customElement("app-root")
 export class AppElement extends LitElement {
-	@state() imageCaptureUrl?: string;
-	@state() realtimeCapture = true;
-
 	@state() hierarchyWidth = 256;
 	@state() resizing = false;
 
 	@state() loading = true;
 
+	@state() blob?: Blob;
 	@state() prompt = "";
+	@state() negPrompt = "";
 	@state() seed = randomSeed();
 
+	@state() inputUrl?: string;
+	@state() outputUrl?: string;
+
+	#lcmClient = new LcmClient(crypto.randomUUID());
+	#outputImageRef = createRef<HTMLImageElement>();
+
+	constructor () {
+		super();
+
+		this.#lcmClient.addEventListener("lcm-image-ready", event => {
+			console.log("lcm-image-ready", event.detail);
+
+			const prevUrl = this.outputUrl;
+			this.outputUrl = URL.createObjectURL(event.detail);
+
+			if (prevUrl)
+				URL.revokeObjectURL(prevUrl);
+		});
+
+		this.#lcmClient.addEventListener("lcm-image-url-ready", event => {
+			console.log("lcm-image-url-ready", event.detail);
+			this.outputUrl = event.detail;
+			this.requestUpdate();
+		});
+	}
+
 	protected override createRenderRoot = () => this;
 
 	override disconnectedCallback(): void {
-		if (this.imageCaptureUrl != null)
-			URL.revokeObjectURL(this.imageCaptureUrl);
+		if (this.inputUrl)
+			URL.revokeObjectURL(this.inputUrl);
+
+		if (this.outputUrl)
+			URL.revokeObjectURL(this.outputUrl);
 
 		super.disconnectedCallback();
 	}
 
+	protected override willUpdate(changes: PropertyValues<this>): void {
+		let updates: Partial<ImageInfo> = {};
+
+		if (changes.has("blob"))
+			updates.blob = this.blob;
+
+		if (changes.has("prompt"))
+			updates.prompt = this.prompt;
+
+		if (changes.has("negPrompt"))
+			updates.negativePrompt = this.negPrompt;
+
+		if (changes.has("seed"))
+			updates.seed = this.seed;
+
+		if (
+			changes.has("blob")
+			|| changes.has("prompt")
+			|| changes.has("negPrompt")
+			|| changes.has("seed")
+		) {
+			this.#lcmClient.update(updates);
+		}
+
+		super.willUpdate(changes);
+	}
+
 	@on("scene-state")
 	onSceneStateChange({ detail: state }: SceneStateEvent): void {
 		console.log("scene state change:", SceneState[state]);
@@ -52,12 +110,9 @@ export class AppElement extends LitElement {
 	}
 
 	@on("entity-changes")
+	@debounce(1000 / 24)
 	onStudioChanges(): void {
-		if (this.realtimeCapture) {
-			this.requestScreenshot_realtime();
-		} else {
-			this.requestScreenshot_delayed();
-		}
+		studio.requestScreenshot();
 	}
 
 	@on("viewport-resized")
@@ -67,27 +122,40 @@ export class AppElement extends LitElement {
 		studio.requestScreenshot();
 	}
 
-	@debounce(1000 / 24, { maxWait: 1000 / 5 })
-	requestScreenshot_realtime() {
-		studio.requestScreenshot();
-	}
-
-	@debounce(1000 / 24)
-	requestScreenshot_delayed() {
-		studio.requestScreenshot();
-	}
-
 	@on("screenshot-ready")
 	onScreenshotReady({ detail: screenshot }: ScreenshotReadyEvent) {
-		const prevUrl = this.imageCaptureUrl;
+		this.blob = new Blob([screenshot.data.buffer], { type: "image/jpg" });
 
-		const blob = new Blob([screenshot.data.buffer], { type: "image/png" });
-		this.imageCaptureUrl = URL.createObjectURL(blob);
+		const prevUrl = this.inputUrl;
+		this.inputUrl = URL.createObjectURL(this.blob);
 
 		if (prevUrl != null)
 			URL.revokeObjectURL(prevUrl);
 	}
 
+	updatePrompt(event: Event): void {
+		this.prompt = (event.target as HTMLInputElement).value;
+	}
+
+	updateNegPrompt(event: Event): void {
+		this.negPrompt = (event.target as HTMLInputElement).value;
+	}
+
+	refreshSeed(): void {
+		this.seed = randomSeed();
+	}
+
+	stopTextInputBubbling(event: KeyboardEvent): void {
+		// We want modifier keys like "Alt" to always be received by the Bevy
+		// canvas because they affect the camera controller behavior, but we do
+		// NOT want Bevy to receive typing inputs like W, A, S, and D while the
+		// user is trying to type into the Prompt field.
+		if (/^(Alt|Control)$/.test(event.key))
+			return;
+
+		event.stopPropagation();
+	}
+
 	onResizeStart(): void {
 		this.resizing = true;
 	}
@@ -101,15 +169,6 @@ export class AppElement extends LitElement {
 		this.hierarchyWidth = Math.max(Math.min(updated, 512), 128);
 	}
 
-	toggleRealtimeCapture(event: InputEvent): void {
-		const { checked } = event.target as HTMLInputElement;
-		this.realtimeCapture = checked;
-	}
-
-	refreshSeed(): void {
-		this.seed = randomSeed();
-	}
-
 	protected override render = () => html`
 		<sts-canvas-provider class="app-root">
 			<div
@@ -145,45 +204,81 @@ export class AppElement extends LitElement {
 					></sts-transform-toolbar>
 				</sts-studio>
 
-				${this.imageCaptureUrl != null ? html`
+				${this.inputUrl || this.outputUrl ? html`
 					<img
+						${ref(this.#outputImageRef)}
 						class="image-capture"
-						src=${this.imageCaptureUrl}
+						src=${(this.outputUrl ?? this.inputUrl)!}
 						alt="Image capture"
 					/>
 				` : html`
 					<div class="image-capture placeholder"></div>
 				`}
 
-				<div class="config config-prompt h-center v-end"
-					@keydown=${(event: KeyboardEvent) => event.stopPropagation()}
-					@keyup=${(event: KeyboardEvent) => event.stopPropagation()}
+				<div class="config config-prompts h-center v-end"
+					@keydown=${this.stopTextInputBubbling}
+					@keyup=${this.stopTextInputBubbling}
 				>
-					<input
-						class="config-field prompt"
-						type="text"
-						aria-label="Prompt"
-						placeholder="Enter a prompt to customize the output"
-					/>
-					<div class="config-field-group">
+					<div class="config-field-group prompt">
+						<sts-icon class="pre-icon green" icon="check"></sts-icon>
 						<input
-							class="config-field seed"
-							type="number"
-							aria-label="Seed"
-							value=${this.seed}
-							min="100"
-							max="10000"
+							class="config-field prompt"
+							type="text"
+							aria-label="Affirmative Prompt"
+							placeholder="Enter an affirmative prompt to customize the output"
+							.value=${this.prompt}
+							@change=${this.updatePrompt}
+							@keydown=${(event: KeyboardEvent) => {
+								if (event.key === "Enter") {
+									event.preventDefault();
+									this.updatePrompt(event);
+								}
+							}}
 						/>
-						<button
-							class="seed-refresh"
-							@click=${this.refreshSeed}
-						>
-							<sts-icon icon="rotate"></sts-icon>
-						</button>
+					</div>
+					<div class="config-row">
+						<div class="config-field-group prompt">
+							<sts-icon class="pre-icon red" icon="xmark"></sts-icon>
+							<input
+								class="config-field prompt"
+								type="text"
+								aria-label="Negative Prompt"
+								placeholder="Enter a negative prompt to customize the output"
+								.value=${this.negPrompt}
+								@change=${this.updateNegPrompt}
+								@keydown=${(event: KeyboardEvent) => {
+									if (event.key === "Enter") {
+										event.preventDefault();
+										this.updateNegPrompt(event);
+									}
+								}}
+							/>
+						</div>
+						<div class="config-field-group">
+							<input
+								class="config-field seed"
+								type="number"
+								aria-label="Seed"
+								.value=${this.seed.toString()}
+								min="1"
+								max="99999"
+							/>
+							<button
+								class="seed-refresh"
+								@click=${this.refreshSeed}
+							>
+								<sts-icon icon="rotate"></sts-icon>
+							</button>
+						</div>
 					</div>
 				</div>
 			</div>
 
+			${/*
+			FIXME: Real-time capture is not really feasible until we can come up
+			       with a more elegant way to exclude the world grid, gizmos, etc.
+			       from the captured image
+
 			<div class="config v-start h-end">
 				<label class="config-checkbox">
 					<input
@@ -194,6 +289,8 @@ export class AppElement extends LitElement {
 					Realtime Capture
 				</label>
 			</div>
+			*/
+			nothing}
 
 			${this.loading ? html`
 				<!-- TODO: Animated spinner or something -- talk to our designers -->
@@ -206,5 +303,5 @@ export class AppElement extends LitElement {
 }
 
 function randomSeed(): number {
-	return Math.round(Math.random() * 5000);
+	return Math.round(Math.random() * 99999);
 }

image-generator-demo/src/app/lcm-client.ts

@@ -0,0 +1,172 @@
+export enum LcmLiveStatus {
+	None = "NONE",
+	Connected = "connected",
+	Disconnected = "disconnected",
+	Wait = "wait",
+	SendFrame = "send_frame",
+	Timeout = "timeout",
+}
+
+export interface ImageInfo {
+	blob: Blob;
+	prompt: string;
+	negativePrompt: string;
+	seed: number;
+	guidanceScale: number;
+}
+
+interface LcmMessageData {
+	status: LcmLiveStatus | "error";
+	message?: string;
+}
+
+export class LcmClient extends WebSocket {
+	static readonly Api = "//b3000.pancake-chimaera.ts.net/api";
+
+	#userId: string;
+
+	get status() { return this.#status; }
+	#status = LcmLiveStatus.None;
+
+	#waiting = false;
+
+	#blob?: Blob;
+	#prompt?: string;
+	#negativePrompt?: string;
+	#seed?: number;
+	#guidanceScale = 0.8;
+
+	constructor (userId: string) {
+		super(`wss:${LcmClient.Api}/ws/${userId}`);
+		this.#userId = userId;
+
+		this.addEventListener("open", () => this.#onOpen());
+		this.addEventListener("close", () => this.#onClose());
+		this.addEventListener("message", event => this.#onMessage(event))
+	}
+
+	update(data: Partial<ImageInfo>): void {
+		if ("blob" in data)
+			this.#blob = data.blob;
+
+		if ("prompt" in data)
+			this.#prompt = data.prompt;
+
+		if ("negativePrompt" in data)
+			this.#negativePrompt = data.negativePrompt;
+
+		if ("seed" in data)
+			this.#seed = data.seed;
+
+		this.#update();
+	}
+
+	#onOpen(): void {
+		console.log(`Marking connection to websocket as open for user ${this.#userId}`);
+		this.#status = LcmLiveStatus.Connected;
+	}
+
+	#onClose(): void {
+		console.log(`Marking connection to websocket as closed for user ${this.#userId}`);
+		this.#status = LcmLiveStatus.Disconnected;
+	}
+
+	#onMessage(event: MessageEvent<string>): void {
+		console.log("onMessage:", event);
+		const data: LcmMessageData = JSON.parse(event.data);
+
+		if (data.status === "error") {
+			this.#status = LcmLiveStatus.Disconnected;
+			console.error(data.message);
+
+			return;
+		}
+
+		this.#status = data.status;
+
+		if (this.#status === LcmLiveStatus.SendFrame && this.#waiting)
+			this.#sendFrame();
+	}
+
+	#update(): void {
+		if (this.#status === LcmLiveStatus.SendFrame) {
+			this.#sendFrame();
+		} else {
+			this.#waiting = true;
+		}
+	}
+
+	async #sendFrame(): Promise<void> {
+		if (!this.#blob || !this.#prompt || !this.#negativePrompt || this.#seed == null) {
+			this.#waiting = true;
+			return;
+		}
+
+		console.log("Sending next frame...");
+
+		this.send(JSON.stringify({ status: "next_frame" }));
+		this.send(JSON.stringify({
+			prompt: this.#prompt,
+			negativePrompt: this.#negativePrompt,
+			seed: this.#seed,
+			guidanceScale: this.#guidanceScale,
+		}));
+		this.send(this.#blob);
+
+		this.#waiting = false;
+
+		try {
+			const url = `https:${LcmClient.Api}/stream/${this.#userId}`;
+
+			// FIXME: This only actually renders an image in the DOM about ~10% of the time
+			this.dispatchEvent(new CustomEvent("lcm-image-url-ready", { detail: url }));
+
+			// const response = await fetch(url, { method: "GET" });
+			// console.log("response:", response);
+
+			// if (!response.ok || !response.body) {
+			// 	console.error(`GET failed with status ${response.status}: "${response.statusText}"`);
+			// 	return;
+			// }
+
+			// const blobParts = [] as Uint8Array[];
+			// const reader = response.body.getReader();
+
+			// // FIXME: This doesn't appear to be an image
+			// // const { value } = await reader.read();
+			// // if (value) blobParts.push(value);
+
+			// // FIXME: this just stops responding after the first `read()` call even
+			// //        though `done` is false
+			// while (true) {
+			// 	console.log("Reading next chunk...");
+			//
+			// 	const { value, done } = await reader.read();
+			// 	console.log({ value, done });
+			//
+			// 	if (value) blobParts.push(value);
+			// 	if (done) break;
+			// }
+
+			// this.dispatchEvent(new CustomEvent<Blob>("lcm-image-ready", {
+			// 	detail: new Blob(blobParts, { type: "image/jpg" }),
+			// }));
+		}
+		catch (err) {
+			console.error(err);
+		}
+	}
+}
+
+export interface LcmClientEventMap extends WebSocketEventMap {
+	"lcm-image-ready": CustomEvent<Blob>;
+	"lcm-image-url-ready": CustomEvent<string>;
+}
+
+export interface LcmClient {
+	addEventListener<K extends keyof LcmClientEventMap>(
+		type: K,
+		listener: (this: LcmClient, ev: LcmClientEventMap[K]) => any,
+		options?: boolean | AddEventListenerOptions
+	): void;
+}