Files
storyteller-bevy/studio-web/src/lib/canvas-provider.element.ts
T
Brooks Palin d51eec52c3 Bootstrap Image Generator application (#53)
This PR begins scaffolding the Stable Diffusion Turbo application (title
TBD), which is a separate frontend for Storyteller Engine which focuses
on image generation.

### Design Overview

* The application renders a "split-screen" view, with Storyteller Studio
  on the left, and the Stable Diffusion Turbo output on the right
* When changes are made in the Studio editor:
  - A screenshot is taken from the Bevy application
  - Screenshot is piped into Stable Diffusion Turbo model
  - Model output is rendered to the screen
* This all happens in as close to real-time as possible

### Work completed in this PR

* The `studio-frontend` project has been (somewhat crudely) refactored
  to split most of its components into reusable libraries
  - `studio-ui` is a use-case-agnostic design system library for core UI
    components. These are "dumb" components which are controlled
    via `@property` inputs and send `CustomEvent`s in response to user
    input.
  - `studio-frontend` is a collection of reusable Storyteller Studio
    widgets. These are "smart" components which depend on
    `@storyteller/studio` directly, manage their own state, and require very
    little configuration or styling. They are designed to be drop-in
    building blocks that can be reused between various frontends.
* The new application has been bootstrapped as `image-generator-demo`
  - Currently, this simply grabs screenshots from the Bevy application and
    renders those screenshots on the right side of the split-screen view. A
    future PR will wire up to the Storyteller.ai backend to pipe these
    screenshots through Stable Diffusion Turbo, and render that model's
    output instead.

### Testing

* Install dependencies as described in the repo's README
* Run `npx nx serve image-generator-demo`
* Open [localhost:4200](http://localhost:4200) in your browser
  - Use the camera controller in the left side of the view to manipulate
    the Bevy scene
  - Verify the right side of the view updates with a fresh screenshot at a
    rate of ~5 fps
* Toggle the "Realtime Capture" checkbox
  - Verify the right side view no longer updates while the user is
    actively manipulating the editor
  - This should allow the app to run smoother on lower-spec hardware

---------

Co-authored-by: Danny McGee <dannymcgee@gmail.com>
2024-02-26 17:13:07 -05:00

125 lines
3.7 KiB
TypeScript

import { Token, UniqueToken, on, provide } from "@storyteller/framework";
import { LitElement, html, unsafeCSS } from "lit";
import { customElement } from "lit/decorators.js";
import styles from "./canvas-provider.element.scss?inline";
export interface CanvasProvider {
readonly host: HTMLElement;
readonly canvasId: string;
readonly canvas: HTMLCanvasElement | undefined;
/**
* Remove the canvas element from the provider host and return it. This can
* be used to temporarily move the canvas element to a different location in
* the DOM tree (e.g., for displaying in a modal dialog).
*
* `returnCanvas` must be called to restore the canvas element to its
* original location before the canvas's new parent element is removed from
* the document. If the canvas element is entirely absent from the document
* at any point after initialization (i.e., if
* `document.getElementById(canvasId)` returns `null`), the
* `@storyteller/studio` app will panic.
*/
takeCanvas(): HTMLCanvasElement | undefined;
/**
* Return the canvas element to the provider after removing it via
* `takeCanvas`.
*
* When `takeCanvas` is used to move the canvas element to a new location,
* this method must be called before the canvas's new parent is removed from
* the DOM tree, to ensure that the canvas element is always accessible to
* `@storyteller/studio`.
*/
returnCanvas(canvas: HTMLCanvasElement): void;
}
export const CANVAS_PROVIDER: Token<CanvasProvider> = new UniqueToken("CanvasProvider");
@customElement("sts-canvas-provider")
@provide(CANVAS_PROVIDER)
export class CanvasProviderElement
extends LitElement
implements CanvasProvider
{
static override styles = unsafeCSS(styles);
get host() { return this; }
get canvasId() { return this.#canvasId; }
get canvas(): HTMLCanvasElement | undefined { return this.#canvas; }
#canvasId = "studio_canvas";
#canvas?: HTMLCanvasElement;
takeCanvas(): HTMLCanvasElement | undefined {
if (!this.#canvas) return;
const canvas = this.removeChild(this.#canvas);
this.#canvas = undefined;
return canvas;
}
returnCanvas(canvas: HTMLCanvasElement) {
this.appendChild(canvas);
this.#canvas = canvas;
}
override connectedCallback(): void {
this.#canvas ??= this.#appendLightDomCanvas();
super.connectedCallback();
}
@on("keydown")
@on("keyup")
_pipeKeyboardEvents(event: KeyboardEvent): void {
// When the user interacts with one of the web UI elements, the Bevy
// canvas will lose focus, preventing it from receiving keyboard events.
// As a workaround, we can listen for any events that have bubbled up to
// the app root and repeat them for the canvas.
//
// This does unfortunately burden the web front-end with some additional
// responsibility: when we handle a keyboard event and _don't_ want the
// Bevy app to receive it, we'll need to call `event.stopPropagation()` in
// the handler to prevent it from reaching this listener.
const {
type,
code, charCode, key, keyCode, location, repeat,
altKey, ctrlKey, metaKey, shiftKey,
} = event;
document
.getElementById(this.#canvasId)
?.dispatchEvent(new KeyboardEvent(type, {
code, charCode, key, keyCode, location, repeat,
altKey, ctrlKey, metaKey, shiftKey,
bubbles: true,
cancelable: true,
composed: true,
}));
}
protected override render = () => html`
<slot></slot>
<slot name="canvas"></slot>
`;
#appendLightDomCanvas(): HTMLCanvasElement {
const canvas = document.createElement("canvas");
canvas.id = this.#canvasId;
canvas.slot = "canvas";
canvas.tabIndex = -1;
this.appendChild(canvas);
return canvas;
}
}
declare global {
export interface HTMLElementTagNameMap {
"sts-canvas-provider": CanvasProviderElement;
}
}