mirror of
https://github.com/storytold/storyteller-bevy.git
synced 2026-10-18 07:35:19 +00:00
d51eec52c3
This PR begins scaffolding the Stable Diffusion Turbo application (title
TBD), which is a separate frontend for Storyteller Engine which focuses
on image generation.
### Design Overview
* The application renders a "split-screen" view, with Storyteller Studio
on the left, and the Stable Diffusion Turbo output on the right
* When changes are made in the Studio editor:
- A screenshot is taken from the Bevy application
- Screenshot is piped into Stable Diffusion Turbo model
- Model output is rendered to the screen
* This all happens in as close to real-time as possible
### Work completed in this PR
* The `studio-frontend` project has been (somewhat crudely) refactored
to split most of its components into reusable libraries
- `studio-ui` is a use-case-agnostic design system library for core UI
components. These are "dumb" components which are controlled
via `@property` inputs and send `CustomEvent`s in response to user
input.
- `studio-frontend` is a collection of reusable Storyteller Studio
widgets. These are "smart" components which depend on
`@storyteller/studio` directly, manage their own state, and require very
little configuration or styling. They are designed to be drop-in
building blocks that can be reused between various frontends.
* The new application has been bootstrapped as `image-generator-demo`
- Currently, this simply grabs screenshots from the Bevy application and
renders those screenshots on the right side of the split-screen view. A
future PR will wire up to the Storyteller.ai backend to pipe these
screenshots through Stable Diffusion Turbo, and render that model's
output instead.
### Testing
* Install dependencies as described in the repo's README
* Run `npx nx serve image-generator-demo`
* Open [localhost:4200](http://localhost:4200) in your browser
- Use the camera controller in the left side of the view to manipulate
the Bevy scene
- Verify the right side of the view updates with a fresh screenshot at a
rate of ~5 fps
* Toggle the "Realtime Capture" checkbox
- Verify the right side view no longer updates while the user is
actively manipulating the editor
- This should allow the app to run smoother on lower-spec hardware
---------
Co-authored-by: Danny McGee <dannymcgee@gmail.com>
125 lines
3.7 KiB
TypeScript
125 lines
3.7 KiB
TypeScript
import { Token, UniqueToken, on, provide } from "@storyteller/framework";
|
|
import { LitElement, html, unsafeCSS } from "lit";
|
|
import { customElement } from "lit/decorators.js";
|
|
|
|
import styles from "./canvas-provider.element.scss?inline";
|
|
|
|
export interface CanvasProvider {
|
|
readonly host: HTMLElement;
|
|
readonly canvasId: string;
|
|
readonly canvas: HTMLCanvasElement | undefined;
|
|
|
|
/**
|
|
* Remove the canvas element from the provider host and return it. This can
|
|
* be used to temporarily move the canvas element to a different location in
|
|
* the DOM tree (e.g., for displaying in a modal dialog).
|
|
*
|
|
* `returnCanvas` must be called to restore the canvas element to its
|
|
* original location before the canvas's new parent element is removed from
|
|
* the document. If the canvas element is entirely absent from the document
|
|
* at any point after initialization (i.e., if
|
|
* `document.getElementById(canvasId)` returns `null`), the
|
|
* `@storyteller/studio` app will panic.
|
|
*/
|
|
takeCanvas(): HTMLCanvasElement | undefined;
|
|
|
|
/**
|
|
* Return the canvas element to the provider after removing it via
|
|
* `takeCanvas`.
|
|
*
|
|
* When `takeCanvas` is used to move the canvas element to a new location,
|
|
* this method must be called before the canvas's new parent is removed from
|
|
* the DOM tree, to ensure that the canvas element is always accessible to
|
|
* `@storyteller/studio`.
|
|
*/
|
|
returnCanvas(canvas: HTMLCanvasElement): void;
|
|
}
|
|
|
|
export const CANVAS_PROVIDER: Token<CanvasProvider> = new UniqueToken("CanvasProvider");
|
|
|
|
@customElement("sts-canvas-provider")
|
|
@provide(CANVAS_PROVIDER)
|
|
export class CanvasProviderElement
|
|
extends LitElement
|
|
implements CanvasProvider
|
|
{
|
|
static override styles = unsafeCSS(styles);
|
|
|
|
get host() { return this; }
|
|
get canvasId() { return this.#canvasId; }
|
|
get canvas(): HTMLCanvasElement | undefined { return this.#canvas; }
|
|
|
|
#canvasId = "studio_canvas";
|
|
#canvas?: HTMLCanvasElement;
|
|
|
|
takeCanvas(): HTMLCanvasElement | undefined {
|
|
if (!this.#canvas) return;
|
|
|
|
const canvas = this.removeChild(this.#canvas);
|
|
this.#canvas = undefined;
|
|
|
|
return canvas;
|
|
}
|
|
|
|
returnCanvas(canvas: HTMLCanvasElement) {
|
|
this.appendChild(canvas);
|
|
this.#canvas = canvas;
|
|
}
|
|
|
|
override connectedCallback(): void {
|
|
this.#canvas ??= this.#appendLightDomCanvas();
|
|
super.connectedCallback();
|
|
}
|
|
|
|
@on("keydown")
|
|
@on("keyup")
|
|
_pipeKeyboardEvents(event: KeyboardEvent): void {
|
|
// When the user interacts with one of the web UI elements, the Bevy
|
|
// canvas will lose focus, preventing it from receiving keyboard events.
|
|
// As a workaround, we can listen for any events that have bubbled up to
|
|
// the app root and repeat them for the canvas.
|
|
//
|
|
// This does unfortunately burden the web front-end with some additional
|
|
// responsibility: when we handle a keyboard event and _don't_ want the
|
|
// Bevy app to receive it, we'll need to call `event.stopPropagation()` in
|
|
// the handler to prevent it from reaching this listener.
|
|
const {
|
|
type,
|
|
code, charCode, key, keyCode, location, repeat,
|
|
altKey, ctrlKey, metaKey, shiftKey,
|
|
} = event;
|
|
|
|
document
|
|
.getElementById(this.#canvasId)
|
|
?.dispatchEvent(new KeyboardEvent(type, {
|
|
code, charCode, key, keyCode, location, repeat,
|
|
altKey, ctrlKey, metaKey, shiftKey,
|
|
bubbles: true,
|
|
cancelable: true,
|
|
composed: true,
|
|
}));
|
|
}
|
|
|
|
protected override render = () => html`
|
|
<slot></slot>
|
|
<slot name="canvas"></slot>
|
|
`;
|
|
|
|
#appendLightDomCanvas(): HTMLCanvasElement {
|
|
const canvas = document.createElement("canvas");
|
|
canvas.id = this.#canvasId;
|
|
canvas.slot = "canvas";
|
|
canvas.tabIndex = -1;
|
|
|
|
this.appendChild(canvas);
|
|
|
|
return canvas;
|
|
}
|
|
}
|
|
|
|
declare global {
|
|
export interface HTMLElementTagNameMap {
|
|
"sts-canvas-provider": CanvasProviderElement;
|
|
}
|
|
}
|