mirror of
https://github.com/storytold/storyteller-bevy.git
synced 2026-10-29 09:08:58 +00:00
Bootstrap Image Generator application (#53)
This PR begins scaffolding the Stable Diffusion Turbo application (title
TBD), which is a separate frontend for Storyteller Engine which focuses
on image generation.
### Design Overview
* The application renders a "split-screen" view, with Storyteller Studio
on the left, and the Stable Diffusion Turbo output on the right
* When changes are made in the Studio editor:
- A screenshot is taken from the Bevy application
- Screenshot is piped into Stable Diffusion Turbo model
- Model output is rendered to the screen
* This all happens in as close to real-time as possible
### Work completed in this PR
* The `studio-frontend` project has been (somewhat crudely) refactored
to split most of its components into reusable libraries
- `studio-ui` is a use-case-agnostic design system library for core UI
components. These are "dumb" components which are controlled
via `@property` inputs and send `CustomEvent`s in response to user
input.
- `studio-frontend` is a collection of reusable Storyteller Studio
widgets. These are "smart" components which depend on
`@storyteller/studio` directly, manage their own state, and require very
little configuration or styling. They are designed to be drop-in
building blocks that can be reused between various frontends.
* The new application has been bootstrapped as `image-generator-demo`
- Currently, this simply grabs screenshots from the Bevy application and
renders those screenshots on the right side of the split-screen view. A
future PR will wire up to the Storyteller.ai backend to pipe these
screenshots through Stable Diffusion Turbo, and render that model's
output instead.
### Testing
* Install dependencies as described in the repo's README
* Run `npx nx serve image-generator-demo`
* Open [localhost:4200](http://localhost:4200) in your browser
- Use the camera controller in the left side of the view to manipulate
the Bevy scene
- Verify the right side of the view updates with a fresh screenshot at a
rate of ~5 fps
* Toggle the "Realtime Capture" checkbox
- Verify the right side view no longer updates while the user is
actively manipulating the editor
- This should allow the app to run smoother on lower-spec hardware
---------
Co-authored-by: Danny McGee <dannymcgee@gmail.com>
This commit is contained in:
Brooks Palin
@@ -1,17 +1,12 @@
|
||||
import { on } from "@storyteller/framework";
|
||||
import * as studio from "@storyteller/studio";
|
||||
import {
|
||||
type EntityDespawnEvent,
|
||||
type EntityMultiSpawnEvent,
|
||||
type EntitySelectEvent,
|
||||
type EntitySpawnEvent,
|
||||
type SceneSavedEvent,
|
||||
type SceneStateEvent,
|
||||
TransformSpace,
|
||||
TransformType,
|
||||
SceneState,
|
||||
StudioMode,
|
||||
} from "@storyteller/studio";
|
||||
import { StudioURLParams } from "@storyteller/studio-web";
|
||||
import { CanvasProvider } from "@storyteller/studio-web/canvas-provider";
|
||||
import { match } from "@storyteller/utility";
|
||||
import { LitElement, html, nothing } from "lit";
|
||||
@@ -20,144 +15,35 @@ import { classMap } from "lit-html/directives/class-map.js";
|
||||
import { Ref, createRef, ref } from "lit-html/directives/ref.js";
|
||||
import { v4 as uuid } from "uuid";
|
||||
|
||||
import "@storyteller/studio-ui/button";
|
||||
import "@storyteller/studio-ui/icon";
|
||||
import "@storyteller/studio-ui/toolbar";
|
||||
import "@storyteller/studio-ui/tree";
|
||||
import "@storyteller/studio-web/canvas-provider";
|
||||
import "@storyteller/studio-web/viewer";
|
||||
import "@storyteller/studio-web/bvh-viewer";
|
||||
import "@storyteller/studio-web/editor";
|
||||
import "@storyteller/studio-web/mixamo-viewer";
|
||||
import "@storyteller/studio-web/scene-hierarchy";
|
||||
import "@storyteller/studio-web/transform-toolbar";
|
||||
import "./anim-editor.element";
|
||||
import "./button.element";
|
||||
import "./help.element";
|
||||
import "./icon.element";
|
||||
import "./toolbar.element";
|
||||
import "./transform-toolbar.element";
|
||||
import "./tree.element";
|
||||
|
||||
import "./app.element.scss";
|
||||
|
||||
interface SceneObject extends Omit<studio.SceneObject, "children"> {
|
||||
children: SceneObject[];
|
||||
}
|
||||
|
||||
@customElement("app-root")
|
||||
export class AppElement extends LitElement {
|
||||
@state() showHelp = false;
|
||||
@state() loading = true;
|
||||
|
||||
@state() transformSpace = TransformSpace.Local;
|
||||
@state() transformType = TransformType.Translate;
|
||||
|
||||
@state() sceneObjects: SceneObject[] = [];
|
||||
@state() selection?: string | null;
|
||||
|
||||
@state() bvhPath?: string;
|
||||
@state() mixamoPath?: string;
|
||||
@state() scenePath?: string;
|
||||
@state() mode?: StudioMode;
|
||||
|
||||
#sceneObjects: studio.SceneObject[] = [];
|
||||
#entityMap = new Map<string, SceneObject>();
|
||||
#params = new StudioURLParams();
|
||||
#canvasProvider: Ref<CanvasProvider> = createRef();
|
||||
|
||||
get #studioCanvas(): HTMLCanvasElement | null {
|
||||
const provider = this.#canvasProvider.value;
|
||||
if (!provider) return null;
|
||||
|
||||
return document.getElementById(provider.canvasId) as HTMLCanvasElement | null;
|
||||
}
|
||||
|
||||
override connectedCallback(): void {
|
||||
super.connectedCallback();
|
||||
|
||||
const params = new URLSearchParams(document.location.search);
|
||||
|
||||
if (params.has("bvh"))
|
||||
this.bvhPath = params.get("bvh")!;
|
||||
|
||||
if (params.has("mixamo"))
|
||||
this.mixamoPath = params.get("mixamo")!;
|
||||
|
||||
if (params.has("scene"))
|
||||
this.scenePath = params.get("scene")!;
|
||||
|
||||
this.mode = match (params.get("mode"), {
|
||||
"studio": () => StudioMode.Editor,
|
||||
"viewer": () => StudioMode.Viewer,
|
||||
_: () => StudioMode.Viewer,
|
||||
});
|
||||
|
||||
this.#redirectBodyFocus();
|
||||
}
|
||||
|
||||
protected override createRenderRoot(): Element | ShadowRoot {
|
||||
return this;
|
||||
}
|
||||
|
||||
@on("keydown")
|
||||
@on("keyup")
|
||||
pipeKeyboardEvents(event: KeyboardEvent): void {
|
||||
// When the user interacts with one of the web UI elements, the Bevy
|
||||
// canvas will lose focus, preventing it from receiving keyboard events.
|
||||
// As a workaround, we can listen for any events that have bubbled up to
|
||||
// the app root and repeat them for the canvas.
|
||||
//
|
||||
// This does unfortunately burden the web front-end with some additional
|
||||
// responsibility: when we handle a keyboard event and _don't_ want the
|
||||
// Bevy app to receive it, we'll need to call `event.stopPropagation()` in
|
||||
// the handler to prevent it from reaching this listener.
|
||||
const {
|
||||
type,
|
||||
code, charCode, key, keyCode, location, repeat,
|
||||
altKey, ctrlKey, metaKey, shiftKey,
|
||||
} = event;
|
||||
|
||||
this.#studioCanvas?.dispatchEvent(new KeyboardEvent(type, {
|
||||
code, charCode, key, keyCode, location, repeat,
|
||||
altKey, ctrlKey, metaKey, shiftKey,
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
composed: true,
|
||||
}));
|
||||
}
|
||||
|
||||
onSceneStateChange({ detail: state }: SceneStateEvent): void {
|
||||
console.log("scene state change:", SceneState[state]);
|
||||
|
||||
this.loading = match (state, {
|
||||
[SceneState.Active]: () => false,
|
||||
_: () => true,
|
||||
});
|
||||
}
|
||||
|
||||
@on("window:entity-select")
|
||||
onEntitySelectionChange({ detail: entity }: EntitySelectEvent): void {
|
||||
const sceneObject = this.#entityMap.get(entity!);
|
||||
console.log("Entity selection changed:", sceneObject);
|
||||
this.selection = entity;
|
||||
}
|
||||
|
||||
@on("window:entity-spawn")
|
||||
onEntitySpawned({ detail: value }: EntitySpawnEvent): void {
|
||||
console.log("Entity spawned:", value);
|
||||
this.#sceneObjects = this.#sceneObjects.concat(value).sort(sortByName);
|
||||
this.#updateHierarchy();
|
||||
}
|
||||
|
||||
@on("window:entity-multi-spawn")
|
||||
onEntitiesSpawned({ detail: values }: EntityMultiSpawnEvent): void {
|
||||
console.log("Entities spawned:", values);
|
||||
this.#sceneObjects = this.#sceneObjects.concat(values).sort(sortByName);
|
||||
this.#updateHierarchy();
|
||||
}
|
||||
|
||||
@on("window:entity-despawn")
|
||||
onEntityDespawned({ detail: entity }: EntityDespawnEvent): void {
|
||||
console.log("Entity despawned:", entity);
|
||||
this.#sceneObjects = this.#sceneObjects.filter(({ id }) => id !== entity);
|
||||
this.#updateHierarchy();
|
||||
}
|
||||
|
||||
@on("window:scene-saved")
|
||||
@on("scene-saved")
|
||||
async onSceneSaved({ detail: scene }: SceneSavedEvent): Promise<void> {
|
||||
const file = new File([scene], "scene.scn.ron", { type: "text/ron" });
|
||||
const formData = new FormData();
|
||||
@@ -179,98 +65,52 @@ export class AppElement extends LitElement {
|
||||
}
|
||||
}
|
||||
|
||||
@on("scene-state")
|
||||
onSceneStateChange({ detail: state }: SceneStateEvent): void {
|
||||
console.log("scene state change:", SceneState[state]);
|
||||
|
||||
this.loading = match (state, {
|
||||
[SceneState.Active]: () => false,
|
||||
_: () => true,
|
||||
});
|
||||
}
|
||||
|
||||
toggleHelp(): void {
|
||||
this.showHelp = !this.showHelp;
|
||||
}
|
||||
|
||||
onTransformSpaceChange({ detail: space }: CustomEvent<TransformSpace>): void {
|
||||
this.transformSpace = space;
|
||||
studio.setTransformSpace(space);
|
||||
}
|
||||
|
||||
onTransformTypeChange({ detail: type }: CustomEvent<TransformType>): void {
|
||||
this.transformType = type;
|
||||
studio.setTransformType(type);
|
||||
}
|
||||
|
||||
// TODO: Get some help from someone who knows what an algorithm is
|
||||
#updateHierarchy(): void {
|
||||
this.#entityMap.clear();
|
||||
|
||||
for (let obj of this.#sceneObjects)
|
||||
this.#entityMap.set(obj.id, { ...obj, children: [] });
|
||||
|
||||
for (let obj of this.#entityMap.values()) {
|
||||
if (obj.parent != null) {
|
||||
const parent = this.#entityMap.get(obj.parent);
|
||||
if (!parent)
|
||||
throw new Error(`Missing parent: ${obj.parent}`);
|
||||
|
||||
parent.children.push(obj);
|
||||
}
|
||||
}
|
||||
|
||||
for (let obj of this.#entityMap.values())
|
||||
obj.children.sort(sortByName);
|
||||
|
||||
this.sceneObjects = Array
|
||||
.from(this.#entityMap.values())
|
||||
.filter(obj => obj.parent == null)
|
||||
.sort(sortByName);
|
||||
|
||||
console.log("Hierarchy:", this.sceneObjects);
|
||||
}
|
||||
|
||||
#redirectBodyFocus(): void {
|
||||
// If user focus lands on the `document.body`, neither the frontend app
|
||||
// nor the Bevy app will properly handle keyboard events. Thie function
|
||||
// runs every tick to ensure that user focus never becomes "stranded" in
|
||||
// this way.
|
||||
if (document.activeElement === document.body) {
|
||||
const canvasId = this.#canvasProvider.value?.canvasId;
|
||||
if (canvasId)
|
||||
document.getElementById(canvasId)?.focus();
|
||||
}
|
||||
|
||||
requestAnimationFrame(() => this.#redirectBodyFocus());
|
||||
}
|
||||
|
||||
protected override render = () => html`
|
||||
<sts-canvas-provider
|
||||
class="app-root"
|
||||
${ref(this.#canvasProvider as any)}
|
||||
>
|
||||
<!-- Scene view -->
|
||||
${this.bvhPath ? html`
|
||||
${this.#params.bvh ? html`
|
||||
<sts-bvh-viewer
|
||||
bvhPath=${this.bvhPath}
|
||||
bvhPath=${this.#params.bvh}
|
||||
skyboxId="test_scene"
|
||||
@scene-state=${this.onSceneStateChange}
|
||||
></sts-bvh-viewer>
|
||||
` : this.mixamoPath ? html`
|
||||
` : this.#params.mixamo ? html`
|
||||
<sts-mixamo-viewer
|
||||
mixamoPath=${this.mixamoPath}
|
||||
mixamoPath=${this.#params.mixamo}
|
||||
skyboxId="test_scene"
|
||||
@scene-state=${this.onSceneStateChange}
|
||||
></sts-mixamo-viewer>
|
||||
` : this.scenePath ? html`
|
||||
` : this.#params.scene ? html`
|
||||
<sts-editor
|
||||
scenePath=${this.scenePath}
|
||||
@scene-state=${this.onSceneStateChange}
|
||||
scenePath=${this.#params.scene}
|
||||
></sts-editor>
|
||||
` : html`
|
||||
<sts-viewer
|
||||
objectId="sample-room.gltf"
|
||||
skyboxId="test_scene"
|
||||
.mode=${this.mode}
|
||||
@scene-state=${this.onSceneStateChange}
|
||||
.mode=${this.#params.mode}
|
||||
></sts-viewer>
|
||||
`}
|
||||
|
||||
<!-- Actions toolbar -->
|
||||
<sts-toolbar class="v-start h-start">
|
||||
<sts-toolbar-group>
|
||||
${this.mode === StudioMode.Editor ? html`
|
||||
${this.#params.mode === StudioMode.Editor ? html`
|
||||
<sts-button
|
||||
icon="floppy-disk"
|
||||
@click=${() => studio.saveScene()}
|
||||
@@ -288,30 +128,10 @@ export class AppElement extends LitElement {
|
||||
</sts-toolbar-group>
|
||||
</sts-toolbar>
|
||||
|
||||
${this.mode === StudioMode.Editor ? html`
|
||||
<!-- Transform toolbar -->
|
||||
<sts-transform-toolbar
|
||||
class="v-start h-center"
|
||||
.type=${this.transformType}
|
||||
@type-change=${this.onTransformTypeChange}
|
||||
.space=${this.transformSpace}
|
||||
@space-change=${this.onTransformSpaceChange}
|
||||
></sts-transform-toolbar>
|
||||
|
||||
<!-- Hierarchy and inspector -->
|
||||
<div class="hierarchy v-start h-end">
|
||||
${this.sceneObjects.map(obj => html`
|
||||
<sts-tree class="scene-object"
|
||||
id=${obj.id}
|
||||
.name=${obj.name}
|
||||
.treeChildren=${obj.children}
|
||||
></sts-tree>
|
||||
`)}
|
||||
</div>
|
||||
` : nothing}
|
||||
|
||||
<!-- Animation timeline -->
|
||||
${this.mode === StudioMode.Editor ? html`
|
||||
<!-- Studio-mode-only elements -->
|
||||
${this.#params.mode === StudioMode.Editor ? html`
|
||||
<sts-transform-toolbar class="v-start h-center"></sts-transform-toolbar>
|
||||
<sts-scene-hierarchy class="hierarchy v-start h-end"></sts-scene-hierarchy>
|
||||
<sts-anim-editor class="timeline v-end h-start h-before-hierarchy"></sts-anim-editor>
|
||||
` : nothing}
|
||||
|
||||
@@ -329,7 +149,3 @@ export class AppElement extends LitElement {
|
||||
</sts-canvas-provider>
|
||||
`;
|
||||
}
|
||||
|
||||
function sortByName(a: { name: string }, b: { name: string }): number {
|
||||
return a.name.localeCompare(b.name);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user