storytold/storyteller-bevy
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-bevy.git synced 2026-10-26 08:38:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

More Image Generator stuff (#59)

Browse Source 
YOLO
This commit is contained in:
Danny McGee
2024-02-28 15:52:01 -05:00
committed by GitHub
parent 04ecc78059
commit f31cf35da1
18 changed files with 462 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
image-generator-demo/src/app/app.element.ts
+156 -59
View File
@@ -7,10 +7,13 @@ import {
StudioMode,
} from "@storyteller/studio";
import { match } from "@storyteller/utility";
import { LitElement, html, nothing } from "lit";
import { LitElement, type PropertyValues, html, nothing } from "lit";
import { customElement, state } from "lit/decorators.js";
import { styleMap } from "lit/directives/style-map.js";
import { classMap } from "lit/directives/class-map.js";
import { createRef, ref } from "lit/directives/ref.js";
import { styleMap } from "lit/directives/style-map.js";
import { ImageInfo, LcmClient } from "./lcm-client";
import "@storyteller/studio-web/canvas-provider";
import "@storyteller/studio-web/scene-hierarchy";
@@ -21,26 +24,81 @@ import "./app.element.scss"
@customElement("app-root")
export class AppElement extends LitElement {
@state() imageCaptureUrl?: string;
@state() realtimeCapture = true;
@state() hierarchyWidth = 256;
@state() resizing = false;
@state() loading = true;
@state() blob?: Blob;
@state() prompt = "";
@state() negPrompt = "";
@state() seed = randomSeed();
@state() inputUrl?: string;
@state() outputUrl?: string;
#lcmClient = new LcmClient(crypto.randomUUID());
#outputImageRef = createRef<HTMLImageElement>();
constructor () {
super();
this.#lcmClient.addEventListener("lcm-image-ready", event => {
console.log("lcm-image-ready", event.detail);
const prevUrl = this.outputUrl;
this.outputUrl = URL.createObjectURL(event.detail);
if (prevUrl)
URL.revokeObjectURL(prevUrl);
});
this.#lcmClient.addEventListener("lcm-image-url-ready", event => {
console.log("lcm-image-url-ready", event.detail);
this.outputUrl = event.detail;
this.requestUpdate();
});
}
protected override createRenderRoot = () => this;
override disconnectedCallback(): void {
if (this.imageCaptureUrl != null)
URL.revokeObjectURL(this.imageCaptureUrl);
if (this.inputUrl)
URL.revokeObjectURL(this.inputUrl);
if (this.outputUrl)
URL.revokeObjectURL(this.outputUrl);
super.disconnectedCallback();
}
protected override willUpdate(changes: PropertyValues<this>): void {
let updates: Partial<ImageInfo> = {};
if (changes.has("blob"))
updates.blob = this.blob;
if (changes.has("prompt"))
updates.prompt = this.prompt;
if (changes.has("negPrompt"))
updates.negativePrompt = this.negPrompt;
if (changes.has("seed"))
updates.seed = this.seed;
if (
changes.has("blob")
|| changes.has("prompt")
|| changes.has("negPrompt")
|| changes.has("seed")
) {
this.#lcmClient.update(updates);
}
super.willUpdate(changes);
}
@on("scene-state")
onSceneStateChange({ detail: state }: SceneStateEvent): void {
console.log("scene state change:", SceneState[state]);
@@ -52,12 +110,9 @@ export class AppElement extends LitElement {
}
@on("entity-changes")
@debounce(1000 / 24)
onStudioChanges(): void {
if (this.realtimeCapture) {
this.requestScreenshot_realtime();
} else {
this.requestScreenshot_delayed();
}
studio.requestScreenshot();
}
@on("viewport-resized")
@@ -67,27 +122,40 @@ export class AppElement extends LitElement {
studio.requestScreenshot();
}
@debounce(1000 / 24, { maxWait: 1000 / 5 })
requestScreenshot_realtime() {
studio.requestScreenshot();
}
@debounce(1000 / 24)
requestScreenshot_delayed() {
studio.requestScreenshot();
}
@on("screenshot-ready")
onScreenshotReady({ detail: screenshot }: ScreenshotReadyEvent) {
const prevUrl = this.imageCaptureUrl;
this.blob = new Blob([screenshot.data.buffer], { type: "image/jpg" });
const blob = new Blob([screenshot.data.buffer], { type: "image/png" });
this.imageCaptureUrl = URL.createObjectURL(blob);
const prevUrl = this.inputUrl;
this.inputUrl = URL.createObjectURL(this.blob);
if (prevUrl != null)
URL.revokeObjectURL(prevUrl);
}
updatePrompt(event: Event): void {
this.prompt = (event.target as HTMLInputElement).value;
}
updateNegPrompt(event: Event): void {
this.negPrompt = (event.target as HTMLInputElement).value;
}
refreshSeed(): void {
this.seed = randomSeed();
}
stopTextInputBubbling(event: KeyboardEvent): void {
// We want modifier keys like "Alt" to always be received by the Bevy
// canvas because they affect the camera controller behavior, but we do
// NOT want Bevy to receive typing inputs like W, A, S, and D while the
// user is trying to type into the Prompt field.
if (/^(Alt|Control)$/.test(event.key))
return;
event.stopPropagation();
}
onResizeStart(): void {
this.resizing = true;
}
@@ -101,15 +169,6 @@ export class AppElement extends LitElement {
this.hierarchyWidth = Math.max(Math.min(updated, 512), 128);
}
toggleRealtimeCapture(event: InputEvent): void {
const { checked } = event.target as HTMLInputElement;
this.realtimeCapture = checked;
}
refreshSeed(): void {
this.seed = randomSeed();
}
protected override render = () => html`
<sts-canvas-provider class="app-root">
<div
@@ -145,45 +204,81 @@ export class AppElement extends LitElement {
></sts-transform-toolbar>
</sts-studio>
${this.imageCaptureUrl != null ? html`
${this.inputUrl || this.outputUrl ? html`
<img
${ref(this.#outputImageRef)}
class="image-capture"
src=${this.imageCaptureUrl}
src=${(this.outputUrl ?? this.inputUrl)!}
alt="Image capture"
/>
` : html`
<div class="image-capture placeholder"></div>
`}
<div class="config config-prompt h-center v-end"
@keydown=${(event: KeyboardEvent) => event.stopPropagation()}
@keyup=${(event: KeyboardEvent) => event.stopPropagation()}
<div class="config config-prompts h-center v-end"
@keydown=${this.stopTextInputBubbling}
@keyup=${this.stopTextInputBubbling}
>
<input
class="config-field prompt"
type="text"
aria-label="Prompt"
placeholder="Enter a prompt to customize the output"
/>
<div class="config-field-group">
<div class="config-field-group prompt">
<sts-icon class="pre-icon green" icon="check"></sts-icon>
<input
class="config-field seed"
type="number"
aria-label="Seed"
value=${this.seed}
min="100"
max="10000"
class="config-field prompt"
type="text"
aria-label="Affirmative Prompt"
placeholder="Enter an affirmative prompt to customize the output"
.value=${this.prompt}
@change=${this.updatePrompt}
@keydown=${(event: KeyboardEvent) => {
if (event.key === "Enter") {
event.preventDefault();
this.updatePrompt(event);
}
}}
/>
<button
class="seed-refresh"
@click=${this.refreshSeed}
>
<sts-icon icon="rotate"></sts-icon>
</button>
</div>
<div class="config-row">
<div class="config-field-group prompt">
<sts-icon class="pre-icon red" icon="xmark"></sts-icon>
<input
class="config-field prompt"
type="text"
aria-label="Negative Prompt"
placeholder="Enter a negative prompt to customize the output"
.value=${this.negPrompt}
@change=${this.updateNegPrompt}
@keydown=${(event: KeyboardEvent) => {
if (event.key === "Enter") {
event.preventDefault();
this.updateNegPrompt(event);
}
}}
/>
</div>
<div class="config-field-group">
<input
class="config-field seed"
type="number"
aria-label="Seed"
.value=${this.seed.toString()}
min="1"
max="99999"
/>
<button
class="seed-refresh"
@click=${this.refreshSeed}
>
<sts-icon icon="rotate"></sts-icon>
</button>
</div>
</div>
</div>
</div>
${/*
FIXME: Real-time capture is not really feasible until we can come up
with a more elegant way to exclude the world grid, gizmos, etc.
from the captured image
<div class="config v-start h-end">
<label class="config-checkbox">
<input
@@ -194,6 +289,8 @@ export class AppElement extends LitElement {
Realtime Capture
</label>
</div>
*/
nothing}
${this.loading ? html`
<!-- TODO: Animated spinner or something -- talk to our designers -->
@@ -206,5 +303,5 @@ export class AppElement extends LitElement {
}
function randomSeed(): number {
return Math.round(Math.random() * 5000);
return Math.round(Math.random() * 99999);
}