scene enqueues to engine compositor

package.json

@@ -26,6 +26,7 @@
     "@vitest/ui": "~0.32.0",
     "chalk": "^5.3.0",
     "glob": "^10.3.10",
+    "idb": "8.0.0",
     "jsdom": "~22.1.0",
     "nx": "16.9.1",
     "prettier": "^2.6.2",
@@ -39,8 +40,7 @@
     "vite-plugin-wasm": "^3.2.2",
     "vite-plugin-web-components-hmr": "^0.1.3",
     "vitest": "~0.32.0",
-    "which": "^4.0.0",
-    "idb": "8.0.0"
+    "which": "^4.0.0"
   },
   "dependencies": {
     "@fortawesome/fontawesome-pro": "^6.4.2",
@@ -49,6 +49,7 @@
     "lodash": "^4.17.21",
     "react": "18.2.0",
     "react-dom": "18.2.0",
-    "tslib": "^2.3.0"
+    "tslib": "^2.3.0",
+    "uuid": "^9.0.1"
   }
 }

studio-frontend/src/app/api/EngineCompositer.ts

@@ -1,6 +1,6 @@
 import MakeRequest from "./MakeRequest";
 
-export interface EnqueueEngineCompositingRequest {
+export interface EngineCompositerRequest {
   uuid_idempotency_token: string,
   media_file_token: string,
   camera?: string,
@@ -8,12 +8,12 @@ export interface EnqueueEngineCompositingRequest {
   skybox?: string,
 }
 
-export interface EnqueueEngineCompositingResponse {
+export interface EngineCompositerResponse {
   success: boolean,
   inference_job_token?: string
 }
 
-export const EnqueueEngineCompositing = MakeRequest<string, EnqueueEngineCompositingRequest, EnqueueEngineCompositingResponse,{}>({
+export const EngineCompositer = MakeRequest<string, EngineCompositerRequest, EngineCompositerResponse,{}>({
   method: "POST",
   routingFunction: () => `/v1/conversion/enqueue_render_engine_scene`,
 });

studio-frontend/src/app/app.element.ts

@@ -40,10 +40,16 @@ import "@storyteller/studio-web/studio";
 import "@storyteller/studio-web/transform-toolbar";
 import "./anim-editor.element";
 import "./help.element";
-import "./temp-video-preview.element";
+import "./job-status.element";
+import "./style-workspace.element";
 
 import "./app.element.scss";
 
+export enum WorkspaceMode {
+	Stage,
+	Style
+}
+
 @customElement("app-root")
 @provide({
 	token: StorytellerApi,
@@ -60,8 +66,12 @@ export class AppElement extends LitElement {
 
 	@state() currentSidebar : string = 'hierarchy';
 
+	@state() workspace : WorkspaceMode = WorkspaceMode.Stage;
+
 	@property({ attribute: false }) api: StorytellerApi =  new StorytellerApi(mockApi ? new StorytellerLocalApi() : new StorytellerRemoteApi());
 
+	@state() sceneToken?: string;
+
 	#addToSceneButtonRef = createRef<ButtonElement>();
 	#videoStylePreviewBtnRef = createRef<ButtonElement>();
 	#mediaLibraryRef = createRef<DialogElement>();
@@ -81,6 +91,8 @@ export class AppElement extends LitElement {
 			
 			const token = await this.api.createScene(file);
 
+			this.sceneToken = token;
+
 			this.dispatchEvent(new CustomEvent("scene-uploaded", {
 				detail: token,
 				bubbles: true,
@@ -131,6 +143,21 @@ export class AppElement extends LitElement {
 		return html`<></>`;
 	}
 
+	switchStage(name, debug) {
+		console.log(`🎚️ from: ${ debug }`, name, WorkspaceMode.Stage);
+		return (event) => {
+			switch(name) {
+				case WorkspaceMode.Stage: 
+					this.workspace = WorkspaceMode.Stage;
+					break;
+				case WorkspaceMode.Style: 
+					this.workspace = WorkspaceMode.Style;
+					break;
+			}
+		}
+	}
+
+
 	protected override render() {
 		return html`
 		<sts-canvas-provider class="app-root">
@@ -154,38 +181,21 @@ export class AppElement extends LitElement {
 
 		<div class="wrapper" style=${styleMap({ "--sidebar-width": `${this.inspectorWidth}px` })}>
 			<!-- Scene view -->
-			<sts-studio
-				class=${classMap({
-					"studio-scene": true,
-					"block-pointer-events": this.resizingPanel,
-				})}
-				.objectId=${this.#params.objectId}
-				.bvhPath=${this.#params.bvh}
-				.mixamoPath=${this.#params.mixamo}
-				.sceneImport=${this.#params.sceneImport}
-				.storytellerScene=${this.#params.scene}
-				.skyboxId=${this.#params.skybox ?? "test_scene"}
-				.mode=${this.#params.mode}
-			></sts-studio>
 
-			<!-- Actions toolbar -->
-			<sts-toolbar class="action-toolbar">
-				<sts-toolbar-group>
-					${this.#params.mode === StudioMode.Editor ? html`
-						<sts-button
-							@click=${() => studio.saveScene()}
-						>
-							Save Scene
-						</sts-button>
-					` : nothing}
-					<sts-button
-						class=${classMap({ active: this.showHelp })}
-						@click=${this.toggleHelp}
-					>
-						Help
-					</sts-button>
-				</sts-toolbar-group>
-			</sts-toolbar>
+			${ this.workspace === WorkspaceMode.Stage ?
+				html`<sts-studio
+					class=${classMap({
+						"studio-scene": true,
+						"block-pointer-events": this.resizingPanel,
+					})}
+					.objectId=${this.#params.objectId}
+					.bvhPath=${this.#params.bvh}
+					.mixamoPath=${this.#params.mixamo}
+					.sceneImport=${this.#params.sceneImport}
+					.storytellerScene=${this.#params.scene}
+					.skyboxId=${this.#params.skybox ?? "test_scene"}
+					.mode=${this.#params.mode}
+				></sts-studio>
 
 			<!-- Studio-mode-only elements -->
 			${this.#params.mode === StudioMode.Editor ? html`
@@ -198,6 +208,8 @@ export class AppElement extends LitElement {
 						</sts-button>
 						<sts-button
 							${ref(this.#videoStylePreviewBtnRef)}
+							name="Style"
+							@click=${ this.switchStage(WorkspaceMode.Style) }
 							icon="films"
 							class="video-style-preview-dropdown"
 						>
@@ -232,22 +244,55 @@ export class AppElement extends LitElement {
 					</section>
 
 					<div class="entity-inspector">
-						<sts-entity-inspector></sts-entity-inspector>
+						<sts-entity-inspector
+						></sts-entity-inspector>
 					</div>
 
-					<sts-anim-editor
-						class="timeline"
-						style=${styleMap({
-					right: `${this.inspectorWidth + 32}px`
-				})}
-					></sts-anim-editor>
 			` : nothing}
 
 
+				` : html`<sts-style-workspace
+						.trigger=${ this.#videoStylePreviewBtnRef }
+						.sceneToken=${ this.sceneToken }
+						.switchStage=${ this.switchStage }
+				></sts-style-workspace>`
+			}
+			<!-- Actions toolbar -->
+			<sts-toolbar class="action-toolbar">
+				<sts-toolbar-group>
+					${this.#params.mode === StudioMode.Editor ? html`
+						<sts-button
+							@click=${() => studio.saveScene()}
+						>
+							Save Scene
+						</sts-button>
+					` : nothing}
+					<sts-button
+						class=${classMap({ active: this.showHelp })}
+						@click=${this.toggleHelp}
+					>
+						Help
+					</sts-button>
+				</sts-toolbar-group>
+			</sts-toolbar>
+
+
+			<sts-anim-editor
+				class="timeline"
+				style=${styleMap({
+					right: `${this.inspectorWidth + 32}px`
+				})}
+			></sts-anim-editor>
+
+			
 			<sts-temp-video-preview
 				.trigger=${ this.#videoStylePreviewBtnRef }>
 			</sts-temp-video-preview>
 
+
+			<sts-job-status>
+			</sts-job-status>
+
 			<!-- Dialogs and modals -->
 			${this.showHelp ? html`
 			<div class="help">

studio-frontend/src/app/inspector-tabs/style-tab.ts

@@ -3,7 +3,7 @@ import { LitElement, TemplateResult, html, unsafeCSS } from "lit";
 import { customElement, property, state } from "lit/decorators.js";
 import { v4 as uuid } from "uuid";
 
-import { EnqueueEngineCompositing } from "../api/EnqueueEngineCompositing";
+import { EngineCompositer } from "../api/EngineCompositer";
 
 import styles from "./style-tab.scss?inline";
 
@@ -132,7 +132,7 @@ export class StyleTab extends LitElement {
 		oAThing(): void { console.log("🍕")};
 
 	compositorEnqueue = (sceneToken: string) => () => {
-		EnqueueEngineCompositing("",{
+		EngineCompositer("",{
 			uuid_idempotency_token: uuid(),
 			media_file_token: sceneToken,
 			camera: "zoom",

studio-frontend/src/app/job-status.element.ts

@@ -0,0 +1,39 @@
+import * as studio from "@storyteller/studio";
+import { LitElement, type PropertyValues, html, nothing, unsafeCSS, TemplateResult } from "lit";
+import { type Ref, createRef, ref } from "lit/directives/ref.js";
+import { customElement, property, state } from "lit/decorators.js";
+import { ifDefined } from "lit/directives/if-defined.js";
+import { styleMap } from "lit/directives/style-map.js";
+import { type ScreenshotReadyEvent } from "@storyteller/studio";
+import { DialogElement } from "@storyteller/studio-ui/dialog";
+import { bind, on } from "@storyteller/framework";
+
+import { VideoStylePreview } from "./api/VideoStylePreview";
+
+import "./style-select.element";
+
+import styles from "./temp-video-preview.element.scss?inline";
+import { STYLE_OPTIONS, type VstStyle } from "./inspector-tabs/style-tab";
+
+enum InspectorMode {
+	Properties,
+	Style
+}
+
+@customElement("sts-job-status")
+export class JobStatus extends LitElement {
+	#JobStatusRef = createRef<DialogElement>();
+
+	protected override render = (): TemplateResult => html`
+		<sts-dialog
+			${ref(this.#JobStatusRef)}
+			skipWrapper
+			.trigger=${ true }
+			.template=${[this, html`
+				<sts-dialog-base class="video-style-preview">
+					Hello world
+				</sts-dialog-base>
+			`] as const}
+		></sts-dialog>
+	`
+}

studio-frontend/src/app/style-workspace.element.scss

@@ -0,0 +1,176 @@
+@use "sass:math";
+
+:host {
+	position: absolute;
+	background: var(--color-page-background);
+	color: #fff;
+	width: 100%;
+	height: 100%;
+	padding: 50px 0; // replace with actual rem value eventually
+}
+
+.video-style-preview {
+
+	&::part(content) {
+		display: flex;
+		flex-direction: column;
+		align-items: center;
+		justify-content: center;
+		gap: 2rem;
+	}
+
+	&__page_container {
+		display: flex;
+		align-items: center;
+		justify-content: center;
+		flex-direction: column;
+		gap: 1rem;
+		width: 100%;
+		height: 100%;
+	}
+
+	&__previews {
+		display: flex;
+		justify-content: center;
+		align-items: center;
+		gap: 1rem;
+		width: 100%;
+	}
+
+	&__panel {
+		// flex: 0 0 50%;
+		flex-direction: column;
+		align-items: center;
+		gap: 1rem;
+		width: 45%;
+		overflow: hidden;
+		background: var(--color-panel-item-name-background);
+		border-radius: .5rem;
+	}
+
+	&__arrow {
+		height: 30px;
+		width: 30px;
+		opacity: .6;
+		// background: red;
+
+		line, polyline {
+			fill: none;
+			stroke: #fff;
+			stroke-width: 4;
+			stroke-linecap: round;
+		}
+	}
+
+	&__heading {
+		margin: 0;
+		font: {
+			family: "Fira Sans";
+			size: 1rem;
+			weight: 500;
+		}
+
+		&.preview-top-heading {
+			padding: .5rem;
+			text-align: center;
+			width: 100%;
+		}
+	}
+
+	&__image {
+		position: relative;
+		display: flex;
+		align-items: center;
+		justify-content: center;
+		width: 100%;
+		height: 0;
+		padding-top: calc((536 / 960) * 100%);
+		background: {
+			size: cover;
+			position: center center;
+		}
+
+		* {
+			position: absolute;
+			top: 50%;
+			left: 50%;
+			transform: translateX(-50%) translateY(-50%);
+		}
+	}
+
+	&__input-set {
+		pointer-events: all;
+		background: var(--color-toolbar-background);
+		border-radius: .5rem;
+		display: flex;
+		flex-direction: column;
+		max-width: calc(100% - 4rem);
+		padding: 1rem;
+		gap: 1rem;
+		overflow: hidden;
+	}
+
+	&__prompts {
+		display: flex;
+		justify-content: center;
+		gap: 1rem;
+	}
+
+	&__form-field {
+		display: flex;
+		flex-direction: column;
+		gap: 0.75rem;
+
+		&--base-style {
+			align-items: center;
+		}
+	}
+
+	&__label {
+		font: {
+			family: "Fira Sans";
+			size: math.div(15, 16) * 1rem;
+			weight: 500;
+		}
+	}
+
+	&__input {
+		pointer-events: all;
+		box-sizing: border-box;
+		display: flex;
+		align-items: center;
+		width: 100%;
+		padding: 0.5rem 0.75rem;
+		border-radius: 0.5rem;
+		gap: 0.75rem;
+
+		background: var(--color-panel-item-name-background);
+		border: none;
+		resize: none;
+		color: #FFFC;
+
+		font: {
+			family: "Fira Sans", sans-serif;
+			size: math.div(15, 16) * 1rem;
+		}
+
+		&:focus {
+			outline: none;
+			border-color: var(--color-accent);
+		}
+	}
+
+	&__buttons {
+		display: flex;
+		gap: 1rem;
+		justify-content: center;
+	}
+
+	&__go-back {
+		position: absolute;
+		top: 0;
+		background-color: var(--color-toolbar-background);
+		padding: var(--padding);
+		border-radius:  0 0 .5rem .5rem;
+	}
+}

studio-frontend/src/app/style-workspace.element.ts

@@ -0,0 +1,267 @@
+import * as studio from "@storyteller/studio";
+import { LitElement, type PropertyValues, html, nothing, unsafeCSS, TemplateResult } from "lit";
+import { type Ref, createRef, ref } from "lit/directives/ref.js";
+import { customElement, property, state } from "lit/decorators.js";
+import { ifDefined } from "lit/directives/if-defined.js";
+import { styleMap } from "lit/directives/style-map.js";
+import { type ScreenshotReadyEvent } from "@storyteller/studio";
+import { DialogElement } from "@storyteller/studio-ui/dialog";
+import { bind, on } from "@storyteller/framework";
+import { v4 as uuidv4 } from "uuid";
+import {
+	EngineCompositer,
+	EngineCompositerResponse
+} from "./api/EngineCompositer"
+import { VideoStylePreview } from "./api/VideoStylePreview";
+
+import styles from "./style-workspace.element.scss?inline";
+
+import "./bubble-select.element";
+import "@storyteller/studio-ui/button";
+import { STYLE_OPTIONS, type VstStyle } from "./inspector-tabs/style-tab";
+
+
+@customElement("sts-style-workspace")
+export class StyleWorkspace extends LitElement {
+	static override styles = unsafeCSS(styles);
+
+	@property({ attribute: false })
+	trigger?: Ref<HTMLElement>;
+
+	@state() sourceBlob?: Blob;
+	@state() resultBlob?: Blob;
+
+	@state() sourceImageUrl?: string;
+	@state() resultImageUrl?: string;
+
+	@state() selectedStyle: VstStyle = "anime_2d_flat";
+	@state() prompt = "";
+	@state() negPrompt = "";
+
+	@property() sceneToken = "";
+
+	#temporaryVideoPreviewRef = createRef<DialogElement>();
+
+	protected override willUpdate(changes: PropertyValues<this>): void {
+		if (changes.has("trigger")) {
+			const prevTrigger = changes.get("trigger");
+			prevTrigger?.value?.removeEventListener("click", this.onDialogOpen);
+
+			this.trigger?.value?.addEventListener("click", this.onDialogOpen);
+		}
+
+		super.willUpdate(changes)
+	}
+
+	override disconnectedCallback(): void {
+		this.trigger?.value?.removeEventListener("click", this.onDialogOpen);
+
+		if (this.sourceImageUrl) {
+			URL.revokeObjectURL(this.sourceImageUrl);
+			this.sourceImageUrl = undefined;
+		}
+
+		if (this.resultImageUrl) {
+			URL.revokeObjectURL(this.resultImageUrl);
+			this.resultImageUrl = undefined;
+		}
+
+		super.disconnectedCallback();
+	}
+
+	requestScreen = () => {
+		studio.requestScreenshot();
+		console.log("🟢");
+	}
+
+	onDialogOpen = () => {
+		studio.requestScreenshot();
+	}
+
+	@on("window:screenshot-ready")
+	async onScreenshotReady (event: ScreenshotReadyEvent) {
+		console.log("🟣",event.detail.data);
+		const file = new File(
+			[event.detail.data],
+			"studio-scene-screen.png",
+			{ type: "image/png" },
+		);
+		this.sourceBlob = file;
+
+		const prevSourceUrl = this.sourceImageUrl;
+		this.sourceImageUrl = URL.createObjectURL(this.sourceBlob);
+
+		if (prevSourceUrl)
+			URL.revokeObjectURL(prevSourceUrl);
+
+		this.resultImageUrl = undefined;
+
+		try {
+			this.resultBlob = await VideoStylePreview({
+				video: file,
+				request:
+					JSON.stringify({
+						"style": this.selectedStyle,
+						"positive_prompt": this.prompt,
+						"negative_prompt": this.negPrompt,
+					})
+			});
+			console.log("🟡", this.resultBlob);
+		}
+		catch (err) {
+			// TODO: Show error message
+			console.error(err);
+			return;
+		}
+
+		const prevResultUrl = this.resultImageUrl;
+		this.resultImageUrl = URL.createObjectURL(this.resultBlob);
+
+		if (prevResultUrl)
+			URL.revokeObjectURL(prevResultUrl);
+	}
+
+	stopTextInputBubbling(event: KeyboardEvent): void {
+		// We want modifier keys like "Alt" to always be received by the Bevy
+		// canvas because they affect the camera controller behavior, but we do
+		// NOT want Bevy to receive typing inputs like W, A, S, and D while the
+		// user is trying to type into the Prompt field.
+		if (/^(Alt|Control)$/.test(event.key))
+			return;
+
+		event.stopPropagation();
+	}
+
+	@on("window:scene-uploaded") getSceneToken({ detail }) {
+		console.log("👺", detail);
+		this.sceneToken = detail;
+	}
+
+	compositorEnqueue() {
+		console.log("🧑🏻‍🎤 scene token", this.sceneToken);
+
+		if (this.sceneToken) {
+			EngineCompositer("",{
+				uuid_idempotency_token: uuidv4(),
+				media_file_token: this.sceneToken,
+			}).then((res: EngineCompositerResponse) => {
+				console.log("👻",res);
+			}); 
+		}
+
+	}
+
+
+	protected override render = () => html`
+		<div class="video-style-preview__page_container">
+			<div class="video-style-preview__previews">
+				<div class="video-style-preview__panel">
+					<h2 class="video-style-preview__heading preview-top-heading">
+						Raw Preview
+					</h2>
+					<div
+						class="
+							video-style-preview__image
+							video-style-preview__image--source"
+						style=${styleMap({
+							"background-image": this.sourceImageUrl
+								? `url(${this.sourceImageUrl})`
+								: undefined,
+						})}
+						src=${ifDefined(this.sourceImageUrl)}
+					>
+						${!this.sourceImageUrl ? html`
+							<h2 class="video-style-preview__heading">
+								Loading...
+							</h2>
+						` : nothing}
+					</div>
+				</div>
+				<svg class="video-style-preview__arrow">
+					<line x1="2" y1="15" x2="28" y2="15"/>
+					<polyline points="18,4 28,15 18,26"/>
+				</svg>
+				<div class="video-style-preview__panel">
+					<h2 class="video-style-preview__heading preview-top-heading">
+						Styled Preview
+					</h2>
+					<div
+						class="
+							video-style-preview__image
+							video-style-preview__image--result"
+						style=${styleMap({
+							"background-image": this.resultImageUrl
+								? `url(${this.resultImageUrl})`
+								: undefined,
+						})}
+					>
+						${!this.resultImageUrl ? html`
+							<h2 class="video-style-preview__heading">
+								Loading...
+							</h2>
+						` : nothing}
+					</div>
+				</div>
+			</div>
+			<div class="video-style-preview__input-set">
+				<sts-bubble-select
+					.options=${STYLE_OPTIONS}
+					.value=${bind(this, "selectedStyle")}
+				></sts-bubble-select>
+				<div class="video-style-preview__prompts">
+					<div class="video-style-preview__form-field">
+						<label
+							class="video-style-preview__label"
+							for="video-style-preview-prompt"
+						>
+							Enter a Prompt
+						</label>
+						<textarea
+							id="video-style-preview-prompt"
+							class="video-style-preview__input"
+							name="prompt"
+							cols="60"
+							rows="8"
+							.value=${this.prompt}
+							@input=${(event: InputEvent) => {
+								this.prompt = (event.target as HTMLTextAreaElement).value;
+							}}
+						></textarea>
+					</div>
+					<div class="video-style-preview__form-field">
+						<label
+							class="video-style-preview__label"
+							for="video-style-preview-neg-prompt"
+						>
+							Negative Prompt
+						</label>
+						<textarea
+							id="video-style-preview-neg-prompt"
+							class="video-style-preview__input"
+							name="neg-prompt"
+							cols="60"
+							rows="8"
+							placeholder=${"Type here to filter out the things you don't "
+								+ "want in your scene..."}
+							.value=${this.negPrompt}
+							@input=${(event: InputEvent) => {
+								this.negPrompt = (event.target as HTMLTextAreaElement).value;
+							}}
+						></textarea>
+					</div>
+				</div>
+				<div class="video-style-preview__buttons">
+					<sts-button @click=${this.requestScreen}>
+						Update preview
+					</sts-button>
+					<sts-button primary @click=${this.compositorEnqueue}>
+						Generate movie
+					</sts-button>
+				</div>
+			</div>
+			<div class="video-style-preview__go-back">
+				<sts-button>Go back</sts-button>
+			</div>
+		</div>
+	`
+}

studio-frontend/src/styles.scss

@@ -8,8 +8,10 @@ html, body {
 	margin: 0;
 	overflow: hidden;
 
+	--color-page-background: #1a1a27;
 	--color-panel-background: #1e1e2c;
 	--color-toolbar-background: #2c2c3b;
+	--color-prompts-background: #4b4b5c;
 	--color-toolbar-background-light: #595970;	
 	--color-panel-item-name-background: #39394d;
 	--color-panel-item-background: #55556b;