Add WIP video style preview dialog (#115)

Co-authored-by: Victor <victor@storyteller.ai>
This commit is contained in:
Danny McGee
2024-03-29 22:33:44 -04:00
committed by GitHub
parent 3dba6cb284
commit fbc914fd7b
19 changed files with 677 additions and 153 deletions
@@ -0,0 +1,23 @@
// import GetApiHost from "./GetApiHost";
// const { formatUrl, host = "" } = GetApiHost();
const MakeMultipartRequest = (endpoint = "", body: any) => {
const formData = new FormData();
Object.keys(body).forEach((key) => formData.append(key, body[key]));
// formData.append('source', "video");
return fetch("https://funnel.tailce84f.ts.net/preview/", {
method: 'POST',
credentials: 'include',
headers: {
'Accept': 'application/json',
},
body: formData,
})
.then(res => res.blob());
}
export default MakeMultipartRequest;
+39 -16
View File
@@ -8,8 +8,8 @@ interface RouteSetup<UrlRouteArgs> {
method: string;
multipart?: boolean;
routingFunction: UrlRoutingFunction<UrlRouteArgs>;
}
}
const METHOD_OMITS_BODY: { [key: string]: boolean } = {
DELETE: false,
GET: true,
@@ -19,25 +19,50 @@ const METHOD_OMITS_BODY: { [key: string]: boolean } = {
PUT: false,
};
const MakeRequest = <UrlRouteArgs, Request, Response, UrlParams>(routeSetup: RouteSetup<UrlRouteArgs>) : (urlRouteArgs: UrlRouteArgs, request: Request, queries?: UrlParams) => Promise<Response> => {
return async function(urlRouteArgs: UrlRouteArgs, request: Request, queries?: any) : Promise<Response> {
const newQueries = queries ? Object.keys(queries).map((key, i) => {
return `${ i ? "&" : "" }${ key }=${ Array.isArray(queries[key]) ? queries[key].join(`&${ key }=`) : queries[key] }`
}).join("") : null;
export default <
UrlRouteArgs,
Request,
Response,
UrlParams extends Record<string, unknown>,
>(routeSetup: RouteSetup<UrlRouteArgs>) => {
return async function (
urlRouteArgs: UrlRouteArgs,
request: Request,
queries?: UrlParams,
overrideHost?: string
): Promise<Response> {
const newQueries = queries
? Object
.keys(queries)
.map((key, i) =>
`${ i ? "&" : "" }${ key }=${
Array.isArray(queries[key])
? (queries[key] as unknown[]).join(`&${ key }=`)
: queries[key]
}`
)
.join("")
: null;
const endpoint = `${ routeSetup.routingFunction(urlRouteArgs) }${ newQueries ? "?" + newQueries : "" }`;
const endpoint = `${
routeSetup.routingFunction(urlRouteArgs)
}${
newQueries ? "?" + newQueries : ""
}`;
const method = routeSetup.method;
const methodOmitsBody = METHOD_OMITS_BODY[method] || false;
console.log("🛼",request)
return fetch(`https://api.fakeyou.com${ endpoint }`, {
method,
headers: {
"Accept": "application/json",
...methodOmitsBody? {} : { "Content-Type": "application/json" }
return fetch(
`${ overrideHost ? overrideHost : "https://api.fakeyou.com" }${ endpoint }`,
{
method,
headers: {
"Accept": "application/json",
...methodOmitsBody? {} : { "Content-Type": "application/json" }
},
credentials: 'include',
credentials: "include",
...methodOmitsBody ? {} : { body: JSON.stringify(request) },
})
.then(res => res.json())
@@ -52,5 +77,3 @@ const MakeRequest = <UrlRouteArgs, Request, Response, UrlParams>(routeSetup: Rou
.catch(e => ({ success : false }));
}
};
export default MakeRequest;
@@ -0,0 +1,10 @@
import MakeMultipartRequest from "./MakeMultipartRequest";
export interface VideoStylePreviewRequest {
video: File,
request: string
}
export const VideoStylePreview = (request: VideoStylePreviewRequest) => {
return MakeMultipartRequest("",request);
}
+18 -5
View File
@@ -37,6 +37,7 @@ import "@storyteller/studio-web/studio";
import "@storyteller/studio-web/transform-toolbar";
import "./anim-editor.element";
import "./help.element";
import "./temp-video-preview.element";
import "./app.element.scss";
import { mock_save_load } from "./util";
@@ -60,6 +61,7 @@ export class AppElement extends LitElement {
private api = new StorytellerApi();
#addToSceneButtonRef = createRef<ButtonElement>();
#videoStylePreviewBtnRef = createRef<ButtonElement>();
#mediaLibraryRef = createRef<DialogElement>();
#inspectorRef = createRef<HTMLDivElement>();
#hierarchyRef = createRef<SceneHierarchyElement>();
@@ -161,7 +163,7 @@ export class AppElement extends LitElement {
<sts-scene-data-provider>
<sts-overlay-provider>
<sts-iframe-messaging-bridge></sts-iframe-messaging-bridge>
${this.#params.mode === StudioMode.Editor ? html`
<sts-add-to-scene-menu
.trigger=${this.#addToSceneButtonRef}
@@ -220,6 +222,12 @@ export class AppElement extends LitElement {
class="add-to-scene-dropdown"
>
</sts-button>
<sts-button
${ref(this.#videoStylePreviewBtnRef)}
icon="films"
class="video-style-preview-dropdown"
>
</sts-button>
<sts-transform-toolbar></sts-transform-toolbar>
</div>
@@ -234,7 +242,7 @@ export class AppElement extends LitElement {
<div class="sidebar-header">
${this.sidebars.map(({title, name}) => {
return html`<sts-button class=${classMap({"current-tab": name === this.currentSidebar, "sidebar-tab": true})}>${title}</sts-button>`
})}
})}
</div>
${this.renderSidebar()}
@@ -252,7 +260,7 @@ export class AppElement extends LitElement {
<div class="entity-inspector">
<sts-entity-inspector></sts-entity-inspector>
</div>
<sts-anim-editor
class="timeline"
style=${styleMap({
@@ -260,14 +268,19 @@ export class AppElement extends LitElement {
})}
></sts-anim-editor>
` : nothing}
<sts-temp-video-preview
.trigger=${ this.#videoStylePreviewBtnRef }>
</sts-temp-video-preview>
<!-- Dialogs and modals -->
${this.showHelp ? html`
<div class="help">
<sts-help></sts-help>
</div>
` : nothing}
</div>
</div>
${this.loading ? html`
@@ -1,36 +1,57 @@
.studio-bubble-select {
@use "sass:math";
:host {
// background: red;
display: flex;
list-style: none;
margin-block-start: 0px;
margin-block-end: 0px;
padding-inline-start: 0px;
gap: 0.6rem;
box-sizing: border-box;
}
li {
flex: 0 0 auto;
border: 2px solid darken(purple, 3%);
padding: 8px 16px;
border-radius: 0.5rem;
background-color: darken(purple, 3%);
cursor: pointer;
outline: none;
transition: all 0.2s ease;
font-size: 15px;
font-weight: 500;
color: rgba(#fff, 0.75);
white-space: nowrap;
*, *::before, *::after {
box-sizing: border-box;
}
&.selected-bubble {
color: #fff;
border-color: red;
background-color: lighten(purple, 6%);
&.fill {
background-color: red;
}
}
.bubble {
position: relative;
flex: 0 0 auto;
display: flex;
align-items: flex-end;
width: 80px;
height: 80px;
padding: 0.5rem;
border-radius: 0.5rem;
background: {
size: cover;
position: center center;
}
outline: #FFF2 solid 2px;
color: rgba(#fff, 0.75);
transition: {
property: background-color, outline-color;
duration: 200ms;
timing-function: ease;
}
white-space: nowrap;
cursor: pointer;
}
&.selected {
outline-color: var(--color-accent);
}
.gradient {
position: absolute;
inset: 0;
background: linear-gradient(to top, #000B, #0000);
border-radius: 0.5rem;
}
.label {
position: relative;
font: {
size: math.div(12, 16) * 1rem;
weight: 500;
}
}
}
@@ -1,28 +1,54 @@
import { LitElement, html, nothing, unsafeCSS } from "lit";
import { customElement, property, state } from "lit/decorators.js";
import { FormFieldBaseElement } from "@storyteller/studio-ui/forms/form-field-base";
import { PropertyValues, html, unsafeCSS } from "lit";
import { customElement, property } from "lit/decorators.js";
import { classMap } from "lit/directives/class-map.js";
import { styleMap } from "lit/directives/style-map.js";
import { type StyleOption, type VstStyle } from "./inspector-tabs/style-tab";
import styles from "./bubble-select.element.scss?inline";
// const abc = "word";
@customElement("sts-bubble-select")
export class BubbleSelect extends LitElement {
export class BubbleSelect extends FormFieldBaseElement {
static override styles = unsafeCSS(styles);
@property() name = "";
@property() onChange = (e: any) => {};
@property() options = [];
@property() value = "";
@property({ reflect: true })
override role = "listbox"
@property({ attribute: false })
options: readonly StyleOption[] = [];
@property() value?: VstStyle;
protected override update(changes: PropertyValues<this>): void {
if (changes.has("value"))
this.internals.setFormValue(this.value ?? "");
super.update(changes);
}
protected override render = () => html`
<ul class="studio-bubble-select">
${ this.options.map(({ label, value: itemValue },i) =>
html`<li
class=${ itemValue === this.value ? "selected-bubble" : "" }
@click=${ ({ target }: any) => {
this.onChange({ target: { name: this.name, value: itemValue } });
} }
>${ label }</li>`
)}
</ul>
${ this.options.map(({ label, value, imageUrl }) => html`
<div
class=${classMap({
"bubble": true,
"selected": value === this.value,
})}
style=${styleMap({
"background-image": `url(${imageUrl})`,
})}
@click=${() => {
// this.onChange({ target: { name: this.name, value: itemValue } });
this.dispatchEvent(new CustomEvent("value-change", {
detail: value,
}))
} }
>
<div class="gradient"></div>
<span class="label">
${ label }
</span>
</div>`
)}
`
}
}
@@ -1,41 +1,56 @@
import { LitElement, html, nothing, unsafeCSS } from "lit";
import { bind, on } from "@storyteller/framework";
import { LitElement, TemplateResult, html, unsafeCSS } from "lit";
import { customElement, property, state } from "lit/decorators.js";
import styles from "./style-tab.scss?inline";
import { EnqueueEngineCompositing } from "../api/EnqueueEngineCompositing";
import { v4 as uuid } from "uuid";
import { drag, on, provide } from "@storyteller/framework";
import { EnqueueEngineCompositing } from "../api/EnqueueEngineCompositing";
import styles from "./style-tab.scss?inline";
export type VstStyle
= "anime_2d_flat"
| "cartoon_3d"
| "ink_bw_style"
| "paper_origami"
;
export interface StyleOption {
label: string;
imageUrl: string;
value: VstStyle;
}
export const STYLE_OPTIONS: readonly StyleOption[] = [
{
label: "2D Anime",
imageUrl: "https://fakeyou.com/images/landing/onboarding/styles/style-2d-anime.webp",
value: "anime_2d_flat",
},
{
label: "3D Cartoon",
imageUrl: "https://fakeyou.com/images/landing/onboarding/styles/style-3d-cartoon.webp",
value: "cartoon_3d",
},
{
label: "Ink B&W",
imageUrl: "https://fakeyou.com/images/landing/onboarding/styles/style-ink-bw.webp",
value: "ink_bw_style",
},
{
label: "Origami",
imageUrl: "https://fakeyou.com/images/landing/onboarding/styles/style-origami.webp",
value: "paper_origami",
},
];
@customElement("sts-style-tab")
export class StyleTab extends LitElement {
static override styles = unsafeCSS(styles);
@state() vstStyle = "anime_2d_flat";
@state() vstStyle: VstStyle = "anime_2d_flat";
@state() posPrompt = "";
@state() negPrompt = "";
@property() sceneToken = "";
styleOptions = [
{
label: "2D Anime",
imageUrl: "/images/landing/onboarding/styles/style-2d-anime.webp",
value: "anime_2d_flat",
},
{
label: "3D Cartoon",
imageUrl: "/images/landing/onboarding/styles/style-3d-cartoon.webp",
value: "cartoon_3d",
},
{
label: "Ink B&W",
imageUrl: "/images/landing/onboarding/styles/style-ink-bw.webp",
value: "ink_bw_style",
},
{
label: "Origami",
imageUrl: "/images/landing/onboarding/styles/style-origami.webp",
value: "paper_origami",
},
];
inputChange = ({ target }: { target: { name: string, value: any } }) => {
// console.log("💁🏼‍♀️", this[target.name]);
// this[target.name] = target.value; // fix later
@@ -69,32 +84,36 @@ export class StyleTab extends LitElement {
console.log("🏎️", this.sceneToken);
}
protected override render = () => html`<section class="inspector-input-set">
${ this.vstStyle }
<button @click=${ this.thingy }>State</button>
<sts-bubble-select
.onChange=${ ({ target }: any) => { this.vstStyle = target.value; } }
.options=${this.styleOptions}
.value=${this.vstStyle}
></sts-bubble-select>
<label htmlFor="inspector-posPrompt">Positive Prompt</label>
<textarea
id="inspector-posPrompt"
name="posPrompt"
@input=${ this.inputChange }
.value=${ this.posPrompt }
></textarea>
<label htmlFor="inspector-posPrompt">Negative Prompt</label>
<textarea
id="inspector-negPrompt"
name="negPrompt"
@input=${ this.inputChange }
.value=${ this.negPrompt }
></textarea>
<sts-button
@click=${this.compositorEnqueue(this.sceneToken)}
>
Generate your movie
</sts-button>
</section>`;
}
protected override render = (): TemplateResult => html`
<section class="inspector-input-set">
${ this.vstStyle }
<button @click=${ this.thingy }>State</button>
<sts-bubble-select
.options=${STYLE_OPTIONS}
.value=${bind(this, "vstStyle")}
></sts-bubble-select>
<label htmlFor="inspector-posPrompt">Positive Prompt</label>
<textarea
id="inspector-posPrompt"
name="posPrompt"
@input=${ this.inputChange }
.value=${ this.posPrompt }
></textarea>
<label htmlFor="inspector-posPrompt">Negative Prompt</label>
<textarea
id="inspector-negPrompt"
name="negPrompt"
@input=${ this.inputChange }
.value=${ this.negPrompt }
></textarea>
<sts-button
@click=${this.compositorEnqueue(this.sceneToken)}
>
Generate your movie
</sts-button>
</section>
`;
}
@@ -1,6 +0,0 @@
.studio-inspector {
color: #fff;
background: blue;
position: absolute;
top: 50%;
}
@@ -0,0 +1,109 @@
@use "sass:math";
.video-style-preview {
width: calc(100vw - 3rem);
height: calc(100vh - 3rem);
&::part(content) {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 2rem;
}
&__side-by-side {
display: flex;
width: 100%;
}
&__panel {
flex: 0 0 50%;
display: flex;
flex-direction: column;
align-items: center;
gap: 1rem;
}
&__heading {
margin: 0;
font: {
family: "Fira Sans";
size: 1rem;
weight: 500;
}
}
&__image {
position: relative;
display: flex;
align-items: center;
justify-content: center;
width: 100%;
height: 0;
padding-top: calc((536 / 960) * 100%);
background: {
size: cover;
position: center center;
}
* {
position: absolute;
top: 50%;
left: 50%;
transform: translateX(-50%) translateY(-50%);
}
}
&__prompts {
display: flex;
gap: 1rem;
}
&__form-field {
display: flex;
flex-direction: column;
gap: 0.75rem;
&--base-style {
align-items: center;
}
}
&__label {
font: {
family: "Fira Sans";
size: math.div(15, 16) * 1rem;
weight: 500;
}
}
&__input {
box-sizing: border-box;
display: flex;
align-items: center;
width: 100%;
padding: 0.5rem 0.75rem;
border-radius: 0.5rem;
gap: 0.75rem;
background: #0004;
border: 2px solid #FFF2;
color: #FFFC;
transition: {
property: background-color, border-color, color;
duration: 33.333ms;
timing-function: linear;
}
font: {
family: "Fira Sans", sans-serif;
size: math.div(15, 16) * 1rem;
}
&:focus {
outline: none;
border-color: var(--color-accent);
}
}
}
@@ -0,0 +1,269 @@
import * as studio from "@storyteller/studio";
import { LitElement, type PropertyValues, html, nothing, unsafeCSS, TemplateResult } from "lit";
import { type Ref, createRef, ref } from "lit/directives/ref.js";
import { customElement, property, state } from "lit/decorators.js";
import { ifDefined } from "lit/directives/if-defined.js";
import { styleMap } from "lit/directives/style-map.js";
import { type ScreenshotReadyEvent } from "@storyteller/studio";
import { DialogElement } from "@storyteller/studio-ui/dialog";
import { bind, on } from "@storyteller/framework";
import { VideoStylePreview } from "./api/VideoStylePreview";
import "./bubble-select.element";
import styles from "./temp-video-preview.element.scss?inline";
import { STYLE_OPTIONS, type VstStyle } from "./inspector-tabs/style-tab";
enum InspectorMode {
Properties,
Style
}
@customElement("sts-temp-video-preview")
export class TempVideoPreview extends LitElement {
@property({ attribute: false })
trigger?: Ref<HTMLElement>;
@state() sourceBlob?: Blob;
@state() resultBlob?: Blob;
@state() sourceImageUrl?: string;
@state() resultImageUrl?: string;
@state() selectedStyle: VstStyle = "anime_2d_flat";
@state() prompt = "";
@state() negPrompt = "";
#temporaryVideoPreviewRef = createRef<DialogElement>();
protected override willUpdate(changes: PropertyValues<this>): void {
if (changes.has("trigger")) {
const prevTrigger = changes.get("trigger");
prevTrigger?.value?.removeEventListener("click", this.onDialogOpen);
this.trigger?.value?.addEventListener("click", this.onDialogOpen);
}
super.willUpdate(changes)
}
override disconnectedCallback(): void {
this.trigger?.value?.removeEventListener("click", this.onDialogOpen);
if (this.sourceImageUrl) {
URL.revokeObjectURL(this.sourceImageUrl);
this.sourceImageUrl = undefined;
}
if (this.resultImageUrl) {
URL.revokeObjectURL(this.resultImageUrl);
this.resultImageUrl = undefined;
}
super.disconnectedCallback();
}
requestScreen = () => {
studio.requestScreenshot();
console.log("🟢");
}
onDialogOpen = () => {
studio.requestScreenshot();
}
@on("window:screenshot-ready")
async onScreenshotReady (event: ScreenshotReadyEvent) {
console.log("🟣",event.detail.data);
const file = new File(
[event.detail.data],
"studio-scene-screen.png",
{ type: "image/png" },
);
this.sourceBlob = file;
const prevSourceUrl = this.sourceImageUrl;
this.sourceImageUrl = URL.createObjectURL(this.sourceBlob);
if (prevSourceUrl)
URL.revokeObjectURL(prevSourceUrl);
this.resultImageUrl = undefined;
try {
this.resultBlob = await VideoStylePreview({
video: file,
request:
JSON.stringify({
"style": this.selectedStyle,
"positive_prompt": this.prompt,
"negative_prompt": this.negPrompt,
})
});
console.log("🟡", this.resultBlob);
}
catch (err) {
// TODO: Show error message
console.error(err);
return;
}
const prevResultUrl = this.resultImageUrl;
this.resultImageUrl = URL.createObjectURL(this.resultBlob);
if (prevResultUrl)
URL.revokeObjectURL(prevResultUrl);
}
stopTextInputBubbling(event: KeyboardEvent): void {
// We want modifier keys like "Alt" to always be received by the Bevy
// canvas because they affect the camera controller behavior, but we do
// NOT want Bevy to receive typing inputs like W, A, S, and D while the
// user is trying to type into the Prompt field.
if (/^(Alt|Control)$/.test(event.key))
return;
event.stopPropagation();
}
protected override render = (): TemplateResult => html`
<sts-dialog
${ref(this.#temporaryVideoPreviewRef)}
skipWrapper
.trigger=${ this.trigger ?? [] }
.template=${[this, html`
<sts-dialog-base
class="video-style-preview"
@close=${() => {
this.#temporaryVideoPreviewRef.value?.close();
}}
@keydown=${this.stopTextInputBubbling}
@keyup=${this.stopTextInputBubbling}
>
<sts-dialog-header icon="films">
Video Style Preview
</sts-dialog-header>
<style>${unsafeCSS(styles)}</style>
<div class="video-style-preview__side-by-side">
<div class="video-style-preview__panel">
<h2 class="video-style-preview__heading">
Raw Preview
</h2>
<div
class="
video-style-preview__image
video-style-preview__image--source"
style=${styleMap({
"background-image": this.sourceImageUrl
? `url(${this.sourceImageUrl})`
: undefined,
})}
src=${ifDefined(this.sourceImageUrl)}
>
${!this.sourceImageUrl ? html`
<h2 class="video-style-preview__heading">
Loading...
</h2>
` : nothing}
</div>
</div>
<div class="video-style-preview__panel">
<h2 class="video-style-preview__heading">
Styled Preview
</h2>
<div
class="
video-style-preview__image
video-style-preview__image--result"
style=${styleMap({
"background-image": this.resultImageUrl
? `url(${this.resultImageUrl})`
: undefined,
})}
>
${!this.resultImageUrl ? html`
<h2 class="video-style-preview__heading">
Loading...
</h2>
` : nothing}
</div>
</div>
</div>
<div class="
video-style-preview__form-field
video-style-preview__form-field--base-style"
>
<label
class="video-style-preview__label"
for="video-style-preview-base-style"
>
Select Base Style
</label>
<sts-bubble-select
id="video-style-preview-base-style"
.options=${STYLE_OPTIONS}
.value=${bind(this, "selectedStyle")}
></sts-bubble-select>
</div>
<div class="video-style-preview__prompts">
<div class="video-style-preview__form-field">
<label
class="video-style-preview__label"
for="video-style-preview-prompt"
>
Enter a Prompt
</label>
<textarea
id="video-style-preview-prompt"
class="video-style-preview__input"
name="prompt"
cols="60"
rows="8"
.value=${this.prompt}
@input=${(event: InputEvent) => {
this.prompt = (event.target as HTMLTextAreaElement).value;
}}
></textarea>
</div>
<div class="video-style-preview__form-field">
<label
class="video-style-preview__label"
for="video-style-preview-neg-prompt"
>
Negative Prompt
</label>
<textarea
id="video-style-preview-neg-prompt"
class="video-style-preview__input"
name="neg-prompt"
cols="60"
rows="8"
placeholder=${"Type here to filter out the things you don't "
+ "want in your scene..."}
.value=${this.negPrompt}
@input=${(event: InputEvent) => {
this.negPrompt = (event.target as HTMLTextAreaElement).value;
}}
></textarea>
</div>
</div>
<sts-dialog-footer>
<sts-button
primary
icon="rotate"
@click=${this.requestScreen}
>
Update
</sts-button>
</sts-dialog-footer>
</sts-dialog-base>
`] as const}
></sts-dialog>
`
}