diff --git a/image-generator-demo/src/app/app.element.ts b/image-generator-demo/src/app/app.element.ts index 7f0b042..96c0ae7 100644 --- a/image-generator-demo/src/app/app.element.ts +++ b/image-generator-demo/src/app/app.element.ts @@ -20,6 +20,13 @@ import "@storyteller/studio-web/viewer"; import "./app.element.scss" +export enum LCMLiveStatus { + CONNECTED = "connected", + DISCONNECTED = "disconnected", + WAIT = "wait", + SEND_FRAME = "send_frame", + TIMEOUT = "timeout", +} class Debounce { static readonly Wait = 1000 / 24; static readonly Realtime = { @@ -38,11 +45,36 @@ class Debounce { } } +interface IImageInfo { + prompt?: string; + negative_prompt?: string; + seed?: number; + guidance_scale?: number; +} + +// imageinfo class + +class ImageInfo { + prompt?: string; + negativePrompt?: string; + seed?: number; + guidanceScale?: number; + + constructor(data: IImageInfo) { + this.prompt = data.prompt; + this.negativePrompt = data.negative_prompt; + this.seed = data.seed; + this.guidanceScale = data.guidance_scale; + } +} + @customElement("app-root") export class AppElement extends LitElement { @state() loading = true; @state() imageCaptureUrl?: string; @state() realtimeCapture = true; + @state() lcmLiveStatus = LCMLiveStatus.DISCONNECTED; + @state() websocket: WebSocket | null = null; params = new StudioURLParams(); viewer = createRef(); @@ -76,8 +108,80 @@ export class AppElement extends LitElement { @on("screenshot-ready") onScreenshotReady({ detail: screenshot }: ScreenshotReadyEvent) { const prevUrl = this.imageCaptureUrl; - const blob = new Blob([screenshot.data.buffer], { type: "image/png" }); + + const innerSend = (blob: Blob | { [key: string]: any }) : void => { + if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { + if (blob instanceof Blob) { + this.websocket.send(blob); + } else { + this.websocket.send(JSON.stringify(blob)); + } + } else { + console.log("WebSocket not connected durign innerSend with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState + " and blob " + blob + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState); + } + }; + + // if websocket is disconnected, connect + if (this.lcmLiveStatus === LCMLiveStatus.DISCONNECTED) { + const userId = crypto.randomUUID(); + this.websocket = new WebSocket("wss://b3000.pancake-chimaera.ts.net/api/ws/" + userId); + this.websocket.onopen = () => { + console.log("Marking connection to websocket as open for user " + userId); + this.lcmLiveStatus = LCMLiveStatus.CONNECTED; + }; + this.websocket.onclose = () => { + console.log("Marking connection to websocket as closed for user " + userId); + this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED; + }; + this.websocket.onmessage = (event) => { + const data = JSON.parse(event.data); + switch (data.status) { + case "connected": + this.lcmLiveStatus = LCMLiveStatus.CONNECTED; + break; + case "send_frame": + this.lcmLiveStatus = LCMLiveStatus.SEND_FRAME; + const default_prompt = "Portrait of The Joker halloween costume, face painting, with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece" + const default_negative_prompt = "black and white, blurry, low resolution, pixelated, pixel art, low quality, low fidelity" + const imageInfo = new ImageInfo({ + prompt: default_prompt, + negative_prompt: default_negative_prompt, + seed: 12345, + guidance_scale: 1.2, + }); + const streamData = blob; + console.log("Sending next frame with streamData" + streamData + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState) + this.websocket?.send(JSON.stringify({ status: "next_frame" })); + // for (const d of streamData) { + innerSend(streamData); + // } + break; + case "wait": + this.lcmLiveStatus = LCMLiveStatus.WAIT; + break; + case "timeout": + console.log("timeout"); + this.lcmLiveStatus = LCMLiveStatus.TIMEOUT; + break; + case "error": + console.log(data.message); + this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED; + // lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED); + // streamId.set(null); + // reject(new Error(data.message)); + break; + } + }; + } + + + if (this.lcmLiveStatus === LCMLiveStatus.CONNECTED) { + + } else { + console.log("WebSocket not connected with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState); + } + this.imageCaptureUrl = URL.createObjectURL(blob); console.log(`New blob URL: ${this.imageCaptureUrl}`);