diff --git a/image-generator-demo/public/link.jpg b/image-generator-demo/public/link.jpg new file mode 100644 index 0000000..00bca1f Binary files /dev/null and b/image-generator-demo/public/link.jpg differ diff --git a/image-generator-demo/src/app/app.element.ts b/image-generator-demo/src/app/app.element.ts index 96c0ae7..fb5e9a5 100644 --- a/image-generator-demo/src/app/app.element.ts +++ b/image-generator-demo/src/app/app.element.ts @@ -76,11 +76,123 @@ export class AppElement extends LitElement { @state() lcmLiveStatus = LCMLiveStatus.DISCONNECTED; @state() websocket: WebSocket | null = null; + @state() blob: any; + @state() linkBlob: any; + params = new StudioURLParams(); viewer = createRef(); + + protected override createRenderRoot = () => this; + constructor() { + super(); + console.log('construct'); + + let that = this; + + // Load an image we can use later + fetch("/link.jpg") + .then(response => response.blob()) + .then(blob => { + console.log("blob", blob); + that.linkBlob = blob; + }); + + + // Set a dom element we can use to render. + const img = document.createElement("img"); + img.setAttribute("id", "video_image") + document.body.appendChild(img); + + } + + + connectWebsocket() { + const userId = crypto.randomUUID(); + this.websocket = new WebSocket("wss://b3000.pancake-chimaera.ts.net/api/ws/" + userId); + this.websocket.onopen = () => { + console.log("Marking connection to websocket as open for user " + userId); + this.lcmLiveStatus = LCMLiveStatus.CONNECTED; + }; + this.websocket.onclose = () => { + console.log("Marking connection to websocket as closed for user " + userId); + this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED; + }; + this.websocket.onmessage = (event) => { + const data = JSON.parse(event.data); + switch (data.status) { + case "connected": + this.lcmLiveStatus = LCMLiveStatus.CONNECTED; + break; + case "send_frame": + this.lcmLiveStatus = LCMLiveStatus.SEND_FRAME; + + const default_prompt = "Portrait of The Joker halloween costume, face painting, with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece" + const default_negative_prompt = "black and white, blurry, low resolution, pixelated, pixel art, low quality, low fidelity" + + const imageInfo = new ImageInfo({ + prompt: default_prompt, + negative_prompt: default_negative_prompt, + seed: 12345, + guidance_scale: 1.2, + }); + + //const streamData = blob; + //console.log("Sending next frame with streamData" + streamData + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState) + + console.log("Sending next frame..."); + this.websocket?.send(JSON.stringify({ status: "next_frame" })); + + this.websocket?.send(JSON.stringify(imageInfo)); + + let blob = new Blob([], { type: "image/jpeg" }); + + if (this.blob) { + blob = this.blob; + } + //if (this.linkBlob) { + // blob = this.linkBlob; + //} + + console.log("blob:", blob); + this.websocket?.send(blob); + + + // After the server gets the image, we need to load the pipeline result. + let src = `https://b3000.pancake-chimaera.ts.net/api/stream/${userId}`; + + //let img = new Image(); + //img.src = src; + + console.log("set source", src); + //document.getElementById("video_image")?.setAttribute("src", src); + document.getElementsByClassName("image-capture")[0]?.setAttribute("src", src); + + + // for (const d of streamData) { + //innerSend(streamData); + // } + break; + case "wait": + this.lcmLiveStatus = LCMLiveStatus.WAIT; + break; + case "timeout": + console.log("timeout"); + this.lcmLiveStatus = LCMLiveStatus.TIMEOUT; + break; + case "error": + console.log(data.message); + this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED; + // lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED); + // streamId.set(null); + // reject(new Error(data.message)); + break; + } + }; + } + override disconnectedCallback(): void { if (this.imageCaptureUrl != null) URL.revokeObjectURL(this.imageCaptureUrl); @@ -110,81 +222,37 @@ export class AppElement extends LitElement { const prevUrl = this.imageCaptureUrl; const blob = new Blob([screenshot.data.buffer], { type: "image/png" }); - const innerSend = (blob: Blob | { [key: string]: any }) : void => { - if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { - if (blob instanceof Blob) { - this.websocket.send(blob); - } else { - this.websocket.send(JSON.stringify(blob)); - } - } else { - console.log("WebSocket not connected durign innerSend with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState + " and blob " + blob + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState); - } - }; + console.log("onScreenshotReady", this.websocket?.readyState); + + //const innerSend = (blob: Blob | { [key: string]: any }) : void => { + // if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { + // if (blob instanceof Blob) { + // this.websocket.send(blob); + // } else { + // this.websocket.send(JSON.stringify(blob)); + // } + // } else { + // console.log("WebSocket not connected durign innerSend with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState + " and blob " + blob + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState); + // } + //}; // if websocket is disconnected, connect - if (this.lcmLiveStatus === LCMLiveStatus.DISCONNECTED) { - const userId = crypto.randomUUID(); - this.websocket = new WebSocket("wss://b3000.pancake-chimaera.ts.net/api/ws/" + userId); - this.websocket.onopen = () => { - console.log("Marking connection to websocket as open for user " + userId); - this.lcmLiveStatus = LCMLiveStatus.CONNECTED; - }; - this.websocket.onclose = () => { - console.log("Marking connection to websocket as closed for user " + userId); - this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED; - }; - this.websocket.onmessage = (event) => { - const data = JSON.parse(event.data); - switch (data.status) { - case "connected": - this.lcmLiveStatus = LCMLiveStatus.CONNECTED; - break; - case "send_frame": - this.lcmLiveStatus = LCMLiveStatus.SEND_FRAME; - const default_prompt = "Portrait of The Joker halloween costume, face painting, with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece" - const default_negative_prompt = "black and white, blurry, low resolution, pixelated, pixel art, low quality, low fidelity" - const imageInfo = new ImageInfo({ - prompt: default_prompt, - negative_prompt: default_negative_prompt, - seed: 12345, - guidance_scale: 1.2, - }); - const streamData = blob; - console.log("Sending next frame with streamData" + streamData + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState) - this.websocket?.send(JSON.stringify({ status: "next_frame" })); - // for (const d of streamData) { - innerSend(streamData); - // } - break; - case "wait": - this.lcmLiveStatus = LCMLiveStatus.WAIT; - break; - case "timeout": - console.log("timeout"); - this.lcmLiveStatus = LCMLiveStatus.TIMEOUT; - break; - case "error": - console.log(data.message); - this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED; - // lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED); - // streamId.set(null); - // reject(new Error(data.message)); - break; - } - }; - } + //if (this.lcmLiveStatus === LCMLiveStatus.DISCONNECTED) { + //} - if (this.lcmLiveStatus === LCMLiveStatus.CONNECTED) { - - } else { - console.log("WebSocket not connected with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState); - } + //if (this.lcmLiveStatus === LCMLiveStatus.CONNECTED) { + //} else { + // console.log("WebSocket not connected with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState); + //} this.imageCaptureUrl = URL.createObjectURL(blob); console.log(`New blob URL: ${this.imageCaptureUrl}`); + this.blob = blob; + + this.connectWebsocket(); + if (prevUrl != null) URL.revokeObjectURL(prevUrl); }