storytold/storyteller-bevy
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-bevy.git synced 2026-10-17 00:13:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

kind of working end to end, but buggy

Browse Source
This commit is contained in:
Brandon Thomas
2024-02-27 04:40:22 -05:00
parent 62f9736f64
commit 4e6610b2e9
2 changed files with 135 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
image-generator-demo/public/link.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 436 KiB

image-generator-demo/src/app/app.element.ts
+135 -67
View File
@@ -76,11 +76,123 @@ export class AppElement extends LitElement {
@state() lcmLiveStatus = LCMLiveStatus.DISCONNECTED;
@state() websocket: WebSocket | null = null;
@state() blob: any;
@state() linkBlob: any;
params = new StudioURLParams();
viewer = createRef<ViewerElement>();
protected override createRenderRoot = () => this;
constructor() {
super();
console.log('construct');
let that = this;
// Load an image we can use later
fetch("/link.jpg")
.then(response => response.blob())
.then(blob => {
console.log("blob", blob);
that.linkBlob = blob;
});
// Set a dom element we can use to render.
const img = document.createElement("img");
img.setAttribute("id", "video_image")
document.body.appendChild(img);
}
connectWebsocket() {
const userId = crypto.randomUUID();
this.websocket = new WebSocket("wss://b3000.pancake-chimaera.ts.net/api/ws/" + userId);
this.websocket.onopen = () => {
console.log("Marking connection to websocket as open for user " + userId);
this.lcmLiveStatus = LCMLiveStatus.CONNECTED;
};
this.websocket.onclose = () => {
console.log("Marking connection to websocket as closed for user " + userId);
this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED;
};
this.websocket.onmessage = (event) => {
const data = JSON.parse(event.data);
switch (data.status) {
case "connected":
this.lcmLiveStatus = LCMLiveStatus.CONNECTED;
break;
case "send_frame":
this.lcmLiveStatus = LCMLiveStatus.SEND_FRAME;
const default_prompt = "Portrait of The Joker halloween costume, face painting, with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece"
const default_negative_prompt = "black and white, blurry, low resolution, pixelated, pixel art, low quality, low fidelity"
const imageInfo = new ImageInfo({
prompt: default_prompt,
negative_prompt: default_negative_prompt,
seed: 12345,
guidance_scale: 1.2,
});
//const streamData = blob;
//console.log("Sending next frame with streamData" + streamData + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState)
console.log("Sending next frame...");
this.websocket?.send(JSON.stringify({ status: "next_frame" }));
this.websocket?.send(JSON.stringify(imageInfo));
let blob = new Blob([], { type: "image/jpeg" });
if (this.blob) {
blob = this.blob;
}
//if (this.linkBlob) {
// blob = this.linkBlob;
//}
console.log("blob:", blob);
this.websocket?.send(blob);
// After the server gets the image, we need to load the pipeline result.
let src = `https://b3000.pancake-chimaera.ts.net/api/stream/${userId}`;
//let img = new Image();
//img.src = src;
console.log("set source", src);
//document.getElementById("video_image")?.setAttribute("src", src);
document.getElementsByClassName("image-capture")[0]?.setAttribute("src", src);
// for (const d of streamData) {
//innerSend(streamData);
// }
break;
case "wait":
this.lcmLiveStatus = LCMLiveStatus.WAIT;
break;
case "timeout":
console.log("timeout");
this.lcmLiveStatus = LCMLiveStatus.TIMEOUT;
break;
case "error":
console.log(data.message);
this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED;
// lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
// streamId.set(null);
// reject(new Error(data.message));
break;
}
};
}
override disconnectedCallback(): void {
if (this.imageCaptureUrl != null)
URL.revokeObjectURL(this.imageCaptureUrl);
@@ -110,81 +222,37 @@ export class AppElement extends LitElement {
const prevUrl = this.imageCaptureUrl;
const blob = new Blob([screenshot.data.buffer], { type: "image/png" });
const innerSend = (blob: Blob | { [key: string]: any }) : void => {
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
if (blob instanceof Blob) {
this.websocket.send(blob);
} else {
this.websocket.send(JSON.stringify(blob));
}
} else {
console.log("WebSocket not connected durign innerSend with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState + " and blob " + blob + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState);
}
};
console.log("onScreenshotReady", this.websocket?.readyState);
//const innerSend = (blob: Blob | { [key: string]: any }) : void => {
// if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
// if (blob instanceof Blob) {
// this.websocket.send(blob);
// } else {
// this.websocket.send(JSON.stringify(blob));
// }
// } else {
// console.log("WebSocket not connected durign innerSend with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState + " and blob " + blob + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState);
// }
//};
// if websocket is disconnected, connect
if (this.lcmLiveStatus === LCMLiveStatus.DISCONNECTED) {
const userId = crypto.randomUUID();
this.websocket = new WebSocket("wss://b3000.pancake-chimaera.ts.net/api/ws/" + userId);
this.websocket.onopen = () => {
console.log("Marking connection to websocket as open for user " + userId);
this.lcmLiveStatus = LCMLiveStatus.CONNECTED;
};
this.websocket.onclose = () => {
console.log("Marking connection to websocket as closed for user " + userId);
this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED;
};
this.websocket.onmessage = (event) => {
const data = JSON.parse(event.data);
switch (data.status) {
case "connected":
this.lcmLiveStatus = LCMLiveStatus.CONNECTED;
break;
case "send_frame":
this.lcmLiveStatus = LCMLiveStatus.SEND_FRAME;
const default_prompt = "Portrait of The Joker halloween costume, face painting, with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece"
const default_negative_prompt = "black and white, blurry, low resolution, pixelated, pixel art, low quality, low fidelity"
const imageInfo = new ImageInfo({
prompt: default_prompt,
negative_prompt: default_negative_prompt,
seed: 12345,
guidance_scale: 1.2,
});
const streamData = blob;
console.log("Sending next frame with streamData" + streamData + " and websocket " + this.websocket + " and websocket.readyState " + this.websocket?.readyState)
this.websocket?.send(JSON.stringify({ status: "next_frame" }));
// for (const d of streamData) {
innerSend(streamData);
// }
break;
case "wait":
this.lcmLiveStatus = LCMLiveStatus.WAIT;
break;
case "timeout":
console.log("timeout");
this.lcmLiveStatus = LCMLiveStatus.TIMEOUT;
break;
case "error":
console.log(data.message);
this.lcmLiveStatus = LCMLiveStatus.DISCONNECTED;
// lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
// streamId.set(null);
// reject(new Error(data.message));
break;
}
};
}
//if (this.lcmLiveStatus === LCMLiveStatus.DISCONNECTED) {
//}
if (this.lcmLiveStatus === LCMLiveStatus.CONNECTED) {
} else {
console.log("WebSocket not connected with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState);
}
//if (this.lcmLiveStatus === LCMLiveStatus.CONNECTED) {
//} else {
// console.log("WebSocket not connected with state" + this.lcmLiveStatus + " and readyState " + this.websocket?.readyState);
//}
this.imageCaptureUrl = URL.createObjectURL(blob);
console.log(`New blob URL: ${this.imageCaptureUrl}`);
this.blob = blob;
this.connectWebsocket();
if (prevUrl != null)
URL.revokeObjectURL(prevUrl);
}