Bootstrap Image Generator application (#53)

This PR begins scaffolding the Stable Diffusion Turbo application (title
TBD), which is a separate frontend for Storyteller Engine which focuses
on image generation.

### Design Overview

* The application renders a "split-screen" view, with Storyteller Studio
  on the left, and the Stable Diffusion Turbo output on the right
* When changes are made in the Studio editor:
  - A screenshot is taken from the Bevy application
  - Screenshot is piped into Stable Diffusion Turbo model
  - Model output is rendered to the screen
* This all happens in as close to real-time as possible

### Work completed in this PR

* The `studio-frontend` project has been (somewhat crudely) refactored
  to split most of its components into reusable libraries
  - `studio-ui` is a use-case-agnostic design system library for core UI
    components. These are "dumb" components which are controlled
    via `@property` inputs and send `CustomEvent`s in response to user
    input.
  - `studio-frontend` is a collection of reusable Storyteller Studio
    widgets. These are "smart" components which depend on
    `@storyteller/studio` directly, manage their own state, and require very
    little configuration or styling. They are designed to be drop-in
    building blocks that can be reused between various frontends.
* The new application has been bootstrapped as `image-generator-demo`
  - Currently, this simply grabs screenshots from the Bevy application and
    renders those screenshots on the right side of the split-screen view. A
    future PR will wire up to the Storyteller.ai backend to pipe these
    screenshots through Stable Diffusion Turbo, and render that model's
    output instead.

### Testing

* Install dependencies as described in the repo's README
* Run `npx nx serve image-generator-demo`
* Open [localhost:4200](http://localhost:4200) in your browser
  - Use the camera controller in the left side of the view to manipulate
    the Bevy scene
  - Verify the right side of the view updates with a fresh screenshot at a
    rate of ~5 fps
* Toggle the "Realtime Capture" checkbox
  - Verify the right side view no longer updates while the user is
    actively manipulating the editor
  - This should allow the app to run smoother on lower-spec hardware

---------

Co-authored-by: Danny McGee <dannymcgee@gmail.com>
This commit is contained in:
Brooks Palin
2024-02-26 15:13:07 -07:00
committed by GitHub
parent 7b06d48c5d
commit d51eec52c3
71 changed files with 1331 additions and 579 deletions
@@ -1,25 +0,0 @@
import fs from "fs";
import {NormalizedOutputOptions} from "rollup";
function copyRecursively(src: string, dst: string) {
if (fs.statSync(src).isDirectory()) {
// folder
if (!fs.existsSync(dst)) {
fs.mkdirSync(dst);
}
for (const file of fs.readdirSync(src)) {
copyRecursively(src + "/" + file, dst + "/" + file);
}
} else {
// file
fs.copyFileSync(src, dst);
}
}
export function buildMiddleWare(options: NormalizedOutputOptions, assets: string[] = []) {
const dst = options.dir;
if (!dst) throw new Error("options.dir is undefined");
for (const dir of assets) {
copyRecursively(dir, dst)
}
}
@@ -1,26 +0,0 @@
// FIXME: Oh my god, did Danny seriously just copy and paste this entire folder
// from `studio-frontend` instead of creating a separate project for it
// and importing it like a decent, law-abiding citizen? Wow, dude really
// needs to get his shit together.
// NOTE: This is an inline fork of this npm package:
// https://github.com/nguyenbatranvan/vite-multiple-assets
//
// The published version at time of writing doesn't include mime types for many
// of the asset types Bevy needs to load.
import {PluginOption, ViteDevServer} from "vite";
import {ServerMiddleWare} from "./server";
import {buildMiddleWare} from "./build";
export default function DynamicPublicDirectory(assets: string[]): PluginOption {
return {
configureServer(server: ViteDevServer) {
return ServerMiddleWare(server, assets)
},
async writeBundle(options) {
buildMiddleWare(options, assets)
},
name: "dynamic assets"
};
}
@@ -1,91 +0,0 @@
import fs from "fs";
import path from "path";
import { ViteDevServer } from "vite";
interface IPropsFile {
name: string;
files: string[];
}
function getFiles(dir, files_): string[] {
files_ = files_ || [];
const files = fs.readdirSync(dir);
for (const i in files) {
const name = dir + "/" + files[i];
if (fs.statSync(name).isDirectory()) {
getFiles(name, files_);
} else {
files_.push(name);
}
}
return files_;
}
const mimeTypes = {
".7z": "application/x-7z-compressed",
".bin": "application/octet-stream",
".css": "text/css",
".eot": "font/eot",
".gif": "image/gif",
".gltf": "model/gltf+json",
".gz": "application/gzip",
".hdr": "image/vnd.radiance",
".html": "text/html",
".ico": "image/x-icon",
".jpg": "image/jpeg",
".js": "text/javascript",
".json": "application/json",
".ktx": "image/ktx",
".ktx2": "image/ktx",
".meta": "text/plain",
".mjs": "text/javascript",
".mp3": "audio/mpeg",
".mp4": "video/mp4",
".otf": "font/otf",
".png": "image/png",
".rar": "application/x-rar-compressed",
".svg": "image/svg+xml",
".tiff": "image/tiff",
".ttf": "font/ttf",
".txt": "text/plain",
".wasm": "application/wasm",
".webm": "video/webm",
".webp": "image/webp",
".wgsl": "text/wgsl",
".woff": "font/woff",
".woff2": "font/woff2",
".xml": "text/xml",
".zip": "application/zip",
}
export function ServerMiddleWare(server: ViteDevServer, assets: string[] = []) {
if (!assets || !assets.length)
return;
const fileObject: IPropsFile[] = [];
for (let i = 0; i < assets.length; i++) {
const files = getFiles(path.join(process.cwd(), `/${assets[i]}`), []);
fileObject.push({
name: assets[i],
files
});
}
return () => {
server.middlewares.use(async (req, res, next) => {
for (let i = 0; i < fileObject.length; i++) {
const file = path.join(process.cwd(), `${fileObject[i].name}/${req.originalUrl}`);
if (fileObject[i].files.some(f => path.relative(f, file) === "")) {
const extension = file.substring(file.lastIndexOf("."));
res.setHeader("Cache-Control", "max-age=31536000, immutable");
res.setHeader("Content-Type", mimeTypes[extension]);
res.writeHead(200);
res.write(fs.readFileSync(path.join(process.cwd(), "/" + fileObject[i].name + req.originalUrl)));
res.end();
break;
}
}
next();
});
};
}
+1 -2
View File
@@ -3,8 +3,7 @@ import react from "@vitejs/plugin-react";
import { defineConfig } from "vite";
import topLevelAwait from "vite-plugin-top-level-await";
import wasm from "vite-plugin-wasm";
import mergeAssetsDirs from "./vite-plugins/multiple-assets";
import mergeAssetsDirs from "../vite-multiple-assets";
export default defineConfig({
cacheDir: "../node_modules/.vite/react-frontend",