Bootstrap Image Generator application (#53)

This PR begins scaffolding the Stable Diffusion Turbo application (title
TBD), which is a separate frontend for Storyteller Engine which focuses
on image generation.

### Design Overview

* The application renders a "split-screen" view, with Storyteller Studio
  on the left, and the Stable Diffusion Turbo output on the right
* When changes are made in the Studio editor:
  - A screenshot is taken from the Bevy application
  - Screenshot is piped into Stable Diffusion Turbo model
  - Model output is rendered to the screen
* This all happens in as close to real-time as possible

### Work completed in this PR

* The `studio-frontend` project has been (somewhat crudely) refactored
  to split most of its components into reusable libraries
  - `studio-ui` is a use-case-agnostic design system library for core UI
    components. These are "dumb" components which are controlled
    via `@property` inputs and send `CustomEvent`s in response to user
    input.
  - `studio-frontend` is a collection of reusable Storyteller Studio
    widgets. These are "smart" components which depend on
    `@storyteller/studio` directly, manage their own state, and require very
    little configuration or styling. They are designed to be drop-in
    building blocks that can be reused between various frontends.
* The new application has been bootstrapped as `image-generator-demo`
  - Currently, this simply grabs screenshots from the Bevy application and
    renders those screenshots on the right side of the split-screen view. A
    future PR will wire up to the Storyteller.ai backend to pipe these
    screenshots through Stable Diffusion Turbo, and render that model's
    output instead.

### Testing

* Install dependencies as described in the repo's README
* Run `npx nx serve image-generator-demo`
* Open [localhost:4200](http://localhost:4200) in your browser
  - Use the camera controller in the left side of the view to manipulate
    the Bevy scene
  - Verify the right side of the view updates with a fresh screenshot at a
    rate of ~5 fps
* Toggle the "Realtime Capture" checkbox
  - Verify the right side view no longer updates while the user is
    actively manipulating the editor
  - This should allow the app to run smoother on lower-spec hardware

---------

Co-authored-by: Danny McGee <dannymcgee@gmail.com>
This commit is contained in:
Brooks Palin
2024-02-26 15:13:07 -07:00
committed by GitHub
parent 7b06d48c5d
commit d51eec52c3
71 changed files with 1331 additions and 579 deletions
+2 -1
View File
@@ -5,7 +5,8 @@
"*.tsx": "${capture}.module.scss, ${capture}.spec.tsx",
},
"rust-analyzer.cargo.features": [
"wasm"
"wasm",
"wasm_screenshot"
],
"files.associations": {
"*.meta": "ron"
+1 -1
View File
@@ -1,6 +1,6 @@
{
"space_editor_core::hotkeys::HotkeySet<space_editor_ui::tools::gizmo::GizmoHotkey>": "{\"space_editor_core::hotkeys::HotkeySet<space_editor_ui::tools::gizmo::GizmoHotkey>\":(bindings:{Rotate:[R],Translate:[G],Scale:[S],Multiple:[ShiftLeft],Clone:[AltLeft],Delete:[X]},name:\"GizmoHotkey\")}",
"space_undo::ChangeChainSettings": "{\"space_undo::ChangeChainSettings\":(max_change_chain_size:200)}",
"space_persistence::PersistenceSettings": "{\"space_persistence::PersistenceSettings\":(load_on_startup:true,save_on_close:true)}",
"space_editor_core::hotkeys::HotkeySet<space_editor_ui::tools::gizmo::GizmoHotkey>": "{\"space_editor_core::hotkeys::HotkeySet<space_editor_ui::tools::gizmo::GizmoHotkey>\":(bindings:{Rotate:[R],Translate:[G],Scale:[S],Multiple:[ShiftLeft],Clone:[AltLeft],Delete:[X]},name:\"GizmoHotkey\")}",
"space_editor_ui::settings::NewWindowSettings": "{\"space_editor_ui::settings::NewWindowSettings\":(new_tab:SameNode)}",
}
+8
View File
@@ -0,0 +1,8 @@
{
"jsc": {
"parser": {
"syntax": "typescript"
},
"target": "ES2020"
}
}
+19
View File
@@ -0,0 +1,19 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>Image Generator Demo</title>
<base href="/" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
<link rel="stylesheet" href="/src/styles.scss" />
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Fira+Sans:wght@400;500;700&display=swap" rel="stylesheet">
</head>
<body>
<app-root></app-root>
<script type="module" src="/src/main.ts"></script>
</body>
</html>
+71
View File
@@ -0,0 +1,71 @@
{
"name": "image-generator-demo",
"$schema": "../node_modules/nx/schemas/project-schema.json",
"projectType": "application",
"sourceRoot": "image-generator-demo/src",
"tags": [],
"targets": {
"build": {
"executor": "@nx/vite:build",
"outputs": [
"{options.outputPath}"
],
"defaultConfiguration": "production",
"options": {
"outputPath": "dist/image-generator-demo"
},
"configurations": {
"development": {
"mode": "development"
},
"production": {
"mode": "production"
}
}
},
"serve": {
"executor": "@nx/vite:dev-server",
"defaultConfiguration": "development",
"dependsOn": [
{
"projects": ["studio"],
"target": "build:image-generator"
}
],
"options": {
"buildTarget": "image-generator-demo:build"
},
"configurations": {
"development": {
"buildTarget": "image-generator-demo:build:development",
"hmr": true
},
"production": {
"buildTarget": "image-generator-demo:build:production",
"hmr": false
}
}
},
"preview": {
"executor": "@nx/vite:preview-server",
"defaultConfiguration": "production",
"dependsOn": [
{
"projects": ["studio"],
"target": "build:image-generator"
}
],
"options": {
"buildTarget": "image-generator-demo:build"
},
"configurations": {
"development": {
"buildTarget": "image-generator-demo:build:development"
},
"production": {
"buildTarget": "image-generator-demo:build:production"
}
}
}
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

@@ -0,0 +1,72 @@
.app-root {
position: fixed;
inset: 0;
display: flex;
}
// Layout utilities
.v-start, .v-end, .h-start, .h-end, .h-center {
position: absolute;
}
.v-start { top: 1rem; }
.v-end { bottom: 1rem; }
.h-start { left: 1rem; }
.h-end { right: 1rem; }
.h-center {
left: 50%;
transform: translateX(-50%);
}
.studio-scene,
.image-capture {
flex: 0 0 50%;
}
.hierarchy {
width: 256px;
max-height: calc(100vh - 2rem);
}
.config {
display: flex;
flex-direction: column;
gap: 0.25em;
padding: 0.5rem 1rem;
background: #0006;
backdrop-filter: blur(16px);
border-radius: 0.5rem;
.checkbox {
display: flex;
align-items: center;
gap: 0.5rem;
height: 1.5rem;
color: #FFFC;
transition: color 0.1s linear;
font: {
family: "Fira Sans", sans-serif;
size: 15px;
weight: 500;
}
&:hover {
color: #FFFF;
}
}
}
.loading {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
background: #222;
color: #FFF;
font: {
family: "Fira Sans", sans-serif;
weight: 700;
}
}
+135
View File
@@ -0,0 +1,135 @@
import { on } from "@storyteller/framework";
import * as studio from "@storyteller/studio";
import {
SceneState,
type SceneStateEvent,
type ScreenshotReadyEvent,
StudioMode,
} from "@storyteller/studio";
import { StudioURLParams, ViewerElement } from "@storyteller/studio-web";
import { match } from "@storyteller/utility";
import { LitElement, html, nothing } from "lit";
import { customElement, state } from "lit/decorators.js";
import { createRef, ref } from "lit/directives/ref.js";
import { DebounceSettings, debounce } from "lodash";
import "@storyteller/studio-web/canvas-provider";
import "@storyteller/studio-web/scene-hierarchy";
import "@storyteller/studio-web/transform-toolbar";
import "@storyteller/studio-web/viewer";
import "./app.element.scss"
class Debounce {
static readonly Wait = 1000 / 24;
static readonly Realtime = {
leading: false,
trailing: true,
maxWait: 1000 / 5,
};
static readonly Delayed = {
leading: false,
trailing: true,
};
static func(config: DebounceSettings) {
return debounce(() => studio.requestScreenshot(), this.Wait, config);
}
}
@customElement("app-root")
export class AppElement extends LitElement {
@state() loading = true;
@state() imageCaptureUrl?: string;
@state() realtimeCapture = true;
params = new StudioURLParams();
viewer = createRef<ViewerElement>();
protected override createRenderRoot = () => this;
override disconnectedCallback(): void {
if (this.imageCaptureUrl != null)
URL.revokeObjectURL(this.imageCaptureUrl);
super.disconnectedCallback();
}
@on("scene-state")
onSceneStateChange({ detail: state }: SceneStateEvent): void {
console.log("scene state change:", SceneState[state]);
this.loading = match (state, {
[SceneState.Active]: () => false,
_: () => true,
});
}
@on("entity-changes")
onStudioChanges(): void {
this.requestScreenshot();
}
requestScreenshot = Debounce.func(Debounce.Realtime);
@on("screenshot-ready")
onScreenshotReady({ detail: screenshot }: ScreenshotReadyEvent) {
const prevUrl = this.imageCaptureUrl;
const blob = new Blob([screenshot.data.buffer], { type: "image/png" });
this.imageCaptureUrl = URL.createObjectURL(blob);
console.log(`New blob URL: ${this.imageCaptureUrl}`);
if (prevUrl != null)
URL.revokeObjectURL(prevUrl);
}
toggleRealtimeCapture(event: InputEvent): void {
const { checked } = event.target as HTMLInputElement;
this.realtimeCapture = checked;
const config = checked ? Debounce.Realtime : Debounce.Delayed;
this.requestScreenshot = Debounce.func(config);
}
protected override render = () => html`
<sts-canvas-provider class="app-root">
<sts-viewer
${ref(this.viewer)}
class="studio-scene"
objectId="base-human-female.gltf"
skyboxId="test_scene"
.mode=${StudioMode.Editor}
></sts-viewer>
${this.imageCaptureUrl != null ? html`
<img
class="image-capture"
src=${this.imageCaptureUrl}
alt="Image capture"
/>
` : nothing}
<sts-scene-hierarchy class="hierarchy h-start v-start"></sts-scene-hierarchy>
<sts-transform-toolbar class="h-center v-start"></sts-transform-toolbar>
<div class="config v-start h-end">
<label class="checkbox">
<input
type="checkbox"
?checked=${this.realtimeCapture}
@input=${this.toggleRealtimeCapture}
/>
Realtime Capture
</label>
</div>
${this.loading ? html`
<!-- TODO: Animated spinner or something -- talk to our designers -->
<div class="loading">
<h1>Loading</h1>
</div>
` : nothing}
</sts-canvas-provider>
`
}
+1
View File
@@ -0,0 +1 @@
import "./app/app.element";
+10
View File
@@ -0,0 +1,10 @@
*, *::before, *::after {
box-sizing: border-box;
}
html, body {
width: 100%;
height: 100%;
margin: 0;
overflow: hidden;
}
+15
View File
@@ -0,0 +1,15 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"outDir": "../dist/out-tsc",
"types": ["node"]
},
"exclude": [
"src/**/*.spec.ts",
"src/**/*.test.ts"
],
"include": [
"src/**/*.ts",
"typings/*.d.ts"
]
}
+34
View File
@@ -0,0 +1,34 @@
{
"extends": "../tsconfig.base.json",
"files": [],
"compilerOptions": {
"target": "ES2020",
"experimentalDecorators": true,
"useDefineForClassFields": false,
"module": "ESNext",
"lib": ["ES2020", "DOM", "DOM.Iterable"],
"skipLibCheck": true,
"moduleResolution": "Bundler",
"allowImportingTsExtensions": true,
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"esModuleInterop": true,
"strict": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"noImplicitAny": true,
"noImplicitOverride": true,
"noImplicitReturns": true,
"types": ["vite/client"]
},
"include": ["src"],
"references": [
{
"path": "./tsconfig.app.json"
}
]
}
+4
View File
@@ -0,0 +1,4 @@
declare module "*.scss?inline" {
const styles: import("lit").CSSResult;
export default styles;
}
+4
View File
@@ -0,0 +1,4 @@
declare module "*.svg?raw" {
const svg: string;
export default svg;
}
+29
View File
@@ -0,0 +1,29 @@
/// <reference types='vitest' />
import { defineConfig } from "vite";
import { nxViteTsPaths } from "@nx/vite/plugins/nx-tsconfig-paths.plugin";
import wasm from "vite-plugin-wasm";
import topLevelAwait from "vite-plugin-top-level-await";
import mergeAssetsDirs from "../vite-multiple-assets";
export default defineConfig({
cacheDir: "../node_modules/.vite/image-generator-demo",
server:{
port: 4200,
host: "localhost",
},
preview:{
port: 4300,
host: "localhost",
},
plugins: [
nxViteTsPaths(),
wasm(),
topLevelAwait(),
mergeAssetsDirs([
"image-generator-demo/public",
"dist/studio/public",
]),
],
});
@@ -1,25 +0,0 @@
import fs from "fs";
import {NormalizedOutputOptions} from "rollup";
function copyRecursively(src: string, dst: string) {
if (fs.statSync(src).isDirectory()) {
// folder
if (!fs.existsSync(dst)) {
fs.mkdirSync(dst);
}
for (const file of fs.readdirSync(src)) {
copyRecursively(src + "/" + file, dst + "/" + file);
}
} else {
// file
fs.copyFileSync(src, dst);
}
}
export function buildMiddleWare(options: NormalizedOutputOptions, assets: string[] = []) {
const dst = options.dir;
if (!dst) throw new Error("options.dir is undefined");
for (const dir of assets) {
copyRecursively(dir, dst)
}
}
@@ -1,26 +0,0 @@
// FIXME: Oh my god, did Danny seriously just copy and paste this entire folder
// from `studio-frontend` instead of creating a separate project for it
// and importing it like a decent, law-abiding citizen? Wow, dude really
// needs to get his shit together.
// NOTE: This is an inline fork of this npm package:
// https://github.com/nguyenbatranvan/vite-multiple-assets
//
// The published version at time of writing doesn't include mime types for many
// of the asset types Bevy needs to load.
import {PluginOption, ViteDevServer} from "vite";
import {ServerMiddleWare} from "./server";
import {buildMiddleWare} from "./build";
export default function DynamicPublicDirectory(assets: string[]): PluginOption {
return {
configureServer(server: ViteDevServer) {
return ServerMiddleWare(server, assets)
},
async writeBundle(options) {
buildMiddleWare(options, assets)
},
name: "dynamic assets"
};
}
@@ -1,91 +0,0 @@
import fs from "fs";
import path from "path";
import { ViteDevServer } from "vite";
interface IPropsFile {
name: string;
files: string[];
}
function getFiles(dir, files_): string[] {
files_ = files_ || [];
const files = fs.readdirSync(dir);
for (const i in files) {
const name = dir + "/" + files[i];
if (fs.statSync(name).isDirectory()) {
getFiles(name, files_);
} else {
files_.push(name);
}
}
return files_;
}
const mimeTypes = {
".7z": "application/x-7z-compressed",
".bin": "application/octet-stream",
".css": "text/css",
".eot": "font/eot",
".gif": "image/gif",
".gltf": "model/gltf+json",
".gz": "application/gzip",
".hdr": "image/vnd.radiance",
".html": "text/html",
".ico": "image/x-icon",
".jpg": "image/jpeg",
".js": "text/javascript",
".json": "application/json",
".ktx": "image/ktx",
".ktx2": "image/ktx",
".meta": "text/plain",
".mjs": "text/javascript",
".mp3": "audio/mpeg",
".mp4": "video/mp4",
".otf": "font/otf",
".png": "image/png",
".rar": "application/x-rar-compressed",
".svg": "image/svg+xml",
".tiff": "image/tiff",
".ttf": "font/ttf",
".txt": "text/plain",
".wasm": "application/wasm",
".webm": "video/webm",
".webp": "image/webp",
".wgsl": "text/wgsl",
".woff": "font/woff",
".woff2": "font/woff2",
".xml": "text/xml",
".zip": "application/zip",
}
export function ServerMiddleWare(server: ViteDevServer, assets: string[] = []) {
if (!assets || !assets.length)
return;
const fileObject: IPropsFile[] = [];
for (let i = 0; i < assets.length; i++) {
const files = getFiles(path.join(process.cwd(), `/${assets[i]}`), []);
fileObject.push({
name: assets[i],
files
});
}
return () => {
server.middlewares.use(async (req, res, next) => {
for (let i = 0; i < fileObject.length; i++) {
const file = path.join(process.cwd(), `${fileObject[i].name}/${req.originalUrl}`);
if (fileObject[i].files.some(f => path.relative(f, file) === "")) {
const extension = file.substring(file.lastIndexOf("."));
res.setHeader("Cache-Control", "max-age=31536000, immutable");
res.setHeader("Content-Type", mimeTypes[extension]);
res.writeHead(200);
res.write(fs.readFileSync(path.join(process.cwd(), "/" + fileObject[i].name + req.originalUrl)));
res.end();
break;
}
}
next();
});
};
}
+1 -2
View File
@@ -3,8 +3,7 @@ import react from "@vitejs/plugin-react";
import { defineConfig } from "vite";
import topLevelAwait from "vite-plugin-top-level-await";
import wasm from "vite-plugin-wasm";
import mergeAssetsDirs from "./vite-plugins/multiple-assets";
import mergeAssetsDirs from "../vite-multiple-assets";
export default defineConfig({
cacheDir: "../node_modules/.vite/react-frontend",
@@ -5,10 +5,10 @@ import { LitElement, type PropertyValues, html, unsafeCSS } from "lit";
import { customElement, state } from "lit/decorators.js";
import { styleMap } from "lit/directives/style-map.js";
import "@storyteller/studio-ui/button";
import "@storyteller/studio-ui/toolbar";
import "@storyteller/studio-ui/tree";
import "./anim-timeline.element";
import "./button.element";
import "./toolbar.element";
import "./tree.element";
import styles from "./anim-editor.element.scss?inline";
-24
View File
@@ -35,33 +35,9 @@ sts-button.active {
// Hierarchy (TODO: Probably move this)
.hierarchy {
overflow: auto;
box-sizing: border-box;
width: $hierarchy-width;
max-height: calc(100vh - 32px);
padding: 10px 15px;
background: #0006;
backdrop-filter: blur(16px);
border: 8px solid transparent;
border-radius: 0.5rem;
pointer-events: auto;
&::-webkit-scrollbar {
width: 8px;
height: 8px;
&-thumb {
border-radius: 2px;
background: #FFF2;
&:hover {
background: #FFF3;
}
}
}
}
.loading {
+31 -215
View File
@@ -1,17 +1,12 @@
import { on } from "@storyteller/framework";
import * as studio from "@storyteller/studio";
import {
type EntityDespawnEvent,
type EntityMultiSpawnEvent,
type EntitySelectEvent,
type EntitySpawnEvent,
type SceneSavedEvent,
type SceneStateEvent,
TransformSpace,
TransformType,
SceneState,
StudioMode,
} from "@storyteller/studio";
import { StudioURLParams } from "@storyteller/studio-web";
import { CanvasProvider } from "@storyteller/studio-web/canvas-provider";
import { match } from "@storyteller/utility";
import { LitElement, html, nothing } from "lit";
@@ -20,144 +15,35 @@ import { classMap } from "lit-html/directives/class-map.js";
import { Ref, createRef, ref } from "lit-html/directives/ref.js";
import { v4 as uuid } from "uuid";
import "@storyteller/studio-ui/button";
import "@storyteller/studio-ui/icon";
import "@storyteller/studio-ui/toolbar";
import "@storyteller/studio-ui/tree";
import "@storyteller/studio-web/canvas-provider";
import "@storyteller/studio-web/viewer";
import "@storyteller/studio-web/bvh-viewer";
import "@storyteller/studio-web/editor";
import "@storyteller/studio-web/mixamo-viewer";
import "@storyteller/studio-web/scene-hierarchy";
import "@storyteller/studio-web/transform-toolbar";
import "./anim-editor.element";
import "./button.element";
import "./help.element";
import "./icon.element";
import "./toolbar.element";
import "./transform-toolbar.element";
import "./tree.element";
import "./app.element.scss";
interface SceneObject extends Omit<studio.SceneObject, "children"> {
children: SceneObject[];
}
@customElement("app-root")
export class AppElement extends LitElement {
@state() showHelp = false;
@state() loading = true;
@state() transformSpace = TransformSpace.Local;
@state() transformType = TransformType.Translate;
@state() sceneObjects: SceneObject[] = [];
@state() selection?: string | null;
@state() bvhPath?: string;
@state() mixamoPath?: string;
@state() scenePath?: string;
@state() mode?: StudioMode;
#sceneObjects: studio.SceneObject[] = [];
#entityMap = new Map<string, SceneObject>();
#params = new StudioURLParams();
#canvasProvider: Ref<CanvasProvider> = createRef();
get #studioCanvas(): HTMLCanvasElement | null {
const provider = this.#canvasProvider.value;
if (!provider) return null;
return document.getElementById(provider.canvasId) as HTMLCanvasElement | null;
}
override connectedCallback(): void {
super.connectedCallback();
const params = new URLSearchParams(document.location.search);
if (params.has("bvh"))
this.bvhPath = params.get("bvh")!;
if (params.has("mixamo"))
this.mixamoPath = params.get("mixamo")!;
if (params.has("scene"))
this.scenePath = params.get("scene")!;
this.mode = match (params.get("mode"), {
"studio": () => StudioMode.Editor,
"viewer": () => StudioMode.Viewer,
_: () => StudioMode.Viewer,
});
this.#redirectBodyFocus();
}
protected override createRenderRoot(): Element | ShadowRoot {
return this;
}
@on("keydown")
@on("keyup")
pipeKeyboardEvents(event: KeyboardEvent): void {
// When the user interacts with one of the web UI elements, the Bevy
// canvas will lose focus, preventing it from receiving keyboard events.
// As a workaround, we can listen for any events that have bubbled up to
// the app root and repeat them for the canvas.
//
// This does unfortunately burden the web front-end with some additional
// responsibility: when we handle a keyboard event and _don't_ want the
// Bevy app to receive it, we'll need to call `event.stopPropagation()` in
// the handler to prevent it from reaching this listener.
const {
type,
code, charCode, key, keyCode, location, repeat,
altKey, ctrlKey, metaKey, shiftKey,
} = event;
this.#studioCanvas?.dispatchEvent(new KeyboardEvent(type, {
code, charCode, key, keyCode, location, repeat,
altKey, ctrlKey, metaKey, shiftKey,
bubbles: true,
cancelable: true,
composed: true,
}));
}
onSceneStateChange({ detail: state }: SceneStateEvent): void {
console.log("scene state change:", SceneState[state]);
this.loading = match (state, {
[SceneState.Active]: () => false,
_: () => true,
});
}
@on("window:entity-select")
onEntitySelectionChange({ detail: entity }: EntitySelectEvent): void {
const sceneObject = this.#entityMap.get(entity!);
console.log("Entity selection changed:", sceneObject);
this.selection = entity;
}
@on("window:entity-spawn")
onEntitySpawned({ detail: value }: EntitySpawnEvent): void {
console.log("Entity spawned:", value);
this.#sceneObjects = this.#sceneObjects.concat(value).sort(sortByName);
this.#updateHierarchy();
}
@on("window:entity-multi-spawn")
onEntitiesSpawned({ detail: values }: EntityMultiSpawnEvent): void {
console.log("Entities spawned:", values);
this.#sceneObjects = this.#sceneObjects.concat(values).sort(sortByName);
this.#updateHierarchy();
}
@on("window:entity-despawn")
onEntityDespawned({ detail: entity }: EntityDespawnEvent): void {
console.log("Entity despawned:", entity);
this.#sceneObjects = this.#sceneObjects.filter(({ id }) => id !== entity);
this.#updateHierarchy();
}
@on("window:scene-saved")
@on("scene-saved")
async onSceneSaved({ detail: scene }: SceneSavedEvent): Promise<void> {
const file = new File([scene], "scene.scn.ron", { type: "text/ron" });
const formData = new FormData();
@@ -179,98 +65,52 @@ export class AppElement extends LitElement {
}
}
@on("scene-state")
onSceneStateChange({ detail: state }: SceneStateEvent): void {
console.log("scene state change:", SceneState[state]);
this.loading = match (state, {
[SceneState.Active]: () => false,
_: () => true,
});
}
toggleHelp(): void {
this.showHelp = !this.showHelp;
}
onTransformSpaceChange({ detail: space }: CustomEvent<TransformSpace>): void {
this.transformSpace = space;
studio.setTransformSpace(space);
}
onTransformTypeChange({ detail: type }: CustomEvent<TransformType>): void {
this.transformType = type;
studio.setTransformType(type);
}
// TODO: Get some help from someone who knows what an algorithm is
#updateHierarchy(): void {
this.#entityMap.clear();
for (let obj of this.#sceneObjects)
this.#entityMap.set(obj.id, { ...obj, children: [] });
for (let obj of this.#entityMap.values()) {
if (obj.parent != null) {
const parent = this.#entityMap.get(obj.parent);
if (!parent)
throw new Error(`Missing parent: ${obj.parent}`);
parent.children.push(obj);
}
}
for (let obj of this.#entityMap.values())
obj.children.sort(sortByName);
this.sceneObjects = Array
.from(this.#entityMap.values())
.filter(obj => obj.parent == null)
.sort(sortByName);
console.log("Hierarchy:", this.sceneObjects);
}
#redirectBodyFocus(): void {
// If user focus lands on the `document.body`, neither the frontend app
// nor the Bevy app will properly handle keyboard events. Thie function
// runs every tick to ensure that user focus never becomes "stranded" in
// this way.
if (document.activeElement === document.body) {
const canvasId = this.#canvasProvider.value?.canvasId;
if (canvasId)
document.getElementById(canvasId)?.focus();
}
requestAnimationFrame(() => this.#redirectBodyFocus());
}
protected override render = () => html`
<sts-canvas-provider
class="app-root"
${ref(this.#canvasProvider as any)}
>
<!-- Scene view -->
${this.bvhPath ? html`
${this.#params.bvh ? html`
<sts-bvh-viewer
bvhPath=${this.bvhPath}
bvhPath=${this.#params.bvh}
skyboxId="test_scene"
@scene-state=${this.onSceneStateChange}
></sts-bvh-viewer>
` : this.mixamoPath ? html`
` : this.#params.mixamo ? html`
<sts-mixamo-viewer
mixamoPath=${this.mixamoPath}
mixamoPath=${this.#params.mixamo}
skyboxId="test_scene"
@scene-state=${this.onSceneStateChange}
></sts-mixamo-viewer>
` : this.scenePath ? html`
` : this.#params.scene ? html`
<sts-editor
scenePath=${this.scenePath}
@scene-state=${this.onSceneStateChange}
scenePath=${this.#params.scene}
></sts-editor>
` : html`
<sts-viewer
objectId="sample-room.gltf"
skyboxId="test_scene"
.mode=${this.mode}
@scene-state=${this.onSceneStateChange}
.mode=${this.#params.mode}
></sts-viewer>
`}
<!-- Actions toolbar -->
<sts-toolbar class="v-start h-start">
<sts-toolbar-group>
${this.mode === StudioMode.Editor ? html`
${this.#params.mode === StudioMode.Editor ? html`
<sts-button
icon="floppy-disk"
@click=${() => studio.saveScene()}
@@ -288,30 +128,10 @@ export class AppElement extends LitElement {
</sts-toolbar-group>
</sts-toolbar>
${this.mode === StudioMode.Editor ? html`
<!-- Transform toolbar -->
<sts-transform-toolbar
class="v-start h-center"
.type=${this.transformType}
@type-change=${this.onTransformTypeChange}
.space=${this.transformSpace}
@space-change=${this.onTransformSpaceChange}
></sts-transform-toolbar>
<!-- Hierarchy and inspector -->
<div class="hierarchy v-start h-end">
${this.sceneObjects.map(obj => html`
<sts-tree class="scene-object"
id=${obj.id}
.name=${obj.name}
.treeChildren=${obj.children}
></sts-tree>
`)}
</div>
` : nothing}
<!-- Animation timeline -->
${this.mode === StudioMode.Editor ? html`
<!-- Studio-mode-only elements -->
${this.#params.mode === StudioMode.Editor ? html`
<sts-transform-toolbar class="v-start h-center"></sts-transform-toolbar>
<sts-scene-hierarchy class="hierarchy v-start h-end"></sts-scene-hierarchy>
<sts-anim-editor class="timeline v-end h-start h-before-hierarchy"></sts-anim-editor>
` : nothing}
@@ -329,7 +149,3 @@ export class AppElement extends LitElement {
</sts-canvas-provider>
`;
}
function sortByName(a: { name: string }, b: { name: string }): number {
return a.name.localeCompare(b.name);
}
@@ -1,25 +0,0 @@
import fs from "fs";
import {NormalizedOutputOptions} from "rollup";
function copyRecursively(src: string, dst: string) {
if (fs.statSync(src).isDirectory()) {
// folder
if (!fs.existsSync(dst)) {
fs.mkdirSync(dst);
}
for (const file of fs.readdirSync(src)) {
copyRecursively(src + "/" + file, dst + "/" + file);
}
} else {
// file
fs.copyFileSync(src, dst);
}
}
export function buildMiddleWare(options: NormalizedOutputOptions, assets: string[] = []) {
const dst = options.dir;
if (!dst) throw new Error("options.dir is undefined");
for (const dir of assets) {
copyRecursively(dir, dst)
}
}
@@ -1,23 +0,0 @@
// NOTE: This is an inline fork of this npm package:
// https://github.com/nguyenbatranvan/vite-multiple-assets
//
// The published version at time of writing doesn't include a mime type for
// WGSL, which is fixed in this copy. This module can be removed and replaced
// with the npm dependency if/when this PR is accepted:
// https://github.com/nguyenbatranvan/vite-multiple-assets/pull/5
import {PluginOption, ViteDevServer} from "vite";
import {ServerMiddleWare} from "./server";
import {buildMiddleWare} from "./build";
export default function DynamicPublicDirectory(assets: string[]): PluginOption {
return {
configureServer(server: ViteDevServer) {
return ServerMiddleWare(server, assets)
},
async writeBundle(options) {
buildMiddleWare(options, assets)
},
name: "dynamic assets"
};
}
@@ -1,93 +0,0 @@
import fs from "fs";
import path from "path";
import { ViteDevServer } from "vite";
interface IPropsFile {
name: string;
files: string[];
}
function getFiles(dir, files_): string[] {
files_ = files_ || [];
const files = fs.readdirSync(dir);
for (const i in files) {
const name = dir + "/" + files[i];
if (fs.statSync(name).isDirectory()) {
getFiles(name, files_);
} else {
files_.push(name);
}
}
return files_;
}
const mimeTypes = {
".7z": "application/x-7z-compressed",
".bin": "application/octet-stream",
".bvh": "text/bvh",
".css": "text/css",
".eot": "font/eot",
".gif": "image/gif",
".glb": "model/gltf-binary",
".gltf": "model/gltf+json",
".gz": "application/gzip",
".hdr": "image/vnd.radiance",
".html": "text/html",
".ico": "image/x-icon",
".jpg": "image/jpeg",
".js": "text/javascript",
".json": "application/json",
".ktx": "image/ktx",
".ktx2": "image/ktx",
".meta": "text/plain",
".mjs": "text/javascript",
".mp3": "audio/mpeg",
".mp4": "video/mp4",
".otf": "font/otf",
".png": "image/png",
".rar": "application/x-rar-compressed",
".svg": "image/svg+xml",
".tiff": "image/tiff",
".ttf": "font/ttf",
".txt": "text/plain",
".wasm": "application/wasm",
".webm": "video/webm",
".webp": "image/webp",
".wgsl": "text/wgsl",
".woff": "font/woff",
".woff2": "font/woff2",
".xml": "text/xml",
".zip": "application/zip",
}
export function ServerMiddleWare(server: ViteDevServer, assets: string[] = []) {
if (!assets || !assets.length)
return;
const fileObject: IPropsFile[] = [];
for (let i = 0; i < assets.length; i++) {
const files = getFiles(path.join(process.cwd(), `/${assets[i]}`), []);
fileObject.push({
name: assets[i],
files
});
}
return () => {
server.middlewares.use(async (req, res, next) => {
for (let i = 0; i < fileObject.length; i++) {
const file = path.join(process.cwd(), `${fileObject[i].name}/${req.originalUrl}`);
if (fileObject[i].files.some(f => path.relative(f, file) === "")) {
const extension = file.substring(file.lastIndexOf("."));
res.setHeader("Cache-Control", "max-age=31536000, immutable");
res.setHeader("Content-Type", mimeTypes[extension]);
res.writeHead(200);
res.write(fs.readFileSync(path.join(process.cwd(), "/" + fileObject[i].name + req.originalUrl)));
res.end();
break;
}
}
next();
});
};
}
+1 -2
View File
@@ -5,8 +5,7 @@ import { defineConfig } from "vite";
import { nxViteTsPaths } from "@nx/vite/plugins/nx-tsconfig-paths.plugin";
import wasm from "vite-plugin-wasm";
import topLevelAwait from "vite-plugin-top-level-await";
import mergeAssetsDirs from "./vite-plugins/multiple-assets";
import mergeAssetsDirs from "../vite-multiple-assets";
export default defineConfig({
cacheDir: "../node_modules/.vite/studio-frontend",
+12
View File
@@ -0,0 +1,12 @@
# studio-ui
This is a UI/component library that can be easily shared between multiple
Storyteller Studio frontend applications.
## Building
Run `nx build studio-ui` to build the library.
## Running unit tests
Run `nx test studio-ui` to execute the unit tests via [Jest](https://jestjs.io).
+12
View File
@@ -0,0 +1,12 @@
{
"name": "@storyteller/studio-ui",
"version": "0.0.1",
"dependencies": {
"@storyteller/framework": "0.0.1",
"@storyteller/studio": "0.0.1",
"lit": "^2.8.0"
},
"main": "./index.js",
"module": "./index.mjs",
"typings": "./index.d.ts"
}
+30
View File
@@ -0,0 +1,30 @@
{
"name": "studio-ui",
"$schema": "../node_modules/nx/schemas/project-schema.json",
"sourceRoot": "studio-ui/src",
"projectType": "library",
"targets": {
"build": {
"executor": "@nx/vite:build",
"outputs": ["{options.outputPath}"],
"dependsOn": [
{
"projects": ["studio"],
"target": "build:web"
}
],
"options": {
"outputPath": "dist/studio-ui"
}
},
"test": {
"executor": "@nx/vite:test",
"outputs": ["{options.reportsDirectory}"],
"options": {
"passWithNoTests": true,
"reportsDirectory": "../coverage/studio-ui"
}
}
},
"tags": []
}
+4
View File
@@ -0,0 +1,4 @@
export * from "./lib/button.element";
export * from "./lib/icon.element";
export * from "./lib/toolbar.element";
export * from "./lib/tree.element";
@@ -23,7 +23,7 @@ export class TreeElement extends LitElement {
@property() name = "";
@property() icon?: string;
@property({ attribute: false }) treeChildren?: Tree[];
@property({ type: Boolean }) expanded = false;
@property({ type: Boolean }) expanded = true;
@state() selected = false;
+22
View File
@@ -0,0 +1,22 @@
{
"extends": "../tsconfig.base.json",
"compilerOptions": {
"module": "commonjs",
"forceConsistentCasingInFileNames": true,
"strict": true,
"noImplicitOverride": true,
"noPropertyAccessFromIndexSignature": true,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true
},
"files": [],
"include": [],
"references": [
{
"path": "./tsconfig.lib.json"
},
{
"path": "./tsconfig.spec.json"
}
]
}
+10
View File
@@ -0,0 +1,10 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"outDir": "../dist/out-tsc",
"declaration": true,
"types": ["node", "vite/client"]
},
"include": ["src/**/*.ts"],
"exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"]
}
+25
View File
@@ -0,0 +1,25 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"outDir": "../../dist/out-tsc",
"types": [
"vitest/globals",
"vitest/importMeta",
"vite/client",
"node",
"vitest"
]
},
"include": [
"vite.config.ts",
"src/**/*.test.ts",
"src/**/*.spec.ts",
"src/**/*.test.tsx",
"src/**/*.spec.tsx",
"src/**/*.test.js",
"src/**/*.spec.js",
"src/**/*.test.jsx",
"src/**/*.spec.jsx",
"src/**/*.d.ts"
]
}
+58
View File
@@ -0,0 +1,58 @@
/// <reference types='vitest' />
import { nxViteTsPaths } from "@nx/vite/plugins/nx-tsconfig-paths.plugin";
import { defineConfig } from "vite";
import dts from "vite-plugin-dts";
import wasm from "vite-plugin-wasm";
import topLevelAwait from "vite-plugin-top-level-await";
import * as path from "path";
export default defineConfig({
cacheDir: "../node_modules/.vite/studio-web",
plugins: [
nxViteTsPaths(),
wasm(),
topLevelAwait(),
dts({
entryRoot: "src",
tsConfigFilePath: path.join(__dirname, "tsconfig.lib.json"),
skipDiagnostics: true,
}),
],
// Uncomment this if you are using workers.
// worker: {
// plugins: [ nxViteTsPaths() ],
// },
// Configuration for building your library.
// See: https://vitejs.dev/guide/build.html#library-mode
build: {
lib: {
// Could also be a dictionary or array of multiple entry points.
entry: "src/index.ts",
name: "studio-ui",
fileName: "index",
// Change this to the formats you want to support.
// Don't forget to update your package.json as well.
formats: ["es", "cjs"],
},
rollupOptions: {
// External packages that should not be bundled into your library.
external: [
"lit",
"@storyteller/framework",
"@storyteller/studio",
],
},
},
test: {
globals: true,
cache: {
dir: "../node_modules/.vitest",
},
environment: "jsdom",
include: ["src/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}"],
},
});
+3
View File
@@ -2,4 +2,7 @@ export * from "./lib/bvh-viewer.element";
export * from "./lib/canvas-provider.element";
export * from "./lib/editor.element";
export * from "./lib/mixamo-viewer.element";
export * from "./lib/scene-hierarchy.element";
export * from "./lib/transform-toolbar.element";
export * from "./lib/url-params";
export * from "./lib/viewer.element";
+30 -1
View File
@@ -1,4 +1,4 @@
import { Token, UniqueToken, provide } from "@storyteller/framework";
import { Token, UniqueToken, on, provide } from "@storyteller/framework";
import { LitElement, html, unsafeCSS } from "lit";
import { customElement } from "lit/decorators.js";
@@ -71,6 +71,35 @@ export class CanvasProviderElement
super.connectedCallback();
}
@on("keydown")
@on("keyup")
_pipeKeyboardEvents(event: KeyboardEvent): void {
// When the user interacts with one of the web UI elements, the Bevy
// canvas will lose focus, preventing it from receiving keyboard events.
// As a workaround, we can listen for any events that have bubbled up to
// the app root and repeat them for the canvas.
//
// This does unfortunately burden the web front-end with some additional
// responsibility: when we handle a keyboard event and _don't_ want the
// Bevy app to receive it, we'll need to call `event.stopPropagation()` in
// the handler to prevent it from reaching this listener.
const {
type,
code, charCode, key, keyCode, location, repeat,
altKey, ctrlKey, metaKey, shiftKey,
} = event;
document
.getElementById(this.#canvasId)
?.dispatchEvent(new KeyboardEvent(type, {
code, charCode, key, keyCode, location, repeat,
altKey, ctrlKey, metaKey, shiftKey,
bubbles: true,
cancelable: true,
composed: true,
}));
}
protected override render = () => html`
<slot></slot>
<slot name="canvas"></slot>
@@ -0,0 +1,24 @@
:host {
box-sizing: border-box;
padding: 0.5rem 1rem;
overflow: auto;
background: #0006;
backdrop-filter: blur(16px);
border: 8px solid transparent;
border-radius: 0.5rem;
&::-webkit-scrollbar {
width: 8px;
height: 8px;
&-thumb {
border-radius: 2px;
background: #FFF2;
&:hover {
background: #FFF3;
}
}
}
}
@@ -0,0 +1,77 @@
import * as studio from "@storyteller/studio";
import {
type EntityDespawnEvent,
type EntityMultiSpawnEvent,
type EntitySpawnEvent,
} from "@storyteller/studio";
import { on } from "@storyteller/framework";
import { LitElement, html, unsafeCSS } from "lit";
import { customElement, state } from "lit/decorators.js";
import styles from "./scene-hierarchy.element.scss?inline";
import "@storyteller/studio-ui/tree";
interface SceneObject extends Omit<studio.SceneObject, "children"> {
children: SceneObject[];
}
@customElement("sts-scene-hierarchy")
export class SceneHierarchyElement extends LitElement {
static override styles = unsafeCSS(styles);
get tree() { return this._tree as readonly SceneObject[]; }
get entityMap() { return this.#entityMap as ReadonlyMap<string, SceneObject>; }
@state() private _tree: SceneObject[] = [];
#sceneObjectsFlat: studio.SceneObject[] = [];
#entityMap = new Map<string, SceneObject>();
@on("window:entity-spawn")
@on("window:entity-multi-spawn")
_onEntitiesSpawned({ detail: entities }: EntitySpawnEvent | EntityMultiSpawnEvent): void {
this.#sceneObjectsFlat = this.#sceneObjectsFlat.concat(entities);
this.#rebuildTree();
}
@on("window:entity-despawn")
_onEntityDespawn({ detail: entity }: EntityDespawnEvent): void {
this.#sceneObjectsFlat = this.#sceneObjectsFlat.filter(({ id }) => id !== entity);
this.#rebuildTree();
}
#rebuildTree(): void {
this.#entityMap.clear();
for (let obj of this.#sceneObjectsFlat)
this.#entityMap.set(obj.id, { ...obj, children: [] });
for (let obj of this.#entityMap.values()) {
if (obj.parent != null) {
const parent = this.#entityMap.get(obj.parent);
if (!parent)
throw new Error(`Missing parent: ${obj.parent}`);
parent.children.push(obj);
}
}
for (let obj of this.#entityMap.values())
obj.children.sort((a, b) => a.name.localeCompare(b.name));
this._tree = Array
.from(this.#entityMap.values())
.filter(obj => obj.parent == null)
.sort((a, b) => a.name.localeCompare(b.name))
}
protected override render = () => html`
${this.tree.map(obj => html`
<sts-tree
id=${obj.id}
.name=${obj.name}
.treeChildren=${obj.children}
></sts-tree>
`)}
`
}
@@ -10,4 +10,5 @@
inset: 0;
width: 100%;
height: 100%;
outline: none !important;
}
+19
View File
@@ -10,11 +10,14 @@ import styles from "./studio-base.element.scss?inline";
export abstract class StudioBaseElement extends LitElement {
static override styles = unsafeCSS(styles);
get canvas() { return this.#canvas; }
@inject(CANVAS_PROVIDER)
_canvasProvider?: CanvasProvider;
#canvas?: HTMLCanvasElement;
#resizeObserver?: ResizeObserver;
#frameRequest?: number;
abstract onStart(viewportSize: ViewportSize): void;
abstract onChanges?(changes: PropertyValues<this>): void;
@@ -30,9 +33,14 @@ export abstract class StudioBaseElement extends LitElement {
if ((this.#canvas = this._canvasProvider.takeCanvas()))
this.appendChild(this.#canvas);
this.#tick();
}
override disconnectedCallback(): void {
if (this.#frameRequest != null)
cancelAnimationFrame(this.#frameRequest);
if (this.#canvas)
this._canvasProvider?.returnCanvas(this.removeChild(this.#canvas));
@@ -54,6 +62,17 @@ export abstract class StudioBaseElement extends LitElement {
studio.resize(new ViewportSize(width, height));
}
#tick(): void {
// If user focus lands on the `document.body`, neither the frontend app
// nor the Bevy app will properly handle keyboard events. Thie function
// runs every tick to ensure that user focus never becomes "stranded" in
// this way.
if (document.activeElement === document.body)
this.#canvas?.focus();
this.#frameRequest = requestAnimationFrame(() => this.#tick());
}
#start(): void {
if (!this.#canvas) return;
@@ -1,7 +1,11 @@
import * as studio from "@storyteller/studio";
import { TransformSpace, TransformType } from "@storyteller/studio";
import { match } from "@storyteller/utility";
import { LitElement, html, unsafeCSS } from "lit";
import { customElement, property } from "lit/decorators.js";
import { customElement, state } from "lit/decorators.js";
import "@storyteller/studio-ui/icon";
import "@storyteller/studio-ui/toolbar";
import styles from "./transform-toolbar.element.scss?inline";
@@ -9,12 +13,15 @@ import styles from "./transform-toolbar.element.scss?inline";
export class TransformToolbarElement extends LitElement {
static override styles = unsafeCSS(styles);
@property({ type: Number }) space = TransformSpace.Local;
@property({ type: Number }) type = TransformType.Translate;
get space() { return this._space; }
@state() private _space = TransformSpace.Local;
get type() { return this._type; }
@state() private _type = TransformType.Translate;
#onSpaceChange(event: InputEvent): void {
const { value } = event.target as HTMLInputElement;
const space = match (value, {
this._space = match (value, {
"local": () => TransformSpace.Local,
"world": () => TransformSpace.World,
_: () => {
@@ -22,11 +29,7 @@ export class TransformToolbarElement extends LitElement {
}
});
this.dispatchEvent(new CustomEvent<TransformSpace>("space-change", {
detail: space,
bubbles: true,
composed: true,
}));
studio.setTransformSpace(this._space);
}
#onTypeChange(event: InputEvent): void {
@@ -40,15 +43,10 @@ export class TransformToolbarElement extends LitElement {
}
});
const type = checked
? this.type | flag
: this.type & ~flag;
if (checked) this._type |= flag;
else this._type &= ~flag;
this.dispatchEvent(new CustomEvent<TransformType>("type-change", {
detail: type,
bubbles: true,
composed: true,
}));
studio.setTransformType(this._type);
}
protected override render = () => html`
@@ -59,7 +57,7 @@ export class TransformToolbarElement extends LitElement {
<input type="radio"
name="transform-space"
value="local"
?checked=${this.space === TransformSpace.Local}
?checked=${this._space === TransformSpace.Local}
@input=${this.#onSpaceChange}
/>
</label>
@@ -68,7 +66,7 @@ export class TransformToolbarElement extends LitElement {
<input type="radio"
name="transform-space"
value="world"
?checked=${this.space === TransformSpace.World}
?checked=${this._space === TransformSpace.World}
@input=${this.#onSpaceChange}
/>
</label>
@@ -78,7 +76,7 @@ export class TransformToolbarElement extends LitElement {
<sts-icon icon="up-down-left-right">Translate</sts-icon>
<input type="checkbox"
name="translate"
?checked=${Boolean(this.type & TransformType.Translate)}
?checked=${Boolean(this._type & TransformType.Translate)}
@input=${this.#onTypeChange}
/>
</label>
@@ -86,7 +84,7 @@ export class TransformToolbarElement extends LitElement {
<sts-icon icon="rotate">Rotate</sts-icon>
<input type="checkbox"
name="rotate"
?checked=${Boolean(this.type & TransformType.Rotate)}
?checked=${Boolean(this._type & TransformType.Rotate)}
@input=${this.#onTypeChange}
/>
</label>
@@ -95,7 +93,7 @@ export class TransformToolbarElement extends LitElement {
<input type="checkbox"
name="scale"
disabled
?checked=${Boolean(this.type & TransformType.Scale)}
?checked=${Boolean(this._type & TransformType.Scale)}
@input=${this.#onTypeChange}
/>
</label>
+20
View File
@@ -0,0 +1,20 @@
import { StudioMode } from "@storyteller/studio";
import { match } from "@storyteller/utility";
export class StudioURLParams {
#_nativeParams?: URLSearchParams;
get #nativeParams() {
return this.#_nativeParams ??= new URLSearchParams(document.location.search);
}
get bvh() { return this.#nativeParams.get("bvh"); }
get mixamo() { return this.#nativeParams.get("mixamo"); }
get scene() { return this.#nativeParams.get("scene"); }
get mode(): StudioMode {
return match(this.#nativeParams.get("mode"), {
"studio": () => StudioMode.Editor,
"viewer": () => StudioMode.Viewer,
_: () => StudioMode.Viewer,
});
}
}
+1
View File
@@ -66,6 +66,7 @@ wasm = [
"bevy_rapier3d/wasm-bindgen",
]
desktop = []
wasm_screenshot = []
[dependencies.bevy]
workspace = true
+11 -3
View File
@@ -27,7 +27,15 @@
"executor": "nx:run-commands",
"inputs": ["assets"],
"options": {
"command": "node scripts/build-web.mjs",
"command": "node scripts/build-web.mjs --features=wasm",
"cwd": "studio"
}
},
"build:image-generator": {
"executor": "nx:run-commands",
"inputs": ["assets"],
"options": {
"command": "node scripts/build-web.mjs --features=wasm,wasm_screenshot",
"cwd": "studio"
}
},
@@ -41,8 +49,8 @@
"headless": {
"executor": "nx:run-commands",
"options": {
"command": "cargo run -p studio --bin studio-headless -- {args}",
}
"command": "cargo run -p studio --bin studio-headless -- {args}"
}
},
"test": {
"executor": "@nxrs/cargo:test",
+7 -2
View File
@@ -13,6 +13,12 @@ async function main() {
const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
const workspaceRoot = path.resolve(__dirname, "../..");
const extraCargoArgs = process.argv.slice(2);
console.log(`Extra cargo args:`);
for (let arg of extraCargoArgs) {
console.log(` ${arg}`);
}
/** @type import("node:child_process").StdioOptions */
const stdio = ["ignore", "inherit", "inherit"];
@@ -26,9 +32,8 @@ async function main() {
"-p", "studio",
"--lib",
"--release",
"--features", "wasm",
"--target", "wasm32-unknown-unknown",
], {
].concat(extraCargoArgs), {
shell: true,
cwd: workspaceRoot,
stdio,
+19 -16
View File
@@ -10,7 +10,7 @@ use crate::{
interaction::Selected,
math::{
Easing, InterpPosition, InterpVelocity, Interpolation, InterpolationSystem, NearlyEq,
Velocity,
NearlyZero, Velocity,
},
};
@@ -78,9 +78,6 @@ fn camera_control_kinematic(
return;
}
let (mut controller, mut xform, proj) = q_camera.single_mut();
let mut window = q_window.single_mut();
let mut any_ongoing = false;
let mut any_started = false;
let mut any_stopped = false;
@@ -92,7 +89,8 @@ fn camera_control_kinematic(
)
}) {
if let InputAction(lifecycle, Action::Camera(motion)) = action {
use CameraMotion::*;
let (mut controller, mut xform, proj) = q_camera.single_mut();
let window = q_window.single();
match lifecycle {
Lifecycle::Start => any_started = true,
@@ -100,20 +98,21 @@ fn camera_control_kinematic(
Lifecycle::Stop => any_stopped = true,
}
use CameraMotion::*;
match motion {
Pan(delta) => do_pan(proj, &window, *delta, &mut xform, &mut controller),
Zoom(delta) => do_zoom(&window, *delta, &mut xform, &mut controller),
Orbit(delta) => do_orbit(&window, *delta, *lifecycle, &mut xform, &mut controller),
Rotate(delta) => do_rotate(&window, *delta, &mut xform, &mut controller),
Pan(delta) => do_pan(proj, window, *delta, &mut xform, &mut controller),
Zoom(delta) => do_zoom(window, *delta, &mut xform, &mut controller),
Orbit(delta) => do_orbit(window, *delta, *lifecycle, &mut xform, &mut controller),
Rotate(delta) => do_rotate(window, *delta, &mut xform, &mut controller),
_ => {}
}
}
}
if any_started {
capture_pointer(&mut window);
capture_pointer(&mut q_window.single_mut());
} else if any_stopped && !any_ongoing {
release_pointer(&mut window);
release_pointer(&mut q_window.single_mut());
}
}
@@ -215,12 +214,16 @@ fn process_focus_inputs(
}
fn camera_control_physical(
mut q_camera: Query<(&mut CameraController, &mut Transform, &Velocity)>,
mut q_camera: Query<(Entity, &mut CameraController, &Velocity)>,
mut q_xforms: Query<&mut Transform>,
) {
for (mut controller, mut xform, velocity) in q_camera.iter_mut() {
let delta_pos = xform.rotation * (**velocity);
xform.translation += delta_pos;
controller.focus += delta_pos;
for (ent, mut controller, velocity) in q_camera.iter_mut() {
if !velocity.nearly_zero() {
let mut xform = q_xforms.get_mut(ent).unwrap();
let delta_pos = xform.rotation * (**velocity);
xform.translation += delta_pos;
controller.focus += delta_pos;
}
}
}
+77
View File
@@ -0,0 +1,77 @@
use bevy::prelude::*;
pub struct ChangeTrackingPlugin;
impl Plugin for ChangeTrackingPlugin {
fn build(&self, app: &mut App) {
app.add_event::<EntityChangedEvent>();
// Add more of these systems for each component that we want to track
app.add_systems(Update, send_change_events::<Transform>);
#[cfg(feature = "wasm")]
app.add_systems(
Update,
wasm::send_change_events::<Transform>.map(bevy::utils::error),
);
}
}
#[derive(Component, Debug)]
pub struct ChangeTracker;
fn send_change_events<T: Component>(
query: Query<Entity, (Changed<T>, With<ChangeTracker>)>,
mut entity_change_writer: EventWriter<EntityChangedEvent>,
) {
for entity in query.iter() {
entity_change_writer.send(EntityChangedEvent(entity));
}
}
#[derive(Event, Deref)]
pub struct EntityChangedEvent(pub Entity);
#[cfg(feature = "wasm")]
mod wasm {
use bevy::{prelude::*, window::PrimaryWindow};
use wasm_bindgen::prelude::wasm_bindgen;
use crate::wasm;
use super::ChangeTracker;
#[wasm_bindgen(typescript_custom_section)]
const ENTITY_CHANGES_EVENT: &str = r#"
export interface EntityChangesEvent extends CustomEvent {
type: "entity-changes";
detail: string[];
}
declare global {
export interface GlobalEventHandlersEventMap {
"entity-changes": EntityChangesEvent;
}
}
"#;
pub(super) fn send_change_events<T: Component>(
q_changed_ents: Query<Entity, (Changed<T>, With<ChangeTracker>)>,
q_window: Query<&Window, With<PrimaryWindow>>,
) -> Result<(), String> {
if q_changed_ents.is_empty() {
return Ok(());
}
let Ok(window) = q_window.get_single() else {
return Ok(());
};
let entities = q_changed_ents
.iter()
.map(|ent| format!("{ent:?}"))
.collect::<Vec<_>>();
wasm::dispatch_to_js(window, "entity-changes", &entities)
}
}
+3
View File
@@ -19,6 +19,7 @@ use wgpu::Extent3d;
mod anim;
mod bvh;
mod camera_controller;
mod change_tracking;
mod debug;
mod design_system;
mod hierarchy;
@@ -279,6 +280,8 @@ pub fn start_internal(
..default()
}),
CommonPlugins,
#[cfg(all(feature = "wasm", feature = "wasm_screenshot"))]
wasm::ScreenshotPlugin,
));
if matches!(studio_mode, StudioMode::Editor) {
+22 -3
View File
@@ -22,7 +22,9 @@ use wasm_bindgen::prelude::wasm_bindgen;
use wgpu::{TextureDescriptor, TextureUsages};
use crate::{
anim::SkeletalJoint,
camera_controller::{CameraController, CameraControllerBundle},
change_tracking::{ChangeTracker, ChangeTrackingPlugin},
debug::DebugPoint,
is_headless, is_not_headless,
math::{Combinable, Extrema},
@@ -37,6 +39,8 @@ pub struct SceneSpawnerSystem;
impl Plugin for ScenePlugin {
fn build(&self, app: &mut App) {
app.add_plugins(ChangeTrackingPlugin);
app.insert_resource(ClearColor(Color::hsl(0.0, 0.0, 0.25)))
.insert_resource(Msaa::Off)
.add_state::<SceneState>()
@@ -56,10 +60,15 @@ impl Plugin for ScenePlugin {
.add_systems(
OnEnter(SceneState::Active),
(
tag_scene_elements,
frame_scene_in_view,
start_capture.run_if(is_headless),
(
tag_scene_elements,
frame_scene_in_view,
start_capture.run_if(is_headless),
),
apply_deferred,
track_scene_elements,
)
.chain()
.in_set(SceneSpawnerSystem),
);
@@ -234,6 +243,7 @@ fn initialize(mut cmd: Commands) {
cmd.spawn((
Name::new("Main Camera"),
MainCamera,
ChangeTracker,
Camera3dBundle {
transform: camera_xform,
tonemapping: Tonemapping::BlenderFilmic,
@@ -483,6 +493,15 @@ fn start_capture(mut captures: Query<&mut ScreenCapture>) {
}
}
fn track_scene_elements(
mut commands: Commands,
scene_elements: Query<Entity, (With<Transform>, With<SkeletalJoint>)>,
) {
for element in scene_elements.iter() {
commands.entity(element).insert(ChangeTracker);
}
}
#[cfg(feature = "wasm")]
mod wasm {
use bevy::{prelude::*, window::PrimaryWindow};
+4
View File
@@ -4,9 +4,13 @@ use parking_lot::ReentrantMutex;
use wasm_bindgen::prelude::wasm_bindgen;
mod conv;
#[cfg(feature = "wasm_screenshot")]
mod screenshot;
mod util;
pub use conv::*;
#[cfg(feature = "wasm_screenshot")]
pub use screenshot::ScreenshotPlugin;
pub(crate) use util::*;
static INITIALIZED: OnceLock<ReentrantMutex<RefCell<bool>>> = OnceLock::new();
+161
View File
@@ -0,0 +1,161 @@
//! Outline of the flow of events here:
//!
//! 1. [`crate::change_tracking`] system sends events to frontend to notify of
//! tracked changes
//! 2. Frontend calls [`request_screenshot`] on a debounce, writing an empty
//! message to [`SCREENSHOT_REQUEST`]
//! 3. [`take_screenshots`] watches the [`SCREENSHOT_REQUEST`] buffer and
//! populates the [`SCREENSHOT`] buffer with a PNG-encoded `Vec<u8>`
//! 4. [`send_screenshots`] watches the [`SCREENSHOT`] buffer and sends an event
//! to the frontend by converting the `Vec<u8>` to a [`js_sys::Uint8Array`]
use bevy::{prelude::*, render::view::screenshot::ScreenshotManager, window::PrimaryWindow};
use image::{codecs::png::PngEncoder, ColorType, ImageEncoder};
use wasm_bindgen::prelude::{wasm_bindgen, JsValue};
use super::{AsJsValue, StaticBuffer, WasmMessageBuffer};
static SCREENSHOT_REQUEST: WasmMessageBuffer<()> = WasmMessageBuffer::new();
static SCREENSHOT: WasmMessageBuffer<ScreenshotNative> = WasmMessageBuffer::new();
pub struct ScreenshotPlugin;
impl Plugin for ScreenshotPlugin {
fn build(&self, app: &mut App) {
app.add_systems(
Update,
(
take_screenshots.map(bevy::utils::error),
send_screenshots.map(bevy::utils::error),
),
);
}
}
#[wasm_bindgen]
#[derive(Clone)]
pub struct Screenshot {
data_pvt: js_sys::Uint8Array,
pub width: usize,
pub height: usize,
}
// This intermediate type is necessary because the `Screenshot` struct
// cannot be safely stored in a `WasmMessageBuffer`
struct ScreenshotNative {
data: Vec<u8>,
width: usize,
height: usize,
}
impl From<ScreenshotNative> for Screenshot {
fn from(value: ScreenshotNative) -> Self {
Self {
data_pvt: js_sys::Uint8Array::from(&value.data[..]),
width: value.width,
height: value.height,
}
}
}
#[wasm_bindgen]
impl Screenshot {
#[wasm_bindgen(getter)]
pub fn data(&self) -> js_sys::Uint8Array {
self.data_pvt.clone()
}
}
impl AsJsValue for Screenshot {
fn as_js(&self) -> Result<JsValue, String> {
Ok(JsValue::from(self.clone()))
}
}
#[wasm_bindgen(js_name = requestScreenshot)]
pub fn request_screenshot() {
SCREENSHOT_REQUEST.write_message(());
}
fn take_screenshots(
mut r_screenshot_mgr: ResMut<ScreenshotManager>,
q_window: Query<Entity, With<PrimaryWindow>>,
) -> Result<(), String> {
if SCREENSHOT_REQUEST.take_message().is_none() {
return Ok(());
}
let Ok(window_ent) = q_window.get_single() else {
return Ok(());
};
r_screenshot_mgr
.take_screenshot(window_ent, |img| {
let img = match img.try_into_dynamic().map(|img| img.to_rgb8()) {
Ok(img) => img,
Err(err) => {
error!("{err}");
return;
}
};
let channel_count = <image::Rgb<u8> as image::Pixel>::CHANNEL_COUNT as usize;
let buffer_len = match Some(channel_count)
.and_then(|size| size.checked_mul(img.width() as usize))
.and_then(|size| size.checked_mul(img.height() as usize))
{
Some(size) => size,
None => {
error!("Failed to compute buffer length for image!");
return;
}
};
let src_buffer = &img.as_raw()[..buffer_len];
let mut png_bytes = vec![];
if let Err(err) = PngEncoder::new(&mut png_bytes).write_image(
src_buffer,
img.width(),
img.height(),
ColorType::Rgb8,
) {
error!("{err}");
return;
}
SCREENSHOT.write_message(ScreenshotNative {
data: png_bytes,
width: img.width() as usize,
height: img.height() as usize,
});
})
.map_err(|err| format!("{err}"))
}
#[wasm_bindgen(typescript_custom_section)]
const SCREENSHOT_READY_EVENT: &str = r#"
export interface ScreenshotReadyEvent extends CustomEvent {
type: "screenshot-ready";
detail: Screenshot;
}
declare global {
export interface GlobalEventHandlersEventMap {
"screenshot-ready": ScreenshotReadyEvent;
}
}
"#;
fn send_screenshots(q_window: Query<&Window, With<PrimaryWindow>>) -> Result<(), String> {
let Some(screenshot) = SCREENSHOT.take_message() else {
return Ok(());
};
let Ok(window) = q_window.get_single() else {
return Ok(());
};
let screenshot = Screenshot::from(screenshot);
super::dispatch_to_js(window, "screenshot-ready", &screenshot)
}
+7
View File
@@ -17,12 +17,19 @@
"paths": {
"@storyteller/framework": ["framework/src/index.ts"],
"@storyteller/studio": ["dist/studio"],
"@storyteller/studio-ui": ["studio-ui/src/index.ts"],
"@storyteller/studio-ui/button": ["studio-ui/src/lib/button.element.ts"],
"@storyteller/studio-ui/icon": ["studio-ui/src/lib/icon.element.ts"],
"@storyteller/studio-ui/toolbar": ["studio-ui/src/lib/toolbar.element.ts"],
"@storyteller/studio-ui/tree": ["studio-ui/src/lib/tree.element.ts"],
"@storyteller/studio-web": ["studio-web/src/index.ts"],
"@storyteller/studio-web/canvas-provider": ["studio-web/src/lib/canvas-provider.element.ts"],
"@storyteller/studio-web/viewer": ["studio-web/src/lib/viewer.element.ts"],
"@storyteller/studio-web/bvh-viewer": ["studio-web/src/lib/bvh-viewer.element.ts"],
"@storyteller/studio-web/editor": ["studio-web/src/lib/editor.element.ts"],
"@storyteller/studio-web/mixamo-viewer": ["studio-web/src/lib/mixamo-viewer.element.ts"],
"@storyteller/studio-web/scene-hierarchy": ["studio-web/src/lib/scene-hierarchy.element.ts"],
"@storyteller/studio-web/transform-toolbar": ["studio-web/src/lib/transform-toolbar.element.ts"],
"@storyteller/utility": ["utility/src/index.ts"]
}
},
+5
View File
@@ -0,0 +1,5 @@
# vite-multiple-assets
This is a simple Vite plugin forked from [`vite-multiple-assets`](https://github.com/nguyenbatranvan/vite-multiple-assets)
and extended to provide MIME types for additional filetypes not commonly used on
the web, but needed for a Bevy 3D application.
+6
View File
@@ -0,0 +1,6 @@
{
"name": "@storyteller/vite-multiple-assets",
"version": "0.0.1",
"dependencies": {},
"main": "./src/index.js"
}
+8
View File
@@ -0,0 +1,8 @@
{
"name": "vite-multiple-assets",
"$schema": "../node_modules/nx/schemas/project-schema.json",
"projectType": "library",
"sourceRoot": "vite-multiple-assets/src",
"targets": {},
"tags": []
}
+16
View File
@@ -0,0 +1,16 @@
import { PluginOption, ViteDevServer } from "vite";
import { ServerMiddleWare } from "./lib/server";
import { buildMiddleWare } from "./lib/build";
export default function DynamicPublicDirectory(assets: string[]): PluginOption {
return {
configureServer(server: ViteDevServer) {
return ServerMiddleWare(server, assets)
},
async writeBundle(options) {
buildMiddleWare(options, assets)
},
name: "dynamic assets"
};
}
+25
View File
@@ -0,0 +1,25 @@
import fs from "fs";
import { NormalizedOutputOptions } from "rollup";
function copyRecursively(src: string, dst: string) {
if (fs.statSync(src).isDirectory()) {
// folder
if (!fs.existsSync(dst)) {
fs.mkdirSync(dst);
}
for (const file of fs.readdirSync(src)) {
copyRecursively(src + "/" + file, dst + "/" + file);
}
} else {
// file
fs.copyFileSync(src, dst);
}
}
export function buildMiddleWare(options: NormalizedOutputOptions, assets: string[] = []) {
const dst = options.dir;
if (!dst) throw new Error("options.dir is undefined");
for (const dir of assets) {
copyRecursively(dir, dst)
}
}
+93
View File
@@ -0,0 +1,93 @@
import fs from "fs";
import path from "path";
import { ViteDevServer } from "vite";
interface IPropsFile {
name: string;
files: string[];
}
function getFiles(dir, files_): string[] {
files_ = files_ || [];
const files = fs.readdirSync(dir);
for (const i in files) {
const name = dir + "/" + files[i];
if (fs.statSync(name).isDirectory()) {
getFiles(name, files_);
} else {
files_.push(name);
}
}
return files_;
}
const mimeTypes = {
".7z": "application/x-7z-compressed",
".bin": "application/octet-stream",
".bvh": "text/bvh",
".css": "text/css",
".eot": "font/eot",
".gif": "image/gif",
".glb": "model/gltf-binary",
".gltf": "model/gltf+json",
".gz": "application/gzip",
".hdr": "image/vnd.radiance",
".html": "text/html",
".ico": "image/x-icon",
".jpg": "image/jpeg",
".js": "text/javascript",
".json": "application/json",
".ktx": "image/ktx",
".ktx2": "image/ktx",
".meta": "text/plain",
".mjs": "text/javascript",
".mp3": "audio/mpeg",
".mp4": "video/mp4",
".otf": "font/otf",
".png": "image/png",
".rar": "application/x-rar-compressed",
".svg": "image/svg+xml",
".tiff": "image/tiff",
".ttf": "font/ttf",
".txt": "text/plain",
".wasm": "application/wasm",
".webm": "video/webm",
".webp": "image/webp",
".wgsl": "text/wgsl",
".woff": "font/woff",
".woff2": "font/woff2",
".xml": "text/xml",
".zip": "application/zip",
}
export function ServerMiddleWare(server: ViteDevServer, assets: string[] = []) {
if (!assets || !assets.length)
return;
const fileObject: IPropsFile[] = [];
for (let i = 0; i < assets.length; i++) {
const files = getFiles(path.join(process.cwd(), `/${assets[i]}`), []);
fileObject.push({
name: assets[i],
files
});
}
return () => {
server.middlewares.use(async (req, res, next) => {
for (let i = 0; i < fileObject.length; i++) {
const file = path.join(process.cwd(), `${fileObject[i].name}/${req.originalUrl}`);
if (fileObject[i].files.some(f => path.relative(f, file) === "")) {
const extension = file.substring(file.lastIndexOf("."));
res.setHeader("Cache-Control", "max-age=31536000, immutable");
res.setHeader("Content-Type", mimeTypes[extension]);
res.writeHead(200);
res.write(fs.readFileSync(path.join(process.cwd(), "/" + fileObject[i].name + req.originalUrl)));
res.end();
break;
}
}
next();
});
};
}
+22
View File
@@ -0,0 +1,22 @@
{
"extends": "../tsconfig.base.json",
"compilerOptions": {
"module": "commonjs",
"forceConsistentCasingInFileNames": true,
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"strict": true,
"noImplicitAny": false,
"skipLibCheck": true
},
"files": [],
"include": [],
"references": [
{
"path": "./tsconfig.lib.json"
},
{
"path": "./tsconfig.spec.json"
}
]
}
+10
View File
@@ -0,0 +1,10 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"outDir": "../dist/out-tsc",
"declaration": true,
"types": ["node", "vite/client"]
},
"include": ["src/**/*.ts"],
"exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"]
}
+25
View File
@@ -0,0 +1,25 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"outDir": "../../dist/out-tsc",
"types": [
"vitest/globals",
"vitest/importMeta",
"vite/client",
"node",
"vitest"
]
},
"include": [
"vite.config.ts",
"src/**/*.test.ts",
"src/**/*.spec.ts",
"src/**/*.test.tsx",
"src/**/*.spec.tsx",
"src/**/*.test.js",
"src/**/*.spec.js",
"src/**/*.test.jsx",
"src/**/*.spec.jsx",
"src/**/*.d.ts"
]
}