From d51eec52c3c762b380203c456b49b9069d322856 Mon Sep 17 00:00:00 2001 From: Brooks Palin Date: Mon, 26 Feb 2024 15:13:07 -0700 Subject: [PATCH] Bootstrap Image Generator application (#53) This PR begins scaffolding the Stable Diffusion Turbo application (title TBD), which is a separate frontend for Storyteller Engine which focuses on image generation. ### Design Overview * The application renders a "split-screen" view, with Storyteller Studio on the left, and the Stable Diffusion Turbo output on the right * When changes are made in the Studio editor: - A screenshot is taken from the Bevy application - Screenshot is piped into Stable Diffusion Turbo model - Model output is rendered to the screen * This all happens in as close to real-time as possible ### Work completed in this PR * The `studio-frontend` project has been (somewhat crudely) refactored to split most of its components into reusable libraries - `studio-ui` is a use-case-agnostic design system library for core UI components. These are "dumb" components which are controlled via `@property` inputs and send `CustomEvent`s in response to user input. - `studio-frontend` is a collection of reusable Storyteller Studio widgets. These are "smart" components which depend on `@storyteller/studio` directly, manage their own state, and require very little configuration or styling. They are designed to be drop-in building blocks that can be reused between various frontends. * The new application has been bootstrapped as `image-generator-demo` - Currently, this simply grabs screenshots from the Bevy application and renders those screenshots on the right side of the split-screen view. A future PR will wire up to the Storyteller.ai backend to pipe these screenshots through Stable Diffusion Turbo, and render that model's output instead. ### Testing * Install dependencies as described in the repo's README * Run `npx nx serve image-generator-demo` * Open [localhost:4200](http://localhost:4200) in your browser - Use the camera controller in the left side of the view to manipulate the Bevy scene - Verify the right side of the view updates with a fresh screenshot at a rate of ~5 fps * Toggle the "Realtime Capture" checkbox - Verify the right side view no longer updates while the user is actively manipulating the editor - This should allow the app to run smoother on lower-spec hardware --------- Co-authored-by: Danny McGee --- .vscode/settings.json | 3 +- editor.ron | 2 +- image-generator-demo/.swcrc | 8 + image-generator-demo/index.html | 19 ++ image-generator-demo/project.json | 71 +++++ image-generator-demo/public/favicon.ico | Bin 0 -> 15086 bytes image-generator-demo/src/app/app.element.scss | 72 +++++ image-generator-demo/src/app/app.element.ts | 135 ++++++++++ image-generator-demo/src/main.ts | 1 + image-generator-demo/src/styles.scss | 10 + image-generator-demo/tsconfig.app.json | 15 ++ image-generator-demo/tsconfig.json | 34 +++ image-generator-demo/typings/scss.d.ts | 4 + image-generator-demo/typings/svg.d.ts | 4 + image-generator-demo/vite.config.ts | 29 +++ .../vite-plugins/multiple-assets/build.ts | 25 -- .../vite-plugins/multiple-assets/index.ts | 26 -- .../vite-plugins/multiple-assets/server.ts | 91 ------- react-frontend/vite.config.ts | 3 +- .../src/app/anim-editor.element.ts | 6 +- studio-frontend/src/app/app.element.scss | 24 -- studio-frontend/src/app/app.element.ts | 246 +++--------------- .../vite-plugins/multiple-assets/build.ts | 25 -- .../vite-plugins/multiple-assets/index.ts | 23 -- .../vite-plugins/multiple-assets/server.ts | 93 ------- studio-frontend/vite.config.ts | 3 +- studio-ui/README.md | 12 + studio-ui/package.json | 12 + studio-ui/project.json | 30 +++ studio-ui/src/index.ts | 4 + .../src/lib}/button.element.scss | 0 .../src/lib}/button.element.ts | 0 .../src/lib}/icon.element.scss | 0 .../app => studio-ui/src/lib}/icon.element.ts | 0 .../src/lib}/toolbar-group.element.scss | 0 .../src/lib}/toolbar.element.scss | 0 .../src/lib}/toolbar.element.ts | 0 .../src/lib}/tree.element.scss | 0 .../app => studio-ui/src/lib}/tree.element.ts | 2 +- studio-ui/tsconfig.json | 22 ++ studio-ui/tsconfig.lib.json | 10 + studio-ui/tsconfig.spec.json | 25 ++ studio-ui/vite.config.ts | 58 +++++ studio-web/src/index.ts | 3 + studio-web/src/lib/canvas-provider.element.ts | 31 ++- .../src/lib/scene-hierarchy.element.scss | 24 ++ studio-web/src/lib/scene-hierarchy.element.ts | 77 ++++++ studio-web/src/lib/studio-base.element.scss | 1 + studio-web/src/lib/studio-base.element.ts | 19 ++ .../src/lib}/transform-toolbar.element.scss | 0 .../src/lib}/transform-toolbar.element.ts | 42 ++- studio-web/src/lib/url-params.ts | 20 ++ studio/Cargo.toml | 1 + studio/project.json | 14 +- studio/scripts/build-web.mjs | 9 +- studio/src/camera_controller.rs | 35 +-- studio/src/change_tracking.rs | 77 ++++++ studio/src/lib.rs | 3 + studio/src/scene.rs | 25 +- studio/src/wasm/mod.rs | 4 + studio/src/wasm/screenshot.rs | 161 ++++++++++++ tsconfig.base.json | 7 + vite-multiple-assets/README.md | 5 + vite-multiple-assets/package.json | 6 + vite-multiple-assets/project.json | 8 + vite-multiple-assets/src/index.ts | 16 ++ vite-multiple-assets/src/lib/build.ts | 25 ++ vite-multiple-assets/src/lib/server.ts | 93 +++++++ vite-multiple-assets/tsconfig.json | 22 ++ vite-multiple-assets/tsconfig.lib.json | 10 + vite-multiple-assets/tsconfig.spec.json | 25 ++ 71 files changed, 1331 insertions(+), 579 deletions(-) create mode 100644 image-generator-demo/.swcrc create mode 100644 image-generator-demo/index.html create mode 100644 image-generator-demo/project.json create mode 100644 image-generator-demo/public/favicon.ico create mode 100644 image-generator-demo/src/app/app.element.scss create mode 100644 image-generator-demo/src/app/app.element.ts create mode 100644 image-generator-demo/src/main.ts create mode 100644 image-generator-demo/src/styles.scss create mode 100644 image-generator-demo/tsconfig.app.json create mode 100644 image-generator-demo/tsconfig.json create mode 100644 image-generator-demo/typings/scss.d.ts create mode 100644 image-generator-demo/typings/svg.d.ts create mode 100644 image-generator-demo/vite.config.ts delete mode 100644 react-frontend/vite-plugins/multiple-assets/build.ts delete mode 100644 react-frontend/vite-plugins/multiple-assets/index.ts delete mode 100644 react-frontend/vite-plugins/multiple-assets/server.ts delete mode 100644 studio-frontend/vite-plugins/multiple-assets/build.ts delete mode 100644 studio-frontend/vite-plugins/multiple-assets/index.ts delete mode 100644 studio-frontend/vite-plugins/multiple-assets/server.ts create mode 100644 studio-ui/README.md create mode 100644 studio-ui/package.json create mode 100644 studio-ui/project.json create mode 100644 studio-ui/src/index.ts rename {studio-frontend/src/app => studio-ui/src/lib}/button.element.scss (100%) rename {studio-frontend/src/app => studio-ui/src/lib}/button.element.ts (100%) rename {studio-frontend/src/app => studio-ui/src/lib}/icon.element.scss (100%) rename {studio-frontend/src/app => studio-ui/src/lib}/icon.element.ts (100%) rename {studio-frontend/src/app => studio-ui/src/lib}/toolbar-group.element.scss (100%) rename {studio-frontend/src/app => studio-ui/src/lib}/toolbar.element.scss (100%) rename {studio-frontend/src/app => studio-ui/src/lib}/toolbar.element.ts (100%) rename {studio-frontend/src/app => studio-ui/src/lib}/tree.element.scss (100%) rename {studio-frontend/src/app => studio-ui/src/lib}/tree.element.ts (97%) create mode 100644 studio-ui/tsconfig.json create mode 100644 studio-ui/tsconfig.lib.json create mode 100644 studio-ui/tsconfig.spec.json create mode 100644 studio-ui/vite.config.ts create mode 100644 studio-web/src/lib/scene-hierarchy.element.scss create mode 100644 studio-web/src/lib/scene-hierarchy.element.ts rename {studio-frontend/src/app => studio-web/src/lib}/transform-toolbar.element.scss (100%) rename {studio-frontend/src/app => studio-web/src/lib}/transform-toolbar.element.ts (72%) create mode 100644 studio-web/src/lib/url-params.ts create mode 100644 studio/src/change_tracking.rs create mode 100644 studio/src/wasm/screenshot.rs create mode 100644 vite-multiple-assets/README.md create mode 100644 vite-multiple-assets/package.json create mode 100644 vite-multiple-assets/project.json create mode 100644 vite-multiple-assets/src/index.ts create mode 100644 vite-multiple-assets/src/lib/build.ts create mode 100644 vite-multiple-assets/src/lib/server.ts create mode 100644 vite-multiple-assets/tsconfig.json create mode 100644 vite-multiple-assets/tsconfig.lib.json create mode 100644 vite-multiple-assets/tsconfig.spec.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 645d66b..4f9b6e6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,7 +5,8 @@ "*.tsx": "${capture}.module.scss, ${capture}.spec.tsx", }, "rust-analyzer.cargo.features": [ - "wasm" + "wasm", + "wasm_screenshot" ], "files.associations": { "*.meta": "ron" diff --git a/editor.ron b/editor.ron index 2f5962d..5ec76bc 100644 --- a/editor.ron +++ b/editor.ron @@ -1,6 +1,6 @@ { + "space_editor_core::hotkeys::HotkeySet": "{\"space_editor_core::hotkeys::HotkeySet\":(bindings:{Rotate:[R],Translate:[G],Scale:[S],Multiple:[ShiftLeft],Clone:[AltLeft],Delete:[X]},name:\"GizmoHotkey\")}", "space_undo::ChangeChainSettings": "{\"space_undo::ChangeChainSettings\":(max_change_chain_size:200)}", "space_persistence::PersistenceSettings": "{\"space_persistence::PersistenceSettings\":(load_on_startup:true,save_on_close:true)}", - "space_editor_core::hotkeys::HotkeySet": "{\"space_editor_core::hotkeys::HotkeySet\":(bindings:{Rotate:[R],Translate:[G],Scale:[S],Multiple:[ShiftLeft],Clone:[AltLeft],Delete:[X]},name:\"GizmoHotkey\")}", "space_editor_ui::settings::NewWindowSettings": "{\"space_editor_ui::settings::NewWindowSettings\":(new_tab:SameNode)}", } \ No newline at end of file diff --git a/image-generator-demo/.swcrc b/image-generator-demo/.swcrc new file mode 100644 index 0000000..f83e5db --- /dev/null +++ b/image-generator-demo/.swcrc @@ -0,0 +1,8 @@ +{ + "jsc": { + "parser": { + "syntax": "typescript" + }, + "target": "ES2020" + } +} diff --git a/image-generator-demo/index.html b/image-generator-demo/index.html new file mode 100644 index 0000000..190aac3 --- /dev/null +++ b/image-generator-demo/index.html @@ -0,0 +1,19 @@ + + + + + Image Generator Demo + + + + + + + + + + + + + + diff --git a/image-generator-demo/project.json b/image-generator-demo/project.json new file mode 100644 index 0000000..719f5b8 --- /dev/null +++ b/image-generator-demo/project.json @@ -0,0 +1,71 @@ +{ + "name": "image-generator-demo", + "$schema": "../node_modules/nx/schemas/project-schema.json", + "projectType": "application", + "sourceRoot": "image-generator-demo/src", + "tags": [], + "targets": { + "build": { + "executor": "@nx/vite:build", + "outputs": [ + "{options.outputPath}" + ], + "defaultConfiguration": "production", + "options": { + "outputPath": "dist/image-generator-demo" + }, + "configurations": { + "development": { + "mode": "development" + }, + "production": { + "mode": "production" + } + } + }, + "serve": { + "executor": "@nx/vite:dev-server", + "defaultConfiguration": "development", + "dependsOn": [ + { + "projects": ["studio"], + "target": "build:image-generator" + } + ], + "options": { + "buildTarget": "image-generator-demo:build" + }, + "configurations": { + "development": { + "buildTarget": "image-generator-demo:build:development", + "hmr": true + }, + "production": { + "buildTarget": "image-generator-demo:build:production", + "hmr": false + } + } + }, + "preview": { + "executor": "@nx/vite:preview-server", + "defaultConfiguration": "production", + "dependsOn": [ + { + "projects": ["studio"], + "target": "build:image-generator" + } + ], + "options": { + "buildTarget": "image-generator-demo:build" + }, + "configurations": { + "development": { + "buildTarget": "image-generator-demo:build:development" + }, + "production": { + "buildTarget": "image-generator-demo:build:production" + } + } + } + } +} diff --git a/image-generator-demo/public/favicon.ico b/image-generator-demo/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..317ebcb2336e0833a22dddf0ab287849f26fda57 GIT binary patch literal 15086 zcmeI332;U^%p|z7g|#(P)qFEA@4f!_@qOK2 z_lJl}!lhL!VT_U|uN7%8B2iKH??xhDa;*`g{yjTFWHvXn;2s{4R7kH|pKGdy(7z!K zgftM+Ku7~24TLlh(!g)gz|foI94G^t2^IO$uvX$3(OR0<_5L2sB)lMAMy|+`xodJ{ z_Uh_1m)~h?a;2W{dmhM;u!YGo=)OdmId_B<%^V^{ovI@y`7^g1_V9G}*f# zNzAtvou}I!W1#{M^@ROc(BZ! z+F!!_aR&Px3_reO(EW+TwlW~tv*2zr?iP7(d~a~yA|@*a89IUke+c472NXM0wiX{- zl`UrZC^1XYyf%1u)-Y)jj9;MZ!SLfd2Hl?o|80Su%Z?To_=^g_Jt0oa#CT*tjx>BI z16wec&AOWNK<#i0Qd=1O$fymLRoUR*%;h@*@v7}wApDl^w*h}!sYq%kw+DKDY)@&A z@9$ULEB3qkR#85`lb8#WZw=@})#kQig9oqy^I$dj&k4jU&^2(M3q{n1AKeGUKPFbr z1^<)aH;VsG@J|B&l>UtU#Ejv3GIqERzYgL@UOAWtW<{p#zy`WyJgpCy8$c_e%wYJL zyGHRRx38)HyjU3y{-4z6)pzb>&Q1pR)B&u01F-|&Gx4EZWK$nkUkOI|(D4UHOXg_- zw{OBf!oWQUn)Pe(=f=nt=zkmdjpO^o8ZZ9o_|4tW1ni+Un9iCW47*-ut$KQOww!;u z`0q)$s6IZO!~9$e_P9X!hqLxu`fpcL|2f^I5d4*a@Dq28;@2271v_N+5HqYZ>x;&O z05*7JT)mUe&%S0@UD)@&8SmQrMtsDfZT;fkdA!r(S=}Oz>iP)w=W508=Rc#nNn7ym z1;42c|8($ALY8#a({%1#IXbWn9-Y|0eDY$_L&j{63?{?AH{);EzcqfydD$@-B`Y3<%IIj7S7rK_N}je^=dEk%JQ4c z!tBdTPE3Tse;oYF>cnrapWq*o)m47X1`~6@(!Y29#>-#8zm&LXrXa(3=7Z)ElaQqj z-#0JJy3Fi(C#Rx(`=VXtJ63E2_bZGCz+QRa{W0e2(m3sI?LOcUBx)~^YCqZ{XEPX)C>G>U4tfqeH8L(3|pQR*zbL1 zT9e~4Tb5p9_G}$y4t`i*4t_Mr9QYvL9C&Ah*}t`q*}S+VYh0M6GxTTSXI)hMpMpIq zD1ImYqJLzbj0}~EpE-aH#VCH_udYEW#`P2zYmi&xSPs_{n6tBj=MY|-XrA;SGA_>y zGtU$?HXm$gYj*!N)_nQ59%lQdXtQZS3*#PC-{iB_sm+ytD*7j`D*k(P&IH2GHT}Eh z5697eQECVIGQAUe#eU2I!yI&%0CP#>%6MWV z@zS!p@+Y1i1b^QuuEF*13CuB zu69dve5k7&Wgb+^s|UB08Dr3u`h@yM0NTj4h7MnHo-4@xmyr7(*4$rpPwsCDZ@2be zRz9V^GnV;;?^Lk%ynzq&K(Aix`mWmW`^152Hoy$CTYVehpD-S1-W^#k#{0^L`V6CN+E z!w+xte;2vu4AmVNEFUOBmrBL>6MK@!O2*N|2=d|Y;oN&A&qv=qKn73lDD zI(+oJAdgv>Yr}8(&@ZuAZE%XUXmX(U!N+Z_sjL<1vjy1R+1IeHt`79fnYdOL{$ci7 z%3f0A*;Zt@ED&Gjm|OFTYBDe%bbo*xXAQsFz+Q`fVBH!N2)kaxN8P$c>sp~QXnv>b zwq=W3&Mtmih7xkR$YA)1Yi?avHNR6C99!u6fh=cL|KQ&PwF!n@ud^n(HNIImHD!h87!i*t?G|p0o+eelJ?B@A64_9%SBhNaJ64EvKgD&%LjLCYnNfc; znj?%*p@*?dq#NqcQFmmX($wms@CSAr9#>hUR^=I+=0B)vvGX%T&#h$kmX*s=^M2E!@N9#m?LhMvz}YB+kd zG~mbP|D(;{s_#;hsKK9lbVK&Lo734x7SIFJ9V_}2$@q?zm^7?*XH94w5Qae{7zOMUF z^?%F%)c1Y)Q?Iy?I>knw*8gYW#ok|2gdS=YYZLiD=CW|Nj;n^x!=S#iJ#`~Ld79+xXpVmUK^B(xO_vO!btA9y7w3L3-0j-y4 z?M-V{%z;JI`bk7yFDcP}OcCd*{Q9S5$iGA7*E1@tfkyjAi!;wP^O71cZ^Ep)qrQ)N z#wqw0_HS;T7x3y|`P==i3hEwK%|>fZ)c&@kgKO1~5<5xBSk?iZV?KI6&i72H6S9A* z=U(*e)EqEs?Oc04)V-~K5AUmh|62H4*`UAtItO$O(q5?6jj+K^oD!04r=6#dsxp?~}{`?&sXn#q2 zGuY~7>O2=!u@@Kfu7q=W*4egu@qPMRM>(eyYyaIE<|j%d=iWNdGsx%c!902v#ngNg z@#U-O_4xN$s_9?(`{>{>7~-6FgWpBpqXb`Ydc3OFL#&I}Irse9F_8R@4zSS*Y*o*B zXL?6*Aw!AfkNCgcr#*yj&p3ZDe2y>v$>FUdKIy_2N~}6AbHc7gA3`6$g@1o|dE>vz z4pl(j9;kyMsjaw}lO?(?Xg%4k!5%^t#@5n=WVc&JRa+XT$~#@rldvN3S1rEpU$;XgxVny7mki3 z-Hh|jUCHrUXuLr!)`w>wgO0N%KTB-1di>cj(x3Bav`7v z3G7EIbU$z>`Nad7Rk_&OT-W{;qg)-GXV-aJT#(ozdmnA~Rq3GQ_3mby(>q6Ocb-RgTUhTN)))x>m&eD;$J5Bg zo&DhY36Yg=J=$Z>t}RJ>o|@hAcwWzN#r(WJ52^g$lh^!63@hh+dR$&_dEGu&^CR*< z!oFqSqO@>xZ*nC2oiOd0eS*F^IL~W-rsrO`J`ej{=ou_q^_(<$&-3f^J z&L^MSYWIe{&pYq&9eGaArA~*kA studio.requestScreenshot(), this.Wait, config); + } +} + +@customElement("app-root") +export class AppElement extends LitElement { + @state() loading = true; + @state() imageCaptureUrl?: string; + @state() realtimeCapture = true; + + params = new StudioURLParams(); + viewer = createRef(); + + protected override createRenderRoot = () => this; + + override disconnectedCallback(): void { + if (this.imageCaptureUrl != null) + URL.revokeObjectURL(this.imageCaptureUrl); + + super.disconnectedCallback(); + } + + @on("scene-state") + onSceneStateChange({ detail: state }: SceneStateEvent): void { + console.log("scene state change:", SceneState[state]); + + this.loading = match (state, { + [SceneState.Active]: () => false, + _: () => true, + }); + } + + @on("entity-changes") + onStudioChanges(): void { + this.requestScreenshot(); + } + + requestScreenshot = Debounce.func(Debounce.Realtime); + + @on("screenshot-ready") + onScreenshotReady({ detail: screenshot }: ScreenshotReadyEvent) { + const prevUrl = this.imageCaptureUrl; + + const blob = new Blob([screenshot.data.buffer], { type: "image/png" }); + this.imageCaptureUrl = URL.createObjectURL(blob); + console.log(`New blob URL: ${this.imageCaptureUrl}`); + + if (prevUrl != null) + URL.revokeObjectURL(prevUrl); + } + + toggleRealtimeCapture(event: InputEvent): void { + const { checked } = event.target as HTMLInputElement; + this.realtimeCapture = checked; + + const config = checked ? Debounce.Realtime : Debounce.Delayed; + this.requestScreenshot = Debounce.func(config); + } + + protected override render = () => html` + + + + ${this.imageCaptureUrl != null ? html` + Image capture + ` : nothing} + + + +
+ +
+ + ${this.loading ? html` + +
+

Loading

+
+ ` : nothing} +
+ ` +} diff --git a/image-generator-demo/src/main.ts b/image-generator-demo/src/main.ts new file mode 100644 index 0000000..cde5d45 --- /dev/null +++ b/image-generator-demo/src/main.ts @@ -0,0 +1 @@ +import "./app/app.element"; diff --git a/image-generator-demo/src/styles.scss b/image-generator-demo/src/styles.scss new file mode 100644 index 0000000..3782467 --- /dev/null +++ b/image-generator-demo/src/styles.scss @@ -0,0 +1,10 @@ +*, *::before, *::after { + box-sizing: border-box; +} + +html, body { + width: 100%; + height: 100%; + margin: 0; + overflow: hidden; +} diff --git a/image-generator-demo/tsconfig.app.json b/image-generator-demo/tsconfig.app.json new file mode 100644 index 0000000..433a657 --- /dev/null +++ b/image-generator-demo/tsconfig.app.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../dist/out-tsc", + "types": ["node"] + }, + "exclude": [ + "src/**/*.spec.ts", + "src/**/*.test.ts" + ], + "include": [ + "src/**/*.ts", + "typings/*.d.ts" + ] +} diff --git a/image-generator-demo/tsconfig.json b/image-generator-demo/tsconfig.json new file mode 100644 index 0000000..724ccfc --- /dev/null +++ b/image-generator-demo/tsconfig.json @@ -0,0 +1,34 @@ +{ + "extends": "../tsconfig.base.json", + "files": [], + "compilerOptions": { + "target": "ES2020", + "experimentalDecorators": true, + "useDefineForClassFields": false, + "module": "ESNext", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "skipLibCheck": true, + + "moduleResolution": "Bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "esModuleInterop": true, + + "strict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitAny": true, + "noImplicitOverride": true, + "noImplicitReturns": true, + + "types": ["vite/client"] + }, + "include": ["src"], + "references": [ + { + "path": "./tsconfig.app.json" + } + ] +} diff --git a/image-generator-demo/typings/scss.d.ts b/image-generator-demo/typings/scss.d.ts new file mode 100644 index 0000000..c057e6a --- /dev/null +++ b/image-generator-demo/typings/scss.d.ts @@ -0,0 +1,4 @@ +declare module "*.scss?inline" { + const styles: import("lit").CSSResult; + export default styles; +} diff --git a/image-generator-demo/typings/svg.d.ts b/image-generator-demo/typings/svg.d.ts new file mode 100644 index 0000000..bc2f8ce --- /dev/null +++ b/image-generator-demo/typings/svg.d.ts @@ -0,0 +1,4 @@ +declare module "*.svg?raw" { + const svg: string; + export default svg; +} diff --git a/image-generator-demo/vite.config.ts b/image-generator-demo/vite.config.ts new file mode 100644 index 0000000..d9af530 --- /dev/null +++ b/image-generator-demo/vite.config.ts @@ -0,0 +1,29 @@ + +/// +import { defineConfig } from "vite"; + +import { nxViteTsPaths } from "@nx/vite/plugins/nx-tsconfig-paths.plugin"; +import wasm from "vite-plugin-wasm"; +import topLevelAwait from "vite-plugin-top-level-await"; +import mergeAssetsDirs from "../vite-multiple-assets"; + +export default defineConfig({ + cacheDir: "../node_modules/.vite/image-generator-demo", + server:{ + port: 4200, + host: "localhost", + }, + preview:{ + port: 4300, + host: "localhost", + }, + plugins: [ + nxViteTsPaths(), + wasm(), + topLevelAwait(), + mergeAssetsDirs([ + "image-generator-demo/public", + "dist/studio/public", + ]), + ], +}); diff --git a/react-frontend/vite-plugins/multiple-assets/build.ts b/react-frontend/vite-plugins/multiple-assets/build.ts deleted file mode 100644 index b478c9d..0000000 --- a/react-frontend/vite-plugins/multiple-assets/build.ts +++ /dev/null @@ -1,25 +0,0 @@ -import fs from "fs"; -import {NormalizedOutputOptions} from "rollup"; - -function copyRecursively(src: string, dst: string) { - if (fs.statSync(src).isDirectory()) { - // folder - if (!fs.existsSync(dst)) { - fs.mkdirSync(dst); - } - for (const file of fs.readdirSync(src)) { - copyRecursively(src + "/" + file, dst + "/" + file); - } - } else { - // file - fs.copyFileSync(src, dst); - } -} - -export function buildMiddleWare(options: NormalizedOutputOptions, assets: string[] = []) { - const dst = options.dir; - if (!dst) throw new Error("options.dir is undefined"); - for (const dir of assets) { - copyRecursively(dir, dst) - } -} diff --git a/react-frontend/vite-plugins/multiple-assets/index.ts b/react-frontend/vite-plugins/multiple-assets/index.ts deleted file mode 100644 index bef1d09..0000000 --- a/react-frontend/vite-plugins/multiple-assets/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -// FIXME: Oh my god, did Danny seriously just copy and paste this entire folder -// from `studio-frontend` instead of creating a separate project for it -// and importing it like a decent, law-abiding citizen? Wow, dude really -// needs to get his shit together. - -// NOTE: This is an inline fork of this npm package: -// https://github.com/nguyenbatranvan/vite-multiple-assets -// -// The published version at time of writing doesn't include mime types for many -// of the asset types Bevy needs to load. - -import {PluginOption, ViteDevServer} from "vite"; -import {ServerMiddleWare} from "./server"; -import {buildMiddleWare} from "./build"; - -export default function DynamicPublicDirectory(assets: string[]): PluginOption { - return { - configureServer(server: ViteDevServer) { - return ServerMiddleWare(server, assets) - }, - async writeBundle(options) { - buildMiddleWare(options, assets) - }, - name: "dynamic assets" - }; -} diff --git a/react-frontend/vite-plugins/multiple-assets/server.ts b/react-frontend/vite-plugins/multiple-assets/server.ts deleted file mode 100644 index 61455c4..0000000 --- a/react-frontend/vite-plugins/multiple-assets/server.ts +++ /dev/null @@ -1,91 +0,0 @@ -import fs from "fs"; -import path from "path"; -import { ViteDevServer } from "vite"; - -interface IPropsFile { - name: string; - files: string[]; -} - -function getFiles(dir, files_): string[] { - files_ = files_ || []; - const files = fs.readdirSync(dir); - for (const i in files) { - const name = dir + "/" + files[i]; - if (fs.statSync(name).isDirectory()) { - getFiles(name, files_); - } else { - files_.push(name); - } - } - return files_; -} - -const mimeTypes = { - ".7z": "application/x-7z-compressed", - ".bin": "application/octet-stream", - ".css": "text/css", - ".eot": "font/eot", - ".gif": "image/gif", - ".gltf": "model/gltf+json", - ".gz": "application/gzip", - ".hdr": "image/vnd.radiance", - ".html": "text/html", - ".ico": "image/x-icon", - ".jpg": "image/jpeg", - ".js": "text/javascript", - ".json": "application/json", - ".ktx": "image/ktx", - ".ktx2": "image/ktx", - ".meta": "text/plain", - ".mjs": "text/javascript", - ".mp3": "audio/mpeg", - ".mp4": "video/mp4", - ".otf": "font/otf", - ".png": "image/png", - ".rar": "application/x-rar-compressed", - ".svg": "image/svg+xml", - ".tiff": "image/tiff", - ".ttf": "font/ttf", - ".txt": "text/plain", - ".wasm": "application/wasm", - ".webm": "video/webm", - ".webp": "image/webp", - ".wgsl": "text/wgsl", - ".woff": "font/woff", - ".woff2": "font/woff2", - ".xml": "text/xml", - ".zip": "application/zip", -} - -export function ServerMiddleWare(server: ViteDevServer, assets: string[] = []) { - if (!assets || !assets.length) - return; - - const fileObject: IPropsFile[] = []; - for (let i = 0; i < assets.length; i++) { - const files = getFiles(path.join(process.cwd(), `/${assets[i]}`), []); - fileObject.push({ - name: assets[i], - files - }); - } - - return () => { - server.middlewares.use(async (req, res, next) => { - for (let i = 0; i < fileObject.length; i++) { - const file = path.join(process.cwd(), `${fileObject[i].name}/${req.originalUrl}`); - if (fileObject[i].files.some(f => path.relative(f, file) === "")) { - const extension = file.substring(file.lastIndexOf(".")); - res.setHeader("Cache-Control", "max-age=31536000, immutable"); - res.setHeader("Content-Type", mimeTypes[extension]); - res.writeHead(200); - res.write(fs.readFileSync(path.join(process.cwd(), "/" + fileObject[i].name + req.originalUrl))); - res.end(); - break; - } - } - next(); - }); - }; -} diff --git a/react-frontend/vite.config.ts b/react-frontend/vite.config.ts index 483252c..248a00b 100644 --- a/react-frontend/vite.config.ts +++ b/react-frontend/vite.config.ts @@ -3,8 +3,7 @@ import react from "@vitejs/plugin-react"; import { defineConfig } from "vite"; import topLevelAwait from "vite-plugin-top-level-await"; import wasm from "vite-plugin-wasm"; - -import mergeAssetsDirs from "./vite-plugins/multiple-assets"; +import mergeAssetsDirs from "../vite-multiple-assets"; export default defineConfig({ cacheDir: "../node_modules/.vite/react-frontend", diff --git a/studio-frontend/src/app/anim-editor.element.ts b/studio-frontend/src/app/anim-editor.element.ts index b108990..47500ed 100644 --- a/studio-frontend/src/app/anim-editor.element.ts +++ b/studio-frontend/src/app/anim-editor.element.ts @@ -5,10 +5,10 @@ import { LitElement, type PropertyValues, html, unsafeCSS } from "lit"; import { customElement, state } from "lit/decorators.js"; import { styleMap } from "lit/directives/style-map.js"; +import "@storyteller/studio-ui/button"; +import "@storyteller/studio-ui/toolbar"; +import "@storyteller/studio-ui/tree"; import "./anim-timeline.element"; -import "./button.element"; -import "./toolbar.element"; -import "./tree.element"; import styles from "./anim-editor.element.scss?inline"; diff --git a/studio-frontend/src/app/app.element.scss b/studio-frontend/src/app/app.element.scss index fc4a85b..f580b98 100644 --- a/studio-frontend/src/app/app.element.scss +++ b/studio-frontend/src/app/app.element.scss @@ -35,33 +35,9 @@ sts-button.active { // Hierarchy (TODO: Probably move this) .hierarchy { - overflow: auto; - - box-sizing: border-box; width: $hierarchy-width; max-height: calc(100vh - 32px); - padding: 10px 15px; - - background: #0006; - backdrop-filter: blur(16px); - border: 8px solid transparent; - border-radius: 0.5rem; - pointer-events: auto; - - &::-webkit-scrollbar { - width: 8px; - height: 8px; - - &-thumb { - border-radius: 2px; - background: #FFF2; - - &:hover { - background: #FFF3; - } - } - } } .loading { diff --git a/studio-frontend/src/app/app.element.ts b/studio-frontend/src/app/app.element.ts index f2ad003..a9c4685 100644 --- a/studio-frontend/src/app/app.element.ts +++ b/studio-frontend/src/app/app.element.ts @@ -1,17 +1,12 @@ import { on } from "@storyteller/framework"; import * as studio from "@storyteller/studio"; import { - type EntityDespawnEvent, - type EntityMultiSpawnEvent, - type EntitySelectEvent, - type EntitySpawnEvent, type SceneSavedEvent, type SceneStateEvent, - TransformSpace, - TransformType, SceneState, StudioMode, } from "@storyteller/studio"; +import { StudioURLParams } from "@storyteller/studio-web"; import { CanvasProvider } from "@storyteller/studio-web/canvas-provider"; import { match } from "@storyteller/utility"; import { LitElement, html, nothing } from "lit"; @@ -20,144 +15,35 @@ import { classMap } from "lit-html/directives/class-map.js"; import { Ref, createRef, ref } from "lit-html/directives/ref.js"; import { v4 as uuid } from "uuid"; +import "@storyteller/studio-ui/button"; +import "@storyteller/studio-ui/icon"; +import "@storyteller/studio-ui/toolbar"; +import "@storyteller/studio-ui/tree"; import "@storyteller/studio-web/canvas-provider"; import "@storyteller/studio-web/viewer"; import "@storyteller/studio-web/bvh-viewer"; import "@storyteller/studio-web/editor"; import "@storyteller/studio-web/mixamo-viewer"; +import "@storyteller/studio-web/scene-hierarchy"; +import "@storyteller/studio-web/transform-toolbar"; import "./anim-editor.element"; -import "./button.element"; import "./help.element"; -import "./icon.element"; -import "./toolbar.element"; -import "./transform-toolbar.element"; -import "./tree.element"; import "./app.element.scss"; -interface SceneObject extends Omit { - children: SceneObject[]; -} - @customElement("app-root") export class AppElement extends LitElement { @state() showHelp = false; @state() loading = true; - @state() transformSpace = TransformSpace.Local; - @state() transformType = TransformType.Translate; - - @state() sceneObjects: SceneObject[] = []; - @state() selection?: string | null; - - @state() bvhPath?: string; - @state() mixamoPath?: string; - @state() scenePath?: string; - @state() mode?: StudioMode; - - #sceneObjects: studio.SceneObject[] = []; - #entityMap = new Map(); + #params = new StudioURLParams(); #canvasProvider: Ref = createRef(); - get #studioCanvas(): HTMLCanvasElement | null { - const provider = this.#canvasProvider.value; - if (!provider) return null; - - return document.getElementById(provider.canvasId) as HTMLCanvasElement | null; - } - - override connectedCallback(): void { - super.connectedCallback(); - - const params = new URLSearchParams(document.location.search); - - if (params.has("bvh")) - this.bvhPath = params.get("bvh")!; - - if (params.has("mixamo")) - this.mixamoPath = params.get("mixamo")!; - - if (params.has("scene")) - this.scenePath = params.get("scene")!; - - this.mode = match (params.get("mode"), { - "studio": () => StudioMode.Editor, - "viewer": () => StudioMode.Viewer, - _: () => StudioMode.Viewer, - }); - - this.#redirectBodyFocus(); - } - protected override createRenderRoot(): Element | ShadowRoot { return this; } - @on("keydown") - @on("keyup") - pipeKeyboardEvents(event: KeyboardEvent): void { - // When the user interacts with one of the web UI elements, the Bevy - // canvas will lose focus, preventing it from receiving keyboard events. - // As a workaround, we can listen for any events that have bubbled up to - // the app root and repeat them for the canvas. - // - // This does unfortunately burden the web front-end with some additional - // responsibility: when we handle a keyboard event and _don't_ want the - // Bevy app to receive it, we'll need to call `event.stopPropagation()` in - // the handler to prevent it from reaching this listener. - const { - type, - code, charCode, key, keyCode, location, repeat, - altKey, ctrlKey, metaKey, shiftKey, - } = event; - - this.#studioCanvas?.dispatchEvent(new KeyboardEvent(type, { - code, charCode, key, keyCode, location, repeat, - altKey, ctrlKey, metaKey, shiftKey, - bubbles: true, - cancelable: true, - composed: true, - })); - } - - onSceneStateChange({ detail: state }: SceneStateEvent): void { - console.log("scene state change:", SceneState[state]); - - this.loading = match (state, { - [SceneState.Active]: () => false, - _: () => true, - }); - } - - @on("window:entity-select") - onEntitySelectionChange({ detail: entity }: EntitySelectEvent): void { - const sceneObject = this.#entityMap.get(entity!); - console.log("Entity selection changed:", sceneObject); - this.selection = entity; - } - - @on("window:entity-spawn") - onEntitySpawned({ detail: value }: EntitySpawnEvent): void { - console.log("Entity spawned:", value); - this.#sceneObjects = this.#sceneObjects.concat(value).sort(sortByName); - this.#updateHierarchy(); - } - - @on("window:entity-multi-spawn") - onEntitiesSpawned({ detail: values }: EntityMultiSpawnEvent): void { - console.log("Entities spawned:", values); - this.#sceneObjects = this.#sceneObjects.concat(values).sort(sortByName); - this.#updateHierarchy(); - } - - @on("window:entity-despawn") - onEntityDespawned({ detail: entity }: EntityDespawnEvent): void { - console.log("Entity despawned:", entity); - this.#sceneObjects = this.#sceneObjects.filter(({ id }) => id !== entity); - this.#updateHierarchy(); - } - - @on("window:scene-saved") + @on("scene-saved") async onSceneSaved({ detail: scene }: SceneSavedEvent): Promise { const file = new File([scene], "scene.scn.ron", { type: "text/ron" }); const formData = new FormData(); @@ -179,98 +65,52 @@ export class AppElement extends LitElement { } } + @on("scene-state") + onSceneStateChange({ detail: state }: SceneStateEvent): void { + console.log("scene state change:", SceneState[state]); + + this.loading = match (state, { + [SceneState.Active]: () => false, + _: () => true, + }); + } + toggleHelp(): void { this.showHelp = !this.showHelp; } - onTransformSpaceChange({ detail: space }: CustomEvent): void { - this.transformSpace = space; - studio.setTransformSpace(space); - } - - onTransformTypeChange({ detail: type }: CustomEvent): void { - this.transformType = type; - studio.setTransformType(type); - } - - // TODO: Get some help from someone who knows what an algorithm is - #updateHierarchy(): void { - this.#entityMap.clear(); - - for (let obj of this.#sceneObjects) - this.#entityMap.set(obj.id, { ...obj, children: [] }); - - for (let obj of this.#entityMap.values()) { - if (obj.parent != null) { - const parent = this.#entityMap.get(obj.parent); - if (!parent) - throw new Error(`Missing parent: ${obj.parent}`); - - parent.children.push(obj); - } - } - - for (let obj of this.#entityMap.values()) - obj.children.sort(sortByName); - - this.sceneObjects = Array - .from(this.#entityMap.values()) - .filter(obj => obj.parent == null) - .sort(sortByName); - - console.log("Hierarchy:", this.sceneObjects); - } - - #redirectBodyFocus(): void { - // If user focus lands on the `document.body`, neither the frontend app - // nor the Bevy app will properly handle keyboard events. Thie function - // runs every tick to ensure that user focus never becomes "stranded" in - // this way. - if (document.activeElement === document.body) { - const canvasId = this.#canvasProvider.value?.canvasId; - if (canvasId) - document.getElementById(canvasId)?.focus(); - } - - requestAnimationFrame(() => this.#redirectBodyFocus()); - } - protected override render = () => html` - ${this.bvhPath ? html` + ${this.#params.bvh ? html` - ` : this.mixamoPath ? html` + ` : this.#params.mixamo ? html` - ` : this.scenePath ? html` + ` : this.#params.scene ? html` ` : html` `} - ${this.mode === StudioMode.Editor ? html` + ${this.#params.mode === StudioMode.Editor ? html` studio.saveScene()} @@ -288,30 +128,10 @@ export class AppElement extends LitElement { - ${this.mode === StudioMode.Editor ? html` - - - - -
- ${this.sceneObjects.map(obj => html` - - `)} -
- ` : nothing} - - - ${this.mode === StudioMode.Editor ? html` + + ${this.#params.mode === StudioMode.Editor ? html` + + ` : nothing} @@ -329,7 +149,3 @@ export class AppElement extends LitElement {
`; } - -function sortByName(a: { name: string }, b: { name: string }): number { - return a.name.localeCompare(b.name); -} diff --git a/studio-frontend/vite-plugins/multiple-assets/build.ts b/studio-frontend/vite-plugins/multiple-assets/build.ts deleted file mode 100644 index b478c9d..0000000 --- a/studio-frontend/vite-plugins/multiple-assets/build.ts +++ /dev/null @@ -1,25 +0,0 @@ -import fs from "fs"; -import {NormalizedOutputOptions} from "rollup"; - -function copyRecursively(src: string, dst: string) { - if (fs.statSync(src).isDirectory()) { - // folder - if (!fs.existsSync(dst)) { - fs.mkdirSync(dst); - } - for (const file of fs.readdirSync(src)) { - copyRecursively(src + "/" + file, dst + "/" + file); - } - } else { - // file - fs.copyFileSync(src, dst); - } -} - -export function buildMiddleWare(options: NormalizedOutputOptions, assets: string[] = []) { - const dst = options.dir; - if (!dst) throw new Error("options.dir is undefined"); - for (const dir of assets) { - copyRecursively(dir, dst) - } -} diff --git a/studio-frontend/vite-plugins/multiple-assets/index.ts b/studio-frontend/vite-plugins/multiple-assets/index.ts deleted file mode 100644 index 066c88a..0000000 --- a/studio-frontend/vite-plugins/multiple-assets/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -// NOTE: This is an inline fork of this npm package: -// https://github.com/nguyenbatranvan/vite-multiple-assets -// -// The published version at time of writing doesn't include a mime type for -// WGSL, which is fixed in this copy. This module can be removed and replaced -// with the npm dependency if/when this PR is accepted: -// https://github.com/nguyenbatranvan/vite-multiple-assets/pull/5 - -import {PluginOption, ViteDevServer} from "vite"; -import {ServerMiddleWare} from "./server"; -import {buildMiddleWare} from "./build"; - -export default function DynamicPublicDirectory(assets: string[]): PluginOption { - return { - configureServer(server: ViteDevServer) { - return ServerMiddleWare(server, assets) - }, - async writeBundle(options) { - buildMiddleWare(options, assets) - }, - name: "dynamic assets" - }; -} diff --git a/studio-frontend/vite-plugins/multiple-assets/server.ts b/studio-frontend/vite-plugins/multiple-assets/server.ts deleted file mode 100644 index de26582..0000000 --- a/studio-frontend/vite-plugins/multiple-assets/server.ts +++ /dev/null @@ -1,93 +0,0 @@ -import fs from "fs"; -import path from "path"; -import { ViteDevServer } from "vite"; - -interface IPropsFile { - name: string; - files: string[]; -} - -function getFiles(dir, files_): string[] { - files_ = files_ || []; - const files = fs.readdirSync(dir); - for (const i in files) { - const name = dir + "/" + files[i]; - if (fs.statSync(name).isDirectory()) { - getFiles(name, files_); - } else { - files_.push(name); - } - } - return files_; -} - -const mimeTypes = { - ".7z": "application/x-7z-compressed", - ".bin": "application/octet-stream", - ".bvh": "text/bvh", - ".css": "text/css", - ".eot": "font/eot", - ".gif": "image/gif", - ".glb": "model/gltf-binary", - ".gltf": "model/gltf+json", - ".gz": "application/gzip", - ".hdr": "image/vnd.radiance", - ".html": "text/html", - ".ico": "image/x-icon", - ".jpg": "image/jpeg", - ".js": "text/javascript", - ".json": "application/json", - ".ktx": "image/ktx", - ".ktx2": "image/ktx", - ".meta": "text/plain", - ".mjs": "text/javascript", - ".mp3": "audio/mpeg", - ".mp4": "video/mp4", - ".otf": "font/otf", - ".png": "image/png", - ".rar": "application/x-rar-compressed", - ".svg": "image/svg+xml", - ".tiff": "image/tiff", - ".ttf": "font/ttf", - ".txt": "text/plain", - ".wasm": "application/wasm", - ".webm": "video/webm", - ".webp": "image/webp", - ".wgsl": "text/wgsl", - ".woff": "font/woff", - ".woff2": "font/woff2", - ".xml": "text/xml", - ".zip": "application/zip", -} - -export function ServerMiddleWare(server: ViteDevServer, assets: string[] = []) { - if (!assets || !assets.length) - return; - - const fileObject: IPropsFile[] = []; - for (let i = 0; i < assets.length; i++) { - const files = getFiles(path.join(process.cwd(), `/${assets[i]}`), []); - fileObject.push({ - name: assets[i], - files - }); - } - - return () => { - server.middlewares.use(async (req, res, next) => { - for (let i = 0; i < fileObject.length; i++) { - const file = path.join(process.cwd(), `${fileObject[i].name}/${req.originalUrl}`); - if (fileObject[i].files.some(f => path.relative(f, file) === "")) { - const extension = file.substring(file.lastIndexOf(".")); - res.setHeader("Cache-Control", "max-age=31536000, immutable"); - res.setHeader("Content-Type", mimeTypes[extension]); - res.writeHead(200); - res.write(fs.readFileSync(path.join(process.cwd(), "/" + fileObject[i].name + req.originalUrl))); - res.end(); - break; - } - } - next(); - }); - }; -} diff --git a/studio-frontend/vite.config.ts b/studio-frontend/vite.config.ts index 4ade41c..2d5913f 100644 --- a/studio-frontend/vite.config.ts +++ b/studio-frontend/vite.config.ts @@ -5,8 +5,7 @@ import { defineConfig } from "vite"; import { nxViteTsPaths } from "@nx/vite/plugins/nx-tsconfig-paths.plugin"; import wasm from "vite-plugin-wasm"; import topLevelAwait from "vite-plugin-top-level-await"; - -import mergeAssetsDirs from "./vite-plugins/multiple-assets"; +import mergeAssetsDirs from "../vite-multiple-assets"; export default defineConfig({ cacheDir: "../node_modules/.vite/studio-frontend", diff --git a/studio-ui/README.md b/studio-ui/README.md new file mode 100644 index 0000000..e3c328d --- /dev/null +++ b/studio-ui/README.md @@ -0,0 +1,12 @@ +# studio-ui + +This is a UI/component library that can be easily shared between multiple +Storyteller Studio frontend applications. + +## Building + +Run `nx build studio-ui` to build the library. + +## Running unit tests + +Run `nx test studio-ui` to execute the unit tests via [Jest](https://jestjs.io). diff --git a/studio-ui/package.json b/studio-ui/package.json new file mode 100644 index 0000000..5812501 --- /dev/null +++ b/studio-ui/package.json @@ -0,0 +1,12 @@ +{ + "name": "@storyteller/studio-ui", + "version": "0.0.1", + "dependencies": { + "@storyteller/framework": "0.0.1", + "@storyteller/studio": "0.0.1", + "lit": "^2.8.0" + }, + "main": "./index.js", + "module": "./index.mjs", + "typings": "./index.d.ts" + } diff --git a/studio-ui/project.json b/studio-ui/project.json new file mode 100644 index 0000000..623beb6 --- /dev/null +++ b/studio-ui/project.json @@ -0,0 +1,30 @@ +{ + "name": "studio-ui", + "$schema": "../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "studio-ui/src", + "projectType": "library", + "targets": { + "build": { + "executor": "@nx/vite:build", + "outputs": ["{options.outputPath}"], + "dependsOn": [ + { + "projects": ["studio"], + "target": "build:web" + } + ], + "options": { + "outputPath": "dist/studio-ui" + } + }, + "test": { + "executor": "@nx/vite:test", + "outputs": ["{options.reportsDirectory}"], + "options": { + "passWithNoTests": true, + "reportsDirectory": "../coverage/studio-ui" + } + } + }, + "tags": [] +} diff --git a/studio-ui/src/index.ts b/studio-ui/src/index.ts new file mode 100644 index 0000000..6cbf9ae --- /dev/null +++ b/studio-ui/src/index.ts @@ -0,0 +1,4 @@ +export * from "./lib/button.element"; +export * from "./lib/icon.element"; +export * from "./lib/toolbar.element"; +export * from "./lib/tree.element"; diff --git a/studio-frontend/src/app/button.element.scss b/studio-ui/src/lib/button.element.scss similarity index 100% rename from studio-frontend/src/app/button.element.scss rename to studio-ui/src/lib/button.element.scss diff --git a/studio-frontend/src/app/button.element.ts b/studio-ui/src/lib/button.element.ts similarity index 100% rename from studio-frontend/src/app/button.element.ts rename to studio-ui/src/lib/button.element.ts diff --git a/studio-frontend/src/app/icon.element.scss b/studio-ui/src/lib/icon.element.scss similarity index 100% rename from studio-frontend/src/app/icon.element.scss rename to studio-ui/src/lib/icon.element.scss diff --git a/studio-frontend/src/app/icon.element.ts b/studio-ui/src/lib/icon.element.ts similarity index 100% rename from studio-frontend/src/app/icon.element.ts rename to studio-ui/src/lib/icon.element.ts diff --git a/studio-frontend/src/app/toolbar-group.element.scss b/studio-ui/src/lib/toolbar-group.element.scss similarity index 100% rename from studio-frontend/src/app/toolbar-group.element.scss rename to studio-ui/src/lib/toolbar-group.element.scss diff --git a/studio-frontend/src/app/toolbar.element.scss b/studio-ui/src/lib/toolbar.element.scss similarity index 100% rename from studio-frontend/src/app/toolbar.element.scss rename to studio-ui/src/lib/toolbar.element.scss diff --git a/studio-frontend/src/app/toolbar.element.ts b/studio-ui/src/lib/toolbar.element.ts similarity index 100% rename from studio-frontend/src/app/toolbar.element.ts rename to studio-ui/src/lib/toolbar.element.ts diff --git a/studio-frontend/src/app/tree.element.scss b/studio-ui/src/lib/tree.element.scss similarity index 100% rename from studio-frontend/src/app/tree.element.scss rename to studio-ui/src/lib/tree.element.scss diff --git a/studio-frontend/src/app/tree.element.ts b/studio-ui/src/lib/tree.element.ts similarity index 97% rename from studio-frontend/src/app/tree.element.ts rename to studio-ui/src/lib/tree.element.ts index 3911fcc..96ce8a0 100644 --- a/studio-frontend/src/app/tree.element.ts +++ b/studio-ui/src/lib/tree.element.ts @@ -23,7 +23,7 @@ export class TreeElement extends LitElement { @property() name = ""; @property() icon?: string; @property({ attribute: false }) treeChildren?: Tree[]; - @property({ type: Boolean }) expanded = false; + @property({ type: Boolean }) expanded = true; @state() selected = false; diff --git a/studio-ui/tsconfig.json b/studio-ui/tsconfig.json new file mode 100644 index 0000000..451a604 --- /dev/null +++ b/studio-ui/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/studio-ui/tsconfig.lib.json b/studio-ui/tsconfig.lib.json new file mode 100644 index 0000000..4d90bb1 --- /dev/null +++ b/studio-ui/tsconfig.lib.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../dist/out-tsc", + "declaration": true, + "types": ["node", "vite/client"] + }, + "include": ["src/**/*.ts"], + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] +} diff --git a/studio-ui/tsconfig.spec.json b/studio-ui/tsconfig.spec.json new file mode 100644 index 0000000..d231383 --- /dev/null +++ b/studio-ui/tsconfig.spec.json @@ -0,0 +1,25 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "types": [ + "vitest/globals", + "vitest/importMeta", + "vite/client", + "node", + "vitest" + ] + }, + "include": [ + "vite.config.ts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.test.tsx", + "src/**/*.spec.tsx", + "src/**/*.test.js", + "src/**/*.spec.js", + "src/**/*.test.jsx", + "src/**/*.spec.jsx", + "src/**/*.d.ts" + ] +} diff --git a/studio-ui/vite.config.ts b/studio-ui/vite.config.ts new file mode 100644 index 0000000..8ac8b66 --- /dev/null +++ b/studio-ui/vite.config.ts @@ -0,0 +1,58 @@ +/// +import { nxViteTsPaths } from "@nx/vite/plugins/nx-tsconfig-paths.plugin"; +import { defineConfig } from "vite"; +import dts from "vite-plugin-dts"; +import wasm from "vite-plugin-wasm"; +import topLevelAwait from "vite-plugin-top-level-await"; +import * as path from "path"; + +export default defineConfig({ + cacheDir: "../node_modules/.vite/studio-web", + + plugins: [ + nxViteTsPaths(), + wasm(), + topLevelAwait(), + dts({ + entryRoot: "src", + tsConfigFilePath: path.join(__dirname, "tsconfig.lib.json"), + skipDiagnostics: true, + }), + ], + + // Uncomment this if you are using workers. + // worker: { + // plugins: [ nxViteTsPaths() ], + // }, + + // Configuration for building your library. + // See: https://vitejs.dev/guide/build.html#library-mode + build: { + lib: { + // Could also be a dictionary or array of multiple entry points. + entry: "src/index.ts", + name: "studio-ui", + fileName: "index", + // Change this to the formats you want to support. + // Don't forget to update your package.json as well. + formats: ["es", "cjs"], + }, + rollupOptions: { + // External packages that should not be bundled into your library. + external: [ + "lit", + "@storyteller/framework", + "@storyteller/studio", + ], + }, + }, + + test: { + globals: true, + cache: { + dir: "../node_modules/.vitest", + }, + environment: "jsdom", + include: ["src/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}"], + }, +}); diff --git a/studio-web/src/index.ts b/studio-web/src/index.ts index 82671ea..fc74141 100644 --- a/studio-web/src/index.ts +++ b/studio-web/src/index.ts @@ -2,4 +2,7 @@ export * from "./lib/bvh-viewer.element"; export * from "./lib/canvas-provider.element"; export * from "./lib/editor.element"; export * from "./lib/mixamo-viewer.element"; +export * from "./lib/scene-hierarchy.element"; +export * from "./lib/transform-toolbar.element"; +export * from "./lib/url-params"; export * from "./lib/viewer.element"; diff --git a/studio-web/src/lib/canvas-provider.element.ts b/studio-web/src/lib/canvas-provider.element.ts index a8a18c9..31f1889 100644 --- a/studio-web/src/lib/canvas-provider.element.ts +++ b/studio-web/src/lib/canvas-provider.element.ts @@ -1,4 +1,4 @@ -import { Token, UniqueToken, provide } from "@storyteller/framework"; +import { Token, UniqueToken, on, provide } from "@storyteller/framework"; import { LitElement, html, unsafeCSS } from "lit"; import { customElement } from "lit/decorators.js"; @@ -71,6 +71,35 @@ export class CanvasProviderElement super.connectedCallback(); } + @on("keydown") + @on("keyup") + _pipeKeyboardEvents(event: KeyboardEvent): void { + // When the user interacts with one of the web UI elements, the Bevy + // canvas will lose focus, preventing it from receiving keyboard events. + // As a workaround, we can listen for any events that have bubbled up to + // the app root and repeat them for the canvas. + // + // This does unfortunately burden the web front-end with some additional + // responsibility: when we handle a keyboard event and _don't_ want the + // Bevy app to receive it, we'll need to call `event.stopPropagation()` in + // the handler to prevent it from reaching this listener. + const { + type, + code, charCode, key, keyCode, location, repeat, + altKey, ctrlKey, metaKey, shiftKey, + } = event; + + document + .getElementById(this.#canvasId) + ?.dispatchEvent(new KeyboardEvent(type, { + code, charCode, key, keyCode, location, repeat, + altKey, ctrlKey, metaKey, shiftKey, + bubbles: true, + cancelable: true, + composed: true, + })); + } + protected override render = () => html` diff --git a/studio-web/src/lib/scene-hierarchy.element.scss b/studio-web/src/lib/scene-hierarchy.element.scss new file mode 100644 index 0000000..7cc3c00 --- /dev/null +++ b/studio-web/src/lib/scene-hierarchy.element.scss @@ -0,0 +1,24 @@ +:host { + box-sizing: border-box; + padding: 0.5rem 1rem; + overflow: auto; + + background: #0006; + backdrop-filter: blur(16px); + border: 8px solid transparent; + border-radius: 0.5rem; + + &::-webkit-scrollbar { + width: 8px; + height: 8px; + + &-thumb { + border-radius: 2px; + background: #FFF2; + + &:hover { + background: #FFF3; + } + } + } +} diff --git a/studio-web/src/lib/scene-hierarchy.element.ts b/studio-web/src/lib/scene-hierarchy.element.ts new file mode 100644 index 0000000..2769db2 --- /dev/null +++ b/studio-web/src/lib/scene-hierarchy.element.ts @@ -0,0 +1,77 @@ +import * as studio from "@storyteller/studio"; +import { + type EntityDespawnEvent, + type EntityMultiSpawnEvent, + type EntitySpawnEvent, +} from "@storyteller/studio"; +import { on } from "@storyteller/framework"; +import { LitElement, html, unsafeCSS } from "lit"; +import { customElement, state } from "lit/decorators.js"; + +import styles from "./scene-hierarchy.element.scss?inline"; + +import "@storyteller/studio-ui/tree"; + +interface SceneObject extends Omit { + children: SceneObject[]; +} + +@customElement("sts-scene-hierarchy") +export class SceneHierarchyElement extends LitElement { + static override styles = unsafeCSS(styles); + + get tree() { return this._tree as readonly SceneObject[]; } + get entityMap() { return this.#entityMap as ReadonlyMap; } + + @state() private _tree: SceneObject[] = []; + #sceneObjectsFlat: studio.SceneObject[] = []; + #entityMap = new Map(); + + @on("window:entity-spawn") + @on("window:entity-multi-spawn") + _onEntitiesSpawned({ detail: entities }: EntitySpawnEvent | EntityMultiSpawnEvent): void { + this.#sceneObjectsFlat = this.#sceneObjectsFlat.concat(entities); + this.#rebuildTree(); + } + + @on("window:entity-despawn") + _onEntityDespawn({ detail: entity }: EntityDespawnEvent): void { + this.#sceneObjectsFlat = this.#sceneObjectsFlat.filter(({ id }) => id !== entity); + this.#rebuildTree(); + } + + #rebuildTree(): void { + this.#entityMap.clear(); + + for (let obj of this.#sceneObjectsFlat) + this.#entityMap.set(obj.id, { ...obj, children: [] }); + + for (let obj of this.#entityMap.values()) { + if (obj.parent != null) { + const parent = this.#entityMap.get(obj.parent); + if (!parent) + throw new Error(`Missing parent: ${obj.parent}`); + + parent.children.push(obj); + } + } + + for (let obj of this.#entityMap.values()) + obj.children.sort((a, b) => a.name.localeCompare(b.name)); + + this._tree = Array + .from(this.#entityMap.values()) + .filter(obj => obj.parent == null) + .sort((a, b) => a.name.localeCompare(b.name)) + } + + protected override render = () => html` + ${this.tree.map(obj => html` + + `)} + ` +} diff --git a/studio-web/src/lib/studio-base.element.scss b/studio-web/src/lib/studio-base.element.scss index e9de20f..87e527a 100644 --- a/studio-web/src/lib/studio-base.element.scss +++ b/studio-web/src/lib/studio-base.element.scss @@ -10,4 +10,5 @@ inset: 0; width: 100%; height: 100%; + outline: none !important; } diff --git a/studio-web/src/lib/studio-base.element.ts b/studio-web/src/lib/studio-base.element.ts index 65db8cd..6d49d96 100644 --- a/studio-web/src/lib/studio-base.element.ts +++ b/studio-web/src/lib/studio-base.element.ts @@ -10,11 +10,14 @@ import styles from "./studio-base.element.scss?inline"; export abstract class StudioBaseElement extends LitElement { static override styles = unsafeCSS(styles); + get canvas() { return this.#canvas; } + @inject(CANVAS_PROVIDER) _canvasProvider?: CanvasProvider; #canvas?: HTMLCanvasElement; #resizeObserver?: ResizeObserver; + #frameRequest?: number; abstract onStart(viewportSize: ViewportSize): void; abstract onChanges?(changes: PropertyValues): void; @@ -30,9 +33,14 @@ export abstract class StudioBaseElement extends LitElement { if ((this.#canvas = this._canvasProvider.takeCanvas())) this.appendChild(this.#canvas); + + this.#tick(); } override disconnectedCallback(): void { + if (this.#frameRequest != null) + cancelAnimationFrame(this.#frameRequest); + if (this.#canvas) this._canvasProvider?.returnCanvas(this.removeChild(this.#canvas)); @@ -54,6 +62,17 @@ export abstract class StudioBaseElement extends LitElement { studio.resize(new ViewportSize(width, height)); } + #tick(): void { + // If user focus lands on the `document.body`, neither the frontend app + // nor the Bevy app will properly handle keyboard events. Thie function + // runs every tick to ensure that user focus never becomes "stranded" in + // this way. + if (document.activeElement === document.body) + this.#canvas?.focus(); + + this.#frameRequest = requestAnimationFrame(() => this.#tick()); + } + #start(): void { if (!this.#canvas) return; diff --git a/studio-frontend/src/app/transform-toolbar.element.scss b/studio-web/src/lib/transform-toolbar.element.scss similarity index 100% rename from studio-frontend/src/app/transform-toolbar.element.scss rename to studio-web/src/lib/transform-toolbar.element.scss diff --git a/studio-frontend/src/app/transform-toolbar.element.ts b/studio-web/src/lib/transform-toolbar.element.ts similarity index 72% rename from studio-frontend/src/app/transform-toolbar.element.ts rename to studio-web/src/lib/transform-toolbar.element.ts index e6ca3fe..8a8f4e1 100644 --- a/studio-frontend/src/app/transform-toolbar.element.ts +++ b/studio-web/src/lib/transform-toolbar.element.ts @@ -1,7 +1,11 @@ +import * as studio from "@storyteller/studio"; import { TransformSpace, TransformType } from "@storyteller/studio"; import { match } from "@storyteller/utility"; import { LitElement, html, unsafeCSS } from "lit"; -import { customElement, property } from "lit/decorators.js"; +import { customElement, state } from "lit/decorators.js"; + +import "@storyteller/studio-ui/icon"; +import "@storyteller/studio-ui/toolbar"; import styles from "./transform-toolbar.element.scss?inline"; @@ -9,12 +13,15 @@ import styles from "./transform-toolbar.element.scss?inline"; export class TransformToolbarElement extends LitElement { static override styles = unsafeCSS(styles); - @property({ type: Number }) space = TransformSpace.Local; - @property({ type: Number }) type = TransformType.Translate; + get space() { return this._space; } + @state() private _space = TransformSpace.Local; + + get type() { return this._type; } + @state() private _type = TransformType.Translate; #onSpaceChange(event: InputEvent): void { const { value } = event.target as HTMLInputElement; - const space = match (value, { + this._space = match (value, { "local": () => TransformSpace.Local, "world": () => TransformSpace.World, _: () => { @@ -22,11 +29,7 @@ export class TransformToolbarElement extends LitElement { } }); - this.dispatchEvent(new CustomEvent("space-change", { - detail: space, - bubbles: true, - composed: true, - })); + studio.setTransformSpace(this._space); } #onTypeChange(event: InputEvent): void { @@ -40,15 +43,10 @@ export class TransformToolbarElement extends LitElement { } }); - const type = checked - ? this.type | flag - : this.type & ~flag; + if (checked) this._type |= flag; + else this._type &= ~flag; - this.dispatchEvent(new CustomEvent("type-change", { - detail: type, - bubbles: true, - composed: true, - })); + studio.setTransformType(this._type); } protected override render = () => html` @@ -59,7 +57,7 @@ export class TransformToolbarElement extends LitElement { @@ -68,7 +66,7 @@ export class TransformToolbarElement extends LitElement { @@ -78,7 +76,7 @@ export class TransformToolbarElement extends LitElement { Translate @@ -86,7 +84,7 @@ export class TransformToolbarElement extends LitElement { Rotate @@ -95,7 +93,7 @@ export class TransformToolbarElement extends LitElement { diff --git a/studio-web/src/lib/url-params.ts b/studio-web/src/lib/url-params.ts new file mode 100644 index 0000000..a8e449a --- /dev/null +++ b/studio-web/src/lib/url-params.ts @@ -0,0 +1,20 @@ +import { StudioMode } from "@storyteller/studio"; +import { match } from "@storyteller/utility"; + +export class StudioURLParams { + #_nativeParams?: URLSearchParams; + get #nativeParams() { + return this.#_nativeParams ??= new URLSearchParams(document.location.search); + } + + get bvh() { return this.#nativeParams.get("bvh"); } + get mixamo() { return this.#nativeParams.get("mixamo"); } + get scene() { return this.#nativeParams.get("scene"); } + get mode(): StudioMode { + return match(this.#nativeParams.get("mode"), { + "studio": () => StudioMode.Editor, + "viewer": () => StudioMode.Viewer, + _: () => StudioMode.Viewer, + }); + } +} diff --git a/studio/Cargo.toml b/studio/Cargo.toml index 3daa843..3892c64 100644 --- a/studio/Cargo.toml +++ b/studio/Cargo.toml @@ -66,6 +66,7 @@ wasm = [ "bevy_rapier3d/wasm-bindgen", ] desktop = [] +wasm_screenshot = [] [dependencies.bevy] workspace = true diff --git a/studio/project.json b/studio/project.json index bffad5a..eac8000 100644 --- a/studio/project.json +++ b/studio/project.json @@ -27,7 +27,15 @@ "executor": "nx:run-commands", "inputs": ["assets"], "options": { - "command": "node scripts/build-web.mjs", + "command": "node scripts/build-web.mjs --features=wasm", + "cwd": "studio" + } + }, + "build:image-generator": { + "executor": "nx:run-commands", + "inputs": ["assets"], + "options": { + "command": "node scripts/build-web.mjs --features=wasm,wasm_screenshot", "cwd": "studio" } }, @@ -41,8 +49,8 @@ "headless": { "executor": "nx:run-commands", "options": { - "command": "cargo run -p studio --bin studio-headless -- {args}", - } + "command": "cargo run -p studio --bin studio-headless -- {args}" + } }, "test": { "executor": "@nxrs/cargo:test", diff --git a/studio/scripts/build-web.mjs b/studio/scripts/build-web.mjs index bb65eea..10b0c47 100644 --- a/studio/scripts/build-web.mjs +++ b/studio/scripts/build-web.mjs @@ -13,6 +13,12 @@ async function main() { const __dirname = url.fileURLToPath(new URL(".", import.meta.url)); const workspaceRoot = path.resolve(__dirname, "../.."); + const extraCargoArgs = process.argv.slice(2); + console.log(`Extra cargo args:`); + for (let arg of extraCargoArgs) { + console.log(` ${arg}`); + } + /** @type import("node:child_process").StdioOptions */ const stdio = ["ignore", "inherit", "inherit"]; @@ -26,9 +32,8 @@ async function main() { "-p", "studio", "--lib", "--release", - "--features", "wasm", "--target", "wasm32-unknown-unknown", - ], { + ].concat(extraCargoArgs), { shell: true, cwd: workspaceRoot, stdio, diff --git a/studio/src/camera_controller.rs b/studio/src/camera_controller.rs index 186f69d..66d4ae4 100644 --- a/studio/src/camera_controller.rs +++ b/studio/src/camera_controller.rs @@ -10,7 +10,7 @@ use crate::{ interaction::Selected, math::{ Easing, InterpPosition, InterpVelocity, Interpolation, InterpolationSystem, NearlyEq, - Velocity, + NearlyZero, Velocity, }, }; @@ -78,9 +78,6 @@ fn camera_control_kinematic( return; } - let (mut controller, mut xform, proj) = q_camera.single_mut(); - let mut window = q_window.single_mut(); - let mut any_ongoing = false; let mut any_started = false; let mut any_stopped = false; @@ -92,7 +89,8 @@ fn camera_control_kinematic( ) }) { if let InputAction(lifecycle, Action::Camera(motion)) = action { - use CameraMotion::*; + let (mut controller, mut xform, proj) = q_camera.single_mut(); + let window = q_window.single(); match lifecycle { Lifecycle::Start => any_started = true, @@ -100,20 +98,21 @@ fn camera_control_kinematic( Lifecycle::Stop => any_stopped = true, } + use CameraMotion::*; match motion { - Pan(delta) => do_pan(proj, &window, *delta, &mut xform, &mut controller), - Zoom(delta) => do_zoom(&window, *delta, &mut xform, &mut controller), - Orbit(delta) => do_orbit(&window, *delta, *lifecycle, &mut xform, &mut controller), - Rotate(delta) => do_rotate(&window, *delta, &mut xform, &mut controller), + Pan(delta) => do_pan(proj, window, *delta, &mut xform, &mut controller), + Zoom(delta) => do_zoom(window, *delta, &mut xform, &mut controller), + Orbit(delta) => do_orbit(window, *delta, *lifecycle, &mut xform, &mut controller), + Rotate(delta) => do_rotate(window, *delta, &mut xform, &mut controller), _ => {} } } } if any_started { - capture_pointer(&mut window); + capture_pointer(&mut q_window.single_mut()); } else if any_stopped && !any_ongoing { - release_pointer(&mut window); + release_pointer(&mut q_window.single_mut()); } } @@ -215,12 +214,16 @@ fn process_focus_inputs( } fn camera_control_physical( - mut q_camera: Query<(&mut CameraController, &mut Transform, &Velocity)>, + mut q_camera: Query<(Entity, &mut CameraController, &Velocity)>, + mut q_xforms: Query<&mut Transform>, ) { - for (mut controller, mut xform, velocity) in q_camera.iter_mut() { - let delta_pos = xform.rotation * (**velocity); - xform.translation += delta_pos; - controller.focus += delta_pos; + for (ent, mut controller, velocity) in q_camera.iter_mut() { + if !velocity.nearly_zero() { + let mut xform = q_xforms.get_mut(ent).unwrap(); + let delta_pos = xform.rotation * (**velocity); + xform.translation += delta_pos; + controller.focus += delta_pos; + } } } diff --git a/studio/src/change_tracking.rs b/studio/src/change_tracking.rs new file mode 100644 index 0000000..a3f4e32 --- /dev/null +++ b/studio/src/change_tracking.rs @@ -0,0 +1,77 @@ +use bevy::prelude::*; + +pub struct ChangeTrackingPlugin; + +impl Plugin for ChangeTrackingPlugin { + fn build(&self, app: &mut App) { + app.add_event::(); + + // Add more of these systems for each component that we want to track + app.add_systems(Update, send_change_events::); + + #[cfg(feature = "wasm")] + app.add_systems( + Update, + wasm::send_change_events::.map(bevy::utils::error), + ); + } +} + +#[derive(Component, Debug)] +pub struct ChangeTracker; + +fn send_change_events( + query: Query, With)>, + mut entity_change_writer: EventWriter, +) { + for entity in query.iter() { + entity_change_writer.send(EntityChangedEvent(entity)); + } +} + +#[derive(Event, Deref)] +pub struct EntityChangedEvent(pub Entity); + +#[cfg(feature = "wasm")] +mod wasm { + use bevy::{prelude::*, window::PrimaryWindow}; + use wasm_bindgen::prelude::wasm_bindgen; + + use crate::wasm; + + use super::ChangeTracker; + + #[wasm_bindgen(typescript_custom_section)] + const ENTITY_CHANGES_EVENT: &str = r#" +export interface EntityChangesEvent extends CustomEvent { + type: "entity-changes"; + detail: string[]; +} + +declare global { + export interface GlobalEventHandlersEventMap { + "entity-changes": EntityChangesEvent; + } +} + "#; + + pub(super) fn send_change_events( + q_changed_ents: Query, With)>, + q_window: Query<&Window, With>, + ) -> Result<(), String> { + if q_changed_ents.is_empty() { + return Ok(()); + } + + let Ok(window) = q_window.get_single() else { + return Ok(()); + }; + + let entities = q_changed_ents + .iter() + .map(|ent| format!("{ent:?}")) + .collect::>(); + + wasm::dispatch_to_js(window, "entity-changes", &entities) + } +} diff --git a/studio/src/lib.rs b/studio/src/lib.rs index da30fb8..48939dd 100644 --- a/studio/src/lib.rs +++ b/studio/src/lib.rs @@ -19,6 +19,7 @@ use wgpu::Extent3d; mod anim; mod bvh; mod camera_controller; +mod change_tracking; mod debug; mod design_system; mod hierarchy; @@ -279,6 +280,8 @@ pub fn start_internal( ..default() }), CommonPlugins, + #[cfg(all(feature = "wasm", feature = "wasm_screenshot"))] + wasm::ScreenshotPlugin, )); if matches!(studio_mode, StudioMode::Editor) { diff --git a/studio/src/scene.rs b/studio/src/scene.rs index 1faaa8c..97575a5 100644 --- a/studio/src/scene.rs +++ b/studio/src/scene.rs @@ -22,7 +22,9 @@ use wasm_bindgen::prelude::wasm_bindgen; use wgpu::{TextureDescriptor, TextureUsages}; use crate::{ + anim::SkeletalJoint, camera_controller::{CameraController, CameraControllerBundle}, + change_tracking::{ChangeTracker, ChangeTrackingPlugin}, debug::DebugPoint, is_headless, is_not_headless, math::{Combinable, Extrema}, @@ -37,6 +39,8 @@ pub struct SceneSpawnerSystem; impl Plugin for ScenePlugin { fn build(&self, app: &mut App) { + app.add_plugins(ChangeTrackingPlugin); + app.insert_resource(ClearColor(Color::hsl(0.0, 0.0, 0.25))) .insert_resource(Msaa::Off) .add_state::() @@ -56,10 +60,15 @@ impl Plugin for ScenePlugin { .add_systems( OnEnter(SceneState::Active), ( - tag_scene_elements, - frame_scene_in_view, - start_capture.run_if(is_headless), + ( + tag_scene_elements, + frame_scene_in_view, + start_capture.run_if(is_headless), + ), + apply_deferred, + track_scene_elements, ) + .chain() .in_set(SceneSpawnerSystem), ); @@ -234,6 +243,7 @@ fn initialize(mut cmd: Commands) { cmd.spawn(( Name::new("Main Camera"), MainCamera, + ChangeTracker, Camera3dBundle { transform: camera_xform, tonemapping: Tonemapping::BlenderFilmic, @@ -483,6 +493,15 @@ fn start_capture(mut captures: Query<&mut ScreenCapture>) { } } +fn track_scene_elements( + mut commands: Commands, + scene_elements: Query, With)>, +) { + for element in scene_elements.iter() { + commands.entity(element).insert(ChangeTracker); + } +} + #[cfg(feature = "wasm")] mod wasm { use bevy::{prelude::*, window::PrimaryWindow}; diff --git a/studio/src/wasm/mod.rs b/studio/src/wasm/mod.rs index 751854a..6ec32e2 100644 --- a/studio/src/wasm/mod.rs +++ b/studio/src/wasm/mod.rs @@ -4,9 +4,13 @@ use parking_lot::ReentrantMutex; use wasm_bindgen::prelude::wasm_bindgen; mod conv; +#[cfg(feature = "wasm_screenshot")] +mod screenshot; mod util; pub use conv::*; +#[cfg(feature = "wasm_screenshot")] +pub use screenshot::ScreenshotPlugin; pub(crate) use util::*; static INITIALIZED: OnceLock>> = OnceLock::new(); diff --git a/studio/src/wasm/screenshot.rs b/studio/src/wasm/screenshot.rs new file mode 100644 index 0000000..b4141c4 --- /dev/null +++ b/studio/src/wasm/screenshot.rs @@ -0,0 +1,161 @@ +//! Outline of the flow of events here: +//! +//! 1. [`crate::change_tracking`] system sends events to frontend to notify of +//! tracked changes +//! 2. Frontend calls [`request_screenshot`] on a debounce, writing an empty +//! message to [`SCREENSHOT_REQUEST`] +//! 3. [`take_screenshots`] watches the [`SCREENSHOT_REQUEST`] buffer and +//! populates the [`SCREENSHOT`] buffer with a PNG-encoded `Vec` +//! 4. [`send_screenshots`] watches the [`SCREENSHOT`] buffer and sends an event +//! to the frontend by converting the `Vec` to a [`js_sys::Uint8Array`] + +use bevy::{prelude::*, render::view::screenshot::ScreenshotManager, window::PrimaryWindow}; +use image::{codecs::png::PngEncoder, ColorType, ImageEncoder}; +use wasm_bindgen::prelude::{wasm_bindgen, JsValue}; + +use super::{AsJsValue, StaticBuffer, WasmMessageBuffer}; + +static SCREENSHOT_REQUEST: WasmMessageBuffer<()> = WasmMessageBuffer::new(); +static SCREENSHOT: WasmMessageBuffer = WasmMessageBuffer::new(); + +pub struct ScreenshotPlugin; + +impl Plugin for ScreenshotPlugin { + fn build(&self, app: &mut App) { + app.add_systems( + Update, + ( + take_screenshots.map(bevy::utils::error), + send_screenshots.map(bevy::utils::error), + ), + ); + } +} + +#[wasm_bindgen] +#[derive(Clone)] +pub struct Screenshot { + data_pvt: js_sys::Uint8Array, + pub width: usize, + pub height: usize, +} + +// This intermediate type is necessary because the `Screenshot` struct +// cannot be safely stored in a `WasmMessageBuffer` +struct ScreenshotNative { + data: Vec, + width: usize, + height: usize, +} + +impl From for Screenshot { + fn from(value: ScreenshotNative) -> Self { + Self { + data_pvt: js_sys::Uint8Array::from(&value.data[..]), + width: value.width, + height: value.height, + } + } +} + +#[wasm_bindgen] +impl Screenshot { + #[wasm_bindgen(getter)] + pub fn data(&self) -> js_sys::Uint8Array { + self.data_pvt.clone() + } +} + +impl AsJsValue for Screenshot { + fn as_js(&self) -> Result { + Ok(JsValue::from(self.clone())) + } +} + +#[wasm_bindgen(js_name = requestScreenshot)] +pub fn request_screenshot() { + SCREENSHOT_REQUEST.write_message(()); +} + +fn take_screenshots( + mut r_screenshot_mgr: ResMut, + q_window: Query>, +) -> Result<(), String> { + if SCREENSHOT_REQUEST.take_message().is_none() { + return Ok(()); + } + + let Ok(window_ent) = q_window.get_single() else { + return Ok(()); + }; + + r_screenshot_mgr + .take_screenshot(window_ent, |img| { + let img = match img.try_into_dynamic().map(|img| img.to_rgb8()) { + Ok(img) => img, + Err(err) => { + error!("{err}"); + return; + } + }; + + let channel_count = as image::Pixel>::CHANNEL_COUNT as usize; + let buffer_len = match Some(channel_count) + .and_then(|size| size.checked_mul(img.width() as usize)) + .and_then(|size| size.checked_mul(img.height() as usize)) + { + Some(size) => size, + None => { + error!("Failed to compute buffer length for image!"); + return; + } + }; + let src_buffer = &img.as_raw()[..buffer_len]; + + let mut png_bytes = vec![]; + if let Err(err) = PngEncoder::new(&mut png_bytes).write_image( + src_buffer, + img.width(), + img.height(), + ColorType::Rgb8, + ) { + error!("{err}"); + return; + } + + SCREENSHOT.write_message(ScreenshotNative { + data: png_bytes, + width: img.width() as usize, + height: img.height() as usize, + }); + }) + .map_err(|err| format!("{err}")) +} + +#[wasm_bindgen(typescript_custom_section)] +const SCREENSHOT_READY_EVENT: &str = r#" +export interface ScreenshotReadyEvent extends CustomEvent { + type: "screenshot-ready"; + detail: Screenshot; +} + +declare global { + export interface GlobalEventHandlersEventMap { + "screenshot-ready": ScreenshotReadyEvent; + } +} + "#; + +fn send_screenshots(q_window: Query<&Window, With>) -> Result<(), String> { + let Some(screenshot) = SCREENSHOT.take_message() else { + return Ok(()); + }; + + let Ok(window) = q_window.get_single() else { + return Ok(()); + }; + + let screenshot = Screenshot::from(screenshot); + + super::dispatch_to_js(window, "screenshot-ready", &screenshot) +} diff --git a/tsconfig.base.json b/tsconfig.base.json index 2d911b3..7cab5a6 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -17,12 +17,19 @@ "paths": { "@storyteller/framework": ["framework/src/index.ts"], "@storyteller/studio": ["dist/studio"], + "@storyteller/studio-ui": ["studio-ui/src/index.ts"], + "@storyteller/studio-ui/button": ["studio-ui/src/lib/button.element.ts"], + "@storyteller/studio-ui/icon": ["studio-ui/src/lib/icon.element.ts"], + "@storyteller/studio-ui/toolbar": ["studio-ui/src/lib/toolbar.element.ts"], + "@storyteller/studio-ui/tree": ["studio-ui/src/lib/tree.element.ts"], "@storyteller/studio-web": ["studio-web/src/index.ts"], "@storyteller/studio-web/canvas-provider": ["studio-web/src/lib/canvas-provider.element.ts"], "@storyteller/studio-web/viewer": ["studio-web/src/lib/viewer.element.ts"], "@storyteller/studio-web/bvh-viewer": ["studio-web/src/lib/bvh-viewer.element.ts"], "@storyteller/studio-web/editor": ["studio-web/src/lib/editor.element.ts"], "@storyteller/studio-web/mixamo-viewer": ["studio-web/src/lib/mixamo-viewer.element.ts"], + "@storyteller/studio-web/scene-hierarchy": ["studio-web/src/lib/scene-hierarchy.element.ts"], + "@storyteller/studio-web/transform-toolbar": ["studio-web/src/lib/transform-toolbar.element.ts"], "@storyteller/utility": ["utility/src/index.ts"] } }, diff --git a/vite-multiple-assets/README.md b/vite-multiple-assets/README.md new file mode 100644 index 0000000..d5b41a7 --- /dev/null +++ b/vite-multiple-assets/README.md @@ -0,0 +1,5 @@ +# vite-multiple-assets + +This is a simple Vite plugin forked from [`vite-multiple-assets`](https://github.com/nguyenbatranvan/vite-multiple-assets) +and extended to provide MIME types for additional filetypes not commonly used on +the web, but needed for a Bevy 3D application. diff --git a/vite-multiple-assets/package.json b/vite-multiple-assets/package.json new file mode 100644 index 0000000..96542b9 --- /dev/null +++ b/vite-multiple-assets/package.json @@ -0,0 +1,6 @@ +{ + "name": "@storyteller/vite-multiple-assets", + "version": "0.0.1", + "dependencies": {}, + "main": "./src/index.js" +} diff --git a/vite-multiple-assets/project.json b/vite-multiple-assets/project.json new file mode 100644 index 0000000..6a6bcf7 --- /dev/null +++ b/vite-multiple-assets/project.json @@ -0,0 +1,8 @@ +{ + "name": "vite-multiple-assets", + "$schema": "../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "sourceRoot": "vite-multiple-assets/src", + "targets": {}, + "tags": [] +} diff --git a/vite-multiple-assets/src/index.ts b/vite-multiple-assets/src/index.ts new file mode 100644 index 0000000..2e13364 --- /dev/null +++ b/vite-multiple-assets/src/index.ts @@ -0,0 +1,16 @@ +import { PluginOption, ViteDevServer } from "vite"; + +import { ServerMiddleWare } from "./lib/server"; +import { buildMiddleWare } from "./lib/build"; + +export default function DynamicPublicDirectory(assets: string[]): PluginOption { + return { + configureServer(server: ViteDevServer) { + return ServerMiddleWare(server, assets) + }, + async writeBundle(options) { + buildMiddleWare(options, assets) + }, + name: "dynamic assets" + }; +} diff --git a/vite-multiple-assets/src/lib/build.ts b/vite-multiple-assets/src/lib/build.ts new file mode 100644 index 0000000..31d30a4 --- /dev/null +++ b/vite-multiple-assets/src/lib/build.ts @@ -0,0 +1,25 @@ +import fs from "fs"; +import { NormalizedOutputOptions } from "rollup"; + +function copyRecursively(src: string, dst: string) { + if (fs.statSync(src).isDirectory()) { + // folder + if (!fs.existsSync(dst)) { + fs.mkdirSync(dst); + } + for (const file of fs.readdirSync(src)) { + copyRecursively(src + "/" + file, dst + "/" + file); + } + } else { + // file + fs.copyFileSync(src, dst); + } +} + +export function buildMiddleWare(options: NormalizedOutputOptions, assets: string[] = []) { + const dst = options.dir; + if (!dst) throw new Error("options.dir is undefined"); + for (const dir of assets) { + copyRecursively(dir, dst) + } +} diff --git a/vite-multiple-assets/src/lib/server.ts b/vite-multiple-assets/src/lib/server.ts new file mode 100644 index 0000000..996bd5e --- /dev/null +++ b/vite-multiple-assets/src/lib/server.ts @@ -0,0 +1,93 @@ +import fs from "fs"; +import path from "path"; +import { ViteDevServer } from "vite"; + +interface IPropsFile { + name: string; + files: string[]; +} + +function getFiles(dir, files_): string[] { + files_ = files_ || []; + const files = fs.readdirSync(dir); + for (const i in files) { + const name = dir + "/" + files[i]; + if (fs.statSync(name).isDirectory()) { + getFiles(name, files_); + } else { + files_.push(name); + } + } + return files_; +} + +const mimeTypes = { + ".7z": "application/x-7z-compressed", + ".bin": "application/octet-stream", + ".bvh": "text/bvh", + ".css": "text/css", + ".eot": "font/eot", + ".gif": "image/gif", + ".glb": "model/gltf-binary", + ".gltf": "model/gltf+json", + ".gz": "application/gzip", + ".hdr": "image/vnd.radiance", + ".html": "text/html", + ".ico": "image/x-icon", + ".jpg": "image/jpeg", + ".js": "text/javascript", + ".json": "application/json", + ".ktx": "image/ktx", + ".ktx2": "image/ktx", + ".meta": "text/plain", + ".mjs": "text/javascript", + ".mp3": "audio/mpeg", + ".mp4": "video/mp4", + ".otf": "font/otf", + ".png": "image/png", + ".rar": "application/x-rar-compressed", + ".svg": "image/svg+xml", + ".tiff": "image/tiff", + ".ttf": "font/ttf", + ".txt": "text/plain", + ".wasm": "application/wasm", + ".webm": "video/webm", + ".webp": "image/webp", + ".wgsl": "text/wgsl", + ".woff": "font/woff", + ".woff2": "font/woff2", + ".xml": "text/xml", + ".zip": "application/zip", +} + +export function ServerMiddleWare(server: ViteDevServer, assets: string[] = []) { + if (!assets || !assets.length) + return; + + const fileObject: IPropsFile[] = []; + for (let i = 0; i < assets.length; i++) { + const files = getFiles(path.join(process.cwd(), `/${assets[i]}`), []); + fileObject.push({ + name: assets[i], + files + }); + } + + return () => { + server.middlewares.use(async (req, res, next) => { + for (let i = 0; i < fileObject.length; i++) { + const file = path.join(process.cwd(), `${fileObject[i].name}/${req.originalUrl}`); + if (fileObject[i].files.some(f => path.relative(f, file) === "")) { + const extension = file.substring(file.lastIndexOf(".")); + res.setHeader("Cache-Control", "max-age=31536000, immutable"); + res.setHeader("Content-Type", mimeTypes[extension]); + res.writeHead(200); + res.write(fs.readFileSync(path.join(process.cwd(), "/" + fileObject[i].name + req.originalUrl))); + res.end(); + break; + } + } + next(); + }); + }; +} diff --git a/vite-multiple-assets/tsconfig.json b/vite-multiple-assets/tsconfig.json new file mode 100644 index 0000000..a99bedc --- /dev/null +++ b/vite-multiple-assets/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "strict": true, + "noImplicitAny": false, + "skipLibCheck": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/vite-multiple-assets/tsconfig.lib.json b/vite-multiple-assets/tsconfig.lib.json new file mode 100644 index 0000000..4d90bb1 --- /dev/null +++ b/vite-multiple-assets/tsconfig.lib.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../dist/out-tsc", + "declaration": true, + "types": ["node", "vite/client"] + }, + "include": ["src/**/*.ts"], + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] +} diff --git a/vite-multiple-assets/tsconfig.spec.json b/vite-multiple-assets/tsconfig.spec.json new file mode 100644 index 0000000..d231383 --- /dev/null +++ b/vite-multiple-assets/tsconfig.spec.json @@ -0,0 +1,25 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "types": [ + "vitest/globals", + "vitest/importMeta", + "vite/client", + "node", + "vitest" + ] + }, + "include": [ + "vite.config.ts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.test.tsx", + "src/**/*.spec.tsx", + "src/**/*.test.js", + "src/**/*.spec.js", + "src/**/*.test.jsx", + "src/**/*.spec.jsx", + "src/**/*.d.ts" + ] +}