From 7b06d48c5d64a90c3a6211d10f70fc4846676c6e Mon Sep 17 00:00:00 2001 From: Danny McGee Date: Fri, 23 Feb 2024 22:04:06 -0500 Subject: [PATCH] Scene storage backend integration (#51) This PR adds Storyteller.ai API integration for saving and loading Storyteller Studio scenes to/from the backend. ### Known issues * The skybox is not currently stored in the serialized scene format --- .github/workflows/pr-validation.yml | 8 ++- ci.sh | 2 +- package-lock.json | 11 +++- package.json | 4 +- rust-toolchain.toml | 2 +- studio-frontend/src/app/app.element.ts | 39 ++++++++---- studio-web/src/index.ts | 2 + studio-web/src/lib/editor.element.ts | 28 +++++++++ studio/src/lib.rs | 87 +++++++++++++++++--------- studio/src/main.rs | 12 ++-- studio/src/path.rs | 40 ++++++++++++ studio/src/remote/reader.rs | 6 +- studio/src/scene.rs | 64 ++++++++++++++++--- tsconfig.base.json | 1 + 14 files changed, 244 insertions(+), 62 deletions(-) create mode 100644 studio-web/src/lib/editor.element.ts create mode 100644 studio/src/path.rs diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index b23ccaa..cbb9e65 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -15,7 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@1.76.0 + with: + components: rustfmt - name: Check Rust formatting run: cargo fmt --all -- --check @@ -35,7 +37,9 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ runner.os }}-cargo-lint-and-test-${{ hashFiles('**/Cargo.toml') }} - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@1.76 + with: + components: clippy - name: Install Linux dependencies run: | sudo apt-get update diff --git a/ci.sh b/ci.sh index bd8c910..5c97c09 100755 --- a/ci.sh +++ b/ci.sh @@ -6,7 +6,7 @@ set -euxo pipefail rustup target add wasm32-unknown-unknown # Emit versions -rustup show +rustup show active-toolchain npm version # Install project dependencies diff --git a/package-lock.json b/package-lock.json index 964117b..9d13de8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,8 @@ "lodash": "^4.17.21", "react": "18.2.0", "react-dom": "18.2.0", - "tslib": "^2.3.0" + "tslib": "^2.3.0", + "uuid": "^9.0.1" }, "devDependencies": { "@nx/js": "16.9.1", @@ -29,6 +30,7 @@ "@types/node": "18.14.2", "@types/react": "18.2.14", "@types/react-dom": "18.2.6", + "@types/uuid": "^9.0.8", "@types/which": "^3.0.2", "@vitejs/plugin-react": "~4.0.0", "@vitest/coverage-c8": "~0.32.0", @@ -4410,6 +4412,12 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.4.tgz", "integrity": "sha512-IDaobHimLQhjwsQ/NMwRVfa/yL7L/wriQPMhw1ZJall0KX6E1oxk29XMDeilW5qTIg5aoiqf5Udy8U/51aNoQQ==" }, + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", + "dev": true + }, "node_modules/@types/which": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/which/-/which-3.0.2.tgz", @@ -10194,7 +10202,6 @@ "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "dev": true, "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" diff --git a/package.json b/package.json index b1c0857..65ddb8a 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@types/node": "18.14.2", "@types/react": "18.2.14", "@types/react-dom": "18.2.6", + "@types/uuid": "^9.0.8", "@types/which": "^3.0.2", "@vitejs/plugin-react": "~4.0.0", "@vitest/coverage-c8": "~0.32.0", @@ -45,6 +46,7 @@ "lodash": "^4.17.21", "react": "18.2.0", "react-dom": "18.2.0", - "tslib": "^2.3.0" + "tslib": "^2.3.0", + "uuid": "^9.0.1" } } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 292fe49..624eb0e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "stable" +channel = "1.76.0" diff --git a/studio-frontend/src/app/app.element.ts b/studio-frontend/src/app/app.element.ts index 5b2158f..f2ad003 100644 --- a/studio-frontend/src/app/app.element.ts +++ b/studio-frontend/src/app/app.element.ts @@ -18,10 +18,12 @@ import { LitElement, html, nothing } from "lit"; import { customElement, state } from "lit/decorators.js"; import { classMap } from "lit-html/directives/class-map.js"; import { Ref, createRef, ref } from "lit-html/directives/ref.js"; +import { v4 as uuid } from "uuid"; import "@storyteller/studio-web/canvas-provider"; import "@storyteller/studio-web/viewer"; import "@storyteller/studio-web/bvh-viewer"; +import "@storyteller/studio-web/editor"; import "@storyteller/studio-web/mixamo-viewer"; import "./anim-editor.element"; import "./button.element"; @@ -50,6 +52,7 @@ export class AppElement extends LitElement { @state() bvhPath?: string; @state() mixamoPath?: string; + @state() scenePath?: string; @state() mode?: StudioMode; #sceneObjects: studio.SceneObject[] = []; @@ -74,6 +77,9 @@ export class AppElement extends LitElement { if (params.has("mixamo")) this.mixamoPath = params.get("mixamo")!; + if (params.has("scene")) + this.scenePath = params.get("scene")!; + this.mode = match (params.get("mode"), { "studio": () => StudioMode.Editor, "viewer": () => StudioMode.Viewer, @@ -152,19 +158,25 @@ export class AppElement extends LitElement { } @on("window:scene-saved") - onSceneSaved({ detail: scene }: SceneSavedEvent): void { - const fileBlob = new Blob([scene], { type: "text/ron" }); - const url = URL.createObjectURL(fileBlob); + async onSceneSaved({ detail: scene }: SceneSavedEvent): Promise { + const file = new File([scene], "scene.scn.ron", { type: "text/ron" }); + const formData = new FormData(); - const anchor = document.createElement("a"); - anchor.href = url; - anchor.download = "scene.scn.ron"; - anchor.click(); + formData.append("uuid_idempotency_token", uuid()); + formData.append("file", file); - setTimeout(() => { - URL.revokeObjectURL(url); - anchor.remove(); - }); + try { + const response = await fetch("https://api.fakeyou.com/v1/engine/create_scene", { + method: "POST", + credentials: "include", + headers: { "Accept": "text/plain" }, + body: formData, + }); + console.log(response); + } + catch (err) { + console.error(err); + } } toggleHelp(): void { @@ -241,6 +253,11 @@ export class AppElement extends LitElement { skyboxId="test_scene" @scene-state=${this.onSceneStateChange} > + ` : this.scenePath ? html` + ` : html` ): void { + if (changes.has("scenePath") && !!this.scenePath) { + // TODO + } + } +} diff --git a/studio/src/lib.rs b/studio/src/lib.rs index 3641593..da30fb8 100644 --- a/studio/src/lib.rs +++ b/studio/src/lib.rs @@ -27,6 +27,7 @@ mod inspector; mod interaction; pub mod math; mod mixamo; +mod path; mod persistance; mod remote; mod scene; @@ -48,7 +49,7 @@ pub use crate::{ mixamo::{MixamoImportEvent, MixamoPlugin}, persistance::PersistancePlugin, remote::RemoteAssetPlugin, - scene::{LoadSceneEvent, ScenePlugin}, + scene::{LoadLibraryObjectEvent, LoadSceneEvent, ScenePlugin}, skybox::{SkyboxId, SkyboxPlugin}, ui::UiLayerPlugin, viewport::ViewportSize, @@ -120,8 +121,10 @@ pub fn start_viewer( viewport_size: ViewportSize, ) { start_internal( - object_id, - skybox_id, + InitSeed::FromLibrary { + object_id: object_id.into(), + skybox_id: skybox_id.into(), + }, canvas_selector, canvas_alt, viewport_size, @@ -132,15 +135,13 @@ pub fn start_viewer( #[cfg_attr(feature = "wasm", wasm_bindgen(js_name = startEditor))] pub fn start_editor( - object_id: &str, - skybox_id: &str, + scene_id: &str, canvas_selector: &str, canvas_alt: &str, viewport_size: ViewportSize, ) { start_internal( - object_id, - skybox_id, + InitSeed::FromScene(scene_id.into()), canvas_selector, canvas_alt, viewport_size, @@ -158,8 +159,10 @@ pub fn start_bvh_viewer( viewport_size: ViewportSize, ) { let mut app = start_internal( - "Mannequin.gltf", - skybox_id, + InitSeed::FromLibrary { + object_id: "Mannequin.gltf".into(), + skybox_id: skybox_id.into(), + }, canvas_selector, canvas_alt, viewport_size, @@ -188,8 +191,10 @@ pub fn start_mixamo_viewer( viewport_size: ViewportSize, ) { let mut app = start_internal( - "Mannequin.gltf", - skybox_id, + InitSeed::FromLibrary { + object_id: "Mannequin.gltf".into(), + skybox_id: skybox_id.into(), + }, canvas_selector, canvas_alt, viewport_size, @@ -216,8 +221,10 @@ pub fn start( studio_mode: StudioMode, ) { start_internal( - object_id, - skybox_id, + InitSeed::FromLibrary { + object_id: object_id.into(), + skybox_id: skybox_id.into(), + }, canvas_selector, canvas_alt, viewport_size, @@ -226,13 +233,24 @@ pub fn start( .run(); } -// TODO: These `start_` functiions should be refactored to better support the -// needs of the "reaodnly viewer" vs "full studio editor" use cases. See -// notes in `studio-web/src/lib/viewer.element.ts` for more info. +pub enum InitSeed { + /// Initialize the application from assets in the "bundled" asset library. + FromLibrary { + /// A filename, including the extension, that can be found under + /// `studio/assets/gltf` (e.g., "Mannequin.gltf") + object_id: String, + /// A skybox name that corresponds to a matching set of assets under + /// `studio/assets/skyboxes`, _not_ including file extensions or + /// "diffuse"/"specular" qualifiers (e.g., "gum_trees_4k") + skybox_id: String, + }, + /// Initialize the application by loading a previously saved `*.scn.ron` + /// scene file. + FromScene(String), +} pub fn start_internal( - object_id: &str, - skybox_id: &str, + seed: InitSeed, canvas_selector: &str, canvas_alt: &str, viewport_size: ViewportSize, @@ -241,9 +259,6 @@ pub fn start_internal( #[cfg(feature = "wasm")] console_error_panic_hook::set_once(); - let object_id = object_id.to_string(); - let skybox_id = skybox_id.to_string(); - let mut app = App::new(); app.insert_resource(HeadlessMode::None); @@ -274,13 +289,25 @@ pub fn start_internal( app.auto_reflected_undo::(); } - app.add_systems( - Startup, - move |mut cmd: Commands, mut ew: EventWriter| { - cmd.insert_resource(SkyboxId(skybox_id.clone())); - ew.send(LoadSceneEvent(object_id.clone())); - }, - ); + match seed { + InitSeed::FromLibrary { + object_id, + skybox_id, + } => { + app.add_systems( + Startup, + move |mut cmd: Commands, mut ew: EventWriter| { + cmd.insert_resource(SkyboxId(skybox_id.clone())); + ew.send(LoadLibraryObjectEvent(object_id.clone())); + }, + ); + } + InitSeed::FromScene(scene) => { + app.add_systems(Startup, move |mut ew: EventWriter| { + ew.send(LoadSceneEvent(scene.clone())); + }); + } + } #[cfg(feature = "wasm")] wasm::initialize(); @@ -331,10 +358,10 @@ pub fn start_headless( Startup, move |mut cmd: Commands, asset_server: Res, - mut ew: EventWriter| { + mut ew: EventWriter| { cmd.insert_resource(SkyboxId(skybox.clone())); cmd.insert_resource(TestBvh(asset_server.load(bvh_dir.clone()))); - ew.send(LoadSceneEvent("Mannequin.gltf".to_string())); + ew.send(LoadLibraryObjectEvent("Mannequin.gltf".to_string())); }, ); diff --git a/studio/src/main.rs b/studio/src/main.rs index 4007aa7..8ef44f3 100644 --- a/studio/src/main.rs +++ b/studio/src/main.rs @@ -1,6 +1,6 @@ use bevy::{app::Update, window::close_on_esc}; use clap::Parser; -use studio::{screenshot::ScreenShotPlugin, StudioMode, ViewportSize}; +use studio::{screenshot::ScreenShotPlugin, InitSeed, StudioMode, ViewportSize}; #[derive(Parser)] #[command(name = "StorytellerStudio")] @@ -16,13 +16,15 @@ struct StudioArgs { fn main() { let args = StudioArgs::parse(); - let model_path = args.model.unwrap_or("Mannequin.gltf".to_string()); - let skybox_path = args.skybox.unwrap_or("gum_trees_4k".to_string()); + let object_id = args.model.unwrap_or("Mannequin.gltf".to_string()); + let skybox_id = args.skybox.unwrap_or("gum_trees_4k".to_string()); // Edit the arguments here to change the default startup scene or skybox let mut app = studio::start_internal( - model_path.as_str(), - skybox_path.as_str(), + InitSeed::FromLibrary { + object_id, + skybox_id, + }, "", // ignored for non-WASM "Storyteller Studio", ViewportSize::default(), diff --git a/studio/src/path.rs b/studio/src/path.rs new file mode 100644 index 0000000..b266cbb --- /dev/null +++ b/studio/src/path.rs @@ -0,0 +1,40 @@ +use std::{ffi::OsStr, path::Path}; + +pub trait PathExtras { + /// See [std::path::Path::file_prefix] + #[must_use] + fn ext_file_prefix(&self) -> Option<&OsStr>; +} + +impl PathExtras for Path { + fn ext_file_prefix(&self) -> Option<&OsStr> { + self.file_name() + .map(split_file_at_dot) + .map(|(before, _after)| before) + } +} + +/// Note: Copied from private fn `std::path::split_file_at_dot` +fn split_file_at_dot(file: &OsStr) -> (&OsStr, Option<&OsStr>) { + let slice = file.as_encoded_bytes(); + if slice == b".." { + return (file, None); + } + + // The unsafety here stems from converting between &OsStr and &[u8] + // and back. This is safe to do because (1) we only look at ASCII + // contents of the encoding and (2) new &OsStr values are produced + // only from ASCII-bounded slices of existing &OsStr values. + let i = match slice[1..].iter().position(|b| *b == b'.') { + Some(i) => i + 1, + None => return (file, None), + }; + let before = &slice[..i]; + let after = &slice[i + 1..]; + unsafe { + ( + OsStr::from_encoded_bytes_unchecked(before), + Some(OsStr::from_encoded_bytes_unchecked(after)), + ) + } +} diff --git a/studio/src/remote/reader.rs b/studio/src/remote/reader.rs index 1753860..f45d933 100644 --- a/studio/src/remote/reader.rs +++ b/studio/src/remote/reader.rs @@ -2,11 +2,13 @@ use std::{io::ErrorKind, path::Path}; use bevy::{ asset::io::{AssetReader, AssetReaderError, PathStream, Reader, VecReader}, + prelude::*, utils::BoxedFuture, }; use reqwest::StatusCode; use super::response::GetMediaFileSuccessResponse; +use crate::path::PathExtras; pub struct RemoteAssetReader; @@ -22,7 +24,7 @@ impl AssetReader for RemoteAssetReader { ) -> BoxedFuture<'a, Result>, AssetReaderError>> { Box::pin(async move { let token = path - .file_stem() + .ext_file_prefix() .ok_or_else(|| { AssetReaderError::Io(std::io::Error::new( ErrorKind::Other, @@ -32,6 +34,8 @@ impl AssetReader for RemoteAssetReader { .to_str() .unwrap(); + info!("Loading asset from media token: '{token}'"); + let response = reqwest::get(format!("{}/{token}", &Self::API_BASE_URL)) .await .map_err(IntoError::::into)?; diff --git a/studio/src/scene.rs b/studio/src/scene.rs index ed32e68..1faaa8c 100644 --- a/studio/src/scene.rs +++ b/studio/src/scene.rs @@ -13,7 +13,10 @@ use bevy::{ scene::InstanceId, }; -use space_editor::{prelude::PrefabMarker, space_prefab::component::SceneAutoChild}; +use space_editor::{ + prelude::{EditorEvent, EditorPrefabPath, PrefabMarker}, + space_prefab::component::SceneAutoChild, +}; #[cfg(feature = "wasm")] use wasm_bindgen::prelude::wasm_bindgen; use wgpu::{TextureDescriptor, TextureUsages}; @@ -37,6 +40,7 @@ impl Plugin for ScenePlugin { app.insert_resource(ClearColor(Color::hsl(0.0, 0.0, 0.25))) .insert_resource(Msaa::Off) .add_state::() + .add_event::() .add_event::() .add_systems( Startup, @@ -45,7 +49,10 @@ impl Plugin for ScenePlugin { initialiaze_headless.run_if(is_headless), ), ) - .add_systems(Update, init_scene_loading.in_set(SceneSpawnerSystem)) + .add_systems( + Update, + (init_object_loading, init_scene_loading).in_set(SceneSpawnerSystem), + ) .add_systems( OnEnter(SceneState::Active), ( @@ -78,7 +85,7 @@ impl Plugin for ScenePlugin { app.add_systems( Update, ( - wasm::dispatch_scene_request, + wasm::dispatch_object_load_request, wasm::notify_state_changes .map(bevy::utils::error) .run_if(resource_changed::>()), @@ -172,6 +179,33 @@ fn auto_add_handle( } } +/// Request the application to load a `gltf`/`glb` scene from the "bundled" +/// asset library in `assets/gltf/`. Expects a filename including the extension. +/// +/// ```ignore +/// fn main() { +/// App::new() +/// .add_plugins(( +/// DefaultPlugins, +/// CommonPlugins, +/// )) +/// .add_systems(Startup, load_mannequin) +/// } +/// +/// fn load_mannequin(mut ew_load_object: EventWriter) { +/// ew_load_object.send(LoadLibraryObjectEvent("Mannequin.gltf".into())); +/// } +/// ``` +#[derive(Event, Deref)] +pub struct LoadLibraryObjectEvent(pub String); + +/// Request the application to load a previously saved `*.scn.ron` dynamic scene +/// file. +/// +/// Expects a full, resolvable path to the file (e.g. `scenes/my-scene.scn.ron` +/// for a local file saved under `studio/assets/scenes`, or +/// `remote://.scn.ron` for a scene saved to the Storyteller.ai +/// backend). #[derive(Event, Deref)] pub struct LoadSceneEvent(pub String); @@ -270,9 +304,9 @@ fn initialiaze_headless( )); } -fn init_scene_loading(mut cmd: Commands, mut er: EventReader) { +fn init_object_loading(mut cmd: Commands, mut er: EventReader) { if let Some(id) = er.read().last() { - info!("Loading scene: {}", id.0); + info!("Loading library asset: {}", id.0); cmd.spawn(( SpatialBundle::default(), space_editor::prelude::GltfPrefab { @@ -285,6 +319,18 @@ fn init_scene_loading(mut cmd: Commands, mut er: EventReader) { } } +fn init_scene_loading( + mut er_load_scene: EventReader, + mut ew_editor: EventWriter, +) { + if let Some(scene_path) = er_load_scene.read().last() { + info!("Loading scene: {}", scene_path.0); + ew_editor.send(EditorEvent::Load(EditorPrefabPath::File( + scene_path.0.clone(), + ))); + } +} + #[derive(Component)] pub struct NoMeshElement; @@ -442,7 +488,7 @@ mod wasm { use bevy::{prelude::*, window::PrimaryWindow}; use wasm_bindgen::prelude::*; - use super::{LoadSceneEvent, SceneState}; + use super::{LoadLibraryObjectEvent, SceneState}; use crate::wasm::{self, StaticBuffer, WasmMessageBuffer}; static OBJECT_LOAD_REQUEST: WasmMessageBuffer = WasmMessageBuffer::new(); @@ -454,9 +500,11 @@ mod wasm { // FIXME: Is it an object or a scene? Inconsistent terminology is // unnecessarily confusing - pub(super) fn dispatch_scene_request(mut ew_load_scene: EventWriter) { + pub(super) fn dispatch_object_load_request( + mut ew_load_scene: EventWriter, + ) { if let Some(id) = OBJECT_LOAD_REQUEST.take_message() { - ew_load_scene.send(LoadSceneEvent(id)); + ew_load_scene.send(LoadLibraryObjectEvent(id)); } } diff --git a/tsconfig.base.json b/tsconfig.base.json index 34d4b4f..2d911b3 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -21,6 +21,7 @@ "@storyteller/studio-web/canvas-provider": ["studio-web/src/lib/canvas-provider.element.ts"], "@storyteller/studio-web/viewer": ["studio-web/src/lib/viewer.element.ts"], "@storyteller/studio-web/bvh-viewer": ["studio-web/src/lib/bvh-viewer.element.ts"], + "@storyteller/studio-web/editor": ["studio-web/src/lib/editor.element.ts"], "@storyteller/studio-web/mixamo-viewer": ["studio-web/src/lib/mixamo-viewer.element.ts"], "@storyteller/utility": ["utility/src/index.ts"] }