storytold/storyteller-bevy
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-bevy.git synced 2026-10-17 00:13:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Scene storage backend integration (#51)

Browse Source 
This PR adds Storyteller.ai API integration for saving and loading
Storyteller Studio scenes to/from the backend.

### Known issues
* The skybox is not currently stored in the serialized scene format
This commit is contained in:
Danny McGee
2024-02-23 22:04:06 -05:00
committed by GitHub
parent 6ca2a992e5
commit 7b06d48c5d
14 changed files with 244 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-validation.yml
+6 -2
View File
@@ -15,7 +15,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: dtolnay/rust-toolchain@1.76.0
with:
components: rustfmt
- name: Check Rust formatting
run: cargo fmt --all -- --check
@@ -35,7 +37,9 @@ jobs:
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-lint-and-test-${{ hashFiles('**/Cargo.toml') }}
- uses: dtolnay/rust-toolchain@stable
- uses: dtolnay/rust-toolchain@1.76
with:
components: clippy
- name: Install Linux dependencies
run: |
sudo apt-get update
ci.sh
+1 -1
View File
@@ -6,7 +6,7 @@ set -euxo pipefail
rustup target add wasm32-unknown-unknown
# Emit versions
rustup show
rustup show active-toolchain
npm version
# Install project dependencies
package-lock.json
Generated
+9 -2
View File
@@ -14,7 +14,8 @@
"lodash": "^4.17.21",
"react": "18.2.0",
"react-dom": "18.2.0",
"tslib": "^2.3.0"
"tslib": "^2.3.0",
"uuid": "^9.0.1"
},
"devDependencies": {
"@nx/js": "16.9.1",
@@ -29,6 +30,7 @@
"@types/node": "18.14.2",
"@types/react": "18.2.14",
"@types/react-dom": "18.2.6",
"@types/uuid": "^9.0.8",
"@types/which": "^3.0.2",
"@vitejs/plugin-react": "~4.0.0",
"@vitest/coverage-c8": "~0.32.0",
@@ -4410,6 +4412,12 @@
"resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.4.tgz",
"integrity": "sha512-IDaobHimLQhjwsQ/NMwRVfa/yL7L/wriQPMhw1ZJall0KX6E1oxk29XMDeilW5qTIg5aoiqf5Udy8U/51aNoQQ=="
},
"node_modules/@types/uuid": {
"version": "9.0.8",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz",
"integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==",
"dev": true
},
"node_modules/@types/which": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/which/-/which-3.0.2.tgz",
@@ -10194,7 +10202,6 @@
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
"dev": true,
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
package.json
+3 -1
View File
@@ -19,6 +19,7 @@
"@types/node": "18.14.2",
"@types/react": "18.2.14",
"@types/react-dom": "18.2.6",
"@types/uuid": "^9.0.8",
"@types/which": "^3.0.2",
"@vitejs/plugin-react": "~4.0.0",
"@vitest/coverage-c8": "~0.32.0",
@@ -45,6 +46,7 @@
"lodash": "^4.17.21",
"react": "18.2.0",
"react-dom": "18.2.0",
"tslib": "^2.3.0"
"tslib": "^2.3.0",
"uuid": "^9.0.1"
}
}
rust-toolchain.toml
+1 -1
View File
@@ -1,2 +1,2 @@
[toolchain]
channel = "stable"
channel = "1.76.0"
studio-frontend/src/app/app.element.ts
+28 -11
View File
@@ -18,10 +18,12 @@ import { LitElement, html, nothing } from "lit";
import { customElement, state } from "lit/decorators.js";
import { classMap } from "lit-html/directives/class-map.js";
import { Ref, createRef, ref } from "lit-html/directives/ref.js";
import { v4 as uuid } from "uuid";
import "@storyteller/studio-web/canvas-provider";
import "@storyteller/studio-web/viewer";
import "@storyteller/studio-web/bvh-viewer";
import "@storyteller/studio-web/editor";
import "@storyteller/studio-web/mixamo-viewer";
import "./anim-editor.element";
import "./button.element";
@@ -50,6 +52,7 @@ export class AppElement extends LitElement {
@state() bvhPath?: string;
@state() mixamoPath?: string;
@state() scenePath?: string;
@state() mode?: StudioMode;
#sceneObjects: studio.SceneObject[] = [];
@@ -74,6 +77,9 @@ export class AppElement extends LitElement {
if (params.has("mixamo"))
this.mixamoPath = params.get("mixamo")!;
if (params.has("scene"))
this.scenePath = params.get("scene")!;
this.mode = match (params.get("mode"), {
"studio": () => StudioMode.Editor,
"viewer": () => StudioMode.Viewer,
@@ -152,19 +158,25 @@ export class AppElement extends LitElement {
}
@on("window:scene-saved")
onSceneSaved({ detail: scene }: SceneSavedEvent): void {
const fileBlob = new Blob([scene], { type: "text/ron" });
const url = URL.createObjectURL(fileBlob);
async onSceneSaved({ detail: scene }: SceneSavedEvent): Promise<void> {
const file = new File([scene], "scene.scn.ron", { type: "text/ron" });
const formData = new FormData();
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = "scene.scn.ron";
anchor.click();
formData.append("uuid_idempotency_token", uuid());
formData.append("file", file);
setTimeout(() => {
URL.revokeObjectURL(url);
anchor.remove();
});
try {
const response = await fetch("https://api.fakeyou.com/v1/engine/create_scene", {
method: "POST",
credentials: "include",
headers: { "Accept": "text/plain" },
body: formData,
});
console.log(response);
}
catch (err) {
console.error(err);
}
}
toggleHelp(): void {
@@ -241,6 +253,11 @@ export class AppElement extends LitElement {
skyboxId="test_scene"
@scene-state=${this.onSceneStateChange}
></sts-mixamo-viewer>
` : this.scenePath ? html`
<sts-editor
scenePath=${this.scenePath}
@scene-state=${this.onSceneStateChange}
></sts-editor>
` : html`
<sts-viewer
objectId="sample-room.gltf"
studio-web/src/index.ts
+2
View File
@@ -1,3 +1,5 @@
export * from "./lib/bvh-viewer.element";
export * from "./lib/canvas-provider.element";
export * from "./lib/editor.element";
export * from "./lib/mixamo-viewer.element";
export * from "./lib/viewer.element";
studio-web/src/lib/editor.element.ts
+28
View File
@@ -0,0 +1,28 @@
import * as studio from "@storyteller/studio";
import { ViewportSize } from "@storyteller/studio";
import { PropertyValues } from "lit";
import { customElement, property } from "lit/decorators.js";
import { StudioBaseElement } from "./studio-base.element";
@customElement("sts-editor")
export class EditorElement extends StudioBaseElement {
@property() scenePath?: string;
onStart(viewportSize: ViewportSize): void {
if (!this.scenePath) return;
studio.startEditor(
this.scenePath,
`#${this._canvasProvider!.canvasId}`,
"Storyteller Studio",
viewportSize,
);
}
onChanges(changes: PropertyValues<this>): void {
if (changes.has("scenePath") && !!this.scenePath) {
// TODO
}
}
}
studio/src/lib.rs
+57 -30
View File
@@ -27,6 +27,7 @@ mod inspector;
mod interaction;
pub mod math;
mod mixamo;
mod path;
mod persistance;
mod remote;
mod scene;
@@ -48,7 +49,7 @@ pub use crate::{
mixamo::{MixamoImportEvent, MixamoPlugin},
persistance::PersistancePlugin,
remote::RemoteAssetPlugin,
scene::{LoadSceneEvent, ScenePlugin},
scene::{LoadLibraryObjectEvent, LoadSceneEvent, ScenePlugin},
skybox::{SkyboxId, SkyboxPlugin},
ui::UiLayerPlugin,
viewport::ViewportSize,
@@ -120,8 +121,10 @@ pub fn start_viewer(
viewport_size: ViewportSize,
) {
start_internal(
object_id,
skybox_id,
InitSeed::FromLibrary {
object_id: object_id.into(),
skybox_id: skybox_id.into(),
},
canvas_selector,
canvas_alt,
viewport_size,
@@ -132,15 +135,13 @@ pub fn start_viewer(
#[cfg_attr(feature = "wasm", wasm_bindgen(js_name = startEditor))]
pub fn start_editor(
object_id: &str,
skybox_id: &str,
scene_id: &str,
canvas_selector: &str,
canvas_alt: &str,
viewport_size: ViewportSize,
) {
start_internal(
object_id,
skybox_id,
InitSeed::FromScene(scene_id.into()),
canvas_selector,
canvas_alt,
viewport_size,
@@ -158,8 +159,10 @@ pub fn start_bvh_viewer(
viewport_size: ViewportSize,
) {
let mut app = start_internal(
"Mannequin.gltf",
skybox_id,
InitSeed::FromLibrary {
object_id: "Mannequin.gltf".into(),
skybox_id: skybox_id.into(),
},
canvas_selector,
canvas_alt,
viewport_size,
@@ -188,8 +191,10 @@ pub fn start_mixamo_viewer(
viewport_size: ViewportSize,
) {
let mut app = start_internal(
"Mannequin.gltf",
skybox_id,
InitSeed::FromLibrary {
object_id: "Mannequin.gltf".into(),
skybox_id: skybox_id.into(),
},
canvas_selector,
canvas_alt,
viewport_size,
@@ -216,8 +221,10 @@ pub fn start(
studio_mode: StudioMode,
) {
start_internal(
object_id,
skybox_id,
InitSeed::FromLibrary {
object_id: object_id.into(),
skybox_id: skybox_id.into(),
},
canvas_selector,
canvas_alt,
viewport_size,
@@ -226,13 +233,24 @@ pub fn start(
.run();
}
// TODO: These `start_` functiions should be refactored to better support the
// needs of the "reaodnly viewer" vs "full studio editor" use cases. See
// notes in `studio-web/src/lib/viewer.element.ts` for more info.
pub enum InitSeed {
/// Initialize the application from assets in the "bundled" asset library.
FromLibrary {
/// A filename, including the extension, that can be found under
/// `studio/assets/gltf` (e.g., "Mannequin.gltf")
object_id: String,
/// A skybox name that corresponds to a matching set of assets under
/// `studio/assets/skyboxes`, _not_ including file extensions or
/// "diffuse"/"specular" qualifiers (e.g., "gum_trees_4k")
skybox_id: String,
},
/// Initialize the application by loading a previously saved `*.scn.ron`
/// scene file.
FromScene(String),
}
pub fn start_internal(
object_id: &str,
skybox_id: &str,
seed: InitSeed,
canvas_selector: &str,
canvas_alt: &str,
viewport_size: ViewportSize,
@@ -241,9 +259,6 @@ pub fn start_internal(
#[cfg(feature = "wasm")]
console_error_panic_hook::set_once();
let object_id = object_id.to_string();
let skybox_id = skybox_id.to_string();
let mut app = App::new();
app.insert_resource(HeadlessMode::None);
@@ -274,13 +289,25 @@ pub fn start_internal(
app.auto_reflected_undo::<Transform>();
}
app.add_systems(
Startup,
move |mut cmd: Commands, mut ew: EventWriter<LoadSceneEvent>| {
cmd.insert_resource(SkyboxId(skybox_id.clone()));
ew.send(LoadSceneEvent(object_id.clone()));
},
);
match seed {
InitSeed::FromLibrary {
object_id,
skybox_id,
} => {
app.add_systems(
Startup,
move |mut cmd: Commands, mut ew: EventWriter<LoadLibraryObjectEvent>| {
cmd.insert_resource(SkyboxId(skybox_id.clone()));
ew.send(LoadLibraryObjectEvent(object_id.clone()));
},
);
}
InitSeed::FromScene(scene) => {
app.add_systems(Startup, move |mut ew: EventWriter<LoadSceneEvent>| {
ew.send(LoadSceneEvent(scene.clone()));
});
}
}
#[cfg(feature = "wasm")]
wasm::initialize();
@@ -331,10 +358,10 @@ pub fn start_headless(
Startup,
move |mut cmd: Commands,
asset_server: Res<AssetServer>,
mut ew: EventWriter<LoadSceneEvent>| {
mut ew: EventWriter<LoadLibraryObjectEvent>| {
cmd.insert_resource(SkyboxId(skybox.clone()));
cmd.insert_resource(TestBvh(asset_server.load(bvh_dir.clone())));
ew.send(LoadSceneEvent("Mannequin.gltf".to_string()));
ew.send(LoadLibraryObjectEvent("Mannequin.gltf".to_string()));
},
);
studio/src/main.rs
+7 -5
View File
@@ -1,6 +1,6 @@
use bevy::{app::Update, window::close_on_esc};
use clap::Parser;
use studio::{screenshot::ScreenShotPlugin, StudioMode, ViewportSize};
use studio::{screenshot::ScreenShotPlugin, InitSeed, StudioMode, ViewportSize};
#[derive(Parser)]
#[command(name = "StorytellerStudio")]
@@ -16,13 +16,15 @@ struct StudioArgs {
fn main() {
let args = StudioArgs::parse();
let model_path = args.model.unwrap_or("Mannequin.gltf".to_string());
let skybox_path = args.skybox.unwrap_or("gum_trees_4k".to_string());
let object_id = args.model.unwrap_or("Mannequin.gltf".to_string());
let skybox_id = args.skybox.unwrap_or("gum_trees_4k".to_string());
// Edit the arguments here to change the default startup scene or skybox
let mut app = studio::start_internal(
model_path.as_str(),
skybox_path.as_str(),
InitSeed::FromLibrary {
object_id,
skybox_id,
},
"", // ignored for non-WASM
"Storyteller Studio",
ViewportSize::default(),
studio/src/path.rs
+40
View File
@@ -0,0 +1,40 @@
use std::{ffi::OsStr, path::Path};
pub trait PathExtras {
/// See [std::path::Path::file_prefix]
#[must_use]
fn ext_file_prefix(&self) -> Option<&OsStr>;
}
impl PathExtras for Path {
fn ext_file_prefix(&self) -> Option<&OsStr> {
self.file_name()
.map(split_file_at_dot)
.map(|(before, _after)| before)
}
}
/// Note: Copied from private fn `std::path::split_file_at_dot`
fn split_file_at_dot(file: &OsStr) -> (&OsStr, Option<&OsStr>) {
let slice = file.as_encoded_bytes();
if slice == b".." {
return (file, None);
}
// The unsafety here stems from converting between &OsStr and &[u8]
// and back. This is safe to do because (1) we only look at ASCII
// contents of the encoding and (2) new &OsStr values are produced
// only from ASCII-bounded slices of existing &OsStr values.
let i = match slice[1..].iter().position(|b| *b == b'.') {
Some(i) => i + 1,
None => return (file, None),
};
let before = &slice[..i];
let after = &slice[i + 1..];
unsafe {
(
OsStr::from_encoded_bytes_unchecked(before),
Some(OsStr::from_encoded_bytes_unchecked(after)),
)
}
}
studio/src/remote/reader.rs
+5 -1
View File
@@ -2,11 +2,13 @@ use std::{io::ErrorKind, path::Path};
use bevy::{
asset::io::{AssetReader, AssetReaderError, PathStream, Reader, VecReader},
prelude::*,
utils::BoxedFuture,
};
use reqwest::StatusCode;
use super::response::GetMediaFileSuccessResponse;
use crate::path::PathExtras;
pub struct RemoteAssetReader;
@@ -22,7 +24,7 @@ impl AssetReader for RemoteAssetReader {
) -> BoxedFuture<'a, Result<Box<Reader<'a>>, AssetReaderError>> {
Box::pin(async move {
let token = path
.file_stem()
.ext_file_prefix()
.ok_or_else(|| {
AssetReaderError::Io(std::io::Error::new(
ErrorKind::Other,
@@ -32,6 +34,8 @@ impl AssetReader for RemoteAssetReader {
.to_str()
.unwrap();
info!("Loading asset from media token: '{token}'");
let response = reqwest::get(format!("{}/{token}", &Self::API_BASE_URL))
.await
.map_err(IntoError::<AssetReaderError>::into)?;
studio/src/scene.rs
+56 -8
View File
@@ -13,7 +13,10 @@ use bevy::{
scene::InstanceId,
};
use space_editor::{prelude::PrefabMarker, space_prefab::component::SceneAutoChild};
use space_editor::{
prelude::{EditorEvent, EditorPrefabPath, PrefabMarker},
space_prefab::component::SceneAutoChild,
};
#[cfg(feature = "wasm")]
use wasm_bindgen::prelude::wasm_bindgen;
use wgpu::{TextureDescriptor, TextureUsages};
@@ -37,6 +40,7 @@ impl Plugin for ScenePlugin {
app.insert_resource(ClearColor(Color::hsl(0.0, 0.0, 0.25)))
.insert_resource(Msaa::Off)
.add_state::<SceneState>()
.add_event::<LoadLibraryObjectEvent>()
.add_event::<LoadSceneEvent>()
.add_systems(
Startup,
@@ -45,7 +49,10 @@ impl Plugin for ScenePlugin {
initialiaze_headless.run_if(is_headless),
),
)
.add_systems(Update, init_scene_loading.in_set(SceneSpawnerSystem))
.add_systems(
Update,
(init_object_loading, init_scene_loading).in_set(SceneSpawnerSystem),
)
.add_systems(
OnEnter(SceneState::Active),
(
@@ -78,7 +85,7 @@ impl Plugin for ScenePlugin {
app.add_systems(
Update,
(
wasm::dispatch_scene_request,
wasm::dispatch_object_load_request,
wasm::notify_state_changes
.map(bevy::utils::error)
.run_if(resource_changed::<State<SceneState>>()),
@@ -172,6 +179,33 @@ fn auto_add_handle<T: Asset>(
}
}
/// Request the application to load a `gltf`/`glb` scene from the "bundled"
/// asset library in `assets/gltf/`. Expects a filename including the extension.
///
/// ```ignore
/// fn main() {
/// App::new()
/// .add_plugins((
/// DefaultPlugins,
/// CommonPlugins,
/// ))
/// .add_systems(Startup, load_mannequin)
/// }
///
/// fn load_mannequin(mut ew_load_object: EventWriter<LoadLibraryObjectEvent>) {
/// ew_load_object.send(LoadLibraryObjectEvent("Mannequin.gltf".into()));
/// }
/// ```
#[derive(Event, Deref)]
pub struct LoadLibraryObjectEvent(pub String);
/// Request the application to load a previously saved `*.scn.ron` dynamic scene
/// file.
///
/// Expects a full, resolvable path to the file (e.g. `scenes/my-scene.scn.ron`
/// for a local file saved under `studio/assets/scenes`, or
/// `remote://<media_token>.scn.ron` for a scene saved to the Storyteller.ai
/// backend).
#[derive(Event, Deref)]
pub struct LoadSceneEvent(pub String);
@@ -270,9 +304,9 @@ fn initialiaze_headless(
));
}
fn init_scene_loading(mut cmd: Commands, mut er: EventReader<LoadSceneEvent>) {
fn init_object_loading(mut cmd: Commands, mut er: EventReader<LoadLibraryObjectEvent>) {
if let Some(id) = er.read().last() {
info!("Loading scene: {}", id.0);
info!("Loading library asset: {}", id.0);
cmd.spawn((
SpatialBundle::default(),
space_editor::prelude::GltfPrefab {
@@ -285,6 +319,18 @@ fn init_scene_loading(mut cmd: Commands, mut er: EventReader<LoadSceneEvent>) {
}
}
fn init_scene_loading(
mut er_load_scene: EventReader<LoadSceneEvent>,
mut ew_editor: EventWriter<EditorEvent>,
) {
if let Some(scene_path) = er_load_scene.read().last() {
info!("Loading scene: {}", scene_path.0);
ew_editor.send(EditorEvent::Load(EditorPrefabPath::File(
scene_path.0.clone(),
)));
}
}
#[derive(Component)]
pub struct NoMeshElement;
@@ -442,7 +488,7 @@ mod wasm {
use bevy::{prelude::*, window::PrimaryWindow};
use wasm_bindgen::prelude::*;
use super::{LoadSceneEvent, SceneState};
use super::{LoadLibraryObjectEvent, SceneState};
use crate::wasm::{self, StaticBuffer, WasmMessageBuffer};
static OBJECT_LOAD_REQUEST: WasmMessageBuffer<String> = WasmMessageBuffer::new();
@@ -454,9 +500,11 @@ mod wasm {
// FIXME: Is it an object or a scene? Inconsistent terminology is
// unnecessarily confusing
pub(super) fn dispatch_scene_request(mut ew_load_scene: EventWriter<LoadSceneEvent>) {
pub(super) fn dispatch_object_load_request(
mut ew_load_scene: EventWriter<LoadLibraryObjectEvent>,
) {
if let Some(id) = OBJECT_LOAD_REQUEST.take_message() {
ew_load_scene.send(LoadSceneEvent(id));
ew_load_scene.send(LoadLibraryObjectEvent(id));
}
}
tsconfig.base.json
+1
View File
@@ -21,6 +21,7 @@
"@storyteller/studio-web/canvas-provider": ["studio-web/src/lib/canvas-provider.element.ts"],
"@storyteller/studio-web/viewer": ["studio-web/src/lib/viewer.element.ts"],
"@storyteller/studio-web/bvh-viewer": ["studio-web/src/lib/bvh-viewer.element.ts"],
"@storyteller/studio-web/editor": ["studio-web/src/lib/editor.element.ts"],
"@storyteller/studio-web/mixamo-viewer": ["studio-web/src/lib/mixamo-viewer.element.ts"],
"@storyteller/utility": ["utility/src/index.ts"]
}