mirror of
https://github.com/storytold/storyteller-bevy.git
synced 2026-10-17 00:13:17 +00:00
Storyteller.ai API integration (#49)
This PR integrates asset retrieval from the Storyteller.ai back-end API, and begins implementing long-term scene saving and storage. ### Media retrieval Instead of fetching raw asset URLs, going forward, media files from remote sources (e.g., in URL parameters) should be specified with a path like the following: ``` remote://<token>.<filetype> ``` where * `token` is a media token (e.g., "m_70b77neq9jsgmy5v31mekzchcbd223") * `filetype` is a file extension appropriate for the type of file being fetched (e.g., "gltf") The `remote` protocol will be recognized by a [custom asset reader](https://github.com/storytold/storyteller-bevy/blob/70726db111f6ccafd47feb055c5546540fef1786/studio/src/remote/reader.rs) which will retrieve the file by calling a Storyteller API endpoint. ### Scene storage Additionally, this PR begins laying the groundwork for storing/retrieving Bevy scenes to/from the backend instead of solely in-memory. This work is still incomplete -- as of this PR, a button has been added to the web UI allowing the user to download the scene file locally for testing/debugging purposes. ### Test URL Media retrieval from the backend can be tested by [visiting this URL](https://deploy-preview-49--storyteller-studio.netlify.app/?mode=viewer&mixamo=remote://m_70b77neq9jsgmy5v31mekzchcbd223.gltf) in Chrome with the `--disable-web-security` flag (to bypass CORS errors)
This commit is contained in:
@@ -28,6 +28,7 @@ mod interaction;
|
||||
pub mod math;
|
||||
mod mixamo;
|
||||
mod persistance;
|
||||
mod remote;
|
||||
mod scene;
|
||||
pub mod screenshot;
|
||||
mod skybox;
|
||||
@@ -46,6 +47,7 @@ pub use crate::{
|
||||
math::InterpolationPlugin,
|
||||
mixamo::{MixamoImportEvent, MixamoPlugin},
|
||||
persistance::PersistancePlugin,
|
||||
remote::RemoteAssetPlugin,
|
||||
scene::{LoadSceneEvent, ScenePlugin},
|
||||
skybox::{SkyboxId, SkyboxPlugin},
|
||||
ui::UiLayerPlugin,
|
||||
@@ -251,6 +253,7 @@ pub fn start_internal(
|
||||
#[cfg(feature = "wasm")]
|
||||
viewport::wasm::ViewportPlugin,
|
||||
WebAssetPlugin,
|
||||
RemoteAssetPlugin,
|
||||
DefaultPlugins.set(WindowPlugin {
|
||||
primary_window: Some(Window {
|
||||
resolution: viewport_size.into(),
|
||||
|
||||
@@ -10,6 +10,13 @@ impl Plugin for PersistancePlugin {
|
||||
EditorTabName::Other("Persistance".to_string()),
|
||||
DebugPersistanceTab::default(),
|
||||
);
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
app.add_systems(Update, wasm::forward_save_scene_events)
|
||||
.add_systems(
|
||||
OnEnter(SaveState::Idle),
|
||||
wasm::on_scene_saved.map(bevy::utils::warn),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,3 +50,64 @@ impl EditorTab for DebugPersistanceTab {
|
||||
"Persistance".into()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "wasm")]
|
||||
mod wasm {
|
||||
use bevy::{prelude::*, window::PrimaryWindow};
|
||||
use space_editor::prelude::{EditorEvent, EditorPrefabPath, PrefabMemoryCache};
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
use crate::wasm::{self, StaticBuffer, WasmMessageBuffer};
|
||||
|
||||
static SAVE_SCENE: WasmMessageBuffer<()> = WasmMessageBuffer::new();
|
||||
|
||||
#[wasm_bindgen(js_name = saveScene)]
|
||||
pub fn save_scene() {
|
||||
SAVE_SCENE.write_message(());
|
||||
}
|
||||
|
||||
pub(super) fn forward_save_scene_events(mut ew_editor_events: EventWriter<EditorEvent>) {
|
||||
if SAVE_SCENE.take_message().is_some() {
|
||||
ew_editor_events.send(EditorEvent::Save(EditorPrefabPath::MemoryCahce));
|
||||
}
|
||||
}
|
||||
|
||||
#[wasm_bindgen(typescript_custom_section)]
|
||||
const SCENE_SAVED_EVENT: &str = r#"
|
||||
export interface SceneSavedEvent extends CustomEvent {
|
||||
type: "scene-saved";
|
||||
detail: string;
|
||||
}
|
||||
|
||||
declare global {
|
||||
export interface GlobalEventHandlersEventMap {
|
||||
"scene-saved": SceneSavedEvent;
|
||||
}
|
||||
}
|
||||
"#;
|
||||
|
||||
pub(super) fn on_scene_saved(
|
||||
r_scene_cache: Res<PrefabMemoryCache>,
|
||||
r_type_registry: Res<AppTypeRegistry>,
|
||||
ra_scenes: Res<Assets<DynamicScene>>,
|
||||
q_window: Query<&Window, With<PrimaryWindow>>,
|
||||
) -> Result<(), String> {
|
||||
let handle = r_scene_cache
|
||||
.scene
|
||||
.as_ref()
|
||||
.ok_or_else(|| "Scene cache is empty!".to_string())?;
|
||||
|
||||
let serialized = match ra_scenes.get(handle) {
|
||||
Some(scene) => scene
|
||||
.serialize_ron(&r_type_registry)
|
||||
.map_err(|err| format!("{err}")),
|
||||
None => Err("No scene found for cached handle!".into()),
|
||||
}?;
|
||||
|
||||
let Ok(window) = q_window.get_single() else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
wasm::dispatch_to_js(window, "scene-saved", &serialized)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
use bevy::{
|
||||
asset::io::{AssetSource, AssetSourceId},
|
||||
prelude::*,
|
||||
};
|
||||
|
||||
use self::reader::RemoteAssetReader;
|
||||
|
||||
mod reader;
|
||||
mod response;
|
||||
|
||||
pub struct RemoteAssetPlugin;
|
||||
|
||||
impl Plugin for RemoteAssetPlugin {
|
||||
fn build(&self, app: &mut App) {
|
||||
app.register_asset_source(
|
||||
AssetSourceId::Name("remote".into()),
|
||||
AssetSource::build()
|
||||
.with_reader(|| Box::new(RemoteAssetReader))
|
||||
.with_writer(|| None),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
use std::{io::ErrorKind, path::Path};
|
||||
|
||||
use bevy::{
|
||||
asset::io::{AssetReader, AssetReaderError, PathStream, Reader, VecReader},
|
||||
utils::BoxedFuture,
|
||||
};
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use super::response::GetMediaFileSuccessResponse;
|
||||
|
||||
pub struct RemoteAssetReader;
|
||||
|
||||
impl RemoteAssetReader {
|
||||
const API_BASE_URL: &'static str = "https://api.fakeyou.com/v1/media_files/file";
|
||||
const BUCKET_BASE_URL: &'static str = "https://storage.googleapis.com/vocodes-public";
|
||||
}
|
||||
|
||||
impl AssetReader for RemoteAssetReader {
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
path: &'a Path,
|
||||
) -> BoxedFuture<'a, Result<Box<Reader<'a>>, AssetReaderError>> {
|
||||
Box::pin(async move {
|
||||
let token = path
|
||||
.file_stem()
|
||||
.ok_or_else(|| {
|
||||
AssetReaderError::Io(std::io::Error::new(
|
||||
ErrorKind::Other,
|
||||
format!("Failed to parse token from '{path:?}'"),
|
||||
))
|
||||
})?
|
||||
.to_str()
|
||||
.unwrap();
|
||||
|
||||
let response = reqwest::get(format!("{}/{token}", &Self::API_BASE_URL))
|
||||
.await
|
||||
.map_err(IntoError::<AssetReaderError>::into)?;
|
||||
|
||||
let response = match response.status() {
|
||||
StatusCode::OK => response
|
||||
.json::<GetMediaFileSuccessResponse>()
|
||||
.await
|
||||
.map_err(IntoError::<AssetReaderError>::into),
|
||||
other => Err(AssetReaderError::Io(std::io::Error::new(
|
||||
ErrorKind::Other,
|
||||
format!("Status Code from endpoint: {other}"),
|
||||
))),
|
||||
}?;
|
||||
|
||||
let url = format!(
|
||||
"{}{}",
|
||||
&Self::BUCKET_BASE_URL,
|
||||
&response.media_file.public_bucket_path
|
||||
);
|
||||
|
||||
let response = reqwest::get(url)
|
||||
.await
|
||||
.map_err(IntoError::<AssetReaderError>::into)?;
|
||||
|
||||
let bytes = match response.status() {
|
||||
// TODO: This is not thread-safe, but some variation of this
|
||||
// would be better than the current solution once Bevy
|
||||
// WASM becomes multi-threaded:
|
||||
// StatusCode::OK => Ok(response.bytes_stream().map(|result| {
|
||||
// result.map_err(|err| std::io::Error::new(ErrorKind::Other, format!("{err}")))
|
||||
// })),
|
||||
StatusCode::OK => Ok(response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(IntoError::<AssetReaderError>::into)?
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>()),
|
||||
other => Err(AssetReaderError::Io(std::io::Error::new(
|
||||
ErrorKind::Other,
|
||||
format!("Status Code from bucket URL: {other}"),
|
||||
))),
|
||||
}?;
|
||||
|
||||
let reader: Box<Reader> = Box::new(VecReader::new(bytes));
|
||||
Ok(reader)
|
||||
})
|
||||
}
|
||||
|
||||
fn read_meta<'a>(
|
||||
&'a self,
|
||||
path: &'a Path,
|
||||
) -> BoxedFuture<'a, Result<Box<Reader<'a>>, AssetReaderError>> {
|
||||
// TODO: Do we want to try to read .meta files?
|
||||
Box::pin(async { Err(AssetReaderError::NotFound(path.into())) })
|
||||
}
|
||||
|
||||
fn read_directory<'a>(
|
||||
&'a self,
|
||||
path: &'a Path,
|
||||
) -> BoxedFuture<'a, Result<Box<PathStream>, AssetReaderError>> {
|
||||
// We won't have to deal with directories
|
||||
Box::pin(async { Err(AssetReaderError::NotFound(path.into())) })
|
||||
}
|
||||
|
||||
fn is_directory<'a>(&'a self, _: &'a Path) -> BoxedFuture<'a, Result<bool, AssetReaderError>> {
|
||||
// We won't have to deal with directories
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
trait IntoError<Output> {
|
||||
fn into(self) -> Output;
|
||||
}
|
||||
|
||||
impl IntoError<AssetReaderError> for reqwest::Error {
|
||||
fn into(self) -> AssetReaderError {
|
||||
AssetReaderError::Io(std::io::Error::new(ErrorKind::Other, format!("{self}")))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
//! Response types copy/pasted straight from the backend source code.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GetMediaFileSuccessResponse {
|
||||
pub success: bool,
|
||||
pub media_file: MediaFileInfo,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct MediaFileInfo {
|
||||
pub token: String,
|
||||
|
||||
/// Type of media will dictate which fields are populated and what
|
||||
/// the frontend should display (eg. video player vs audio player).
|
||||
pub media_type: String,
|
||||
|
||||
/// If the file was generated as part of a batch, this is the token for the batch.
|
||||
pub maybe_batch_token: Option<String>,
|
||||
|
||||
/// URL to the media file
|
||||
pub public_bucket_path: String,
|
||||
|
||||
//// Provenance Data
|
||||
//pub origin_product: MediaFileOriginProductCategory,
|
||||
//pub origin_category: MediaFileOriginCategory,
|
||||
//pub maybe_origin_model_type: Option<MediaFileOriginModelType>,
|
||||
//pub maybe_origin_model_token: Option<String>,
|
||||
//pub maybe_origin_filename: Option<String>,
|
||||
/// Possible model info (if generated by inference)
|
||||
pub maybe_model_weight_info: Option<GetMediaFileModelInfo>,
|
||||
|
||||
/// User info
|
||||
pub maybe_creator_user: Option<UserDetailsLight>,
|
||||
|
||||
pub creator_set_visibility: String,
|
||||
|
||||
/// Text transcripts for TTS, etc.
|
||||
pub maybe_text_transcript: Option<String>,
|
||||
|
||||
/// We can simulate media files for "tts_results" records.
|
||||
/// If this is set as true, this informs the frontend and API callers not to treat
|
||||
/// the token as a media file and improperly assume it can be used with the rest of
|
||||
/// the media file APIs.
|
||||
pub is_emulated_media_file: bool,
|
||||
|
||||
/// Statistics about the media file
|
||||
pub stats: SimpleEntityStats,
|
||||
|
||||
pub created_at: String,
|
||||
pub updated_at: String,
|
||||
//pub maybe_moderator_fields: Option<ModeratorFields>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct GetMediaFileModelInfo {
|
||||
pub weight_token: String,
|
||||
pub weight_type: String,
|
||||
pub weight_category: String,
|
||||
pub title: String,
|
||||
|
||||
/// Cover images are small descriptive images that can be set for any model.
|
||||
/// If a cover image is set, this is the path to the asset.
|
||||
pub maybe_cover_image_public_bucket_path: Option<String>,
|
||||
|
||||
/// NB: Technically this shouldn't be a nullable field, but since there are so many
|
||||
/// joins we'll relax that this must exist (for now).
|
||||
pub maybe_weight_creator: Option<UserDetailsLight>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct UserDetailsLight {
|
||||
/// The token for the user
|
||||
pub user_token: String,
|
||||
|
||||
/// The unique username someone logs in with
|
||||
/// As of 2023-08-23, this is always lowercase
|
||||
pub username: String,
|
||||
|
||||
/// As of 2023-08-23, this is the username with capitalization
|
||||
/// (In the future, a display name can be customized by the user.)
|
||||
pub display_name: String,
|
||||
|
||||
/// Email hash for Gravatar
|
||||
/// Always set for now since login is email/username+password.
|
||||
/// In the future this will need to become an optional *OR* be filled with a bogus hash.
|
||||
pub gravatar_hash: String,
|
||||
|
||||
/// For users without a gravatar, we show one of our own avatars.
|
||||
pub default_avatar: DefaultAvatarInfo,
|
||||
// In the future, we'll also support user-uploaded avatars that we store on our servers.
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct DefaultAvatarInfo {
|
||||
pub image_index: u8,
|
||||
pub color_index: u8,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct SimpleEntityStats {
|
||||
/// Number of positive ratings (or "likes") for this item
|
||||
pub positive_rating_count: u32,
|
||||
/// Number of bookmarks for this item
|
||||
pub bookmark_count: u32,
|
||||
}
|
||||
Reference in New Issue
Block a user