Storyteller.ai API integration (#49)

This PR integrates asset retrieval from the Storyteller.ai back-end API,
and begins implementing long-term scene saving and storage.

### Media retrieval
Instead of fetching raw asset URLs, going forward, media files from
remote sources (e.g., in URL parameters) should be specified with a path
like the following:
```
remote://<token>.<filetype>
```
where
* `token` is a media token (e.g., "m_70b77neq9jsgmy5v31mekzchcbd223")
* `filetype` is a file extension appropriate for the type of file being
  fetched (e.g., "gltf")

The `remote` protocol will be recognized by a [custom asset
reader](https://github.com/storytold/storyteller-bevy/blob/70726db111f6ccafd47feb055c5546540fef1786/studio/src/remote/reader.rs)
which will retrieve the file by calling a Storyteller API endpoint.

### Scene storage
Additionally, this PR begins laying the groundwork for
storing/retrieving Bevy scenes to/from the backend instead of solely
in-memory. This work is still incomplete -- as of this PR, a button has
been added to the web UI allowing the user to download the scene
file locally for testing/debugging purposes.

### Test URL
Media retrieval from the backend can be tested by [visiting this
URL](https://deploy-preview-49--storyteller-studio.netlify.app/?mode=viewer&mixamo=remote://m_70b77neq9jsgmy5v31mekzchcbd223.gltf)
in Chrome with the `--disable-web-security` flag (to bypass CORS errors)
This commit is contained in:
Danny McGee
2024-02-23 21:05:32 -05:00
committed by GitHub
parent fffcb38ec6
commit 6ca2a992e5
10 changed files with 770 additions and 27 deletions
+22
View File
@@ -0,0 +1,22 @@
use bevy::{
asset::io::{AssetSource, AssetSourceId},
prelude::*,
};
use self::reader::RemoteAssetReader;
mod reader;
mod response;
pub struct RemoteAssetPlugin;
impl Plugin for RemoteAssetPlugin {
fn build(&self, app: &mut App) {
app.register_asset_source(
AssetSourceId::Name("remote".into()),
AssetSource::build()
.with_reader(|| Box::new(RemoteAssetReader))
.with_writer(|| None),
);
}
}
+114
View File
@@ -0,0 +1,114 @@
use std::{io::ErrorKind, path::Path};
use bevy::{
asset::io::{AssetReader, AssetReaderError, PathStream, Reader, VecReader},
utils::BoxedFuture,
};
use reqwest::StatusCode;
use super::response::GetMediaFileSuccessResponse;
pub struct RemoteAssetReader;
impl RemoteAssetReader {
const API_BASE_URL: &'static str = "https://api.fakeyou.com/v1/media_files/file";
const BUCKET_BASE_URL: &'static str = "https://storage.googleapis.com/vocodes-public";
}
impl AssetReader for RemoteAssetReader {
fn read<'a>(
&'a self,
path: &'a Path,
) -> BoxedFuture<'a, Result<Box<Reader<'a>>, AssetReaderError>> {
Box::pin(async move {
let token = path
.file_stem()
.ok_or_else(|| {
AssetReaderError::Io(std::io::Error::new(
ErrorKind::Other,
format!("Failed to parse token from '{path:?}'"),
))
})?
.to_str()
.unwrap();
let response = reqwest::get(format!("{}/{token}", &Self::API_BASE_URL))
.await
.map_err(IntoError::<AssetReaderError>::into)?;
let response = match response.status() {
StatusCode::OK => response
.json::<GetMediaFileSuccessResponse>()
.await
.map_err(IntoError::<AssetReaderError>::into),
other => Err(AssetReaderError::Io(std::io::Error::new(
ErrorKind::Other,
format!("Status Code from endpoint: {other}"),
))),
}?;
let url = format!(
"{}{}",
&Self::BUCKET_BASE_URL,
&response.media_file.public_bucket_path
);
let response = reqwest::get(url)
.await
.map_err(IntoError::<AssetReaderError>::into)?;
let bytes = match response.status() {
// TODO: This is not thread-safe, but some variation of this
// would be better than the current solution once Bevy
// WASM becomes multi-threaded:
// StatusCode::OK => Ok(response.bytes_stream().map(|result| {
// result.map_err(|err| std::io::Error::new(ErrorKind::Other, format!("{err}")))
// })),
StatusCode::OK => Ok(response
.bytes()
.await
.map_err(IntoError::<AssetReaderError>::into)?
.into_iter()
.collect::<Vec<_>>()),
other => Err(AssetReaderError::Io(std::io::Error::new(
ErrorKind::Other,
format!("Status Code from bucket URL: {other}"),
))),
}?;
let reader: Box<Reader> = Box::new(VecReader::new(bytes));
Ok(reader)
})
}
fn read_meta<'a>(
&'a self,
path: &'a Path,
) -> BoxedFuture<'a, Result<Box<Reader<'a>>, AssetReaderError>> {
// TODO: Do we want to try to read .meta files?
Box::pin(async { Err(AssetReaderError::NotFound(path.into())) })
}
fn read_directory<'a>(
&'a self,
path: &'a Path,
) -> BoxedFuture<'a, Result<Box<PathStream>, AssetReaderError>> {
// We won't have to deal with directories
Box::pin(async { Err(AssetReaderError::NotFound(path.into())) })
}
fn is_directory<'a>(&'a self, _: &'a Path) -> BoxedFuture<'a, Result<bool, AssetReaderError>> {
// We won't have to deal with directories
Box::pin(async { Ok(false) })
}
}
trait IntoError<Output> {
fn into(self) -> Output;
}
impl IntoError<AssetReaderError> for reqwest::Error {
fn into(self) -> AssetReaderError {
AssetReaderError::Io(std::io::Error::new(ErrorKind::Other, format!("{self}")))
}
}
+107
View File
@@ -0,0 +1,107 @@
//! Response types copy/pasted straight from the backend source code.
use serde::Deserialize;
#[derive(Deserialize, Debug)]
pub struct GetMediaFileSuccessResponse {
pub success: bool,
pub media_file: MediaFileInfo,
}
#[derive(Deserialize, Debug)]
pub struct MediaFileInfo {
pub token: String,
/// Type of media will dictate which fields are populated and what
/// the frontend should display (eg. video player vs audio player).
pub media_type: String,
/// If the file was generated as part of a batch, this is the token for the batch.
pub maybe_batch_token: Option<String>,
/// URL to the media file
pub public_bucket_path: String,
//// Provenance Data
//pub origin_product: MediaFileOriginProductCategory,
//pub origin_category: MediaFileOriginCategory,
//pub maybe_origin_model_type: Option<MediaFileOriginModelType>,
//pub maybe_origin_model_token: Option<String>,
//pub maybe_origin_filename: Option<String>,
/// Possible model info (if generated by inference)
pub maybe_model_weight_info: Option<GetMediaFileModelInfo>,
/// User info
pub maybe_creator_user: Option<UserDetailsLight>,
pub creator_set_visibility: String,
/// Text transcripts for TTS, etc.
pub maybe_text_transcript: Option<String>,
/// We can simulate media files for "tts_results" records.
/// If this is set as true, this informs the frontend and API callers not to treat
/// the token as a media file and improperly assume it can be used with the rest of
/// the media file APIs.
pub is_emulated_media_file: bool,
/// Statistics about the media file
pub stats: SimpleEntityStats,
pub created_at: String,
pub updated_at: String,
//pub maybe_moderator_fields: Option<ModeratorFields>,
}
#[derive(Deserialize, Debug)]
pub struct GetMediaFileModelInfo {
pub weight_token: String,
pub weight_type: String,
pub weight_category: String,
pub title: String,
/// Cover images are small descriptive images that can be set for any model.
/// If a cover image is set, this is the path to the asset.
pub maybe_cover_image_public_bucket_path: Option<String>,
/// NB: Technically this shouldn't be a nullable field, but since there are so many
/// joins we'll relax that this must exist (for now).
pub maybe_weight_creator: Option<UserDetailsLight>,
}
#[derive(Deserialize, Debug)]
pub struct UserDetailsLight {
/// The token for the user
pub user_token: String,
/// The unique username someone logs in with
/// As of 2023-08-23, this is always lowercase
pub username: String,
/// As of 2023-08-23, this is the username with capitalization
/// (In the future, a display name can be customized by the user.)
pub display_name: String,
/// Email hash for Gravatar
/// Always set for now since login is email/username+password.
/// In the future this will need to become an optional *OR* be filled with a bogus hash.
pub gravatar_hash: String,
/// For users without a gravatar, we show one of our own avatars.
pub default_avatar: DefaultAvatarInfo,
// In the future, we'll also support user-uploaded avatars that we store on our servers.
}
#[derive(Deserialize, Debug)]
pub struct DefaultAvatarInfo {
pub image_index: u8,
pub color_index: u8,
}
#[derive(Deserialize, Debug)]
pub struct SimpleEntityStats {
/// Number of positive ratings (or "likes") for this item
pub positive_rating_count: u32,
/// Number of bookmarks for this item
pub bookmark_count: u32,
}