storytold/storyteller-bevy
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-bevy.git synced 2026-10-20 07:54:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into add-inspector-v2

Browse Source
This commit is contained in:
rewin
2024-02-29 22:02:00 +03:00
parent 9f128e599f f31cf35da1
commit 82e7b76bf0
116 changed files with 5010 additions and 1297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
studio/Cargo.toml
+12 -4
View File
@@ -24,11 +24,12 @@ bevy_mod_outline = "0.6.0"
bevy_web_asset = "0.7.0"
bitflags = "2.4.0"
console_error_panic_hook = { version = "0.1.7", optional = true }
itertools = "0.12.1"
js-sys = { version = "0.3.65", optional = true }
parking_lot = "0.12.1"
serde = { version = "1.0.192", optional = true }
serde = "1.0.192"
serde-wasm-bindgen = { version = "0.6.1", optional = true }
serde_json = { version = "1.0.108", optional = true }
serde_json = "1.0.108"
studio-bvh = { path = "../studio-bvh" }
wasm-bindgen = "0.2.84"
web-sys = { version = "0.3.65", optional = true, features = [
@@ -47,13 +48,20 @@ image = "0.24.8"
bytemuck = "1.14.1"
space_editor = {git = "https://github.com/rewin123/space_editor", rev = "ce83f86dc902787ab16213f3209276a784654df6"}
anyhow.workspace = true
serde_repr = "0.1.18"
[dependencies.reqwest]
version = "0.11.24"
features = [
"default",
"json",
"stream",
]
[features]
wasm = [
"dep:console_error_panic_hook",
"dep:js-sys",
"dep:serde",
"dep:serde_json",
"dep:serde-wasm-bindgen",
"dep:web-sys",
"bevy_rapier3d/wasm-bindgen",
studio/assets/scenes/test.ron
+69
View File
File diff suppressed because one or more lines are too long
studio/project.json
+3 -3
View File
@@ -27,7 +27,7 @@
"executor": "nx:run-commands",
"inputs": ["assets"],
"options": {
"command": "node scripts/build-web.mjs",
"command": "node scripts/build-web.mjs --features=wasm",
"cwd": "studio"
}
},
@@ -41,8 +41,8 @@
"headless": {
"executor": "nx:run-commands",
"options": {
"command": "cargo run -p studio --bin studio-headless -- {args}",
}
"command": "cargo run -p studio --bin studio-headless -- {args}"
}
},
"test": {
"executor": "@nxrs/cargo:test",
studio/scripts/build-web.mjs
+7 -2
View File
@@ -13,6 +13,12 @@ async function main() {
const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
const workspaceRoot = path.resolve(__dirname, "../..");
const extraCargoArgs = process.argv.slice(2);
console.log(`Extra cargo args:`);
for (let arg of extraCargoArgs) {
console.log(` ${arg}`);
}
/** @type import("node:child_process").StdioOptions */
const stdio = ["ignore", "inherit", "inherit"];
@@ -26,9 +32,8 @@ async function main() {
"-p", "studio",
"--lib",
"--release",
"--features", "wasm",
"--target", "wasm32-unknown-unknown",
], {
].concat(extraCargoArgs), {
shell: true,
cwd: workspaceRoot,
stdio,
studio/src/anim/camera.rs
+208
View File
@@ -0,0 +1,208 @@
use bevy::prelude::*;
use itertools::Itertools;
use space_editor::space_prefab::editor_registry::EditorRegistryExt;
use crate::math::{InvLerp, Lerp};
use super::GlobalSeekTime;
pub struct CameraAnimPlugin;
impl Plugin for CameraAnimPlugin {
fn build(&self, app: &mut App) {
app.register_type::<CameraKeyframe>()
.editor_registry::<CameraKeyframe>()
.register_type::<CameraTargetKeyframe>()
.editor_registry::<CameraTargetKeyframe>();
app.add_systems(Update, draw_camera_keyframes);
#[cfg(feature = "wasm")]
app.add_systems(
Update,
(
wasm::spawn_camera_keyframes,
wasm::notify_keyframe_changes.map(bevy::utils::error),
wasm::process_update_keyframe,
),
);
}
}
#[derive(Component, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Component)]
pub struct CameraKeyframe(pub f32);
#[derive(Component, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Component)]
pub struct CameraTargetKeyframe(pub f32);
fn draw_camera_keyframes(
mut gizmos: Gizmos,
r_seek_time: Res<GlobalSeekTime>,
q_cam_keyframe_xforms: Query<&GlobalTransform, With<CameraKeyframe>>,
q_cam_target_keyframe_xforms: Query<&GlobalTransform, With<CameraTargetKeyframe>>,
q_cam_keyframes: Query<(&CameraKeyframe, &GlobalTransform)>,
) {
if q_cam_keyframes.is_empty() {
return;
}
for world_xform in q_cam_keyframe_xforms
.iter()
.chain(q_cam_target_keyframe_xforms.iter())
{
let origin = world_xform.translation();
let x_axis = world_xform.right() * 0.25;
let y_axis = world_xform.up() * 0.25;
let z_axis = world_xform.back() * 0.25;
gizmos.line(origin, origin + x_axis, Color::RED);
gizmos.line(origin, origin + y_axis, Color::GREEN);
gizmos.line(origin, origin + z_axis, Color::BLUE);
}
let seek_time = **r_seek_time;
let keyframes_sorted = q_cam_keyframes
.iter()
.sorted_by(|(a, _), (b, _)| a.partial_cmp(&b.0).unwrap())
.collect_vec();
let (timestamp_first, xform_first) = keyframes_sorted.first().unwrap();
let (timestamp_last, xform_last) = keyframes_sorted.last().unwrap();
if seek_time <= timestamp_first.0 {
let (_, rot, pos) = xform_first.to_scale_rotation_translation();
gizmos.sphere(pos, rot, 0.25, Color::WHITE);
} else if seek_time >= timestamp_last.0 {
let (_, rot, pos) = xform_last.to_scale_rotation_translation();
gizmos.sphere(pos, rot, 0.25, Color::WHITE);
} else if let Some(((timestamp_a, xform_a), (timestamp_b, xform_b))) = keyframes_sorted
.iter()
.tuple_windows()
.find(|((a, _), (b, _))| a.0 <= seek_time && b.0 >= seek_time)
{
let t = seek_time.inv_lerp(timestamp_a.0, timestamp_b.0);
let (_, rot_a, pos_a) = xform_a.to_scale_rotation_translation();
let (_, rot_b, pos_b) = xform_b.to_scale_rotation_translation();
let pos = t.lerp(pos_a, pos_b);
let rot = Quat::slerp(rot_a, rot_b, t);
gizmos.sphere(pos, rot, 0.25, Color::WHITE);
}
}
#[cfg(feature = "wasm")]
mod wasm {
use std::sync::Arc;
use bevy::{prelude::*, window::PrimaryWindow};
use space_editor::prelude::space_undo::{self, AddedEntity};
use wasm_bindgen::prelude::*;
use crate::{
camera_controller::CameraController,
wasm::{self, FromStr, StaticBuffer, WasmMessageBuffer},
};
use super::{CameraKeyframe, CameraTargetKeyframe};
static ADD_KEYFRAME: WasmMessageBuffer<f32> = WasmMessageBuffer::new();
static UPDATE_KEYFRAME: WasmMessageBuffer<(Entity, f32)> = WasmMessageBuffer::new();
#[wasm_bindgen(js_name = addCameraKeyframe)]
pub fn add_camera_keyframe(timestamp: f32) {
ADD_KEYFRAME.write_message(timestamp);
}
pub(super) fn spawn_camera_keyframes(
mut cmd: Commands,
mut ew_changes: EventWriter<space_undo::NewChange>,
q_camera: Query<(&CameraController, &GlobalTransform)>,
) {
if let Some(timestamp) = ADD_KEYFRAME.take_message() {
let Ok((cam_controller, world_xform)) = q_camera.get_single() else {
return;
};
let entity = cmd
.spawn((
Name::new("Camera Target Keyframe"),
TransformBundle::from_transform(Transform::from_translation(
cam_controller.focus,
)),
CameraTargetKeyframe(timestamp),
))
.id();
ew_changes.send(space_undo::NewChange {
change: Arc::new(AddedEntity { entity }),
});
let entity = cmd
.spawn((
Name::new("Camera Keyframe"),
TransformBundle::from_transform(Transform::from(*world_xform)),
CameraKeyframe(timestamp),
))
.id();
ew_changes.send(space_undo::NewChange {
change: Arc::new(AddedEntity { entity }),
});
}
}
#[wasm_bindgen(js_name = updateKeyframe)]
pub fn update_keyframe(entity: &str, timestamp: f32) {
let entity = Entity::from_str(entity).unwrap();
UPDATE_KEYFRAME.write_message((entity, timestamp));
}
pub(super) fn process_update_keyframe(
mut q_camera_keyframes: Query<&mut CameraKeyframe>,
mut q_camera_target_keyframes: Query<&mut CameraTargetKeyframe>,
) {
if let Some((ent, timestamp)) = UPDATE_KEYFRAME.take_message() {
if let Ok(mut keyframe) = q_camera_keyframes.get_mut(ent) {
**keyframe = timestamp;
} else if let Ok(mut keyframe) = q_camera_target_keyframes.get_mut(ent) {
**keyframe = timestamp;
} else {
warn!("No matching keyframe found for entity: {ent:?}");
}
}
}
#[wasm_bindgen(typescript_custom_section)]
const CAMERA_KEYFRAMES_EVENT: &str = r#"
export interface CameraKeyframesEvent extends CustomEvent {
type: "camera-keyframes";
detail: Record<string, number>;
}
declare global {
export interface GlobalEventHandlersEventMap {
"camera-keyframes": CameraKeyframesEvent;
}
}
"#;
pub(super) fn notify_keyframe_changes(
mut l_notified: Local<serde_json::Map<String, serde_json::Value>>,
q_keyframes: Query<(Entity, &CameraKeyframe)>,
q_window: Query<&Window, With<PrimaryWindow>>,
) -> Result<(), String> {
if l_notified.len() == q_keyframes.iter().count() {
return Ok(());
}
let Ok(window) = q_window.get_single() else {
return Ok(());
};
l_notified.clear();
for (ent, timestamp) in q_keyframes.iter() {
l_notified.insert(format!("{ent:?}"), timestamp.0.into());
}
wasm::dispatch_to_js(window, "camera-keyframes", &*l_notified)
}
}
studio/src/anim/mod.rs
+36 -3
View File
@@ -6,9 +6,10 @@ use bevy::{
};
use space_editor::space_prefab::editor_registry::EditorRegistryExt;
use crate::hierarchy;
use crate::{hierarchy, scene::SceneElement};
pub use retargeting::*;
mod camera;
mod retargeting;
pub struct AnimPlugin;
@@ -28,11 +29,19 @@ impl Plugin for AnimPlugin {
.editor_registry::<InvBindPose>();
app.add_systems(Update, process_skinned_meshes);
app.insert_resource(GlobalSeekTime(0.));
app.add_plugins(retargeting::RetargetingPlugin);
app.add_plugins((retargeting::RetargetingPlugin, camera::CameraAnimPlugin));
#[cfg(feature = "wasm")]
app.add_systems(Update, wasm::update_global_seek_time);
}
}
#[derive(Resource, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Resource)]
pub struct GlobalSeekTime(pub f32);
#[derive(Component, Clone, Copy, Debug, Default, Reflect)]
#[reflect(Component)]
pub struct Skeleton;
@@ -122,7 +131,7 @@ fn process_skinned_meshes(
// skeleton
let joints = joints.iter().copied().collect::<Vec<_>>();
if let Some(root) = hierarchy::nearest_common_ancestor(&joints, &q_parents) {
cmd.entity(root).insert(Skeleton);
cmd.entity(root).insert((Skeleton, SceneElement::Skeleton));
}
// Tag each joint and find its bind poses
@@ -132,6 +141,7 @@ fn process_skinned_meshes(
cmd.entity(joint).insert((
SkeletalJoint(Color::BLACK),
SceneElement::Bone,
BindPoseLocal(bind_pose_local),
BindPoseGlobal(bind_pose_global.affine()),
InvBindPose(bind_pose_global.affine().inverse()),
@@ -139,3 +149,26 @@ fn process_skinned_meshes(
}
}
}
#[cfg(feature = "wasm")]
mod wasm {
use bevy::prelude::*;
use wasm_bindgen::prelude::*;
use crate::wasm::{StaticBuffer, WasmMessageBuffer};
use super::GlobalSeekTime;
static SEEK_TIME: WasmMessageBuffer<f32> = WasmMessageBuffer::new();
#[wasm_bindgen(js_name = seekTime)]
pub fn seek_time(timestamp: f32) {
SEEK_TIME.write_message(timestamp);
}
pub(super) fn update_global_seek_time(mut r_seek_time: ResMut<GlobalSeekTime>) {
if let Some(timestamp) = SEEK_TIME.take_message() {
**r_seek_time = timestamp;
}
}
}
studio/src/camera_controller.rs
+19 -16
View File
@@ -10,7 +10,7 @@ use crate::{
interaction::Selected,
math::{
Easing, InterpPosition, InterpVelocity, Interpolation, InterpolationSystem, NearlyEq,
Velocity,
NearlyZero, Velocity,
},
};
@@ -78,9 +78,6 @@ fn camera_control_kinematic(
return;
}
let (mut controller, mut xform, proj) = q_camera.single_mut();
let mut window = q_window.single_mut();
let mut any_ongoing = false;
let mut any_started = false;
let mut any_stopped = false;
@@ -92,7 +89,8 @@ fn camera_control_kinematic(
)
}) {
if let InputAction(lifecycle, Action::Camera(motion)) = action {
use CameraMotion::*;
let (mut controller, mut xform, proj) = q_camera.single_mut();
let window = q_window.single();
match lifecycle {
Lifecycle::Start => any_started = true,
@@ -100,20 +98,21 @@ fn camera_control_kinematic(
Lifecycle::Stop => any_stopped = true,
}
use CameraMotion::*;
match motion {
Pan(delta) => do_pan(proj, &window, *delta, &mut xform, &mut controller),
Zoom(delta) => do_zoom(&window, *delta, &mut xform, &mut controller),
Orbit(delta) => do_orbit(&window, *delta, *lifecycle, &mut xform, &mut controller),
Rotate(delta) => do_rotate(&window, *delta, &mut xform, &mut controller),
Pan(delta) => do_pan(proj, window, *delta, &mut xform, &mut controller),
Zoom(delta) => do_zoom(window, *delta, &mut xform, &mut controller),
Orbit(delta) => do_orbit(window, *delta, *lifecycle, &mut xform, &mut controller),
Rotate(delta) => do_rotate(window, *delta, &mut xform, &mut controller),
_ => {}
}
}
}
if any_started {
capture_pointer(&mut window);
capture_pointer(&mut q_window.single_mut());
} else if any_stopped && !any_ongoing {
release_pointer(&mut window);
release_pointer(&mut q_window.single_mut());
}
}
@@ -215,12 +214,16 @@ fn process_focus_inputs(
}
fn camera_control_physical(
mut q_camera: Query<(&mut CameraController, &mut Transform, &Velocity)>,
mut q_camera: Query<(Entity, &mut CameraController, &Velocity)>,
mut q_xforms: Query<&mut Transform>,
) {
for (mut controller, mut xform, velocity) in q_camera.iter_mut() {
let delta_pos = xform.rotation * (**velocity);
xform.translation += delta_pos;
controller.focus += delta_pos;
for (ent, mut controller, velocity) in q_camera.iter_mut() {
if !velocity.nearly_zero() {
let mut xform = q_xforms.get_mut(ent).unwrap();
let delta_pos = xform.rotation * (**velocity);
xform.translation += delta_pos;
controller.focus += delta_pos;
}
}
}
studio/src/change_tracking.rs
+92
View File
@@ -0,0 +1,92 @@
use bevy::prelude::*;
use crate::scene::{SceneElement, SceneSpawnerSystem, SceneState};
pub struct ChangeTrackingPlugin;
impl Plugin for ChangeTrackingPlugin {
fn build(&self, app: &mut App) {
app.add_event::<EntityChangedEvent>();
// Add more of these systems for each component that we want to track
app.add_systems(Update, send_change_events::<Transform>);
app.add_systems(
OnEnter(SceneState::Active),
track_scene_elements.after(SceneSpawnerSystem),
);
#[cfg(feature = "wasm")]
app.add_systems(
Update,
wasm::send_change_events::<Transform>.map(bevy::utils::error),
);
}
}
#[derive(Component, Debug)]
pub struct ChangeTracker;
fn send_change_events<T: Component>(
query: Query<Entity, (Changed<T>, With<ChangeTracker>)>,
mut entity_change_writer: EventWriter<EntityChangedEvent>,
) {
for entity in query.iter() {
entity_change_writer.send(EntityChangedEvent(entity));
}
}
fn track_scene_elements(
mut cmd: Commands,
q_scene_elements: Query<Entity, (With<Transform>, With<SceneElement>, Without<ChangeTracker>)>,
) {
for element in q_scene_elements.iter() {
cmd.entity(element).insert(ChangeTracker);
}
}
#[derive(Event, Deref)]
pub struct EntityChangedEvent(pub Entity);
#[cfg(feature = "wasm")]
mod wasm {
use bevy::{prelude::*, window::PrimaryWindow};
use wasm_bindgen::prelude::wasm_bindgen;
use crate::wasm;
use super::ChangeTracker;
#[wasm_bindgen(typescript_custom_section)]
const ENTITY_CHANGES_EVENT: &str = r#"
export interface EntityChangesEvent extends CustomEvent {
type: "entity-changes";
detail: string[];
}
declare global {
export interface GlobalEventHandlersEventMap {
"entity-changes": EntityChangesEvent;
}
}
"#;
pub(super) fn send_change_events<T: Component>(
q_changed_ents: Query<Entity, (Changed<T>, With<ChangeTracker>)>,
q_window: Query<&Window, With<PrimaryWindow>>,
) -> Result<(), String> {
if q_changed_ents.is_empty() {
return Ok(());
}
let Ok(window) = q_window.get_single() else {
return Ok(());
};
let entities = q_changed_ents
.iter()
.map(|ent| format!("{ent:?}"))
.collect::<Vec<_>>();
wasm::dispatch_to_js(window, "entity-changes", &entities)
}
}
studio/src/headless.rs
+165
View File
@@ -0,0 +1,165 @@
use std::time::Duration;
use bevy::{
app::{AppExit, ScheduleRunnerPlugin},
prelude::*,
};
use space_editor::prelude::{EditorEvent, EditorPrefabPath};
use wgpu::Extent3d;
use crate::{
bvh::{BvhAnimCompleteEvent, TestBvh},
screenshot::{ScreenCapture, ScreenShotPlugin},
BvhPlugin, CommonPlugins, LoadLibraryObjectEvent, LoadSceneEvent, MainCamera,
MixamoImportEvent, MixamoPlugin, SkyboxId, StudioMode, WorldGridPlugin,
};
#[derive(Debug, Resource, Clone)]
pub enum HeadlessMode {
Bvh(String, String),
Scene(String, String),
Mixamo(String, String),
None,
}
pub fn start_headless(
mode: HeadlessMode,
fps: u32,
outout_dir: &str,
frames_to_capture: u32,
image_res: Extent3d,
) -> App {
let mut app = App::new();
app.insert_resource(mode.clone());
app.insert_resource(StudioMode::Headless);
app.insert_resource(HeadlessSettings {
max_frames: frames_to_capture,
output_dims: image_res,
output_dir: outout_dir.to_string(),
});
app.add_plugins((
ScheduleRunnerPlugin::run_loop(Duration::from_secs_f64(1.0 / fps as f64)),
DefaultPlugins.set(WindowPlugin {
primary_window: None,
exit_condition: bevy::window::ExitCondition::DontExit,
close_when_requested: true,
}),
ScreenShotPlugin,
CommonPlugins.build().disable::<WorldGridPlugin>(),
));
match mode {
HeadlessMode::Bvh(bvh_dir, skybox) => {
app.add_plugins(BvhPlugin);
app.add_systems(
Startup,
move |mut cmd: Commands,
asset_server: Res<AssetServer>,
mut ew: EventWriter<LoadLibraryObjectEvent>| {
cmd.insert_resource(SkyboxId(skybox.clone()));
cmd.insert_resource(TestBvh(asset_server.load(bvh_dir.clone())));
ew.send(LoadLibraryObjectEvent("Mannequin.gltf".to_string()));
},
);
app.add_systems(PostUpdate, check_for_bvh_capture_complete);
}
HeadlessMode::Scene(scene, skybox) => {
app.add_systems(
Startup,
move |mut cmd: Commands, mut ew: EventWriter<EditorEvent>| {
cmd.insert_resource(SkyboxId(skybox.clone()));
ew.send(EditorEvent::Load(EditorPrefabPath::File(scene.clone())))
},
);
}
HeadlessMode::Mixamo(mixamo_path, skybox) => {
app.add_plugins(MixamoPlugin);
app.add_systems(
Startup,
move |mut commands: Commands,
mut ew: EventWriter<MixamoImportEvent>,
mut load_scene_event: EventWriter<LoadSceneEvent>| {
ew.send(MixamoImportEvent(mixamo_path.clone()));
load_scene_event.send(LoadSceneEvent("Mannequin.gltf".to_string()));
commands.insert_resource(SkyboxId(skybox.clone()));
},
);
app.add_systems(PostUpdate, check_for_mixamo_capture_complete);
app.run();
}
HeadlessMode::None => {}
}
app
}
pub fn is_headless(studio_state: Res<StudioMode>) -> bool {
*studio_state == StudioMode::Headless
}
pub fn is_not_headless(studio_state: Res<StudioMode>) -> bool {
*studio_state != StudioMode::Headless
}
#[derive(Resource)]
pub struct HeadlessSettings {
pub output_dims: Extent3d,
pub output_dir: String,
pub max_frames: u32,
}
impl Default for HeadlessSettings {
fn default() -> Self {
HeadlessSettings {
output_dims: Extent3d {
width: 1280,
height: 720,
depth_or_array_layers: 1,
},
output_dir: "./screen_shots/".to_string(),
max_frames: u32::MAX,
}
}
}
fn check_for_bvh_capture_complete(
mut close_app: EventWriter<AppExit>,
mut main_camera: Query<&mut ScreenCapture, With<MainCamera>>,
bvh_event_reader: EventReader<BvhAnimCompleteEvent>,
) {
let Ok(mut screen_capture) = main_camera.get_single_mut() else {
return;
};
println!("Frames captured {}", screen_capture.frames_captured());
if !screen_capture.should_capture() || !bvh_event_reader.is_empty() {
screen_capture.stop_capturing();
close_app.send(AppExit);
}
}
fn check_for_mixamo_capture_complete(
mut close_app: EventWriter<AppExit>,
mut main_camera: Query<&mut ScreenCapture, With<MainCamera>>,
mixamo_anims: Query<&AnimationPlayer>,
) {
let Ok(mut capture) = main_camera.get_single_mut() else {
return;
};
for anim in mixamo_anims.iter() {
println!(
"Time elapsed {} | Frames Captured : {}",
anim.elapsed(),
capture.frames_captured()
);
if anim.completions() != 0 {
capture.stop_capturing();
close_app.send(AppExit);
}
}
}
studio/src/input.rs
+19 -5
View File
@@ -79,6 +79,8 @@ pub enum Selection {
Clear,
/// Focus the camera on the currently selected entity
Focus,
/// Delete the currently selected entity and its descendants
Delete,
}
impl From<Selection> for Action {
fn from(value: Selection) -> Self {
@@ -100,6 +102,12 @@ fn translate_selection_events(keyboard: Res<Input<KeyCode>>, mut input: EventWri
} else if keyboard.just_released(KeyCode::F) {
input.send(InputAction(Lifecycle::Stop, Selection::Focus.into()));
}
if keyboard.just_pressed(KeyCode::Delete) {
input.send(InputAction(Lifecycle::Start, Selection::Delete.into()));
} else if keyboard.just_released(KeyCode::Delete) {
input.send(InputAction(Lifecycle::Stop, Selection::Delete.into()));
}
}
fn translate_debug_toggle(keyboard: Res<Input<KeyCode>>, mut input: EventWriter<InputAction>) {
@@ -169,11 +177,17 @@ fn translate_dcc_camera_events(
else if (keyboard.just_released(KeyCode::AltLeft) && !keyboard.pressed(KeyCode::AltRight))
|| (keyboard.just_released(KeyCode::AltRight) && !keyboard.pressed(KeyCode::AltLeft))
{
input.send(InputAction(
Lifecycle::Stop,
CameraMotion::Orbit(Vec2::ZERO).into(),
));
input.send(InputAction(Lifecycle::Stop, CameraMotion::Zoom(0.).into()));
// Stop Camera Orbit
if mouse_buttons.pressed(MouseButton::Left) {
input.send(InputAction(
Lifecycle::Stop,
CameraMotion::Orbit(Vec2::ZERO).into(),
));
}
// Stop Camera Zoom
else if mouse_buttons.pressed(MouseButton::Right) {
input.send(InputAction(Lifecycle::Stop, CameraMotion::Zoom(0.).into()));
}
}
// Camera Rotation
else if mouse_buttons.pressed(MouseButton::Right) {
studio/src/inspector/wasm.rs
+35 -54
View File
@@ -4,10 +4,7 @@ use bevy::{prelude::*, utils::HashSet, window::PrimaryWindow};
use serde::Serialize;
use wasm_bindgen::prelude::*;
use crate::{
scene::SceneElement,
wasm::{self, AsJson},
};
use crate::{scene::SceneElement, wasm};
#[wasm_bindgen(typescript_custom_section)]
const TYPE_DECLARATIONS: &str = r#"
@@ -16,6 +13,7 @@ export interface SceneObject {
id: string;
/** A user-friendly name for displaying in the UI */
name: string;
type: SceneElement;
/** The IDs of this object's children in the hierarchy */
children: string[];
/** The ID of this object's parent in the hierarchy, if it has one */
@@ -52,7 +50,7 @@ export interface EntityMultiSpawnEvent extends CustomEvent {
}
declare global {
export interface HTMLElementEventMap {
export interface GlobalEventHandlersEventMap {
"entity-spawn": EntitySpawnEvent;
"entity-despawn": EntityDespawnEvent;
"entity-multi-spawn": EntityMultiSpawnEvent;
@@ -64,50 +62,50 @@ declare global {
pub(super) struct SceneObject {
id: String,
name: String,
#[serde(rename = "type")]
type_: SceneElement,
children: Vec<String>,
parent: Option<String>,
transform: serde_json::Value,
#[serde(rename(serialize = "globalTransform"))]
global_transform: serde_json::Value,
}
struct SceneObjectBuilder<'a> {
struct SceneObjectBuilder {
id: Entity,
name: Option<String>,
type_: Option<SceneElement>,
children: Option<Vec<Entity>>,
parent: Option<Entity>,
transform: Option<&'a Transform>,
global_transform: Option<&'a GlobalTransform>,
}
impl<'a> SceneObjectBuilder<'a> {
impl SceneObjectBuilder {
fn new(id: Entity) -> Self {
Self {
id,
name: None,
type_: None,
children: None,
parent: None,
transform: None,
global_transform: None,
}
}
fn finish(self) -> SceneObject {
SceneObject {
id: format!("{:?}", self.id),
name: self.name.unwrap(),
type_: self.type_.unwrap(),
children: self
.children
.map(|children| children.iter().map(|ent| format!("{ent:?}")).collect())
.unwrap_or_default(),
parent: self.parent.map(|ent| format!("{ent:?}")),
transform: self.transform.unwrap().as_json(),
global_transform: self.global_transform.unwrap().as_json(),
}
}
fn with_name(mut self, name: String) -> Self {
self.name = Some(name);
self
}
fn with_type(mut self, type_: SceneElement) -> Self {
self.type_ = Some(type_);
self
}
fn with_children(mut self, children: Vec<Entity>) -> Self {
self.children = Some(children);
self
@@ -116,18 +114,10 @@ impl<'a> SceneObjectBuilder<'a> {
self.parent = Some(parent);
self
}
fn with_transform(mut self, transform: &'a Transform) -> Self {
self.transform = Some(transform);
self
}
fn with_global_transform(mut self, global_transform: &'a GlobalTransform) -> Self {
self.global_transform = Some(global_transform);
self
}
}
impl<'a> SceneObject {
fn build(id: Entity) -> SceneObjectBuilder<'a> {
impl SceneObject {
fn build(id: Entity) -> SceneObjectBuilder {
SceneObjectBuilder::new(id)
}
}
@@ -140,8 +130,8 @@ pub(super) fn notify_entity_spawn_changes(
mut l_despawned_entities: Local<Vec<Entity>>,
mut l_missing_parents: Local<VecDeque<Entity>>,
q_window: Query<&Window, With<PrimaryWindow>>,
q_inspectables: Query<(Entity, &Transform, &GlobalTransform), With<SceneElement>>,
q_others: Query<(Entity, &Transform, &GlobalTransform), Without<SceneElement>>,
q_inspectables: Query<(Entity, &SceneElement)>,
q_others: Query<Entity, Without<SceneElement>>,
q_names: Query<&Name>,
q_parents: Query<&Children>,
q_children: Query<&Parent>,
@@ -149,26 +139,21 @@ pub(super) fn notify_entity_spawn_changes(
l_missing_parents.clear();
l_new_objects.clear();
l_new_objects.extend(
q_inspectables
.iter()
.filter_map(|(ent, xform, world_xform)| {
build_scene_object(
&mut l_notified,
&mut l_missing_parents,
ent,
xform,
world_xform,
&q_inspectables,
&q_names,
&q_parents,
&q_children,
)
}),
);
l_new_objects.extend(q_inspectables.iter().filter_map(|(ent, &type_)| {
build_scene_object(
&mut l_notified,
&mut l_missing_parents,
ent,
type_,
&q_inspectables,
&q_names,
&q_parents,
&q_children,
)
}));
while let Some(parent) = l_missing_parents.pop_front() {
let Ok((ent, xform, world_xform)) = q_others.get(parent) else {
let Ok(ent) = q_others.get(parent) else {
error!("Failed to find missing parent: {parent:?}");
l_notified.insert(parent);
@@ -179,8 +164,7 @@ pub(super) fn notify_entity_spawn_changes(
&mut l_notified,
&mut l_missing_parents,
ent,
xform,
world_xform,
SceneElement::Generic,
&q_inspectables,
&q_names,
&q_parents,
@@ -218,9 +202,8 @@ fn build_scene_object(
notified: &mut HashSet<Entity>,
missing_parents: &mut VecDeque<Entity>,
ent: Entity,
xform: &Transform,
world_xform: &GlobalTransform,
q_inspectables: &Query<(Entity, &Transform, &GlobalTransform), With<SceneElement>>,
type_: SceneElement,
q_inspectables: &Query<(Entity, &SceneElement)>,
q_names: &Query<&Name>,
q_parents: &Query<&Children>,
q_children: &Query<&Parent>,
@@ -231,9 +214,7 @@ fn build_scene_object(
notified.insert(ent);
let mut result = SceneObject::build(ent)
.with_transform(xform)
.with_global_transform(world_xform);
let mut result = SceneObject::build(ent).with_type(type_);
if let Ok(children) = q_parents.get(ent) {
result = result.with_children(children.iter().cloned().collect());
studio/src/interaction/gizmos/common.rs
+1 -1
View File
@@ -6,7 +6,7 @@ use crate::interaction::TransformSpace;
use super::material::GizmoMaterial;
#[derive(Component, Reflect)]
pub(super) struct TransformGizmo {
pub struct TransformGizmo {
pub(super) target: Entity,
pub(super) transform_space: TransformSpace,
}
studio/src/interaction/gizmos/mod.rs
+1
View File
@@ -11,6 +11,7 @@ use self::{
rotation::{spawn_rotation_gizmo, update_ss_ring_rotation},
translation::spawn_translation_gizmo,
};
pub use common::TransformGizmo;
use super::{Selection, TransformMode, TransformType};
studio/src/interaction/selection.rs
+35 -9
View File
@@ -1,8 +1,11 @@
use std::sync::Arc;
use bevy::{prelude::*, render::mesh::skinning::SkinnedMesh};
use bevy_mod_outline::{
AutoGenerateOutlineNormalsPlugin, OutlineBundle, OutlinePlugin, OutlineVolume,
};
use bevy_mod_picking::{prelude::Pointer, selection::Select, PickableBundle};
use space_editor::prelude::space_undo::{self, NewChange, RemovedEntity};
use crate::{
input::{self, Action, InputAction, Lifecycle},
@@ -13,7 +16,13 @@ pub struct SelectionPlugin;
impl Plugin for SelectionPlugin {
fn build(&self, app: &mut App) {
app.add_plugins((OutlinePlugin, AutoGenerateOutlineNormalsPlugin))
.add_systems(Update, update_selection)
.add_systems(
Update,
(
update_selection,
process_delete_inputs.run_if(resource_exists::<Selection>()),
),
)
.add_systems(
PostUpdate,
draw_selection_outline.run_if(resource_changed_or_removed::<Selection>()),
@@ -149,6 +158,28 @@ fn update_selection(
}
}
fn process_delete_inputs(
mut cmd: Commands,
mut er_input: EventReader<InputAction>,
mut ew_changes: EventWriter<space_undo::NewChange>,
r_selection: Res<Selection>,
) {
if er_input.read().any(|action| {
matches!(
action,
InputAction(Lifecycle::Stop, Action::Selection(input::Selection::Delete))
)
}) {
let entity = **r_selection;
cmd.entity(entity).despawn_recursive();
cmd.remove_resource::<Selection>();
ew_changes.send(NewChange {
change: Arc::new(RemovedEntity { entity }),
});
}
}
fn draw_selection_outline(
mut cmd: Commands,
q_selected: Query<&Children, With<Selected>>,
@@ -193,7 +224,7 @@ mod wasm {
use wasm_bindgen::prelude::*;
use super::{Selected, Selection};
use crate::wasm::{self, StaticBuffer, WasmMessageBuffer};
use crate::wasm::{self, FromStr, StaticBuffer, WasmMessageBuffer};
#[wasm_bindgen(typescript_custom_section)]
const EVENT_DECLARATION: &str = r#"
@@ -207,7 +238,7 @@ export interface EntitySelectEvent extends CustomEvent {
}
declare global {
export interface HTMLElementEventMap {
export interface GlobalEventHandlersEventMap {
"entity-select": EntitySelectEvent;
}
}
@@ -225,12 +256,7 @@ declare global {
#[wasm_bindgen]
pub fn select(entity: &str) {
let mut split = entity.split('v');
let index = split.next().unwrap().parse::<u32>().unwrap();
let generation = split.next().unwrap().parse::<u32>().unwrap();
let bits = (generation as u64) << 32 | index as u64;
let entity = Entity::from_bits(bits);
let entity = Entity::from_str(entity).unwrap();
SELECTION_REQUEST.write_message(entity);
}
studio/src/lib.rs
+103 -257
View File
@@ -1,33 +1,31 @@
#![allow(clippy::type_complexity, clippy::too_many_arguments)]
use std::time::Duration;
use bevy::{
app::{AppExit, PluginGroupBuilder, ScheduleRunnerPlugin},
prelude::*,
};
use bevy::{app::PluginGroupBuilder, prelude::*};
use bevy_mod_picking::DefaultPickingPlugins;
use bevy_web_asset::WebAssetPlugin;
use bvh::{BvhAnimCompleteEvent, TestBvh};
use screenshot::{ScreenCapture, ScreenShotPlugin};
use space_editor::prelude::{EditorEvent, EditorPrefabPath};
use scene::ImportSceneEvent;
#[cfg(feature = "wasm")]
use serde_repr::{Deserialize_repr, Serialize_repr};
#[cfg(feature = "wasm")]
use wasm_bindgen::prelude::*;
use wgpu::Extent3d;
mod anim;
mod bvh;
mod camera_controller;
mod change_tracking;
mod debug;
mod design_system;
pub mod headless;
mod hierarchy;
mod input;
mod inspector;
mod interaction;
pub mod math;
mod mixamo;
mod path;
mod persistance;
mod remote;
mod scene;
pub mod screenshot;
mod skybox;
@@ -46,7 +44,8 @@ pub use crate::{
math::InterpolationPlugin,
mixamo::{MixamoImportEvent, MixamoPlugin},
persistance::PersistancePlugin,
scene::{LoadSceneEvent, ScenePlugin},
remote::RemoteAssetPlugin,
scene::{LoadLibraryObjectEvent, LoadSceneEvent, ScenePlugin},
skybox::{SkyboxId, SkyboxPlugin},
ui::UiLayerPlugin,
viewport::ViewportSize,
@@ -59,7 +58,12 @@ mod wasm;
#[derive(Component)]
pub(crate) struct MainCamera;
#[cfg_attr(feature = "wasm", wasm_bindgen)]
#[cfg_attr(
feature = "wasm",
wasm_bindgen,
repr(u8),
derive(Deserialize_repr, Serialize_repr)
)]
#[derive(Resource, Clone, Copy, PartialEq, Eq, Debug)]
pub enum StudioMode {
Editor,
@@ -100,169 +104,65 @@ impl PluginGroup for EditorPlugins {
}
}
#[cfg_attr(feature = "wasm", wasm_bindgen(js_name = startViewer))]
/// Start the application as a read-only viewer for a single library object.
///
/// @param object_id ID of the library object to load.
/// @param skybox_id ID of the Skybox to use
/// @param canvas_selector A CSS selector for the canvas that will host the Bevy
/// application. This should be unique (i.e. `querySelectorAll` should not
/// return more than one element), so using an ID selector is a good idea.
/// @param canvas_alt `alt` text for the HTML canvas element.
/// @param viewport_size The size of the canvas in CSS `px` units.
pub fn start_viewer(
object_id: &str,
skybox_id: &str,
canvas_selector: &str,
canvas_alt: &str,
viewport_size: ViewportSize,
) {
start_internal(
object_id,
skybox_id,
canvas_selector,
canvas_alt,
viewport_size,
StudioMode::Viewer,
)
.run();
#[derive(Debug)]
pub enum InitParams {
/// Initialize the application by importing an asset and processing it to
/// work with Storyteller Studio's feature-set.
FromAsset {
asset: InitAsset,
/// A skybox name that corresponds to a matching set of assets under
/// `studio/assets/skyboxes`, _not_ including file extensions or
/// "diffuse"/"specular" qualifiers (e.g., "gum_trees_4k")
skybox_id: String,
},
/// Initialize the application by loading a previously saved Storyteller
/// Studio scene.
FromScene(String),
}
#[cfg_attr(feature = "wasm", wasm_bindgen(js_name = startEditor))]
pub fn start_editor(
object_id: &str,
skybox_id: &str,
canvas_selector: &str,
canvas_alt: &str,
viewport_size: ViewportSize,
) {
start_internal(
object_id,
skybox_id,
canvas_selector,
canvas_alt,
viewport_size,
StudioMode::Editor,
)
.run();
#[derive(Debug)]
pub enum InitAsset {
/// Initialize the application from an asset in the "bundled" asset library.
/// Takes a filename, including the extension, that can be found under
/// `studio/assets/gltf` (e.g., "base-human-female.gltf").
LibraryObject(String),
/// Initialize the application by loading an arbitrary glTF scene from any
/// source (e.g., the Storyteller.ai backend)
ImportedScene(String),
/// Initialize the application by retargeting a BVH animation (currently
/// only supports the MocapNET skeleton) onto the bundled `Mannequin.gltf`
/// model.
BvhAnim(String),
/// Initialize the application by retargeting a Mixamo animation onto the
/// bundled `Mannequin.gltf` model.
MixamoAnim(String),
/// Initialize the application by loading an arbitary model given an identifier
MediaAsset(String),
}
#[cfg_attr(feature = "wasm", wasm_bindgen(js_name = startBvhViewer))]
pub fn start_bvh_viewer(
bvh_path: &str,
skybox_id: &str,
canvas_selector: &str,
canvas_alt: &str,
viewport_size: ViewportSize,
) {
let mut app = start_internal(
"Mannequin.gltf",
skybox_id,
canvas_selector,
canvas_alt,
viewport_size,
StudioMode::Viewer,
);
let bvh_path = bvh_path.to_string();
app.add_plugins(BvhPlugin);
app.add_systems(
Startup,
move |mut cmd: Commands, r_assets: Res<AssetServer>| {
cmd.insert_resource(bvh::TestBvh(r_assets.load(bvh_path.clone())));
},
);
app.run();
}
#[cfg_attr(feature = "wasm", wasm_bindgen(js_name = startMixamoViewer))]
pub fn start_mixamo_viewer(
mixamo_path: &str,
skybox_id: &str,
canvas_selector: &str,
canvas_alt: &str,
viewport_size: ViewportSize,
) {
let mut app = start_internal(
"Mannequin.gltf",
skybox_id,
canvas_selector,
canvas_alt,
viewport_size,
StudioMode::Viewer,
);
let mixamo_path = mixamo_path.to_string();
app.add_plugins(MixamoPlugin);
app.add_systems(Startup, move |mut ew: EventWriter<MixamoImportEvent>| {
ew.send(MixamoImportEvent(mixamo_path.clone()));
});
app.run();
}
#[cfg_attr(feature = "wasm", wasm_bindgen)]
pub fn start(
object_id: &str,
skybox_id: &str,
canvas_selector: &str,
canvas_alt: &str,
viewport_size: ViewportSize,
studio_mode: StudioMode,
) {
start_internal(
object_id,
skybox_id,
canvas_selector,
canvas_alt,
viewport_size,
studio_mode,
)
.run();
}
// TODO: These `start_` functiions should be refactored to better support the
// needs of the "reaodnly viewer" vs "full studio editor" use cases. See
// notes in `studio-web/src/lib/viewer.element.ts` for more info.
pub fn start_internal(
object_id: &str,
skybox_id: &str,
canvas_selector: &str,
canvas_alt: &str,
viewport_size: ViewportSize,
studio_mode: StudioMode,
) -> App {
pub fn start(init: InitParams, mode: StudioMode, window_config: Option<Window>) -> App {
#[cfg(feature = "wasm")]
console_error_panic_hook::set_once();
let object_id = object_id.to_string();
let skybox_id = skybox_id.to_string();
let mut app = App::new();
app.insert_resource(studio_mode);
app.insert_resource(mode);
app.add_plugins((
#[cfg(feature = "wasm")]
viewport::wasm::ViewportPlugin,
WebAssetPlugin,
RemoteAssetPlugin,
DefaultPlugins.set(WindowPlugin {
primary_window: Some(Window {
resolution: viewport_size.into(),
title: canvas_alt.into(),
canvas: Some(canvas_selector.into()),
..default()
}),
primary_window: Some(window_config.unwrap_or_default()),
..default()
}),
CommonPlugins,
#[cfg(feature = "wasm")]
wasm::ScreenshotPlugin,
));
if matches!(studio_mode, StudioMode::Editor) {
if matches!(mode, StudioMode::Editor) {
app.add_plugins((DefaultPickingPlugins, EditorPlugins));
// Only for registering components for undo
@@ -270,13 +170,16 @@ pub fn start_internal(
app.auto_reflected_undo::<Transform>();
}
app.add_systems(
Startup,
move |mut cmd: Commands, mut ew: EventWriter<LoadSceneEvent>| {
cmd.insert_resource(SkyboxId(skybox_id.clone()));
ew.send(LoadSceneEvent(object_id.clone()));
},
);
match init {
InitParams::FromAsset { asset, skybox_id } => {
init_from_asset(&mut app, asset, skybox_id);
}
InitParams::FromScene(scene) => {
app.add_systems(Startup, move |mut ew: EventWriter<LoadSceneEvent>| {
ew.send(LoadSceneEvent(scene.clone()));
});
}
}
#[cfg(feature = "wasm")]
wasm::initialize();
@@ -284,109 +187,52 @@ pub fn start_internal(
app
}
pub enum HeadlessMode {
Bvh(String, String),
Scene(String, String),
}
pub fn start_headless(
mode: HeadlessMode,
fps: u32,
outout_dir: &str,
frames_to_capture: u32,
image_res: Extent3d,
) -> App {
let mut app = App::new();
app.insert_resource(StudioMode::Headless);
app.insert_resource(HeadlessSettings {
frames_to_capture,
output_dims: image_res,
output_dir: outout_dir.to_string(),
fn init_from_asset(app: &mut App, asset: InitAsset, skybox_id: String) {
app.add_systems(Startup, move |mut cmd: Commands| {
cmd.insert_resource(SkyboxId(skybox_id.clone()));
});
app.add_plugins((
ScheduleRunnerPlugin::run_loop(Duration::from_secs_f64(1.0 / fps as f64)),
DefaultPlugins.set(WindowPlugin {
primary_window: None,
exit_condition: bevy::window::ExitCondition::DontExit,
close_when_requested: true,
}),
ScreenShotPlugin,
CommonPlugins.build().disable::<WorldGridPlugin>(),
));
match mode {
HeadlessMode::Bvh(bvh_dir, skybox) => {
app.add_plugins(BvhPlugin);
match asset {
InitAsset::LibraryObject(id) => {
app.add_systems(
Startup,
move |mut cmd: Commands,
asset_server: Res<AssetServer>,
mut ew: EventWriter<LoadSceneEvent>| {
cmd.insert_resource(SkyboxId(skybox.clone()));
cmd.insert_resource(TestBvh(asset_server.load(bvh_dir.clone())));
ew.send(LoadSceneEvent("Mannequin.gltf".to_string()));
},
);
app.add_systems(PostUpdate, check_for_capture_complete);
}
HeadlessMode::Scene(scene, skybox) => {
app.add_systems(
Startup,
move |mut cmd: Commands, mut ew: EventWriter<EditorEvent>| {
cmd.insert_resource(SkyboxId(skybox.clone()));
ew.send(EditorEvent::Load(EditorPrefabPath::File(scene.clone())))
move |mut ew: EventWriter<LoadLibraryObjectEvent>| {
ew.send(LoadLibraryObjectEvent(id.clone()));
},
);
}
}
anim @ (InitAsset::BvhAnim(_) | InitAsset::MixamoAnim(_)) => {
app.add_systems(
Startup,
move |mut ew: EventWriter<LoadLibraryObjectEvent>| {
ew.send(LoadLibraryObjectEvent("Mannequin.gltf".into()));
},
);
app
}
pub fn is_headless(studio_state: Res<StudioMode>) -> bool {
*studio_state == StudioMode::Headless
}
pub fn is_not_headless(studio_state: Res<StudioMode>) -> bool {
*studio_state != StudioMode::Headless
}
#[derive(Resource)]
pub struct HeadlessSettings {
pub output_dims: Extent3d,
pub output_dir: String,
pub frames_to_capture: u32,
}
impl Default for HeadlessSettings {
fn default() -> Self {
HeadlessSettings {
output_dims: Extent3d {
width: 1280,
height: 720,
depth_or_array_layers: 1,
},
output_dir: "./screen_shots/".to_string(),
frames_to_capture: 1,
match anim {
InitAsset::BvhAnim(anim) => {
app.add_plugins(BvhPlugin);
app.add_systems(
Startup,
move |mut cmd: Commands, r_assets: Res<AssetServer>| {
cmd.insert_resource(bvh::TestBvh(r_assets.load(anim.clone())));
},
);
}
InitAsset::MixamoAnim(anim) => {
app.add_plugins(MixamoPlugin);
app.add_systems(Startup, move |mut ew: EventWriter<MixamoImportEvent>| {
ew.send(MixamoImportEvent(anim.clone()));
});
}
_ => unreachable!(),
}
}
}
}
fn check_for_capture_complete(
mut commands: Commands,
mut close_app: EventWriter<AppExit>,
main_camera: Query<(Entity, &ScreenCapture), With<MainCamera>>,
bvh_event_reader: EventReader<BvhAnimCompleteEvent>,
) {
let Ok((entity, screen_capture)) = main_camera.get_single() else {
return;
};
if screen_capture.is_complete() || !bvh_event_reader.is_empty() {
commands.entity(entity).remove::<ScreenCapture>();
close_app.send(AppExit);
InitAsset::ImportedScene(scene) => {
app.add_systems(Startup, move |mut ew: EventWriter<ImportSceneEvent>| {
ew.send(ImportSceneEvent(scene.clone()));
});
}
InitAsset::MediaAsset(_) => todo!(),
}
}
studio/src/main-headless.rs
+39 -1
View File
@@ -1,6 +1,6 @@
use bevy::render::render_resource::Extent3d;
use clap::{Args, Parser, Subcommand};
use studio::{start_headless, HeadlessMode};
use studio::headless::{start_headless, HeadlessMode};
#[derive(Parser)]
#[command(name = "StorytellerStudioHeadless")]
@@ -43,6 +43,19 @@ enum Subcommands {
#[arg(long, value_parser = clap::value_parser!(u32).range(1..))]
frames: Option<u32>,
},
Mixamo {
mixamo_dir: String,
#[arg(long, short = 'o', short)]
out: Option<String>,
#[arg(long)]
skybox: Option<String>,
#[arg(long = "res", value_parser = extent3d_parser)]
output_resolution: Option<Extent3d>,
#[arg(long, value_parser = clap::value_parser!(u32).range(1..))]
fps: Option<u32>,
#[arg(long, value_parser = clap::value_parser!(u32).range(1..))]
frames: Option<u32>,
},
}
#[derive(Args, Clone)]
@@ -102,6 +115,31 @@ fn main() {
output_res,
)
}
Subcommands::Mixamo {
mixamo_dir,
out,
skybox,
output_resolution,
fps,
frames,
} => {
let output_dir = out.unwrap_or("./screen_shots/".to_string());
let output_res = output_resolution.unwrap_or(Extent3d {
width: 1280,
height: 720,
depth_or_array_layers: 1,
});
let fps = fps.unwrap_or(24);
let frames = frames.unwrap_or(u32::MAX);
let skybox = skybox.unwrap_or("gum_trees_4k".to_string());
start_headless(
HeadlessMode::Mixamo(mixamo_dir, skybox),
fps,
&output_dir,
frames,
output_res,
)
}
};
app.run();
studio/src/main.rs
+16 -10
View File
@@ -1,6 +1,9 @@
use bevy::{app::Update, window::close_on_esc};
use bevy::{
app::Update,
window::{close_on_esc, Window},
};
use clap::Parser;
use studio::{screenshot::ScreenShotPlugin, StudioMode, ViewportSize};
use studio::{screenshot::ScreenShotPlugin, InitAsset, InitParams, StudioMode};
#[derive(Parser)]
#[command(name = "StorytellerStudio")]
@@ -16,17 +19,20 @@ struct StudioArgs {
fn main() {
let args = StudioArgs::parse();
let model_path = args.model.unwrap_or("Mannequin.gltf".to_string());
let skybox_path = args.skybox.unwrap_or("gum_trees_4k".to_string());
let object_id = args.model.unwrap_or("Mannequin.gltf".to_string());
let skybox_id = args.skybox.unwrap_or("gum_trees_4k".to_string());
// Edit the arguments here to change the default startup scene or skybox
let mut app = studio::start_internal(
model_path.as_str(),
skybox_path.as_str(),
"", // ignored for non-WASM
"Storyteller Studio",
ViewportSize::default(),
let mut app = studio::start(
InitParams::FromAsset {
asset: InitAsset::LibraryObject(object_id),
skybox_id,
},
StudioMode::Editor,
Some(Window {
title: "Storyteller Studio".into(),
..Default::default()
}),
);
// Add any additional systems, resources, etc. for local debugging
studio/src/path.rs
+40
View File
@@ -0,0 +1,40 @@
use std::{ffi::OsStr, path::Path};
pub trait PathExtras {
/// See [std::path::Path::file_prefix]
#[must_use]
fn ext_file_prefix(&self) -> Option<&OsStr>;
}
impl PathExtras for Path {
fn ext_file_prefix(&self) -> Option<&OsStr> {
self.file_name()
.map(split_file_at_dot)
.map(|(before, _after)| before)
}
}
/// Note: Copied from private fn `std::path::split_file_at_dot`
fn split_file_at_dot(file: &OsStr) -> (&OsStr, Option<&OsStr>) {
let slice = file.as_encoded_bytes();
if slice == b".." {
return (file, None);
}
// The unsafety here stems from converting between &OsStr and &[u8]
// and back. This is safe to do because (1) we only look at ASCII
// contents of the encoding and (2) new &OsStr values are produced
// only from ASCII-bounded slices of existing &OsStr values.
let i = match slice[1..].iter().position(|b| *b == b'.') {
Some(i) => i + 1,
None => return (file, None),
};
let before = &slice[..i];
let after = &slice[i + 1..];
unsafe {
(
OsStr::from_encoded_bytes_unchecked(before),
Some(OsStr::from_encoded_bytes_unchecked(after)),
)
}
}
studio/src/persistance/mod.rs
+68
View File
@@ -10,6 +10,13 @@ impl Plugin for PersistancePlugin {
EditorTabName::Other("Persistance".to_string()),
DebugPersistanceTab::default(),
);
#[cfg(feature = "wasm")]
app.add_systems(Update, wasm::forward_save_scene_events)
.add_systems(
OnEnter(SaveState::Idle),
wasm::on_scene_saved.map(bevy::utils::warn),
);
}
}
@@ -43,3 +50,64 @@ impl EditorTab for DebugPersistanceTab {
"Persistance".into()
}
}
#[cfg(feature = "wasm")]
mod wasm {
use bevy::{prelude::*, window::PrimaryWindow};
use space_editor::prelude::{EditorEvent, EditorPrefabPath, PrefabMemoryCache};
use wasm_bindgen::prelude::*;
use crate::wasm::{self, StaticBuffer, WasmMessageBuffer};
static SAVE_SCENE: WasmMessageBuffer<()> = WasmMessageBuffer::new();
#[wasm_bindgen(js_name = saveScene)]
pub fn save_scene() {
SAVE_SCENE.write_message(());
}
pub(super) fn forward_save_scene_events(mut ew_editor_events: EventWriter<EditorEvent>) {
if SAVE_SCENE.take_message().is_some() {
ew_editor_events.send(EditorEvent::Save(EditorPrefabPath::MemoryCahce));
}
}
#[wasm_bindgen(typescript_custom_section)]
const SCENE_SAVED_EVENT: &str = r#"
export interface SceneSavedEvent extends CustomEvent {
type: "scene-saved";
detail: string;
}
declare global {
export interface GlobalEventHandlersEventMap {
"scene-saved": SceneSavedEvent;
}
}
"#;
pub(super) fn on_scene_saved(
r_scene_cache: Res<PrefabMemoryCache>,
r_type_registry: Res<AppTypeRegistry>,
ra_scenes: Res<Assets<DynamicScene>>,
q_window: Query<&Window, With<PrimaryWindow>>,
) -> Result<(), String> {
let handle = r_scene_cache
.scene
.as_ref()
.ok_or_else(|| "Scene cache is empty!".to_string())?;
let serialized = match ra_scenes.get(handle) {
Some(scene) => scene
.serialize_ron(&r_type_registry)
.map_err(|err| format!("{err}")),
None => Err("No scene found for cached handle!".into()),
}?;
let Ok(window) = q_window.get_single() else {
return Ok(());
};
wasm::dispatch_to_js(window, "scene-saved", &serialized)
}
}
studio/src/remote/mod.rs
+22
View File
@@ -0,0 +1,22 @@
use bevy::{
asset::io::{AssetSource, AssetSourceId},
prelude::*,
};
use self::reader::RemoteAssetReader;
mod reader;
mod response;
pub struct RemoteAssetPlugin;
impl Plugin for RemoteAssetPlugin {
fn build(&self, app: &mut App) {
app.register_asset_source(
AssetSourceId::Name("remote".into()),
AssetSource::build()
.with_reader(|| Box::new(RemoteAssetReader))
.with_writer(|| None),
);
}
}
studio/src/remote/reader.rs
+118
View File
@@ -0,0 +1,118 @@
use std::{io::ErrorKind, path::Path};
use bevy::{
asset::io::{AssetReader, AssetReaderError, PathStream, Reader, VecReader},
prelude::*,
utils::BoxedFuture,
};
use reqwest::StatusCode;
use super::response::GetMediaFileSuccessResponse;
use crate::path::PathExtras;
pub struct RemoteAssetReader;
impl RemoteAssetReader {
const API_BASE_URL: &'static str = "https://api.fakeyou.com/v1/media_files/file";
const BUCKET_BASE_URL: &'static str = "https://storage.googleapis.com/vocodes-public";
}
impl AssetReader for RemoteAssetReader {
fn read<'a>(
&'a self,
path: &'a Path,
) -> BoxedFuture<'a, Result<Box<Reader<'a>>, AssetReaderError>> {
Box::pin(async move {
let token = path
.ext_file_prefix()
.ok_or_else(|| {
AssetReaderError::Io(std::io::Error::new(
ErrorKind::Other,
format!("Failed to parse token from '{path:?}'"),
))
})?
.to_str()
.unwrap();
info!("Loading asset from media token: '{token}'");
let response = reqwest::get(format!("{}/{token}", &Self::API_BASE_URL))
.await
.map_err(IntoError::<AssetReaderError>::into)?;
let response = match response.status() {
StatusCode::OK => response
.json::<GetMediaFileSuccessResponse>()
.await
.map_err(IntoError::<AssetReaderError>::into),
other => Err(AssetReaderError::Io(std::io::Error::new(
ErrorKind::Other,
format!("Status Code from endpoint: {other}"),
))),
}?;
let url = format!(
"{}{}",
&Self::BUCKET_BASE_URL,
&response.media_file.public_bucket_path
);
let response = reqwest::get(url)
.await
.map_err(IntoError::<AssetReaderError>::into)?;
let bytes = match response.status() {
// TODO: This is not thread-safe, but some variation of this
// would be better than the current solution once Bevy
// WASM becomes multi-threaded:
// StatusCode::OK => Ok(response.bytes_stream().map(|result| {
// result.map_err(|err| std::io::Error::new(ErrorKind::Other, format!("{err}")))
// })),
StatusCode::OK => Ok(response
.bytes()
.await
.map_err(IntoError::<AssetReaderError>::into)?
.into_iter()
.collect::<Vec<_>>()),
other => Err(AssetReaderError::Io(std::io::Error::new(
ErrorKind::Other,
format!("Status Code from bucket URL: {other}"),
))),
}?;
let reader: Box<Reader> = Box::new(VecReader::new(bytes));
Ok(reader)
})
}
fn read_meta<'a>(
&'a self,
path: &'a Path,
) -> BoxedFuture<'a, Result<Box<Reader<'a>>, AssetReaderError>> {
// TODO: Do we want to try to read .meta files?
Box::pin(async { Err(AssetReaderError::NotFound(path.into())) })
}
fn read_directory<'a>(
&'a self,
path: &'a Path,
) -> BoxedFuture<'a, Result<Box<PathStream>, AssetReaderError>> {
// We won't have to deal with directories
Box::pin(async { Err(AssetReaderError::NotFound(path.into())) })
}
fn is_directory<'a>(&'a self, _: &'a Path) -> BoxedFuture<'a, Result<bool, AssetReaderError>> {
// We won't have to deal with directories
Box::pin(async { Ok(false) })
}
}
trait IntoError<Output> {
fn into(self) -> Output;
}
impl IntoError<AssetReaderError> for reqwest::Error {
fn into(self) -> AssetReaderError {
AssetReaderError::Io(std::io::Error::new(ErrorKind::Other, format!("{self}")))
}
}
studio/src/remote/response.rs
+107
View File
@@ -0,0 +1,107 @@
//! Response types copy/pasted straight from the backend source code.
use serde::Deserialize;
#[derive(Deserialize, Debug)]
pub struct GetMediaFileSuccessResponse {
pub success: bool,
pub media_file: MediaFileInfo,
}
#[derive(Deserialize, Debug)]
pub struct MediaFileInfo {
pub token: String,
/// Type of media will dictate which fields are populated and what
/// the frontend should display (eg. video player vs audio player).
pub media_type: String,
/// If the file was generated as part of a batch, this is the token for the batch.
pub maybe_batch_token: Option<String>,
/// URL to the media file
pub public_bucket_path: String,
//// Provenance Data
//pub origin_product: MediaFileOriginProductCategory,
//pub origin_category: MediaFileOriginCategory,
//pub maybe_origin_model_type: Option<MediaFileOriginModelType>,
//pub maybe_origin_model_token: Option<String>,
//pub maybe_origin_filename: Option<String>,
/// Possible model info (if generated by inference)
pub maybe_model_weight_info: Option<GetMediaFileModelInfo>,
/// User info
pub maybe_creator_user: Option<UserDetailsLight>,
pub creator_set_visibility: String,
/// Text transcripts for TTS, etc.
pub maybe_text_transcript: Option<String>,
/// We can simulate media files for "tts_results" records.
/// If this is set as true, this informs the frontend and API callers not to treat
/// the token as a media file and improperly assume it can be used with the rest of
/// the media file APIs.
pub is_emulated_media_file: bool,
/// Statistics about the media file
pub stats: SimpleEntityStats,
pub created_at: String,
pub updated_at: String,
//pub maybe_moderator_fields: Option<ModeratorFields>,
}
#[derive(Deserialize, Debug)]
pub struct GetMediaFileModelInfo {
pub weight_token: String,
pub weight_type: String,
pub weight_category: String,
pub title: String,
/// Cover images are small descriptive images that can be set for any model.
/// If a cover image is set, this is the path to the asset.
pub maybe_cover_image_public_bucket_path: Option<String>,
/// NB: Technically this shouldn't be a nullable field, but since there are so many
/// joins we'll relax that this must exist (for now).
pub maybe_weight_creator: Option<UserDetailsLight>,
}
#[derive(Deserialize, Debug)]
pub struct UserDetailsLight {
/// The token for the user
pub user_token: String,
/// The unique username someone logs in with
/// As of 2023-08-23, this is always lowercase
pub username: String,
/// As of 2023-08-23, this is the username with capitalization
/// (In the future, a display name can be customized by the user.)
pub display_name: String,
/// Email hash for Gravatar
/// Always set for now since login is email/username+password.
/// In the future this will need to become an optional *OR* be filled with a bogus hash.
pub gravatar_hash: String,
/// For users without a gravatar, we show one of our own avatars.
pub default_avatar: DefaultAvatarInfo,
// In the future, we'll also support user-uploaded avatars that we store on our servers.
}
#[derive(Deserialize, Debug)]
pub struct DefaultAvatarInfo {
pub image_index: u8,
pub color_index: u8,
}
#[derive(Deserialize, Debug)]
pub struct SimpleEntityStats {
/// Number of positive ratings (or "likes") for this item
pub positive_rating_count: u32,
/// Number of bookmarks for this item
pub bookmark_count: u32,
}
studio/src/scene.rs
+130 -38
View File
@@ -12,19 +12,24 @@ use bevy::{
render::{camera::RenderTarget, primitives::Aabb},
scene::InstanceId,
};
use space_editor::{prelude::PrefabMarker, space_prefab::component::SceneAutoChild};
#[cfg(feature = "wasm")]
use serde_repr::{Deserialize_repr, Serialize_repr};
use space_editor::{
prelude::{EditorEvent, EditorPrefabPath, PrefabMarker},
space_prefab::component::SceneAutoChild,
};
#[cfg(feature = "wasm")]
use wasm_bindgen::prelude::wasm_bindgen;
use wgpu::{TextureDescriptor, TextureUsages};
use crate::{
camera_controller::{CameraController, CameraControllerBundle},
change_tracking::{ChangeTracker, ChangeTrackingPlugin},
debug::DebugPoint,
is_headless, is_not_headless,
headless::{is_headless, is_not_headless, HeadlessSettings},
math::{Combinable, Extrema},
screenshot::ScreenCapture,
HeadlessSettings, MainCamera,
MainCamera,
};
pub struct ScenePlugin;
@@ -34,10 +39,14 @@ pub struct SceneSpawnerSystem;
impl Plugin for ScenePlugin {
fn build(&self, app: &mut App) {
app.add_plugins(ChangeTrackingPlugin);
app.insert_resource(ClearColor(Color::hsl(0.0, 0.0, 0.25)))
.insert_resource(Msaa::Off)
.add_state::<SceneState>()
.add_event::<LoadLibraryObjectEvent>()
.add_event::<LoadSceneEvent>()
.add_event::<ImportSceneEvent>()
.add_systems(
Startup,
(
@@ -45,10 +54,20 @@ impl Plugin for ScenePlugin {
initialiaze_headless.run_if(is_headless),
),
)
.add_systems(Update, init_scene_loading.in_set(SceneSpawnerSystem))
.add_systems(
Update,
(init_object_loading, init_scene_import, init_scene_loading)
.in_set(SceneSpawnerSystem),
)
.add_systems(
OnEnter(SceneState::Active),
(tag_scene_elements, frame_scene_in_view).in_set(SceneSpawnerSystem),
(
tag_scene_elements,
frame_scene_in_view,
start_capture.run_if(is_headless),
)
.chain()
.in_set(SceneSpawnerSystem),
);
app.add_systems(Update, tag_scene_elements);
@@ -73,7 +92,7 @@ impl Plugin for ScenePlugin {
app.add_systems(
Update,
(
wasm::dispatch_scene_request,
wasm::dispatch_object_load_request,
wasm::notify_state_changes
.map(bevy::utils::error)
.run_if(resource_changed::<State<SceneState>>()),
@@ -83,16 +102,20 @@ impl Plugin for ScenePlugin {
}
fn minimal_scene_setup(mut cmd: Commands) {
cmd.spawn(DirectionalLightBundle {
directional_light: DirectionalLight {
shadows_enabled: false,
illuminance: 20000.0,
cmd.spawn((
Name::new("Directional Light"),
DirectionalLightBundle {
directional_light: DirectionalLight {
shadows_enabled: false,
illuminance: 20000.0,
..default()
},
transform: Transform::default().looking_at(Vec3::NEG_ONE, Vec3::Y),
..default()
},
transform: Transform::default().looking_at(Vec3::NEG_ONE, Vec3::Y),
..default()
})
.insert(PrefabMarker);
PrefabMarker,
SceneElement::Light,
));
}
#[cfg_attr(feature = "wasm", wasm_bindgen)]
@@ -167,17 +190,58 @@ fn auto_add_handle<T: Asset>(
}
}
/// Request the application to load a `gltf`/`glb` scene from the "bundled"
/// asset library in `assets/gltf/`. Expects a filename including the extension.
///
/// ```ignore
/// fn main() {
/// App::new()
/// .add_plugins((
/// DefaultPlugins,
/// CommonPlugins,
/// ))
/// .add_systems(Startup, load_mannequin)
/// }
///
/// fn load_mannequin(mut ew_load_object: EventWriter<LoadLibraryObjectEvent>) {
/// ew_load_object.send(LoadLibraryObjectEvent("Mannequin.gltf".into()));
/// }
/// ```
#[derive(Event, Deref)]
pub struct LoadLibraryObjectEvent(pub String);
/// Request the application to load a previously saved Storyteller Studio scene.
///
/// Expects a full, resolvable path to the file (e.g. `scenes/my-scene.scn.ron`
/// for a local file saved under `studio/assets/scenes`, or
/// `remote://<media_token>.scn.ron` for a scene saved to the Storyteller.ai
/// backend).
#[derive(Event, Deref)]
pub struct LoadSceneEvent(pub String);
// TODOL This could be converted into an enum that indicates what "kind" of
// element this is (e.g. StaticMesh, SkinnedMesh, Bone, Light, Camera).
// We don't have an immediate need for that just yet, but likely will
// soon.
/// Request the application to import and process a glTF scene from an arbitrary
/// asset source (e.g., an http path or a Storyteller.ai media token).
#[derive(Event, Deref)]
pub struct ImportSceneEvent(pub String);
/// A tag that identifies its entity as a user-facing scene element like a prop,
/// a light, or a camera.
#[derive(Component)]
pub struct SceneElement;
#[cfg_attr(
feature = "wasm",
repr(u8),
wasm_bindgen,
derive(Serialize_repr, Deserialize_repr)
)]
#[derive(Component, Clone, Copy, Debug, PartialEq, Eq)]
pub enum SceneElement {
Generic,
Mesh,
Skeleton,
Bone,
Light,
#[allow(dead_code)] // TODO: Will be used for camera animation
Camera,
}
#[derive(Component)]
pub struct SceneData {
@@ -195,6 +259,7 @@ fn initialize(mut cmd: Commands) {
cmd.spawn((
Name::new("Main Camera"),
MainCamera,
ChangeTracker,
Camera3dBundle {
transform: camera_xform,
tonemapping: Tonemapping::BlenderFilmic,
@@ -261,17 +326,15 @@ fn initialiaze_headless(
..default()
},
CameraControllerBundle::new(focus, camera_xform.translation),
ScreenCapture::number_of_frames(
headless_settings.frames_to_capture,
&headless_settings.output_dir,
),
ScreenCapture::from_out(&headless_settings.output_dir),
));
}
fn init_scene_loading(mut cmd: Commands, mut er: EventReader<LoadSceneEvent>) {
fn init_object_loading(mut cmd: Commands, mut er: EventReader<LoadLibraryObjectEvent>) {
if let Some(id) = er.read().last() {
info!("Loading scene: {}", id.0);
info!("Loading library asset: {}", id.0);
cmd.spawn((
Name::new(id.0.clone()),
SpatialBundle::default(),
space_editor::prelude::GltfPrefab {
path: format!("gltf/{}", id.0),
@@ -283,11 +346,33 @@ fn init_scene_loading(mut cmd: Commands, mut er: EventReader<LoadSceneEvent>) {
}
}
#[derive(Component)]
pub struct NoMeshElement;
fn init_scene_import(mut cmd: Commands, mut er: EventReader<ImportSceneEvent>) {
if let Some(path) = er.read().last() {
info!("Importing asset: {}", path.0);
cmd.spawn((
Name::new("Scene"), // TODO: Maybe use the "basename" part of the asset path?
SpatialBundle::default(),
space_editor::prelude::GltfPrefab {
path: path.0.clone(),
scene: "Scene0".to_string(),
},
PrefabMarker,
SceneRoot,
));
}
}
#[derive(Component)]
pub struct InitedKointSkinnedMesh;
fn init_scene_loading(
mut er_load_scene: EventReader<LoadSceneEvent>,
mut ew_editor: EventWriter<EditorEvent>,
) {
if let Some(scene_path) = er_load_scene.read().last() {
info!("Loading scene: {}", scene_path.0);
ew_editor.send(EditorEvent::Load(EditorPrefabPath::File(
scene_path.0.clone(),
)));
}
}
fn tag_scene_elements(
mut cmd: Commands,
@@ -305,7 +390,6 @@ fn tag_scene_elements(
With<Name>,
Without<Handle<Mesh>>,
Without<SceneElement>,
Without<NoMeshElement>,
),
>,
mesh_entities: Query<Entity, With<Handle<Mesh>>>,
@@ -319,9 +403,9 @@ fn tag_scene_elements(
}
}
if has_mesh {
cmd.entity(e).insert(SceneElement);
cmd.entity(e).insert(SceneElement::Mesh);
} else {
cmd.entity(e).insert(NoMeshElement);
cmd.entity(e).insert(SceneElement::Generic);
}
}
}
@@ -429,12 +513,18 @@ fn frame_scene_in_view(
cam_xform.translation = cam.focus + cam_xform.back() * radius;
}
fn start_capture(mut captures: Query<&mut ScreenCapture>) {
for mut capture in captures.iter_mut() {
capture.start_capturing();
}
}
#[cfg(feature = "wasm")]
mod wasm {
use bevy::{prelude::*, window::PrimaryWindow};
use wasm_bindgen::prelude::*;
use super::{LoadSceneEvent, SceneState};
use super::{LoadLibraryObjectEvent, SceneState};
use crate::wasm::{self, StaticBuffer, WasmMessageBuffer};
static OBJECT_LOAD_REQUEST: WasmMessageBuffer<String> = WasmMessageBuffer::new();
@@ -446,9 +536,11 @@ mod wasm {
// FIXME: Is it an object or a scene? Inconsistent terminology is
// unnecessarily confusing
pub(super) fn dispatch_scene_request(mut ew_load_scene: EventWriter<LoadSceneEvent>) {
pub(super) fn dispatch_object_load_request(
mut ew_load_scene: EventWriter<LoadLibraryObjectEvent>,
) {
if let Some(id) = OBJECT_LOAD_REQUEST.take_message() {
ew_load_scene.send(LoadSceneEvent(id));
ew_load_scene.send(LoadLibraryObjectEvent(id));
}
}
@@ -460,7 +552,7 @@ export interface SceneStateEvent extends CustomEvent {
}
declare global {
export interface HTMLElementEventMap {
export interface GlobalEventHandlersEventMap {
"scene-state": SceneStateEvent;
}
}
studio/src/screenshot.rs
+46 -37
View File
@@ -30,8 +30,7 @@ impl Plugin for ScreenShotPlugin {
fn build(&self, app: &mut App) {
app.add_systems(
Update,
(test_system, update_screen_capture_on_extract_step)
.run_if(in_state(SceneState::Active)),
(update_screen_capture_on_extract_step).run_if(in_state(SceneState::Active)),
);
let Ok(render_world) = app.get_sub_app_mut(bevy::render::RenderApp) else {
@@ -65,60 +64,71 @@ impl Plugin for ScreenShotPlugin {
}
}
fn test_system(
mut commands: Commands,
camera: Query<Entity, With<Camera>>,
input: Res<Input<KeyCode>>,
) {
if input.just_pressed(KeyCode::F10) {
for entity in camera.iter() {
commands
.entity(entity)
.insert(ScreenCapture::number_of_frames(16, "./scrnshot/"));
}
}
}
//==============================================================================
// Sceen Shot Component
//==============================================================================
#[derive(Component, Clone)]
pub struct ScreenCapture {
number_of_frames: u32,
frames: u32,
max_frames: u32,
to_directory: String,
capture_depth: bool,
currently_captureing: bool,
}
impl ScreenCapture {
pub fn number_of_frames(frames: u32, output_dir: &str) -> Self {
pub fn from_out(output_dir: &str) -> Self {
ScreenCapture {
number_of_frames: frames,
frames: 0,
to_directory: output_dir.to_string(),
capture_depth: false,
currently_captureing: false,
max_frames: u32::MAX,
}
}
pub fn with_max_frames(mut self, max_frames: u32) -> Self {
self.max_frames = max_frames;
self
}
pub fn capture_depth(mut self) -> Self {
self.capture_depth = true;
self
}
pub fn is_complete(&self) -> bool {
println!("Number of Frames Left: {}", self.number_of_frames);
self.number_of_frames == 1
pub fn is_capturing(&self) -> bool {
self.currently_captureing
}
pub fn stop_capturing(&mut self) {
self.currently_captureing = false;
}
pub fn start_capturing(&mut self) {
self.currently_captureing = true;
}
pub fn should_capture(&self) -> bool {
self.frames < self.max_frames && self.is_capturing()
}
pub fn frames_captured(&self) -> u32 {
self.frames
}
pub fn max_frames(&self) -> u32 {
self.max_frames
}
}
fn update_screen_capture_on_extract_step(
mut commands: Commands,
mut screen_captures: Query<(Entity, &mut ScreenCapture)>,
) {
for (entity, mut screen_capture) in screen_captures.iter_mut() {
screen_capture.number_of_frames -= 1;
if screen_capture.number_of_frames == 0 {
commands.entity(entity).remove::<ScreenCapture>();
}
fn update_screen_capture_on_extract_step(mut screen_captures: Query<&mut ScreenCapture>) {
for mut screen_capture in screen_captures.iter_mut() {
screen_capture.frames += 1;
// if screen_capture.number_of_frames == 0 {
// commands.entity(entity).remove::<ScreenCapture>();
// }
}
}
@@ -128,7 +138,9 @@ fn extract_screen_capture_components(
screen_capure_comps: Extract<Query<(Entity, &ScreenCapture), With<Camera>>>,
) {
for (entity, screen_capture) in screen_capure_comps.iter() {
commands.get_or_spawn(entity).insert(screen_capture.clone());
if screen_capture.should_capture() {
commands.get_or_spawn(entity).insert(screen_capture.clone());
}
}
}
@@ -152,7 +164,7 @@ fn prepare_screen_capture_buffers(
device: Res<RenderDevice>,
) {
for (entity, view_target, screen_capture) in views.iter() {
if screen_capture.number_of_frames > 0 {
if screen_capture.frames > 0 {
let texture = view_target.main_texture();
let size = texture.size();
let format = texture.format();
@@ -174,8 +186,6 @@ fn prepare_screen_capture_buffers(
main_bytes_per_row: bytes_per_row,
main_padded_bytes_per_row: padded_bytes_per_row,
});
println!("Buffers Created!");
}
}
}
@@ -219,7 +229,6 @@ impl ViewNode for ScreenCaptureNode {
},
view_query.1.main_texture().size(),
);
println!("Saved to buffer!");
Ok(())
}
}
@@ -255,7 +264,7 @@ fn save_buffer_to_file(
let path = format!(
"{}img_{:0.5}.png",
screen_capture.to_directory, screen_capture.number_of_frames
screen_capture.to_directory, screen_capture.frames
);
save_buffer::<Rgba<u8>>(&main_image_values, &buffers.main_buffer_extent, &path);
studio/src/skybox.rs
+15 -6
View File
@@ -39,12 +39,21 @@ struct SkyboxAssets {
}
fn load_skybox(mut cmd: Commands, assets: Res<AssetServer>, id: Res<SkyboxId>) {
let paths = SkyboxPaths::new(&**id);
cmd.insert_resource(SkyboxAssets {
cubemap: assets.load(paths.cubemap),
env_diffuse: assets.load(paths.env_diffuse),
env_specular: assets.load(paths.env_specular),
});
let id = &**id;
if let Ok(color) = Color::hex(id) {
cmd.insert_resource(ClearColor(color));
cmd.insert_resource(AmbientLight {
color,
brightness: 1.0,
})
} else {
let paths = SkyboxPaths::new(id);
cmd.insert_resource(SkyboxAssets {
cubemap: assets.load(paths.cubemap),
env_diffuse: assets.load(paths.env_diffuse),
env_specular: assets.load(paths.env_specular),
});
}
}
fn skybox_assets_loaded(
studio/src/viewport.rs
+2 -1
View File
@@ -1,10 +1,11 @@
use core::fmt;
use bevy::window::WindowResolution;
use serde::{Deserialize, Serialize};
use wasm_bindgen::prelude::*;
#[wasm_bindgen]
#[derive(Clone, Copy)]
#[derive(Clone, Copy, Serialize, Deserialize)]
pub struct ViewportSize {
pub width: f32,
pub height: f32,
studio/src/wasm/conv.rs
+41
View File
@@ -85,6 +85,12 @@ impl AsJson for GlobalTransform {
}
}
impl AsJson for serde_json::Map<String, JsonValue> {
fn as_json(&self) -> JsonValue {
JsonValue::Object(self.clone())
}
}
impl<T> AsJsValue for T
where
T: AsJson,
@@ -140,3 +146,38 @@ impl AsJsValue for u32 {
Ok(JsValue::from_f64(*self as f64))
}
}
/// A crate-local version of `std::str::FromStr` that can be implemented for
/// foreign types
pub trait FromStr: Sized {
type Err;
fn from_str(s: &str) -> Result<Self, Self::Err>;
}
impl FromStr for Entity {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let fmt_err =
|| format!("Expected a string with format '<integer>v<integer>', found '{s}'");
let mut split = s.split('v');
let index = split
.next()
.unwrap()
.parse::<u32>()
.map_err(|_| fmt_err())?;
let generation = split
.next()
.unwrap()
.parse::<u32>()
.map_err(|_| fmt_err())?;
let bits = (generation as u64) << 32 | index as u64;
Ok(Entity::from_bits(bits))
}
}
studio/src/wasm/init.rs
+155
View File
@@ -0,0 +1,155 @@
use bevy::prelude::*;
use serde::{Deserialize, Serialize};
use wasm_bindgen::prelude::*;
use crate::{InitAsset, InitParams, StudioMode, ViewportSize};
#[wasm_bindgen(typescript_custom_section)]
const WASM_ENTRY_POINTS: &str = r#"
export interface InitOptions {
mode: StudioMode;
/** Required for all entry-points except `initFromStorytellerScene()` */
skyboxId?: string;
canvasSelector: string;
canvasAlt: string;
viewportSize: ViewportSize;
}
export function initFromLibrary(objectId: string, options: InitOptions): void;
export function initFromBvh(animPath: string, options: InitOptions): void;
export function initFromMixamo(animPath: string, options: InitOptions): void;
export function initFromSceneImport(scenePath: string, options: InitOptions): void;
export function initFromStorytellerScene(scenePath: string, options: InitOptions): void;
"#;
#[derive(Serialize, Deserialize)]
struct WasmInitOptions {
#[serde(rename = "skyboxId")]
skybox_id: Option<String>,
mode: StudioMode,
#[serde(rename = "canvasSelector")]
canvas_selector: String,
#[serde(rename = "canvasAlt")]
canvas_alt: String,
#[serde(rename = "viewportSize")]
viewport_size: ViewportSize,
}
#[wasm_bindgen(skip_typescript, js_name = initFromLibrary)]
pub fn init_from_library(object_id: String, options: JsValue) -> Result<(), JsValue> {
let options = serde_wasm_bindgen::from_value::<WasmInitOptions>(options)?;
let skybox_id = options
.skybox_id
.expect("`skyboxId` is required for `initFromLibrary`");
crate::start(
InitParams::FromAsset {
asset: InitAsset::LibraryObject(object_id),
skybox_id,
},
options.mode,
Some(Window {
resolution: options.viewport_size.into(),
title: options.canvas_alt,
canvas: Some(options.canvas_selector),
..default()
}),
)
.run();
Ok(())
}
#[wasm_bindgen(skip_typescript, js_name = initFromBvh)]
pub fn init_from_bvh(anim_path: String, options: JsValue) -> Result<(), JsValue> {
let options = serde_wasm_bindgen::from_value::<WasmInitOptions>(options)?;
let skybox_id = options
.skybox_id
.expect("`skyboxId` is required for `initFromBvh`");
crate::start(
InitParams::FromAsset {
asset: InitAsset::BvhAnim(anim_path),
skybox_id,
},
options.mode,
Some(Window {
resolution: options.viewport_size.into(),
title: options.canvas_alt,
canvas: Some(options.canvas_selector),
..default()
}),
)
.run();
Ok(())
}
#[wasm_bindgen(skip_typescript, js_name = initFromMixamo)]
pub fn init_from_mixamo(anim_path: String, options: JsValue) -> Result<(), JsValue> {
let options = serde_wasm_bindgen::from_value::<WasmInitOptions>(options)?;
let skybox_id = options
.skybox_id
.expect("`skyboxId` is required for `initFromMixamo`");
crate::start(
InitParams::FromAsset {
asset: InitAsset::MixamoAnim(anim_path),
skybox_id,
},
options.mode,
Some(Window {
resolution: options.viewport_size.into(),
title: options.canvas_alt,
canvas: Some(options.canvas_selector),
..default()
}),
)
.run();
Ok(())
}
#[wasm_bindgen(skip_typescript, js_name = initFromSceneImport)]
pub fn init_from_scene_import(scene_path: String, options: JsValue) -> Result<(), JsValue> {
let options = serde_wasm_bindgen::from_value::<WasmInitOptions>(options)?;
let skybox_id = options
.skybox_id
.expect("`skyboxId` is required for `initFromSceneImport`");
crate::start(
InitParams::FromAsset {
asset: InitAsset::ImportedScene(scene_path),
skybox_id,
},
options.mode,
Some(Window {
resolution: options.viewport_size.into(),
title: options.canvas_alt,
canvas: Some(options.canvas_selector),
..default()
}),
)
.run();
Ok(())
}
#[wasm_bindgen(skip_typescript, js_name = initFromStorytellerScene)]
pub fn init_from_storyteller_scene(scene_path: String, options: JsValue) -> Result<(), JsValue> {
let options = serde_wasm_bindgen::from_value::<WasmInitOptions>(options)?;
crate::start(
InitParams::FromScene(scene_path),
options.mode,
Some(Window {
resolution: options.viewport_size.into(),
title: options.canvas_alt,
canvas: Some(options.canvas_selector),
..default()
}),
)
.run();
Ok(())
}
studio/src/wasm/mod.rs
+3
View File
@@ -4,9 +4,12 @@ use parking_lot::ReentrantMutex;
use wasm_bindgen::prelude::wasm_bindgen;
mod conv;
mod init;
mod screenshot;
mod util;
pub use conv::*;
pub use screenshot::ScreenshotPlugin;
pub(crate) use util::*;
static INITIALIZED: OnceLock<ReentrantMutex<RefCell<bool>>> = OnceLock::new();
studio/src/wasm/screenshot.rs
+211
View File
@@ -0,0 +1,211 @@
//! Outline of the flow of events here:
//!
//! 1. [`crate::change_tracking`] system sends events to frontend to notify of
//! tracked changes
//! 2. Frontend calls [`request_screenshot`] on a debounce, writing an empty
//! message to [`SCREENSHOT_REQUEST`]
//! 3. [`take_screenshots`] watches the [`SCREENSHOT_REQUEST`] buffer and
//! populates the [`SCREENSHOT`] buffer with a PNG-encoded `Vec<u8>`
//! 4. [`send_screenshots`] watches the [`SCREENSHOT`] buffer and sends an event
//! to the frontend by converting the `Vec<u8>` to a [`js_sys::Uint8Array`]
use bevy::{prelude::*, render::view::screenshot::ScreenshotManager, window::PrimaryWindow};
use image::{codecs::jpeg::JpegEncoder, ColorType, ImageEncoder};
use wasm_bindgen::prelude::{wasm_bindgen, JsValue};
use crate::{interaction::gizmos::TransformGizmo, world_grid::WorldGridPlane};
use super::{AsJsValue, StaticBuffer, WasmMessageBuffer};
static SCREENSHOT_REQUEST: WasmMessageBuffer<()> = WasmMessageBuffer::new();
static SCREENSHOT: WasmMessageBuffer<ScreenshotNative> = WasmMessageBuffer::new();
pub struct ScreenshotPlugin;
impl Plugin for ScreenshotPlugin {
fn build(&self, app: &mut App) {
app.add_event::<ScreenshotRequestedEvent>();
app.add_systems(
Update,
(
(
prepare_screenshot,
apply_deferred,
take_screenshots.map(bevy::utils::error),
)
.chain(),
send_screenshots.map(bevy::utils::error),
),
);
}
}
#[derive(Event)]
pub struct ScreenshotRequestedEvent;
#[wasm_bindgen]
#[derive(Clone)]
pub struct Screenshot {
data_pvt: js_sys::Uint8Array,
pub width: usize,
pub height: usize,
}
// This intermediate type is necessary because the `Screenshot` struct
// cannot be safely stored in a `WasmMessageBuffer`
struct ScreenshotNative {
data: Vec<u8>,
width: usize,
height: usize,
}
impl From<ScreenshotNative> for Screenshot {
fn from(value: ScreenshotNative) -> Self {
Self {
data_pvt: js_sys::Uint8Array::from(&value.data[..]),
width: value.width,
height: value.height,
}
}
}
#[wasm_bindgen]
impl Screenshot {
#[wasm_bindgen(getter)]
pub fn data(&self) -> js_sys::Uint8Array {
self.data_pvt.clone()
}
}
impl AsJsValue for Screenshot {
fn as_js(&self) -> Result<JsValue, String> {
Ok(JsValue::from(self.clone()))
}
}
#[wasm_bindgen(js_name = requestScreenshot)]
pub fn request_screenshot() {
info!("[wasm::screenshot] Screenshot request received");
SCREENSHOT_REQUEST.write_message(());
}
fn prepare_screenshot(
mut q_editor_visuals: Query<&mut Visibility, Or<(With<WorldGridPlane>, With<TransformGizmo>)>>,
mut ew: EventWriter<ScreenshotRequestedEvent>,
) {
if SCREENSHOT_REQUEST.take_message().is_none() {
return;
}
info!(
"[wasm::screenshot] Hiding {} editor visuals",
q_editor_visuals.iter().count()
);
for mut visibility in q_editor_visuals.iter_mut() {
*visibility = Visibility::Hidden;
}
ew.send(ScreenshotRequestedEvent);
}
fn take_screenshots(
mut r_screenshot_mgr: ResMut<ScreenshotManager>,
q_window: Query<Entity, With<PrimaryWindow>>,
mut er: EventReader<ScreenshotRequestedEvent>,
) -> Result<(), String> {
if er.is_empty() {
return Ok(());
}
let Ok(window_ent) = q_window.get_single() else {
return Ok(());
};
er.clear();
info!("[wasm::screenshot] Queueing screenshot request");
r_screenshot_mgr
.take_screenshot(window_ent, |img| {
let img = match img.try_into_dynamic().map(|img| img.to_rgb8()) {
Ok(img) => img,
Err(err) => {
error!("{err}");
return;
}
};
let channel_count = <image::Rgb<u8> as image::Pixel>::CHANNEL_COUNT as usize;
let buffer_len = match Some(channel_count)
.and_then(|size| size.checked_mul(img.width() as usize))
.and_then(|size| size.checked_mul(img.height() as usize))
{
Some(size) => size,
None => {
error!("Failed to compute buffer length for image!");
return;
}
};
let src_buffer = &img.as_raw()[..buffer_len];
let mut jpg_bytes = vec![];
if let Err(err) = JpegEncoder::new_with_quality(&mut jpg_bytes, 70).write_image(
src_buffer,
img.width(),
img.height(),
ColorType::Rgb8,
) {
error!("{err}");
return;
}
SCREENSHOT.write_message(ScreenshotNative {
data: jpg_bytes,
width: img.width() as usize,
height: img.height() as usize,
});
})
.map_err(|err| format!("{err}"))
}
#[wasm_bindgen(typescript_custom_section)]
const SCREENSHOT_READY_EVENT: &str = r#"
export interface ScreenshotReadyEvent extends CustomEvent {
type: "screenshot-ready";
detail: Screenshot;
}
declare global {
export interface GlobalEventHandlersEventMap {
"screenshot-ready": ScreenshotReadyEvent;
}
}
"#;
fn send_screenshots(
q_window: Query<&Window, With<PrimaryWindow>>,
mut q_editor_visuals: Query<&mut Visibility, Or<(With<WorldGridPlane>, With<TransformGizmo>)>>,
) -> Result<(), String> {
let Some(screenshot) = SCREENSHOT.take_message() else {
return Ok(());
};
let Ok(window) = q_window.get_single() else {
return Ok(());
};
let screenshot = Screenshot::from(screenshot);
super::dispatch_to_js(window, "screenshot-ready", &screenshot)?;
info!(
"[wasm::screenshot] Revealing {} previously hidden editor visuals",
q_editor_visuals.iter().count()
);
for mut visibility in q_editor_visuals.iter_mut() {
*visibility = Visibility::Inherited;
}
Ok(())
}
studio/src/world_grid.rs
+1 -1
View File
@@ -17,7 +17,7 @@ impl Plugin for WorldGridPlugin {
}
#[derive(Component)]
struct WorldGridPlane;
pub struct WorldGridPlane;
#[derive(Asset, AsBindGroup, TypePath, Clone)]
struct ProceduralGridMaterial {}