From 61fa16bdba8f819f5f24bdbd8f1452865c0dd9df Mon Sep 17 00:00:00 2001 From: Danny McGee Date: Wed, 28 Feb 2024 15:47:26 -0500 Subject: [PATCH] Expose skeletal joints to frontend hierarchy (#56) This PR exposes skeletal joint entities to the frontend hierarchy, allowing the user to pose rigged models by selecting the joints and transforming them. Additionally, entity-type information (e.g., Mesh, Skeleton, Bone, Light, etc.) has been added to the data structure passed to the frontend, which is rendered with icons in the tree view. --- studio-ui/src/lib/icon.element.ts | 6 ++ studio-ui/src/lib/tree.element.scss | 6 +- studio-ui/src/lib/tree.element.ts | 6 +- studio-web/src/lib/scene-hierarchy.element.ts | 24 ++++- studio/src/anim/mod.rs | 5 +- studio/src/inspector/wasm.rs | 87 ++++++++----------- studio/src/scene.rs | 58 ++++++++----- utility/src/lib/match.ts | 2 +- 8 files changed, 107 insertions(+), 87 deletions(-) diff --git a/studio-ui/src/lib/icon.element.ts b/studio-ui/src/lib/icon.element.ts index 5921924..b31bcb4 100644 --- a/studio-ui/src/lib/icon.element.ts +++ b/studio-ui/src/lib/icon.element.ts @@ -21,6 +21,9 @@ import farRotate from "@fortawesome/fontawesome-pro/svgs/regular/arrows-rotate.s import farScale from "@fortawesome/fontawesome-pro/svgs/regular/arrow-up-right-and-arrow-down-left-from-center.svg?raw"; import fasCaretRight from "@fortawesome/fontawesome-pro/svgs/solid/caret-right.svg?raw"; import fasFloppyDisk from "@fortawesome/fontawesome-pro/svgs/solid/floppy-disk.svg?raw"; +import farSkeleton from "@fortawesome/fontawesome-pro/svgs/regular/skeleton.svg?raw"; +import farBone from "@fortawesome/fontawesome-pro/svgs/regular/bone.svg?raw"; +import fasBrightness from "@fortawesome/fontawesome-pro/svgs/solid/brightness.svg?raw"; import styles from "./icon.element.scss?inline"; @@ -47,6 +50,7 @@ function renderSvg(iconId: string) { return RENDER_RESULTS.get(iconId)!; const svgSrc = match (iconId, { + "bone": () => farBone, "bwd-fast": () => fasBwdFast, "bwd-step": () => fasBwdStep, "circle-question": () => fasCircleQuestion, @@ -57,11 +61,13 @@ function renderSvg(iconId: string) { "fwd-fast": () => fasFwdFast, "fwd-step": () => fasFwdStep, "key": () => fasKey, + "light": () => fasBrightness, "globe": () => farGlobe, "play": () => fasPlay, "question": () => fasQuestion, "rotate": () => farRotate, "scale": () => farScale, + "skeleton": () => farSkeleton, "up-down-left-right": () => farUpDownLeftRight, "video": () => farVideo, _: () => null, diff --git a/studio-ui/src/lib/tree.element.scss b/studio-ui/src/lib/tree.element.scss index 82f5558..8a55b74 100644 --- a/studio-ui/src/lib/tree.element.scss +++ b/studio-ui/src/lib/tree.element.scss @@ -70,7 +70,7 @@ header { } } -.icon.expanded { +.caret.expanded { transform: rotate(90deg); } @@ -83,6 +83,10 @@ header { padding-inline-start: 5px; } +.icon { + opacity: 0.75; +} + .children { margin-inline-start: 7px; padding-inline-start: 9px; diff --git a/studio-ui/src/lib/tree.element.ts b/studio-ui/src/lib/tree.element.ts index 96ce8a0..f8621ab 100644 --- a/studio-ui/src/lib/tree.element.ts +++ b/studio-ui/src/lib/tree.element.ts @@ -13,6 +13,7 @@ import styles from "./tree.element.scss?inline"; export interface Tree { id?: string; name: string; + icon?: string; children?: Tree[]; } @@ -55,7 +56,7 @@ export class TreeElement extends LitElement { > ${this.icon ? html` - + ` : nothing} ${this.name} @@ -79,6 +80,7 @@ export class TreeElement extends LitElement { `)} diff --git a/studio-web/src/lib/scene-hierarchy.element.ts b/studio-web/src/lib/scene-hierarchy.element.ts index 2769db2..7b14c1a 100644 --- a/studio-web/src/lib/scene-hierarchy.element.ts +++ b/studio-web/src/lib/scene-hierarchy.element.ts @@ -1,10 +1,12 @@ +import { on } from "@storyteller/framework"; import * as studio from "@storyteller/studio"; import { + SceneElement, type EntityDespawnEvent, type EntityMultiSpawnEvent, type EntitySpawnEvent, } from "@storyteller/studio"; -import { on } from "@storyteller/framework"; +import { match } from "@storyteller/utility"; import { LitElement, html, unsafeCSS } from "lit"; import { customElement, state } from "lit/decorators.js"; @@ -12,7 +14,8 @@ import styles from "./scene-hierarchy.element.scss?inline"; import "@storyteller/studio-ui/tree"; -interface SceneObject extends Omit { +interface SceneObject extends Omit { + icon?: string; children: SceneObject[]; } @@ -43,8 +46,20 @@ export class SceneHierarchyElement extends LitElement { #rebuildTree(): void { this.#entityMap.clear(); - for (let obj of this.#sceneObjectsFlat) - this.#entityMap.set(obj.id, { ...obj, children: [] }); + for (let obj of this.#sceneObjectsFlat) { + this.#entityMap.set(obj.id, { + ...obj, + icon: match (obj.type, { + [SceneElement.Generic]: () => "cube", + [SceneElement.Mesh]: () => "cube", // TODO ? + [SceneElement.Skeleton]: () => "skeleton", + [SceneElement.Bone]: () => "bone", + [SceneElement.Light]: () => "light", + [SceneElement.Camera]: () => "video", + }), + children: [] + }); + } for (let obj of this.#entityMap.values()) { if (obj.parent != null) { @@ -70,6 +85,7 @@ export class SceneHierarchyElement extends LitElement { `)} diff --git a/studio/src/anim/mod.rs b/studio/src/anim/mod.rs index 6c512fb..ce98564 100644 --- a/studio/src/anim/mod.rs +++ b/studio/src/anim/mod.rs @@ -6,7 +6,7 @@ use bevy::{ }; use space_editor::space_prefab::editor_registry::EditorRegistryExt; -use crate::hierarchy; +use crate::{hierarchy, scene::SceneElement}; pub use retargeting::*; mod camera; @@ -131,7 +131,7 @@ fn process_skinned_meshes( // skeleton let joints = joints.iter().copied().collect::>(); if let Some(root) = hierarchy::nearest_common_ancestor(&joints, &q_parents) { - cmd.entity(root).insert(Skeleton); + cmd.entity(root).insert((Skeleton, SceneElement::Skeleton)); } // Tag each joint and find its bind poses @@ -141,6 +141,7 @@ fn process_skinned_meshes( cmd.entity(joint).insert(( SkeletalJoint(Color::BLACK), + SceneElement::Bone, BindPoseLocal(bind_pose_local), BindPoseGlobal(bind_pose_global.affine()), InvBindPose(bind_pose_global.affine().inverse()), diff --git a/studio/src/inspector/wasm.rs b/studio/src/inspector/wasm.rs index 66c2548..286619a 100644 --- a/studio/src/inspector/wasm.rs +++ b/studio/src/inspector/wasm.rs @@ -4,10 +4,7 @@ use bevy::{prelude::*, utils::HashSet, window::PrimaryWindow}; use serde::Serialize; use wasm_bindgen::prelude::*; -use crate::{ - scene::SceneElement, - wasm::{self, AsJson}, -}; +use crate::{scene::SceneElement, wasm}; #[wasm_bindgen(typescript_custom_section)] const TYPE_DECLARATIONS: &str = r#" @@ -16,6 +13,7 @@ export interface SceneObject { id: string; /** A user-friendly name for displaying in the UI */ name: string; + type: SceneElement; /** The IDs of this object's children in the hierarchy */ children: string[]; /** The ID of this object's parent in the hierarchy, if it has one */ @@ -64,50 +62,50 @@ declare global { pub(super) struct SceneObject { id: String, name: String, + #[serde(rename = "type")] + type_: SceneElement, children: Vec, parent: Option, - transform: serde_json::Value, - #[serde(rename(serialize = "globalTransform"))] - global_transform: serde_json::Value, } -struct SceneObjectBuilder<'a> { +struct SceneObjectBuilder { id: Entity, name: Option, + type_: Option, children: Option>, parent: Option, - transform: Option<&'a Transform>, - global_transform: Option<&'a GlobalTransform>, } -impl<'a> SceneObjectBuilder<'a> { +impl SceneObjectBuilder { fn new(id: Entity) -> Self { Self { id, name: None, + type_: None, children: None, parent: None, - transform: None, - global_transform: None, } } fn finish(self) -> SceneObject { SceneObject { id: format!("{:?}", self.id), name: self.name.unwrap(), + type_: self.type_.unwrap(), children: self .children .map(|children| children.iter().map(|ent| format!("{ent:?}")).collect()) .unwrap_or_default(), parent: self.parent.map(|ent| format!("{ent:?}")), - transform: self.transform.unwrap().as_json(), - global_transform: self.global_transform.unwrap().as_json(), } } fn with_name(mut self, name: String) -> Self { self.name = Some(name); self } + fn with_type(mut self, type_: SceneElement) -> Self { + self.type_ = Some(type_); + self + } fn with_children(mut self, children: Vec) -> Self { self.children = Some(children); self @@ -116,18 +114,10 @@ impl<'a> SceneObjectBuilder<'a> { self.parent = Some(parent); self } - fn with_transform(mut self, transform: &'a Transform) -> Self { - self.transform = Some(transform); - self - } - fn with_global_transform(mut self, global_transform: &'a GlobalTransform) -> Self { - self.global_transform = Some(global_transform); - self - } } -impl<'a> SceneObject { - fn build(id: Entity) -> SceneObjectBuilder<'a> { +impl SceneObject { + fn build(id: Entity) -> SceneObjectBuilder { SceneObjectBuilder::new(id) } } @@ -140,8 +130,8 @@ pub(super) fn notify_entity_spawn_changes( mut l_despawned_entities: Local>, mut l_missing_parents: Local>, q_window: Query<&Window, With>, - q_inspectables: Query<(Entity, &Transform, &GlobalTransform), With>, - q_others: Query<(Entity, &Transform, &GlobalTransform), Without>, + q_inspectables: Query<(Entity, &SceneElement)>, + q_others: Query>, q_names: Query<&Name>, q_parents: Query<&Children>, q_children: Query<&Parent>, @@ -149,26 +139,21 @@ pub(super) fn notify_entity_spawn_changes( l_missing_parents.clear(); l_new_objects.clear(); - l_new_objects.extend( - q_inspectables - .iter() - .filter_map(|(ent, xform, world_xform)| { - build_scene_object( - &mut l_notified, - &mut l_missing_parents, - ent, - xform, - world_xform, - &q_inspectables, - &q_names, - &q_parents, - &q_children, - ) - }), - ); + l_new_objects.extend(q_inspectables.iter().filter_map(|(ent, &type_)| { + build_scene_object( + &mut l_notified, + &mut l_missing_parents, + ent, + type_, + &q_inspectables, + &q_names, + &q_parents, + &q_children, + ) + })); while let Some(parent) = l_missing_parents.pop_front() { - let Ok((ent, xform, world_xform)) = q_others.get(parent) else { + let Ok(ent) = q_others.get(parent) else { error!("Failed to find missing parent: {parent:?}"); l_notified.insert(parent); @@ -179,8 +164,7 @@ pub(super) fn notify_entity_spawn_changes( &mut l_notified, &mut l_missing_parents, ent, - xform, - world_xform, + SceneElement::Generic, &q_inspectables, &q_names, &q_parents, @@ -218,9 +202,8 @@ fn build_scene_object( notified: &mut HashSet, missing_parents: &mut VecDeque, ent: Entity, - xform: &Transform, - world_xform: &GlobalTransform, - q_inspectables: &Query<(Entity, &Transform, &GlobalTransform), With>, + type_: SceneElement, + q_inspectables: &Query<(Entity, &SceneElement)>, q_names: &Query<&Name>, q_parents: &Query<&Children>, q_children: &Query<&Parent>, @@ -231,9 +214,7 @@ fn build_scene_object( notified.insert(ent); - let mut result = SceneObject::build(ent) - .with_transform(xform) - .with_global_transform(world_xform); + let mut result = SceneObject::build(ent).with_type(type_); if let Ok(children) = q_parents.get(ent) { result = result.with_children(children.iter().cloned().collect()); diff --git a/studio/src/scene.rs b/studio/src/scene.rs index 875373a..5c51325 100644 --- a/studio/src/scene.rs +++ b/studio/src/scene.rs @@ -12,7 +12,8 @@ use bevy::{ render::{camera::RenderTarget, primitives::Aabb}, scene::InstanceId, }; - +#[cfg(feature = "wasm")] +use serde_repr::{Deserialize_repr, Serialize_repr}; use space_editor::{ prelude::{EditorEvent, EditorPrefabPath, PrefabMarker}, space_prefab::component::SceneAutoChild, @@ -106,16 +107,20 @@ impl Plugin for ScenePlugin { } fn minimal_scene_setup(mut cmd: Commands) { - cmd.spawn(DirectionalLightBundle { - directional_light: DirectionalLight { - shadows_enabled: false, - illuminance: 20000.0, + cmd.spawn(( + Name::new("Directional Light"), + DirectionalLightBundle { + directional_light: DirectionalLight { + shadows_enabled: false, + illuminance: 20000.0, + ..default() + }, + transform: Transform::default().looking_at(Vec3::NEG_ONE, Vec3::Y), ..default() }, - transform: Transform::default().looking_at(Vec3::NEG_ONE, Vec3::Y), - ..default() - }) - .insert(PrefabMarker); + PrefabMarker, + SceneElement::Light, + )); } #[cfg_attr(feature = "wasm", wasm_bindgen)] @@ -224,14 +229,24 @@ pub struct LoadSceneEvent(pub String); #[derive(Event, Deref)] pub struct ImportSceneEvent(pub String); -// TODOL This could be converted into an enum that indicates what "kind" of -// element this is (e.g. StaticMesh, SkinnedMesh, Bone, Light, Camera). -// We don't have an immediate need for that just yet, but likely will -// soon. /// A tag that identifies its entity as a user-facing scene element like a prop, /// a light, or a camera. -#[derive(Component)] -pub struct SceneElement; +#[cfg_attr( + feature = "wasm", + repr(u8), + wasm_bindgen, + derive(Serialize_repr, Deserialize_repr) +)] +#[derive(Component, Clone, Copy, Debug, PartialEq, Eq)] +pub enum SceneElement { + Generic, + Mesh, + Skeleton, + Bone, + Light, + #[allow(dead_code)] // TODO: Will be used for camera animation + Camera, +} #[derive(Component)] pub struct SceneData { @@ -324,6 +339,7 @@ fn init_object_loading(mut cmd: Commands, mut er: EventReader) { if let Some(path) = er.read().last() { info!("Importing asset: {}", path.0); cmd.spawn(( + Name::new("Scene"), // TODO: Maybe use the "basename" part of the asset path? SpatialBundle::default(), space_editor::prelude::GltfPrefab { path: path.0.clone(), @@ -362,12 +379,6 @@ fn init_scene_loading( } } -#[derive(Component)] -pub struct NoMeshElement; - -#[derive(Component)] -pub struct InitedKointSkinnedMesh; - fn tag_scene_elements( mut cmd: Commands, // A "glTF mesh" is actually a container for one or more "glTF primitives", @@ -384,7 +395,6 @@ fn tag_scene_elements( With, Without>, Without, - Without, ), >, mesh_entities: Query>>, @@ -398,9 +408,9 @@ fn tag_scene_elements( } } if has_mesh { - cmd.entity(e).insert(SceneElement); + cmd.entity(e).insert(SceneElement::Mesh); } else { - cmd.entity(e).insert(NoMeshElement); + cmd.entity(e).insert(SceneElement::Generic); } } } diff --git a/utility/src/lib/match.ts b/utility/src/lib/match.ts index c93700c..86fbb05 100644 --- a/utility/src/lib/match.ts +++ b/utility/src/lib/match.ts @@ -83,7 +83,7 @@ export function match>( subject: K | null | undefined, matcher: T, ): MatchReturn { - if (subject && subject in matcher) + if (subject != null && subject in matcher) return matcher[subject]!(); if ("_" in matcher)