Mixamo animation support (#37)

This PR adds support for Mixamo FBX imports (via backend conversion to
glTF/glB) and some miscellaneous improvements to the animation systems.

### User-facing improvements
* Retargeting math has been fixed to eliminate jittering issues
* Retargeter can now handle animated bone translations when present
  (generally this is only used for root motion)

### Housekeeping / Code improvements

* Animation-related components and system logic have been moved to a
  dedicated `anim` module and `AnimPlugin`
* Retargeting stuff has been moved from the `bvh` module to
  `anim::retargeting`
* Component-based solution for marking an entity as an animation
  retargeting source makes it easy to extend the retargeting functionality
  to new types of source animations

### Limitations

Currently the code assumes that there will be at most one animation
source at a time. That single animation (if present) will be retargeted
to all other skeletons in the scene. A component-based solution for
directing a specific target skeleton to use a specific source animation
is possible, but will require some additional consideration around the
implementation design for the read-only "viewer" modes.

### Test animations:

* Hip-Hop Dancing (Mixamo): ?mode=viewer&mixamo=mocap/hip-hop-dancing.gltf
* Shuffle (MocapNET): ?mode=viewer&bvh=mocap/shuffle.bvh
* Scott (MocapNET): ?mode=viewer&bvh=mocap/scott.bvh
* Scott 1 (MocapNET): ?mode=viewer&bvh=mocap/scott1.bvh
This commit is contained in:
Danny McGee
2024-02-09 13:21:16 -05:00
committed by GitHub
parent d1b4aab8ca
commit a5aeff7a3a
13 changed files with 4109 additions and 299 deletions
+141
View File
@@ -0,0 +1,141 @@
use bevy::{
math::Affine3A,
prelude::*,
render::{mesh::skinning::SkinnedMesh, view::NoFrustumCulling},
utils::{EntityHashMap, EntityHashSet},
};
use space_editor::space_prefab::editor_registry::EditorRegistryExt;
use crate::hierarchy;
pub use retargeting::*;
mod retargeting;
pub struct AnimPlugin;
impl Plugin for AnimPlugin {
fn build(&self, app: &mut App) {
app.register_type::<Skeleton>()
.editor_registry::<Skeleton>()
.register_type::<SkeletalJoint>()
.editor_registry::<SkeletalJoint>()
.register_type::<ProcessedSkinnedMesh>()
.editor_registry::<ProcessedSkinnedMesh>()
.register_type::<BindPoseLocal>()
.editor_registry::<BindPoseLocal>()
.register_type::<BindPoseGlobal>()
.editor_registry::<BindPoseGlobal>()
.register_type::<InvBindPose>()
.editor_registry::<InvBindPose>();
app.add_systems(Update, process_skinned_meshes);
app.add_plugins(retargeting::RetargetingPlugin);
}
}
#[derive(Component, Clone, Copy, Debug, Default, Reflect)]
#[reflect(Component)]
pub struct Skeleton;
#[derive(Component, Clone, Copy, Debug, Default, Reflect)]
#[reflect(Component)]
pub struct SkeletalJoint(pub Color);
#[derive(Component, Clone, Copy, Debug, Default, Reflect)]
#[reflect(Component)]
pub struct ProcessedSkinnedMesh;
#[derive(Component, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Component)]
pub struct BindPoseLocal(pub Transform);
#[derive(Component, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Component)]
pub struct BindPoseGlobal(pub Affine3A);
#[derive(Component, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Component)]
pub struct InvBindPose(pub Affine3A);
fn process_skinned_meshes(
mut cmd: Commands,
q_skinned_meshes: Query<(Entity, &SkinnedMesh), Without<ProcessedSkinnedMesh>>,
mut q_xforms: ParamSet<(TransformHelper, Query<&Transform>)>,
q_parents: Query<Option<&Parent>>,
) {
if q_skinned_meshes.is_empty() {
return;
}
// Map of skinned-mesh parent entities to the set of joints bound to those
// skinned meshes. We use the skinned-mesh parent because individual skinned
// meshes in glTF format are often only parts of the whole "model," in which
// case the parts will be siblings in the hierarchy.
//
// I.e., considering a hierarchy like this:
// ```
// └ RootNode
// ├ Body [SkinnedMesh]
// ├ Head [SkinnedMesh]
// ├ Eyes [SkinnedMesh]
// └ Hips [Joint]
// ├ UpperLeg_L [Joint]
// │ └ ...
// ├ UpperLeg_R [Joint]
// │ └ ...
// └ Spine [Joint]
// └ ...
// ```
// The hashmap should contain an entry like this:
// ```
// {
// RootNode: { Hips, UpperLeg_L, UpperLeg_R, Spine, ... }
// }
// ```
// Building this hashmap before processing each joint avoids redundancy in
// the common case where the same joints are bound to multiple skinned meshes
let mut joint_collections = EntityHashMap::<Entity, EntityHashSet<Entity>>::default();
for (ent, skinned_mesh) in q_skinned_meshes.iter() {
cmd.entity(ent)
.insert(NoFrustumCulling)
.insert(ProcessedSkinnedMesh);
let Some(parent) = q_parents.get(ent).ok().flatten() else {
continue;
};
let parent_ent = **parent;
let joints = match joint_collections.get_mut(&parent_ent) {
Some(set) => set,
None => {
let (_, set) =
joint_collections.insert_unique_unchecked(parent_ent, Default::default());
set
}
};
joints.extend(skinned_mesh.joints.iter());
}
for joints in joint_collections.values() {
// Find the common ancestor of this joint collection and tag it as the
// skeleton
let joints = joints.iter().copied().collect::<Vec<_>>();
if let Some(root) = hierarchy::nearest_common_ancestor(&joints, &q_parents) {
cmd.entity(root).insert(Skeleton);
}
// Tag each joint and find its bind poses
for &joint in joints.iter() {
let bind_pose_global = q_xforms.p0().compute_global_transform(joint).unwrap();
let bind_pose_local = *q_xforms.p1().get(joint).unwrap();
cmd.entity(joint).insert((
SkeletalJoint(Color::BLACK),
BindPoseLocal(bind_pose_local),
BindPoseGlobal(bind_pose_global.affine()),
InvBindPose(bind_pose_global.affine().inverse()),
));
}
}
}
+431
View File
@@ -0,0 +1,431 @@
use bevy::{prelude::*, utils::HashMap};
use space_editor::space_prefab::editor_registry::EditorRegistryExt;
use crate::math::NearlyEq;
use super::{BindPoseGlobal, BindPoseLocal, InvBindPose, SkeletalJoint, Skeleton};
pub(super) struct RetargetingPlugin;
impl Plugin for RetargetingPlugin {
fn build(&self, app: &mut App) {
app.register_type::<SkeletonHeight>()
.editor_registry::<SkeletonHeight>()
.register_type::<RetargetMap>();
app.insert_resource(RetargetMap::default());
app.add_systems(Update, copy_pose);
}
}
#[derive(Component, Clone, Copy, Debug, Default, Reflect)]
#[reflect(Component)]
pub struct RetargetSource;
#[derive(Component, Clone, Copy, Debug, Default, Reflect, Deref, DerefMut)]
#[reflect(Component)]
pub struct SkeletonHeight(f32);
fn copy_pose(
mut cmd: Commands,
r_retarget_map: Res<RetargetMap>,
q_src_root: Query<Entity, With<RetargetSource>>,
q_tgt_roots: Query<Entity, (With<Skeleton>, Without<RetargetSource>)>,
q_parents: Query<&Parent>,
q_children: Query<&Children>,
q_joint_names: Query<&Name, With<SkeletalJoint>>,
q_inv_bind_poses: Query<&InvBindPose>,
q_world_bind_poses: Query<&BindPoseGlobal>,
q_local_bind_poses: Query<&BindPoseLocal>,
q_skel_height: Query<&SkeletonHeight>,
mut q_xforms: Query<&mut Transform>,
q_world_xforms: Query<&GlobalTransform>,
) {
let src_root = match q_src_root.get_single() {
Ok(src_root) => src_root,
Err(err) => {
error!("{err}");
return;
}
};
let src_height = match q_skel_height.get(src_root) {
Ok(skel_height) => **skel_height,
Err(_) => {
let Some(value) = compute_skeleton_height(src_root, &q_children, &q_world_bind_poses)
else {
return;
};
cmd.entity(src_root).insert(SkeletonHeight(value));
value
}
};
let src_joints = q_children
.iter_descendants(src_root)
.filter_map(|ent| match q_joint_names.get(ent) {
Ok(name) => Some((ent, name)),
Err(_) => None,
})
.collect::<Vec<_>>();
// FIXME:
// This is very much not fast, and probably shouldn't be done every frame.
// Tentative plan to optimize this without adding too much additional
// complexity:
//
// 1) Add some sort of "AnimationRecorder" component to the target root
// 2) Beginning with `src_anim.seek_time == 0.0`, record each local
// rotation and translation performed on the target bones as a new
// keyframe on an `AnimationClip` held by `AnimationRecorder`
// 3) When `src_anim.seek_time == src_anim.duration`, finalize the new
// `AnimationClip` and add it to the asset server
// 4) Stop running this system and instead start playing the new animation
// with an `AnimationPlayer` component on the target root
for tgt_root in q_tgt_roots.iter() {
let tgt_height = match q_skel_height.get(tgt_root) {
Ok(skel_height) => **skel_height,
Err(_) => {
let Some(value) =
compute_skeleton_height(tgt_root, &q_children, &q_world_bind_poses)
else {
continue;
};
cmd.entity(tgt_root).insert(SkeletonHeight(value));
value
}
};
let xlation_scale = tgt_height / src_height;
for tgt_joint in q_children.iter_descendants(tgt_root) {
let Ok(tgt_name) = q_joint_names.get(tgt_joint) else {
continue;
};
// Look up the source bone in the retarget map
// TODO: Snatching O(n) performance from the jaws of an O(1) data
// structure here by using a many-source to single-target
// hashmap, but then looking up the source from the target
// instead of the other way around
let Some(src_joint) = src_joints.iter().copied().find_map(|(joint, src_name)| {
match r_retarget_map.src_to_tgt.get(src_name.as_str()) {
Some(name) if name == tgt_name.as_str() => Some(joint),
_ => None,
}
}) else {
continue;
};
// Get the query data needed to compute the target bone's local rotation
let Ok(src_inv_bind_pose) = q_inv_bind_poses.get(src_joint) else {
continue;
};
let Ok(src_world_xform) = q_world_xforms.get(src_joint) else {
continue;
};
let Ok(tgt_world_bind_pose) = q_world_bind_poses.get(tgt_joint) else {
continue;
};
// The math here was adapted from this writeup:
// https://wickedengine.net/2022/09/23/animation-retargeting/
//
// ```pseudocode
// // Apply target-local rotation
// tgt_working_xform = src_world_xform * src_inv_bind_pose
// tgt_working_xform = tgt_inv_parent_xform * tgt_working_xform * tgt_world_bind_pose
// tgt_xform.rotation = tgt_working_xform.rotation
//
// // Check if the translation component is animated
// if src_local_xform.translation != src_local_bind_pose.translation {
// // Apply target-local translation
// src_world_xlation = src_inv_bind_pose * src_local_xform.translation
// tgt_local_xlation = tgt_inv_parent_xform * src_world_xlation
// tgt_xform.translation = tgt_local_xlation
// }
// ```
let mut tgt_working_xform = src_world_xform.affine() * (**src_inv_bind_pose);
// The parent bone's world transform must be computed each iteration,
// since it will be affected by the rotations applied on the previous
// iteration.
let tgt_parent = q_parents.get(tgt_joint);
let parent_xform = tgt_parent
.and_then(|parent| q_xforms.get(**parent))
.copied()
.unwrap_or_default();
let mut parent_world_xform = GlobalTransform::from(parent_xform);
if let Ok(parent) = tgt_parent {
for ancestor in q_parents.iter_ancestors(**parent) {
let xform = q_xforms.get(ancestor).copied().unwrap_or_default();
parent_world_xform = xform * parent_world_xform;
}
}
let tgt_inv_parent_xform = parent_world_xform.affine().inverse();
tgt_working_xform = tgt_inv_parent_xform * tgt_working_xform * (**tgt_world_bind_pose);
// Apply the local rotation
if let Ok(mut tgt_xform) = q_xforms.get_mut(tgt_joint) {
tgt_xform.rotation = Quat::from_affine3(&tgt_working_xform);
}
// Get the query data needed to compute the target bone's local translation
let Ok(src_local_bind_pose) = q_local_bind_poses.get(src_joint) else {
continue;
};
let Ok(src_world_bind_pose) = q_world_bind_poses.get(src_joint) else {
continue;
};
let Ok(src_local_xlation) = q_xforms.get(src_joint).map(|xform| xform.translation)
else {
continue;
};
// If the source bone translation != its bind pose translation,
// this translation component is animated (likely root motion)
if !src_local_xlation.nearly_eq_within(src_local_bind_pose.translation, 1e-3) {
let src_world_xlation = src_world_bind_pose.transform_vector3(src_local_xlation);
let tgt_local_xlation = tgt_inv_parent_xform.transform_vector3(src_world_xlation);
// Apply the local translation
if let Ok(mut tgt_xform) = q_xforms.get_mut(tgt_joint) {
tgt_xform.translation = tgt_local_xlation * xlation_scale;
}
}
}
}
}
fn compute_skeleton_height(
root: Entity,
q_children: &Query<&Children>,
q_world_bind_poses: &Query<&BindPoseGlobal>,
) -> Option<f32> {
let joint_positions = q_children
.iter_descendants(root)
.filter_map(|joint| {
q_world_bind_poses
.get(joint)
.map(|bind_pose| bind_pose.translation)
.ok()
})
.collect::<Vec<_>>();
if joint_positions.is_empty() {
return None;
}
let y_max = joint_positions
.iter()
.fold(f32::MIN, |accum, current| f32::max(accum, current.y));
let y_min = joint_positions
.iter()
.fold(f32::MAX, |accum, current| f32::min(accum, current.y));
Some(y_max - y_min)
}
#[derive(Resource, Clone, Debug, Reflect)]
#[reflect(Resource)]
pub struct RetargetMap {
pub src_to_tgt: HashMap<String, String>,
}
impl RetargetMap {
const MOCAP_NET: [(&str, &str); 80] = [
// Spine
("LowerBack", "DEF-spine"),
("hip", "DEF-spine"),
("abdomen", "DEF-spine.002"),
("chest", "DEF-spine.003"),
("Neck", "DEF-spine.004"),
("neck", "DEF-spine.004"),
("Neck1", "DEF-spine.005"),
("neck1", "DEF-spine.005"),
("Head", "DEF-spine.006"),
("head", "DEF-spine.006"),
// Right arm
("rcollar", "DEF-shoulder.R"),
("rCollar", "DEF-shoulder.R"),
("rshoulder", "DEF-upper_arm.R"),
("rShldr", "DEF-upper_arm.R"),
("relbow", "DEF-forearm.R"),
("rForeArm", "DEF-forearm.R"),
("rhand", "DEF-hand.R"),
("rHand", "DEF-hand.R"),
// Right fingers
// -- Index
("metacarpal1.r", "DEF-palm.01.R"),
("finger2-1.r", "DEF-f_index.01.R"),
("finger2-2.r", "DEF-f_index.02.R"),
("finger2-3.r", "DEF-f_index.03.R"),
// -- Middle
("metacarpal2.r", "DEF-palm.02.R"),
("finger3-1.r", "DEF-f_middle.01.R"),
("finger3-2.r", "DEF-f_middle.02.R"),
("finger3-3.r", "DEF-f_middle.03.R"),
// -- Ring
("metacarpal3.r", "DEF-palm.03.R"),
("finger4-1.r", "DEF-f_ring.01.R"),
("finger4-2.r", "DEF-f_ring.02.R"),
("finger4-3.r", "DEF-f_ring.03.R"),
// -- Pinky
("metacarpal4.r", "DEF-palm.04.R"),
("finger5-1.r", "DEF-f_pinky.01.R"),
("finger5-2.r", "DEF-f_pinky.02.R"),
("finger5-3.r", "DEF-f_pinky.03.R"),
// -- Thumb
("rthumb", "DEF-thumb.01.R"),
("finger1-2.r", "DEF-thumb.02.R"),
("finger1-3.r", "DEF-thumb.03.R"),
// Right leg
("rhip", "DEF-thigh.R"),
("rThigh", "DEF-thigh.R"),
("rknee", "DEF-shin.R"),
("rShin", "DEF-shin.R"),
("rfoot", "DEF-foot.R"),
("rFoot", "DEF-foot.R"),
("RightToeBase", "DEF-toe.R"),
// Left arm
("lcollar", "DEF-shoulder.L"),
("lCollar", "DEF-shoulder.L"),
("lshoulder", "DEF-upper_arm.L"),
("lShldr", "DEF-upper_arm.L"),
("lelbow", "DEF-forearm.L"),
("lForeArm", "DEF-forearm.L"),
("lhand", "DEF-hand.L"),
("lHand", "DEF-hand.L"),
// Left fingers
// -- Index
("metacarpal1.l", "DEF-palm.01.L"),
("finger2-1.l", "DEF-f_index.01.L"),
("finger2-2.l", "DEF-f_index.02.L"),
("finger2-3.l", "DEF-f_index.03.L"),
// -- Middle
("metacarpal2.l", "DEF-palm.02.L"),
("finger3-1.l", "DEF-f_middle.01.L"),
("finger3-2.l", "DEF-f_middle.02.L"),
("finger3-3.l", "DEF-f_middle.03.L"),
// -- Ring
("metacarpal3.l", "DEF-palm.03.L"),
("finger4-1.l", "DEF-f_ring.01.L"),
("finger4-2.l", "DEF-f_ring.02.L"),
("finger4-3.l", "DEF-f_ring.03.L"),
// -- Pinky
("metacarpal4.l", "DEF-palm.04.L"),
("finger5-1.l", "DEF-f_pinky.01.L"),
("finger5-2.l", "DEF-f_pinky.02.L"),
("finger5-3.l", "DEF-f_pinky.03.L"),
// -- Thumb
("lthumb", "DEF-thumb.01.L"),
("finger1-2.l", "DEF-thumb.02.L"),
("finger1-3.l", "DEF-thumb.03.L"),
// Left leg
("lhip", "DEF-thigh.L"),
("lThigh", "DEF-thigh.L"),
("lknee", "DEF-shin.L"),
("lShin", "DEF-shin.L"),
("lfoot", "DEF-foot.L"),
("lFoot", "DEF-foot.L"),
("LeftToeBase", "DEF-toe.L"),
// Pelvis
("RHipJoint", "DEF-pelvis.R"),
("LHipJoint", "DEF-pelvis.L"),
];
const MIXAMO: [(&str, &str); 60] = [
// Spine
("mixamorig:Hips", "DEF-spine"),
("mixamorig:Spine", "DEF-spine.001"),
("mixamorig:Spine1", "DEF-spine.002"),
("mixamorig:Spine2", "DEF-spine.003"),
("mixamorig:Neck", "DEF-spine.004"),
("mixamorig:Head", "DEF-spine.006"),
// Right arm
("mixamorig:RightShoulder", "DEF-shoulder.R"),
("mixamorig:RightArm", "DEF-upper_arm.R"),
("mixamorig:RightForeArm", "DEF-forearm.R"),
("mixamorig:RightHand", "DEF-hand.R"),
// Right fingers
// -- index
("mixamorig:RightHandIndex1", "DEF-palm.01.R"),
("mixamorig:RightHandIndex2", "DEF-f_index.01.R"),
("mixamorig:RightHandIndex3", "DEF-f_index.02.R"),
("mixamorig:RightHandIndex4", "DEF-f_index.03.R"),
// -- middle
("mixamorig:RightHandMiddle1", "DEF-palm.02.R"),
("mixamorig:RightHandMiddle2", "DEF-f_middle.01.R"),
("mixamorig:RightHandMiddle3", "DEF-f_middle.02.R"),
("mixamorig:RightHandMiddle4", "DEF-f_middle.03.R"),
// -- ring
("mixamorig:RightHandRing1", "DEF-palm.03.R"),
("mixamorig:RightHandRing2", "DEF-f_ring.01.R"),
("mixamorig:RightHandRing3", "DEF-f_ring.02.R"),
("mixamorig:RightHandRing4", "DEF-f_ring.03.R"),
// -- pinky
("mixamorig:RightHandPinky1", "DEF-palm.04.R"),
("mixamorig:RightHandPinky2", "DEF-f_pinky.01.R"),
("mixamorig:RightHandPinky3", "DEF-f_pinky.02.R"),
("mixamorig:RightHandPinky4", "DEF-f_pinky.03.R"),
// -- thumb
("mixamorig:RightHandThumb1", "DEF-thumb.01.R"),
("mixamorig:RightHandThumb2", "DEF-thumb.02.R"),
("mixamorig:RightHandThumb3", "DEF-thumb.03.R"),
// Right Leg
("mixamorig:RightUpLeg", "DEF-thigh.R"),
("mixamorig:RightLeg", "DEF-shin.R"),
("mixamorig:RightFoot", "DEF-foot.R"),
("mixamorig:RightToeBase", "DEF-toe.R"),
// Left arm
("mixamorig:LeftShoulder", "DEF-shoulder.L"),
("mixamorig:LeftArm", "DEF-upper_arm.L"),
("mixamorig:LeftForeArm", "DEF-forearm.L"),
("mixamorig:LeftHand", "DEF-hand.L"),
// Left fingers
// -- index
("mixamorig:LeftHandIndex1", "DEF-palm.01.L"),
("mixamorig:LeftHandIndex2", "DEF-f_index.01.L"),
("mixamorig:LeftHandIndex3", "DEF-f_index.02.L"),
("mixamorig:LeftHandIndex4", "DEF-f_index.03.L"),
// -- middle
("mixamorig:LeftHandMiddle1", "DEF-palm.02.L"),
("mixamorig:LeftHandMiddle2", "DEF-f_middle.01.L"),
("mixamorig:LeftHandMiddle3", "DEF-f_middle.02.L"),
("mixamorig:LeftHandMiddle4", "DEF-f_middle.03.L"),
// -- ring
("mixamorig:LeftHandRing1", "DEF-palm.03.L"),
("mixamorig:LeftHandRing2", "DEF-f_ring.01.L"),
("mixamorig:LeftHandRing3", "DEF-f_ring.02.L"),
("mixamorig:LeftHandRing4", "DEF-f_ring.03.L"),
// -- pinky
("mixamorig:LeftHandPinky1", "DEF-palm.04.L"),
("mixamorig:LeftHandPinky2", "DEF-f_pinky.01.L"),
("mixamorig:LeftHandPinky3", "DEF-f_pinky.02.L"),
("mixamorig:LeftHandPinky4", "DEF-f_pinky.03.L"),
// -- thumb
("mixamorig:LeftHandThumb1", "DEF-thumb.01.L"),
("mixamorig:LeftHandThumb2", "DEF-thumb.02.L"),
("mixamorig:LeftHandThumb3", "DEF-thumb.03.L"),
// Left Leg
("mixamorig:LeftUpLeg", "DEF-thigh.L"),
("mixamorig:LeftLeg", "DEF-shin.L"),
("mixamorig:LeftFoot", "DEF-foot.L"),
("mixamorig:LeftToeBase", "DEF-toe.L"),
];
}
impl Default for RetargetMap {
fn default() -> Self {
Self {
src_to_tgt: Self::MOCAP_NET
.iter()
.chain(Self::MIXAMO.iter())
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect(),
}
}
}