Mixamo animation support (#37)

This PR adds support for Mixamo FBX imports (via backend conversion to
glTF/glB) and some miscellaneous improvements to the animation systems.

### User-facing improvements
* Retargeting math has been fixed to eliminate jittering issues
* Retargeter can now handle animated bone translations when present
  (generally this is only used for root motion)

### Housekeeping / Code improvements

* Animation-related components and system logic have been moved to a
  dedicated `anim` module and `AnimPlugin`
* Retargeting stuff has been moved from the `bvh` module to
  `anim::retargeting`
* Component-based solution for marking an entity as an animation
  retargeting source makes it easy to extend the retargeting functionality
  to new types of source animations

### Limitations

Currently the code assumes that there will be at most one animation
source at a time. That single animation (if present) will be retargeted
to all other skeletons in the scene. A component-based solution for
directing a specific target skeleton to use a specific source animation
is possible, but will require some additional consideration around the
implementation design for the read-only "viewer" modes.

### Test animations:

* Hip-Hop Dancing (Mixamo): ?mode=viewer&mixamo=mocap/hip-hop-dancing.gltf
* Shuffle (MocapNET): ?mode=viewer&bvh=mocap/shuffle.bvh
* Scott (MocapNET): ?mode=viewer&bvh=mocap/scott.bvh
* Scott 1 (MocapNET): ?mode=viewer&bvh=mocap/scott1.bvh
This commit is contained in:
Danny McGee
2024-02-09 13:21:16 -05:00
committed by GitHub
parent d1b4aab8ca
commit a5aeff7a3a
13 changed files with 4109 additions and 299 deletions
+141
View File
@@ -0,0 +1,141 @@
use bevy::{
math::Affine3A,
prelude::*,
render::{mesh::skinning::SkinnedMesh, view::NoFrustumCulling},
utils::{EntityHashMap, EntityHashSet},
};
use space_editor::space_prefab::editor_registry::EditorRegistryExt;
use crate::hierarchy;
pub use retargeting::*;
mod retargeting;
pub struct AnimPlugin;
impl Plugin for AnimPlugin {
fn build(&self, app: &mut App) {
app.register_type::<Skeleton>()
.editor_registry::<Skeleton>()
.register_type::<SkeletalJoint>()
.editor_registry::<SkeletalJoint>()
.register_type::<ProcessedSkinnedMesh>()
.editor_registry::<ProcessedSkinnedMesh>()
.register_type::<BindPoseLocal>()
.editor_registry::<BindPoseLocal>()
.register_type::<BindPoseGlobal>()
.editor_registry::<BindPoseGlobal>()
.register_type::<InvBindPose>()
.editor_registry::<InvBindPose>();
app.add_systems(Update, process_skinned_meshes);
app.add_plugins(retargeting::RetargetingPlugin);
}
}
#[derive(Component, Clone, Copy, Debug, Default, Reflect)]
#[reflect(Component)]
pub struct Skeleton;
#[derive(Component, Clone, Copy, Debug, Default, Reflect)]
#[reflect(Component)]
pub struct SkeletalJoint(pub Color);
#[derive(Component, Clone, Copy, Debug, Default, Reflect)]
#[reflect(Component)]
pub struct ProcessedSkinnedMesh;
#[derive(Component, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Component)]
pub struct BindPoseLocal(pub Transform);
#[derive(Component, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Component)]
pub struct BindPoseGlobal(pub Affine3A);
#[derive(Component, Clone, Copy, Debug, Default, Deref, DerefMut, Reflect)]
#[reflect(Component)]
pub struct InvBindPose(pub Affine3A);
fn process_skinned_meshes(
mut cmd: Commands,
q_skinned_meshes: Query<(Entity, &SkinnedMesh), Without<ProcessedSkinnedMesh>>,
mut q_xforms: ParamSet<(TransformHelper, Query<&Transform>)>,
q_parents: Query<Option<&Parent>>,
) {
if q_skinned_meshes.is_empty() {
return;
}
// Map of skinned-mesh parent entities to the set of joints bound to those
// skinned meshes. We use the skinned-mesh parent because individual skinned
// meshes in glTF format are often only parts of the whole "model," in which
// case the parts will be siblings in the hierarchy.
//
// I.e., considering a hierarchy like this:
// ```
// └ RootNode
// ├ Body [SkinnedMesh]
// ├ Head [SkinnedMesh]
// ├ Eyes [SkinnedMesh]
// └ Hips [Joint]
// ├ UpperLeg_L [Joint]
// │ └ ...
// ├ UpperLeg_R [Joint]
// │ └ ...
// └ Spine [Joint]
// └ ...
// ```
// The hashmap should contain an entry like this:
// ```
// {
// RootNode: { Hips, UpperLeg_L, UpperLeg_R, Spine, ... }
// }
// ```
// Building this hashmap before processing each joint avoids redundancy in
// the common case where the same joints are bound to multiple skinned meshes
let mut joint_collections = EntityHashMap::<Entity, EntityHashSet<Entity>>::default();
for (ent, skinned_mesh) in q_skinned_meshes.iter() {
cmd.entity(ent)
.insert(NoFrustumCulling)
.insert(ProcessedSkinnedMesh);
let Some(parent) = q_parents.get(ent).ok().flatten() else {
continue;
};
let parent_ent = **parent;
let joints = match joint_collections.get_mut(&parent_ent) {
Some(set) => set,
None => {
let (_, set) =
joint_collections.insert_unique_unchecked(parent_ent, Default::default());
set
}
};
joints.extend(skinned_mesh.joints.iter());
}
for joints in joint_collections.values() {
// Find the common ancestor of this joint collection and tag it as the
// skeleton
let joints = joints.iter().copied().collect::<Vec<_>>();
if let Some(root) = hierarchy::nearest_common_ancestor(&joints, &q_parents) {
cmd.entity(root).insert(Skeleton);
}
// Tag each joint and find its bind poses
for &joint in joints.iter() {
let bind_pose_global = q_xforms.p0().compute_global_transform(joint).unwrap();
let bind_pose_local = *q_xforms.p1().get(joint).unwrap();
cmd.entity(joint).insert((
SkeletalJoint(Color::BLACK),
BindPoseLocal(bind_pose_local),
BindPoseGlobal(bind_pose_global.affine()),
InvBindPose(bind_pose_global.affine().inverse()),
));
}
}
}