import { Color, Matrix4, type Object3D, Quaternion, Vector3, type WebXRManager, } from "three"; import { SplatMesh } from "./SplatMesh"; // Experimental WebXR hand tracking and movement const DEFAULT_MOVE_INERTIA = 0.5; const DEFAULT_ROTATE_INERTIA = 0.5; const TOUCH_BIAS = 0.0; export enum JointEnum { w = "wrist", t0 = "thumb-metacarpal", t1 = "thumb-phalanx-proximal", t2 = "thumb-phalanx-distal", t3 = "thumb-tip", i0 = "index-finger-metacarpal", i1 = "index-finger-phalanx-proximal", i2 = "index-finger-phalanx-intermediate", i3 = "index-finger-phalanx-distal", i4 = "index-finger-tip", m0 = "middle-finger-metacarpal", m1 = "middle-finger-phalanx-proximal", m2 = "middle-finger-phalanx-intermediate", m3 = "middle-finger-phalanx-distal", m4 = "middle-finger-tip", r0 = "ring-finger-metacarpal", r1 = "ring-finger-phalanx-proximal", r2 = "ring-finger-phalanx-intermediate", r3 = "ring-finger-phalanx-distal", r4 = "ring-finger-tip", p0 = "pinky-finger-metacarpal", p1 = "pinky-finger-phalanx-proximal", p2 = "pinky-finger-phalanx-intermediate", p3 = "pinky-finger-phalanx-distal", p4 = "pinky-finger-tip", } export type JointId = keyof typeof JointEnum; export const JOINT_IDS = Object.keys(JointEnum) as JointId[]; export const NUM_JOINTS = JOINT_IDS.length; export const JOINT_INDEX: { [key in JointId]: number } = { w: 0, t0: 1, t1: 2, t2: 3, t3: 4, i0: 5, i1: 6, i2: 7, i3: 8, i4: 9, m0: 10, m1: 11, m2: 12, m3: 13, m4: 14, r0: 15, r1: 16, r2: 17, r3: 18, r4: 19, p0: 20, p1: 21, p2: 22, p3: 23, p4: 24, }; export const JOINT_RADIUS: { [key in JointId]: number } = { w: 0.02, t0: 0.02, t1: 0.014, t2: 0.0115, t3: 0.0085, i0: 0.022, i1: 0.012, i2: 0.0085, i3: 0.0075, i4: 0.0065, m0: 0.021, m1: 0.012, m2: 0.008, m3: 0.0075, m4: 0.0065, r0: 0.019, r1: 0.011, r2: 0.0075, r3: 0.007, r4: 0.006, p0: 0.012, p1: 0.01, p2: 0.007, p3: 0.0065, p4: 0.0055, }; export const JOINT_SEGMENTS: JointId[][] = [ ["w", "t0", "t1", "t2", "t3"], ["w", "i0", "i1", "i2", "i3", "i4"], ["w", "m0", "m1", "m2", "m3", "m4"], ["w", "r0", "r1", "r2", "r3", "r4"], ["w", "p0", "p1", "p2", "p3", "p4"], ]; export const JOINT_SEGMENT_STEPS: number[][] = [ [8, 10, 8, 6], [8, 19, 14, 8, 6], [8, 19, 14, 8, 6], [8, 19, 14, 8, 6], [8, 19, 14, 8, 6], ]; export const JOINT_TIPS: JointId[] = ["t3", "i4", "m4", "r4", "p4"]; export const FINGER_TIPS: JointId[] = ["i4", "m4", "r4", "p4"]; export enum Hand { left = "left", right = "right", } export const HANDS = Object.keys(Hand) as Hand[]; export type Joint = { position: Vector3; quaternion: Quaternion; radius: number; }; export type HandJoints = { [key in JointId]?: Joint }; export type HandsJoints = { [key in Hand]?: HandJoints }; export class XrHands { hands: HandsJoints = {}; last: HandsJoints = {}; values: Record = {}; tests: Record = {}; lastTests: Record = {}; updated = false; update({ xr, xrFrame }: { xr: WebXRManager; xrFrame: XRFrame }) { const xrSession = xr.getSession(); if (!xrSession) { return; } const referenceSpace = xr.getReferenceSpace(); if (!referenceSpace) { return; } if (!xrFrame.getJointPose) { return; } this.last = this.hands; this.lastTests = this.tests; this.hands = {}; this.values = {}; this.tests = {}; for (const inputSource of xrSession.inputSources) { if (!inputSource.hand) { continue; } const hand = inputSource.handedness as Hand; this.hands[hand] = {}; // Iterate over JointId for (const jointId of JOINT_IDS) { const jointSpace = inputSource.hand.get(JointEnum[jointId]); if (jointSpace) { const jointPose = xrFrame.getJointPose(jointSpace, referenceSpace); if (jointPose) { const { position, orientation } = jointPose.transform; this.hands[hand][jointId] = { position: new Vector3(position.x, position.y, position.z), quaternion: new Quaternion( orientation.x, orientation.y, orientation.z, orientation.w, ), radius: jointPose.radius || 0.001, }; } } } } for (const hand of HANDS) { for (const { key, value } of [ { key: `${hand}AllTips`, value: this.allTipsTouching(hand) }, { key: `${hand}IndexThumb`, value: this.touching(hand, "i4", hand, "t3"), }, { key: `${hand}MiddleThumb`, value: this.touching(hand, "m4", hand, "t3"), }, { key: `${hand}RingThumb`, value: this.touching(hand, "r4", hand, "t3"), }, { key: `${hand}PinkyThumb`, value: this.touching(hand, "p4", hand, "t3"), }, { key: `${hand}TriTips`, value: this.triTipsTouching(hand) }, ]) { this.values[key] = value; this.tests[key] = value === 1.0 ? true : value === 0.0 ? false : (this.lastTests[key] ?? false); } } } makeGhostMesh(): SplatMesh { const center = new Vector3(); const scales = new Vector3(0.01, 0.01, 0.01); const quaternion = new Quaternion(0, 0, 0, 1); const color = new Color(1, 1, 1); const CYCLE = Math.PI * 3; const WHITE = new Color(1, 1, 1); let opacity = 1.0; const mesh = new SplatMesh({ onFrame: () => { let splatIndex = 0; for (const handedness of HANDS) { const xrHand = this.hands[handedness]; for (const [index, segment] of JOINT_SEGMENTS.entries()) { for (let i = 1; i < segment.length; ++i) { const segmentSplats = JOINT_SEGMENT_STEPS[index][i - 1] * 2; const lastSegment = i + 1 === segment.length; const jointA = xrHand?.[segment[i - 1]]; const jointB = xrHand?.[segment[i]]; for (let j = 0; j < segmentSplats; ++j) { const t = (j + 0.5) / segmentSplats; opacity = 0.0; if (jointA && jointB) { center.copy(jointA.position).lerp(jointB.position, t); quaternion .copy(jointA.quaternion) .slerp(jointB.quaternion, t); const radiusA = JOINT_RADIUS[segment[i - 1]]; const radiusB = JOINT_RADIUS[segment[i]]; let radius = (1 - t) * radiusA + t * radiusB; if (lastSegment && t > 0.8) { // Round out finger tips radius *= Math.sqrt(1 - ((t - 0.8) / 0.2) ** 2); } scales.set(0.65 * radius, 0.5 * radius, 0.003); color.set( 0.55 + 0.45 * Math.sin(center.x * CYCLE), 0.55 + 0.45 * Math.sin(center.y * CYCLE), 0.55 + 0.45 * Math.sin(center.z * CYCLE), ); if (handedness === "right") { color.set(1 - color.r, 1 - color.g, 1 - color.b); } opacity = 0.75; } mesh.packedSplats.setSplat( splatIndex, center, scales, quaternion, opacity, color, ); splatIndex += 1; } } } } mesh.packedSplats.numSplats = splatIndex; mesh.packedSplats.needsUpdate = true; mesh.numSplats = splatIndex; mesh.updateVersion(); }, }); return mesh; } distance( handA: Hand, jointA: JointId, handB: Hand, jointB: JointId, last = false, ): number { const hA = last ? this.last[handA] : this.hands[handA]; const hB = last ? this.last[handB] : this.hands[handB]; const jA = hA?.[jointA]; const jB = hB?.[jointB]; if (!jA || !jB) { return Number.POSITIVE_INFINITY; } return jA.position.distanceTo(jB.position); } separation( handA: Hand, jointA: JointId, handB: Hand, jointB: JointId, last = false, ): number { const d = this.distance(handA, jointA, handB, jointB, last); if (d === Number.POSITIVE_INFINITY) { return Number.POSITIVE_INFINITY; } return d - JOINT_RADIUS[jointA] - JOINT_RADIUS[jointB]; } touching( handA: Hand, jointA: JointId, handB: Hand, jointB: JointId, last = false, ): number { const d = this.separation(handA, jointA, handB, jointB, last); if (d === Number.POSITIVE_INFINITY) { return Number.POSITIVE_INFINITY; } return 1 - Math.max(0, Math.min(1, d / 0.01 - TOUCH_BIAS)); } allTipsTouching(hand: Hand, last = false): number { return Math.min( this.touching(hand, "t3", hand, "i4", last), this.touching(hand, "i4", hand, "m4", last), this.touching(hand, "m4", hand, "r4", last), this.touching(hand, "r4", hand, "p4", last), // this.touching(hand, "p4", hand, "t3", last), ); } triTipsTouching(hand: Hand, last = false): number { return Math.min( this.touching(hand, "t3", hand, "i4", last), this.touching(hand, "i4", hand, "m4", last), this.touching(hand, "m4", hand, "t3", last), ); } } export class HandMovement { xrHands: XrHands; control: Object3D; moveInertia: number; rotateInertia: number; lastGrip: { [key in Hand]?: Vector3 } = {}; lastPivot: Vector3 = new Vector3(); rotateVelocity = 0; velocity: Vector3 = new Vector3(); constructor({ xrHands, control, moveInertia, rotateInertia, }: { xrHands: XrHands; control: Object3D; moveInertia?: number; rotateInertia?: number; }) { this.xrHands = xrHands; this.control = control; this.moveInertia = moveInertia ?? DEFAULT_MOVE_INERTIA; this.rotateInertia = rotateInertia ?? DEFAULT_ROTATE_INERTIA; } update(deltaTime: number) { const grip: { [key in Hand]?: Vector3 } = {}; for (const handedness of HANDS) { const hand = this.xrHands.hands[handedness]; if (hand && this.xrHands.tests[`${handedness}MiddleThumb`]) { grip[handedness] = new Vector3() .add(hand.t3?.position ?? new Vector3()) .add(hand.i4?.position ?? new Vector3()) .add(hand.m4?.position ?? new Vector3()) .add(hand.r4?.position ?? new Vector3()) .add(hand.p4?.position ?? new Vector3()) .multiplyScalar(1 / 5); } } if (grip.left && grip.right && this.lastGrip.left && this.lastGrip.right) { const mid = grip.left.clone().add(grip.right).multiplyScalar(0.5); const lastMid = this.lastGrip.left .clone() .add(this.lastGrip.right) .multiplyScalar(0.5); this.lastPivot = mid; const delta = mid.clone().applyMatrix4(this.control.matrix); delta.sub(lastMid.clone().applyMatrix4(this.control.matrix)); delta.multiplyScalar(1 / deltaTime); this.velocity.lerp(delta, 1 - Math.exp(-20 * deltaTime)); const angle = Math.atan2(grip.left.z - mid.z, grip.left.x - mid.x); const lastAngle = Math.atan2( this.lastGrip.left.z - lastMid.z, this.lastGrip.left.x - lastMid.x, ); // Find closest rotation over circle between angle and lastAngle let closestAngle = angle - lastAngle; if (closestAngle > Math.PI) { closestAngle -= Math.PI * 2; } else if (closestAngle < -Math.PI) { closestAngle += Math.PI * 2; } const rotateVelocity = closestAngle / deltaTime; const blend = Math.exp(-20 * deltaTime); this.rotateVelocity = this.rotateVelocity * blend + rotateVelocity * (1 - blend); } else { this.rotateVelocity *= Math.exp(-deltaTime / this.rotateInertia); if (grip.left && this.lastGrip.left) { const delta = grip.left.clone().applyMatrix4(this.control.matrix); delta.sub(this.lastGrip.left.clone().applyMatrix4(this.control.matrix)); delta.multiplyScalar(1 / deltaTime); this.velocity.lerp(delta, 1 - Math.exp(-20 * deltaTime)); } else if (grip.right && this.lastGrip.right) { const delta = grip.right.clone().applyMatrix4(this.control.matrix); delta.sub( this.lastGrip.right.clone().applyMatrix4(this.control.matrix), ); delta.multiplyScalar(1 / deltaTime); this.velocity.lerp(delta, 1 - Math.exp(-20 * deltaTime)); } else { this.velocity.multiplyScalar(Math.exp(-deltaTime / this.moveInertia)); } } const negPivot = this.lastPivot.clone().negate(); const rotate = new Matrix4() .makeTranslation(negPivot) .premultiply(new Matrix4().makeRotationY(this.rotateVelocity * deltaTime)) .premultiply(new Matrix4().makeTranslation(this.lastPivot)); this.control.matrix.multiply(rotate); this.control.matrix.decompose( this.control.position, this.control.quaternion, this.control.scale, ); this.control.updateMatrixWorld(true); this.control.position.sub(this.velocity.clone().multiplyScalar(deltaTime)); this.lastGrip = grip; } }