Initial commit

src/ForgeRenderer.ts

@@ -0,0 +1,856 @@
+import * as THREE from "three";
+
+import { ForgeViewpoint, type ForgeViewpointOptions } from "./ForgeViewpoint";
+import { PackedSplats } from "./PackedSplats";
+import { RgbaArray } from "./RgbaArray";
+import { type GeneratorMapping, SplatAccumulator } from "./SplatAccumulator";
+import { SplatEdit } from "./SplatEdit";
+import { SplatGenerator, SplatModifier } from "./SplatGenerator";
+import { SplatGeometry } from "./SplatGeometry";
+import { SplatMesh } from "./SplatMesh";
+import {
+  DynoVec3,
+  DynoVec4,
+  Gsplat,
+  TPackedSplats,
+  dynoBlock,
+  readPackedSplat,
+  transformGsplat,
+} from "./dyno";
+import { getShaders } from "./shaders";
+import {
+  averagePositions,
+  averageQuaternions,
+  cloneClock,
+  withinCoorientDist,
+} from "./utils";
+
+// ForgeRenderer aggregates splats from multiple generators into a single
+// accumulated collection per frame. In normal operation we only need a
+// maximum of 3 accumulators: One currently being viewed, one currently
+// being sorted, and one more for generating the next frame. Accumulators
+// must be "released" by each viewpoint using it, so in unusual cases
+// such as slow render-outs, we may want to allow more than 3 so the
+// pipeline can continue generating new frames, but we limit to a maximum
+// of 5 to avoid excessive memory usage.
+const MAX_ACCUMULATORS = 5;
+
+// Scene.onBeforeRender monkey-patch to
+// inject a ForgeRenderer into a scene with SplatMeshes if there isn't
+// one already. Restore original Scene.onBeforeRenderer and Scene.add when done.
+let hasSplatMesh = false;
+let hasForgeRenderer = false;
+
+let forgeRendererInstance: ForgeRenderer;
+
+const sceneAdd = THREE.Scene.prototype.add;
+THREE.Scene.prototype.add = function (object) {
+  hasSplatMesh = hasSplatMesh || object instanceof SplatMesh;
+  hasForgeRenderer = hasForgeRenderer || object instanceof ForgeRenderer;
+  sceneAdd.call(this, object);
+  return this;
+};
+
+const sceneOnBeforeRender = THREE.Scene.prototype.onBeforeRender;
+THREE.Scene.prototype.onBeforeRender = function (renderer) {
+  if (!hasSplatMesh) {
+    return;
+  }
+  if (!hasForgeRenderer) {
+    const forge = forgeRendererInstance || new ForgeRenderer({ renderer });
+    this.add(forge);
+  }
+  THREE.Scene.prototype.onBeforeRender = sceneOnBeforeRender;
+  THREE.Scene.prototype.add = sceneAdd;
+};
+
+export type ForgeRendererOptions = {
+  // Pass in your THREE.WebGLRenderer instance so Forge can perform work
+  // outside the usual render loop. Should be created with antialias: false
+  // (default setting) as WebGL anti-aliasing doesn't improve Gaussian Splatting
+  // rendering and significantly reduces performance.
+  renderer: THREE.WebGLRenderer;
+  // Pass in a THREE.Clock to synchronize time-based effects across different
+  // systems. Alternatively, you can set the ForgeRenderer properties time and
+  // deltaTime directly. (default: new THREE.Clock)
+  clock?: THREE.Clock;
+  // Controls whether to check and automatically update Gsplat collection after
+  // each frame render. (default: true)
+  autoUpdate?: boolean;
+  // Controls whether to update the Gsplats before or after rendering. For WebXR
+  // this must be false in order to complete rendering as soon as possible.
+  // (default: false)
+  preUpdate?: boolean;
+  // Distance threshold for ForgeRenderer movement triggering a Gsplat update at
+  // the new origin. (default: 1.0)
+  originDistance?: number;
+  // Maximum standard deviations from the center to render Gaussians. Values
+  // Math.sqrt(5)..Math.sqrt(8) produce good results and can be tweaked for
+  // performance. (default: Math.sqrt(8))
+  maxStdDev?: number;
+  // Enable 2D Gaussian splatting rendering ability. When this mode is enabled,
+  // any scale x/y/z component that is exactly 0 (minimum quantized value) results
+  // in the other two non-0 axis being interpreted as an oriented 2D Gaussian Splat,
+  // rather instead of the usual projected 3DGS Z-slice. When reading PLY files,
+  // scale values less than e^-20 will be interpreted as 0. (default: true)
+  enable2DGS?: boolean;
+  // Scalar value to add to 2D splat covariance diagonal, effectively blurring +
+  // enlarging splats. In scenes trained without the Gsplat anti-aliasing tweak
+  // this value was typically 0.3, but with anti-aliasing it is 0.0 (default: 0.0)
+  preBlurAmount?: number;
+  // Scalar value to add to 2D splat covarianve diagonal, with opacity adjustment
+  // to correctly account for "blurring" when anti-aliasing. Typically 0.3
+  // (equivalent to approx 0.5 pixel radius) in scenes trained with anti-aliasing.
+  blurAmount?: number;
+  // Modulate Gaussian kernel falloff. 0 means "no falloff, flat shading",
+  // while 1 is the normal Gaussian kernel. (default: 1.0)
+  falloff?: number;
+  // X/Y clipping boundary factor for Gsplat centers against view frustum.
+  // 1.0 clips any centers that are exactly out of bounds, while 1.4 clips
+  // centers that are 40% beyond the bounds. (default: 1.4)
+  clipXY?: number;
+  // Configures the ForgeViewpointOptions for the default ForgeViewpoint
+  // associated with this ForgeRenderer. Notable option: sortRadial (sort by
+  // radial distance or Z-depth)
+  view?: ForgeViewpointOptions;
+};
+
+export class ForgeRenderer extends THREE.Mesh {
+  renderer: THREE.WebGLRenderer;
+  material: THREE.ShaderMaterial;
+  uniforms: ReturnType<typeof ForgeRenderer.makeUniforms>;
+
+  autoUpdate: boolean;
+  preUpdate: boolean;
+  originDistance: number;
+  maxStdDev: number;
+  enable2DGS: boolean;
+  preBlurAmount: number;
+  blurAmount: number;
+  falloff: number;
+  clipXY: number;
+
+  time?: number;
+  deltaTime?: number;
+  clock: THREE.Clock;
+
+  // Latest Gsplat collection being displayed
+  active: SplatAccumulator;
+  // Free list of accumulators for reuse
+  private freeAccumulators: SplatAccumulator[];
+  // Total number of accumulators currently allocated
+  private accumulatorCount: number;
+  // Default ForgeViewpoint used for rendering to the canvas
+  defaultView: ForgeViewpoint;
+  // List of ForgeViewpoints with autoUpdate enabled
+  autoViewpoints: ForgeViewpoint[] = [];
+
+  // Dynos used to transform Gsplats to the accumulator coordinate system
+  private rotateToAccumulator = new DynoVec4({ value: new THREE.Quaternion() });
+  private translateToAccumulator = new DynoVec3({ value: new THREE.Vector3() });
+  private modifier: SplatModifier;
+
+  // Last rendered frame number so we know when we're rendering a new frame
+  private lastFrame = -1;
+  // Last update timestamp to compute deltaTime
+  private lastUpdateTime: number | null = null;
+  // List of cameras used for the current viewpoint (for WebXR)
+  private defaultCameras: THREE.Matrix4[] = [];
+
+  // Should be set to the defaultView, but can be temporarily changed to another
+  // viewpoint using prepareViewpoint() for rendering from a different viewpoint.
+  viewpoint: ForgeViewpoint;
+
+  // Holds data needed to perform a scheduled Gsplat update.
+  private pendingUpdate: {
+    scene: THREE.Scene;
+    originToWorld: THREE.Matrix4;
+  } | null = null;
+
+  // Internal ForgeViewpoint used for environment map rendering.
+  private envViewpoint: ForgeViewpoint | null = null;
+
+  // Data and buffers used for environment map rendering
+  private static cubeRender: {
+    target: THREE.WebGLCubeRenderTarget;
+    camera: THREE.CubeCamera;
+    near: number;
+    far: number;
+  } | null = null;
+  private static pmrem: THREE.PMREMGenerator | null = null;
+
+  constructor(options: ForgeRendererOptions) {
+    const uniforms = ForgeRenderer.makeUniforms();
+    const shaders = getShaders();
+    const material = new THREE.ShaderMaterial({
+      glslVersion: THREE.GLSL3,
+      vertexShader: shaders.splatVertex,
+      fragmentShader: shaders.splatFragment,
+      uniforms,
+      transparent: true,
+      blending: THREE.NormalBlending,
+      depthTest: true,
+      depthWrite: false,
+      side: THREE.DoubleSide,
+    });
+
+    super(EMPTY_GEOMETRY, material);
+    // Disable frustum culling because we want to always draw them all
+    // and cull Gsplats individually in the shader
+    this.frustumCulled = false;
+
+    this.renderer = options.renderer;
+    this.material = material;
+    this.uniforms = uniforms;
+
+    // Create a Gsplat modifier that takes the output of any SplatGenerator
+    // and transforms them into the accumulator's coordinate system
+    const modifier = dynoBlock(
+      { gsplat: Gsplat },
+      { gsplat: Gsplat },
+      ({ gsplat }) => {
+        if (!gsplat) {
+          throw new Error("gsplat not defined");
+        }
+        gsplat = transformGsplat(gsplat, {
+          rotate: this.rotateToAccumulator,
+          translate: this.translateToAccumulator,
+        });
+        return { gsplat };
+      },
+    );
+    this.modifier = new SplatModifier(modifier);
+
+    this.autoUpdate = options.autoUpdate ?? true;
+    this.preUpdate = options.preUpdate ?? false;
+    this.originDistance = options.originDistance ?? 1;
+    this.maxStdDev = options.maxStdDev ?? Math.sqrt(8.0);
+    this.enable2DGS = options.enable2DGS ?? true;
+    this.preBlurAmount = options.preBlurAmount ?? 0.0;
+    this.blurAmount = options.blurAmount ?? 0.3;
+    this.falloff = options.falloff ?? 1.0;
+    this.clipXY = options.clipXY ?? 1.4;
+
+    this.active = new SplatAccumulator();
+    this.accumulatorCount = 1;
+    this.freeAccumulators = [];
+    // Start with the minimum of 2 total accumulators
+    for (let count = 0; count < 1; ++count) {
+      this.freeAccumulators.push(new SplatAccumulator());
+      this.accumulatorCount += 1;
+    }
+
+    // Create a default ForgeViewpoint that is used when we call render()
+    // on the scene and has the sorted Gsplat collection from that viewpoint.
+    this.defaultView = new ForgeViewpoint({
+      ...options.view,
+      autoUpdate: true,
+      forge: this,
+    });
+    this.viewpoint = this.defaultView;
+    this.prepareViewpoint(this.viewpoint);
+
+    this.clock = options.clock ? cloneClock(options.clock) : new THREE.Clock();
+
+    forgeRendererInstance = this;
+  }
+
+  static makeUniforms() {
+    // Create uniforms used for Gsplat vertex and fragment shaders
+    const uniforms = {
+      // Size of render viewport in pixels
+      renderSize: { value: new THREE.Vector2() },
+      // Total number of Gsplats in packedSplats to render
+      numSplats: { value: 0 },
+      // SplatAccumulator to view transformation quaternion
+      renderToViewQuat: { value: new THREE.Quaternion() },
+      // SplatAccumulator to view transformation translation
+      renderToViewPos: { value: new THREE.Vector3() },
+      // Maximum distance (in stddevs) from Gsplat center to render
+      maxStdDev: { value: 1.0 },
+      // Enable interpreting 0-thickness Gsplats as 2DGS
+      enable2DGS: { value: false },
+      // Add to projected 2D splat covariance diagonal (thickens and brightens)
+      preBlurAmount: { value: 0.0 },
+      // Add to 2D splat covariance diagonal and adjust opacity (anti-aliasing)
+      blurAmount: { value: 0.3 },
+      // Modulate Gaussian kernal falloff. 0 means "no falloff, flat shading",
+      // 1 is normal e^-x^2 falloff.
+      falloff: { value: 1.0 },
+      // Clip Gsplats that are clipXY times beyond the +-1 frustum bounds
+      clipXY: { value: 1.4 },
+      // Gsplat collection to render
+      packedSplats: { type: "t", value: PackedSplats.getEmpty() },
+      // Time in seconds for time-based effects
+      time: { value: 0 },
+      // Delta time in seconds since last frame
+      deltaTime: { value: 0 },
+      // Whether to encode Gsplat with linear RGB (for environment mapping)
+      encodeLinear: { value: false },
+      // Debug flag that alternates each frame
+      debugFlag: { value: false },
+    };
+    return uniforms;
+  }
+
+  private canAllocAccumulator(): boolean {
+    // Returns true if can allocate an accumulator immediately
+    return (
+      this.freeAccumulators.length > 0 ||
+      this.accumulatorCount < MAX_ACCUMULATORS
+    );
+  }
+
+  private maybeAllocAccumulator(): SplatAccumulator | null {
+    // Allocate an accumulator immediately if possible, else return null
+    let accumulator = this.freeAccumulators.pop();
+    if (accumulator === undefined) {
+      if (this.accumulatorCount >= MAX_ACCUMULATORS) {
+        return null;
+      }
+      accumulator = new SplatAccumulator();
+      this.accumulatorCount += 1;
+    }
+    accumulator.refCount = 1;
+    return accumulator;
+  }
+
+  releaseAccumulator(accumulator: SplatAccumulator) {
+    // Decrement reference count and recycle if no longer in use
+    accumulator.refCount -= 1;
+    if (accumulator.refCount === 0) {
+      this.freeAccumulators.push(accumulator);
+    }
+  }
+
+  newViewpoint(options: ForgeViewpointOptions) {
+    // Create a new ForgeViewpoint for this ForgeRenderer.
+    // Note that every ForgeRenderer has an initial forge.defaultView: ForgeViewpoint
+    // from construction, which is used for the default canvas render loop.
+    // Calling this method allows you to create additional viewpoints, which can be
+    // updated automatically each frame (performing Gsplat sorting every time there
+    // is an update), or updated on-demand for controlled rendering for video render
+    // or similar applications.
+    return new ForgeViewpoint({ ...options, forge: this });
+  }
+
+  onBeforeRender(
+    renderer: THREE.WebGLRenderer,
+    scene: THREE.Scene,
+    camera: THREE.Camera,
+  ) {
+    // Called by Three.js before rendering this ForgeRenderer.
+    // At this point we can't modify the geometry or material, all these must
+    // be set in the scene already before this is called. Update the uniforms
+    // to render the Gsplats from the current active viewpoint.
+    const time = this.time ?? this.clock.getElapsedTime();
+    const deltaTime = time - (this.viewpoint.lastTime ?? time);
+    this.viewpoint.lastTime = time;
+
+    const frame = renderer.info.render.frame;
+    const isNewFrame = frame !== this.lastFrame;
+    this.lastFrame = frame;
+
+    const viewpoint = this.viewpoint;
+    if (viewpoint === this.defaultView) {
+      // When rendering is triggered on the default viewpoint,
+      // perform automatic updates.
+      if (isNewFrame) {
+        if (!renderer.xr.isPresenting) {
+          // Non-WebXR mode, just a single camera
+          this.defaultView.viewToWorld = camera.matrixWorld.clone();
+          this.defaultCameras = [this.defaultView.viewToWorld];
+        } else {
+          // In WebXR mode we are called multiple times, once for each eye,
+          // so use their average to compute the sort center.
+          const cameras = renderer.xr.getCamera().cameras;
+          this.defaultCameras = [
+            cameras[0].matrixWorld,
+            cameras[1].matrixWorld,
+          ];
+          this.defaultView.viewToWorld =
+            averageOriginToWorlds(this.defaultCameras) ?? new THREE.Matrix4();
+        }
+      }
+
+      if (this.autoUpdate) {
+        this.update({ scene, viewToWorld: this.defaultView.viewToWorld });
+      }
+    }
+
+    // Update uniforms for rendering
+
+    if (isNewFrame) {
+      // Keep these uniforms the same for both eyes if in WebXR
+      this.uniforms.time.value = time;
+      this.uniforms.deltaTime.value = deltaTime;
+      // Alternating debug flag that can aid in visual debugging
+      this.uniforms.debugFlag.value = (performance.now() / 1000.0) % 2.0 < 1.0;
+    }
+
+    if (viewpoint.target) {
+      // Rendering to a texture target, so its dimensions
+      this.uniforms.renderSize.value.set(
+        viewpoint.target.width,
+        viewpoint.target.height,
+      );
+    } else {
+      // Rendering to the canvas or WebXR
+      const renderSize = renderer.getSize(this.uniforms.renderSize.value);
+      if (renderSize.x === 1 && renderSize.y === 1) {
+        // WebXR mode on Apple Vision Pro returns 1x1 when presenting.
+        // Use a different means to figure out the render size.
+        const baseLayer = renderer.xr.getSession()?.renderState.baseLayer;
+        if (baseLayer) {
+          renderSize.x = baseLayer.framebufferWidth;
+          renderSize.y = baseLayer.framebufferHeight;
+        }
+      }
+    }
+
+    // Update uniforms from instance properties
+    this.uniforms.encodeLinear.value = viewpoint.encodeLinear;
+    this.uniforms.maxStdDev.value = this.maxStdDev;
+    this.uniforms.enable2DGS.value = this.enable2DGS;
+    this.uniforms.preBlurAmount.value = this.preBlurAmount;
+    this.uniforms.blurAmount.value = this.blurAmount;
+    this.uniforms.falloff.value = this.falloff;
+    this.uniforms.clipXY.value = this.clipXY;
+
+    // Calculate the transform from the accumulator to the current camera
+    const accumToWorld =
+      viewpoint.display?.accumulator.toWorld ?? new THREE.Matrix4();
+    const worldToCamera = camera.matrixWorld.clone().invert();
+    const originToCamera = accumToWorld.clone().premultiply(worldToCamera);
+    originToCamera.decompose(
+      this.uniforms.renderToViewPos.value,
+      this.uniforms.renderToViewQuat.value,
+      new THREE.Vector3(),
+    );
+  }
+
+  // Update the uniforms for the given viewpoint.
+  // Note that the client expects to be able to call render() at any point
+  // to update the canvas, so we must switch the viewpoint back to
+  // defaultView when we're finished.
+  prepareViewpoint(viewpoint?: ForgeViewpoint) {
+    this.viewpoint = viewpoint ?? this.viewpoint;
+
+    if (this.viewpoint.display) {
+      const { accumulator, geometry } = this.viewpoint.display;
+      this.uniforms.numSplats.value = accumulator.splats.numSplats;
+      this.uniforms.packedSplats.value = accumulator.splats.getTexture();
+      this.geometry = geometry;
+    } else {
+      // No Gsplats to display for this viewpoint yet
+      this.uniforms.numSplats.value = 0;
+      this.uniforms.packedSplats.value = PackedSplats.getEmpty();
+      this.geometry = EMPTY_GEOMETRY;
+    }
+  }
+
+  // If forge.autoUpdate is false then you must manually call
+  // forge.update({ scene }) to have the scene Gsplats be re-generated.
+  update({
+    scene,
+    viewToWorld,
+  }: { scene: THREE.Scene; viewToWorld?: THREE.Matrix4 }) {
+    // Compute the transform for the ForgeRenderer to use as origin
+    // for Gsplat generation and accumulation.
+    const originToWorld = this.matrixWorld.clone();
+    // Either do the update now, or in the next "tick" depending on preUpdate
+    if (this.preUpdate) {
+      this.updateInternal({ scene, originToWorld, viewToWorld });
+    } else {
+      // Pass the update parameters to be performed on the next tick
+      this.pendingUpdate = {
+        scene,
+        originToWorld,
+      };
+      setTimeout(() => {
+        if (this.pendingUpdate) {
+          const { scene, originToWorld } = this.pendingUpdate;
+          this.pendingUpdate = null;
+          this.updateInternal({ scene, originToWorld, viewToWorld });
+        }
+      }, 1);
+    }
+  }
+
+  updateInternal({
+    scene,
+    originToWorld,
+    viewToWorld,
+  }: {
+    scene: THREE.Scene;
+    originToWorld?: THREE.Matrix4;
+    viewToWorld?: THREE.Matrix4;
+  }): boolean {
+    if (!this.canAllocAccumulator()) {
+      // We don't have any available accumulators because of sorting
+      // back pressure, so don't update this time but try again next time.
+      // Signal update not attempted.
+      return false;
+    }
+
+    // Figure out the frame of the ForgeRenderer and current view
+    if (!originToWorld) {
+      originToWorld = this.active.toWorld;
+    }
+    viewToWorld = viewToWorld ?? originToWorld.clone();
+
+    const time = this.time ?? this.clock.getElapsedTime();
+    const deltaTime = time - (this.lastUpdateTime ?? time);
+    this.lastUpdateTime = time;
+
+    // Create a lookup from last active SplatGenerator to Gsplat mapping record
+    const activeMapping = this.active.mapping.reduce((map, record) => {
+      map.set(record.node, record);
+      return map;
+    }, new Map<SplatGenerator, GeneratorMapping>());
+
+    // Traverse visible scene to find all SplatGenerators and global SplatEdits
+    const { generators, globalEdits } = this.compileScene(scene);
+
+    // Let all SplatGenerators run their frameUpdate() method
+    for (const object of generators) {
+      object.frameUpdate?.({
+        object,
+        time,
+        deltaTime,
+        viewToWorld,
+        globalEdits,
+      });
+    }
+
+    // Make sure we have new version numbers for any objects with either
+    // generator or numSplats that have changed since the last frame.
+    for (const object of generators) {
+      const current = activeMapping.get(object);
+      const numSplats = object.generator ? object.numSplats : 0;
+      if (
+        object.generator !== current?.generator ||
+        numSplats !== current?.count
+      ) {
+        object.updateVersion();
+      }
+    }
+
+    // Check if the origin is within the maximum allowed distance before
+    // we trigger an update.
+    const originUpdate = !withinCoorientDist({
+      matrix1: originToWorld,
+      matrix2: this.active.toWorld,
+      maxDistance: this.originDistance,
+    });
+
+    // Check if we need any update at all
+    const needsUpdate =
+      originUpdate ||
+      generators.length !== activeMapping.size ||
+      generators.some((g) => g.version !== activeMapping.get(g)?.version);
+
+    let accumulator: SplatAccumulator | null = null;
+    if (needsUpdate) {
+      // Need to update, so allocate an accumulator
+      accumulator = this.maybeAllocAccumulator();
+      if (!accumulator) {
+        // This should never happen since we checked canAllocAccumulator() above
+        throw new Error("Unreachable");
+      }
+
+      // Compute whether our view frame has changed enough to warrant
+      // doing a Gsplat sort. Check both distance epsilon and
+      // minimum co-orientation (dot product of quaternions)
+      const originChanged = !withinCoorientDist({
+        matrix1: originToWorld,
+        matrix2: this.active.toWorld,
+        maxDistance: 0.00001,
+        minCoorient: 0.99999,
+      });
+
+      // Compute an ordering of the generators with the rough goal
+      // of keeping unchanging generators near the front to minimize
+      // the number of Gsplats that need to be regenerated.
+      const sorted = generators
+        .map((g, gIndex): [number, number, SplatGenerator] => {
+          const lastGen = activeMapping.get(g);
+          // If no previous generator, sort by absolute version, which will
+          // tend to push frequently updated generators toward the end
+          return !lastGen
+            ? [Number.POSITIVE_INFINITY, g.version, g]
+            : // Sort by version deltas then by previous ordering in the mapping,
+              // attempting to keep unchanging generators near the front
+              // to improve our chances of avoiding a re-generation.
+              [g.version - lastGen.version, lastGen.base, g];
+        })
+        .sort((a, b) => {
+          // Sort by first then second element of the tuple
+          if (a[0] !== b[0]) {
+            return a[0] - b[0];
+          }
+          return a[1] - b[1];
+        });
+      const genOrder = sorted.map(([_version, _seq, g]) => g);
+
+      // Compute sequential layout of generated splats
+      const splatCounts = genOrder.map((g) => g.numSplats);
+      const { maxSplats, mapping } =
+        accumulator.splats.generateMapping(splatCounts);
+      const newGenerators = genOrder.map((node, gIndex) => {
+        const { base, count } = mapping[gIndex];
+        return {
+          node,
+          generator: node.generator,
+          version: node.version,
+          base,
+          count,
+        };
+      });
+
+      // Compute worldToAccumulator origin transform (no scale)
+      originToWorld
+        .clone()
+        .invert()
+        .decompose(
+          this.translateToAccumulator.value,
+          this.rotateToAccumulator.value,
+          new THREE.Vector3(),
+        );
+
+      // Generate the Gsplats according to the mapping that need updating
+      accumulator.ensureGenerate(maxSplats);
+      const generated = accumulator.generateSplats({
+        renderer: this.renderer,
+        modifier: this.modifier,
+        generators: newGenerators,
+        forceUpdate: originChanged,
+        originToWorld,
+      });
+
+      // Update splat version number
+      accumulator.splatsVersion = this.active.splatsVersion + 1;
+      // Increment the mapping version if the mapping isn't identical to before
+      const hasCorrespondence = accumulator.hasCorrespondence(this.active);
+      accumulator.mappingVersion =
+        this.active.mappingVersion + (hasCorrespondence ? 0 : 1);
+
+      // Release the old accumulator and make the new one active
+      this.releaseAccumulator(this.active);
+      this.active = accumulator;
+      this.prepareViewpoint();
+    }
+
+    // Let the system breath before potentially triggering sorts
+    setTimeout(() => {
+      // Notify all auto-updating viewpoints that we updated the Gsplats
+      for (const view of this.autoViewpoints) {
+        view.autoPoll({ accumulator: accumulator ?? undefined });
+      }
+    }, 1);
+
+    // Signal update was performed
+    return true;
+  }
+
+  private compileScene(scene: THREE.Scene): {
+    generators: SplatGenerator[];
+    globalEdits: SplatEdit[];
+  } {
+    // Take a snapshot of the SplatGenerators and SplatEdits in the scene
+    // to be used to run an update.
+    const generators: SplatGenerator[] = [];
+    scene.traverse((node) => {
+      if (node instanceof SplatGenerator) {
+        generators.push(node);
+      }
+    });
+
+    const globalEdits = new Set<SplatEdit>();
+    scene.traverseVisible((node) => {
+      if (node instanceof SplatEdit) {
+        let ancestor = node.parent;
+        while (ancestor != null && !(ancestor instanceof SplatMesh)) {
+          ancestor = ancestor.parent;
+        }
+        if (ancestor == null) {
+          // Not part of a SplatMesh so it's a global edit
+          globalEdits.add(node);
+        }
+      }
+    });
+    return { generators, globalEdits: Array.from(globalEdits) };
+  }
+
+  // Renders out the scene to an environment map that can be used for
+  // Image-based lighting or similar applications. First optionally updates Gsplats,
+  // sorts them with respect to the provided worldCenter, renders 6 cube faces,
+  // then pre-filters them using THREE.PMREMGenerator and returns a THREE.Texture
+  // that can assigned directly to a THREE.MeshStandardMaterial.envMap property.
+  async renderEnvMap({
+    renderer,
+    scene,
+    worldCenter,
+    size = 256,
+    near = 0.1,
+    far = 1000,
+    hideObjects = [],
+    update = false,
+  }: {
+    renderer?: THREE.WebGLRenderer;
+    scene: THREE.Scene;
+    worldCenter: THREE.Vector3;
+    size?: number;
+    near?: number;
+    far?: number;
+    hideObjects?: THREE.Object3D[];
+    update?: boolean;
+  }): Promise<THREE.Texture> {
+    if (!this.envViewpoint) {
+      this.envViewpoint = this.newViewpoint({ sort360: true });
+    }
+    if (
+      !ForgeRenderer.cubeRender ||
+      ForgeRenderer.cubeRender.target.width !== size ||
+      ForgeRenderer.cubeRender.near !== near ||
+      ForgeRenderer.cubeRender.far !== far
+    ) {
+      if (ForgeRenderer.cubeRender) {
+        ForgeRenderer.cubeRender.target.dispose();
+      }
+      const target = new THREE.WebGLCubeRenderTarget(size, {
+        format: THREE.RGBAFormat,
+        generateMipmaps: true,
+        minFilter: THREE.LinearMipMapLinearFilter,
+      });
+      const camera = new THREE.CubeCamera(near, far, target);
+      ForgeRenderer.cubeRender = { target, camera, near, far };
+    }
+
+    if (!ForgeRenderer.pmrem) {
+      ForgeRenderer.pmrem = new THREE.PMREMGenerator(renderer ?? this.renderer);
+    }
+
+    // Prepare the viewpoint, sorting Gsplats for this view origin.
+    const viewToWorld = new THREE.Matrix4().setPosition(worldCenter);
+    await this.envViewpoint?.prepare({ scene, viewToWorld, update });
+
+    const { target, camera } = ForgeRenderer.cubeRender;
+    camera.position.copy(worldCenter);
+
+    // Save the visibility state of objects we want to hide before render
+    const objectVisibility = new Map<THREE.Object3D, boolean>();
+    for (const object of hideObjects) {
+      objectVisibility.set(object, object.visible);
+      object.visible = false;
+    }
+
+    // Update the CubeCamera, which performs 6 cube face renders
+    this.prepareViewpoint(this.envViewpoint);
+    camera.update(renderer ?? this.renderer, scene);
+
+    // Restore viewpoint to default and object visibility
+    this.prepareViewpoint(this.defaultView);
+    for (const [object, visible] of objectVisibility.entries()) {
+      object.visible = visible;
+    }
+
+    // Pre-filter the cube map using THREE.PMREMGenerator
+    return ForgeRenderer.pmrem?.fromCubemap(target.texture).texture;
+  }
+
+  // Utility function to recursively set the envMap property for any
+  // THREE.MeshStandardMaterial within the subtree of root.
+  recurseSetEnvMap(root: THREE.Object3D, envMap: THREE.Texture) {
+    root.traverse((node) => {
+      if (node instanceof THREE.Mesh) {
+        if (Array.isArray(node.material)) {
+          for (const material of node.material) {
+            if (material instanceof THREE.MeshStandardMaterial) {
+              material.envMap = envMap;
+            }
+          }
+        } else {
+          if (node.material instanceof THREE.MeshStandardMaterial) {
+            node.material.envMap = envMap;
+          }
+        }
+      }
+    });
+  }
+
+  // Utility function that helps extract the Gsplat RGBA values from a
+  // SplatGenerator, including the result of any real-time RGBA SDF edits applied
+  // to a SplatMesh. This effectively "bakes" any computed RGBA values, which can
+  // now be used as a pipeline input via SplatMesh.splatRgba to inject these
+  // baked values into the Gsplat data.
+  getRgba({
+    generator,
+    rgba,
+  }: { generator: SplatGenerator; rgba?: RgbaArray }): RgbaArray {
+    const mapping = this.active.mapping.find(({ node }) => node === generator);
+    if (!mapping) {
+      throw new Error("Generator not found");
+    }
+
+    rgba = rgba ?? new RgbaArray();
+    rgba.fromPackedSplats({
+      packedSplats: this.active.splats,
+      base: mapping.base,
+      count: mapping.count,
+      renderer: this.renderer,
+    });
+    return rgba;
+  }
+
+  // Utility function that builds on getRgba({ generator }) and additionally
+  // reads back the RGBA values to the CPU in a Uint8Array with packed RGBA
+  // in that byte order.
+  async readRgba({
+    generator,
+    rgba,
+  }: { generator: SplatGenerator; rgba?: RgbaArray }): Promise<Uint8Array> {
+    rgba = this.getRgba({ generator, rgba });
+    return rgba.read();
+  }
+}
+
+const EMPTY_GEOMETRY = new SplatGeometry(new Uint32Array(1), 0);
+
+const reorderSplats = dynoBlock(
+  { packedSplats: TPackedSplats, index: "int" },
+  { gsplat: Gsplat },
+  ({ packedSplats, index }) => {
+    if (!packedSplats || !index) {
+      throw new Error("Invalid input");
+    }
+    const gsplat = readPackedSplat(packedSplats, index);
+    return { gsplat };
+  },
+);
+
+function averageOriginToWorlds(
+  originToWorlds: THREE.Matrix4[],
+): THREE.Matrix4 | null {
+  if (originToWorlds.length === 0) {
+    return null;
+  }
+
+  const position = new THREE.Vector3();
+  const quaternion = new THREE.Quaternion();
+  const scale = new THREE.Vector3();
+
+  const positions: THREE.Vector3[] = [];
+  const quaternions: THREE.Quaternion[] = [];
+  for (const matrix of originToWorlds) {
+    matrix.decompose(position, quaternion, scale);
+    positions.push(position);
+    quaternions.push(quaternion);
+  }
+
+  return new THREE.Matrix4().compose(
+    averagePositions(positions),
+    averageQuaternions(quaternions),
+    new THREE.Vector3(1, 1, 1),
+  );
+}