h264enc: pay back single-pass VBR/CBR overshoot within a short clip (#3200)

The 1-pass rate controller corrects its qscale by the bits spent over the
target divided by one second of bits. A clip that starts over target (the
opening I frame, a model that is still learning) then settles at a constant
excess instead of paying it back: 2 s of a 1080p mandelbrot zoom with the
export defaults (High, Balanced, VBR 20 Mbit/s, max 30, two B-frames) came
out at +9.8% at 30 fps and +13.5% at 60 fps.

The correction horizon is now a quarter second of bits (still clamped to
0.5..2x). Measured on the same clips: +4.1% and +4.0%; an 8 s clip goes from
+3.4% to +0.6%; clips that were already on target or under are unchanged
within 1%. Single-pass CBR/VBR streams change, so the two pinned slice hashes
that use them are updated.

Tested with a new ffmpeg-generated 2 s 1080p test (skipped without ffmpeg)
asserting the bitrate lands within 6% of target at 30 and 60 fps; it failed
before.

Fixes #3200

crates/h264enc/src/ratecontrol.rs

@@ -199,7 +199,11 @@ impl RateControl {
                 let s = self.sum_s + s_here + ps;
                 let wanted = bpf * (self.frames as f64 + 1.0 + pn);
                 let mut rf = s / wanted;
-                let abr_buffer = bpf * self.fps.max(1.0);
+                // A quarter second of bits: with a longer horizon the correction is so weak that a short clip
+                // that starts over target never pays the excess back (+10..14% on 2 s of high motion at 1 s).
+                // Never shorter than one frame's budget: at a low frame rate the horizon would be a fraction of a
+                // frame and the correction would overshoot in the other direction every frame.
+                let abr_buffer = (bpf * self.fps.max(1.0) * 0.25).max(bpf);
                 let overflow = (1.0 + (self.total_bits - bpf * self.frames as f64) / abr_buffer).clamp(0.5, 2.0);
                 rf *= overflow;
                 let q = rf * cplx.powf(1.0 - QCOMP) * type_factor(t);

crates/h264enc/tests/it/main.rs

@@ -2,3 +2,4 @@ mod common;
 mod crf_level_and_note;
 mod ffmpeg_oracle;
 mod pinned_decisions;
+mod vbr_overshoot;

crates/h264enc/tests/it/pinned_decisions.rs

@@ -45,9 +45,9 @@ fn cfg(profile: Profile, preset: Preset, rate: RateControl) -> EncoderConfig {
 fn coded_slices_are_pinned() {
     let cases = [
         ("high balanced crf", cfg(Profile::High, Preset::Balanced, RateControl::Crf(23.0)), 0xecd1_35a9_3a46_f7d8u64),
-        ("high quality vbr", cfg(Profile::High, Preset::Quality, RateControl::Vbr { target_kbps: 800, max_kbps: 1200 }), 0x251e_c302_fd8a_ed38),
+        ("high quality vbr", cfg(Profile::High, Preset::Quality, RateControl::Vbr { target_kbps: 800, max_kbps: 1200 }), 0x2129_41dd_a9c5_f087),
         ("main speed qp", cfg(Profile::Main, Preset::Speed, RateControl::Qp(30)), 0xa123_355d_d580_4e7b),
-        ("baseline balanced cbr", cfg(Profile::Baseline, Preset::Balanced, RateControl::Cbr { kbps: 600 }), 0x13cb_e982_aee0_13f8),
+        ("baseline balanced cbr", cfg(Profile::Baseline, Preset::Balanced, RateControl::Cbr { kbps: 600 }), 0x21ce_6999_8c31_8b0a),
     ];
     let got: Vec<u64> = cases.iter().map(|(_, c, _)| slices_hash(c.clone(), 20)).collect();
     for ((name, _, _), got) in cases.iter().zip(&got) {

crates/h264enc/tests/it/vbr_overshoot.rs

@@ -0,0 +1,76 @@
+//! Single-pass VBR must land near its target bitrate even on a short, high-motion clip.
+
+use crate::common::{self, Yuv};
+use filmcraft_h264enc::*;
+use std::io::Read;
+use std::process::{Command, Stdio};
+
+/// Encode two seconds of a zooming fractal at 1080p, streaming the frames from ffmpeg into the encoder, and return
+/// the resulting bitrate; `None` when there is no ffmpeg.
+fn encode_clip(fps: usize) -> Option<f64> {
+    let (w, h) = (1920usize, 1080usize);
+    let ffmpeg = common::ffmpeg()?;
+    let mut child = Command::new(ffmpeg)
+        .args(["-v", "error", "-f", "lavfi", "-i"])
+        .arg(format!("mandelbrot=size={w}x{h}:rate={fps}"))
+        .args(["-frames:v", &(2 * fps).to_string(), "-f", "rawvideo", "-pix_fmt", "yuv420p", "-"])
+        .stdout(Stdio::piped())
+        .spawn()
+        .expect("start ffmpeg");
+    let mut stdout = child.stdout.take().expect("ffmpeg stdout");
+    // the export defaults: High, Balanced, VBR 20 Mbit/s with a 30 Mbit/s peak, two B-frames, a slice per four MB rows
+    let mut c = EncoderConfig::new(w as u32, h as u32, fps as u32, 1);
+    c.profile = Profile::High;
+    c.preset = Preset::Balanced;
+    c.bframes = 2;
+    c.slices = 17;
+    c.rate = RateControl::Vbr { target_kbps: 20_000, max_kbps: 30_000 };
+    let mut enc = Encoder::new(c).unwrap();
+    let mut buf = vec![0u8; w * h * 3 / 2];
+    let (mut bytes, mut n) = (0usize, 0usize);
+    while stdout.read_exact(&mut buf).is_ok() {
+        let f = Yuv { w, h, y: buf[..w * h].to_vec(), u: buf[w * h..w * h * 5 / 4].to_vec(), v: buf[w * h * 5 / 4..].to_vec() };
+        bytes += enc.encode(&f.frame(), n as i64).unwrap().iter().map(|p| p.data.len()).sum::<usize>();
+        n += 1;
+    }
+    let status = child.wait().expect("wait for ffmpeg");
+    assert!(status.success(), "ffmpeg exited with {status}");
+    assert_eq!(n, 2 * fps, "ffmpeg produced {n} frames");
+    bytes += enc.flush().iter().map(|p| p.data.len()).sum::<usize>();
+    Some(bytes as f64 * 8.0 / (n as f64 / fps as f64) / 1000.0)
+}
+
+#[test]
+fn default_vbr_lands_near_its_target_on_a_short_high_motion_clip() {
+    for fps in [30usize, 60] {
+        let Some(kbps) = encode_clip(fps) else { return };
+        // +9.8% (30 fps) and +13.5% (60 fps) before the rate controller's correction horizon was shortened
+        assert!((kbps / 20_000.0 - 1.0).abs() <= 0.06, "{fps} fps: {kbps:.0} kbit/s against a 20000 kbit/s target");
+    }
+}
+
+/// Noisy blocks whose detail changes from frame to frame, so each frame costs a different number of bits.
+fn busy_frame(w: usize, h: usize, i: usize) -> Yuv {
+    let detail = 1 + (i * 7) % 5;
+    let px = |x: usize, y: usize| ((x * 31 + y * 17 + i * 13) ^ (x * y / detail)) as u8;
+    let y: Vec<u8> = (0..w * h).map(|k| px(k % w, k / w)).collect();
+    Yuv { w, h, y, u: vec![128; w * h / 4], v: vec![128; w * h / 4] }
+}
+
+#[test]
+fn one_fps_vbr_does_not_swing_the_qp_from_frame_to_frame() {
+    let (w, h) = (320usize, 240usize);
+    let mut c = EncoderConfig::new(w as u32, h as u32, 1, 1);
+    c.bframes = 0;
+    c.rate = RateControl::Vbr { target_kbps: 400, max_kbps: 600 };
+    let mut enc = Encoder::new(c).unwrap();
+    let mut qps = Vec::new();
+    for i in 0..24 {
+        let f = busy_frame(w, h, i);
+        qps.extend(enc.encode(&f.frame(), i as i64).unwrap().iter().map(|p| i32::from(p.qp)));
+    }
+    qps.extend(enc.flush().iter().map(|p| i32::from(p.qp)));
+    assert_eq!(qps.len(), 24);
+    let swing = qps.windows(2).skip(1).map(|p| (p[1] - p[0]).abs()).max().unwrap_or(0);
+    assert!(swing <= 4, "QP swings by {swing} between frames: {qps:?}");
+}