Merge origin/main into gpu-rendering

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
2026-10-10 08:27:57 -04:00
93 changed files with 11123 additions and 101 deletions
Generated
+31
View File
@@ -1679,6 +1679,7 @@ dependencies = [
"filmcraft-render",
"filmcraft-time",
"filmcraft-ui-egui",
"glob",
"log",
"muda",
"objc2 0.6.4",
@@ -1887,6 +1888,7 @@ dependencies = [
"filmcraft-testkit",
"filmcraft-text",
"filmcraft-time",
"filmcraft-tts",
"image",
"log",
"serde",
@@ -2233,6 +2235,28 @@ dependencies = [
"serde",
]
[[package]]
name = "filmcraft-tts"
version = "0.5.0"
dependencies = [
"candle-core",
"candle-nn",
"filmcraft-audio-dsp",
"filmcraft-tts-text",
"serde",
"serde_json",
"thiserror 2.0.21",
"zip",
]
[[package]]
name = "filmcraft-tts-text"
version = "0.5.0"
dependencies = [
"serde",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-ui-egui"
version = "0.5.0"
@@ -2258,6 +2282,7 @@ dependencies = [
"filmcraft-scopes",
"filmcraft-text",
"filmcraft-time",
"filmcraft-tts",
"image",
"log",
"pollster 0.4.0",
@@ -2838,6 +2863,12 @@ dependencies = [
"vello_common",
]
[[package]]
name = "glob"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
[[package]]
name = "glow"
version = "0.17.0"
+2
View File
@@ -44,6 +44,8 @@ filmcraft-interchange = { path = "crates/interchange" }
filmcraft-text = { path = "crates/text" }
filmcraft-captions = { path = "crates/captions" }
filmcraft-speech = { path = "crates/speech" }
filmcraft-tts = { path = "crates/tts" }
filmcraft-tts-text = { path = "crates/tts-text" }
filmcraft-render = { path = "crates/render" }
filmcraft-gpu = { path = "crates/gpu" }
filmcraft-scopes = { path = "crates/scopes" }
+7 -1
View File
@@ -259,9 +259,15 @@ async fn cli() {
register_hardware_decoders();
register_gpu_frame_renderer();
let a = Args::parse(std::env::args().skip(1));
// `--help` anywhere (`filmcraft-cli --help`, `filmcraft-cli export --help`) prints the reference:
// the parser takes it as a flag option, so it never reaches the subcommand match.
if a.flag("--help") {
out!("{HELP}");
return;
}
let Some(cmd) = a.pos(0) else { usage("missing subcommand") };
match cmd {
"help" | "--help" | "-h" => out!("{HELP}"),
"help" | "-h" => out!("{HELP}"),
"version" => outln!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")),
"probe" => {
let path = a.pos(1).unwrap_or_else(|| usage("probe <media> [--image-sequence]"));
+16
View File
@@ -24,6 +24,22 @@ fn help_and_usage_errors() {
assert_eq!(cli(&["exec", "file.newBin", "notkv"]).status.code(), Some(2));
}
#[test]
fn long_help_matches_help_command() {
let reference = cli(&["help"]);
assert!(reference.status.success());
let long_help = cli(&["--help"]);
assert!(long_help.status.success(), "--help failed: {}", String::from_utf8_lossy(&long_help.stderr));
assert_eq!(long_help.stdout, reference.stdout);
assert!(long_help.stderr.is_empty());
let short_help = cli(&["-h"]);
assert!(short_help.status.success());
assert_eq!(short_help.stdout, reference.stdout);
let sub_help = cli(&["export", "--help"]);
assert!(sub_help.status.success(), "export --help failed: {}", String::from_utf8_lossy(&sub_help.stderr));
assert_eq!(sub_help.stdout, reference.stdout);
}
#[test]
fn probe_reports_mpeg_transport_and_program_streams() {
let Some(ffmpeg) = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"].into_iter().find(|p| std::path::Path::new(p).exists()) else {
+5
View File
@@ -14,6 +14,8 @@ workspace = true
default = []
# Speech-to-text (Text panel ▸ Transcribe) with Whisper on the CPU, and model downloads.
whisper = ["filmcraft-engine/whisper", "filmcraft-engine/speech-download"]
# Natural Text to Speech voices (Kokoro-82M on the CPU, downloaded on first use).
neural-voices = ["filmcraft-engine/neural-voices"]
[dependencies]
filmcraft-engine = { workspace = true }
@@ -33,6 +35,9 @@ cpal = "0.16"
chrono = { version = "0.4", default-features = false, features = ["clock"] }
sys-locale = "0.3"
[dev-dependencies]
glob = "0.3"
[target.'cfg(target_os = "macos")'.dependencies]
muda = "0.21"
objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSProcessInfo", "NSString", "NSArray"] }
+112 -10
View File
@@ -1,9 +1,10 @@
//! cpal audio output: the playback master clock. Settings ▸ Audio Hardware picks the host
//! ("Device Class"), the output device, the I/O buffer size and the sample rate.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use filmcraft_engine::settings::AudioHardwarePrefs;
use filmcraft_ui_egui::{AudioDevices, AudioOut};
@@ -17,6 +18,31 @@ pub struct CpalOut {
hw: AudioHardwarePrefs,
/// Sequence sample rate for "Attempt to force hardware to document sample rate".
document_rate: Option<u32>,
/// Status-bar note: the host's default device could not be opened and a fallback device is
/// in use, or no output device was found (playing without sound).
note: Option<Note>,
/// The fallback picked when the default device could not be opened, as (host, device) names,
/// so later lookups (`devices`, `configure`, `start`) reuse it instead of probing every PCM.
fallback: Mutex<Option<(String, String)>>,
}
/// A hardware note for the status bar, translated when it is shown.
#[derive(Clone, Debug, PartialEq)]
enum Note {
/// The default output device cannot be opened; this named device is used instead.
Fallback(String),
/// No output device at all: playback runs without sound.
NoDevice,
}
impl Note {
fn text(&self) -> String {
use filmcraft_ui_egui::i18n::{fmt, t};
match self {
Note::Fallback(device) => fmt(t("Audio hardware: the default output device cannot be opened; using '{device}'"), &[("device", device)]),
Note::NoDevice => t("No audio output device found: playing without sound (check Settings ▸ Audio Hardware)").to_string(),
}
}
}
/// The host named in the settings (empty or unknown = the default host).
@@ -30,21 +56,76 @@ fn host(name: &str) -> cpal::Host {
cpal::default_host()
}
/// The output device named in the settings (empty or missing = the host's default).
fn output_device(h: &cpal::Host, name: &str) -> Option<cpal::Device> {
/// The output device the settings ask for, with a status-bar note when a fallback is in use.
struct DeviceChoice {
device: cpal::Device,
/// The host's default device could not be opened: this device is the fallback.
fallback: Option<Note>,
}
/// Fallback order when the default device cannot be opened: the sound-server bridges first
/// (`pipewire`, `pulse`: they share the card with every other app), then other named PCMs, and
/// raw `hw:`/`plughw:` devices last (opening one takes the card exclusively and silences the
/// rest of the desktop).
fn fallback_rank(name: &str) -> u8 {
let n = name.to_ascii_lowercase();
if n == "pipewire" || n.starts_with("pipewire:") {
0
} else if n == "pulse" || n.starts_with("pulse:") {
1
} else if n.starts_with("hw:") || n.starts_with("plughw:") {
3
} else {
2
}
}
/// The output device named in the settings (empty or missing = the host's default). When the
/// default cannot be opened — a broken ALSA `default` (missing `99-pipewire-default.conf`,
/// #23/#106) — a fallback that can, so the app is not silently mute. The fallback list is probed
/// only after the default fails, in [`fallback_rank`] order, and the pick is cached in `cache`.
fn output_device(h: &cpal::Host, name: &str, cache: &Mutex<Option<(String, String)>>) -> Option<DeviceChoice> {
if !name.is_empty()
&& let Ok(mut devs) = h.output_devices()
&& let Some(d) = devs.find(|d| d.name().is_ok_and(|n| n == name))
{
return Some(d);
return Some(DeviceChoice { device: d, fallback: None });
}
h.default_output_device()
if let Some(d) = h.default_output_device()
&& d.default_output_config().is_ok()
{
return Some(DeviceChoice { device: d, fallback: None });
}
let host_name = h.id().name().to_string();
let mut cache = cache.lock().unwrap_or_else(PoisonError::into_inner);
let mut devs: Vec<(String, cpal::Device)> = h.output_devices().ok()?.filter_map(|d| d.name().ok().map(|n| (n, d))).collect();
if let Some((cached_host, cached)) = cache.as_ref()
&& *cached_host == host_name
&& let Some(i) = devs.iter().position(|(n, _)| n == cached)
{
let (found, d) = devs.swap_remove(i);
return Some(DeviceChoice { device: d, fallback: Some(Note::Fallback(found)) });
}
devs.sort_by_key(|(n, _)| fallback_rank(n));
let (found, d) = devs.into_iter().find(|(_, d)| d.default_output_config().is_ok())?;
log::warn!("the default audio output device cannot be opened; falling back to '{found}'");
*cache = Some((host_name, found.clone()));
Some(DeviceChoice { device: d, fallback: Some(Note::Fallback(found)) })
}
impl CpalOut {
pub fn new() -> Self {
let host = cpal::default_host();
let cfg = host.default_output_device().and_then(|dev| dev.default_output_config().ok());
let fallback = Mutex::new(None);
let choice = output_device(&host, "", &fallback);
let cfg = choice.as_ref().and_then(|c| c.device.default_output_config().ok());
let note = match choice {
Some(c) => c.fallback,
None => {
log::warn!("no audio output device found");
Some(Note::NoDevice)
}
};
Self {
stream: None,
played: Arc::new(AtomicU64::new(0)),
@@ -53,6 +134,8 @@ impl CpalOut {
channels: cfg.as_ref().map_or(2, |c| c.channels()),
hw: AudioHardwarePrefs::default(),
document_rate: None,
note,
fallback,
}
}
@@ -124,7 +207,8 @@ impl AudioOut for CpalOut {
fn start(&mut self, mut fill: Box<dyn FnMut(&mut [f32], usize) + Send>) -> Result<u32, String> {
self.stop();
let host = host(&self.hw.device_class);
let dev = output_device(&host, &self.hw.default_output).ok_or("no output device")?;
let DeviceChoice { device: dev, fallback } = output_device(&host, &self.hw.default_output, &self.fallback).ok_or("no output device")?;
self.note = fallback;
let cfg = self.config(&dev)?;
let channels = cfg.channels() as usize;
if channels == 0 || cfg.sample_rate().0 == 0 {
@@ -185,7 +269,8 @@ impl AudioOut for CpalOut {
let h = host(&self.hw.device_class);
let outputs = h.output_devices().map(|d| d.filter_map(|x| x.name().ok()).collect()).unwrap_or_default();
let inputs = h.input_devices().map(|d| d.filter_map(|x| x.name().ok()).collect()).unwrap_or_default();
let output_channels = output_device(&h, &self.hw.default_output).and_then(|d| self.config(&d).ok()).map(|c| c.channels()).unwrap_or(0);
let output_channels =
output_device(&h, &self.hw.default_output, &self.fallback).and_then(|c| self.config(&c.device).ok()).map(|cfg| cfg.channels()).unwrap_or(0);
AudioDevices { hosts: cpal::available_hosts().iter().map(|h| h.name().to_string()).collect(), inputs, outputs, output_channels }
}
fn configure(&mut self, hw: &AudioHardwarePrefs, document_rate: Option<u32>) {
@@ -193,13 +278,17 @@ impl AudioOut for CpalOut {
self.document_rate = document_rate;
// the rate playback mixes at must be known before `start`
let host = host(&self.hw.device_class);
if let Some(dev) = output_device(&host, &self.hw.default_output)
&& let Ok(cfg) = self.config(&dev)
if let Some(choice) = output_device(&host, &self.hw.default_output, &self.fallback)
&& let Ok(cfg) = self.config(&choice.device)
{
self.rate = cfg.sample_rate().0;
self.channels = cfg.channels();
self.note = choice.fallback;
}
}
fn note(&self) -> Option<String> {
self.note.as_ref().map(Note::text)
}
}
#[cfg(test)]
@@ -207,6 +296,19 @@ mod tests {
use super::*;
use cpal::Sample;
#[test]
fn fallback_prefers_sound_servers_over_raw_hardware() {
let mut names = vec!["plughw:CARD=PCH,DEV=0", "hw:CARD=PCH,DEV=0", "sysdefault:CARD=PCH", "pulse", "pipewire"];
names.sort_by_key(|n| fallback_rank(n));
assert_eq!(names, ["pipewire", "pulse", "sysdefault:CARD=PCH", "plughw:CARD=PCH,DEV=0", "hw:CARD=PCH,DEV=0"]);
}
#[test]
fn notes_name_the_fallback_device() {
assert!(Note::Fallback("pipewire".into()).text().contains("'pipewire'"));
assert!(!Note::NoDevice.text().is_empty());
}
#[test]
fn integer_conversion_clips_and_preserves_interleaved_channels() {
let mut signed = [0_i16; 8];
+329
View File
@@ -0,0 +1,329 @@
//! Dev-time Linux desktop entry: the shell icon fallback.
//!
//! The Linux shell (Wayland *and* X11) resolves the window to a `.desktop` file by app id (set in
//! `main.rs` via `eframe`'s `with_app_id`) and takes the taskbar / window-list icon from its
//! `Icon=` entry in the hicolor theme. Packaging installs both
//! (`packaging/linux/ai.storyteller.filmcraft.desktop` and the hicolor PNGs) — but a source-tree
//! run has neither, so KDE/GNOME fall back to the generic Wayland icon. The fix is to ensure a
//! user-level entry for those runs: when no entry for the app id exists in any XDG data dir,
//! install one (plus the hicolor icon PNGs, read from this checkout's `assets/app-icon`, so
//! nothing is embedded in the binary) into `$XDG_DATA_HOME`/`~/.local/share`. An existing
//! packaged entry is never overridden; the user-owned dev copy is rewritten when it points at
//! another binary (a dev build moves between checkouts). The shell picks the icon up at the *next* app start;
//! `update-desktop-database`/`kbuildsycoca` is not needed for icon lookup.
//!
//! Opt-in only: nothing is written unless `FILMCRAFT_DEV_DESKTOP_ENTRY=1` is set, so ordinary
//! `cargo run`s (agents, worktrees, CI) never touch the user's profile. Packaged runs are
//! excluded even then — AppImage (`APPIMAGE`) mounts at a fresh `/tmp/.mount_*` each launch,
//! so its `Exec` path changes every time and it ships its own entry, and a Flatpak
//! (`FLATPAK_ID`) installs its entry through the manifest. `packaging/linux/install.sh` writes the packaged
//! `Exec=` the same two-layer way as here; keep the two in sync.
//!
//! Every failure is logged and skipped — a missing dev icon must never stop the app
//! ([`AGENTS.md`](/AGENTS.md) §0 "never crash").
/// Install the user-level desktop entry and icons for a source-tree run (Linux).
pub fn ensure_dev_desktop_entry() {
// Packaged and sandboxed runs never write into the user's profile: an AppImage mounts at a
// fresh /tmp/.mount_* every launch (the Exec path changes each time) and a Flatpak installs
// its own entry.
if std::env::var_os("APPIMAGE").is_some() || std::env::var_os("FLATPAK_ID").is_some() {
return;
}
// Opt-in only: a plain `cargo run` (agents, worktrees, other checkouts) must not write into
// the user's profile, and a packaged release never does (packages and install.sh install
// the real entry).
if std::env::var_os("FILMCRAFT_DEV_DESKTOP_ENTRY").is_none_or(|v| v != "1") {
return;
}
if let Err(e) = ensure() {
log::warn!("desktop entry (taskbar icon) not installed: {e}");
}
}
/// Install missing files; `Ok` whether anything was written or an entry already existed.
fn ensure() -> Result<(), DevIconError> {
let file_name = format!("{}.desktop", crate::APP_ID);
let user_dir = user_data_dir()?;
let exe = std::env::current_exe().map_err(DevIconError::Io)?;
// Not `to_string_lossy`: a substituted U+FFFD would install an `Exec` that can never start
// the app, and the same lossy string would match on every later run, so the broken entry
// would never be rewritten or warned about.
let exe = exe.to_str().ok_or(DevIconError::BadExecPath)?;
if exe.contains('=') || exe.contains('%') || exe.chars().any(|c| c.is_control()) {
// A desktop entry cannot carry these (install.sh rejects them for the same reason):
// '=' separates key and value, '%' introduces a field code, control characters are
// not valid in the string layer.
return Err(DevIconError::BadExecPath);
}
for dir in search_dirs(&user_dir) {
let candidate = dir.join("applications").join(&file_name);
if !candidate.is_file() {
continue;
}
// A packaged entry (any system dir) already gives the shell its icon; only our own
// user-level copy is kept fresh.
if dir != user_dir {
return Ok(());
}
let body = std::fs::read_to_string(&candidate).map_err(DevIconError::Io)?;
if desktop_file_exec_is_ours(&body, exe) {
return Ok(());
}
break; // a dev entry from another checkout: fall through and rewrite it
}
for size in ICON_SIZES {
let rel = std::path::Path::new("hicolor").join(format!("{size}x{size}")).join("apps").join(format!("{}.png", crate::APP_ID));
let dest = user_dir.join("icons").join(&rel);
if !dest.is_file()
&& let Some(parent) = dest.parent()
{
// The checkout this binary was built from; a missing icon only costs the icon.
let src = std::path::Path::new(ICON_DIR).join(&rel);
if !src.is_file() {
log::warn!("dev desktop entry: icon {} not found; skipping it", src.display());
continue;
}
std::fs::create_dir_all(parent).map_err(DevIconError::Io)?;
std::fs::copy(&src, &dest).map_err(DevIconError::Io)?;
}
}
let applications = user_dir.join("applications");
std::fs::create_dir_all(&applications).map_err(DevIconError::Io)?;
let body = desktop_file_content(crate::APP_ID, "FilmCraft", exe);
std::fs::write(applications.join(&file_name), body).map_err(DevIconError::Io)?;
Ok(())
}
/// The hicolor icon sizes installed for the taskbar (a subset large enough for panel and topbar;
/// the shell picks the nearest size, so 256 and 512 cover both).
const ICON_SIZES: [u16; 2] = [256, 512];
/// The app icons in the source tree this binary was built from (a path, not embedded bytes:
/// the entry is a dev-checkout convenience and release binaries must not carry the PNGs).
const ICON_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../assets/app-icon");
/// The user data dir we install into: `$XDG_DATA_HOME`, else `$HOME/.local/share` (XDG base
/// directory spec). Untrusted environment: an empty or relative `XDG_DATA_HOME` means the
/// default, and a missing or relative `HOME` is an error — never a guess that would write
/// into the working directory.
fn user_data_dir() -> Result<std::path::PathBuf, DevIconError> {
if let Some(v) = std::env::var_os("XDG_DATA_HOME")
&& !v.is_empty()
&& std::path::Path::new(&v).is_absolute()
{
return Ok(std::path::PathBuf::from(v));
}
match std::env::var_os("HOME").filter(|h| !h.is_empty() && std::path::Path::new(h).is_absolute()) {
Some(home) => Ok(std::path::PathBuf::from(home).join(".local").join("share")),
None => Err(DevIconError::NoHome),
}
}
/// The XDG data dirs to search for an existing entry, user dir first, then `$XDG_DATA_DIRS` (or
/// the spec default `/usr/local/share:/usr/share`). Only absolute paths are taken; the spec
/// requires them and a relative one would read the working directory.
fn search_dirs(user: &std::path::Path) -> Vec<std::path::PathBuf> {
let mut dirs = vec![user.to_path_buf()];
let system = std::env::var_os("XDG_DATA_DIRS")
.filter(|v| !v.is_empty())
.map(|v| v.to_string_lossy().into_owned())
.unwrap_or_else(|| "/usr/local/share:/usr/share".into());
for dir in system.split(':') {
if std::path::Path::new(dir).is_absolute() {
dirs.push(std::path::PathBuf::from(dir));
}
}
dirs
}
/// Whether the `.desktop` body's `Exec` points at our binary exactly (a stale dev entry from
/// another checkout — `/a/filmcraft-old` is not `/a/filmcraft` — does not, and is rewritten; a
/// packaged `Exec=filmcraft` never matches a source-tree path).
fn desktop_file_exec_is_ours(body: &str, exe: &str) -> bool {
exec_line(body).is_some_and(|arg| arg == exe)
}
/// The first `Exec=` argument (the executable), unescaped through both Desktop Entry layers:
/// first the string layer of the file value (`\\` `\s` `\n` `\t` `\r`, per
/// <https://specifications.freedesktop.org/desktop-entry/latest/>), then the shell layer of the
/// argument (double quotes, and `\` before `` ` `"` `\` `$` inside them). `None` when there is no
/// `Exec=` line or its first argument cannot be parsed.
fn exec_line(body: &str) -> Option<String> {
let value = body.lines().find(|l| l.starts_with("Exec=")).and_then(|l| l.strip_prefix("Exec="))?;
let value = unescape_string_value(value);
if let Some(rest) = value.strip_prefix('"') {
let mut out = String::new();
let mut chars = rest.chars();
while let Some(c) = chars.next() {
match c {
'"' => return Some(out),
'\\' => match chars.next() {
Some(e @ ('\\' | '"' | '`' | '$')) => out.push(e),
Some(e) => {
out.push('\\');
out.push(e);
}
None => return None, // a dangling escape ends the argument
},
c => out.push(c),
}
}
None // no closing quote
} else {
Some(value.split(' ').next().unwrap_or_default().to_string())
}
}
/// Undo the desktop entry's string layer: `\\` `\s` `\n` `\t` `\r` are the defined sequences; a
/// backslash before anything else is kept as the character itself (lenient, like desktop file
/// parsers in the wild).
fn unescape_string_value(value: &str) -> String {
let mut out = String::with_capacity(value.len());
let mut chars = value.chars();
while let Some(c) = chars.next() {
if c != '\\' {
out.push(c);
continue;
}
match chars.next() {
Some('\\') => out.push('\\'),
Some('s') => out.push(' '),
Some('n') => out.push('\n'),
Some('t') => out.push('\t'),
Some('r') => out.push('\r'),
Some(e) => out.push(e),
None => out.push('\\'),
}
}
out
}
/// The `.desktop` body: same fields as `packaging/linux/ai.storyteller.filmcraft.desktop`, with
/// `Exec` pointing at this exact binary (a dev build isn't on `$PATH`, so `TryExec` is left out)
/// and the icon by theme name.
fn desktop_file_content(app_id: &str, name: &str, exe: &str) -> String {
let mut out = String::new();
out.push_str("[Desktop Entry]\n");
out.push_str("Type=Application\n");
out.push_str(&format!("Name={name}\n"));
out.push_str("GenericName=Video Editor\n");
out.push_str("Comment=Edit video, color and sound\n");
out.push_str(&format!("Exec=\"{}\" %F\n", escape_exec_argument(exe)));
out.push_str(&format!("Icon={app_id}\n"));
out.push_str("Terminal=false\n");
out.push_str("StartupNotify=true\n");
out.push_str(&format!("StartupWMClass={app_id}\n"));
out.push_str("Categories=AudioVideo;Video;AudioVideoEditing;\n");
out.push_str("Keywords=video;editor;film;timeline;color;grading;nle;\n");
out
}
/// Escape `s` for the `Exec=` value's two layers, like `packaging/linux/install.sh`: first the
/// shell layer (`` ` `` `"` `$` `\` get a backslash, and the argument is double-quoted), then
/// the string layer of the file value (every backslash doubled).
fn escape_exec_argument(s: &str) -> String {
let mut shell = String::with_capacity(s.len() + 2);
for c in s.chars() {
if matches!(c, '`' | '"' | '$' | '\\') {
shell.push('\\');
}
shell.push(c);
}
shell.replace('\\', "\\\\")
}
/// Everything that can keep the dev icon out of the taskbar; logged, never fatal.
#[derive(Debug)]
enum DevIconError {
Io(std::io::Error),
NoHome,
BadExecPath,
}
impl std::fmt::Display for DevIconError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DevIconError::Io(e) => write!(f, "{e}"),
DevIconError::NoHome => write!(f, "no home directory for the user data dir"),
DevIconError::BadExecPath => {
write!(f, "the binary path has a character a desktop entry cannot carry (=, % or a control character), or is not valid UTF-8")
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The entry mirrors the packaged one, derived from the real file so the two cannot drift:
/// minus `TryExec` (a dev build isn't on `$PATH`), plus the quoted dev `Exec` path.
/// The icons the entry installs exist in the checkout.
#[test]
fn icon_sources_exist_in_the_checkout() {
for size in ICON_SIZES {
let p = std::path::Path::new(ICON_DIR).join(format!("hicolor/{size}x{size}/apps/ai.storyteller.filmcraft.png"));
assert!(p.is_file(), "{}", p.display());
}
}
#[test]
fn desktop_file_matches_the_packaged_fields() {
let packaged = include_str!("../../../packaging/linux/ai.storyteller.filmcraft.desktop");
let expected = packaged
.lines()
.filter(|l| !l.starts_with("TryExec="))
.map(|l| if l.starts_with("Exec=") { "Exec=\"/target/debug/filmcraft\" %F" } else { l })
.collect::<Vec<_>>()
.join("\n")
+ "\n";
let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", "/target/debug/filmcraft");
assert_eq!(body, expected);
}
/// Both layers are escaped, like install.sh: shell first (`` ` `` `"` `$` `\` get a
/// backslash), then *every* backslash doubled for the file value — including the escape
/// backslashes just added.
#[test]
fn exec_escapes_both_desktop_entry_layers() {
assert_eq!(escape_exec_argument("/opt/film craft/filmcraft"), "/opt/film craft/filmcraft");
// a\b → shell a\\b → string a\\\\b
assert_eq!(escape_exec_argument("a\\b"), "a\\\\\\\\b");
// a$b → shell a\$b → string a\\$b
assert_eq!(escape_exec_argument("a$b"), "a\\\\$b");
}
/// A written entry reads back as the exact binary it was written for, through both layers.
#[test]
fn exec_round_trips_through_both_layers() {
for exe in ["/home/dev/Projects/filmcraft/target/debug/filmcraft", "/opt/film craft/filmcraft", "/a$b\"c`d\\e"] {
let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", exe);
assert_eq!(exec_line(&body).as_deref(), Some(exe));
assert!(desktop_file_exec_is_ours(&body, exe));
}
}
/// Only the exact executable counts: another checkout's path, a path with our path as a
/// prefix, and a packaged `Exec=filmcraft` are all someone else's entry.
#[test]
fn desktop_file_exec_is_ours_matches_only_our_binary() {
let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", "/a/filmcraft");
assert!(!desktop_file_exec_is_ours(&body, "/a/filmcraft-old"));
assert!(!desktop_file_exec_is_ours(&body, "/b/filmcraft"));
assert!(!desktop_file_exec_is_ours("Exec=filmcraft %F\n", "/a/filmcraft"));
assert!(!desktop_file_exec_is_ours("Name=filmcraft\n", "/a/filmcraft"));
assert!(!desktop_file_exec_is_ours("Exec=\"dangling\n", "/a/filmcraft"));
}
/// The string layer is undone before the argument is split: `\s` becomes a space, which
/// ends a bare argument (the shell layer's quoting, not `\s`, is how a space travels), and
/// `\\` is one backslash. install.sh always quotes, so a space in our own `Exec` never
/// splits.
#[test]
fn exec_reads_the_string_layer() {
assert_eq!(exec_line("Exec=/a\\sb %F\n").as_deref(), Some("/a"));
assert_eq!(exec_line("Exec=\"/a\\sb\" %F\n").as_deref(), Some("/a b"));
assert_eq!(exec_line("Exec=/a\\\\b %F\n").as_deref(), Some("/a\\b"));
}
}
+70
View File
@@ -0,0 +1,70 @@
//! Extension filters for native file dialogs.
//!
//! Every filtered file dialog (import, relink, open file/preset, open/save project) builds
//! its filter with [`extensions`]. On Linux and the BSDs, rfd's XDG portal, GTK and Zenity
//! backends turn each extension into a case-sensitive `*.{ext}` glob, so camera files such
//! as `shot.MP4` or `shot.Mp4` would be hidden. The helper appends one bracket pattern per
//! extension (`mp4` -> `[mM][pP]4`) after the literal extensions, which stay first so save
//! dialogs keep their normal default suffix. macOS and Windows dialogs already ignore case
//! and receive the plain list.
pub fn extensions(exts: &[&str]) -> Vec<String> {
let extensions: Vec<_> = exts.iter().map(|ext| (*ext).to_owned()).collect();
// XDG portal/Zenity globs are case-sensitive. Keep the literal extensions first so
// Save dialogs can still infer a normal default suffix, then cover mixed case too.
// https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.FileChooser.html
#[cfg(not(any(target_os = "windows", target_os = "macos")))]
{
let mut extensions = extensions;
for ext in exts {
if !ext.bytes().any(|b| b.is_ascii_alphabetic()) {
continue;
}
let mut pattern = String::new();
for c in ext.chars() {
if c.is_ascii_alphabetic() {
pattern.extend(['[', c.to_ascii_lowercase(), c.to_ascii_uppercase(), ']']);
} else {
pattern.push(c);
}
}
extensions.push(pattern);
}
extensions
}
// Native macOS and Windows filters take ordinary extensions.
#[cfg(any(target_os = "windows", target_os = "macos"))]
extensions
}
#[cfg(test)]
mod tests {
use super::extensions;
// rfd's XDG portal and Zenity backends turn each extension into `*.{ext}`.
// Exercise those case-sensitive patterns against filenames, as the picker does.
#[cfg(not(any(target_os = "windows", target_os = "macos")))]
#[test]
fn camera_files_are_visible_in_every_capitalization() {
let filters: Vec<_> = extensions(&["mp4", "mov", "m2ts", "wav", "fcproj"]).iter().map(|ext| glob::Pattern::new(&format!("*.{ext}")).unwrap()).collect();
for name in ["camera.mp4", "camera.MP4", "camera.Mp4", "clip.MOV", "clip.moV", "take.M2TS", "sound.WaV", "Edit.FCPROJ", "Edit.FcProj"] {
assert!(filters.iter().any(|f| f.matches(name)), "the picker hides {name}");
}
for name in ["camera.mp4.bak", "camera.mp44", "camera.xmp4", "notes.txt", "mp4"] {
assert!(!filters.iter().any(|f| f.matches(name)), "the picker admits {name}");
}
}
#[test]
fn save_filters_keep_the_literal_default_extension() {
assert_eq!(extensions(&["mp4", "mov"]).first().map(String::as_str), Some("mp4"));
assert!(extensions(&[]).is_empty());
assert_eq!(extensions(&["*", "", "123"]), ["*", "", "123"]);
}
#[cfg(any(target_os = "windows", target_os = "macos"))]
#[test]
fn native_filters_receive_only_literal_extensions() {
assert_eq!(extensions(&["mp4", "mov", "fcproj"]), ["mp4", "mov", "fcproj"]);
}
}
+18 -6
View File
@@ -28,6 +28,9 @@ mod args;
mod audio;
mod audio_in;
mod control_server;
#[cfg(target_os = "linux")]
mod dev_icon;
mod file_filters;
#[cfg(any(target_os = "windows", test))]
mod graphics;
mod logging;
@@ -107,6 +110,8 @@ fn main() -> eframe::Result {
}
}
app_nap::disable();
#[cfg(target_os = "linux")]
dev_icon::ensure_dev_desktop_entry();
// Panics anywhere go to <data dir>/Logs/crash-<day>.log with a backtrace; the UI pass and
// frame workers catch them and keep running (see filmcraft_ui_egui::crash).
filmcraft_ui_egui::crash::install(log_dir);
@@ -200,7 +205,7 @@ fn main() -> eframe::Result {
app.audio = Some(Box::new(audio::CpalOut::new()));
app.hooks.pick_files = Some(Box::new(|exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), exts)
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_files()
.unwrap_or_default()
.into_iter()
@@ -209,18 +214,21 @@ fn main() -> eframe::Result {
}));
// Link Media ▸ Locate…, Attach Proxies, Reconnect Full Resolution: one path, not imported.
app.hooks.pick_file_for_relink = Some(Box::new(|exts: &[&str], _hint| {
rfd::FileDialog::new().add_filter(filmcraft_ui_egui::i18n::t("Media"), exts).pick_file().map(|p| p.to_string_lossy().to_string())
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save = Some(Box::new(|name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &["fcproj"])
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save_as = Some(Box::new(|filter: &str, exts: &[&str], name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), exts)
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
@@ -230,6 +238,7 @@ fn main() -> eframe::Result {
Some(Box::new(|dir: &str| rfd::FileDialog::new().set_directory(dir).pick_folder().map(|p| p.to_string_lossy().into_owned())));
app.hooks.open_path = Some(Box::new(open_path));
// Settings ▸ General ▸ Interface Language ▸ System Language (#218).
app.hooks.cursor_screen_position = Some(Box::new(filmcraft_platform::cursor::cursor_screen_position));
app.hooks.system_languages = Some(Box::new(|| sys_locale::get_locales().collect()));
// Settings ▸ Appearance ▸ Appearance Mode ▸ Sync with system on Linux desktops whose
// compositor reports no theme to winit (no polling: see appearance.rs).
@@ -238,11 +247,14 @@ fn main() -> eframe::Result {
window_raise::raise_without_focus();
}));
app.hooks.pick_open_file = Some(Box::new(|filter: &str, exts: &[&str]| {
rfd::FileDialog::new().add_filter(filmcraft_ui_egui::i18n::t(filter), exts).pick_file().map(|p| p.to_string_lossy().to_string())
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_open_project = Some(Box::new(|| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &["fcproj"])
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
+9
View File
@@ -22,6 +22,11 @@ real_name = "Brandon Thomas"
display_name = "Brandon Thomas"
sync_github_name = false
[people.ahmedhosnypro]
real_name = "Ahmed Hosny"
display_name = "Ahmed Hosny"
sync_github_name = false
[people.sanyamsingh001]
display_name = "sanyamsingh001"
sync_github_name = false
@@ -41,6 +46,10 @@ sync_github_name = true
[people.ItzBloxy]
display_name = "Bloxy"
[people.Hathor875]
real_name = "Krzysztof Cieślik"
sync_github_name = true
[people.ReReddo]
display_name = "ReReddo"
sync_github_name = false
+41 -3
View File
@@ -139,6 +139,30 @@ pub struct TypeParams {
pub text: String,
}
#[derive(Debug, Deserialize, JsonSchema)]
pub struct MediaImportParams {
/// Absolute paths to media files (MP4/MOV, WAV/MP3/FLAC/AIFF/Ogg, PNG/JPEG/…).
pub paths: Vec<String>,
/// Bin ID to import into (optional).
#[serde(default)]
pub bin: Option<u64>,
/// Import numbered stills as an image sequence (optional, default false).
#[serde(default)]
pub image_sequence: Option<bool>,
}
/// `file.import` params for a `media_import` call.
fn media_import_params(p: MediaImportParams) -> Value {
let mut params = json!({"paths": p.paths});
if let Some(bin) = p.bin {
params["bin"] = json!(bin);
}
if let Some(seq) = p.image_sequence {
params["imageSequence"] = json!(seq);
}
params
}
#[derive(Debug, Default, Deserialize, JsonSchema)]
pub struct ElementsParams {
/// Only element ids starting with this prefix (e.g. `timeline.clip.`).
@@ -381,9 +405,8 @@ impl FilmcraftMcp {
description = "Import media files by absolute path (MP4/MOV, WAV/MP3/FLAC/AIFF/Ogg, PNG/JPEG/…).",
annotations(read_only_hint = false, destructive_hint = false, idempotent_hint = false, open_world_hint = false)
)]
async fn media_import(&self, Parameters(p): Parameters<TypeParams>) -> Result<CallToolResult, McpError> {
let paths: Vec<&str> = p.text.split('\n').map(str::trim).filter(|s| !s.is_empty()).collect();
wrap(self.run("file.import", json!({"paths": paths})).await)
async fn media_import(&self, Parameters(p): Parameters<MediaImportParams>) -> Result<CallToolResult, McpError> {
wrap(self.run("file.import", media_import_params(p)).await)
}
#[tool(
@@ -735,6 +758,21 @@ mod tests {
}
}
/// #499: `media_import` takes `paths` (a required array), and forwards `bin` / `image_sequence` to `file.import`.
#[test]
fn media_import_schema_and_params() {
let m = FilmcraftMcp::headless(Session::default());
let def = m.tool_router.get("media_import").unwrap();
let schema = Value::Object((*def.input_schema).clone());
assert_eq!(schema["properties"]["paths"]["type"], "array", "{schema}");
assert!(schema["required"].as_array().unwrap().iter().any(|r| r == "paths"), "{schema}");
let p: MediaImportParams = serde_json::from_value(json!({"paths": ["/a.png"], "bin": 7, "image_sequence": true})).unwrap();
assert_eq!(media_import_params(p), json!({"paths": ["/a.png"], "bin": 7, "imageSequence": true}));
let p: MediaImportParams = serde_json::from_value(json!({"paths": ["/a.wav"]})).unwrap();
assert_eq!(media_import_params(p), json!({"paths": ["/a.wav"]}));
assert!(serde_json::from_value::<MediaImportParams>(json!({"text": "/a.wav"})).is_err());
}
fn demo() -> Session {
let mut s = Session::default();
s.execute("file.openDemoProject", json!({})).unwrap();
+1
View File
@@ -22,6 +22,7 @@ pub mod frame_size;
pub mod multicam;
pub mod through;
pub mod transcript;
pub mod transitions;
use std::collections::HashMap;
+401
View File
@@ -0,0 +1,401 @@
//! Applied transitions: where one may sit over its cut, its alignment, and moving / resizing /
//! removing one (Effect Controls' Duration and Alignment, dragging its edges or its middle in the
//! Timeline, Delete).
//!
//! Premiere (Align transitions; Change transition duration): a transition between two clips stays
//! over their cut, "Center at Cut", "Start at Cut", "End at Cut" or, once dragged off-centre,
//! "Custom Start". Changing its duration moves both ends equally for Center at Cut / Custom Start,
//! only the end for Start at Cut and only the beginning for End at Cut. A one-sided transition (a
//! fade from or to nothing) starts or ends at its clip's edge.
//!
//! Premiere limits a transition by the media its clips have beyond the cut (handles). FilmCraft
//! does not check handles when applying one, so here a transition may cover at most the clips it
//! joins: from the outgoing clip's start to the incoming clip's end.
use filmcraft_project::{Sequence, Track, Transition, TransitionAlign, TransitionId};
use filmcraft_time::Tick;
use crate::{Edge, EditError, Result};
/// How a transition sits on its cut, as Effect Controls' Alignment menu shows it.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Alignment {
Center,
Start,
End,
/// Premiere's "Custom Start": dragged to a place over the cut that is none of the others.
Custom,
}
impl Alignment {
pub const MENU: [Alignment; 3] = [Alignment::Center, Alignment::Start, Alignment::End];
pub fn id(self) -> &'static str {
match self {
Alignment::Center => "center",
Alignment::Start => "start",
Alignment::End => "end",
Alignment::Custom => "custom",
}
}
pub fn label(self) -> &'static str {
match self {
Alignment::Center => "Center at Cut",
Alignment::Start => "Start at Cut",
Alignment::End => "End at Cut",
Alignment::Custom => "Custom Start",
}
}
/// `center`, `start` or `end` (Custom Start is reached by moving a transition, not chosen).
pub fn parse(s: &str) -> Option<Alignment> {
Alignment::MENU.into_iter().find(|a| a.id().eq_ignore_ascii_case(s) || a.label().eq_ignore_ascii_case(s))
}
}
/// Where a transition may be: `(earliest start, cut, latest end)`. Between two clips that is the
/// outgoing clip's start, the cut and the incoming clip's end; for a fade to or from nothing the cut
/// is its clip's edge. `None` when its clips are gone.
pub fn bounds(track: &Track, tr: &Transition) -> Option<(Tick, Tick, Tick)> {
let from = tr.from.and_then(|c| track.item(c));
let to = tr.to.and_then(|c| track.item(c));
match (from, to) {
(Some(f), Some(t)) => Some((f.start, t.start, t.end())),
(None, Some(t)) => Some((t.start, t.start, t.end())),
(Some(f), None) => Some((f.start, f.end(), f.end())),
(None, None) => None,
}
}
/// A transition's alignment on `cut`. Centred means within half a `frame` of the cut, as
/// centring an odd number of frames lands on a frame boundary.
pub fn alignment(tr: &Transition, cut: Tick, frame: Tick) -> Alignment {
match (tr.from, tr.to) {
(None, Some(_)) => return Alignment::Start,
(Some(_), None) => return Alignment::End,
_ => {}
}
if tr.start == cut {
Alignment::Start
} else if tr.end() == cut {
Alignment::End
} else if (tr.start.0.saturating_mul(2).saturating_add(tr.duration.0) - cut.0.saturating_mul(2)).abs() <= frame.0.max(0) {
Alignment::Center
} else {
Alignment::Custom
}
}
fn find(seq: &Sequence, id: TransitionId) -> Option<(&Track, &Transition)> {
seq.all_tracks().find_map(|t| t.transitions.iter().find(|x| x.id == id).map(|x| (t, x)))
}
/// Where a transition would start for `duration` and `align`, keeping its cut (and, for Center
/// at Cut / Custom Start, its centre; `snap` puts a centred start on a frame).
pub fn start_for(seq: &Sequence, id: TransitionId, duration: Tick, align: Alignment, frame: Tick, snap: impl Fn(Tick) -> Tick) -> Option<Tick> {
let (track, tr) = find(seq, id)?;
let (_, cut, _) = bounds(track, tr)?;
let one_sided = alignment(tr, cut, frame);
let align = match one_sided {
Alignment::Start | Alignment::End if tr.from.is_none() || tr.to.is_none() => one_sided,
_ => align,
};
Some(match align {
Alignment::Start => cut,
Alignment::End => cut - duration,
Alignment::Center => snap(cut - duration.mul_ratio(1, 2)),
// Custom Start: the ends move equally, around where it is now
Alignment::Custom => snap(tr.start + tr.duration.mul_ratio(1, 2) - duration.mul_ratio(1, 2)),
})
}
/// Move / resize a transition to cover `start .. start + duration`. Refused (leaving the sequence
/// as it was) when its track is locked, it would be shorter than `min`, it would leave its clips,
/// or it would no longer touch its cut (a one-sided one must keep its edge).
pub fn set_span(seq: &mut Sequence, id: TransitionId, start: Tick, duration: Tick, min: Tick, frame: Tick) -> Result<()> {
let (track, tr) = find(seq, id).ok_or_else(|| EditError::Other(format!("no transition {}", id.0)))?;
if track.locked {
return Err(EditError::Locked);
}
let (lo, cut, hi) = bounds(track, tr).ok_or_else(|| EditError::Other("the transition's clips are gone".into()))?;
if duration < min.max(Tick(1)) {
return Err(EditError::Other("a transition lasts at least one frame".into()));
}
let end = start.0.checked_add(duration.0).map(Tick).ok_or_else(|| EditError::Other("transition out of range".into()))?;
if start < lo || end > hi {
return Err(EditError::Other("a transition can't extend past the clips it joins".into()));
}
let on_cut = match (tr.from, tr.to) {
(None, Some(_)) => start == cut,
(Some(_), None) => end == cut,
_ => start <= cut && cut <= end,
};
if !on_cut {
return Err(EditError::Other(if tr.from.is_some() && tr.to.is_some() {
"a transition between two clips must stay over their cut".into()
} else {
"a one-sided transition stays on its clip's edge".into()
}));
}
let tid = track.id;
let Some(x) = seq.track_mut(tid).and_then(|t| t.transitions.iter_mut().find(|x| x.id == id)) else {
return Err(EditError::Other(format!("no transition {}", id.0)));
};
x.start = start;
x.duration = duration;
// the stored alignment (interchange): Custom Start has no variant, its span says it all
x.align = match alignment(x, cut, frame) {
Alignment::Start => TransitionAlign::StartAtCut,
Alignment::End => TransitionAlign::EndAtCut,
Alignment::Center | Alignment::Custom => TransitionAlign::CenterAtCut,
};
Ok(())
}
/// Where dragging a transition by `delta` in the Timeline puts it, as `(start, duration)`. Its
/// middle (`edge` None) slides it over the cut. An end trims that end only (Premiere shows the
/// Trim-In / Trim-Out icon there): the other end stays, whatever the alignment, so a Center at
/// Cut transition becomes Custom Start. (A typed duration, in Effect Controls or Set Transition
/// Duration, follows the alignment instead: see [`start_for`].) A fade to or from nothing keeps
/// its clip-edge end, and does not slide. Always clamped to where it may be ([`bounds`], over the
/// cut, at least one `frame`). `None` when its clips are gone.
pub fn drag_span(track: &Track, tr: &Transition, edge: Option<Edge>, delta: Tick, frame: Tick) -> Option<(Tick, Tick)> {
let (lo, cut, hi) = bounds(track, tr)?;
let frame = frame.max(Tick(1));
let (s0, e0, d0) = (tr.start, tr.end(), tr.duration);
let one_sided = tr.from.is_none() || tr.to.is_none();
let unchanged = Some((s0, d0));
match edge {
None if one_sided => unchanged,
None => {
let (min_s, max_s) = (lo.max(cut - d0), (hi - d0).min(cut));
if min_s > max_s { unchanged } else { Some(((s0 + delta).clamp(min_s, max_s), d0)) }
}
Some(Edge::In) if tr.from.is_none() => unchanged,
Some(Edge::In) => {
let max_s = cut.min(e0 - frame).max(lo);
let st = (s0 + delta).clamp(lo, max_s);
Some((st, e0 - st))
}
Some(Edge::Out) if tr.to.is_none() => unchanged,
Some(Edge::Out) => {
let min_e = cut.max(s0 + frame).min(hi);
let en = (e0 + delta).clamp(min_e, hi);
Some((s0, en - s0))
}
}
}
/// Remove transitions (on unlocked tracks). Returns how many went.
pub fn remove(seq: &mut Sequence, ids: &[TransitionId]) -> Result<usize> {
if seq.all_tracks().any(|t| t.locked && t.transitions.iter().any(|x| ids.contains(&x.id))) {
return Err(EditError::Locked);
}
let mut n = 0;
for t in seq.all_tracks_mut() {
let before = t.transitions.len();
t.transitions.retain(|x| !ids.contains(&x.id));
n += before - t.transitions.len();
}
if n == 0 { Err(EditError::Nothing) } else { Ok(n) }
}
#[cfg(test)]
mod tests {
use super::*;
use filmcraft_project::{ClipId, ItemId, Label, Project, SequenceSettings, TrackItem, find_effect};
use filmcraft_time::FrameRate;
const R: FrameRate = FrameRate::FPS_24;
fn f(n: i64) -> Tick {
R.tick_of(n)
}
fn clip(id: u64, start: i64, dur: i64) -> TrackItem {
TrackItem {
id: ClipId(id),
item: ItemId(1),
name: format!("c{id}"),
label: Label::Iris,
start: f(start),
duration: f(dur),
source_in: Tick::ZERO,
speed: 1.0,
reverse: false,
enabled: true,
link: None,
group: None,
effects: vec![],
markers: vec![],
gain_db: 0.0,
frame_hold: None,
scale_to_frame: false,
essential: None,
multicam: None,
time_interpolation: Default::default(),
hold_filters: false,
field_options: None,
source_channels: Vec::new(),
audio_stream: 0,
graphic: None,
}
}
fn transition(id: u64, start: i64, dur: i64, from: Option<u64>, to: Option<u64>) -> Transition {
// any effect: these tests are about where the transition sits
let effect = find_effect("opacity").unwrap().instance();
Transition {
id: TransitionId(id),
effect,
start: f(start),
duration: f(dur),
from: from.map(ClipId),
to: to.map(ClipId),
align: TransitionAlign::CenterAtCut,
reverse: false,
}
}
/// V1: c1 0..48, c2 48..96, c3 96..144; a centred 24-frame transition on 48 (id 1), a fade in
/// at 0 (id 2) and a fade out at 144 (id 3).
fn seq() -> Sequence {
let mut p = Project::new("t");
let s = p.new_sequence("s", SequenceSettings { frame_rate: R, ..Default::default() }, 1, 0, None);
let mut q = p.sequence(s).unwrap().clone();
let t = &mut q.video_tracks[0];
t.items = vec![clip(1, 0, 48), clip(2, 48, 48), clip(3, 96, 48)];
t.transitions = vec![transition(1, 36, 24, Some(1), Some(2)), transition(2, 0, 12, None, Some(1)), transition(3, 132, 12, Some(3), None)];
q
}
fn span(q: &Sequence, id: u64) -> (Tick, Tick) {
let x = q.video_tracks[0].transitions.iter().find(|x| x.id.0 == id).unwrap();
(x.start, x.duration)
}
fn align(q: &Sequence, id: u64) -> Alignment {
let (t, x) = find(q, TransitionId(id)).unwrap();
let (_, cut, _) = bounds(t, x).unwrap();
alignment(x, cut, f(1))
}
#[test]
fn alignment_reads_the_span() {
let mut q = seq();
assert_eq!((align(&q, 1), align(&q, 2), align(&q, 3)), (Alignment::Center, Alignment::Start, Alignment::End));
set_span(&mut q, TransitionId(1), f(48), f(24), f(1), f(1)).unwrap();
assert_eq!(align(&q, 1), Alignment::Start);
set_span(&mut q, TransitionId(1), f(24), f(24), f(1), f(1)).unwrap();
assert_eq!(align(&q, 1), Alignment::End);
set_span(&mut q, TransitionId(1), f(40), f(24), f(1), f(1)).unwrap();
assert_eq!(align(&q, 1), Alignment::Custom);
// an odd centred duration sits half a frame off the cut and is still Center at Cut
set_span(&mut q, TransitionId(1), f(42), f(13), f(1), f(1)).unwrap();
assert_eq!(align(&q, 1), Alignment::Center);
}
#[test]
fn duration_changes_follow_the_alignment() {
let q = seq();
let snap = |t: Tick| R.snap(t);
// Center at Cut: both ends move
assert_eq!(start_for(&q, TransitionId(1), f(36), Alignment::Center, f(1), snap), Some(f(30)));
// Start at Cut: only the end moves; End at Cut: only the beginning
assert_eq!(start_for(&q, TransitionId(1), f(36), Alignment::Start, f(1), snap), Some(f(48)));
assert_eq!(start_for(&q, TransitionId(1), f(36), Alignment::End, f(1), snap), Some(f(12)));
// a one-sided transition keeps its edge whatever is asked
assert_eq!(start_for(&q, TransitionId(2), f(20), Alignment::Center, f(1), snap), Some(f(0)));
assert_eq!(start_for(&q, TransitionId(3), f(20), Alignment::Center, f(1), snap), Some(f(124)));
}
#[test]
fn custom_start_resizes_around_its_middle() {
let mut q = seq();
set_span(&mut q, TransitionId(1), f(40), f(20), f(1), f(1)).unwrap();
assert_eq!(start_for(&q, TransitionId(1), f(10), Alignment::Custom, f(1), |t| R.snap(t)), Some(f(45)));
}
#[test]
fn spans_stay_on_the_cut_and_inside_the_clips() {
let mut q = seq();
let before = q.clone();
for (id, start, dur) in [
(1, -1, 24), // before the outgoing clip
(1, 40, 60), // past the incoming clip's end
(1, 10, 20), // entirely before the cut
(1, 50, 10), // entirely after it
(1, 40, 0), // empty
(2, 1, 12), // a fade in must start at its clip's start
(3, 120, 12), // a fade out must end at its clip's end
(9, 0, 12), // no such transition
] {
assert!(set_span(&mut q, TransitionId(id), f(start), f(dur), f(1), f(1)).is_err(), "{id} {start} {dur}");
assert_eq!(q, before, "refused changes leave the sequence as it was");
}
// the whole of both clips is allowed
set_span(&mut q, TransitionId(1), f(0), f(96), f(1), f(1)).unwrap();
assert_eq!(span(&q, 1), (f(0), f(96)));
// Tick::MAX never overflows
assert!(set_span(&mut q, TransitionId(1), Tick(i64::MAX - 1), Tick(i64::MAX), f(1), f(1)).is_err());
}
fn drag(q: &Sequence, id: u64, edge: Option<Edge>, frames: i64) -> (i64, i64) {
let (t, x) = find(q, TransitionId(id)).unwrap();
let (st, d) = drag_span(t, x, edge, f(frames), f(1)).unwrap();
(R.frame_at(st), R.frame_at(d))
}
#[test]
fn dragging_a_transition() {
let mut q = seq();
// centred (36..60 over the cut at 48): an end trims that end only, the other stays
assert_eq!(drag(&q, 1, Some(Edge::In), -4), (32, 28));
assert_eq!(drag(&q, 1, Some(Edge::Out), -10), (36, 14));
assert_eq!(drag(&q, 1, Some(Edge::Out), -100), (36, 12), "the end stops at the cut");
assert_eq!(drag(&q, 1, Some(Edge::Out), 100), (36, 60), "and at the incoming clip's end");
assert_eq!(drag(&q, 1, Some(Edge::In), -100), (0, 60), "the start at the outgoing clip's start");
// the middle slides it, staying over the cut and inside the clips
assert_eq!(drag(&q, 1, None, 5), (41, 24));
assert_eq!(drag(&q, 1, None, 100), (48, 24));
assert_eq!(drag(&q, 1, None, -100), (24, 24));
// not centred: the other end stays
set_span(&mut q, TransitionId(1), f(40), f(20), f(1), f(1)).unwrap();
assert_eq!(drag(&q, 1, Some(Edge::In), -4), (36, 24));
assert_eq!(drag(&q, 1, Some(Edge::In), 100), (48, 12), "the start stops at the cut");
assert_eq!(drag(&q, 1, Some(Edge::Out), 6), (40, 26));
assert_eq!(drag(&q, 1, Some(Edge::Out), -100), (40, 8), "the end stops at the cut");
// a fade in keeps its start, a fade out its end; neither slides
assert_eq!(drag(&q, 2, Some(Edge::In), 3), (0, 12));
assert_eq!(drag(&q, 2, Some(Edge::Out), 3), (0, 15));
assert_eq!(drag(&q, 2, None, 3), (0, 12));
assert_eq!(drag(&q, 3, Some(Edge::Out), 3), (132, 12));
assert_eq!(drag(&q, 3, Some(Edge::In), -3), (129, 15));
// whatever comes out is a span set_span accepts
for (id, edge, n) in [(1, Some(Edge::In), -70), (1, None, 33), (2, Some(Edge::Out), 99), (3, Some(Edge::In), -99)] {
let (t, x) = find(&q, TransitionId(id)).unwrap();
let (st, d) = drag_span(t, x, edge, f(n), f(1)).unwrap();
let mut c = q.clone();
set_span(&mut c, TransitionId(id), st, d, f(1), f(1)).unwrap();
}
}
#[test]
fn locked_tracks_refuse() {
let mut q = seq();
q.video_tracks[0].locked = true;
assert_eq!(set_span(&mut q, TransitionId(1), f(40), f(20), f(1), f(1)), Err(EditError::Locked));
assert_eq!(remove(&mut q, &[TransitionId(1)]), Err(EditError::Locked));
}
#[test]
fn remove_takes_only_the_named_transitions() {
let mut q = seq();
assert_eq!(remove(&mut q, &[TransitionId(1), TransitionId(3)]), Ok(2));
let left: Vec<u64> = q.video_tracks[0].transitions.iter().map(|x| x.id.0).collect();
assert_eq!(left, vec![2]);
assert_eq!(q.video_tracks[0].items.len(), 3, "the clips stay");
assert_eq!(remove(&mut q, &[TransitionId(1)]), Err(EditError::Nothing));
}
}
+3
View File
@@ -15,6 +15,8 @@ default = []
whisper = ["filmcraft-speech/whisper"]
# Speech-model downloads over HTTPS.
speech-download = ["filmcraft-speech/download"]
# Natural voices for Text to Speech: Kokoro-82M on candle (CPU) and their download.
neural-voices = ["filmcraft-tts/kokoro", "speech-download"]
[dependencies]
filmcraft-time = { workspace = true }
@@ -35,6 +37,7 @@ filmcraft-format = { workspace = true }
filmcraft-interchange = { workspace = true }
filmcraft-captions = { workspace = true }
filmcraft-speech = { workspace = true }
filmcraft-tts = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
+1
View File
@@ -949,6 +949,7 @@ fn consolidate_duplicates(s: &mut Session, _: &Value) -> Result<Value> {
for it in pr.items.values_mut() {
match &mut it.kind {
ItemKind::Sequence(q) => {
let q = std::sync::Arc::make_mut(q);
for t in q.all_tracks_mut() {
for i in &mut t.items {
if let Some(k) = map.get(&i.item) {
+246 -20
View File
@@ -81,6 +81,15 @@ pub(crate) fn has_selection(s: &Session) -> std::result::Result<(), String> {
}
/// Clips or captions selected (Clear / Ripple Delete work on either).
fn has_any_selection(s: &Session) -> std::result::Result<(), String> {
has_seq(s)?;
if s.state.selection.is_empty() && s.state.caption_selection.is_empty() && s.state.transition_selection.is_empty() {
Err("nothing selected".into())
} else {
Ok(())
}
}
/// Clips or captions selected (Ripple Delete: a transition has no length of its own to close).
fn has_clip_or_caption_selection(s: &Session) -> std::result::Result<(), String> {
has_seq(s)?;
if s.state.selection.is_empty() && s.state.caption_selection.is_empty() { Err("nothing selected".into()) } else { Ok(()) }
}
@@ -1204,6 +1213,11 @@ fn build() -> Vec<CommandSpec> {
cmd!("edit.pasteInsert", "Paste Insert", ["Edit"], Some("Cmd+Shift+V"), "{}", has_clipboard, |s, _| paste(s, true)),
cmd!("edit.clear", "Clear", ["Edit"], Some("Backspace"), r#"{"clips":[id]?}"#, has_any_selection, |s, p| {
if p.get("clips").is_none() && p.get("clip").is_none() && s.state.selection.is_empty() {
// a selected transition (clicked in the Timeline) goes, as a clip would
if !s.state.transition_selection.is_empty() {
let ids = s.state.transition_selection.clone();
return remove_transitions(s, &ids, "Clear");
}
let caps = s.state.caption_selection.clone();
return crate::captions::delete(s, &caps, false);
}
@@ -1215,7 +1229,7 @@ fn build() -> Vec<CommandSpec> {
})?;
Ok(Value::Null)
}),
cmd!("edit.rippleDelete", "Ripple Delete", ["Edit"], Some("Shift+Delete"), r#"{"clips":[id]?}"#, has_any_selection, |s, p| {
cmd!("edit.rippleDelete", "Ripple Delete", ["Edit"], Some("Shift+Delete"), r#"{"clips":[id]?}"#, has_clip_or_caption_selection, |s, p| {
if p.get("clips").is_none() && p.get("clip").is_none() && s.state.selection.is_empty() {
let caps = s.state.caption_selection.clone();
return crate::captions::delete(s, &caps, true);
@@ -1363,10 +1377,70 @@ fn build() -> Vec<CommandSpec> {
let sel = clips_p(s, p);
let linked = sel.iter().all(|c| s.active_sequence().and_then(|q| q.find_item(*c)).is_some_and(|(_, i)| i.link.is_some()));
s.edit_sequence(if linked { "Unlink" } else { "Link" }, |q, ctx, _| {
let l = if linked { None } else { Some(ctx.alloc()) };
for c in &sel {
if let Some((_, i)) = q.find_item_mut(*c) {
i.link = l;
if linked {
for c in &sel {
if let Some((_, i)) = q.find_item_mut(*c) {
i.link = None;
}
}
} else {
// each selected clip with its track kind, worked out once
type Spec = (ClipId, filmcraft_project::ItemId, filmcraft_time::TimeRange);
let (mut v_clips, mut audio_specs): (Vec<Spec>, Vec<Spec>) = (Vec::new(), Vec::new());
for c in &sel {
if let Some((tid, i)) = q.find_item(*c) {
if q.video_tracks.iter().any(|t| t.id == tid) {
v_clips.push((*c, i.item, i.range()));
} else {
audio_specs.push((*c, i.item, i.range()));
}
}
}
if !audio_specs.is_empty() && v_clips.len() > 1 {
let mut linked_audios = std::collections::HashSet::new();
for (vid, item_id, vrange) in &v_clips {
// Pair with overlapping audio clips from the same item, or overlapping audio clips if no same-item audio
let matching: Vec<ClipId> = audio_specs
.iter()
.filter(|(aid, aitem, arange)| !linked_audios.contains(aid) && aitem == item_id && arange.overlaps(vrange))
.map(|(aid, _, _)| *aid)
.collect();
let to_link = if !matching.is_empty() {
matching
} else {
audio_specs
.iter()
.filter(|(aid, _, arange)| !linked_audios.contains(aid) && arange.overlaps(vrange))
.map(|(aid, _, _)| *aid)
.collect()
};
// a video clip with no audio partner stays unlinked: a one-member link would read as "linked"
let l = (!to_link.is_empty()).then(|| ctx.alloc());
if let Some((_, vi)) = q.find_item_mut(*vid) {
vi.link = l;
}
for aid in to_link {
linked_audios.insert(aid);
if let Some((_, ai)) = q.find_item_mut(aid) {
ai.link = l;
}
}
}
// selected audio clips with no overlapping video stay unlinked
for (aid, _, _) in audio_specs {
if !linked_audios.contains(&aid)
&& let Some((_, ai)) = q.find_item_mut(aid)
{
ai.link = None;
}
}
} else {
let l = ctx.alloc();
for c in &sel {
if let Some((_, i)) = q.find_item_mut(*c) {
i.link = Some(l);
}
}
}
}
Ok(())
@@ -1557,10 +1631,20 @@ fn build() -> Vec<CommandSpec> {
"Edit Transition Settings",
[],
None,
r#"{"transition":id,"params":{param:value}?,"reverse":bool?,"reset":bool?}"#,
r#"{"transition":id,"params":{param:value}?,"reverse":bool?,"reset":bool?,"duration":ticks?|"frames":n?,"align":"center"|"start"|"end"?|"start":ticks?,"merge":bool?,"begin":bool?}"#,
has_seq,
set_transition
),
cmd!("sequence.removeTransition", "Clear Transition", [], None, r#"{"transitions":[id]?}"#, has_seq, |s, p| {
let ids: Vec<TransitionId> = match p.get("transitions").or_else(|| p.get("transition")) {
Some(v) => transition_ids_p(v).ok_or_else(|| bad("sequence.removeTransition", "`transitions` is a list of transition ids"))?,
None => s.state.transition_selection.clone(),
};
if ids.is_empty() {
return Err(bad("sequence.removeTransition", "no transition given or selected"));
}
remove_transitions(s, &ids, "Clear Transition")
}),
cmd!("sequence.closeGap", "Close Gap", ["Sequence"], None, r#"{"track":"V1"|id,"time":ticks}"#, has_seq, |s, p| {
let tr = track_p(s, p, "track", "sequence.closeGap")?.ok_or_else(|| bad("sequence.closeGap", "need `track`"))?;
let t = time_p(s, p, "").unwrap_or(s.playhead());
@@ -2052,13 +2136,18 @@ fn build() -> Vec<CommandSpec> {
Ok(json!({"clips": ids.iter().map(|c| c.0).collect::<Vec<_>>()}))
}
),
cmd!("timeline.select", "Select Clips", [], None, r#"{"clips":[id],"add":bool,"toggle":bool}"#, has_seq, |s, p| {
cmd!("timeline.select", "Select Clips", [], None, r#"{"clips":[id],"transitions":[id]?,"add":bool,"toggle":bool}"#, has_seq, |s, p| {
if let Some(ids) = p.get("transitions") {
return select_transitions(s, ids, bool_p(p, "add").unwrap_or(false), bool_p(p, "toggle").unwrap_or(false));
}
let clips: Vec<ClipId> =
p.get("clips").and_then(Value::as_array).map(|a| a.iter().filter_map(|v| v.as_u64().map(ClipId)).collect()).unwrap_or_default();
let clips = with_links(s, &clips);
// selecting clips leaves trim mode (Premiere: clip and edit point selections are exclusive)
s.state.edit_points.clear();
s.state.trim_shift = Default::default();
// and a clip selection replaces a transition one
s.state.transition_selection.clear();
if bool_p(p, "toggle").unwrap_or(false) {
for c in clips {
if let Some(i) = s.state.selection.iter().position(|x| *x == c) {
@@ -2432,13 +2521,14 @@ fn build() -> Vec<CommandSpec> {
"Set Effect Parameter",
[],
None,
r##"{"clip":id,"effect":"motion"|index,"param":str,"mask":n?,"value":num|[x,y]|"#rrggbb"|bool|path,"time":ticks?,"merge":bool?,"begin":bool?}"##,
r##"{"clip":id,"effect":"motion"|index,"param":str,"mask":n?,"value":num|[x,y]|"#rrggbb"|bool|path|"option name" (a choice takes its index or its name),"time":ticks? (where the keyframe goes when the parameter is animated; a parameter that is not animated gets a new static value whatever the time),"keyframe":bool? (true: write a keyframe at `time` and animate the parameter),"merge":bool?,"begin":bool?}"##,
has_seq,
|s, p| {
let c = clip_p(p, "clip").ok_or_else(|| bad("effects.setParam", "need `clip`"))?;
let pid = str_p(p, "param").ok_or_else(|| bad("effects.setParam", "need `param`"))?.to_string();
let val = p.get("value").cloned().ok_or_else(|| bad("effects.setParam", "need `value`"))?;
let eff = p.get("effect").cloned().unwrap_or(json!("motion"));
let keyframe = bool_p(p, "keyframe").unwrap_or(false);
let ph = s.playhead();
let tl = time_p(s, p, "").unwrap_or(ph);
let pq = p.clone();
@@ -2447,7 +2537,7 @@ fn build() -> Vec<CommandSpec> {
if bool_p(p, "begin").unwrap_or(false) {
s.history.merge_key = None;
}
s.edit_sequence_as("Change Effect Parameter", merge.as_deref(), |q, _, _| {
let keyframes = s.edit_sequence_as("Change Effect Parameter", merge.as_deref(), |q, _, _| {
let (_, it) = q.find_item_mut(c).ok_or(filmcraft_edit::EditError::NoItem(c))?;
let mt = it.source_time_at(tl.clamp(it.start, (it.end() - Tick(1)).max(it.start)));
let e = match &eff {
@@ -2456,6 +2546,22 @@ fn build() -> Vec<CommandSpec> {
_ => None,
}
.ok_or_else(|| bad("effects.setParam", "no such effect on clip"))?;
// the parameter's definition (a mask's own parameters have none)
let def = if pq.get("mask").is_none_or(Value::is_null) { e.def().and_then(|d| d.param(&pid)) } else { None };
// a choice takes its option name as well as its index, as transition parameters do
let val = match (def.map(|d| &d.kind), &val) {
(Some(filmcraft_project::ParamKind::Choice(opts)), Value::String(name)) => {
let i = opts
.iter()
.position(|o| o.eq_ignore_ascii_case(name))
.ok_or_else(|| bad("effects.setParam", format!("`{pid}` must be one of {opts:?}")))?;
json!(i)
}
_ => val.clone(),
};
if keyframe && def.is_some_and(|d| !d.animatable) {
return Err(bad("effects.setParam", format!("`{pid}` cannot be animated")));
}
if eff == json!("opacity") || eff == json!("motion") {
e.enabled = true;
}
@@ -2467,10 +2573,16 @@ fn build() -> Vec<CommandSpec> {
}
let prm = crate::masks::target_param(e, &pq, &pid).ok_or_else(|| bad("effects.setParam", format!("no param `{pid}`")))?;
let v = json_to_param(&prm.value, &val).ok_or_else(|| bad("effects.setParam", "value has the wrong type"))?;
prm.set_at(mt, v);
Ok(())
if keyframe {
// the stopwatch and the value in one step
prm.put_keyframe(mt, v);
} else {
prm.set_at(mt, v);
}
Ok(prm.keyframes.len())
})?;
Ok(Value::Null)
// 0 keyframes: the value is static (`time` was not used)
Ok(json!({"keyframes": keyframes}))
}
),
cmd!("effects.toggleAnimation", "Toggle Animation", [], None, r#"{"clip":id,"effect":str|index,"param":str,"mask":n?}"#, has_seq, |s, p| {
@@ -2718,6 +2830,7 @@ fn build() -> Vec<CommandSpec> {
v.extend(crate::scopes::commands());
v.extend(crate::remix::commands());
v.extend(crate::voiceover::commands());
v.extend(crate::narration::commands());
v.extend(crate::keyboard::commands());
v.extend(crate::project_panel::commands());
v.extend(crate::media_browser::commands());
@@ -3200,13 +3313,109 @@ fn set_transition_params(e: &mut filmcraft_project::EffectInstance, params: &Val
Ok(())
}
/// `sequence.setTransition`: edit an applied transition's settings (Effect Controls).
/// A JSON list of transition ids (or one id).
fn transition_ids_p(v: &Value) -> Option<Vec<TransitionId>> {
match v {
Value::Array(a) => a.iter().map(|x| x.as_u64().map(TransitionId)).collect(),
x => x.as_u64().map(|i| vec![TransitionId(i)]),
}
}
/// `timeline.select {"transitions": […]}`: select transitions (clicking one in the Timeline). It
/// replaces the clip, edit point and caption selections, as a clip selection replaces it.
fn select_transitions(s: &mut Session, ids: &Value, add: bool, toggle: bool) -> Result<Value> {
const CMD: &str = "timeline.select";
let ids = transition_ids_p(ids).ok_or_else(|| bad(CMD, "`transitions` is a list of transition ids"))?;
let q = s.active_sequence().ok_or(EngineError::NoSequence)?;
if let Some(missing) = ids.iter().find(|id| !q.all_tracks().any(|t| t.transitions.iter().any(|x| x.id == **id))) {
return Err(bad(CMD, format!("no transition {}", missing.0)));
}
if toggle {
for id in ids {
match s.state.transition_selection.iter().position(|x| *x == id) {
Some(i) => {
s.state.transition_selection.remove(i);
}
None => s.state.transition_selection.push(id),
}
}
} else if add {
for id in ids {
if !s.state.transition_selection.contains(&id) {
s.state.transition_selection.push(id);
}
}
} else {
s.state.transition_selection = ids;
}
s.state.selection.clear();
s.state.caption_selection.clear();
s.state.edit_points.clear();
s.state.trim_shift = Default::default();
Ok(json!({"selection": [], "transitionSelection": s.state.transition_selection.iter().map(|t| t.0).collect::<Vec<_>>()}))
}
/// Remove transitions in one undo step and drop them from the selection.
fn remove_transitions(s: &mut Session, ids: &[TransitionId], label: &str) -> Result<Value> {
let ids = ids.to_vec();
let n = s.edit_sequence(label, |q, _, st| {
let n = edit::transitions::remove(q, &ids)?;
st.transition_selection.retain(|x| !ids.contains(x));
Ok(n)
})?;
Ok(json!({"removed": n}))
}
/// `sequence.setTransition`: edit an applied transition (Effect Controls, the Timeline): its
/// settings, Reverse, and where it sits. `duration` / `frames` resize it the way its alignment
/// says (Center at Cut and Custom Start: both ends; Start at Cut: the end; End at Cut: the
/// beginning), `align` re-aligns it on its cut, `start` moves it (dragging it over the cut; with
/// `duration`, dragging one edge). One undo step; with `merge`, a drag's steps share one.
fn set_transition(s: &mut Session, p: &Value) -> Result<Value> {
const CMD: &str = "sequence.setTransition";
let id = p.get("transition").and_then(Value::as_u64).ok_or_else(|| bad(CMD, "need `transition`"))?;
let tid = TransitionId(id);
let q = s.active_sequence().ok_or(EngineError::NoSequence)?;
let found = q.video_tracks.iter().chain(q.audio_tracks.iter()).find_map(|tr| tr.transitions.iter().find(|x| x.id.0 == id)).cloned();
let mut cur = found.ok_or_else(|| bad(CMD, format!("no transition {id}")))?;
let rate = q.settings.frame_rate.sane();
let frame = rate.frame_duration();
let found = q.all_tracks().find_map(|tr| tr.transitions.iter().find(|x| x.id == tid).map(|x| (tr, x)));
let (track, cur) = found.ok_or_else(|| bad(CMD, format!("no transition {id}")))?;
let mut cur = cur.clone();
let (lo, cut, hi) = edit::transitions::bounds(track, &cur).ok_or_else(|| bad(CMD, "the transition's clips are gone"))?;
// times outside the clips it joins are refused before any frame arithmetic (hostile input)
let room = Tick(hi.0.saturating_sub(lo.0));
let duration = match (p.get("frames").and_then(Value::as_i64), p.get("duration").and_then(Value::as_i64)) {
(Some(_), Some(_)) => return Err(bad(CMD, "give `duration` or `frames`, not both")),
(Some(f), None) if (1..=10_000_000).contains(&f) && rate.tick_of(f) <= room => Some(rate.tick_of(f)),
(None, Some(d)) if d > 0 && d <= room.0 => Some(rate.snap_nearest(Tick(d)).max(frame)),
(Some(_), None) | (None, Some(_)) => return Err(bad(CMD, "a transition lasts at least one frame and can't extend past the clips it joins")),
(None, None) => None,
};
let align = match str_p(p, "align") {
Some(a) => Some(edit::transitions::Alignment::parse(a).ok_or_else(|| bad(CMD, format!("`align` is center, start or end, not `{a}`")))?),
None => None,
};
let start = match p.get("start").and_then(Value::as_i64) {
Some(t) if (lo.0..=hi.0).contains(&t) => Some(rate.snap_nearest(Tick(t))),
Some(_) => return Err(bad(CMD, "a transition can't extend past the clips it joins")),
None => None,
};
if start.is_some() && align.is_some() {
return Err(bad(CMD, "give `start` or `align`, not both"));
}
let span = if duration.is_some() || align.is_some() || start.is_some() {
let d = duration.unwrap_or(cur.duration);
let st = match start {
Some(st) => st,
None => {
let al = align.unwrap_or_else(|| edit::transitions::alignment(&cur, cut, frame));
edit::transitions::start_for(q, tid, d, al, frame, |t| rate.snap(t)).ok_or_else(|| bad(CMD, format!("no transition {id}")))?
}
};
Some((st, d))
} else {
None
};
if p.get("reset").and_then(Value::as_bool).unwrap_or(false)
&& let Some(def) = cur.effect.def()
{
@@ -3219,16 +3428,32 @@ fn set_transition(s: &mut Session, p: &Value) -> Result<Value> {
cur.reverse = r;
}
let name = cur.effect.def().map_or("Transition", |d| d.name).to_string();
s.edit_sequence(&format!("Edit {name}"), |q, _, _| {
if bool_p(p, "begin").unwrap_or(false) {
s.history.merge_key = None;
}
let merge = bool_p(p, "merge").unwrap_or(false).then(|| format!("setTransition:{id}"));
let after = s.edit_sequence_as(&format!("Edit {name}"), merge.as_deref(), |q, _, _| {
if let Some((st, d)) = span {
edit::transitions::set_span(q, tid, st, d, frame, frame)?;
}
let mut after = None;
for tr in q.video_tracks.iter_mut().chain(q.audio_tracks.iter_mut()) {
if let Some(x) = tr.transitions.iter_mut().find(|x| x.id.0 == id) {
if let Some(x) = tr.transitions.iter_mut().find(|x| x.id == tid) {
x.effect = cur.effect.clone();
x.reverse = cur.reverse;
after = Some((x.start, x.duration));
}
}
Ok(())
Ok(after)
})?;
Ok(json!({"transition": id, "effect": cur.effect.effect, "reverse": cur.reverse}))
let (st, d) = after.unwrap_or((cur.start, cur.duration));
let al = s
.active_sequence()
.and_then(|q| {
q.all_tracks().find_map(|tr| tr.transitions.iter().find(|x| x.id == tid).and_then(|x| edit::transitions::bounds(tr, x).map(|b| (x.clone(), b.1))))
})
.map(|(x, cut)| edit::transitions::alignment(&x, cut, frame).id());
Ok(json!({"transition": id, "effect": cur.effect.effect, "reverse": cur.reverse, "start": st.0, "duration": d.0, "align": al}))
}
fn apply_transition(s: &mut Session, p: &Value, kind: TrackKind) -> Result<Value> {
@@ -3345,7 +3570,7 @@ pub fn inspect_sequence(s: &Session, id: ItemId, q: &filmcraft_project::Sequence
"enabled": i.enabled, "link": i.link, "label": i.label.name(),
"effects": i.effects.iter().map(|e| json!({"effect": e.effect, "enabled": e.enabled, "masks": e.masks.len(), "params": e.params.iter().map(|(k, p)| (k.clone(), json!({"value": format!("{:?}", p.value), "keyframes": p.keyframes.len()}))).collect::<serde_json::Map<_, _>>()})).collect::<Vec<_>>(),
})).collect::<Vec<_>>(),
"transitions": t.transitions.iter().map(|x| json!({"id": x.id.0, "effect": x.effect.effect, "start": x.start.0, "duration": x.duration.0, "from": x.from.map(|c| c.0), "to": x.to.map(|c| c.0), "reverse": x.reverse, "params": x.effect.params.iter().map(|(k, v)| (k.clone(), serde_json::to_value(&v.value).unwrap_or_default())).collect::<serde_json::Map<_, _>>()})).collect::<Vec<_>>(),
"transitions": t.transitions.iter().map(|x| json!({"id": x.id.0, "effect": x.effect.effect, "start": x.start.0, "duration": x.duration.0, "from": x.from.map(|c| c.0), "to": x.to.map(|c| c.0), "reverse": x.reverse, "align": filmcraft_edit::transitions::bounds(t, x).map(|b| filmcraft_edit::transitions::alignment(x, b.1, rate.frame_duration()).id()), "params": x.effect.params.iter().map(|(k, v)| (k.clone(), serde_json::to_value(&v.value).unwrap_or_default())).collect::<serde_json::Map<_, _>>()})).collect::<Vec<_>>(),
})
};
json!({
@@ -3362,6 +3587,7 @@ pub fn inspect_sequence(s: &Session, id: ItemId, q: &filmcraft_project::Sequence
"video": q.video_tracks.iter().map(tr).collect::<Vec<_>>(),
"audio": q.audio_tracks.iter().map(tr).collect::<Vec<_>>(),
"selection": s.state.selection.iter().map(|c| c.0).collect::<Vec<_>>(),
"transitionSelection": s.state.transition_selection.iter().map(|t| t.0).collect::<Vec<_>>(),
})
}
@@ -211,3 +211,56 @@ fn project_delete_takes_explicit_bins_and_items_without_a_selection() {
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(s.project.items.len(), before);
}
#[test]
fn link_multiple_video_and_audio_pairs_creates_one_link_per_pair() {
let mut s = demo();
let v_clips = v1(&s);
let a_clips = a1(&s);
assert!(v_clips.len() >= 3 && a_clips.len() >= 3);
let (v0, v1, v2) = (v_clips[0].id, v_clips[1].id, v_clips[2].id);
let (a0, a1, a2) = (a_clips[0].id, a_clips[1].id, a_clips[2].id);
// First, unlink all three pairs
let r = s.execute("clip.link", json!({"clips": [v0.0, a0.0, v1.0, a1.0, v2.0, a2.0]})).unwrap();
assert_eq!(r["linked"], false);
for cid in [v0, a0, v1, a1, v2, a2] {
assert_eq!(clip(&s, cid).unwrap().link, None);
}
// Now, run clip.link on all six clips. In Premiere parity (#507),
// it should create 3 separate links (one per pair), not one big link for all six.
let r = s.execute("clip.link", json!({"clips": [v0.0, a0.0, v1.0, a1.0, v2.0, a2.0]})).unwrap();
assert_eq!(r["linked"], true);
let l0 = clip(&s, v0).unwrap().link;
let la0 = clip(&s, a0).unwrap().link;
let l1 = clip(&s, v1).unwrap().link;
let la1 = clip(&s, a1).unwrap().link;
let l2 = clip(&s, v2).unwrap().link;
let la2 = clip(&s, a2).unwrap().link;
assert!(l0.is_some() && l1.is_some() && l2.is_some());
assert_eq!(l0, la0, "v0 linked to a0");
assert_eq!(l1, la1, "v1 linked to a1");
assert_eq!(l2, la2, "v2 linked to a2");
assert_ne!(l0, l1, "different pairs must not share the same link id");
assert_ne!(l1, l2, "different pairs must not share the same link id");
assert_ne!(l0, l2, "different pairs must not share the same link id");
}
/// A video clip with no overlapping selected audio, and audio with no overlapping video, stay
/// unlinked: a one-member link would make the clip read as "linked" with no partner.
#[test]
fn link_leaves_clips_without_a_partner_unlinked() {
let mut s = demo();
let (v, a) = (v1(&s), a1(&s));
let (v0, v1, a0, a2) = (v[0].id, v[1].id, a[0].id, a[2].id);
s.execute("clip.link", json!({"clips": [v0.0, v1.0, a0.0, a2.0]})).unwrap();
s.execute("clip.link", json!({"clips": [v0.0, v1.0, a0.0, a2.0]})).unwrap();
assert!(clip(&s, v0).unwrap().link.is_some());
assert_eq!(clip(&s, v0).unwrap().link, clip(&s, a0).unwrap().link);
assert_eq!(clip(&s, v1).unwrap().link, None, "v1 has no overlapping audio in the selection");
assert_eq!(clip(&s, a2).unwrap().link, None, "a2 has no overlapping video in the selection");
}
+1 -1
View File
@@ -291,7 +291,7 @@ fn set_controls(s: &mut Session, clip: ClipId, values: &serde_json::Map<String,
props.insert(c.param.clone(), v.clone());
}
}
set_props(s, clip, ei, &props, tl, "Change Template Property")?;
set_props(s, clip, ei, &props, tl, "Change Template Property", None)?;
n += 1;
}
Ok(n)
+22 -8
View File
@@ -14,7 +14,7 @@ use filmcraft_render::graphic_clip::{item_layer_specs, layer_local_bounds, layer
use filmcraft_time::{Tick, TimeRange};
use serde_json::{Value, json};
use crate::commands::{CommandSpec, always, bad, f64_p, has_seq, str_p, time_p, u64_p};
use crate::commands::{CommandSpec, always, bad, bool_p, f64_p, has_seq, str_p, time_p, u64_p};
use crate::{EngineError, Result, Session};
type Run = fn(&mut Session, &Value) -> Result<Value>;
@@ -262,10 +262,19 @@ fn apply_props(e: &mut EffectInstance, props: &serde_json::Map<String, Value>, m
}
/// Set properties `props` on a layer (keyframe-aware at time `tl`). Changing the text keeps
/// per-character styles on their characters.
pub(crate) fn set_props(s: &mut Session, clip: ClipId, eidx: usize, props: &serde_json::Map<String, Value>, tl: Tick, label: &str) -> Result<()> {
/// per-character styles on their characters. `merge` folds consecutive calls with the same key
/// into one undo step (a drag is one undoable change, like `effects.setParam`).
pub(crate) fn set_props(
s: &mut Session,
clip: ClipId,
eidx: usize,
props: &serde_json::Map<String, Value>,
tl: Tick,
label: &str,
merge: Option<&str>,
) -> Result<()> {
let props = props.clone();
s.edit_sequence(label, |q, _, _| {
s.edit_sequence_as(label, merge, |q, _, _| {
let (_, it) = q.find_item_mut(clip).ok_or(filmcraft_edit::EditError::NoItem(clip))?;
let mt = it.source_time_at(tl.clamp(it.start, it.end() - Tick(1)));
let e = it.effects.get_mut(eidx).ok_or_else(|| bad("graphics.set", "no such layer"))?;
@@ -744,7 +753,7 @@ fn menu_commands() -> Vec<CommandSpec> {
props.insert("sides".into(), json!(n));
let ph = s.playhead();
let before = s.history.undo.len();
set_props(s, clip, ei, &props, ph, "Change Graphic Property")?;
set_props(s, clip, ei, &props, ph, "Change Graphic Property", None)?;
if s.history.undo.len() > before {
s.history.undo.pop();
}
@@ -963,7 +972,7 @@ pub fn commands() -> Vec<CommandSpec> {
let mut props = serde_json::Map::new();
props.insert("text".into(), Value::String(text));
let ph = s.playhead();
set_props(s, clip, ei, &props, ph, "Edit Text")?;
set_props(s, clip, ei, &props, ph, "Edit Text", None)?;
if merge && s.history.undo.len() >= 2 {
// keep the snapshot from before the typing session
s.history.undo.pop();
@@ -976,14 +985,19 @@ pub fn commands() -> Vec<CommandSpec> {
"Set Graphic Properties",
&[],
None,
r##"{"clip":id?,"layer":n|name?,"props":{"font":"Inter","font_style":"Bold","size":120,"align":"center","tracking":50,"leading":0,"fill_color":"#ffcc00","stroke":true,"stroke_width":6,"background":true,"shadow":true,"position":[x,y],"scale":100,"rotation":0,"opacity":100,…},"time":ticks?}"##,
r##"{"clip":id?,"layer":n|name?,"props":{"font":"Inter","font_style":"Bold","size":120,"align":"center","tracking":50,"leading":0,"fill_color":"#ffcc00","stroke":true,"stroke_width":6,"background":true,"shadow":true,"position":[x,y],"scale":100,"rotation":0,"opacity":100,…},"time":ticks?,"merge":bool?,"begin":bool?}"##,
has_graphic,
|s, p| {
let clip = target_clip(s, p).ok_or_else(|| bad("graphics.set", "no graphic clip"))?;
let (_, ei) = layer_effect_index(s, clip, p)?;
let props = p.get("props").and_then(Value::as_object).ok_or_else(|| bad("graphics.set", "need `props`"))?.clone();
let tl = time_p(s, p, "").unwrap_or_else(|| s.playhead());
set_props(s, clip, ei, &props, tl, "Change Graphic Property")?;
// `merge`: a drag of a property is one undo step; `begin` starts a new one (#201)
let merge = bool_p(p, "merge").unwrap_or(false).then(|| format!("graphics.set:{}:{ei}", clip.0));
if bool_p(p, "begin").unwrap_or(false) {
s.history.merge_key = None;
}
set_props(s, clip, ei, &props, tl, "Change Graphic Property", merge.as_deref())?;
Ok(Value::Null)
},
),
+32
View File
@@ -153,6 +153,38 @@ fn typing_coalesces_into_one_undo_step_and_props_set() {
assert_eq!(text_of(&layers(&s, clip)[0]), "");
}
/// A drag of a property is one undo step: `graphics.set` with `merge` folds consecutive calls
/// into the step the first change of the press began (`begin`), so one undo restores the
/// original value (the color picker in Essential Graphics was committing one step per frame).
#[test]
fn a_property_drag_is_one_undo_step() {
let mut s = demo();
let r = s.execute("graphics.newText", json!({"text": "Title"})).unwrap();
let clip = ClipId(r["clip"].as_u64().unwrap());
s.execute("graphics.set", json!({"clip": clip.0, "layer": 0, "props": {"shadow_color": [0.0, 0.0, 0.0, 1.0]}})).unwrap();
let color = |s: &Session| -> [f32; 4] {
let q = s.active_sequence().unwrap();
let (_, it) = q.find_item(clip).unwrap();
let e = it.effects.iter().find(|e| e.effect == "graphic_text").unwrap();
e.param("shadow_color").unwrap().value_at(Tick::ZERO).as_color().unwrap()
};
let before = s.history.undo.len();
// the drag: the first change of the press begins the step, the rest fold into it
for (i, c) in [[0.1, 0.0, 0.0, 1.0], [0.2, 0.0, 0.0, 1.0], [0.3, 0.0, 0.0, 1.0], [0.4, 0.0, 0.0, 1.0]].iter().enumerate() {
s.execute("graphics.set", json!({"clip": clip.0, "layer": 0, "props": {"shadow_color": c}, "merge": true, "begin": i == 0})).unwrap();
}
assert_eq!(s.history.undo.len(), before + 1, "one undo step for the whole drag");
assert_eq!(color(&s)[0], 0.4);
// a second drag (a new press) begins its own step
s.execute("graphics.set", json!({"clip": clip.0, "layer": 0, "props": {"shadow_color": [1.0, 0.0, 0.0, 1.0]}, "merge": true, "begin": true})).unwrap();
s.execute("graphics.set", json!({"clip": clip.0, "layer": 0, "props": {"shadow_color": [0.0, 1.0, 0.0, 1.0]}, "merge": true})).unwrap();
assert_eq!(s.history.undo.len(), before + 2, "a new press begins a new step");
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(color(&s)[0], 0.4, "undo takes back only the second drag");
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(color(&s)[0], 0.0, "and then the first");
}
#[test]
fn align_and_distribute_layers() {
let mut s = demo();
+13
View File
@@ -29,6 +29,7 @@ pub mod media_browser;
pub mod media_pool;
pub mod mixer;
pub mod multicam;
pub mod narration;
pub mod panels;
pub mod perf;
pub mod presets;
@@ -261,6 +262,10 @@ pub struct EditorState {
/// Selected captions (caption tracks / Captions panel).
#[serde(default)]
pub caption_selection: Vec<ClipId>,
/// Selected transitions (clicked in the Timeline): Effect Controls shows one, Delete removes
/// them. Selecting clips clears it and selecting transitions clears the clip selection.
#[serde(default)]
pub transition_selection: Vec<filmcraft_project::TransitionId>,
/// Selected layers (indices among the graphic layers, 0 = back) of the selected graphic clip.
#[serde(default)]
pub graphic_layers: Vec<usize>,
@@ -349,6 +354,10 @@ pub struct Session {
pub mcrec: multicam::Recorder,
/// Voice-over recording: the input device and the take in progress.
pub voiceover: voiceover::VoiceOver,
/// The last `tts.preview` result, for the host to play (Text to Speech ▸ Preview).
pub tts_preview: Option<Arc<filmcraft_tts::Audio>>,
/// Synthesized narrations (`tts.render` fills it from a background job).
pub tts_cache: narration::SynthCache,
/// Dynamic (J/K/L) trimming and trim-mode loop playback in progress.
pub trim_play: trim::TrimPlayback,
/// Keyboard shortcuts (active bindings, presets; `shortcuts.*` commands).
@@ -451,6 +460,8 @@ impl Session {
mixrec: Default::default(),
mcrec: Default::default(),
voiceover: Default::default(),
tts_preview: None,
tts_cache: Default::default(),
trim_play: Default::default(),
shortcuts: shortcuts::Shortcuts::new(),
offline: Default::default(),
@@ -1065,6 +1076,8 @@ mod mixer_tests;
#[cfg(test)]
mod multicam_tests;
#[cfg(test)]
mod narration_tests;
#[cfg(test)]
mod nest_editing_tests;
#[cfg(test)]
mod nest_fidelity_tests;
+533
View File
@@ -0,0 +1,533 @@
//! Text to Speech (narrations): write what the narrator says, pick a voice, and get an audio clip.
//!
//! | command | does |
//! |---|---|
//! | `tts.voices` | languages, voices, vocal pitches, pace limits and the pause marker |
//! | `tts.preview` | synthesize without changing the project (`sample: true` speaks the voice's sample sentence); the host plays [`Session::tts_preview`]; `path` also writes a WAV |
//! | `tts.create` | synthesize, write `Narration <n>.wav`, import it and place it at the playhead (one undo step) |
//! | `tts.edit` | change a narration clip's script or voice settings and regenerate it in place (one undo step) |
//! | `tts.inspect` | the narration settings of a clip or item |
//! | `tts.render` | synthesize in a background job into the cache, so the UI's create / edit / preview return at once |
//! | `tts.downloadVoices` | download the natural voices (Kokoro-82M, 9 US English voices, CMUdict; 336 MB, SHA-256 checked) as a background job |
//!
//! **Where a new narration goes.** At `time` (default: the playhead) on `track` if given (it must
//! be free there), else on the first targeted audio track that is free for the narration's length,
//! else the first free audio track, else on a new audio track. Existing clips are never overwritten.
//!
//! **Editing keeps the clip's length** (user decision, 2026-10-07). The clip keeps its start,
//! duration and speed. When the new speech is longer than the clip, the clip still ends where it
//! did but the file holds the whole speech, so trimming the clip's end out reveals the rest. When it
//! is shorter, the file is padded with silence to the clip's length. The clip's source In is reset
//! to the start of the new speech. Other clips (including other clips of the old narration) are not
//! changed. Each edit writes a new file; the old item and its narration stay, so undo is exact.
//!
//! Files are named `Narration <n>.wav` (mono 32-bit float at the voice's rate, 24 kHz) and saved
//! where voice-over recordings go (Scratch Disks ▸ Captured, else next to the project, else
//! `Narrations` in the data or temporary directory); an edit writes next to the previous file.
use std::sync::{Arc, Mutex, PoisonError};
use serde_json::{Value, json};
use filmcraft_project::{ClipId, ItemId, ItemKind, MediaRef, Narration, TrackId, TrackKind, VocalPitch};
use filmcraft_time::{Tick, TimeRange};
use filmcraft_tts as tts;
use crate::commands::{CommandSpec, always, bad, has_seq, str_p, time_p, u64_p};
use crate::{EngineError, Result, Session};
/// Item name prefix of generated narrations.
const BASE_NAME: &str = "Narration";
/// The narration of a project item.
pub fn narration_of(s: &Session, item: ItemId) -> Option<&Narration> {
s.project.narrations.get(&item)
}
/// The narration clip a command works on: `clip`, else the first selected clip whose item is a
/// narration.
pub fn target_clip(s: &Session, p: &Value) -> Option<ClipId> {
let seq = s.active_sequence()?;
let is_narration = |c: ClipId| seq.find_item(c).is_some_and(|(_, it)| s.project.narrations.contains_key(&it.item));
match u64_p(p, "clip") {
Some(c) => Some(ClipId(c)).filter(|c| is_narration(*c)),
None => s.state.selection.iter().copied().find(|c| is_narration(*c)),
}
}
fn has_narration_clip(s: &Session) -> std::result::Result<(), String> {
has_seq(s)?;
target_clip(s, &Value::Null).map(|_| ()).ok_or_else(|| "select a narration clip".into())
}
/// Script and settings from params, on top of `base`.
fn settings(cmd: &str, p: &Value, base: Option<&Narration>) -> Result<Narration> {
// a setting of the wrong type is an error, never silently replaced by a default
for key in ["text", "voice", "pitch"] {
if p.get(key).is_some_and(|v| !v.is_string()) {
return Err(bad(cmd, format!("`{key}` must be a string")));
}
}
let text = match str_p(p, "text") {
Some(t) => t.to_string(),
None => base.map(|b| b.text.clone()).ok_or_else(|| bad(cmd, "`text` is required"))?,
};
let voice = str_p(p, "voice").map(str::to_string).or_else(|| base.map(|b| b.voice.clone())).unwrap_or_else(|| tts::default_voice_id().to_string());
let info = tts::voices_in(None).into_iter().find(|v| v.id == voice).ok_or_else(|| bad(cmd, format!("unknown voice `{voice}` (see tts.voices)")))?;
let pitch = match str_p(p, "pitch") {
Some(v) => VocalPitch::from_id(v).ok_or_else(|| bad(cmd, format!("unknown pitch `{v}` (extraLow, low, default, high, extraHigh)")))?,
None => base.map(|b| b.pitch).unwrap_or_default(),
};
let pace = match p.get("pace") {
Some(v) => v.as_f64().ok_or_else(|| bad(cmd, "`pace` must be a number"))?,
None => base.map_or(1.0, |b| b.pace),
};
let n = Narration { text, language: info.language.to_string(), voice, pitch, pace, speech_duration: Tick::ZERO };
n.check().map_err(|e| bad(cmd, e))?;
Ok(n)
}
/// Synthesized narrations by settings, newest last (`tts.render` fills it in the background so
/// the UI's create / edit / preview don't block).
pub type SynthCache = Arc<Mutex<Vec<(String, Arc<tts::Audio>)>>>;
const CACHE_ENTRIES: usize = 6;
fn cache_key(n: &Narration) -> String {
format!("{}\u{1f}{}\u{1f}{}\u{1f}{}", n.voice, n.pitch.id(), n.pace.to_bits(), n.text)
}
fn tts_err(cmd: &str, e: tts::TtsError) -> EngineError {
match e {
tts::TtsError::NotInstalled => bad(cmd, "the natural voices are not downloaded yet: Text to Speech ▸ Download natural voices (tts.downloadVoices)"),
e => bad(cmd, e.to_string()),
}
}
/// Synthesize `n` (no cache).
fn render(n: &Narration, models_dir: Option<&std::path::Path>) -> std::result::Result<tts::Audio, tts::TtsError> {
let voice = tts::voice_in(&n.voice, models_dir)?;
voice.synthesize(&n.text, &tts::Params { semitones: n.pitch.semitones(), pace: n.pace })
}
fn synthesize(s: &Session, cmd: &str, n: &Narration) -> Result<Arc<tts::Audio>> {
let key = cache_key(n);
if let Some(a) = s.tts_cache.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(k, _)| *k == key).map(|(_, a)| a.clone()) {
return Ok(a);
}
let a = Arc::new(render(n, crate::transcript::models_dir().as_deref()).map_err(|e| tts_err(cmd, e))?);
cache_put(&s.tts_cache, key, a.clone());
Ok(a)
}
fn cache_put(c: &SynthCache, key: String, a: Arc<tts::Audio>) {
let mut c = c.lock().unwrap_or_else(PoisonError::into_inner);
c.retain(|(k, _)| *k != key);
c.push((key, a));
let extra = c.len().saturating_sub(CACHE_ENTRIES);
c.drain(..extra);
}
fn package_json() -> Value {
let dir = crate::transcript::models_dir();
json!({
"id": tts::catalog::PACKAGE_ID,
"name": tts::catalog::PACKAGE_NAME,
"installed": dir.as_deref().is_some_and(tts::catalog::installed),
"size": tts::catalog::size(),
"missingBytes": dir.as_deref().map_or(tts::catalog::size(), tts::catalog::missing_bytes),
"license": tts::catalog::LICENSE,
"source": tts::catalog::SOURCE,
"available": cfg!(feature = "neural-voices"),
})
}
fn voices(_s: &mut Session, _p: &Value) -> Result<Value> {
let dir = crate::transcript::models_dir();
Ok(json!({
"package": package_json(),
"languages": tts::LANGUAGES.iter().map(|(id, name)| json!({"id": id, "name": name})).collect::<Vec<_>>(),
"voices": tts::voices_in(dir.as_deref()),
"defaultVoice": tts::default_voice_id(),
"pitches": VocalPitch::ALL.iter().map(|p| json!({"id": p.id(), "label": p.label(), "semitones": p.semitones()})).collect::<Vec<_>>(),
"pace": {"min": tts::MIN_PACE, "max": tts::MAX_PACE, "default": 1.0},
"pauseMarker": tts::script::PAUSE_MARKER,
"maxTextBytes": tts::MAX_TEXT_BYTES,
"sampleSentence": tts::SAMPLE_SENTENCE,
}))
}
fn preview(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.preview";
let base = target_clip(s, p).and_then(|c| s.active_sequence()?.find_item(c).map(|(_, it)| it.item)).and_then(|i| s.project.narrations.get(&i).cloned());
let mut q = p.clone();
if q.get("sample").and_then(Value::as_bool).unwrap_or(false) {
q["text"] = json!(tts::SAMPLE_SENTENCE);
}
let n = settings(cmd, &q, base.as_ref())?;
let audio = synthesize(s, cmd, &n)?;
let seconds = audio.seconds();
let samples = audio.samples.len();
let rate = audio.sample_rate;
let mut path = Value::Null;
if let Some(dest) = str_p(p, "path") {
let bytes = crate::voiceover::write_wav_f32(&audio.samples, rate);
s.services.write_file(dest, &bytes).map_err(|e| EngineError::Other(format!("{dest}: {e}")))?;
path = json!(dest);
}
s.tts_preview = Some(audio);
Ok(json!({"seconds": seconds, "samples": samples, "sampleRate": rate, "voice": n.voice, "path": path}))
}
/// Write `samples` as a new WAV and import it (its own undo step; callers collapse history).
/// `near`: an edit writes next to the narration's previous file unless `dir` is given.
fn write_and_import(s: &mut Session, p: &Value, samples: &[f32], rate: u32, near: Option<ItemId>) -> Result<(ItemId, String)> {
let beside = near.filter(|_| p.get("dir").is_none()).and_then(|i| match &s.project.items.get(&i)?.kind {
ItemKind::Media(m) => match &m.media {
MediaRef::File { path } => std::path::Path::new(path).parent().filter(|d| !d.as_os_str().is_empty()).map(|d| d.to_string_lossy().into_owned()),
MediaRef::Generator(_) => None,
},
_ => None,
});
let dir = beside.unwrap_or_else(|| crate::voiceover::media_dir(s, p, "Narrations"));
let path = crate::voiceover::unique_wav_path(s, &dir, BASE_NAME)?;
if !cfg!(target_arch = "wasm32") {
std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{dir}: {e}")))?;
}
let bytes = crate::voiceover::write_wav_f32(samples, rate);
s.services.write_file(&path, &bytes).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
let item = crate::commands::import_bytes(s, &path, bytes.into(), None)?;
Ok((item, path))
}
fn track_is_free(s: &Session, track: TrackId, range: TimeRange) -> bool {
s.active_sequence()
.and_then(|q| q.audio_tracks.iter().find(|t| t.id == track))
.is_some_and(|t| !t.locked && !t.items.iter().any(|it| it.start < range.end() && range.start < it.end()))
}
/// The track a new narration of `range` goes to (None: add a track).
fn placement(s: &Session, p: &Value, range: TimeRange) -> Result<Option<TrackId>> {
let cmd = "tts.create";
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
if let Some(v) = p.get("track") {
let id =
crate::mixer::strip_ref(seq, v).filter(|id| seq.audio_tracks.iter().any(|t| t.id == *id)).ok_or_else(|| bad(cmd, format!("no audio track {v}")))?;
if !track_is_free(s, id, range) {
return Err(bad(cmd, format!("audio track {v} is locked or has clips there")));
}
return Ok(Some(id));
}
let targeted = s.targeting().targeted;
let order = seq.audio_tracks.iter().filter(|t| targeted.contains(&t.id)).chain(seq.audio_tracks.iter().filter(|t| !targeted.contains(&t.id)));
Ok(order.map(|t| t.id).find(|id| track_is_free(s, *id, range)))
}
fn create(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.create";
let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?;
let mut n = settings(cmd, p, None)?;
let at = time_p(s, p, "").unwrap_or(s.playhead());
if at < Tick::ZERO {
return Err(bad(cmd, "the start time cannot be negative"));
}
let audio = synthesize(s, cmd, &n)?;
let rate = audio.sample_rate.max(1);
let dur = Tick::from_units(audio.samples.len() as i64, i64::from(rate));
n.speech_duration = dur;
let range = TimeRange::new(at, dur);
let track = placement(s, p, range)?;
let n0 = s.history.undo.len();
let result = (|| -> Result<Value> {
let track = match track {
Some(t) => t,
None => {
let before: Vec<TrackId> = s.active_sequence().map(|q| q.audio_tracks.iter().map(|t| t.id).collect()).unwrap_or_default();
s.execute("sequence.addTracks", json!({"video": 0, "audio": 1, "audioAfter": "last"}))?;
s.active_sequence()
.and_then(|q| q.audio_tracks.iter().map(|t| t.id).find(|id| !before.contains(id)))
.ok_or_else(|| EngineError::Other("could not add an audio track".into()))?
}
};
let (item, path) = write_and_import(s, p, &audio.samples, rate, None)?;
let seq_rate = s.project.sequence(seq_id).map(|q| q.settings.frame_rate).unwrap_or_default();
let narration = n.clone();
let clip = s.edit("New Narration", |pr, st| {
let mut it = pr
.make_track_item(item, TrackKind::Audio, at, TimeRange::new(Tick::ZERO, dur), seq_rate)
.ok_or(EngineError::Other("the narration is not importable".into()))?;
it.duration = dur;
let id = it.id;
let seq = pr.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?;
let t = seq.audio_tracks.iter_mut().find(|t| t.id == track).ok_or_else(|| bad(cmd, "the narration track was deleted"))?;
let pos = t.items.partition_point(|x| x.start <= at);
t.items.insert(pos, it);
seq.check().map_err(EngineError::Other)?;
pr.narrations.insert(item, narration);
st.selection = vec![id];
Ok(id)
})?;
let seq = s.project.sequence(seq_id).ok_or(EngineError::NoSequence)?;
Ok(json!({
"item": item.0,
"clip": clip.0,
"track": crate::mixer::strip_label(seq, track),
"start": at.0,
"duration": dur.0,
"speechDuration": dur.0,
"path": path,
"sampleRate": rate,
}))
})();
crate::clip_ops::collapse_history(s, n0, "New Narration");
if result.is_err() && s.history.undo.len() > n0 {
// a step failed after the track was added or the file imported: roll back to before
let _ = s.undo();
s.history.redo.clear();
}
result
}
/// Samples of media a clip of `duration` at `speed` plays, rounded up (exact for 100 %).
fn needed_samples(duration: Tick, speed: f64, rate: u32) -> usize {
let rate = i64::from(rate.max(1));
let media = if speed.is_finite() && speed != 0.0 && (speed.abs() - 1.0).abs() > 1e-12 {
// speed is a ratio, not edit time: scale the exact tick count
Tick::from_seconds_f64(duration.seconds() * speed.abs())
} else {
duration
};
let mut n = media.to_units_floor(rate);
if Tick::from_units(n, rate) < media {
n = n.saturating_add(1);
}
let cap = (tts::MAX_SECONDS * 2.0) as i64 * rate;
n.clamp(0, cap) as usize
}
fn edit(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.edit";
let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?;
let clip = target_clip(s, p).ok_or_else(|| bad(cmd, "select a narration clip (or pass `clip`)"))?;
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
let (track, it) = seq.find_item(clip).ok_or_else(|| bad(cmd, "no such clip"))?;
if seq.audio_tracks.iter().any(|t| t.id == track && t.locked) {
return Err(bad(cmd, "the clip's track is locked"));
}
let (old_item, clip_dur, speed) = (it.item, it.duration, it.speed);
let base = s.project.narrations.get(&old_item).cloned().ok_or_else(|| bad(cmd, "the clip is not a narration"))?;
let mut n = settings(cmd, p, Some(&base))?;
let audio = synthesize(s, cmd, &n)?;
let rate = audio.sample_rate.max(1);
let speech = audio.samples.len();
n.speech_duration = Tick::from_units(speech as i64, i64::from(rate));
// the clip keeps its length: pad the file with silence to what the clip plays
let needed = needed_samples(clip_dur, speed, rate);
let mut samples = audio.samples.clone();
if needed > samples.len() {
samples.try_reserve(needed - samples.len()).map_err(|e| bad(cmd, format!("unable to allocate the narration: {e}")))?;
samples.resize(needed, 0.0);
}
let file_len = samples.len();
let n0 = s.history.undo.len();
let result = (|| -> Result<Value> {
let (item, path) = write_and_import(s, p, &samples, rate, Some(old_item))?;
let name = s.project.items.get(&item).map(|i| i.name.clone()).unwrap_or_default();
let narration = n.clone();
s.edit("Edit Narration", |pr, _| {
let seq = pr.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?;
let (_, it) = seq.find_item_mut(clip).ok_or_else(|| bad(cmd, "the clip was deleted"))?;
it.item = item;
it.name = name;
it.source_in = Tick::ZERO;
seq.check().map_err(EngineError::Other)?;
pr.narrations.insert(item, narration);
Ok(())
})?;
Ok(json!({
"clip": clip.0,
"item": item.0,
"previousItem": old_item.0,
"duration": clip_dur.0,
"speechDuration": n.speech_duration.0,
"fileDuration": Tick::from_units(file_len as i64, i64::from(rate)).0,
"path": path,
"sampleRate": rate,
}))
})();
crate::clip_ops::collapse_history(s, n0, "Edit Narration");
if result.is_err() && s.history.undo.len() > n0 {
let _ = s.undo();
s.history.redo.clear();
}
result
}
fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.inspect";
let (item, clip) = match u64_p(p, "item") {
Some(i) => (ItemId(i), None),
None => {
let c = target_clip(s, p).ok_or_else(|| bad(cmd, "pass `item` or `clip`, or select a narration clip"))?;
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
(seq.find_item(c).map(|(_, it)| it.item).ok_or_else(|| bad(cmd, "no such clip"))?, Some(c.0))
}
};
let n = s.project.narrations.get(&item).ok_or_else(|| bad(cmd, "not a narration"))?;
Ok(json!({"item": item.0, "clip": clip, "narration": n}))
}
fn new_job(s: &mut Session, label: String) -> crate::Job {
let id = s.jobs.iter().map(|j| j.id).max().unwrap_or(0) + 1;
crate::Job { id, label, progress: Default::default(), result: Default::default() }
}
fn spawn(s: &mut Session, job: crate::Job, wait: bool, name: &str, run: impl FnOnce() + Send + 'static) -> Result<u64> {
let id = job.id;
let run = crate::export_tools::guard_job(job.progress.clone(), job.result.clone(), run);
s.jobs.push(job);
if wait || cfg!(target_arch = "wasm32") {
run();
} else {
std::thread::Builder::new().name(name.into()).spawn(run).map_err(|e| EngineError::Other(e.to_string()))?;
}
Ok(id)
}
type JobResult = std::sync::Mutex<Option<std::result::Result<filmcraft_export::Report, String>>>;
fn finish(prog: &filmcraft_export::Progress, res: &JobResult, r: std::result::Result<String, String>, t0: web_time::Instant) {
use std::sync::atomic::Ordering;
let secs = t0.elapsed().as_secs_f64();
let r = match r {
Ok(status) => {
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = status;
Ok(filmcraft_export::Report { path: String::new(), frames: 0, seconds: secs, bytes: 0, render_fps: 0.0, extra_files: Vec::new() })
}
Err(e) => {
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = e.clone();
*prog.error.lock().unwrap_or_else(PoisonError::into_inner) = Some(e.clone());
Err(e)
}
};
prog.finished.store(true, Ordering::Relaxed);
*res.lock().unwrap_or_else(PoisonError::into_inner) = Some(r);
}
/// Synthesize in the background into the cache, so `tts.create` / `tts.edit` / `tts.preview` with
/// the same settings return at once.
fn render_job(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.render";
let base = target_clip(s, p).and_then(|c| s.active_sequence()?.find_item(c).map(|(_, it)| it.item)).and_then(|i| s.project.narrations.get(&i).cloned());
let mut q = p.clone();
if q.get("sample").and_then(Value::as_bool).unwrap_or(false) {
q["text"] = json!(tts::SAMPLE_SENTENCE);
}
let n = settings(cmd, &q, base.as_ref())?;
let key = cache_key(&n);
if s.tts_cache.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|(k, _)| *k == key) {
return Ok(json!({"job": null, "cached": true}));
}
let job = new_job(s, "Synthesizing narration".into());
let (prog, res, cache) = (job.progress.clone(), job.result.clone(), s.tts_cache.clone());
let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);
let id = spawn(s, job, wait, "filmcraft-tts", move || {
let t0 = web_time::Instant::now();
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = "Synthesizing…".into();
let r = match render(&n, crate::transcript::models_dir().as_deref()) {
Ok(a) => {
let secs = a.seconds();
cache_put(&cache, key, Arc::new(a));
Ok(format!("{secs:.1} s of speech"))
}
Err(tts::TtsError::NotInstalled) => Err("the natural voices are not downloaded yet".to_string()),
Err(e) => Err(e.to_string()),
};
finish(&prog, &res, r, t0);
})?;
Ok(json!({"job": id, "cached": false}))
}
/// Download the natural voices (Kokoro-82M, its voices and CMUdict) in the background.
fn download_voices(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.downloadVoices";
let dir = crate::transcript::models_dir().ok_or_else(|| bad(cmd, "no data directory for voice models"))?;
if tts::catalog::installed(&dir) {
return Ok(json!({"job": null, "installed": true}));
}
if !cfg!(feature = "neural-voices") {
return Err(bad(cmd, "natural voices are not available in this build (built without the `neural-voices` feature)"));
}
let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);
let job = new_job(s, format!("Downloading {}", tts::catalog::PACKAGE_NAME));
job.progress.total.store(tts::catalog::missing_bytes(&dir), std::sync::atomic::Ordering::Relaxed);
let (prog, res) = (job.progress.clone(), job.result.clone());
let id = spawn(s, job, wait, "filmcraft-tts-download", move || {
let t0 = web_time::Instant::now();
let r = download_package(&dir, &prog);
finish(&prog, &res, r.map(|()| "Natural voices installed".to_string()), t0);
})?;
Ok(json!({"job": id, "installed": false}))
}
#[cfg(feature = "neural-voices")]
fn download_package(dir: &std::path::Path, prog: &filmcraft_export::Progress) -> std::result::Result<(), String> {
use std::sync::atomic::Ordering;
let files: Vec<filmcraft_speech::models::ModelFile> =
tts::catalog::FILES.iter().map(|f| filmcraft_speech::models::ModelFile { name: f.name, url: f.url, sha256: f.sha256, size: f.size }).collect();
let mut progress = |done: u64, total: u64, file: &str| {
prog.done.store(done, Ordering::Relaxed);
prog.total.store(total, Ordering::Relaxed);
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = format!("Downloading {file}");
!prog.cancel.load(Ordering::Relaxed)
};
filmcraft_speech::models::download_files(&tts::catalog::package_dir(dir), &files, &mut progress).map_err(|e| e.to_string())
}
#[cfg(not(feature = "neural-voices"))]
fn download_package(_: &std::path::Path, _: &filmcraft_export::Progress) -> std::result::Result<(), String> {
Err("natural voices are not available in this build".into())
}
fn spec(
id: &'static str,
label: &'static str,
params: &'static str,
enabled: fn(&Session) -> std::result::Result<(), String>,
run: fn(&mut Session, &Value) -> Result<Value>,
journal: bool,
) -> CommandSpec {
CommandSpec { id, label, menu: &[], shortcut: None, params, enabled, run, journal }
}
pub fn commands() -> Vec<CommandSpec> {
vec![
spec("tts.voices", "List Narration Voices", "{}", always, voices, false),
spec(
"tts.preview",
"Preview Narration",
r#"{"text":str?,"voice":str?,"pitch":"default"?,"pace":f64?,"sample":bool?,"clip":id?,"path":str?}"#,
always,
preview,
false,
),
spec(
"tts.create",
"New Narration",
r#"{"text":str,"voice":str?,"pitch":"extraLow|low|default|high|extraHigh"?,"pace":0.5..2?,"time":ticks?,"track":"A1"|id?,"dir":str?}"#,
has_seq,
create,
true,
),
spec("tts.edit", "Edit Narration", r#"{"clip":id?,"text":str?,"voice":str?,"pitch":str?,"pace":f64?,"dir":str?}"#, has_narration_clip, edit, true),
spec("tts.inspect", "Inspect Narration", r#"{"clip":id?,"item":id?}"#, always, inspect, false),
spec(
"tts.render",
"Synthesize Narration",
r#"{"text":str?,"voice":str?,"pitch":str?,"pace":f64?,"sample":bool?,"clip":id?,"wait":bool?}"#,
always,
render_job,
false,
),
spec("tts.downloadVoices", "Download Natural Voices", r#"{"wait":bool?}"#, always, download_voices, false),
]
}
+425
View File
@@ -0,0 +1,425 @@
//! Tests of [`crate::narration`]: Text to Speech narrations are placed without overwriting, keep
//! their clip length when edited, and are one exact undo step.
use super::*;
use filmcraft_project::{ClipId, ItemId};
use serde_json::json;
const RATE: i64 = filmcraft_tts::SAMPLE_RATE as i64;
fn tmp(name: &str) -> String {
let d = std::env::temp_dir().join(format!("filmcraft-tts-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
d.to_string_lossy().into_owned()
}
fn demo() -> Session {
let mut s = Session::default();
s.execute("file.openDemoProject", json!({})).unwrap();
s
}
/// The 32-bit float samples of a WAV written by `write_wav_f32`.
fn wav_samples(path: &str) -> Vec<f32> {
let b = std::fs::read(path).unwrap();
b[44..].as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect()
}
fn clip(s: &Session, id: u64) -> filmcraft_project::TrackItem {
s.active_sequence().unwrap().find_item(ClipId(id)).unwrap().1.clone()
}
fn all_clips(s: &Session) -> Vec<filmcraft_project::TrackItem> {
s.active_sequence().unwrap().all_tracks().flat_map(|t| t.items.clone()).collect()
}
#[test]
fn voices_lists_the_built_in_voices() {
let mut s = Session::default();
let r = s.execute("tts.voices", json!({})).unwrap();
let ids: Vec<&str> = r["voices"].as_array().unwrap().iter().map(|v| v["id"].as_str().unwrap()).collect();
assert!(ids.contains(&"basic-female") && ids.contains(&"basic-male"), "{ids:?}");
assert_eq!(r["languages"][0]["id"], "en-US");
assert_eq!(r["pitches"].as_array().unwrap().len(), 5);
assert_eq!(r["pauseMarker"], "[pause 1s]");
}
#[test]
fn create_places_the_narration_at_the_playhead_as_one_undo_step() {
let mut s = demo();
let dir = tmp("create");
s.execute("playhead.set", json!({"seconds": 2.0})).unwrap();
let t0 = s.playhead();
assert!(t0 > Tick::ZERO);
let before = all_clips(&s);
let undo0 = s.history.undo.len();
let r = s.execute("tts.create", json!({"text": "Welcome to FilmCraft.", "voice": "basic-male", "pitch": "low", "pace": 1.2, "dir": dir})).unwrap();
let c = clip(&s, r["clip"].as_u64().unwrap());
assert_eq!(c.start, t0);
// the clip is exactly as long as the speech (sample-exact) and plays it from the start
let samples = wav_samples(r["path"].as_str().unwrap());
assert_eq!(c.duration, Tick::from_units(samples.len() as i64, RATE));
assert_eq!(c.source_in, Tick::ZERO);
assert_eq!(r["speechDuration"], c.duration.0);
let n = s.project.narrations.get(&ItemId(r["item"].as_u64().unwrap())).unwrap();
assert_eq!((n.text.as_str(), n.voice.as_str(), n.pitch, n.pace), ("Welcome to FilmCraft.", "basic-male", VocalPitch::Low, 1.2));
assert_eq!(n.speech_duration, c.duration);
assert_eq!(s.state.selection, vec![c.id]);
// nothing that was there before moved or changed
let after = all_clips(&s);
for b in &before {
assert!(after.contains(b), "clip {:?} changed", b.id);
}
assert_eq!(after.len(), before.len() + 1);
// one undo step that removes the clip and the narration; redo brings both back
assert_eq!(s.history.undo.len(), undo0 + 1);
assert_eq!(s.history.undo.last().unwrap().0, "New Narration");
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(all_clips(&s), before);
assert!(s.project.narrations.is_empty());
s.execute("edit.redo", json!({})).unwrap();
assert_eq!(clip(&s, c.id.0), c);
assert_eq!(s.project.narrations.len(), 1);
let _ = std::fs::remove_dir_all(&dir);
}
use filmcraft_project::VocalPitch;
use filmcraft_time::Tick;
#[test]
fn create_never_overwrites_and_adds_a_track_when_every_track_is_busy() {
let mut s = demo();
let dir = tmp("busy");
// fill every audio track over 0–60 s with a narration-length clip so none is free at 0
let tracks = s.active_sequence().unwrap().audio_tracks.len();
let first = s.execute("tts.create", json!({"text": "One two three.", "time": 0, "dir": dir})).unwrap();
let busy_len = Tick(first["duration"].as_i64().unwrap());
for _ in 1..tracks + 2 {
s.execute("tts.create", json!({"text": "One two three.", "time": 0, "dir": dir})).unwrap();
}
// every create landed on its own track at 0: no two narrations overlap on one track
let seq = s.active_sequence().unwrap();
for t in &seq.audio_tracks {
let mut items: Vec<_> = t.items.iter().map(|i| (i.start, i.end())).collect();
items.sort();
for w in items.windows(2) {
assert!(w[0].1 <= w[1].0, "overlap on {:?}", t.name);
}
}
assert!(seq.audio_tracks.len() > tracks, "a new audio track was added");
assert!(busy_len > Tick::ZERO);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn create_on_an_occupied_or_unknown_track_is_refused_and_changes_nothing() {
let mut s = demo();
let dir = tmp("occupied");
let r = s.execute("tts.create", json!({"text": "Hello.", "time": 0, "track": "A1", "dir": dir}));
if s.active_sequence().unwrap().audio_tracks[0].items.iter().any(|i| i.start <= Tick::ZERO && Tick::ZERO < i.end()) {
assert!(r.is_err(), "A1 is busy at 0");
}
let undo0 = s.history.undo.len();
let items0 = s.project.items.len();
assert!(s.execute("tts.create", json!({"text": "Hello.", "track": "A99", "dir": dir})).is_err());
s.execute("timeline.setTrack", json!({"track": "A2", "locked": true})).unwrap();
let undo1 = s.history.undo.len();
assert!(s.execute("tts.create", json!({"text": "Hello.", "track": "A2", "time": 0, "dir": dir})).is_err());
assert_eq!(s.history.undo.len(), undo1);
assert!(undo1 >= undo0);
assert_eq!(s.project.items.len(), items0, "no file was imported");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn editing_to_longer_speech_keeps_the_clip_length_and_the_whole_speech() {
let mut s = demo();
let dir = tmp("longer");
let r = s.execute("tts.create", json!({"text": "Hi.", "time": 0, "dir": dir})).unwrap();
let c0 = clip(&s, r["clip"].as_u64().unwrap());
let undo0 = s.history.undo.len();
let e = s.execute("tts.edit", json!({"clip": c0.id.0, "text": "Hi. This narration is now a great deal longer than it was before."})).unwrap();
let c1 = clip(&s, c0.id.0);
assert_eq!((c1.id, c1.start, c1.duration, c1.speed), (c0.id, c0.start, c0.duration, c0.speed), "position and length kept");
assert_ne!(c1.item, c0.item, "a new file");
assert_eq!(c1.source_in, Tick::ZERO);
let n = s.project.narrations.get(&c1.item).unwrap();
assert!(n.speech_duration > c1.duration, "the speech is longer than the clip");
// the file holds the whole speech, nothing padded
let samples = wav_samples(e["path"].as_str().unwrap());
assert_eq!(Tick::from_units(samples.len() as i64, RATE), n.speech_duration);
assert_eq!(e["fileDuration"], n.speech_duration.0);
// the old item and its narration are untouched (other clips of it, undo)
assert_eq!(s.project.narrations.get(&c0.item).unwrap().text, "Hi.");
assert_eq!(s.history.undo.len(), undo0 + 1);
assert_eq!(s.history.undo.last().unwrap().0, "Edit Narration");
// the edited file sits next to the first one
assert!(e["path"].as_str().unwrap().starts_with(&dir), "{}", e["path"]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn editing_to_shorter_speech_pads_the_file_with_exact_silence() {
let mut s = demo();
let dir = tmp("shorter");
let r = s.execute("tts.create", json!({"text": "This first version of the narration is fairly long, with several words.", "time": 0, "dir": dir})).unwrap();
let id = r["clip"].as_u64().unwrap();
let c0 = clip(&s, id);
let e = s.execute("tts.edit", json!({"clip": id, "text": "Short."})).unwrap();
let c1 = clip(&s, id);
assert_eq!((c1.start, c1.duration), (c0.start, c0.duration));
let n = s.project.narrations.get(&c1.item).unwrap().clone();
assert!(n.speech_duration < c1.duration);
let samples = wav_samples(e["path"].as_str().unwrap());
let speech = n.speech_duration.to_units_floor(RATE) as usize;
// the file covers the clip, and everything after the speech is exactly zero
assert!(Tick::from_units(samples.len() as i64, RATE) >= c1.duration);
assert!(samples.len() > speech);
assert!(samples[speech..].iter().all(|&x| x == 0.0));
assert!(samples[..speech].iter().any(|&x| x != 0.0));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn undo_and_redo_of_an_edit_are_exact() {
let mut s = demo();
let dir = tmp("undo");
let r = s.execute("tts.create", json!({"text": "Before.", "time": 0, "dir": dir})).unwrap();
let id = r["clip"].as_u64().unwrap();
let before = (*s.project).clone();
s.execute("tts.edit", json!({"clip": id, "text": "After.", "voice": "basic-male", "pitch": "extraHigh", "pace": 0.7})).unwrap();
let after = (*s.project).clone();
assert_ne!(before, after);
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(*s.project, before);
let n = s.project.narrations.get(&clip(&s, id).item).unwrap();
assert_eq!((n.text.as_str(), n.voice.as_str()), ("Before.", "basic-female"));
s.execute("edit.redo", json!({})).unwrap();
assert_eq!(*s.project, after);
let n = s.project.narrations.get(&clip(&s, id).item).unwrap();
assert_eq!((n.text.as_str(), n.pitch, n.pace), ("After.", VocalPitch::ExtraHigh, 0.7));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn edit_without_changes_keeps_settings_and_settings_inherit() {
let mut s = demo();
let dir = tmp("inherit");
let r = s.execute("tts.create", json!({"text": "Keep me.", "voice": "basic-male", "pitch": "high", "pace": 1.5, "time": 0, "dir": dir})).unwrap();
let id = r["clip"].as_u64().unwrap();
s.execute("tts.edit", json!({"clip": id, "pace": 0.8})).unwrap();
let n = s.project.narrations.get(&clip(&s, id).item).unwrap();
assert_eq!((n.text.as_str(), n.voice.as_str(), n.pitch, n.pace), ("Keep me.", "basic-male", VocalPitch::High, 0.8));
let i = s.execute("tts.inspect", json!({"clip": id})).unwrap();
assert_eq!(i["narration"]["pace"], 0.8);
assert_eq!(i["narration"]["pitch"], "high");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn edit_is_disabled_without_a_narration_clip() {
let mut s = demo();
let dir = tmp("enabled");
s.state.selection.clear();
assert!(!s.is_enabled("tts.edit"));
assert!(s.execute("tts.edit", json!({"text": "x"})).is_err());
// a non-narration clip is refused
let other = s.active_sequence().unwrap().all_tracks().find_map(|t| t.items.first().map(|i| i.id)).unwrap();
assert!(s.execute("tts.edit", json!({"clip": other.0, "text": "x"})).is_err());
let r = s.execute("tts.create", json!({"text": "Enable me.", "time": 0, "dir": dir})).unwrap();
assert!(s.is_enabled("tts.edit"), "the new narration is selected");
s.state.selection.clear();
assert!(s.execute("tts.edit", json!({"clip": r["clip"], "text": "By id."})).is_ok());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn edit_on_a_locked_track_is_refused() {
let mut s = demo();
let dir = tmp("locked");
let r = s.execute("tts.create", json!({"text": "Locked.", "time": 0, "dir": dir})).unwrap();
let track = r["track"].as_str().unwrap().to_string();
s.execute("timeline.setTrack", json!({"track": track, "locked": true})).unwrap();
let snap = (*s.project).clone();
assert!(s.execute("tts.edit", json!({"clip": r["clip"], "text": "Changed."})).is_err());
assert_eq!(*s.project, snap);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn hostile_parameters_are_refused_without_changes() {
let mut s = demo();
let dir = tmp("hostile");
let snap = (*s.project).clone();
let undo0 = s.history.undo.len();
let big = "word ".repeat(300_000);
let cases = [
json!({}),
json!({"text": ""}),
json!({"text": " [pause 2s] "}),
json!({"text": "👍"}),
json!({"text": 42}),
json!({"text": big}),
json!({"text": "hi", "voice": "nope"}),
json!({"text": "hi", "voice": 7}),
json!({"text": "hi", "pitch": "loud"}),
json!({"text": "hi", "pace": "fast"}),
json!({"text": "hi", "pace": 5.0}),
json!({"text": "hi", "pace": -1}),
json!({"text": "hi", "pace": 1e308}),
json!({"text": "hi", "time": -5}),
json!({"text": "hi", "track": {"x": 1}}),
json!({"text": "hi", "track": "V1"}),
];
for p in cases {
let mut q = p.clone();
q["dir"] = json!(dir);
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| s.execute("tts.create", q.clone())));
assert!(r.is_ok(), "panicked on {p}");
assert!(r.unwrap().is_err(), "accepted {p}");
assert_eq!(*s.project, snap, "changed by {p}");
assert_eq!(s.history.undo.len(), undo0);
}
for p in [json!({"clip": 999_999_999}), json!({"clip": "x"}), json!({"clip": -1})] {
assert!(s.execute("tts.edit", p.clone()).is_err(), "{p}");
assert!(s.execute("tts.inspect", p.clone()).is_err(), "{p}");
}
assert!(s.execute("tts.preview", json!({"text": "hi", "pace": 9})).is_err());
// no sequence open
let mut empty = Session::default();
assert!(empty.execute("tts.create", json!({"text": "hi", "dir": dir})).is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn preview_changes_nothing_and_hands_the_audio_to_the_host() {
let mut s = demo();
let dir = tmp("preview");
std::fs::create_dir_all(&dir).unwrap();
let snap = (*s.project).clone();
let undo0 = s.history.undo.len();
let path = format!("{dir}/p.wav");
let r = s.execute("tts.preview", json!({"text": "Preview me.", "path": path})).unwrap();
assert_eq!(*s.project, snap);
assert_eq!(s.history.undo.len(), undo0);
let a = s.tts_preview.clone().unwrap();
assert_eq!(r["samples"], a.samples.len());
assert_eq!(wav_samples(&path), a.samples);
// the sample sentence for "Hear this voice"
let r = s.execute("tts.preview", json!({"sample": true, "voice": "basic-male"})).unwrap();
assert_eq!(r["voice"], "basic-male");
assert!(r["seconds"].as_f64().unwrap() > 1.0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn narrations_survive_save_and_reload() {
let mut s = demo();
let dir = tmp("save");
let r = s.execute("tts.create", json!({"text": "Saved [pause 500ms] and loaded.", "pitch": "extraLow", "time": 0, "dir": dir})).unwrap();
let bytes = filmcraft_format::encode(&s.project, false);
let loaded = filmcraft_format::decode(&bytes).unwrap();
assert_eq!(loaded.project.narrations, s.project.narrations);
let n = loaded.project.narrations.get(&ItemId(r["item"].as_u64().unwrap())).unwrap();
assert_eq!((n.text.as_str(), n.pitch), ("Saved [pause 500ms] and loaded.", VocalPitch::ExtraLow));
// a project without narrations writes no `narrations` key (older files are unchanged)
let plain = String::from_utf8(filmcraft_format::encode(&demo().project, false)).unwrap();
assert!(!plain.contains("narrations"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_narration_is_heard_in_the_mix() {
let mut s = demo();
let dir = tmp("mix");
let r = s.execute("tts.create", json!({"text": "Can you hear me now?", "time": 0, "dir": dir})).unwrap();
let track = r["track"].as_str().unwrap().to_string();
let n = s.active_sequence().unwrap().audio_tracks.len();
for i in 1..=n {
if format!("A{i}") != track {
s.execute("mixer.setStrip", json!({"strip": format!("A{i}"), "muted": true})).unwrap();
}
}
let provider = s.media.provider(s.project.clone(), s.services.clone());
let sr = s.active_sequence().unwrap().settings.sample_rate as usize;
let mix = filmcraft_render::audio::mix_sequence(&s.project, s.active_sequence().unwrap(), 0, sr, &provider).channels;
let rms = (mix[0].iter().map(|x| f64::from(x * x)).sum::<f64>() / sr as f64).sqrt();
assert!(rms > 0.01, "narration audible, rms {rms}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn render_fills_the_cache_that_create_and_preview_use() {
let mut s = demo();
let dir = tmp("render");
let p = json!({"text": "Rendered in the background.", "voice": "basic-male", "pace": 1.1});
let r = s.execute("tts.render", json!({"text": "Rendered in the background.", "voice": "basic-male", "pace": 1.1, "wait": true})).unwrap();
let job = r["job"].as_u64().expect("a job");
let j = s.jobs.iter().find(|j| j.id == job).unwrap();
assert!(j.progress.finished.load(std::sync::atomic::Ordering::Relaxed));
assert!(matches!(&*j.result.lock().unwrap(), Some(Ok(_))));
// the same settings are now cached: render again is a no-op, preview hands back the same audio
assert_eq!(s.execute("tts.render", p.clone()).unwrap()["cached"], true);
s.execute("tts.preview", p.clone()).unwrap();
let cached = s.tts_cache.lock().unwrap().last().unwrap().1.clone();
assert!(std::sync::Arc::ptr_eq(s.tts_preview.as_ref().unwrap(), &cached));
let mut q = p.clone();
q["dir"] = json!(dir);
q["time"] = json!(0);
let c = s.execute("tts.create", q).unwrap();
assert_eq!(c["speechDuration"], Tick::from_units(cached.samples.len() as i64, RATE).0);
// a failing render reports the error in the job
let r = s.execute("tts.render", json!({"text": "x", "voice": "kokoro-heart", "wait": true}));
if let Ok(r) = r {
let job = r["job"].as_u64().unwrap();
let j = s.jobs.iter().find(|j| j.id == job).unwrap();
assert!(matches!(&*j.result.lock().unwrap(), Some(Err(_))) || cfg!(feature = "neural-voices"));
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn natural_voices_are_listed_and_explain_what_is_missing() {
let mut s = demo();
let v = s.execute("tts.voices", json!({})).unwrap();
let ids: Vec<&str> = v["voices"].as_array().unwrap().iter().map(|x| x["id"].as_str().unwrap()).collect();
assert!(ids.contains(&"kokoro-heart") && ids.contains(&"kokoro-michael"), "{ids:?}");
assert_eq!(v["package"]["id"], "kokoro-82m");
assert!(v["package"]["size"].as_u64().unwrap() > 300_000_000);
assert_eq!(v["package"]["available"], cfg!(feature = "neural-voices"));
if !v["package"]["installed"].as_bool().unwrap() {
let e = s.execute("tts.create", json!({"text": "Hi.", "voice": "kokoro-heart"})).unwrap_err().to_string();
assert!(e.contains("download") || e.contains("not available"), "{e}");
}
if !cfg!(feature = "neural-voices") && !v["package"]["installed"].as_bool().unwrap() {
let e = s.execute("tts.downloadVoices", json!({})).unwrap_err().to_string();
assert!(e.contains("not available in this build"), "{e}");
}
}
#[test]
fn a_project_subset_keeps_only_the_narrations_of_kept_items() {
let mut s = demo();
let dir = tmp("subset");
let r = s.execute("tts.create", json!({"text": "Hello.", "voice": "basic-male", "dir": dir})).unwrap();
let item = ItemId(r["item"].as_u64().unwrap());
let others: std::collections::BTreeSet<ItemId> = s.project.items.keys().copied().filter(|i| *i != item).collect();
assert!(crate::project_tools::project_subset(&s.project, &others).narrations.is_empty());
let with: std::collections::BTreeSet<ItemId> = [item].into();
assert!(crate::project_tools::project_subset(&s.project, &with).narrations.contains_key(&item));
}
#[test]
fn unique_wav_path_gives_up_instead_of_looping_forever() {
let mut s = demo();
let first = crate::voiceover::unique_wav_path(&s, "/nonexistent-dir", "Narration").unwrap();
assert!(first.ends_with("Narration 1.wav"), "{first}");
// every candidate name taken by a project item: an error, not an endless loop
let template = s.project.items.values().next().cloned().unwrap();
for k in 1..=99_999u32 {
let mut it = template.clone();
it.name = format!("Narration {k}.wav");
std::sync::Arc::make_mut(&mut s.project).items.insert(ItemId(1_000_000 + u64::from(k)), it);
}
assert!(crate::voiceover::unique_wav_path(&s, "/nonexistent-dir", "Narration").is_err());
}
+1
View File
@@ -756,6 +756,7 @@ pub fn project_subset(p: &Project, keep: &BTreeSet<ItemId>) -> Project {
out.items.retain(|id, _| keep.contains(id));
prune(&mut out.root, keep);
out.transcripts.retain(|id, _| keep.contains(id));
out.narrations.retain(|id, _| keep.contains(id));
out.search_bins.clear();
out
}
+1 -1
View File
@@ -428,7 +428,7 @@ fn simplify(s: &mut Session, p: &Value) -> Result<Value> {
let report = simplify_sequence(pr, &mut q, &o);
q.check().map_err(EngineError::Other)?;
let bin = pr.root.parent_of(seq_id).filter(|b| b.0 != 0);
let id = pr.add_item(&name, src.label, ItemKind::Sequence(Box::new(q)), bin);
let id = pr.add_item(&name, src.label, ItemKind::Sequence(std::sync::Arc::new(q)), bin);
st.active_sequence = Some(id);
if !st.open_sequences.contains(&id) {
st.open_sequences.push(id);
+4
View File
@@ -486,6 +486,8 @@ pub struct TimelinePrefs {
pub match_frame_sets_in: bool,
pub restore_open_sequences: bool,
pub add_tracks_automatically: bool,
/// Files dropped onto the Timeline are placed directly on it.
pub drop_import_to_timeline: bool,
}
impl Default for TimelinePrefs {
@@ -512,6 +514,7 @@ impl Default for TimelinePrefs {
match_frame_sets_in: false,
restore_open_sequences: true,
add_tracks_automatically: true,
drop_import_to_timeline: true,
}
}
}
@@ -1080,6 +1083,7 @@ static CATEGORIES: &[Category] = &[
b("timeline.matchFrameSetsIn", "Match frame sets in point", false),
b("timeline.restoreOpenSequences", "Restore open sequences when opening projects", false),
b("timeline.addTracksAutomatically", "Add tracks automatically when editing source clips onto the Timeline", false),
b("timeline.dropImportToTimeline", "Place files dropped onto the Timeline directly on the Timeline", true),
],
},
Category {
+41
View File
@@ -552,6 +552,47 @@ fn hardware_decoding_setting_drives_the_decoder_switch() {
assert!(settings::field("playback.hardwareDecoding").is_some_and(|f| f.wired && matches!(f.kind, Kind::Choice(_))));
}
/// Settings ▸ Timeline ▸ "Place files dropped onto the Timeline directly on the Timeline": on by default.
#[test]
fn drop_import_to_timeline_defaults_to_on() {
assert!(settings::TimelinePrefs::default().drop_import_to_timeline);
assert!(Preferences::default().timeline.drop_import_to_timeline);
}
/// The key is saved under `timeline.dropImportToTimeline`, read back by `prefs.get`, and persisted.
#[test]
fn drop_import_to_timeline_round_trips_through_prefs() {
let dir = tmp_dir("drop-import-to-timeline");
let path = dir.join("preferences.json");
let mut s = Session { prefs_path: Some(path.clone()), ..Session::default() };
set(&mut s, "timeline.dropImportToTimeline", json!(false));
assert!(!s.prefs.timeline.drop_import_to_timeline);
assert_eq!(s.execute("prefs.get", json!({"key": "timeline.dropImportToTimeline"})).unwrap(), json!(false));
assert!(!Preferences::load(&path).timeline.drop_import_to_timeline);
// a non-boolean is rejected and the stored value is kept
assert!(s.execute("prefs.set", json!({"key": "timeline.dropImportToTimeline", "value": "no"})).is_err());
assert!(!s.prefs.timeline.drop_import_to_timeline);
std::fs::remove_dir_all(dir).unwrap();
}
/// Preferences files written before the key existed load it as on.
#[test]
fn drop_import_to_timeline_defaults_to_on_for_older_prefs() {
assert!(serde_json::from_value::<settings::TimelinePrefs>(json!({})).unwrap().drop_import_to_timeline);
let old = Preferences::from_value(json!({"version": 2, "timeline": {"stillImageDuration": 3}}));
assert!(old.timeline.drop_import_to_timeline);
assert_eq!(old.timeline.still_image_duration, 3.0);
}
/// The key is a Boolean field of the Timeline category in the settings registry.
#[test]
fn drop_import_to_timeline_is_in_the_settings_registry() {
let (cat, f) = settings::fields().into_iter().find(|(_, f)| f.key == "timeline.dropImportToTimeline").expect("registry field");
assert_eq!(cat, "timeline");
assert!(matches!(f.kind, Kind::Bool));
assert_eq!(settings::field("timeline.dropImportToTimeline").map(|f| f.key), Some("timeline.dropImportToTimeline"));
}
/// New users keep the dark Darkest theme (following the system is opt-in); a file saved before
/// appearance modes keeps showing its Color Theme as a fixed Dark or Light mode.
#[test]
+1
View File
@@ -426,6 +426,7 @@ fn category_of(id: &str, menu: &[&str]) -> String {
"view" => "View".into(),
"app" | "help" | "mode" => "Application".into(),
"captions" => "Captions".into(),
"tts" => "Text to Speech".into(),
_ => menu.first().map(|s| s.to_string()).unwrap_or_else(|| {
let mut c = prefix.chars();
c.next().map(|f| f.to_uppercase().collect::<String>() + c.as_str()).unwrap_or_default()
+1 -1
View File
@@ -94,7 +94,7 @@ fn marker_list(p: &mut Project, item: ItemId) -> Result<&mut Vec<Marker>> {
let it = p.item_mut(item).ok_or_else(|| EngineError::Other("Source item is unavailable".into()))?;
match &mut it.kind {
filmcraft_project::ItemKind::Media(m) => Ok(&mut m.markers),
filmcraft_project::ItemKind::Sequence(q) => Ok(&mut q.markers),
filmcraft_project::ItemKind::Sequence(q) => Ok(&mut std::sync::Arc::make_mut(q).markers),
_ => Err(EngineError::Other("Source marker editing requires media or a sequence".into())),
}
}
+194
View File
@@ -326,6 +326,92 @@ fn add_keyframe_toggles_the_keyframe_at_the_playhead() {
assert_eq!(scale(&s), (0, 100.0));
}
/// `effects.setParam` writes a keyframe only on an animated parameter. An agent that sends a value
/// with a `time` to a parameter that is not animated gets a static value (the result says so:
/// 0 keyframes); `"keyframe": true` is the stopwatch and the value in one step.
#[test]
fn set_param_writes_a_keyframe_when_asked_or_animated() {
let mut s = demo();
let c = s.active_sequence().unwrap().video_tracks[0].items[0].id.0;
// (keyframes, static value, value at 0.5 s, value at 2.5 s)
let scale = |s: &Session| {
let it = s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(c)).unwrap().1;
let p = &it.effect("motion").unwrap().params["scale"];
let at = |seconds: f64| p.f64_at(it.source_time_at(Tick::from_seconds_f64(seconds)));
(p.keyframes.len(), p.value.as_f64().unwrap(), at(0.5), at(2.5))
};
let set = |value: f64, seconds: f64| json!({"clip": c, "effect": "motion", "param": "scale", "value": value, "seconds": seconds});
// not animated: the time is not used and the value is static
let r = s.execute("effects.setParam", set(80.0, 0.5)).unwrap();
assert_eq!(r["keyframes"], 0, "{r}");
assert_eq!(scale(&s), (0, 80.0, 80.0, 80.0));
// `keyframe`: the parameter becomes animated at that time
let mut first = set(60.0, 0.5);
first["keyframe"] = json!(true);
let r = s.execute("effects.setParam", first).unwrap();
assert_eq!(r["keyframes"], 1, "{r}");
assert_eq!(scale(&s).0, 1);
// animated now, so a plain set at another time is a second keyframe
let r = s.execute("effects.setParam", set(120.0, 2.5)).unwrap();
assert_eq!(r["keyframes"], 2, "{r}");
let (n, _, a, b) = scale(&s);
assert_eq!((n, a, b), (2, 60.0, 120.0));
// a keyframe that is already there is replaced, not doubled
let mut again = set(70.0, 0.5);
again["keyframe"] = json!(true);
assert_eq!(s.execute("effects.setParam", again).unwrap()["keyframes"], 2);
assert_eq!(scale(&s).2, 70.0);
s.undo();
assert_eq!(scale(&s).2, 60.0);
s.undo();
assert_eq!(scale(&s).0, 1);
s.redo();
s.redo();
let (n, _, a, b) = scale(&s);
assert_eq!((n, a, b), (2, 70.0, 120.0));
// a parameter that cannot be animated refuses the keyframe and keeps its value
let blend = json!({"clip": c, "effect": "opacity", "param": "blend", "value": 8, "keyframe": true});
let e = s.execute("effects.setParam", blend).unwrap_err().to_string();
assert!(e.contains("cannot be animated"), "{e}");
let it = s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(c)).unwrap().1;
assert_eq!(it.effect("opacity").unwrap().params["blend"].value, filmcraft_project::ParamValue::Choice(0));
// `keyframe` that is not a bool means no keyframe request
let mut odd = set(70.0, 0.5);
odd["keyframe"] = json!("yes");
assert_eq!(s.execute("effects.setParam", odd).unwrap()["keyframes"], 2);
}
/// A choice parameter takes its option name as well as its index, as transition parameters do
/// (`effects.list {"detail": true}` lists the names).
#[test]
fn set_param_takes_a_choice_by_name() {
use filmcraft_project::ParamValue;
let mut s = demo();
let c = s.active_sequence().unwrap().video_tracks[0].items[0].id.0;
let blend =
|s: &Session| s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(c)).unwrap().1.effect("opacity").unwrap().params["blend"].value.clone();
let set = |value: serde_json::Value| json!({"clip": c, "effect": "opacity", "param": "blend", "value": value});
s.execute("effects.setParam", set(json!("Screen"))).unwrap();
assert_eq!(blend(&s), ParamValue::Choice(8));
s.execute("effects.setParam", set(json!("linear dodge (add)"))).unwrap();
assert_eq!(blend(&s), ParamValue::Choice(10), "names are matched without case");
s.execute("effects.setParam", set(json!(12))).unwrap();
assert_eq!(blend(&s), ParamValue::Choice(12), "an index works as before");
s.undo();
assert_eq!(blend(&s), ParamValue::Choice(10));
s.redo();
assert_eq!(blend(&s), ParamValue::Choice(12));
// a name that is not an option is an error that lists the options and changes nothing
let e = s.execute("effects.setParam", set(json!("Glow"))).unwrap_err().to_string();
assert!(e.contains("Screen") && e.contains("Overlay"), "{e}");
for bad in [json!(""), json!(null), json!([8]), json!({"name": "Screen"}), json!(-1), json!(1.5)] {
assert!(s.execute("effects.setParam", set(bad.clone())).is_err(), "{bad}");
}
assert_eq!(blend(&s), ParamValue::Choice(12));
// a name is still the wrong type for a parameter that is not a choice
assert!(s.execute("effects.setParam", json!({"clip": c, "effect": "motion", "param": "scale", "value": "Screen"})).is_err());
}
/// Dragging the Volume line or one of its keyframes in the timeline (#223) sends a keyframe edit
/// every frame; `merge` keeps the whole drag one undo step and `begin` starts the next one.
#[test]
@@ -731,6 +817,114 @@ fn dragging_an_effect_parameter_is_one_undo_step() {
assert_eq!(opacity(&s), start, "and then the whole first one");
}
/// The demo's first video transition between two clips: (id, start, duration, cut).
fn demo_crossing(s: &Session) -> (u64, Tick, Tick, Tick) {
let q = s.active_sequence().unwrap();
q.video_tracks
.iter()
.find_map(|t| {
t.transitions.iter().find(|x| x.from.is_some() && x.to.is_some()).map(|x| (x.id.0, x.start, x.duration, t.item(x.to.unwrap()).unwrap().start))
})
.expect("the demo has a transition between two clips")
}
fn transition_ids(s: &Session) -> Vec<u64> {
s.active_sequence().unwrap().all_tracks().flat_map(|t| t.transitions.iter().map(|x| x.id.0)).collect()
}
/// #430: a transition can be selected (it replaces the clip selection and back), Delete removes
/// it and leaves its clips, undo brings it back; Ripple Delete is not for a transition alone.
#[test]
fn transitions_select_and_delete() {
let mut s = demo();
let (tid, ..) = demo_crossing(&s);
let clips = s.active_sequence().unwrap().all_tracks().map(|t| t.items.len()).sum::<usize>();
let some_clip = s.active_sequence().unwrap().video_tracks[0].items[0].id.0;
s.execute("timeline.select", json!({"clips": [some_clip]})).unwrap();
let r = s.execute("timeline.select", json!({"transitions": [tid]})).unwrap();
assert_eq!(r["transitionSelection"], json!([tid]));
assert!(s.state.selection.is_empty(), "selecting a transition replaces the clip selection");
assert!(s.execute("timeline.select", json!({"transitions": [987_654]})).is_err(), "unknown ids are refused");
assert!(s.execute("edit.rippleDelete", json!({})).is_err(), "Ripple Delete needs clips");
let before = (*s.project).clone();
s.execute("edit.clear", json!({})).unwrap();
assert!(!transition_ids(&s).contains(&tid), "Delete removes the selected transition");
assert_eq!(s.active_sequence().unwrap().all_tracks().map(|t| t.items.len()).sum::<usize>(), clips, "and leaves the clips");
assert!(s.state.transition_selection.is_empty());
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(*s.project, before, "one undo step");
// selecting clips clears a transition selection
s.execute("timeline.select", json!({"transitions": [tid]})).unwrap();
s.execute("timeline.select", json!({"clips": [some_clip]})).unwrap();
assert!(s.state.transition_selection.is_empty());
// by id, without a selection
s.execute("sequence.removeTransition", json!({"transitions": [tid]})).unwrap();
assert!(!transition_ids(&s).contains(&tid));
assert!(s.execute("sequence.removeTransition", json!({"transitions": [tid]})).is_err(), "nothing left to remove");
assert!(s.execute("sequence.removeTransition", json!({})).is_err(), "nothing given or selected");
}
/// #224 / Effect Controls: Duration follows the alignment, Alignment re-aligns on the cut, Start
/// moves it (Custom Start), all within the clips it joins; a drag is one undo step.
#[test]
fn transition_duration_alignment_and_position() {
let mut s = demo();
let rate = s.sequence_rate();
let f = |n: i64| rate.tick_of(n);
let (tid, _, _, cut) = demo_crossing(&s);
let set = |s: &mut Session, p: Value| {
let mut p = p;
p["transition"] = json!(tid);
s.execute("sequence.setTransition", p)
};
let r = set(&mut s, json!({"align": "start"})).unwrap();
assert_eq!((r["start"].as_i64(), r["align"].as_str()), (Some(cut.0), Some("start")));
// Start at Cut: only the end moves
let r = set(&mut s, json!({"frames": 10})).unwrap();
assert_eq!((r["start"].as_i64(), r["duration"].as_i64()), (Some(cut.0), Some(f(10).0)));
// End at Cut: only the beginning moves
let r = set(&mut s, json!({"align": "end", "frames": 8})).unwrap();
assert_eq!((r["start"].as_i64(), r["duration"].as_i64(), r["align"].as_str()), (Some((cut - f(8)).0), Some(f(8).0), Some("end")));
// Center at Cut: both ends
let r = set(&mut s, json!({"align": "center", "frames": 12})).unwrap();
assert_eq!((r["start"].as_i64(), r["align"].as_str()), (Some((cut - f(6)).0), Some("center")));
// dragging it over the cut: Custom Start; then a new duration keeps its middle
let r = set(&mut s, json!({"start": (cut - f(3)).0})).unwrap();
assert_eq!(r["align"], json!("custom"));
let r = set(&mut s, json!({"frames": 6})).unwrap();
assert_eq!((r["start"].as_i64(), r["duration"].as_i64()), (Some(cut.0), Some(f(6).0)), "centre at cut + 3 frames, 6 frames long");
// dragging one edge: start and duration together
let r = set(&mut s, json!({"start": (cut - f(4)).0, "duration": f(10).0})).unwrap();
assert_eq!((r["start"].as_i64(), r["duration"].as_i64()), (Some((cut - f(4)).0), Some(f(10).0)));
// refused, and nothing changes
let before = (*s.project).clone();
let history = s.history.undo.len();
for p in [
json!({"frames": 0}),
json!({"frames": 100_000_000}),
json!({"duration": -5}),
json!({"duration": i64::MAX}),
json!({"frames": 10, "duration": f(10).0}),
json!({"align": "middle"}),
json!({"align": "start", "start": cut.0}),
json!({"start": i64::MIN}),
json!({"start": (cut + f(1)).0, "duration": f(1).0}),
] {
assert!(set(&mut s, p.clone()).is_err(), "{p}");
assert_eq!(*s.project, before, "{p}");
assert_eq!(s.history.undo.len(), history, "{p}");
}
assert!(s.execute("sequence.setTransition", json!({"transition": 987_654, "frames": 5})).is_err());
// a drag (merge) is one undo step
let history = s.history.undo.len();
for (i, n) in [11, 12, 13, 14].into_iter().enumerate() {
set(&mut s, json!({"frames": n, "merge": true, "begin": i == 0})).unwrap();
}
assert_eq!(s.history.undo.len(), history + 1);
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(*s.project, before);
}
/// #484: Enable flips each selected clip on its own, as in Premiere. With one enabled and one
/// disabled clip selected it swaps them, instead of first making both the same.
#[test]
+23 -11
View File
@@ -386,6 +386,13 @@ fn sync(s: &mut Session, p: &Value) -> Result<Value> {
/// Directory recordings are written to.
fn record_dir(s: &Session, p: &Value) -> String {
media_dir(s, p, "Voice-over Recordings")
}
/// Directory generated media (recordings, narrations) is written to: `dir`, else Project
/// Settings ▸ Scratch Disks ▸ Captured, else next to the project, else `fallback` in the data or
/// temporary directory.
pub(crate) fn media_dir(s: &Session, p: &Value, fallback: &str) -> String {
if let Some(d) = str_p(p, "dir") {
return d.to_string();
}
@@ -396,7 +403,21 @@ fn record_dir(s: &Session, p: &Value) -> String {
return d.to_string_lossy().into_owned();
}
let base = s.prefs_path.as_ref().and_then(|p| p.parent()).map(|d| d.to_path_buf()).unwrap_or_else(|| crate::temp_dir().join("FilmCraft"));
base.join("Voice-over Recordings").to_string_lossy().into_owned()
base.join(fallback).to_string_lossy().into_owned()
}
/// `<dir>/<base> <n>.wav` with the first `n` used by neither a project item nor an existing file;
/// an error once `n` passes 99 999 (bounded: `file_size` is a filesystem call per try).
pub(crate) fn unique_wav_path(s: &Session, dir: &str, base: &str) -> Result<String> {
let names: std::collections::HashSet<String> = s.project.items.values().map(|i| i.name.clone()).collect();
for k in 1..=99_999u32 {
let file = format!("{base} {k}.wav");
let path = std::path::Path::new(dir).join(&file).to_string_lossy().into_owned();
if !names.contains(&file) && s.services.file_size(&path).is_err() {
return Ok(path);
}
}
Err(EngineError::Other(format!("{dir}: no free file name for \"{base} <n>.wav\"")))
}
fn stop(s: &mut Session, p: &Value) -> Result<Value> {
@@ -449,16 +470,7 @@ fn stop(s: &mut Session, p: &Value) -> Result<Value> {
// write the file
let dir = record_dir(s, p);
let base = s.prefs.voice_over.name.clone();
let names: std::collections::HashSet<String> = s.project.items.values().map(|i| i.name.clone()).collect();
let mut k = 1;
let path = loop {
let file = format!("{base} {k}.wav");
let path = std::path::Path::new(&dir).join(&file).to_string_lossy().into_owned();
if !names.contains(&file) && s.services.file_size(&path).is_err() {
break path;
}
k += 1;
};
let path = unique_wav_path(s, &dir, &base)?;
if !cfg!(target_arch = "wasm32") {
std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{dir}: {e}")))?;
}
+10 -2
View File
@@ -35,6 +35,8 @@
//! (`Project::source_graphics`). No-op step; older builds would drop these fields when saving.
//! - **v13**: all container audio streams, and per-timeline-clip stream selection. Old single-stream
//! `audio` objects read as stream zero; older builds must refuse new multi-stream projects.
//! - **v14** (M7.9): narrations (`Project::narrations`, Text to Speech). No-op step; older builds
//! would drop them when saving.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
@@ -56,7 +58,7 @@ pub type Migration = fn(Value) -> Result<Value, String>;
/// `MIGRATIONS[i]` upgrades schema `i + 1` to `i + 2`. Append one function per schema bump; never
/// edit a shipped one.
pub const MIGRATIONS: &[Migration] =
&[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13];
&[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13, v13_to_v14];
/// The schema version this build writes (and the newest it reads).
pub const SCHEMA_VERSION: u32 = MIGRATIONS.len() as u32 + 1;
@@ -224,6 +226,7 @@ fn validate_loaded(mut loaded: Loaded) -> Result<Loaded, FormatError> {
m.info.audio_streams.truncate(max);
}
filmcraft_project::ItemKind::Sequence(sequence) => {
let sequence = std::sync::Arc::make_mut(sequence);
for clip in sequence.audio_tracks.iter_mut().flat_map(|t| t.items.iter_mut()) {
if clip.audio_stream >= max {
log::warn!("sequence `{name}`: clip `{}` uses audio stream {}, reset to 0", clip.name, clip.audio_stream);
@@ -346,13 +349,18 @@ fn v12_to_v13(doc: Value) -> Result<Value, String> {
Ok(doc)
}
/// v13 → v14: narrations (Text to Speech). Existing data needs no change.
fn v13_to_v14(doc: Value) -> Result<Value, String> {
Ok(doc)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn schema_version_matches_table() {
assert_eq!(SCHEMA_VERSION, 13);
assert_eq!(SCHEMA_VERSION, 14);
}
#[test]
+1 -1
View File
@@ -131,7 +131,7 @@ pub fn big_project(n: usize) -> Project {
let seq_id = ItemId(21);
let mut next = 1_000_000u64;
let seq = match &mut p.items.get_mut(&seq_id).unwrap().kind {
ItemKind::Sequence(s) => s,
ItemKind::Sequence(s) => std::sync::Arc::make_mut(s),
_ => unreachable!(),
};
let v0 = seq.video_tracks[0].items[0].clone();
+1 -1
View File
@@ -387,7 +387,7 @@ impl Builder {
/// Add an empty sequence item (filled later with [`Builder::put_sequence`]).
pub fn reserve_sequence(&mut self, name: &str, settings: SequenceSettings, bin: Option<BinId>) -> ItemId {
let seq = empty_sequence(settings);
self.p.add_item(name, Label::Forest, ItemKind::Sequence(Box::new(seq)), bin)
self.p.add_item(name, Label::Forest, ItemKind::Sequence(std::sync::Arc::new(seq)), bin)
}
pub fn put_sequence(&mut self, id: ItemId, mut seq: Sequence) {
+36 -1
View File
@@ -110,6 +110,24 @@ struct Imp<'a, 'i, 'r> {
in_progress: HashSet<String>,
links: Vec<PendingLink>,
xml_clip_ids: HashMap<(ItemId, String), ClipId>,
/// `(sequence key, file key)` pairs where some audio clip item of that file uses `sourcetrack/trackindex` > 1,
/// i.e. the file's channels are split onto mono tracks in that sequence (#463).
split_files: HashSet<(String, String)>,
}
/// The `(sequence key, file key)` an audio clip item belongs to, if it references a file.
fn clip_seq_file(c: Node) -> Option<(String, String)> {
let seq = c.ancestors().skip(1).find(|a| a.has_tag_name("sequence"))?;
Some((node_key(seq), node_key(child(c, "file")?)))
}
/// The 1-based `sourcetrack/trackindex` of an audio clip item.
fn audio_track_index(c: Node) -> Option<i64> {
let st = child(c, "sourcetrack")?;
if child_text(st, "mediatype").is_some_and(|m| m != "audio") {
return None;
}
child_i64(st, "trackindex")
}
fn node_key(n: Node) -> String {
@@ -136,9 +154,15 @@ pub(crate) fn import(text_in: &str, opts: &ImportOptions, report: &mut Report) -
in_progress: HashSet::new(),
links: Vec::new(),
xml_clip_ids: HashMap::new(),
split_files: HashSet::new(),
};
for n in root.descendants().filter(|n| n.is_element()) {
match n.tag_name().name() {
"clipitem" if audio_track_index(n).is_some_and(|i| i > 1) => {
if let Some(key) = clip_seq_file(n) {
imp.split_files.insert(key);
}
}
"file" if elements(n).next().is_some() => {
imp.files.entry(node_key(n)).or_insert(n);
}
@@ -482,6 +506,17 @@ impl<'a, 'i> Imp<'a, 'i, '_> {
if let Some(l) = path_text(c, &["labels", "label2"]).and_then(label_from) {
ti.label = l;
}
// `sourcetrack/trackindex` (1-based) picks the source channel of a mono track split off a multichannel file (#463).
// Index 1 is also what a stereo clip on a stereo track carries, so it means channel 0 only when another clip
// item of the same file in this sequence uses a higher index (the file is split); otherwise it keeps the
// default (empty) channel mapping.
if kind == TrackKind::Audio
&& let Some(idx) = audio_track_index(c)
&& (idx > 1 || (idx == 1 && clip_seq_file(c).is_some_and(|k| self.split_files.contains(&k))))
&& let Ok(ch) = u16::try_from(idx - 1)
{
ti.source_channels = vec![ch];
}
if let Some(mode) = child_text(c, "compositemode") {
match BLEND_XML.iter().find(|(n, _)| n.eq_ignore_ascii_case(mode)) {
Some((_, v)) if *v != 0 => set_param(&mut ti, "opacity", "blend", Param::new(ParamValue::Choice(*v))),
@@ -962,7 +997,7 @@ impl Exp<'_, '_> {
if kind == TrackKind::Audio {
self.w.open("sourcetrack", &[]);
self.w.text("mediatype", "audio");
self.w.text("trackindex", 1);
self.w.text("trackindex", c.source_channels.first().map_or(1, |ch| u32::from(*ch).saturating_add(1)));
self.w.close();
}
if let Some(blend) = param(c, "opacity", "blend").and_then(|p| p.value.as_f64()).map(|v| v as u32).filter(|v| *v != 0) {
+2
View File
@@ -438,6 +438,7 @@ pub fn merge_into(target: &mut Project, fragment: Imported, bin: Option<BinId>)
it.created = it.id.0;
match &mut it.kind {
ItemKind::Sequence(seq) => {
let seq = std::sync::Arc::make_mut(seq);
for m in &mut seq.markers {
m.id = MarkerId(target.alloc_id());
}
@@ -510,6 +511,7 @@ pub fn rebase_source_timecode(project: &mut Project, item: ItemId, start_tc_fram
}
for it in project.items.values_mut() {
if let ItemKind::Sequence(seq) = &mut it.kind {
let seq = std::sync::Arc::make_mut(seq);
for t in seq.all_tracks_mut() {
for ti in t.items.iter_mut().filter(|ti| ti.item == item) {
ti.source_in -= off;
+73
View File
@@ -527,3 +527,76 @@ fn sub_frame_clips_keep_equal_source_and_record_spans() {
assert_eq!(out_f - in_f, end_f - start_f, "start {start}, in {src}, duration {dur} (tenths of a second): {spans:?}");
}
}
/// #463: a stereo file split onto two mono tracks (A1 = channel 1, A2 = channel 2) lost the right channel,
/// because every audio clip item was written with `sourcetrack/trackindex` 1 and the importer ignored it.
#[test]
fn split_mono_tracks_keep_their_source_channel() {
let r = FrameRate::FPS_23_976;
let mut p = Project::new("P");
let a = media(&mut p, "/m/a.wav", false, true, r);
let s = sequence(&mut p, "t", r, false);
let left = clip(&mut p, s, TrackKind::Audio, 0, a, 0, 24, 0);
let right = clip(&mut p, s, TrackKind::Audio, 1, a, 0, 24, 0);
for (id, ch) in [(left, 0u16), (right, 1u16)] {
let (_, it) = p.sequence_mut(s).unwrap().find_item_mut(id).unwrap();
it.source_channels = vec![ch];
}
let (imp, xml, _) = roundtrip(&p, s, Format::Fcp7Xml, &ExportOptions::default());
assert!(xml.contains("<trackindex>2</trackindex>"), "{xml}");
let seq = imp.project.sequence(only_seq(&imp)).unwrap();
let channels = |t: usize| seq.tracks(TrackKind::Audio)[t].items.iter().map(|i| i.source_channels.clone()).collect::<Vec<_>>();
assert_eq!(channels(0), vec![vec![0u16]], "{xml}");
assert_eq!(channels(1), vec![vec![1u16]], "{xml}");
let (bytes, _) = export(&imp.project, only_seq(&imp), Format::Fcp7Xml, &ExportOptions::default()).expect("export");
assert!(String::from_utf8(bytes).unwrap().contains("<trackindex>2</trackindex>"));
}
#[test]
fn audio_sourcetrack_trackindex_2_imports_as_source_channel_1() {
let doc = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE xmeml>
<xmeml version="4"><sequence id="sequence-1"><name>S</name><duration>48</duration>{RATE}
<media><audio><track><clipitem id="clipitem-1"><name>take</name>{RATE}<start>0</start><end>48</end><in>0</in><out>48</out>
<file id="file-1"><name>take.wav</name><pathurl>file://localhost/media/take.wav</pathurl>{RATE}<duration>48</duration><media><audio><channelcount>2</channelcount></audio></media></file>
<sourcetrack><mediatype>audio</mediatype><trackindex>2</trackindex></sourcetrack>
</clipitem></track></audio></media>
</sequence></xmeml>"#
);
let (imp, _) = import(doc.as_bytes(), Format::Fcp7Xml, None).unwrap();
let seq = imp.project.sequence(only_seq(&imp)).unwrap();
assert_eq!(seq.tracks(TrackKind::Audio)[0].items[0].source_channels, vec![1u16]);
}
/// #463: Premiere writes a stereo file split onto two mono tracks as clip items with `trackindex` 1 and 2;
/// a file used only with `trackindex` 1 is a stereo clip and keeps the default (all channels) mapping.
#[test]
fn audio_trackindex_1_is_channel_0_only_when_the_file_is_split() {
let doc = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE xmeml>
<xmeml version="4"><sequence id="sequence-1"><name>S</name><duration>48</duration>{RATE}
<media><audio>
<track><clipitem id="clipitem-1"><name>take</name>{RATE}<start>0</start><end>48</end><in>0</in><out>48</out>
<file id="file-1"><name>take.wav</name><pathurl>file://localhost/media/take.wav</pathurl>{RATE}<duration>48</duration><media><audio><channelcount>2</channelcount></audio></media></file>
<sourcetrack><mediatype>audio</mediatype><trackindex>1</trackindex></sourcetrack>
</clipitem>
<clipitem id="clipitem-3"><name>other</name>{RATE}<start>48</start><end>96</end><in>0</in><out>48</out>
<file id="file-2"><name>other.wav</name><pathurl>file://localhost/media/other.wav</pathurl>{RATE}<duration>48</duration><media><audio><channelcount>2</channelcount></audio></media></file>
<sourcetrack><mediatype>audio</mediatype><trackindex>1</trackindex></sourcetrack>
</clipitem></track>
<track><clipitem id="clipitem-2"><name>take</name>{RATE}<start>0</start><end>48</end><in>0</in><out>48</out>
<file id="file-1"/>
<sourcetrack><mediatype>audio</mediatype><trackindex>2</trackindex></sourcetrack>
</clipitem></track>
</audio></media>
</sequence></xmeml>"#
);
let (imp, _) = import(doc.as_bytes(), Format::Fcp7Xml, None).unwrap();
let seq = imp.project.sequence(only_seq(&imp)).unwrap();
let a1 = &seq.tracks(TrackKind::Audio)[0].items;
assert_eq!(a1[0].source_channels, vec![0u16]);
assert!(a1[1].source_channels.is_empty(), "{:?}", a1[1].source_channels);
assert_eq!(seq.tracks(TrackKind::Audio)[1].items[0].source_channels, vec![1u16]);
}
+1
View File
@@ -50,6 +50,7 @@ windows = { version = "0.62", features = [
"Win32_Media_MediaFoundation",
"Win32_System_Com",
"Win32_System_LibraryLoader",
"Win32_UI_WindowsAndMessaging",
] }
# Linux: libva (VA-API) is loaded at run time with libloading (ISC; already in the lockfile through
+28
View File
@@ -0,0 +1,28 @@
//! The mouse position in screen pixels, for when the windowing layer has none.
//!
//! While files are dragged in from the OS file manager, winit on Windows delivers no cursor events
//! (it only reports hovered and dropped files), so the UI cannot learn where a drop landed.
//!
//! This is the one non-media OS call in this crate: `crates/platform` is the only crate allowed
//! `unsafe` (AGENTS.md §0.3, ADR 0001), so the single `GetCursorPos` call lives here rather than
//! in the UI. It is read-only, has a safe `Option`-returning API, and every other platform (and a
//! failed call) falls back to the last pointer position egui saw. Drop it once winit reports the
//! pointer position during an OS drag-and-drop.
/// The cursor's position on the virtual screen in physical pixels, or `None` where the OS cannot
/// say or this platform's windowing layer already reports it during a drag.
#[cfg(target_os = "windows")]
#[allow(unsafe_code)]
pub fn cursor_screen_position() -> Option<(i32, i32)> {
use windows::Win32::Foundation::POINT;
use windows::Win32::UI::WindowsAndMessaging::GetCursorPos;
let mut p = POINT { x: 0, y: 0 };
// SAFETY: `p` is a valid, writable POINT for the duration of the call.
unsafe { GetCursorPos(&mut p) }.ok()?;
Some((p.x, p.y))
}
#[cfg(not(target_os = "windows"))]
pub fn cursor_screen_position() -> Option<(i32, i32)> {
None
}
+1
View File
@@ -34,6 +34,7 @@
mod annexb;
#[cfg_attr(not(any(target_os = "windows", target_os = "linux")), allow(dead_code))]
mod biplanar;
pub mod cursor;
#[cfg(target_os = "macos")]
pub mod hardware_encode;
pub mod hybrid;
+19
View File
@@ -126,6 +126,25 @@ fn layouts_match_the_header() {
(112, 4),
"NV_ENC_CONFIG_HEVC_VUI_PARAMETERS"
);
// The header typedefs the HEVC VUI to the H.264 one; pin the fields session.rs writes, both
// inside the VUI and where they land in NV_ENC_CONFIG_HEVC (hevcVUIParameters at 64), so a
// binding mismatch cannot hide behind the typedef (#442: HEVC timing came out reversed).
for (field, offset, name) in [
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, videoSignalTypePresentFlag), 8, "videoSignalTypePresentFlag"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, videoFormat), 12, "videoFormat"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, videoFullRangeFlag), 16, "videoFullRangeFlag"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, colourDescriptionPresentFlag), 20, "colourDescriptionPresentFlag"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, colourPrimaries), 24, "colourPrimaries"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, transferCharacteristics), 28, "transferCharacteristics"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, colourMatrix), 32, "colourMatrix"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, timingInfoPresentFlag), 52, "timingInfoPresentFlag"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, numUnitInTicks), 56, "numUnitInTicks"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, timeScale), 60, "timeScale"),
(offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, reserved), 64, "reserved"),
] {
assert_eq!(field, offset, "NV_ENC_CONFIG_HEVC_VUI_PARAMETERS.{name}");
assert_eq!(offset_of!(NV_ENC_CONFIG_HEVC, hevcVUIParameters) + field, 64 + offset, "NV_ENC_CONFIG_HEVC.hevcVUIParameters.{name}");
}
assert_eq!((size_of::<NV_ENC_CONFIG_HEVC>(), align_of::<NV_ENC_CONFIG_HEVC>()), (1560, 8), "NV_ENC_CONFIG_HEVC");
assert_eq!(offset_of!(NV_ENC_CONFIG_HEVC, level), 0, "NV_ENC_CONFIG_HEVC.level");
assert_eq!(offset_of!(NV_ENC_CONFIG_HEVC, tier), 4, "NV_ENC_CONFIG_HEVC.tier");
+5 -3
View File
@@ -416,9 +416,11 @@ impl Session {
vui.colourPrimaries = u32::from(p.signal.primaries);
vui.transferCharacteristics = u32::from(p.signal.transfer);
vui.colourMatrix = u32::from(p.signal.matrix);
vui.timingInfoPresentFlag = 1;
vui.numUnitInTicks = p.fps.1;
vui.timeScale = p.fps.0;
// Let NVENC derive timing from frameRateNum/frameRateDen. Some drivers
// reverse the explicitly supplied HEVC VUI timing pair.
vui.timingInfoPresentFlag = 0;
vui.numUnitInTicks = 0;
vui.timeScale = 0;
}
}
+16
View File
@@ -321,3 +321,19 @@ fn a_keyframe_every_one_or_two_pictures_is_written_without_b_frames() {
assert_eq!(n + dec.flush().len(), 12, "keyint {keyint}");
}
}
#[test]
fn vui_timing_matches_integer_and_fractional_frame_rates() {
let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
for fps in [(24, 1), (25, 1), (30_000, 1_001)] {
let mut cfg = config(640, 360);
cfg.fps = fps;
cfg.bframes = false;
let Some((enc, _, _)) = encode_all(&cfg, 2) else { return };
let sps_nal = enc.parameter_sets().0;
let sps = Sps::parse(&unescape_rbsp(&sps_nal[2..])).unwrap();
let (units, scale) = sps.vui.and_then(|v| v.timing).expect("HEVC VUI timing");
assert!(units > 0 && scale > 0);
assert_eq!(u64::from(scale) * u64::from(fps.1), u64::from(units) * u64::from(fps.0), "timing for {fps:?}");
}
}
+37 -3
View File
@@ -20,6 +20,7 @@ pub mod keyframe;
pub mod mask;
pub mod mixer;
pub mod multicam;
pub mod narration;
pub mod transcript;
pub mod vtransition;
@@ -39,6 +40,7 @@ pub use keyframe::{Interpolation, Keyframe, Param, ParamValue};
pub use mask::{Mask, MaskMode, MaskPath, MaskVertex, TrackMethod};
pub use mixer::{AutomationMode, InputMap, MixerStrip, TrackSend};
pub use multicam::{Camera, MergedClip, MulticamAudio, MulticamSel, MulticamSource};
pub use narration::{Narration, VocalPitch};
pub use transcript::{Speaker, Transcript, Word};
macro_rules! id_type {
@@ -262,7 +264,9 @@ impl MediaClip {
#[allow(clippy::large_enum_variant)]
pub enum ItemKind {
Media(MediaClip),
Sequence(Box<Sequence>),
/// Shared so project snapshots used by undo and background readers do not deep-copy every
/// timeline. Mutators detach with `Arc::make_mut`.
Sequence(std::sync::Arc<Sequence>),
/// A subclip: a media item restricted to a range.
Subclip {
parent: ItemId,
@@ -327,7 +331,7 @@ impl ProjectItem {
}
pub fn as_sequence_mut(&mut self) -> Option<&mut Sequence> {
match &mut self.kind {
ItemKind::Sequence(s) => Some(s),
ItemKind::Sequence(s) => Some(std::sync::Arc::make_mut(s)),
_ => None,
}
}
@@ -1245,6 +1249,10 @@ pub struct Project {
/// project item. Schema v12.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub source_graphics: BTreeMap<ItemId, SourceGraphic>,
/// Narrations (Text to Speech): the script and voice settings each generated WAV item was
/// made from, keyed by that item. Schema v14.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub narrations: BTreeMap<ItemId, Narration>,
}
/// How a sequence is shown in the Timeline panel: its zoom, scroll position and track heights.
@@ -1339,6 +1347,7 @@ impl Project {
transcripts: BTreeMap::new(),
search_bins: Vec::new(),
source_graphics: BTreeMap::new(),
narrations: BTreeMap::new(),
}
}
@@ -1502,7 +1511,7 @@ impl Project {
let id = TrackId(self.alloc_id());
seq.audio_tracks.push(Track::new(id, TrackKind::Audio, format!("Audio {}", i + 1)));
}
self.add_item(name, Label::Forest, ItemKind::Sequence(Box::new(seq)), bin)
self.add_item(name, Label::Forest, ItemKind::Sequence(std::sync::Arc::new(seq)), bin)
}
/// Re-base an item's media time: whatever was at media time `t` is at `t - delta` afterwards
@@ -1524,6 +1533,7 @@ impl Project {
}
for it in self.items.values_mut() {
if let ItemKind::Sequence(seq) = &mut it.kind {
let seq = std::sync::Arc::make_mut(seq);
for t in seq.all_tracks_mut() {
for ti in t.items.iter_mut().filter(|ti| users.contains(&ti.item)) {
ti.source_in -= delta;
@@ -1984,6 +1994,30 @@ mod tests {
assert_eq!(p.root.parent_of(i), None);
}
#[test]
fn project_clone_shares_sequences_until_mutated() {
fn sequence_arc(p: &Project, id: ItemId) -> &std::sync::Arc<Sequence> {
match &p.item(id).unwrap().kind {
ItemKind::Sequence(q) => q,
_ => unreachable!(),
}
}
let (mut p, _, a) = demo_project();
let b = p.new_sequence("Sequence 02", SequenceSettings::default(), 1, 1, None);
let before = p.clone();
assert!(std::sync::Arc::ptr_eq(sequence_arc(&p, a), sequence_arc(&before, a)));
assert!(std::sync::Arc::ptr_eq(sequence_arc(&p, b), sequence_arc(&before, b)));
p.sequence_mut(a).unwrap().mark_in = Some(Tick(1));
assert!(!std::sync::Arc::ptr_eq(sequence_arc(&p, a), sequence_arc(&before, a)));
assert!(std::sync::Arc::ptr_eq(sequence_arc(&p, b), sequence_arc(&before, b)));
assert_eq!(before.sequence(a).unwrap().mark_in, None);
assert_eq!(Project::from_json(&p.to_json()).unwrap(), p);
}
#[test]
fn nest_cycle_finds_sequences_that_contain_themselves() {
let (mut p, clip, a) = demo_project();
+152
View File
@@ -0,0 +1,152 @@
//! Narrations: audio generated from text (the Text to Speech panel).
//!
//! A [`Narration`] belongs to the project item of the WAV file that was generated for it
//! (`Project::narrations`, keyed by the item id), so every clip of that item can be clicked to
//! reopen the script and voice settings. Editing a narration writes a new WAV (a new item) and
//! swaps the clip over to it; the old item and its narration stay, so undo restores them exactly.
use filmcraft_time::Tick;
use serde::{Deserialize, Serialize};
/// Longest script accepted, in bytes.
pub const MAX_TEXT_BYTES: usize = 64 * 1024;
/// Pace limits (speaking rate multiplier).
pub const MIN_PACE: f64 = 0.5;
pub const MAX_PACE: f64 = 2.0;
/// Text to Speech ▸ Advanced ▸ Vocal pitch.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum VocalPitch {
ExtraLow,
Low,
#[default]
Default,
High,
ExtraHigh,
}
impl VocalPitch {
pub const ALL: [VocalPitch; 5] = [VocalPitch::ExtraLow, VocalPitch::Low, VocalPitch::Default, VocalPitch::High, VocalPitch::ExtraHigh];
pub fn label(self) -> &'static str {
match self {
VocalPitch::ExtraLow => "Extra low",
VocalPitch::Low => "Low",
VocalPitch::Default => "Default",
VocalPitch::High => "High",
VocalPitch::ExtraHigh => "Extra high",
}
}
/// The command / serde id (`extraLow`, `low`, `default`, `high`, `extraHigh`).
pub fn id(self) -> &'static str {
match self {
VocalPitch::ExtraLow => "extraLow",
VocalPitch::Low => "low",
VocalPitch::Default => "default",
VocalPitch::High => "high",
VocalPitch::ExtraHigh => "extraHigh",
}
}
pub fn from_id(s: &str) -> Option<VocalPitch> {
let n = s.to_ascii_lowercase().replace([' ', '_', '-'], "");
Self::ALL.into_iter().find(|p| p.id().to_ascii_lowercase() == n)
}
/// Pitch shift in semitones.
pub fn semitones(self) -> f64 {
match self {
VocalPitch::ExtraLow => -6.0,
VocalPitch::Low => -3.0,
VocalPitch::Default => 0.0,
VocalPitch::High => 3.0,
VocalPitch::ExtraHigh => 6.0,
}
}
}
/// The script and voice settings a narration was generated from.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Narration {
/// The script, with pause markers (`[pause 1s]`) as typed.
pub text: String,
/// BCP 47 language tag (`en-US`).
pub language: String,
/// Voice id (`tts.voices`).
pub voice: String,
#[serde(default)]
pub pitch: VocalPitch,
/// Speaking rate, [`MIN_PACE`]–[`MAX_PACE`].
pub pace: f64,
/// Length of the speech in the generated file. The file can be longer: when an edit made the
/// speech shorter than its clip, the file is padded with silence to the clip's length.
pub speech_duration: Tick,
}
impl Narration {
/// Check values read from a project file or a command.
pub fn check(&self) -> Result<(), String> {
if self.text.len() > MAX_TEXT_BYTES {
return Err(format!("the script is longer than {} KB", MAX_TEXT_BYTES / 1024));
}
if !(MIN_PACE..=MAX_PACE).contains(&self.pace) {
return Err(format!("pace must be between {MIN_PACE} and {MAX_PACE}"));
}
if self.speech_duration < Tick::ZERO {
return Err("negative speech duration".into());
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn sample() -> Narration {
Narration {
text: "Hello there. [pause 1s] Welcome.".into(),
language: "en-US".into(),
voice: "basic-female".into(),
pitch: VocalPitch::High,
pace: 1.25,
speech_duration: Tick::from_seconds_f64(2.5),
}
}
#[test]
fn round_trips_through_json() {
let n = sample();
let s = serde_json::to_string(&n).unwrap();
assert!(s.contains("\"pitch\":\"high\""), "{s}");
assert_eq!(serde_json::from_str::<Narration>(&s).unwrap(), n);
}
#[test]
fn pitch_ids_round_trip() {
for p in VocalPitch::ALL {
assert_eq!(VocalPitch::from_id(p.id()), Some(p));
}
assert_eq!(VocalPitch::from_id("Extra High"), Some(VocalPitch::ExtraHigh));
assert_eq!(VocalPitch::from_id("loud"), None);
}
#[test]
fn check_rejects_hostile_values() {
let mut n = sample();
n.pace = f64::NAN;
assert!(n.check().is_err());
n.pace = 3.0;
assert!(n.check().is_err());
let mut n = sample();
n.text = "a".repeat(MAX_TEXT_BYTES + 1);
assert!(n.check().is_err());
let mut n = sample();
n.speech_duration = Tick(-1);
assert!(n.check().is_err());
assert!(sample().check().is_ok());
}
}
+11 -4
View File
@@ -222,15 +222,22 @@ pub type DownloadProgress<'a> = &'a mut dyn FnMut(u64, u64, &str) -> bool;
/// verifier.
#[cfg(feature = "download")]
pub fn download(models_dir: &Path, m: &ModelInfo, progress: DownloadProgress) -> Result<(), crate::SpeechError> {
download_files(&model_dir(models_dir, m), m.files, progress)
}
/// [`download`] for any pinned file list (the neural voices use it too): missing files of
/// `files` go into `dir`, each verified against its SHA-256 and size before it is renamed into
/// place.
#[cfg(feature = "download")]
pub fn download_files(dir: &Path, files: &[ModelFile], progress: DownloadProgress) -> Result<(), crate::SpeechError> {
use crate::SpeechError;
use sha2::Digest;
use std::io::{Read, Write};
let dir = model_dir(models_dir, m);
std::fs::create_dir_all(&dir)?;
let total = missing_bytes(models_dir, m);
std::fs::create_dir_all(dir)?;
let total: u64 = files.iter().filter(|f| !std::fs::metadata(dir.join(f.name)).is_ok_and(|md| md.len() == f.size)).map(|f| f.size).sum();
let agent = agent();
let mut done = 0u64;
for f in m.files {
for f in files {
let dest = dir.join(f.name);
if std::fs::metadata(&dest).is_ok_and(|md| md.len() == f.size) {
continue;
+14
View File
@@ -0,0 +1,14 @@
[package]
name = "filmcraft-tts-text"
description = "FilmCraft text-to-speech text front end: text normalization (numbers, money, dates, times, abbreviations, acronyms, pause markers) into plain spoken words, English and Spanish"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
thiserror = { workspace = true }
+170
View File
@@ -0,0 +1,170 @@
# filmcraft-tts-text
Text-to-speech text front end for FilmCraft (L2, phases P2a + P2b). Takes a
narration script and returns plain spoken words, structured as tokens: `Word`,
`Pause` (from `[pause 1s]` markers), `SentenceEnd` and `Punct`
(`normalize`), and turns those words into Kokoro phoneme symbols (`phonemize`).
```rust
use filmcraft_tts_text::{normalize, Lang, Token};
let tokens = normalize("Call 555-0134 by 3:30 p.m. on Oct. 7th, it's $3.50 (20% off)", Lang::EnUs)?;
// → Call | five five five zero one three four | by | three thirty p m | on |
// October seventh | , | it's | three dollars and fifty cents | ( | twenty percent off | )
```
English only (`Lang::EnUs`, `Lang::EnGb`), per the amended brief. Handles:
cardinals to 999 999 999 999, negatives, decimals, ordinals, money (`$`, `€`,
`£`), percent, years (1100–2099), clock times, numeric and abbreviated dates,
phone-like digit groups, common abbreviations (Mr., Mrs., Dr., St., Mt., vs.,
etc., e.g., i.e., a.m., p.m.), spelled acronyms, `&`/`+`/`@`/`#`/`/`, and pause
markers. Never crashes: >1 MiB input is `NormalizeError::TooLong`, oversized or
leading-zero numbers are read digit by digit, malformed pause markers become
plain text, and all slicing is boundary-safe with `get()`.
## Rules where behaviour is a judgement call
- **Cardinals have no "and"**: "1,234" → "one thousand two hundred thirty-four".
British "one thousand two hundred **and** thirty-four" is not implemented.
- **Decimals read digit by digit** after "point" ("3.14" → "three point one four").
- **Years**: a standalone 4-digit number in 1100–2099: 1100–1999 →
century + remainder ("nineteen ninety-nine", "nineteen oh five",
"nineteen hundred"); 2000–2009 → "two thousand five"; 2010–2099 →
"twenty twenty-six". 2100+ reads as an ordinary cardinal.
- **St.** is "saint" before a capitalised name, "street" otherwise; **Dr.** is
always "doctor" (drive/dr. disambiguation is out of scope, as the brief allows).
- **Acronyms**: unpronounceable all-caps words of 2–5 letters are spelled letter
by letter ("TDA" → T D A, "FBI" → F B I): a leading consonant cluster of 2+ or
an interior run of 3+ consonants makes an acronym unpronounceable, as does
having no vowels at all. Pronounceable ones (NASA, UNESCO and a small list)
stay words. Single letters are Word tokens that G2P should read as letter names.
- **a.m./p.m.** become the letter words "a m"/"p m". Lowercase "pm" without dots
is spelled; lowercase "am" is left alone (it's the English verb).
- **Fractions**: 1/2 → "one half"; other denominators ≤ 12 → "N ordinal(s)"
("three quarters"); anything else → "N over M". Dates need a 4-digit year —
"10/7" alone reads as a fraction.
- **Digit groups**: hyphen-joined digits read digit by digit when any group has
a leading zero or there are 7+ digits in total ("555-0134"); otherwise two
small groups read "A to B" ("2-3" → "two to three").
- **Pause markers**: `[pause <n>s|ms]`, capped at 10 s; malformed markers
([pause], [pause x], unclosed) stay plain text — never an error.
- **Money**: "$3.50" → "three dollars and fifty cents" (€ → euros/cents, £ →
pounds/pence); a lone `$`/`€`/`£` with no number stays a punctuation token.
- **Sentence ends**: runs of `.!?` are one `SentenceEnd`; abbreviation and
number dots are consumed by their rules and never end a sentence.
## Limitations
- English only (amended brief). Spanish was cut from this crate's scope.
- No "and" in British cardinals; no British-ism beyond date order and £.
- Pronunciation is not attempted here — this crate stops at spoken words.
- Time hours 0–12 read as cardinals ("0:15" → "zero fifteen"), not "twelve".
- Ordinal suffixes are validated but "1st"/"1 th"-style spacing is not accepted.
- "may" is never treated as a month abbreviation (it's a word); other dotted
month abbreviations (Jan.–Dec.) are.
## References
None (rules written from scratch). Clean-room: no TTS/normalizer source was read.
# P2b — English pronunciation (`phonemize`)
```rust
pub fn phonemize(text: &str, lang: Lang, lex: &Lexicon, overrides: &Overrides)
-> Result<Vec<Chunk>, NormalizeError>;
pub enum Chunk { Phonemes(String), Pause { millis: u32 } }
pub const MAX_CHUNK_CHARS: usize = 400;
pub const SYMBOLS: &str; // every character phonemize may emit
```
`phonemize` normalizes the script, then phonemizes each sentence into one
`Chunk::Phonemes` (split at the last `,`/`;`/`:` before 400 chars, else at the
last space, else hard-split). Normalizer pauses become `Chunk::Pause`. One space
between words; punctuation from the normalizer attaches to the preceding word
with no space (`həlˈO, wˈɜɹld.`); a sentence terminator becomes a `.`.
## The ARPAbet → Kokoro table (American English)
AA→ɑ, AE→æ, AH0→ə, AH1/2→ʌ, AO→ɔ, AW→W, AY→I, EH→ɛ, ER0→ɚ, ER1/2→ɜɹ, EY→A,
IH→ɪ, IY→i, OW→O, OY→Y, UH→ʊ, UW→u; B→b, CH→ʧ, D→d, DH→ð, F→f, G→ɡ (U+0261,
not ASCII g), HH→h, JH→ʤ, K→k, L→l, M→m, N→n, NG→ŋ, P→p, R→ɹ, S→s, SH→ʃ, T→t,
TH→θ, V→v, W→w, Y→j, Z→z, ZH→ʒ. `A I O W Y` are Kokoro's single-symbol
diphthongs (eɪ aɪ oʊ aʊ ɔɪ).
## Stress
`ˈ` (primary) / `ˌ` (secondary) go immediately before the stressed vowel symbol,
not before the syllable: HELLO (HH AH0 L OW1) → `həlˈO`. Single-syllable
function words (a, an, the, of, to, and, in, is, it, for, on, with, at, by, as,
be, or, but, from, that, this) get no stress mark. Spelled-out single letters
use the letter's name from the dictionary (T → `tˈi`; CMUdict lists "A" as the
article first and the letter name as an alternate, and the letter-name alternate
is preferred).
## Lookup order
1. Overrides (case-insensitive; ≤ 1 000 entries of ≤ 200 chars honoured, excess
ignored). The value is either raw Kokoro phonemes between slashes
(`/tˌidˌiˈA/` — accepted only if every character is in `SYMBOLS`, otherwise
the override is ignored) or a plain-English respelling ("tee dee ay") that
goes through the same dictionary + rules.
2. Dictionary (first pronunciation). The dictionary is CMUdict in its text
format, parsed by `Lexicon::parse_cmudict` (alternates `WORD(2)`, `;;;`
comments, `#` trailing comments; malformed lines skipped and counted by
`skipped()`; input over 16 MiB or 500 000 entries is `LexError`).
3. Morphology on a dictionary stem: plural/possessive `-s`/`-'s` with voicing
(s / z / ɪz), `-ed` (t / d / ɪd), `-ing` (stem may regain its silent e:
making → make), `-er`, `-ly`, and the prefixes `un-`, `re-`.
4. Hyphenated words: each part goes through the same pipeline.
5. Letter-to-sound rules.
## Letter-to-sound rules (written from scratch, clean-room)
A left-to-right scanner with longest-match rules: `tion`/`sion` (stress on the
vowel before the suffix), `ture`/`sure`, `ough` (→ ɔ; +f word-final; "ought" →
ɔt), `igh`, `tch`, `dge`, `ch`, `sh`, `th`, `ph`, `ck`, `qu`, `wh`, `gh` (g at
the start, f word-final, silent before t), `ng` final, `nk`, initial `kn`/`gn`
(silent k/g), `wr`, final `mb`, `gu` (silent u), word-initial `ps`, `sch`,
`sten` (listen), final consonant+`le` → əl, vowel digraphs (ee/ea/ie→i, oo/ue/ui→u,
ou→W, ow→O, oi/oy→Y, ai/ay/ei→A, au/aw→ɔ, oa/oe→O), silent final e with
lengthening of the last vowel (cake → `kˈAk`), doubled consonants keeping the
vowel short (little → `lˈɪtəl`), soft c/g before e/i/y, `x` → ks, final y → i.
Primary stress lands on the first syllable unless a suffix rule moves it.
Every word with letters produces pronounceable output — never empty.
## Measured accuracy
Held-out measurement against the real CMUdict (`cmusphinx/cmudict`, commit
`74790861f652b15e4ac49015a90074ad62a27690`, SHA-256
`81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22`): every 50th
entry held out, lexicon rebuilt without them, letter-to-sound output compared
with the dictionary pronunciation. On 2026-10-08, machine: mjcamacho's Linux
workstation (rustc 1.99.0, release build): **229 held-out words, 9.1 % exact
word accuracy, 37.9 % phoneme error rate** (Levenshtein over phoneme symbols).
The test runs whenever `FILMCRAFT_CMUDICT` points at a CMUdict file and prints
the numbers; it asserts only that it runs. CMUdict is downloaded with the voices
at run time, never committed (BSD-style licence, attributed in the download
dialog).
## Limitations
- Homographs get their first CMUdict pronunciation only: "read" is always
/ɹiːd/, never /ɹɛd/; live/live, lead/lead, bow/bow are wrong half the time.
Overrides are the mitigation.
- Names, jargon and irregular spellings are the letter-to-sound rules' weak
spot (colonel, yacht, place names). Measured accuracy above is the honest
number.
- No flap-T or connected-speech effects ("water" is /ˈwɔtɚ/-style from the
dictionary, rules give ˈwɔtɚ only if the dictionary has it).
- `ow` is read /oʊ/ everywhere (so "wow" → wˈO), `ough` has one reading plus
word-final f; "ia"/"ea" ambiguities resolved to one value each.
- The sentence terminator always becomes `.` (the normalizer collapses `!`/`?`
into `SentenceEnd`).
- English only.
## References
None (rules written from scratch). Clean-room: no G2P/TTS project source was
read. Allowed sources used: CMUdict's own data and licence, published IPA
charts, and Wikipedia's ARPAbet and English phonology pages.
+26
View File
@@ -0,0 +1,26 @@
//! Text front end for FilmCraft's Text to Speech (phase P2a).
//!
//! [`normalize`] turns a narration script into plain spoken words, structured as
//! [`Token`]s: words, pauses ([`Token::Pause`], from `[pause 1s]`-style markers),
//! sentence ends ([`Token::SentenceEnd`]) and kept punctuation ([`Token::Punct`]).
//! It handles numbers (cardinals, ordinals, negatives, decimals), money (`$`, `€`,
//! `£`), percent, years, clock times, dates, phone-like digit groups, common
//! abbreviations, acronyms and the `&`/`+`/`@`/`#`/`/` symbols — English
//! ([`Lang::EnUs`] / [`Lang::EnGb`]).
//!
//! This crate is the first stage of the text pipeline; the grapheme-to-phoneme step
//! (P2b) consumes its words. It is pure Rust, depends only on `serde` and
//! `thiserror`, and never crashes on hostile input (see `AGENTS.md` §0): input over
//! 1 MiB is a [`NormalizeError::TooLong`], oversized numbers are read digit by digit,
//! and malformed pause markers fall back to plain text rather than erroring.
//!
//! The rules were written from scratch (clean-room); see the crate README for the
//! judgement calls and limitations.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
mod normalize;
pub mod phonemize;
pub use normalize::{Lang, NormalizeError, Token, normalize};
pub use phonemize::{Chunk, LexError, Lexicon, MAX_CHUNK_CHARS, Overrides, SYMBOLS, phonemize};
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+33
View File
@@ -0,0 +1,33 @@
[package]
name = "filmcraft-tts"
description = "FilmCraft text to speech: the Voice trait, the voice catalogue, script parsing (pause markers), the built-in formant voice and (feature kokoro) Kokoro-82M neural voices on candle"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[lints]
workspace = true
[features]
default = []
# Kokoro-82M neural voices on the CPU via candle (pure Rust). Off by default and in wasm builds.
kokoro = ["dep:candle-core", "dep:candle-nn", "dep:zip", "dep:serde_json", "dep:filmcraft-tts-text", "dep:filmcraft-audio-dsp"]
[dependencies]
filmcraft-tts-text = { workspace = true, optional = true }
filmcraft-audio-dsp = { workspace = true, optional = true }
serde = { workspace = true }
thiserror = { workspace = true }
candle-core = { version = "0.9", default-features = false, optional = true }
candle-nn = { version = "0.9", default-features = false, optional = true }
zip = { version = "7", default-features = false, optional = true }
serde_json = { workspace = true, optional = true }
[[example]]
name = "kokoro_say"
required-features = ["kokoro"]
[[example]]
name = "narrate"
required-features = ["kokoro"]
+83
View File
@@ -0,0 +1,83 @@
# filmcraft-tts
Text to speech for FilmCraft's Text to Speech panel (narrations, L2). Engine commands (`tts.*`) and
clip behaviour: `crates/engine/src/narration.rs`.
- `Voice` trait (script in, mono 24 kHz `f32` out), the voice catalogue (`voices`, `voice`), and
the limits (64 KB script, 30-minute narration, pace 0.5–2×, pitch ±12 semitones).
- `script`: pause markers `[pause 1s]` / `[pause 500ms]` (case-insensitive, capped at 10 s;
malformed markers are read as text).
- `formant`: the built-in voices **Basic Female** and **Basic Male**, an original source–filter
formant synthesizer: spelling rules → phone targets → glottal pulse train through three two-pole
resonators, plus filtered noise for fricatives and bursts. No download, works in the web build.
- `kokoro` (feature `kokoro`): **Kokoro-82M neural voices** in pure Rust on candle (CPU): ALBERT
text encoder, StyleTTS 2 prosody predictor (durations, F0, energy) and decoder, iSTFTNet
vocoder with a harmonic-plus-noise source. Weights are read from the official PyTorch
checkpoint with candle's restricted pickle reader (no code from the file runs); voice packs from
their zip archive. `config.json` is checked against the architecture this code implements.
- `neural` (feature `kokoro`): the natural voices as `Voice`s: script → `filmcraft-tts-text`
(normalizer, CMUdict + letter-to-sound, ≤ 400-symbol chunks, pauses) → Kokoro per chunk → exact
silence for pauses, 120 ms between sentences → vocal pitch with `filmcraft-audio-dsp`'s pitch
shifter (Kokoro has no pitch input). Pace is Kokoro's speed.
- `catalog`: the natural-voice package (Kokoro-82M, 9 US English voices graded C+ or better on the
model card, CMUdict; 336 MB), every file pinned to a revision and SHA-256. Downloaded by the
engine (`tts.downloadVoices`) after the user confirms; never bundled or committed.
Everything is deterministic (the vocoder's noise is seeded). Without features the crate depends
only on `serde` and `thiserror` (and checks for wasm).
## Measured quality and speed (M7.12, 2026-10-08)
50 original sentences (`tests/data/sentences.txt`: numbers, money, times, dates, phone numbers,
acronyms, names) spoken by each voice through the full pipeline, transcribed by Whisper-base
(FilmCraft's own, `filmcraft-speech`), word error rate after lowercasing and removing punctuation
(number formatting such as "six" vs "6" still counts as an error):
| Voice | WER | | Voice | WER |
|---|---|---|---|---|
| Heart | 2.1 % | | Puck | 2.8 % |
| Aoede | 2.4 % | | Michael | 3.2 % |
| Nicole | 2.8 % | | Kore | 3.4 % |
| Bella | 3.0 % | | Fenrir, Sarah | 3.6 % |
| Basic Female (built in) | 96.6 % | | | |
Speed: 2.2–2.3× real time on one desktop CPU (Linux x86-64, load average ≈ 15 from parallel
builds), e.g. 186 s of speech in 84 s; model load 0.5 s. Reproduce with the `narrate` example and
`filmcraft-speech`'s `transcribe` example.
## Limitations
- The built-in voices are **robotic** and nearly unintelligible to a recogniser (above); they are
the no-download fallback and the test voice.
- English (United States) only. Words outside CMUdict go through letter-to-sound rules and are
often mispronounced (see `crates/tts-text/README.md`); homographs take the first pronunciation.
- About 2× faster than real time on the CPU: a one-minute narration takes about 30 s. The UI runs
synthesis as a background job.
- Kokoro's model card states the weights are Apache-2.0 but the repository has no LICENSE file.
- Vocal pitch shifts the voice's base pitch (and its formants slightly); pace scales every duration.
## Tests
`cargo test -p filmcraft-tts`: determinism, valid samples (finite, peak ≤ 0.8, audible RMS), pause
markers produce exactly the requested number of zero samples, pace 2× / 0.5× gives 0.5× / 2× the
length (±5 % / ±10 %), measured pitch (autocorrelation) of the female voice is ≥ 1.5× the male, and
+6 semitones raises it by √2 (±0.12); empty, emoji-only, non-Latin, pause-only and over-long scripts
are refused before rendering; hostile scripts never panic.
Model-dependent tests run when `FILMCRAFT_KOKORO_DIR` names a downloaded package directory
(determinism, whole frames, pace changes length, refusals); the loaders have hostile-input tests
(truncated or oversized voice data, wrong byte order, NaN, non-zip files, mismatched config).
## References
- G. E. Peterson and H. L. Barney, "Control Methods Used in a Study of the Vowels", JASA 24 (1952):
the published average formant frequencies used for the vowels.
- D. H. Klatt, "Software for a cascade/parallel formant synthesizer", JASA 67 (1980): the standard
two-pole digital resonator form. No code was read or copied; everything else is original.
- Kokoro (implemented from papers, `config.json` and the checkpoint's tensor names and shapes; no
reference code read): Y. A. Li et al., "StyleTTS 2" (NeurIPS 2023); T. Kaneko et al., "iSTFTNet"
(ICASSP 2022); Z. Lan et al., "ALBERT" (ICLR 2020); J. Kong et al., "HiFi-GAN" (NeurIPS 2020);
X. Wang et al., "Neural source-filter waveform models" (2019); L. Ziyin et al., "Neural Networks
Fail to Learn Periodic Functions and How to Fix It" (Snake, NeurIPS 2020). Weights: hexgrad,
Kokoro-82M, Apache-2.0.
+46
View File
@@ -0,0 +1,46 @@
//! `cargo run -p filmcraft-tts --release --features kokoro --example kokoro_say -- <model dir> <voice.pt> "<phonemes>" out.wav [speed]`
//!
//! Speaks Kokoro phonemes with a voice pack and writes a 24 kHz mono 32-bit float WAV.
fn main() -> Result<(), Box<dyn std::error::Error>> {
let a: Vec<String> = std::env::args().collect();
let (Some(dir), Some(voice), Some(ph), Some(out)) = (a.get(1), a.get(2), a.get(3), a.get(4)) else {
eprintln!("usage: kokoro_say <model dir> <voice.pt> <phonemes> <out.wav> [speed]");
std::process::exit(2);
};
let speed: f64 = a.get(5).and_then(|s| s.parse().ok()).unwrap_or(1.0);
let t0 = std::time::Instant::now();
let k = filmcraft_tts::kokoro::Kokoro::load(std::path::Path::new(dir))?;
let v = filmcraft_tts::kokoro::VoicePack::load(std::path::Path::new(voice))?;
let t1 = std::time::Instant::now();
let s = k.speak(ph, &v, speed, 1)?;
let t2 = std::time::Instant::now();
let rate = filmcraft_tts::kokoro::KOKORO_RATE;
let mut w = Vec::with_capacity(44 + s.len() * 4);
let data = (s.len() * 4) as u32;
w.extend_from_slice(b"RIFF");
w.extend_from_slice(&(36 + data).to_le_bytes());
w.extend_from_slice(b"WAVEfmt ");
w.extend_from_slice(&16u32.to_le_bytes());
w.extend_from_slice(&3u16.to_le_bytes());
w.extend_from_slice(&1u16.to_le_bytes());
w.extend_from_slice(&rate.to_le_bytes());
w.extend_from_slice(&(rate * 4).to_le_bytes());
w.extend_from_slice(&4u16.to_le_bytes());
w.extend_from_slice(&32u16.to_le_bytes());
w.extend_from_slice(b"data");
w.extend_from_slice(&data.to_le_bytes());
for x in &s {
w.extend_from_slice(&x.to_le_bytes());
}
std::fs::write(out, w)?;
let secs = s.len() as f64 / f64::from(rate);
eprintln!(
"load {:.2}s, synth {:.2}s for {secs:.2}s of audio (×{:.1} real time), peak {:.3}",
(t1 - t0).as_secs_f64(),
(t2 - t1).as_secs_f64(),
secs / (t2 - t1).as_secs_f64(),
s.iter().fold(0f32, |m, x| m.max(x.abs()))
);
Ok(())
}
+49
View File
@@ -0,0 +1,49 @@
//! `cargo run -p filmcraft-tts --release --features kokoro --example narrate -- <models dir> <voice id> <script.txt> <out.wav>`
//!
//! Speaks a script file with any FilmCraft voice (built in or natural) through the full pipeline
//! (normalizer, pronunciation, Kokoro) and writes a mono 32-bit float WAV.
fn main() -> Result<(), Box<dyn std::error::Error>> {
let a: Vec<String> = std::env::args().collect();
let (Some(models), Some(id), Some(script), Some(out)) = (a.get(1), a.get(2), a.get(3), a.get(4)) else {
eprintln!("usage: narrate <models dir> <voice id> <script.txt> <out.wav>");
std::process::exit(2);
};
let text = std::fs::read_to_string(script)?;
let t0 = std::time::Instant::now();
let v = filmcraft_tts::voice_in(id, Some(std::path::Path::new(models)))?;
let t1 = std::time::Instant::now();
let audio = v.synthesize(&text, &filmcraft_tts::Params::default())?;
let t2 = std::time::Instant::now();
let rate = audio.sample_rate;
let data = (audio.samples.len() * 4) as u32;
let mut w = Vec::with_capacity(44 + data as usize);
for part in [
&b"RIFF"[..],
&(36 + data).to_le_bytes(),
b"WAVEfmt ",
&16u32.to_le_bytes(),
&3u16.to_le_bytes(),
&1u16.to_le_bytes(),
&rate.to_le_bytes(),
&(rate * 4).to_le_bytes(),
&4u16.to_le_bytes(),
&32u16.to_le_bytes(),
b"data",
&data.to_le_bytes(),
] {
w.extend_from_slice(part);
}
for x in &audio.samples {
w.extend_from_slice(&x.to_le_bytes());
}
std::fs::write(out, w)?;
eprintln!(
"{id}: load {:.2}s, synth {:.2}s for {:.2}s of audio (×{:.1} real time)",
(t1 - t0).as_secs_f64(),
(t2 - t1).as_secs_f64(),
audio.seconds(),
audio.seconds() / (t2 - t1).as_secs_f64()
);
Ok(())
}
+154
View File
@@ -0,0 +1,154 @@
//! The downloadable neural voices: Kokoro-82M, its voice packs and the CMUdict pronunciation
//! dictionary. Every file is pinned to a revision and checked against its SHA-256 when downloaded
//! (the downloader is `filmcraft_speech::models::download_files`, run by the engine after the
//! user confirms). Nothing here is bundled with FilmCraft or committed to the repository.
//!
//! Licences: Kokoro-82M weights and voices by hexgrad, Apache-2.0 (declared on the model card; the
//! repository has no separate LICENSE file). CMUdict, Carnegie Mellon University, BSD-style.
//! Pins verified 2026-10-07 against the Hugging Face API and by `sha256sum`.
use std::path::{Path, PathBuf};
use serde::Serialize;
use crate::{Gender, VoiceInfo};
/// One file of the neural voice package.
#[derive(Clone, Copy, Debug, Serialize)]
pub struct ModelFile {
/// Local file name inside the package directory.
pub name: &'static str,
pub url: &'static str,
pub sha256: &'static str,
pub size: u64,
}
/// Directory name of the package under the models directory.
pub const PACKAGE_ID: &str = "kokoro-82m";
pub const PACKAGE_NAME: &str = "Natural voices (Kokoro-82M)";
pub const LICENSE: &str = "Apache-2.0 (Kokoro-82M by hexgrad); CMUdict: BSD-style (Carnegie Mellon University)";
pub const SOURCE: &str = "https://huggingface.co/hexgrad/Kokoro-82M";
macro_rules! hf {
($name:literal, $path:literal, $sha:literal, $size:expr) => {
ModelFile {
name: $name,
url: concat!("https://huggingface.co/hexgrad/Kokoro-82M/resolve/f3ff3571791e39611d31c381e3a41a3af07b4987/", $path),
sha256: $sha,
size: $size,
}
};
}
pub static FILES: &[ModelFile] = &[
hf!("config.json", "config.json", "5abb01e2403b072bf03d04fde160443e209d7a0dad49a423be15196b9b43c17f", 2351),
hf!("kokoro-v1_0.pth", "kokoro-v1_0.pth", "496dba118d1a58f5f3db2efc88dbdc216e0483fc89fe6e47ee1f2c53f18ad1e4", 327_212_226),
ModelFile {
name: "cmudict.dict",
url: "https://raw.githubusercontent.com/cmusphinx/cmudict/74790861f652b15e4ac49015a90074ad62a27690/cmudict.dict",
sha256: "81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22",
size: 3_618_488,
},
hf!("af_heart.pt", "voices/af_heart.pt", "0ab5709b8ffab19bfd849cd11d98f75b60af7733253ad0d67b12382a102cb4ff", 523_425),
hf!("af_bella.pt", "voices/af_bella.pt", "8cb64e02fcc8de0327a8e13817e49c76c945ecf0052ceac97d3081480e8e48d6", 523_425),
hf!("af_nicole.pt", "voices/af_nicole.pt", "c5561808bcf5250fe8c5f5de32caf2d94f27e57e95befdb098c5c85991d4c5da", 523_430),
hf!("af_aoede.pt", "voices/af_aoede.pt", "c03bd1a4c3716c2d8eaa3d50022f62d5c31cfbd6e15933a00b17fefe13841cc4", 523_425),
hf!("af_kore.pt", "voices/af_kore.pt", "8bfbc512321c3db49dff984ac675fa5ac7eaed5a96cc31104d3a9080e179d69d", 523_420),
hf!("af_sarah.pt", "voices/af_sarah.pt", "49bd364ea3be9eb3e9685e8f9a15448c4883112a7c0ff7ab139fa4088b08cef9", 523_425),
hf!("am_michael.pt", "voices/am_michael.pt", "9a443b79a4b22489a5b0ab7c651a0bcd1a30bef675c28333f06971abbd47bd37", 523_435),
hf!("am_fenrir.pt", "voices/am_fenrir.pt", "98e507eca1db08230ae3b6232d59c10aec9630022d19accac4f5d12fcec3c37a", 523_430),
hf!("am_puck.pt", "voices/am_puck.pt", "dd1d8973f4ce4b7d8ae407c77a435f485dabc052081b80ea75c4f30b84f36223", 523_420),
];
/// A neural voice: catalogue entry and its voice-pack file.
#[derive(Clone, Copy, Debug)]
pub struct NeuralVoice {
pub info: VoiceInfo,
pub file: &'static str,
}
const NEURAL_LICENSE: &str = "Apache-2.0 (Kokoro-82M by hexgrad)";
macro_rules! voice {
($id:literal, $name:literal, $gender:ident, $file:literal, $desc:literal) => {
NeuralVoice {
info: VoiceInfo {
id: $id,
name: $name,
language: "en-US",
gender: Gender::$gender,
engine: "neural",
description: $desc,
license: NEURAL_LICENSE,
installed: false,
},
file: $file,
}
};
}
/// US English voices the model card grades C+ or better, best first.
pub static VOICES: &[NeuralVoice] = &[
voice!("kokoro-heart", "Heart", Female, "af_heart.pt", "Natural voice, warm (model card grade A)."),
voice!("kokoro-bella", "Bella", Female, "af_bella.pt", "Natural voice, bright (grade A−)."),
voice!("kokoro-nicole", "Nicole", Female, "af_nicole.pt", "Natural voice, soft (grade B−)."),
voice!("kokoro-michael", "Michael", Male, "am_michael.pt", "Natural voice (grade C+)."),
voice!("kokoro-fenrir", "Fenrir", Male, "am_fenrir.pt", "Natural voice, deep (grade C+)."),
voice!("kokoro-puck", "Puck", Male, "am_puck.pt", "Natural voice, lively (grade C+)."),
voice!("kokoro-aoede", "Aoede", Female, "af_aoede.pt", "Natural voice (grade C+)."),
voice!("kokoro-kore", "Kore", Female, "af_kore.pt", "Natural voice (grade C+)."),
voice!("kokoro-sarah", "Sarah", Female, "af_sarah.pt", "Natural voice (grade C+)."),
];
pub fn find(id: &str) -> Option<&'static NeuralVoice> {
VOICES.iter().find(|v| v.info.id == id)
}
/// Total download size.
pub fn size() -> u64 {
FILES.iter().map(|f| f.size).sum()
}
pub fn package_dir(models_dir: &Path) -> PathBuf {
models_dir.join(PACKAGE_ID)
}
/// Every file present with its pinned size (contents were verified when downloaded).
pub fn installed(models_dir: &Path) -> bool {
let d = package_dir(models_dir);
FILES.iter().all(|f| std::fs::metadata(d.join(f.name)).is_ok_and(|m| m.len() == f.size))
}
/// Bytes still to download.
pub fn missing_bytes(models_dir: &Path) -> u64 {
let d = package_dir(models_dir);
FILES.iter().filter(|f| !std::fs::metadata(d.join(f.name)).is_ok_and(|m| m.len() == f.size)).map(|f| f.size).sum()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn files_are_pinned_and_voices_resolve() {
for f in FILES {
assert_eq!(f.sha256.len(), 64, "{}", f.name);
assert!(f.sha256.bytes().all(|b| b.is_ascii_hexdigit()));
assert!(
f.url.starts_with("https://")
&& (f.url.contains("/f3ff3571791e39611d31c381e3a41a3af07b4987/") || f.url.contains("/74790861f652b15e4ac49015a90074ad62a27690/")),
"{}",
f.url
);
assert!(!f.name.contains('/'), "flat package layout: {}", f.name);
}
for v in VOICES {
assert!(FILES.iter().any(|f| f.name == v.file), "{} has no file", v.info.id);
assert_eq!(find(v.info.id).map(|x| x.file), Some(v.file));
}
assert!(size() > 330_000_000 && size() < 340_000_000, "{}", size());
let dir = std::env::temp_dir().join(format!("filmcraft-tts-cat-{}", std::process::id()));
assert!(!installed(&dir));
assert_eq!(missing_bytes(&dir), size());
}
}
+736
View File
@@ -0,0 +1,736 @@
//! The built-in voices: an original source–filter formant synthesizer.
//!
//! Text is turned into phones by simple English spelling rules ([`phones`]): digraphs (`th`, `sh`,
//! `ee`, `oo`, `igh`…), a silent final `e` that lengthens the vowel before it (`make`, `time`),
//! and digits read as words. Each phone is a target (three formant frequencies and bandwidths,
//! voicing and noise amplitudes, a duration). The renderer moves between targets with short
//! transitions, drives a cascade of three two-pole resonators with a glottal pulse train for voiced
//! sounds, and a resonator at the phone's noise frequency with white noise for fricatives, bursts
//! and aspiration. Pitch follows a falling line across each sentence and rises at a question mark.
//!
//! Vowel formants are the published adult averages of Peterson & Barney, "Control Methods Used in
//! a Study of the Vowels" (JASA 24, 1952). The digital resonator is the standard two-pole form
//! described in D. H. Klatt, "Software for a cascade/parallel formant synthesizer" (JASA 67, 1980).
//! Everything else is original. The result is intelligible-ish and clearly robotic: it is the
//! no-download fallback, not the main voice.
use crate::script::{self, Segment};
use crate::{Audio, Gender, MAX_SECONDS, Params, SAMPLE_RATE, TtsError, Voice, VoiceInfo, check_text};
/// A built-in voice: catalogue entry plus its speaking parameters.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct VoiceSpec {
pub info: VoiceInfo,
/// Base pitch in Hz.
pub f0: f64,
/// Formant scale (vocal tract length): 1 = adult male averages.
pub formant_scale: f64,
/// Breath noise mixed into voicing, 0–1.
pub breath: f32,
}
const LICENSE: &str = "Built into FilmCraft (project licence)";
pub static VOICES: &[VoiceSpec] = &[
VoiceSpec {
info: VoiceInfo {
id: "basic-female",
name: "Basic Female",
language: "en-US",
gender: Gender::Female,
engine: "basic",
description: "Built-in voice. No download; robotic.",
license: LICENSE,
installed: true,
},
f0: 205.0,
formant_scale: 1.17,
breath: 0.06,
},
VoiceSpec {
info: VoiceInfo {
id: "basic-male",
name: "Basic Male",
language: "en-US",
gender: Gender::Male,
engine: "basic",
description: "Built-in voice. No download; robotic.",
license: LICENSE,
installed: true,
},
f0: 112.0,
formant_scale: 1.0,
breath: 0.03,
},
];
/// A built-in voice ready to speak.
pub struct FormantVoice {
spec: VoiceSpec,
}
impl FormantVoice {
pub fn new(spec: VoiceSpec) -> Self {
FormantVoice { spec }
}
}
impl Voice for FormantVoice {
fn info(&self) -> VoiceInfo {
self.spec.info
}
fn synthesize(&self, text: &str, params: &Params) -> Result<Audio, TtsError> {
check_text(text)?;
params.check()?;
let segments = script::parse(text);
// phones per speech segment, and the length check before anything is rendered
let mut plan: Vec<Piece> = Vec::with_capacity(segments.len());
let mut total_ms = 0.0f64;
let mut spoken = false;
for seg in &segments {
match seg {
Segment::Pause { millis } => {
total_ms += f64::from(*millis);
plan.push(Piece::Pause(*millis));
}
Segment::Speech(s) => {
let ph = phones(s);
spoken |= ph.iter().any(|p| p.kind != Kind::Silence);
total_ms += ph.iter().map(|p| f64::from(p.dur_ms)).sum::<f64>() / params.pace;
plan.push(Piece::Speech(ph));
}
}
if total_ms > MAX_SECONDS * 1000.0 {
return Err(TtsError::AudioTooLong);
}
}
if !spoken {
return Err(TtsError::Empty);
}
let estimate = (total_ms / 1000.0 * f64::from(SAMPLE_RATE)).ceil() as usize;
let mut out: Vec<f32> = Vec::new();
out.try_reserve(estimate.saturating_add(SAMPLE_RATE as usize)).map_err(|e| TtsError::Invalid(format!("unable to allocate the narration: {e}")))?;
for piece in &plan {
match piece {
Piece::Pause(ms) => {
let n = (f64::from(*ms) / 1000.0 * f64::from(SAMPLE_RATE)).round() as usize;
out.resize(out.len().saturating_add(n), 0.0);
}
Piece::Speech(ph) => render(&self.spec, ph, params, &mut out),
}
}
// one gain for the whole narration so loudness doesn't jump between pieces
let peak = out.iter().fold(0.0f32, |m, s| m.max(s.abs()));
if peak > 0.0 && peak.is_finite() {
let g = 0.8 / peak;
for s in &mut out {
*s *= g;
}
} else if !peak.is_finite() {
return Err(TtsError::Invalid("synthesis produced invalid samples".into()));
}
Ok(Audio { samples: out, sample_rate: SAMPLE_RATE })
}
}
enum Piece {
Pause(u32),
Speech(Vec<Phone>),
}
// ------------------------------------------------------------------------------------- phones
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Kind {
Vowel,
Glide,
Nasal,
Fricative,
Closure,
Burst,
Silence,
}
/// One synthesis target.
#[derive(Clone, Copy, Debug, PartialEq)]
struct Phone {
kind: Kind,
/// Formant frequencies (adult male, scaled per voice) and bandwidths, Hz.
f: [f32; 3],
bw: [f32; 3],
/// Voicing amplitude 0–1.
voice: f32,
/// Noise amplitude 0–1, through a resonator at `noise_f` / `noise_bw`.
noise: f32,
noise_f: f32,
noise_bw: f32,
dur_ms: f32,
/// Pitch multiplier at this phone (stress, sentence end).
pitch: f32,
/// Ends a sentence (pitch reset after it).
sentence_end: bool,
}
const BW: [f32; 3] = [80.0, 100.0, 140.0];
/// Neutral tract: what silences and closures move towards.
const NEUTRAL: [f32; 3] = [500.0, 1500.0, 2500.0];
fn base(kind: Kind, f: [f32; 3], dur_ms: f32) -> Phone {
Phone { kind, f, bw: BW, voice: 0.0, noise: 0.0, noise_f: 3000.0, noise_bw: 2000.0, dur_ms, pitch: 1.0, sentence_end: false }
}
fn vowel(f: [f32; 3], dur_ms: f32) -> Phone {
Phone { voice: 1.0, ..base(Kind::Vowel, f, dur_ms) }
}
fn glide(f: [f32; 3]) -> Phone {
Phone { voice: 0.8, ..base(Kind::Glide, f, 65.0) }
}
fn nasal(f: [f32; 3]) -> Phone {
Phone { voice: 0.45, bw: [60.0, 200.0, 300.0], ..base(Kind::Nasal, f, 75.0) }
}
fn fricative(noise_f: f32, noise_bw: f32, noise: f32, voiced: bool, dur_ms: f32) -> Phone {
Phone { voice: if voiced { 0.35 } else { 0.0 }, noise, noise_f, noise_bw, ..base(Kind::Fricative, NEUTRAL, dur_ms) }
}
fn silence(dur_ms: f32) -> Phone {
base(Kind::Silence, NEUTRAL, dur_ms)
}
/// A stop: closure then burst.
fn stop(out: &mut Vec<Phone>, burst_f: f32, voiced: bool) {
out.push(Phone { voice: if voiced { 0.12 } else { 0.0 }, ..base(Kind::Closure, NEUTRAL, if voiced { 45.0 } else { 55.0 }) });
out.push(Phone { noise: 0.9, noise_f: burst_f, noise_bw: 1800.0, ..base(Kind::Burst, NEUTRAL, if voiced { 15.0 } else { 30.0 }) });
}
// Peterson & Barney (1952) adult male averages: F1, F2, F3.
const AA: [f32; 3] = [730.0, 1090.0, 2440.0];
const AE: [f32; 3] = [660.0, 1720.0, 2410.0];
const AH: [f32; 3] = [640.0, 1190.0, 2390.0];
const AO: [f32; 3] = [570.0, 840.0, 2410.0];
const EH: [f32; 3] = [530.0, 1840.0, 2480.0];
const ER: [f32; 3] = [490.0, 1350.0, 1690.0];
const IH: [f32; 3] = [390.0, 1990.0, 2550.0];
const IY: [f32; 3] = [270.0, 2290.0, 3010.0];
const UH: [f32; 3] = [440.0, 1020.0, 2240.0];
const UW: [f32; 3] = [300.0, 870.0, 2240.0];
fn diphthong(out: &mut Vec<Phone>, a: [f32; 3], b: [f32; 3], stressed: bool) {
let d = if stressed { 95.0 } else { 70.0 };
out.push(vowel(a, d));
out.push(vowel(b, d * 0.8));
}
const DIGITS: [&str; 10] = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"];
fn is_vowel(c: char) -> bool {
matches!(c, 'a' | 'e' | 'i' | 'o' | 'u')
}
/// Phones for a piece of speech (no pause markers).
fn phones(text: &str) -> Vec<Phone> {
let mut out = Vec::new();
let mut word = String::new();
let mut question_from = 0usize;
for c in text.chars().chain(std::iter::once(' ')) {
let c = c.to_ascii_lowercase();
if c.is_ascii_alphabetic() || c == '\'' {
if c != '\'' {
word.push(c);
}
continue;
}
if !word.is_empty() {
speak_word(&word, &mut out);
word.clear();
}
match c {
'0'..='9' => {
let d = DIGITS.get(c as usize - '0' as usize).copied().unwrap_or("");
speak_word(d, &mut out);
out.push(silence(30.0));
}
',' | ';' | ':' | '-' | '(' | ')' => out.push(silence(170.0)),
'.' | '!' | '?' => {
if c == '?' {
// rise over the last word of the question
let n = out.len();
let from =
out.get(question_from..).map_or(n, |s| s.iter().rposition(|p| p.kind == Kind::Silence).map_or(question_from, |i| question_from + i));
for p in out.get_mut(from..).into_iter().flatten() {
p.pitch *= 1.25;
}
}
if let Some(last) = out.last_mut() {
last.sentence_end = true;
}
out.push(silence(330.0));
question_from = out.len();
}
_ => {
if out.last().is_some_and(|p| p.kind != Kind::Silence) {
out.push(silence(25.0));
}
}
}
}
out
}
/// Spelling rules for one lowercase ASCII word.
fn speak_word(w: &str, out: &mut Vec<Phone>) {
let c: Vec<char> = w.chars().collect();
let n = c.len();
let at = |i: usize| c.get(i).copied().unwrap_or(' ');
// a final silent e makes the vowel before the last consonant long: make, time, home, cute
let magic_e = n >= 3 && at(n - 1) == 'e' && !is_vowel(at(n - 2)) && is_vowel(at(n - 3));
let mut stressed = true;
let mut i = 0;
while i < n {
let (a, b, d) = (at(i), at(i + 1), at(i + 2));
let long = magic_e && i == n - 3;
let vd = if stressed { 115.0 } else { 80.0 };
let before = out.len();
let step = match a {
'a' if b == 'i' || b == 'y' => {
diphthong(out, EH, IY, stressed);
2
}
'a' if b == 'u' || b == 'w' => {
out.push(vowel(AO, vd));
2
}
'a' if b == 'r' => {
out.push(vowel(AA, vd));
out.push(glide([420.0, 1300.0, 1600.0]));
2
}
'a' if long => {
diphthong(out, EH, IY, stressed);
1
}
'a' => {
out.push(vowel(if stressed { AE } else { AH }, vd));
1
}
'e' if b == 'e' || b == 'a' => {
out.push(vowel(IY, vd + 20.0));
2
}
'e' if b == 'r' => {
out.push(vowel(ER, vd));
2
}
'e' if b == 'y' && i + 2 == n => {
out.push(vowel(IY, vd));
2
}
'e' if i == n - 1 && n > 2 => 1, // silent final e
'e' => {
out.push(vowel(if i == n - 1 { IY } else { EH }, vd));
1
}
'i' if b == 'g' && d == 'h' => {
diphthong(out, AA, IY, stressed);
3
}
'i' if long || (i == n - 1 && n <= 2) => {
diphthong(out, AA, IY, stressed);
1
}
'i' if b == 'r' => {
out.push(vowel(ER, vd));
2
}
'i' => {
out.push(vowel(IH, vd));
1
}
'o' if b == 'o' => {
out.push(vowel(if d == 'k' { UH } else { UW }, vd + 15.0));
2
}
'o' if b == 'u' || b == 'w' => {
diphthong(out, AA, UW, stressed);
2
}
'o' if b == 'i' || b == 'y' => {
diphthong(out, AO, IY, stressed);
2
}
'o' if b == 'a' || long || i == n - 1 => {
diphthong(out, AO, UW, stressed);
if b == 'a' { 2 } else { 1 }
}
'o' if b == 'r' => {
out.push(vowel(AO, vd));
out.push(glide([420.0, 1300.0, 1600.0]));
2
}
'o' => {
out.push(vowel(AA, vd));
1
}
'u' if long => {
out.push(glide([280.0, 2200.0, 3000.0]));
out.push(vowel(UW, vd));
1
}
'u' if b == 'r' => {
out.push(vowel(ER, vd));
2
}
'u' => {
out.push(vowel(AH, vd));
1
}
'y' if i == 0 => {
out.push(glide([280.0, 2200.0, 3000.0]));
1
}
'y' => {
if i == n - 1 && n <= 3 {
diphthong(out, AA, IY, stressed);
} else {
out.push(vowel(if i == n - 1 { IY } else { IH }, vd));
}
1
}
't' if b == 'h' => {
out.push(fricative(6000.0, 4000.0, 0.25, false, 80.0));
2
}
's' if b == 'h' => {
out.push(fricative(2600.0, 1400.0, 0.7, false, 110.0));
2
}
'c' if b == 'h' => {
stop(out, 4000.0, false);
out.push(fricative(2600.0, 1400.0, 0.7, false, 70.0));
2
}
'p' if b == 'h' => {
out.push(fricative(5000.0, 4000.0, 0.3, false, 90.0));
2
}
'w' if b == 'h' => {
out.push(glide([300.0, 610.0, 2200.0]));
2
}
'n' if b == 'g' => {
out.push(nasal([250.0, 2300.0, 2700.0]));
2
}
'c' if b == 'k' => {
stop(out, 2000.0, false);
2
}
'q' if b == 'u' => {
stop(out, 2000.0, false);
out.push(glide([300.0, 610.0, 2200.0]));
2
}
'c' if matches!(b, 'e' | 'i' | 'y') => {
out.push(fricative(6000.0, 2000.0, 0.6, false, 100.0));
1
}
'c' | 'k' | 'q' => {
stop(out, 2000.0, false);
1
}
'g' if matches!(b, 'e' | 'i') && i > 0 => {
stop(out, 3500.0, true);
out.push(fricative(2600.0, 1400.0, 0.5, true, 60.0));
1
}
'g' => {
stop(out, 2000.0, true);
1
}
'j' => {
stop(out, 3500.0, true);
out.push(fricative(2600.0, 1400.0, 0.5, true, 60.0));
1
}
'b' => {
stop(out, 800.0, true);
1
}
'd' => {
stop(out, 3500.0, true);
1
}
'p' => {
stop(out, 800.0, false);
1
}
't' => {
stop(out, 4000.0, false);
1
}
'x' => {
stop(out, 2000.0, false);
out.push(fricative(6000.0, 2000.0, 0.6, false, 90.0));
1
}
'f' => {
out.push(fricative(5000.0, 4000.0, 0.3, false, 90.0));
1
}
'v' => {
out.push(fricative(5000.0, 4000.0, 0.25, true, 70.0));
1
}
's' => {
out.push(fricative(6000.0, 2000.0, 0.6, false, 100.0));
1
}
'z' => {
out.push(fricative(6000.0, 2000.0, 0.45, true, 85.0));
1
}
'h' => {
out.push(fricative(1500.0, 2500.0, 0.3, false, 55.0));
1
}
'l' => {
out.push(glide([360.0, 1300.0, 2700.0]));
1
}
'r' => {
out.push(glide([420.0, 1300.0, 1600.0]));
1
}
'w' => {
out.push(glide([300.0, 610.0, 2200.0]));
1
}
'm' => {
out.push(nasal([250.0, 1100.0, 2400.0]));
1
}
'n' => {
out.push(nasal([250.0, 1700.0, 2600.0]));
1
}
_ => 1,
};
let added_vowel = out.get(before..).is_some_and(|s| s.iter().any(|p| p.kind == Kind::Vowel));
if added_vowel && stressed {
for p in out.get_mut(before..).into_iter().flatten() {
p.pitch *= 1.08;
}
stressed = false;
}
// doubled consonants are one sound: "ll", "ss", "tt"
let mut skip = step;
while !is_vowel(a) && at(i + skip) == a {
skip += 1;
}
i += skip;
}
}
// ------------------------------------------------------------------------------------- render
/// Two-pole resonator (unity gain at DC).
#[derive(Clone, Copy, Default)]
struct Resonator {
a: f32,
b: f32,
c: f32,
y1: f32,
y2: f32,
}
impl Resonator {
fn set(&mut self, f: f32, bw: f32) {
let t = 1.0 / SAMPLE_RATE as f32;
let f = f.clamp(50.0, SAMPLE_RATE as f32 * 0.45);
let bw = bw.clamp(20.0, 8000.0);
self.c = -(-2.0 * std::f32::consts::PI * bw * t).exp();
self.b = 2.0 * (-std::f32::consts::PI * bw * t).exp() * (2.0 * std::f32::consts::PI * f * t).cos();
self.a = 1.0 - self.b - self.c;
}
fn tick(&mut self, x: f32) -> f32 {
let y = self.a * x + self.b * self.y1 + self.c * self.y2;
self.y2 = self.y1;
self.y1 = y;
y
}
}
/// Deterministic white noise (xorshift32), −1..1.
struct Noise(u32);
impl Noise {
fn next(&mut self) -> f32 {
let mut x = self.0;
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
self.0 = x;
(x as f32 / u32::MAX as f32) * 2.0 - 1.0
}
}
/// Parameters are updated every `BLOCK` samples.
const BLOCK: usize = 48;
/// Transition into each phone, ms.
const TRANSITION_MS: f32 = 30.0;
fn lerp(a: f32, b: f32, t: f32) -> f32 {
a + (b - a) * t
}
fn render(spec: &VoiceSpec, ph: &[Phone], params: &Params, out: &mut Vec<f32>) {
let sr = SAMPLE_RATE as f32;
let shift = 2f64.powf(params.semitones / 12.0);
let f0_base = (spec.f0 * shift) as f32;
let fscale = (spec.formant_scale * 2f64.powf(params.semitones / 12.0 * 0.15)) as f32;
let pace = params.pace as f32;
let mut res = [Resonator::default(); 3];
let mut nres = Resonator::default();
let mut noise = Noise(0x9E37_79B9);
let mut phase = 0.0f32;
let mut prev_glottal = 0.0f32;
let mut dc = (0.0f32, 0.0f32);
// sentence pitch line: from 1.1 down to 0.85 of the base over each sentence
let sentence_len = |from: usize| -> f32 { ph.get(from..).map_or(0, |s| s.iter().position(|p| p.sentence_end).map_or(s.len(), |e| e + 1)) as f32 };
let mut sentence_start = 0usize;
let mut sentence_n = sentence_len(0).max(1.0);
let mut prev = silence(0.0);
for (k, p) in ph.iter().enumerate() {
let n = ((p.dur_ms / pace) / 1000.0 * sr).round().max(1.0) as usize;
let trans = ((TRANSITION_MS / pace) / 1000.0 * sr).min(n as f32 * 0.5).max(1.0);
let line = 1.1 - 0.25 * ((k - sentence_start) as f32 / sentence_n);
let mut i = 0;
while i < n {
let t = ((i as f32) / trans).min(1.0);
let f = [0, 1, 2].map(|j| lerp(prev.f[j], p.f[j], t) * fscale);
for (j, r) in res.iter_mut().enumerate() {
r.set(f[j], p.bw[j] * fscale.sqrt());
}
nres.set(lerp(prev.noise_f, p.noise_f, t), lerp(prev.noise_bw, p.noise_bw, t));
let voice = lerp(prev.voice, p.voice, t);
let namp = lerp(prev.noise, p.noise, t);
let f0 = (f0_base * line * lerp(prev.pitch, p.pitch, t)).clamp(40.0, 600.0);
let end = (i + BLOCK).min(n);
for _ in i..end {
phase += f0 / sr;
if phase >= 1.0 {
phase -= 1.0;
}
// glottal flow pulse (open 60 % of the period), differentiated for lip radiation
let g = if phase < 0.6 { (std::f32::consts::PI * phase / 0.6).sin().powi(2) } else { 0.0 };
let dg = g - prev_glottal;
prev_glottal = g;
let breath = noise.next() * spec.breath * g;
let mut v = (dg * 6.0 + breath) * voice;
for r in &mut res {
v = r.tick(v);
}
let nz = nres.tick(noise.next()) * namp * 0.6;
// DC blocker
let x = v + nz;
let y = x - dc.0 + 0.995 * dc.1;
dc = (x, y);
out.push(y);
}
i = end;
}
prev = *p;
if p.sentence_end {
sentence_start = k + 1;
sentence_n = sentence_len(k + 1).max(1.0);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::voice;
fn say(id: &str, text: &str, p: Params) -> Audio {
voice(id).unwrap().synthesize(text, &p).unwrap()
}
/// Dominant pitch of the middle of `s` by autocorrelation, 60–500 Hz.
fn pitch(s: &[f32]) -> f64 {
let mid = &s[s.len() / 3..s.len() * 2 / 3];
let sr = f64::from(SAMPLE_RATE);
let (lo, hi) = ((sr / 500.0) as usize, (sr / 60.0) as usize);
let best = (lo..hi).max_by(|&a, &b| {
let ca: f64 = mid.iter().zip(&mid[a..]).map(|(x, y)| f64::from(x * y)).sum();
let cb: f64 = mid.iter().zip(&mid[b..]).map(|(x, y)| f64::from(x * y)).sum();
ca.total_cmp(&cb)
});
sr / best.unwrap() as f64
}
#[test]
fn speaks_deterministically_with_valid_samples() {
let a = say("basic-female", "Hello world, this is FilmCraft. Ready?", Params::default());
let b = say("basic-female", "Hello world, this is FilmCraft. Ready?", Params::default());
assert_eq!(a, b);
assert_eq!(a.sample_rate, SAMPLE_RATE);
assert!(a.seconds() > 1.0 && a.seconds() < 8.0, "{}", a.seconds());
assert!(a.samples.iter().all(|s| s.is_finite() && s.abs() <= 0.8001));
let rms = (a.samples.iter().map(|s| f64::from(s * s)).sum::<f64>() / a.samples.len() as f64).sqrt();
assert!(rms > 0.03, "too quiet: {rms}");
}
#[test]
fn pause_markers_are_exact_silence() {
let plain = say("basic-male", "One. Two.", Params::default());
let paused = say("basic-male", "One. [pause 1s] Two.", Params::default());
assert_eq!(paused.samples.len(), plain.samples.len() + SAMPLE_RATE as usize);
let one = say("basic-male", "One. ", Params::default()).samples.len();
assert!(paused.samples[one..one + SAMPLE_RATE as usize].iter().all(|&s| s == 0.0));
}
#[test]
fn pace_scales_duration() {
let text = "The quick brown fox jumps over the lazy dog.";
let n1 = say("basic-female", text, Params::default()).samples.len() as f64;
let n2 = say("basic-female", text, Params { pace: 2.0, ..Default::default() }).samples.len() as f64;
let nh = say("basic-female", text, Params { pace: 0.5, ..Default::default() }).samples.len() as f64;
assert!((n2 / n1 - 0.5).abs() < 0.05, "{}", n2 / n1);
assert!((nh / n1 - 2.0).abs() < 0.1, "{}", nh / n1);
}
#[test]
fn pitch_follows_voice_and_setting() {
let text = "aaaaaaaaaaaaaaaaaaaa";
let female = pitch(&say("basic-female", text, Params::default()).samples);
let male = pitch(&say("basic-male", text, Params::default()).samples);
let high = pitch(&say("basic-male", text, Params { semitones: 6.0, ..Default::default() }).samples);
assert!(female > male * 1.5, "female {female} male {male}");
assert!((high / male - 2f64.powf(0.5)).abs() < 0.12, "high {high} male {male}");
}
#[test]
fn refuses_what_it_cannot_say() {
let v = voice("basic-female").unwrap();
assert_eq!(v.synthesize("", &Params::default()), Err(TtsError::Empty));
assert_eq!(v.synthesize("👍 🎬", &Params::default()), Err(TtsError::Empty));
assert_eq!(v.synthesize("日本語", &Params::default()), Err(TtsError::Empty));
assert_eq!(v.synthesize("[pause 2s]", &Params::default()), Err(TtsError::Empty));
assert!(matches!(v.synthesize("hi", &Params { pace: f64::NAN, ..Default::default() }), Err(TtsError::Invalid(_))));
// 64 KB of letters would be over the 30-minute limit: refused before rendering
assert_eq!(v.synthesize(&"a ".repeat(crate::MAX_TEXT_BYTES / 2), &Params { pace: 0.5, ..Default::default() }), Err(TtsError::AudioTooLong));
}
#[test]
fn hostile_scripts_never_panic() {
let v = voice("basic-male").unwrap();
for s in ["a\0b", "x\u{301}y", "'''", "9999999999999999999999", "[pause 1s]a[pause", "Ünïcödé façade", "?!?!?!", "a?b?c?", "zzzzzzzz", "e"] {
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| v.synthesize(s, &Params::default())));
assert!(r.is_ok(), "panicked on {s:?}");
}
}
}
+163
View File
@@ -0,0 +1,163 @@
//! The small DSP pieces of the vocoder: a 20-point STFT / inverse STFT (hop 5, periodic Hann
//! window, centred with reflection padding) and a seeded random source, so synthesis is
//! deterministic.
/// FFT size, hop and the number of one-sided bins.
pub(crate) const N_FFT: usize = 20;
pub(crate) const HOP: usize = 5;
pub(crate) const BINS: usize = N_FFT / 2 + 1;
fn hann() -> [f64; N_FFT] {
let mut w = [0f64; N_FFT];
for (i, v) in w.iter_mut().enumerate() {
*v = 0.5 - 0.5 * (std::f64::consts::TAU * i as f64 / N_FFT as f64).cos();
}
w
}
/// Reflect index `i` (may be negative or past the end) into `0..n`.
fn reflect(i: isize, n: usize) -> usize {
if n <= 1 {
return 0;
}
let period = 2 * (n as isize - 1);
let mut j = i.rem_euclid(period);
if j >= n as isize {
j = period - j;
}
j as usize
}
/// Magnitude and phase, bin-major: `mag[b * frames + f]`. Frames = `len / HOP + 1`.
pub(crate) fn stft(x: &[f32]) -> (Vec<f32>, Vec<f32>) {
let n = x.len();
let frames = n / HOP + 1;
let w = hann();
let half = (N_FFT / 2) as isize;
let mut mag = vec![0f32; BINS * frames];
let mut ph = vec![0f32; BINS * frames];
let mut seg = [0f64; N_FFT];
for f in 0..frames {
for (k, s) in seg.iter_mut().enumerate() {
let i = (f * HOP) as isize + k as isize - half;
*s = f64::from(x.get(reflect(i, n)).copied().unwrap_or(0.0)) * w[k];
}
for b in 0..BINS {
let (mut re, mut im) = (0f64, 0f64);
for (k, s) in seg.iter().enumerate() {
let a = -std::f64::consts::TAU * (b * k) as f64 / N_FFT as f64;
re += s * a.cos();
im += s * a.sin();
}
if let Some(m) = mag.get_mut(b * frames + f) {
*m = re.hypot(im) as f32;
}
if let Some(p) = ph.get_mut(b * frames + f) {
*p = im.atan2(re) as f32;
}
}
}
(mag, ph)
}
/// Inverse of [`stft`] from magnitude and phase (bin-major); output length `HOP · (frames − 1)`.
pub(crate) fn istft(mag: &[f32], phase: &[f32]) -> Vec<f32> {
let frames = mag.len() / BINS;
if frames == 0 || phase.len() < mag.len() {
return Vec::new();
}
let w = hann();
let total = N_FFT + HOP * (frames - 1);
let mut out = vec![0f64; total];
let mut env = vec![0f64; total];
let mut frame = [0f64; N_FFT];
for f in 0..frames {
// inverse real DFT of the one-sided spectrum
for (k, v) in frame.iter_mut().enumerate() {
let mut acc = 0f64;
for b in 0..BINS {
let m = f64::from(mag.get(b * frames + f).copied().unwrap_or(0.0));
let p = f64::from(phase.get(b * frames + f).copied().unwrap_or(0.0));
let (re, im) = (m * p.cos(), m * p.sin());
let a = std::f64::consts::TAU * (b * k) as f64 / N_FFT as f64;
let c = if b == 0 || b == N_FFT / 2 { 1.0 } else { 2.0 };
acc += c * (re * a.cos() - im * a.sin());
}
*v = acc / N_FFT as f64;
}
for k in 0..N_FFT {
let at = f * HOP + k;
if let (Some(o), Some(e)) = (out.get_mut(at), env.get_mut(at)) {
*o += frame[k] * w[k];
*e += w[k] * w[k];
}
}
}
let start = N_FFT / 2;
let len = HOP * (frames - 1);
(start..start + len)
.map(|i| match (out.get(i), env.get(i)) {
(Some(o), Some(e)) if *e > 1e-11 => (o / e) as f32,
(Some(o), _) => *o as f32,
_ => 0.0,
})
.collect()
}
/// Deterministic random numbers (SplitMix64).
pub(crate) struct Rng(u64);
impl Rng {
pub(crate) fn new(seed: u64) -> Rng {
Rng(seed ^ 0x9E37_79B9_7F4A_7C15)
}
fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = self.0;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
/// Uniform in [0, 1).
pub(crate) fn uniform(&mut self) -> f64 {
(self.next() >> 11) as f64 / (1u64 << 53) as f64
}
/// Standard normal (Box–Muller).
pub(crate) fn normal(&mut self) -> f64 {
let u1 = self.uniform().max(1e-12);
let u2 = self.uniform();
(-2.0 * u1.ln()).sqrt() * (std::f64::consts::TAU * u2).cos()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn istft_inverts_stft() {
let x: Vec<f32> = (0..600).map(|i| ((i as f32) * 0.07).sin() * 0.5 + ((i as f32) * 0.31).cos() * 0.2).collect();
let (m, p) = stft(&x);
assert_eq!(m.len(), BINS * (600 / HOP + 1));
let y = istft(&m, &p);
assert_eq!(y.len(), 600);
let err = x.iter().zip(&y).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max);
assert!(err < 1e-4, "max error {err}");
}
#[test]
fn rng_is_deterministic_and_normalish() {
let mut a = Rng::new(7);
let mut b = Rng::new(7);
let xs: Vec<f64> = (0..20000).map(|_| a.normal()).collect();
assert!((0..20000).all(|i| xs[i] == b.normal()));
let mean = xs.iter().sum::<f64>() / xs.len() as f64;
let var = xs.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / xs.len() as f64;
assert!(mean.abs() < 0.03 && (var - 1.0).abs() < 0.05, "{mean} {var}");
}
#[test]
fn reflect_pads_like_numpy() {
assert_eq!((-3..8).map(|i| reflect(i, 5)).collect::<Vec<_>>(), vec![3, 2, 1, 0, 1, 2, 3, 4, 3, 2, 1]);
}
}
+362
View File
@@ -0,0 +1,362 @@
//! Building blocks shared by the Kokoro modules. Tensors are `[1, channels, time]` unless noted;
//! everything runs in f32 on the CPU.
use candle_core::{D, Tensor};
use super::weights::{Weights, err};
use crate::TtsError;
pub(crate) type R<T> = Result<T, TtsError>;
/// 1-D convolution with optional bias.
pub(crate) struct Conv {
w: Tensor,
b: Option<Tensor>,
pad: usize,
stride: usize,
dil: usize,
groups: usize,
}
impl Conv {
pub(crate) fn load(wt: &Weights, name: &str, pad: usize, stride: usize, dil: usize, groups: usize) -> R<Conv> {
let w = wt.get(&format!("{name}.weight"))?;
if w.rank() != 3 {
return Err(err(format!("{name}: conv weight must be 3-D")));
}
let b = wt.get(&format!("{name}.bias")).ok();
Ok(Conv { w, b, pad, stride, dil, groups })
}
/// `pad = dil · (k − 1) / 2` ("same" length for stride 1).
pub(crate) fn same(wt: &Weights, name: &str, dil: usize) -> R<Conv> {
let k = wt.get(&format!("{name}.weight"))?.dims().get(2).copied().unwrap_or(1);
Conv::load(wt, name, dil * (k.saturating_sub(1)) / 2, 1, dil, 1)
}
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
let y = x.conv1d(&self.w, self.pad, self.stride, self.dil, self.groups).map_err(err)?;
add_bias(y, self.b.as_ref())
}
}
fn add_bias(y: Tensor, b: Option<&Tensor>) -> R<Tensor> {
match b {
Some(b) => {
let c = b.dims1().map_err(err)?;
y.broadcast_add(&b.reshape((1, c, 1)).map_err(err)?).map_err(err)
}
None => Ok(y),
}
}
/// Transposed 1-D convolution.
pub(crate) struct ConvT {
w: Tensor,
b: Option<Tensor>,
pad: usize,
out_pad: usize,
stride: usize,
groups: usize,
}
impl ConvT {
pub(crate) fn load(wt: &Weights, name: &str, pad: usize, out_pad: usize, stride: usize, groups: usize) -> R<ConvT> {
Ok(ConvT { w: wt.get(&format!("{name}.weight"))?, b: wt.get(&format!("{name}.bias")).ok(), pad, out_pad, stride, groups })
}
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
let y = x.conv_transpose1d(&self.w, self.pad, self.out_pad, self.stride, 1, self.groups).map_err(err)?;
add_bias(y, self.b.as_ref())
}
}
/// Linear layer on the last dimension.
pub(crate) struct Linear {
w: Tensor,
b: Option<Tensor>,
}
impl Linear {
pub(crate) fn load(wt: &Weights, name: &str) -> R<Linear> {
Ok(Linear { w: wt.get(&format!("{name}.weight"))?.t().map_err(err)?, b: wt.get(&format!("{name}.bias")).ok() })
}
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
let y = x.broadcast_matmul(&self.w).map_err(err)?;
match &self.b {
Some(b) => y.broadcast_add(b).map_err(err),
None => Ok(y),
}
}
}
pub(crate) fn leaky(x: &Tensor, slope: f64) -> R<Tensor> {
// max(x, slope · x) for 0 < slope < 1
x.maximum(&(x * slope).map_err(err)?).map_err(err)
}
/// Normalise over the last dimension (layer norm without affine).
pub(crate) fn norm_last(x: &Tensor, eps: f64) -> R<Tensor> {
let mean = x.mean_keepdim(D::Minus1).map_err(err)?;
let xc = x.broadcast_sub(&mean).map_err(err)?;
let var = xc.sqr().map_err(err)?.mean_keepdim(D::Minus1).map_err(err)?;
xc.broadcast_div(&(var + eps).map_err(err)?.sqrt().map_err(err)?).map_err(err)
}
/// Layer norm over the last dimension with weight and bias.
pub(crate) struct LayerNorm {
g: Tensor,
b: Tensor,
eps: f64,
}
impl LayerNorm {
pub(crate) fn load(wt: &Weights, gamma: &str, beta: &str, eps: f64) -> R<LayerNorm> {
Ok(LayerNorm { g: wt.get(gamma)?, b: wt.get(beta)?, eps })
}
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
norm_last(x, self.eps)?.broadcast_mul(&self.g).map_err(err)?.broadcast_add(&self.b).map_err(err)
}
}
/// Split a style projection `[1, 2C]` into `(1 + gamma, beta)`.
fn style_affine(fc: &Linear, s: &Tensor) -> R<(Tensor, Tensor)> {
let h = fc.fwd(s)?;
let c2 = h.dim(1).map_err(err)?;
let c = c2 / 2;
let gamma = (h.narrow(1, 0, c).map_err(err)? + 1.0).map_err(err)?;
let beta = h.narrow(1, c, c).map_err(err)?;
Ok((gamma, beta))
}
/// Adaptive instance norm: instance-normalise each channel over time, then scale and shift by the
/// style (StyleTTS 2).
pub(crate) struct AdaIn {
fc: Linear,
}
impl AdaIn {
pub(crate) fn load(wt: &Weights, name: &str) -> R<AdaIn> {
Ok(AdaIn { fc: Linear::load(wt, &format!("{name}.fc"))? })
}
pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R<Tensor> {
let (gamma, beta) = style_affine(&self.fc, s)?;
let c = gamma.dim(1).map_err(err)?;
let n = norm_last(x, 1e-5)?;
n.broadcast_mul(&gamma.reshape((1, c, 1)).map_err(err)?).map_err(err)?.broadcast_add(&beta.reshape((1, c, 1)).map_err(err)?).map_err(err)
}
}
/// Adaptive layer norm on `[1, T, C]`: layer-normalise over channels, then style scale and shift.
pub(crate) struct AdaLayerNorm {
fc: Linear,
}
impl AdaLayerNorm {
pub(crate) fn load(wt: &Weights, name: &str) -> R<AdaLayerNorm> {
Ok(AdaLayerNorm { fc: Linear::load(wt, &format!("{name}.fc"))? })
}
pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R<Tensor> {
let (gamma, beta) = style_affine(&self.fc, s)?;
norm_last(x, 1e-5)?.broadcast_mul(&gamma).map_err(err)?.broadcast_add(&beta).map_err(err)
}
}
/// Residual block with adaptive instance norm (StyleTTS 2 `AdainResBlk1d`), optionally doubling
/// the time resolution.
pub(crate) struct AdaResBlock {
norm1: AdaIn,
norm2: AdaIn,
conv1: Conv,
conv2: Conv,
shortcut: Option<Conv>,
pool: Option<ConvT>,
}
impl AdaResBlock {
pub(crate) fn load(wt: &Weights, name: &str, upsample: bool) -> R<AdaResBlock> {
let shortcut = wt.get(&format!("{name}.conv1x1.weight")).ok().map(|w| Conv { w, b: None, pad: 0, stride: 1, dil: 1, groups: 1 });
let pool = if upsample {
let c = wt.get(&format!("{name}.pool.weight"))?.dim(0).map_err(err)?;
Some(ConvT::load(wt, &format!("{name}.pool"), 1, 1, 2, c)?)
} else {
None
};
Ok(AdaResBlock {
norm1: AdaIn::load(wt, &format!("{name}.norm1"))?,
norm2: AdaIn::load(wt, &format!("{name}.norm2"))?,
conv1: Conv::same(wt, &format!("{name}.conv1"), 1)?,
conv2: Conv::same(wt, &format!("{name}.conv2"), 1)?,
shortcut,
pool,
})
}
pub(crate) fn upsamples(&self) -> bool {
self.pool.is_some()
}
pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R<Tensor> {
let mut r = leaky(&self.norm1.fwd(x, s)?, 0.2)?;
if let Some(p) = &self.pool {
r = p.fwd(&r)?;
}
r = self.conv1.fwd(&r)?;
r = leaky(&self.norm2.fwd(&r, s)?, 0.2)?;
r = self.conv2.fwd(&r)?;
let mut sc = x.clone();
if self.pool.is_some() {
sc = upsample2(&sc)?;
}
if let Some(c) = &self.shortcut {
sc = c.fwd(&sc)?;
}
((r + sc).map_err(err)? / std::f64::consts::SQRT_2).map_err(err)
}
}
/// Nearest-neighbour ×2 along time.
pub(crate) fn upsample2(x: &Tensor) -> R<Tensor> {
let (b, c, t) = x.dims3().map_err(err)?;
x.unsqueeze(3).map_err(err)?.broadcast_as((b, c, t, 2)).map_err(err)?.reshape((b, c, t * 2)).map_err(err)
}
/// Snake activation `x + sin²(αx) / α` with a learned α per channel.
pub(crate) fn snake(x: &Tensor, alpha: &Tensor) -> R<Tensor> {
let ax = x.broadcast_mul(alpha).map_err(err)?;
let s2 = ax.sin().map_err(err)?.sqr().map_err(err)?;
let inv = (alpha.clone() + 1e-9).map_err(err)?.recip().map_err(err)?;
(x + s2.broadcast_mul(&inv).map_err(err)?).map_err(err)
}
/// Generator residual block (HiFi-GAN "resblock 1" with adaptive instance norm and Snake).
pub(crate) struct SnakeResBlock {
convs1: Vec<Conv>,
convs2: Vec<Conv>,
adain1: Vec<AdaIn>,
adain2: Vec<AdaIn>,
alpha1: Vec<Tensor>,
alpha2: Vec<Tensor>,
}
impl SnakeResBlock {
pub(crate) fn load(wt: &Weights, name: &str, dilations: [usize; 3]) -> R<SnakeResBlock> {
let mut b = SnakeResBlock { convs1: vec![], convs2: vec![], adain1: vec![], adain2: vec![], alpha1: vec![], alpha2: vec![] };
for (j, d) in dilations.iter().enumerate() {
b.convs1.push(Conv::same(wt, &format!("{name}.convs1.{j}"), *d)?);
b.convs2.push(Conv::same(wt, &format!("{name}.convs2.{j}"), 1)?);
b.adain1.push(AdaIn::load(wt, &format!("{name}.adain1.{j}"))?);
b.adain2.push(AdaIn::load(wt, &format!("{name}.adain2.{j}"))?);
b.alpha1.push(wt.get(&format!("{name}.alpha1.{j}"))?);
b.alpha2.push(wt.get(&format!("{name}.alpha2.{j}"))?);
}
Ok(b)
}
pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R<Tensor> {
let mut x = x.clone();
for j in 0..self.convs1.len() {
let (Some(c1), Some(c2), Some(n1), Some(n2), Some(a1), Some(a2)) =
(self.convs1.get(j), self.convs2.get(j), self.adain1.get(j), self.adain2.get(j), self.alpha1.get(j), self.alpha2.get(j))
else {
break;
};
let mut t = snake(&n1.fwd(&x, s)?, a1)?;
t = c1.fwd(&t)?;
t = snake(&n2.fwd(&t, s)?, a2)?;
t = c2.fwd(&t)?;
x = (x + t).map_err(err)?;
}
Ok(x)
}
}
/// One LSTM direction (PyTorch gate order i, f, g, o), run in plain loops for speed.
struct LstmDir {
w_ih: Tensor,
w_hh: Vec<f32>,
bias: Vec<f32>,
hidden: usize,
}
impl LstmDir {
fn load(wt: &Weights, name: &str, suffix: &str) -> R<LstmDir> {
let w_ih = wt.get(&format!("{name}.weight_ih_l0{suffix}"))?;
let w_hh = wt.get(&format!("{name}.weight_hh_l0{suffix}"))?;
let (g4, hidden) = w_hh.dims2().map_err(err)?;
if g4 != 4 * hidden {
return Err(err(format!("{name}: bad LSTM shape")));
}
let b_ih: Vec<f32> = wt.get(&format!("{name}.bias_ih_l0{suffix}"))?.to_vec1().map_err(err)?;
let b_hh: Vec<f32> = wt.get(&format!("{name}.bias_hh_l0{suffix}"))?.to_vec1().map_err(err)?;
let bias = b_ih.iter().zip(&b_hh).map(|(a, b)| a + b).collect();
// row-major [4H, H] → transposed [H, 4H] so the recurrent product walks memory forwards
let hh: Vec<Vec<f32>> = w_hh.to_vec2().map_err(err)?;
let mut w_t = vec![0f32; hidden * g4];
for (r, row) in hh.iter().enumerate() {
for (c, v) in row.iter().enumerate() {
if let Some(slot) = w_t.get_mut(c * g4 + r) {
*slot = *v;
}
}
}
Ok(LstmDir { w_ih: w_ih.t().map_err(err)?, w_hh: w_t, bias, hidden })
}
/// `x`: `[T, I]` → `[T, H]` (in input order).
fn run(&self, x: &Tensor, reverse: bool) -> R<Vec<f32>> {
let (t_len, _) = x.dims2().map_err(err)?;
let h = self.hidden;
let g4 = 4 * h;
let pre: Vec<f32> = x.matmul(&self.w_ih).map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?;
let mut out = vec![0f32; t_len * h];
let (mut hs, mut cs) = (vec![0f32; h], vec![0f32; h]);
let mut gates = vec![0f32; g4];
let sig = |v: f32| 1.0 / (1.0 + (-v).exp());
for step in 0..t_len {
let t = if reverse { t_len - 1 - step } else { step };
let Some(p) = pre.get(t * g4..(t + 1) * g4) else { break };
for ((g, a), b) in gates.iter_mut().zip(p).zip(&self.bias) {
*g = a + b;
}
for (k, hv) in hs.iter().enumerate() {
if *hv == 0.0 {
continue;
}
let Some(row) = self.w_hh.get(k * g4..(k + 1) * g4) else { break };
for (g, w) in gates.iter_mut().zip(row) {
*g += hv * w;
}
}
for j in 0..h {
let (i, f, gg, o) = (gates[j], gates[h + j], gates[2 * h + j], gates[3 * h + j]);
let c = sig(f) * cs[j] + sig(i) * gg.tanh();
cs[j] = c;
hs[j] = sig(o) * c.tanh();
}
if let Some(slot) = out.get_mut(t * h..(t + 1) * h) {
slot.copy_from_slice(&hs);
}
}
Ok(out)
}
}
/// Bidirectional single-layer LSTM on `[1, T, I]` → `[1, T, 2H]`.
pub(crate) struct BiLstm {
fwd: LstmDir,
bwd: LstmDir,
}
impl BiLstm {
pub(crate) fn load(wt: &Weights, name: &str) -> R<BiLstm> {
Ok(BiLstm { fwd: LstmDir::load(wt, name, "")?, bwd: LstmDir::load(wt, name, "_reverse")? })
}
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
let x2 = x.squeeze(0).map_err(err)?.contiguous().map_err(err)?;
let t = x2.dim(0).map_err(err)?;
let h = self.fwd.hidden;
let a = self.fwd.run(&x2, false)?;
let b = self.bwd.run(&x2, true)?;
let mut out = Vec::with_capacity(t * 2 * h);
for i in 0..t {
out.extend_from_slice(a.get(i * h..(i + 1) * h).unwrap_or(&[]));
out.extend_from_slice(b.get(i * h..(i + 1) * h).unwrap_or(&[]));
}
Tensor::from_vec(out, (1, t, 2 * h), x.device()).map_err(err)
}
}
+113
View File
@@ -0,0 +1,113 @@
//! Kokoro-82M neural voices (feature `kokoro`): pure-Rust inference on candle (CPU).
//!
//! Implemented from the papers the model card cites (StyleTTS 2, iSTFTNet; with ALBERT, HiFi-GAN,
//! the neural source-filter model and the Snake activation they build on), the released
//! `config.json` and the tensor names and shapes in the checkpoint. No reference code was read.
//! Weights (Apache-2.0, hexgrad) are downloaded on first use and never bundled.
//!
//! [`Kokoro::speak`] takes Kokoro phonemes (from `filmcraft-tts-text`), at most
//! [`MAX_PHONEMES`] per call, and a voice pack.
mod dsp;
mod layers;
mod model;
mod weights;
use std::path::Path;
use crate::TtsError;
pub use model::SAMPLE_RATE as KOKORO_RATE;
/// Longest phoneme string one forward pass takes (the position table has 512 rows, two of them
/// for the boundary tokens).
pub const MAX_PHONEMES: usize = 510;
/// A loaded Kokoro model.
pub struct Kokoro {
model: model::Model,
vocab: std::collections::HashMap<char, u32>,
}
/// A loaded voice pack.
pub struct VoicePack(Vec<f32>);
impl VoicePack {
pub fn load(path: &Path) -> Result<VoicePack, TtsError> {
weights::read_voice(path).map(VoicePack)
}
}
impl Kokoro {
/// Load `config.json` and `kokoro-v1_0.pth` from `dir`.
pub fn load(dir: &Path) -> Result<Kokoro, TtsError> {
let cfg = weights::read_config(&dir.join("config.json"))?;
let w = weights::Weights::load(&dir.join("kokoro-v1_0.pth"))?;
Ok(Kokoro { model: model::Model::load(&w)?, vocab: cfg.vocab })
}
/// Phoneme ids of `phonemes`; symbols outside the vocabulary are dropped.
pub fn ids(&self, phonemes: &str) -> Vec<u32> {
phonemes.chars().filter_map(|c| self.vocab.get(&c).copied()).collect()
}
/// Speak `phonemes` with `voice` at `speed` (0.5–2). Deterministic for a given `seed`.
pub fn speak(&self, phonemes: &str, voice: &VoicePack, speed: f64, seed: u64) -> Result<Vec<f32>, TtsError> {
if !(0.5..=2.0).contains(&speed) {
return Err(TtsError::Invalid("speed must be between 0.5 and 2".into()));
}
let ids = self.ids(phonemes);
if ids.is_empty() {
return Err(TtsError::Empty);
}
if ids.len() > MAX_PHONEMES {
return Err(TtsError::Invalid(format!("at most {MAX_PHONEMES} phonemes per pass")));
}
let style = weights::style_for(&voice.0, ids.len(), self.model.device())?;
let out = self.model.forward(&ids, &style, speed, seed)?;
if out.iter().any(|s| !s.is_finite()) {
return Err(TtsError::Model("the model produced invalid samples".into()));
}
Ok(out)
}
}
/// Tests with the real model run when `FILMCRAFT_KOKORO_DIR` names a directory holding
/// `config.json`, `kokoro-v1_0.pth` and `af_heart.pt` (the downloaded package); they skip otherwise.
#[cfg(test)]
mod tests {
use super::*;
fn model() -> Option<(Kokoro, VoicePack)> {
let dir = std::path::PathBuf::from(std::env::var_os("FILMCRAFT_KOKORO_DIR")?);
let voice = [dir.join("af_heart.pt"), dir.join("voices/af_heart.pt")].into_iter().find(|p| p.exists())?;
Some((Kokoro::load(&dir).unwrap(), VoicePack::load(&voice).unwrap()))
}
#[test]
fn speaks_deterministically_and_pace_changes_the_length() {
let Some((k, v)) = model() else { return };
let ph = "həlˈO wˈɜɹld.";
let a = k.speak(ph, &v, 1.0, 1).unwrap();
assert_eq!(a, k.speak(ph, &v, 1.0, 1).unwrap(), "deterministic");
assert_eq!(a.len() % 600, 0, "whole frames");
let secs = a.len() as f64 / f64::from(KOKORO_RATE);
assert!((0.5..3.0).contains(&secs), "{secs}");
let peak = a.iter().fold(0f32, |m, x| m.max(x.abs()));
assert!(peak > 0.05 && peak < 1.0, "{peak}");
let fast = k.speak(ph, &v, 2.0, 1).unwrap();
let ratio = fast.len() as f64 / a.len() as f64;
assert!((0.35..0.7).contains(&ratio), "{ratio}");
}
#[test]
fn refuses_bad_input() {
let Some((k, v)) = model() else { return };
assert_eq!(k.speak("", &v, 1.0, 1).err(), Some(TtsError::Empty));
assert_eq!(k.speak("日本語👍", &v, 1.0, 1).err(), Some(TtsError::Empty), "nothing in the vocabulary");
assert!(k.speak("a", &v, 3.0, 1).is_err());
assert!(k.speak("a", &v, f64::NAN, 1).is_err());
assert!(k.speak(&"ə".repeat(MAX_PHONEMES + 1), &v, 1.0, 1).is_err());
assert_eq!(k.ids("a日b"), k.ids("ab"));
}
}
+385
View File
@@ -0,0 +1,385 @@
//! The Kokoro-82M forward pass (inference only, batch of one).
//!
//! 1. **Text** – phoneme ids go through an ALBERT encoder (one shared layer applied 12 times) and
//! a linear map to 512 channels (`bert`, `bert_encoder`), and separately through an embedding,
//! three convolutions and a bidirectional LSTM (`text_encoder`).
//! 2. **Prosody** – the duration encoder (LSTMs with style-adaptive layer norm), a duration head
//! (50 bins, sigmoid, summed), then pitch (F0) and energy (N) curves at twice the frame rate from
//! adaptive residual blocks (`predictor`).
//! 3. **Decoder** – text features expanded to frames by the durations, combined with F0 and N in
//! adaptive residual blocks (`decoder.encode`, `decoder.decode`).
//! 4. **Vocoder** – iSTFTNet: a harmonic-plus-noise source from F0 (9 harmonics), two upsampling
//! stages (×10, ×6) with source injection and Snake residual blocks, then magnitude and phase for
//! an inverse STFT (n_fft 20, hop 5) at 24 kHz.
//!
//! Style: a voice pack row of 256 values; the first 128 condition the decoder, the last 128 the
//! prosody predictor.
use candle_core::{D, Device, Tensor};
use super::layers::*;
use super::weights::{Weights, err};
const HEADS: usize = 12;
const LAYERS: usize = 12;
pub const SAMPLE_RATE: u32 = 24_000;
/// Output samples per predicted duration frame (2 F0 frames × 60 upsampling × hop 5).
pub(crate) const SAMPLES_PER_FRAME: usize = 600;
struct Albert {
word: Tensor,
pos: Tensor,
ttype: Tensor,
emb_ln: LayerNorm,
map_in: Linear,
q: Linear,
k: Linear,
v: Linear,
dense: Linear,
attn_ln: LayerNorm,
ffn: Linear,
ffn_out: Linear,
full_ln: LayerNorm,
}
impl Albert {
fn load(w: &Weights) -> R<Albert> {
let l = "bert.encoder.albert_layer_groups.0.albert_layers.0";
Ok(Albert {
word: w.shaped("bert.embeddings.word_embeddings.weight", &[178, 128])?,
pos: w.shaped("bert.embeddings.position_embeddings.weight", &[512, 128])?,
ttype: w.shaped("bert.embeddings.token_type_embeddings.weight", &[2, 128])?,
emb_ln: LayerNorm::load(w, "bert.embeddings.LayerNorm.weight", "bert.embeddings.LayerNorm.bias", 1e-12)?,
map_in: Linear::load(w, "bert.encoder.embedding_hidden_mapping_in")?,
q: Linear::load(w, &format!("{l}.attention.query"))?,
k: Linear::load(w, &format!("{l}.attention.key"))?,
v: Linear::load(w, &format!("{l}.attention.value"))?,
dense: Linear::load(w, &format!("{l}.attention.dense"))?,
attn_ln: LayerNorm::load(w, &format!("{l}.attention.LayerNorm.weight"), &format!("{l}.attention.LayerNorm.bias"), 1e-12)?,
ffn: Linear::load(w, &format!("{l}.ffn"))?,
ffn_out: Linear::load(w, &format!("{l}.ffn_output"))?,
full_ln: LayerNorm::load(w, &format!("{l}.full_layer_layer_norm.weight"), &format!("{l}.full_layer_layer_norm.bias"), 1e-12)?,
})
}
/// `ids`: `[T]` → `[1, T, 768]`.
fn fwd(&self, ids: &Tensor) -> R<Tensor> {
let t = ids.dim(0).map_err(err)?;
let e = self.word.index_select(ids, 0).map_err(err)?;
let e = (e + self.pos.narrow(0, 0, t).map_err(err)?).map_err(err)?;
let e = e.broadcast_add(&self.ttype.narrow(0, 0, 1).map_err(err)?).map_err(err)?;
let mut x = self.map_in.fwd(&self.emb_ln.fwd(&e)?)?.unsqueeze(0).map_err(err)?; // [1, T, 768]
let hd = 768 / HEADS;
let scale = 1.0 / (hd as f64).sqrt();
let split = |y: Tensor| -> R<Tensor> { y.reshape((1, t, HEADS, hd)).map_err(err)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err) };
for _ in 0..LAYERS {
let q = split(self.q.fwd(&x)?)?;
let k = split(self.k.fwd(&x)?)?;
let v = split(self.v.fwd(&x)?)?;
let att = (q.matmul(&k.t().map_err(err)?).map_err(err)? * scale).map_err(err)?;
let att = candle_nn::ops::softmax_last_dim(&att).map_err(err)?;
let ctx = att.matmul(&v).map_err(err)?.transpose(1, 2).map_err(err)?.reshape((1, t, 768)).map_err(err)?;
let a = self.attn_ln.fwd(&(x + self.dense.fwd(&ctx)?).map_err(err)?)?;
let f = self.ffn_out.fwd(&self.ffn.fwd(&a)?.gelu().map_err(err)?)?;
x = self.full_ln.fwd(&(f + a).map_err(err)?)?;
}
Ok(x)
}
}
struct TextEncoder {
emb: Tensor,
convs: Vec<(Conv, LayerNorm)>,
lstm: BiLstm,
}
impl TextEncoder {
fn load(w: &Weights) -> R<TextEncoder> {
let mut convs = Vec::new();
for i in 0..3 {
let c = Conv::same(w, &format!("text_encoder.cnn.{i}.0"), 1)?;
let n = LayerNorm::load(w, &format!("text_encoder.cnn.{i}.1.gamma"), &format!("text_encoder.cnn.{i}.1.beta"), 1e-5)?;
convs.push((c, n));
}
Ok(TextEncoder { emb: w.shaped("text_encoder.embedding.weight", &[178, 512])?, convs, lstm: BiLstm::load(w, "text_encoder.lstm")? })
}
/// `[T]` → `[1, 512, T]`.
fn fwd(&self, ids: &Tensor) -> R<Tensor> {
let mut x = self.emb.index_select(ids, 0).map_err(err)?.t().map_err(err)?.unsqueeze(0).map_err(err)?; // [1, 512, T]
for (c, n) in &self.convs {
x = c.fwd(&x)?;
// layer norm over channels
x = n.fwd(&x.transpose(1, 2).map_err(err)?)?.transpose(1, 2).map_err(err)?;
x = leaky(&x, 0.2)?;
}
let y = self.lstm.fwd(&x.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?)?; // [1, T, 512]
y.transpose(1, 2).map_err(err)
}
}
struct Predictor {
dur_lstms: Vec<BiLstm>,
dur_norms: Vec<AdaLayerNorm>,
lstm: BiLstm,
dur_proj: Linear,
shared: BiLstm,
f0: Vec<AdaResBlock>,
n: Vec<AdaResBlock>,
f0_proj: Conv,
n_proj: Conv,
}
impl Predictor {
fn load(w: &Weights) -> R<Predictor> {
let mut dur_lstms = Vec::new();
let mut dur_norms = Vec::new();
for i in 0..3 {
dur_lstms.push(BiLstm::load(w, &format!("predictor.text_encoder.lstms.{}", 2 * i))?);
dur_norms.push(AdaLayerNorm::load(w, &format!("predictor.text_encoder.lstms.{}", 2 * i + 1))?);
}
let blocks = |k: &str| -> R<Vec<AdaResBlock>> { (0..3).map(|i| AdaResBlock::load(w, &format!("predictor.{k}.{i}"), i == 1)).collect() };
Ok(Predictor {
dur_lstms,
dur_norms,
lstm: BiLstm::load(w, "predictor.lstm")?,
dur_proj: Linear::load(w, "predictor.duration_proj.linear_layer")?,
shared: BiLstm::load(w, "predictor.shared")?,
f0: blocks("F0")?,
n: blocks("N")?,
f0_proj: Conv::load(w, "predictor.F0_proj", 0, 1, 1, 1)?,
n_proj: Conv::load(w, "predictor.N_proj", 0, 1, 1, 1)?,
})
}
/// Duration encoder: `d_en` `[1, 512, T]`, style `[1, 128]` → `[1, T, 640]`.
fn encode(&self, d_en: &Tensor, s: &Tensor) -> R<Tensor> {
let t = d_en.dim(2).map_err(err)?;
let s_t = s.unsqueeze(1).map_err(err)?.broadcast_as((1, t, 128)).map_err(err)?.contiguous().map_err(err)?;
let mut x = Tensor::cat(&[&d_en.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?, &s_t], 2).map_err(err)?;
for (l, n) in self.dur_lstms.iter().zip(&self.dur_norms) {
let y = n.fwd(&l.fwd(&x)?, s)?;
x = Tensor::cat(&[&y, &s_t], 2).map_err(err)?;
}
Ok(x)
}
/// Frames per input token, at `speed`.
fn durations(&self, d: &Tensor, speed: f64) -> R<Vec<usize>> {
let x = self.lstm.fwd(d)?;
let logits = self.dur_proj.fwd(&x)?;
let dur = candle_nn::ops::sigmoid(&logits).map_err(err)?.sum(D::Minus1).map_err(err)?.squeeze(0).map_err(err)?;
let v: Vec<f32> = dur.to_vec1().map_err(err)?;
Ok(v.iter().map(|d| ((f64::from(*d) / speed).round().clamp(1.0, 200.0)) as usize).collect())
}
/// F0 and N curves `[1, 1, 2F]` from frame features `en` `[1, 640, F]`.
fn f0_n(&self, en: &Tensor, s: &Tensor) -> R<(Tensor, Tensor)> {
let x = self.shared.fwd(&en.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?;
let mut f = x.clone();
for b in &self.f0 {
f = b.fwd(&f, s)?;
}
let mut n = x;
for b in &self.n {
n = b.fwd(&n, s)?;
}
Ok((self.f0_proj.fwd(&f)?, self.n_proj.fwd(&n)?))
}
}
struct Generator {
l_linear: Linear,
noise_convs: Vec<Conv>,
noise_res: Vec<SnakeResBlock>,
ups: Vec<ConvT>,
resblocks: Vec<SnakeResBlock>,
conv_post: Conv,
}
impl Generator {
fn load(w: &Weights) -> R<Generator> {
let g = "decoder.generator";
let dil = [1, 3, 5];
Ok(Generator {
l_linear: Linear::load(w, &format!("{g}.m_source.l_linear"))?,
noise_convs: vec![Conv::load(w, &format!("{g}.noise_convs.0"), 3, 6, 1, 1)?, Conv::load(w, &format!("{g}.noise_convs.1"), 0, 1, 1, 1)?],
noise_res: vec![SnakeResBlock::load(w, &format!("{g}.noise_res.0"), dil)?, SnakeResBlock::load(w, &format!("{g}.noise_res.1"), dil)?],
ups: vec![ConvT::load(w, &format!("{g}.ups.0"), 5, 0, 10, 1)?, ConvT::load(w, &format!("{g}.ups.1"), 3, 0, 6, 1)?],
resblocks: (0..6).map(|i| SnakeResBlock::load(w, &format!("{g}.resblocks.{i}"), dil)).collect::<R<_>>()?,
conv_post: Conv::same(w, &format!("{g}.conv_post"), 1)?,
})
}
/// `x` `[1, 512, 2F]`, style `[1, 128]`, F0 per half-frame → samples.
fn fwd(&self, x: &Tensor, s: &Tensor, f0: &[f32], seed: u64) -> R<Vec<f32>> {
let dev = x.device();
let source = harmonic_source(f0, &self.l_linear, seed, dev)?;
let (mag, phase) = super::dsp::stft(&source);
let frames = mag.len() / super::dsp::BINS;
let mut har = mag;
har.extend_from_slice(&phase);
let har = Tensor::from_vec(har, (1, 2 * super::dsp::BINS, frames), dev).map_err(err)?;
let mut x = x.clone();
for i in 0..2 {
let (Some(nc), Some(nr), Some(up)) = (self.noise_convs.get(i), self.noise_res.get(i), self.ups.get(i)) else { break };
x = leaky(&x, 0.1)?;
let xs = nr.fwd(&nc.fwd(&har)?, s)?;
x = up.fwd(&x)?;
if i == 1 {
// reflection pad one frame on the left
let first = x.narrow(2, 1, 1).map_err(err)?;
x = Tensor::cat(&[&first, &x], 2).map_err(err)?;
}
let t = x.dim(2).map_err(err)?.min(xs.dim(2).map_err(err)?);
x = (x.narrow(2, 0, t).map_err(err)? + xs.narrow(2, 0, t).map_err(err)?).map_err(err)?;
let mut acc: Option<Tensor> = None;
for j in 0..3 {
let Some(rb) = self.resblocks.get(i * 3 + j) else { break };
let y = rb.fwd(&x, s)?;
acc = Some(match acc {
Some(a) => (a + y).map_err(err)?,
None => y,
});
}
x = (acc.ok_or_else(|| err("missing resblocks"))? / 3.0).map_err(err)?;
}
x = leaky(&x, 0.01)?;
let y = self.conv_post.fwd(&x)?.squeeze(0).map_err(err)?; // [22, T]
let bins = super::dsp::BINS;
let mag: Vec<f32> = y.narrow(0, 0, bins).map_err(err)?.exp().map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?;
let ph: Vec<f32> = y.narrow(0, bins, bins).map_err(err)?.sin().map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?;
Ok(super::dsp::istft(&mag, &ph))
}
}
/// Neural source filter: 9 sine harmonics of F0 (voiced) or noise (unvoiced), mixed to one
/// channel by a learned linear layer and tanh. F0 is given per half-frame and held for 300 samples.
fn harmonic_source(f0: &[f32], l_linear: &Linear, seed: u64, dev: &Device) -> R<Vec<f32>> {
const HARM: usize = 9;
const HOLD: usize = SAMPLES_PER_FRAME / 2;
let n = f0.len() * HOLD;
let mut rng = super::dsp::Rng::new(seed);
// random start phase for the overtones, none for the fundamental
let start: Vec<f64> = (0..HARM).map(|h| if h == 0 { 0.0 } else { rng.uniform() }).collect();
let mut phase = start;
let mut sines = vec![0f32; n * HARM];
for (i, row) in sines.as_chunks_mut::<HARM>().0.iter_mut().enumerate() {
let f = f64::from(f0.get(i / HOLD).copied().unwrap_or(0.0)).max(0.0);
let voiced = f > 10.0;
let noise_amp = if voiced { 0.003 } else { 0.1 / 3.0 };
for (h, out) in row.iter_mut().enumerate() {
phase[h] = (phase[h] + f * (h + 1) as f64 / f64::from(SAMPLE_RATE)).fract();
let sine = if voiced { 0.1 * (std::f64::consts::TAU * phase[h]).sin() } else { 0.0 };
*out = (sine + noise_amp * rng.normal()) as f32;
}
}
let t = Tensor::from_vec(sines, (n, HARM), dev).map_err(err)?;
let y = l_linear.fwd(&t)?.tanh().map_err(err)?.flatten_all().map_err(err)?;
y.to_vec1().map_err(err)
}
struct Decoder {
f0_conv: Conv,
n_conv: Conv,
encode: AdaResBlock,
decode: Vec<AdaResBlock>,
asr_res: Conv,
vocoder: Generator,
}
impl Decoder {
fn load(w: &Weights) -> R<Decoder> {
Ok(Decoder {
f0_conv: Conv::load(w, "decoder.F0_conv", 1, 2, 1, 1)?,
n_conv: Conv::load(w, "decoder.N_conv", 1, 2, 1, 1)?,
encode: AdaResBlock::load(w, "decoder.encode", false)?,
decode: (0..4).map(|i| AdaResBlock::load(w, &format!("decoder.decode.{i}"), i == 3)).collect::<R<_>>()?,
asr_res: Conv::load(w, "decoder.asr_res.0", 0, 1, 1, 1)?,
vocoder: Generator::load(w)?,
})
}
fn fwd(&self, asr: &Tensor, f0: &Tensor, n: &Tensor, s: &Tensor, seed: u64) -> R<Vec<f32>> {
let f0_curve: Vec<f32> = f0.flatten_all().map_err(err)?.to_vec1().map_err(err)?;
let f = self.f0_conv.fwd(f0)?;
let nn = self.n_conv.fwd(n)?;
let frames = asr.dim(2).map_err(err)?.min(f.dim(2).map_err(err)?).min(nn.dim(2).map_err(err)?);
let asr = asr.narrow(2, 0, frames).map_err(err)?;
let f = f.narrow(2, 0, frames).map_err(err)?;
let nn = nn.narrow(2, 0, frames).map_err(err)?;
let mut x = self.encode.fwd(&Tensor::cat(&[&asr, &f, &nn], 1).map_err(err)?, s)?;
let res = self.asr_res.fwd(&asr)?;
let mut concat = true;
for b in &self.decode {
if concat {
x = Tensor::cat(&[&x, &res, &f, &nn], 1).map_err(err)?;
}
x = b.fwd(&x, s)?;
if b.upsamples() {
concat = false;
}
}
self.vocoder.fwd(&x, s, &f0_curve, seed)
}
}
/// The loaded model.
pub(crate) struct Model {
albert: Albert,
bert_enc: Linear,
text: TextEncoder,
pred: Predictor,
dec: Decoder,
dev: Device,
}
impl Model {
pub(crate) fn load(w: &Weights) -> R<Model> {
Ok(Model {
albert: Albert::load(w)?,
bert_enc: Linear::load(w, "bert_encoder")?,
text: TextEncoder::load(w)?,
pred: Predictor::load(w)?,
dec: Decoder::load(w)?,
dev: Device::Cpu,
})
}
pub(crate) fn device(&self) -> &Device {
&self.dev
}
/// Phoneme ids (without the boundary tokens; at most 510) and a 256-value style → samples.
pub(crate) fn forward(&self, ids: &[u32], style: &Tensor, speed: f64, seed: u64) -> R<Vec<f32>> {
let mut padded = Vec::with_capacity(ids.len() + 2);
padded.push(0u32);
padded.extend_from_slice(ids);
padded.push(0);
let t = padded.len();
let ids_t = Tensor::from_vec(padded, t, &self.dev).map_err(err)?;
let s_dec = style.narrow(1, 0, 128).map_err(err)?;
let s_pro = style.narrow(1, 128, 128).map_err(err)?;
let bert = self.albert.fwd(&ids_t)?;
let d_en = self.bert_enc.fwd(&bert)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?; // [1, 512, T]
let d = self.pred.encode(&d_en, &s_pro)?; // [1, T, 640]
let dur = self.pred.durations(&d, speed)?;
let frames: usize = dur.iter().sum();
// alignment [T, F]: token i covers its run of frames
let mut aln = vec![0f32; t * frames];
let mut at = 0usize;
for (i, n) in dur.iter().enumerate() {
for k in at..at + n {
if let Some(v) = aln.get_mut(i * frames + k) {
*v = 1.0;
}
}
at += n;
}
let aln = Tensor::from_vec(aln, (1, t, frames), &self.dev).map_err(err)?;
let en = d.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?.matmul(&aln).map_err(err)?; // [1, 640, F]
let (f0, n) = self.pred.f0_n(&en, &s_pro)?;
let t_en = self.text.fwd(&ids_t)?; // [1, 512, T]
let asr = t_en.matmul(&aln).map_err(err)?; // [1, 512, F]
self.dec.fwd(&asr, &f0, &n, &s_dec, seed)
}
}
+266
View File
@@ -0,0 +1,266 @@
//! Loading the Kokoro-82M checkpoint, its config and voice packs.
//!
//! The official release is a PyTorch zip checkpoint (`kokoro-v1_0.pth`) holding five state
//! dictionaries (`bert`, `bert_encoder`, `predictor`, `text_encoder`, `decoder`), each with a
//! `module.` prefix. They are read with candle's restricted pickle reader, which only rebuilds
//! tensors and never runs code from the file. Weight-normalised layers (`weight_g`, `weight_v`) are
//! fused into a plain weight on load. Voice packs (`voices/<id>.pt`) hold one bare float32 tensor
//! `[510, 1, 256]`, read straight from the zip archive.
use std::collections::HashMap;
use std::io::Read;
use std::path::Path;
use candle_core::{DType, Device, Tensor};
use crate::TtsError;
pub(crate) fn err(e: impl std::fmt::Display) -> TtsError {
TtsError::Model(e.to_string())
}
/// The architecture this code implements; a checkpoint whose config differs is refused.
pub(crate) struct Config {
pub vocab: HashMap<char, u32>,
}
/// Values of `config.json` this implementation is written for.
const EXPECT: &[(&str, &str)] = &[
("hidden_dim", "512"),
("style_dim", "128"),
("n_token", "178"),
("max_dur", "50"),
("n_layer", "3"),
("text_encoder_kernel_size", "5"),
("istftnet.gen_istft_n_fft", "20"),
("istftnet.gen_istft_hop_size", "5"),
("istftnet.upsample_rates", "[10,6]"),
("istftnet.upsample_kernel_sizes", "[20,12]"),
("istftnet.resblock_kernel_sizes", "[3,7,11]"),
("istftnet.upsample_initial_channel", "512"),
("plbert.hidden_size", "768"),
("plbert.num_attention_heads", "12"),
("plbert.num_hidden_layers", "12"),
("plbert.intermediate_size", "2048"),
];
pub(crate) fn read_config(path: &Path) -> Result<Config, TtsError> {
let bytes = std::fs::read(path).map_err(|e| err(format!("{}: {e}", path.display())))?;
if bytes.len() > 1 << 20 {
return Err(err("config.json is too large"));
}
let v: serde_json::Value = serde_json::from_slice(&bytes).map_err(err)?;
for (key, want) in EXPECT {
let mut cur = &v;
for part in key.split('.') {
cur = cur.get(part).ok_or_else(|| err(format!("config.json: missing `{key}`")))?;
}
let got = serde_json::to_string(cur).map_err(err)?;
if got != *want {
return Err(err(format!("config.json: `{key}` is {got}, this build supports {want}")));
}
}
let table = v.get("vocab").and_then(|t| t.as_object()).ok_or_else(|| err("config.json: missing `vocab`"))?;
let mut vocab = HashMap::new();
for (sym, id) in table {
let mut chars = sym.chars();
let (Some(c), None) = (chars.next(), chars.next()) else { continue };
let Some(id) = id.as_u64().filter(|i| *i > 0 && *i < 178) else { continue };
vocab.insert(c, id as u32);
}
if vocab.len() < 100 {
return Err(err("config.json: the phoneme vocabulary is incomplete"));
}
Ok(Config { vocab })
}
/// All model tensors by full name (`decoder.generator.conv_post.weight`…), weight norm fused.
pub(crate) struct Weights {
map: HashMap<String, Tensor>,
}
const PARTS: [&str; 5] = ["bert", "bert_encoder", "predictor", "text_encoder", "decoder"];
impl Weights {
pub(crate) fn load(path: &Path) -> Result<Weights, TtsError> {
let mut raw: HashMap<String, Tensor> = HashMap::new();
for part in PARTS {
let tensors = candle_core::pickle::read_all_with_key(path, Some(part)).map_err(|e| err(format!("{}: {e}", path.display())))?;
if tensors.is_empty() {
return Err(err(format!("{}: no `{part}` weights", path.display())));
}
for (name, t) in tensors {
let name = name.strip_prefix("module.").unwrap_or(&name);
let t = t.to_dtype(DType::F32).map_err(err)?;
raw.insert(format!("{part}.{name}"), t);
}
}
// fuse weight norm: w = g · v / ‖v‖ (norm over every dim but the first)
let mut map = HashMap::with_capacity(raw.len());
let gs: Vec<String> = raw.keys().filter(|k| k.ends_with(".weight_g")).cloned().collect();
for g_name in gs {
let base = g_name.trim_end_matches("_g").to_string();
let v_name = format!("{base}_v");
let (Some(g), Some(v)) = (raw.remove(&g_name), raw.remove(&v_name)) else {
return Err(err(format!("{g_name} without {v_name}")));
};
let norm = v.sqr().map_err(err)?.sum_keepdim(1).map_err(err)?.sum_keepdim(2).map_err(err)?.sqrt().map_err(err)?;
let w = v.broadcast_div(&norm).map_err(err)?.broadcast_mul(&g).map_err(err)?;
map.insert(base, w);
}
map.extend(raw);
Ok(Weights { map })
}
pub(crate) fn get(&self, name: &str) -> Result<Tensor, TtsError> {
self.map.get(name).cloned().ok_or_else(|| err(format!("missing weight `{name}`")))
}
/// `get` with an expected shape.
pub(crate) fn shaped(&self, name: &str, dims: &[usize]) -> Result<Tensor, TtsError> {
let t = self.get(name)?;
if t.dims() != dims {
return Err(err(format!("weight `{name}` is {:?}, expected {dims:?}", t.dims())));
}
Ok(t)
}
}
/// Number of style vectors in a voice pack (one per input length 1..=510).
pub(crate) const PACK_LEN: usize = 510;
pub(crate) const STYLE: usize = 256;
/// Read a voice pack: a zip whose `<name>/data/0` holds `[510, 1, 256]` little-endian float32.
pub(crate) fn read_voice(path: &Path) -> Result<Vec<f32>, TtsError> {
let f = std::fs::File::open(path).map_err(|e| err(format!("{}: {e}", path.display())))?;
let mut z = zip::ZipArchive::new(f).map_err(|e| err(format!("{}: {e}", path.display())))?;
let name = (0..z.len())
.filter_map(|i| z.by_index(i).ok().map(|e| e.name().to_string()))
.find(|n| n.ends_with("/data/0"))
.ok_or_else(|| err(format!("{}: not a voice pack", path.display())))?;
let order = (0..z.len()).filter_map(|i| z.by_index(i).ok().map(|e| e.name().to_string())).find(|n| n.ends_with("/byteorder"));
if let Some(o) = order {
let mut s = String::new();
z.by_name(&o).map_err(err)?.take(16).read_to_string(&mut s).map_err(err)?;
if s.trim() != "little" {
return Err(err(format!("{}: unsupported byte order `{s}`", path.display())));
}
}
let want = PACK_LEN * STYLE * 4;
let mut e = z.by_name(&name).map_err(err)?;
if e.size() != want as u64 {
return Err(err(format!("{}: voice data is {} bytes, expected {want}", path.display(), e.size())));
}
let mut buf = Vec::with_capacity(want);
e.by_ref().take(want as u64).read_to_end(&mut buf).map_err(err)?;
if buf.len() != want {
return Err(err(format!("{}: truncated voice data", path.display())));
}
let v: Vec<f32> = buf.as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect();
if v.iter().any(|x| !x.is_finite()) {
return Err(err(format!("{}: voice data is not finite", path.display())));
}
Ok(v)
}
/// The style vector for an input of `n` phonemes: `[1, 256]`.
pub(crate) fn style_for(pack: &[f32], n: usize, dev: &Device) -> Result<Tensor, TtsError> {
let row = n.clamp(1, PACK_LEN) - 1;
let s = pack.get(row * STYLE..(row + 1) * STYLE).ok_or_else(|| err("voice pack too short"))?;
Tensor::from_slice(s, (1, STYLE), dev).map_err(err)
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
fn tmp(name: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("filmcraft-kokoro-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
/// A voice-pack-shaped zip with `data` as `x/data/0`.
fn pack(dir: &Path, data: &[u8], order: &str) -> std::path::PathBuf {
let p = dir.join("v.pt");
let mut z = zip::ZipWriter::new(std::fs::File::create(&p).unwrap());
let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
z.start_file("x/data.pkl", o).unwrap();
z.write_all(b"not used").unwrap();
z.start_file("x/byteorder", o).unwrap();
z.write_all(order.as_bytes()).unwrap();
z.start_file("x/data/0", o).unwrap();
z.write_all(data).unwrap();
z.finish().unwrap();
p
}
fn floats(n: usize, v: f32) -> Vec<u8> {
std::iter::repeat_n(v, n).flat_map(f32::to_le_bytes).collect()
}
#[test]
fn reads_a_well_formed_voice_pack_and_picks_the_row_by_length() {
let d = tmp("ok");
let mut data = floats(PACK_LEN * STYLE, 0.0);
// row 4 (inputs of 5 phonemes) starts with 7.0
data[4 * STYLE * 4..4 * STYLE * 4 + 4].copy_from_slice(&7.0f32.to_le_bytes());
let v = read_voice(&pack(&d, &data, "little")).unwrap();
assert_eq!(v.len(), PACK_LEN * STYLE);
let s = style_for(&v, 5, &Device::Cpu).unwrap();
assert_eq!(s.dims(), &[1, STYLE]);
assert_eq!(s.to_vec2::<f32>().unwrap()[0][0], 7.0);
// lengths outside 1..=510 clamp instead of failing
assert!(style_for(&v, 0, &Device::Cpu).is_ok());
assert!(style_for(&v, 100_000, &Device::Cpu).is_ok());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn hostile_voice_packs_are_errors_not_panics() {
let d = tmp("bad");
let cases: Vec<(&str, std::path::PathBuf)> =
vec![("short", pack(&d, &floats(10, 0.0), "little")), ("long", pack(&d.join(".."), &floats(PACK_LEN * STYLE + 1, 0.0), "little"))];
for (what, p) in cases {
assert!(read_voice(&p).is_err(), "{what}");
}
let p = pack(&d, &floats(PACK_LEN * STYLE, 0.0), "big");
assert!(read_voice(&p).is_err(), "big endian");
let p = pack(&d, &floats(PACK_LEN * STYLE, f32::NAN), "little");
assert!(read_voice(&p).is_err(), "NaN");
let junk = d.join("junk.pt");
std::fs::write(&junk, b"PK\x03\x04 definitely not a zip").unwrap();
assert!(read_voice(&junk).is_err());
std::fs::write(&junk, b"").unwrap();
assert!(read_voice(&junk).is_err());
assert!(read_voice(&d.join("missing.pt")).is_err());
// a checkpoint that isn't one
assert!(Weights::load(&junk).is_err());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn config_must_match_the_implemented_architecture() {
let d = tmp("cfg");
let p = d.join("config.json");
let good = r#"{"hidden_dim":512,"style_dim":128,"n_token":178,"max_dur":50,"n_layer":3,"text_encoder_kernel_size":5,
"istftnet":{"gen_istft_n_fft":20,"gen_istft_hop_size":5,"upsample_rates":[10,6],"upsample_kernel_sizes":[20,12],"resblock_kernel_sizes":[3,7,11],"upsample_initial_channel":512},
"plbert":{"hidden_size":768,"num_attention_heads":12,"num_hidden_layers":12,"intermediate_size":2048},"vocab":VOCAB}"#;
let vocab: String = format!("{{{}}}", (0..110).map(|i| format!("\"{}\":{}", char::from_u32(0x100 + i).unwrap(), i + 1)).collect::<Vec<_>>().join(","));
std::fs::write(&p, good.replace("VOCAB", &vocab)).unwrap();
assert_eq!(read_config(&p).unwrap().vocab.len(), 110);
std::fs::write(&p, good.replace("VOCAB", &vocab).replace("\"hidden_dim\":512", "\"hidden_dim\":256")).unwrap();
assert!(read_config(&p).err().unwrap().to_string().contains("hidden_dim"));
std::fs::write(&p, good.replace("VOCAB", "{\"a\":1}")).unwrap();
assert!(read_config(&p).is_err(), "tiny vocabulary");
for junk in ["", "{", "[]", "null", "{\"vocab\":5}"] {
std::fs::write(&p, junk).unwrap();
assert!(read_config(&p).is_err(), "{junk}");
}
std::fs::write(&p, vec![b' '; (1 << 20) + 1]).unwrap();
assert!(read_config(&p).is_err(), "oversized");
let _ = std::fs::remove_dir_all(&d);
}
}
+222
View File
@@ -0,0 +1,222 @@
//! Text to speech for FilmCraft's Text to Speech panel (narrations).
//!
//! - [`Voice`]: the trait every voice implements: a script in, mono samples out.
//! - [`voices`] / [`voice`]: the voice catalogue (id, name, language, gender, engine, licence).
//! - [`script`]: splits a script into speech and pause markers (`[pause 1s]`, `[pause 500ms]`).
//! - [`formant`]: the built-in voices, an original source–filter formant synthesizer. They need
//! no download and work everywhere (also in the web build); they sound robotic. The neural voices
//! (Kokoro-82M, downloaded on first use) are added behind a feature in a later milestone.
//!
//! Everything is deterministic: the same script and settings give the same samples.
pub mod catalog;
pub mod formant;
#[cfg(feature = "kokoro")]
pub mod kokoro;
#[cfg(feature = "kokoro")]
pub mod neural;
pub mod script;
use serde::Serialize;
/// Sample rate of every voice's output.
pub const SAMPLE_RATE: u32 = 24_000;
/// Longest narration a voice produces, in seconds.
pub const MAX_SECONDS: f64 = 30.0 * 60.0;
/// Longest script accepted, in bytes (same as `filmcraft_project::narration::MAX_TEXT_BYTES`).
pub const MAX_TEXT_BYTES: usize = 64 * 1024;
/// Pace limits.
pub const MIN_PACE: f64 = 0.5;
pub const MAX_PACE: f64 = 2.0;
/// Pitch shift limit in semitones (either way).
pub const MAX_SEMITONES: f64 = 12.0;
#[derive(Debug, thiserror::Error, PartialEq)]
pub enum TtsError {
#[error("unknown voice `{0}`")]
UnknownVoice(String),
#[error("the script has nothing to say")]
Empty,
#[error("the script is longer than {} KB", MAX_TEXT_BYTES / 1024)]
TextTooLong,
#[error("the narration would be longer than {} minutes", MAX_SECONDS / 60.0)]
AudioTooLong,
#[error("{0}")]
Invalid(String),
#[error("voice model: {0}")]
Model(String),
#[error("the natural voices are not downloaded yet")]
NotInstalled,
#[error("natural voices are not available in this build (built without the `kokoro` feature)")]
Unavailable,
}
/// How a voice speaks.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Params {
/// Pitch shift in semitones, ±[`MAX_SEMITONES`].
pub semitones: f64,
/// Speaking rate, [`MIN_PACE`]–[`MAX_PACE`] (1 = normal).
pub pace: f64,
}
impl Default for Params {
fn default() -> Self {
Params { semitones: 0.0, pace: 1.0 }
}
}
impl Params {
pub fn check(&self) -> Result<(), TtsError> {
if !(MIN_PACE..=MAX_PACE).contains(&self.pace) {
return Err(TtsError::Invalid(format!("pace must be between {MIN_PACE} and {MAX_PACE}")));
}
if !(-MAX_SEMITONES..=MAX_SEMITONES).contains(&self.semitones) {
return Err(TtsError::Invalid(format!("pitch must be within ±{MAX_SEMITONES} semitones")));
}
Ok(())
}
}
/// Mono audio at [`SAMPLE_RATE`].
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Audio {
pub samples: Vec<f32>,
pub sample_rate: u32,
}
impl Audio {
pub fn seconds(&self) -> f64 {
self.samples.len() as f64 / f64::from(self.sample_rate.max(1))
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum Gender {
Female,
Male,
}
/// A voice in the catalogue.
#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct VoiceInfo {
pub id: &'static str,
pub name: &'static str,
/// BCP 47 tag.
pub language: &'static str,
pub gender: Gender,
/// `basic` (built in) or `neural` (downloaded).
pub engine: &'static str,
pub description: &'static str,
pub license: &'static str,
/// Usable now (built in, or downloaded).
pub installed: bool,
}
/// A voice that turns a script into audio.
pub trait Voice: Send + Sync {
fn info(&self) -> VoiceInfo;
/// Speak `text` (pause markers allowed). Fails on an empty or over-long script, or bad params.
fn synthesize(&self, text: &str, params: &Params) -> Result<Audio, TtsError>;
}
/// Languages offered (English only for now).
pub const LANGUAGES: &[(&str, &str)] = &[("en-US", "English (United States)")];
/// Every voice FilmCraft knows: the built-in voices, then the neural voices (installed when their
/// package is downloaded into `models_dir`).
pub fn voices_in(models_dir: Option<&std::path::Path>) -> Vec<VoiceInfo> {
let installed = models_dir.is_some_and(catalog::installed) && cfg!(feature = "kokoro");
formant::VOICES.iter().map(|v| v.info).chain(catalog::VOICES.iter().map(|v| VoiceInfo { installed, ..v.info })).collect()
}
/// The built-in voices (always available).
pub fn voices() -> Vec<VoiceInfo> {
formant::VOICES.iter().map(|v| v.info).collect()
}
/// The voice with this id.
pub fn voice(id: &str) -> Result<Box<dyn Voice>, TtsError> {
formant::VOICES
.iter()
.find(|v| v.info.id == id)
.map(|v| Box::new(formant::FormantVoice::new(*v)) as Box<dyn Voice>)
.ok_or_else(|| TtsError::UnknownVoice(id.to_string()))
}
/// The voice `id`: built in, or (feature `kokoro`) a neural voice from the package in `models_dir`.
pub fn voice_in(id: &str, models_dir: Option<&std::path::Path>) -> Result<Box<dyn Voice>, TtsError> {
if catalog::find(id).is_some() {
#[cfg(feature = "kokoro")]
{
let dir = models_dir.ok_or(TtsError::NotInstalled)?;
return neural::load(id, dir).map(|v| Box::new(v) as Box<dyn Voice>);
}
#[cfg(not(feature = "kokoro"))]
{
let _ = models_dir;
return Err(TtsError::Unavailable);
}
}
voice(id)
}
/// The voice used when none is chosen.
pub fn default_voice_id() -> &'static str {
formant::VOICES.first().map(|v| v.info.id).unwrap_or("basic-female")
}
/// The sentence "Hear this voice" plays.
pub const SAMPLE_SENTENCE: &str = "Hello! This is how I sound when I read your script.";
/// Validate a script before synthesis: length, and that it says something.
pub fn check_text(text: &str) -> Result<(), TtsError> {
if text.len() > MAX_TEXT_BYTES {
return Err(TtsError::TextTooLong);
}
let says_something = script::parse(text).iter().any(|s| matches!(s, script::Segment::Speech(t) if t.chars().any(char::is_alphanumeric)));
if !says_something {
return Err(TtsError::Empty);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn catalogue_ids_are_unique_and_resolvable() {
let v = voices();
assert!(v.len() >= 2);
for (i, a) in v.iter().enumerate() {
assert!(v.iter().skip(i + 1).all(|b| b.id != a.id), "duplicate {}", a.id);
assert_eq!(voice(a.id).unwrap().info().id, a.id);
}
assert_eq!(voice("nope").err(), Some(TtsError::UnknownVoice("nope".into())));
assert!(voice(default_voice_id()).is_ok());
}
#[test]
fn params_are_checked() {
assert!(Params::default().check().is_ok());
for p in [
Params { pace: f64::NAN, ..Default::default() },
Params { pace: 0.1, ..Default::default() },
Params { semitones: 40.0, ..Default::default() },
Params { semitones: f64::INFINITY, ..Default::default() },
] {
assert!(p.check().is_err(), "{p:?}");
}
}
#[test]
fn empty_scripts_are_refused() {
assert_eq!(check_text(""), Err(TtsError::Empty));
assert_eq!(check_text(" [pause 1s] ... "), Err(TtsError::Empty));
assert_eq!(check_text(&"a".repeat(MAX_TEXT_BYTES + 1)), Err(TtsError::TextTooLong));
assert!(check_text("Hi").is_ok());
}
}
+133
View File
@@ -0,0 +1,133 @@
//! The neural voices (feature `kokoro`): a script goes through the text front end
//! (`filmcraft-tts-text`: normalizer, CMUdict + letter-to-sound, chunks of at most 400 phonemes
//! and pause markers), each chunk through Kokoro-82M, pauses become exact silence, and the vocal
//! pitch is applied afterwards with the pitch shifter from `filmcraft-audio-dsp` (Kokoro has no
//! pitch control). Pace is Kokoro's speed. The model, the dictionary and voice packs are loaded
//! once per package directory and shared.
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock, PoisonError};
use filmcraft_audio_dsp::AudioEffect;
use filmcraft_tts_text::{Chunk, Lang, Lexicon, Overrides};
use crate::catalog;
use crate::kokoro::{KOKORO_RATE, Kokoro, VoicePack};
use crate::{Audio, MAX_SECONDS, Params, TtsError, Voice, VoiceInfo, check_text};
/// Silence between sentences, ms.
const SENTENCE_GAP_MS: u32 = 120;
struct Loaded {
dir: PathBuf,
model: Arc<Kokoro>,
lexicon: Arc<Lexicon>,
packs: HashMap<&'static str, Arc<VoicePack>>,
}
fn cache() -> &'static Mutex<Option<Loaded>> {
static C: OnceLock<Mutex<Option<Loaded>>> = OnceLock::new();
C.get_or_init(Default::default)
}
/// A neural voice ready to speak.
pub struct NeuralVoice {
info: VoiceInfo,
model: Arc<Kokoro>,
lexicon: Arc<Lexicon>,
pack: Arc<VoicePack>,
}
/// Load (or reuse) the package in `models_dir` and the voice `id`.
pub fn load(id: &str, models_dir: &Path) -> Result<NeuralVoice, TtsError> {
let v = catalog::find(id).ok_or_else(|| TtsError::UnknownVoice(id.to_string()))?;
if !catalog::installed(models_dir) {
return Err(TtsError::NotInstalled);
}
let dir = catalog::package_dir(models_dir);
let mut guard = cache().lock().unwrap_or_else(PoisonError::into_inner);
if guard.as_ref().is_none_or(|l| l.dir != dir) {
let model = Arc::new(Kokoro::load(&dir)?);
let bytes = std::fs::read(dir.join("cmudict.dict")).map_err(|e| TtsError::Model(format!("cmudict.dict: {e}")))?;
let lexicon = Arc::new(Lexicon::parse_cmudict(&bytes).map_err(|e| TtsError::Model(format!("cmudict.dict: {e}")))?);
*guard = Some(Loaded { dir: dir.clone(), model, lexicon, packs: HashMap::new() });
}
let loaded = guard.as_mut().ok_or_else(|| TtsError::Model("voice package not loaded".into()))?;
let pack = match loaded.packs.get(v.file) {
Some(p) => p.clone(),
None => {
let p = Arc::new(VoicePack::load(&dir.join(v.file))?);
loaded.packs.insert(v.file, p.clone());
p
}
};
Ok(NeuralVoice { info: VoiceInfo { installed: true, ..v.info }, model: loaded.model.clone(), lexicon: loaded.lexicon.clone(), pack })
}
impl Voice for NeuralVoice {
fn info(&self) -> VoiceInfo {
self.info
}
fn synthesize(&self, text: &str, params: &Params) -> Result<Audio, TtsError> {
check_text(text)?;
params.check()?;
let chunks = filmcraft_tts_text::phonemize(text, Lang::EnUs, &self.lexicon, &Overrides::default()).map_err(|e| TtsError::Invalid(e.to_string()))?;
let rate = KOKORO_RATE;
let max = (MAX_SECONDS * f64::from(rate)) as usize;
let mut out: Vec<f32> = Vec::new();
let mut spoke = false;
for (i, c) in chunks.iter().enumerate() {
match c {
Chunk::Pause { millis } => {
let n = (u64::from(*millis) * u64::from(rate) / 1000) as usize;
out.resize(out.len().saturating_add(n), 0.0);
}
Chunk::Phonemes(p) => {
if self.model.ids(p).is_empty() {
continue;
}
if spoke {
out.resize(out.len().saturating_add((SENTENCE_GAP_MS * rate / 1000) as usize), 0.0);
}
let s = self.model.speak(p, &self.pack, params.pace, i as u64 + 1)?;
out.extend_from_slice(&s);
spoke = true;
}
}
if out.len() > max {
return Err(TtsError::AudioTooLong);
}
}
if !spoke {
return Err(TtsError::Empty);
}
if params.semitones.abs() > 1e-6 {
out = pitch_shift(out, params.semitones, rate);
}
// headroom: never clip
let peak = out.iter().fold(0f32, |m, x| m.max(x.abs()));
if peak > 0.98 {
let g = 0.98 / peak;
out.iter_mut().for_each(|x| *x *= g);
}
Ok(Audio { samples: out, sample_rate: rate })
}
}
/// Shift pitch by `semitones` keeping the length (latency compensated).
fn pitch_shift(mut x: Vec<f32>, semitones: f64, rate: u32) -> Vec<f32> {
let mut ps = filmcraft_audio_dsp::effects::PitchShifter::new(rate as f32, 1);
ps.set_param("semitones", semitones as f32);
ps.reset();
let lat = ps.latency();
let n = x.len();
x.resize(n + lat, 0.0);
for block in x.chunks_mut(4096) {
ps.process(&mut [block]);
}
x.drain(..lat.min(x.len()));
x.truncate(n);
x
}
+105
View File
@@ -0,0 +1,105 @@
//! Script parsing: speech text and pause markers.
//!
//! A pause marker is `[pause <n>s]` or `[pause <n>ms]` (case-insensitive, `n` a decimal number,
//! spaces optional: `[pause 1.5s]`, `[Pause 500 ms]`). Pauses are capped at [`MAX_PAUSE_MS`]. A
//! bracket that is not a well-formed marker is ordinary text. The Text to Speech panel's
//! *Add pause* button inserts `[pause 1s]`.
/// Longest single pause.
pub const MAX_PAUSE_MS: u32 = 10_000;
/// A piece of a script.
#[derive(Clone, Debug, PartialEq)]
pub enum Segment {
Speech(String),
Pause { millis: u32 },
}
/// The marker *Add pause* inserts.
pub const PAUSE_MARKER: &str = "[pause 1s]";
/// Split `text` into speech and pauses, in order. Empty speech pieces are dropped.
pub fn parse(text: &str) -> Vec<Segment> {
let mut out = Vec::new();
let mut speech = String::new();
let mut rest = text;
while let Some(open) = rest.find('[') {
let (before, from) = rest.split_at(open);
speech.push_str(before);
let close = from.find(']');
match (close, close.and_then(|c| from.get(..=c)).and_then(pause_millis)) {
(Some(close), Some(millis)) => {
flush(&mut out, &mut speech);
out.push(Segment::Pause { millis });
rest = from.get(close + 1..).unwrap_or("");
}
_ => {
speech.push('[');
rest = from.get(1..).unwrap_or("");
}
}
}
speech.push_str(rest);
flush(&mut out, &mut speech);
out
}
fn flush(out: &mut Vec<Segment>, speech: &mut String) {
if !speech.trim().is_empty() {
out.push(Segment::Speech(std::mem::take(speech)));
}
speech.clear();
}
/// `[pause 1.5s]` → 1500. None if `marker` is not a pause marker.
fn pause_millis(marker: &str) -> Option<u32> {
let inner = marker.strip_prefix('[')?.strip_suffix(']')?.trim();
let lower = inner.to_ascii_lowercase();
let arg = lower.strip_prefix("pause")?.trim();
let (num, scale) = if let Some(n) = arg.strip_suffix("ms") { (n.trim(), 1.0) } else { (arg.strip_suffix('s')?.trim(), 1000.0) };
if num.is_empty() || num.len() > 12 || !num.chars().all(|c| c.is_ascii_digit() || c == '.') {
return None;
}
let v: f64 = num.parse().ok()?;
let ms = (v * scale).round();
if !ms.is_finite() || ms < 0.0 {
return None;
}
Some(ms.min(f64::from(MAX_PAUSE_MS)) as u32)
}
#[cfg(test)]
mod tests {
use super::*;
fn sp(s: &str) -> Segment {
Segment::Speech(s.into())
}
#[test]
fn splits_speech_and_pauses() {
assert_eq!(parse("Hello. [pause 1s] World"), vec![sp("Hello. "), Segment::Pause { millis: 1000 }, sp(" World")]);
assert_eq!(parse("[Pause 500 ms]Go"), vec![Segment::Pause { millis: 500 }, sp("Go")]);
assert_eq!(parse("a[pause 1.5s][pause 2s]b"), vec![sp("a"), Segment::Pause { millis: 1500 }, Segment::Pause { millis: 2000 }, sp("b")]);
assert_eq!(parse(PAUSE_MARKER), vec![Segment::Pause { millis: 1000 }]);
}
#[test]
fn malformed_markers_are_text() {
assert_eq!(parse("[pause]"), vec![sp("[pause]")]);
assert_eq!(parse("[pause 1x] a"), vec![sp("[pause 1x] a")]);
assert_eq!(parse("[pause 1s"), vec![sp("[pause 1s")]);
assert_eq!(parse("[note] [pause -1s]"), vec![sp("[note] [pause -1s]")]);
assert_eq!(parse("[[pause 1s]]"), vec![sp("["), Segment::Pause { millis: 1000 }, sp("]")]);
}
#[test]
fn hostile_inputs_never_panic_and_pauses_are_capped() {
assert_eq!(parse("[pause 999999999999s]"), vec![Segment::Pause { millis: MAX_PAUSE_MS }]);
assert_eq!(parse("[pause 1e9s]"), vec![sp("[pause 1e9s]")]);
for s in ["", "[", "]", "[[[[", "日本語[pause 1s]👍", "\u{301}[", "[pause 1.2.3s]", "\0[pause 0s]"] {
let _ = parse(s);
}
assert_eq!(parse(" "), vec![]);
}
}
+50
View File
@@ -0,0 +1,50 @@
Welcome to the project list, where every job you create is stored.
Click the blue button at the top of the page to start a new project.
The filter menu lets you sort projects by date, by name, or by owner.
Each card shows when the project was created and when it was last modified.
Our team finished the review on Tuesday morning.
The meeting starts at 3:30 p.m. in the main conference room.
It costs $3.50 to ship each sample.
Sales grew by 20% over the last quarter.
The building was finished in 1999 and renovated in 2015.
Please call 555-0134 if you have any questions.
The report is due on October 7th.
She bought 12 apples, 6 oranges, and a bag of grapes.
Doctor Smith lives on Main Street near the library.
The NASA mission launched before sunrise.
The TDA tool exports every diagram as a PDF.
Turn left at the second light and park behind the bank.
A quiet river runs through the center of the old town.
The children played in the garden until it got dark.
Remember to save your work before you close the window.
This video will show you how to edit a narration clip.
The weather tomorrow will be sunny with a light breeze.
He read the book in a single afternoon.
The price dropped to $1,250 after the holiday sale.
Our office opens at nine and closes at five.
The first chapter explains the basic ideas.
Water boils at one hundred degrees Celsius.
They traveled by train from the coast to the mountains.
The software update fixes several small problems.
Every frame is rendered in less than a second.
Choose a voice, type your script, and press save.
The museum is closed on Mondays and public holidays.
A good narrator speaks slowly and clearly.
The new library has more than ten thousand books.
Write down the steps so that anyone can repeat them.
The engineer checked every valve before the test.
Our company was founded in 1987 in a small garage.
The second version runs twice as fast as the first.
Bring a jacket, because the evening will be cold.
The camera follows the bird as it flies over the lake.
Thank you for watching, and see you in the next video.
The orchestra played for almost two hours.
Most users finish the setup in about five minutes.
The road was closed because of the heavy snow.
Please confirm your email address to continue.
The garden is full of roses, tulips, and sunflowers.
Our goal is to make editing simple for everyone.
The pilot announced that we would land early.
Fresh bread is baked here every morning.
The team celebrated the launch with a small party.
That is everything you need to know to get started.
+1
View File
@@ -16,6 +16,7 @@ filmcraft-color = { workspace = true }
filmcraft-frame = { workspace = true }
filmcraft-media = { workspace = true }
filmcraft-project = { workspace = true }
filmcraft-tts = { workspace = true }
filmcraft-edit = { workspace = true }
filmcraft-render = { workspace = true }
filmcraft-text = { workspace = true }
+10
View File
@@ -10,6 +10,7 @@
//! - `ui.elements {prefix?}`: registered widgets (optionally filtered by id prefix)
//! - `ui.set {tool?, workspace?, mode?, theme?, focused?, playbackRes?, timeline?:{pps,scroll}}`
//! - `ui.panel.show {panel}` / `ui.panel.close {panel}`
//! - `ui.set {tts: {voice, pitch, pace, text…}}`: patch the Text to Speech panel draft
//! - `ui.click {id | x,y, button?, count?, modifiers?}` / `ui.move {x,y}` / `ui.scroll {x,y,dx,dy}`
//! - `ui.drag {from:{id|x,y}, to:{id|x,y}, steps?, modifiers?}`: synthetic press-move-release
//! - `ui.key {key, command?, shift?, alt?, ctrl?}` / `ui.type {text}`
@@ -123,6 +124,15 @@ pub fn handle(app: &mut FilmcraftApp, ctx: &egui::Context, req: &ControlRequest)
ok(serde_json::to_value(app.auto.query(prefix)).unwrap_or_default())
}
"ui.set" => {
// Text to Speech panel draft (voice, pitch, pace, text…), merged into the current one
if let Some(patch) = p.get("tts") {
let mut cur = serde_json::to_value(&app.ui.tts).unwrap_or_default();
merge(&mut cur, patch);
match serde_json::from_value(cur) {
Ok(d) => app.ui.tts = d,
Err(e) => return err(format!("bad `tts`: {e}")),
}
}
if let Some(t) = s("tool") {
match Tool::from_name(t) {
Some(t) => app.ui.tool = t,
+6 -3
View File
@@ -30,6 +30,7 @@ pub enum PanelKind {
LumetriScopes,
EssentialGraphics,
EssentialSound,
TextToSpeech,
Properties,
Text,
Events,
@@ -39,7 +40,7 @@ pub enum PanelKind {
}
impl PanelKind {
pub const ALL: [PanelKind; 26] = [
pub const ALL: [PanelKind; 27] = [
PanelKind::Project,
PanelKind::MediaBrowser,
PanelKind::Libraries,
@@ -60,6 +61,7 @@ impl PanelKind {
PanelKind::LumetriScopes,
PanelKind::EssentialGraphics,
PanelKind::EssentialSound,
PanelKind::TextToSpeech,
PanelKind::Properties,
PanelKind::Text,
PanelKind::Events,
@@ -89,6 +91,7 @@ impl PanelKind {
PanelKind::LumetriScopes => "Lumetri Scopes",
PanelKind::EssentialGraphics => "Essential Graphics",
PanelKind::EssentialSound => "Essential Sound",
PanelKind::TextToSpeech => "Text to Speech",
PanelKind::Properties => "Properties",
PanelKind::Text => "Text",
PanelKind::Events => "Events",
@@ -268,7 +271,7 @@ pub fn workspace(name: &str) -> DockNode {
hsplit(Ratio(0.5), tabs(&[AudioTrackMixer, Source, AudioClipMixer, EffectControls], 0), tabs(&[Program], 0)),
hsplit(Ratio(0.3), tabs(&[Project, MediaBrowser, Effects, Markers, History], 0), bottom_editing()),
),
tabs(&[EssentialSound], 0),
tabs(&[EssentialSound, TextToSpeech], 0),
),
"Captions and Graphics" => hsplit(
FixedB(330.0),
@@ -284,7 +287,7 @@ pub fn workspace(name: &str) -> DockNode {
hsplit(
Ratio(0.5),
tabs(&[Source, EffectControls, AudioClipMixer, Metadata, LumetriScopes, Text], 0),
hsplit(Ratio(0.6), tabs(&[Program], 0), tabs(&[LumetriColor, EssentialGraphics, EssentialSound, Properties, AudioTrackMixer], 0)),
hsplit(Ratio(0.6), tabs(&[Program], 0), tabs(&[LumetriColor, EssentialGraphics, EssentialSound, TextToSpeech, Properties, AudioTrackMixer], 0)),
),
hsplit(Ratio(0.26), tabs(&[Project, MediaBrowser, Libraries, Info, Effects, Markers, History], 0), bottom_editing()),
),
+7
View File
@@ -1253,6 +1253,9 @@
Active Monitor Monitor activo
Add Añadir
Add tracks automatically when editing source clips onto the Timeline Añadir pistas automáticamente al editar clips de origen en la línea de tiempo
Place files dropped onto the Timeline directly on the Timeline Colocar los archivos soltados en la línea de tiempo directamente en ella
1 file 1 archivo
{n} files {n} archivos
Additional Scale Escala adicional
Additive Dissolve Disolución aditiva
Additive Dissolve (Legacy) Disolución aditiva (heredada)
@@ -3488,6 +3491,8 @@
New Sequence Nueva secuencia
Tracks Pistas
tracks pistas
Clear Transition Borrar transición
Set Transition Duration Establecer duración de la transición
# System appearance modes and theme cards.
Active Activo
@@ -3505,3 +3510,5 @@
Solid Sólido
GPU Rendering Renderizado por GPU
Composite on the GPU when available Componer en la GPU cuando esté disponible
Text to Speech Texto a voz
Edit Narration Editar narración
Can't render this file because it contains an unexpected character in line 3 and column 95.
+7
View File
@@ -1253,6 +1253,9 @@
Active Monitor アクティブなモニター
Add 加算
Add tracks automatically when editing source clips onto the Timeline ソースクリップをタイムラインに配置するとき、トラックを自動的に追加する
Place files dropped onto the Timeline directly on the Timeline タイムラインにドロップしたファイルをタイムラインに直接配置
1 file 1 個のファイル
{n} files {n} 個のファイル
Additional Scale 追加スケール
Additive Dissolve 加算ディゾルブ
Additive Dissolve (Legacy) 加算ディゾルブ (従来版)
@@ -3498,3 +3501,7 @@
Solid べた塗り
GPU Rendering GPU レンダリング
Composite on the GPU when available 利用可能な場合は GPU で合成
Text to Speech テキスト読み上げ
Edit Narration ナレーションを編集
Clear Transition トランジションを消去
Set Transition Duration トランジションのデュレーションを設定
Can't render this file because it contains an unexpected character in line 11 and column 59.
+7
View File
@@ -1246,6 +1246,9 @@
Active Monitor 活动监视器
Add 添加
Add tracks automatically when editing source clips onto the Timeline 将源剪辑编辑到时间轴时自动添加轨道
Place files dropped onto the Timeline directly on the Timeline 将拖放到时间轴上的文件直接放置在时间轴上
1 file 1 个文件
{n} files {n} 个文件
Additional Scale 附加缩放
Additive Dissolve 添加式溶解
Additive Dissolve (Legacy) 添加式溶解(旧版)
@@ -3487,3 +3490,7 @@
Sync with system 跟随系统
GPU Rendering GPU 渲染
Composite on the GPU when available 可用时在 GPU 上合成
Text to Speech 文字转语音
Edit Narration 编辑旁白
Clear Transition 清除过渡
Set Transition Duration 设置过渡持续时间
Can't render this file because it contains an unexpected character in line 3 and column 95.
+126 -3
View File
@@ -88,6 +88,11 @@ pub trait AudioOut {
/// effect at the next `start`; `document_rate` is the sequence sample rate for "Attempt to
/// force hardware to document sample rate".
fn configure(&mut self, _hw: &filmcraft_engine::settings::AudioHardwarePrefs, _document_rate: Option<u32>) {}
/// A hardware note for the status bar — e.g. the default output device could not be opened
/// and a fallback device is in use, or no output device was found — or None.
fn note(&self) -> Option<String> {
None
}
}
/// What the platform audio layer can open (Settings ▸ Audio Hardware).
@@ -105,6 +110,8 @@ pub struct AudioDevices {
#[derive(Default)]
pub struct HostHooks {
pub pick_files: Option<Box<dyn FnMut(&[&str]) -> Vec<String>>>,
/// The cursor in physical screen pixels. Where the windowing layer reports no pointer while files are dragged in from the OS (Windows), a drop is placed where this says it landed.
pub cursor_screen_position: Option<Box<dyn Fn() -> Option<(i32, i32)>>>,
pub pick_save: Option<Box<dyn FnMut(&str) -> Option<String>>>,
pub pick_open_project: Option<Box<dyn FnMut() -> Option<String>>>,
/// Save dialog with a filter: (filter name, extensions, suggested file name) → path.
@@ -170,6 +177,8 @@ pub enum Dialog {
AddTracks,
/// Sequence ▸ Sequence Settings….
SequenceSettings,
/// Set Transition Duration (double-click a transition in the Timeline).
TransitionDuration,
}
#[derive(Default)]
@@ -277,6 +286,10 @@ pub struct FilmcraftApp {
/// The sequence whose view `ui.timeline` holds, and that view as it was last exchanged with
/// `session.state.timeline_views` (see `sync_timeline_view`).
timeline_view_of: Option<filmcraft_engine::project::ItemId>,
/// Items imported by an OS file drop, to place on the Timeline at the drop point this frame.
pending_timeline_drop: Option<(Vec<filmcraft_engine::project::ItemId>, egui::Pos2)>,
/// Files being dragged over the window from the OS (count, pointer): the Timeline previews where they would land.
file_drag_hover: Option<(usize, egui::Pos2)>,
timeline_view_last: Option<filmcraft_engine::project::SequenceView>,
pub fps: f32,
last_time: f64,
@@ -465,6 +478,8 @@ impl FilmcraftApp {
integrated_titlebar: false,
last_timeline_width: 1000.0,
timeline_view_of: None,
pending_timeline_drop: None,
file_drag_hover: None,
timeline_view_last: None,
fps: 60.0,
last_time: 0.0,
@@ -597,6 +612,9 @@ impl FilmcraftApp {
self.start_audio();
}
}
if let Some(note) = self.audio.as_ref().and_then(|a| a.note()) {
self.ui.status = note;
}
if prev.is_none() && !self.workspace_restored {
// reopen the workspace in use when the app last closed
self.workspace_restored = true;
@@ -695,6 +713,11 @@ impl FilmcraftApp {
}
if !self.ui.dock.contains(p) {
let near = match p {
// a side panel beside Properties (or Essential Sound), never over the Program monitor
PanelKind::TextToSpeech => [PanelKind::Properties, PanelKind::EssentialSound, PanelKind::EssentialGraphics]
.into_iter()
.find(|k| self.ui.dock.contains(*k))
.unwrap_or(PanelKind::Program),
PanelKind::LumetriColor | PanelKind::EssentialGraphics | PanelKind::EssentialSound | PanelKind::Properties => PanelKind::Program,
PanelKind::Source
| PanelKind::EffectControls
@@ -814,6 +837,34 @@ impl FilmcraftApp {
panels::voiceover::on_play(self);
}
/// Play the last Text to Speech preview (`Session::tts_preview`) on the audio output, stopping
/// timeline playback first. Starting playback again takes the output back.
pub fn play_tts_preview(&mut self) -> Result<f64, String> {
let audio = self.session.tts_preview.clone().ok_or("nothing to preview")?;
if self.playback.playing {
self.stop();
}
let a = self.audio.as_mut().ok_or("no audio output (check Settings ▸ Audio Hardware)")?;
let sr = a.sample_rate().max(1);
a.stop();
let step = f64::from(audio.sample_rate.max(1)) / f64::from(sr);
let seconds = audio.seconds();
let mut pos = 0.0f64;
let fill = Box::new(move |buf: &mut [f32], ch: usize| {
for frame in buf.chunks_mut(ch.max(1)) {
let i = pos as usize;
let f = (pos - i as f64) as f32;
let x0 = audio.samples.get(i).copied().unwrap_or(0.0);
let x1 = audio.samples.get(i + 1).copied().unwrap_or(0.0);
frame.fill(x0 + (x1 - x0) * f);
pos += step;
}
});
a.start(fill)?;
self.playback.audio_clock = false;
Ok(seconds)
}
pub fn stop(&mut self) {
self.playback.playing = false;
self.playback.stop_at = None;
@@ -862,7 +913,12 @@ impl FilmcraftApp {
})
};
match a.start(fill) {
Ok(_) => self.playback.audio_clock = true,
Ok(_) => {
self.playback.audio_clock = true;
if let Some(note) = self.audio.as_ref().and_then(|audio| audio.note()) {
self.ui.status = note;
}
}
Err(e) => {
log::warn!("audio output unavailable: {e}");
self.playback.audio_clock = false;
@@ -1162,8 +1218,17 @@ impl FilmcraftApp {
r
}
/// Import dropped files.
/// Import dropped files. With Timeline ▸ drop imports to the timeline, the new items are also
/// queued to be placed at the pointer (see `timeline::interact`).
fn handle_drops(&mut self, ctx: &egui::Context) {
// While files hover, the OS sends no cursor events: poll the cursor each frame to draw the Timeline preview.
let hovering = ctx.input(|i| i.raw.hovered_files.len());
self.file_drag_hover = if hovering > 0 && self.session.prefs.timeline.drop_import_to_timeline {
ctx.request_repaint();
self.drop_pointer(ctx).map(|p| (hovering, p))
} else {
None
};
let dropped = ctx.input(|i| i.raw.dropped_files.clone());
let mut paths = Vec::new();
for f in dropped {
@@ -1173,10 +1238,33 @@ impl FilmcraftApp {
}
}
if !paths.is_empty() {
let _ = self.session.execute("file.import", json!({"paths": paths, "bin": self.import_bin().0}));
let pointer = self.drop_pointer(ctx);
let imported = self.session.execute("file.import", json!({"paths": paths, "bin": self.import_bin().0}));
if let Ok(v) = imported
&& let Some(pos) = pointer
&& self.session.prefs.timeline.drop_import_to_timeline
{
let items: Vec<filmcraft_engine::project::ItemId> =
v["items"].as_array().map(|a| a.iter().filter_map(|i| i.as_u64()).map(filmcraft_engine::project::ItemId).collect()).unwrap_or_default();
if !items.is_empty() {
self.pending_timeline_drop = Some((items, pos));
}
}
}
}
/// Where files dragged in from the OS are over the window. The OS reports no cursor motion during such a drag
/// (winit, Windows), so ask it through the host hook, else use the last pointer position egui saw.
fn drop_pointer(&self, ctx: &egui::Context) -> Option<egui::Pos2> {
let os_pointer = self
.hooks
.cursor_screen_position
.as_ref()
.and_then(|f| f())
.and_then(|px| ctx.input(|i| i.viewport().inner_rect.map(|r| screen_px_to_ui(px, i.pixels_per_point, r.min))));
os_pointer.or_else(|| ctx.input(|i| i.pointer.hover_pos().or(i.pointer.latest_pos())))
}
// ---------------------------------------------------------------- input
fn handle_shortcuts(&mut self, ctx: &egui::Context) {
@@ -1388,6 +1476,11 @@ impl FilmcraftApp {
self.advance_playback(&ctx);
self.advance_source_playback(&ctx);
self.scrub_audio(&ctx);
// Text to Speech: a selected narration clip loads into the panel
panels::tts::follow_selection(self);
if self.ui.tts.pending.is_some() || self.ui.tts.download_job.is_some() {
ctx.request_repaint_after(std::time::Duration::from_millis(100));
}
let t = self.tokens;
let full = ui.max_rect();
ui.painter().rect_filled(full, 0.0, t.app_bg);
@@ -1403,6 +1496,8 @@ impl FilmcraftApp {
state::Mode::Import => panels::import_mode::show(self, ui, body),
state::Mode::Export => panels::export_mode::show(self, ui, body),
}
// the Timeline has had its chance to place a file drop this frame; anything left stays a bin import
self.pending_timeline_drop = None;
panels::dialogs::show(self, &ctx);
if !self.ui.show_status_bar {
// no bar to draw the job in, but a finished preview render still plays
@@ -1951,3 +2046,31 @@ mod audio_recovery_tests {
app.stop();
}
}
/// A physical-pixel screen position as a point in the window's UI space, given the window's content origin in points.
///
/// Assumes the whole virtual screen uses the window's `pixels_per_point`: with monitors of mixed
/// DPI the point can be off when the cursor and the window's origin are on different monitors.
fn screen_px_to_ui(px: (i32, i32), pixels_per_point: f32, content_min: egui::Pos2) -> egui::Pos2 {
let ppp = if pixels_per_point.is_finite() && pixels_per_point > 0.0 { pixels_per_point } else { 1.0 };
egui::pos2(px.0 as f32 / ppp - content_min.x, px.1 as f32 / ppp - content_min.y)
}
#[cfg(test)]
mod screen_px_tests {
use super::screen_px_to_ui;
#[test]
fn maps_screen_pixels_into_the_window() {
let p = screen_px_to_ui((1300, 900), 2.0, egui::pos2(100.0, 50.0));
assert_eq!((p.x, p.y), (550.0, 400.0));
}
#[test]
fn a_hostile_scale_does_not_divide_by_zero() {
for ppp in [0.0, -1.0, f32::NAN, f32::INFINITY] {
let p = screen_px_to_ui((10, 20), ppp, egui::pos2(0.0, 0.0));
assert!(p.x.is_finite() && p.y.is_finite());
}
}
}
+6
View File
@@ -67,6 +67,12 @@ pub fn show(app: &mut FilmcraftApp, ctx: &egui::Context) {
}
return;
}
Dialog::TransitionDuration => {
if !crate::panels::transition_controls::duration_dialog(app, ctx) && app.dialog == Some(d) {
app.dialog = None;
}
return;
}
Dialog::SequenceSettings => {
if !crate::panels::sequence_settings::show(app, ctx) && app.dialog == Some(d) {
app.dialog = None;
+151 -2
View File
@@ -28,6 +28,19 @@ fn timeline_time_of(it: &TrackItem, m: Tick) -> Tick {
it.start + Tick(((m - it.source_in).0 as f64 / it.speed.abs().max(1e-6)) as i64)
}
// The press frame can include movement before the button went down. Start from the press
// origin so that only gesture motion enters the preview, including same-frame movement.
fn graph_drag_offset(ui: &egui::Ui, response: &egui::Response, previous: egui::Vec2) -> egui::Vec2 {
if response.drag_started() {
response
.interact_pointer_pos()
.zip(ui.input(|i| i.pointer.press_origin()))
.map_or_else(|| response.drag_delta(), |(position, origin)| position - origin)
} else {
previous + response.drag_delta()
}
}
fn selected_clip(app: &FilmcraftApp) -> Option<(ClipId, TrackItem, TrackKind)> {
selected_clips(app).into_iter().next()
}
@@ -59,6 +72,11 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
let t = app.tokens;
let clips = selected_clips(app);
let Some((clip, it, _)) = clips.first().cloned() else {
// a transition clicked in the Timeline (#430)
if let Some(id) = crate::panels::transition_controls::selected(app) {
crate::panels::transition_controls::effect_controls(app, ui, rect, id);
return;
}
crate::dock::placeholder(ui, rect, &t, tl!("(no clip selected)"));
return;
};
@@ -979,7 +997,7 @@ pub(crate) fn graph_rows(
p.circle_filled(hp, 3.5, t.keyframe_handle);
let hr = ui.interact(Rect::from_center_size(hp, vec2(10.0, 10.0)), hid.with("h"), Sense::drag());
if hr.dragged() {
let nx = hdx + hr.drag_delta().x;
let nx = graph_drag_offset(ui, &hr, vec2(hdx, 0.0)).x;
ui.data_mut(|d| d.insert_temp(hid, nx));
}
if hr.drag_stopped() {
@@ -991,7 +1009,7 @@ pub(crate) fn graph_rows(
}
p.circle_filled(c, 4.5, if resp.dragged() { t.hot_text } else { t.plot_handle });
if resp.dragged() {
let ny = dy + resp.drag_delta().y;
let ny = graph_drag_offset(ui, &resp, vec2(0.0, dy)).y;
ui.data_mut(|d| d.insert_temp(id, ny));
p.text(c + vec2(8.0, -10.0), Align2::LEFT_BOTTOM, format!("{:.dec$}", v_of(c.y)), Tokens::ui(10.5), t.hot_text);
}
@@ -1172,3 +1190,134 @@ mod lane_tests {
assert_eq!(labels, ["00:01:00:00", "00:01:02:00", "00:01:04:00"]);
}
}
#[cfg(test)]
mod graph_drag_tests {
use super::*;
use filmcraft_project::{Interpolation, Keyframe, Param};
struct Driver {
app: FilmcraftApp,
ctx: egui::Context,
clip: ClipId,
time: f64,
}
impl Driver {
fn new() -> Self {
let mut session = filmcraft_engine::Session::default();
session.execute("file.openDemoProject", json!({})).unwrap();
let seq = session.state.active_sequence.unwrap();
let clip = session.active_sequence().unwrap().video_tracks[0].items[0].id;
let it = std::sync::Arc::make_mut(&mut session.project).sequence_mut(seq).unwrap().find_item_mut(clip).unwrap().1;
let mut a = Keyframe::new(it.source_in + Tick(it.duration.0 / 4), ParamValue::Float(40.0));
let mut b = Keyframe::new(it.source_in + Tick(it.duration.0 * 3 / 4), ParamValue::Float(80.0));
a.interp = Interpolation::Bezier;
b.interp = Interpolation::Bezier;
it.effects
.iter_mut()
.find(|e| e.effect == "motion")
.unwrap()
.params
.insert("scale".into(), Param { value: ParamValue::Float(40.0), keyframes: vec![a, b] });
let app = FilmcraftApp::new(session);
let ctx = egui::Context::default();
crate::theme::install(&ctx, &app.tokens);
Self { app, ctx, clip, time: 0.0 }
}
fn param(&self) -> Param {
self.app.session.active_sequence().unwrap().find_item(self.clip).unwrap().1.effect("motion").unwrap().params["scale"].clone()
}
fn undo_len(&mut self) -> usize {
self.app.session.execute("history.list", json!({})).unwrap()["undo"].as_array().unwrap().len()
}
// Draw the actual scalar graph and execute its emitted command, as the panel does.
// Current paint shapes give a real held-frame position rather than previous telemetry.
fn frame(&mut self, events: Vec<egui::Event>) -> [Pos2; 2] {
self.time += 0.1;
self.app.auto.begin_frame();
let mut out = self.ctx.run_ui(egui::RawInput { time: Some(self.time), events, ..Default::default() }, |ui| {
let it = self.app.session.active_sequence().unwrap().find_item(self.clip).unwrap().1.clone();
let index = it.effects.iter().position(|e| e.effect == "motion").unwrap();
let effect = &it.effects[index];
let pd = effect.def().unwrap().param("scale").unwrap();
let body = Rect::from_min_size(pos2(8.0, 8.0), vec2(320.0, 200.0));
let lane = Rect::from_min_max(pos2(360.0, 8.0), pos2(760.0, 220.0));
let mut actions = Vec::new();
graph_rows(&mut self.app, ui, body, self.clip, index, None, pd, &effect.params["scale"], &lane, &it, &mut actions);
run(&mut self.app, ui.ctx(), actions);
});
out.textures_delta.clear();
fn first_circle(shape: &egui::Shape, radius: f32) -> Option<Pos2> {
match shape {
egui::Shape::Circle(circle) if circle.radius == radius => Some(circle.center),
egui::Shape::Vec(shapes) => shapes.iter().find_map(|shape| first_circle(shape, radius)),
_ => None,
}
}
[4.5, 3.5].map(|radius| {
out.shapes.iter().find_map(|shape| first_circle(&shape.shape, radius)).expect("scalar keyframe and outgoing influence are painted")
})
}
}
fn button(pos: Pos2, pressed: bool) -> egui::Event {
egui::Event::PointerButton { pos, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }
}
#[test]
fn scalar_graph_press_starts_at_the_pointer_origin_and_release_is_one_undo_step() {
for influence in [false, true] {
for batched in [true, false] {
let mut d = Driver::new();
let before = d.param();
let undo = d.undo_len();
let target = usize::from(influence);
let start = d.frame(vec![])[target];
// A different prior cursor makes the pre-press movement observable.
d.frame(vec![egui::Event::PointerMoved(start - vec2(80.0, 40.0))]);
if batched {
d.frame(vec![egui::Event::PointerMoved(start), button(start, true)]);
} else {
d.frame(vec![egui::Event::PointerMoved(start)]);
d.frame(vec![button(start, true)]);
}
let pressed = d.frame(vec![])[target];
assert!(pressed.distance(start) < 0.05, "press cannot jump: influence={influence}, batched={batched}, start={start:?}, pressed={pressed:?}");
assert_eq!(d.param(), before);
assert_eq!(d.undo_len(), undo);
let offset = if influence { vec2(20.0, 0.0) } else { vec2(0.0, -20.0) };
d.frame(vec![egui::Event::PointerMoved(start + offset)]);
let preview = d.frame(vec![])[target];
assert!(
preview.distance(start + offset) < 0.05,
"preview follows only gesture motion: influence={influence}, batched={batched}, start={start:?}, preview={preview:?}"
);
assert_eq!(d.param(), before, "held preview cannot commit");
assert_eq!(d.undo_len(), undo);
d.frame(vec![button(start + offset, false)]);
d.frame(vec![]);
let changed = d.param();
assert_eq!(d.undo_len(), undo + 1);
assert_eq!(changed.keyframes[0].time, before.keyframes[0].time);
assert_eq!(changed.keyframes[1], before.keyframes[1]);
if influence {
assert_eq!(changed.keyframes[0].value, before.keyframes[0].value);
assert!(changed.keyframes[0].out_influence > before.keyframes[0].out_influence);
} else {
assert!(changed.keyframes[0].value.as_f64().unwrap() > before.keyframes[0].value.as_f64().unwrap());
assert_eq!(changed.keyframes[0].out_influence, before.keyframes[0].out_influence);
}
d.app.session.undo().expect("one graph gesture to undo");
assert_eq!(d.param(), before);
assert_eq!(d.undo_len(), undo);
d.app.session.redo().expect("the graph gesture to redo");
assert_eq!(d.param(), changed);
assert_eq!(d.undo_len(), undo + 1);
}
}
}
}
+11 -1
View File
@@ -1594,7 +1594,17 @@ pub fn properties(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
for (id, r, label) in autos {
app.auto.add(&id, r, &label);
}
for (cmd, p) in actions {
for (cmd, mut p) in actions {
// a drag of a property is one undo step: the first change of each press begins a new one,
// the rest fold into it (like Effect Controls, #201)
if cmd == "graphics.set" && ui.ctx().input(|i| i.pointer.any_down()) {
let key = egui::Id::new("gfx-props-drag-step");
let press = ui.ctx().input(|i| i.pointer.press_start_time());
let begun = ui.ctx().data(|d| d.get_temp::<Option<f64>>(key)).flatten();
p["merge"] = json!(true);
p["begin"] = json!(begun != press);
ui.ctx().data_mut(|d| d.insert_temp(key, press));
}
if let Err(e) = app.session.execute(&cmd, p) {
app.ui.status = e.to_string();
}
+3
View File
@@ -48,7 +48,9 @@ pub mod timeline_captions;
pub mod timeline_hit;
pub mod timeline_volume;
pub mod tools;
pub mod transition_controls;
pub mod trim_monitor;
pub mod tts;
pub mod voiceover;
pub mod workspaces;
@@ -98,6 +100,7 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rect)
PanelKind::ReferenceMonitor => reference::show(app, ui, rect),
PanelKind::Text => text::show(app, ui, rect),
PanelKind::EssentialSound => essential_sound::show(app, ui, rect),
PanelKind::TextToSpeech => tts::show(app, ui, rect),
other => crate::dock::placeholder(ui, rect, &app.tokens, &tlf!("{panel} — coming in a later milestone", panel = crate::i18n::t(other.title()))),
}
}
+163 -6
View File
@@ -130,6 +130,16 @@ pub enum Drag {
/// As in [`Drag::Trim`].
from: Tick,
},
/// A transition dragged by an end (`edge`: its duration) or its middle (slides it over the
/// cut); `start` / `duration` are where it is shown now, committed on release (#224).
Transition {
id: filmcraft_project::TransitionId,
edge: Option<filmcraft_edit::Edge>,
/// As in [`Drag::Trim`].
from: Tick,
start: Tick,
duration: Tick,
},
Pan {
last: Pos2,
},
@@ -424,13 +434,22 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
}
// selected edit points (trim mode): red brackets for ripple, yellow for roll / regular trim
super::trim_monitor::paint_edit_points(app, &p, &seq, tr, r, &layout);
// transitions
// transitions (one being dragged where the drag puts it; the selected ones outlined)
for trn in &tr.transitions {
let x0 = layout.x_of(trn.start);
let x1 = layout.x_of(trn.end());
let (start, end) = match &app.tl.drag {
Some(Drag::Transition { id, start, duration, .. }) if *id == trn.id => (*start, *start + *duration),
_ => (trn.start, trn.end()),
};
let x0 = layout.x_of(start);
let x1 = layout.x_of(end);
let tr_rect = Rect::from_min_max(pos2(x0, r.rect.min.y + 17.0), pos2(x1, r.rect.max.y - 1.0));
draw_transition(&p, tr_rect, trn, &t);
let selected = app.session.state.transition_selection.contains(&trn.id);
draw_transition(&p, tr_rect, trn, selected, &t);
app.auto.add(&format!("timeline.transition.{}", trn.id.0), tr_rect, &trn.effect.effect);
// its ends, for dragging the duration (the same zones as the hit test)
let zone = super::timeline_hit::EDGE_PX.min(tr_rect.width() / 3.0);
app.auto.add(&format!("timeline.transition.{}.in", trn.id.0), Rect::from_min_max(tr_rect.min, pos2(x0 + zone, tr_rect.max.y)), "start");
app.auto.add(&format!("timeline.transition.{}.out", trn.id.0), Rect::from_min_max(pos2(x1 - zone, tr_rect.min.y), tr_rect.max), "end");
}
}
@@ -902,7 +921,7 @@ fn paint_hatch(p: &egui::Painter, r: Rect) {
}
}
fn draw_transition(p: &egui::Painter, r: Rect, trn: &filmcraft_project::Transition, _t: &Tokens) {
fn draw_transition(p: &egui::Painter, r: Rect, trn: &filmcraft_project::Transition, selected: bool, t: &Tokens) {
let cp = p.with_clip_rect(r.intersect(p.clip_rect()));
cp.rect_filled(r, 0.0, Color32::from_black_alpha(90));
paint_hatch(p, r);
@@ -924,6 +943,9 @@ fn draw_transition(p: &egui::Painter, r: Rect, trn: &filmcraft_project::Transiti
cp.add(egui::Shape::line(b, Stroke::new(1.0, Color32::WHITE)));
}
cp.rect_stroke(r, 0.0, Stroke::new(1.0, Color32::from_rgb(0xeb, 0xeb, 0xeb)), StrokeKind::Inside);
if selected {
cp.rect_stroke(r, 0.0, Stroke::new(2.0, t.accent), StrokeKind::Inside);
}
if r.width() > 34.0 && !audio {
let name = trn.effect.def().map(|d| d.name).unwrap_or(&trn.effect.effect);
let tr = Rect::from_min_size(r.min + vec2(2.0, 2.0), vec2((r.width() - 4.0).min(80.0), 13.0));
@@ -1475,7 +1497,7 @@ pub fn hit_json(app: &FilmcraftApp, pos: Pos2, mods: egui::Modifiers) -> Value {
Hit::Clip { track, clip, edge } => {
json!({"hit": "clip", "track": track.0, "clip": clip.0, "edge": edge.map(|e| format!("{e:?}")), "kind": kind, "time": t.0})
}
Hit::Transition { track, id } => json!({"hit": "transition", "track": track.0, "transition": id.0}),
Hit::Transition { track, id, edge } => json!({"hit": "transition", "track": track.0, "transition": id.0, "edge": edge.map(|e| format!("{e:?}"))}),
Hit::Empty { track } => json!({"hit": "empty", "track": track.0, "time": t.0}),
Hit::None => json!({"hit": "none"}),
}
@@ -1826,6 +1848,8 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: &
let g = grab_at(seq, layout, p, tool, mods, roll_ripple);
let cur = match (tool, g) {
(_, Grab::Edge { .. }) => CursorIcon::ResizeColumn,
(Tool::Selection, Grab::Other(Hit::Transition { edge: Some(_), .. })) => CursorIcon::ResizeColumn,
(Tool::Selection, Grab::Other(Hit::Transition { edge: None, .. })) => CursorIcon::Grab,
(_, Grab::Volume { .. }) if tool == Tool::Pen || mods.command => CursorIcon::Crosshair,
(_, Grab::Volume { .. }) => CursorIcon::ResizeVertical,
(Tool::Razor, Grab::Other(Hit::Clip { .. })) => CursorIcon::Crosshair,
@@ -1943,6 +1967,7 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: &
} else if !sel.contains(&clip) {
if mods.alt {
app.session.state.selection = vec![clip];
app.session.state.transition_selection.clear();
} else {
let _ = app.session.execute("timeline.select", json!({"clips": [clip.0]}));
}
@@ -1954,6 +1979,22 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: &
None
}
}
// click a transition to select it (Shift adds / removes); drag an end to change its
// duration or its middle to slide it over the cut (#430, #224)
(Tool::Selection, Grab::Other(Hit::Transition { id, edge, .. })) => {
// a drag of a selected transition (Shift too) moves it rather than toggling it out first
if !(app.session.state.transition_selection.contains(&id) && resp.drag_started()) {
let r = app.session.execute("timeline.select", json!({"transitions": [id.0], "toggle": mods.shift}));
if let Err(e) = r {
app.ui.status = e.to_string();
}
}
let span = seq.all_tracks().find_map(|tr| tr.transitions.iter().find(|x| x.id == id)).map(|x| (x.start, x.duration));
match span {
Some((start, duration)) if resp.drag_started() => Some(Drag::Transition { id, edge, from: t, start, duration }),
_ => None,
}
}
(_, Grab::Other(Hit::Transition { .. })) => None,
(_, Grab::Other(Hit::Empty { .. })) => {
if resp.drag_started() {
@@ -1961,6 +2002,7 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: &
} else {
app.session.state.selection.clear();
app.session.state.edit_points.clear();
app.session.state.transition_selection.clear();
None
}
}
@@ -2048,6 +2090,14 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: &
let origin = ctx.input(|i| i.pointer.press_origin()).map(|o| layout.tick_at(o.x)).unwrap_or(t_here);
Some(Drag::Slide { clip, delta: rate.snap_nearest(t_here - origin) })
}
Drag::Transition { id, edge, from, start, duration } => {
let delta = rate.snap_nearest(t_here - from);
let span = seq.all_tracks().find_map(|tr| {
tr.transitions.iter().find(|x| x.id == id).and_then(|x| filmcraft_edit::transitions::drag_span(tr, x, edge, delta, rate.frame_duration()))
});
let (start, duration) = span.unwrap_or((start, duration));
Some(Drag::Transition { id, edge, from, start, duration })
}
Drag::Volume(v) => Some(Drag::Volume(super::timeline_volume::drag(app, ui, seq, layout, v, p.y))),
Drag::Marquee { start } => {
let r = Rect::from_two_pos(start, p);
@@ -2095,6 +2145,10 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: &
Drag::Roll { left, right, delta, .. } if delta != Tick::ZERO => Some(app.session.execute("timeline.roll", json!({"left": left.0, "right": right.0, "delta": delta.0}))),
Drag::Slip { clip, delta } if delta != Tick::ZERO => Some(app.session.execute("timeline.slip", json!({"clip": clip.0, "delta": delta.0}))),
Drag::Slide { clip, delta } if delta != Tick::ZERO => Some(app.session.execute("timeline.slide", json!({"clip": clip.0, "delta": delta.0}))),
Drag::Transition { id, start, duration, .. } => {
let moved = seq.all_tracks().find_map(|tr| tr.transitions.iter().find(|x| x.id == id)).is_some_and(|x| (x.start, x.duration) != (start, duration));
moved.then(|| app.session.execute("sequence.setTransition", json!({"transition": id.0, "start": start.0, "duration": duration.0})))
}
Drag::Marquee { start } => {
if let Some(end) = resp.interact_pointer_pos() {
let r = Rect::from_two_pos(start, end);
@@ -2118,6 +2172,15 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: &
}
}
// ---- double-click a transition: Set Transition Duration (Premiere)
if resp.double_clicked()
&& tool == Tool::Selection
&& let Some(p) = resp.interact_pointer_pos()
&& let Hit::Transition { id, .. } = hit(seq, layout, p)
{
crate::panels::transition_controls::open_duration(app, id);
}
// ---- double-click a nested sequence clip: open it in its own Timeline tab
if resp.double_clicked()
&& tool == Tool::Selection
@@ -2342,9 +2405,65 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: &
}
}
}
// ---- OS file drops imported with "drop imports to the timeline": placed at the drop point, one after another
if let Some((items, p)) = app.pending_timeline_drop.clone()
&& layout.content.contains(p)
{
app.pending_timeline_drop = None;
let row = layout.row_at(p.y).cloned().or_else(|| layout.rows.iter().find(|r| r.kind == TrackKind::Video).cloned());
if let Some(row) = row {
let (vt, at) = match row.kind {
TrackKind::Video => (Some(row.track.0), seq.audio_tracks.get(row.index).or(seq.audio_tracks.first()).map(|t| t.id.0)),
TrackKind::Audio => (seq.video_tracks.get(row.index).or(seq.video_tracks.first()).map(|t| t.id.0), Some(row.track.0)),
};
let still = app.session.prefs.timeline.still_duration(rate);
let durations: Vec<Tick> = items
.iter()
.map(|&item| {
let is_still = app.session.project.item(item).and_then(|i| i.as_media()).is_some_and(|m| m.info.kind == filmcraft_media::MediaKind::Still);
app.session.project.item(item).map(|i| i.duration()).filter(|d| d.0 > 0 && !is_still).unwrap_or(still)
})
.collect();
let start = snap(app, seq, layout, rate.snap_nearest(layout.tick_at(p.x).max(Tick::ZERO)), &[]);
for (item, t) in items.iter().zip(chain_starts(start, &durations)) {
let params = json!({"item": item.0, "track": vt, "audioTrack": at, "time": t.0, "insert": false});
if let Err(e) = app.session.execute("timeline.place", params) {
app.ui.status = e.to_string();
}
}
}
}
// preview of OS files being dragged over the Timeline: a placeholder where the drop would start
if let Some((count, p)) = app.file_drag_hover
&& layout.content.contains(p)
{
let row = layout.row_at(p.y).cloned().or_else(|| layout.rows.iter().find(|r| r.kind == TrackKind::Video).cloned());
if let Some(row) = row {
let t = snap(app, seq, layout, rate.snap_nearest(layout.tick_at(p.x).max(Tick::ZERO)), &[]);
let x = layout.x_of(t);
let r = Rect::from_min_max(pos2(x, row.rect.min.y + 1.0), pos2((x + 120.0).min(layout.content.max.x), row.rect.max.y - 1.0));
ui.painter().rect_filled(r, 3.0, Color32::from_white_alpha(40));
ui.painter().rect_stroke(r, 3.0, Stroke::new(1.5, Color32::WHITE), StrokeKind::Inside);
let label = if count == 1 { tl!("1 file").to_string() } else { tlf!("{n} files", n = count) };
ui.painter().text(r.left_top() + vec2(4.0, 2.0), Align2::LEFT_TOP, label, Tokens::ui(10.0), Color32::WHITE);
}
}
app.auto.add("timeline.tracks", layout.content, "tracks");
}
/// Start tick of each clip when the clips are laid end to end from `start`. Saturates rather than overflows.
pub(crate) fn chain_starts(start: Tick, durations: &[Tick]) -> Vec<Tick> {
let mut t = start;
durations
.iter()
.map(|d| {
let at = t;
t = Tick(t.0.saturating_add(d.0.max(0)));
at
})
.collect()
}
#[cfg(test)]
mod waveform_tests {
use super::waveform_display_gain;
@@ -2367,3 +2486,41 @@ mod waveform_tests {
assert!((db(waveform_display_gain(1.0, -6.0)) - (-6.0)).abs() < 0.1);
}
}
#[cfg(test)]
mod chain_starts_tests {
use super::chain_starts;
use filmcraft_time::Tick;
#[test]
fn empty_list_has_no_starts() {
assert!(chain_starts(Tick(100), &[]).is_empty());
}
#[test]
fn one_clip_starts_at_the_drop_point() {
assert_eq!(chain_starts(Tick(7), &[Tick(30)]), vec![Tick(7)]);
}
#[test]
fn several_clips_are_laid_end_to_end() {
let durations = [Tick(10), Tick(25), Tick(5)];
assert_eq!(chain_starts(Tick(100), &durations), vec![Tick(100), Tick(110), Tick(135)]);
}
#[test]
fn start_near_i64_max_saturates_without_panicking() {
let starts = chain_starts(Tick(i64::MAX - 5), &[Tick(10), Tick(10), Tick(10)]);
assert_eq!(starts, vec![Tick(i64::MAX - 5), Tick(i64::MAX), Tick(i64::MAX)]);
}
#[test]
fn zero_length_clips_share_one_start() {
assert_eq!(chain_starts(Tick(42), &[Tick(0), Tick(0), Tick(9)]), vec![Tick(42), Tick(42), Tick(42)]);
}
#[test]
fn negative_durations_count_as_zero() {
assert_eq!(chain_starts(Tick(5), &[Tick(-3), Tick(4)]), vec![Tick(5), Tick(5)]);
}
}
+25 -4
View File
@@ -50,9 +50,21 @@ pub(crate) fn edge_geometry(seq: &Sequence, layout: &Layout, track: TrackId, cli
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Hit {
Ruler,
Clip { track: TrackId, clip: ClipId, edge: Option<Edge> },
Transition { track: TrackId, id: TransitionId },
Empty { track: TrackId },
Clip {
track: TrackId,
clip: ClipId,
edge: Option<Edge>,
},
/// A transition; `edge` is its start (`In`) or end (`Out`) when the press is on one of its
/// ends (dragging it changes the duration), `None` on its middle (dragging slides it).
Transition {
track: TrackId,
id: TransitionId,
edge: Option<Edge>,
},
Empty {
track: TrackId,
},
None,
}
@@ -69,7 +81,16 @@ pub fn hit(seq: &Sequence, layout: &Layout, pos: Pos2) -> Hit {
let x0 = layout.x_of(trn.start);
let x1 = layout.x_of(trn.end());
if pos.x >= x0 && pos.x <= x1 && pos.y > row.rect.min.y + 17.0 {
return Hit::Transition { track: row.track, id: trn.id };
// the ends are edge zones, narrower on a short transition so its middle stays grabbable
let zone = EDGE_PX.min((x1 - x0) / 3.0);
let edge = if pos.x - x0 <= zone {
Some(Edge::In)
} else if x1 - pos.x <= zone {
Some(Edge::Out)
} else {
None
};
return Hit::Transition { track: row.track, id: trn.id, edge };
}
}
if let Some((clip, edge)) = edge_at(tr, layout, pos.x) {
@@ -0,0 +1,305 @@
//! A selected transition (#430): its Effect Controls (Duration, Alignment, Reverse and its own
//! settings, as Premiere's Change transition settings / Align transitions / Change transition
//! duration pages describe them) and the Set Transition Duration dialog a double-click on it in
//! the Timeline opens. Every change is one `sequence.setTransition`; dragging a value is one undo
//! step.
//!
//! Not here yet: the A/B preview thumbnails with their Start / End sliders, Show Actual Sources,
//! and dragging the transition in Effect Controls' own time ruler (the Timeline does that).
//!
//! Automation ids: `effectControls.transition.duration`, `.alignment` (+ `.alignment.option.<center|
//! start|end>` while open), `.reverse`, `.reset`, `.param.<id>` (a point's are `.param.<id>.x` /
//! `.y`; a list's entries `.param.<id>.option.<n>` while open); the dialog's
//! `transitionDuration.value`, `.ok`, `.cancel`.
use egui::{Rect, RichText, vec2};
use filmcraft_edit::transitions::{self as trn, Alignment};
use filmcraft_project::{ParamKind, ParamValue, Transition, TransitionId};
use filmcraft_time::FrameRate;
use serde_json::{Value, json};
use crate::FilmcraftApp;
use crate::state::TransitionDurationDraft;
type Elems = Vec<(String, Rect, String)>;
/// The selected transition, when it still exists, with its alignment and how many frames it may
/// last at most (the clips it joins).
fn lookup(app: &FilmcraftApp, id: TransitionId) -> Option<(Transition, Alignment, i64)> {
let seq = app.session.active_sequence()?;
let rate = seq.settings.frame_rate.sane();
seq.all_tracks().find_map(|tr| {
let x = tr.transitions.iter().find(|x| x.id == id)?;
let (lo, cut, hi) = trn::bounds(tr, x)?;
let max = rate.frame_at(hi - lo).max(1);
Some((x.clone(), trn::alignment(x, cut, rate.frame_duration()), max))
})
}
/// The first selected transition that still exists.
pub fn selected(app: &FilmcraftApp) -> Option<TransitionId> {
app.session.state.transition_selection.iter().copied().find(|id| lookup(app, *id).is_some())
}
fn timecode(frames: f64, rate: FrameRate, drop_frame: bool) -> String {
filmcraft_time::format_timecode_frames(frames.round() as i64, rate, drop_frame)
}
/// A duration field in frames that reads and takes timecode (or a plain frame count).
fn duration_field(ui: &mut egui::Ui, frames: &mut i64, max: i64, rate: FrameRate, drop_frame: bool) -> egui::Response {
ui.add(egui::DragValue::new(frames).range(1..=max.max(1)).speed(0.25).custom_formatter(move |v, _| timecode(v, rate, drop_frame)).custom_parser(move |s| {
match s.trim().parse::<i64>() {
Ok(n) => Some(n as f64),
Err(_) => filmcraft_time::parse_timecode(s, rate, drop_frame, 0).ok().map(|n| n as f64),
}
}))
}
/// Effect Controls for a selected transition. `rect` is the panel.
pub fn effect_controls(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, id: TransitionId) {
let t = app.tokens;
let Some((x, align, max_frames)) = lookup(app, id) else {
crate::dock::placeholder(ui, rect, &t, tl!("(no clip selected)"));
return;
};
let Some(seq) = app.session.active_sequence() else { return };
let (rate, drop_frame) = (seq.settings.frame_rate.sane(), seq.settings.drop_frame);
let (fw, fh) = (f64::from(seq.settings.width), f64::from(seq.settings.height));
let seq_name = app.session.state.active_sequence.and_then(|i| app.session.project.item(i)).map(|i| i.name.clone()).unwrap_or_default();
let def = x.effect.def();
let name = def.map_or(x.effect.effect.as_str(), |d| d.name).to_string();
let one_sided = x.from.is_none() || x.to.is_none();
let mut changes: Vec<Value> = Vec::new();
let mut elems: Elems = Vec::new();
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink2(vec2(10.0, 6.0))).id_salt(("tr-ec", id.0)));
child.label(RichText::new(format!("{seq_name} · {name}")).color(t.text_dim));
child.add_space(4.0);
egui::ScrollArea::vertical().id_salt(("tr-ec-scroll", id.0)).auto_shrink([false, false]).show(&mut child, |ui| {
egui::Grid::new(("tr-ec-grid", id.0)).num_columns(2).spacing([14.0, 8.0]).show(ui, |ui| {
ui.label(tl!("Duration"));
let mut frames = rate.frame_at(x.duration).max(1);
let r = duration_field(ui, &mut frames, max_frames, rate, drop_frame);
elems.push(("effectControls.transition.duration".into(), r.rect, timecode(frames as f64, rate, drop_frame)));
if r.changed() {
changes.push(json!({"frames": frames, "merge": true, "begin": r.drag_started() || !r.dragged()}));
}
ui.end_row();
ui.label(tl!("Alignment"));
let shown = align.label();
let r = ui
.add_enabled_ui(!one_sided, |ui| {
egui::ComboBox::from_id_salt(("tr-align", id.0))
.selected_text(shown)
.width(170.0)
.show_ui(ui, |ui| {
for a in Alignment::MENU {
let o = ui.selectable_label(a == align, a.label());
if o.clicked() && a != align {
changes.push(json!({"align": a.id()}));
}
elems.push((format!("effectControls.transition.alignment.option.{}", a.id()), o.rect, a.label().to_string()));
}
})
.response
})
.inner;
elems.push(("effectControls.transition.alignment".into(), r.rect, shown.to_string()));
ui.end_row();
ui.label(tl!("Reverse"));
let mut rev = x.reverse;
let r = ui.checkbox(&mut rev, "");
elems.push(("effectControls.transition.reverse".into(), r.rect, rev.to_string()));
if r.changed() {
changes.push(json!({"reverse": rev}));
}
ui.end_row();
for pd in def.map(|d| d.params.as_slice()).unwrap_or(&[]) {
let value = x.effect.params.get(pd.id).map_or(&pd.default, |p| &p.value);
ui.label(crate::i18n::t(pd.label));
if let Some(v) = param_widget(ui, &mut elems, pd.id, &pd.kind, value, (fw, fh)) {
changes.push(v);
}
ui.end_row();
}
});
ui.add_space(8.0);
let r = ui.button(tl!("Reset"));
elems.push(("effectControls.transition.reset".into(), r.rect, "Reset".into()));
if r.clicked() {
changes.push(json!({"reset": true}));
}
});
for (eid, r, l) in elems {
app.auto.add(&eid, r, &l);
}
for mut c in changes {
c["transition"] = json!(id.0);
if let Err(e) = app.session.execute("sequence.setTransition", c) {
app.ui.status = e.to_string();
}
}
}
/// One of a transition's own settings. Returns the change, as `sequence.setTransition` params.
fn param_widget(ui: &mut egui::Ui, elems: &mut Elems, pid: &str, kind: &ParamKind, value: &ParamValue, frame: (f64, f64)) -> Option<Value> {
let eid = format!("effectControls.transition.param.{pid}");
let drag = |r: &egui::Response| json!({"merge": true, "begin": r.drag_started() || !r.dragged()});
match (kind, value) {
(ParamKind::Float { min, max, soft_min, soft_max, unit, decimals }, ParamValue::Float(v)) => {
let mut v = *v;
let speed = ((soft_max - soft_min) / 200.0).max(0.001);
let r = ui.add(egui::DragValue::new(&mut v).range(*min..=*max).speed(speed).max_decimals(usize::from(*decimals)).suffix(*unit));
elems.push((eid, r.rect, format!("{v}")));
r.changed().then(|| {
let mut c = drag(&r);
c["params"] = json!({ pid: v });
c
})
}
(ParamKind::Angle, ParamValue::Float(v)) => {
let mut v = *v;
let r = ui.add(egui::DragValue::new(&mut v).speed(0.5).suffix("°"));
elems.push((eid, r.rect, format!("{v}")));
r.changed().then(|| {
let mut c = drag(&r);
c["params"] = json!({ pid: v });
c
})
}
(ParamKind::Bool, ParamValue::Bool(b)) => {
let mut b = *b;
let r = ui.checkbox(&mut b, "");
elems.push((eid, r.rect, b.to_string()));
r.changed().then(|| json!({"params": { pid: b }}))
}
(ParamKind::Choice(opts), ParamValue::Choice(c)) => {
let shown = opts.get(*c as usize).copied().unwrap_or("");
let mut picked = None;
let mut options = Vec::new();
let r = egui::ComboBox::from_id_salt(("tr-param", pid.to_string())).selected_text(crate::i18n::t(shown)).show_ui(ui, |ui| {
for (i, o) in opts.iter().enumerate() {
let r = ui.selectable_label(i == *c as usize, crate::i18n::t(o));
if r.clicked() && i != *c as usize {
picked = Some(i);
}
options.push((format!("{eid}.option.{i}"), r.rect, o.to_string()));
}
});
elems.extend(options);
elems.push((eid, r.response.rect, shown.to_string()));
picked.map(|i| json!({"params": { pid: i }}))
}
(ParamKind::Color, ParamValue::Color(c)) => {
let mut c = *c;
let r = ui.color_edit_button_rgba_unmultiplied(&mut c);
elems.push((eid, r.rect, format!("{c:?}")));
r.changed().then(|| {
let mut v = drag(&r);
v["params"] = json!({ pid: [c[0], c[1], c[2], c[3]] });
v
})
}
(ParamKind::Point, ParamValue::Vec2(p)) => {
// an automatic point (NaN) is the frame's centre until it is moved
let (mut px, mut py) = (if p.x.is_nan() { frame.0 / 2.0 } else { p.x }, if p.y.is_nan() { frame.1 / 2.0 } else { p.y });
let mut out = None;
ui.horizontal(|ui| {
let rx = ui.add(egui::DragValue::new(&mut px).speed(1.0).prefix("x "));
let ry = ui.add(egui::DragValue::new(&mut py).speed(1.0).prefix("y "));
elems.push((format!("{eid}.x"), rx.rect, format!("{px}")));
elems.push((format!("{eid}.y"), ry.rect, format!("{py}")));
for r in [&rx, &ry] {
if r.changed() {
let mut v = drag(r);
v["params"] = json!({ pid: [px, py] });
out = Some(v);
}
}
});
out
}
(_, v) => {
let r = ui.label(RichText::new(format!("{v:?}")).weak());
elems.push((eid, r.rect, String::new()));
None
}
}
}
/// Double-click a transition in the Timeline: Set Transition Duration.
pub fn open_duration(app: &mut FilmcraftApp, id: TransitionId) {
let Some((x, ..)) = lookup(app, id) else { return };
let rate = app.session.active_sequence().map(|q| q.settings.frame_rate.sane()).unwrap_or_default();
app.ui.transition_duration = TransitionDurationDraft { transition: id.0, frames: rate.frame_at(x.duration).max(1) };
let _ = app.session.execute("timeline.select", json!({"transitions": [id.0]}));
app.dialog = Some(crate::Dialog::TransitionDuration);
}
/// The Set Transition Duration dialog. Returns whether it stays open.
pub fn duration_dialog(app: &mut FilmcraftApp, ctx: &egui::Context) -> bool {
let mut d = app.ui.transition_duration.clone();
let id = TransitionId(d.transition);
let Some((_, _, max_frames)) = lookup(app, id) else { return false };
let Some(seq) = app.session.active_sequence() else { return false };
let (rate, drop_frame) = (seq.settings.frame_rate.sane(), seq.settings.drop_frame);
let accent = app.tokens.accent;
let (mut keep, mut apply) = (true, false);
let mut elems: Elems = Vec::new();
egui::Window::new(tl!("Set Transition Duration"))
.collapsible(false)
.resizable(false)
.default_width(260.0)
.anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0])
.show(ctx, |ui| {
ui.horizontal(|ui| {
ui.label(tl!("Duration:"));
let r = duration_field(ui, &mut d.frames, max_frames, rate, drop_frame);
elems.push(("transitionDuration.value".into(), r.rect, timecode(d.frames as f64, rate, drop_frame)));
});
ui.add_space(8.0);
// one row high: a right-to-left layout would otherwise take the window's whole height
let row = egui::vec2(ui.available_width().max(220.0), ui.spacing().interact_size.y);
ui.allocate_ui_with_layout(row, egui::Layout::right_to_left(egui::Align::Center), |ui| {
let o = ui.add(egui::Button::new(RichText::new(tl!("OK")).color(egui::Color32::WHITE)).fill(accent));
elems.push(("transitionDuration.ok".into(), o.rect, "OK".into()));
apply |= o.clicked();
let c = ui.button(tl!("Cancel"));
elems.push(("transitionDuration.cancel".into(), c.rect, "Cancel".into()));
keep &= !c.clicked();
});
});
for (eid, r, l) in elems {
app.auto.add(&eid, r, &l);
}
if ctx.input(|i| i.key_pressed(egui::Key::Escape)) {
keep = false;
}
if ctx.input(|i| i.key_pressed(egui::Key::Enter)) && !ctx.egui_wants_keyboard_input() {
apply = true;
}
if apply {
let cur = lookup(app, id).map(|(x, ..)| rate.frame_at(x.duration));
if cur != Some(d.frames)
&& let Err(e) = app.session.execute("sequence.setTransition", json!({"transition": id.0, "frames": d.frames}))
{
app.ui.status = e.to_string();
}
keep = false;
}
app.ui.transition_duration = d;
keep
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn durations_read_as_timecode() {
assert_eq!(timecode(24.0, FrameRate::FPS_24, false), "00:00:01:00");
assert_eq!(timecode(30.0, FrameRate::FPS_29_97, true), "00;00;01;00");
}
}
+465
View File
@@ -0,0 +1,465 @@
//! Text to Speech panel: write what the narrator says, pick a language, voice, vocal pitch and pace,
//! hear it, and save it as a narration clip. Selecting a narration clip on the timeline loads its
//! script and settings here (and brings the panel forward), and Save regenerates it in place.
//! Every action is a `tts.*` command, so the panel, MCP and the control channel share one path.
//!
//! Synthesis runs as a background job (`tts.render`); when it finishes, the panel runs the action
//! (preview, create or edit), which then uses the cached audio, so the window never stalls. The
//! natural voices need a one-time download that the user confirms in the panel (size, source,
//! licence); it runs as a job with progress and Cancel.
//!
//! Automation ids: `tts.language`, `tts.voice`, `tts.hearVoice`, `tts.advanced` (twirl),
//! `tts.pitch`, `tts.pace`, `tts.text`, `tts.addPause`, `tts.preview`, `tts.save`, `tts.new`,
//! `tts.download`, `tts.download.confirm`, `tts.download.cancel`.
use egui::{Align2, Rect, Sense, Stroke, StrokeKind, pos2, vec2};
use filmcraft_project::{ClipId, VocalPitch};
use serde::{Deserialize, Serialize};
use serde_json::json;
use crate::FilmcraftApp;
use crate::icons::{self, Icon};
use crate::theme::Tokens;
const PAD: f32 = 14.0;
const FOOTER_H: f32 = 48.0;
/// What the panel is writing (UI state, serde so agents can read it).
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub struct TtsDraft {
pub text: String,
pub language: String,
pub voice: String,
pub pitch: VocalPitch,
pub pace: f64,
pub advanced_open: bool,
/// The narration clip and item the draft was loaded from (None: a new narration).
pub loaded_from: Option<(u64, u64)>,
/// An action waiting for its synthesis job.
pub pending: Option<Pending>,
/// The download confirmation is showing.
pub confirm_download: bool,
/// The running natural-voice download job.
pub download_job: Option<u64>,
}
/// A command to run when the synthesis job `job` finishes.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Pending {
pub cmd: String,
pub params: serde_json::Value,
/// Play the preview afterwards.
pub play: bool,
pub job: u64,
}
impl Default for TtsDraft {
fn default() -> Self {
TtsDraft {
text: String::new(),
language: "en-US".into(),
voice: if natural_installed() { "kokoro-heart".into() } else { filmcraft_tts::default_voice_id().into() },
pitch: VocalPitch::Default,
pace: 1.0,
advanced_open: true,
loaded_from: None,
pending: None,
confirm_download: false,
download_job: None,
}
}
}
impl TtsDraft {
fn params(&self) -> serde_json::Value {
json!({"text": self.text, "voice": self.voice, "pitch": self.pitch.id(), "pace": self.pace})
}
}
/// The natural voice package is downloaded.
fn natural_installed() -> bool {
filmcraft_engine::transcript::models_dir().as_deref().is_some_and(filmcraft_tts::catalog::installed)
}
/// State of job `id`: `None` while running, else its result.
fn job_outcome(app: &FilmcraftApp, id: u64) -> Option<Result<(), String>> {
use std::sync::atomic::Ordering;
let Some(j) = app.session.jobs.iter().find(|j| j.id == id) else { return Some(Err("the job disappeared".into())) };
if !j.progress.finished.load(Ordering::Relaxed) {
return None;
}
Some(match &*j.result.lock().unwrap_or_else(std::sync::PoisonError::into_inner) {
Some(Ok(_)) => Ok(()),
Some(Err(e)) => Err(e.clone()),
None => Err("the job stopped".into()),
})
}
/// Start an action: synthesize in the background (or reuse the cache), then run `cmd`.
fn start(app: &mut FilmcraftApp, cmd: &str, params: serde_json::Value, play: bool) {
let mut render = params.clone();
if let Some(o) = render.as_object_mut() {
o.remove("dir");
o.remove("time");
}
match app.session.execute("tts.render", render) {
Ok(r) => match r["job"].as_u64() {
Some(job) => {
app.ui.tts.pending = Some(Pending { cmd: cmd.to_string(), params, play, job });
app.ui.status = "Synthesizing narration…".into();
}
None => run_now(app, cmd, params, play),
},
Err(e) => app.ui.status = e.to_string(),
}
}
fn run_now(app: &mut FilmcraftApp, cmd: &str, params: serde_json::Value, play: bool) {
match app.session.execute(cmd, params) {
Ok(r) => {
if play {
if let Err(e) = app.play_tts_preview() {
app.ui.status = e;
}
} else {
app.ui.status = match cmd {
"tts.create" => format!("Narration added on {}", r["track"].as_str().unwrap_or("an audio track")),
_ => "Narration updated".into(),
};
}
}
Err(e) => app.ui.status = e.to_string(),
}
}
/// Finish a pending action whose synthesis job is done.
fn poll(app: &mut FilmcraftApp) {
if let Some(p) = app.ui.tts.pending.clone() {
match job_outcome(app, p.job) {
None => {}
Some(Ok(())) => {
app.ui.tts.pending = None;
run_now(app, &p.cmd, p.params, p.play);
}
Some(Err(e)) => {
app.ui.tts.pending = None;
app.ui.status = format!("Text to Speech: {e}");
}
}
}
if let Some(j) = app.ui.tts.download_job
&& let Some(r) = job_outcome(app, j)
{
app.ui.tts.download_job = None;
app.ui.status = match r {
Ok(()) => "Natural voices installed".into(),
Err(e) => format!("Download of the natural voices failed: {e}"),
};
}
}
/// The selected narration clip and its item, if any.
fn selected_narration(app: &FilmcraftApp) -> Option<(ClipId, u64)> {
let clip = filmcraft_engine::narration::target_clip(&app.session, &serde_json::Value::Null)?;
let item = app.session.active_sequence()?.find_item(clip)?.1.item;
Some((clip, item.0))
}
/// Load the selected narration clip into the draft when the selection moves to a different one,
/// and bring the panel forward. Runs every frame, also when the panel is hidden.
pub fn follow_selection(app: &mut FilmcraftApp) {
poll(app);
let sel = selected_narration(app);
match sel {
Some((clip, item)) if app.ui.tts.loaded_from != Some((clip.0, item)) => {
let first_time = app.ui.tts.loaded_from.map(|(c, _)| c) != Some(clip.0);
if let Some(n) = app.session.project.narrations.get(&filmcraft_project::ItemId(item)).cloned() {
let d = &mut app.ui.tts;
d.text = n.text;
d.language = n.language;
d.voice = n.voice;
d.pitch = n.pitch;
d.pace = n.pace;
d.loaded_from = Some((clip.0, item));
if first_time {
app.show_panel(crate::dock::PanelKind::TextToSpeech);
}
}
}
None if app.ui.tts.loaded_from.is_some() => {
// deselected: keep the text so it can be saved as a new narration
app.ui.tts.loaded_from = None;
}
_ => {}
}
}
enum Action {
Run(&'static str, serde_json::Value, bool),
New,
Download,
CancelDownload,
}
pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
let t = app.tokens;
let painter = ui.painter().clone();
painter.rect_filled(rect, 0.0, t.panel_bg);
let models = filmcraft_engine::transcript::models_dir();
let voices = filmcraft_tts::voices_in(models.as_deref());
let installed = models.as_deref().is_some_and(filmcraft_tts::catalog::installed);
let busy = app.ui.tts.pending.is_some();
let download = app
.ui
.tts
.download_job
.and_then(|id| app.session.jobs.iter().find(|j| j.id == id))
.map(|j| (j.progress.fraction(), j.progress.status.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone()));
let editing = app.ui.tts.loaded_from.map(|(c, _)| c);
let mut actions: Vec<Action> = Vec::new();
let auto = &mut app.auto;
let d = &mut app.ui.tts;
let body = Rect::from_min_max(pos2(rect.min.x + PAD, rect.min.y + 8.0), pos2(rect.max.x - PAD, rect.max.y - FOOTER_H));
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(body).id_salt("tts-body"));
egui::ScrollArea::vertical().auto_shrink([false, false]).show(&mut child, |ui| {
ui.set_max_width(ui.available_width() - 6.0);
let w = ui.available_width();
// mode line: editing a clip, or writing a new narration
ui.horizontal(|ui| {
let mode = if editing.is_some() { "Editing the selected narration" } else { "New narration" };
ui.label(egui::RichText::new(mode).color(t.text_dim).size(11.5));
if editing.is_some() {
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
let b = ui.small_button("New");
auto.add("tts.new", visible(ui, b.rect), "New narration");
if b.clicked() {
actions.push(Action::New);
}
});
}
});
ui.add_space(6.0);
label(ui, &t, "Language");
let lang_name = filmcraft_tts::LANGUAGES.iter().find(|(id, _)| *id == d.language).map_or(d.language.as_str(), |(_, n)| n);
let cb = egui::ComboBox::from_id_salt("tts-language").selected_text(lang_name).width(w).show_ui(ui, |ui| {
for (id, name) in filmcraft_tts::LANGUAGES {
if ui.selectable_label(d.language == *id, *name).clicked() {
d.language = id.to_string();
}
}
});
auto.add("tts.language", visible(ui, cb.response.rect), "Language");
ui.add_space(10.0);
label(ui, &t, "Voice");
let voice_name = voices.iter().find(|v| v.id == d.voice).map_or(d.voice.as_str(), |v| v.name);
let cb = egui::ComboBox::from_id_salt("tts-voice").selected_text(voice_name).width(w).show_ui(ui, |ui| {
for (engine, heading) in [("neural", "Natural voices"), ("basic", "Basic voices (built in)")] {
ui.label(egui::RichText::new(heading).color(t.text_faint).size(11.0));
for v in voices.iter().filter(|v| v.language == d.language && v.engine == engine) {
let label = if v.engine == "neural" && !v.installed { format!("{} (download)", v.name) } else { v.name.to_string() };
let r = ui.selectable_label(d.voice == v.id, label).on_hover_text(v.description);
if r.clicked() {
d.voice = v.id.to_string();
}
}
}
});
auto.add("tts.voice", visible(ui, cb.response.rect), "Voice");
let natural = filmcraft_tts::catalog::find(&d.voice).is_some();
if natural && !installed {
ui.add_space(6.0);
egui::Frame::new().fill(t.field_bg).stroke(egui::Stroke::new(1.0, t.field_border)).corner_radius(4.0).inner_margin(8.0).show(ui, |ui| {
ui.set_width(w - 18.0);
match &download {
Some((frac, status)) => {
ui.label(egui::RichText::new(status).color(t.text).size(11.5));
ui.add(egui::ProgressBar::new(*frac).show_percentage());
let b = ui.small_button("Cancel");
auto.add("tts.download.cancel", visible(ui, b.rect), "Cancel download");
if b.clicked() {
actions.push(Action::CancelDownload);
}
}
None if d.confirm_download => {
let mb = filmcraft_tts::catalog::size() as f64 / 1e6;
ui.label(egui::RichText::new(format!("Download the natural voices? {mb:.0} MB, once.")).color(t.text).size(12.0).strong());
ui.label(
egui::RichText::new(format!(
"From {} and github.com/cmusphinx/cmudict (pronunciations). Licence: {}. Every file is checked against a pinned SHA-256.",
filmcraft_tts::catalog::SOURCE,
filmcraft_tts::catalog::LICENSE
))
.color(t.text_dim)
.size(11.0),
);
ui.horizontal(|ui| {
let ok = ui.button("Download");
auto.add("tts.download.confirm", visible(ui, ok.rect), "Download");
let no = ui.button("Cancel");
if ok.clicked() {
actions.push(Action::Download);
}
if no.clicked() {
d.confirm_download = false;
}
});
}
None => {
ui.label(egui::RichText::new("Natural voices need a one-time download.").color(t.text).size(11.5));
let b = ui.button(format!("Download natural voices ({:.0} MB)…", filmcraft_tts::catalog::size() as f64 / 1e6));
auto.add("tts.download", visible(ui, b.rect), "Download natural voices");
if b.clicked() {
d.confirm_download = true;
}
}
}
});
}
let voice_ready = !natural || installed;
ui.add_space(4.0);
let (r, resp) = ui.allocate_exact_size(vec2(130.0, 22.0), Sense::click());
let col = if resp.hovered() { t.accent_hover } else { t.accent };
icons::paint(ui.painter(), Rect::from_center_size(pos2(r.min.x + 8.0, r.center().y), vec2(14.0, 14.0)), Icon::Play, col);
ui.painter().text(pos2(r.min.x + 22.0, r.center().y), Align2::LEFT_CENTER, "Hear this voice", Tokens::semibold(12.0), col);
auto.add("tts.hearVoice", visible(ui, r), "Hear this voice");
if resp.clicked() && voice_ready && !busy {
actions.push(Action::Run("tts.preview", json!({"sample": true, "voice": d.voice, "pitch": d.pitch.id(), "pace": d.pace}), true));
}
ui.add_space(8.0);
let (hr, open) = crate::widgets::section_header(ui, egui::Id::new("tts-advanced"), "Advanced", d.advanced_open, &t, true);
auto.add("tts.advanced", visible(ui, hr.rect), "Advanced");
d.advanced_open = open;
if open {
ui.add_space(4.0);
label(ui, &t, "Vocal pitch");
let cb = egui::ComboBox::from_id_salt("tts-pitch").selected_text(d.pitch.label()).width(w).show_ui(ui, |ui| {
for p in VocalPitch::ALL {
if ui.selectable_label(d.pitch == p, p.label()).clicked() {
d.pitch = p;
}
}
});
auto.add("tts.pitch", visible(ui, cb.response.rect), "Vocal pitch");
ui.add_space(10.0);
label(ui, &t, "Pace");
ui.spacing_mut().slider_width = w - 56.0;
let s = ui.add(egui::Slider::new(&mut d.pace, filmcraft_tts::MIN_PACE..=filmcraft_tts::MAX_PACE).step_by(0.05).fixed_decimals(2).suffix("×"));
auto.add("tts.pace", visible(ui, s.rect), "Pace");
let (r, _) = ui.allocate_exact_size(vec2(w - 56.0, 14.0), Sense::hover());
for (x, l) in [(0.0, "0.5×"), (1.0 / 3.0, "1×"), (1.0, "2×")] {
let px = r.min.x + 6.0 + (r.width() - 12.0) * x;
ui.painter().text(pos2(px, r.center().y), Align2::CENTER_CENTER, l, Tokens::ui(10.5), t.text_faint);
}
}
ui.add_space(12.0);
ui.horizontal(|ui| {
ui.label(egui::RichText::new("Text").color(t.text).size(12.5).strong());
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
let (ir, i) = ui.allocate_exact_size(vec2(14.0, 14.0), Sense::hover());
icons::paint(ui.painter(), ir, Icon::Info, t.text_faint);
i.on_hover_text("Add pause inserts [pause 1s]. Change the number for a longer or shorter pause, e.g. [pause 500ms] or [pause 2s].");
let b = ui.small_button("Add pause");
auto.add("tts.addPause", visible(ui, b.rect), "Add pause");
if b.clicked() {
if !d.text.is_empty() && !d.text.ends_with(' ') {
d.text.push(' ');
}
d.text.push_str(filmcraft_tts::script::PAUSE_MARKER);
d.text.push(' ');
}
});
});
ui.add_space(4.0);
let te = egui::TextEdit::multiline(&mut d.text)
.hint_text("Type what you want the voice to say")
.desired_width(w)
.desired_rows(8)
.char_limit(filmcraft_tts::MAX_TEXT_BYTES);
let r = ui.add(te);
auto.add("tts.text", visible(ui, r.rect), "Text");
});
// footer: Preview | Save
let fy = rect.max.y - FOOTER_H + 10.0;
let half = (rect.width() - PAD * 2.0 - 10.0) / 2.0;
let natural = filmcraft_tts::catalog::find(&d.voice).is_some();
let can_say = filmcraft_tts::check_text(&d.text).is_ok() && (!natural || installed) && !busy;
let pr = Rect::from_min_size(pos2(rect.min.x + PAD, fy), vec2(half, 28.0));
if button(ui, &t, pr, Icon::Play, if busy { "Synthesizing…" } else { "Preview" }, false, can_say, "tts-preview").clicked() && can_say {
actions.push(Action::Run("tts.preview", d.params(), true));
}
auto.add("tts.preview", pr, "Preview");
let sr = Rect::from_min_size(pos2(pr.max.x + 10.0, fy), vec2(half, 28.0));
let can_save = can_say && app.session.active_sequence().is_some();
if button(ui, &t, sr, Icon::Plus, if editing.is_some() { "Save" } else { "Add to timeline" }, true, can_save, "tts-save").clicked() && can_save {
let mut p = d.params();
match editing {
Some(c) => {
p["clip"] = json!(c);
actions.push(Action::Run("tts.edit", p, false));
}
None => actions.push(Action::Run("tts.create", p, false)),
}
}
auto.add("tts.save", sr, "Save");
for a in actions {
match a {
Action::New => {
let keep = (app.ui.tts.language.clone(), app.ui.tts.voice.clone(), app.ui.tts.pitch, app.ui.tts.pace);
app.session.state.selection.clear();
let dl = app.ui.tts.download_job;
app.ui.tts = TtsDraft { language: keep.0, voice: keep.1, pitch: keep.2, pace: keep.3, download_job: dl, ..Default::default() };
}
Action::Run(cmd, p, play) => start(app, cmd, p, play),
Action::Download => {
app.ui.tts.confirm_download = false;
match app.session.execute("tts.downloadVoices", json!({})) {
Ok(r) => app.ui.tts.download_job = r["job"].as_u64(),
Err(e) => app.ui.status = e.to_string(),
}
}
Action::CancelDownload => {
if let Some(id) = app.ui.tts.download_job
&& let Some(j) = app.session.jobs.iter().find(|j| j.id == id)
{
j.progress.cancel.store(true, std::sync::atomic::Ordering::Relaxed);
}
}
}
}
}
/// The part of `r` the scroll area shows (what a click can reach).
fn visible(ui: &egui::Ui, r: Rect) -> Rect {
r.intersect(ui.clip_rect())
}
fn label(ui: &mut egui::Ui, t: &Tokens, text: &str) {
ui.label(egui::RichText::new(text).color(t.text).size(12.5).strong());
ui.add_space(2.0);
}
#[allow(clippy::too_many_arguments)]
fn button(ui: &mut egui::Ui, t: &Tokens, r: Rect, icon: Icon, text: &str, primary: bool, active: bool, id: &str) -> egui::Response {
let resp = ui.interact(r, egui::Id::new(id), Sense::click());
let p = ui.painter();
let (bg, fg) = match (primary, active) {
(true, true) => (if resp.hovered() { t.accent_hover } else { t.accent }, egui::Color32::WHITE),
(_, false) => (t.field_bg, t.text_faint),
(false, true) => (if resp.hovered() { t.hover } else { t.field_bg }, t.text),
};
p.rect_filled(r, 4.0, bg);
if !primary {
p.rect_stroke(r, 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside);
}
let g = p.layout_no_wrap(text.to_string(), Tokens::semibold(12.5), fg);
let total = 18.0 + g.size().x;
let x0 = r.center().x - total / 2.0;
icons::paint(p, Rect::from_center_size(pos2(x0 + 7.0, r.center().y), vec2(14.0, 14.0)), icon, fg);
p.galley(pos2(x0 + 18.0, r.center().y - g.size().y / 2.0), g, fg);
resp
}
+17
View File
@@ -411,6 +411,9 @@ pub struct UiState {
/// Essential Sound sub-tab: "Edit" or "Browse".
#[serde(default)]
pub essential_sound_tab: String,
/// Text to Speech panel: the script and voice settings being written or edited.
#[serde(default)]
pub tts: crate::panels::tts::TtsDraft,
/// Export mode: settings, preset, destination, range, the Preset Manager and Quick Export.
#[serde(default)]
pub export: crate::panels::export_mode::ExportUi,
@@ -454,6 +457,9 @@ pub struct UiState {
/// Sequence Settings dialog draft.
#[serde(default)]
pub sequence_settings: SequenceSettingsDraft,
/// Set Transition Duration dialog draft (double-click a transition).
#[serde(default)]
pub transition_duration: TransitionDurationDraft,
/// On-monitor text editing (Type tool / double-click on a text layer).
#[serde(default)]
pub gfx_edit: Option<GfxEdit>,
@@ -757,6 +763,15 @@ impl Default for AddTracksDraft {
}
}
/// The Set Transition Duration dialog (double-click a transition in the Timeline): which
/// transition, and its duration in frames.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct TransitionDurationDraft {
pub transition: u64,
pub frames: i64,
}
/// The Sequence Settings dialog (Sequence ▸ Sequence Settings…), filled from the active sequence
/// when it opens (`panels::sequence_settings::open`). `tab`: `general`, `color` or `vr`; the
/// timebase is `fps_num`/`fps_den`; `mix`: `Stereo`, `Mono`, `5.1` or `Adaptive`; `working_space`:
@@ -890,6 +905,7 @@ impl Default for UiState {
status: String::new(),
eyedropper: None,
essential_sound_tab: "Edit".into(),
tts: Default::default(),
export: Default::default(),
text_tab: captions_tab(),
caption_search: String::new(),
@@ -904,6 +920,7 @@ impl Default for UiState {
add_tracks: AddTracksDraft::default(),
delete_tracks: DeleteTracksDraft::default(),
sequence_settings: SequenceSettingsDraft::default(),
transition_duration: TransitionDurationDraft::default(),
gfx_edit: None,
pen_points: vec![],
link_media: None,
+78
View File
@@ -152,3 +152,81 @@ fn type_tool_edit_style_and_move() {
d.ok("ui.set", json!({"tool": "Selection"}));
d.shot("shape-layer");
}
/// The value of `param` on the text layer of graphic clip `clip`, at its first frame.
fn text_param(d: &mut Driver, clip: u64, param: &str) -> Option<filmcraft_project::ParamValue> {
let s = &d.harness.state().session;
let (_, it) = s.active_sequence()?.find_item(filmcraft_project::ClipId(clip))?;
let e = it.effects.iter().find(|e| e.effect == "graphic_text")?;
Some(e.param(param)?.value_at(filmcraft_time::Tick::ZERO))
}
/// A drag of a property in the Properties panel is one undo step, so one Cmd+Z restores the
/// original value (a drag was committing one undo step per frame, so undo only stepped back one
/// frame's worth; the shadow color picker showed it as "sometimes undo works, sometimes not").
#[test]
fn a_property_drag_is_one_undo_step() {
let mut d = Driver::demo();
d.ok("ui.set", json!({"workspace": "Captions and Graphics"}));
let r = d.exec("graphics.newText", json!({"text": "Title"}));
let clip = r["clip"].as_u64().unwrap();
d.exec("graphics.selectLayer", json!({"clip": clip, "layers": [0]}));
d.frames(4);
// the layer's Opacity number (visible in the Align and Transform section)
d.exec("graphics.set", json!({"clip": clip, "layer": 0, "props": {"opacity": 50}}));
d.frames(2);
let opacity = |d: &mut Driver| -> f64 { text_param(d, clip, "opacity").and_then(|v| v.as_f64()).unwrap() };
let before = opacity(&mut d);
let undo0 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len();
// drag the Opacity number to the left: several move frames, one undo step
let n = d.rect("graphics.prop.opacity");
let (cx, cy) = (n[0] + n[2] / 2.0, n[1] + n[3] / 2.0);
d.ok("ui.drag", json!({"from": {"x": cx, "y": cy}, "to": {"x": cx - 60.0, "y": cy}, "steps": 6}));
d.frames(4);
let after = opacity(&mut d);
assert!(after < before - 5.0, "the drag changed the opacity: {before} -> {after}");
let undo1 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len();
assert_eq!(undo1, undo0 + 1, "a property drag is one undo step");
d.exec("edit.undo", json!({}));
assert_eq!(opacity(&mut d), before, "one undo restores the original value");
}
/// The shadow color picker popup: a drag in the 2D picker is one undo step, so one Cmd+Z
/// restores the original color (a drag was committing one undo step per frame, so undo only
/// stepped back one frame's worth — the reported "sometimes undo works, sometimes not").
#[test]
fn shadow_color_drag_is_one_undo_step() {
let mut d = Driver::demo();
d.ok("ui.set", json!({"workspace": "Captions and Graphics"}));
let r = d.exec("graphics.newText", json!({"text": "Title"}));
let clip = r["clip"].as_u64().unwrap();
d.exec("graphics.selectLayer", json!({"clip": clip, "layers": [0]}));
d.frames(4);
let color = |d: &mut Driver| text_param(d, clip, "shadow_color").and_then(|v| v.as_color());
// scroll the properties panel so the shadow row is visible, then open the picker
d.ok("ui.scroll", json!({"x": 1400.0, "y": 500.0, "dx": 0.0, "dy": -400.0}));
d.frames(3);
let sw = d.rect("graphics.prop.shadow_color");
d.ok("ui.click", json!({"x": sw[0] + sw[2] / 2.0, "y": sw[1] + sw[3] / 2.0}));
d.frames(3);
// the popup flips above the swatch (it does not fit below); find a point in its 2D picker
let undo0 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len();
let mut hit = None;
for (dx, dy) in [(30.0, 30.0), (150.0, 30.0), (30.0, 150.0), (150.0, 150.0)] {
let (px, py) = (sw[0] + dx, sw[1] - dy);
let before = color(&mut d);
d.ok("ui.drag", json!({"from": {"x": px, "y": py}, "to": {"x": px + 60.0, "y": py + 40.0}, "steps": 4}));
d.frames(3);
let after = color(&mut d);
if after != before {
hit = Some((before, after));
break;
}
}
let (before, after) = hit.expect("no drag point hit the color picker");
assert_ne!(after, before, "the drag changed the shadow color");
let undo1 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len();
assert_eq!(undo1, undo0 + 1, "a color drag is one undo step");
d.exec("edit.undo", json!({}));
assert_eq!(color(&mut d), before, "one undo restores the original color");
}
+25 -1
View File
@@ -16,6 +16,7 @@ struct FakeOut {
played: Arc<AtomicU64>,
per_poll: u64,
started: bool,
note: Option<String>,
}
impl AudioOut for FakeOut {
@@ -33,6 +34,9 @@ impl AudioOut for FakeOut {
fn played_frames(&self) -> Option<u64> {
self.started.then(|| self.played.fetch_add(self.per_poll, Ordering::SeqCst) + self.per_poll)
}
fn note(&self) -> Option<String> {
self.note.clone()
}
}
/// Play for `steps` × 0.25 s of simulated time with the given output; the playhead in seconds.
@@ -41,7 +45,7 @@ fn play_with(per_poll: u64, steps: usize) -> (f64, bool) {
s.execute("file.openDemoProject", json!({})).unwrap();
s.set_playhead(Tick::ZERO);
let mut app = FilmcraftApp::new(s);
app.audio = Some(Box::new(FakeOut { played: Arc::new(AtomicU64::new(0)), per_poll, started: false }));
app.audio = Some(Box::new(FakeOut { played: Arc::new(AtomicU64::new(0)), per_poll, started: false, note: None }));
let mut h = Harness::builder().with_size(egui::vec2(1280.0, 800.0)).with_step_dt(0.25).build_eframe(move |_cc| app);
h.step();
h.state_mut().play(1.0);
@@ -68,3 +72,23 @@ fn a_playing_audio_device_drives_the_clock() {
assert!(t > 0.2, "{t:.2} s");
assert!(audio_clock);
}
/// A fallback note from the audio device (broken ALSA `default`, #23/#106) reaches the status
/// bar on the first frame, and survives while playback runs.
#[test]
fn a_fallback_note_from_the_audio_device_reaches_the_status_bar() {
let mut s = Session::default();
s.execute("file.openDemoProject", json!({})).unwrap();
s.set_playhead(Tick::ZERO);
let note = "Audio hardware: the default output device cannot be opened; using 'sofhdadsp'".to_string();
let mut app = FilmcraftApp::new(s);
app.audio = Some(Box::new(FakeOut { played: Arc::new(AtomicU64::new(0)), per_poll: 4_800, started: false, note: Some(note.clone()) }));
let mut h = Harness::builder().with_size(egui::vec2(1280.0, 800.0)).with_step_dt(0.25).build_eframe(move |_cc| app);
h.step();
assert_eq!(h.state_mut().ui.status, note, "the note surfaces on the first frame (apply_prefs)");
h.state_mut().play(1.0);
for _ in 0..16 {
h.step();
}
assert_eq!(h.state_mut().ui.status, note, "the note survives while playback runs");
}
+154
View File
@@ -0,0 +1,154 @@
//! Files dropped from the OS onto the Timeline. With Preferences ▸ Timeline ▸ "Place files dropped
//! onto the Timeline directly on the Timeline" (`drop_import_to_timeline`) they are also placed at
//! the pointer; with it off they are only imported to the bin.
use std::path::{Path, PathBuf};
use egui_kittest::Harness;
use filmcraft_engine::Session;
use filmcraft_ui_egui::FilmcraftApp;
use serde_json::json;
fn step(h: &mut Harness<'static, FilmcraftApp>) {
let ctx = h.ctx.clone();
let mut raw = std::mem::take(h.input_mut());
eframe::App::raw_input_hook(h.state_mut(), &ctx, &mut raw);
*h.input_mut() = raw;
ctx.request_repaint();
h.step();
}
fn harness() -> Harness<'static, FilmcraftApp> {
let mut s = Session::default();
s.execute("file.openDemoProject", json!({})).unwrap();
s.execute("file.newSequence", json!({"name":"Drop test","width":1280,"height":720,"fps":24})).unwrap();
let mut h = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).build_eframe(move |_cc| FilmcraftApp::new(s));
for _ in 0..4 {
step(&mut h);
}
h
}
fn pointer_to(h: &mut Harness<'static, FilmcraftApp>, pos: egui::Pos2) {
h.input_mut().events.push(egui::Event::PointerMoved(pos));
step(h);
}
fn control_rect(h: &Harness<'static, FilmcraftApp>, id: &str) -> egui::Rect {
let [x, y, w, height] = h.state().auto.elements.iter().find(|e| e.id == id).or_else(|| h.state().auto.find(id)).unwrap().rect;
egui::Rect::from_min_size(egui::pos2(x, y), egui::vec2(w, height))
}
/// A point 60 px into the Timeline's clip area on the named track row.
fn point_on_row(h: &Harness<'static, FilmcraftApp>, row_control: &str) -> egui::Pos2 {
let row = control_rect(h, row_control);
let panel = control_rect(h, "panel.Timeline");
egui::pos2(panel.min.x + h.state().ui.timeline.header_w + 60.0, row.center().y)
}
/// A file the OS dropped on the window, as the native integration hands it to egui.
#[derive(Debug)]
struct OsFile(PathBuf);
impl egui::DroppedFile for OsFile {
fn path(&self) -> &Path {
&self.0
}
fn bytes(&self) -> Result<Vec<u8>, String> {
std::fs::read(&self.0).map_err(|e| e.to_string())
}
}
/// Drops `path` with the pointer over `row_control`, as the OS does: the pointer is hovering first,
/// then one frame carries the dropped file. The file is cleared afterwards, as egui only reports it once.
fn drop_on_row(h: &mut Harness<'static, FilmcraftApp>, row_control: &str, path: &Path) {
let at = point_on_row(h, row_control);
pointer_to(h, at);
h.input_mut().dropped_files.push(std::sync::Arc::new(OsFile(path.to_path_buf())));
step(h);
h.input_mut().dropped_files.clear();
step(h);
}
/// (video clips, audio clips) on the active sequence.
fn clip_counts(h: &Harness<'static, FilmcraftApp>) -> (usize, usize) {
let q = h.state().session.active_sequence().unwrap();
(q.video_tracks.iter().map(|t| t.items.len()).sum(), q.audio_tracks.iter().map(|t| t.items.len()).sum())
}
fn fixture(name: &str) -> PathBuf {
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/ui-egui");
std::fs::create_dir_all(&dir).unwrap();
dir.join(name)
}
/// A small generated PNG still.
fn still(name: &str) -> PathBuf {
let p = fixture(name);
image::RgbaImage::from_pixel(64, 36, image::Rgba([40, 140, 220, 255])).save(&p).unwrap();
p
}
/// A 0.5 s mono 48 kHz 16-bit WAV tone (audio only).
fn tone(name: &str) -> PathBuf {
let p = fixture(name);
let rate = 48_000u32;
let n = rate / 2;
let mut data = Vec::with_capacity(n as usize * 2);
for i in 0..n {
let s = (f32::sin(2.0 * std::f32::consts::PI * 440.0 * i as f32 / rate as f32) * 0.5 * i16::MAX as f32) as i16;
data.extend_from_slice(&s.to_le_bytes());
}
let mut wav = Vec::with_capacity(44 + data.len());
wav.extend_from_slice(b"RIFF");
wav.extend_from_slice(&(36 + data.len() as u32).to_le_bytes());
wav.extend_from_slice(b"WAVEfmt ");
wav.extend_from_slice(&16u32.to_le_bytes());
wav.extend_from_slice(&1u16.to_le_bytes()); // PCM
wav.extend_from_slice(&1u16.to_le_bytes()); // mono
wav.extend_from_slice(&rate.to_le_bytes());
wav.extend_from_slice(&(rate * 2).to_le_bytes());
wav.extend_from_slice(&2u16.to_le_bytes());
wav.extend_from_slice(&16u16.to_le_bytes());
wav.extend_from_slice(b"data");
wav.extend_from_slice(&(data.len() as u32).to_le_bytes());
wav.extend_from_slice(&data);
std::fs::write(&p, wav).unwrap();
p
}
#[test]
fn still_dropped_on_the_video_row_is_placed_at_the_pointer() {
let mut h = harness();
let items_before = h.state().session.project.items.len();
drop_on_row(&mut h, "timeline.track.V1.target", &still("drop_still_video_row.png"));
assert_eq!(h.state().session.project.items.len(), items_before + 1, "the still is imported to the bin");
assert_eq!(clip_counts(&h), (1, 0), "status: {}", h.state().ui.status);
}
#[test]
fn still_dropped_on_an_audio_row_goes_to_the_video_track_without_error() {
// a video-only item over an audio row: the picture takes the matching video track, no error
let mut h = harness();
drop_on_row(&mut h, "timeline.track.A1.target", &still("drop_still_audio_row.png"));
assert_eq!(clip_counts(&h), (1, 0), "status: {}", h.state().ui.status);
}
#[test]
fn tone_dropped_on_a_video_row_goes_to_the_audio_track_without_error() {
// an audio-only item over a video row: the sound takes the matching audio track, no error
let mut h = harness();
drop_on_row(&mut h, "timeline.track.V1.target", &tone("drop_tone_video_row.wav"));
assert_eq!(clip_counts(&h), (0, 1), "status: {}", h.state().ui.status);
}
#[test]
fn with_drop_import_off_the_file_is_only_imported() {
let mut h = harness();
h.state_mut().session.prefs.timeline.drop_import_to_timeline = false;
let items_before = h.state().session.project.items.len();
drop_on_row(&mut h, "timeline.track.V1.target", &still("drop_still_pref_off.png"));
assert_eq!(h.state().session.project.items.len(), items_before + 1, "the still is imported to the bin");
assert_eq!(clip_counts(&h), (0, 0), "status: {}", h.state().ui.status);
}
+276
View File
@@ -0,0 +1,276 @@
//! Headless UI tests of applied transitions in the Timeline (#430, #224): click one to select it
//! and see it in Effect Controls, Delete it, drag its end or its middle, double-click it for Set
//! Transition Duration. The real `FilmcraftApp` under `egui_kittest`, driven over the control
//! channel by automation id.
//!
//! Set `FILMCRAFT_UI_SNAPSHOT_DIR=<dir>` to also render the window offscreen with wgpu and write
//! `transition-*.png` there; without it no GPU is needed.
use std::sync::mpsc::{Sender, channel};
use egui_kittest::Harness;
use filmcraft_engine::Session;
use filmcraft_ui_egui::FilmcraftApp;
use filmcraft_ui_egui::control::ControlRequest;
use serde_json::{Value, json};
struct Driver {
harness: Harness<'static, FilmcraftApp>,
tx: Sender<ControlRequest>,
snapshots: Option<std::path::PathBuf>,
}
/// The demo's first video transition between two clips.
#[derive(Clone, Copy, Debug)]
struct Crossing {
id: u64,
start: i64,
duration: i64,
cut: i64,
}
impl Driver {
fn demo() -> Self {
let mut session = Session::default();
session.execute("file.openDemoProject", json!({})).expect("demo project");
let (tx, rx) = channel();
let app = FilmcraftApp::new(session).with_control(rx);
let snapshots = std::env::var_os("FILMCRAFT_UI_SNAPSHOT_DIR").map(std::path::PathBuf::from);
// 60 fps steps, so a double-click's two clicks fall within egui's double-click time
let mut b = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_step_dt(1.0 / 60.0).with_max_steps(10_000);
if snapshots.is_some() {
b = b.wgpu();
}
let harness = b.build_eframe(move |_cc| app);
let mut d = Driver { harness, tx, snapshots };
d.frames(4);
d.ok("ui.panel.show", json!({"panel": "EffectControls"}));
d.frames(3);
d
}
fn frames(&mut self, n: usize) {
for _ in 0..n {
let ctx = self.harness.ctx.clone();
let mut raw = std::mem::take(self.harness.input_mut());
eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw);
*self.harness.input_mut() = raw;
self.harness.step();
}
}
fn call(&mut self, method: &str, params: Value) -> Value {
let (req, reply) = ControlRequest::new(method, params.clone());
self.tx.send(req).unwrap();
for _ in 0..600 {
self.frames(1);
if let Ok(v) = reply.try_recv() {
return v;
}
}
panic!("no reply to {method} {params}");
}
fn ok(&mut self, method: &str, params: Value) -> Value {
let v = self.call(method, params.clone());
assert_eq!(v["ok"], json!(true), "{method} {params} failed: {v}");
v["result"].clone()
}
fn exec(&mut self, command: &str, params: Value) -> Value {
self.ok("engine.execute", json!({"command": command, "params": params}))
}
fn find(&mut self, id: &str) -> Option<(f64, f64, f64, f64, String)> {
self.frames(2);
let v = self.ok("ui.elements", json!({"prefix": id}));
let e = v.as_array().unwrap().iter().find(|e| e["id"] == id)?.clone();
let r: Vec<f64> = e["rect"].as_array().unwrap().iter().map(|x| x.as_f64().unwrap()).collect();
Some((r[0], r[1], r[2], r[3], e["label"].as_str().unwrap_or_default().to_string()))
}
/// With `FILMCRAFT_UI_SNAPSHOT_DIR` set, render the whole window offscreen to `<name>.png`.
fn snapshot(&mut self, name: &str) {
let Some(dir) = self.snapshots.clone() else { return };
self.frames(2);
match self.harness.render() {
Ok(img) => {
std::fs::create_dir_all(&dir).unwrap();
img.save(dir.join(format!("{name}.png"))).unwrap();
}
Err(e) => eprintln!("snapshot {name} skipped: {e}"),
}
}
fn label(&mut self, id: &str) -> String {
self.find(id).unwrap_or_else(|| panic!("no element {id}")).4
}
fn click(&mut self, id: &str) {
self.find(id).unwrap_or_else(|| panic!("no element {id}"));
self.ok("ui.click", json!({"id": id}));
self.frames(3);
}
fn crossing(&mut self) -> Crossing {
let q = self.exec("sequence.inspect", json!({}));
for tr in q["video"].as_array().unwrap() {
for x in tr["transitions"].as_array().unwrap() {
if let (Some(_), Some(to)) = (x["from"].as_u64(), x["to"].as_u64()) {
let cut = tr["items"].as_array().unwrap().iter().find(|i| i["clip"] == to).unwrap()["start"].as_i64().unwrap();
return Crossing { id: x["id"].as_u64().unwrap(), start: x["start"].as_i64().unwrap(), duration: x["duration"].as_i64().unwrap(), cut };
}
}
}
panic!("the demo has a transition between two clips");
}
fn transition(&mut self, id: u64) -> Option<Value> {
let q = self.exec("sequence.inspect", json!({}));
q["video"].as_array().unwrap().iter().flat_map(|t| t["transitions"].as_array().unwrap().clone()).find(|x| x["id"] == id)
}
fn rate(&mut self) -> (i64, i64) {
let q = self.exec("sequence.inspect", json!({}));
(q["settings"]["frame_rate"]["num"].as_i64().unwrap(), q["settings"]["frame_rate"]["den"].as_i64().unwrap())
}
}
/// Ticks in one frame of `rate`.
fn frame_ticks((num, den): (i64, i64)) -> i64 {
254_016_000_000 * den / num
}
/// Clicking a transition selects it (and only it) and opens it in Effect Controls; Delete removes
/// it, leaving its clips; undo brings it back.
#[test]
fn click_selects_a_transition_and_delete_removes_it() {
let mut d = Driver::demo();
let c = d.crossing();
let tid = format!("timeline.transition.{}", c.id);
d.click(&tid);
let q = d.exec("sequence.inspect", json!({}));
assert_eq!(q["transitionSelection"], json!([c.id]), "{q}");
assert_eq!(q["selection"], json!([]), "a transition replaces the clip selection");
// Effect Controls shows it: duration as timecode, its alignment, Reverse
let (num, den) = d.rate();
let rate = filmcraft_engine::time::FrameRate { num, den };
let frames = c.duration / frame_ticks((num, den));
assert_eq!(d.label("effectControls.transition.duration"), filmcraft_engine::time::format_timecode_frames(frames, rate, false));
assert_eq!(d.label("effectControls.transition.alignment"), "Center at Cut");
assert_eq!(d.label("effectControls.transition.reverse"), "false");
d.snapshot("transition-selected");
let clips_before = q["video"].as_array().unwrap().iter().map(|t| t["items"].as_array().unwrap().len()).sum::<usize>();
// Delete
d.ok("ui.key", json!({"key": "Delete"}));
d.frames(3);
assert!(d.transition(c.id).is_none(), "Delete removes the selected transition");
let q = d.exec("sequence.inspect", json!({}));
assert_eq!(q["video"].as_array().unwrap().iter().map(|t| t["items"].as_array().unwrap().len()).sum::<usize>(), clips_before, "its clips stay");
assert!(d.find("effectControls.transition.duration").is_none(), "Effect Controls lets go of it");
d.exec("edit.undo", json!({}));
assert!(d.transition(c.id).is_some(), "undo brings it back");
// clicking a clip selects the clip instead
let clip = q["video"][0]["items"][0]["clip"].as_u64().unwrap();
d.exec("timeline.select", json!({"transitions": [c.id]}));
d.exec("timeline.select", json!({"clips": [clip]}));
assert_eq!(d.exec("sequence.inspect", json!({}))["transitionSelection"], json!([]));
}
/// Dragging a transition's end trims that end only (#224); dragging its middle slides it over the
/// cut (Custom Start). Each drag is one undo step.
#[test]
fn dragging_a_transition_end_or_middle() {
let mut d = Driver::demo();
let c = d.crossing();
let frame = frame_ticks(d.rate());
let undo_before = d.exec("sequence.inspect", json!({}));
let (x, y, w, h, _) = d.find(&format!("timeline.transition.{}.out", c.id)).expect("the transition's end");
let (fx, fy) = (x + w / 2.0, y + h / 2.0);
d.ok("ui.drag", json!({"from": {"x": fx, "y": fy}, "to": {"x": fx + 40.0, "y": fy}, "steps": 8}));
d.frames(3);
let x1 = d.transition(c.id).unwrap();
let (s1, d1) = (x1["start"].as_i64().unwrap(), x1["duration"].as_i64().unwrap());
assert!(d1 > c.duration, "dragging the end out lengthens it: {} -> {d1}", c.duration);
assert_eq!(s1, c.start, "the start stays where it was");
assert_eq!((d1 - c.duration) % frame, 0, "by whole frames");
assert_eq!(x1["align"], json!("custom"), "no longer centred: Custom Start");
// the middle: slide it right of centre
let (x, y, w, h, _) = d.find(&format!("timeline.transition.{}", c.id)).unwrap();
let (mx, my) = (x + w / 2.0, y + h / 2.0);
// more than egui's 6 px click tolerance, or the press is a click
d.ok("ui.drag", json!({"from": {"x": mx, "y": my}, "to": {"x": mx + 14.0, "y": my}, "steps": 6}));
d.frames(3);
let x2 = d.transition(c.id).unwrap();
assert_eq!(x2["duration"].as_i64(), Some(d1), "sliding keeps the duration");
assert!(x2["start"].as_i64().unwrap() > s1, "and moves it right");
assert!(x2["start"].as_i64().unwrap() <= c.cut, "still over the cut");
// off-centre (Custom Start), or Start at Cut if the slide reached the cut, where it stops
assert!(matches!(x2["align"].as_str(), Some("custom" | "start")), "{}", x2["align"]);
// two drags, two undo steps
d.exec("edit.undo", json!({}));
d.exec("edit.undo", json!({}));
let back = d.transition(c.id).unwrap();
assert_eq!((back["start"].as_i64(), back["duration"].as_i64()), (Some(c.start), Some(c.duration)), "{undo_before}");
}
/// Double-clicking a transition opens Set Transition Duration, whose OK sets the duration
/// (following the alignment); Effect Controls' Alignment menu re-aligns it on the cut.
#[test]
fn set_transition_duration_and_alignment_menu() {
let mut d = Driver::demo();
let c = d.crossing();
let frame = frame_ticks(d.rate());
// double-click it
let (x0, y0, w, h, _) = d.find(&format!("timeline.transition.{}", c.id)).unwrap();
d.ok("ui.click", json!({"x": x0 + w / 2.0, "y": y0 + h / 2.0, "count": 2}));
d.frames(3);
let frames = c.duration / frame;
assert!(d.find("transitionDuration.value").is_some(), "Set Transition Duration opened");
assert_eq!(d.harness.state().ui.transition_duration.frames, frames, "with its duration");
assert_eq!(d.exec("sequence.inspect", json!({}))["transitionSelection"], json!([c.id]), "and selected it");
d.snapshot("transition-duration-dialog");
// Center at Cut: a new duration moves both ends
d.harness.state_mut().ui.transition_duration.frames = 10;
d.frames(2);
d.click("transitionDuration.ok");
assert!(d.find("transitionDuration.value").is_none(), "closed");
let x = d.transition(c.id).unwrap();
assert_eq!((x["start"].as_i64(), x["duration"].as_i64(), x["align"].as_str()), (Some(c.cut - 5 * frame), Some(10 * frame), Some("center")));
// the Alignment menu: Start at Cut
d.click("effectControls.transition.alignment");
d.click("effectControls.transition.alignment.option.start");
let x = d.transition(c.id).unwrap();
assert_eq!((x["start"].as_i64(), x["duration"].as_i64(), x["align"].as_str()), (Some(c.cut), Some(10 * frame), Some("start")));
assert_eq!(d.label("effectControls.transition.alignment"), "Start at Cut");
// Reverse
d.click("effectControls.transition.reverse");
assert_eq!(d.transition(c.id).unwrap()["reverse"], json!(true));
}
/// A transition's own settings are in Effect Controls and change it (Push: Direction, Motion Blur).
#[test]
fn transition_settings_in_effect_controls() {
let mut d = Driver::demo();
let q = d.exec("sequence.inspect", json!({}));
let push = q["video"]
.as_array()
.unwrap()
.iter()
.flat_map(|t| t["transitions"].as_array().unwrap().clone())
.find(|x| x["effect"] == "push")
.expect("the demo has a Push");
let id = push["id"].as_u64().unwrap();
d.click(&format!("timeline.transition.{id}"));
for p in ["direction", "motion_blur"] {
assert!(d.find(&format!("effectControls.transition.param.{p}")).is_some(), "{p} shown");
}
d.snapshot("transition-push-settings");
d.click("effectControls.transition.param.direction");
d.click("effectControls.transition.param.direction.option.0");
assert_eq!(d.transition(id).unwrap()["params"]["direction"], json!({"Choice": 0}));
d.click("effectControls.transition.reset");
let def = filmcraft_engine::project::vtransition::find_transition("push", filmcraft_engine::project::EffectKind::VideoTransition).unwrap();
let want = serde_json::to_value(&def.instance().params.get("direction").unwrap().value).unwrap();
assert_eq!(d.transition(id).unwrap()["params"]["direction"], want, "Reset puts Push's own defaults back");
}
+205
View File
@@ -0,0 +1,205 @@
//! Headless UI tests of the Text to Speech panel: typing a script, Add pause, adding the narration
//! to the timeline, clicking the narration clip to load it back, and saving an edit in place.
use std::sync::mpsc::{Sender, channel};
use egui_kittest::Harness;
use filmcraft_engine::Session;
use filmcraft_ui_egui::FilmcraftApp;
use filmcraft_ui_egui::control::ControlRequest;
use serde_json::{Value, json};
struct Driver {
harness: Harness<'static, FilmcraftApp>,
tx: Sender<ControlRequest>,
}
impl Driver {
fn demo(scratch: &str) -> Self {
let mut session = Session::default();
session.execute("file.openDemoProject", json!({})).expect("demo project");
session.execute("file.projectSettings.scratchDisks", json!({"captured": scratch})).expect("scratch disk");
let (tx, rx) = channel();
let app = FilmcraftApp::new(session).with_control(rx);
let harness = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000).build_eframe(move |_cc| app);
let mut d = Driver { harness, tx };
d.frames(4);
d
}
fn frames(&mut self, n: usize) {
for _ in 0..n {
let ctx = self.harness.ctx.clone();
let mut raw = std::mem::take(self.harness.input_mut());
eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw);
*self.harness.input_mut() = raw;
self.harness.step();
}
}
fn call(&mut self, method: &str, params: Value) -> Value {
let (req, reply) = ControlRequest::new(method, params.clone());
self.tx.send(req).unwrap();
for _ in 0..600 {
self.frames(1);
if let Ok(v) = reply.try_recv() {
return v;
}
}
panic!("no reply to {method} {params}");
}
fn ok(&mut self, method: &str, params: Value) -> Value {
let v = self.call(method, params.clone());
assert_eq!(v["ok"], json!(true), "{method} {params} failed: {v}");
v["result"].clone()
}
fn exec(&mut self, command: &str, params: Value) -> Value {
self.ok("engine.execute", json!({"command": command, "params": params}))
}
fn click(&mut self, id: &str) {
self.ok("ui.click", json!({"id": id}));
self.frames(3);
}
fn type_text(&mut self, text: &str) {
self.ok("ui.type", json!({"text": text}));
self.frames(3);
}
fn ui(&mut self) -> Value {
self.ok("ui.inspect", json!({}))
}
fn draft(&mut self) -> Value {
self.ui()["ui"]["tts"].clone()
}
/// Let a background synthesis job finish.
fn settle(&mut self) {
for _ in 0..400 {
if self.draft()["pending"].is_null() {
return;
}
std::thread::sleep(std::time::Duration::from_millis(10));
self.frames(1);
}
panic!("synthesis never finished");
}
fn ids(&mut self, prefix: &str) -> Vec<String> {
let v = self.ok("ui.elements", json!({"prefix": prefix}));
v.as_array().unwrap().iter().filter_map(|e| e["id"].as_str().map(str::to_string)).collect()
}
fn narration_clips(&mut self) -> Vec<u64> {
let seq = self.exec("sequence.inspect", json!({}));
let mut out = Vec::new();
for t in seq["audio"].as_array().into_iter().flatten() {
for it in t["items"].as_array().into_iter().flatten() {
let id = it["clip"].as_u64().unwrap();
if self.call("engine.execute", json!({"command": "tts.inspect", "params": {"clip": id}}))["ok"] == json!(true) {
out.push(id);
}
}
}
out
}
}
fn scratch(name: &str) -> String {
let d = std::env::temp_dir().join(format!("filmcraft-tts-ui-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d.to_string_lossy().into_owned()
}
#[test]
fn write_add_click_to_edit_and_save_in_place() {
let dir = scratch("flow");
let mut d = Driver::demo(&dir);
d.exec("window.panel.TextToSpeech", json!({}));
d.frames(3);
// the panel opens beside Properties, leaving the Program monitor visible
assert!(!d.ids("program.").is_empty(), "Program monitor still shown");
let ids = d.ids("tts.");
for id in ["tts.language", "tts.voice", "tts.hearVoice", "tts.advanced", "tts.pitch", "tts.pace", "tts.text", "tts.addPause", "tts.preview", "tts.save"] {
assert!(ids.iter().any(|i| i == id), "missing {id} in {ids:?}");
}
// type a script and add a pause
d.click("tts.text");
d.type_text("Hello from the panel.");
d.click("tts.addPause");
d.click("tts.text");
d.ok("ui.key", json!({"key": "End"}));
d.type_text("Goodbye.");
let text = d.draft()["text"].as_str().unwrap().to_string();
assert!(text.starts_with("Hello from the panel. [pause 1s]") && text.ends_with("Goodbye."), "{text:?}");
// add it to the timeline
d.exec("playhead.set", json!({"seconds": 1.0}));
d.click("tts.save");
d.settle();
let clips = d.narration_clips();
let status = d.ui()["ui"]["status"].clone();
assert_eq!(clips.len(), 1, "one narration clip; status {status}");
let clip = clips[0];
let n = d.exec("tts.inspect", json!({"clip": clip}));
assert_eq!(n["narration"]["text"].as_str().unwrap(), text);
assert_eq!(d.ui()["selection"], json!([clip]), "the new clip is selected");
assert_eq!(d.draft()["loadedFrom"][0], json!(clip), "the panel now edits it");
// deselect and start fresh, then click the clip on the timeline: the panel loads it back
d.exec("edit.deselectAll", json!({}));
d.frames(3);
assert!(d.draft()["loadedFrom"].is_null());
d.click("tts.text");
d.ok("ui.key", json!({"key": "Cmd+A"}));
d.type_text("Something else");
let at = d.ok("ui.timeline.locate", json!({"clip": clip}));
d.ok("ui.click", json!({"x": at["x"], "y": at["y"]}));
d.frames(4);
assert_eq!(d.ui()["selection"], json!([clip]));
assert_eq!(d.draft()["text"].as_str().unwrap(), text, "clicking the clip loads its script");
// change the script and save: same clip, new audio, one undo step
let before = d.exec("tts.inspect", json!({"clip": clip}))["item"].clone();
d.click("tts.text");
d.ok("ui.key", json!({"key": "Cmd+A"}));
d.type_text("A new script for the same clip.");
d.click("tts.save");
d.settle();
let after = d.exec("tts.inspect", json!({"clip": clip}));
assert_eq!(after["narration"]["text"], "A new script for the same clip.");
assert_ne!(after["item"], before);
assert_eq!(d.narration_clips(), vec![clip]);
let hist = d.exec("history.list", json!({}));
assert_eq!(hist["undo"].as_array().unwrap().last().unwrap(), "Edit Narration");
d.exec("edit.undo", json!({}));
d.frames(3);
assert_eq!(d.exec("tts.inspect", json!({"clip": clip}))["narration"]["text"].as_str().unwrap(), text);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn settings_controls_and_hear_this_voice() {
let dir = scratch("controls");
let mut d = Driver::demo(&dir);
d.exec("window.panel.TextToSpeech", json!({}));
d.frames(3);
// Save is disabled with an empty script
d.click("tts.save");
assert!(d.narration_clips().is_empty());
// Hear this voice runs a preview (the headless harness has no audio output: a status line)
d.click("tts.hearVoice");
d.settle();
let status = d.ui()["ui"]["status"].as_str().unwrap_or_default().to_string();
assert!(status.contains("audio output") || status.is_empty(), "{status}");
// Advanced folds away and back
assert_eq!(d.draft()["advancedOpen"], true);
d.click("tts.advanced");
assert_eq!(d.draft()["advancedOpen"], false);
assert!(!d.ids("tts.").iter().any(|i| i == "tts.pitch"));
d.click("tts.advanced");
assert!(d.ids("tts.").iter().any(|i| i == "tts.pitch"));
let _ = std::fs::remove_dir_all(&dir);
}
+3 -2
View File
@@ -61,7 +61,7 @@ claude mcp add filmcraft -- filmcraft-cli mcp
| `render_preview` | both | same as `render_frame` |
| `project_inspect` | both | bins and items with ids, types, durations; active sequence |
| `sequence_inspect` | both | the active sequence: tracks, clips (`start` / `end` and `sourceIn` / `sourceOut` in ticks, frames, `speed`, `reverse`: the clip plays its `sourceIn` to `sourceOut` stretch backward, `gainDb`), effects, transitions, markers (name and `comment`), playhead, selection |
| `media_import` | both | import files by absolute path (`text`, one path per line) |
| `media_import` | both | import files by absolute path (`paths`, an array; optional `bin`, `image_sequence`) |
| `render_frame` | both | PNG of the program frame at `seconds` (headless renders; bridge screenshots the Program monitor). Read-only: the playhead and selection are left as they were |
| `ui_inspect` | bridge | UI state: tool, workspace, panels, zoom, playback, fps |
| `ui_elements` | bridge | on-screen interactive elements with id, label and rect (`prefix` filter) |
@@ -182,7 +182,8 @@ Notes:
## 2b. Command-line interface
Every command is also one shell call away. Options go anywhere; output is JSON; exit status is 0 on
success, 1 when a command fails and 2 on a usage error. `filmcraft-cli help` prints the reference.
success, 1 when a command fails and 2 on a usage error. `filmcraft-cli help`, `filmcraft-cli --help`
and `filmcraft-cli -h` print the reference and exit successfully.
If the reader of stdout closes the pipe early (`filmcraft-cli commands | head`), the CLI drops the rest
of its output but still finishes the work, saves included, and the exit status still reports
failures; any other stdout write error is reported and makes the status 1.
+7 -1
View File
@@ -24,7 +24,7 @@ Design principles:
L5 ui-egui · automation · platform
L4 engine
L3 render · gpu · export · golden (test-only)
L2 edit · codecs · interchange · captions · speech
L2 edit · codecs · interchange · captions · speech · tts · tts-text
L1 frame · media · project · audio-dsp · text
L0 foundation: time · geom · color · bitstream · testkit (dev-dependency only)
codecs/containers: isobmff · matroska · mxf · cfb · mpegts · ogg · h264 · h264enc · hevc · vp9 · av1 · mpeg2v · prores · dnx · apv · aac · ac3 · opus
@@ -62,6 +62,8 @@ and `filmcraft-cli`.
| `text` | L1 | text engine: font database (bundled OFL fonts + system fonts), shaping (harfrust), bidi, line breaking, paragraph layout, glyph/path rasteriser, strokes ([crates/text/README.md](../crates/text/README.md)) |
| `edit` | L2 | pure edit algebra (insert, overwrite, razor, ripple, roll, slip, slide, rate stretch…; text-based editing: `edit::transcript`) |
| `speech` | L2 | speech-to-text: `Transcriber` trait, Whisper model catalogue + verified downloader (feature `download`), pure-Rust Whisper inference on candle with word timestamps (feature `whisper`), speaker labelling ([transcripts.md](transcripts.md)) |
| `tts` | L2 | text to speech: `Voice` trait, voice catalogue, pause markers, built-in formant voices; Kokoro-82M natural voices on candle and their pinned package catalogue (feature `kokoro`). Same-layer edge to `tts-text` |
| `tts-text` | L2 | text front end for narration: normalizer (numbers, money, dates, acronyms, pauses) and English pronunciation (CMUdict + letter-to-sound) to Kokoro phonemes |
| `codecs` | L2 | container + codec hub: MP4/MOV, MKV, MXF, Ogg and MPEG TS / PS / video elementary stream sources, GOP-aware seeking, decoder registry, audio decoding |
| `interchange` | L2 | EDL, FCP7 XML, FCPXML, OTIO, AAF (on `cfb`) and OMF 2.0 import/export (no file I/O; the engine supplies rendered audio essence) ([README](../crates/interchange/README.md)) |
| `render` | L3 | sequence evaluation, CPU compositor, video effects (`effects`, `vfx`; effects needing other frames or tracks read them through `vfx::FxEnv`), transitions, audio mix |
@@ -455,6 +457,10 @@ stereo / mono track ──5.1 panner───┼─► 5.1 submix / 5.1 Mix ─
**Voice-over recording** (`engine::voiceover`, `audio.voiceover.*`). The record point R is the playhead, or the In point when In/Out are set (punch-in; punch-out at Out). Playback starts the pre-roll before it (C = max(0, R − pre-roll)) and the input is captured from C. When the UI's audio clock really starts, `audio.voiceover.sync` restarts the capture there. `audio.voiceover.stop` keeps the audio from R to min(stop, Out), writes a mono 32-bit float WAV `<Name> <n>.wav` (Scratch Disks ▸ Captured, else next to the project, else the data or temporary directory), imports it and overwrites it onto the record track at R as one undo step. The record track is the given one, else the record-armed one, else the first targeted one. Input goes through the `AudioInput` trait: cpal in the desktop app (`apps/filmcraft/src/audio_in.rs`, on its own thread), and `SyntheticInput` headless, which produces exactly the samples the timeline asks for so recordings land sample-accurately. Voice-Over Record Settings (Source, Input channel, Name, Countdown Sound Cues, Pre-/Post-roll) are preferences (`voiceOver`). The track header's microphone records or stops, and a right-click opens the settings dialog. Countdown beeps (1 kHz, 100 ms, each whole second of pre-roll and at R) are mixed into playback, an overlay counts down, and playback stops at Out + post-roll.
### 5.1.2a Text to Speech (narrations)
**Narrations** (`engine::narration`, `tts.*`; voices in `filmcraft-tts`). The Text to Speech panel (Window ▸ Text to Speech) writes a script with pause markers (`[pause 1s]`), a voice, a vocal pitch and a pace; `tts.create` synthesizes it, writes `Narration <n>.wav` (mono 32-bit float, 24 kHz, where voice-over recordings go), imports it and places it at the playhead on the first free targeted audio track, else the first free audio track, else a new one, as one undo step. The settings are kept per generated item in `Project::narrations` (schema v14). Selecting a narration clip loads it into the panel; `tts.edit` writes a new file (beside the previous one) and swaps the clip to it, keeping the clip's start, duration and speed: longer speech is kept whole in the file but plays only to the clip's end; shorter speech is padded with silence to the clip's length. The old item and narration stay, so undo is exact. `tts.preview` synthesizes without changing the project and the UI plays `Session::tts_preview` on the audio output. Built-in voices are an original formant synthesizer (robotic, no download). The natural voices (feature `neural-voices` of the desktop app, off by default like `whisper`) are Kokoro-82M on candle (`filmcraft-tts`) fed by `filmcraft-tts-text` (normalizer, CMUdict + letter-to-sound); their 336 MB package is downloaded once after the user confirms (`tts.downloadVoices`, a job). The UI synthesizes with `tts.render` (a job filling `Session::tts_cache`) and then runs create / edit / preview, which use the cache.
### 5.1.3 Remix
```text
+1
View File
@@ -40,6 +40,7 @@ cargo xtask web --serve 8765 # the web app on http://12
|---|---|
| `FILMCRAFT_CONTROL_PORT` | Same as `--control <port>` |
| `FILMCRAFT_CPU_COMPOSITE=1` | Disables the GPU compositor |
| `FILMCRAFT_DEV_DESKTOP_ENTRY=1` | Linux, opt-in: a run from a source tree installs a user-level desktop entry (and the icons from `assets/app-icon`) so the taskbar shows the FilmCraft icon. Nothing is written without it; packaged runs never write one |
| `FILMCRAFT_DATA_DIR` | Same as `--data-dir <dir>`: auto-save, crash recovery, settings and logs |
| `RUST_LOG` | Log levels for standard error and the log file (see [Logs](#logs)) |
+1 -1
View File
@@ -89,7 +89,7 @@ Timecode and Clip Name effects also draw with the text engine.
| `graphics.newShape` | (agents) | `shape` (rectangle/ellipse/polygon/path), `position` (the shape's centre), `size`, `points`, `clip` (add to this graphic), `seconds` (5), `track`, `time` |
| `graphics.setTextType` | Text Properties ▸ Text Layer Type | `clip`, `layer`, `type` (`point` / `paragraph`); the text stays where it is |
| `graphics.setText` | typing on the monitor | `clip`, `layer`, `text`, `merge` (coalesce one typing session into one undo step) |
| `graphics.set` | Properties panel | `clip`, `layer`, `props` {parameter id or camelCase alias: value; choices by index or name}, `time` |
| `graphics.set` | Properties panel | `clip`, `layer`, `props` {parameter id or camelCase alias: value; choices by index or name}, `time`, `merge` + `begin` (a drag is one undo step: send `merge: true` on every change and `begin: true` on the first change of each press) |
| `graphics.selectLayer` | layer list / monitor click | `clip`, `layers` |
| `graphics.deleteLayer`, `graphics.arrangeLayer` | layer list | `clip`, `layer`, `to` (front/back/forward/backward/index) |
| `graphics.align` | Align and Transform | `align` (left/hcenter/right/top/vcenter/bottom), `to` (frame/group/selection), `layers` |
+1
View File
@@ -31,6 +31,7 @@ FILMCRAFT_REQUIRE_ORACLES=1 cargo test --workspace # CI: missing ffmpeg fails
| 5.1 / multichannel | `crates/audio-dsp/src/channels.rs`, `crates/render/src/mixer_tests.rs`, `crates/export/src/surround_tests.rs`, `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/audio51_ui.rs` | BS.775 downmix / upmix coefficients exact, mixdown types; 5.1 panner point gains equal-power (Σg² = 1 over a 21×21 grid × 3 centre settings), speaker positions exact, 5.1 through the default puck bit-exact identity; graph: stereo/mono tracks into a 5.1 Mix (L/R, C, rear right, room centre −6 dB each, LFE = mean), 5.1 track through a 5.1 Mix and folded into stereo, a 5.1 submix, `pan51` automation block-invariant, six-channel meters; export: 6-ch WAV (`WAVE_FORMAT_EXTENSIBLE`, mask 0x3F) and ProRes MOV (6 × 24-bit PCM) decoded by ffmpeg **sample-exact**, ffprobe `channels` 6 / `channel_layout` 5.1; AAC 5.1 in MP4: every decoded channel's own tone ≥ 30 dB above the others' (no swapped channels), stereo export of a 5.1 Mix = BS.775 within 2·10⁻⁶; `sequence.settings {mix}`, `file.newSequence {mix, trackType}`, panner lanes, Write pass records the puck; UI: puck drag (one undo step), Center / LFE, six meter bars |
| Audio Clip Mixer automation, Track Mixer view | `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/audio51_ui.rs` | Touch: a 201-point fader ramp thins to ≤ 6 clip keyframes, values within 0.1 dB, live override while held, values outside the gesture unchanged, one undo step; Latch holds until stop; Write records volume and pan from the start; a gesture across two clips writes both; Read commits once without a pass; UI: Clip Mixer mode dropdown, a recorded fader drag writes clip keyframes; Show/Hide Tracks, Meter Input(s) Only, transport record button |
| Voice-over | `crates/engine/src/voiceover_tests.rs`, `crates/ui-egui/tests/voiceover_ui.rs` | synthetic click train lands in `mix_sequence` exactly at R + k·period (pre-roll dropped, exact sample count), punch-in between In/Out (ramp source, every sample checked), one undo step + undo/redo, take numbering, record-arm/targeting track choice, sync/discard/zero-length/locked cases, cue times; UI: right-click Mic → Voice-Over Record Settings dialog (edits persist on OK, Cancel discards), the Mic button starts and stops a take on its track with pre-roll and overlay |
| Text to Speech | `crates/tts/src/**/*.rs` (Kokoro tests need `FILMCRAFT_KOKORO_DIR`), `crates/tts-text`, `crates/engine/src/narration_tests.rs`, `crates/ui-egui/tests/tts_ui.rs` | built-in voices: determinism, finite samples, exact-silence pause markers, pace scales length, measured pitch per voice and per semitone, hostile scripts; engine: narration placed at the playhead with clip length = speech (sample-exact), never overwrites (busy tracks → next free track → new track), occupied/locked/unknown track refused with no change, edit keeps start/length/speed (longer speech: whole speech in the file; shorter: file padded with exact zeros), edits write beside the previous file, undo/redo restore the project exactly, enabled only with a narration clip (or `clip`), 16 hostile parameter sets refused without change, preview changes nothing, save/reload keeps narrations, the narration is audible in `mix_sequence`; UI: type + Add pause + Add to timeline, the new clip is selected and loaded, clicking the clip on the timeline loads its script, Save edits it in place as one undo step, Advanced twirl, panel opens beside Properties; natural voices: 50-sentence Whisper-base WER 2.1–3.6 % for all 9 voices (crate README), loaders fuzzed with hostile files, `tts.render` cache, download refused without the feature |
| Remix | `crates/audio-dsp/src/remix.rs`, `crates/render/src/remix.rs`, `crates/engine/src/remix_tests.rs`, `crates/ui-egui/tests/remix_ui.rs` | generated rhythmic music (Rust-synthesised drums, bass, chord sections at 90–128 BPM, 22.05/44.1/48 kHz): tempo within 0.3 BPM, every beat within 10 ms (measured ≤ 8.9 ms); remix targets from 0.4× to 2.2× the source within one beat (measured ≤ 12 ms at default sliders), every cut on a detected beat, intro and outro kept, pieces ≥ 4 beats, deterministic; silence, tone and noise refused; rendering independent of request cuts, source-exact outside the 20 ms equal-power crossfades, no clicks; the engine's mix equals the source pieces (< 1e-5); undo, redo and revert exact; overlaps refused; Remix Properties dialog and Remix tool drag driven by automation id |
| Oracle | `crates/*/tests/*oracle*.rs`, `conformance.rs` | compare with ffmpeg/ffprobe (§2) |
| Robustness / fuzz | `*/tests/robustness.rs`, `*/tests/fuzz.rs` | seeded mutation and truncation of real and synthetic files; nothing may panic |
+3
View File
@@ -60,6 +60,8 @@ const LAYERS: &[(&str, u8)] = &[
("audio", 2),
("captions", 2),
("speech", 2),
("tts", 2),
("tts-text", 2),
("interchange", 2),
("render", 3),
("gpu", 3),
@@ -93,6 +95,7 @@ const SAME_LAYER: &[(&str, &str)] = &[
("scopes", "gpu"),
("gpu", "render"),
("cli", "filmcraft"),
("tts", "tts-text"),
];
/// Crates that must not appear below L5 (UI toolkits, windowing, OS audio/menus).