mirror of
https://github.com/hypit-ai/hypit.git
synced 2026-11-20 20:40:29 +00:00
1126 lines
50 KiB
TypeScript
1126 lines
50 KiB
TypeScript
import { spawn, spawnSync } from "node:child_process";
|
|
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import test from "node:test";
|
|
import { blobTypes } from "@hypit/blob";
|
|
import { mediaTypes, sealTimelineVisual, sealTimelineAudio, synchronizedMediaSampleFrames, verifyMediaInspection, verifyMuxedMedia, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
|
|
import type { MediaAudioStream, MediaInspection, MuxedMedia, SynchronizedMedia, TimelineAudio } from "@hypit/media";
|
|
import { assertSpeechEvidenceAudioIdentity, speechEvidenceCapabilities, speechEvidenceTypes } from "@hypit/speech-evidence";
|
|
import type { SpeechEvidenceAudio } from "@hypit/speech-evidence";
|
|
import assert from "node:assert/strict";
|
|
import {
|
|
MemoryResourceStore,
|
|
EndpointRegistry,
|
|
} from "@hypit/executor";
|
|
import type { EndpointRegistration } from "@hypit/executor";
|
|
import type { ImmediateEndpointHandler } from "@hypit/endpoint";
|
|
import {
|
|
mediaOperationsCapabilities,
|
|
sealAudioProgramPlan,
|
|
sealMediaSelectionRequest,
|
|
selectMediaStreams,
|
|
} from "@hypit/media-operations";
|
|
import { canonicalize } from "@hypit/protocol";
|
|
import type { CapabilityRef, CanonicalValue, Need, TypeRef } from "@hypit/protocol";
|
|
|
|
import { createLocalMediaProvider } from "../src/index.js";
|
|
import { localMediaToolchainProgram } from "../src/program.js";
|
|
|
|
const hasMediaBinaries = spawnSync("ffmpeg", ["-version"], { stdio: "ignore", windowsHide: true }).status === 0
|
|
&& spawnSync("ffprobe", ["-version"], { stdio: "ignore", windowsHide: true }).status === 0;
|
|
|
|
test("the local media Provider declares its external toolchain without owning a second daemon", async () => {
|
|
const program = localMediaToolchainProgram({
|
|
id: "media",
|
|
ffmpegPath: "ffmpeg",
|
|
ffprobePath: "ffprobe",
|
|
});
|
|
assert.equal(program.id, "media");
|
|
assert.equal(program.installation, undefined);
|
|
assert.equal(program.start, undefined);
|
|
const state = await program.probe();
|
|
assert.equal(state.state, hasMediaBinaries ? "ready" : "down");
|
|
});
|
|
|
|
// Five 64x48 lossless frames at 100 ms each. Later frames are partial alpha
|
|
// rectangles, so this fixture exercises WebP blend and canvas persistence.
|
|
const animatedWebp = Buffer.from([
|
|
"UklGRjQEAABXRUJQVlA4WAoAAAASAAAAPwAALwAAQU5JTQYAAAAAbPz/AABBTk1GpgIAAAAAAAAAAD8AAC8AAGQAAAJWUDhMjQIAAC8/wAsAp8EQkiRJeQ9KwxzeARbKgjBq20iQd/egFMzCOSgH58AUyjwIBCEy0kojzX8gd0AYXnAa/lMIRRkGIQiDELwMG5RBcaEUJQhCDyEIwkotw3l4P34IQlA+o4RdINm27bZt6KLGC9aIhJAnMGqge/f/fxzee1C06Iwyiei/A7dtI6nGCh0PvOk1g3nD02opOIu6Seg2ofsrKS4wj3qYs66vH6+l1G/59+0qqXlC1wktVyYwAaBPqCorqTI7pYgRASOzP2iulqdwCiEYGHS9ZlFpnpIKo+bxoJmAiej6XvoxX3C/qs4gqG6I+WMS5nQpMgWEUfqq30sg0I72QFk473xR9L1MtLXzzhtjgttxAYyCAaPzzpd35Z3zq2UI5mWXg3Z59+kauK9Sox42v7B9NlkcFxfQ9/Hq17l7LrtPv57NALIADBFZThD1XetdU1Vk2SYiy/eJEPI3D4xk+T/FOA+zmYElA85Z4gTnGj+0RWUp2hmPhwOWwgi/y8NoiROWuIBEvnMgS7Yeftdk9bkj23hveDx44QL63PH4hlqOcyGA7S63ZKn+HmpL2rbUuD9iPYznAtq2VH981WKcH0nbKttGu6rqhG2VbUVf20eyl9j1R8I+KfsUAby+a/uQsvFKeb73KL9c0/jPItqxXFW2g1vDD8a8OHABjHjzHHjj8X3dlc+uYQAb2hHyrnS0cUUZ0dYRdbuh3TYzmdmJAmFEDoQR8G7H4yu9EzDA+edOhItLnjvBo7SZFzx32UkqU8891HPfy2dPAMB0752FwnTvHW1P9d5J2VO9dy6wKxk+B/3eSduzye3/gOdOabrvNm6LSZvGdN+dE178mKYTix8NHeCE8H/UxT9/7q4BAEFOTUZSAAAAAAAAAAAABwAACQAAZAAAAFZQOEw6AAAALwdAAhA/EIraSIFWzCkFgchDCg+BQJK/ypDPf4C3zAiyOaCQkSTmFA7mCd6k+0M0hIj+JwfeYjBtNEFOTUZOAAAAAAAAAQAACQAABwAAZAAAAFZQOEw2AAAALwnAARA3MGxq2jZgCrD8NaH0mKr5D8A1EMgogKBt25jCxuUIqvE4fxolEdH/GGRxg/T6q+kDQU5NRl4AAAABAAABAAAJAAAJAABkAAAAVlA4TEYAAAAvCUACEEcgFkzmfwUYmEK2EaCDmT/PfTBTNf9BQOA5mQ8F4KiRJEeKdUCq1r0dkMV01I9ERP+jbSWJ4gre4grOQv78FnwHQU5NRkQAAAACAAACAAAHAAAJAABkAAAAVlA4TCwAAAAvB0ACECcQAoFkmPyJhpmq+Y8gbqCQjSRoGNZgzuDz5zuEiP5nKqXKvxvuAA==",
|
|
].join(""), "base64");
|
|
|
|
async function run(executable: string, args: readonly string[]): Promise<void> {
|
|
await new Promise<void>((resolve, reject) => {
|
|
const child = spawn(executable, [...args], { stdio: ["ignore", "ignore", "pipe"], windowsHide: true });
|
|
let stderr = "";
|
|
child.stderr.on("data", (chunk: Buffer) => { stderr += chunk.toString(); });
|
|
child.on("error", reject);
|
|
child.on("close", (code) => code === 0 ? resolve() : reject(new Error(`${executable} ${String(code)}: ${stderr}`)));
|
|
});
|
|
}
|
|
|
|
function runBytes(executable: string, args: readonly string[], input?: Uint8Array): Buffer {
|
|
const result = spawnSync(executable, [...args], {
|
|
...(input === undefined ? {} : { input }),
|
|
maxBuffer: 32 * 1024 * 1024,
|
|
windowsHide: true,
|
|
});
|
|
assert.equal(result.status, 0, Buffer.from(result.stderr ?? []).toString());
|
|
return Buffer.from(result.stdout ?? []);
|
|
}
|
|
|
|
function rampWav(sampleFrames: number): Buffer {
|
|
const dataBytes = sampleFrames * 2 * 2;
|
|
const bytes = Buffer.alloc(44 + dataBytes);
|
|
bytes.write("RIFF", 0); bytes.writeUInt32LE(36 + dataBytes, 4); bytes.write("WAVE", 8);
|
|
bytes.write("fmt ", 12); bytes.writeUInt32LE(16, 16); bytes.writeUInt16LE(1, 20);
|
|
bytes.writeUInt16LE(2, 22); bytes.writeUInt32LE(48_000, 24); bytes.writeUInt32LE(48_000 * 4, 28);
|
|
bytes.writeUInt16LE(4, 32); bytes.writeUInt16LE(16, 34); bytes.write("data", 36); bytes.writeUInt32LE(dataBytes, 40);
|
|
for (let frame = 0; frame < sampleFrames; frame += 1) {
|
|
const value = frame + 1;
|
|
bytes.writeInt16LE(value, 44 + frame * 4);
|
|
bytes.writeInt16LE(value, 46 + frame * 4);
|
|
}
|
|
return bytes;
|
|
}
|
|
|
|
function pcm16StereoAudibleSpan(bytes: Uint8Array): {
|
|
readonly firstSample: number;
|
|
readonly endSampleExclusive: number;
|
|
readonly sampleFrames: number;
|
|
} {
|
|
const wav = Buffer.from(bytes);
|
|
assert.equal(wav.toString("ascii", 0, 4), "RIFF");
|
|
assert.equal(wav.toString("ascii", 8, 12), "WAVE");
|
|
let dataStart = -1;
|
|
let dataBytes = 0;
|
|
for (let offset = 12; offset + 8 <= wav.length;) {
|
|
const id = wav.toString("ascii", offset, offset + 4);
|
|
const size = wav.readUInt32LE(offset + 4);
|
|
if (id === "fmt ") {
|
|
assert.equal(wav.readUInt16LE(offset + 8), 1);
|
|
assert.equal(wav.readUInt16LE(offset + 10), 2);
|
|
assert.equal(wav.readUInt32LE(offset + 12), 48_000);
|
|
assert.equal(wav.readUInt16LE(offset + 22), 16);
|
|
}
|
|
if (id === "data") {
|
|
dataStart = offset + 8;
|
|
dataBytes = size;
|
|
break;
|
|
}
|
|
offset += 8 + size + (size % 2);
|
|
}
|
|
assert(dataStart >= 0 && dataStart + dataBytes <= wav.length, "normalized WAV has no complete data chunk");
|
|
const sampleFrames = Math.floor(dataBytes / 4);
|
|
let firstSample = sampleFrames;
|
|
let endSampleExclusive = 0;
|
|
for (let frame = 0; frame < sampleFrames; frame += 1) {
|
|
const offset = dataStart + frame * 4;
|
|
if (Math.max(Math.abs(wav.readInt16LE(offset)), Math.abs(wav.readInt16LE(offset + 2))) > 64) {
|
|
firstSample = Math.min(firstSample, frame);
|
|
endSampleExclusive = frame + 1;
|
|
}
|
|
}
|
|
assert(firstSample < endSampleExclusive, "normalized WAV contains no audible samples");
|
|
return { firstSample, endSampleExclusive, sampleFrames };
|
|
}
|
|
|
|
function presentationSampleFrames(stream: MediaAudioStream): number {
|
|
assert(stream.startPts !== undefined && stream.endPts !== undefined);
|
|
const startNumerator = BigInt(stream.startPts.ticks) * BigInt(stream.startPts.timeBase.numerator);
|
|
const startDenominator = BigInt(stream.startPts.timeBase.denominator);
|
|
const endNumerator = BigInt(stream.endPts.ticks) * BigInt(stream.endPts.timeBase.numerator);
|
|
const endDenominator = BigInt(stream.endPts.timeBase.denominator);
|
|
const numerator = (endNumerator * startDenominator - startNumerator * endDenominator) * 48_000n;
|
|
const denominator = endDenominator * startDenominator;
|
|
return Number((numerator * 2n + denominator) / (denominator * 2n));
|
|
}
|
|
|
|
async function fixture(dataRoot: string): Promise<string> {
|
|
const primary = join(dataRoot, "primary.mp4");
|
|
const cover = join(dataRoot, "cover.jpg");
|
|
const source = join(dataRoot, "source.mp4");
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y",
|
|
"-f", "lavfi", "-i", "testsrc2=s=160x96:r=24:d=1",
|
|
"-f", "lavfi", "-i", "sine=frequency=1000:sample_rate=32000:duration=1.05",
|
|
"-map", "0:v:0", "-map", "1:a:0",
|
|
"-c:v", "libx264", "-pix_fmt", "yuv420p",
|
|
"-c:a", "aac",
|
|
primary,
|
|
]);
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i", "color=c=red:s=160x96", "-frames:v", "1", cover,
|
|
]);
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-i", primary, "-i", cover,
|
|
"-map", "0:v:0", "-map", "0:a:0", "-map", "1:v:0",
|
|
"-c", "copy", "-disposition:v:1", "attached_pic", source,
|
|
]);
|
|
return source;
|
|
}
|
|
|
|
function need(
|
|
id: string,
|
|
capability: CapabilityRef,
|
|
returns: TypeRef,
|
|
constraints: CanonicalValue,
|
|
): Need {
|
|
return {
|
|
id,
|
|
capability,
|
|
returns,
|
|
constraints,
|
|
result: `record:${id}`,
|
|
};
|
|
}
|
|
|
|
async function fulfillInline(resources: MemoryResourceStore, request: Need): Promise<CanonicalValue> {
|
|
const provider = await handlerFor(request);
|
|
const result = await provider.handler({
|
|
command: { kind: "fulfill-need", id: `command:${request.id}`, need: request },
|
|
need: request,
|
|
resources,
|
|
credentials: {},
|
|
});
|
|
assert.equal(result.value.kind, "inline");
|
|
return result.value.kind === "inline" ? result.value.value : null;
|
|
}
|
|
|
|
async function handlerFor(request: Need): Promise<{ handler: ImmediateEndpointHandler; registration: EndpointRegistration }> {
|
|
const registry = new EndpointRegistry();
|
|
await createLocalMediaProvider({ processTimeoutMs: 30_000 }).install(registry);
|
|
const resolution = registry.resolve(request);
|
|
assert.equal(resolution.status, "resolved");
|
|
assert.equal(resolution.registration.kind, "immediate");
|
|
return { handler: resolution.registration.handler, registration: resolution.registration };
|
|
}
|
|
|
|
test("local media opts only reusable display materialization into transient execution", () => {
|
|
const provider = createLocalMediaProvider({});
|
|
assert.deepEqual(provider.offers.filter((offer) => offer.transient === true)
|
|
.map((offer) => offer.capability.name), [
|
|
"inspect-media",
|
|
"normalize-media",
|
|
"transform-media",
|
|
"extract-media-audio",
|
|
"extract-media-frame",
|
|
"render-still-video",
|
|
]);
|
|
assert.deepEqual(provider.offers.filter((offer) => offer.transient !== true)
|
|
.map((offer) => offer.capability.name), [
|
|
"project-speech-evidence-audio",
|
|
"render-timeline-audio",
|
|
"mux-program-media",
|
|
]);
|
|
});
|
|
|
|
async function inspectArtifact(resources: MemoryResourceStore, source: Awaited<ReturnType<MemoryResourceStore["put"]>>) {
|
|
const constraints = canonicalize({ source });
|
|
const request = need("need:media-inspect", mediaOperationsCapabilities.inspect,
|
|
mediaTypes.inspection, constraints);
|
|
const provider = await handlerFor(request);
|
|
const result = await provider.handler({
|
|
command: { kind: "fulfill-need", id: "command:media-inspect", need: request },
|
|
need: request,
|
|
resources,
|
|
credentials: {},
|
|
});
|
|
assert.equal(result.value.kind, "inline");
|
|
const value = result.value.kind === "inline" ? result.value.value : null;
|
|
verifyMediaInspection(value);
|
|
return value as unknown as MediaInspection;
|
|
}
|
|
|
|
async function normalizeArtifact(args: {
|
|
resources: MemoryResourceStore;
|
|
source: Awaited<ReturnType<MemoryResourceStore["put"]>>;
|
|
inspection: MediaInspection;
|
|
selection: ReturnType<typeof selectMediaStreams>;
|
|
frameRate: { readonly numerator: number; readonly denominator: number };
|
|
}): Promise<SynchronizedMedia> {
|
|
const constraints = canonicalize({
|
|
source: args.source,
|
|
inspection: args.inspection,
|
|
selection: args.selection,
|
|
frameRate: args.frameRate,
|
|
audio: { sampleRate: 48_000, channels: 2, codec: "pcm_s16le", loudness: "preserve" },
|
|
});
|
|
const request = need("need:media-normalize", mediaOperationsCapabilities.normalize,
|
|
mediaTypes.synchronized, constraints);
|
|
const provider = await handlerFor(request);
|
|
const result = await provider.handler({
|
|
command: { kind: "fulfill-need", id: "command:media-normalize", need: request },
|
|
need: request,
|
|
resources: args.resources,
|
|
credentials: {},
|
|
});
|
|
assert.equal(result.value.kind, "inline");
|
|
const value = result.value.kind === "inline" ? result.value.value : null;
|
|
verifySynchronizedMedia(value);
|
|
return value as unknown as SynchronizedMedia;
|
|
}
|
|
|
|
test("local media Provider enumerates attached pictures and jointly normalizes 32k AAC without inventing speech", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-media-local-"));
|
|
try {
|
|
const sourcePath = await fixture(root);
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(await readFile(sourcePath), "video/mp4");
|
|
const typedInspection = await inspectArtifact(resources, source);
|
|
assert.deepEqual(typedInspection.streams.map((stream) => [stream.index, stream.kind,
|
|
stream.kind === "video" ? stream.role : stream.codecName]), [
|
|
[0, "video", "moving"],
|
|
[1, "audio", "aac"],
|
|
[2, "video", "attached-picture"],
|
|
]);
|
|
const audio = typedInspection.streams.find((stream) => stream.kind === "audio");
|
|
assert.equal(audio?.kind === "audio" && audio.sampleRate, 32_000);
|
|
|
|
const selectionRequest = sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" },
|
|
audio: { mode: "default" },
|
|
spanAuthority: "video",
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
});
|
|
const selection = selectMediaStreams(typedInspection, selectionRequest);
|
|
assert.equal(selection.videoStreamIndex, 0);
|
|
assert.equal(selection.audioStreamIndex, 1);
|
|
const typed = await normalizeArtifact({ resources, source, inspection: typedInspection,
|
|
selection, frameRate: selectionRequest.frameRate });
|
|
assert.equal(typed.frameDomain.frameCount, 30);
|
|
assert.equal(synchronizedMediaSampleFrames(typed), 48_000);
|
|
assert.equal("basisResourceId" in typed, false);
|
|
assert.equal("narrativeDigest" in typed, false);
|
|
assert.equal(await resources.has(typed.visual!.artifact.resource), true);
|
|
assert.equal(await resources.has(typed.audio!.artifact.resource), true);
|
|
|
|
const visualBytes = await resources.get(typed.visual!.artifact.resource);
|
|
assert(visualBytes !== undefined);
|
|
const visualPath = join(root, "normalized-visual.mp4");
|
|
await writeFile(visualPath, visualBytes);
|
|
const probePath = join(root, "normalized-probe.json");
|
|
await new Promise<void>((resolve, reject) => {
|
|
const child = spawn("ffprobe", ["-v", "error", "-show_streams", "-of", "json", visualPath], {
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
windowsHide: true,
|
|
});
|
|
const chunks: Buffer[] = [];
|
|
let stderr = "";
|
|
child.stdout.on("data", (chunk: Buffer) => chunks.push(chunk));
|
|
child.stderr.on("data", (chunk: Buffer) => { stderr += chunk.toString(); });
|
|
child.on("error", reject);
|
|
child.on("close", async (code) => {
|
|
if (code !== 0) reject(new Error(stderr));
|
|
else {
|
|
await writeFile(probePath, Buffer.concat(chunks));
|
|
resolve();
|
|
}
|
|
});
|
|
});
|
|
const outputProbe = JSON.parse((await readFile(probePath)).toString("utf8")) as {
|
|
streams: Array<{ codec_type: string; nb_frames?: string }>;
|
|
};
|
|
assert.deepEqual(outputProbe.streams.map((stream) => stream.codec_type), ["video"]);
|
|
assert.equal(outputProbe.streams[0]?.nb_frames, "30");
|
|
const audioOutput = await inspectArtifact(resources, typed.audio!.artifact);
|
|
const normalizedAudio = audioOutput.streams.find((stream) => stream.kind === "audio");
|
|
assert.equal(normalizedAudio?.kind === "audio" && normalizedAudio.sampleRate, 48_000);
|
|
assert.equal(normalizedAudio?.kind === "audio" && normalizedAudio.channels, 2);
|
|
assert.equal(normalizedAudio?.kind === "audio" && normalizedAudio.decodedSampleFrames, 48_000);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("local media normalization materializes rotation and sample aspect before Spatial", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-display-geometry-"));
|
|
try {
|
|
const base = join(root, "base.mp4");
|
|
const sourcePath = join(root, "rotated.mp4");
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=160x96:r=4:d=0.5",
|
|
"-vf", "setsar=2/1", "-c:v", "libx264", "-pix_fmt", "yuv420p", base,
|
|
]);
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-display_rotation", "90", "-i", base, "-c", "copy", sourcePath,
|
|
]);
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(await readFile(sourcePath), "video/mp4");
|
|
const inspection = await inspectArtifact(resources, source);
|
|
const inputVideo = inspection.streams.find((stream) => stream.kind === "video");
|
|
assert.equal(inputVideo?.kind, "video");
|
|
if (inputVideo?.kind !== "video") return;
|
|
assert.deepEqual(inputVideo.sampleAspectRatio, { numerator: 2, denominator: 1 });
|
|
assert.equal(inputVideo.rotationDegrees, 90);
|
|
const request = sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" }, audio: { mode: "none" },
|
|
spanAuthority: "video", frameRate: { numerator: 4, denominator: 1 },
|
|
});
|
|
const selection = selectMediaStreams(inspection, request);
|
|
const normalized = await normalizeArtifact({ resources, source, inspection, selection, frameRate: request.frameRate });
|
|
assert.equal(normalized.visual?.width, 96);
|
|
assert.equal(normalized.visual?.height, 320);
|
|
const output = await inspectArtifact(resources, normalized.visual!.artifact);
|
|
const outputVideo = output.streams.find((stream) => stream.kind === "video");
|
|
assert.equal(outputVideo?.kind, "video");
|
|
if (outputVideo?.kind !== "video") return;
|
|
assert.deepEqual(outputVideo.sampleAspectRatio, { numerator: 1, denominator: 1 });
|
|
assert.equal(outputVideo.rotationDegrees, 0);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("animated WebP keeps its authored frame timing before fixed-rate normalization", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(animatedWebp, "image/webp");
|
|
const inspection = await inspectArtifact(resources, source);
|
|
assert.deepEqual(inspection.container.formatNames, ["webp", "webp-animation"]);
|
|
assert.deepEqual(inspection.streams.map((stream) => [stream.kind, stream.kind === "video" ? stream.role : undefined,
|
|
stream.decodedUnitCount]), [["video", "moving", 5]]);
|
|
const video = inspection.streams[0];
|
|
assert.equal(video?.kind, "video");
|
|
assert.equal(video?.startPts?.ticks, "0");
|
|
assert.equal(video?.endPts?.ticks, "500");
|
|
const request = sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" },
|
|
audio: { mode: "none" },
|
|
spanAuthority: "video",
|
|
frameRate: { numerator: 20, denominator: 1 },
|
|
});
|
|
const selection = selectMediaStreams(inspection, request);
|
|
const normalized = await normalizeArtifact({ resources, source, inspection, selection, frameRate: request.frameRate });
|
|
assert.equal(normalized.frameDomain.frameCount, 10);
|
|
assert.equal(normalized.visual?.width, 64);
|
|
assert.equal(normalized.visual?.height, 48);
|
|
const output = await inspectArtifact(resources, normalized.visual!.artifact);
|
|
assert.equal(output.streams[0]?.decodedUnitCount, 10);
|
|
});
|
|
|
|
test("animated GIF keeps its authored frame timing before fixed-rate normalization", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-gif-"));
|
|
try {
|
|
const path = join(root, "animated.gif");
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=64x48:r=10:d=0.5",
|
|
"-frames:v", "5", path,
|
|
]);
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(await readFile(path), "image/gif");
|
|
const inspection = await inspectArtifact(resources, source);
|
|
assert.deepEqual(inspection.streams.map((stream) => [stream.kind, stream.kind === "video" ? stream.role : undefined,
|
|
stream.decodedUnitCount]), [["video", "moving", 5]]);
|
|
const request = sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" },
|
|
audio: { mode: "none" },
|
|
spanAuthority: "video",
|
|
frameRate: { numerator: 20, denominator: 1 },
|
|
});
|
|
const selection = selectMediaStreams(inspection, request);
|
|
const normalized = await normalizeArtifact({ resources, source, inspection, selection, frameRate: request.frameRate });
|
|
assert.equal(normalized.frameDomain.frameCount, 10);
|
|
const output = await inspectArtifact(resources, normalized.visual!.artifact);
|
|
assert.equal(output.streams[0]?.decodedUnitCount, 10);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("local media Provider derives one exact 16 kHz mono WhisperX evidence artifact without a hidden second transcode", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-evidence-"));
|
|
try {
|
|
const sourcePath = join(root, "speech-master.wav");
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y",
|
|
"-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=2",
|
|
"-af", "atrim=start_sample=0:end_sample=48001,asetpts=N/SR/TB,aformat=sample_rates=48000:channel_layouts=stereo",
|
|
"-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", sourcePath,
|
|
]);
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(await readFile(sourcePath), "audio/wav");
|
|
const constraints = canonicalize({
|
|
domainId: "provider-media-test-domain",
|
|
source,
|
|
sourceSampleFrames: 48_001,
|
|
evidenceSampleFrames: 16_000,
|
|
});
|
|
const request = need(
|
|
"need:speech-evidence-audio",
|
|
speechEvidenceCapabilities.projectAudio,
|
|
speechEvidenceTypes.audio,
|
|
constraints,
|
|
);
|
|
const value = await fulfillInline(resources, request);
|
|
assertSpeechEvidenceAudioIdentity(value as unknown as SpeechEvidenceAudio);
|
|
const evidence = value as unknown as SpeechEvidenceAudio;
|
|
assert.equal(evidence.domainId, "provider-media-test-domain");
|
|
assert.equal(evidence.sampleFrames, 16_000);
|
|
const inspected = await inspectArtifact(resources, evidence.artifact);
|
|
const audio = inspected.streams.find((stream) => stream.kind === "audio");
|
|
assert.equal(audio?.kind, "audio");
|
|
assert.equal(audio?.kind === "audio" && audio.codecName, "pcm_s16le");
|
|
assert.equal(audio?.kind === "audio" && audio.sampleRate, 16_000);
|
|
assert.equal(audio?.kind === "audio" && audio.channels, 1);
|
|
assert.equal(audio?.kind === "audio" && audio.decodedSampleFrames, 16_000);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true }).catch(() => {});
|
|
}
|
|
});
|
|
|
|
test("local media Provider preserves one source A/V origin when audio starts later than picture", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-offset-"));
|
|
try {
|
|
const sourcePath = join(root, "offset.mkv");
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-copyts",
|
|
"-f", "lavfi", "-i", "testsrc2=s=160x96:r=24:d=1",
|
|
"-f", "lavfi", "-i", "sine=frequency=700:sample_rate=48000:duration=0.6",
|
|
"-filter_complex", "[0:v]setpts=PTS+0.4/TB[v];[1:a]asetpts=PTS+0.6/TB[a]",
|
|
"-map", "[v]", "-map", "[a]",
|
|
"-c:v", "libx264", "-pix_fmt", "yuv420p",
|
|
"-c:a", "pcm_s16le", "-avoid_negative_ts", "disabled",
|
|
sourcePath,
|
|
]);
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(await readFile(sourcePath), "video/x-matroska");
|
|
const inspection = await inspectArtifact(resources, source);
|
|
const request = sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" },
|
|
audio: { mode: "default" },
|
|
spanAuthority: "video",
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
});
|
|
const selection = selectMediaStreams(inspection, request);
|
|
const normalized = await normalizeArtifact({ resources, source, inspection, selection, frameRate: request.frameRate });
|
|
const video = inspection.streams.find((stream) => stream.kind === "video" && stream.index === selection.videoStreamIndex)!;
|
|
const audio = inspection.streams.find((stream) => stream.kind === "audio" && stream.index === selection.audioStreamIndex)!;
|
|
assert(video.startPts !== undefined && audio.startPts !== undefined && audio.endPts !== undefined);
|
|
const seconds = (value: NonNullable<typeof video.startPts>) =>
|
|
Number(value.ticks) * value.timeBase.numerator / value.timeBase.denominator;
|
|
const expectedHead = Math.round((seconds(audio.startPts) - seconds(video.startPts)) * 48_000);
|
|
const expectedEnd = Math.round((seconds(audio.endPts) - seconds(video.startPts)) * 48_000);
|
|
const expectedTail = 48_000 - Math.min(48_000, expectedEnd);
|
|
assert.equal(normalized.frameDomain.frameCount, 30);
|
|
const bytes = await resources.get(normalized.audio!.artifact.resource);
|
|
assert(bytes !== undefined);
|
|
const audible = pcm16StereoAudibleSpan(bytes);
|
|
assert.equal(audible.sampleFrames, 48_000);
|
|
assert.ok(Math.abs(audible.firstSample - expectedHead) <= 64,
|
|
`normalized audio begins at ${audible.firstSample}, expected ${expectedHead}`);
|
|
assert.ok(Math.abs(audible.endSampleExclusive - (48_000 - expectedTail)) <= 64,
|
|
`normalized audio ends at ${audible.endSampleExclusive}, expected ${48_000 - expectedTail}`);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a video-only generated MP4 remains visual-only and cannot satisfy a requested audio stream", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-silent-"));
|
|
try {
|
|
const sourcePath = join(root, "silent.mp4");
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=160x96:r=24:d=0.5",
|
|
"-frames:v", "12", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-an", sourcePath,
|
|
]);
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(await readFile(sourcePath), "video/mp4");
|
|
const inspection = await inspectArtifact(resources, source);
|
|
const request = sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" },
|
|
audio: { mode: "none" },
|
|
spanAuthority: "video",
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
});
|
|
const selection = selectMediaStreams(inspection, request);
|
|
const normalized = await normalizeArtifact({ resources, source, inspection, selection, frameRate: request.frameRate });
|
|
assert.equal(normalized.frameDomain.frameCount, 15);
|
|
assert.ok(normalized.visual);
|
|
assert.equal(normalized.audio, undefined);
|
|
|
|
const invalid = sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" },
|
|
audio: { mode: "default" },
|
|
spanAuthority: "video",
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
});
|
|
assert.throws(() => selectMediaStreams(inspection, invalid), /no eligible stream/u);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("local media Provider transforms A/V and extracts ordinary audio and frame Artifacts", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-ordinary-ops-"));
|
|
try {
|
|
const sourcePath = await fixture(root);
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(await readFile(sourcePath), "video/mp4");
|
|
const inspection = await inspectArtifact(resources, source);
|
|
const selectionRequest = sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" },
|
|
audio: { mode: "default" },
|
|
spanAuthority: "video",
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
});
|
|
const selection = selectMediaStreams(inspection, selectionRequest);
|
|
const normalized = await normalizeArtifact({
|
|
resources, source, inspection, selection, frameRate: selectionRequest.frameRate,
|
|
});
|
|
const executeArtifact = async (request: Need) => {
|
|
const provider = await handlerFor(request);
|
|
const result = await provider.handler({
|
|
command: { kind: "fulfill-need", id: `command:${request.id}`, need: request },
|
|
need: request,
|
|
resources,
|
|
credentials: {},
|
|
});
|
|
assert.equal(result.value.kind, "blob");
|
|
return result.value.kind === "blob" ? result.value : (() => { throw new Error("expected Blob"); })();
|
|
};
|
|
|
|
const transformed = await executeArtifact(need(
|
|
"need:transform",
|
|
mediaOperationsCapabilities.transform,
|
|
blobTypes.blob,
|
|
canonicalize({
|
|
media: normalized,
|
|
program: {
|
|
operations: [
|
|
{ kind: "trim", tailSec: 0.2 },
|
|
{ kind: "retime", rate: 2 },
|
|
],
|
|
},
|
|
}),
|
|
));
|
|
const transformedInspection = await inspectArtifact(resources, transformed);
|
|
assert.equal(transformedInspection.streams.find((item) => item.kind === "video")?.decodedUnitCount, 12);
|
|
|
|
const audioIndex = inspection.streams.find((item) => item.kind === "audio")!.index;
|
|
const extractedAudio = await executeArtifact(need(
|
|
"need:extract-audio",
|
|
mediaOperationsCapabilities.extractAudio,
|
|
blobTypes.blob,
|
|
canonicalize({
|
|
source,
|
|
streamIndex: audioIndex,
|
|
output: { container: "wav", codec: "pcm_s16le", sampleRate: 48_000, channels: 2 },
|
|
}),
|
|
));
|
|
assert.equal(extractedAudio.mediaType, "audio/wav");
|
|
const audioInspection = await inspectArtifact(resources, extractedAudio);
|
|
const audio = audioInspection.streams.find((item) => item.kind === "audio");
|
|
assert.ok(audio?.kind === "audio" && audio.sampleRate === 48_000 && audio.channels === 2);
|
|
|
|
const video = inspection.streams.find((item) => item.kind === "video" && item.role === "moving")!;
|
|
const extractedFrame = await executeArtifact(need(
|
|
"need:extract-frame",
|
|
mediaOperationsCapabilities.extractFrame,
|
|
blobTypes.blob,
|
|
canonicalize({
|
|
source,
|
|
streamIndex: video.index,
|
|
sourceFrameCount: video.decodedUnitCount,
|
|
at: { kind: "last" },
|
|
output: { format: "png" },
|
|
}),
|
|
));
|
|
assert.equal(extractedFrame.mediaType, "image/png");
|
|
const frameInspection = await inspectArtifact(resources, extractedFrame);
|
|
assert.equal(frameInspection.streams.find((item) => item.kind === "video")?.decodedUnitCount, 1);
|
|
|
|
const stillVideo = await executeArtifact(need(
|
|
"need:render-still-video",
|
|
mediaOperationsCapabilities.renderStill,
|
|
blobTypes.blob,
|
|
canonicalize({
|
|
request: {
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
frameCount: 15,
|
|
output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
|
|
segments: [{ startFrame: 0, endFrameExclusive: 15, source: extractedFrame }],
|
|
},
|
|
}),
|
|
));
|
|
assert.equal(stillVideo.mediaType, "video/mp4");
|
|
const stillInspection = await inspectArtifact(resources, stillVideo);
|
|
assert.equal(stillInspection.streams.length, 1);
|
|
assert.equal(stillInspection.streams[0]?.kind, "video");
|
|
assert.equal(stillInspection.streams[0]?.decodedUnitCount, 15);
|
|
|
|
const spread = await executeArtifact(need(
|
|
"need:render-still-spread",
|
|
mediaOperationsCapabilities.renderStill,
|
|
blobTypes.blob,
|
|
canonicalize({
|
|
request: {
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
frameCount: 15,
|
|
output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
|
|
segments: [
|
|
{ startFrame: 0, endFrameExclusive: 10, source: extractedFrame },
|
|
{ startFrame: 10, endFrameExclusive: 15, source: extractedFrame },
|
|
],
|
|
},
|
|
}),
|
|
));
|
|
const spreadInspection = await inspectArtifact(resources, spread);
|
|
assert.equal(spreadInspection.streams.length, 1);
|
|
assert.equal(spreadInspection.streams[0]?.decodedUnitCount, 15, "two pictures spread over the same frame count");
|
|
const twentyFourSpread = await executeArtifact(need(
|
|
"need:render-still-24fps-spread",
|
|
mediaOperationsCapabilities.renderStill,
|
|
blobTypes.blob,
|
|
canonicalize({
|
|
request: {
|
|
frameRate: { numerator: 24, denominator: 1 },
|
|
frameCount: 144,
|
|
output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
|
|
segments: [
|
|
{ startFrame: 0, endFrameExclusive: 24, source: extractedFrame },
|
|
{ startFrame: 24, endFrameExclusive: 96, source: extractedFrame },
|
|
{ startFrame: 96, endFrameExclusive: 144, source: extractedFrame },
|
|
],
|
|
},
|
|
}),
|
|
));
|
|
const twentyFourInspection = await inspectArtifact(resources, twentyFourSpread);
|
|
assert.equal(twentyFourInspection.streams[0]?.decodedUnitCount, 144,
|
|
"three Still segments at 24 fps must preserve the complete frame domain");
|
|
const stillSelection = selectMediaStreams(stillInspection, sealMediaSelectionRequest({
|
|
video: { mode: "primary-moving" },
|
|
audio: { mode: "none" },
|
|
spanAuthority: "video",
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
}));
|
|
const stillNormalized = await normalizeArtifact({
|
|
resources,
|
|
source: stillVideo,
|
|
inspection: stillInspection,
|
|
selection: stillSelection,
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
});
|
|
assert.equal(stillNormalized.frameDomain.frameCount, 15);
|
|
assert.equal(stillNormalized.audio, undefined);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true }).catch(() => {});
|
|
}
|
|
});
|
|
|
|
test("StillVideo converts authored RGB pictures to tagged BT.709 in both render paths", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-bt709-"));
|
|
try {
|
|
const width = 1280;
|
|
const height = 720;
|
|
const expected = [0x20, 0xc0, 0x40] as const;
|
|
const rgb = Buffer.alloc(width * height * 3);
|
|
for (let offset = 0; offset < rgb.length; offset += 3) {
|
|
rgb[offset] = expected[0];
|
|
rgb[offset + 1] = expected[1];
|
|
rgb[offset + 2] = expected[2];
|
|
}
|
|
const png = runBytes("ffmpeg", [
|
|
"-v", "error", "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", `${width}x${height}`,
|
|
"-i", "pipe:0", "-frames:v", "1", "-f", "image2pipe", "-c:v", "png", "pipe:1",
|
|
], rgb);
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(png, "image/png");
|
|
const render = async (id: string, segments: readonly {
|
|
readonly startFrame: number;
|
|
readonly endFrameExclusive: number;
|
|
readonly source: typeof source;
|
|
}[]) => {
|
|
const request = need(
|
|
`need:${id}`,
|
|
mediaOperationsCapabilities.renderStill,
|
|
blobTypes.blob,
|
|
canonicalize({
|
|
request: {
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
frameCount: 4,
|
|
output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
|
|
segments,
|
|
},
|
|
}),
|
|
);
|
|
const provider = await handlerFor(request);
|
|
const result = await provider.handler({
|
|
command: { kind: "fulfill-need", id: `command:${id}`, need: request },
|
|
need: request,
|
|
resources,
|
|
credentials: {},
|
|
});
|
|
assert.equal(result.value.kind, "blob");
|
|
assert.equal(result.value.kind === "blob" && result.value.mediaType, "video/mp4");
|
|
const artifact = result.value.kind === "blob" ? result.value : undefined;
|
|
assert(artifact !== undefined);
|
|
const bytes = await resources.get(artifact.resource);
|
|
assert(bytes !== undefined);
|
|
const path = join(root, `${id}.mp4`);
|
|
await writeFile(path, bytes);
|
|
return path;
|
|
};
|
|
const paths = await Promise.all([
|
|
render("single", [{ startFrame: 0, endFrameExclusive: 4, source }]),
|
|
render("spread", [
|
|
{ startFrame: 0, endFrameExclusive: 2, source },
|
|
{ startFrame: 2, endFrameExclusive: 4, source },
|
|
]),
|
|
]);
|
|
for (const path of paths) {
|
|
const probe = spawnSync("ffprobe", [
|
|
"-v", "error", "-select_streams", "v:0",
|
|
"-show_entries", "stream=color_range,color_space,color_transfer,color_primaries",
|
|
"-of", "json", path,
|
|
], { encoding: "utf8", windowsHide: true });
|
|
assert.equal(probe.status, 0, probe.stderr);
|
|
const stream = (JSON.parse(probe.stdout) as { streams: Record<string, string>[] }).streams[0]!;
|
|
assert.equal(stream.color_range, "tv");
|
|
assert.equal(stream.color_space, "bt709");
|
|
assert.equal(stream.color_transfer, "bt709");
|
|
assert.equal(stream.color_primaries, "bt709");
|
|
|
|
const decoded = runBytes("ffmpeg", [
|
|
"-v", "error", "-i", path, "-frames:v", "1",
|
|
"-vf", "scale=in_color_matrix=bt709:in_range=tv:out_range=pc,format=rgb24",
|
|
"-f", "rawvideo", "pipe:1",
|
|
]);
|
|
const center = (Math.floor(height / 2) * width + Math.floor(width / 2)) * 3;
|
|
expected.forEach((channel, index) => {
|
|
assert.ok(Math.abs(decoded[center + index]! - channel) <= 3,
|
|
`${path} channel ${index} decoded as ${decoded[center + index]}, expected ${channel}`);
|
|
});
|
|
}
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true }).catch(() => {});
|
|
}
|
|
});
|
|
|
|
test("local media Provider renders one frame-domain audio plan and muxes exactly one video-only visual with it", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-program-"));
|
|
try {
|
|
const firstPath = join(root, "first.wav");
|
|
const secondPath = join(root, "second.wav");
|
|
const visualPath = join(root, "visual.mp4");
|
|
await Promise.all([
|
|
run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i",
|
|
"sine=frequency=440:sample_rate=48000:duration=0.5",
|
|
"-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", firstPath,
|
|
]),
|
|
run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i",
|
|
"sine=frequency=880:sample_rate=48000:duration=0.5",
|
|
"-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", secondPath,
|
|
]),
|
|
run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=160x96:r=30:d=1",
|
|
"-frames:v", "30", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-an", visualPath,
|
|
]),
|
|
]);
|
|
|
|
const resources = new MemoryResourceStore();
|
|
const [first, second, visualArtifact] = await Promise.all([
|
|
resources.put(await readFile(firstPath), "audio/wav"),
|
|
resources.put(await readFile(secondPath), "audio/wav"),
|
|
resources.put(await readFile(visualPath), "video/mp4"),
|
|
]);
|
|
const plan = sealAudioProgramPlan({
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
frameCount: 30,
|
|
sampleRate: 48_000,
|
|
sampleFrames: 48_000,
|
|
clips: [
|
|
{
|
|
id: "speech:first",
|
|
artifact: first,
|
|
targetStartSample: 0,
|
|
targetEndSampleExclusive: 24_000,
|
|
sourceSampleFrames: 24_000,
|
|
sourceStartSample: 0,
|
|
sourceEndSampleExclusive: 24_000,
|
|
sourceLoop: false,
|
|
sourcePhaseSample: 0,
|
|
playbackRate: 1,
|
|
mixStartSample: 0,
|
|
mixEndSampleExclusive: 24_000,
|
|
gain: 1,
|
|
fadeInSamples: 0,
|
|
fadeOutSamples: 0,
|
|
},
|
|
{
|
|
id: "speech:second",
|
|
artifact: second,
|
|
targetStartSample: 24_000,
|
|
targetEndSampleExclusive: 48_000,
|
|
sourceSampleFrames: 24_000,
|
|
sourceStartSample: 0,
|
|
sourceEndSampleExclusive: 24_000,
|
|
sourceLoop: false,
|
|
sourcePhaseSample: 0,
|
|
playbackRate: 1,
|
|
mixStartSample: 24_000,
|
|
mixEndSampleExclusive: 48_000,
|
|
gain: 1,
|
|
fadeInSamples: 0,
|
|
fadeOutSamples: 0,
|
|
},
|
|
],
|
|
mix: { normalize: false, limiter: "none" },
|
|
});
|
|
const audioRequest = need(
|
|
"need:render-program-audio",
|
|
mediaOperationsCapabilities.renderAudio,
|
|
mediaTypes.timelineAudio,
|
|
canonicalize({ plan }),
|
|
);
|
|
const audioValue = await fulfillInline(resources, audioRequest);
|
|
verifyTimelineAudio(audioValue);
|
|
const audio = audioValue as unknown as TimelineAudio;
|
|
assert.equal(audio.sampleFrames, 48_000);
|
|
assert.equal(await resources.has(audio.artifact.resource), true);
|
|
|
|
const visual = sealTimelineVisual({
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
frameCount: 30,
|
|
canvas: { width: 160, height: 96 },
|
|
artifact: visualArtifact,
|
|
});
|
|
const muxRequest = need(
|
|
"need:mux-program-media",
|
|
mediaOperationsCapabilities.mux,
|
|
mediaTypes.muxed,
|
|
canonicalize({ visual, audio }),
|
|
);
|
|
const muxValue = await fulfillInline(resources, muxRequest);
|
|
verifyMuxedMedia(muxValue);
|
|
const muxed = muxValue as unknown as MuxedMedia;
|
|
assert.equal(muxed.presentationSampleFrames, 48_000);
|
|
assert.equal(await resources.has(muxed.artifact.resource), true);
|
|
|
|
const inspection = await inspectArtifact(resources, muxed.artifact);
|
|
const videos = inspection.streams.filter((stream) => stream.kind === "video");
|
|
const audios = inspection.streams.filter((stream) => stream.kind === "audio");
|
|
assert.equal(inspection.streams.length, 2);
|
|
assert.equal(videos.length, 1);
|
|
assert.equal(videos[0]?.decodedUnitCount, 30);
|
|
assert.equal(audios.length, 1);
|
|
assert.equal(audios[0]?.kind === "audio" && audios[0].sampleRate, 48_000);
|
|
assert.equal(audios[0]?.kind === "audio" && audios[0].channels, 2);
|
|
assert.equal(audios[0]?.kind === "audio" && presentationSampleFrames(audios[0]), 48_000,
|
|
"AAC packet duration/padding metadata must preserve the authoritative presentation span");
|
|
assert.ok(audios[0]?.kind === "audio" && audios[0].decodedSampleFrames >= 48_000,
|
|
"AAC coding frames may include padding, but must cover the complete presentation span");
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("local media Provider preserves the audio tail for a 30000/1001 frame domain", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-ntsc-mux-"));
|
|
try {
|
|
const visualPath = join(root, "visual.mp4");
|
|
const audioPath = join(root, "audio.wav");
|
|
const frameRate = { numerator: 30_000, denominator: 1_001 };
|
|
const frameCount = 913;
|
|
const sampleFrames = 1_462_261;
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=160x96:r=30000/1001",
|
|
"-frames:v", String(frameCount), "-c:v", "libx264", "-pix_fmt", "yuv420p", "-an", visualPath,
|
|
]);
|
|
await run("ffmpeg", [
|
|
"-v", "error", "-y", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000",
|
|
"-af", `atrim=end_sample=${sampleFrames}`, "-ac", "2", "-c:a", "pcm_s16le", audioPath,
|
|
]);
|
|
|
|
const resources = new MemoryResourceStore();
|
|
const [audioArtifact, visualArtifact] = await Promise.all([
|
|
resources.put(await readFile(audioPath), "audio/wav"),
|
|
resources.put(await readFile(visualPath), "video/mp4"),
|
|
]);
|
|
const audio = sealTimelineAudio({ artifact: audioArtifact, sampleFrames });
|
|
const visual = sealTimelineVisual({
|
|
frameRate,
|
|
frameCount,
|
|
canvas: { width: 160, height: 96 },
|
|
artifact: visualArtifact,
|
|
});
|
|
const muxValue = await fulfillInline(resources, need(
|
|
"need:mux-ntsc-program-media",
|
|
mediaOperationsCapabilities.mux,
|
|
mediaTypes.muxed,
|
|
canonicalize({ visual, audio }),
|
|
));
|
|
verifyMuxedMedia(muxValue);
|
|
const muxed = muxValue as unknown as MuxedMedia;
|
|
const inspection = await inspectArtifact(resources, muxed.artifact);
|
|
const audioStream = inspection.streams.find((stream): stream is MediaAudioStream => stream.kind === "audio");
|
|
assert.ok(audioStream);
|
|
const presentedSamples = presentationSampleFrames(audioStream);
|
|
assert.ok(Math.abs(presentedSamples - sampleFrames) < 64,
|
|
`AAC presentation span must retain the complete NTSC timeline audio (delta ${presentedSamples - sampleFrames})`);
|
|
assert.ok(audioStream.decodedSampleFrames >= sampleFrames,
|
|
"AAC coding frames must cover the complete NTSC timeline audio");
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("local media Provider executes an end-aligned loop from the exact authored sample phase", {
|
|
skip: !hasMediaBinaries,
|
|
}, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-loop-"));
|
|
try {
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(rampWav(100), "audio/wav");
|
|
const plan = sealAudioProgramPlan({
|
|
frameRate: { numerator: 30, denominator: 1 },
|
|
frameCount: 30,
|
|
sampleRate: 48_000,
|
|
sampleFrames: 48_000,
|
|
clips: [{
|
|
id: "loop:end",
|
|
artifact: source,
|
|
targetStartSample: 0,
|
|
targetEndSampleExclusive: 250,
|
|
sourceSampleFrames: 100,
|
|
sourceStartSample: 0,
|
|
sourceEndSampleExclusive: 100,
|
|
sourceLoop: true,
|
|
sourcePhaseSample: 50,
|
|
playbackRate: 1,
|
|
mixStartSample: 0,
|
|
mixEndSampleExclusive: 250,
|
|
gain: 1,
|
|
fadeInSamples: 0,
|
|
fadeOutSamples: 0,
|
|
}],
|
|
mix: { normalize: false, limiter: "none" },
|
|
});
|
|
const value = await fulfillInline(resources, need(
|
|
"need:render-loop-audio",
|
|
mediaOperationsCapabilities.renderAudio,
|
|
mediaTypes.timelineAudio,
|
|
canonicalize({ plan }),
|
|
));
|
|
verifyTimelineAudio(value);
|
|
const audio = value as unknown as TimelineAudio;
|
|
assert.equal(audio.sampleFrames, 48_000);
|
|
const wav = await resources.get(audio.artifact.resource);
|
|
assert(wav);
|
|
const wavPath = join(root, "loop.wav");
|
|
const rawPath = join(root, "loop.raw");
|
|
await writeFile(wavPath, wav);
|
|
await run("ffmpeg", ["-v", "error", "-y", "-i", wavPath, "-f", "s16le", "-acodec", "pcm_s16le", rawPath]);
|
|
const raw = await readFile(rawPath);
|
|
assert.equal(raw.readInt16LE(0), 51, "end alignment must begin at source phase 50");
|
|
assert.equal(raw.readInt16LE(49 * 4), 100);
|
|
assert.equal(raw.readInt16LE(50 * 4), 1, "the complete authored source interval must loop");
|
|
assert.equal(raw.readInt16LE(249 * 4), 100);
|
|
assert.equal(raw.readInt16LE(250 * 4), 0, "silence is absence after the audible target interval");
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("audio range preserves loop phase, tempo and intersected fades from the full programme", { skip: !hasMediaBinaries }, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-audio-range-"));
|
|
try {
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(rampWav(4800), "audio/wav");
|
|
const plan = sealAudioProgramPlan({
|
|
frameRate: { numerator: 30, denominator: 1 }, frameCount: 60,
|
|
sampleRate: 48000, sampleFrames: 96000,
|
|
clips: [{ id: "loop", artifact: source, targetStartSample: 0, targetEndSampleExclusive: 96000,
|
|
sourceSampleFrames: 4800, sourceStartSample: 0, sourceEndSampleExclusive: 4800,
|
|
sourceLoop: true, sourcePhaseSample: 1700, playbackRate: 1.25,
|
|
mixStartSample: 0, mixEndSampleExclusive: 96000, gain: 0.8,
|
|
fadeInSamples: 48000, fadeOutSamples: 30000 }], mix: { normalize: false, limiter: "none" },
|
|
});
|
|
const pcm = async (name: string, range?: { startFrame: number; endFrameExclusive: number }) => {
|
|
const value = await fulfillInline(resources, need(`need:${name}`, mediaOperationsCapabilities.renderAudio,
|
|
mediaTypes.timelineAudio, canonicalize({ plan, ...(range === undefined ? {} : { range }) })));
|
|
verifyTimelineAudio(value);
|
|
const audio = value as unknown as TimelineAudio;
|
|
const path = join(root, `${name}.wav`), raw = join(root, `${name}.raw`);
|
|
await writeFile(path, (await resources.get(audio.artifact.resource))!);
|
|
await run("ffmpeg", ["-v", "error", "-y", "-i", path, "-f", "s16le", raw]);
|
|
return { bytes: await readFile(raw), samples: audio.sampleFrames };
|
|
};
|
|
const full = await pcm("full");
|
|
const range = await pcm("range", { startFrame: 15, endFrameExclusive: 45 });
|
|
assert.equal(range.samples, 48000);
|
|
const expected = full.bytes.subarray(24000 * 4, 72000 * 4);
|
|
assert.equal(range.bytes.length, expected.length);
|
|
let maximumError = 0;
|
|
for (let offset = 0; offset < expected.length; offset += 2) {
|
|
maximumError = Math.max(maximumError, Math.abs(range.bytes.readInt16LE(offset) - expected.readInt16LE(offset)));
|
|
}
|
|
// Different final filter block boundaries can round float PCM to the adjacent s16 value.
|
|
assert.ok(maximumError <= 1, `Selected audio differs by ${maximumError} PCM units`);
|
|
} finally { await rm(root, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test("audio envelopes and audible regions preserve sample phase through gaps and range renders", { skip: !hasMediaBinaries }, async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "hypit-audio-presentation-"));
|
|
try {
|
|
const resources = new MemoryResourceStore();
|
|
const source = await resources.put(rampWav(4800), "audio/wav");
|
|
const plan = sealAudioProgramPlan({
|
|
frameRate: { numerator: 30, denominator: 1 }, frameCount: 12, sampleRate: 48000, sampleFrames: 19200,
|
|
clips: [{ id: "voice", artifact: source, targetStartSample: 1600, targetEndSampleExclusive: 17600,
|
|
sourceSampleFrames: 4800, sourceStartSample: 0, sourceEndSampleExclusive: 4800,
|
|
sourceLoop: true, sourcePhaseSample: 0, playbackRate: 1,
|
|
mixStartSample: 1600, mixEndSampleExclusive: 17600, gain: 1,
|
|
fadeInSamples: 0, fadeOutSamples: 0,
|
|
gainEnvelope: [{ sample: 1600, gain: 0 }, { sample: 17600, gain: 1 }],
|
|
audibility: [{ startSample: 1600, endSampleExclusive: 4800 }, { startSample: 8000, endSampleExclusive: 17600 }],
|
|
}], mix: { normalize: false, limiter: "none" },
|
|
});
|
|
const pcm = async (name: string, range?: { startFrame: number; endFrameExclusive: number }) => {
|
|
const value = await fulfillInline(resources, need(`need:${name}`, mediaOperationsCapabilities.renderAudio,
|
|
mediaTypes.timelineAudio, canonicalize({ plan, ...(range === undefined ? {} : { range }) })));
|
|
verifyTimelineAudio(value);
|
|
const path = join(root, `${name}.wav`), raw = join(root, `${name}.raw`);
|
|
await writeFile(path, (await resources.get((value as unknown as TimelineAudio).artifact.resource))!);
|
|
await run("ffmpeg", ["-v", "error", "-y", "-i", path, "-f", "s16le", raw]);
|
|
return readFile(raw);
|
|
};
|
|
const full = await pcm("full");
|
|
for (const sample of [0, 1599, 4800, 6400, 7999, 17600, 18000]) assert.equal(full.readInt16LE(sample * 4), 0);
|
|
for (const sample of [1600, 2401, 4799, 8000, 8001, 12000, 17599]) {
|
|
const local = sample - 1600;
|
|
const expected = (local % 4800 + 1) * local / 16000;
|
|
assert.ok(Math.abs(full.readInt16LE(sample * 4) - expected) <= 1, `Wrong envelope or source phase at ${sample}`);
|
|
}
|
|
const selected = await pcm("selected", { startFrame: 4, endFrameExclusive: 10 });
|
|
const expected = full.subarray(6400 * 4, 16000 * 4);
|
|
assert.equal(selected.length, expected.length);
|
|
for (let index = 0; index < selected.length; index += 2) {
|
|
assert.ok(Math.abs(selected.readInt16LE(index) - expected.readInt16LE(index)) <= 1);
|
|
}
|
|
} finally { await rm(root, { recursive: true, force: true }); }
|
|
});
|