mirror of
https://github.com/hypit-ai/hypit.git
synced 2026-11-18 20:20:34 +00:00
541 lines
21 KiB
TypeScript
541 lines
21 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
|
|
// This suite verifies the Runtime scheduler and Executor boundary.
|
|
|
|
import {
|
|
ProducerRegistry,
|
|
Executor,
|
|
EndpointRegistry,
|
|
} from "@hypit/executor";
|
|
import type { AsyncEndpoint } from "@hypit/endpoint";
|
|
import {
|
|
applyOperationUpdate,
|
|
InProcessBuildScheduler,
|
|
} from "@hypit/runtime";
|
|
import type { BuildStore, OperationSnapshot, OperationStore, OperationUpdate, RuntimeCommandExecutor, RuntimeExecutionContext, ScheduledBuild, ScheduledBuildResult } from "@hypit/runtime";
|
|
import type { BuildView } from "@hypit/kernel";
|
|
import { capabilities, createGreetingBuild, createParallelGreetingBuild, producers as greetingProducers, types } from "../execution-scenario.js";
|
|
import { completeChainStep, createChainDefinition } from "../execution-scenario.js";
|
|
|
|
function submission(id: string, state: BuildView): ScheduledBuild {
|
|
return {
|
|
id,
|
|
definition: { format: "hypit.build-definition@1", program: state.program, plan: state.plan },
|
|
facts: [],
|
|
};
|
|
}
|
|
|
|
function continuation(result: ScheduledBuildResult): ScheduledBuild {
|
|
return { id: result.id, definition: result.definition, facts: result.facts };
|
|
}
|
|
|
|
function memoryOperations(): OperationStore {
|
|
const values = new Map<string, OperationSnapshot>();
|
|
return {
|
|
async create(operation) {
|
|
values.set(operation.id, structuredClone(operation));
|
|
return structuredClone(operation);
|
|
},
|
|
async read(id) {
|
|
const value = values.get(id);
|
|
return value === undefined ? undefined : structuredClone(value);
|
|
},
|
|
async list(query) {
|
|
return [...values.values()].filter((item) =>
|
|
(query.build === undefined || item.build === query.build)
|
|
&& (query.command === undefined || item.command === query.command)
|
|
&& (query.endpoint === undefined || item.endpoint === query.endpoint));
|
|
},
|
|
async update(id, update: OperationUpdate) {
|
|
const current = values.get(id);
|
|
if (current === undefined) throw new Error(`Operation ${id} does not exist`);
|
|
const next = applyOperationUpdate(current, structuredClone(update));
|
|
values.set(id, next);
|
|
return structuredClone(next);
|
|
},
|
|
};
|
|
}
|
|
|
|
test("a durable Scheduler keeps preparation and Record lookup on the indexed Build execution path", async () => {
|
|
const definition = createChainDefinition(1);
|
|
let appends = 0;
|
|
const builds: BuildStore = {
|
|
async create() { throw new Error("unused"); },
|
|
async read() { throw new Error("unused"); },
|
|
async append(_build, fact) {
|
|
appends++;
|
|
assert.equal(fact.kind, "producer-applied");
|
|
},
|
|
};
|
|
let preparations = 0;
|
|
let executions = 0;
|
|
const executor: RuntimeCommandExecutor = {
|
|
prepare(execution) {
|
|
preparations++;
|
|
return {
|
|
state: execution.view(),
|
|
runnable: execution.commands().map((command) => ({ command, resources: [] })),
|
|
blocked: [],
|
|
};
|
|
},
|
|
async executeCommand(execution, descriptor) {
|
|
executions++;
|
|
assert.equal(descriptor.command.kind, "invoke-producer");
|
|
return { status: "completed", event: completeChainStep(descriptor.command) };
|
|
},
|
|
};
|
|
const [result] = await new InProcessBuildScheduler(executor, { buildStore: builds }).run([{
|
|
id: "indexed-chain",
|
|
definition,
|
|
facts: [],
|
|
}]);
|
|
assert.equal(result?.status, "complete");
|
|
assert.equal(preparations, 1);
|
|
assert.equal(executions, 1);
|
|
assert.equal(appends, 1);
|
|
});
|
|
|
|
|
|
function configuredExecutor(options: {
|
|
readonly resource: string;
|
|
readonly defaultConcurrency: number;
|
|
readonly observe: (active: number) => void;
|
|
readonly endpointId?: string;
|
|
}) {
|
|
const producers = new ProducerRegistry();
|
|
const endpoints = new EndpointRegistry();
|
|
registerGreetingProducers(producers);
|
|
let active = 0;
|
|
let calls = 0;
|
|
endpoints.registerImmediateEndpoint(
|
|
options.endpointId ?? "fixture.seedance",
|
|
capabilities.generation,
|
|
types.generated,
|
|
async () => {
|
|
calls += 1;
|
|
active += 1;
|
|
options.observe(active);
|
|
await new Promise<void>((resolve) => setTimeout(resolve, 10));
|
|
active -= 1;
|
|
return {
|
|
value: (`Generated ${calls}`),
|
|
};
|
|
},
|
|
{
|
|
scheduling: {
|
|
resources: [{
|
|
id: options.resource,
|
|
limit: options.defaultConcurrency,
|
|
}],
|
|
},
|
|
},
|
|
);
|
|
return { executor: new Executor({ producers, endpoints }), endpoints, getCalls: () => calls };
|
|
}
|
|
|
|
function registerGreetingProducers(producers: ProducerRegistry): void {
|
|
producers.registerProducer(greetingProducers.makePrompt, ({ inputs }) => {
|
|
const intent = inputs.intent!;
|
|
const name = (intent.value as { readonly name: string }).name;
|
|
return { kind: "output", value: (`Greet ${name}`) };
|
|
});
|
|
producers.registerProducer(greetingProducers.requestText, ({ inputs }) => {
|
|
const prompt = inputs.prompt!;
|
|
return { kind: "need", constraints: { prompt: prompt.value } };
|
|
});
|
|
producers.registerProducer(greetingProducers.assemble, ({ inputs }) => {
|
|
const generated = inputs.generated!;
|
|
return { kind: "output", value: ({ text: generated.value }) };
|
|
});
|
|
}
|
|
|
|
function asyncExecutor(
|
|
endpoint: AsyncEndpoint,
|
|
operations: OperationStore,
|
|
) {
|
|
const producers = new ProducerRegistry();
|
|
registerGreetingProducers(producers);
|
|
const endpoints = new EndpointRegistry();
|
|
endpoints.registerAsyncEndpoint(
|
|
"generation.local",
|
|
capabilities.generation,
|
|
types.generated,
|
|
endpoint,
|
|
{
|
|
scheduling: {
|
|
resources: [
|
|
{ id: "pool:fixture.account", limit: 1 },
|
|
{ id: "capacity:fixture.account/generation", limit: 1 },
|
|
],
|
|
},
|
|
},
|
|
);
|
|
return new Executor({ producers, endpoints, operations });
|
|
}
|
|
|
|
test("one local Scheduler shares an Endpoint resource across multiple Builds", async () => {
|
|
let maximumActive = 0;
|
|
const { executor, getCalls } = configuredExecutor({
|
|
resource: "pool:fixture.account",
|
|
defaultConcurrency: 1,
|
|
observe(active) {
|
|
maximumActive = Math.max(maximumActive, active);
|
|
},
|
|
});
|
|
const scheduler = new InProcessBuildScheduler(executor);
|
|
const results = await scheduler.run([
|
|
submission("video-a", createGreetingBuild()),
|
|
submission("video-b", createGreetingBuild()),
|
|
]);
|
|
|
|
assert.deepEqual(results.map((result) => result.status), ["complete", "complete"]);
|
|
assert.equal(getCalls(), 2);
|
|
assert.equal(maximumActive, 1);
|
|
assert.equal(results.every((result) =>
|
|
result.outcomes.some((entry) => entry.resources.includes("pool:fixture.account"))), true);
|
|
});
|
|
|
|
test("independent paid commands inside one Build may fill the same resource without duplicating their shared upstream", async () => {
|
|
let maximumActive = 0;
|
|
const { executor, getCalls } = configuredExecutor({
|
|
resource: "pool:fixture.account",
|
|
defaultConcurrency: 2,
|
|
observe(active) {
|
|
maximumActive = Math.max(maximumActive, active);
|
|
},
|
|
});
|
|
const [result] = await new InProcessBuildScheduler(executor).run([{
|
|
...submission("two-shots", createParallelGreetingBuild()),
|
|
}]);
|
|
|
|
assert.equal(result?.status, "complete");
|
|
assert.equal(getCalls(), 2);
|
|
assert.equal(maximumActive, 2);
|
|
assert.equal(result?.state.plan.steps.filter((step) =>
|
|
step.producer.name === greetingProducers.makePrompt.name).length, 1);
|
|
assert.equal(result?.state.records.filter((record) =>
|
|
record.type.name === types.generated.name).length, 2);
|
|
});
|
|
|
|
test("a failure stops new work and preserves a result from an already running sibling", async () => {
|
|
const operations = memoryOperations();
|
|
const producers = new ProducerRegistry();
|
|
const endpoints = new EndpointRegistry();
|
|
registerGreetingProducers(producers);
|
|
let calls = 0;
|
|
const endpoint: AsyncEndpoint = {
|
|
async start() {
|
|
calls += 1;
|
|
if (calls === 1) {
|
|
return {
|
|
status: "failed",
|
|
failure: { code: "PROVIDER_REJECTED", message: "the first provider request was rejected" },
|
|
};
|
|
}
|
|
await new Promise<void>((resolve) => setTimeout(resolve, 20));
|
|
return {
|
|
status: "completed",
|
|
result: "late sibling result",
|
|
};
|
|
},
|
|
poll() {
|
|
throw new Error("completed fixture operations are not polled");
|
|
},
|
|
};
|
|
endpoints.registerAsyncEndpoint(
|
|
"generation.parallel",
|
|
capabilities.generation,
|
|
types.generated,
|
|
endpoint,
|
|
{ scheduling: { resources: [{ id: "pool:fixture.account", limit: 2 }] } },
|
|
);
|
|
|
|
const [result] = await new InProcessBuildScheduler(new Executor({ producers, endpoints, operations })).run([{
|
|
...submission("parallel-failure", createParallelGreetingBuild(3)),
|
|
}]);
|
|
|
|
assert.equal(calls, 2);
|
|
assert.equal(result?.status, "failed");
|
|
assert.equal(result?.state.diagnostics.at(-1)?.code, "PROVIDER_REJECTED");
|
|
assert.match(result?.state.diagnostics.at(-1)?.message ?? "", /first provider request was rejected/);
|
|
assert.equal(result?.outcomes.some((outcome) => outcome.status === "error"), false);
|
|
assert.deepEqual(result?.state.records.find((record) => record.type.name === types.generated.name)?.value,
|
|
("late sibling result"));
|
|
});
|
|
|
|
test("an asynchronous Endpoint starts once and is polled until complete", async () => {
|
|
const operations = memoryOperations();
|
|
let starts = 0;
|
|
let polls = 0;
|
|
let operationId: string | undefined;
|
|
const endpoint: AsyncEndpoint = {
|
|
start({ operation }) {
|
|
starts += 1;
|
|
operationId = operation;
|
|
return { status: "pending", handle: { remoteJob: "job-1" } };
|
|
},
|
|
poll({ operation, handle }) {
|
|
polls += 1;
|
|
assert.equal(operation, operationId);
|
|
assert.deepEqual(handle, { remoteJob: "job-1" });
|
|
return {
|
|
status: "completed",
|
|
result: {
|
|
value: ("Hello after polling"),
|
|
},
|
|
};
|
|
},
|
|
};
|
|
|
|
const firstExecutor = asyncExecutor(endpoint, operations);
|
|
const [first] = await new InProcessBuildScheduler(firstExecutor)
|
|
.run([submission("video", createGreetingBuild())]);
|
|
assert.equal(first?.status, "paused");
|
|
assert.equal(starts, 1);
|
|
assert.equal(polls, 0);
|
|
assert.equal(first?.state.records.some((record) => record.id === "generated:root"), false);
|
|
const pending = first?.outcomes.find((entry) => entry.status === "pending");
|
|
assert.ok(pending?.operation);
|
|
assert.equal((await operations.read(pending.operation))?.status, "running");
|
|
|
|
const secondExecutor = asyncExecutor(endpoint, operations);
|
|
const [second] = await new InProcessBuildScheduler(secondExecutor)
|
|
.run([continuation(first!)]);
|
|
|
|
assert.equal(second?.status, "complete");
|
|
assert.equal(starts, 1);
|
|
assert.equal(polls, 1);
|
|
assert.equal((await operations.read(pending.operation))?.status, "completed");
|
|
});
|
|
|
|
test("a persisted remote acknowledgement survives a later submission failure", async () => {
|
|
const operations = memoryOperations();
|
|
let starts = 0;
|
|
let polls = 0;
|
|
const endpoint: AsyncEndpoint = {
|
|
async start(context) {
|
|
starts += 1;
|
|
await context.checkpoint?.({
|
|
status: "accepted",
|
|
handle: { remoteJob: "accepted-before-local-failure" },
|
|
receipt: { id: "accepted-before-local-failure" },
|
|
});
|
|
throw new Error("local work failed after the remote acknowledgement");
|
|
},
|
|
poll({ handle }) {
|
|
polls += 1;
|
|
assert.deepEqual(handle, { remoteJob: "accepted-before-local-failure" });
|
|
return { status: "completed", result: "recovered without another submission" };
|
|
},
|
|
};
|
|
|
|
const firstExecutor = asyncExecutor(endpoint, operations);
|
|
const [first] = await new InProcessBuildScheduler(firstExecutor)
|
|
.run([submission("checkpointed-submission", createGreetingBuild())]);
|
|
assert.equal(first?.status, "paused");
|
|
assert.equal(starts, 1);
|
|
const pending = first?.outcomes.find((entry) => entry.status === "pending");
|
|
assert.ok(pending?.operation);
|
|
const acknowledged = await operations.read(pending.operation);
|
|
assert.equal(acknowledged?.status, "running");
|
|
assert.deepEqual(acknowledged?.receipt, { id: "accepted-before-local-failure" });
|
|
|
|
const secondExecutor = asyncExecutor(endpoint, operations);
|
|
const [second] = await new InProcessBuildScheduler(secondExecutor).run([continuation(first!)]);
|
|
assert.equal(second?.status, "complete");
|
|
assert.equal(starts, 1);
|
|
assert.equal(polls, 1);
|
|
});
|
|
|
|
test("asynchronous actions expose progress before returning without treating it as acknowledgement", async () => {
|
|
const operations = memoryOperations();
|
|
const progress: unknown[] = [];
|
|
const log: unknown[] = [];
|
|
let reportReady!: () => void;
|
|
let finishAction!: () => void;
|
|
const reported = new Promise<void>((resolve) => { reportReady = resolve; });
|
|
const finish = new Promise<void>((resolve) => { finishAction = resolve; });
|
|
const executor = asyncExecutor({
|
|
async start(context) {
|
|
await context.reportProgress?.({ phase: "Preparing inputs", completed: 0, total: 2, unit: "files" });
|
|
await context.reportProgress?.({ phase: "Preparing inputs", completed: 1, total: 2, unit: "files" });
|
|
reportReady();
|
|
await finish;
|
|
return { status: "pending", handle: { job: "one" }, receipt: { id: "one" } };
|
|
},
|
|
async poll(context) {
|
|
await context.reportProgress?.({ phase: "Reading remote result" });
|
|
return { status: "ready", handle: context.handle };
|
|
},
|
|
async collect(context) {
|
|
await context.reportProgress?.({ phase: "Receiving output" });
|
|
return { status: "completed", result: "done" };
|
|
},
|
|
}, operations);
|
|
const context: RuntimeExecutionContext = {
|
|
build: "progress-build",
|
|
reportProgress: async (value) => { progress.push(value); },
|
|
recordExecution: async (value) => { log.push(value); },
|
|
};
|
|
const running = new InProcessBuildScheduler(executor, {
|
|
executionContext: () => ({
|
|
reportProgress: context.reportProgress!,
|
|
recordExecution: context.recordExecution!,
|
|
}),
|
|
}).run([submission(context.build, createGreetingBuild())]);
|
|
try {
|
|
await reported;
|
|
assert.deepEqual(progress, [0, 1].map((completed) => ({ endpoint: "generation.local",
|
|
progress: { phase: "Preparing inputs", completed, total: 2, unit: "files" },
|
|
})));
|
|
assert.deepEqual(log, [
|
|
{ endpoint: "generation.local", kind: "started" },
|
|
{ endpoint: "generation.local", kind: "phase", phase: "Preparing inputs" },
|
|
]);
|
|
const [pending] = await operations.list({ build: context.build });
|
|
assert.equal(pending!.status, "submitting");
|
|
assert.equal(pending!.receipt, undefined);
|
|
} finally { finishAction(); }
|
|
await running;
|
|
const [submitted] = await operations.list({ build: context.build });
|
|
assert.equal(submitted!.status, "running");
|
|
assert.equal(submitted!.receipt!.id, "one");
|
|
const ready = await executor.advanceOperation(submitted!, context);
|
|
const received = await executor.advanceOperation(ready, context);
|
|
assert.equal(received.status, "received");
|
|
assert.deepEqual(received.value, "done");
|
|
assert.deepEqual(progress.slice(2), ["Reading remote result", "Receiving output"].map((phase) => ({
|
|
endpoint: "generation.local", progress: { phase },
|
|
})));
|
|
});
|
|
|
|
test("a temporary collection failure keeps remote completion and retries collection", async () => {
|
|
const operations = memoryOperations();
|
|
let collections = 0;
|
|
const executor = asyncExecutor({
|
|
start: () => ({ status: "pending", handle: { job: "one" }, receipt: { id: "one" } }),
|
|
poll: (context) => ({ status: "ready", handle: context.handle, receipt: { id: "one" } }),
|
|
collect: (context) => {
|
|
collections += 1;
|
|
if (collections === 1) {
|
|
return { status: "pending", handle: context.handle, progress: { phase: "retrying-collection" } };
|
|
}
|
|
return { status: "completed", result: "done", receipt: { id: "one" } };
|
|
},
|
|
}, operations);
|
|
const [first] = await new InProcessBuildScheduler(executor).run([submission("video", createGreetingBuild())]);
|
|
const operationId = first?.outcomes.find((entry) => entry.status === "pending")?.operation;
|
|
assert.ok(operationId);
|
|
const submitted = await operations.read(operationId);
|
|
assert.ok(submitted);
|
|
|
|
const ready = await executor.advanceOperation(submitted, { build: "video" });
|
|
assert.equal(ready.status, "collecting");
|
|
const retry = await executor.advanceOperation(ready, { build: "video" });
|
|
assert.equal(retry.status, "collecting");
|
|
assert.equal(retry.progress?.phase, "retrying-collection");
|
|
const received = await executor.advanceOperation(retry, { build: "video" });
|
|
assert.equal(collections, 2);
|
|
assert.equal(received.status, "received");
|
|
assert.deepEqual(received.value, "done");
|
|
});
|
|
|
|
test("a submission error ends its attempt and a new Build can submit normally", async () => {
|
|
const operations = memoryOperations();
|
|
let starts = 0;
|
|
const endpoint: AsyncEndpoint = {
|
|
start() {
|
|
starts++;
|
|
if (starts === 1) throw new Error("submission timed out without a receipt");
|
|
return { status: "pending", handle: { job: "second" }, receipt: { id: "second" } };
|
|
},
|
|
poll() { return { status: "completed", result: "done" }; },
|
|
};
|
|
const scheduler = new InProcessBuildScheduler(asyncExecutor(endpoint, operations));
|
|
const [failed] = await scheduler.run([submission("first", createGreetingBuild())]);
|
|
assert.equal(failed?.status, "failed");
|
|
assert.equal((await operations.list({ build: "first" }))[0]?.status, "failed");
|
|
await scheduler.run([continuation(failed!)]);
|
|
assert.equal(starts, 1);
|
|
const [next] = await scheduler.run([submission("second", createGreetingBuild())]);
|
|
const [completed] = await scheduler.run([continuation(next!)]);
|
|
assert.equal(completed?.status, "complete");
|
|
assert.equal(starts, 2);
|
|
});
|
|
|
|
test("wakeAt prevents early polling and Runtime cancellation becomes a terminal Core failure", async () => {
|
|
const operations = memoryOperations();
|
|
let polls = 0;
|
|
let cancels = 0;
|
|
const wakeAt = Date.now() + 60_000;
|
|
const endpoint: AsyncEndpoint = {
|
|
start() {
|
|
return { status: "pending", handle: { remoteJob: "job-wait" }, wakeAt };
|
|
},
|
|
poll() {
|
|
polls += 1;
|
|
throw new Error("wakeAt must stop early polling");
|
|
},
|
|
cancel({ handle }) {
|
|
cancels += 1;
|
|
assert.deepEqual(handle, { remoteJob: "job-wait" });
|
|
return { status: "confirmed" };
|
|
},
|
|
};
|
|
const executor = asyncExecutor(endpoint, operations);
|
|
const scheduler = new InProcessBuildScheduler(executor);
|
|
const [first] = await scheduler.run([submission("cancel-video", createGreetingBuild())]);
|
|
const operationId = first?.outcomes.find((item) => item.status === "pending")?.operation;
|
|
assert.ok(operationId);
|
|
|
|
const [early] = await scheduler.run([continuation(first!)]);
|
|
assert.equal(early?.outcomes.at(-1)?.wakeAt, wakeAt);
|
|
assert.equal(polls, 0);
|
|
|
|
const operation = await operations.read(operationId);
|
|
assert.ok(operation);
|
|
await executor.cancelOperation(operation);
|
|
assert.equal(cancels, 1);
|
|
const [cancelled] = await scheduler.run([continuation(early!)]);
|
|
assert.equal(cancelled?.status, "failed");
|
|
assert.equal(cancelled?.state.diagnostics.at(-1)?.code, "CANCELLED");
|
|
});
|
|
|
|
test("request quantities govern concurrent admission, independent of whole-Need count", async () => {
|
|
const producerRegistry = new ProducerRegistry();
|
|
registerGreetingProducers(producerRegistry);
|
|
const endpoints = new EndpointRegistry();
|
|
let active = 0, maximum = 0;
|
|
endpoints.registerImmediateEndpoint("weighted", capabilities.generation, types.generated, async () => {
|
|
active += 4; maximum = Math.max(maximum, active);
|
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
active -= 4;
|
|
return "done";
|
|
}, { scheduling: { resources: [{ id: "browsers", limit: 6 }], unitsForRequest: () => ({ browsers: 4 }) } });
|
|
const results = await new InProcessBuildScheduler(new Executor({ producers: producerRegistry, endpoints }))
|
|
.run(["a", "b"].map((id) => submission(id, createGreetingBuild())));
|
|
assert.ok(results.every((result) => result.status === "complete"));
|
|
assert.equal(maximum, 4, "two four-worker requests cannot fit in six slots");
|
|
});
|
|
|
|
test("a poll transport error fails once and retains the acknowledged receipt", async () => {
|
|
const operations = memoryOperations();
|
|
let starts = 0, polls = 0;
|
|
const endpoint: AsyncEndpoint = {
|
|
start() { starts++; return { status: "pending", handle: { job: "one" }, receipt: { id: "one" } }; },
|
|
poll() { polls++; throw new Error("connection lost"); },
|
|
};
|
|
const scheduler = new InProcessBuildScheduler(asyncExecutor(endpoint, operations));
|
|
const [first] = await scheduler.run([submission("poll-error", createGreetingBuild())]);
|
|
const [second] = await scheduler.run([continuation(first!)]);
|
|
const [operation] = await operations.list({ build: "poll-error" });
|
|
assert.equal(second?.status, "failed");
|
|
assert.equal(operation?.status, "failed");
|
|
assert.equal(operation?.receipt?.id, "one");
|
|
assert.match(operation!.failure!.message, /connection lost/);
|
|
await scheduler.run([continuation(second!)]);
|
|
assert.equal(starts, 1);
|
|
assert.equal(polls, 1);
|
|
});
|