项目文件夹

文件
wehub-resource-sync c48612c494
CI / E2E Tests (push) Has been cancelled
CI / Lint, Typecheck & Unit Tests (push) Has been cancelled
Docs Build / Build docs site (push) Has been cancelled
Publish @openmaic packages / Build, validate & publish (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 13:03:23 +08:00

404 行
17 KiB
TypeScript

此文件含有模棱两可的 Unicode 字符
此文件含有可能会与其他字符混淆的 Unicode 字符。 如果您是想特意这样的,可以安全地忽略该警告。 使用 Escape 按钮显示他们。
/**
* `resolveActionTimeline` — the index-domain → time-domain expansion.
*
* Playback drives actions by a `(sceneIndex, actionIndex)` cursor; a faithful
* video exporter needs the same sequence laid out on a wall-clock. This pure
* function formalizes the semantics the app's `PlaybackEngine.processNext`
* switch expresses as control flow:
*
* - **Blocking** actions (speech, whiteboard, widget, video, discussion) hold
* the cursor until they complete — the next action starts after them.
* - **Fire-and-forget** actions (spotlight, laser) do not block: playback
* continues immediately, and the effect persists visually for
* {@link EFFECT_AUTO_CLEAR_MS} before auto-clearing.
*
* The blocking/non-blocking partition is read from the DSL's
* {@link FIRE_AND_FORGET_ACTIONS} rather than hardcoded here, so the two stay
* in lockstep. Durations come from the shared {@link timing} spec.
*
* Pure, no runtime dependencies beyond `@openmaic/dsl`.
*/
import type {
Action,
SceneCore,
SpeechAction,
PlayVideoAction,
WbDrawCodeAction,
WbDrawTextAction,
WbDrawTableAction,
WbEditCodeAction,
DiscussionAction,
WbClearAction,
} from '@openmaic/dsl';
import { FIRE_AND_FORGET_ACTIONS } from '@openmaic/dsl';
import { EMPTY_SCENE_DWELL } from './cursor';
import {
EFFECT_AUTO_CLEAR_MS,
DISCUSSION_TRIGGER_DELAY_MS,
DISCUSSION_AUTO_SKIP_MS,
MAX_VIDEO_WAIT_MS,
WB_OPEN_MS,
WB_DRAW_MS,
WB_EDIT_MS,
WB_DELETE_MS,
WB_CLOSE_MS,
WIDGET_MS,
wbDrawCodeMs,
wbClearMs,
estimateSpeechDurationMs,
} from './timing';
const FIRE_AND_FORGET = new Set<string>(FIRE_AND_FORGET_ACTIONS);
/**
* Synthetic segment emitted when a whiteboard mutation runs while the board is
* closed. The app's `ActionEngine.execute()` awaits `ensureWhiteboardOpen()`
* (a `WB_OPEN_MS` open animation) before any `wb_*` action other than
* `wb_open`/`wb_close`, so the exporter must lay down that same beat or every
* later segment starts `WB_OPEN_MS` too early. Distinct id so consumers can
* tell it apart from an authored `wb_open`.
*/
export const IMPLICIT_WB_OPEN: Action = {
id: '__implicit_wb_open__',
type: 'wb_open',
} as Action;
/** Whiteboard mutations that trigger an implicit auto-open when the board is closed. */
function isImplicitOpenTrigger(type: string): boolean {
return type.startsWith('wb_') && type !== 'wb_open' && type !== 'wb_close';
}
export interface ResolveTimelineOptions {
/** Playback speed multiplier applied to speech dwell (estimate and real audio alike). Default 1. */
playbackSpeed?: number;
/**
* Real narration duration (ms) for a speech action when pre-generated audio
* exists. Return `null`/`undefined` to fall back to the deterministic
* {@link estimateSpeechDurationMs}. The exporter, which knows each clip's
* stored audio duration (issue #861), supplies this. The returned length is
* the clip's natural (1×) duration; the timeline divides it by `playbackSpeed`
* to match the live `AudioPlayer.setPlaybackRate` path.
*/
getAudioDurationMs?: (action: SpeechAction) => number | null | undefined;
/**
* Real video duration (ms) for a play_video action. Return `null`/`undefined`
* when unknown — the {@link ResolveTimelineOptions.onUnresolvedVideoDuration}
* policy then decides. A resolved value is capped at {@link MAX_VIDEO_WAIT_MS}.
*/
getVideoDurationMs?: (action: PlayVideoAction) => number | null | undefined;
/**
* What to do when a `play_video` duration is unresolved (no `getVideoDurationMs`,
* or it returned nullish). `play_video` blocks live playback until the video
* ends, so a silent 0 would shift every later action early — hence the default
* is `'throw'` (fail loudly). `'cap'` assumes the {@link MAX_VIDEO_WAIT_MS}
* safety cap; `'zero'` opts back into no-dwell explicitly.
*/
onUnresolvedVideoDuration?: 'throw' | 'cap' | 'zero';
/**
* Live whiteboard element count when a wb_clear runs (the clear animation
* scales with it). Defaults to 0 when not supplied, which yields a 0ms dwell
* (an empty clear is a no-op in the engine); the exporter, which replays
* whiteboard state, can provide the true count.
*/
getClearElementCount?: (action: WbClearAction) => number;
/**
* Whether a discussion action is skipped outright by the engine (already
* consumed, or its `agentId` isn't in the selected set) — in which case it
* contributes no dwell. Depends on runtime state the pure timeline can't see,
* so the caller supplies it; defaults to "not skipped"
* ({@link DISCUSSION_TRIGGER_DELAY_MS}).
*/
isDiscussionSkipped?: (action: DiscussionAction) => boolean;
/**
* Whether a `wb_edit_code` action is a no-op the engine skips without delay
* (target block missing / not a code element / stale line refs). Depends on
* live whiteboard state, so the caller supplies it; defaults to a normal edit
* ({@link WB_EDIT_MS}).
*/
isEditCodeNoop?: (action: WbEditCodeAction) => boolean;
/**
* Whether the whiteboard is already open when the timeline starts. Defaults to
* `false`, matching the engine's post-`resetPlaybackVisualState()` state, so
* the first whiteboard mutation triggers an implicit {@link IMPLICIT_WB_OPEN}
* beat.
*/
whiteboardOpen?: boolean;
}
export interface TimelineSegment {
action: Action;
sceneId: string;
sceneIndex: number;
actionIndex: number;
/** Wall-clock start (ms) relative to the start of playback. */
startMs: number;
/** How long the action is visually present (ms). */
durationMs: number;
/**
* How much the playback cursor advances (ms) before the next action starts.
* Equal to `durationMs` for blocking actions, `0` for fire-and-forget.
*/
advancesCursorMs: number;
/** Whether the action blocks the cursor (false only for fire-and-forget). */
blocking: boolean;
}
/** Line count of a code block, matching the app's `code.split('\n')` typing anim. */
function codeLineCount(code: string): number {
return code.split('\n').length;
}
/**
* Resolve a `play_video` action's blocking duration. Live playback treats
* `play_video` as blocking — it waits until the video ends or {@link MAX_VIDEO_WAIT_MS}
* fires — so an unknown duration must NOT silently become a zero-length segment
* (that would shift every later action early). When `getVideoDurationMs` returns
* a value it's capped at the wait limit; when it's missing/unresolved the
* {@link ResolveTimelineOptions.onUnresolvedVideoDuration} policy decides:
* `'throw'` (default — fail loudly), `'cap'` (assume the max wait), or `'zero'`
* (opt in to the old no-dwell behavior explicitly).
*/
function resolveVideoDurationMs(action: PlayVideoAction, opts: ResolveTimelineOptions): number {
const resolved = opts.getVideoDurationMs?.(action);
if (resolved != null) return Math.min(resolved, MAX_VIDEO_WAIT_MS);
switch (opts.onUnresolvedVideoDuration ?? 'throw') {
case 'zero':
return 0;
case 'cap':
return MAX_VIDEO_WAIT_MS;
case 'throw':
default:
throw new Error(
`resolveActionTimeline: play_video "${action.elementId}" has no resolved duration. ` +
`play_video is blocking, so a missing duration would silently shift later actions ` +
`early. Supply getVideoDurationMs, or set onUnresolvedVideoDuration to 'cap' or 'zero'.`,
);
}
}
/**
* The visual duration (ms) of a single action — how long it is present on
* screen. For blocking actions this is also how long the cursor waits.
*/
function actionDurationMs(action: Action, opts: ResolveTimelineOptions): number {
switch (action.type) {
case 'speech': {
const speed = opts.playbackSpeed ?? 1;
// The live path plays pre-generated audio at `AudioPlayer.setPlaybackRate(speed)`,
// so a stored clip's wall-clock dwell is its length divided by speed — the
// same scaling the no-audio estimate applies. Keeping the two paths in
// lockstep is what stops non-1× exports from drifting.
const audio = opts.getAudioDurationMs?.(action);
if (audio != null) return audio / speed;
return estimateSpeechDurationMs(action.text, { speed });
}
case 'spotlight':
case 'laser':
return EFFECT_AUTO_CLEAR_MS;
case 'discussion': {
// A discussion the engine skips outright — already consumed, or its agent
// isn't selected — contributes no dwell (`processNext` recurses with no
// timer). That skip depends on runtime state (consumed set / selected
// agents), so the caller signals it via `isDiscussionSkipped`.
if (opts.isDiscussionSkipped?.(action as DiscussionAction)) return 0;
// Otherwise: the trigger delay before the ProactiveCard shows, then — in
// unattended playback/export — the card's own auto-skip countdown before
// playback continues. (An attended viewer joining/skipping the card early
// is interactive and out of scope; this models the deterministic
// no-interaction dwell.)
return DISCUSSION_TRIGGER_DELAY_MS + DISCUSSION_AUTO_SKIP_MS;
}
case 'play_video':
return resolveVideoDurationMs(action as PlayVideoAction, opts);
case 'wb_open':
return WB_OPEN_MS;
case 'wb_draw_text': {
// The engine no-ops (no delay) when there's nothing to draw:
// `executeWbDrawText` returns early on empty content.
const content = (action as WbDrawTextAction).content ?? '';
return content ? WB_DRAW_MS : 0;
}
case 'wb_draw_table': {
// `executeWbDrawTable` returns early (no delay) when the table has no rows
// or no columns.
const data = (action as WbDrawTableAction).data;
const rows = data?.length ?? 0;
const cols = rows > 0 ? (data[0]?.length ?? 0) : 0;
return rows === 0 || cols === 0 ? 0 : WB_DRAW_MS;
}
case 'wb_draw_shape':
case 'wb_draw_chart':
case 'wb_draw_latex':
case 'wb_draw_line':
return WB_DRAW_MS;
case 'wb_draw_code':
return wbDrawCodeMs(codeLineCount((action as WbDrawCodeAction).code));
case 'wb_edit_code':
// `executeWbEditCode` returns before its delay when the edit can't apply
// (missing/non-code target, or stale lineId/lineIds). That depends on live
// whiteboard state the pure timeline can't see, so the caller signals a
// no-op via `isEditCodeNoop`; otherwise it's the normal edit animation.
return opts.isEditCodeNoop?.(action as WbEditCodeAction) ? 0 : WB_EDIT_MS;
case 'wb_clear': {
// The engine early-returns with no delay when the board is already empty
// (`executeWbClear`: elementCount === 0 → return), so an empty clear has a
// 0ms dwell, not the `wbClearMs(0)` animation floor.
const count = opts.getClearElementCount?.(action as WbClearAction) ?? 0;
return count === 0 ? 0 : wbClearMs(count);
}
case 'wb_delete':
return WB_DELETE_MS;
case 'wb_close':
return WB_CLOSE_MS;
case 'widget_highlight':
case 'widget_setState':
case 'widget_annotation':
case 'widget_reveal':
return WIDGET_MS;
default:
return 0;
}
}
/**
* Expand a scene list into an ordered wall-clock timeline. Scenes and their
* actions are visited in order; a scene with no actions yields one
* {@link EMPTY_SCENE_DWELL} beat (a blank speech clip's dwell) so it still
* shows, mirroring {@link resolvePlaybackCursor}.
*
* Whiteboard auto-open is modeled: a `wb_*` mutation (draw/edit/clear/delete)
* that runs while the board is closed is preceded by a synthetic
* {@link IMPLICIT_WB_OPEN} beat ({@link WB_OPEN_MS}), exactly as the engine's
* `ensureWhiteboardOpen` does. The open state carries across scenes and is
* toggled by `wb_open` / `wb_close`; seed it via {@link ResolveTimelineOptions.whiteboardOpen}.
*
* @returns segments in play order, each stamped with `startMs`, its visual
* `durationMs`, and how far it `advancesCursorMs`.
*
* Typed against {@link SceneCore} (only `id` + `actions` are read), so an
* app-widened `Scene` (extra content kinds) is accepted without casting.
*/
export function resolveActionTimeline(
scenes: SceneCore[],
opts: ResolveTimelineOptions = {},
): TimelineSegment[] {
const segments: TimelineSegment[] = [];
let clockMs = 0;
// Mirrors the engine's whiteboard-open flag: false after
// resetPlaybackVisualState(), flipped by wb_open / wb_close, and read to
// decide whether a wb_* mutation must first pay the implicit open animation.
let whiteboardOpen = opts.whiteboardOpen ?? false;
const push = (action: Action, sceneId: string, sceneIndex: number, actionIndex: number) => {
const durationMs = actionDurationMs(action, opts);
const blocking = !FIRE_AND_FORGET.has(action.type);
const advancesCursorMs = blocking ? durationMs : 0;
segments.push({
action,
sceneId,
sceneIndex,
actionIndex,
startMs: clockMs,
durationMs,
advancesCursorMs,
blocking,
});
clockMs += advancesCursorMs;
};
const pushWithWhiteboard = (
action: Action,
sceneId: string,
sceneIndex: number,
actionIndex: number,
) => {
// A wb_* mutation on a closed board is auto-preceded by an open animation
// (engine: `execute` awaits `ensureWhiteboardOpen`). Emit that beat first so
// later segments don't start WB_OPEN_MS early.
if (!whiteboardOpen && isImplicitOpenTrigger(action.type)) {
push(IMPLICIT_WB_OPEN, sceneId, sceneIndex, actionIndex);
whiteboardOpen = true;
}
push(action, sceneId, sceneIndex, actionIndex);
if (action.type === 'wb_open') whiteboardOpen = true;
else if (action.type === 'wb_close') whiteboardOpen = false;
};
scenes.forEach((scene, sceneIndex) => {
const actions = scene.actions ?? [];
if (actions.length === 0) {
// Empty scene → one synthetic dwell beat, exactly as the cursor yields.
push(EMPTY_SCENE_DWELL, scene.id, sceneIndex, 0);
return;
}
actions.forEach((action, actionIndex) => {
pushWithWhiteboard(action, scene.id, sceneIndex, actionIndex);
});
});
clampFireAndForgetLifetimes(segments, clockMs);
return segments;
}
/**
* Correct the visual `durationMs` of fire-and-forget effects (spotlight/laser)
* in place. Their nominal lifetime is {@link EFFECT_AUTO_CLEAR_MS}, but the
* engine cuts it short — and occasionally extends it:
*
* - **Scene boundary / completion.** The app plays one `PlaybackEngine` per
* scene: switching scenes tears the engine down (`stop()` → `clearEffects()`)
* and each scene ends via the completion path (`getCurrentAction()` returns
* null → `clearEffects()`). Either way effects never outlive their scene, so
* in this continuous clock a spotlight late in a scene (or the final action of
* the lecture) is cleared at the next scene's start / at completion, not a
* flat 5s later.
* - **Shared auto-clear timer.** `ActionEngine.scheduleEffectClear` uses one
* timer that each new effect *resets*, and `clearAllEffects` drops every active
* effect together. So back-to-back effects (within `EFFECT_AUTO_CLEAR_MS` of
* each other) all live until the last one's fire + `EFFECT_AUTO_CLEAR_MS` —
* the earlier effect is *extended*, not cleared on its own schedule.
*
* `advancesCursorMs` (0 for these) is untouched — only the visual hint changes.
*
* @param completionMs the final cursor clock (when the last scene's actions are
* exhausted and playback completes → `clearEffects`).
*/
function clampFireAndForgetLifetimes(segments: TimelineSegment[], completionMs: number): void {
// First segment startMs per scene index → the wall-clock of that scene's
// boundary clearEffects. Every scene yields ≥1 segment, so all indices exist.
const sceneStartMs = new Map<number, number>();
for (const seg of segments) {
if (!sceneStartMs.has(seg.sceneIndex)) sceneStartMs.set(seg.sceneIndex, seg.startMs);
}
segments.forEach((seg, i) => {
if (seg.blocking) return; // only fire-and-forget effects have a clearable lifetime
// The next clearEffects boundary after this effect: the start of the next
// scene, or completion if this is the last scene.
const nextSceneStart = sceneStartMs.get(seg.sceneIndex + 1);
const boundaryMs = nextSceneStart ?? completionMs;
// Shared auto-clear deadline, chained through any later effects in THIS scene
// that fire before it elapses (each resets the shared timer). Segments are in
// non-decreasing startMs order, so a forward walk suffices.
let deadlineMs = seg.startMs + EFFECT_AUTO_CLEAR_MS;
for (let j = i + 1; j < segments.length; j++) {
const other = segments[j];
if (other.sceneIndex !== seg.sceneIndex) break; // cleared at the boundary anyway
if (other.blocking) continue; // blocking actions don't touch the effect timer
// Chain breaks at exact equality too: the earlier effect's clear timer was
// registered first (same 5000ms delay as the reading timer that triggers
// the later effect), so it fires before the later effect resets it. Its
// predecessor is therefore cleared at exactly `deadlineMs`, not extended.
if (other.startMs >= deadlineMs) break; // timer already fired; chain broken
deadlineMs = Math.max(deadlineMs, other.startMs + EFFECT_AUTO_CLEAR_MS);
}
const clearMs = Math.min(boundaryMs, deadlineMs);
seg.durationMs = Math.max(0, clearMs - seg.startMs);
});
}