/** * `resolveActionTimeline` — the index-domain → time-domain expansion. * * Playback drives actions by a `(sceneIndex, actionIndex)` cursor; a faithful * video exporter needs the same sequence laid out on a wall-clock. This pure * function formalizes the semantics the app's `PlaybackEngine.processNext` * switch expresses as control flow: * * - **Blocking** actions (speech, whiteboard, widget, video, discussion) hold * the cursor until they complete — the next action starts after them. * - **Fire-and-forget** actions (spotlight, laser) do not block: playback * continues immediately, and the effect persists visually for * {@link EFFECT_AUTO_CLEAR_MS} before auto-clearing. * * The blocking/non-blocking partition is read from the DSL's * {@link FIRE_AND_FORGET_ACTIONS} rather than hardcoded here, so the two stay * in lockstep. Durations come from the shared {@link timing} spec. * * Pure, no runtime dependencies beyond `@openmaic/dsl`. */ import type { Action, SceneCore, SpeechAction, PlayVideoAction, WbDrawCodeAction, WbDrawTextAction, WbDrawTableAction, WbEditCodeAction, DiscussionAction, WbClearAction, } from '@openmaic/dsl'; import { FIRE_AND_FORGET_ACTIONS } from '@openmaic/dsl'; import { EMPTY_SCENE_DWELL } from './cursor'; import { EFFECT_AUTO_CLEAR_MS, DISCUSSION_TRIGGER_DELAY_MS, DISCUSSION_AUTO_SKIP_MS, MAX_VIDEO_WAIT_MS, WB_OPEN_MS, WB_DRAW_MS, WB_EDIT_MS, WB_DELETE_MS, WB_CLOSE_MS, WIDGET_MS, wbDrawCodeMs, wbClearMs, estimateSpeechDurationMs, } from './timing'; const FIRE_AND_FORGET = new Set(FIRE_AND_FORGET_ACTIONS); /** * Synthetic segment emitted when a whiteboard mutation runs while the board is * closed. The app's `ActionEngine.execute()` awaits `ensureWhiteboardOpen()` * (a `WB_OPEN_MS` open animation) before any `wb_*` action other than * `wb_open`/`wb_close`, so the exporter must lay down that same beat or every * later segment starts `WB_OPEN_MS` too early. Distinct id so consumers can * tell it apart from an authored `wb_open`. */ export const IMPLICIT_WB_OPEN: Action = { id: '__implicit_wb_open__', type: 'wb_open', } as Action; /** Whiteboard mutations that trigger an implicit auto-open when the board is closed. */ function isImplicitOpenTrigger(type: string): boolean { return type.startsWith('wb_') && type !== 'wb_open' && type !== 'wb_close'; } export interface ResolveTimelineOptions { /** Playback speed multiplier applied to speech dwell (estimate and real audio alike). Default 1. */ playbackSpeed?: number; /** * Real narration duration (ms) for a speech action when pre-generated audio * exists. Return `null`/`undefined` to fall back to the deterministic * {@link estimateSpeechDurationMs}. The exporter, which knows each clip's * stored audio duration (issue #861), supplies this. The returned length is * the clip's natural (1×) duration; the timeline divides it by `playbackSpeed` * to match the live `AudioPlayer.setPlaybackRate` path. */ getAudioDurationMs?: (action: SpeechAction) => number | null | undefined; /** * Real video duration (ms) for a play_video action. Return `null`/`undefined` * when unknown — the {@link ResolveTimelineOptions.onUnresolvedVideoDuration} * policy then decides. A resolved value is capped at {@link MAX_VIDEO_WAIT_MS}. */ getVideoDurationMs?: (action: PlayVideoAction) => number | null | undefined; /** * What to do when a `play_video` duration is unresolved (no `getVideoDurationMs`, * or it returned nullish). `play_video` blocks live playback until the video * ends, so a silent 0 would shift every later action early — hence the default * is `'throw'` (fail loudly). `'cap'` assumes the {@link MAX_VIDEO_WAIT_MS} * safety cap; `'zero'` opts back into no-dwell explicitly. */ onUnresolvedVideoDuration?: 'throw' | 'cap' | 'zero'; /** * Live whiteboard element count when a wb_clear runs (the clear animation * scales with it). Defaults to 0 when not supplied, which yields a 0ms dwell * (an empty clear is a no-op in the engine); the exporter, which replays * whiteboard state, can provide the true count. */ getClearElementCount?: (action: WbClearAction) => number; /** * Whether a discussion action is skipped outright by the engine (already * consumed, or its `agentId` isn't in the selected set) — in which case it * contributes no dwell. Depends on runtime state the pure timeline can't see, * so the caller supplies it; defaults to "not skipped" * ({@link DISCUSSION_TRIGGER_DELAY_MS}). */ isDiscussionSkipped?: (action: DiscussionAction) => boolean; /** * Whether a `wb_edit_code` action is a no-op the engine skips without delay * (target block missing / not a code element / stale line refs). Depends on * live whiteboard state, so the caller supplies it; defaults to a normal edit * ({@link WB_EDIT_MS}). */ isEditCodeNoop?: (action: WbEditCodeAction) => boolean; /** * Whether the whiteboard is already open when the timeline starts. Defaults to * `false`, matching the engine's post-`resetPlaybackVisualState()` state, so * the first whiteboard mutation triggers an implicit {@link IMPLICIT_WB_OPEN} * beat. */ whiteboardOpen?: boolean; } export interface TimelineSegment { action: Action; sceneId: string; sceneIndex: number; actionIndex: number; /** Wall-clock start (ms) relative to the start of playback. */ startMs: number; /** How long the action is visually present (ms). */ durationMs: number; /** * How much the playback cursor advances (ms) before the next action starts. * Equal to `durationMs` for blocking actions, `0` for fire-and-forget. */ advancesCursorMs: number; /** Whether the action blocks the cursor (false only for fire-and-forget). */ blocking: boolean; } /** Line count of a code block, matching the app's `code.split('\n')` typing anim. */ function codeLineCount(code: string): number { return code.split('\n').length; } /** * Resolve a `play_video` action's blocking duration. Live playback treats * `play_video` as blocking — it waits until the video ends or {@link MAX_VIDEO_WAIT_MS} * fires — so an unknown duration must NOT silently become a zero-length segment * (that would shift every later action early). When `getVideoDurationMs` returns * a value it's capped at the wait limit; when it's missing/unresolved the * {@link ResolveTimelineOptions.onUnresolvedVideoDuration} policy decides: * `'throw'` (default — fail loudly), `'cap'` (assume the max wait), or `'zero'` * (opt in to the old no-dwell behavior explicitly). */ function resolveVideoDurationMs(action: PlayVideoAction, opts: ResolveTimelineOptions): number { const resolved = opts.getVideoDurationMs?.(action); if (resolved != null) return Math.min(resolved, MAX_VIDEO_WAIT_MS); switch (opts.onUnresolvedVideoDuration ?? 'throw') { case 'zero': return 0; case 'cap': return MAX_VIDEO_WAIT_MS; case 'throw': default: throw new Error( `resolveActionTimeline: play_video "${action.elementId}" has no resolved duration. ` + `play_video is blocking, so a missing duration would silently shift later actions ` + `early. Supply getVideoDurationMs, or set onUnresolvedVideoDuration to 'cap' or 'zero'.`, ); } } /** * The visual duration (ms) of a single action — how long it is present on * screen. For blocking actions this is also how long the cursor waits. */ function actionDurationMs(action: Action, opts: ResolveTimelineOptions): number { switch (action.type) { case 'speech': { const speed = opts.playbackSpeed ?? 1; // The live path plays pre-generated audio at `AudioPlayer.setPlaybackRate(speed)`, // so a stored clip's wall-clock dwell is its length divided by speed — the // same scaling the no-audio estimate applies. Keeping the two paths in // lockstep is what stops non-1× exports from drifting. const audio = opts.getAudioDurationMs?.(action); if (audio != null) return audio / speed; return estimateSpeechDurationMs(action.text, { speed }); } case 'spotlight': case 'laser': return EFFECT_AUTO_CLEAR_MS; case 'discussion': { // A discussion the engine skips outright — already consumed, or its agent // isn't selected — contributes no dwell (`processNext` recurses with no // timer). That skip depends on runtime state (consumed set / selected // agents), so the caller signals it via `isDiscussionSkipped`. if (opts.isDiscussionSkipped?.(action as DiscussionAction)) return 0; // Otherwise: the trigger delay before the ProactiveCard shows, then — in // unattended playback/export — the card's own auto-skip countdown before // playback continues. (An attended viewer joining/skipping the card early // is interactive and out of scope; this models the deterministic // no-interaction dwell.) return DISCUSSION_TRIGGER_DELAY_MS + DISCUSSION_AUTO_SKIP_MS; } case 'play_video': return resolveVideoDurationMs(action as PlayVideoAction, opts); case 'wb_open': return WB_OPEN_MS; case 'wb_draw_text': { // The engine no-ops (no delay) when there's nothing to draw: // `executeWbDrawText` returns early on empty content. const content = (action as WbDrawTextAction).content ?? ''; return content ? WB_DRAW_MS : 0; } case 'wb_draw_table': { // `executeWbDrawTable` returns early (no delay) when the table has no rows // or no columns. const data = (action as WbDrawTableAction).data; const rows = data?.length ?? 0; const cols = rows > 0 ? (data[0]?.length ?? 0) : 0; return rows === 0 || cols === 0 ? 0 : WB_DRAW_MS; } case 'wb_draw_shape': case 'wb_draw_chart': case 'wb_draw_latex': case 'wb_draw_line': return WB_DRAW_MS; case 'wb_draw_code': return wbDrawCodeMs(codeLineCount((action as WbDrawCodeAction).code)); case 'wb_edit_code': // `executeWbEditCode` returns before its delay when the edit can't apply // (missing/non-code target, or stale lineId/lineIds). That depends on live // whiteboard state the pure timeline can't see, so the caller signals a // no-op via `isEditCodeNoop`; otherwise it's the normal edit animation. return opts.isEditCodeNoop?.(action as WbEditCodeAction) ? 0 : WB_EDIT_MS; case 'wb_clear': { // The engine early-returns with no delay when the board is already empty // (`executeWbClear`: elementCount === 0 → return), so an empty clear has a // 0ms dwell, not the `wbClearMs(0)` animation floor. const count = opts.getClearElementCount?.(action as WbClearAction) ?? 0; return count === 0 ? 0 : wbClearMs(count); } case 'wb_delete': return WB_DELETE_MS; case 'wb_close': return WB_CLOSE_MS; case 'widget_highlight': case 'widget_setState': case 'widget_annotation': case 'widget_reveal': return WIDGET_MS; default: return 0; } } /** * Expand a scene list into an ordered wall-clock timeline. Scenes and their * actions are visited in order; a scene with no actions yields one * {@link EMPTY_SCENE_DWELL} beat (a blank speech clip's dwell) so it still * shows, mirroring {@link resolvePlaybackCursor}. * * Whiteboard auto-open is modeled: a `wb_*` mutation (draw/edit/clear/delete) * that runs while the board is closed is preceded by a synthetic * {@link IMPLICIT_WB_OPEN} beat ({@link WB_OPEN_MS}), exactly as the engine's * `ensureWhiteboardOpen` does. The open state carries across scenes and is * toggled by `wb_open` / `wb_close`; seed it via {@link ResolveTimelineOptions.whiteboardOpen}. * * @returns segments in play order, each stamped with `startMs`, its visual * `durationMs`, and how far it `advancesCursorMs`. * * Typed against {@link SceneCore} (only `id` + `actions` are read), so an * app-widened `Scene` (extra content kinds) is accepted without casting. */ export function resolveActionTimeline( scenes: SceneCore[], opts: ResolveTimelineOptions = {}, ): TimelineSegment[] { const segments: TimelineSegment[] = []; let clockMs = 0; // Mirrors the engine's whiteboard-open flag: false after // resetPlaybackVisualState(), flipped by wb_open / wb_close, and read to // decide whether a wb_* mutation must first pay the implicit open animation. let whiteboardOpen = opts.whiteboardOpen ?? false; const push = (action: Action, sceneId: string, sceneIndex: number, actionIndex: number) => { const durationMs = actionDurationMs(action, opts); const blocking = !FIRE_AND_FORGET.has(action.type); const advancesCursorMs = blocking ? durationMs : 0; segments.push({ action, sceneId, sceneIndex, actionIndex, startMs: clockMs, durationMs, advancesCursorMs, blocking, }); clockMs += advancesCursorMs; }; const pushWithWhiteboard = ( action: Action, sceneId: string, sceneIndex: number, actionIndex: number, ) => { // A wb_* mutation on a closed board is auto-preceded by an open animation // (engine: `execute` awaits `ensureWhiteboardOpen`). Emit that beat first so // later segments don't start WB_OPEN_MS early. if (!whiteboardOpen && isImplicitOpenTrigger(action.type)) { push(IMPLICIT_WB_OPEN, sceneId, sceneIndex, actionIndex); whiteboardOpen = true; } push(action, sceneId, sceneIndex, actionIndex); if (action.type === 'wb_open') whiteboardOpen = true; else if (action.type === 'wb_close') whiteboardOpen = false; }; scenes.forEach((scene, sceneIndex) => { const actions = scene.actions ?? []; if (actions.length === 0) { // Empty scene → one synthetic dwell beat, exactly as the cursor yields. push(EMPTY_SCENE_DWELL, scene.id, sceneIndex, 0); return; } actions.forEach((action, actionIndex) => { pushWithWhiteboard(action, scene.id, sceneIndex, actionIndex); }); }); clampFireAndForgetLifetimes(segments, clockMs); return segments; } /** * Correct the visual `durationMs` of fire-and-forget effects (spotlight/laser) * in place. Their nominal lifetime is {@link EFFECT_AUTO_CLEAR_MS}, but the * engine cuts it short — and occasionally extends it: * * - **Scene boundary / completion.** The app plays one `PlaybackEngine` per * scene: switching scenes tears the engine down (`stop()` → `clearEffects()`) * and each scene ends via the completion path (`getCurrentAction()` returns * null → `clearEffects()`). Either way effects never outlive their scene, so * in this continuous clock a spotlight late in a scene (or the final action of * the lecture) is cleared at the next scene's start / at completion, not a * flat 5s later. * - **Shared auto-clear timer.** `ActionEngine.scheduleEffectClear` uses one * timer that each new effect *resets*, and `clearAllEffects` drops every active * effect together. So back-to-back effects (within `EFFECT_AUTO_CLEAR_MS` of * each other) all live until the last one's fire + `EFFECT_AUTO_CLEAR_MS` — * the earlier effect is *extended*, not cleared on its own schedule. * * `advancesCursorMs` (0 for these) is untouched — only the visual hint changes. * * @param completionMs the final cursor clock (when the last scene's actions are * exhausted and playback completes → `clearEffects`). */ function clampFireAndForgetLifetimes(segments: TimelineSegment[], completionMs: number): void { // First segment startMs per scene index → the wall-clock of that scene's // boundary clearEffects. Every scene yields ≥1 segment, so all indices exist. const sceneStartMs = new Map(); for (const seg of segments) { if (!sceneStartMs.has(seg.sceneIndex)) sceneStartMs.set(seg.sceneIndex, seg.startMs); } segments.forEach((seg, i) => { if (seg.blocking) return; // only fire-and-forget effects have a clearable lifetime // The next clearEffects boundary after this effect: the start of the next // scene, or completion if this is the last scene. const nextSceneStart = sceneStartMs.get(seg.sceneIndex + 1); const boundaryMs = nextSceneStart ?? completionMs; // Shared auto-clear deadline, chained through any later effects in THIS scene // that fire before it elapses (each resets the shared timer). Segments are in // non-decreasing startMs order, so a forward walk suffices. let deadlineMs = seg.startMs + EFFECT_AUTO_CLEAR_MS; for (let j = i + 1; j < segments.length; j++) { const other = segments[j]; if (other.sceneIndex !== seg.sceneIndex) break; // cleared at the boundary anyway if (other.blocking) continue; // blocking actions don't touch the effect timer // Chain breaks at exact equality too: the earlier effect's clear timer was // registered first (same 5000ms delay as the reading timer that triggers // the later effect), so it fires before the later effect resets it. Its // predecessor is therefore cleared at exactly `deadlineMs`, not extended. if (other.startMs >= deadlineMs) break; // timer already fired; chain broken deadlineMs = Math.max(deadlineMs, other.startMs + EFFECT_AUTO_CLEAR_MS); } const clearMs = Math.min(boundaryMs, deadlineMs); seg.durationMs = Math.max(0, clearMs - seg.startMs); }); }