elizaos--eliza
426e9eeabd
Voice Workbench / headless workbench (mocked backends) (push) Has been cancelled
Voice Workbench / real acoustic lane (nightly, provisioned only) (push) Has been cancelled
ci / test (push) Has been cancelled
ci / lint-and-format (push) Has been cancelled
ci / build (push) Has been cancelled
ci / dev-startup (push) Has been cancelled
gitleaks / gitleaks (push) Has been cancelled
Markdown Links / Relative Markdown Links (push) Has been cancelled
Quality (Extended) / Homepage Build (PR smoke) (push) Has been cancelled
Quality (Extended) / Comment-only diff guard (push) Has been cancelled
Quality (Extended) / Format + Type Safety Ratchet (push) Has been cancelled
Quality (Extended) / Develop Gate (secret scan + UI determinism) (push) Has been cancelled
Quality (Extended) / Develop Gate (lint) (push) Has been cancelled
Chat shell gestures / Chat shell gesture + parity e2e (push) Has been cancelled
Cloud Gateway Discord / Test (push) Has been cancelled
Benchmark Bridge Tests / benchmark (bunx @biomejs/biome check packages/lifeops-bench/src, benchmark-lint) (push) Has been cancelled
Benchmark Bridge Tests / benchmark (bunx vitest run --config packages/lifeops-bench/vitest.config.ts --root packages/lifeops-bench --passWithNoTests, benchmark-tests) (push) Has been cancelled
Build Agent Image / build-and-push (push) Has been cancelled
Dev Smoke / bun run dev onboarding chat (push) Has been cancelled
Dev Smoke / Vite HMR dependency-level smoke (push) Has been cancelled
Electrobun Submodule Guard / electrobun gitlink is fetchable (push) Has been cancelled
Publish @elizaos/example-code / check_npm (push) Has been cancelled
Publish @elizaos/example-code / publish_npm (push) Has been cancelled
Publish @elizaos/plugin-elizacloud / verify_version (push) Has been cancelled
Publish @elizaos/plugin-elizacloud / publish_npm (push) Has been cancelled
Sandbox Live Smoke / Sandbox live smoke (push) Has been cancelled
Snap Build & Test / Build Snap (amd64) (push) Has been cancelled
Snap Build & Test / Build Snap (arm64) (push) Has been cancelled
Test Packaging / elizaos CLI global-install smoke (node + bun) (push) Has been cancelled
Cloud Gateway Webhook / Test (push) Has been cancelled
Cloud Tests / lint-and-types (push) Has been cancelled
Cloud Tests / unit-tests (push) Has been cancelled
Cloud Tests / integration-tests (push) Has been cancelled
Cloud Tests / e2e-tests (push) Has been cancelled
CodeQL Advanced / Analyze (javascript-typescript) (push) Has been cancelled
Deploy Apps Worker (Product 2) / Determine environment (push) Has been cancelled
Deploy Apps Worker (Product 2) / Deploy apps worker to apps-control host (${{ needs.determine-env.outputs.environment }}) (push) Has been cancelled
Deploy Eliza Provisioning Worker / Determine environment (push) Has been cancelled
Deploy Eliza Provisioning Worker / Deploy worker to Hetzner host (${{ needs.determine-env.outputs.environment }} @ ${{ needs.determine-env.outputs.deployment_sha }}) (push) Has been cancelled
Dev Smoke / Classify changed paths (push) Has been cancelled
supply-chain / sbom (push) Has been cancelled
supply-chain / vulnerability-scan (push) Has been cancelled
Build, Push & Deploy to Phala Cloud / build-and-push (push) Has been cancelled
Test Packaging / Validate Packaging Configs (push) Has been cancelled
Test Packaging / Build & Test PyPI Package (push) Has been cancelled
Test Packaging / PyPI on Python ${{ matrix.python }} (push) Has been cancelled
Test Packaging / Pack & Test JS Tarballs (push) Has been cancelled
UI Fixture E2E / ui-fixture-e2e (push) Has been cancelled
UI Fixture E2E / fixture-e2e (push) Has been cancelled
UI Story Gate / story-gate (push) Has been cancelled
vault-ci / test (macos-latest) (push) Has been cancelled
vault-ci / test (ubuntu-latest) (push) Has been cancelled
vault-ci / test (windows-latest) (push) Has been cancelled
vault-ci / app-core wiring tests (push) Has been cancelled
verify-patches / verify patches/CHECKSUMS.sha256 (push) Has been cancelled
Voice Benchmark Smoke / voice-emotion fixture smoke (push) Has been cancelled
Voice Benchmark Smoke / voiceagentbench fixture smoke (push) Has been cancelled
Voice Benchmark Smoke / voicebench-quality unit smoke (push) Has been cancelled
Voice Benchmark Smoke / voicebench TypeScript unit (no audio) (push) Has been cancelled
Voice Benchmark Smoke / voice bench smoke summary (push) Has been cancelled
Windows CI / windows ([bun run --cwd packages/app-core test bun run --cwd packages/elizaos test bun run --cwd packages/cloud/shared test], app-and-cli) (push) Has been cancelled
Windows CI / windows ([bun run --cwd packages/scenario-runner test bun run --cwd packages/vault test bun run --cwd packages/security test bun run --cwd plugins/plugin-coding-tools test], framework-packages) (push) Has been cancelled
Windows CI / windows ([bun run --cwd plugins/plugin-elizacloud test bun run --cwd plugins/plugin-discord test bun run --cwd plugins/plugin-anthropic test bun run --cwd plugins/plugin-openai test bun run --cwd plugins/plugin-app-control test bun run --cwd plugins/pl… (push) Has been cancelled
Windows CI / windows ([node packages/scripts/run-turbo.mjs run build --filter=@elizaos/core --filter=@elizaos/shared --filter=@elizaos/agent --concurrency=4 node packages/scripts/run-bash-linux-only.mjs scripts/verify-riscv64-buildpaths.sh node packages/scripts/run… (push) Has been cancelled
Windows CI / windows ([node packages/scripts/run-turbo.mjs run typecheck --filter=@elizaos/core --filter=@elizaos/shared --filter=@elizaos/cloud-shared --concurrency=4 bun run --cwd packages/core test bun run --cwd packages/shared test], core-runtime, 75) (push) Has been cancelled
1161 行
50 KiB
Swift
1161 行
50 KiB
Swift
import Foundation
|
|
import Capacitor
|
|
import Speech
|
|
import AVFoundation
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Thread-safe Audio Buffer Queue
|
|
// ═══════════════════════════════════════════════════════════════════════════════
|
|
|
|
/// Audio tap callbacks fire on a realtime audio thread. We deep-copy buffers into a
|
|
/// lock-protected queue and drain them on a main-thread timer for the speech recognition
|
|
/// request. This keeps the audio callback tiny and avoids blocking the realtime thread.
|
|
/// Pattern: thread-safe lock-protected queue drained on a main-thread timer.
|
|
private final class AudioBufferQueue: @unchecked Sendable {
|
|
private let lock = NSLock()
|
|
private var buffers: [AVAudioPCMBuffer] = []
|
|
|
|
func enqueue(_ buffer: AVAudioPCMBuffer) {
|
|
guard let copy = buffer.deepCopy() else { return }
|
|
lock.lock()
|
|
buffers.append(copy)
|
|
lock.unlock()
|
|
}
|
|
|
|
func drain() -> [AVAudioPCMBuffer] {
|
|
lock.lock()
|
|
let result = buffers
|
|
buffers.removeAll(keepingCapacity: true)
|
|
lock.unlock()
|
|
return result
|
|
}
|
|
|
|
func clear() {
|
|
lock.lock()
|
|
buffers.removeAll(keepingCapacity: false)
|
|
lock.unlock()
|
|
}
|
|
}
|
|
|
|
private extension AVAudioPCMBuffer {
|
|
func deepCopy() -> AVAudioPCMBuffer? {
|
|
let fmt = format
|
|
let len = frameLength
|
|
guard let copy = AVAudioPCMBuffer(pcmFormat: fmt, frameCapacity: len) else { return nil }
|
|
copy.frameLength = len
|
|
if let src = floatChannelData, let dst = copy.floatChannelData {
|
|
for ch in 0..<Int(fmt.channelCount) { dst[ch].update(from: src[ch], count: Int(len)) }
|
|
return copy
|
|
}
|
|
if let src = int16ChannelData, let dst = copy.int16ChannelData {
|
|
for ch in 0..<Int(fmt.channelCount) { dst[ch].update(from: src[ch], count: Int(len)) }
|
|
return copy
|
|
}
|
|
if let src = int32ChannelData, let dst = copy.int32ChannelData {
|
|
for ch in 0..<Int(fmt.channelCount) { dst[ch].update(from: src[ch], count: Int(len)) }
|
|
return copy
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Wake Word Gate (inlined from SwabbleKit)
|
|
// ═══════════════════════════════════════════════════════════════════════════════
|
|
|
|
/// Speech segment with timing data from SFSpeechRecognizer.
|
|
private struct WakeSegment {
|
|
let text: String
|
|
let start: TimeInterval
|
|
let duration: TimeInterval
|
|
let range: Range<String.Index>?
|
|
var end: TimeInterval { start + duration }
|
|
}
|
|
|
|
private struct GateConfig {
|
|
var triggers: [String]
|
|
var minPostTriggerGap: TimeInterval
|
|
var minCommandLength: Int
|
|
|
|
init(triggers: [String], minPostTriggerGap: TimeInterval = 0.45, minCommandLength: Int = 1) {
|
|
self.triggers = triggers
|
|
self.minPostTriggerGap = minPostTriggerGap
|
|
self.minCommandLength = minCommandLength
|
|
}
|
|
}
|
|
|
|
private struct GateMatch {
|
|
let triggerWord: String
|
|
let triggerEndTime: TimeInterval
|
|
let postGap: TimeInterval
|
|
let command: String
|
|
}
|
|
|
|
/// Wake word detection engine. Matches trigger words against speech segments using timing
|
|
/// data to confirm a deliberate pause after the trigger, then extracts the command text.
|
|
/// Supports fuzzy matching via Levenshtein edit distance so that imprecise recognition
|
|
/// (e.g. "melody" for trigger "eliza") still fires.
|
|
private enum WakeGate {
|
|
|
|
// MARK: Token types
|
|
|
|
private struct Token {
|
|
let normalized: String
|
|
let start: TimeInterval
|
|
let end: TimeInterval
|
|
let range: Range<String.Index>?
|
|
}
|
|
|
|
private struct TriggerTokens {
|
|
let original: String
|
|
let tokens: [String]
|
|
}
|
|
|
|
// MARK: Primary timing-based match
|
|
|
|
/// Match trigger words against speech segments using timing data.
|
|
/// Looks for trigger tokens, confirms a post-trigger gap, and extracts the command.
|
|
static func match(transcript: String, segments: [WakeSegment], config: GateConfig) -> GateMatch? {
|
|
let triggers = normalizeTriggers(config.triggers)
|
|
guard !triggers.isEmpty else { return nil }
|
|
let tokens = normalizeSegments(segments)
|
|
guard !tokens.isEmpty else { return nil }
|
|
|
|
struct Candidate {
|
|
let trigger: String; let index: Int; let triggerEnd: TimeInterval; let gap: TimeInterval
|
|
}
|
|
|
|
var best: Candidate?
|
|
for trig in triggers {
|
|
let count = trig.tokens.count
|
|
guard count > 0, tokens.count > count else { continue }
|
|
for i in 0...(tokens.count - count - 1) {
|
|
let exact = (0..<count).allSatisfy { tokens[i + $0].normalized == trig.tokens[$0] }
|
|
let fuzzy = !exact && (0..<count).allSatisfy {
|
|
fuzzyTokenMatch(tokens[i + $0].normalized, trig.tokens[$0])
|
|
}
|
|
guard exact || fuzzy else { continue }
|
|
let trigEnd = tokens[i + count - 1].end
|
|
let gap = tokens[i + count].start - trigEnd
|
|
guard gap >= config.minPostTriggerGap else { continue }
|
|
if let b = best, i <= b.index { continue }
|
|
best = Candidate(trigger: trig.original, index: i, triggerEnd: trigEnd, gap: gap)
|
|
}
|
|
}
|
|
|
|
guard let best else { return nil }
|
|
let cmd = commandText(transcript: transcript, segments: segments, triggerEndTime: best.triggerEnd)
|
|
.trimmingCharacters(in: wsPunct)
|
|
guard cmd.count >= config.minCommandLength else { return nil }
|
|
return GateMatch(triggerWord: best.trigger, triggerEndTime: best.triggerEnd,
|
|
postGap: best.gap, command: cmd)
|
|
}
|
|
|
|
// MARK: Command text extraction
|
|
|
|
/// Extract command text from segments appearing after the trigger end time.
|
|
static func commandText(transcript: String, segments: [WakeSegment], triggerEndTime: TimeInterval) -> String {
|
|
let threshold = triggerEndTime + 0.001
|
|
for seg in segments where seg.start >= threshold {
|
|
if normalizeToken(seg.text).isEmpty { continue }
|
|
if let range = seg.range {
|
|
return String(transcript[range.lowerBound...]).trimmingCharacters(in: wsPunct)
|
|
}
|
|
break
|
|
}
|
|
return segments
|
|
.filter { $0.start >= threshold && !normalizeToken($0.text).isEmpty }
|
|
.map(\.text).joined(separator: " ")
|
|
.trimmingCharacters(in: wsPunct)
|
|
}
|
|
|
|
/// Find the first trigger word and return everything after it.
|
|
/// Supports fuzzy matching so "melody" matches trigger "eliza".
|
|
static func textAfterTrigger(_ text: String, triggers: [String]) -> String {
|
|
let words = text.split(whereSeparator: \.isWhitespace).map(String.init)
|
|
guard !words.isEmpty else { return "" }
|
|
for trigger in triggers {
|
|
let tw = trigger.split(whereSeparator: \.isWhitespace)
|
|
.map { normalizeToken(String($0)) }.filter { !$0.isEmpty }
|
|
guard !tw.isEmpty, words.count >= tw.count else { continue }
|
|
for i in 0...(words.count - tw.count) {
|
|
let matched = (0..<tw.count).allSatisfy { j in
|
|
let w = normalizeToken(words[i + j])
|
|
return w == tw[j] || fuzzyTokenMatch(w, tw[j])
|
|
}
|
|
if matched {
|
|
let afterIdx = i + tw.count
|
|
return afterIdx < words.count
|
|
? words[afterIdx...].joined(separator: " ").trimmingCharacters(in: wsPunct)
|
|
: ""
|
|
}
|
|
}
|
|
}
|
|
return text
|
|
}
|
|
|
|
// MARK: Text-only helpers (no timing data required)
|
|
|
|
/// Quick text-only check for trigger presence.
|
|
static func matchesTextOnly(text: String, triggers: [String]) -> Bool {
|
|
guard !text.isEmpty else { return false }
|
|
let lower = text.lowercased()
|
|
for trigger in triggers {
|
|
let token = trigger.trimmingCharacters(in: wsPunct).lowercased()
|
|
if token.isEmpty { continue }
|
|
if lower.contains(token) { return true }
|
|
// Fuzzy: check individual words
|
|
let words = lower.split(whereSeparator: \.isWhitespace).map(String.init)
|
|
if words.contains(where: { fuzzyTokenMatch($0, token) }) { return true }
|
|
}
|
|
return false
|
|
}
|
|
|
|
/// Check if transcript begins with a trigger word.
|
|
static func startsWithTrigger(transcript: String, triggers: [String]) -> Bool {
|
|
let words = transcript.split(whereSeparator: \.isWhitespace)
|
|
.map { normalizeToken(String($0)) }.filter { !$0.isEmpty }
|
|
guard !words.isEmpty else { return false }
|
|
for trigger in triggers {
|
|
let tw = trigger.split(whereSeparator: \.isWhitespace)
|
|
.map { normalizeToken(String($0)) }.filter { !$0.isEmpty }
|
|
guard !tw.isEmpty, words.count >= tw.count else { continue }
|
|
if zip(tw, words.prefix(tw.count)).allSatisfy({ $0 == $1 || fuzzyTokenMatch($0, $1) }) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
/// Text-only command extraction fallback (when timing data is absent or unreliable).
|
|
static func textOnlyCommand(transcript: String, triggers: [String], minCommandLength: Int) -> String? {
|
|
guard matchesTextOnly(text: transcript, triggers: triggers),
|
|
startsWithTrigger(transcript: transcript, triggers: triggers) else { return nil }
|
|
let after = textAfterTrigger(transcript, triggers: triggers)
|
|
return after.count >= minCommandLength ? after : nil
|
|
}
|
|
|
|
// MARK: Fuzzy matching via Levenshtein distance
|
|
|
|
/// Returns true if two normalized tokens are "close enough" to be considered a match.
|
|
/// Threshold: ceil(maxLen / 3). e.g. "eliza" (7) ↔ "melody" (6) → threshold 3, distance 3 → match.
|
|
static func fuzzyTokenMatch(_ a: String, _ b: String) -> Bool {
|
|
if a == b { return true }
|
|
let maxLen = max(a.count, b.count)
|
|
guard maxLen > 2 else { return false } // Very short words → exact only
|
|
let threshold = max(1, (maxLen + 1) / 3)
|
|
return editDistance(a, b) <= threshold
|
|
}
|
|
|
|
private static func editDistance(_ a: String, _ b: String) -> Int {
|
|
let ac = Array(a), bc = Array(b)
|
|
let m = ac.count, n = bc.count
|
|
if m == 0 { return n }
|
|
if n == 0 { return m }
|
|
var prev = Array(0...n), curr = Array(repeating: 0, count: n + 1)
|
|
for i in 1...m {
|
|
curr[0] = i
|
|
for j in 1...n {
|
|
curr[j] = ac[i - 1] == bc[j - 1]
|
|
? prev[j - 1]
|
|
: 1 + min(prev[j], curr[j - 1], prev[j - 1])
|
|
}
|
|
swap(&prev, &curr)
|
|
}
|
|
return prev[n]
|
|
}
|
|
|
|
// MARK: Normalization helpers
|
|
|
|
private static func normalizeTriggers(_ triggers: [String]) -> [TriggerTokens] {
|
|
triggers.compactMap { trig in
|
|
let t = trig.split(whereSeparator: \.isWhitespace)
|
|
.map { normalizeToken(String($0)) }.filter { !$0.isEmpty }
|
|
return t.isEmpty ? nil : TriggerTokens(original: trig, tokens: t)
|
|
}
|
|
}
|
|
|
|
private static func normalizeSegments(_ segments: [WakeSegment]) -> [Token] {
|
|
segments.compactMap { seg in
|
|
let n = normalizeToken(seg.text)
|
|
return n.isEmpty ? nil : Token(normalized: n, start: seg.start, end: seg.end, range: seg.range)
|
|
}
|
|
}
|
|
|
|
static func normalizeToken(_ t: String) -> String {
|
|
t.trimmingCharacters(in: wsPunct).lowercased()
|
|
}
|
|
|
|
private static let wsPunct = CharacterSet.whitespacesAndNewlines.union(.punctuationCharacters)
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════════
|
|
// MARK: - SFTranscription → WakeSegment
|
|
// ═══════════════════════════════════════════════════════════════════════════════
|
|
|
|
private extension SFTranscription {
|
|
func toWakeSegments(transcript: String) -> [WakeSegment] {
|
|
segments.map { seg in
|
|
WakeSegment(text: seg.substring, start: seg.timestamp, duration: seg.duration,
|
|
range: Range(seg.substringRange, in: transcript))
|
|
}
|
|
}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Swabble Plugin
|
|
// ═══════════════════════════════════════════════════════════════════════════════
|
|
|
|
/**
|
|
* Swabble Plugin for Capacitor
|
|
*
|
|
* Provides voice wake word detection and continuous speech-to-text using Apple's
|
|
* Speech framework.
|
|
*
|
|
* State machine: idle → listening → triggered → capturing → listening
|
|
*/
|
|
@objc(SwabblePlugin)
|
|
public class SwabblePlugin: CAPPlugin, CAPBridgedPlugin {
|
|
public let identifier = "SwabblePlugin"
|
|
public let jsName = "Swabble"
|
|
public let pluginMethods: [CAPPluginMethod] = [
|
|
CAPPluginMethod(name: "start", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "stop", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "isListening", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "getConfig", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "updateConfig", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "checkPermissions", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "requestPermissions", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "getAudioDevices", returnType: CAPPluginReturnPromise),
|
|
CAPPluginMethod(name: "setAudioDevice", returnType: CAPPluginReturnPromise),
|
|
]
|
|
|
|
// ── State Machine ──────────────────────────────────────────────────────
|
|
|
|
private enum State: String {
|
|
case idle, listening, triggered, capturing
|
|
}
|
|
|
|
private var state: State = .idle {
|
|
didSet {
|
|
guard state != oldValue else { return }
|
|
notifyListeners("stateChange", data: ["state": state.rawValue])
|
|
}
|
|
}
|
|
|
|
// ── Audio & Speech ─────────────────────────────────────────────────────
|
|
|
|
private var audioEngine: AVAudioEngine?
|
|
private var speechRecognizer: SFSpeechRecognizer?
|
|
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
|
|
private var recognitionTask: SFSpeechRecognitionTask?
|
|
private let bufferQueue = AudioBufferQueue()
|
|
private var drainTimer: Timer?
|
|
private var captureTimer: Timer?
|
|
private var recognitionGeneration: Int = 0
|
|
|
|
// ── Audio Level Tracking (written from audio thread, read on main) ─────
|
|
|
|
private let audioLevelLock = NSLock()
|
|
private var _pendingRMS: Double = 0
|
|
private var _pendingPeak: Double = 0
|
|
private var noiseFloorRMS: Double = 1e-4
|
|
private var lastSpeechTime: Date?
|
|
private var lastAudioLevelEmitTime: Date?
|
|
|
|
// Voice-activity detection tunables (from classic VoiceWakeRuntime)
|
|
private let minSpeechRMS: Double = 1e-3
|
|
private let speechBoostFactor: Double = 6.0
|
|
private let audioLevelEmitInterval: TimeInterval = 0.066 // ~15 Hz
|
|
|
|
// ── Capture State ──────────────────────────────────────────────────────
|
|
|
|
private var captureStartTime: Date?
|
|
private var capturedTranscript: String = ""
|
|
private var activeTriggerEndTime: TimeInterval?
|
|
private var heardBeyondTrigger: Bool = false
|
|
private var lastTranscript: String = ""
|
|
private var lastTranscriptTime: Date?
|
|
private var cooldownUntil: Date?
|
|
|
|
// Capture tunables (from classic VoiceWakeRuntime)
|
|
private let silenceWindow: TimeInterval = 2.0
|
|
private let triggerOnlySilenceWindow: TimeInterval = 5.0
|
|
private let captureHardStop: TimeInterval = 120.0
|
|
private let debounceAfterSend: TimeInterval = 0.35
|
|
private let triggerPauseWindow: TimeInterval = 0.55
|
|
private let restartDelay: TimeInterval = 0.5
|
|
|
|
// ── Configuration ──────────────────────────────────────────────────────
|
|
|
|
private var config: PluginConfig?
|
|
|
|
struct PluginConfig {
|
|
var triggers: [String]
|
|
var minPostTriggerGap: TimeInterval
|
|
var minCommandLength: Int
|
|
var locale: String
|
|
var sampleRate: Double
|
|
|
|
init(from obj: JSObject) {
|
|
self.triggers = (obj["triggers"] as? [String]) ?? ["eliza"]
|
|
self.minPostTriggerGap = (obj["minPostTriggerGap"] as? Double) ?? 0.45
|
|
self.minCommandLength = (obj["minCommandLength"] as? Int) ?? 1
|
|
self.locale = (obj["locale"] as? String) ?? Locale.current.identifier
|
|
self.sampleRate = (obj["sampleRate"] as? Double) ?? 16000
|
|
}
|
|
|
|
func toJSObject() -> JSObject {
|
|
[
|
|
"triggers": triggers,
|
|
"minPostTriggerGap": minPostTriggerGap,
|
|
"minCommandLength": minCommandLength,
|
|
"locale": locale,
|
|
"sampleRate": sampleRate,
|
|
]
|
|
}
|
|
|
|
fileprivate var gateConfig: GateConfig {
|
|
GateConfig(triggers: triggers, minPostTriggerGap: minPostTriggerGap,
|
|
minCommandLength: minCommandLength)
|
|
}
|
|
}
|
|
|
|
// ── Notification Observers ─────────────────────────────────────────────
|
|
|
|
private var interruptionObserver: NSObjectProtocol?
|
|
private var routeChangeObserver: NSObjectProtocol?
|
|
private var mediaResetObserver: NSObjectProtocol?
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Lifecycle
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
override public func load() {
|
|
super.load()
|
|
setupNotificationObservers()
|
|
}
|
|
|
|
private func setupNotificationObservers() {
|
|
interruptionObserver = NotificationCenter.default.addObserver(
|
|
forName: AVAudioSession.interruptionNotification,
|
|
object: nil, queue: .main
|
|
) { [weak self] note in self?.handleAudioInterruption(note) }
|
|
|
|
routeChangeObserver = NotificationCenter.default.addObserver(
|
|
forName: AVAudioSession.routeChangeNotification,
|
|
object: nil, queue: .main
|
|
) { [weak self] _ in self?.handleRouteChange() }
|
|
|
|
mediaResetObserver = NotificationCenter.default.addObserver(
|
|
forName: AVAudioSession.mediaServicesWereResetNotification,
|
|
object: nil, queue: .main
|
|
) { [weak self] _ in self?.handleMediaServicesReset() }
|
|
}
|
|
|
|
deinit {
|
|
[interruptionObserver, routeChangeObserver, mediaResetObserver]
|
|
.compactMap { $0 }
|
|
.forEach { NotificationCenter.default.removeObserver($0) }
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Plugin Methods
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
@objc func start(_ call: CAPPluginCall) {
|
|
guard let configObj = call.getObject("config") else {
|
|
call.reject("Missing config parameter")
|
|
return
|
|
}
|
|
let cfg = PluginConfig(from: configObj)
|
|
config = cfg
|
|
|
|
SFSpeechRecognizer.requestAuthorization { [weak self] status in
|
|
DispatchQueue.main.async {
|
|
guard let self else { return }
|
|
switch status {
|
|
case .authorized:
|
|
self.beginListening(config: cfg, call: call)
|
|
case .denied, .restricted:
|
|
call.resolve(["started": false, "error": "Speech recognition not authorized"])
|
|
case .notDetermined:
|
|
call.resolve(["started": false, "error": "Speech recognition authorization pending"])
|
|
@unknown default:
|
|
call.resolve(["started": false, "error": "Unknown authorization status"])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
@objc func stop(_ call: CAPPluginCall) {
|
|
stopInternal()
|
|
call.resolve()
|
|
}
|
|
|
|
@objc func isListening(_ call: CAPPluginCall) {
|
|
call.resolve(["listening": state != .idle])
|
|
}
|
|
|
|
@objc func getConfig(_ call: CAPPluginCall) {
|
|
if let config {
|
|
call.resolve(["config": config.toJSObject()])
|
|
} else {
|
|
call.resolve(["config": NSNull()])
|
|
}
|
|
}
|
|
|
|
@objc func updateConfig(_ call: CAPPluginCall) {
|
|
guard let obj = call.getObject("config") else {
|
|
call.reject("Missing config parameter")
|
|
return
|
|
}
|
|
if var cfg = config {
|
|
if let t = obj["triggers"] as? [String] { cfg.triggers = t }
|
|
if let g = obj["minPostTriggerGap"] as? Double { cfg.minPostTriggerGap = g }
|
|
if let l = obj["minCommandLength"] as? Int { cfg.minCommandLength = l }
|
|
if let loc = obj["locale"] as? String { cfg.locale = loc }
|
|
if let sr = obj["sampleRate"] as? Double { cfg.sampleRate = sr }
|
|
config = cfg
|
|
}
|
|
call.resolve()
|
|
}
|
|
|
|
@objc public override func checkPermissions(_ call: CAPPluginCall) {
|
|
let speech = SFSpeechRecognizer.authorizationStatus()
|
|
let mic = AVAudioSession.sharedInstance().recordPermission
|
|
call.resolve([
|
|
"microphone": micPermissionString(mic),
|
|
"speechRecognition": speechPermissionString(speech),
|
|
])
|
|
}
|
|
|
|
@objc public override func requestPermissions(_ call: CAPPluginCall) {
|
|
SFSpeechRecognizer.requestAuthorization { [weak self] speechStatus in
|
|
AVAudioSession.sharedInstance().requestRecordPermission { micGranted in
|
|
DispatchQueue.main.async {
|
|
guard let self else { return }
|
|
call.resolve([
|
|
"microphone": self.micPermissionString(micGranted ? .granted : .denied),
|
|
"speechRecognition": self.speechPermissionString(speechStatus),
|
|
])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
@objc func getAudioDevices(_ call: CAPPluginCall) {
|
|
let session = AVAudioSession.sharedInstance()
|
|
let inputs = session.availableInputs ?? []
|
|
let currentUID = session.currentRoute.inputs.first?.uid
|
|
let devices: [[String: Any]] = inputs.map { port in
|
|
["id": port.uid, "name": port.portName, "isDefault": port.uid == currentUID]
|
|
}
|
|
call.resolve(["devices": devices])
|
|
}
|
|
|
|
@objc func setAudioDevice(_ call: CAPPluginCall) {
|
|
guard let deviceId = call.getString("deviceId") else {
|
|
call.reject("Missing deviceId")
|
|
return
|
|
}
|
|
let session = AVAudioSession.sharedInstance()
|
|
guard let inputs = session.availableInputs,
|
|
let preferred = inputs.first(where: { $0.uid == deviceId }) else {
|
|
call.reject("Audio device not found")
|
|
return
|
|
}
|
|
do {
|
|
try session.setPreferredInput(preferred)
|
|
call.resolve()
|
|
} catch {
|
|
call.reject("Failed to set audio device: \(error.localizedDescription)")
|
|
}
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Recognition Lifecycle
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
private func beginListening(config: PluginConfig, call: CAPPluginCall) {
|
|
// Clean up any prior session without emitting idle stateChange
|
|
stopInternal(emitIdle: false)
|
|
|
|
speechRecognizer = SFSpeechRecognizer(locale: Locale(identifier: config.locale))
|
|
guard let speechRecognizer, speechRecognizer.isAvailable else {
|
|
call.resolve(["started": false,
|
|
"error": "Speech recognizer unavailable for locale \(config.locale)"])
|
|
return
|
|
}
|
|
|
|
do {
|
|
try configureAudioSession()
|
|
try startRecognitionPipeline()
|
|
state = .listening
|
|
call.resolve(["started": true])
|
|
} catch {
|
|
call.resolve(["started": false, "error": error.localizedDescription])
|
|
}
|
|
}
|
|
|
|
/// Start audio engine + recognition task + drain timer.
|
|
private func startRecognitionPipeline() throws {
|
|
recognitionGeneration &+= 1
|
|
let generation = recognitionGeneration
|
|
|
|
let request = SFSpeechAudioBufferRecognitionRequest()
|
|
request.shouldReportPartialResults = true
|
|
request.taskHint = .dictation
|
|
if #available(iOS 16, *) { request.addsPunctuation = true }
|
|
recognitionRequest = request
|
|
|
|
if audioEngine == nil { audioEngine = AVAudioEngine() }
|
|
guard let audioEngine else { throw SwabbleError.audioEngineUnavailable }
|
|
|
|
let inputNode = audioEngine.inputNode
|
|
let format = inputNode.outputFormat(forBus: 0)
|
|
guard format.channelCount > 0, format.sampleRate > 0 else {
|
|
throw SwabbleError.noAudioInput
|
|
}
|
|
inputNode.removeTap(onBus: 0)
|
|
|
|
// Audio tap: copy buffer (thread-safe) and store RMS for main thread to read.
|
|
inputNode.installTap(onBus: 0, bufferSize: 1024, format: format) { [weak self] buffer, _ in
|
|
guard let self else { return }
|
|
self.bufferQueue.enqueue(buffer)
|
|
if let (rms, peak) = Self.calculateRMS(buffer: buffer) {
|
|
self.audioLevelLock.lock()
|
|
self._pendingRMS = rms
|
|
self._pendingPeak = peak
|
|
self.audioLevelLock.unlock()
|
|
}
|
|
}
|
|
|
|
audioEngine.prepare()
|
|
try audioEngine.start()
|
|
startDrainTimer()
|
|
|
|
recognitionTask = speechRecognizer?.recognitionTask(with: request) {
|
|
[weak self, generation] result, error in
|
|
DispatchQueue.main.async {
|
|
self?.handleRecognitionResult(result: result, error: error, generation: generation)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Soft restart: keep audio engine running, just restart recognition request + task.
|
|
/// Used when the recognizer hits its ~1-minute limit or encounters a transient error.
|
|
private func softRestartRecognition() {
|
|
recognitionGeneration &+= 1
|
|
let generation = recognitionGeneration
|
|
|
|
recognitionTask?.cancel()
|
|
recognitionTask = nil
|
|
recognitionRequest?.endAudio()
|
|
recognitionRequest = nil
|
|
|
|
guard state != .idle, let speechRecognizer, speechRecognizer.isAvailable else { return }
|
|
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + restartDelay) { [weak self] in
|
|
guard let self, self.state != .idle else { return }
|
|
|
|
let request = SFSpeechAudioBufferRecognitionRequest()
|
|
request.shouldReportPartialResults = true
|
|
request.taskHint = .dictation
|
|
if #available(iOS 16, *) { request.addsPunctuation = true }
|
|
self.recognitionRequest = request
|
|
|
|
let gen = self.recognitionGeneration
|
|
self.recognitionTask = self.speechRecognizer?.recognitionTask(with: request) {
|
|
[weak self] result, error in
|
|
DispatchQueue.main.async {
|
|
self?.handleRecognitionResult(result: result, error: error, generation: gen)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Hard restart: tear everything down and rebuild.
|
|
/// Used after audio interruptions or media services reset.
|
|
private func hardRestartRecognition() {
|
|
haltRecognitionPipeline()
|
|
guard let config, state != .idle else { return }
|
|
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + restartDelay) { [weak self] in
|
|
guard let self, self.state != .idle else { return }
|
|
self.speechRecognizer = SFSpeechRecognizer(locale: Locale(identifier: config.locale))
|
|
do {
|
|
try self.configureAudioSession()
|
|
try self.startRecognitionPipeline()
|
|
} catch {
|
|
self.emitError(code: "restart_failed", message: error.localizedDescription,
|
|
recoverable: true)
|
|
// Exponential backoff retry
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { [weak self] in
|
|
self?.hardRestartRecognition()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Halt the speech recognition pipeline. Audio engine, tap, and drain timer are stopped.
|
|
private func haltRecognitionPipeline() {
|
|
recognitionGeneration &+= 1
|
|
recognitionTask?.cancel()
|
|
recognitionTask = nil
|
|
recognitionRequest?.endAudio()
|
|
recognitionRequest = nil
|
|
stopDrainTimer()
|
|
bufferQueue.clear()
|
|
audioEngine?.inputNode.removeTap(onBus: 0)
|
|
audioEngine?.stop()
|
|
audioEngine = nil
|
|
}
|
|
|
|
/// Full stop - return to idle and release all resources.
|
|
private func stopInternal(emitIdle: Bool = true) {
|
|
haltRecognitionPipeline()
|
|
stopCaptureTimer()
|
|
speechRecognizer = nil
|
|
capturedTranscript = ""
|
|
captureStartTime = nil
|
|
activeTriggerEndTime = nil
|
|
heardBeyondTrigger = false
|
|
lastTranscript = ""
|
|
lastTranscriptTime = nil
|
|
lastSpeechTime = nil
|
|
cooldownUntil = nil
|
|
noiseFloorRMS = 1e-4
|
|
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
|
if emitIdle { state = .idle }
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Recognition Result Handling
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
private func handleRecognitionResult(result: SFSpeechRecognitionResult?,
|
|
error: Error?, generation: Int) {
|
|
// Drop stale callbacks from superseded recognition sessions
|
|
guard generation == recognitionGeneration else { return }
|
|
|
|
if let error {
|
|
if state != .idle {
|
|
emitError(code: "recognition_error", message: error.localizedDescription,
|
|
recoverable: true)
|
|
softRestartRecognition()
|
|
}
|
|
return
|
|
}
|
|
|
|
guard let result else { return }
|
|
let transcript = result.bestTranscription.formattedString
|
|
let isFinal = result.isFinal
|
|
let segments = result.bestTranscription.toWakeSegments(transcript: transcript)
|
|
let confidence = result.bestTranscription.segments.last?.confidence ?? 0
|
|
|
|
// Build JS segments for the transcript event
|
|
let jsSegments: [[String: Any]] = segments.map { seg in
|
|
["text": seg.text, "start": seg.start, "duration": seg.duration, "isFinal": isFinal]
|
|
}
|
|
notifyListeners("transcript", data: [
|
|
"transcript": transcript, "segments": jsSegments,
|
|
"isFinal": isFinal, "confidence": Double(confidence),
|
|
])
|
|
|
|
if !transcript.isEmpty {
|
|
lastTranscript = transcript
|
|
lastTranscriptTime = Date()
|
|
}
|
|
|
|
switch state {
|
|
case .listening:
|
|
handleListeningResult(transcript: transcript, segments: segments,
|
|
isFinal: isFinal, confidence: confidence)
|
|
case .capturing:
|
|
handleCapturingResult(transcript: transcript, segments: segments, isFinal: isFinal)
|
|
case .triggered, .idle:
|
|
break
|
|
}
|
|
|
|
// When recognition ends naturally (time limit), soft-restart to keep listening
|
|
if isFinal, state != .idle {
|
|
softRestartRecognition()
|
|
}
|
|
}
|
|
|
|
// ── Listening state: look for wake word ────────────────────────────────
|
|
|
|
private func handleListeningResult(transcript: String, segments: [WakeSegment],
|
|
isFinal: Bool, confidence: Float) {
|
|
guard let config else { return }
|
|
if let cooldown = cooldownUntil, Date() < cooldown { return }
|
|
|
|
// 1) Timing-based match (preferred: uses post-trigger gap from segment timing)
|
|
if let match = WakeGate.match(transcript: transcript, segments: segments,
|
|
config: config.gateConfig) {
|
|
triggerWakeWord(match: match, transcript: transcript, confidence: confidence)
|
|
return
|
|
}
|
|
|
|
// 2) Text-only fallback on final results (timing data absent/unreliable)
|
|
if isFinal,
|
|
let command = SwabbleWakeBridgeContract.textOnlyCommand(
|
|
transcript: transcript,
|
|
triggers: config.triggers,
|
|
minCommandLength: config.minCommandLength
|
|
) {
|
|
let trigger = config.triggers.first ?? ""
|
|
let fallback = GateMatch(triggerWord: trigger, triggerEndTime: 0,
|
|
postGap: 0, command: command)
|
|
triggerWakeWord(match: fallback, transcript: transcript, confidence: confidence)
|
|
return
|
|
}
|
|
|
|
// 3) Trigger-only detection: user said just the wake word and paused
|
|
if isTriggerOnly(transcript: transcript) {
|
|
scheduleTriggerOnlyCheck(transcript: transcript)
|
|
}
|
|
}
|
|
|
|
// ── Capturing state: accumulate post-trigger speech ────────────────────
|
|
|
|
private func handleCapturingResult(transcript: String, segments: [WakeSegment],
|
|
isFinal: Bool) {
|
|
guard let config else { return }
|
|
|
|
// Use timing data if available, fall back to text-based extraction
|
|
let command: String
|
|
if let trigEnd = activeTriggerEndTime, !segments.isEmpty {
|
|
let timed = WakeGate.commandText(transcript: transcript, segments: segments,
|
|
triggerEndTime: trigEnd)
|
|
command = timed.isEmpty
|
|
? WakeGate.textAfterTrigger(transcript, triggers: config.triggers)
|
|
: timed
|
|
} else {
|
|
command = WakeGate.textAfterTrigger(transcript, triggers: config.triggers)
|
|
}
|
|
|
|
if !command.isEmpty {
|
|
capturedTranscript = command
|
|
if !heardBeyondTrigger { heardBeyondTrigger = true }
|
|
}
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Wake Word Trigger & Capture
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
private func triggerWakeWord(match: GateMatch, transcript: String, confidence: Float) {
|
|
state = .triggered
|
|
notifyListeners("wakeWord", data: SwabbleWakeBridgeContract.wakeWordPayload(
|
|
match: SwabbleBridgeMatch(
|
|
triggerWord: match.triggerWord,
|
|
triggerEndTime: match.triggerEndTime,
|
|
postGap: match.postGap,
|
|
command: match.command
|
|
),
|
|
transcript: transcript,
|
|
confidence: Double(confidence)
|
|
))
|
|
beginCapture(initialCommand: match.command, triggerEndTime: match.triggerEndTime)
|
|
}
|
|
|
|
private func beginCapture(initialCommand: String, triggerEndTime: TimeInterval) {
|
|
state = .capturing
|
|
capturedTranscript = initialCommand
|
|
captureStartTime = Date()
|
|
activeTriggerEndTime = triggerEndTime
|
|
heardBeyondTrigger = !initialCommand.isEmpty
|
|
lastSpeechTime = Date()
|
|
cooldownUntil = nil
|
|
startCaptureTimer()
|
|
}
|
|
|
|
private func startCaptureTimer() {
|
|
stopCaptureTimer()
|
|
captureTimer = Timer.scheduledTimer(withTimeInterval: 0.2, repeats: true) { [weak self] _ in
|
|
self?.checkCaptureTimeout()
|
|
}
|
|
}
|
|
|
|
private func stopCaptureTimer() {
|
|
captureTimer?.invalidate()
|
|
captureTimer = nil
|
|
}
|
|
|
|
private func checkCaptureTimeout() {
|
|
guard state == .capturing else { stopCaptureTimer(); return }
|
|
let now = Date()
|
|
|
|
// Hard stop after maximum capture duration
|
|
if let start = captureStartTime, now.timeIntervalSince(start) >= captureHardStop {
|
|
finalizeCapture()
|
|
return
|
|
}
|
|
|
|
// Silence detection: different thresholds based on whether we heard post-trigger speech
|
|
let threshold = heardBeyondTrigger ? silenceWindow : triggerOnlySilenceWindow
|
|
if let lastSpeech = lastSpeechTime, now.timeIntervalSince(lastSpeech) >= threshold {
|
|
finalizeCapture()
|
|
}
|
|
}
|
|
|
|
private func finalizeCapture() {
|
|
guard state == .capturing else { return }
|
|
stopCaptureTimer()
|
|
cooldownUntil = Date().addingTimeInterval(debounceAfterSend)
|
|
|
|
let finalText = capturedTranscript.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if !finalText.isEmpty {
|
|
notifyListeners("transcript", data: [
|
|
"transcript": finalText, "segments": [] as [[String: Any]],
|
|
"isFinal": true, "confidence": 1.0,
|
|
])
|
|
}
|
|
|
|
// Reset capture state
|
|
capturedTranscript = ""
|
|
captureStartTime = nil
|
|
activeTriggerEndTime = nil
|
|
heardBeyondTrigger = false
|
|
lastSpeechTime = nil
|
|
|
|
// Return to listening
|
|
state = .listening
|
|
softRestartRecognition()
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Trigger-Only Detection
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
/// Check if the transcript is just the trigger word with no command after it.
|
|
private func isTriggerOnly(transcript: String) -> Bool {
|
|
guard let config else { return false }
|
|
return SwabbleWakeBridgeContract.isTriggerOnly(
|
|
transcript: transcript,
|
|
triggers: config.triggers
|
|
)
|
|
}
|
|
|
|
/// If the transcript hasn't changed after the pause window, start capture.
|
|
private func scheduleTriggerOnlyCheck(transcript: String) {
|
|
let snapshotTime = lastTranscriptTime
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + triggerPauseWindow) { [weak self] in
|
|
guard let self, self.state == .listening else { return }
|
|
guard self.lastTranscriptTime == snapshotTime, self.lastTranscript == transcript else { return }
|
|
guard self.isTriggerOnly(transcript: transcript) else { return }
|
|
if let cooldown = self.cooldownUntil, Date() < cooldown { return }
|
|
|
|
let trigger = self.config?.triggers.first ?? ""
|
|
self.state = .triggered
|
|
self.notifyListeners("wakeWord", data: SwabbleWakeBridgeContract.triggerOnlyPayload(
|
|
trigger: trigger,
|
|
transcript: transcript
|
|
))
|
|
self.beginCapture(initialCommand: "", triggerEndTime: 0)
|
|
}
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Audio Level Processing
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
private static func calculateRMS(buffer: AVAudioPCMBuffer) -> (rms: Double, peak: Double)? {
|
|
guard let channelData = buffer.floatChannelData?[0] else { return nil }
|
|
let count = Int(buffer.frameLength)
|
|
guard count > 0 else { return nil }
|
|
var sum: Double = 0
|
|
var peak: Float = 0
|
|
for i in 0..<count {
|
|
let sample = abs(channelData[i])
|
|
sum += Double(sample) * Double(sample)
|
|
if sample > peak { peak = sample }
|
|
}
|
|
return (sqrt(sum / Double(count)), Double(peak))
|
|
}
|
|
|
|
/// Called from the drain timer on the main thread. Updates noise floor, speech detection,
|
|
/// and emits throttled audioLevel events.
|
|
private func processAudioLevel() {
|
|
audioLevelLock.lock()
|
|
let rms = _pendingRMS
|
|
let peak = _pendingPeak
|
|
audioLevelLock.unlock()
|
|
guard rms > 0 else { return }
|
|
|
|
// Adaptive noise floor: fast decay (quiet room), slow rise (speech/noise)
|
|
let alpha: Double = rms < noiseFloorRMS ? 0.08 : 0.01
|
|
noiseFloorRMS = max(1e-7, noiseFloorRMS + (rms - noiseFloorRMS) * alpha)
|
|
|
|
// Mark speech when audio is clearly above adaptive threshold
|
|
let threshold = max(minSpeechRMS, noiseFloorRMS * speechBoostFactor)
|
|
if rms >= threshold {
|
|
lastSpeechTime = Date()
|
|
}
|
|
|
|
// Throttle audioLevel events to ~15 Hz
|
|
let now = Date()
|
|
if let lastEmit = lastAudioLevelEmitTime, now.timeIntervalSince(lastEmit) < audioLevelEmitInterval {
|
|
return
|
|
}
|
|
lastAudioLevelEmitTime = now
|
|
|
|
let normalized = min(1.0, max(0.0, rms / max(minSpeechRMS, threshold)))
|
|
notifyListeners("audioLevel", data: ["level": normalized, "peak": min(1.0, peak)])
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Drain Timer
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
private func startDrainTimer() {
|
|
stopDrainTimer()
|
|
// 40ms interval matches the classic iOS implementation's drain cadence
|
|
drainTimer = Timer.scheduledTimer(withTimeInterval: 0.04, repeats: true) { [weak self] _ in
|
|
guard let self else { return }
|
|
let buffers = self.bufferQueue.drain()
|
|
if let request = self.recognitionRequest {
|
|
for buf in buffers { request.append(buf) }
|
|
}
|
|
self.processAudioLevel()
|
|
}
|
|
}
|
|
|
|
private func stopDrainTimer() {
|
|
drainTimer?.invalidate()
|
|
drainTimer = nil
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Audio Session Configuration
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
private func configureAudioSession() throws {
|
|
let session = AVAudioSession.sharedInstance()
|
|
try session.setCategory(.playAndRecord, mode: .measurement, options: [
|
|
.duckOthers, .mixWithOthers, .allowBluetooth, .defaultToSpeaker,
|
|
])
|
|
try session.setActive(true, options: [])
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Audio Interruption Handling
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
private func handleAudioInterruption(_ notification: Notification) {
|
|
guard let info = notification.userInfo,
|
|
let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
|
|
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
|
|
|
|
switch type {
|
|
case .began:
|
|
guard state != .idle else { return }
|
|
// Another app (phone call, Siri) took audio focus. Halt gracefully.
|
|
haltRecognitionPipeline()
|
|
stopCaptureTimer()
|
|
capturedTranscript = ""
|
|
captureStartTime = nil
|
|
activeTriggerEndTime = nil
|
|
heardBeyondTrigger = false
|
|
state = .idle
|
|
emitError(code: "audio_interrupted",
|
|
message: "Audio session interrupted by another app", recoverable: true)
|
|
|
|
case .ended:
|
|
let options: AVAudioSession.InterruptionOptions
|
|
if let raw = info[AVAudioSessionInterruptionOptionKey] as? UInt {
|
|
options = AVAudioSession.InterruptionOptions(rawValue: raw)
|
|
} else {
|
|
options = []
|
|
}
|
|
if options.contains(.shouldResume), let config {
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in
|
|
self?.autoRestart(config: config)
|
|
}
|
|
}
|
|
|
|
@unknown default:
|
|
break
|
|
}
|
|
}
|
|
|
|
private func handleRouteChange() {
|
|
// Audio route changed (headphones, Bluetooth). Restart to pick up new device.
|
|
guard state != .idle else { return }
|
|
hardRestartRecognition()
|
|
}
|
|
|
|
private func handleMediaServicesReset() {
|
|
let savedConfig = config
|
|
stopInternal()
|
|
emitError(code: "media_reset", message: "Media services were reset", recoverable: true)
|
|
if let savedConfig {
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { [weak self] in
|
|
self?.autoRestart(config: savedConfig)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Restart from idle after an interruption or reset.
|
|
private func autoRestart(config: PluginConfig) {
|
|
guard state == .idle else { return }
|
|
self.config = config
|
|
speechRecognizer = SFSpeechRecognizer(locale: Locale(identifier: config.locale))
|
|
guard let speechRecognizer, speechRecognizer.isAvailable else {
|
|
emitError(code: "restart_failed", message: "Speech recognizer unavailable",
|
|
recoverable: false)
|
|
return
|
|
}
|
|
do {
|
|
try configureAudioSession()
|
|
try startRecognitionPipeline()
|
|
state = .listening
|
|
} catch {
|
|
emitError(code: "restart_failed", message: error.localizedDescription,
|
|
recoverable: false)
|
|
}
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// MARK: - Helpers
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
private func emitError(code: String, message: String, recoverable: Bool) {
|
|
notifyListeners("error", data: [
|
|
"code": code, "message": message, "recoverable": recoverable,
|
|
])
|
|
}
|
|
|
|
private func micPermissionString(_ status: AVAudioSession.RecordPermission) -> String {
|
|
switch status {
|
|
case .granted: return "granted"
|
|
case .denied: return "denied"
|
|
case .undetermined: return "prompt"
|
|
@unknown default: return "prompt"
|
|
}
|
|
}
|
|
|
|
private func speechPermissionString(_ status: SFSpeechRecognizerAuthorizationStatus) -> String {
|
|
switch status {
|
|
case .authorized: return "granted"
|
|
case .denied, .restricted: return "denied"
|
|
case .notDetermined: return "prompt"
|
|
@unknown default: return "prompt"
|
|
}
|
|
}
|
|
|
|
private enum SwabbleError: LocalizedError {
|
|
case audioEngineUnavailable
|
|
case noAudioInput
|
|
|
|
var errorDescription: String? {
|
|
switch self {
|
|
case .audioEngineUnavailable: return "Unable to create audio engine"
|
|
case .noAudioInput: return "No audio input available"
|
|
}
|
|
}
|
|
}
|
|
}
|