项目文件夹

文件
T

1051 行
40 KiB
Rust

//! Whisper audio transcription with Candle inference.
//!
//! This module provides complete Whisper transcription functionality including:
//! - Audio decoding (MP3, WAV, FLAC, etc.) via symphonia
//! - Resampling to 16kHz via rubato
//! - Whisper model inference via candle-transformers
//! - Automatic model download from HuggingFace Hub
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use crate::MemvidError;
// These are only used when whisper feature is enabled
#[cfg(feature = "whisper")]
use crate::Result;
#[cfg(feature = "whisper")]
use std::path::Path;
// ============================================================================
// Model Registry
// ============================================================================
/// Available Whisper models with verified HuggingFace model IDs
#[derive(Debug, Clone)]
pub struct WhisperModelInfo {
/// Model identifier for HuggingFace
pub model_id: &'static str,
/// Human-readable name
pub name: &'static str,
/// Approximate model size in MB
pub size_mb: f32,
/// Whether this is the default model
pub is_default: bool,
/// Language (e.g., "en" for English-only models, "multilingual" for others)
pub language: &'static str,
}
/// Available Whisper models registry
pub static WHISPER_MODELS: &[WhisperModelInfo] = &[
WhisperModelInfo {
model_id: "openai/whisper-small.en",
name: "whisper-small-en",
size_mb: 244.0,
is_default: true,
language: "en",
},
WhisperModelInfo {
model_id: "openai/whisper-small",
name: "whisper-small",
size_mb: 244.0,
is_default: false,
language: "multilingual",
},
];
/// Get model info by name, defaults to whisper-small-en
pub fn get_whisper_model_info(name: &str) -> &'static WhisperModelInfo {
WHISPER_MODELS
.iter()
.find(|m| m.name == name || m.model_id == name)
.unwrap_or_else(|| {
WHISPER_MODELS
.iter()
.find(|m| m.is_default)
.expect("default whisper model")
})
}
/// Get the default model info
pub fn default_whisper_model_info() -> &'static WhisperModelInfo {
WHISPER_MODELS
.iter()
.find(|m| m.is_default)
.expect("default whisper model exists")
}
// ============================================================================
// Whisper Model Configuration
// ============================================================================
/// Configuration for Whisper model initialization
#[derive(Debug, Clone)]
pub struct WhisperConfig {
/// Model name (e.g., "whisper-small-en")
pub model_name: String,
/// Directory where models are cached
pub models_dir: PathBuf,
/// Whether to run in offline mode (no downloads)
pub offline: bool,
}
impl Default for WhisperConfig {
fn default() -> Self {
let models_dir = std::env::var("MEMVID_MODELS_DIR")
.ok()
.map(PathBuf::from)
.or_else(|| dirs_next::home_dir().map(|d| d.join(".memvid/models")))
.unwrap_or_else(|| PathBuf::from(".memvid/models"));
let model_name = std::env::var("MEMVID_WHISPER_MODEL")
.unwrap_or_else(|_| "whisper-small-en".to_string());
let offline = std::env::var("MEMVID_OFFLINE").is_ok();
Self {
model_name,
models_dir,
offline,
}
}
}
// ============================================================================
// Whisper Error Types
// ============================================================================
/// Whisper-specific errors
#[derive(Debug, thiserror::Error)]
pub enum WhisperError {
/// Model not found
#[error("Whisper model '{model}' not found. {hint}")]
ModelNotFound { model: String, hint: String },
/// Audio decode failed
#[error("Failed to decode audio at {path:?}: {cause}")]
AudioDecodeError { path: PathBuf, cause: String },
/// Audio bytes decode failed
#[error("Failed to decode audio bytes: {cause}")]
AudioBytesDecodeError { cause: String },
/// Inference error
#[error("Whisper inference error: {cause}")]
InferenceError { cause: String },
/// Model download failed
#[error("Failed to download Whisper model: {cause}")]
DownloadError { cause: String },
}
impl From<WhisperError> for MemvidError {
fn from(err: WhisperError) -> Self {
MemvidError::ExtractionFailed {
reason: err.to_string().into_boxed_str(),
}
}
}
// ============================================================================
// Transcription Result
// ============================================================================
/// Result of audio transcription
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TranscriptionResult {
/// The transcribed text
pub text: String,
/// Language detected or specified
pub language: String,
/// Duration of audio in seconds
pub duration_secs: f32,
/// Optional timestamps for segments
#[serde(default)]
pub segments: Vec<TranscriptionSegment>,
}
/// A segment of transcription with timestamps
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TranscriptionSegment {
/// Start time in seconds
pub start: f32,
/// End time in seconds
pub end: f32,
/// Transcribed text for this segment
pub text: String,
}
// ============================================================================
// Audio Decoding (Feature-gated)
// ============================================================================
#[cfg(feature = "whisper")]
mod audio {
use super::*;
use std::fs::File;
use symphonia::core::audio::SampleBuffer;
use symphonia::core::codecs::DecoderOptions;
use symphonia::core::formats::FormatOptions;
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
/// Whisper sample rate (always 16kHz)
pub const WHISPER_SAMPLE_RATE: u32 = 16000;
/// Decode audio file to f32 samples, resampling to 16kHz mono
pub fn decode_audio_file(path: &Path) -> Result<(Vec<f32>, f32)> {
let file = File::open(path).map_err(|e| WhisperError::AudioDecodeError {
path: path.to_path_buf(),
cause: e.to_string(),
})?;
let mss = MediaSourceStream::new(Box::new(file), Default::default());
// Create a hint based on file extension
let mut hint = Hint::new();
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
hint.with_extension(ext);
}
// Probe the media source
let format_opts = FormatOptions::default();
let metadata_opts = MetadataOptions::default();
let probed = symphonia::default::get_probe()
.format(&hint, mss, &format_opts, &metadata_opts)
.map_err(|e| WhisperError::AudioDecodeError {
path: path.to_path_buf(),
cause: format!("Failed to probe audio format: {}", e),
})?;
let mut format = probed.format;
// Find the first audio track
let track = format
.tracks()
.iter()
.find(|t| t.codec_params.codec != symphonia::core::codecs::CODEC_TYPE_NULL)
.ok_or_else(|| WhisperError::AudioDecodeError {
path: path.to_path_buf(),
cause: "No audio track found".to_string(),
})?;
let track_id = track.id;
let sample_rate = track.codec_params.sample_rate.unwrap_or(44100);
let channels = track.codec_params.channels.map(|c| c.count()).unwrap_or(2);
// Create decoder
let decoder_opts = DecoderOptions::default();
let mut decoder = symphonia::default::get_codecs()
.make(&track.codec_params, &decoder_opts)
.map_err(|e| WhisperError::AudioDecodeError {
path: path.to_path_buf(),
cause: format!("Failed to create decoder: {}", e),
})?;
let mut samples: Vec<f32> = Vec::new();
// Decode all packets
loop {
let packet = match format.next_packet() {
Ok(p) => p,
Err(symphonia::core::errors::Error::IoError(e))
if e.kind() == std::io::ErrorKind::UnexpectedEof =>
{
break;
}
Err(_) => break,
};
if packet.track_id() != track_id {
continue;
}
let decoded = match decoder.decode(&packet) {
Ok(d) => d,
Err(_) => continue,
};
let spec = *decoded.spec();
let num_frames = decoded.frames();
if num_frames == 0 {
continue;
}
let mut sample_buf = SampleBuffer::<f32>::new(num_frames as u64, spec);
sample_buf.copy_interleaved_ref(decoded);
let interleaved = sample_buf.samples();
// Convert to mono by averaging channels
if channels > 1 {
for chunk in interleaved.chunks(channels) {
let mono: f32 = chunk.iter().sum::<f32>() / channels as f32;
samples.push(mono);
}
} else {
samples.extend_from_slice(interleaved);
}
}
let duration_secs = samples.len() as f32 / sample_rate as f32;
// Log pre-resampling stats
let pre_min = samples.iter().cloned().fold(f32::INFINITY, f32::min);
let pre_max = samples.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let pre_rms = (samples.iter().map(|x| x * x).sum::<f32>() / samples.len() as f32).sqrt();
tracing::info!(
sample_rate = sample_rate,
channels = channels,
samples_before = samples.len(),
pre_min = pre_min,
pre_max = pre_max,
pre_rms = pre_rms,
"Audio before resampling"
);
// High-quality sinc resampling to 16kHz
let samples = if sample_rate != WHISPER_SAMPLE_RATE {
let resampled = resample_sinc(&samples, sample_rate, WHISPER_SAMPLE_RATE);
// Log post-resampling stats
let post_min = resampled.iter().cloned().fold(f32::INFINITY, f32::min);
let post_max = resampled.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let post_rms =
(resampled.iter().map(|x| x * x).sum::<f32>() / resampled.len() as f32).sqrt();
tracing::info!(
samples_after = resampled.len(),
post_min = post_min,
post_max = post_max,
post_rms = post_rms,
"Audio after resampling"
);
resampled
} else {
tracing::info!("Audio already at 16kHz, no resampling needed");
samples
};
Ok((samples, duration_secs))
}
/// High-quality sinc resampling using rubato
fn resample_sinc(samples: &[f32], from_rate: u32, to_rate: u32) -> Vec<f32> {
use rubato::{FftFixedIn, Resampler};
if from_rate == to_rate {
return samples.to_vec();
}
// Create resampler
let chunk_size = 1024;
let mut resampler = FftFixedIn::<f32>::new(
from_rate as usize,
to_rate as usize,
chunk_size,
2, // sub_chunks for quality
1, // mono
)
.expect("Failed to create resampler");
let mut output = Vec::new();
let mut pos = 0;
// Process in chunks
while pos < samples.len() {
let end = (pos + chunk_size).min(samples.len());
let chunk = &samples[pos..end];
// Pad if needed
let input_chunk: Vec<f32> = if chunk.len() < chunk_size {
let mut padded = chunk.to_vec();
padded.resize(chunk_size, 0.0);
padded
} else {
chunk.to_vec()
};
let input = vec![input_chunk];
let resampled = resampler.process(&input, None).expect("Resampling failed");
if !resampled.is_empty() && !resampled[0].is_empty() {
output.extend_from_slice(&resampled[0]);
}
pos += chunk_size;
}
// Trim to expected length
let expected_len = (samples.len() as f64 * to_rate as f64 / from_rate as f64) as usize;
output.truncate(expected_len);
output
}
}
#[cfg(feature = "whisper")]
pub use audio::*;
// ============================================================================
// Whisper Transcriber (Candle Inference)
// ============================================================================
#[cfg(feature = "whisper")]
mod inference {
use super::*;
use candle_core::{DType, Device, IndexOp, Tensor};
use candle_nn::VarBuilder;
use candle_transformers::models::whisper::{self as m, Config, audio};
use hf_hub::{Repo, RepoType, api::sync::Api};
use tokenizers::Tokenizer;
/// Whisper model wrapper for transcription
pub struct WhisperTranscriber {
model: Model,
tokenizer: Tokenizer,
config: Config,
mel_filters: Vec<f32>,
device: Device,
}
#[allow(dead_code)]
enum Model {
Normal(m::model::Whisper),
Quantized(m::quantized_model::Whisper),
}
impl WhisperTranscriber {
/// Create a new WhisperTranscriber, downloading the model if needed
pub fn new(config: &WhisperConfig) -> Result<Self> {
// Use GPU if available: Metal (macOS) or CUDA (NVIDIA)
let device = Self::select_device();
tracing::info!(device = ?device, "Using device for Whisper");
let model_id = match config.model_name.as_str() {
"whisper-small-en" => "openai/whisper-small.en",
"whisper-small" => "openai/whisper-small",
"whisper-tiny.en" => "openai/whisper-tiny.en",
"whisper-tiny" => "openai/whisper-tiny",
"whisper-base.en" => "openai/whisper-base.en",
"whisper-base" => "openai/whisper-base",
"whisper-medium.en" => "openai/whisper-medium.en",
"whisper-medium" => "openai/whisper-medium",
"whisper-large-v3" => "openai/whisper-large-v3",
other => other, // Allow direct model IDs
};
tracing::info!(model_id = model_id, "Loading Whisper model");
let api = Api::new().map_err(|e| WhisperError::DownloadError {
cause: e.to_string(),
})?;
let repo = api.repo(Repo::with_revision(
model_id.to_string(),
RepoType::Model,
"main".to_string(),
));
// Download model files
let config_path = repo
.get("config.json")
.map_err(|e| WhisperError::DownloadError {
cause: format!("Failed to download config.json: {}", e),
})?;
let tokenizer_path =
repo.get("tokenizer.json")
.map_err(|e| WhisperError::DownloadError {
cause: format!("Failed to download tokenizer.json: {}", e),
})?;
let model_path =
repo.get("model.safetensors")
.map_err(|e| WhisperError::DownloadError {
cause: format!("Failed to download model.safetensors: {}", e),
})?;
// Load config
let config_str = std::fs::read_to_string(&config_path).map_err(|e| {
WhisperError::InferenceError {
cause: format!("Failed to read config: {}", e),
}
})?;
let model_config: Config =
serde_json::from_str(&config_str).map_err(|e| WhisperError::InferenceError {
cause: format!("Failed to parse config: {}", e),
})?;
// Load tokenizer
let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|e| {
WhisperError::InferenceError {
cause: format!("Failed to load tokenizer: {}", e),
}
})?;
// Load mel filters
let mel_bytes = match model_config.num_mel_bins {
80 => include_bytes!("melfilters.bytes").as_slice(),
128 => include_bytes!("melfilters128.bytes").as_slice(),
n => {
return Err(WhisperError::InferenceError {
cause: format!("Unsupported number of mel bins: {}", n),
}
.into());
}
};
let mut mel_filters = vec![0f32; mel_bytes.len() / 4];
<byteorder::LittleEndian as byteorder::ByteOrder>::read_f32_into(
mel_bytes,
&mut mel_filters,
);
// Load model weights
let vb = unsafe {
VarBuilder::from_mmaped_safetensors(&[model_path], DType::F32, &device).map_err(
|e| WhisperError::InferenceError {
cause: format!("Failed to load model weights: {}", e),
},
)?
};
let model = Model::Normal(m::model::Whisper::load(&vb, model_config.clone()).map_err(
|e| WhisperError::InferenceError {
cause: format!("Failed to load Whisper model: {}", e),
},
)?);
tracing::info!("Whisper model loaded successfully");
Ok(Self {
model,
tokenizer,
config: model_config,
mel_filters,
device,
})
}
/// Select the best available device (GPU if available, otherwise CPU)
fn select_device() -> Device {
// Try Metal (macOS Apple Silicon / AMD)
#[cfg(feature = "metal")]
{
if let Ok(device) = Device::new_metal(0) {
tracing::info!("Metal GPU available");
return device;
}
}
// Try CUDA (NVIDIA GPUs)
#[cfg(feature = "cuda")]
{
if let Ok(device) = Device::new_cuda(0) {
tracing::info!("CUDA GPU available");
return device;
}
}
// Fallback to CPU
tracing::info!("Using CPU (no GPU acceleration)");
Device::Cpu
}
/// Transcribe an audio file
pub fn transcribe_file(&mut self, path: &Path) -> Result<TranscriptionResult> {
// Decode audio to PCM
let (pcm_data, duration_secs) = super::decode_audio_file(path)?;
// Check audio statistics
let audio_min = pcm_data.iter().cloned().fold(f32::INFINITY, f32::min);
let audio_max = pcm_data.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let audio_mean = pcm_data.iter().sum::<f32>() / pcm_data.len() as f32;
let audio_rms =
(pcm_data.iter().map(|x| x * x).sum::<f32>() / pcm_data.len() as f32).sqrt();
tracing::info!(
duration = duration_secs,
samples = pcm_data.len(),
min = audio_min,
max = audio_max,
mean = audio_mean,
rms = audio_rms,
"Audio decoded"
);
self.transcribe_pcm(&pcm_data, duration_secs)
}
/// Transcribe PCM audio samples (16kHz mono f32)
pub fn transcribe_pcm(
&mut self,
pcm_data: &[f32],
duration_secs: f32,
) -> Result<TranscriptionResult> {
// Whisper processes audio in 30-second chunks
const CHUNK_LENGTH: usize = 30 * 16000; // 30 seconds at 16kHz
const N_FRAMES: usize = 3000; // frames per chunk
const SAMPLE_RATE: f32 = 16000.0;
// Detect and trim leading silence
let silence_threshold = 0.01; // RMS threshold for silence
let window_size = 1600; // 100ms windows at 16kHz
let start_sample = find_speech_start(pcm_data, silence_threshold, window_size);
let end_sample = find_speech_end(pcm_data, silence_threshold, window_size);
let trimmed_start = start_sample as f32 / SAMPLE_RATE;
let trimmed_end = end_sample as f32 / SAMPLE_RATE;
tracing::info!(
start_sample = start_sample,
end_sample = end_sample,
trimmed_start_sec = trimmed_start,
trimmed_end_sec = trimmed_end,
original_duration = duration_secs,
"Trimmed silence"
);
// Use trimmed audio
let pcm_data = &pcm_data[start_sample..end_sample];
let _trimmed_duration = pcm_data.len() as f32 / SAMPLE_RATE;
let mut all_text = String::new();
let mut segments = Vec::new();
// Process audio in chunks
let num_chunks = (pcm_data.len() + CHUNK_LENGTH - 1) / CHUNK_LENGTH;
for chunk_idx in 0..num_chunks {
let chunk_start = chunk_idx * CHUNK_LENGTH;
let chunk_end = (chunk_start + CHUNK_LENGTH).min(pcm_data.len());
let chunk = &pcm_data[chunk_start..chunk_end];
// Adjust timestamps to account for trimmed silence
let start_time = trimmed_start + chunk_start as f32 / SAMPLE_RATE;
let end_time = trimmed_start + chunk_end as f32 / SAMPLE_RATE;
tracing::info!(
chunk = chunk_idx + 1,
total = num_chunks,
start = start_time,
end = end_time,
"Processing chunk"
);
// Reset decoder KV cache for each new chunk
match &mut self.model {
Model::Normal(m) => m.decoder.reset_kv_cache(),
Model::Quantized(m) => m.decoder.reset_kv_cache(),
}
// Convert chunk to mel spectrogram
let mel = audio::pcm_to_mel(&self.config, chunk, &self.mel_filters);
let n_mels = self.config.num_mel_bins;
let mel_len = mel.len();
let n_frames = mel_len / n_mels;
if chunk_idx == 0 {
// Print config for debugging
tracing::info!(
num_mel_bins = self.config.num_mel_bins,
max_source_positions = self.config.max_source_positions,
max_target_positions = self.config.max_target_positions,
"Model config"
);
// Mel statistics
let mel_min = mel.iter().cloned().fold(f32::INFINITY, f32::min);
let mel_max = mel.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let mel_mean = mel.iter().sum::<f32>() / mel.len() as f32;
tracing::info!(
mel_len = mel_len,
n_mels = n_mels,
n_frames = n_frames,
chunk_samples = chunk.len(),
expected_frames = 3000,
mel_min = mel_min,
mel_max = mel_max,
mel_mean = mel_mean,
"Mel spectrogram computed"
);
}
// Ensure we have exactly 3000 frames (pad or truncate)
// NOTE: mel array from pcm_to_mel is stored as [mel_bin_0_all_frames, mel_bin_1_all_frames, ...]
// So each mel bin has n_frames contiguous values: mel[bin * n_frames + frame]
let mel = if n_frames < N_FRAMES {
// Pad each mel bin's frames with zeros to reach N_FRAMES
let mut padded = vec![0.0f32; n_mels * N_FRAMES];
for bin in 0..n_mels {
let src_start = bin * n_frames;
let dst_start = bin * N_FRAMES;
padded[dst_start..dst_start + n_frames]
.copy_from_slice(&mel[src_start..src_start + n_frames]);
}
padded
} else if n_frames > N_FRAMES {
// Truncate each mel bin's frames to N_FRAMES
let mut truncated = vec![0.0f32; n_mels * N_FRAMES];
for bin in 0..n_mels {
let src_start = bin * n_frames;
let dst_start = bin * N_FRAMES;
truncated[dst_start..dst_start + N_FRAMES]
.copy_from_slice(&mel[src_start..src_start + N_FRAMES]);
}
truncated
} else {
mel
};
let mel =
Tensor::from_vec(mel, (1, n_mels, N_FRAMES), &self.device).map_err(|e| {
WhisperError::InferenceError {
cause: format!("Failed to create mel tensor: {}", e),
}
})?;
if chunk_idx == 0 {
let mel_shape = mel.shape();
tracing::info!(
mel_shape = ?mel_shape,
"Mel tensor shape"
);
}
// Run encoder
let audio_features = match &mut self.model {
Model::Normal(m) => m.encoder.forward(&mel, true),
Model::Quantized(m) => m.encoder.forward(&mel, true),
}
.map_err(|e| WhisperError::InferenceError {
cause: format!("Encoder forward failed: {}", e),
})?;
if chunk_idx == 0 {
let af_shape = audio_features.shape();
tracing::info!(
audio_features_shape = ?af_shape,
"Audio features from encoder"
);
}
// Get special token IDs
let sot_token = self.token_id(m::SOT_TOKEN)?;
let transcribe_token = self.token_id(m::TRANSCRIBE_TOKEN)?;
let eot_token = self.token_id(m::EOT_TOKEN)?;
let no_timestamps_token = self.token_id(m::NO_TIMESTAMPS_TOKEN)?;
if chunk_idx == 0 {
let en_token = self.tokenizer.token_to_id("<|en|>");
tracing::info!(
sot = sot_token,
transcribe = transcribe_token,
eot = eot_token,
no_timestamps = no_timestamps_token,
en_token = ?en_token,
"Special tokens"
);
}
// Build initial prompt
// For English-only models (*.en), we DON'T use language token
// For multilingual models, we add language token after sot_token
let has_language_token = self.tokenizer.token_to_id("<|en|>").is_some();
// English-only models have vocab size 51864, multilingual have 51865
let is_english_only = self.config.vocab_size == 51864;
let tokens = if is_english_only {
// English-only: SOT -> transcribe -> notimestamps
vec![sot_token, transcribe_token, no_timestamps_token]
} else if has_language_token {
// Multilingual: SOT -> language -> transcribe -> notimestamps
let language_token = self.token_id("<|en|>")?;
vec![
sot_token,
language_token,
transcribe_token,
no_timestamps_token,
]
} else {
// Fallback
vec![sot_token, transcribe_token, no_timestamps_token]
};
if chunk_idx == 0 {
tracing::info!(
is_english_only = is_english_only,
vocab_size = self.config.vocab_size,
prompt_tokens = ?tokens,
"Initial prompt"
);
}
let mut all_tokens = tokens.clone();
// Autoregressive decoding with token suppression
let sample_len = self.config.max_target_positions / 2;
let mut repeat_count = 0;
let mut last_token: Option<u32> = None;
// Build suppression mask
let suppress_tokens = &self.config.suppress_tokens;
for i in 0..sample_len {
// For autoregressive decoding with KV cache:
// - First iteration: pass all prompt tokens, flush_kv_cache=true
// - Subsequent iterations: pass only the new token, flush_kv_cache=false
let tokens_tensor = Tensor::new(all_tokens.as_slice(), &self.device)
.and_then(|t| t.unsqueeze(0))
.map_err(|e| WhisperError::InferenceError {
cause: format!("Failed to create tokens tensor: {}", e),
})?;
if chunk_idx == 0 && i < 3 {
tracing::info!(
step = i,
all_tokens_len = all_tokens.len(),
tokens_shape = ?tokens_tensor.shape(),
"Decoder input"
);
}
// Get hidden states from decoder, then project to vocabulary
// Always pass all tokens (candle doesn't use KV cache the same way as PyTorch)
let logits = match &mut self.model {
Model::Normal(m) => {
let hidden = m
.decoder
.forward(&tokens_tensor, &audio_features, true)
.map_err(|e| WhisperError::InferenceError {
cause: format!("Decoder forward failed: {}", e),
})?;
m.decoder.final_linear(&hidden).map_err(|e| {
WhisperError::InferenceError {
cause: format!("Final linear failed: {}", e),
}
})?
}
Model::Quantized(m) => {
let hidden = m
.decoder
.forward(&tokens_tensor, &audio_features, true)
.map_err(|e| WhisperError::InferenceError {
cause: format!("Decoder forward failed: {}", e),
})?;
m.decoder.final_linear(&hidden).map_err(|e| {
WhisperError::InferenceError {
cause: format!("Final linear failed: {}", e),
}
})?
}
};
if chunk_idx == 0 && i == 0 {
tracing::info!(
logits_shape = ?logits.shape(),
"Decoder output logits"
);
}
// Get logits for last position
let (_, seq_len, _) =
logits.dims3().map_err(|e| WhisperError::InferenceError {
cause: format!("Failed to get logits dims: {}", e),
})?;
let mut logits_vec = logits
.i((0, seq_len - 1, ..))
.and_then(|t| t.to_vec1::<f32>())
.map_err(|e| WhisperError::InferenceError {
cause: format!("Failed to extract logits: {}", e),
})?;
// Apply token suppression from config
for &token_id in suppress_tokens.iter() {
if (token_id as usize) < logits_vec.len() {
logits_vec[token_id as usize] = f32::NEG_INFINITY;
}
}
// Suppress EOT token for first few steps to allow generation
if all_tokens.len() < 10 {
logits_vec[eot_token as usize] = f32::NEG_INFINITY;
}
// Suppress all special tokens during generation:
// - SOT (50257), language tokens (50258-50261), task tokens (50358-50359),
// - no_timestamps (50362), and timestamp tokens (50363+)
logits_vec[sot_token as usize] = f32::NEG_INFINITY;
logits_vec[transcribe_token as usize] = f32::NEG_INFINITY;
logits_vec[no_timestamps_token as usize] = f32::NEG_INFINITY;
// Suppress all tokens from 50257 onward (special tokens) except those in normal vocab
for token_id in 50257..logits_vec.len() {
logits_vec[token_id] = f32::NEG_INFINITY;
}
if chunk_idx == 0 && i == 0 {
tracing::info!(
suppress_count = suppress_tokens.len(),
eot_suppressed = all_tokens.len() < 10,
"Applied token suppression"
);
}
// Find argmax
let next_token = logits_vec
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| {
a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(idx, _)| idx as u32)
.unwrap_or(eot_token);
if chunk_idx == 0 && i < 5 {
let max_logit =
logits_vec.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let min_logit = logits_vec.iter().cloned().fold(f32::INFINITY, f32::min);
tracing::info!(
step = i,
next_token = next_token,
max_logit = max_logit,
min_logit = min_logit,
"Decoding step"
);
}
if next_token == eot_token || next_token >= self.config.vocab_size as u32 {
if chunk_idx == 0 && i < 5 {
tracing::info!(
next_token = next_token,
eot = eot_token,
"Stopping: EOT or invalid token"
);
}
break;
}
// Check for excessive repetition (stop if same token repeats >3 times)
if Some(next_token) == last_token {
repeat_count += 1;
if repeat_count > 3 {
tracing::debug!("Breaking due to token repetition");
break;
}
} else {
repeat_count = 0;
}
last_token = Some(next_token);
all_tokens.push(next_token);
}
// Decode tokens to text for this chunk
let prompt_len = if is_english_only { 3 } else { 4 };
if chunk_idx == 0 {
tracing::info!(
prompt_tokens = ?&all_tokens[..prompt_len],
generated_tokens = ?&all_tokens[prompt_len..],
total = all_tokens.len(),
"Generated tokens for chunk"
);
}
let chunk_text = self
.tokenizer
.decode(&all_tokens[prompt_len..], true) // Skip prompt tokens
.map_err(|e| WhisperError::InferenceError {
cause: format!("Failed to decode tokens: {}", e),
})?;
let trimmed_text = chunk_text.trim();
if !trimmed_text.is_empty() {
if !all_text.is_empty() {
all_text.push(' ');
}
all_text.push_str(trimmed_text);
segments.push(TranscriptionSegment {
start: start_time,
end: end_time,
text: trimmed_text.to_string(),
});
}
}
Ok(TranscriptionResult {
text: all_text.trim().to_string(),
language: "en".to_string(),
duration_secs,
segments,
})
}
fn token_id(&self, token: &str) -> Result<u32> {
self.tokenizer.token_to_id(token).ok_or_else(|| {
WhisperError::InferenceError {
cause: format!("Token '{}' not found in vocabulary", token),
}
.into()
})
}
}
/// Find the sample index where speech starts (after leading silence)
fn find_speech_start(samples: &[f32], threshold: f32, window_size: usize) -> usize {
for i in (0..samples.len()).step_by(window_size) {
let end = (i + window_size).min(samples.len());
let window = &samples[i..end];
let rms = (window.iter().map(|x| x * x).sum::<f32>() / window.len() as f32).sqrt();
if rms > threshold {
// Found speech, go back a bit to not cut off the start
return i.saturating_sub(window_size);
}
}
0 // No silence found, return start
}
/// Find the sample index where speech ends (before trailing silence)
fn find_speech_end(samples: &[f32], threshold: f32, window_size: usize) -> usize {
for i in (0..samples.len()).rev().step_by(window_size) {
let start = i.saturating_sub(window_size);
let window = &samples[start..=i.min(samples.len() - 1)];
let rms = (window.iter().map(|x| x * x).sum::<f32>() / window.len() as f32).sqrt();
if rms > threshold {
// Found speech, go forward a bit to not cut off the end
return (i + window_size).min(samples.len());
}
}
samples.len() // No silence found, return end
}
}
#[cfg(feature = "whisper")]
pub use inference::WhisperTranscriber;
// ============================================================================
// Tests
// ============================================================================
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn whisper_model_registry() {
let default = default_whisper_model_info();
assert_eq!(default.name, "whisper-small-en");
assert!(default.is_default);
assert_eq!(default.language, "en");
// Unknown model returns default
let unknown = get_whisper_model_info("nonexistent");
assert_eq!(unknown.name, "whisper-small-en");
}
#[test]
fn whisper_config_defaults() {
let config = WhisperConfig::default();
assert_eq!(config.model_name, "whisper-small-en");
}
}