# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations import asyncio import os from dataclasses import dataclass, replace from typing import Literal import aiohttp from livekit.agents import ( APIConnectionError, APIStatusError, APITimeoutError, LanguageCode, tts, utils, ) from livekit.agents.types import ( DEFAULT_API_CONNECT_OPTIONS, NOT_GIVEN, APIConnectOptions, NotGivenOr, ) from livekit.agents.utils import is_given SUPPORTED_OUTPUT_FORMATS = { 8000: "raw-8khz-16bit-mono-pcm", 16000: "raw-16khz-16bit-mono-pcm", 22050: "raw-22050hz-16bit-mono-pcm", 24000: "raw-24khz-16bit-mono-pcm", 44100: "raw-44100hz-16bit-mono-pcm", 48000: "raw-48khz-16bit-mono-pcm", } @dataclass class ProsodyConfig: rate: Literal["x-slow", "slow", "medium", "fast", "x-fast"] | float | None = None volume: Literal["silent", "x-soft", "soft", "medium", "loud", "x-loud"] | float | None = None pitch: Literal["x-low", "low", "medium", "high", "x-high"] | None = None def validate(self) -> None: if self.rate: if isinstance(self.rate, float) and not 0.5 <= self.rate <= 2: raise ValueError("Prosody rate must be between 0.5 and 2") if isinstance(self.rate, str) and self.rate not in [ "x-slow", "slow", "medium", "fast", "x-fast", ]: raise ValueError( "Prosody rate must be one of 'x-slow', 'slow', 'medium', 'fast', 'x-fast'" ) if self.volume: if isinstance(self.volume, float) and not 0 <= self.volume <= 100: raise ValueError("Prosody volume must be between 0 and 100") if isinstance(self.volume, str) and self.volume not in [ "silent", "x-soft", "soft", "medium", "loud", "x-loud", ]: raise ValueError( "Prosody volume must be one of 'silent', 'x-soft', 'soft', 'medium', 'loud', 'x-loud'" # noqa: E501 ) if self.pitch and self.pitch not in [ "x-low", "low", "medium", "high", "x-high", ]: raise ValueError( "Prosody pitch must be one of 'x-low', 'low', 'medium', 'high', 'x-high'" ) def __post_init__(self) -> None: self.validate() @dataclass class StyleConfig: style: str degree: float | None = None def validate(self) -> None: if self.degree is not None and not 0.1 <= self.degree <= 2.0: raise ValueError("Style degree must be between 0.1 and 2.0") def __post_init__(self) -> None: self.validate() @dataclass class _TTSOptions: sample_rate: int subscription_key: str | None region: str | None voice: str language: LanguageCode | None speech_endpoint: str | None deployment_id: str | None prosody: NotGivenOr[ProsodyConfig] style: NotGivenOr[StyleConfig] lexicon_uri: NotGivenOr[str] auth_token: str | None = None def get_endpoint_url(self) -> str: base = ( self.speech_endpoint or f"https://{self.region}.tts.speech.microsoft.com/cognitiveservices/v1" ) if self.deployment_id: return f"{base}?deploymentId={self.deployment_id}" return base class TTS(tts.TTS): def __init__( self, *, voice: str = "en-US-JennyNeural", language: str | None = None, sample_rate: int = 24000, prosody: NotGivenOr[ProsodyConfig] = NOT_GIVEN, style: NotGivenOr[StyleConfig] = NOT_GIVEN, lexicon_uri: NotGivenOr[str] = NOT_GIVEN, speech_key: str | None = None, speech_region: str | None = None, speech_endpoint: str | None = None, deployment_id: str | None = None, speech_auth_token: str | None = None, http_session: aiohttp.ClientSession | None = None, ) -> None: super().__init__( capabilities=tts.TTSCapabilities(streaming=False), sample_rate=sample_rate, num_channels=1, ) if sample_rate not in SUPPORTED_OUTPUT_FORMATS: raise ValueError( f"Unsupported sample rate {sample_rate}. Supported: {list(SUPPORTED_OUTPUT_FORMATS)}" # noqa: E501 ) if not speech_key: speech_key = os.environ.get("AZURE_SPEECH_KEY") if not speech_region: speech_region = os.environ.get("AZURE_SPEECH_REGION") if not speech_endpoint: speech_endpoint = os.environ.get("AZURE_SPEECH_ENDPOINT") has_endpoint = bool(speech_endpoint) has_key_and_region = bool(speech_key and speech_region) has_token_and_region = bool(speech_auth_token and speech_region) if not (has_endpoint or has_key_and_region or has_token_and_region): raise ValueError( "Authentication requires one of: speech_endpoint (AZURE_SPEECH_ENDPOINT), " "speech_key & speech_region (AZURE_SPEECH_KEY & AZURE_SPEECH_REGION), " "or speech_auth_token & speech_region." ) if is_given(prosody): prosody.validate() if is_given(style): style.validate() self._session = http_session self._opts = _TTSOptions( sample_rate=sample_rate, subscription_key=speech_key, region=speech_region, speech_endpoint=speech_endpoint, voice=voice, deployment_id=deployment_id, language=LanguageCode(language) if language else None, prosody=prosody, style=style, lexicon_uri=lexicon_uri, auth_token=speech_auth_token, ) @property def model(self) -> str: return "unknown" @property def provider(self) -> str: return "Azure TTS" def update_options( self, *, voice: NotGivenOr[str] = NOT_GIVEN, language: NotGivenOr[str] = NOT_GIVEN, prosody: NotGivenOr[ProsodyConfig] = NOT_GIVEN, style: NotGivenOr[StyleConfig] = NOT_GIVEN, lexicon_uri: NotGivenOr[str] = NOT_GIVEN, ) -> None: if is_given(voice): self._opts.voice = voice if is_given(language): self._opts.language = LanguageCode(language) if is_given(prosody): prosody.validate() self._opts.prosody = prosody if is_given(style): style.validate() self._opts.style = style if is_given(lexicon_uri): self._opts.lexicon_uri = lexicon_uri def _ensure_session(self) -> aiohttp.ClientSession: if not self._session: self._session = utils.http_context.http_session() return self._session def synthesize( self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> tts.ChunkedStream: return ChunkedStream(tts=self, input_text=text, conn_options=conn_options) class ChunkedStream(tts.ChunkedStream): def __init__(self, *, tts: TTS, input_text: str, conn_options: APIConnectOptions) -> None: super().__init__(tts=tts, input_text=input_text, conn_options=conn_options) self._tts: TTS = tts self._opts = replace(tts._opts) def _build_ssml(self) -> str: lang = self._opts.language or "en-US" ssml = ( f'' ) ssml += f'' if is_given(self._opts.lexicon_uri): ssml += f'' if is_given(self._opts.style): degree = f' styledegree="{self._opts.style.degree}"' if self._opts.style.degree else "" ssml += f'' if is_given(self._opts.prosody): p = self._opts.prosody rate_attr = f' rate="{p.rate}"' if p.rate is not None else "" vol_attr = f' volume="{p.volume}"' if p.volume is not None else "" pitch_attr = f' pitch="{p.pitch}"' if p.pitch is not None else "" ssml += f"{self.input_text}" else: ssml += self.input_text if is_given(self._opts.style): ssml += "" ssml += "" return ssml async def _run(self, output_emitter: tts.AudioEmitter) -> None: headers = { "Content-Type": "application/ssml+xml", "X-Microsoft-OutputFormat": SUPPORTED_OUTPUT_FORMATS[self._opts.sample_rate], "User-Agent": "LiveKit Agents", } if self._opts.auth_token: headers["Authorization"] = f"Bearer {self._opts.auth_token}" elif self._opts.subscription_key: headers["Ocp-Apim-Subscription-Key"] = self._opts.subscription_key output_emitter.initialize( request_id=utils.shortuuid(), sample_rate=self._opts.sample_rate, num_channels=1, mime_type="audio/pcm", ) try: async with self._tts._ensure_session().post( url=self._opts.get_endpoint_url(), headers=headers, data=self._build_ssml(), timeout=aiohttp.ClientTimeout(total=30, sock_connect=self._conn_options.timeout), ) as resp: resp.raise_for_status() async for data, _ in resp.content.iter_chunks(): output_emitter.push(data) except asyncio.TimeoutError: raise APITimeoutError() from None except aiohttp.ClientResponseError as e: raise APIStatusError( message=e.message, status_code=e.status, request_id=None, body=None, ) from None except Exception as e: raise APIConnectionError(str(e)) from e