/livekit-plugins/livekit-plugins-respeecher/livekit/plugins/respeecher/tts.py

# Copyright 2025 LiveKit, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import annotations

import asyncio
import base64
import json
import os
import weakref
from dataclasses import dataclass, replace

import aiohttp

from livekit.agents import (
    APIConnectionError,
    APIConnectOptions,
    APIError,
    APIStatusError,
    APITimeoutError,
    tokenize,
    tts,
    utils,
)
from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS, NOT_GIVEN, NotGivenOr
from livekit.agents.utils import is_given

from .log import logger
from .models import TTSEncoding, TTSModels, Voice, VoiceSettings
from .version import __version__

API_VERSION = __version__
API_AUTH_HEADER = "X-API-Key"
API_VERSION_HEADER = "LiveKit-Plugin-Respeecher-Version"
API_BASE_URL = "https://api.respeecher.com/v1"

DEFAULT_VOICES: dict[str, str] = {
    "/public/tts/en-rt": "samantha",
    "/public/tts/ua-rt": "olesia-conversation",
}


@dataclass
class _TTSOptions:
    model: TTSModels | str
    encoding: TTSEncoding
    sample_rate: int
    voice_id: str
    voice_settings: NotGivenOr[VoiceSettings]
    api_key: str
    base_url: str


async def list_voices(
    *,
    model: TTSModels | str = "/public/tts/en-rt",
    api_key: NotGivenOr[str] = NOT_GIVEN,
    base_url: str = API_BASE_URL,
    http_session: aiohttp.ClientSession | None = None,
) -> list[Voice]:
    """List available voices for the given Respeecher model.

    Args:
        model: The Respeecher TTS model whose voices should be listed.
        api_key: Respeecher API key. If not provided, uses RESPEECHER_API_KEY env variable.
        base_url: The base URL for the Respeecher API.
        http_session: Optional aiohttp session to use for the request.
    """
    resolved_api_key = api_key if is_given(api_key) else os.environ.get("RESPEECHER_API_KEY")
    if not resolved_api_key:
        raise ValueError("RESPEECHER_API_KEY must be set")

    session = http_session or utils.http_context.http_session()
    async with session.get(
        f"{base_url}{model}/voices",
        headers={
            API_AUTH_HEADER: resolved_api_key,
            API_VERSION_HEADER: API_VERSION,
        },
    ) as resp:
        resp.raise_for_status()
        data = await resp.json()
        voices = [Voice(voice_data) for voice_data in data]
        if not voices:
            raise APIError("No voices are available")
        return voices


class TTS(tts.TTS):
    def __init__(
        self,
        *,
        voice_id: NotGivenOr[str] = NOT_GIVEN,
        api_key: NotGivenOr[str] = NOT_GIVEN,
        model: TTSModels | str = "/public/tts/en-rt",
        encoding: TTSEncoding = "pcm_s16le",
        voice_settings: NotGivenOr[VoiceSettings] = NOT_GIVEN,
        sample_rate: int = 24000,
        tokenizer: NotGivenOr[tokenize.SentenceTokenizer] = NOT_GIVEN,
        http_session: aiohttp.ClientSession | None = None,
        base_url: str = API_BASE_URL,
    ) -> None:
        """
        Create a new instance of Respeecher TTS.

        Args:
            voice_id: ID of the voice to use. If not provided, a model-specific default is used (see `DEFAULT_VOICES`). Each model exposes a different set of voices; call the module-level `list_voices()` helper to discover the IDs available for the chosen model.
            api_key: Respeecher API key. If not provided, uses RESPEECHER_API_KEY env variable.
            model: The Respeecher TTS model to use.
            encoding: Audio encoding format.
            voice_settings: Optional voice settings including sampling parameters.
            sample_rate: Audio sample rate in Hz.
            http_session: Optional aiohttp session to use for requests.
            base_url: The base URL for the Respeecher API.
        """

        super().__init__(
            capabilities=tts.TTSCapabilities(
                streaming=True,
                aligned_transcript=False,
            ),
            sample_rate=sample_rate,
            num_channels=1,
        )

        respeecher_api_key = api_key if is_given(api_key) else os.environ.get("RESPEECHER_API_KEY")
        if not respeecher_api_key:
            raise ValueError("RESPEECHER_API_KEY must be set")

        resolved_voice_id = voice_id if is_given(voice_id) else DEFAULT_VOICES.get(model)
        if not resolved_voice_id:
            raise ValueError(
                f"voice_id is required for model {model!r} (no default voice is configured); "
                "pass voice_id explicitly or use one of the supported models."
            )

        self._opts = _TTSOptions(
            model=model,
            encoding=encoding,
            sample_rate=sample_rate,
            voice_id=resolved_voice_id,
            voice_settings=voice_settings,
            api_key=respeecher_api_key,
            base_url=base_url,
        )
        self._session = http_session
        self._streams = weakref.WeakSet[SynthesizeStream]()
        self._sentence_tokenizer = (
            tokenizer if is_given(tokenizer) else tokenize.blingfire.SentenceTokenizer()
        )
        self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse](
            connect_cb=self._connect_ws,
            close_cb=self._close_ws,
        )
        self._retired_pools: list[utils.ConnectionPool[aiohttp.ClientWebSocketResponse]] = []

    @property
    def model(self) -> str:
        return self._opts.model

    @property
    def provider(self) -> str:
        return "Respeecher"

    async def _connect_ws(self, timeout: float) -> aiohttp.ClientWebSocketResponse:
        session = self._ensure_session()
        # WebSocket protocol does not support custom headers, using query parameter
        ws_url = self._opts.base_url.replace("https://", "wss://").replace("http://", "ws://")
        if not ws_url.startswith("wss://"):
            logger.error("Insecure WebSocket connection detected, wss:// required")
            raise APIConnectionError("Secure WebSocket connection (wss://) required")

        full_ws_url = f"{ws_url}{self._opts.model}/tts/websocket?api_key={self._opts.api_key}&source={API_VERSION_HEADER}&version={API_VERSION}"
        return await asyncio.wait_for(session.ws_connect(full_ws_url), timeout)

    async def _close_ws(self, ws: aiohttp.ClientWebSocketResponse) -> None:
        await ws.close()

    def _ensure_session(self) -> aiohttp.ClientSession:
        if not self._session:
            self._session = utils.http_context.http_session()
        return self._session

    def update_options(
        self,
        *,
        voice_id: NotGivenOr[str] = NOT_GIVEN,
        voice_settings: NotGivenOr[VoiceSettings] = NOT_GIVEN,
        model: NotGivenOr[TTSModels | str] = NOT_GIVEN,
    ) -> None:
        """Update TTS options"""
        if is_given(model) and model != self._opts.model:
            self._opts.model = model
            # The model is baked into the WebSocket URL, so existing pooled
            # connections can't serve the new model. Retire the old pool
            # (letting any in-flight stream finish using its connection) and
            # route new requests through a fresh pool. The retired pool is
            # closed during aclose().
            self._retired_pools.append(self._pool)
            self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse](
                connect_cb=self._connect_ws,
                close_cb=self._close_ws,
            )

        if is_given(voice_id):
            self._opts.voice_id = voice_id
        if is_given(voice_settings):
            self._opts.voice_settings = voice_settings

    def synthesize(
        self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
    ) -> ChunkedStream:
        return ChunkedStream(tts=self, input_text=text, conn_options=conn_options)

    def prewarm(self) -> None:
        self._pool.prewarm()

    def stream(
        self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
    ) -> SynthesizeStream:
        stream = SynthesizeStream(tts=self, conn_options=conn_options)
        self._streams.add(stream)
        return stream

    async def aclose(self) -> None:
        for stream in list(self._streams):
            await stream.aclose()

        self._streams.clear()
        await self._pool.aclose()
        for pool in self._retired_pools:
            await pool.aclose()
        self._retired_pools.clear()


class ChunkedStream(tts.ChunkedStream):
    """Synthesize text using Respeecher HTTPS endpoint"""

    def __init__(self, *, tts: TTS, input_text: str, conn_options: APIConnectOptions) -> None:
        super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
        self._tts: TTS = tts
        self._opts = replace(tts._opts)

    async def _run(self, output_emitter: tts.AudioEmitter) -> None:
        """Run the TTS synthesis"""
        json_data = {
            "transcript": self._input_text,
            "voice": {
                "id": self._opts.voice_id,
            },
            "output_format": {
                "sample_rate": self._opts.sample_rate,
                "encoding": self._opts.encoding,
            },
        }

        if is_given(self._opts.voice_settings) and self._opts.voice_settings.sampling_params:
            json_data["voice"]["sampling_params"] = self._opts.voice_settings.sampling_params  # type: ignore[index]

        http_url = f"{self._opts.base_url}{self._opts.model}/tts/bytes"

        try:
            async with self._tts._ensure_session().post(
                http_url,
                headers={
                    API_AUTH_HEADER: self._opts.api_key,
                    API_VERSION_HEADER: API_VERSION,
                    "Content-Type": "application/json",
                },
                json=json_data,
                timeout=aiohttp.ClientTimeout(total=30, sock_connect=self._conn_options.timeout),
            ) as resp:
                resp.raise_for_status()

                output_emitter.initialize(
                    request_id=utils.shortuuid(),
                    sample_rate=self._opts.sample_rate,
                    num_channels=1,
                    # /tts/bytes returns WAV-wrapped PCM; the WebSocket stream returns raw PCM.
                    mime_type="audio/wav",
                )

                async for data, _ in resp.content.iter_chunks():
                    output_emitter.push(data)

                output_emitter.flush()
        except asyncio.TimeoutError:
            raise APITimeoutError() from None
        except aiohttp.ClientResponseError as e:
            raise APIStatusError(
                message=e.message, status_code=e.status, request_id=None, body=None
            ) from None
        except Exception as e:
            raise APIConnectionError() from e


class SynthesizeStream(tts.SynthesizeStream):
    """Streamed API using WebSocket for real-time synthesis"""

    def __init__(self, *, tts: TTS, conn_options: APIConnectOptions):
        super().__init__(tts=tts, conn_options=conn_options)
        self._tts: TTS = tts
        self._opts = replace(tts._opts)

    async def aclose(self) -> None:
        await super().aclose()

    async def _run(self, output_emitter: tts.AudioEmitter) -> None:
        context_id = utils.shortuuid()
        output_emitter.initialize(
            request_id=context_id,
            sample_rate=self._opts.sample_rate,
            num_channels=1,
            stream=True,
            mime_type="audio/pcm",
        )
        output_emitter.start_segment(segment_id=context_id)

        sent_tokenizer_stream = self._tts._sentence_tokenizer.stream()
        input_ended = False

        def _voice_payload() -> dict[str, object]:
            voice: dict[str, object] = {"id": self._opts.voice_id}
            if is_given(self._opts.voice_settings) and self._opts.voice_settings.sampling_params:
                voice["sampling_params"] = self._opts.voice_settings.sampling_params
            return voice

        async def _input_task() -> None:
            async for data in self._input_ch:
                if isinstance(data, self._FlushSentinel):
                    sent_tokenizer_stream.flush()
                    continue
                sent_tokenizer_stream.push_text(data)
            sent_tokenizer_stream.end_input()

        async def _sentence_stream_task(ws: aiohttp.ClientWebSocketResponse) -> None:
            nonlocal input_ended
            output_format = {
                "encoding": self._opts.encoding,
                "sample_rate": self._opts.sample_rate,
            }
            async for sent in sent_tokenizer_stream:
                self._mark_started()
                await ws.send_str(
                    json.dumps(
                        {
                            "context_id": context_id,
                            "transcript": sent.token if sent.token else " ",
                            "voice": _voice_payload(),
                            "continue": True,
                            "output_format": output_format,
                        }
                    )
                )

            await ws.send_str(
                json.dumps(
                    {
                        "context_id": context_id,
                        "transcript": " ",
                        "voice": _voice_payload(),
                        "continue": False,
                        "output_format": output_format,
                    }
                )
            )
            input_ended = True

        async def _recv_task(ws: aiohttp.ClientWebSocketResponse) -> None:
            while True:
                msg = await ws.receive(timeout=self._conn_options.timeout)
                if msg.type in (
                    aiohttp.WSMsgType.CLOSED,
                    aiohttp.WSMsgType.CLOSE,
                    aiohttp.WSMsgType.CLOSING,
                ):
                    raise APIStatusError(
                        "Respeecher connection closed unexpectedly", request_id=context_id
                    )

                if msg.type == aiohttp.WSMsgType.ERROR:
                    raise APIConnectionError(
                        f"Respeecher WebSocket transport error: {ws.exception()}"
                    )

                if msg.type != aiohttp.WSMsgType.TEXT:
                    logger.warning("Unexpected Respeecher message type %s", msg.type)
                    continue

                data = json.loads(msg.data)

                if data.get("context_id") != context_id:
                    logger.warning(
                        "Received a message with context_id=%s instead of expected %s",
                        data.get("context_id"),
                        context_id,
                    )
                    continue

                if data.get("type") == "error":
                    raise APIError(f"Respeecher returned error: {data.get('error')}")

                if data.get("type") == "chunk":
                    audio_data = base64.b64decode(data["data"])
                    output_emitter.push(audio_data)

                elif data.get("type") == "done":
                    if input_ended:
                        break

        try:
            async with self._tts._pool.connection(timeout=self._conn_options.timeout) as ws:
                tasks = [
                    asyncio.create_task(_input_task()),
                    asyncio.create_task(_sentence_stream_task(ws)),
                    asyncio.create_task(_recv_task(ws)),
                ]

                try:
                    await asyncio.gather(*tasks)
                finally:
                    await sent_tokenizer_stream.aclose()
                    await utils.aio.gracefully_cancel(*tasks)
        except asyncio.TimeoutError:
            raise APITimeoutError() from None
        except aiohttp.ClientResponseError as e:
            raise APIStatusError(
                message=e.message, status_code=e.status, request_id=None, body=None
            ) from None
        except APIError:
            raise
        except Exception as e:
            raise APIConnectionError() from e
        finally:
            output_emitter.end_segment()