项目文件夹

文件
wehub-resource-sync c3749daf48
Tests / test-linux (3.13) (push) Failing after 0s
Tests / test-linux (3.11) (push) Failing after 1s
Tests / lint (push) Failing after 0s
Tests / test-linux (3.9) (push) Failing after 1s
Docker / build (push) Failing after 1s
Docker / build-gpu (push) Failing after 2s
Tests / test-windows (push) Has been cancelled
Tests / test-macos (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:03:03 +08:00

950 行
36 KiB
Python

"""
MemPalace configuration system.
Priority: env vars > config file (~/.mempalace/config.json) > defaults
"""
import json
import os
import re
from datetime import date, datetime
from functools import lru_cache
from pathlib import Path
# ── Input validation ──────────────────────────────────────────────────────────
# Shared sanitizers for wing/room/entity names. Prevents path traversal,
# excessively long strings, and special characters that could cause issues
# in file paths, SQLite, or ChromaDB metadata.
MAX_NAME_LENGTH = 128
_SAFE_NAME_RE = re.compile(r"^(?:[^\W_]|[^\W_][\w .'-]{0,126}[^\W_])$")
# MCP clients (e.g. Claude Desktop, WorkBuddy) occasionally relay lone UTF-16
# surrogates (U+D800–U+DFFF) when proxying binary-in-Unicode or corrupted
# clipboard input. Python's ``str.encode('utf-8')`` raises on these, which
# crashes ChromaDB add/upsert with -32000. See issue #1235.
_LONE_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
def strip_lone_surrogates(text: str) -> str:
"""Replace lone UTF-16 surrogates with U+FFFD so the string is legal UTF-8 (#1235)."""
return _LONE_SURROGATE_RE.sub("", text)
# Tool output mined from real transcripts routinely embeds a NUL character
# (U+0000) — e.g. captured Bash output where a reader raced a background
# writer, or genuine binary/NUL-delimited command output. A document
# containing one is otherwise valid, well-formed text (unlike a lone
# surrogate, which is invalid UTF-8), but handing it to ChromaDB's
# SQLite/FTS5 layer can corrupt the FTS5 inverted index for the *whole*
# collection (``PRAGMA quick_check`` reports "malformed inverted index for
# FTS5 table"), not just fail to store that one document. Stripping it
# before it reaches the chromadb client is the same defense-in-depth this
# module already applies to lone surrogates (#1235) — sanitize input we
# don't control before it reaches a datastore we don't control.
def strip_nul_bytes(text: str) -> str:
"""Replace embedded NUL characters with U+FFFD before ChromaDB storage."""
return text.replace("\x00", "")
def normalize_wing_name(name: str) -> str:
"""Lower-case + collapse separators (`-`, ` `) to `_` for wing slugs.
The same rule is applied by ``init`` when persisting `topics_by_wing`
and when writing `mempalace.yaml`, so the miner's lookup matches at
mine time regardless of the source dirname.
Leading/trailing separators are stripped so a path-encoded dirname like
``-home-user-proj`` yields ``home_user_proj`` rather than a leading-
underscore slug that ``sanitize_name`` (and thus the MCP write tools)
would reject.
"""
return name.lower().replace(" ", "_").replace("-", "_").strip("_")
def sanitize_name(value: str, field_name: str = "name") -> str:
"""Validate and sanitize a wing/room/entity name.
Raises ValueError if the name is invalid.
"""
if not isinstance(value, str) or not value.strip():
raise ValueError(f"{field_name} must be a non-empty string")
value = value.strip()
if len(value) > MAX_NAME_LENGTH:
raise ValueError(f"{field_name} exceeds maximum length of {MAX_NAME_LENGTH} characters")
# Block path traversal
if ".." in value or "/" in value or "\\" in value:
raise ValueError(f"{field_name} contains invalid path characters")
# Block null bytes
if "\x00" in value:
raise ValueError(f"{field_name} contains null bytes")
# Enforce safe character set
if not _SAFE_NAME_RE.match(value):
raise ValueError(f"{field_name} contains invalid characters")
return value
def sanitize_kg_value(value: str, field_name: str = "value") -> str:
"""Validate a knowledge-graph entity name (subject or object).
More permissive than sanitize_name — allows punctuation like commas,
colons, and parentheses that are common in natural-language KG values.
Only blocks null bytes and over-length strings.
Not used for wing/room names (which have filesystem constraints) or
predicates (which should be simple relationship identifiers).
"""
if not isinstance(value, str) or not value.strip():
raise ValueError(f"{field_name} must be a non-empty string")
value = value.strip()
if len(value) > MAX_NAME_LENGTH:
raise ValueError(f"{field_name} exceeds maximum length of {MAX_NAME_LENGTH} characters")
if "\x00" in value:
raise ValueError(f"{field_name} contains null bytes")
return strip_lone_surrogates(value)
# ISO-8601 temporal validator for knowledge-graph temporal parameters
# (as_of, valid_from, valid_to, ended).
#
# The KG stores temporal values as TEXT. Lexicographic comparisons are only
# safe when datetime values use one canonical shape. Accept full dates for
# legacy compatibility and exact UTC datetimes for sub-day precision.
#
# Accepted:
# YYYY-MM-DD
# YYYY-MM-DDTHH:MM:SSZ
# YYYY-MM-DDTHH:MM:SS+00:00 (normalized to ...Z)
#
# Rejected:
# partial dates, naive datetimes, non-UTC timezone offsets, fractional
# seconds, and SQLite-style space-separated datetimes.
_ISO_DATE_RE = re.compile(r"^\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])$")
_ISO_UTC_DATETIME_RE = re.compile(
r"^\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])"
r"T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:Z|\+00:00)$"
)
def _validate_iso_temporal_calendar(value: str) -> None:
"""Reject impossible calendar values after regex shape validation."""
if _ISO_DATE_RE.match(value):
date.fromisoformat(value)
return
if _ISO_UTC_DATETIME_RE.match(value):
datetime.fromisoformat(value.replace("Z", "+00:00"))
return
raise ValueError
def sanitize_iso_temporal(value, field_name: str = "date"):
"""Validate an ISO-8601 date or canonical UTC datetime string.
Accepts ``None`` and ``""`` as pass-through values.
Accepted non-empty string forms:
- ``YYYY-MM-DD``
- ``YYYY-MM-DDTHH:MM:SSZ``
- ``YYYY-MM-DDTHH:MM:SS+00:00`` normalized to ``...Z``
Partial dates are rejected because KG queries compare TEXT temporal values.
Non-canonical datetime forms are rejected because mixed temporal string
formats can silently return wrong KG query results.
"""
if value is None or value == "":
return value
if not isinstance(value, str):
raise ValueError(f"{field_name} must be a string")
value = value.strip()
try:
_validate_iso_temporal_calendar(value)
except ValueError:
raise ValueError(
f"{field_name}={value!r} is not a valid ISO-8601 date or UTC datetime "
"(expected YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ)"
) from None
if value.endswith("+00:00"):
value = f"{value[:-6]}Z"
return value
def sanitize_iso_date(value, field_name: str = "date"):
"""Backward-compatible wrapper for ISO temporal validation.
Historically this accepted only full dates. It now also accepts canonical
UTC datetimes, but the old name is kept so existing imports continue to
work.
"""
return sanitize_iso_temporal(value, field_name)
def sanitize_content(value: str, max_length: int = 100_000) -> str:
"""Validate drawer/diary content length."""
if not isinstance(value, str) or not value.strip():
raise ValueError("content must be a non-empty string")
if len(value) > max_length:
raise ValueError(f"content exceeds maximum length of {max_length} characters")
if "\x00" in value:
raise ValueError("content contains null bytes")
return strip_lone_surrogates(value)
DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
DEFAULT_COLLECTION_NAME = "mempalace_drawers"
DEFAULT_BACKEND = "chroma"
DEFAULT_MILVUS_CONSISTENCY_LEVEL = "Strong"
_MILVUS_CONSISTENCY_LEVELS = {
"strong": "Strong",
"session": "Session",
"bounded": "Bounded",
"eventually": "Eventually",
}
# How many timestamped palace backups to retain before the oldest are
# pruned. Applies to the accumulating backups written by ``mempalace
# migrate`` and ``mempalace repair max-seq-id`` — see
# ``MempalaceConfig.max_backups``.
DEFAULT_MAX_BACKUPS = 10
def normalize_milvus_consistency_level(value) -> str:
raw = str(value).strip() if value else DEFAULT_MILVUS_CONSISTENCY_LEVEL
normalized = _MILVUS_CONSISTENCY_LEVELS.get(raw.lower())
if normalized:
return normalized
allowed = ", ".join(_MILVUS_CONSISTENCY_LEVELS.values())
raise ValueError(f"milvus_consistency_level must be one of: {allowed}")
def sqlite_read_uri(db_path: str) -> str:
"""Return a read-only ``file:`` URI for ``sqlite3.connect(..., uri=True)``.
A bare ``f"file:{db_path}?mode=ro"`` mis-parses paths containing spaces or
other URI-reserved characters — common in real home directories (a Windows
user folder like ``First Last``, many macOS paths). ``pathname2url``
percent-encodes the path and normalizes separators so the database opens on
every platform.
"""
from urllib.request import pathname2url
db_path = os.fspath(db_path)
return f"file:{pathname2url(db_path)}?mode=ro"
@lru_cache(maxsize=1)
def get_configured_collection_name() -> str:
"""Return the configured drawer collection name without repeated config-file reads."""
return MempalaceConfig().collection_name
# Single source of truth for chunking defaults. ``mempalace.miner``
# imports these so the legacy module-level ``CHUNK_SIZE`` /
# ``CHUNK_OVERLAP`` / ``MIN_CHUNK_SIZE`` constants stay in sync with
# ``MempalaceConfig.chunk_*``. Putting them here (not in miner.py) keeps
# the config layer self-contained and avoids circular imports.
DEFAULT_CHUNK_SIZE = 800
DEFAULT_CHUNK_OVERLAP = 100
DEFAULT_MIN_CHUNK_SIZE = 50
DEFAULT_TOPIC_WINGS = [
"emotions",
"consciousness",
"memory",
"technical",
"identity",
"family",
"creative",
]
DEFAULT_HALL_KEYWORDS = {
"emotions": [
"scared",
"afraid",
"worried",
"happy",
"sad",
"love",
"hate",
"feel",
"cry",
"tears",
],
"consciousness": [
"consciousness",
"conscious",
"aware",
"real",
"genuine",
"soul",
"exist",
"alive",
],
"memory": ["memory", "remember", "forget", "recall", "archive", "palace", "store"],
"technical": [
"code",
"python",
"script",
"bug",
"error",
"function",
"api",
"database",
"server",
],
"identity": ["identity", "name", "who am i", "persona", "self"],
"family": [
"family",
"kids",
"children",
"daughter",
"son",
"parent",
"mother",
"father",
],
"creative": [
"game",
"gameplay",
"player",
"app",
"design",
"art",
"music",
"story",
],
}
class MempalaceConfig:
"""Configuration manager for MemPalace.
Load order: env vars > config file > defaults.
"""
def __init__(self, config_dir=None):
"""Initialize config.
Args:
config_dir: Override config directory (useful for testing).
Defaults to ~/.mempalace.
"""
self._config_dir = (
Path(config_dir) if config_dir else Path(os.path.expanduser("~/.mempalace"))
)
self._config_file = self._config_dir / "config.json"
self._people_map_file = self._config_dir / "people_map.json"
self._file_config = {}
if self._config_file.exists():
try:
with open(self._config_file, "r") as f:
self._file_config = json.load(f)
except (json.JSONDecodeError, OSError):
self._file_config = {}
@property
def palace_path(self):
"""Path to the memory palace data directory."""
env_val = os.environ.get("MEMPALACE_PALACE_PATH") or os.environ.get("MEMPAL_PALACE_PATH")
if env_val:
# Normalize: expand ~ and collapse .. to match the CLI --palace
# code path (mcp_server.py:62) and prevent surprise redirection
# when the env var contains unresolved components.
return os.path.abspath(os.path.expanduser(env_val))
return os.path.expanduser(self._file_config.get("palace_path", DEFAULT_PALACE_PATH))
@property
def tunnel_file(self):
"""Path to the tunnel file, sibling of palace_path."""
return os.path.join(os.path.dirname(self.palace_path), "tunnels.json")
@property
def hallway_file(self):
"""Path to the hallway file, sibling of palace_path.
Mirrors ``tunnel_file`` so within-wing hallway state is scoped to the
configured palace and survives palace rebuilds (it does not live in
ChromaDB which can be recreated). Prior to this property the path was
hardcoded under ``~/.mempalace/hallways.json`` and multiple palaces on
one host silently shared one file (see ``hallways._legacy_hallway_file``).
"""
return os.path.join(os.path.dirname(self.palace_path), "hallways.json")
@property
def collection_name(self):
"""ChromaDB collection name."""
return self._file_config.get("collection_name", DEFAULT_COLLECTION_NAME)
@property
def backend(self):
"""Storage backend name.
Read from ``config.json`` first, then ``MEMPALACE_BACKEND``, then
``"chroma"`` for backwards compatibility with existing palaces.
"""
cfg_val = self._file_config.get("backend")
if cfg_val:
return str(cfg_val).strip().lower()
env_val = os.environ.get("MEMPALACE_BACKEND")
if env_val:
return env_val.strip().lower()
return DEFAULT_BACKEND
@property
def qdrant_url(self):
"""Qdrant endpoint for the opt-in ``qdrant`` backend.
Defaults to localhost so selecting Qdrant never silently sends memory
to a remote service. Users can point at a LAN or cloud endpoint via
config or ``MEMPALACE_QDRANT_URL`` when they deliberately choose that.
"""
env_val = os.environ.get("MEMPALACE_QDRANT_URL")
if env_val:
return env_val.strip()
return str(self._file_config.get("qdrant_url", "http://localhost:6333")).strip()
@property
def qdrant_api_key(self):
"""API key for the opt-in ``qdrant`` backend, if configured."""
env_val = os.environ.get("MEMPALACE_QDRANT_API_KEY")
if env_val:
return env_val
value = self._file_config.get("qdrant_api_key")
return str(value) if value else None
@property
def qdrant_namespace(self):
"""Optional Qdrant collection namespace/prefix."""
env_val = os.environ.get("MEMPALACE_QDRANT_NAMESPACE")
if env_val:
return env_val.strip()
value = self._file_config.get("qdrant_namespace")
return str(value).strip() if value else None
@property
def qdrant_timeout(self):
"""Qdrant HTTP timeout in seconds."""
env_val = os.environ.get("MEMPALACE_QDRANT_TIMEOUT")
raw = env_val if env_val is not None else self._file_config.get("qdrant_timeout", 10.0)
try:
timeout = float(raw)
except (TypeError, ValueError):
timeout = 10.0
return timeout if timeout > 0 else 10.0
@property
def milvus_uri(self):
"""Milvus endpoint for the opt-in ``milvus`` backend.
Defaults to ``None`` so selecting Milvus uses per-palace Milvus Lite at
``<palace>/milvus.db``. Set this only to deliberately use a shared
Milvus server, Zilliz Cloud, or a custom local Lite file.
"""
env_val = os.environ.get("MEMPALACE_MILVUS_URI")
if env_val:
return env_val.strip()
value = self._file_config.get("milvus_uri")
return str(value).strip() if value else None
@property
def milvus_token(self):
"""Token for the opt-in ``milvus`` backend, if configured."""
env_val = os.environ.get("MEMPALACE_MILVUS_TOKEN")
if env_val:
return env_val
value = self._file_config.get("milvus_token")
return str(value) if value else None
@property
def milvus_db_name(self):
"""Optional Milvus database name for the opt-in ``milvus`` backend."""
env_val = os.environ.get("MEMPALACE_MILVUS_DB_NAME")
if env_val:
return env_val.strip()
value = self._file_config.get("milvus_db_name")
return str(value).strip() if value else None
@property
def milvus_namespace(self):
"""Optional Milvus collection namespace/prefix."""
env_val = os.environ.get("MEMPALACE_MILVUS_NAMESPACE")
if env_val:
return env_val.strip()
value = self._file_config.get("milvus_namespace")
return str(value).strip() if value else None
@property
def milvus_consistency_level(self):
"""Milvus read consistency level for the opt-in ``milvus`` backend."""
env_val = os.environ.get("MEMPALACE_MILVUS_CONSISTENCY_LEVEL")
if env_val:
return normalize_milvus_consistency_level(env_val)
value = self._file_config.get("milvus_consistency_level")
return normalize_milvus_consistency_level(value)
@property
def pgvector_dsn(self):
"""Postgres DSN for the opt-in ``pgvector`` backend.
Defaults to a localhost DSN so selecting pgvector never silently sends
memory to a remote database. Point at a LAN or cloud Postgres via config
or ``MEMPALACE_PGVECTOR_DSN`` only when deliberately chosen.
"""
env_val = os.environ.get("MEMPALACE_PGVECTOR_DSN")
if env_val:
return env_val.strip()
return str(
self._file_config.get("pgvector_dsn", "postgresql://localhost:5432/mempalace")
).strip()
@property
def pgvector_namespace(self):
"""Optional pgvector table namespace/prefix for multi-tenant isolation."""
env_val = os.environ.get("MEMPALACE_PGVECTOR_NAMESPACE")
if env_val:
return env_val.strip()
value = self._file_config.get("pgvector_namespace")
return str(value).strip() if value else None
@property
def people_map(self):
"""Mapping of name variants to canonical names."""
if self._people_map_file.exists():
try:
with open(self._people_map_file, "r") as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
pass
return self._file_config.get("people_map", {})
@property
def hooks_auto_save(self):
"""Whether the stop/precompact hooks should block for auto-save.
When False, hooks pass through without blocking — equivalent to
disabling auto-save while keeping hook scripts installed.
"""
env_val = os.environ.get("MEMPALACE_HOOKS_AUTO_SAVE")
if env_val is not None:
return env_val.lower() not in ("false", "0", "no")
hooks = self._file_config.get("hooks", {})
return hooks.get("auto_save", True)
@property
def topic_wings(self):
"""List of topic wing names."""
return self._file_config.get("topic_wings", DEFAULT_TOPIC_WINGS)
@property
def hall_keywords(self):
"""Mapping of hall names to keyword lists."""
return self._file_config.get("hall_keywords", DEFAULT_HALL_KEYWORDS)
@staticmethod
def _try_coerce_int(value, minimum=None):
"""Coerce a raw config value to int, or ``None`` if it cannot be a
valid setting.
bool, empty/garbage string, non-numeric, and below-``minimum``
values all return ``None``. Shared by ``_coerce_config_int``
(which substitutes a documented default) and
``min_chunk_size_explicit`` (which must distinguish "unusable"
from "explicitly set" without crashing the convo path).
"""
if isinstance(value, bool):
return None
try:
if isinstance(value, str):
value = value.strip()
if not value:
return None
value = int(value)
except (TypeError, ValueError, OverflowError):
# OverflowError: JSON ``1e1000`` parses to float('inf'), and
# ``int(inf)`` raises it — still just garbage config, not a crash.
return None
if minimum is not None and value < minimum:
return None
return value
def _coerce_config_int(self, key: str, default: int, minimum=None) -> int:
"""Read an int config value, falling back to ``default`` on bad input.
Hand-edited ``config.json`` is the most common source of garbage:
a string, a bool, a negative number, or a JSON null. None of those
should crash mining or hang ``chunk_text()`` — fall back silently
to the documented default rather than letting a typo break ingest.
"""
coerced = self._try_coerce_int(self._file_config.get(key, default), minimum)
return default if coerced is None else coerced
def _validated_chunk_config(self):
"""Return ``(chunk_size, chunk_overlap, min_chunk_size)`` post-validation.
Enforces the invariants the miner relies on:
* ``chunk_size >= 1``
* ``0 <= chunk_overlap < chunk_size`` — equality would loop forever
* ``min_chunk_size <= chunk_size`` — otherwise no chunk is ever
large enough to file, and ingest silently produces 0 drawers
Repairs (rather than raises) on violation so a single bad
config.json key doesn't take ingest down.
"""
chunk_size = self._coerce_config_int("chunk_size", DEFAULT_CHUNK_SIZE, minimum=1)
chunk_overlap = self._coerce_config_int("chunk_overlap", DEFAULT_CHUNK_OVERLAP, minimum=0)
min_chunk_size = self._coerce_config_int(
"min_chunk_size", DEFAULT_MIN_CHUNK_SIZE, minimum=0
)
if chunk_overlap >= chunk_size:
chunk_overlap = (
DEFAULT_CHUNK_OVERLAP
if DEFAULT_CHUNK_OVERLAP < chunk_size
else max(0, chunk_size - 1)
)
if min_chunk_size > chunk_size:
min_chunk_size = (
DEFAULT_MIN_CHUNK_SIZE if DEFAULT_MIN_CHUNK_SIZE <= chunk_size else chunk_size
)
return chunk_size, chunk_overlap, min_chunk_size
@property
def chunk_size(self) -> int:
"""Characters per drawer chunk (validated, ``>= 1``)."""
return self._validated_chunk_config()[0]
@property
def chunk_overlap(self) -> int:
"""Overlap between adjacent chunks (validated, ``< chunk_size``)."""
return self._validated_chunk_config()[1]
@property
def min_chunk_size(self) -> int:
"""Minimum chunk size — skip smaller chunks (validated, ``<= chunk_size``)."""
return self._validated_chunk_config()[2]
@property
def min_chunk_size_explicit(self):
"""Validated ``min_chunk_size`` iff the user explicitly set it.
Returns the coerced int when ``config.json`` defines a usable
``min_chunk_size`` (``>= 0`` and ``<= chunk_size``); ``None`` when
the key is absent/null or the value is unusable. ``convo_miner``
relies on the ``None`` sentinel to keep its lower 30-char floor
(more permissive than the 50-char project default, so short
exchanges are not dropped) for untuned users while still honoring
an explicit override —
replacing the raw, unvalidated ``_file_config`` reach that crashed
convo ingest on a bad key (#1024 review).
"""
raw = self._file_config.get("min_chunk_size")
if raw is None:
return None
coerced = self._try_coerce_int(raw, minimum=0)
if coerced is None or coerced > self.chunk_size:
return None
return coerced
@property
def entity_languages(self):
"""Languages whose entity-detection patterns should be applied.
Reads from env var ``MEMPALACE_ENTITY_LANGUAGES`` (comma-separated)
first, then the ``entity_languages`` field in ``config.json``,
defaulting to ``["en"]``.
"""
env_val = os.environ.get("MEMPALACE_ENTITY_LANGUAGES") or os.environ.get(
"MEMPAL_ENTITY_LANGUAGES"
)
if env_val:
return [s.strip() for s in env_val.split(",") if s.strip()] or ["en"]
cfg = self._file_config.get("entity_languages")
if isinstance(cfg, list) and cfg:
return [str(s) for s in cfg]
return ["en"]
def set_entity_languages(self, languages):
"""Persist the entity-detection language list to ``config.json``."""
normalized = [s.strip() for s in languages if s and s.strip()]
if not normalized:
normalized = ["en"]
self._file_config["entity_languages"] = normalized
self._config_dir.mkdir(parents=True, exist_ok=True)
try:
with open(self._config_file, "w", encoding="utf-8") as f:
json.dump(self._file_config, f, indent=2, ensure_ascii=False)
except OSError:
pass
try:
self._config_file.chmod(0o600)
except (OSError, NotImplementedError):
pass
return normalized
@property
def embedding_device(self):
"""Hardware device for the ONNX embedding model.
Values: ``"auto"`` (default), ``"cpu"``, ``"cuda"``, ``"coreml"``,
``"dml"``. Read from env ``MEMPALACE_EMBEDDING_DEVICE`` first, then
``embedding_device`` in ``config.json``, then ``"auto"``.
``auto`` resolves to the first available accelerator at runtime via
:mod:`mempalace.embedding`; requesting an unavailable accelerator
logs a warning and falls back to CPU.
"""
env_val = os.environ.get("MEMPALACE_EMBEDDING_DEVICE")
if env_val:
return env_val.strip().lower()
return str(self._file_config.get("embedding_device", "auto")).strip().lower()
@property
def embedding_model(self):
"""Embedding model identifier.
Values: ``"minilm"`` (ChromaDB's all-MiniLM-L6-v2 — English-only),
``"embeddinggemma"`` (multilingual, 100+ languages, default for
new installs since onboarding writes the choice). Read from env
``MEMPALACE_EMBEDDING_MODEL`` first, then ``embedding_model`` in
``config.json``, then ``"minilm"`` as a back-compat fallback for
palaces created before onboarding asked the question.
Switching models on an existing palace requires re-embedding
(different vector space) — ChromaDB rejects reads when the persisted
EF name doesn't match. Run ``mempalace repair rebuild-index`` after
changing this value.
"""
env_val = os.environ.get("MEMPALACE_EMBEDDING_MODEL")
if env_val:
return env_val.strip().lower()
return str(self._file_config.get("embedding_model", "minilm")).strip().lower()
@property
def embedding_threads(self) -> int:
"""Cap on the embedder's ONNX Runtime intra-op thread pool (#1068).
ChromaDB's ONNX embedder builds its ``InferenceSession`` with no thread
cap, so the intra-op pool defaults to the physical core count and a
background ``mine`` pins every core — stacked Stop-hook fires turn into
thermal events. ``OMP_NUM_THREADS`` is inert here (ORT owns its own
pool), so the cap is applied via ``SessionOptions`` in
:mod:`mempalace.embedding`.
Read from env ``MEMPALACE_EMBEDDING_THREADS`` first, then
``embedding_threads`` in ``config.json``. Semantics:
- unset / ``"auto"`` → half the logical CPUs (min 1), so a background
mine leaves the machine usable out of the box.
- a positive integer → exactly that many intra-op threads.
- ``0`` or negative → uncapped: ORT's default (physical core count),
for users who want maximum indexing throughput.
"""
raw = os.environ.get("MEMPALACE_EMBEDDING_THREADS")
if raw is None:
raw = self._file_config.get("embedding_threads")
if raw is None or str(raw).strip().lower() in ("", "auto"):
return max(1, (os.cpu_count() or 2) // 2)
try:
val = int(str(raw).strip())
except (TypeError, ValueError):
return max(1, (os.cpu_count() or 2) // 2)
return val if val > 0 else 0
def set_embedding_model(self, model: str) -> None:
"""Persist the embedding-model choice to ``config.json``.
Onboarding calls this once on first run. Accepts ``"minilm"`` or
``"embeddinggemma"``; other values are normalized to lowercase and
passed through (``embedding.get_embedding_function`` falls back to
minilm for unrecognized values).
"""
self._file_config["embedding_model"] = str(model).strip().lower()
self._config_dir.mkdir(parents=True, exist_ok=True)
try:
with open(self._config_file, "w", encoding="utf-8") as f:
json.dump(self._file_config, f, indent=2, ensure_ascii=False)
except OSError:
pass
try:
self._config_file.chmod(0o600)
except (OSError, NotImplementedError):
pass
def set_backend(self, backend: str) -> None:
"""Persist the storage backend choice to ``config.json``."""
backend = str(backend).strip().lower()
from .backends import get_backend_class
get_backend_class(backend)
self._file_config["backend"] = backend
self._config_dir.mkdir(parents=True, exist_ok=True)
try:
with open(self._config_file, "w", encoding="utf-8") as f:
json.dump(self._file_config, f, indent=2, ensure_ascii=False)
except OSError:
pass
try:
self._config_file.chmod(0o600)
except (OSError, NotImplementedError):
pass
@property
def topic_tunnel_min_count(self):
"""Minimum number of overlapping confirmed topics required to create
a cross-wing tunnel between two wings.
Default is ``1`` — any single shared topic produces a tunnel. Bump
to ``2+`` if your projects share lots of common-tech labels (Python,
Docker, Git) and you want only meaningfully overlapping wings to
link. Reads ``MEMPALACE_TOPIC_TUNNEL_MIN_COUNT`` env first, then the
config-file value, then ``1``.
"""
env_val = os.environ.get("MEMPALACE_TOPIC_TUNNEL_MIN_COUNT")
if env_val:
try:
parsed = int(env_val)
if parsed >= 1:
return parsed
except ValueError:
pass
cfg_val = self._file_config.get("topic_tunnel_min_count")
try:
parsed = int(cfg_val) if cfg_val is not None else 1
except (TypeError, ValueError):
parsed = 1
return max(1, parsed)
@property
def max_backups(self) -> int:
"""Number of timestamped palace backups to retain before pruning.
Applies to the accumulating, timestamped backups created by
``mempalace migrate`` (``<palace>.pre-migrate.<timestamp>``) and
``mempalace repair max-seq-id``
(``chroma.sqlite3.max-seq-id-backup-<timestamp>``). Each of those
commands writes a fresh full-size copy every run and historically
never deleted the old ones, so on a machine that mines or repairs on
a schedule the backup set could silently grow until it filled the
disk. After each backup is written, copies beyond this count (oldest
first) are removed.
Reads ``MEMPALACE_MAX_BACKUPS`` env first, then ``max_backups`` in
``config.json``, then the default of ``10``. A value of ``0`` disables
pruning and keeps every backup (use when an external retention policy
manages cleanup). Negative or non-numeric values fall back to the
default rather than crashing migrate/repair.
"""
env_val = os.environ.get("MEMPALACE_MAX_BACKUPS")
if env_val is not None:
coerced = self._try_coerce_int(env_val, minimum=0)
if coerced is not None:
return coerced
coerced = self._try_coerce_int(
self._file_config.get("max_backups", DEFAULT_MAX_BACKUPS), minimum=0
)
return DEFAULT_MAX_BACKUPS if coerced is None else coerced
@property
def hook_silent_save(self):
"""Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""
return self._file_config.get("hooks", {}).get("silent_save", True)
@property
def hook_desktop_toast(self):
"""Whether the stop hook shows a desktop notification via notify-send."""
return self._file_config.get("hooks", {}).get("desktop_toast", False)
@property
def hook_use_daemon(self):
"""Whether hooks should submit save/mine work to the opt-in daemon."""
env_val = os.environ.get("MEMPALACE_HOOKS_DAEMON")
if env_val is not None:
return env_val.lower() in ("true", "1", "yes", "on")
value = self._file_config.get("hooks", {}).get("daemon", False)
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.lower() in ("true", "1", "yes", "on")
return value == 1
def set_hook_setting(self, key: str, value: bool):
"""Update a hook setting and write config to disk."""
if "hooks" not in self._file_config:
self._file_config["hooks"] = {}
self._file_config["hooks"][key] = value
try:
with open(self._config_file, "w", encoding="utf-8") as f:
json.dump(self._file_config, f, indent=2, ensure_ascii=False)
except OSError:
pass
def init(self):
"""Create config directory and write default config.json if it doesn't exist."""
self._config_dir.mkdir(parents=True, exist_ok=True)
# Restrict directory permissions to owner only (Unix)
try:
self._config_dir.chmod(0o700)
except (OSError, NotImplementedError):
pass # Windows doesn't support Unix permissions
if not self._config_file.exists():
# Chunking parameters (chunk_size, chunk_overlap, min_chunk_size)
# are intentionally NOT written here — convo_miner.py distinguishes
# "user has tuned this" from "user is on defaults" by checking
# ``_file_config.get("min_chunk_size") is None``. Writing the
# miner.py defaults (50) into config.json breaks that detection
# and silently overrides convo_miner's stricter 30-char floor,
# dropping legitimate short conversation exchanges. Module-level
# defaults already apply correctly when these keys are absent.
default_config = {
"palace_path": DEFAULT_PALACE_PATH,
"collection_name": DEFAULT_COLLECTION_NAME,
"topic_wings": DEFAULT_TOPIC_WINGS,
"hall_keywords": DEFAULT_HALL_KEYWORDS,
}
with open(self._config_file, "w") as f:
json.dump(default_config, f, indent=2)
# Restrict config file to owner read/write only
try:
self._config_file.chmod(0o600)
except (OSError, NotImplementedError):
pass
return self._config_file
def save_people_map(self, people_map):
"""Write people_map.json to config directory.
Args:
people_map: Dict mapping name variants to canonical names.
"""
self._config_dir.mkdir(parents=True, exist_ok=True)
with open(self._people_map_file, "w") as f:
json.dump(people_map, f, indent=2)
try:
self._people_map_file.chmod(0o600)
except (OSError, NotImplementedError):
pass
return self._people_map_file