package indexer import ( "bytes" "context" "encoding/json" "encoding/xml" "errors" "fmt" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/pelletier/go-toml/v2" "go.uber.org/zap" "golang.org/x/sync/semaphore" "github.com/zzet/gortex/internal/codegen" "github.com/zzet/gortex/internal/codeowners" "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/contracts" "github.com/zzet/gortex/internal/embedding" "github.com/zzet/gortex/internal/entrypoints" "github.com/zzet/gortex/internal/excludes" "github.com/zzet/gortex/internal/fixtures" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/intern" "github.com/zzet/gortex/internal/licenses" "github.com/zzet/gortex/internal/modules" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/crashpool" "github.com/zzet/gortex/internal/pathkey" "github.com/zzet/gortex/internal/progress" "github.com/zzet/gortex/internal/reach" "github.com/zzet/gortex/internal/resolver" "github.com/zzet/gortex/internal/search" "github.com/zzet/gortex/internal/search/trigram" "github.com/zzet/gortex/internal/semantic" gortexsql "github.com/zzet/gortex/internal/sql" "github.com/zzet/gortex/internal/todos" ) // IndexResult holds the outcome of an indexing operation. type IndexResult struct { NodeCount int `json:"node_count"` EdgeCount int `json:"edge_count"` // FileCount is the total number of language files the indexer // observed for this repo — i.e. how big the repo is on disk, not // how much work this pass did. Stamped onto RepoMetadata so // `daemon status` shows a stable file count across both full-track // and incremental-reconcile paths. FileCount int `json:"file_count"` // StaleFileCount is the number of files that were actually // re-indexed in this pass (only populated by IncrementalReindex // — full-index passes treat every file as stale and would // duplicate FileCount). Used by the janitor / reconcile log to // report "how much work did the snapshot delta require". StaleFileCount int `json:"stale_file_count,omitempty"` // FailedFiles lists files an incremental pass could not index even // after one retry — a parse error, or a file locked or removed // mid-pass. A caller can replay them explicitly. Whether the next // incremental pass retries them on its own depends on the failure: // a file whose bytes could not even be read (locked, permission // denied, removed mid-walk) never gets an mtime recorded, so it // stays stale and is retried on every subsequent pass; a file that // was read but failed to parse (syntax error, crash-isolation // quarantine, extraction timeout) DOES get its current on-disk // mtime recorded, so it is retried only once its content changes // again — this keeps a warm restart from perpetually treating the // whole repo as changed because of one unparseable file. Empty on // a clean pass and on full-index passes. FailedFiles []string `json:"failed_files,omitempty"` // QuarantinedFiles is the number of files held in the parser // crash-isolation quarantine after this pass — files that // SIGSEGV'd / hung / panicked the parser and were skipped with a // Meta["parse_error"] node. Zero unless crash isolation is on. QuarantinedFiles int `json:"quarantined_files,omitempty"` // SkippedFiles is the number of files skipped by the size cap // (MaxFileSize), the per-file extraction timeout (MaxExtractMillis), // or the content-admission policy (index.content — oversized documents // and, by default, binary/vector data assets). Each is recorded in the // graph as a synthetic file node carrying skipped_due_to_size / // skipped_due_to_timeout / skipped_due_to_content telemetry. Zero // unless one of those gates fires. SkippedFiles int `json:"skipped_files,omitempty"` // DeletedFileCount is the number of previously-indexed files that // were evicted this pass because they no longer exist on disk (only // populated by IncrementalReindex). Together with StaleFileCount it // lets a batch caller — the daemon warmup loop in particular — decide // whether a repo actually changed since the last shutdown: when both // are zero across every repo, the persisted graph already carries // every resolved / derived edge and the global resolution passes can // be skipped entirely (the warm-restart fast path). DeletedFileCount int `json:"deleted_file_count,omitempty"` // FullRetrack is true when this result came from a whole-repo // re-track (IndexCtx) rather than an incremental pass — i.e. the // changed-file set is unknown and StaleFileCount does NOT reflect // "how many files changed" (it keeps its normal incremental-work // meaning and is 0 here). Callers that gate global re-resolution on // "did this repo change" must OR in FullRetrack alongside // StaleFileCount / DeletedFileCount. FullRetrack bool `json:"full_retrack,omitempty"` DurationMs int64 `json:"duration_ms"` Errors []IndexError `json:"errors,omitempty"` // RepoPrefix is the prefix the repo was actually registered under. // It usually equals config.ResolvePrefix(entry), but a git worktree // tracked as an independent instance gets a derived `@` // prefix that the caller cannot recompute from the (by-value) entry // it passed in — callers that need to attach a watcher or report the // outcome read it from here. Populated by the multi-repo track and // reconcile paths; empty in single-repo mode. RepoPrefix string `json:"repo_prefix,omitempty"` } // EdgeSanityViolated reports the post-reindex sanity-check failure: an // index pass that observed source files and extracted symbol nodes from // them, yet produced zero edges. A populated graph with no edges still // looks indexed but answers every "who calls X" / "what imports Y" // query with nothing — a wholesale edge-extraction failure (a broken // grammar, an aborted reindex) worth surfacing rather than shipping // silently. Even a single one-function file yields containment edges, // so a real repo only trips this when extraction failed across the // board. func (r *IndexResult) EdgeSanityViolated() bool { return r != nil && r.FileCount > 0 && r.NodeCount > 0 && r.EdgeCount == 0 } // IndexError records a per-file parsing failure. type IndexError struct { FilePath string `json:"file_path"` Error string `json:"error"` } // Indexer walks a repository and populates the graph. type Indexer struct { graph graph.Store // indexCount tracks how many IndexCtx calls this Indexer has // completed. Gates the cold-start shadow-swap: each per-repo // Indexer in MultiIndexer is fresh (indexCount==0), so all of // them take the shadow path regardless of what sibling repos // have already drained into the shared disk store. Per-repo- // prefixed stub IDs make the concurrent drains conflict-free. indexCount atomic.Int32 registry *parser.Registry resolver *resolver.Resolver search search.Backend config config.IndexConfig transforms *transformPipeline excludes *excludes.Matcher excludeOnce sync.Once dirIgnore *excludes.Hierarchical dirIgnoreOnce sync.Once rootPath string // projectName is the repo's own name (go.mod module / package.json / // dir), computed once per index. Stripped from the BM25-indexed file // path so a query word matching it doesn't earn a useless uniform // path-field boost across every document. "" disables the de-weighting. projectName string logger *zap.Logger // Crash-isolation parser pool, lazily created and then reused // across single-file re-indexes so the watcher hot path never // forks a worker subprocess per file. parsePool *crashpool.Pool parseQuar *crashpool.Quarantine parsePoolMu sync.Mutex // Trigram code-search index, lazily built on first GrepText call // and rebuilt only when indexGen advances past the build it was // made from. indexGen is bumped by every full or incremental // index, so a burst of searches between reindexes hits a warm // index. indexGen atomic.Uint64 trigramSearcher *trigram.Searcher trigramGen uint64 trigramMu sync.Mutex // repoPrefix is set in multi-repo mode to prefix all file paths and node IDs. // When empty, the indexer operates in single-repo mode (backward compatible). repoPrefix string // workspaceID is the hard graph boundary slug for this repo. // Stamped onto every node emitted by this indexer via applyRepoPrefix // so query-time scoping doesn't have to look it up by repo prefix. // Defaults at the MultiIndexer layer to the per-repo `.gortex.yaml` // `workspace:` slug, falling back to repoPrefix when no slug is // declared (so legacy configs keep working). workspaceID string // projectID is the soft sub-boundary slug. Defaults to the repo // prefix in single-project repos. Monorepos resolve a per-file // projectID via the `projects[]` paths-glob mapping in // `.gortex.yaml`; until that lookup is wired in, every node from // this indexer carries the repo-default value. projectID string // contractRegistry holds detected API contracts (HTTP routes, gRPC, etc.). contractRegistry *contracts.Registry // trackedRepoModules maps repo names to Go module paths for cross-repo dependency detection. // Populated by MultiIndexer from go.mod files of tracked repos. trackedRepoModules map[string]string // embedder is the optional embedding provider for semantic search. embedder embedding.Provider // skipVectorBuild, when true, makes buildSearchIndex populate only // the text index and never run the embedding pass — even with an // embedder set. The daemon flips it on for the warmup re-index loop // when a snapshot already carries the workspace vector index, so // the graph is not re-embedded only to have the cached index // overwrite it. Off by default; a normal index always builds // vectors when an embedder is present. skipVectorBuild bool // bulkVectorSink holds the disk store captured at the bulk-load // shadow swap, so buildSearchIndex can still persist the vector // index to the backend while idx.graph points at the in-memory // shadow (which does not implement graph.VectorSearcher). Without // it the embedding pass under the bulk loader builds vectors only // in the in-process HNSW — they never reach the `vectors` table and // are lost on the next daemon restart, forcing a paid re-embed. // Set during the shadow swap, cleared when idx.graph is restored. bulkVectorSink graph.VectorSearcher // contentSink mirrors bulkVectorSink for the content full-text index: // the disk store captured at the shadow swap, so the per-file content // stream reaches content_fts on disk even while idx.graph points at the // in-memory shadow (which does not implement graph.ContentSearcher). // Set during the shadow swap, cleared when idx.graph is restored. contentSink graph.ContentSearcher // embedChunkOpts tunes the AST sub-chunking buildSearchIndex applies // to large symbols before embedding. The zero value makes the // chunker fall back to its package defaults. embedChunkOpts embedding.ChunkOptions // embedMaxSymbols overrides the built-in cap on how many texts the // vector index will hold before buildSearchIndex skips the embed // pass. Zero keeps the built-in default. embedMaxSymbols int // embedAPIConcurrency bounds how many embedding requests run in // parallel against an API-backed embedder. Zero keeps the built-in // default. Ignored for in-process embedders, which serialise on an // inference mutex. embedAPIConcurrency int // lastVectorBuildErr records why the most recent buildSearchIndex pass // shipped text-only instead of a vector index (chunk-embed failure, // all-vectors-invalid, or the symbol-count guard). Nil after a build that // produced a vector index. Read via LastVectorBuildError once a build has // finished — it lets `gortex eval embedders` report the real cause instead // of a bare "no vector data". lastVectorBuildErr error // semanticMgr is the optional semantic enrichment manager. semanticMgr *semantic.Manager // resolverLSPHelper, when non-nil, is the resolve-time LSP // helper installed on idx.resolver. Held here so MultiIndexer // can mirror it onto the global post-pass resolver in // RunDeferredPassesAll. See SetResolverLSPHelper. resolverLSPHelper resolver.LSPHelper // npmAliasOnce builds npmAlias lazily on the first resolve-time // import-rewrite request. Lazy because the repo root and prefix // are set after New(); by the time the resolver runs they are // final. npmAliasOnce sync.Once npmAlias *npmAliasIndex // workspaceMembersOnce builds workspaceMembers lazily on the first // resolve-time package-manager-workspace lookup. Lazy for the same // reason as npmAliasOnce — the repo root and prefix are final only // after New(). workspaceMembersOnce sync.Once workspaceMembers *workspaceMembershipIndex // Mtime tracking and parse error retention for index health diagnostics. parseErrors []IndexError fileMtimes map[string]int64 lastIndexTime time.Time totalDetected int mtimeMu sync.RWMutex // contractCache memoizes the contract-extractor output per file. // Keyed by graph file path (with repo prefix); value is the file's // disk mtime when last extracted plus the contracts that came out. // extractContracts replays cache hits to skip the read + 8-extractor // run for files that haven't changed since the last extraction — // the dominant cost on repos with tens of thousands of source files. contractCache map[string]*contractCacheEntry contractCacheMu sync.RWMutex // upgradeOnce gates the BM25→Bleve auto-upgrade to exactly one // goroutine per indexer lifetime. Without this, every post-threshold // IndexCtx — which fires once per tracked repo during multi-repo // warmup — would spawn a fresh upgradeSearchToBleve goroutine. // Each rebuilds ~N-doc Bleve indexes (≈32 KiB/doc), so overlapping // upgrades peak memory far above steady-state and waste CPU on // rebuilds that the next Swap immediately discards. Also counts // scheduled upgrades so tests can observe gating decisions // without relying on log scrapes or timing. upgradeOnce sync.Once upgradeSpawnedMu sync.Mutex upgradeSpawned int // deferResolve, when set, makes IndexCtx skip the cross-cutting passes // (per-repo ResolveAll / semantic enrichment / contract extraction + // commit) so the multi-repo orchestrator can run them serially after // the parallel fan-out joins. Without this, two goroutines indexing // different repos into the shared graph race on Edge.Meta during the // resolver's mutation phase vs. the contract pass's graph walk via // AllEdges(). deferResolve bool pendingContractReg *contracts.Registry // pendingEnrich is raised by an index pass that did real work — IndexCtx // that observed files (or a whole-repo re-track) and IncrementalReindex / // IncrementalReindexPaths that re-indexed or evicted at least one file. It // is cleared only after runDeferredEnrich completes a fully non-partial // semantic enrichment for this repo. The daemon warmup enriches every // indexer it collected, so this gates the (multi-minute LSP hover) pass to // repos that actually changed: an unchanged repo on a warm restart would // otherwise re-confirm nothing for 10+ minutes. A partial / abandoned / // failed enrich leaves it set so a later deferred pass retries. pendingEnrich atomic.Bool // fullReindexed is raised by a whole-repo (re-)parse — IndexCtx, reached // via a full re-track, a cold TrackRepo, or a snapshot-partial forced full // walk — which evicts and re-creates every node and edge for the repo. That // drops the LSP hover-enrichment edges a static re-parse cannot reproduce, // so the deferred-enrichment pass must re-run even when the persisted // completion marker still records the repo's HEAD on a clean tree. It // threads Force into RepoEnrichState so enrichMarkerCurrent stops gating the // pass out. A fresh Indexer per daemon run starts it false, so it only ever // reflects work this run performed. fullReindexed atomic.Bool // reparsedThisRun is the scoped analogue of fullReindexed: it is raised by a // scoped incremental pass that re-parsed at least one stale file this run — // IncrementalReindexPaths / IncrementalReindex with a non-zero // StaleFileCount. A scoped re-parse evicts and re-creates just the changed // files' nodes, dropping THEIR hover-enrichment edges exactly as a whole-repo // re-parse drops every node's, so the deferred pass must likewise run past // the completion marker at an unchanged clean HEAD — otherwise the re-parsed // files' LSP edges stay durably gone until the repo's HEAD moves or its tree // goes dirty. runDeferredEnrich ORs it into RepoEnrichState.Force; because the // hover provider skips already-stamped nodes, the marker bypass re-hovers only // the freshly-unstamped re-parsed files, not the whole repo, so the cost stays // bounded. fullReindexed stays clear on the scoped paths — this flag keeps the // two claims distinct. A fresh Indexer per daemon run starts it false. reparsedThisRun atomic.Bool // deferGlobalPasses, when set, makes IndexCtx and IncrementalReindex // skip the graph-wide derivation passes (InferImplements, // InferOverrides, markTestSymbolsAndEmitEdges). These passes walk the // entire shared graph, so running them per-repo inside a batch loop // (warmup, ReconcileAll) is O(R · global_size) — quadratic for repo // counts in the hundreds. The batch caller is responsible for invoking // RunGlobalGraphPasses exactly once at the end. Has no effect on the // deferResolve path (multi-repo IndexCtx already skips those passes). deferGlobalPasses bool // skipResolveInDeferred, when set, makes RunDeferredPasses skip the // per-repo resolver.ResolveAll() call. ResolveAll walks the entire // shared graph, so paying it once per indexer across hundreds of // repos is O(R · E). MultiIndexer.RunDeferredPassesAll sets this // flag on every indexer and runs a single resolver.New(graph).ResolveAll // once at the end, which picks up every placeholder edge at once. // Has no effect on direct (non-batch) callers of RunDeferredPasses. skipResolveInDeferred bool // codeownersOnce ensures the repo-level CODEOWNERS file is parsed // exactly once per indexer lifetime. The rule list is derived // from .github/CODEOWNERS / CODEOWNERS / docs/CODEOWNERS at // first use and applied per-file by applyCoverageDomains; an // absent file produces empty rules and a no-op pass. codeownersOnce sync.Once codeownersRules []codeowners.Rule // cloneIndex maintains the clone-detection CMS + length-stratified // LSH live across single-file edits, so a steady-state reindex // updates EdgeSimilarTo edges in O(edited file) instead of the // whole-graph detectClonesAndEmitEdges recompute. Constructed empty // (built=false) — a batch/global clone pass calls Rebuild to seed it, // after which indexFile drives EvictFuncs/UpdateFuncs. While un-built // indexFile falls back to the whole-graph pass. cloneIndex *incrementalCloneIndex // affectedByPasses / affectedByFilesResolved / affectedByDropped // count the affected-by re-resolution activity (see affected_by.go): // passes that found a signature delta and ran, referencing files // re-resolved by them, and files dropped by the fan-out cap. // Exposed via AffectedByCounts so tests and diagnostics can observe // that a body-only edit triggered no fan-out. affectedByPasses atomic.Int64 affectedByFilesResolved atomic.Int64 affectedByDropped atomic.Int64 } // contractCacheEntry is a cached contract-extraction result for one file. type contractCacheEntry struct { mtimeNano int64 contracts []contracts.Contract } // New creates an Indexer that writes through the supplied graph.Store. // Any backend (in-memory, SQLite-on-disk, remote) is acceptable — the // indexer's mutation paths go through the Store interface methods only, // so swapping backends is a zero-code-change configuration choice for // callers. func New(g graph.Store, reg *parser.Registry, cfg config.IndexConfig, logger *zap.Logger) *Indexer { idx := &Indexer{ graph: g, registry: reg, resolver: resolver.New(g), // Wrap in Swappable so the auto-upgrade to Bleve at large // corpus sizes can happen in a background goroutine without // racing with concurrent searches. Subsequent reassignments to // idx.search (Hybrid wrap, etc.) should use swap helpers below. // // When the backing store implements graph.SymbolSearcher // (today only store_sqlite), the initial backend is a thin // adapter that forwards Search to the store's native FTS. // The in-process Bleve / BM25 build path is then bypassed // entirely — saving ~100MB heap on a Vscode-scale repo and // putting search in the same address space as the rest of // the graph queries. search: search.NewSwappable(initialSearchBackend(g)), config: cfg, transforms: newTransformPipeline(cfg.Transforms, logger), logger: logger, fileMtimes: make(map[string]int64), contractCache: make(map[string]*contractCacheEntry), cloneIndex: newIncrementalCloneIndex(), } // Resolve JS/TS imports declared through an npm alias against the // local index. The index is built lazily on first use — the repo // root and prefix are not final until after New(). idx.resolver.SetNpmAliasResolver(idx.resolveNpmAliasImport) // Expand JS/TS tsconfig/jsconfig path-alias imports (`@/lib/x`) // against the local index so cross-directory alias imports resolve // to their real file. Same lazy-build rationale. idx.resolver.SetPathAliasResolver(idx.resolvePathAliasImport) // Break same-named import collisions in favour of the importer's // own package-manager workspace member. Same lazy-build rationale. idx.resolver.SetWorkspaceMembership(idx.indexerWorkspaceMembership) return idx } // resolveNpmAliasImport is the resolver.NpmAliasResolver installed on // this Indexer's resolver. It rewrites a JS/TS import specifier that // matches an npm-alias dependency key in the importing file's // nearest-ancestor package.json. Returns "" (no rewrite) when no // alias applies. The backing npmAliasIndex is built once, lazily. func (idx *Indexer) resolveNpmAliasImport(callerFile, specifier string) string { idx.npmAliasOnce.Do(func() { idx.npmAlias = newNpmAliasIndex(map[string]string{idx.repoPrefix: idx.rootPath}) }) return idx.npmAlias.Resolve(callerFile, specifier) } // swappable returns the search backend cast to *search.Swappable. Panics // if the invariant (idx.search is always a Swappable) is ever broken — // that would be a programmer error in this file, not a runtime condition. func (idx *Indexer) swappable() *search.Swappable { if sw, ok := idx.search.(*search.Swappable); ok { return sw } panic("indexer: search backend is not *search.Swappable — invariant violated") } // searchIndexFields returns the text fields fed to the BM25 search // backend for a node. For an ordinary code symbol that is its // name, file path, and signature. For a KindDoc prose-section node // the body is what carries the search signal, so the section text // (Meta["section_text"]) is indexed alongside the breadcrumb name // -- a prose query then ranks the section, not just a heading match. func searchIndexFields(n *graph.Node, projectName string) []string { // The project-name path segment is stripped from the INDEXED path (not the // stored FilePath) so it contributes no uniform path-field boost. indexedPath := search.StripProjectNameFromPath(n.FilePath, projectName) if n.Kind == graph.KindDoc { body, _ := n.Meta["section_text"].(string) return []string{n.Name, indexedPath, body} } sig, _ := n.Meta["signature"].(string) // A symbol's doc comment is the natural-language statement of what it // does — precisely the vocabulary a task-intent query carries ("union // the two sequences", "performs matching on the ignore files") when it // names no identifier verbatim. The identifier and signature alone // answer name lookups; the doc summary is what lets an intent query // reach the definition at all. Only the leading summary is indexed // (docSummary), so a long doc block's examples and edge-case prose // can't dilute the name/signature tokens under BM25 length // normalisation. Empty doc → empty field, dropped by the caller, so a // symbol with no doc indexes exactly as before. doc, _ := n.Meta["doc"].(string) return []string{n.Name, indexedPath, sig, docSummary(doc)} } // docSummaryMaxRunes bounds how much of a doc comment enters the search // document. The leading summary is the highest-signal part; the rest of a // long doc — parameter lists, example code blocks, edge-case notes — is // lower-signal and, unbounded, would dominate the token bag and dilute the // identifier under BM25 length normalisation. The bound keeps the doc's // contribution on the order of a signature's. It is a generic document-size // budget, independent of any repository or corpus. const docSummaryMaxRunes = 280 // docSummary returns the leading summary of a doc comment for the search // index: the first paragraph (up to the first blank line), trimmed and // bounded to docSummaryMaxRunes. Doc comments across languages put the // one-line / one-paragraph summary first and defer detail and examples to // later paragraphs, so the first paragraph is the highest-signal slice. // Empty in → empty out. func docSummary(doc string) string { if doc == "" { return "" } // Normalise CRLF so the blank-line cut is newline-style-agnostic. if strings.Contains(doc, "\r") { doc = strings.ReplaceAll(doc, "\r\n", "\n") doc = strings.ReplaceAll(doc, "\r", "\n") } if i := strings.Index(doc, "\n\n"); i >= 0 { doc = doc[:i] } doc = strings.TrimSpace(doc) if r := []rune(doc); len(r) > docSummaryMaxRunes { doc = strings.TrimSpace(string(r[:docSummaryMaxRunes])) } return doc } // vectorSearcherDelegate is the search.VectorDelegate-shaped // adapter the indexer hands to VectorBackend.SetDelegate when the // underlying store implements graph.VectorSearcher. SimilarTo just // forwards — search.VectorDelegate is defined to return // graph.VectorHit slices directly, so there's no translation work // here, just a small struct so the in-process search package // doesn't depend on graph.VectorSearcher's full surface. type vectorSearcherDelegate struct { s graph.VectorSearcher } func (d *vectorSearcherDelegate) SimilarTo(vec []float32, limit int) ([]graph.VectorHit, error) { if d == nil || d.s == nil { return nil, nil } return d.s.SimilarTo(vec, limit) } // initialSearchBackend picks the search.Backend the indexer wraps // in its Swappable on construction. When the underlying store // implements graph.SymbolSearcher (today only store_sqlite), a // thin adapter routes Search calls through the store's native FTS // — the in-process BM25 / Bleve build path is bypassed entirely. // Otherwise falls through to search.NewAuto which picks BM25 for // small corpora and auto-upgrades to Bleve once the size warrants // it. func initialSearchBackend(g graph.Store) search.Backend { if s, ok := g.(graph.SymbolSearcher); ok { return search.NewSymbolSearcherBackend(s) } return search.NewAuto() } // isSymbolSearcherBackend reports whether the swappable's currently // active backend is the SymbolSearcher adapter. Used to suppress // the Bleve auto-upgrade goroutine — if the active backend is // already a native FTS, upgrading to Bleve would re-index the same // corpus into a parallel in-process Bleve and silently swap it in, // defeating the FTS path and pinning the ~100MB heap the FTS // integration was meant to release. func isSymbolSearcherBackend(b search.Backend) bool { if b == nil { return false } if sw, ok := b.(*search.Swappable); ok { b = sw.Inner() } _, ok := b.(*search.SymbolSearcherBackend) return ok } // lessEdgeKey orders edges by their logical key (from, to, kind, // file_path, line) — the same tuple the sqlite edges table's // UNIQUE(from_id, …) index is built on. Draining edges in this order on // the cold bulk load keeps that index's inserts local instead of random, // reducing B-tree page splits. func lessEdgeKey(a, b *graph.Edge) bool { if a.From != b.From { return a.From < b.From } if a.To != b.To { return a.To < b.To } if a.Kind != b.Kind { return a.Kind < b.Kind } if a.FilePath != b.FilePath { return a.FilePath < b.FilePath } return a.Line < b.Line } // ftsTokensFor produces the pre-tokenised text the backend FTS path // indexes. Mirrors searchIndexFields' field selection but joins // every field through search.Tokenize (camelCase / snake_case / // path-segment splitter) so the resulting token list matches the // in-process BM25 corpus contract — the same query produces the // same recall against either backend. Joined with spaces so the // downstream COPY FROM sees a single STRING column value. func ftsTokensFor(n *graph.Node, projectName string) string { fields := searchIndexFields(n, projectName) if n.QualName != "" { // QualName carries the dotted form (`pkg.Sub.Type.Method`) // that adds qualifier-hop recall ("auth" matching // "auth.ValidateToken"). searchIndexFields omits it for // the legacy BM25 path (which folds qual into the // name-token bag separately), so we add it explicitly here. fields = append(fields, n.QualName) } tokens := make([]string, 0, 16) for _, f := range fields { if f == "" { continue } tokens = append(tokens, search.Tokenize(f)...) } if len(tokens) == 0 { return "" } return strings.Join(tokens, " ") } // shouldIndexForSearch reports whether a node should be added to the // text search index (BM25/Bleve). File and Import nodes are never // searchable symbols. Beyond that, config.SkipSearch filters out // (language, kind) pairs that would only add noise — JSON/YAML/TOML // keys, CSS tokens, Terraform blocks, shell/build variables. All three // text-index call sites (buildSearchIndex bulk loop, indexFile // incremental add, upgradeSearchToBleve repopulate) must go through // this predicate so they can't drift. func (idx *Indexer) shouldIndexForSearch(n *graph.Node) bool { // Cross-daemon proxy-edge nodes stand in for remote symbols; they // are never surfaced in local name search. Inert until // edge-minting is enabled. if graph.IsProxyNode(n) { return false } if n.Kind == graph.KindFile || n.Kind == graph.KindImport { return false } // KindLocal nodes are intra-function bindings emitted to satisfy // rel-table FK constraints on the dataflow edges that target // locals. They have a real Name (the variable identifier) but // surfacing them in BM25 would flood every search for common // names like `err`, `data`, `n`, `i`. Excluded unconditionally. if n.Kind == graph.KindLocal { return false } // KindBuiltin nodes are language intrinsics (append / len / // string / int / ...). Surfacing them in name search would // drown every other hit on common identifiers — agents already // know `string` / `append`. They remain queryable by kind and // by ID for the analytics passes that care. if n.Kind == graph.KindBuiltin { return false } // CONTENT (data_class="content") section nodes live in the dedicated // content index (content_fts), never the symbol search — keeping the // symbol corpus code-only and bounded. Markdown prose (KindDoc without // data_class=content) is unaffected and still honours IndexProse below. if isContentNode(n) { return false } // Prose-section nodes are searchable only when prose indexing is // enabled (search.index_prose); the rest of the graph is // unaffected by the toggle. if n.Kind == graph.KindDoc && !idx.config.IndexProse { return false } if config.ShouldSkipSearch(idx.config.SkipSearch, n.Language, string(n.Kind)) { return false } return true } // upgradeSearchToBleve constructs a Bleve backend from the current graph // and atomically swaps it in. Designed to run in a background goroutine // triggered by IndexCtx after the initial index completes. Does nothing // if Bleve construction fails (caller already hit AutoThreshold but the // in-memory backend keeps serving correctly, just with worse memory // characteristics). // bleveUpgradeEntry is one row of the snapshot the upgrade goroutine // works from. Snapshotting (id, name, file, signature) up front in // the foreground — before the goroutine starts reading them — keeps // the goroutine race-free against subsequent Index calls' Meta-writing // passes (reach.BuildIndex, ResolveTemporalCalls, ...). type bleveUpgradeEntry struct { id string // fields is the BM25 text payload for the node, as produced by // searchIndexFields: name + file + signature for a code symbol, // name + file + section body for a KindDoc prose section. fields []string } // snapshotBleveEntries captures every node currently eligible for the // search index plus its `signature` Meta string. Called synchronously // from IndexCtx after every Node.Meta mutating pass has returned, so // the read of n.Meta happens with no concurrent writer. func (idx *Indexer) snapshotBleveEntries() []bleveUpgradeEntry { nodes := idx.graph.AllNodes() out := make([]bleveUpgradeEntry, 0, len(nodes)) for _, n := range nodes { if !idx.shouldIndexForSearch(n) { continue } out = append(out, bleveUpgradeEntry{id: n.ID, fields: searchIndexFields(n, idx.projectName)}) } return out } func (idx *Indexer) upgradeSearchToBleve(snapshot []bleveUpgradeEntry) { // Defensive early-return: if the active text backend is already // Bleve, there is nothing to upgrade. IndexCtx's sync.Once guard // prevents re-entry from the auto-upgrade path, but direct // callers (tests, manual invocation from tooling) could still // hit this function twice; a second run would pointlessly // rebuild a full Bleve index and Swap it over an identical one. inner := idx.swappable().Inner() if _, ok := inner.(*search.BleveBackend); ok { return } if hyb, ok := inner.(*search.HybridBackend); ok { if _, ok := hyb.TextBackend().(*search.BleveBackend); ok { return } } // Opt-in disk backend. Scorch stores the inverted index on disk // (~10-20× less heap than upsidedown+gtreap) at the cost of file // I/O during build. Users point GORTEX_BLEVE_DISK_DIR at a // writable path; we manage the file lifecycle inside it. diskDir := os.Getenv("GORTEX_BLEVE_DISK_DIR") var ( blv *search.BleveBackend err error ) if diskDir != "" { blv, err = search.NewBleveDisk(diskDir) if err != nil { idx.logger.Warn("search: bleve disk construction failed, falling back to in-memory", zap.String("dir", diskDir), zap.Error(err)) } } if blv == nil { blv, err = search.NewBleve() if err != nil { idx.logger.Warn("search: bleve construction failed, staying on in-memory", zap.Error(err)) return } } // Use the foreground snapshot the spawner captured rather than // re-walking idx.graph here: the goroutine outlives the spawning // IndexCtx call, and subsequent Index calls' Meta-writing passes // (reach.BuildIndex, ResolveTemporalCalls, ...) mutate Node.Meta // on the same Node objects. Reading sig from a live n.Meta here // would race with those writes. for _, e := range snapshot { blv.Add(e.id, e.fields...) } // Preserve the vector index if one is wired up. The previous inner // is normally a HybridBackend wrapping text + vector + embedder; // swapping in raw Bleve would let Swap's old.Close() run on the // old Hybrid, which closes only its text side (hybrid.Close) but // leaves the resulting inner — raw *BleveBackend — unwrapped, so // every downstream hybrid/semantic query silently degrades to // BM25 until the daemon restarts. Rewrap the fresh Bleve in a new // Hybrid carrying the old vector + embedder. The vector backend // itself is never closed by Hybrid.Close, so it stays alive even // after the old Hybrid is torn down by Swap. sw := idx.swappable() var replacement search.Backend = blv if oldHybrid, ok := sw.Inner().(*search.HybridBackend); ok { replacement = search.NewHybrid(blv, oldHybrid.VectorIndex(), oldHybrid.Embedder()) } sw.Swap(replacement) mode := "memory" if blv.DiskPath() != "" { mode = "disk" } idx.logger.Info("search: upgraded to Bleve backend (background)", zap.Int("symbols", blv.Count()), zap.String("mode", mode), zap.String("disk_path", blv.DiskPath())) } // Graph returns the underlying graph. func (idx *Indexer) Graph() graph.Store { return idx.graph } // Search returns the search backend. func (idx *Indexer) Search() search.Backend { return idx.search } // Registry returns the parser registry shared across this indexer. // Exposed for the editor-overlay middleware: the overlay layer-build // pass parses each pushed buffer through the same per-language // extractor the indexer uses, ensuring overlay-derived nodes match // base-derived nodes byte-for-byte for the same input. func (idx *Indexer) Registry() *parser.Registry { return idx.registry } // ContractRegistry returns the contract registry populated during indexing. func (idx *Indexer) ContractRegistry() *contracts.Registry { return idx.contractRegistry } // SetContractRegistry installs reg as the indexer's contract registry. // Used by the daemon warmup path to rehydrate the registry from a // snapshot when IncrementalReindex skipped extraction (no stale files // → extractContracts never ran → idx.contractRegistry stays nil after // reconcile, which used to leave multi-repo `contracts` queries silently // empty). Callers should only install when ContractRegistry() is nil; // installing over a freshly-extracted registry would roll state backward. func (idx *Indexer) SetContractRegistry(reg *contracts.Registry) { idx.contractRegistry = reg } // SetTrackedRepoModules sets the map of tracked repo names to Go module paths. // This enables the GoModExtractor to detect cross-repo dependencies. func (idx *Indexer) SetTrackedRepoModules(m map[string]string) { idx.trackedRepoModules = m } // SetDeferResolve toggles whether IndexCtx defers the cross-cutting passes // to a later RunDeferredPasses call. See the deferResolve field comment. func (idx *Indexer) SetDeferResolve(v bool) { idx.deferResolve = v } // SetSkipResolveInDeferred toggles whether RunDeferredPasses calls // idx.resolver.ResolveAll. The MultiIndexer batch driver sets this so // the per-repo resolver pass — which walks the entire shared graph — // runs exactly once globally instead of R times. See the // skipResolveInDeferred field comment. func (idx *Indexer) SetSkipResolveInDeferred(v bool) { idx.skipResolveInDeferred = v } // SetDeferGlobalPasses toggles whether the graph-wide derivation passes // (InferImplements, InferOverrides, markTestSymbolsAndEmitEdges) run // inline at the end of IndexCtx / IncrementalReindex. Set true when the // caller drives a batch (e.g. daemon warmup) and will invoke // RunGlobalGraphPasses once at the end. See the deferGlobalPasses field // comment. func (idx *Indexer) SetDeferGlobalPasses(v bool) { idx.deferGlobalPasses = v } // RunGlobalGraphPasses runs the graph-wide derivation passes once // against the indexer's shared graph. Safe to call against a graph that // already has these edges — InferImplements / InferOverrides skip // existing parents, and graph.AddEdge dedupes by edgeKey so EdgeTests // re-emission is a no-op. Logs counts for telemetry. Use when batching // multiple per-repo TrackRepoCtx / IncrementalReindex calls under // SetDeferGlobalPasses(true). func (idx *Indexer) RunGlobalGraphPasses(ctx context.Context) { if idx.graph == nil { return } reporter := progress.FromContext(ctx) if added := idx.resolver.InferImplements(); added > 0 { idx.logger.Info("inferred implements (global)", zap.Int("added", added)) } if added := idx.resolver.InferOverrides(); added > 0 { idx.logger.Info("inferred overrides (global)", zap.Int("added", added)) } marked, emitted := markTestSymbolsAndEmitEdges(idx.graph) if marked > 0 || emitted > 0 { idx.logger.Info("test edges emitted (global)", zap.Int("test_symbols", marked), zap.Int("edges", emitted), ) } if re, ep, fa := synthesizeCapabilityEdges(idx.graph); re > 0 || ep > 0 || fa > 0 { idx.logger.Info("capability edges emitted (global)", zap.Int("reads_env", re), zap.Int("executes_process", ep), zap.Int("accesses_field", fa), ) } reporter.Report("clone detection pass (global)", 0, 0) if cs := detectClonesAndEmitEdgesCtx(ctx, idx.graph, idx.repoPrefix, idx.cloneThreshold()); cs.Items > 0 { idx.logger.Info("clone edges emitted (global)", zap.Int("items", cs.Items), zap.Int("clone_pairs", cs.Pairs), zap.Int("edges", cs.Edges), zap.Int("skipped_buckets", cs.SkippedBuckets), zap.Int("skipped_bucket_items", cs.SkippedBucketItems), zap.Int("diffused_pairs", cs.DiffusedPairs), zap.Int("diffused_edges", cs.DiffusedEdges), ) } // Seed the incremental clone index from the freshly-baselined // signatures + sidecar so steady-state single-file edits after this // batch go incremental instead of re-running the whole-graph pass. if idx.cloneIndex != nil { idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix) } // Framework dynamic-dispatch synthesis (gRPC stubs, Temporal // workflow→activity, in-process / native event channels, native // bridges). Runs after InferImplements/InferOverrides (the // interface-satisfaction signals several synthesizers depend on) and // before DetectCrossRepoEdges so a cross-repo synthesized call gets // its parallel cross_repo_calls edge. reporter.Report("framework dispatch synthesis (global)", 0, 0) if rep := resolver.RunFrameworkSynthesizers(idx.graph); rep.Total > 0 { idx.logger.Info("framework dispatch calls synthesized (global)", zap.Int("edges", rep.Total), zap.Any("per_synthesizer", rep.Per), ) } // External-call placeholder synthesis (opt-in). Runs after the // resolver and the gRPC/Temporal stub passes so every edge that // could land on a real node already has; the leftover external // terminals are then materialised into synthetic call-chain nodes. reporter.Report("external-call synthesis (global)", 0, 0) if extCalls := resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled()); extCalls > 0 { idx.logger.Info("external-call placeholders synthesized (global)", zap.Int("edges", extCalls), ) } // Speculative dynamic-dispatch synthesis (opt-in, default off). Mints // best-guess hidden-by-default call edges for blind-spot shapes. if spec := resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled()); spec > 0 { idx.logger.Info("speculative dispatch edges synthesized (global)", zap.Int("edges", spec), ) } // Content -> code "why" links. Runs before DetectCrossRepoEdges so a // chunk that motivates a symbol in another repo gets its parallel // cross_repo_motivates edge minted by the cross-repo pass below. reporter.Report("content links (global)", 0, 0) idx.linkContentToCode() // Cross-repo edge layer. Runs after InferImplements / InferOverrides // so cross-repo implements / extends edges pick up their parallel // cross_repo_* edges. No-op on single-repo graphs (no RepoPrefix). reporter.Report("cross-repo edges (global)", 0, 0) if crossRepoEdges := resolver.DetectCrossRepoEdges(idx.graph); crossRepoEdges > 0 { idx.logger.Info("cross-repo edges emitted (global)", zap.Int("edges", crossRepoEdges), ) } // Reachability index — used to be precomputed here for every // impact seed. The eager pass was retired because the breakeven // math doesn't work: on a 200 k-seed graph (k8s) the build took // ~2000 s of cold-index wall time to save ~10 ms per // AnalyzeImpact call, requiring ~200 k queries to pay off — well // beyond any realistic agent session. Lookups are now // compute-on-first-use; we just invalidate the cache so any // surviving stamps from a previous build don't shadow the fresh // graph state. reach.InvalidateIndex() } // cloneThreshold returns the configured Jaccard similarity cutoff for // clone detection (0 = use the clones package default). func (idx *Indexer) cloneThreshold() float64 { return idx.config.Coverage.ClonesThreshold() } // RunDeferredPasses runs the per-repo cross-cutting passes that IndexCtx // skipped in deferred mode: per-repo ResolveAll, semantic enrichment, and // contract extraction + commit. Safe to call only after IndexCtx has // populated the graph for this repo. Idempotent — second calls are a no-op // because the pending registry is cleared at the end. // // The graph-wide derivation passes (InferImplements, InferOverrides, // markTestSymbolsAndEmitEdges) intentionally do NOT run here. They walk // the entire shared graph, so the multi-repo orchestrator must invoke // MultiIndexer.RunGlobalGraphPasses exactly once after every repo has // finished its deferred per-repo work. func (idx *Indexer) RunDeferredPasses(ctx context.Context) { if idx.pendingContractReg == nil { return } reporter := progress.FromContext(ctx) tphase := time.Now() var dGoMod, dResolve, dEnrich, dContract time.Duration idx.runDeferredGoMod() dGoMod = time.Since(tphase) tphase = time.Now() // Per-repo resolver.ResolveAll walks the entire shared graph; with R // repos and E edges that's O(R · E). The MultiIndexer batch driver // sets skipResolveInDeferred so this runs exactly once globally // (resolver.New(mi.graph).ResolveAll after every per-repo deferred // pass has committed contracts). Direct (non-batch) callers leave // the flag false and pay the standard single-repo cost. if !idx.skipResolveInDeferred { reporter.Report("resolving references", 0, 0) idx.populateCppIncludeDirs(false) idx.resolver.ResolveAll() } dResolve = time.Since(tphase) tphase = time.Now() reporter.Report("semantic enrichment", 0, 0) idx.runDeferredEnrich() dEnrich = time.Since(tphase) tphase = time.Now() reporter.Report("extracting contracts", 0, 0) idx.runDeferredContracts() dContract = time.Since(tphase) idx.logger.Info("DEFERRED-TIMING per-repo", zap.String("repo", idx.repoPrefix), zap.Duration("gomod", dGoMod), zap.Duration("resolve", dResolve), zap.Duration("enrich", dEnrich), zap.Duration("contract_commit", dContract)) } // runDeferredGoMod materialises dep:: contract nodes from go.mod // BEFORE ResolveAll so the resolver's import bridge can re-target Go imports // of declared modules to their dep contract node instead of an external:: // stub. Split out of RunDeferredPasses so the batch driver can run it // serially across repos ahead of the parallel enrichment phase. func (idx *Indexer) runDeferredGoMod() { if idx.pendingContractReg == nil { return } idx.extractGoModContracts(idx.pendingContractReg) } // runDeferredEnrich runs semantic enrichment for this repo. Safe to run // concurrently across repos: the manager fetches a per-repo LSP provider // instance (keyed by the repo's workspace), the go-types stash is keyed by // repo root, tstypes is stateless, and every provider serialises its graph // mutations on the backend resolve mutex. func (idx *Indexer) runDeferredEnrich() { if idx.semanticMgr == nil || !idx.semanticMgr.Enabled() || !idx.semanticMgr.HasProviders() { return } // Gate the pass to repos that actually changed. The daemon warmup collects // every indexer and enriches them all, but an unchanged repo's persisted // graph already carries its enrichment edges — re-running gopls hover for // it confirms nothing over many minutes. GORTEX_WARMUP_FORCE_ENRICH=1 // bypasses the gate for a full re-enrich. forced := os.Getenv("GORTEX_WARMUP_FORCE_ENRICH") == "1" if !idx.pendingEnrich.Load() { if !forced { idx.logger.Info("deferred enrichment skipped", zap.String("repo", idx.repoPrefix), zap.String("reason", "unchanged")) return } idx.logger.Info("deferred enrichment forced despite no pending changes", zap.String("repo", idx.repoPrefix)) } // Key by the repo prefix so a repo-scoped provider can scope file // selection to this repo (empty in single-repo mode). roots := map[string]string{idx.repoPrefix: idx.rootPath} // Compute the repo's git freshness ONCE and thread it in so the manager's // per-provider skip gate and the completion-marker write agree on the // identical (sha, dirty): a provider whose persisted marker still matches // HEAD on a clean tree is skipped instead of re-running its hover pass. sha, dirty := repoHeadAndDirty(idx.rootPath) // A re-parse this run evicted the persisted hover edges of the re-parsed // files, so force the pass past the completion-marker gate — an unchanged // clean HEAD would otherwise skip re-enrichment and leave those files' LSP // edges durably gone. fullReindexed covers a whole-repo re-parse (IndexCtx); // reparsedThisRun covers a scoped incremental re-parse that dropped only the // changed files' edges. Either forces the pass; the provider re-hovers only // the freshly-unstamped nodes, so a scoped force stays bounded to those files. opts := semantic.EnrichOptions{ RepoState: map[string]semantic.RepoEnrichState{ idx.repoPrefix: {SHA: sha, Dirty: dirty, Force: idx.fullReindexed.Load() || idx.reparsedThisRun.Load()}, }, } results, partialRepos, err := idx.semanticMgr.EnrichAll(idx.graph, roots, opts) if err != nil { idx.logger.Warn("semantic enrichment failed", zap.Error(err)) return } for _, r := range results { idx.logger.Info("semantic enrichment result", zap.String("provider", r.Provider), zap.String("language", r.Language), zap.Int("confirmed", r.EdgesConfirmed), zap.Int("added", r.EdgesAdded), zap.Int("refuted", r.EdgesRefuted), zap.Float64("coverage", r.CoveragePercent), ) } // Clear the pending marker only when every provider that ran for this repo // finished non-partial. A partial / abandoned / failed pass leaves it set // so a later deferred pass (or the next restart, once the repo changes // again) retries the enrichment rather than trusting an incomplete graph. if !partialRepos[idx.repoPrefix] { idx.pendingEnrich.Store(false) // Persist a whole-repo completion marker at this HEAD so the next warm // restart can tell, with one lookup, that this repo's enrichment finished // and MaybeSeedPendingEnrich need not resume it. A partial / abandoned // pass takes the other branch and writes no marker, so the absent marker // re-arms the pass on the next start. No-op on a dirty tree / empty sha. idx.semanticMgr.RecordRepoEnrichmentComplete(idx.graph, idx.repoPrefix, sha, dirty) } } // MaybeSeedPendingEnrich re-arms the deferred-enrichment gate for a repo whose // persisted enrichment is known-incomplete at the current clean HEAD, so a warm // restart resumes a semantic pass a prior process left partial or abandoned. // // pendingEnrich otherwise reflects only re-indexing work performed THIS run, so // an unchanged repo whose first enrichment was cut short by the per-repo // deadline would short-circuit runDeferredEnrich on every subsequent restart — // its whole-repo completion marker is absent (a partial pass writes none) yet no // file changed to raise the flag. The daemon warmup calls this after the parse // phase, before draining the deferred passes. // // Returns whether this repo will enrich (already pending, or newly seeded). A // no-op — false — for a repo without semantic providers, on a non-git tree (no // reliable freshness signal), on a backend that does not persist enrichment // state (it re-indexes from scratch each restart anyway), when the completion // marker already records the current HEAD, or on a dirty tree (the marker is // never written or trusted against uncommitted content, and resuming every // restart while the tree stays dirty would defeat the warm-restart fast path). func (idx *Indexer) MaybeSeedPendingEnrich() bool { if idx.semanticMgr == nil || !idx.semanticMgr.Enabled() || !idx.semanticMgr.HasProviders() { return false } if idx.pendingEnrich.Load() { // This run's re-indexing work already armed the gate. return true } // Cheap probe first: only the sha is needed to tell a repo whose marker is // already current (the common warm-restart case) from one that must resume, // so the slower git status shell-out is deferred to the resume path below. sha := repoHead(idx.rootPath) if sha == "" { return false } current, persisted := idx.semanticMgr.RepoEnrichmentMarkerState(idx.graph, idx.repoPrefix, sha) if !persisted || current { return false } if _, dirty := repoHeadAndDirty(idx.rootPath); dirty { return false } idx.logger.Info("deferred enrichment re-armed: persisted enrichment incomplete", zap.String("repo", idx.repoPrefix), zap.String("sha", sha)) idx.pendingEnrich.Store(true) return true } // runDeferredContracts extracts and commits this repo's contract nodes and // clears the pending registration. extractGoModContracts already ran via // runDeferredGoMod. Mutates the shared graph and walks repo edges, so the // batch driver runs it serially after the parallel enrichment phase. The // go-types LookupTypeAtLine it relies on reads the per-repo stash, so it is // correct even though every repo's enrichment ran before any contracts. func (idx *Indexer) runDeferredContracts() { if idx.pendingContractReg == nil { return } idx.extractExternalModules() idx.extractDIContracts(idx.pendingContractReg) idx.commitContracts(idx.pendingContractReg) idx.pendingContractReg = nil } // RootPath returns the root path used for relative path computation. func (idx *Indexer) RootPath() string { return idx.rootPath } // storeRootPath records the repository root, skipping a redundant // self-assignment. The watcher goroutine reads idx.rootPath without a // lock, so re-storing the identical value from a concurrent reindex or // mtime-census pass would be a data race for no observable change. func (idx *Indexer) storeRootPath(absRoot string) { if idx.rootPath != absRoot { idx.rootPath = absRoot } } // populateCppIncludeDirs reconstructs each C/C++ source file's include search // path from compile_commands.json and hands it to the resolver, so a quoted // include binds against the real `-I` dir set (deterministic collision-breaking) // before the suffix-unique fallback. forceReload drops the cache first, so an // incremental reindex picks up an edited compile_commands.json without a // daemon restart. Keys/dirs are prefixed in multi-repo mode to match file IDs. func (idx *Indexer) populateCppIncludeDirs(forceReload bool) { if idx.resolver == nil || idx.rootPath == "" { return } if forceReload { clearCppIncludeDirCache(idx.rootPath) } prefix := "" if idx.repoPrefix != "" { prefix = idx.repoPrefix + "/" } prefixDirs := func(dirs []string) []string { if prefix == "" || len(dirs) == 0 { return dirs } pd := make([]string, len(dirs)) for i, d := range dirs { pd[i] = prefix + d } return pd } tus := loadCompileCommands(idx.rootPath) if len(tus) == 0 { // No compile DB: fall back to the conventional include-root heuristic // so the ordered probe still runs for repos without a compile DB. idx.resolver.SetCppIncludeDirs(nil) idx.resolver.SetCppFallbackIncludeDirs(prefixDirs(heuristicIncludeDirs(idx.rootPath))) return } perFile := make(map[string][]string, len(tus)) for f, tu := range tus { perFile[prefix+f] = prefixDirs(tu.includeDirs) } idx.resolver.SetCppIncludeDirs(perFile) idx.resolver.SetCppFallbackIncludeDirs(nil) } // ResolveFilePath maps a graph file path (repo-relative in single-repo mode) // to an absolute filesystem path. Returns "" when no root is set so callers // can refuse rather than open against the daemon process CWD. Implements // analysis.SourceReader. func (idx *Indexer) ResolveFilePath(graphPath string) string { if graphPath == "" { return "" } if filepath.IsAbs(graphPath) { return filepath.Clean(graphPath) } if idx.rootPath == "" { return "" } // In multi-repo mode the lone Indexer is wrapped by MultiIndexer // (which exposes RepoRoot/ResolveFilePath); single-repo callers // hit this path directly. rel := graphPath if idx.repoPrefix != "" { rel = strings.TrimPrefix(rel, idx.repoPrefix+"/") } return filepath.Clean(filepath.Join(idx.rootPath, rel)) } // relKey reduces an absolute path under the repo root to the canonical // repo-relative key the graph and the mtime map are indexed by: forward // slashes, and Unicode NFC. // // The NFC fold is load-bearing. A file with a non-ASCII name is handed // to the indexer in different byte forms depending on the source — the // filesystem walk (filepath.WalkDir) yields decomposed NFD on macOS, // while the git watcher decodes `git diff` output that git stored as // precomposed NFC. Keying the bulk walk under one form and an // incremental patch under the other would split a single file across // two graph keys: the watcher's evict would miss the walk's node, // IncrementalReindex would see the file as both deleted and freshly // created, and the daemon would carry a stale duplicate. Folding every // key through one form here removes that whole class of mismatch. // // On a path that is not under rootPath (filepath.Rel fails) the input // is returned slash-normalised and NFC-folded so the result is still a // stable key, just not repo-relative. func (idx *Indexer) relKey(absPath string) string { rel, err := filepath.Rel(idx.rootPath, absPath) if err != nil { return pathkey.Normalize(filepath.ToSlash(absPath)) } return pathkey.Normalize(filepath.ToSlash(rel)) } // RelKey exposes relKey to in-package collaborators (the watcher) that // hold an absolute filesystem path and need the canonical repo-relative // graph key for it — e.g. to look up a file's nodes before and after a // re-index. Going through one helper keeps the watcher's key in lockstep // with the keys IndexFile / EvictFile write. func (idx *Indexer) RelKey(absPath string) string { return idx.relKey(absPath) } // SetRepoPrefix sets the repository prefix for multi-repo mode. // When non-empty, all node IDs and file paths are prefixed with "/". func (idx *Indexer) SetRepoPrefix(prefix string) { idx.repoPrefix = prefix } // RepoPrefix returns the current repository prefix. func (idx *Indexer) RepoPrefix() string { return idx.repoPrefix } // SetWorkspaceID sets the workspace slug stamped onto nodes emitted // by this indexer. Empty means "no workspace declared" — the // applyRepoPrefix path will fall back to RepoPrefix so multi-repo // configs without `.gortex.yaml::workspace:` keep working. func (idx *Indexer) SetWorkspaceID(id string) { idx.workspaceID = id } // WorkspaceID returns the workspace slug this indexer stamps on nodes. func (idx *Indexer) WorkspaceID() string { return idx.workspaceID } // SetProjectID sets the project slug stamped onto nodes emitted by // this indexer. Single-project repos pass their repo name (the // MultiIndexer default); monorepos compute a per-file slug from the // `projects[]` mapping (follow-up work). func (idx *Indexer) SetProjectID(id string) { idx.projectID = id } // ProjectID returns the project slug this indexer stamps on nodes. func (idx *Indexer) ProjectID() string { return idx.projectID } // SetEmbedder sets the embedding provider for semantic search. // When set, buildSearchIndex will create a HybridBackend with vector search. func (idx *Indexer) SetEmbedder(p embedding.Provider) { idx.embedder = p } // SetSkipVectorBuild toggles the embedding pass in buildSearchIndex. // When true, buildSearchIndex builds only the text index — used by the // daemon warmup path when a snapshot already carries the workspace // vector index, so the graph is not needlessly re-embedded. When false // (the default) an indexer with an embedder set always builds vectors. func (idx *Indexer) SetSkipVectorBuild(skip bool) { idx.skipVectorBuild = skip } // SetEmbeddingChunkOptions tunes the AST sub-chunking applied to large // symbols before embedding (threshold and window line counts). The // zero value leaves the chunker on its built-in defaults. func (idx *Indexer) SetEmbeddingChunkOptions(opts embedding.ChunkOptions) { idx.embedChunkOpts = opts } // SetEmbeddingMaxSymbols overrides the cap on how many texts the vector // index will hold before buildSearchIndex skips the embed pass. Zero // keeps the built-in default. func (idx *Indexer) SetEmbeddingMaxSymbols(n int) { idx.embedMaxSymbols = n } // SetEmbeddingAPIConcurrency overrides how many embedding requests run // in parallel against an API-backed embedder. Zero keeps the built-in // default. Has no effect on in-process embedders. func (idx *Indexer) SetEmbeddingAPIConcurrency(n int) { idx.embedAPIConcurrency = n } // LastVectorBuildError returns why the most recent index build produced no // vector index (chunk-embed failure, all vectors invalid, or the symbol-count // guard), or nil when a vector index was built or the embedder was unset. Read // it after an index build completes; it is not safe to call concurrently with // one. func (idx *Indexer) LastVectorBuildError() error { return idx.lastVectorBuildErr } // SetSemanticManager sets the semantic enrichment manager. // When set, the indexer runs semantic enrichment after resolution. func (idx *Indexer) SetSemanticManager(m *semantic.Manager) { idx.semanticMgr = m } // SemanticManager returns the semantic enrichment manager. func (idx *Indexer) SemanticManager() *semantic.Manager { return idx.semanticMgr } // SetResolverLSPHelper installs a resolve-time LSP helper on the // underlying Resolver. The helper is consulted from inside // resolveEdge for languages whose extensions the helper claims // (TS/JS/JSX/TSX today via tsserver); see internal/resolver/ // lsp_helper.go for the contract. // // Pass nil to detach. Must be called before ResolveAll / ResolveFile; // the resolver caches no LSP state across passes, so mid-pass swaps // are racy and not supported. func (idx *Indexer) SetResolverLSPHelper(h resolver.LSPHelper) { if idx.resolver != nil { idx.resolver.SetLSPHelper(h) } idx.resolverLSPHelper = h } // ResolverLSPHelper returns the currently installed resolver-time LSP // helper, or nil. Exported so MultiIndexer can mirror the helper onto // the global post-pass resolver in RunDeferredPassesAll. func (idx *Indexer) ResolverLSPHelper() resolver.LSPHelper { return idx.resolverLSPHelper } // ExportVectorIndex returns the serialized vector index bytes, dims, and count. // Returns nil, 0, 0 if no vector index is active. func (idx *Indexer) ExportVectorIndex() ([]byte, int, int) { hybrid, ok := idx.swappable().Inner().(*search.HybridBackend) if !ok { return nil, 0, 0 } vec := hybrid.VectorIndex() if vec == nil || vec.Count() == 0 { return nil, 0, 0 } var buf bytes.Buffer if err := vec.Save(&buf); err != nil { idx.logger.Warn("failed to export vector index", zap.Error(err)) return nil, 0, 0 } return buf.Bytes(), vec.Dims(), vec.Count() } // ImportVectorIndex restores a vector index from serialized data and wraps // the current text search backend into a HybridBackend. func (idx *Indexer) ImportVectorIndex(data []byte, dims, count int) error { if len(data) == 0 || idx.embedder == nil { return nil } // Validate dimensions match the current embedder to avoid mismatches // when switching providers (e.g., GloVe 50d → ONNX 384d). embedderDims := idx.embedder.Dimensions() if embedderDims > 0 && embedderDims != dims { idx.logger.Info("vector index dims mismatch, will re-embed", zap.Int("cached_dims", dims), zap.Int("embedder_dims", embedderDims)) return nil // skip import, buildSearchIndex will re-embed } vec := search.NewVector(dims) if err := vec.LoadFrom(bytes.NewReader(data)); err != nil { return fmt.Errorf("import vector index: %w", err) } vec.SetCount(count) sw := idx.swappable() sw.Swap(search.NewHybrid(sw.Inner(), vec, idx.embedder)) idx.logger.Info("restored vector index from cache", zap.Int("vectors", count), zap.Int("dims", dims)) return nil } // prefixPath prepends the repoPrefix to a relative path when in multi-repo mode. // Returns the path unchanged when repoPrefix is empty. func (idx *Indexer) prefixPath(relPath string) string { if idx.repoPrefix == "" { return relPath } return idx.repoPrefix + "/" + relPath } // graphFilePaths maps reindex file paths (absolute or root-relative, as // passed to IndexFile) to the canonical graph file-path form // (prefixPath(relKey)) that GetFileNodes is keyed by — so file-scoped // passes can look up the just-reindexed files' nodes. func (idx *Indexer) graphFilePaths(files []string) []string { out := make([]string, 0, len(files)) for _, f := range files { abs := f if !filepath.IsAbs(abs) && idx.rootPath != "" { abs = filepath.Join(idx.rootPath, f) } out = append(out, idx.prefixPath(idx.relKey(abs))) } return out } // applyRepoPrefix transforms nodes and edges produced by an extractor to include // the repo prefix in IDs and file paths. Sets Node.RepoPrefix on all nodes. // This is a no-op when repoPrefix is empty (single-repo mode). // // Edge targets beginning with "unresolved::" are a sentinel meaning "the // resolver will replace this with a real node ID after all files are // indexed." Prefixing them turns "unresolved::fetchUsers" into // "web/unresolved::fetchUsers", which the resolver's HasPrefix check on // "unresolved::" misses — leaving every call edge permanently unresolved // in multi-repo mode and breaking get_callers / get_call_chain across all // languages. Skip prefixing on unresolved targets; the resolver will land // the edge on a real ID that already carries its own correct prefix // (possibly cross-repo, which the resolver marks explicitly). // todoTags returns the configured TODO marker set or the default // (TODO/FIXME/HACK/XXX/NOTE). Reads from the IndexConfig the indexer // already holds — IsCoverageEnabled gating happens at the call site. func (idx *Indexer) todoTags() []string { if tags := idx.config.Coverage.Todos.Tags; len(tags) > 0 { return tags } return []string{"TODO", "FIXME", "HACK", "XXX", "NOTE"} } // todoMaxText returns the configured cap on stored TODO text or the // 200-char default. func (idx *Indexer) todoMaxText() int { if n := idx.config.Coverage.Todos.MaxText; n > 0 { return n } return 200 } // loadCodeownersRules lazily parses the repo's CODEOWNERS file. The // sync.Once guarantees one parse per indexer; applyCoverageDomains // is then a pure rule-match per file. Errors silently produce an // empty rule set — the ownership domain is implicitly gated on // file presence rather than failing extraction when the file is // missing or malformed. func (idx *Indexer) loadCodeownersRules() []codeowners.Rule { idx.codeownersOnce.Do(func() { rules, _, ok := codeowners.LoadFromRepo(idx.rootPath) if !ok { return } idx.codeownersRules = rules }) return idx.codeownersRules } // applyCoverageDomains runs the per-file coverage extractors // (todos, licenses, ownership) and applies the post-extraction // strip pass for domains the language extractor always emits but // the user has gated off (function_shape). Appended/stripped // nodes/edges flow through the same applyRepoPrefix / graph.AddNode // pipeline as the language extractor's output. Called from both // the bulk index worker pool (IndexCtx) and the incremental // indexFile path. // // relPath is the unprefixed file path; lang is the detected // language; src is the file bytes. func (idx *Indexer) applyCoverageDomains(relPath, lang string, src []byte, result *parser.ExtractionResult) { if idx.config.Coverage.IsEnabled("todos") { findings := todos.Scan(src, idx.todoTags(), idx.todoMaxText()) todoNodes, todoEdges := todos.BuildGraphArtifacts(relPath, findings, lang) result.Nodes = append(result.Nodes, todoNodes...) result.Edges = append(result.Edges, todoEdges...) } if idx.config.Coverage.IsEnabled("licenses") { if spdx := licenses.Scan(src); spdx != "" { licNodes, licEdges := licenses.BuildGraphArtifacts(relPath, spdx, lang) result.Nodes = append(result.Nodes, licNodes...) result.Edges = append(result.Edges, licEdges...) } } if idx.config.Coverage.IsEnabled("ownership") { if rules := idx.loadCodeownersRules(); len(rules) > 0 { if owners := codeowners.MatchFile(relPath, rules); len(owners) > 0 { teamNodes, teamEdges := codeowners.BuildGraphArtifacts(relPath, owners, lang) result.Nodes = append(result.Nodes, teamNodes...) result.Edges = append(result.Edges, teamEdges...) } } } if idx.config.Coverage.IsEnabled("codegen") { if marker := codegen.Scan(src); marker.Generated { // Stamp the marker on the file node when the language // extractor produced one. Generated files without a // file-shaped result node still get the EdgeGeneratedBy // edge so downstream walks pick them up. for _, n := range result.Nodes { if n.Kind == graph.KindFile && n.FilePath == relPath { if n.Meta == nil { n.Meta = map[string]any{} } codegen.MarkFileNode(n.Meta, marker) break } } result.Edges = append(result.Edges, codegen.BuildGraphArtifacts(relPath, marker)...) } // Annotation-driven codegen: Lombok / MapStruct / Kotlin // compiler plugins generate members that never appear in // source. Flag the annotated symbols so they stay visible. if extra, st := codegen.MarkAnnotatedGenerated(result.Nodes, result.Edges); st.NodesMarked > 0 { result.Edges = append(result.Edges, extra...) // Materialize the actual Lombok accessor members (getX/setX/ // builder/log) so `obj.getName()` resolves to a real graph node. if lnodes, ledges := codegen.MaterializeLombokAccessors(result.Nodes); len(lnodes) > 0 { result.Nodes = append(result.Nodes, lnodes...) result.Edges = append(result.Edges, ledges...) } } } // Framework entry points (Alembic migrations / Next.js pages / // ASP.NET host files): symbols reachable only from a runtime. // Stamped so dead-code analysis treats them as live roots. entrypoints.Detect(relPath, lang, result.Nodes, result.Edges) if !idx.config.Coverage.IsEnabled("function_shape") { stripFunctionShape(result) } if !idx.config.Coverage.IsEnabled("type_shape") { stripTypeShape(result) } if !idx.config.Coverage.IsEnabled("constants") { revertConstantsToVariables(result) } if !idx.config.Coverage.IsEnabled("concurrency") { stripConcurrencyEdges(result) } if idx.config.Coverage.IsEnabled("fixtures") { applyFixtureClassification(relPath, lang, result) } if !idx.config.Coverage.IsEnabled("observability") { stripObservabilityArtifacts(result) } if !idx.config.Coverage.IsEnabled("pubsub") { stripPubsubArtifacts(result) } if !idx.config.Coverage.IsEnabled("flags") { stripFlagArtifacts(result) } if !idx.config.Coverage.IsEnabled("configs") { stripConfigArtifacts(result) } // SQL migration / DDL extraction is high-signal — CREATE TABLE in a // dedicated migration file or a `gortex db schema` dump is // unambiguous, unlike the noisy code-side string-literal SQL the // `sql` domain gates. Run it always; the strip below removes only the // gated code-side SQL, preserving migration-origin schema nodes. applyMigrationExtraction(relPath, src, result) if !idx.config.Coverage.IsEnabled("sql") { stripSQLArtifacts(result) } if idx.config.Coverage.IsEnabled("clones") { applyCloneSignatures(src, result) } } // applyMigrationExtraction parses CREATE TABLE DDL out of SQL migration // files (path under migrate/ or migrations/) and `gortex db schema` dumps // (recognised by their generated-header marker, wherever they're saved). // Each declared table becomes a KindTable node with its columns as // KindColumn nodes (EdgeMemberOf column → table), alongside the synthetic // KindMigration node for the file, with EdgeProvides edges from migration // → table so reverse-walk queries answer "which migrations create this // table". A generated dump carries the real dialect in its header; // hand-written migrations stay "generic" since the .sql file alone doesn't // tell us which dialect the application targets. All emitted nodes carry // Meta["origin"]="migration" so they survive the sql-domain strip — this // schema DDL is unambiguous and high-signal, unlike the code-side // string-literal SQL the `sql` coverage gate exists to suppress. func applyMigrationExtraction(relPath string, src []byte, result *parser.ExtractionResult) { generated := gortexsql.IsGeneratedSchema(src) if !gortexsql.IsMigrationPath(relPath) && !generated { return } tables := gortexsql.ExtractCreateTablesWithColumns(string(src)) if len(tables) == 0 { return } dialect := "generic" if generated { if d := gortexsql.GeneratedSchemaDialect(src); d != "" { dialect = d } } migrationID := gortexsql.MigrationNodeID(relPath) result.Nodes = append(result.Nodes, &graph.Node{ ID: migrationID, Kind: graph.KindMigration, Name: filepath.Base(relPath), FilePath: relPath, Language: "sql", Meta: map[string]any{ "dialect": dialect, "origin": "migration", }, }) for _, t := range tables { tableID := gortexsql.TableNodeID(dialect, t.Schema, t.Table) meta := map[string]any{ "table": t.Table, "dialect": dialect, "origin": "migration", } if t.Schema != "" { meta["schema"] = t.Schema } result.Nodes = append(result.Nodes, &graph.Node{ ID: tableID, Kind: graph.KindTable, Name: t.Table, FilePath: relPath, Language: "sql", Meta: meta, }) result.Edges = append(result.Edges, &graph.Edge{ From: migrationID, To: tableID, Kind: graph.EdgeProvides, FilePath: relPath, Origin: graph.OriginASTResolved, }) for _, c := range t.Columns { colID := gortexsql.ColumnNodeID(dialect, t.Schema, t.Table, c.Name) cmeta := map[string]any{ "table": t.Table, "dialect": dialect, "origin": "migration", } if t.Schema != "" { cmeta["schema"] = t.Schema } if c.Type != "" { cmeta["type"] = c.Type } result.Nodes = append(result.Nodes, &graph.Node{ ID: colID, Kind: graph.KindColumn, Name: c.Name, FilePath: relPath, Language: "sql", Meta: cmeta, }) result.Edges = append(result.Edges, &graph.Edge{ From: colID, To: tableID, Kind: graph.EdgeMemberOf, FilePath: relPath, Origin: graph.OriginASTResolved, }) } } } // stripSQLArtifacts drops KindTable + KindMigration nodes plus // the matching EdgeQueries / EdgeProvides edges when the sql // coverage domain is gated off. Mirrors the strip passes for // flags / configs / observability — endpoint-aware so any // leftover edges to stripped nodes are pruned. SQL extraction // defaults off because string-literal pattern matching against // db.Get / db.Query / db.Exec produces false positives when // domain code shares method names (cache.Get, etc.). func stripSQLArtifacts(result *parser.ExtractionResult) { stripped := make(map[string]struct{}) keptNodes := result.Nodes[:0] for _, n := range result.Nodes { if (n.Kind == graph.KindTable || n.Kind == graph.KindMigration) && !isMigrationOriginNode(n) { stripped[n.ID] = struct{}{} continue } // SQL-context KindString registry nodes are the short-circuit // input for the SQL extractor; gated off alongside the rest // of the SQL domain so a disabled gate leaves no SQL-related // residue in the graph. if n.Kind == graph.KindString { if ctx, _ := n.Meta["context"].(string); ctx == "sql" { stripped[n.ID] = struct{}{} continue } } keptNodes = append(keptNodes, n) } result.Nodes = keptNodes keptEdges := result.Edges[:0] for _, e := range result.Edges { if e.Kind == graph.EdgeQueries { continue } if _, ok := stripped[e.From]; ok { continue } if _, ok := stripped[e.To]; ok { continue } keptEdges = append(keptEdges, e) } result.Edges = keptEdges } // stripConfigArtifacts drops KindConfigKey nodes plus // EdgeReadsConfig / EdgeWritesConfig edges when the configs // coverage domain is gated off. Endpoint-aware so any leftover // edges to stripped key nodes are pruned. // // Infrastructure-origin config keys (Meta["origin"] in {"k8s", // "dockerfile"}) are preserved because they are emitted by the K8s // manifest, Kustomize, and Dockerfile extractors, which have no // dedicated coverage flag and always run. Stripping them would // also strip the EdgeUsesEnv edges those extractors produce (which // target the same node IDs), defeating the cross-ref between // container env declarations and code-side `os.Getenv` reads. func stripConfigArtifacts(result *parser.ExtractionResult) { stripped := make(map[string]struct{}) keptNodes := result.Nodes[:0] for _, n := range result.Nodes { if n.Kind == graph.KindConfigKey && !isInfraOriginConfigKey(n) && !isDocFrontmatterConfigKey(n) { stripped[n.ID] = struct{}{} continue } keptNodes = append(keptNodes, n) } result.Nodes = keptNodes keptEdges := result.Edges[:0] for _, e := range result.Edges { if e.Kind == graph.EdgeReadsConfig || e.Kind == graph.EdgeWritesConfig { continue } if _, ok := stripped[e.To]; ok { continue } keptEdges = append(keptEdges, e) } result.Edges = keptEdges } // isInfraOriginConfigKey reports whether a KindConfigKey node was // emitted by the K8s / Kustomize / Dockerfile extractors. These // nodes carry Meta["origin"] = "k8s" or "dockerfile" by convention. // The code-side extractors (Go os.Getenv, Python os.environ, viper, // struct-tag, …) leave Meta["origin"] empty. // isMigrationOriginNode reports whether a node was emitted by migration / // live-DB DDL extraction (Meta["origin"]=="migration"). These schema nodes // are high-signal and survive the sql-domain strip — the gate only removes // the noisy code-side string-literal SQL. func isMigrationOriginNode(n *graph.Node) bool { if n == nil || n.Meta == nil { return false } o, _ := n.Meta["origin"].(string) return o == "migration" } func isInfraOriginConfigKey(n *graph.Node) bool { if n == nil || n.Kind != graph.KindConfigKey || n.Meta == nil { return false } origin, _ := n.Meta["origin"].(string) return origin == "k8s" || origin == "dockerfile" } // isDocFrontmatterConfigKey reports whether a KindConfigKey node is a // document's frontmatter metadata (Quarto .qmd, …) rather than code / // infra configuration. These keys ride with the document / prose ingest, // which is independent of the `configs` coverage domain, so they survive // the strip the same way infra-origin keys do — keeping a .qmd's declared // title / format / params searchable by default. Add new frontmatter // sources to the switch as prose extractors gain frontmatter support. func isDocFrontmatterConfigKey(n *graph.Node) bool { if n == nil || n.Kind != graph.KindConfigKey || n.Meta == nil { return false } switch src, _ := n.Meta["source"].(string); src { case "quarto_frontmatter": return true } return false } // stripFlagArtifacts drops KindFlag nodes and EdgeTogglesFlag // edges when the flags coverage domain is gated off. Mirrors the // observability strip — endpoint-aware so any leftover edges that // pointed to a removed flag node are also dropped. func stripFlagArtifacts(result *parser.ExtractionResult) { stripped := make(map[string]struct{}) keptNodes := result.Nodes[:0] for _, n := range result.Nodes { if n.Kind == graph.KindFlag { stripped[n.ID] = struct{}{} continue } keptNodes = append(keptNodes, n) } result.Nodes = keptNodes keptEdges := result.Edges[:0] for _, e := range result.Edges { if e.Kind == graph.EdgeTogglesFlag { continue } if _, ok := stripped[e.To]; ok { continue } keptEdges = append(keptEdges, e) } result.Edges = keptEdges } // stripObservabilityArtifacts drops the log/metric/trace KindEvent // nodes and their EdgeEmits edges when the observability coverage // domain is gated off. Used for the same reason as the function-shape // and type-shape strips: the language extractor always emits, and the // indexer prunes per-file before applyRepoPrefix so the gate stays a // pure-config dial without parser plumbing. // // Pub/sub KindEvent nodes (Meta["event_kind"]="pubsub") are a // separately-gated domain — they share the KindEvent kind and the // EdgeEmits edge (publish side) but belong to the `pubsub` coverage // domain, so this pass leaves them and any EdgeEmits/EdgeListensOn // edge targeting them untouched. stripPubsubArtifacts owns those. func stripObservabilityArtifacts(result *parser.ExtractionResult) { stripped := make(map[string]struct{}) pubsubNodes := make(map[string]struct{}) keptNodes := result.Nodes[:0] for _, n := range result.Nodes { if isPubsubEventNode(n) { pubsubNodes[n.ID] = struct{}{} keptNodes = append(keptNodes, n) continue } if n.Kind == graph.KindEvent { stripped[n.ID] = struct{}{} continue } // log_message-context KindString registry nodes are the // string-side shadow of log KindEvent emissions. They gate // alongside the rest of the observability domain so a // disabled gate leaves no log residue in the graph. if n.Kind == graph.KindString { if ctx, _ := n.Meta["context"].(string); ctx == "log_message" { stripped[n.ID] = struct{}{} continue } } keptNodes = append(keptNodes, n) } result.Nodes = keptNodes keptEdges := result.Edges[:0] for _, e := range result.Edges { if _, ok := stripped[e.To]; ok { continue } // The observability gate strips every EdgeEmits — the publish // side of the log/metric/trace layer. The one exception is an // EdgeEmits whose target is a pub/sub topic node: that's the // publish side of the separately-gated pubsub domain, so it // survives here and is owned by stripPubsubArtifacts. if e.Kind == graph.EdgeEmits { if _, ok := pubsubNodes[e.To]; !ok { continue } } keptEdges = append(keptEdges, e) } result.Edges = keptEdges } // stripPubsubArtifacts drops the pub/sub KindEvent topic nodes // (Meta["event_kind"]="pubsub"), every EdgeListensOn edge (a // pubsub-only edge kind), and any EdgeEmits edge whose target is a // pub/sub topic node, when the pubsub coverage domain is gated off. // Endpoint-aware so a publish edge into a stripped topic node doesn't // dangle. Mirrors stripObservabilityArtifacts — the two domains share // KindEvent + EdgeEmits but are toggled independently. func stripPubsubArtifacts(result *parser.ExtractionResult) { stripped := make(map[string]struct{}) keptNodes := result.Nodes[:0] for _, n := range result.Nodes { if isPubsubEventNode(n) { stripped[n.ID] = struct{}{} continue } keptNodes = append(keptNodes, n) } result.Nodes = keptNodes keptEdges := result.Edges[:0] for _, e := range result.Edges { if e.Kind == graph.EdgeListensOn { continue } if _, ok := stripped[e.To]; ok { continue } keptEdges = append(keptEdges, e) } result.Edges = keptEdges } // isPubsubEventNode reports whether a node is a pub/sub topic node — a // KindEvent carrying Meta["event_kind"]="pubsub". Distinguishes the // pub/sub domain from observability (log/metric/trace) events, which // share KindEvent but are gated separately. func isPubsubEventNode(n *graph.Node) bool { if n == nil || n.Kind != graph.KindEvent || n.Meta == nil { return false } kind, _ := n.Meta["event_kind"].(string) return kind == "pubsub" } // applyFixtureClassification reclassifies the language extractor's // emitted file node from KindFile to KindFixture when the file // lives under a testdata/ directory. When the language extractor // produced no file node (file types without a registered // extractor), a standalone KindFixture node is emitted instead. // // Reference edges from test functions to fixtures are out of scope // for v1 — agents can already filter by kind to enumerate fixtures. func applyFixtureClassification(relPath, lang string, result *parser.ExtractionResult) { for _, n := range result.Nodes { if n.Kind == graph.KindFile && n.FilePath == relPath { if fixtures.ReclassifyFileToFixture(n) { return } break } } result.Nodes = append(result.Nodes, fixtures.BuildGraphArtifacts(relPath, lang)...) } // stripConcurrencyEdges removes the EdgeSpawns / EdgeSends / // EdgeRecvs edges introduced by the concurrency coverage domain. // EdgeCalls is left in place — spawns are emitted in addition to // the corresponding call edge, so dropping just the spawn edge // reverts to the pre-coverage call graph without losing reachability. func stripConcurrencyEdges(result *parser.ExtractionResult) { keptEdges := result.Edges[:0] for _, e := range result.Edges { switch e.Kind { case graph.EdgeSpawns, graph.EdgeSends, graph.EdgeRecvs: continue } keptEdges = append(keptEdges, e) } result.Edges = keptEdges } // revertConstantsToVariables re-classifies KindConstant / // KindEnumMember nodes back to KindVariable when the constants // coverage domain is gated off. Unlike stripFunctionShape / // stripTypeShape this is a re-classification, not a removal — // users who disable the domain still want their `const` and `iota` // declarations in the graph, just under the original kind that // pre-coverage code expected. func revertConstantsToVariables(result *parser.ExtractionResult) { for _, n := range result.Nodes { switch n.Kind { case graph.KindConstant, graph.KindEnumMember: n.Kind = graph.KindVariable } } } // stripTypeShape removes the alias / composition edges introduced // by the type_shape coverage domain (EdgeAliases, EdgeComposes). // EdgeExtends is left in place — it's an existing edge kind whose // newtype-derived emissions fall under the spec's "EdgeExtends // continues to mean newtype / inheritance / interface extension" // guarantee, not a new domain signal. func stripTypeShape(result *parser.ExtractionResult) { keptEdges := result.Edges[:0] for _, e := range result.Edges { switch e.Kind { case graph.EdgeAliases, graph.EdgeComposes: continue } keptEdges = append(keptEdges, e) } result.Edges = keptEdges } // stripFunctionShape removes the param/closure/generic_param nodes // and their associated edges from a per-file extraction result. // Used when the function_shape coverage domain is gated off — the // language extractor always emits these for resolution-internal // reasons, and we drop them after the extractor returns rather than // wire a config dependency through the parser layer. func stripFunctionShape(result *parser.ExtractionResult) { stripped := make(map[string]struct{}) keptNodes := result.Nodes[:0] for _, n := range result.Nodes { if isFunctionShapeNode(n.Kind) { stripped[n.ID] = struct{}{} continue } keptNodes = append(keptNodes, n) } result.Nodes = keptNodes keptEdges := result.Edges[:0] for _, e := range result.Edges { if isFunctionShapeEdge(e.Kind) { continue } if _, ok := stripped[e.From]; ok { continue } if _, ok := stripped[e.To]; ok { continue } keptEdges = append(keptEdges, e) } result.Edges = keptEdges } func isFunctionShapeNode(kind graph.NodeKind) bool { switch kind { case graph.KindParam, graph.KindClosure, graph.KindGenericParam: return true } return false } func isFunctionShapeEdge(kind graph.EdgeKind) bool { switch kind { case graph.EdgeParamOf, graph.EdgeReturns, graph.EdgeTypedAs, graph.EdgeCaptures: return true } return false } func (idx *Indexer) applyRepoPrefix(nodes []*graph.Node, edges []*graph.Edge) { // Stamp WorkspaceID / ProjectID on every node emitted by this // indexer regardless of mode — single-repo and multi-repo both // need the boundary slugs for query scoping and contract // matching. Single-repo callers can leave them empty; the // MultiIndexer path always sets them via SetWorkspaceID / // SetProjectID before calling Index. if idx.workspaceID != "" || idx.projectID != "" { for _, n := range nodes { if idx.workspaceID != "" && n.WorkspaceID == "" { n.WorkspaceID = idx.workspaceID } if idx.projectID != "" && n.ProjectID == "" { n.ProjectID = idx.projectID } } } if idx.repoPrefix == "" { return } prefix := idx.repoPrefix + "/" const unresolvedMarker = "unresolved::" // Intern every minted string. A node ID is referenced once on the // node and again on every edge endpoint that points at it; a file // path recurs on every node and edge in that file. Without // interning each reference is a distinct `prefix + s` allocation — // interning collapses them to one shared backing array, and edge // endpoints end up sharing storage with the node ID they name. // // The file path is identical for every node and edge extracted from // the same file — thousands of them. Concatenating `prefix + path` // per reference would mint thousands of throwaway strings before // interning collapses their storage. This per-call cache computes // the interned prefixed path once per distinct raw path, so a file // with N symbols pays one concatenation instead of N. prefixedPath := make(map[string]string) internFilePath := func(raw string) string { if c, ok := prefixedPath[raw]; ok { return c } c := intern.String(prefix + raw) prefixedPath[raw] = c return c } for _, n := range nodes { n.ID = intern.String(prefix + n.ID) n.FilePath = internFilePath(n.FilePath) n.RepoPrefix = idx.repoPrefix // Name, Language and Kind are low-cardinality and recur across // thousands of nodes — method/function names like String, New, // Get…, the ~20 distinct languages, and the fixed set of node // kinds. Interning collapses each to a single backing array; it // also shrinks the byName secondary index, whose keys are these // same strings. n.Name = intern.String(n.Name) n.Language = intern.String(n.Language) n.Kind = graph.NodeKind(intern.String(string(n.Kind))) } for _, e := range edges { e.From = intern.String(prefix + e.From) if strings.HasPrefix(e.To, unresolvedMarker) { // Unresolved targets carry no prefix, but many edges name // the same unresolved symbol — still worth interning. e.To = intern.String(e.To) } else { e.To = intern.String(prefix + e.To) } e.FilePath = internFilePath(e.FilePath) } } // Index walks root and populates the graph using a concurrent worker pool. // // This is the backwards-compatible entry point; it delegates to IndexCtx with // a background context. Callers wanting progress notifications or cancellation // should use IndexCtx directly. func (idx *Indexer) Index(root string) (*IndexResult, error) { return idx.IndexCtx(context.Background(), root) } // IndexCtx is Index with a context, enabling progress reporting. The reporter // is pulled from ctx via progress.FromContext — attach one with // progress.WithReporter to receive stage updates. If no reporter is attached, // stage calls are silently dropped. // clampParseWeight maps a file size to a parse-admission weight bounded to // [1, budget]. A file larger than the whole budget is admitted alone // (weight == budget) so the bytes-in-flight semaphore can never deadlock. func clampParseWeight(size, budget int64) int64 { if size < 1 { return 1 } if size > budget { return budget } return size } func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexResult, retErr error) { start := time.Now() reporter := progress.FromContext(ctx) // Cold/full-index GC tuning: raise the GC percent window and install a // cgroup-aware soft memory limit for the duration of the index, then // restore the prior settings on every exit path. The knobs affect only GC // timing and peak RSS during the allocation burst of a full index — the // graph content is identical with them on or off. Disable for A/B runs // with GORTEX_INDEX_GC_TUNE=0; see gc_tune.go. restoreGCTuning := applyIndexGCTuning(idx.logger) defer restoreGCTuning() absRoot, err := filepath.Abs(root) if err != nil { return nil, err } idx.rootPath = absRoot idx.projectName = search.DetectProjectName(absRoot) reporter.Report("walking files", 0, 0) // Collect files. Files over IndexConfig.MaxFileSize are skipped // during the walk — they're nearly always generated/minified code // that dominates parse time without contributing useful signal. // A single summary warning reports how many were skipped so the // user knows when the cap is biting. // // Each surviving file is captured with its walk-time ModTime so // the worker (contract-cache mtime stamp) and the post-parse // fileMtimes loop don't have to os.Stat again. d.Info() is one // syscall per file regardless; trading one walk-time stat for two // later stats is the net win. maxSize := idx.config.MaxFileSize // Corpus-admission gate: drops oversized document assets and (by // default) binary/vector data artifacts at the walk, before they are // read and extracted, so a content-heavy repo can't pull gigabytes of // non-source files into the parse pipeline and OOM (#120). Inert for // all-code repos. contentGate := idx.newContentAdmissionGate() // Git-aware admission (opt-in): when index.skip_untracked_assets is on, // drop asset-class files git does not track — uncommitted RAG corpora / // datasets / build outputs that .gitignore can't catch (#120). Inert // when off, on a non-git repo, or when git is unavailable. untrackedGate := idx.newUntrackedAssetGate(ctx, absRoot) var files []walkedFile var skippedLarge int var skippedBytes int64 var skippedBySize []skippedFile var skippedByContent []skippedFile var skippedContentBytes int64 var parseFailedFiles []skippedFile err = filepath.WalkDir(absRoot, func(path string, d os.DirEntry, err error) error { if err != nil { return nil } if d.IsDir() { if idx.shouldPruneDir(path, absRoot) { return filepath.SkipDir } return nil } lang, ok := idx.effectiveLanguage(path, nil) if !ok { return nil } if idx.shouldExclude(path, absRoot, false) { return nil } info, statErr := d.Info() if statErr != nil { // Couldn't read FileInfo (race with deletion, broken // symlink, …). Skip — the worker would fail too. return nil } if maxSize > 0 && info.Size() > maxSize { skippedLarge++ skippedBytes += info.Size() rel, _ := filepath.Rel(absRoot, path) skippedBySize = append(skippedBySize, skippedFile{ relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), }) return nil } if reason, skip := untrackedGate.skip(lang, path); skip { skippedContentBytes += info.Size() rel, _ := filepath.Rel(absRoot, path) skippedByContent = append(skippedByContent, skippedFile{ relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), reason: reason, }) return nil } if reason, skip := contentGate.skip(lang, info.Size()); skip { skippedContentBytes += info.Size() rel, _ := filepath.Rel(absRoot, path) skippedByContent = append(skippedByContent, skippedFile{ relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), reason: reason, }) return nil } files = append(files, walkedFile{ path: path, lang: lang, size: info.Size(), mtimeNano: info.ModTime().UnixNano(), }) return nil }) if err != nil { return nil, err } if skippedLarge > 0 { idx.logger.Info("indexer: skipped large files over MaxFileSize", zap.Int("count", skippedLarge), zap.Int64("total_bytes", skippedBytes), zap.Int64("limit_bytes", maxSize)) } if len(skippedByContent) > 0 { idx.logger.Info("indexer: skipped content/data assets by admission policy", zap.Int("count", len(skippedByContent)), zap.Int64("total_bytes", skippedContentBytes)) } reporter.Report("walking files", len(files), len(files)) // In-memory shadow for cold-start indexing on disk-backed stores. // Disk backends pay ms-level per-call cost on every read; running // the resolver against the disk store turns its ~100k+ point // lookups into many minutes of wall time. Instead, swap idx.graph // to an in-memory *Graph for the whole IndexCtx pipeline — parse, // resolve, all subpasses, every per-edge MERGE/MATCH stays in // memory at nanosecond latency. At the end, dump the final state // to the disk backend via one BulkLoad cycle, so the disk has the // post-resolve graph and the bench's query workload runs against // the persisted state. // // Guards: // - Backend must implement graph.BulkLoader (the on-disk backend opts in). // - Store must be empty (NodeCount == 0 && EdgeCount == 0). The // final dump is BulkLoad's INSERT-only fast path — running it // against a non-empty store would corrupt or duplicate. // Incremental / re-index flows fall through to the per-call // AddBatch path against the disk store directly. // - File count is below the shadow-max threshold (see // shadowMaxFileCount). Above the threshold the shadow's RAM // footprint would exceed available memory — Linux / Firefox // at full scale (~10M+ edges) would push the shadow past // 20GB. Override with GORTEX_SHADOW_MAX_FILES. // - The swap happens before the parse worker pool starts and is // committed before IndexCtx returns. retErr from the named // return suppresses the commit when the pipeline errored — // the disk store stays empty rather than capturing partial // state. var diskTarget graph.Store var inMemShadow *graph.Graph bl, blOK := idx.graph.(graph.BulkLoader) // Per-Indexer sentinel: each *Indexer is constructed fresh // (per-repo in MultiIndexer, once in single-repo daemons), so // "this Indexer has indexed before" is the right question to // gate the shadow-swap on. The legacy gate looked at the // disk store's NodeCount, but in MultiIndexer the disk store // holds data from sibling repos that already drained — the // gate would mis-fire and force the big repo onto the per-row // path. With per-repo-prefixed stub IDs (internal/graph/stub.go) // concurrent shadow drains no longer conflict on PRIMARY KEY, // so disk-non-empty is safe. firstIndex := idx.indexCount.Load() == 0 belowShadowMax := len(files) <= shadowMaxFileCount() // The file-count ceiling is blind to the few-huge-files shape: a // content repo of a few hundred PDFs / text dumps / spreadsheets is // far under shadowMaxFileCount yet holds multiple GB that explode // into hundreds of thousands of section nodes. Gate the in-memory // shadow on total input bytes too, so such a repo falls through to // the bounded per-call disk path instead of pinning the whole // post-parse graph in RAM and OOMing (see #120). var totalFileBytes int64 for i := range files { totalFileBytes += files[i].size } maxShadowBytes := shadowMaxBytes() belowShadowBytes := totalFileBytes <= maxShadowBytes shadowTaken := blOK && firstIndex && belowShadowMax && belowShadowBytes preNodes := idx.graph.NodeCount() preEdges := idx.graph.EdgeCount() idx.logger.Info("indexer: shadow-swap decision", zap.String("repo", idx.RepoPrefix()), zap.Bool("bulk_loader", blOK), zap.Bool("first_index", firstIndex), zap.Int("pre_nodes", preNodes), zap.Int("pre_edges", preEdges), zap.Int("files", len(files)), zap.Int("shadow_max_files", shadowMaxFileCount()), zap.Bool("below_shadow_max", belowShadowMax), zap.Int64("total_file_bytes", totalFileBytes), zap.Int64("shadow_max_bytes", maxShadowBytes), zap.Bool("below_shadow_bytes", belowShadowBytes), zap.Bool("shadow_taken", shadowTaken), ) if shadowTaken { // Warm-restart safety. `firstIndex` is a PER-INDEXER sentinel, and // a fresh per-repo Indexer is constructed on every daemon restart, // so firstIndex is true on every restart — even when the // persistent disk store already holds this repo's nodes from a // prior run. The shadow drain below ends in BulkLoad's INSERT-only // COPY, which (per this function's own contract) "running against a // non-empty store would corrupt or duplicate". A duplicate-primary- // key bulk load against the persisted rows would fail warmup, and // because the repo's mtimes never get persisted when warmup dies // first, the failure re-fires on the next restart: a crash loop. // Evicting the repo's existing rows first makes the bulk load land // on a clean slate. EvictRepo self-guards with a count query, so this is a // cheap no-op for the genuine first-index cases (true cold start, // a newly-tracked repo) where the disk store has no rows for this // prefix. preNodes>0 short-circuits the call entirely on the // first repo of a cold start (empty store). if preNodes > 0 { if n, e := idx.graph.EvictRepo(idx.RepoPrefix()); n > 0 || e > 0 { idx.logger.Info("indexer: evicted stale repo rows before bulk reload (warm restart)", zap.String("repo", idx.RepoPrefix()), zap.Int("nodes", n), zap.Int("edges", e)) } } idx.indexCount.Add(1) diskTarget = idx.graph inMemShadow = graph.New() idx.graph = inMemShadow // Capture the disk store as the vector sink: buildSearchIndex runs // while idx.graph is the shadow (no VectorSearcher), so without this // the embedded vectors never land on disk. The `vectors` table has no // FK to `nodes`, so upserting before FlushBulk persists the nodes is // safe. Cleared when idx.graph is restored below. idx.bulkVectorSink, _ = diskTarget.(graph.VectorSearcher) // Same capture for the content index: the per-file content stream // must reach content_fts on disk while idx.graph is the shadow. idx.contentSink, _ = diskTarget.(graph.ContentSearcher) // The resolver was constructed at indexer.New with the disk // Store. Redirect it at the shadow too, otherwise ResolveAll // reads from the empty disk Store, finds no pending edges, // and short-circuits — silently disabling every resolver pass // (module attribution, relative imports, edge in-place // resolution, …) for any backend that takes the shadow path. if idx.resolver != nil { idx.resolver.SetGraph(inMemShadow) } defer func() { if retErr != nil { idx.graph = diskTarget idx.bulkVectorSink = nil idx.contentSink = nil if idx.resolver != nil { idx.resolver.SetGraph(diskTarget) } return } reporter.Report("persisting bulk graph", 0, 0) drainStart := time.Now() shadowNodeCount := inMemShadow.NodeCount() shadowEdgeCount := inMemShadow.EdgeCount() idx.logger.Info("indexer: drain start (shadow → disk)", zap.String("repo", idx.RepoPrefix()), zap.Int("shadow_nodes", shadowNodeCount), zap.Int("shadow_edges", shadowEdgeCount), ) bl.BeginBulkLoad() // Drain the shadow shard-by-shard so the indexer's hold on // the 11-GB Linux-scale graph is released progressively // instead of pinned until persist returns. The drain // iterators free each shard's node/edge maps as they // advance, so peak RAM during the persist window is // roughly the chunk buffer + the backend's working set, // not full shadow + the disk backend's bulk-COPY buffer. // // Collect (id, tokens) for every search-eligible node as // the drain yields them — feeds the backend's native FTS // at FlushBulk time when the store implements // graph.SymbolSearcher. Nodes that fail // shouldIndexForSearch (KindFile / KindImport / // KindLocal / KindBuiltin / skip-search lang+kind pairs) // are excluded so the FTS corpus matches the in-process // BM25 corpus exactly. searcher, hasFTS := diskTarget.(graph.SymbolSearcher) var ftsItems []graph.SymbolFTSItem if hasFTS { // Pre-size to the shadow's node count to avoid grow // churn on a 600k-node Vscode-shape repo. ftsItems = make([]graph.SymbolFTSItem, 0, inMemShadow.NodeCount()) } // Key-ordered bulk drain. The nodes table is WITHOUT ROWID // (its primary key IS the B-tree), so inserting in ascending // id order makes each insert an append to the rightmost leaf // instead of a random split mid-tree — far fewer page splits // on the cold load. The edges table's UNIQUE(from_id, …) index // benefits the same way from from-id-ordered inserts. So the // whole drain is collected, sorted once, then chunk-written. // // This block only runs on the first/empty cold index (the // shadow-swap branch above), and the shadow node/edge sets // already fit in RAM (the branch is gated on the shadow byte / // file budget). DrainNodes / DrainEdges free each shard as they // yield, so holding the drained set in one slice to sort costs // no more peak RAM than the shadow already did. const persistChunk = 100000 allNodes := make([]*graph.Node, 0, shadowNodeCount) for n := range inMemShadow.DrainNodes() { if hasFTS && idx.shouldIndexForSearch(n) { ftsItems = append(ftsItems, graph.SymbolFTSItem{ NodeID: n.ID, Tokens: ftsTokensFor(n, idx.projectName), }) } allNodes = append(allNodes, n) } sort.Slice(allNodes, func(i, j int) bool { return allNodes[i].ID < allNodes[j].ID }) for start := 0; start < len(allNodes); start += persistChunk { end := min(start+persistChunk, len(allNodes)) diskTarget.AddBatch(allNodes[start:end], nil) } allNodes = nil allEdges := make([]*graph.Edge, 0, shadowEdgeCount) for e := range inMemShadow.DrainEdges() { allEdges = append(allEdges, e) } sort.Slice(allEdges, func(i, j int) bool { return lessEdgeKey(allEdges[i], allEdges[j]) }) for start := 0; start < len(allEdges); start += persistChunk { end := min(start+persistChunk, len(allEdges)) diskTarget.AddBatch(nil, allEdges[start:end]) } allEdges = nil flushStart := time.Now() idx.logger.Info("indexer: FlushBulk start", zap.String("repo", idx.RepoPrefix()), zap.Duration("drain_elapsed", flushStart.Sub(drainStart)), ) if ferr := bl.FlushBulk(); ferr != nil { retErr = fmt.Errorf("indexer: persist bulk graph: %w", ferr) } idx.logger.Info("indexer: FlushBulk complete", zap.String("repo", idx.RepoPrefix()), zap.Duration("flush_elapsed", time.Since(flushStart)), zap.Duration("total_drain", time.Since(drainStart)), zap.Int("nodes", shadowNodeCount), zap.Int("edges", shadowEdgeCount), ) // Build the backend FTS after the bulk load completes so // CREATE_FTS_INDEX has the full corpus to scan in one // pass. BulkUpsertSymbolFTS does its own // extension-install dance, so this is the only place the // indexer needs to know about SymbolSearcher. if hasFTS && len(ftsItems) > 0 { reporter.Report("building symbol fts", 0, 0) if ferr := searcher.BulkUpsertSymbolFTS(idx.RepoPrefix(), ftsItems); ferr != nil { idx.logger.Warn("indexer: bulk symbol FTS upsert failed", zap.Error(ferr)) } else if ferr := searcher.BuildSymbolIndex(); ferr != nil { idx.logger.Warn("indexer: backend FTS build failed", zap.Error(ferr)) } reporter.Report("building symbol fts", 1, 1) } reporter.Report("persisting bulk graph", 1, 1) idx.graph = diskTarget idx.bulkVectorSink = nil idx.contentSink = nil // Mirror of the SetGraph(inMemShadow) above: the resolver // must follow the graph pointer back to the disk store, or // every post-index per-file resolve (the watcher save path, // incremental reindex) reads the drained — now empty — // shadow and silently resolves nothing. if idx.resolver != nil { idx.resolver.SetGraph(diskTarget) } }() } else if diskTarget == nil && idx.graph.NodeCount() == 0 && idx.graph.EdgeCount() == 0 { if _, isBulk := idx.graph.(graph.BulkLoader); isBulk && firstIndex && (!belowShadowMax || !belowShadowBytes) { idx.logger.Info("indexer: skipping in-memory shadow; building against disk store (bounded RAM)", zap.Int("files", len(files)), zap.Int("file_threshold", shadowMaxFileCount()), zap.Bool("over_file_count", !belowShadowMax), zap.Int64("total_file_bytes", totalFileBytes), zap.Int64("byte_threshold", maxShadowBytes), zap.Bool("over_byte_budget", !belowShadowBytes)) } } // Content-index rebuild strategy. The crash-safe path (on-disk store) // deletes each file's prior content rows as that file re-streams // (contentWipeFile, invoked at the per-file AddBatch sites below) and // sweeps every OTHER content row after the authoritative mtime replace — // so a mid-parse kill leaves a mix of old+new content per file instead of // the empty table a repo-wide pre-wipe would leave. contentStreamedFiles // records which files actually streamed content this walk (the wipe's own // argument, i.e. the node FilePath content_fts carries); the end sweep // keeps exactly that set, so files that vanished from disk AND files that // still exist but no longer yield content sections (doc emptied, // classification changed) are both reaped in one scan — the transitions // the old repo-wide pre-wipe used to cover. Backends without the per-file // capability keep that old behaviour: one repo-wide wipe up front (a // cold-store no-op; a cheap per-repo DELETE on a warm one). var ( contentWipeFile func(filePath string) contentStreamedMu sync.Mutex contentStreamedFiles map[string]struct{} ) if cs := idx.contentSearcher(); cs != nil { if w, ok := cs.(interface { WipeContentFileInRepo(repoPrefix, filePath string) error }); ok { repoPrefix := idx.RepoPrefix() contentStreamedFiles = make(map[string]struct{}) contentWipeFile = func(filePath string) { if filePath == "" { return } contentStreamedMu.Lock() contentStreamedFiles[filePath] = struct{}{} contentStreamedMu.Unlock() if err := w.WipeContentFileInRepo(repoPrefix, filePath); err != nil { idx.logger.Warn("indexer: per-file content wipe failed", zap.Error(err)) } } } else if err := cs.WipeContent(idx.RepoPrefix()); err != nil { idx.logger.Warn("indexer: content index wipe failed", zap.Error(err)) } } // Worker pool. workers := idx.config.Workers if workers <= 0 { workers = 1 } // idx.config.Workers defaults to the host's runtime.NumCPU(); inside a // CPU-limited container that exceeds the cgroup CPU quota and the pool // over-subscribes, so CFS throttling drags throughput down. Clamp the // effective pool size to the quota when one is present (lowers only, floor // of 1, host-identical when unquotaed). GORTEX_INDEX_CPU_CLAMP=0 opts out. if cpuClampEnabled() { workers = clampWorkersToCPUQuota(workers, cgroupCPUQuota()) } // Optional crash isolation: run tree-sitter extraction in worker // subprocesses so a grammar SIGSEGV / OOM / hang on one // pathological file is contained — the bad file is quarantined and // the pass still completes. Off unless index.crash_isolation / // GORTEX_PARSER_ISOLATION is set. var parsePool *crashpool.Pool var quarantine *crashpool.Quarantine if idx.crashIsolationEnabled() { quarantine = crashpool.LoadQuarantine(filepath.Join(absRoot, ".gortex", "parser-quarantine.json")) if p, perr := idx.newParsePool(workers); perr != nil { idx.logger.Warn("indexer: crash isolation requested but parser pool unavailable; parsing in-process", zap.Error(perr)) } else { parsePool = p defer parsePool.Close() idx.logger.Info("indexer: parser crash isolation enabled", zap.Int("workers", workers)) } } // Workers parse files, write the resulting nodes/edges to the // sharded graph, and run per-file contract extractors on the same // src bytes — all in one pass. Reusing src avoids the 10k+ disk // re-reads the old "parse then extractContracts" flow did; running // the contract extractors per-worker parallelises what used to be // a serial post-pass; language-filtered dispatch skips extractors // that can't match (HTTP on .css, OpenAPI on .ts, etc.). const parseReportEvery = 50 totalFiles := len(files) _, contractExtractorsByLang := idx.buildPerFileContractExtractors() contractReg := contracts.NewRegistry() var contractMu sync.Mutex var errMu sync.Mutex var errors []IndexError var processed int64 var fileCount int64 var skippedByTimeout int64 var skippedByMinified int64 // Bound peak parse memory: a weighted, bytes-in-flight semaphore // admits each worker by its file size before it reads + extracts, so // a cluster of large files (PDFs / office docs in a content repo) // serialises instead of all workers materialising whole files and // their parse trees at once. Code files are tiny and flow freely; // only genuinely large inputs queue. budget <= 0 disables the cap. parseBudget := idx.config.MaxParseBytesInFlight var parseSem *semaphore.Weighted if parseBudget > 0 { parseSem = semaphore.NewWeighted(parseBudget) } // In addition to the bytes-in-flight budget above, cap how many // genuinely large files are *read* concurrently: a few huge PDFs / // spreadsheets / vector artifacts can dominate RSS before extraction // even starts. Small source files bypass the gate and keep full // throughput. largeReadGate := make(chan struct{}, largeFileReadParallelism(workers)) readFile := func(wf walkedFile) ([]byte, error) { if wf.size < largeFileReadThresholdBytes { return os.ReadFile(wf.path) } largeReadGate <- struct{}{} defer func() { <-largeReadGate }() return os.ReadFile(wf.path) } // recordStreamedMtime persists a file's mtime incrementally, in batches, // as its nodes land on disk during a full track — so a first-ever index // killed mid-parse RESUMES on the next boot (the flushed files reconcile // clean, the remainder re-parses) instead of re-tracking a large repo // from scratch every time it dies under memory pressure. // // INVARIANT: an mtime row must land only AFTER its file's nodes are // durably in the store, or a fresh mtime would falsely imply indexed // nodes. The `idx.graph.(graph.FileMtimeWriter)` assertion is what // enforces it: it succeeds ONLY on the direct-to-disk path, where // idx.graph is the on-disk store and AddBatch has already made the // file's nodes durable. On the whole-repo in-memory shadow and the // streaming-flush per-chunk shadow, idx.graph is a plain graph.Graph // (no FileMtimeWriter) whose nodes reach disk only at a later drain, so // this self-skips and those paths persist at their own drain points (the // final ReplaceFileMtimes / the streaming-flush per-chunk persist). The // final ReplaceFileMtimes still runs and is authoritative — it also // prunes deleted files — so these batches only bound the work a crash // can waste; the leftover under-threshold tail rides that final replace. var ( streamMtimeMu sync.Mutex streamMtimes = make(map[string]int64, mtimeStreamPersistEvery) ) recordStreamedMtime := func(absPath string, mtimeNano int64) { if mtimeNano <= 0 { return } w, ok := idx.graph.(graph.FileMtimeWriter) if !ok { return } var flush map[string]int64 streamMtimeMu.Lock() streamMtimes[idx.relKey(absPath)] = mtimeNano if len(streamMtimes) >= mtimeStreamPersistEvery { flush = streamMtimes streamMtimes = make(map[string]int64, mtimeStreamPersistEvery) } streamMtimeMu.Unlock() if flush != nil { if err := w.BulkSetFileMtimes(idx.repoPrefix, flush); err != nil { idx.logger.Warn("indexer: incremental mtime batch persist failed", zap.String("repo", idx.repoPrefix), zap.Error(err)) } } } // parseChunk runs the per-file worker pool over the supplied // slice. Closure over outer state (errors, counters, contract // registry, parsePool, quarantine) so it can be called multiple // times — once for the non-streaming path, repeatedly for the // streaming-flush large-repo path where each call processes a // bounded slice into a per-chunk in-memory shadow. parseChunk := func(chunkFiles []walkedFile) { fileCh := make(chan walkedFile, workers*4) var wg sync.WaitGroup for range workers { wg.Add(1) go func() { defer wg.Done() var localContracts []contracts.Contract for wf := range fileCh { path := wf.path p := atomic.AddInt64(&processed, 1) if p == 1 || p%parseReportEvery == 0 { reporter.Report("parsing", int(p), totalFiles) } // Admit this file into extraction under the // bytes-in-flight budget before reading it, so large // content files serialise instead of all workers // materialising whole files at once. var weight int64 if parseSem != nil { weight = clampParseWeight(wf.size, parseBudget) if aerr := parseSem.Acquire(ctx, weight); aerr != nil { return } } relPath, _ := filepath.Rel(absRoot, path) // Streaming content extractors (PDF / office docs) read the // file themselves — one page/slide/sheet at a time — instead // of materialising the whole file. Only the in-process route // streams; the crash-isolation subprocess route keeps bytes. if walkExt, found := idx.registry.GetByLanguage(wf.lang); found && parsePool == nil { if se, ok := walkExt.(parser.StreamingExtractor); ok { result, serr := idx.extractStreaming(se, path, relPath) if parseSem != nil { parseSem.Release(weight) } if serr != nil { errMu.Lock() errors = append(errors, IndexError{FilePath: path, Error: serr.Error()}) if result == nil { parseFailedFiles = append(parseFailedFiles, skippedFile{relPath: relPath, lang: wf.lang, cause: serr.Error()}) } errMu.Unlock() } if result == nil { continue } idx.applyRepoPrefix(result.Nodes, result.Edges) if contentWipeFile != nil { contentWipeFile(firstContentFilePath(result.Nodes)) } idx.streamContentSections(result.Nodes) idx.graph.AddBatch(result.Nodes, result.Edges) idx.persistConstValues(result) recordStreamedMtime(path, wf.mtimeNano) continue } } src, err := readFile(wf) if err != nil { errMu.Lock() errors = append(errors, IndexError{FilePath: path, Error: err.Error()}) errMu.Unlock() if parseSem != nil { parseSem.Release(weight) } continue } // Reuse the walk-time language. The walk's // effectiveLanguage call already consulted shebang // bytes via readSniffPrefix (512-byte probe), so a // re-detect against the full src would change the // answer only on the vanishingly rare case where a // language marker lives past byte 512 — and any such // case is content-sniffing-by-luck rather than spec'd // behaviour. The fallback below covers the truly // pathological case where the walk-time language has // no extractor registered (effectively dead code). lang := wf.lang ext, _ := idx.registry.GetByLanguage(lang) if ext == nil { if relang, ok := idx.effectiveLanguage(path, src); ok { lang = relang ext, _ = idx.registry.GetByLanguage(lang) } } if ext == nil { if parseSem != nil { parseSem.Release(weight) } continue } // Pre-ingestion transforms: rewrite the bytes before // extraction (BOM strip, minified-bundle expansion, a // PDF→markdown command, …). src = idx.transforms.run(relPath, src) result, skipped, err := idx.extractFile(parsePool, quarantine, path, relPath, lang, ext, src) if parseSem != nil { parseSem.Release(weight) } if err != nil { errMu.Lock() errors = append(errors, IndexError{FilePath: path, Error: err.Error()}) errMu.Unlock() } if result == nil { // A full-index parse failure that produced no nodes: // record it for a skip-node post-pass so the file // stays visible instead of vanishing. (The live-modify // path never reaches here — it keeps a file's prior // nodes through a transient parse failure.) if err != nil { errMu.Lock() parseFailedFiles = append(parseFailedFiles, skippedFile{relPath: relPath, lang: lang, cause: err.Error()}) errMu.Unlock() } continue } if skipped && len(result.Nodes) > 0 { if _, ok := result.Nodes[0].Meta["skipped_due_to_timeout"]; ok { atomic.AddInt64(&skippedByTimeout, 1) } if _, ok := result.Nodes[0].Meta["skipped_due_to_minified"]; ok { atomic.AddInt64(&skippedByMinified, 1) } } // Append coverage artifacts (todos / licenses / // ownership) before applyRepoPrefix so they get the // same multi-repo namespacing treatment as // language-extractor output. Skipped for quarantined / // timed-out files — the coverage scanners would re-read // a source the parser could not survive. if !skipped { idx.applyCoverageDomains(relPath, lang, src, result) } idx.applyRepoPrefix(result.Nodes, result.Edges) // Find the file node (if the extractor produced one) // and collect its outgoing edges — contract extractors // take the file-scope edge set (imports, etc.), not // every intra-file edge. var fileNodeID, fileGraphPath string for _, n := range result.Nodes { if n.Kind == graph.KindFile { fileNodeID = n.ID fileGraphPath = n.FilePath break } } var fileScopeEdges []*graph.Edge if fileNodeID != "" { for _, e := range result.Edges { if e.From == fileNodeID { fileScopeEdges = append(fileScopeEdges, e) } } } // Stream this file's content (data_class=content) section // bodies into the dedicated content index and lean the // nodes to a snippet BEFORE AddBatch, so the bulk text // never enters the graph, the symbol search, or the // materialising code passes. if contentWipeFile != nil { contentWipeFile(firstContentFilePath(result.Nodes)) } idx.streamContentSections(result.Nodes) // Batch the per-file insert into one shard-grouped pass // so each shard's lock is acquired at most once per // file instead of N + 2·E times. Profiling showed 69 // of 102 workers blocked on lockTwoWrite under the // per-edge path during cold-start warmup. idx.graph.AddBatch(result.Nodes, result.Edges) idx.persistConstValues(result) idx.persistFileMeta(relPath, src, result) recordStreamedMtime(path, wf.mtimeNano) if !skipped && fileGraphPath != "" { exts := contractExtractorsByLang[lang] if len(exts) > 0 { c := idx.runContractExtractorsForFile( fileGraphPath, src, result.Nodes, fileScopeEdges, exts, result.Tree) localContracts = append(localContracts, c...) // Populate the per-file contract cache so a // later IncrementalReindex can skip this file // on a cache hit. Mtime comes from the walk- // time d.Info() — no extra stat here. if wf.mtimeNano > 0 { idx.contractCacheMu.Lock() idx.contractCache[fileGraphPath] = &contractCacheEntry{ mtimeNano: wf.mtimeNano, contracts: c, } idx.contractCacheMu.Unlock() } } } // Release the parse tree now that the per-file // contract pass is done. Post-passes that need a // tree for this file (cross-file handler resolution) // re-parse on demand. Nil-safe. result.Tree.Release() atomic.AddInt64(&fileCount, 1) } if len(localContracts) > 0 { contractMu.Lock() for _, c := range localContracts { contractReg.Add(c) } contractMu.Unlock() } }() } for _, f := range chunkFiles { fileCh <- f } close(fileCh) wg.Wait() } // Dispatch the largest files first. Both dispatch paths below // consume this slice in order: the plain path feeds it straight to // the worker pool, and the streaming-flush path carves it into // chunks from the front. Feeding the biggest file to the workers // first lets its long parse overlap with the tail of small files, // instead of the whole index waiting on a large file that would // otherwise be dispatched last. The sort is stable and a pure // permutation, so the set of indexed files and the resulting graph // are identical — only the dispatch order changes. sortBySizeDesc(files) // Streaming-flush path: above shadowMaxFileCount with a // BulkLoader-capable backend, we can't fit the whole shadow in // RAM but we can still amortise the per-file disk-write cost by // chunking. Each chunk runs against its own throwaway shadow, // then flushes via BulkLoad to disk. Resolve runs against the // disk store afterwards (per-call, slower than the shadow path // but bounded RAM). Activated by GORTEX_STREAMING_FLUSH=1; off // by default since it requires the disk-only resolver path // (~tens of minutes on huge repos) that we haven't yet // optimised end-to-end. if diskTarget == nil && streamingFlushActive(idx.graph, len(files)) { bl, _ := idx.graph.(graph.BulkLoader) streamingDisk := idx.graph chunkSize := streamingChunkSize() idx.logger.Info("indexer: streaming-flush parse", zap.Int("files", len(files)), zap.Int("chunk_size", chunkSize)) for chunkStart := 0; chunkStart < len(files); chunkStart += chunkSize { chunkEnd := min(chunkStart+chunkSize, len(files)) chunkShadow := graph.New() idx.graph = chunkShadow parseChunk(files[chunkStart:chunkEnd]) // Flush chunk to disk. bl.BeginBulkLoad() streamingDisk.AddBatch(chunkShadow.AllNodes(), chunkShadow.AllEdges()) if err := bl.FlushBulk(); err != nil { return nil, fmt.Errorf("indexer: streaming-flush chunk %d..%d: %w", chunkStart, chunkEnd, err) } // This chunk's nodes are durable on disk now, so persist its // files' mtimes — a kill mid-track then resumes from this chunk // boundary instead of re-tracking the whole large first index // from scratch. The in-worker recordStreamedMtime is a no-op on // this path (idx.graph was the throwaway per-chunk shadow, not a // FileMtimeWriter); the persist happens here, after the drain, // keeping the "fresh mtime implies durable nodes" invariant. if w, ok := streamingDisk.(graph.FileMtimeWriter); ok { batch := make(map[string]int64, chunkEnd-chunkStart) for _, f := range files[chunkStart:chunkEnd] { if f.mtimeNano > 0 { batch[idx.relKey(f.path)] = f.mtimeNano } } if len(batch) > 0 { if err := w.BulkSetFileMtimes(idx.repoPrefix, batch); err != nil { idx.logger.Warn("indexer: streaming-flush chunk mtime persist failed", zap.String("repo", idx.repoPrefix), zap.Error(err)) } } } } // After all chunks, idx.graph points at the disk store so // the resolver and subpasses read/mutate the merged state. idx.graph = streamingDisk } else { parseChunk(files) } // Finalise the content index after the per-file streaming appends so // its FTS5 segments are merged before the first content query. if cs := idx.contentSearcher(); cs != nil { if err := cs.BuildContentIndex(); err != nil { idx.logger.Warn("indexer: content index build failed", zap.Error(err)) } } if processed > 0 { reporter.Report("parsing", int(processed), totalFiles) } // Emit synthetic file nodes for files dropped by the size cap so // they stay visible in the graph with skip telemetry attached // instead of vanishing silently. idx.emitSizeSkipNodes(skippedBySize) idx.emitContentSkipNodes(skippedByContent) idx.emitParseFailedSkipNodes(parseFailedFiles) // Populate fileMtimes for all detected files. Keyed through // relKey so the mtime map agrees with the graph's file-node keys // (and with the incremental / git-watcher paths) on the NFC form // of every non-ASCII filename. Mtimes are the walk-time values // captured via d.Info(); no per-file os.Stat round-trip here. idx.mtimeMu.Lock() idx.fileMtimes = make(map[string]int64, len(files)) for _, f := range files { if f.mtimeNano > 0 { idx.fileMtimes[idx.relKey(f.path)] = f.mtimeNano } } mtimeSnapshot := make(map[string]int64, len(idx.fileMtimes)) for k, v := range idx.fileMtimes { mtimeSnapshot[k] = v } idx.mtimeMu.Unlock() // Persist the per-file mtimes through the store's optional // FileMtime sidecar table. On the on-disk backend this lets warm // restarts seed ReconcileRepoCtx without having to read them back // out of the gob+gzip metadata snapshot; on the in-memory // backend the capability isn't implemented and the assertion // short-circuits. // // Multi-repo bug: when the shadow-swap path is active, idx.graph // is the in-memory shadow graph at this point — graph.Graph does // NOT implement FileMtimeWriter, so the type assertion fails and // persistence is silently skipped. The actual disk store is // the local diskTarget variable; checking it first ensures warm- // restart-skip-reindex actually works. The defer that swaps // idx.graph back to diskTarget runs LATER, when IndexCtx returns, // so we can't rely on it here. Falls through to idx.graph for the // non-shadow path. mtimeTarget := graph.Store(idx.graph) if diskTarget != nil { mtimeTarget = diskTarget } // Full-index persist is AUTHORITATIVE: replace the repo's entire mtime // set so files deleted since the last index are pruned. An upsert-only // write (BulkSetFileMtimes) leaves deleted-file rows behind, and warm- // restart reconcile then detects them as phantom deletions on every // restart — forcing a full re-track that never converges. Prefer the // replace capability; fall back to upsert for backends without it. if len(mtimeSnapshot) > 0 { var perr error persisted := false if r, ok := mtimeTarget.(graph.FileMtimeReplacer); ok { perr, persisted = r.ReplaceFileMtimes(idx.repoPrefix, mtimeSnapshot), true } else if w, ok := mtimeTarget.(graph.FileMtimeWriter); ok { perr, persisted = w.BulkSetFileMtimes(idx.repoPrefix, mtimeSnapshot), true } if persisted { if perr != nil { idx.logger.Warn("persist file mtimes failed", zap.String("repo", idx.repoPrefix), zap.Error(perr)) } else { idx.logger.Info("persisted file mtimes", zap.String("repo", idx.repoPrefix), zap.Int("count", len(mtimeSnapshot))) } } // Crash-safe content path: reap every content row this walk did NOT // re-stream. keep is contentStreamedFiles — the files that actually // produced content sections this run — NOT the surviving-file mtime // set: a file can survive on disk yet stop yielding content (doc // emptied, classification changed), and keying keep off mtimes would // protect its stale rows forever. Recorded keys are the wipe's own // argument (the node FilePath content_fts carries), so the comparison // matches the stored rows in single- and multi-repo form alike. A walk // that streamed NO content falls back to the repo-wide wipe: the repo // has zero content files now, and the sweep's empty-keep guard (a // never-wipe-from-empty safety net) would otherwise no-op and leave // every stale row behind. Only when the per-file wipe path is active; // on the repo-wide-wipe fallback the up-front pre-wipe already cleared // both transitions. Runs only under the completed-walk guard above // (len(mtimeSnapshot) > 0), so a killed parse never triggers it. if contentWipeFile != nil { contentStreamedMu.Lock() keep := contentStreamedFiles contentStreamedMu.Unlock() if len(keep) == 0 { if cs := idx.contentSearcher(); cs != nil { if err := cs.WipeContent(idx.RepoPrefix()); err != nil { idx.logger.Warn("indexer: content wipe of contentless repo failed", zap.Error(err)) } } } else if sw, ok := idx.contentSearcher().(interface { DeleteContentFilesForRepoNotIn(repoPrefix string, keep map[string]struct{}) error }); ok { if err := sw.DeleteContentFilesForRepoNotIn(idx.repoPrefix, keep); err != nil { idx.logger.Warn("indexer: content sweep of stale files failed", zap.Error(err)) } } } } // Retain parse errors and record index metadata. idx.parseErrors = errors idx.totalDetected = len(files) idx.lastIndexTime = time.Now() if idx.deferResolve { // Multi-repo orchestrator runs these serially after wg.Wait() // to avoid races on the shared graph between this goroutine's // ResolveAll mutation phase and a sibling goroutine's contract // pass walking AllEdges. See SetDeferResolve. idx.pendingContractReg = contractReg } else { // Materialise dep:: contract nodes from go.mod BEFORE // ResolveAll so the resolver's import bridge can re-target Go // imports of declared modules to their dep contract node. idx.extractGoModContracts(contractReg) reporter.Report("resolving references", 0, 0) // Resolve cross-file references. idx.populateCppIncludeDirs(true) idx.resolver.ResolveAll() // Infer structural interface satisfaction + method-level // overrides. Skipped under deferGlobalPasses so a batch caller // (warmup, ReconcileAll) can run them once at the end against // the final shared graph instead of paying the O(global) walk // per repo. InferOverrides depends on InferImplements running // first. if !idx.deferGlobalPasses { reporter.Report("inferring interfaces", 0, 0) idx.resolver.InferImplements() idx.resolver.InferOverrides() } // Semantic enrichment (SCIP, go/types, LSP). if idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders() { reporter.Report("semantic enrichment", 0, 0) // Key by the repo prefix so a repo-scoped provider can scope // file selection to this repo (empty in single-repo mode). roots := map[string]string{idx.repoPrefix: absRoot} // The inline full-index path does not gate or persist enrichment // markers (a zero EnrichOptions): the deferred warmup path owns the // skip-on-restart optimisation. results, _, err := idx.semanticMgr.EnrichAll(idx.graph, roots, semantic.EnrichOptions{}) if err != nil { idx.logger.Warn("semantic enrichment failed", zap.Error(err)) } else if len(results) > 0 { for _, r := range results { idx.logger.Info("semantic enrichment result", zap.String("provider", r.Provider), zap.String("language", r.Language), zap.Int("confirmed", r.EdgesConfirmed), zap.Int("added", r.EdgesAdded), zap.Int("refuted", r.EdgesRefuted), zap.Float64("coverage", r.CoveragePercent), ) } } } } reporter.Report("building search index", 0, 0) // Build search index. idx.buildSearchIndex() if !idx.deferResolve { // Contracts were already extracted inline during parse (per file, // per worker). Here we just finish up. extractGoModContracts // already ran (see the !deferResolve branch above) so dep // nodes were available during ResolveAll's import-bridge pass; // commitContracts is idempotent for those. reporter.Report("extracting contracts", 0, 0) idx.extractExternalModules() idx.extractDIContracts(contractReg) idx.commitContracts(contractReg) // Test-edge pass — runs once the call graph is final. Skipped // under deferGlobalPasses so a batch caller can fold this into // one global pass after the per-repo loop. if !idx.deferGlobalPasses { reporter.Report("test edge pass", 0, 0) marked, emitted := markTestSymbolsAndEmitEdges(idx.graph) if marked > 0 || emitted > 0 { idx.logger.Info("test edges emitted", zap.Int("test_symbols", marked), zap.Int("edges", emitted), ) } if re, ep, fa := synthesizeCapabilityEdges(idx.graph); re > 0 || ep > 0 || fa > 0 { idx.logger.Info("capability edges emitted", zap.Int("reads_env", re), zap.Int("executes_process", ep), zap.Int("accesses_field", fa), ) } reporter.Report("clone detection pass", 0, 0) if cs := detectClonesAndEmitEdgesCtx(ctx, idx.graph, idx.repoPrefix, idx.cloneThreshold()); cs.Items > 0 { idx.logger.Info("clone edges emitted", zap.Int("items", cs.Items), zap.Int("clone_pairs", cs.Pairs), zap.Int("edges", cs.Edges), zap.Int("skipped_buckets", cs.SkippedBuckets), zap.Int("skipped_bucket_items", cs.SkippedBucketItems), zap.Int("diffused_pairs", cs.DiffusedPairs), zap.Int("diffused_edges", cs.DiffusedEdges), ) } // Seed the incremental clone index from the freshly-baselined // signatures + sidecar so steady-state single-file edits go // incremental (EvictFuncs/UpdateFuncs) instead of re-running // this whole-graph pass per file. The batch pass remains the // re-baseline (corrects CMS drift) and owns diffusion. if idx.cloneIndex != nil { idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix) } // Framework dynamic-dispatch synthesis — runs once the call // graph and interface inference are final. Skipped under // deferGlobalPasses; the batch caller folds it into // RunGlobalGraphPasses. reporter.Report("framework dispatch synthesis", 0, 0) if rep := resolver.RunFrameworkSynthesizers(idx.graph); rep.Total > 0 { idx.logger.Info("framework dispatch calls synthesized", zap.Int("edges", rep.Total), zap.Any("per_synthesizer", rep.Per), ) } // External-call placeholder synthesis (opt-in) — runs after // the resolver and stub passes so only genuinely un-indexed // external targets are left to materialise. reporter.Report("external-call synthesis", 0, 0) if extCalls := resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled()); extCalls > 0 { idx.logger.Info("external-call placeholders synthesized", zap.Int("edges", extCalls), ) } if spec := resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled()); spec > 0 { idx.logger.Info("speculative dispatch edges synthesized", zap.Int("edges", spec), ) } // Reachability index — used to be precomputed for every // impact seed here. The eager pass was retired because the // breakeven was untenable on monorepo graphs (k8s: // ~2000 s build to save ~10 ms per query, ~200 k-query // breakeven). reach.Lookup now computes the BFS on first // access per seed and caches the result. The // InvalidateIndex call bumps the build counter so any // stale stamps from a prior build (e.g. snapshot reload // before a partial mutation) no longer shadow the live // graph state. reach.InvalidateIndex() } } // Auto-upgrade to Bleve if above threshold. Run in the background // so the foreground IndexCtx returns immediately — populating // Bleve with 50k+ symbols takes 30-60s and adding that to the // initial-index latency was the dominant tail. Searches against // idx.search keep hitting the in-memory backend until the swap // completes; nothing observes a half-built Bleve. // // upgradeOnce gates the spawn so multi-repo warmup, which calls // IndexCtx once per tracked repo, doesn't launch one upgrade // goroutine per post-threshold repo. One per indexer lifetime. // // Skip the upgrade when the active search backend is the // SymbolSearcher adapter: the disk store's native FTS is // already serving search at engine-native latency, and // spawning a parallel Bleve build would (a) waste ~100MB heap // re-indexing the same corpus and (b) silently swap the // adapter out for Bleve on completion — defeating the whole // FTS path. The Swappable's current backend tells us which // branch we're on. if !isSymbolSearcherBackend(idx.search) && idx.search.Count() >= search.AutoThreshold { idx.upgradeOnce.Do(func() { reporter.Report("scheduling search backend upgrade", 0, 0) idx.upgradeSpawnedMu.Lock() idx.upgradeSpawned++ idx.upgradeSpawnedMu.Unlock() // Snapshot upfront so the background goroutine doesn't // read Node.Meta concurrently with subsequent Index // calls' Meta-writing passes (reach.BuildIndex, // ResolveTemporalCalls, ...). snapshot := idx.snapshotBleveEntries() go idx.upgradeSearchToBleve(snapshot) }) } // Persist the parser quarantine so a file that crashed the parser // stays skipped across daemon restarts until its content changes. if quarantine != nil { if err := quarantine.Save(); err != nil { idx.logger.Warn("indexer: failed to persist parser quarantine", zap.Error(err)) } else if n := quarantine.Len(); n > 0 { idx.logger.Info("indexer: parser quarantine", zap.Int("files", n)) } } reporter.Report("indexing complete", int(fileCount), len(files)) // Persist the Merkle baseline so the next incremental pass diffs // against content hashes rather than re-indexing the whole repo. workspaceFP := "" if idx.merkleEnabled() { paths := make([]string, len(files)) for i, wf := range files { paths[i] = wf.path } workspaceFP = idx.saveMerkleBaseline(absRoot, paths) } idx.indexGen.Add(1) // invalidate the trigram search cache nodes, edges := idx.repoNodeEdgeCount() idx.persistRepoIndexState(diskTarget, absRoot, workspaceFP, nodes, edges) result = &IndexResult{ NodeCount: nodes, EdgeCount: edges, FileCount: int(fileCount), QuarantinedFiles: quarantine.Len(), SkippedFiles: len(skippedBySize) + len(skippedByContent) + int(skippedByTimeout) + int(skippedByMinified), DurationMs: time.Since(start).Milliseconds(), Errors: errors, } idx.warnIfEdgeSanityViolated(result) // A whole-repo (re-)track that observed files did work: mark the repo for // deferred semantic enrichment. FullRetrack is stamped by the multi-repo // caller after this returns, so FileCount carries the signal here. if result.FileCount > 0 || result.FullRetrack { idx.pendingEnrich.Store(true) // IndexCtx re-parsed every file, dropping this repo's hover-enrichment // edges — force the deferred pass past the completion-marker gate so // they are restored even at an unchanged clean HEAD. idx.fullReindexed.Store(true) } return result, nil } // repoNodeEdgeCount returns this indexer's contribution to the graph, // scoped to its repoPrefix in multi-repo mode. In single-repo mode // (empty prefix) every node carries an empty RepoPrefix anyway, so the // graph totals equal the repo's contribution and we use the cheap // global accessors. The multi-repo path uses RepoMemoryEstimate which // walks only this repo's byRepo bucket — O(repo size), not O(graph) — // so callers that stamp RepoMetadata.NodeCount/EdgeCount no longer // freeze the workspace-wide total at TrackRepo time. func (idx *Indexer) repoNodeEdgeCount() (int, int) { if idx.repoPrefix == "" { return idx.graph.NodeCount(), idx.graph.EdgeCount() } est := idx.graph.RepoMemoryEstimate(idx.repoPrefix) return est.NodeCount, est.EdgeCount } // warnIfEdgeSanityViolated logs a loud warning when an index pass // produced files and symbol nodes but no edges — see // IndexResult.EdgeSanityViolated. func (idx *Indexer) warnIfEdgeSanityViolated(r *IndexResult) { if r.EdgeSanityViolated() { idx.logger.Warn("indexer: edge-sanity check failed — index has files and nodes but zero edges; edge extraction likely failed wholesale", zap.Int("files", r.FileCount), zap.Int("nodes", r.NodeCount), zap.Int("edges", r.EdgeCount)) } } // IndexFile parses a single file and patches the graph (evict then // add), including per-file resolver work for cross-file references. // Use in the single-event fsnotify path where each edit is isolated. func (idx *Indexer) IndexFile(filePath string) error { return idx.indexFile(filePath, true) } // IndexFileNoResolve is IndexFile minus the per-file resolver call. // Callers in batch paths (storm mode, branch-switch reconcile, git // diff dispatch) use this when they will run resolver.ResolveAll() // once at the end of the batch; otherwise a 500-file checkout pays // the per-file resolver cost 500 times instead of once. func (idx *Indexer) IndexFileNoResolve(filePath string) error { return idx.indexFile(filePath, false) } func (idx *Indexer) indexFile(filePath string, resolve bool) error { absPath, err := filepath.Abs(filePath) if err != nil { return err } // relKey gives the canonical key (slash form, Unicode NFC). Using // it here keeps an incremental re-index — which may be driven by a // git-watcher path in NFC or an FSEvents path in NFD — landing on // the exact graph key the bulk walk created, so the evict below // finds the file's existing nodes instead of leaking a duplicate. relPath := idx.relKey(absPath) // In multi-repo mode, the graph stores prefixed file paths. graphPath := idx.prefixPath(relPath) // Parse-then-swap: we must NOT evict the file's existing nodes/edges // and search entries until we hold a usable parse result. Evicting // first leaves the file at zero nodes whenever the on-disk bytes are // transiently unparseable (a save mid-edit) — a failed extraction // then returns early and the symbols stay nuked. Capturing the old // state up front and deferring the actual eviction to evictExisting() // keeps the file stale-but-present on failure (stale beats empty) and // shrinks the no-nodes window to the gap between evict and AddBatch. // // oldFuncIDs holds this file's function/method node IDs so the // incremental clone index can drop their CMS/LSH contributions — // EvictFile removes the nodes (and their clone_sig) from the graph, // so it must be captured before evictExisting runs. var oldFuncIDs []string evictExisting := func() { for _, n := range idx.graph.GetFileNodes(graphPath) { if n.Kind != graph.KindFile && n.Kind != graph.KindImport { idx.search.Remove(n.ID) } if n.Kind == graph.KindFunction || n.Kind == graph.KindMethod { oldFuncIDs = append(oldFuncIDs, n.ID) } } idx.restubIncomingRefs(graphPath) idx.graph.EvictFile(graphPath) } src, err := os.ReadFile(absPath) if err != nil { return err } lang, ok := idx.effectiveLanguage(absPath, src) if !ok { return nil } ext, _ := idx.registry.GetByLanguage(lang) if ext == nil { return nil } // Honour the size cap on the incremental path too: an over-cap // file gets a synthetic skip node, not a parse — matching the // bulk IndexCtx walk. This IS a successful result, so it evicts the // prior state and installs the synthetic node, same as before. if maxSize := idx.config.MaxFileSize; maxSize > 0 && int64(len(src)) > maxSize { n := sizeSkipNode(skippedFile{ relPath: filepath.ToSlash(relPath), lang: lang, size: int64(len(src)), }, maxSize) idx.applyRepoPrefix([]*graph.Node{n}, nil) evictExisting() idx.graph.AddBatch([]*graph.Node{n}, nil) return nil } // Honour the content-admission gate on the incremental path too, so a // document over its cap (or a data asset, by default) the cold walk // would skip doesn't get parsed back in when the watcher re-indexes it // — same synthetic-skip-node treatment as the size cap above. if reason, skip := idx.newContentAdmissionGate().skip(lang, int64(len(src))); skip { n := contentSkipNode(skippedFile{ relPath: filepath.ToSlash(relPath), lang: lang, size: int64(len(src)), reason: reason, }) idx.applyRepoPrefix([]*graph.Node{n}, nil) evictExisting() idx.graph.AddBatch([]*graph.Node{n}, nil) return nil } // Pre-ingestion transforms — same pipeline as the bulk path. src = idx.transforms.run(relPath, src) // Crash isolation for the incremental path: a file the user just // saved that SIGSEGVs the parser is quarantined instead of taking // the daemon down with it. The pool is long-lived and shared, so // the watcher hot path never forks a worker subprocess per file. var pool *crashpool.Pool var quarantine *crashpool.Quarantine if idx.crashIsolationEnabled() { pool, quarantine = idx.sharedParsePool() } result, skipped, err := idx.extractFile(pool, quarantine, absPath, relPath, lang, ext, src) if quarantine != nil && quarantine.Len() > 0 { _ = quarantine.Save() } if result == nil { // No usable parse result (transient parse failure, quarantine, // timeout). Do NOT evict — the file's prior nodes/edges/search // entries stay intact. A stale-but-present file beats an empty // one, and the next successful re-index swaps cleanly. // // The bytes were read successfully (src above), so this is a // stable fact about the file's current on-disk content, not a // transient "couldn't even open it" failure (that case returns // earlier, before relPath/mtime bookkeeping, and deliberately // leaves the mtime unrecorded so it keeps retrying). Recording // the mtime here keeps a warm restart's HasChangesSinceMtimes // from perpetually seeing this one unparseable file as "the // repo changed" and routing the whole repo through the // expensive shadow re-track path on every restart. idx.recordFileMtime(relPath, absPath) return err } // Affected-by snapshot: the symbol shapes and reverse-reference // sources the post-resolve signature-delta pass compares against, // captured BEFORE eviction — EvictFile drops in-edges from // unchanged files and replaces this file's nodes, so neither is // recoverable afterwards. Skipped on the no-resolve and // deferred-batch paths, whose callers run a full resolve (and // persistAllRefFacts) once at the end of the batch. // // Also skipped for a quarantined / timed-out / minified-skipped file: // its synthetic result carries zero symbols, so the delta would read // every prior symbol as removed and fan out to re-resolve the whole // reverse graph on a transient parse failure. A failure that yields no // symbols is not the same as a symbol genuinely deleted from source. var abSnap *affectedBySnapshot var reuseIdx map[reuseKey]*reuseVal var priorUnresolved []*graph.Edge if resolve && !idx.deferGlobalPasses && !skipped { abSnap = idx.snapshotAffectedBy(graphPath) // Snapshot the file's outgoing edges before eviction: resolved ones so // the re-parse recovers unchanged resolutions (reuseIdx), and the ones // still unresolved so the forward pass can skip re-trying them // (priorUnresolved). Together this makes a save re-resolve only the // references it actually changed instead of the whole file. reuseIdx, priorUnresolved = captureIncrementalState(idx.graph, graphPath) } // We hold a usable result: evict the old state now, then add the // new — the window where the file has no nodes is just this gap. evictExisting() // Coverage extractors (todos, licenses, ownership). Same call // site exists in the bulk IndexCtx worker pool — see // applyCoverageDomains. Skipped for a quarantined / timed-out file. if !skipped { idx.applyCoverageDomains(relPath, lang, src, result) } idx.applyRepoPrefix(result.Nodes, result.Edges) // Reuse prior resolutions for edges whose source-side shape is unchanged // (the common case on a small edit), so the resolver below only handles // genuinely-new references instead of re-resolving the whole file. The // about-to-be-added node IDs let the reuse recover same-file targets that // eviction removed and this AddBatch re-adds under identical IDs — without // them a same-file call's resolution + tier is lost to a full re-resolve on // every structural save. newNodeIDs := make(map[string]struct{}, len(result.Nodes)) for _, n := range result.Nodes { if n != nil { newNodeIDs[n.ID] = struct{}{} } } if reused := applyResolvedOutEdges(idx.graph, result.Edges, reuseIdx, newNodeIDs); reused > 0 { idx.logger.Debug("indexer: reused prior resolutions", zap.Int("edges", reused), zap.String("file", graphPath)) } // Content (incremental): clear this file's prior content rows, then // re-stream + lean — mirrors the full-index per-file path so an edited // content file leaves no stale rows and doesn't revert to full text on // the node. if cs := idx.contentSearcher(); cs != nil { if fp := firstContentFilePath(result.Nodes); fp != "" { if err := cs.WipeContentFile(fp); err != nil { idx.logger.Warn("indexer: content index file wipe failed", zap.Error(err)) } } } idx.streamContentSections(result.Nodes) idx.graph.AddBatch(result.Nodes, result.Edges) idx.persistConstValues(result) idx.persistFileMeta(relPath, src, result) // Add new symbols to search index. shouldIndexForSearch enforces // the same SkipSearch filter used by the bulk and upgrade paths. // When the backing store implements graph.SymbolSearcher we // also mirror each upsert into its native FTS, so an // incremental reindex doesn't fall out of sync with the // bulk-built corpus. searcher, _ := idx.graph.(graph.SymbolSearcher) for _, n := range result.Nodes { if !idx.shouldIndexForSearch(n) { continue } idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...) if searcher != nil { if err := searcher.UpsertSymbolFTS(n.ID, ftsTokensFor(n, idx.projectName)); err != nil { idx.logger.Debug("indexer: backend FTS upsert failed", zap.String("id", n.ID), zap.Error(err)) } } } if resolve { // Forward pass (this file's outgoing references) plus the // reverse pass binding callers in OTHER files that reference a // symbol (re)defined here — a symbol newly defined or changed // here leaves callers elsewhere pointing at the unresolved stub // restubIncomingRefs left when the prior concrete node was // evicted. Scoped to this file's names — not a whole-graph // ResolveAll — and run as one combined pass so the resolver's // per-pass indexes are built once per save, not twice. // // Skip re-resolving references that were already unresolved before the // edit and are unchanged: they stay parked on their stubs for the // incoming pass, so a small edit to a reference-heavy file no longer // re-runs the candidate cascade on thousands of stdlib/external calls. idx.resolver.SetIncrementalSkip(priorUnresolved) idx.resolver.ResolveFileAndIncoming(graphPath) idx.resolver.SetIncrementalSkip(nil) // CPG-lite dataflow placeholders for this file: inter- // procedural callees may have just been lifted by // ResolveFile, so re-run the dataflow materialisation pass // to keep arg_of / returns_to edges in sync with the // freshly resolved EdgeCalls graph. Scoped to this file's // out-edges — not a whole-graph AllEdges scan — so an // incremental edit stays O(file), not O(all edges). idx.materializeDataflowParamsForFile(graphPath, result.Edges) // Clone detection. EvictFile above removed this file's // EdgeSimilarTo edges in both directions. When the incremental // clone index is built, re-bank just this file's bodies // (EvictFuncs the old ids, UpdateFuncs the fresh nodes) — an // O(edited file) update that restores the same edge set the // whole-graph pass would. Until a batch/global pass has seeded // the index (built=false) we fall back to the full recompute. // Skipped under deferGlobalPasses — a batch caller (ReconcileAll, // warmup) runs the global pass once at the end. if !idx.deferGlobalPasses { if idx.cloneIndex != nil { // Seed the incremental clone index on first use — e.g. after a // warm restart that loaded the graph from disk without a full // re-index, `built` is false. Without this we'd fall to the // whole-graph recompute on EVERY save; instead pay it once and // then run the O(edited file) update. if !idx.cloneIndex.built { idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix) } idx.cloneIndex.EvictFuncs(idx.graph, oldFuncIDs) idx.cloneIndex.UpdateFuncs(idx.graph, idx.repoPrefix, cloneFuncNodes(result.Nodes), idx.cloneThreshold()) } else { detectClonesAndEmitEdges(idx.graph, idx.repoPrefix, idx.cloneThreshold()) } } // in-memory backend. Skipped for a quarantined / timed-out / // minified file: its synthetic result yields no facts, so a // delete-then-set would durably drop the file's real facts on a // transient parse failure and leave them gone until a clean // reparse — abSnap is nil here too, so the affected-by pass that // would also fan out is already a no-op. if !skipped { idx.persistRefFactsForFiles([]string{graphPath}) // Affected-by re-resolution: if this save changed a symbol's // signature or kind, or removed a symbol, re-resolve the files // that referenced it — bounded, synchronous, and gated on the // signature delta so a body-only edit fans out to nothing. idx.reresolveAffectedBy(graphPath, abSnap, result.Nodes) // Incremental semantic enrichment for this single file. Mirrors the // full-index EnrichAll call but scoped to the saved file, so a // watcher save re-runs the type resolvers (and any watch-enabled // LSP / compiler provider) instead of leaving the file's edges at // their pre-enrichment tier until the next full reindex. Gated // internally on Config.EnrichOnWatch; a no-op when disabled. providersPresent := idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders() reEnriched := false if providersPresent { if _, err := idx.semanticMgr.EnrichFile(idx.graph, idx.rootPath, graphPath); err != nil { idx.logger.Debug("indexer: incremental semantic enrichment failed", zap.String("file", graphPath), zap.Error(err)) } else { reEnriched = idx.semanticMgr.EnrichesOnWatch() } } // Record whether this live re-parse left the file below the // enrichment tier: providers exist (so there IS an lsp/ast tier to // fall short of) but the save did not re-run enrichment. When set, // find_usages / get_callers flag their default text_matched // suppression as re-verification-pending so a hidden-but-real usage // is diagnosable rather than silently dropped. Cleared when // enrichment did re-run for the file. idx.setReparsePendingEnrichment(graphPath, providersPresent && !reEnriched) } } // Update mtime for this file. relPath is already the canonical // key (relKey applied slash + NFC), so the mtime entry lines up // with the graph file-node key and with the bulk-walk mtimes. idx.recordFileMtime(relPath, absPath) return nil } // recordFileMtime restamps the recorded mtime for relPath (a canonical // relKey, not repo-prefixed) from absPath's current on-disk mtime, both // in the in-memory map and — when the backend supports it — the store's // FileMtime sidecar. Per-file write is ~1ms on the on-disk backend; // trivial under steady-state file-watcher load. A missing/unstatable // file is a no-op; a persist error is logged but non-fatal, since the // in-memory map (which the current process trusts) is already correct. func (idx *Indexer) recordFileMtime(relPath, absPath string) { info, err := os.Stat(absPath) if err != nil { return } mtime := info.ModTime().UnixNano() idx.mtimeMu.Lock() idx.fileMtimes[relPath] = mtime idx.mtimeMu.Unlock() if w, ok := idx.graph.(graph.FileMtimeWriter); ok { if err := w.BulkSetFileMtimes(idx.repoPrefix, map[string]int64{relPath: mtime}); err != nil { idx.logger.Warn("persist file mtime failed", zap.String("repo", idx.repoPrefix), zap.String("file", relPath), zap.Error(err)) } } } // StructuralSymbols parses a file from its current on-disk content and // returns the structural symbols it defines — functions, methods, // types, interfaces, constants, variables, fields, enum members — and // nothing else. It is a read-only probe: the graph and the search // index are left completely untouched, no mtime is stamped, and no // resolver runs. The watcher uses it to decide whether a save is // structurally inert (a comment / whitespace / config-value edit that // changes no symbol) and can skip the destructive evict + reindex. // // The second return reports whether the file was parseable at all: a // file with no detectable language, an over-cap file, or a read error // yields (nil, false). A genuinely empty source file yields // (empty-slice, true). func (idx *Indexer) StructuralSymbols(filePath string) ([]*graph.Node, bool) { absPath, err := filepath.Abs(filePath) if err != nil { return nil, false } relPath, err := filepath.Rel(idx.rootPath, absPath) if err != nil { relPath = filePath } src, err := os.ReadFile(absPath) if err != nil { return nil, false } lang, ok := idx.effectiveLanguage(absPath, src) if !ok { return nil, false } ext, _ := idx.registry.GetByLanguage(lang) if ext == nil { return nil, false } // An over-cap file is never structurally parsed on the indexing // path either (it gets a synthetic skip node), so the watcher // cannot prove inertness for it — fall through to a real reindex. if maxSize := idx.config.MaxFileSize; maxSize > 0 && int64(len(src)) > maxSize { return nil, false } // Same pre-ingestion transforms as indexFile so the probe parses // exactly the bytes the real index pass would. src = idx.transforms.run(relPath, src) var pool *crashpool.Pool var quarantine *crashpool.Quarantine if idx.crashIsolationEnabled() { pool, quarantine = idx.sharedParsePool() } result, skipped, err := idx.extractFile(pool, quarantine, absPath, relPath, lang, ext, src) if quarantine != nil && quarantine.Len() > 0 { _ = quarantine.Save() } // A skipped (quarantined / timed-out) file produces only a // synthetic node — not the real symbol set — so inertness cannot // be proven and the caller must reindex normally. if result == nil || skipped || err != nil { return nil, false } out := make([]*graph.Node, 0, len(result.Nodes)) for _, n := range result.Nodes { if isStructuralKind(n.Kind) { out = append(out, n) } } return out, true } // isStructuralKind reports whether a node kind represents a structural // code symbol — the kinds whose presence, name, or signature define a // file's graph shape. File and import nodes (graph bookkeeping), // params, closures, and the coverage-domain kinds (todos, licenses, // strings, …) are deliberately excluded: a change confined to those // does not alter the structural graph the watcher cares about. func isStructuralKind(k graph.NodeKind) bool { switch k { case graph.KindFunction, graph.KindMethod, graph.KindType, graph.KindInterface, graph.KindVariable, graph.KindConstant, graph.KindField, graph.KindEnumMember: return true default: return false } } // ResolveAll re-runs the global cross-file reference resolver and // interface-implementation inference. Exposed for batch paths that // defer per-file resolver work until the end of a batch. func (idx *Indexer) ResolveAll() { idx.populateCppIncludeDirs(true) idx.resolver.ResolveAll() idx.resolver.InferImplements() idx.resolver.InferOverrides() // Framework dynamic-dispatch synthesis (gRPC / Temporal / event // channels / native bridges) depends on InferImplements (the // interface-satisfaction signals) having run first. resolver.RunFrameworkSynthesizers(idx.graph) // External-call placeholder synthesis (opt-in) — runs after the // resolver and stub passes so only genuinely un-indexed external // targets remain to materialise. resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled()) resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled()) // CPG-lite dataflow rewriting must run after the call resolver // has lifted unresolved:: targets; arg_of edges then point at // real function/method nodes whose param nodes can be found, // and returns_to placeholders join cleanly against the // now-resolved EdgeCalls edge at the same caller+line. idx.materializeDataflowParams() // Seed the durable reference-facts sidecar from the fully-resolved graph // (no-op on the in-memory backend). idx.persistAllRefFacts() } // EvictFile removes all nodes and edges belonging to filePath. // // filePath may arrive in any Unicode form — the git watcher derives it // from `git diff` output (NFC), while an FSEvents-driven evict carries // the filesystem's form (NFD on macOS). relKey folds both to the // canonical NFC key the graph indexed the file under, so the eviction // actually finds the file's nodes rather than silently no-opping and // leaving a stale subtree behind. func (idx *Indexer) EvictFile(filePath string) (int, int) { absPath := filePath if !filepath.IsAbs(absPath) { absPath = filepath.Join(idx.rootPath, filePath) } relPath := idx.relKey(absPath) // In multi-repo mode, the graph stores prefixed file paths. graphPath := idx.prefixPath(relPath) // Remove from search index. for _, n := range idx.graph.GetFileNodes(graphPath) { if n.Kind != graph.KindFile && n.Kind != graph.KindImport { idx.search.Remove(n.ID) } } idx.restubIncomingRefs(graphPath) idx.evictEnrichment(graphPath) idx.deleteRefFactsForFiles(idx.repoPrefix, []string{graphPath}) return idx.graph.EvictFile(graphPath) } // ReresolveFileScoped forces the scoped re-resolution + LSP re-verify a normal // IndexFile performs, WITHOUT re-parsing and WITHOUT the IsStale gate — used by // the watcher's shape-degradation self-heal, where the file's mtime is already // current (IndexFile just ran) yet its resolved edges came out degraded, so a // plain IncrementalReindexPaths would stale-gate it out. Re-runs only this // file's forward + incoming resolve and, when a watch-enabled provider is // wired, its incremental enrichment. O(file), no whole-graph pass. No-op when // the file has no nodes (evicted since it was enqueued). func (idx *Indexer) ReresolveFileScoped(filePath string) error { absPath := filePath if !filepath.IsAbs(absPath) { absPath = filepath.Join(idx.rootPath, filePath) } graphPath := idx.prefixPath(idx.relKey(absPath)) if len(idx.graph.GetFileNodes(graphPath)) == 0 { return nil // file gone / evicted; nothing to re-resolve } idx.resolver.ResolveFileAndIncoming(graphPath) providersPresent := idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders() reEnriched := false if providersPresent { if _, err := idx.semanticMgr.EnrichFile(idx.graph, idx.rootPath, graphPath); err != nil { idx.logger.Debug("indexer: forced scoped enrichment failed", zap.String("file", graphPath), zap.Error(err)) } else { reEnriched = idx.semanticMgr.EnrichesOnWatch() } } // Keep the find_usages staleness marker consistent with the enrichment // that actually ran during this forced re-resolve (mirrors indexFile). idx.setReparsePendingEnrichment(graphPath, providersPresent && !reEnriched) return nil } // restubIncomingRefs rewrites every resolved reference edge that points // INTO a symbol of graphPath from a surviving (other-file) source back // to an `unresolved::` stub, in place, BEFORE the file's nodes are // evicted. Graph eviction otherwise drops those incoming caller edges // wholesale (it removes the edge from the surviving source's out-edge // bucket) and nothing recreates them until a cold reindex — so editing // or deleting a definition silently strips its callers' edges and // find_usages / get_callers go blank. Re-stubbing detaches the edges // from the soon-to-be-evicted nodes so they survive as pending stubs; // ResolveIncomingForFile (after a re-index) rebinds them to the file's // fresh symbols, or they stay unresolved — the correct state once the // symbol is gone. Only name-resolvable reference kinds are re-stubbed; // structural and enrichment edges are left to be dropped. Backend- // agnostic: GetInEdges + ReindexEdges are the same Store primitives the // resolver uses, so this behaves identically on the in-memory and disk // stores. // evictEnrichment drops the per-node enrichment sidecar rows (churn, // coverage, release, blame — change A) for a file's nodes on the // delete/rename paths only, so a removed file leaves no orphan // enrichment. Capability-gated. A modify re-indexes the same node IDs // (enrichment stays valid) so it is NOT cascaded there. func (idx *Indexer) evictEnrichment(graphPath string) { nodes := idx.graph.GetFileNodes(graphPath) if len(nodes) == 0 { return } ids := make([]string, 0, len(nodes)) for _, n := range nodes { ids = append(ids, n.ID) } if w, ok := idx.graph.(graph.ChurnEnrichmentWriter); ok { _ = w.DeleteChurn(ids) } if w, ok := idx.graph.(graph.CoverageEnrichmentWriter); ok { _ = w.DeleteCoverage(ids) } if w, ok := idx.graph.(graph.ReleaseEnrichmentWriter); ok { _ = w.DeleteReleases(ids) } if w, ok := idx.graph.(graph.BlameEnrichmentWriter); ok { _ = w.DeleteBlame(ids) } } func (idx *Indexer) restubIncomingRefs(graphPath string) { nodes := idx.graph.GetFileNodes(graphPath) if len(nodes) == 0 { return } evicted := make(map[string]struct{}, len(nodes)) for _, n := range nodes { evicted[n.ID] = struct{}{} } var batch []graph.EdgeReindex for _, n := range nodes { if n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) { continue } stub := graph.UnresolvedMarker + n.Name for _, e := range idx.graph.GetInEdges(n.ID) { if e == nil || !graph.IsResolvableRefEdge(e.Kind) { continue } if _, fromEvicted := evicted[e.From]; fromEvicted { continue // intra-file edge: the source is evicted too } if graph.IsUnresolvedTarget(e.To) { continue // already a pending stub } oldTo := e.To // Stash + clear the edge's resolved provenance before restubbing: // an `unresolved::` stub must not keep advertising a resolved // tier. The incoming-resolve pass restores it verbatim if the stub // rebinds to the same target (idempotent re-parse); a deleted or // moved target leaves the stub honestly unresolved. graph.StashRestubProvenance(e) e.To = stub batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) } } if len(batch) > 0 { idx.graph.ReindexEdges(batch) } } // embeddingDimsOrDefault returns the embedder's reported vector width, // falling back to a neutral placeholder only when the provider cannot // state its width yet (Dimensions() == 0, the APIProvider-before-first- // call case). The fallback is never persisted: buildSearchIndex and // ImportVectorIndex both overwrite it with the true width taken from a // real vector / the cached header. Kept as a named helper so the // vector-dimension default has one definition instead of a scattered // magic number. func embeddingDimsOrDefault(p embedding.Provider) int { if p == nil { return 0 } if d := p.Dimensions(); d > 0 { return d } // Provider has not committed to a width. 384 matches the default // transformer backend (MiniLM-L6-v2); a static GloVe provider // always reports its real 50 and never reaches this branch. return 384 } // collectEmbedTexts walks the nodes and produces the parallel texts / // ids slices the embedding pass consumes, plus a chunkMap recording // which synthetic IDs are chunks of which symbol. // // A symbol whose source span exceeds the configured chunk threshold is // read from disk and split into AST windows by embedding.ChunkSymbol; // each window contributes one text (its body, prefixed with the // symbol's kind + name for a little lexical grounding) under a // synthetic ID "#chunkK", and chunkMap[syntheticID] = symbolID. // A symbol below the threshold — or one whose file can't be read — // contributes a single metadata text under its own ID, exactly as the // pre-chunking pipeline did. The returned skipped count is the number // of nodes dropped by the SkipEmbed rules. func (idx *Indexer) collectEmbedTexts(nodes []*graph.Node) (texts []string, ids []string, chunkMap map[string]string, skipped int) { chunkMap = make(map[string]string) opts := idx.embedChunkOpts threshold := opts.ThresholdLines if threshold <= 0 { threshold = embedding.DefaultChunkThresholdLines } // fileCache memoizes one os.ReadFile per source file — many symbols // share a file, and the chunker only needs the bytes once. fileCache := make(map[string][]byte) readFile := func(graphPath string) []byte { if cached, ok := fileCache[graphPath]; ok { return cached } var data []byte if abs := idx.ResolveFilePath(graphPath); abs != "" { if b, err := os.ReadFile(abs); err == nil { data = b } } fileCache[graphPath] = data // cache misses too (nil) — don't re-stat return data } for _, n := range nodes { if n.Kind == graph.KindFile || n.Kind == graph.KindImport { continue } // CONTENT section bodies are served by the content index, not the // vector store — excluding them keeps the embed-text count (and the // 100k auto-disable check) code-only, so a content-heavy repo no // longer drowns the embedding pass in hundreds of thousands of // section texts. if isContentNode(n) { skipped++ continue } if config.ShouldSkipEmbed(idx.config.SkipEmbed, n.Language, string(n.Kind)) { skipped++ continue } sig, _ := n.Meta["signature"].(string) metaText := fmt.Sprintf("%s %s %s %s", n.Kind, n.Name, sig, n.FilePath) // Decide whether to sub-chunk: the symbol must declare a // multi-line span past the threshold and its file must be // readable. Anything else falls back to the metadata vector. span := n.EndLine - n.StartLine + 1 body := extractSymbolBody(n, readFile, threshold) if span <= threshold || len(body) == 0 { texts = append(texts, metaText) ids = append(ids, n.ID) continue } windows := embedding.ChunkSymbol(body, n.Language, n.ID, opts) if len(windows) <= 1 { // The chunker decided one window was enough (short body, // no splitter, parse failure) — embed it as a single // metadata + body vector under the symbol's own ID. texts = append(texts, metaText+" "+windows[0].Text) ids = append(ids, n.ID) continue } for _, w := range windows { chunkID := fmt.Sprintf("%s#chunk%d", n.ID, w.WindowIndex) texts = append(texts, fmt.Sprintf("%s %s %s", n.Kind, n.Name, w.Text)) ids = append(ids, chunkID) chunkMap[chunkID] = n.ID } } return texts, ids, chunkMap, skipped } // extractSymbolBody returns the source text of a symbol's span, read // from its file via readFile and sliced by the node's 1-based // StartLine..EndLine. Returns nil when the file is unreadable, the // line range is unusable, or the symbol is at or below the threshold // (small symbols never need their body — the caller embeds metadata). func extractSymbolBody(n *graph.Node, readFile func(string) []byte, threshold int) []byte { if n.StartLine <= 0 || n.EndLine < n.StartLine { return nil } if n.EndLine-n.StartLine+1 <= threshold { return nil } data := readFile(n.FilePath) if len(data) == 0 { return nil } return sliceLines(data, n.StartLine, n.EndLine) } // sliceLines returns the bytes of the 1-based inclusive line range // [start,end] of src. An out-of-range request is clamped; an empty // result is returned for a range that lands entirely past EOF. func sliceLines(src []byte, start, end int) []byte { if start < 1 { start = 1 } line := 1 startByte := -1 endByte := len(src) for i := 0; i < len(src); i++ { if line == start && startByte < 0 { startByte = i } if src[i] == '\n' { line++ if line == end+1 { endByte = i + 1 // include the trailing newline break } } } if startByte < 0 { return nil } if endByte < startByte { endByte = len(src) } return src[startByte:endByte] } // defaultEmbedAPIConcurrency bounds parallel embedding requests // against an API-backed embedder when embedding.api_concurrency is // unset. Four is a conservative default that overlaps round-trips // without tripping typical hosted-API rate limits. const defaultEmbedAPIConcurrency = 4 // embedChunkBatch is one unit of work for the embedding pool: the // texts of one chunk plus the index that fixes where its vectors land // in the result slice. Carrying the index makes completion order // irrelevant — workers write by index, never append. type embedChunkBatch struct { index int texts []string } // embedAllChunks embeds every text, returning the vectors in the same // order as texts. The work is split into batches of batchSize texts. // // For an API-backed embedder the batches run through a bounded worker // pool — a hosted embedding round-trip dominates index time, so // overlapping requests is a real speedup. In-process embedders // (Hugot / ONNX / GoMLX / static) serialise on an inference mutex, so // concurrency buys them nothing and they keep the simple serial path. // // The abort-on-any-error contract is preserved in both modes: the // first batch failure cancels the group and embedAllChunks returns the // error with no partial result, exactly as the old serial loop did. // embedFn already layers the deadline-halving retry on top of each // batch. func (idx *Indexer) embedAllChunks( texts []string, batchSize int, embedFn func(ctx context.Context, items []string) ([][]float32, error), ) ([][]float32, error) { if len(texts) == 0 { return nil, nil } // Split into batches up front so both the serial and parallel // paths iterate the same units. var batches []embedChunkBatch for start := 0; start < len(texts); start += batchSize { end := start + batchSize if end > len(texts) { end = len(texts) } batches = append(batches, embedChunkBatch{index: len(batches), texts: texts[start:end]}) } // Per-batch result slots, pre-sized so workers write by index and // completion order never matters. results := make([][][]float32, len(batches)) // Only run the pool for an embedder that declares itself safe and // worthwhile to call concurrently — the API-backed provider, where // overlapped HTTP round-trips are a real win. In-process backends // (Hugot / ONNX / GoMLX) hold an inference mutex, so a pool would // only add scheduling overhead; they keep the serial path. apiBacked := false if c, ok := idx.embedder.(interface{ Concurrent() bool }); ok { apiBacked = c.Concurrent() } concurrency := idx.embedAPIConcurrency if concurrency <= 0 { concurrency = defaultEmbedAPIConcurrency } if concurrency > len(batches) { concurrency = len(batches) } if !apiBacked || concurrency <= 1 { // Serial path — unchanged behaviour for in-process embedders. ctx := context.Background() for _, b := range batches { vecs, err := embedFn(ctx, b.texts) if err != nil { return nil, err } results[b.index] = vecs } return flattenEmbedResults(results), nil } // Parallel path — bounded worker pool for an API-backed embedder. // A cancellable group context means the first failure stops every // in-flight worker; the indexer's existing per-batch retry still // runs underneath embedFn. ctx, cancel := context.WithCancel(context.Background()) defer cancel() jobs := make(chan embedChunkBatch) var ( wg sync.WaitGroup errOnce sync.Once firstErr error ) fail := func(err error) { errOnce.Do(func() { firstErr = err cancel() // abort siblings on the first error }) } for w := 0; w < concurrency; w++ { wg.Add(1) go func() { defer wg.Done() for b := range jobs { if ctx.Err() != nil { return // group already aborted } vecs, err := embedFn(ctx, b.texts) if err != nil { fail(err) return } // Write into the pre-sized slot — no shared append, so // no lock and order is fixed by b.index. results[b.index] = vecs } }() } idx.logger.Info("embedding vector index with a concurrent API pool", zap.Int("workers", concurrency), zap.Int("batches", len(batches))) for _, b := range batches { if ctx.Err() != nil { break // stop feeding once aborted } select { case jobs <- b: case <-ctx.Done(): } } close(jobs) wg.Wait() if firstErr != nil { return nil, firstErr } return flattenEmbedResults(results), nil } // flattenEmbedResults concatenates per-batch vector slices back into a // single slice aligned with the original texts order. func flattenEmbedResults(results [][][]float32) [][]float32 { total := 0 for _, r := range results { total += len(r) } out := make([][]float32, 0, total) for _, r := range results { out = append(out, r...) } return out } // buildSearchIndex populates the search backend from the current graph. // When an embedder is set, also builds a vector index and wraps both // in a HybridBackend with RRF fusion. // // In multi-repo mode the search backend is shared across every repo // (Indexer.search is wired to MultiIndexer.search at construction). // Walking g.AllNodes() and re-Add()ing every node would mean each // freshly-tracked repo pays an O(workspace) re-index pass over all // previously-tracked repos' nodes — quadratic in repo count and the // dominant cost of warming up a 260-repo workspace. So when this // indexer carries a non-empty repoPrefix we walk only that repo's // byRepo bucket; the other repos' entries are already in the shared // backend from when they were tracked. Single-repo mode keeps the // AllNodes() path because nodes there carry an empty RepoPrefix and // GetRepoNodes("") would miss them. func (idx *Indexer) buildSearchIndex() { // Start every build from a clean vector-build error: the degraded vector // paths below set it, and a successful build (or a benign skip / no embedder) // leaves it nil, so LastVectorBuildError always reflects the current pass. idx.lastVectorBuildErr = nil // Code-only enumeration: content (data_class=content) sections live in // the content index, never the symbol search or the vector store, so the // FTS loop below and collectEmbedTexts both skip them anyway. Fetching // the non-content set up front means a content-heavy repo's hundreds of // thousands of sections never enter memory here (the disk backend filters // them in SQL), instead of being materialised only to be skipped. nodes := graph.RepoCodeNodes(idx.graph, idx.repoPrefix) // Install the learned sub-word boundary table on the BM25 layer // before populating postings, so the optional sparse-ngram // tokenizer's split points are data-driven and — crucially — // identical on the index path here and the query path later. The // table must be in place before the first Add for postings and // queries to agree; it is a no-op for non-BM25 backends and a no-op // for search overall unless GORTEX_SPARSE_NGRAM is set. search.InstallNgramBoundaries(idx.search, search.BuildNgramBoundaries(idx.graph)) // Build text index. The SkipSearch filter (wired through // idx.shouldIndexForSearch) drops config-key-style variable nodes // that would only pad the index — see docs on IndexConfig.SkipSearch. for _, n := range nodes { if !idx.shouldIndexForSearch(n) { continue } idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...) } // Build vector index if embedder is available. if idx.embedder == nil { return } // skipVectorBuild short-circuits the embedding pass: the text index // above is fully populated, but a caller (the daemon warmup loop // after a snapshot restore) has signalled that the workspace vector // index will be supplied separately, so re-embedding here would be // wasted work immediately overwritten by the cached index. if idx.skipVectorBuild { return } // Provisional dimensionality: trust the embedder's own report. // A provider that can't state its width yet (an APIProvider before // its first call returns 0) gets a neutral placeholder — the value // is overwritten below from the first real vector, so it never // reaches the persisted index. The old hard-coded 300 was wrong for // the default static GloVe provider (50d) and misrepresented the // index width in the interim; deriving from Dimensions() keeps it // honest for every provider. dims := embeddingDimsOrDefault(idx.embedder) // Collect texts and IDs for batch embedding. Nodes matching // Semantic.SkipEmbed (e.g. CSS custom properties, terraform blocks, // YAML/TOML/shell config vars) are kept in the text index but // excluded from the vector index — embedding them is pure cost // with no semantic payoff and on big monorepos dominates RAM. // // A symbol whose source span exceeds the chunk threshold is split // into AST windows: each window is embedded as its own vector under // a synthetic ID ("#chunkK"), and chunkMap records the // chunk → parent mapping so query-time de-chunking maps a chunk hit // back to the symbol. A small symbol stays a single metadata-only // vector under its own ID. chunkMap is empty when nothing was split. texts, ids, chunkMap, skipped := idx.collectEmbedTexts(nodes) if skipped > 0 { idx.logger.Info("skipped embedding for low-value nodes", zap.Int("count", skipped), zap.Int("embedded", len(texts))) } if len(texts) == 0 { return } // Embedding scaling guards. Hard-cap the vector index for repos // big enough that the cost no longer pays off — BM25 alone is a // fine fallback and an OOM during initial index is much worse than // missing the semantic boost. Chunk the EmbedBatch calls so any // single API request stays small (matters for hosted embedders // with per-request token limits). // // embedChunkTimeout is generous because ONNX inference (Hugot) has // long tail latency: a 60s budget made one in ~30 chunks miss its // deadline, which under the old fail-fast policy threw away every // already-embedded chunk and silently degraded to BM25 with no // signal to the user. 5 minutes covers observed worst-case spikes // without changing steady-state behaviour. On a true hang the // caller can still cancel the parent indexing call. const ( defaultEmbedMaxSymbols = 100_000 embedChunkSize = 500 embedChunkTimeout = 5 * time.Minute ) // The cap is over the embeddable-text count, which with AST // sub-chunking can exceed the symbol count. embedding.max_symbols // overrides the built-in default for users with memory headroom. embedMaxSymbols := defaultEmbedMaxSymbols if idx.embedMaxSymbols > 0 { embedMaxSymbols = idx.embedMaxSymbols } // Env override wins over both the default and the config-wired cap — // a reliable knob independent of the config plumbing. Lets an operator // lift the vector-index size guard for a large repo without editing // (or debugging) the layered config. if env := os.Getenv("GORTEX_EMBEDDINGS_MAX_SYMBOLS"); env != "" { if n, err := strconv.Atoi(strings.TrimSpace(env)); err == nil && n > 0 { embedMaxSymbols = n } } if len(texts) > embedMaxSymbols { idx.logger.Warn("vector index disabled — embedding text count exceeds threshold", zap.Int("texts", len(texts)), zap.Int("threshold", embedMaxSymbols), zap.String("hint", "BM25 text search remains active; raise embedding.max_symbols if you have memory headroom")) idx.lastVectorBuildErr = fmt.Errorf("embedding text count %d exceeds threshold %d (raise embedding.max_symbols)", len(texts), embedMaxSymbols) return } // embedWithRetry runs one chunk under ctx; on a context-deadline // failure it splits the chunk in half and retries each half once. // A single slow batch shouldn't throw away every already-embedded // chunk and silently demote the backend to BM25. ctx is the group // context, so once one chunk fails everywhere the in-flight retries // here see the cancellation and stop too. var embedWithRetry func(ctx context.Context, items []string) ([][]float32, error) embedWithRetry = func(ctx context.Context, items []string) ([][]float32, error) { chunkCtx, cancel := context.WithTimeout(ctx, embedChunkTimeout) out, err := idx.embedder.EmbedBatch(chunkCtx, items) cancel() if err == nil { return out, nil } // Only retry on deadline-style failures; auth/protocol errors // won't get better with smaller batches. A cancellation from // the group context (a sibling chunk already failed) is not a // retry case either. if ctx.Err() != nil || !errors.Is(err, context.DeadlineExceeded) || len(items) <= 1 { return nil, err } idx.logger.Warn("embed chunk timed out, retrying with halved batch", zap.Int("size", len(items)), zap.Error(err)) mid := len(items) / 2 left, lerr := embedWithRetry(ctx, items[:mid]) if lerr != nil { return nil, lerr } right, rerr := embedWithRetry(ctx, items[mid:]) if rerr != nil { return nil, rerr } return append(left, right...), nil } // Embed every chunk. For an API-backed embedder the chunks are run // through a bounded worker pool (a hosted round-trip dominates // indexing time, so overlapping requests is a real win); local // in-process backends serialise on an inference mutex, so they keep // the simple serial path. Either way the abort-on-any-error // contract holds — one chunk failure means no vector index ships. vectors, err := idx.embedAllChunks(texts, embedChunkSize, embedWithRetry) if err != nil { // A partial vector index would mis-score later queries (some // symbols semantically findable, others not) — bail to // text-only search rather than ship an inconsistent hybrid // backend. idx.logger.Warn("vector index aborted on chunk failure", zap.Error(err)) idx.lastVectorBuildErr = fmt.Errorf("chunk embedding failed: %w", err) return } // Detect actual dimensions from first vector. if len(vectors) > 0 && len(vectors[0]) > 0 { dims = len(vectors[0]) } vecBackend := search.NewVector(dims) // VectorSearcher capability bridging: if the underlying store // has a native HNSW, install it as the in-process backend's // delegate — Add becomes a no-op, Search forwards to the // engine, and we don't allocate `dim × 4 × N` bytes of heap // for a parallel in-process HNSW. The indexer still drives // the writes (BulkUpsertEmbeddings below) so the engine // index lands with the same corpus the in-process one would // have built. vecSearcher, _ := idx.graph.(graph.VectorSearcher) if vecSearcher != nil { vecBackend.SetDelegate(&vectorSearcherDelegate{s: vecSearcher}) } // persistSink is where the embedded vectors are written to the backend // so they survive a restart. Normally it is the active graph (a disk // store). Under the bulk loader idx.graph is the in-memory shadow, which // does not implement VectorSearcher; fall back to the disk store captured // at the shadow swap (bulkVectorSink) so the vector index still reaches // the `vectors` table instead of living only in the in-process HNSW and // vanishing on the next restart (which would force a paid re-embed). persistSink := vecSearcher if persistSink == nil { persistSink = idx.bulkVectorSink } var backendItems []graph.VectorItem if persistSink != nil { backendItems = make([]graph.VectorItem, 0, len(vectors)) } // Add only well-formed vectors. A nil or wrong-width vector would poison // the index — a mis-scored or panicking query — so drop it, count it, and // keep a sample of offending node IDs for the log. var droppedVectors int var droppedSample []string for i, vec := range vectors { if len(vec) != dims { droppedVectors++ if len(droppedSample) < 5 { droppedSample = append(droppedSample, ids[i]) } continue } vecBackend.Add(ids[i], vec) if persistSink != nil { backendItems = append(backendItems, graph.VectorItem{ NodeID: ids[i], Vec: vec, }) } } // If every vector was invalid there is nothing to search on. Ship text-only // rather than a silently empty vector index that mis-scores every query — // the same all-or-nothing contract as the chunk-failure abort above. if len(vectors) > 0 && vecBackend.Count() == 0 { idx.logger.Warn("vector index aborted — all embeddings invalid", zap.Int("dropped", droppedVectors), zap.Int("dimensions", dims), zap.Strings("sample_ids", droppedSample)) idx.lastVectorBuildErr = fmt.Errorf("all %d embedding vectors were invalid (want width %d)", droppedVectors, dims) return } if persistSink != nil && len(backendItems) > 0 { if err := persistSink.BulkUpsertEmbeddings(backendItems); err != nil { idx.logger.Warn("indexer: backend vector bulk upsert failed", zap.Error(err)) } else if err := persistSink.BuildVectorIndex(dims); err != nil { idx.logger.Warn("indexer: backend vector index build failed", zap.Error(err)) } } // Install the chunk → parent-symbol mapping so HybridBackend can // de-chunk vector hits back to symbols at query time. Empty when no // symbol was large enough to split. if len(chunkMap) > 0 { vecBackend.SetChunkMap(chunkMap) } // Wrap text + vector into hybrid backend, swapping it in atomically // so any concurrent searches keep seeing a coherent backend. // // Unwrap any existing HybridBackend to its text side before // re-wrapping. Without this, buildSearchIndex called again (e.g. // once per tracked repo during daemon warmup) would stack a fresh // Hybrid on top of the previous one — nested Hybrids retain all // their stale vector indexes, ballooning live memory by an order // of magnitude. The text backend (BM25 or Bleve) has already been // updated with every node via idx.search.Add above; a single // Hybrid wrapping it + the latest vecBackend is all we need. sw := idx.swappable() inner := sw.Inner() if hyb, ok := inner.(*search.HybridBackend); ok { inner = hyb.TextBackend() } sw.Swap(search.NewHybrid(inner, vecBackend, idx.embedder)) if droppedVectors > 0 { idx.logger.Warn("indexer: dropped invalid embedding vectors", zap.Int("dropped", droppedVectors), zap.Int("dimensions", dims), zap.Strings("sample_ids", droppedSample)) } fields := []zap.Field{ zap.Int("vectors", vecBackend.Count()), zap.Int("chunk_vectors", len(chunkMap)), zap.Int("dimensions", dims), zap.Int("dropped", droppedVectors), } // Surface the actual token spend of a paid embedding pass when the // backend reports usage (API providers do; in-process ones don't). // Without this the cost of an embedding run is invisible after the fact. if acc, ok := idx.embedder.(interface{ TokensUsed() int64 }); ok { if tokens := acc.TokensUsed(); tokens > 0 { fields = append(fields, zap.Int64("embed_tokens", tokens)) } } idx.logger.Info("vector index built", fields...) } // dirIgnoreFiles are the per-directory ignore-file basenames honored by // the index walk, siblings to .gitignore: Gortex's own .gortexignore // plus ripgrep's .ignore and .rgignore. Patterns in each file are // scoped to the directory that contains it; later filenames win, so a // directory's .rgignore overrides its .ignore on a conflicting path. var dirIgnoreFiles = []string{".gortexignore", ".ignore", ".rgignore"} // shouldExclude reports whether a path is excluded by the effective // ignore list. The flat matcher is built lazily from idx.config.Exclude, // which is populated by ConfigManager.GetRepoConfig with the full // layered list (builtin + global + RepoEntry + workspace). A path is // also excluded by any per-directory ignore file (dirIgnoreFiles) // present in one of its ancestor directories. isDir lets a trailing- // slash pattern prune a directory subtree instead of only its files. func (idx *Indexer) shouldExclude(path, root string, isDir bool) bool { // .claude/ and .kiro/ are Builtin-excluded wholesale, but may hold an // MCP server config the MCP-config-as-graph feature targets (the // extractor's own docs name .kiro/mcp.json). Descend those subtrees // and index only the MCP config files within them — everything else // stays excluded, so the agent-state noise never reaches the graph. if rel, err := filepath.Rel(root, path); err == nil && excludes.InAgentConfigDir(rel) { if isDir { return false } return !excludes.IsMCPConfigFile(rel) } if m := idx.excludeMatcher(); m != nil && m.MatchAbsDir(path, root, isDir) { return true } return idx.dirIgnoreMatcher(root).Match(path, isDir) } // shouldPruneDir reports whether the index walk may skip a directory // subtree wholesale (filepath.SkipDir) instead of descending it. A // directory is prunable only when it is excluded AND no re-include ("!") // pattern targets anything beneath it. go-gitignore's "*" matches across // "/", so a blanket like "wp-content/plugins/*" reports the parent // directory "wp-content/plugins" itself as excluded; pruning it would // skip a later "!wp-content/plugins/foo/" re-include before the walk ever // reaches the child. Mirroring git, we keep descending such a directory // and let the per-file shouldExclude check filter its contents. func (idx *Indexer) shouldPruneDir(path, root string) bool { if !idx.shouldExclude(path, root, true) { return false } if m := idx.excludeMatcher(); m != nil { if rel, err := filepath.Rel(root, path); err == nil && m.HasNegatedDescendant(filepath.ToSlash(rel)) { return false } } if idx.dirIgnoreMatcher(root).HasNegatedDescendant(path) { return false } return true } // dirIgnoreMatcher returns the per-directory ignore matcher, built lazily // against the repo root the index walk is anchored at. func (idx *Indexer) dirIgnoreMatcher(root string) *excludes.Hierarchical { idx.dirIgnoreOnce.Do(func() { idx.dirIgnore = excludes.NewHierarchical(root, dirIgnoreFiles...) }) return idx.dirIgnore } func (idx *Indexer) excludeMatcher() *excludes.Matcher { idx.excludeOnce.Do(func() { patterns := idx.config.Exclude // A nil/empty list from upstream means "no layering was applied" // (e.g. a direct caller of indexer.New without ConfigManager). // Fall back to the builtin baseline so the walk still skips the // obvious non-source dirs. if len(patterns) == 0 { patterns = excludes.Builtin } idx.excludes = excludes.New(patterns) }) return idx.excludes } // ParseErrors returns the parse errors from the last full index. func (idx *Indexer) ParseErrors() []IndexError { return idx.parseErrors } // FileMtimes returns a copy of the file modification time map. func (idx *Indexer) FileMtimes() map[string]int64 { idx.mtimeMu.RLock() defer idx.mtimeMu.RUnlock() out := make(map[string]int64, len(idx.fileMtimes)) for k, v := range idx.fileMtimes { out[k] = v } return out } // RefreshFileMtime restamps the recorded modification time for a file // from its current on-disk mtime, without re-indexing it. The watcher // calls this when a save turned out to be structurally inert and the // reindex was skipped: the graph is already correct, but the recorded // mtime must advance past the save so the poller's mtime sweep does // not keep re-flagging the same untouched file. A file absent from // disk or never indexed is a no-op. // // The in-memory map is what the current process's poller and // IsStale checks trust, but a warm restart trusts the persisted // FileMtime sidecar instead — without also writing through to it here, // a single inert save during a session left the persisted row at its // pre-save value, so the next restart's HasChangesSinceMtimes saw this // file as changed and re-tracked the whole repo. Mirrors the per-file // indexFile persist (recordFileMtime); a no-op on the in-memory backend. func (idx *Indexer) RefreshFileMtime(filePath string) { absPath, err := filepath.Abs(filePath) if err != nil { return } info, err := os.Stat(absPath) if err != nil { return } // relKey (slash + NFC) so the lookup hits the same fileMtimes // entry the index walk created for a non-ASCII filename. key := idx.relKey(absPath) mtime := info.ModTime().UnixNano() idx.mtimeMu.Lock() _, tracked := idx.fileMtimes[key] if tracked { idx.fileMtimes[key] = mtime } idx.mtimeMu.Unlock() if !tracked { return } if w, ok := idx.graph.(graph.FileMtimeWriter); ok { if err := w.BulkSetFileMtimes(idx.repoPrefix, map[string]int64{key: mtime}); err != nil { idx.logger.Warn("persist file mtime failed", zap.String("repo", idx.repoPrefix), zap.String("file", key), zap.Error(err)) } } } // pruneDeletedFileMtimes drops the persisted mtime rows for files the // incremental reindex just confirmed deleted. The in-memory map is already // pruned by the caller; this keeps the store's FileMtime sidecar in step so // a later warm restart does not re-discover them as phantom deletions and // force a full re-track. A no-op when the backend lacks the capability // (the in-memory backend) or the list is empty. func (idx *Indexer) pruneDeletedFileMtimes(deleted []string) { if len(deleted) == 0 { return } if d, ok := idx.graph.(graph.FileMtimeDeleter); ok { if err := d.DeleteFileMtimes(idx.repoPrefix, deleted); err != nil { idx.logger.Warn("prune deleted file mtimes failed", zap.String("repo", idx.repoPrefix), zap.Error(err)) } } } // SetFileMtimes restores the file modification time map from a persisted snapshot. func (idx *Indexer) SetFileMtimes(mtimes map[string]int64) { idx.mtimeMu.Lock() defer idx.mtimeMu.Unlock() idx.fileMtimes = make(map[string]int64, len(mtimes)) for k, v := range mtimes { idx.fileMtimes[k] = v } } // SetRootPath sets the root path for relative path computation. func (idx *Indexer) SetRootPath(root string) { abs, err := filepath.Abs(root) if err != nil { abs = root } idx.storeRootPath(abs) } // IncrementalReindexPaths re-indexes only the files reachable from the // supplied paths, instead of walking the whole repository root. // // Each path may be absolute or relative to root, and may be a file or a // directory; directories are walked recursively with the same // exclude / language filters as a full pass. Within that scoped file // set the behaviour matches IncrementalReindex: only files that are // stale (mtime or, in Merkle mode, content) are re-indexed, and a file // previously tracked under one of the scoped paths but now absent from // disk is evicted. // // When paths is empty the call degrades to IncrementalReindex(root) — // callers can therefore pass an optional path list unconditionally. func (idx *Indexer) IncrementalReindexPaths(root string, paths []string) (*IndexResult, error) { if len(paths) == 0 { return idx.IncrementalReindex(root) } start := time.Now() absRoot, err := filepath.Abs(root) if err != nil { return nil, err } idx.storeRootPath(absRoot) // scopeRels holds the repo-relative slash-paths the caller asked to // reindex — used both to drive the discovery walk and to bound // deletion detection to the scoped subtree. scopeRels := make(map[string]bool) // diskFiles is the set of in-scope language files currently on // disk; staleFiles is the subset that changed since the last pass. diskFiles := make(map[string]bool) var staleFiles []string merkleMode := idx.merkleEnabled() for _, p := range paths { absPath := p if !filepath.IsAbs(absPath) { absPath = filepath.Join(absRoot, filepath.FromSlash(p)) } absPath = filepath.Clean(absPath) // A path outside the repo root is rejected: scoping is a // narrowing operation, never an escape hatch to index files // the repo doesn't own. rel, relErr := filepath.Rel(absRoot, absPath) if relErr != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { return nil, fmt.Errorf("incremental reindex: path %q is outside repository root %q", p, absRoot) } // Canonical key (slash + NFC) so scopeRels matches fileMtimes // keys when deletion detection intersects the two below. scopeRels[idx.relKey(absPath)] = true info, statErr := os.Stat(absPath) if statErr != nil { // A path that no longer exists is not an error: it may be // a deleted file the caller still wants evicted. Deletion // detection below handles it via scopeRels. if errors.Is(statErr, os.ErrNotExist) { continue } return nil, fmt.Errorf("incremental reindex: stat %q: %w", p, statErr) } if info.IsDir() { walkErr := filepath.WalkDir(absPath, func(path string, d os.DirEntry, err error) error { if err != nil { return nil } if d.IsDir() { if idx.shouldPruneDir(path, absRoot) { return filepath.SkipDir } return nil } if _, ok := idx.effectiveLanguage(path, nil); !ok { return nil } if idx.shouldExclude(path, absRoot, false) { return nil } // relKey (slash + NFC) keeps the disk set keyed // consistently with fileMtimes for non-ASCII names. relPath := idx.relKey(path) diskFiles[relPath] = true if !merkleMode && idx.IsStale(relPath) { staleFiles = append(staleFiles, path) } return nil }) if walkErr != nil { return nil, walkErr } continue } // Single file. Apply the same language / exclude gate so a // caller can't force a non-source or excluded file in. if _, ok := idx.effectiveLanguage(absPath, nil); !ok { continue } if idx.shouldExclude(absPath, absRoot, false) { continue } // relKey (slash + NFC) — same canonical key the graph and // fileMtimes use, so a non-ASCII path passed in here matches // regardless of the Unicode form the caller supplied. relPath := idx.relKey(absPath) diskFiles[relPath] = true if !merkleMode && idx.IsStale(relPath) { staleFiles = append(staleFiles, absPath) } } // In Merkle mode the per-file mtime check is skipped; the stale set // comes from a content-addressed tree diff over the whole repo, // then intersected back down to the requested scope. if merkleMode { for _, abs := range idx.merkleStaleFiles(absRoot, diskFiles) { rel, relErr := filepath.Rel(absRoot, abs) if relErr != nil { continue } if diskFiles[filepath.ToSlash(rel)] { staleFiles = append(staleFiles, abs) } } } // Deletion detection, bounded to the scoped subtree. A file tracked // in fileMtimes that sits under one of the requested paths but is // absent from this scoped discovery walk is a deletion candidate; // the same stat-before-evict guard as IncrementalReindex applies so // a newly-excluded or transiently-unreachable file is preserved. idx.mtimeMu.RLock() var candidates []string for relPath := range idx.fileMtimes { if diskFiles[relPath] { continue } if relPathInScope(relPath, scopeRels) { candidates = append(candidates, relPath) } } idx.mtimeMu.RUnlock() var deletedFiles []string for _, relPath := range candidates { absPath := filepath.Join(absRoot, filepath.FromSlash(relPath)) _, statErr := os.Stat(absPath) if statErr == nil { continue } if errors.Is(statErr, os.ErrNotExist) { deletedFiles = append(deletedFiles, relPath) continue } idx.logger.Warn("incremental reindex: stat failed during scoped deletion detection, preserving", zap.String("rel", relPath), zap.Error(statErr)) } for _, relPath := range deletedFiles { graphPath := idx.prefixPath(relPath) idx.restubIncomingRefs(graphPath) idx.evictEnrichment(graphPath) idx.graph.EvictFile(graphPath) idx.mtimeMu.Lock() delete(idx.fileMtimes, relPath) idx.mtimeMu.Unlock() } // Prune the persisted mtime rows for deleted files too, so the next // warm restart does not see them as phantom deletions (the in-memory // delete above does not reach the store's sidecar table). idx.pruneDeletedFileMtimes(deletedFiles) // Re-index stale files with the same one-shot retry as the // whole-root path — a file locked or mid-write when the walk caught // it gets a second chance before landing on FailedFiles. var failedFiles []string for _, f := range staleFiles { if err := idx.IndexFile(f); err != nil { idx.logger.Debug("incremental reindex: failed to index file", zap.String("file", f), zap.Error(err)) failedFiles = append(failedFiles, f) } } if len(failedFiles) > 0 { retry := failedFiles failedFiles = nil for _, f := range retry { if err := idx.IndexFile(f); err != nil { idx.logger.Warn("incremental reindex: file failed after retry", zap.String("file", f), zap.Error(err)) failedFiles = append(failedFiles, f) } } } // Re-infer interface implementations and re-run stub-call passes — // eviction may have dropped edges. Skipped under deferGlobalPasses // so a batch caller runs one global pass at the end. if !idx.deferGlobalPasses && (len(staleFiles) > 0 || len(deletedFiles) > 0) { // Scoped inference passes re-derive only the affected types/interfaces // (add-parity with the full pass); fall back to whole-graph when // scoping is disabled. if !idx.runScopedInferencePasses(staleFiles) { idx.resolver.InferImplements() idx.resolver.InferOverrides() } // Capability (reads_env / executes_process / accesses_field) and // framework-dispatch synthesis derive from code structure; skip them // when the reconcile touched only non-code files (docs/config) and // removed nothing — they cannot change any edge in that case, and // eviction already handled any deletion. Same idempotent re-derive // RunGlobalGraphPasses runs at full index. if len(deletedFiles) > 0 || idx.staleFilesAffectDerivedEdges(staleFiles) { synthesizeCapabilityEdges(idx.graph) resolver.RunFrameworkSynthesizers(idx.graph) } // Incremental: synthesize external calls only for the reindexed // files (O(edited files)), not a full-graph recompute. resolver.SynthesizeExternalCallsForFiles(idx.graph, idx.externalCallSynthesisEnabled(), idx.graphFilePaths(staleFiles)) } // Skip the search-index rebuild on a zero-change reconcile when the // backend already persists its search structures (the on-disk // backend keeps its FTS index and vector embeddings on disk). // buildSearchIndex re-reads every node (GetRepoNodes) and re-embeds // them, then BulkUpsertEmbeddings re-writes the embedding rows. On a // warm restart that work is pure recompute of already-persisted data. // When nothing changed there is nothing to re-embed, so skip it // entirely — the persisted index is authoritative. The in-memory // backends (BM25 / Bleve) must still rebuild from the replayed // snapshot, so they keep the unconditional path. if len(staleFiles) > 0 || len(deletedFiles) > 0 || !isSymbolSearcherBackend(idx.search) { idx.buildSearchIndex() } if len(staleFiles) > 0 || len(deletedFiles) > 0 { idx.extractContracts() idx.indexGen.Add(1) // files changed — invalidate the trigram cache } nodes, edges := idx.repoNodeEdgeCount() result := &IndexResult{ NodeCount: nodes, EdgeCount: edges, FileCount: len(diskFiles), StaleFileCount: len(staleFiles), DeletedFileCount: len(deletedFiles), FailedFiles: failedFiles, DurationMs: time.Since(start).Milliseconds(), } idx.warnIfEdgeSanityViolated(result) // An incremental pass that re-indexed or evicted at least one file did // work — mark the repo for deferred semantic enrichment. A zero-change // reconcile leaves the marker untouched so an unchanged repo is skipped. if result.StaleFileCount > 0 || result.DeletedFileCount > 0 { idx.pendingEnrich.Store(true) } // A re-parsed stale file's hover-enrichment edges were evicted with its old // nodes, so force the deferred pass past the completion marker (a deletion // evicts nodes but creates no fresh unstamped ones, so it alone does not). if result.StaleFileCount > 0 { idx.reparsedThisRun.Store(true) } return result, nil } // relPathInScope reports whether a repo-relative slash-path falls under // any of the scoped paths — either an exact file match or anywhere // inside a scoped directory. func relPathInScope(relPath string, scope map[string]bool) bool { if scope[relPath] { return true } for s := range scope { if s == "." { return true } if strings.HasPrefix(relPath, s+"/") { return true } } return false } // IncrementalReindex walks the file tree and re-indexes only files that changed // since the last snapshot. It also evicts nodes for deleted files. func (idx *Indexer) IncrementalReindex(root string) (*IndexResult, error) { start := time.Now() absRoot, err := filepath.Abs(root) if err != nil { return nil, err } idx.storeRootPath(absRoot) // Collect files currently on disk. diskFiles := make(map[string]bool) var staleFiles []string // Merkle mode replaces per-file mtime staleness with a // content-addressed Merkle-tree diff computed after the walk. merkleMode := idx.merkleEnabled() err = filepath.WalkDir(absRoot, func(path string, d os.DirEntry, err error) error { if err != nil { return nil } if d.IsDir() { if idx.shouldPruneDir(path, absRoot) { return filepath.SkipDir } return nil } if _, ok := idx.effectiveLanguage(path, nil); !ok { return nil } if idx.shouldExclude(path, absRoot, false) { return nil } // relKey (slash + NFC) so the disk set is keyed identically // to fileMtimes — otherwise a non-ASCII file the snapshot // stored under one Unicode form and this walk observes under // another would be seen as both deleted and newly created. relPath := idx.relKey(path) diskFiles[relPath] = true if !merkleMode && idx.IsStale(relPath) { staleFiles = append(staleFiles, path) } return nil }) if err != nil { return nil, err } // In Merkle mode the per-file mtime check above is skipped; the // stale set comes from a content-addressed tree diff instead. if merkleMode { staleFiles = idx.merkleStaleFiles(absRoot, diskFiles) } // Detect deleted files. A file that's tracked in fileMtimes but // absent from the current discovery walk is a candidate, but // "absent from discovery" is not the same as "absent from disk": // // - The exclude list (.gortex.yaml, builtin, workspace) may have // grown since the last index — every newly-excluded file would // be classified as deleted. // - A language extractor's Extensions() may have changed across // versions — files whose ext is no longer detected would be // classified as deleted. // - WalkDir swallowed a transient error (EACCES, EIO, NFS hiccup, // ELOOP) — the file is unreachable this pass but still on disk. // // All three would purge legitimate graph state on every daemon // restart. Stat the candidate first: only treat ENOENT/ENOTDIR as // deletion; preserve on success (file exists, just not discovered) // and on transient errors. The cost is one extra stat per // previously-indexed-but-not-discovered file, which is bounded by // the size of the exclusion delta. idx.mtimeMu.RLock() var candidates []string for relPath := range idx.fileMtimes { if !diskFiles[relPath] { candidates = append(candidates, relPath) } } idx.mtimeMu.RUnlock() var deletedFiles []string for _, relPath := range candidates { absPath := filepath.Join(absRoot, relPath) _, err := os.Stat(absPath) if err == nil { // File exists on disk; it was excluded or its extension is // no longer detected. Preserve. continue } if errors.Is(err, os.ErrNotExist) { deletedFiles = append(deletedFiles, relPath) continue } // Transient error — preserve to be safe. idx.logger.Warn("incremental reindex: stat failed during deletion detection, preserving", zap.String("rel", relPath), zap.Error(err)) } // Evict only files that are truly absent from disk. for _, relPath := range deletedFiles { graphPath := idx.prefixPath(relPath) idx.restubIncomingRefs(graphPath) idx.evictEnrichment(graphPath) idx.graph.EvictFile(graphPath) idx.mtimeMu.Lock() delete(idx.fileMtimes, relPath) idx.mtimeMu.Unlock() } // Prune the persisted mtime rows for deleted files too, so the next // warm restart does not see them as phantom deletions (the in-memory // delete above does not reach the store's sidecar table). idx.pruneDeletedFileMtimes(deletedFiles) // Re-index stale files. A file that fails — most often because it // was locked or mid-write when the walk caught it — is collected // and retried once below. A failure that survives the retry is // surfaced on IndexResult.FailedFiles so the caller can replay it. // A file whose bytes couldn't be read at all keeps no mtime // recorded, so it stays stale for the next incremental pass; a // file that read but failed to parse gets its mtime recorded (see // indexFile's result==nil branch), so it stops being retried until // its content changes again — see IndexResult.FailedFiles. var failedFiles []string for _, f := range staleFiles { if err := idx.IndexFile(f); err != nil { idx.logger.Debug("incremental reindex: failed to index file", zap.String("file", f), zap.Error(err)) failedFiles = append(failedFiles, f) } } if len(failedFiles) > 0 { retry := failedFiles failedFiles = nil for _, f := range retry { if err := idx.IndexFile(f); err != nil { idx.logger.Warn("incremental reindex: file failed after retry", zap.String("file", f), zap.Error(err)) failedFiles = append(failedFiles, f) } } } // Re-infer interface implementations (edges may have been lost // during eviction). Skipped under deferGlobalPasses so a batch // caller (ReconcileAll, warmup) can run a single global pass at // the end instead of paying O(global) per repo. if !idx.deferGlobalPasses { // Scoped inference passes re-derive only the affected types/interfaces // (add-parity with the full pass); fall back to whole-graph when off. if !idx.runScopedInferencePasses(staleFiles) { idx.resolver.InferImplements() idx.resolver.InferOverrides() } // Capability edges (reads_env / executes_process / accesses_field) // and framework dynamic-dispatch synthesis are whole-graph recomputes // that derive edges only from structural nodes in the changed files. // When nothing structural changed — no stale code file (a doc/config // edit or a true zero-change reconcile) and no deletion — they can // produce no new edge, so re-running them is pure cost: the dominant // waste in a no-op IncrementalReindex, which the periodic janitor runs // per repo per tick. Gate them on the same predicate the path-scoped // IncrementalReindexPaths already uses. Deletions still trigger a // re-run because an evicted file may have been a dispatch endpoint. if len(deletedFiles) > 0 || idx.staleFilesAffectDerivedEdges(staleFiles) { synthesizeCapabilityEdges(idx.graph) resolver.RunFrameworkSynthesizers(idx.graph) } // External-call synthesis (opt-in) — file-scoped to the reindexed // files (O(edited files)), not a full-graph recompute. Eviction // already dropped a removed file's synthetic edges; a re-indexed // file's fresh external terminals are re-materialised here. resolver.SynthesizeExternalCallsForFiles(idx.graph, idx.externalCallSynthesisEnabled(), idx.graphFilePaths(staleFiles)) // Clone detection is not re-run here: each stale file was // re-indexed through IndexFile above, whose resolve pass // already recomputed EdgeSimilarTo against the fresh graph, // and deleted files self-clean via EvictFile's bidirectional // edge removal. Under deferGlobalPasses the batch caller runs // the global clone pass once at the end. } // Rebuild search index to ensure consistency — but skip it on a // zero-change reconcile against a backend that persists its search // structures natively (the on-disk backend). See the matching guard // in the other incremental path: re-embedding is wasted work and // there is nothing to rebuild when no file changed. if len(staleFiles) > 0 || len(deletedFiles) > 0 || !isSymbolSearcherBackend(idx.search) { idx.buildSearchIndex() } // Update totalDetected so index_health reports correctly after cache restore. if idx.totalDetected == 0 { idx.totalDetected = len(diskFiles) } // Re-extract contracts only if stale files were re-indexed. if len(staleFiles) > 0 || len(deletedFiles) > 0 { idx.extractContracts() idx.indexGen.Add(1) // files changed — invalidate the trigram cache } nodes, edges := idx.repoNodeEdgeCount() result := &IndexResult{ NodeCount: nodes, EdgeCount: edges, FileCount: len(diskFiles), StaleFileCount: len(staleFiles), DeletedFileCount: len(deletedFiles), FailedFiles: failedFiles, DurationMs: time.Since(start).Milliseconds(), } idx.warnIfEdgeSanityViolated(result) // An incremental pass that re-indexed or evicted at least one file did // work — mark the repo for deferred semantic enrichment. A zero-change // reconcile leaves the marker untouched so an unchanged repo is skipped. if result.StaleFileCount > 0 || result.DeletedFileCount > 0 { idx.pendingEnrich.Store(true) } // A re-parsed stale file's hover-enrichment edges were evicted with its old // nodes, so force the deferred pass past the completion marker (a deletion // evicts nodes but creates no fresh unstamped ones, so it alone does not). if result.StaleFileCount > 0 { idx.reparsedThisRun.Store(true) } return result, nil } // LastIndexTime returns the timestamp of the last full index. func (idx *Indexer) LastIndexTime() time.Time { return idx.lastIndexTime } // TotalDetected returns the total number of files detected during the last full index. func (idx *Indexer) TotalDetected() int { return idx.totalDetected } // buildPerFileContractExtractors returns the set of extractors that // operate on a single source file (everything except GoModExtractor, // which runs once against go.mod at the repo root) plus a language → // [extractors] map so callers can skip extractors whose // SupportedLanguages() doesn't include a given file's language. // Building the language map once avoids doing the string-membership // check per file. func (idx *Indexer) buildPerFileContractExtractors() ([]contracts.Extractor, map[string][]contracts.Extractor) { extractors := []contracts.Extractor{ &contracts.HTTPExtractor{ClientAliases: idx.config.HTTPClientAliases}, &contracts.GRPCExtractor{}, &contracts.ThriftExtractor{}, &contracts.GraphQLExtractor{}, &contracts.TRPCExtractor{}, &contracts.OpenAPIExtractor{}, &contracts.TopicExtractor{}, &contracts.WebSocketExtractor{}, &contracts.NestMicroserviceExtractor{}, &contracts.EnvVarExtractor{}, } // Config-driven event bus: only registered when the user declared // boundaries (index.event_bus / CODEGRAPH_EVENT_CONFIG), so the default // extractor set is unchanged. if b := idx.eventBusBoundaries(); len(b) > 0 { extractors = append(extractors, &contracts.EventBusExtractor{Boundaries: b}) } byLang := make(map[string][]contracts.Extractor) for _, ex := range extractors { for _, lang := range ex.SupportedLanguages() { byLang[lang] = append(byLang[lang], ex) } } return extractors, byLang } // runContractExtractorsForFile applies the given extractors to a single // file and returns the raw contracts (with RepoPrefix already set). // Called both inline from parse workers and from the full-walk // extractContracts path — they share the same per-file work. func (idx *Indexer) runContractExtractorsForFile( graphPath string, src []byte, fileNodes []*graph.Node, fileEdges []*graph.Edge, exts []contracts.Extractor, tree *parser.ParseTree, ) []contracts.Contract { if len(exts) == 0 { return nil } // Contracts from synthetic test/bench fixtures are kept (so drift // checks can flag a stale test pinned to an obsolete production // contract) but tagged with is_test=true and a test_source // category so the dashboard can filter them out by default. testSource := fixtures.TestContractSource(graphPath) var out []contracts.Contract // The graph store backs graph-wide constant resolution for endpoint // arguments (a route path / queue / topic referenced by a const). Resolved // once per call; nil when the backend can't satisfy the reader (const // dereference is then disabled and store-aware extractors degrade to their // tree-aware behaviour). var endpointStore contracts.EndpointConstStore if es, ok := idx.graph.(contracts.EndpointConstStore); ok { endpointStore = es } for _, ex := range exts { var found []contracts.Contract if sae, ok := ex.(contracts.StoreAwareExtractor); ok { found = sae.ExtractWithStore(graphPath, src, fileNodes, fileEdges, tree, endpointStore, idx.repoPrefix) } else if tae, ok := ex.(contracts.TreeAwareExtractor); ok && tree != nil { found = tae.ExtractWithTree(graphPath, src, fileNodes, fileEdges, tree) } else { found = ex.Extract(graphPath, src, fileNodes, fileEdges) } for i := range found { found[i].RepoPrefix = idx.repoPrefix // Stamp the workspace / project slugs alongside the repo // prefix so the matcher's boundary check has the data it // needs without a second registry walk. Empty slugs // default to RepoPrefix at Match time via // Contract.EffectiveWorkspace / EffectiveProject. if idx.workspaceID != "" { found[i].WorkspaceID = idx.workspaceID } if idx.projectID != "" { found[i].ProjectID = idx.projectID } if testSource != "" { if found[i].Meta == nil { found[i].Meta = map[string]any{} } found[i].Meta["is_test"] = true found[i].Meta["test_source"] = testSource } } out = append(out, found...) } return out } // commitContracts writes contract nodes + provides/consumes edges for // every contract in reg, and sets idx.contractRegistry to reg. Called // once per index pass after all per-file contracts have been collected // (inline from parse workers) plus go.mod has been processed. func (idx *Indexer) commitContracts(reg *contracts.Registry) { // Upgrade bare type names in contract Meta (e.g. "UserResp") to // full symbol IDs (e.g. "pkg/resp.go::UserResp") now that the // graph is complete. During extraction the enricher only saw // the handler's file-scoped node list, so types declared in a // sibling file stayed as bare names. reg.UpgradeBareTypeRefs(func(name, repoHint string) []string { matches := idx.graph.FindNodesByName(name) var same, others []string for _, n := range matches { if n.Kind != graph.KindType { continue } if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") { same = append(same, n.ID) continue } others = append(others, n.ID) } if len(same) > 0 { return same } return others }) // Cross-file handler resolution. When a route is registered with // a handler identifier that the file-scoped extractor couldn't // resolve (`h.ServeArchive` in router.go wiring a method defined // in archive_handler.go), the contract's SymbolID fell back to // the enclosing router function and schema extraction ran // against the router's body — which has every route's bindings // piled on top of each other. Re-run enrichment with the // correct per-handler scope now that the graph is complete. idx.resolveProviderHandlers(reg) // Cross-file route-prefix joining. A FastAPI APIRouter declared in // one file (`router = APIRouter(prefix="/users")`) is mounted under // a second prefix elsewhere (`app.include_router(router, // prefix="/api")`), so a route declared `@router.get("/{id}")` // belongs at /api/users/{id}, not /{id}. Rewrite the affected // provider contract IDs to the joined path before the matcher pairs // them with consumers. Also handles Express app.use mounts and // NestJS @Controller class prefixes. Reads source straight off disk // (cached per file) — the same access pattern resolveProviderHandlers // uses for cross-file handler bodies. // // Mount sites (a main.py that only calls include_router) often carry // no route contracts, so the scan-file set comes from the graph's // py/ts/js file nodes, not the registry — but only when at least one // prefix-eligible route contract exists, so non-FastAPI/Express/Nest // repos pay nothing. if scanFiles := idx.routerPrefixScanFiles(reg); len(scanFiles) > 0 { srcCache := make(map[string][]byte) contracts.JoinRouterPrefixes(reg, scanFiles, func(filePath string) []byte { if data, ok := srcCache[filePath]; ok { return data } data := idx.contractFileSrc(filePath) srcCache[filePath] = data return data }) } // Spring application(-profile)?.{yml,properties} config-key graph: emit the // value-redacted config-key nodes and reads_config edges from the @Value / // @ConfigurationProperties beans the Java extractor stamped. Cheap to skip // on non-Spring repos (no config files + no stamped beans = no work). contracts.BindSpringConfig(idx.graph, idx.contractFileSrc) // Trace response variables back to their call-site return types. // Handles `source, err := h.svc.Get(...)` → response_type is // whatever `h.svc.Get` returns. The enricher can't do this // without graph access; this pass reads each method's signature // directly off the graph node, parses the first non-error // return type, and resolves it to a symbol ID. idx.resolveCallReturnTypes(reg) // Snapshot field-level shapes for every type that's referenced as // a contract's request / response body. This is Stage 2 — without // per-field data Stage 3 (validation, breaking-change detection) // has nothing to diff. We de-duplicate by symbol ID so heavy // fan-in types (a User DTO used by 40 routes) only get parsed // once per index pass. idx.snapshotContractShapes(reg) // Fold each type's snapshotted Shape into the envelope rows that // reference it. The dashboard renders these rows as the response // JSON shape (e.g. `{ workspace: { id: string }, repos: [{ name: string }] }`) // instead of the bare type-symbol-ID, which answers nothing about // the wire format. idx.inlineEnvelopeShapes(reg) all := reg.All() nodes := make([]*graph.Node, 0, len(all)) edges := make([]*graph.Edge, 0, len(all)) for _, c := range all { // dep:: nodes were materialised by extractGoModContracts // before ResolveAll (so the import bridge could find them); // re-emitting them here would PK-collide on backends whose bulk // load is INSERT-only (the on-disk backend). The pre-pass is the single // writer for that contract type. if c.Type == contracts.ContractDependency { continue } nodes = append(nodes, &graph.Node{ ID: c.ID, Kind: graph.KindContract, Name: c.ID, FilePath: c.FilePath, Language: "contract", RepoPrefix: c.RepoPrefix, WorkspaceID: c.EffectiveWorkspace(), ProjectID: c.EffectiveProject(), Meta: map[string]any{ "type": string(c.Type), "role": string(c.Role), "symbol_id": c.SymbolID, "line": c.Line, "confidence": c.Confidence, "contract_meta": c.Meta, }, }) if c.SymbolID == "" { continue } edgeKind := graph.EdgeProvides if c.Role == contracts.RoleConsumer { edgeKind = graph.EdgeConsumes } edges = append(edges, &graph.Edge{ From: c.SymbolID, To: c.ID, Kind: edgeKind, FilePath: c.FilePath, Line: c.Line, }) // Framework-layer EdgeHandlesRoute. Emitted alongside // EdgeProvides for HTTP / gRPC / WS / GraphQL / topic // providers so `analyze kind=routes` and other // framework-aware tools walk one targeted edge instead // of filtering EdgeProvides by contract type. Consumers // (callers of routes) and non-route contract types (env, // OpenAPI specs, DI tokens) intentionally skip this // edge — they aren't route handlers. if c.Role == contracts.RoleProvider && isRouteContractType(c.Type) { edges = append(edges, &graph.Edge{ From: c.SymbolID, To: c.ID, Kind: graph.EdgeHandlesRoute, FilePath: c.FilePath, Line: c.Line, Meta: map[string]any{ "contract_type": string(c.Type), }, }) } } bulkStart := time.Now() idx.bulkCommit(nodes, edges) bulkElapsed := time.Since(bulkStart) idx.contractRegistry = reg repo := idx.rootPath if idx.repoPrefix != "" { repo = idx.repoPrefix } idx.logger.Info("contracts extracted", zap.String("repo", repo), zap.Int("count", len(all)), zap.Duration("commit_bulk_elapsed", bulkElapsed)) } // bulkCommit writes nodes + edges in one AddBatch call. The bulk // load path is intentionally NOT used here: contract IDs often // coincide with existing source-symbol IDs (a route handler shows // up as both a Go function and an HTTP-contract anchor), and the // on-disk backend's bulk load is INSERT-only on the node table so // any collision fails the whole batch. AddBatch's non-bulk path // upserts every row so duplicates are absorbed in place; the // per-call cost is amortised by the chunked write path the backend // uses internally. func (idx *Indexer) bulkCommit(nodes []*graph.Node, edges []*graph.Edge) { if len(nodes) == 0 && len(edges) == 0 { return } idx.graph.AddBatch(nodes, edges) } // routerPrefixScanFiles returns the set of source files // JoinRouterPrefixes must scan for router definitions and mount sites // (APIRouter / include_router / app.use). Returns nil when no HTTP // contract uses a prefix-joining framework (FastAPI / Express / NestJS), // so unrelated repos skip the file enumeration entirely. When eligible, // it enumerates py / ts / js / tsx / jsx file nodes from the graph — the // mount file (a FastAPI main.py) frequently has no route contract of its // own and so can't be discovered from the registry alone. func (idx *Indexer) routerPrefixScanFiles(reg *contracts.Registry) []string { eligible := false for _, c := range reg.All() { if c.Type != contracts.ContractHTTP || c.Meta == nil { continue } switch fw, _ := c.Meta["framework"].(string); fw { case "fastapi/flask", "express", "nestjs": eligible = true } if eligible { break } } if !eligible { return nil } var out []string for _, n := range idx.graph.AllNodes() { if n.Kind != graph.KindFile { continue } path := n.FilePath if path == "" { path = n.ID } switch { case strings.HasSuffix(path, ".py"), strings.HasSuffix(path, ".ts"), strings.HasSuffix(path, ".tsx"), strings.HasSuffix(path, ".js"), strings.HasSuffix(path, ".jsx"): out = append(out, path) } } return out } // contractFileSrc reads the on-disk source for a contract FilePath // (which is repo-prefixed when the indexer uses a repo prefix). Returns // nil when the file can't be read. Mirrors the disk-resolution logic in // resolveProviderHandlers so cross-file passes share one access pattern. func (idx *Indexer) contractFileSrc(filePath string) []byte { diskPath := filePath if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") { diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/") } diskPath = filepath.Join(idx.rootPath, diskPath) data, err := os.ReadFile(diskPath) if err != nil { return nil } return data } // isRouteContractType reports whether a ContractType corresponds to a // real network-route handler (HTTP / gRPC / WebSocket / GraphQL / // topic). Used to gate EdgeHandlesRoute emission so the framework-layer // edge stays focused on actual handlers and excludes env / OpenAPI / // dependency / DI-token contracts that share the EdgeProvides edge but // aren't routes in the agent-asks-which-handler-serves-X sense. func isRouteContractType(t contracts.ContractType) bool { switch t { case contracts.ContractHTTP, contracts.ContractGRPC, contracts.ContractThrift, contracts.ContractGraphQL, contracts.ContractTopic, contracts.ContractWS: return true } return false } // resolveProviderHandlers finds the actual handler for every HTTP // provider contract whose per-file extraction couldn't resolve the // handler identifier (typically routers in one file wiring handlers // defined in sibling files). For each such contract: // // - Take Meta["handler_trail"] — the full expression between the // HandleFunc parens, which carries every handler candidate // (wrappers + inner handler). Fall back to "handler_ident" // when no trail was captured (older contracts, simple consumer // patterns). // - Enumerate candidates in source order and look each up in the // graph; take the innermost (last) one that resolves. That // picks h.ServeArchive out of WithAuth(h.ServeArchive) instead // of the WithAuth wrapper. // - Re-run EnrichHTTPContract against the handler's file with the // handler's line range so the enricher sees its actual body // instead of the router's. // - Drop `handler_ident` / `handler_trail` from meta afterwards — // they were internal resolution hints. func (idx *Indexer) resolveProviderHandlers(reg *contracts.Registry) { type pending struct { contractID string trail string fallback string repoHint string // srcDir is the directory of the contract's registration site // (the file with the HandleFunc call). Used by lookupHandler // as a tie-breaker when two same-repo functions share a name // across packages — e.g. `Handler.handleContracts` in the // `server` pkg vs `Server.handleContracts` in `mcp`. A // `recv.method` call from inside `server/handler.go` resolves // to the same-package method, not the cross-package one. srcDir string } var todo []pending for _, c := range reg.All() { if c.Role != contracts.RoleProvider || c.Type != contracts.ContractHTTP { continue } trail, _ := c.Meta["handler_trail"].(string) fallback, _ := c.Meta["handler_ident"].(string) if trail == "" && fallback == "" { continue } // Skip contracts where schema is already populated — the // initial file-scoped pass worked. if src, _ := c.Meta["schema_source"].(string); src == "extracted" || src == "partial" { continue } todo = append(todo, pending{ contractID: c.ID, trail: trail, fallback: fallback, repoHint: c.RepoPrefix, srcDir: filepath.Dir(c.FilePath), }) } // Always strip the internal handler hints from Meta at the end of // this pass — successful or not. They were only ever intended as // per-pass resolution scratchpad: when the cross-file lookup // succeeds we delete them in the patched-contract loop below; when // it fails (no candidate, ambiguous, etc.) they used to leak to // the dashboard as values like `handler_trail: "/users", listUsers` // — useless to a reader. This cleanup runs unconditionally so // downstream consumers never see internal extractor state. defer func() { for _, c := range reg.All() { if c.Meta == nil { continue } if _, hasIdent := c.Meta["handler_ident"]; !hasIdent { if _, hasTrail := c.Meta["handler_trail"]; !hasTrail { continue } } items := reg.ByID(c.ID) for i := range items { if items[i].Meta == nil { continue } delete(items[i].Meta, "handler_ident") delete(items[i].Meta, "handler_trail") } reg.ReplaceByID(c.ID, items) } }() if len(todo) == 0 { return } // Cache file source + node list per file path — a single router // often refers to dozens of handlers in the same sibling file. fileSrc := make(map[string][]byte) fileNodes := make(map[string][]*graph.Node) // fileTrees caches per-file ParseTree handles parsed lazily below // so a router referencing many handlers in the same sibling file // only parses that file once. fileTrees := make(map[string]*parser.ParseTree) defer func() { for _, t := range fileTrees { t.Release() } }() resolved := 0 for _, p := range todo { handlerNode := idx.resolveInnermostHandler(p.trail, p.fallback, p.repoHint, p.srcDir) if handlerNode == nil { continue } src, ok := fileSrc[handlerNode.FilePath] if !ok { diskPath := handlerNode.FilePath if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") { diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/") } diskPath = filepath.Join(idx.rootPath, diskPath) data, err := os.ReadFile(diskPath) if err != nil { fileSrc[handlerNode.FilePath] = nil continue } fileSrc[handlerNode.FilePath] = data src = data } if src == nil { continue } nodes, ok := fileNodes[handlerNode.FilePath] if !ok { nodes = idx.graph.GetFileNodes(handlerNode.FilePath) fileNodes[handlerNode.FilePath] = nodes } lang := detectLangFromPath(handlerNode.FilePath) tree, treeReady := fileTrees[handlerNode.FilePath] if !treeReady { tree = contracts.ParseTreeForLang(lang, src) fileTrees[handlerNode.FilePath] = tree } // Re-run enrichment. EnrichHTTPContractWithTree reads the // contract's SymbolID to locate the handler body range — swap // it in temporarily to the resolved handler so the lookup // works. With a tree the AST overlay runs after the regex // pass and overrides Meta keys it can confidently produce. matches := reg.ByID(p.contractID) if len(matches) == 0 { continue } for i, c := range matches { if c.Role != contracts.RoleProvider { continue } // Operate on a copy; Registry entries are values. patched := c patched.SymbolID = handlerNode.ID patched.FilePath = handlerNode.FilePath if patched.Meta == nil { patched.Meta = map[string]any{} } // Drop prior path_params so the enricher's fresh pass // repopulates consistently (path hasn't changed, but we // want the call-path to be identical to Stage 1). lines := splitLines(src) contracts.EnrichHTTPContractWithTree(&patched, lines, nodes, lang, tree) delete(patched.Meta, "handler_ident") delete(patched.Meta, "handler_trail") matches[i] = patched resolved++ } // Write back the mutated set. The registry doesn't have an // "update" API; we use AddAll semantics via Set-like // operations. Simpler: clear then re-add all roles to this ID. reg.ReplaceByID(p.contractID, matches) } if resolved > 0 { idx.logger.Info("resolved cross-file provider handlers", zap.Int("count", resolved), zap.Int("considered", len(todo))) } } // resolveInnermostHandler picks the innermost handler candidate from // the call trail that resolves to a real function or method in the // graph. Walks candidates in source order and keeps the LAST // successful lookup — for `WithAuth(h.ServeArchive)` that's // `h.ServeArchive`, not the `WithAuth` wrapper. Falls back to the // single identifier when no trail is available (e.g. simple bare // `r.GET("/x", listUsers)` patterns). func (idx *Indexer) resolveInnermostHandler(trail, fallback, repoHint, srcDir string) *graph.Node { candidates := contracts.HandlerCandidatesInTrail(trail) if len(candidates) == 0 && fallback != "" { candidates = []string{fallback} } var best *graph.Node for _, c := range candidates { if n := idx.lookupHandler(c, repoHint, srcDir); n != nil { best = n } } return best } // lookupHandler maps a raw identifier from a route pattern to the // graph node for the handler function / method. // // - "h.ServeArchive" → method named "ServeArchive", prefer same repo. // - "ServeArchive" → function or method of that name. // - "pkg.Foo" → same as first form, package-qualified call. // // Returns nil when no candidate resolves unambiguously. func (idx *Indexer) lookupHandler(ident, repoHint, srcDir string) *graph.Node { // Strip a leading receiver / package qualifier — "h.ServeArchive" // → "ServeArchive". name := ident if i := strings.LastIndex(name, "."); i >= 0 { name = name[i+1:] } if name == "" { return nil } candidates := idx.graph.FindNodesByName(name) if len(candidates) == 0 { return nil } var sameRepo, other []*graph.Node for _, n := range candidates { if n.Kind != graph.KindFunction && n.Kind != graph.KindMethod { continue } if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") { sameRepo = append(sameRepo, n) continue } other = append(other, n) } if len(sameRepo) == 1 { return sameRepo[0] } if len(sameRepo) == 0 && len(other) == 1 { return other[0] } // Multiple candidates — try same-package tie-break before giving up. // A `recv.method` call inside `pkg/foo.go` resolves to a method // declared in the same package; cross-package lookalikes (e.g. // `Server.handleContracts` in `mcp` vs `Handler.handleContracts` // in `server`) are filtered out. Without this, both routers and // MCP-side handlers compete for the same name and the resolver // falls back to the enclosing function (`registerRoutes`). if srcDir != "" { pool := sameRepo if len(pool) == 0 { pool = other } var samePkg []*graph.Node for _, n := range pool { if filepath.Dir(n.FilePath) == srcDir { samePkg = append(samePkg, n) } } if len(samePkg) == 1 { return samePkg[0] } } return nil // ambiguous } func splitLines(src []byte) []string { return strings.Split(string(src), "\n") } // detectLangFromPath mirrors internal/contracts.detectLanguage so the // enricher's language-gate fires correctly for the handler's own file. func detectLangFromPath(path string) string { switch { case strings.HasSuffix(path, ".go"): return "go" case strings.HasSuffix(path, ".ts"), strings.HasSuffix(path, ".tsx"): return "typescript" case strings.HasSuffix(path, ".js"), strings.HasSuffix(path, ".jsx"): return "javascript" case strings.HasSuffix(path, ".py"): return "python" case strings.HasSuffix(path, ".java"): return "java" case strings.HasSuffix(path, ".kt"), strings.HasSuffix(path, ".kts"): return "kotlin" case strings.HasSuffix(path, ".dart"): return "dart" } return "" } // responseHelperCallRe pulls the third argument out of a JSON-response // helper call, e.g. `respondJSON(w, http.StatusOK, source)` → "source", // `WriteJSON(w, 200, &result)` → "result". Matches every helper name // the Go enricher knows about so the two pipes stay in sync. var responseHelperCallRe = regexp.MustCompile( `(?:[A-Za-z_]\w*\.)?(?:[Rr]espond|[Ww]rite|[Ss]end|[Rr]ender)(?:JSON|Json)\(\s*\w+\s*,\s*[^,]+?\s*,\s*&?([A-Za-z_]\w*)\s*\)`, ) // receiverMatchesHint decides whether a method node could plausibly // be the target of a call whose receiver chain includes `hint` as // its penultimate segment. For `h.tucks.Update`, the hint is // "tucks" and we accept receivers whose name (stripped of pointer // marker) contains "tucks" case-insensitively: // // *TucksStore.Update ✓ (receiver "TucksStore" contains "tucks") // *PostgresTuckStore.Update ✓ (contains "tuck") // *EmailSources.Update ✗ (no "tucks") // // The hint may itself be the receiver variable (`h` in `h.Update(...)`) // when the call has only two segments; in that case any same-repo // method named `Update` passes — but the upstream `len(matches) != 1` // check still demands uniqueness, which is the real guard. func receiverMatchesHint(n *graph.Node, hint string) bool { if hint == "" { return true } // Method ID looks like "/::Receiver.Method". Extract // "Receiver" by splitting once on `::` then taking the type part // before the last `.`. idParts := strings.Split(n.ID, "::") if len(idParts) < 2 { return true // conservative: no receiver info available → don't filter out } last := idParts[len(idParts)-1] dot := strings.LastIndex(last, ".") if dot < 0 { return true // plain function, no receiver } recv := strings.TrimPrefix(last[:dot], "*") // Handle both singular and plural forms: "tucks" in hint matches // "TuckStore" by containing "tuck", and "tuck" hint matches // "Tucks" too. Strip a trailing `s` from the longer side to let // singular/plural pairs match. return strings.Contains(strings.ToLower(recv), strings.ToLower(hint)) || strings.Contains(strings.ToLower(recv), strings.ToLower(strings.TrimSuffix(hint, "s"))) || strings.Contains(strings.ToLower(recv), strings.ToLower(hint+"s")) } // parseFirstNonErrorReturnType walks a Go function signature and // returns the first return type that isn't `error`. Signatures as // stored in the graph have the form: // // func ((s *Store)) Get(args) (*EmailSource, error) // func list() []*User // func save(x Foo) error // // Regex-based extraction struggles with the receiver's `((...))` // nesting and with multi-paren return groups — we parse with an // explicit bracket-depth counter so every shape above is handled // the same way. // resolveCallReturnTypes is the graph-aware companion to the // regex-based schema enricher. For every HTTP provider contract whose // response couldn't be pinned syntactically (response_expr is set, // response_type is empty), we: // // - Pull the bound variable's name out of the helper-call expression. // - Read the handler's body from disk. // - Find the variable's declaration line and parse the RHS call. // - Look up the called method by name in the graph (preferring the // same file / same repo). // - Parse the method's signature meta for the first non-error // return type, strip `*` / `[]`, resolve to a type node's ID. // - Patch the contract's meta in place. // // This is the proper tracing the name-based heuristic only // approximated — it follows the variable to its definition instead // of guessing from its name. func (idx *Indexer) resolveCallReturnTypes(reg *contracts.Registry) { resolved := 0 bfCache := newBodyFactsCache(idx) defer bfCache.Close() for _, c := range reg.All() { if c.Role != contracts.RoleProvider || c.Type != contracts.ContractHTTP { continue } handler := idx.graph.GetNode(c.SymbolID) bf := bfCache.For(handler) // Path 1: bare-variable response (`return WriteJSON(w, code, resp)`). // Trace the variable to its binding call's return type — and, // failing that, the literal/builtin shape of its declaration — // then stamp response_type / response_repeated. Accepts // response_expr in two forms: // - Bare identifier ("result") — emitted by the // AST overlay (or the post-fix Go enricher) when the value // is a plain var. // - Full helper call ("WriteJSON(w, …)") — older // extraction output, kept compatible by extracting the // third arg via responseHelperCallRe. if rt, _ := c.Meta["response_type"].(string); rt == "" { respExpr, _ := c.Meta["response_expr"].(string) varName := "" switch { case respExpr == "": // nothing to work with case isLikelyIdentifier(respExpr): varName = respExpr default: if m := responseHelperCallRe.FindStringSubmatch(respExpr); len(m) >= 2 { varName = m[1] } } if varName != "" { typeID, repeated := idx.lookupVarTypeForContract(c, bf, varName) if typeID != "" { items := reg.ByID(c.ID) changed := false for i := range items { if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID { continue } if items[i].Meta == nil { items[i].Meta = map[string]any{} } items[i].Meta["response_type"] = typeID if repeated { items[i].Meta["response_repeated"] = true } items[i].Meta["schema_source"] = "extracted" delete(items[i].Meta, "response_expr") changed = true } if changed { reg.ReplaceByID(c.ID, items) resolved++ } } } } // Path 2: envelope response (`map[string]any{"workspace": ws, // "repos": repos}`). For each row that didn't resolve a type // syntactically, trace its expression to a binding call's // return type and patch the row in place. Pulled out as a // separate pass so a contract whose top-level response_type // stays unresolvable can still get per-field signal — which is // the whole point of the envelope view. envRaw, ok := c.Meta["response_envelope"].([]map[string]any) if !ok { continue } if !envelopeNeedsResolution(envRaw) { continue } envChanged := false for ri := range envRaw { if t, _ := envRaw[ri]["type"].(string); t != "" { continue } expr, _ := envRaw[ri]["expr"].(string) if expr == "" { continue } // Strip a leading `&` / `*` so the binding lookup sees // the underlying identifier. ident := strings.TrimLeft(expr, "&*") if !isLikelyIdentifier(ident) { continue } typeID, repeated := idx.lookupVarTypeForContract(c, bf, ident) if typeID != "" { envRaw[ri]["type"] = typeID if repeated { envRaw[ri]["repeated"] = true } envChanged = true } } if !envChanged { continue } // Promote schema_source to "extracted" if every row now has a // type (or this is a single-key envelope whose lone field // resolved). Otherwise leave it as "partial" — we have more // info than before but it's not exhaustive. items := reg.ByID(c.ID) patched := false for i := range items { if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID { continue } if items[i].Meta == nil { items[i].Meta = map[string]any{} } items[i].Meta["response_envelope"] = envRaw if envelopeFullyTyped(envRaw) { items[i].Meta["schema_source"] = "extracted" } patched = true } if patched { reg.ReplaceByID(c.ID, items) resolved++ } } if resolved > 0 { idx.logger.Info("resolved response types from call signatures", zap.Int("count", resolved)) } } func envelopeNeedsResolution(env []map[string]any) bool { for _, row := range env { if t, _ := row["type"].(string); t == "" { return true } } return false } func envelopeFullyTyped(env []map[string]any) bool { if len(env) == 0 { return false } for _, row := range env { if t, _ := row["type"].(string); t == "" { return false } } return true } // upgradeBareTypeName looks up `name` in the graph and returns the // matching type node's ID, preferring same-repo matches. Falls back // to the input string when no graph type is found, so callers can // still surface primitives ("string", "int", "bool") and external // types ("map[string]int") that have no graph node. The shape- // inlining pass will leave those as-is since lookupShape requires a // `::` separator. func (idx *Indexer) upgradeBareTypeName(name, repoHint string) string { if name == "" { return name } if strings.Contains(name, "::") { return name // already a graph ID } candidates := idx.graph.FindNodesByName(name) var fallback *graph.Node for _, n := range candidates { if n.Kind != graph.KindType { continue } if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") { return n.ID } if fallback == nil { fallback = n } } if fallback != nil { return fallback.ID } return name } // isLikelyIdentifier accepts the bare-identifier and dotted-path // forms that traceVarTypeFromBody can match against a binding line. // Compound expressions ("len(repos)", "&Foo{}") are out of scope — // they'd need a more thorough RHS parser than the regex chain here. func isLikelyIdentifier(s string) bool { if s == "" { return false } for i, r := range s { switch { case r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z'): continue case i > 0 && r >= '0' && r <= '9': continue case r == '.' && i > 0: continue default: return false } } return true } // lookupVarTypeForContract resolves a variable to its return type // using BodyFacts (AST-driven, structurally correct). Returns // (typeID, repeated) or ("", false) when the binding can't be // resolved. // // AST-only: phase 1b deleted the body-text regex fallback. Languages // without a BodyFactsFactory get nopBodyFacts (which returns empty // Bindings), so this function is a no-op for non-Go contracts. // Their per-file regex enricher in schema_enrich_.go still runs // and populates request_type / response_type via the framework // detectors — only the post-pass cross-handler trace is AST-only. func (idx *Indexer) lookupVarTypeForContract( c contracts.Contract, bf contracts.BodyFacts, varName string, ) (string, bool) { if bf == nil { return "", false } b := bf.VarBinding(varName) // Highest tier: BindingResolver (go/types via goanalysis.Provider) // returns compiler-resolved types. When --semantic is enabled and // the provider has run, this is authoritative for any binding // whose source line we tracked. if br := contracts.CurrentBindingResolver(); br != nil && b.Line > 0 { if typeName, ok := br.LookupTypeAtLine(c.FilePath, b.Line); ok && typeName != "" { return idx.upgradeBareTypeName(typeName, c.RepoPrefix), b.Repeated } } switch b.Kind { case contracts.BindingMethodCall, contracts.BindingFuncCall: if b.CallExpr != "" { if typeID, repeated, _ := idx.resolveCallExprToType(b.CallExpr, c.RepoPrefix); typeID != "" { return typeID, repeated } } default: // Composite / slice / map / literal / path-value / // header-value / form-value / query-get — already typed. if b.TypeID != "" { return idx.upgradeBareTypeName(b.TypeID, c.RepoPrefix), b.Repeated } } return "", false } // resolveCallExprToType walks the graph from a call expression like // `h.svc.GetRepos` to the called method/function's first non-error // return type, returning (typeID, repeated, pointer). Empty string // when the call is ambiguous (multiple candidates with disagreeing // signatures) or the receiver doesn't match any graph node. // // Extracted from traceVarTypeFromBodyWithShape so BodyFacts-driven // callers can reuse the graph walk without going through the regex // path. The regex-driven traceVarTypeFromBodyWithShape is the // fallback for non-Go languages until they ship a BodyFacts // implementation. func (idx *Indexer) resolveCallExprToType(callExpr, repoHint string) (string, bool, bool) { if callExpr == "" { return "", false, false } // Split the call path. `h.tucks.Update` → ["h", "tucks", "Update"]. // The last segment is the method name; the penultimate is the // receiver field / package, which we use to disambiguate when // multiple methods share the name. parts := strings.Split(callExpr, ".") methodName := parts[len(parts)-1] if methodName == "" { return "", false, false } var receiverHint string if len(parts) >= 2 { receiverHint = parts[len(parts)-2] } candidates := idx.graph.FindNodesByName(methodName) var matches []*graph.Node for _, n := range candidates { if n.Kind != graph.KindMethod && n.Kind != graph.KindFunction { continue } if repoHint != "" && !strings.HasPrefix(n.ID, repoHint+"/") { continue } if receiverHint != "" && !receiverMatchesHint(n, receiverHint) { continue } matches = append(matches, n) } if len(matches) == 0 { return "", false, false } // Interface + implementation stacks often produce multiple // receivers that share the same method signature — a production // postgres store and a mock test store both implement // `emailSources.Update(...) (*EmailSource, error)`. Parse every // candidate's signature; if they all agree on the first // non-error return type, use that. Otherwise bail so we don't // attribute a wrong type silently. var retType string for _, m := range matches { if m.Meta == nil { continue } sig, _ := m.Meta["signature"].(string) t := parseFirstNonErrorReturnType(sig) if t == "" { continue } // De-prioritise mock receivers: if a non-mock candidate // later disagrees we want that to be the authoritative one. if retType == "" { retType = t continue } if t != retType { // Candidates disagree — can't tell which wins. The // caller sees the raw expression and can drill in. return "", false, false } } if retType == "" { return "", false, false } // Capture slice/pointer flags before stripping so the caller can // render `[Foo]` / `*Foo` correctly. Order matters: a return type // like `[]*Foo` is reported as repeated AND pointer. repeated := strings.HasPrefix(retType, "[]") pointer := strings.HasPrefix(retType, "*") || (repeated && strings.HasPrefix(retType[2:], "*")) // Strip `*` / `[]` / package qualifier so resolveTypeByName can // match the plain type-node name. retType = strings.TrimLeft(retType, "*[]") if dot := strings.LastIndex(retType, "."); dot >= 0 { retType = retType[dot+1:] } // Look up the type node, preferring same-repo matches. typeCandidates := idx.graph.FindNodesByName(retType) var bestType *graph.Node for _, n := range typeCandidates { if n.Kind != graph.KindType { continue } if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") { bestType = n break } if bestType == nil { bestType = n } } if bestType != nil { return bestType.ID, repeated, pointer } // Bare name — downstream UpgradeBareTypeRefs can still upgrade // it later, but we return it as-is so the consumer sees something // real. return retType, repeated, pointer } func parseFirstNonErrorReturnType(sig string) string { sig = strings.TrimSpace(sig) if !strings.HasPrefix(sig, "func") { return "" } sig = strings.TrimSpace(strings.TrimPrefix(sig, "func")) // Optional receiver. Two forms to recognise: // `((*Recv)) Name(params)` — gortex's stored double-paren form // `(r *Recv) Name(params)` — standard Go source form // In both cases a function name follows the receiver parens. // Anonymous function types (`func(a, b) (c, d)`) have no // receiver — the first `(` opens the parameter list and is // followed by another `(` for the return group or end-of-string. // Disambiguate by peeking past the first balanced `(...)` group // for an identifier letter. if strings.HasPrefix(sig, "(") { end := findBalancedParenEnd(sig) if end < 0 { return "" } afterFirstGroup := strings.TrimSpace(sig[end+1:]) if len(afterFirstGroup) > 0 { r := afterFirstGroup[0] isIdent := (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_' if isIdent { sig = afterFirstGroup } } } // Skip the function name — everything up to the parameter list's // opening `(`. if i := strings.Index(sig, "("); i >= 0 { sig = sig[i:] } else { return "" } // Skip parameter list. end := findBalancedParenEnd(sig) if end < 0 { return "" } sig = strings.TrimSpace(sig[end+1:]) if sig == "" { return "" } // Return clause — either `(T1, T2, ...)` or a bare single type. var inner string if strings.HasPrefix(sig, "(") { end := findBalancedParenEnd(sig) if end < 0 { return "" } inner = sig[1:end] } else { inner = sig } return firstNonErrorReturnField(inner) } // findBalancedParenEnd returns the index of the `)` that closes the // `(` at s[0]. Returns -1 when the parens don't balance. func findBalancedParenEnd(s string) int { if len(s) == 0 || s[0] != '(' { return -1 } depth := 0 for i := 0; i < len(s); i++ { switch s[i] { case '(': depth++ case ')': depth-- if depth == 0 { return i } } } return -1 } // firstNonErrorReturnField splits a return-clause body by top-level // commas and returns the first type expression that isn't `error`. // Named return parameters (`result *User`) are handled by taking the // last whitespace-separated token as the type. func firstNonErrorReturnField(inner string) string { var fields []string depth := 0 start := 0 for i := 0; i < len(inner); i++ { switch inner[i] { case '(', '[': depth++ case ')', ']': depth-- case ',': if depth == 0 { fields = append(fields, strings.TrimSpace(inner[start:i])) start = i + 1 } } } if last := strings.TrimSpace(inner[start:]); last != "" { fields = append(fields, last) } for _, f := range fields { t := f // Named return: `ctx context.Context`, `result *User`. Grab // the last whitespace-separated token — that's the type. if parts := strings.Fields(f); len(parts) > 1 { t = parts[len(parts)-1] } if t == "error" || strings.HasSuffix(t, ".error") { continue } return t } return "" } // snapshotContractShapes walks every request_type / response_type // reference in the registry, loads each referenced type node's source, // and attaches the extracted Shape to the node's Meta["shape"]. // // We: // - Collect the unique set of symbol IDs — a popular DTO might be a // request/response on dozens of routes and we want to parse its // source once. // - Read each file once (cached in the source map). // - Skip nodes whose ID doesn't look like a symbol (bare names that // couldn't be upgraded) — those have nothing to dereference. // - Skip type nodes that already have a shape attached from a prior // pass on the same session (ETag-style short-circuit). func (idx *Indexer) snapshotContractShapes(reg *contracts.Registry) { symbols := make(map[string]struct{}) for _, c := range reg.All() { for _, key := range []string{"request_type", "response_type"} { v, _ := c.Meta[key].(string) if v == "" || !strings.Contains(v, "::") { continue } symbols[v] = struct{}{} } // Envelope rows reference types the dashboard wants expanded // just as much as the top-level response_type does — without // snapshotting them here, the inlineEnvelopeShapes pass below // finds no shape to fold into the row. if env, ok := c.Meta["response_envelope"].([]map[string]any); ok { for _, row := range env { v, _ := row["type"].(string) if v == "" || !strings.Contains(v, "::") { continue } symbols[v] = struct{}{} } } } if len(symbols) == 0 { return } srcCache := make(map[string][]byte) attached := 0 for id := range symbols { node := idx.graph.GetNode(id) if node == nil { continue } // Accept both KindType and KindInterface — TypeScript / // Java / Kotlin model their type defs as interfaces, and // the dashboard wants their fields expanded just like Go // struct types. Limiting to KindType silently dropped every // TS interface shape extraction. if node.Kind != graph.KindType && node.Kind != graph.KindInterface { continue } if _, done := node.Meta["shape"]; done { continue } src, ok := srcCache[node.FilePath] if !ok { // File paths in the graph are repo-prefixed; trim the // prefix for disk I/O. diskPath := node.FilePath if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") { diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/") } diskPath = filepath.Join(idx.rootPath, diskPath) data, err := os.ReadFile(diskPath) if err != nil { srcCache[node.FilePath] = nil continue } srcCache[node.FilePath] = data src = data } if src == nil { continue } shape := contracts.ExtractShape(node.FilePath, src, node.StartLine, node.EndLine) if shape == nil { continue } if node.Meta == nil { node.Meta = map[string]any{} } node.Meta["shape"] = shape attached++ } if attached > 0 { idx.logger.Info("contract shapes snapshotted", zap.Int("types", attached), zap.Int("examined", len(symbols))) } } // inlineEnvelopeShapes folds each type node's snapshotted shape onto // every response_envelope row that references it. After this pass an // envelope row carries the full JSON-rendering data: // // { // "name": "repos", // "expr": "repos", // "type": "/service.go::Repo", // "shape": { "kind": "struct", "fields": [...] } // } // // so the dashboard can render the actual response shape instead of a // bare type-symbol-ID. Idempotent: rows that already carry "shape" // are skipped, which lets cross-pass calls (re-extraction, snapshot // restore) re-run cheaply. // // Also handles the top-level response_type / request_type: the // referenced type's shape is mirrored onto Meta["response_shape"] / // Meta["request_shape"] so plain-typed responses (no map envelope) // also expose their JSON object shape on the dashboard. func (idx *Indexer) inlineEnvelopeShapes(reg *contracts.Registry) { inlined := 0 for _, c := range reg.All() { changed := false // Envelope rows. if env, ok := c.Meta["response_envelope"].([]map[string]any); ok && len(env) > 0 { for ri, row := range env { if _, has := row["shape"]; has { continue } if shape := idx.lookupShape(row["type"]); shape != nil { env[ri]["shape"] = shape changed = true } } if changed { items := reg.ByID(c.ID) for i := range items { if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID { continue } if items[i].Meta == nil { items[i].Meta = map[string]any{} } items[i].Meta["response_envelope"] = env } reg.ReplaceByID(c.ID, items) } } // Top-level request/response shapes — same idea, applied to a // plain `response_type: "::Foo"` so the schema view can // render the JSON object even when there's no envelope wrapper. topChanged := false for metaKey, shapeKey := range map[string]string{ "response_type": "response_shape", "request_type": "request_shape", } { if _, has := c.Meta[shapeKey]; has { continue } if shape := idx.lookupShape(c.Meta[metaKey]); shape != nil { items := reg.ByID(c.ID) for i := range items { if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID { continue } if items[i].Meta == nil { items[i].Meta = map[string]any{} } items[i].Meta[shapeKey] = shape } reg.ReplaceByID(c.ID, items) topChanged = true } } if changed || topChanged { inlined++ } } if inlined > 0 { idx.logger.Info("response envelopes hydrated with shapes", zap.Int("contracts", inlined)) } } // lookupShape resolves a Meta type reference to the snapshotted shape // stored on its graph node. Accepts string IDs (the only form used in // today's pipeline); other shapes pass through as nil so callers can // chain without type-asserting upstream. func (idx *Indexer) lookupShape(raw any) any { id, ok := raw.(string) if !ok || id == "" || !strings.Contains(id, "::") { return nil } node := idx.graph.GetNode(id) if node == nil || node.Meta == nil { return nil } shape, ok := node.Meta["shape"] if !ok || shape == nil { return nil } return shape } // extractExternalModules parses the repo's go.mod once and writes // KindModule nodes plus EdgeDependsOnModule edges into the graph. // A synthetic KindFile node is emitted for go.mod itself so the // edges have a real source endpoint that survives applyRepoPrefix. // Safe to call when no go.mod exists. Other manifest formats // (package.json, pnpm-lock, requirements.txt, Cargo.toml, …) are // future additions — each lands as another switch case here. // // Import-node → module-node edges (per the broader coverage spec) // are deferred to v2; the v1 file-level edge is already enough for // agents asking "what does this repo depend on". func (idx *Indexer) extractExternalModules() { if !idx.config.Coverage.IsEnabled("modules") { return } // Walk known manifest formats at the repo root. Each manifest // produces an independent Spec list and gets its own synthetic // file node — the file→module edge stays scoped to the // originating manifest so cross-ecosystem queries (e.g. "what // does package.json declare") don't bleed into go.mod's // answer. manifests := []struct { path string parse func([]byte) []modules.Spec ownPathFromSrc func([]byte) string }{ { path: "go.mod", parse: modules.ParseGoMod, ownPathFromSrc: readGoModModulePath, }, { path: "package.json", parse: modules.ParsePackageJSON, ownPathFromSrc: readPackageJSONOwnName, }, { path: "package-lock.json", parse: modules.ParsePackageLockJSON, ownPathFromSrc: nil, // package-lock has no own-name notion separate from package.json }, { path: "yarn.lock", parse: modules.ParseYarnLock, ownPathFromSrc: nil, }, { path: "pnpm-lock.yaml", parse: modules.ParsePnpmLock, ownPathFromSrc: nil, }, { path: "pyproject.toml", parse: modules.ParsePyProject, ownPathFromSrc: readPyProjectOwnName, }, { path: "requirements.txt", parse: modules.ParseRequirementsTxt, ownPathFromSrc: nil, // requirements.txt has no own-name notion }, { path: "Cargo.toml", parse: modules.ParseCargoToml, ownPathFromSrc: readCargoTomlOwnName, }, { path: "pom.xml", parse: modules.ParsePomXML, ownPathFromSrc: readPomXMLOwnName, }, } for _, m := range manifests { idx.extractOneModuleManifest(m.path, m.parse, m.ownPathFromSrc) } // After per-manifest module extraction, detect whether this repo's // root is a package-manager workspace and materialise its // root→member edges. idx.extractPackageWorkspace() } // extractOneModuleManifest reads a single manifest file from the // repo root, parses it via the supplied parser, and writes the // resulting nodes/edges + import-link edges into the graph. Used // from extractExternalModules's per-manifest dispatch. func (idx *Indexer) extractOneModuleManifest(relPath string, parse func([]byte) []modules.Spec, ownPathFromSrc func([]byte) string) { manifestAbs := filepath.Join(idx.rootPath, relPath) src, err := os.ReadFile(manifestAbs) if err != nil { return } specs := parse(src) nodes, edges := modules.BuildGraphArtifacts(relPath, specs) if len(nodes) == 0 && len(edges) == 0 { return } // Synthetic file node for the manifest — it isn't represented // through the language-extractor pipeline (no extractor is // registered for the .mod extension; package.json may have one // but the JSON walker doesn't emit a synthetic file node we // can reuse), so the EdgeDependsOnModule edges would otherwise // dangle from a missing source endpoint after applyRepoPrefix // runs in multi-repo mode. manifestNode := &graph.Node{ ID: relPath, Kind: graph.KindFile, Name: filepath.Base(relPath), FilePath: relPath, Language: manifestLanguage(relPath), } allNodes := append([]*graph.Node{manifestNode}, nodes...) idx.applyRepoPrefix(allNodes, edges) idx.graph.AddBatch(allNodes, edges) // Connect each KindImport node to its matching module via // longest-prefix path resolution. Repo-internal imports (the // own-module path) are filtered inside LinkImports — when the // manifest doesn't expose an own-name, the filter is a no-op // which is safe (no own-module imports to match against). var ownModulePath string if ownPathFromSrc != nil { ownModulePath = ownPathFromSrc(src) } // Scope the walk to this repo's own import nodes. The unscoped // LinkImports walks g.AllNodes(); under a warmup loop across // hundreds of repos that's O(R · N). The per-repo byRepo bucket // keeps this O(repo size). repoNodes := idx.graph.GetRepoNodes(idx.repoPrefix) importNodes := make([]*graph.Node, 0, len(repoNodes)) for _, n := range repoNodes { if n.Kind == graph.KindImport { importNodes = append(importNodes, n) } } modules.LinkImportsIn(idx.graph, importNodes, specs, ownModulePath) } // manifestLanguage returns the language tag stamped on a manifest's // synthetic file node. Used purely for Brief listings — the kind // field carries the structural type. func manifestLanguage(relPath string) string { switch filepath.Base(relPath) { case "go.mod": return "go" case "package.json", "package-lock.json": return "json" case "yarn.lock": return "yarn" case "pnpm-lock.yaml": return "yaml" case "pyproject.toml", "Cargo.toml": return "toml" case "requirements.txt": return "text" case "pom.xml": return "xml" } return "" } // readPomXMLOwnName builds the project's own Maven coordinate // (groupId:artifactId) so LinkImports can filter self-references. // Java workspace setups where a sibling module imports the parent // project shouldn't accidentally resolve to an external dep with // the same coordinate. Returns "" when either field is missing — // the LinkImports filter treats "" as no own-module filter, which // is safe. func readPomXMLOwnName(src []byte) string { var manifest struct { GroupID string `xml:"groupId"` ArtifactID string `xml:"artifactId"` } if err := xml.Unmarshal(src, &manifest); err != nil { return "" } if manifest.GroupID == "" || manifest.ArtifactID == "" { return "" } return manifest.GroupID + ":" + manifest.ArtifactID } // readCargoTomlOwnName reads the crate's own name from // `[package] name`. Used for LinkImports's own-module filter so // workspace-internal crate references don't accidentally match // external crates with the same name. func readCargoTomlOwnName(src []byte) string { var manifest struct { Package struct { Name string `toml:"name"` } `toml:"package"` } if err := toml.Unmarshal(src, &manifest); err != nil { return "" } return manifest.Package.Name } // readPyProjectOwnName returns the package's own name from the // pyproject.toml `[project] name` field. Used by LinkImports's // own-module filter so a workspace package's internal imports // don't accidentally collide with external pypi names. Returns "" // on parse error or when the field is absent. func readPyProjectOwnName(src []byte) string { var manifest struct { Project struct { Name string `toml:"name"` } `toml:"project"` } if err := toml.Unmarshal(src, &manifest); err != nil { return "" } return manifest.Project.Name } // readPackageJSONOwnName extracts the manifest's `name` field — the // npm equivalent of go.mod's `module` directive. Returns "" on // missing or malformed JSON; LinkImports treats "" as "no own- // module filter", which is safe because internal-package matches // (e.g. workspaces) won't accidentally collide with external deps // at the longest-prefix scan. func readPackageJSONOwnName(src []byte) string { var manifest struct { Name string `json:"name"` } if err := json.Unmarshal(src, &manifest); err != nil { return "" } return manifest.Name } // readGoModModulePath extracts the `module ` directive value from // go.mod source. Mirrors the inline parse in coverage.ReadModulePath // — we keep both copies tiny rather than introducing a one-import // shared helper that would force a layering compromise (coverage // shouldn't depend on indexer; indexer shouldn't depend on coverage). func readGoModModulePath(src []byte) string { for _, line := range strings.Split(string(src), "\n") { line = strings.TrimSpace(line) if strings.HasPrefix(line, "module ") { return strings.TrimSpace(strings.TrimPrefix(line, "module ")) } } return "" } // extractGoModContracts runs the go.mod-specific extractor once against // the repo root (go.mod isn't represented as a file node in the graph). // Results are added to reg. Safe to call when no go.mod exists. // // Also materialises the dep:: contracts as graph nodes // immediately, so the resolver's import-bridge (Resolver.lookupDepModule) // can find them during ResolveAll. commitContracts later AddNode is // idempotent — it skips nodes that already exist — so this doesn't // double-emit. We only do this for type=dependency; everything else // goes through the normal commit path which depends on a resolved // graph (UpgradeBareTypeRefs, resolveProviderHandlers). func (idx *Indexer) extractGoModContracts(reg *contracts.Registry) { goModPath := filepath.Join(idx.rootPath, "go.mod") goModSrc, err := os.ReadFile(goModPath) if err != nil { return } goModExtractor := &contracts.GoModExtractor{TrackedRepos: idx.trackedRepoModules} goModFilePath := "go.mod" if idx.repoPrefix != "" { goModFilePath = idx.repoPrefix + "/go.mod" } found := goModExtractor.Extract(goModFilePath, goModSrc, nil, nil) reg.AddAllScoped(found, idx.repoPrefix, idx.workspaceID, idx.projectID) var nodes []*graph.Node for i := range found { c := found[i] if c.Type != contracts.ContractDependency { continue } if idx.graph.GetNode(c.ID) != nil { continue } nodes = append(nodes, &graph.Node{ ID: c.ID, Kind: graph.KindContract, Name: c.ID, FilePath: c.FilePath, Language: "contract", RepoPrefix: idx.repoPrefix, Meta: map[string]any{"type": string(c.Type), "role": string(c.Role)}, }) } if len(nodes) > 0 { idx.graph.AddBatch(nodes, nil) } } // extractContracts scans all file nodes in the graph and runs contract // extractors to detect API contracts (HTTP routes, gRPC services, // GraphQL, topics, etc.). Detected contracts are added as graph nodes // with provides/consumes edges. // // This full-walk path is used by IncrementalReindex (where many files // are already cached). IndexCtx instead runs the per-file work inline // with parsing — see the worker loop — and skips this function. func (idx *Indexer) extractContracts() { reg := contracts.NewRegistry() _, byLang := idx.buildPerFileContractExtractors() // Track which file nodes we saw this pass so we can prune stale // cache entries afterwards (files that left the graph). seenFiles := make(map[string]struct{}) // Multi-repo mode: walk only this repo's nodes. The previous // AllNodes()-then-filter pass paid an O(global) walk per repo, // which compounded with hundreds of tracked siblings. var nodes []*graph.Node if idx.repoPrefix != "" { nodes = idx.graph.GetRepoNodes(idx.repoPrefix) } else { nodes = idx.graph.AllNodes() } // Pre-bucket the already-fetched node slice by FilePath so the // per-file body can look up its co-located nodes in O(1) instead // of firing a fresh GetFileNodes query per file. Likewise pre- // fetch every out-edge whose source is in this repo as ONE backend // call and bucket by From so the per-file body can replace // GetOutEdges(fileNode.ID) — on disk backends the per-file query // path was the second-largest source of round-trips in // deferred_passes (after the DI walk). nodesByFile := make(map[string][]*graph.Node, len(nodes)) for _, n := range nodes { if n == nil { continue } nodesByFile[n.FilePath] = append(nodesByFile[n.FilePath], n) } var edgesByFrom map[string][]*graph.Edge if idx.repoPrefix != "" { repoEdges := idx.graph.GetRepoEdges(idx.repoPrefix) edgesByFrom = make(map[string][]*graph.Edge, len(nodes)) for _, e := range repoEdges { if e == nil { continue } edgesByFrom[e.From] = append(edgesByFrom[e.From], e) } } for _, fileNode := range nodes { if fileNode.Kind != graph.KindFile { continue } // In multi-repo mode the byRepo bucket is already scoped, but // the path-prefix strip below still needs to run. relPath := fileNode.FilePath if idx.repoPrefix != "" { if !strings.HasPrefix(relPath, idx.repoPrefix+"/") { continue } relPath = strings.TrimPrefix(relPath, idx.repoPrefix+"/") } absPath := filepath.Join(idx.rootPath, relPath) info, statErr := os.Stat(absPath) if statErr != nil { continue } seenFiles[fileNode.FilePath] = struct{}{} currentMtime := info.ModTime().UnixNano() // Cache hit: replay the previously-extracted contracts without // re-reading the file or re-running the 8 extractors. This is // the dominant savings path on repos with many files where most // haven't changed since the last extraction (e.g. live re-index // after a single-file edit). idx.contractCacheMu.RLock() cached, ok := idx.contractCache[fileNode.FilePath] idx.contractCacheMu.RUnlock() if ok && cached.mtimeNano == currentMtime { for _, c := range cached.contracts { reg.Add(c) } continue } src, err := os.ReadFile(absPath) if err != nil { continue } var fileNodes []*graph.Node var fileEdges []*graph.Edge if idx.repoPrefix != "" { fileNodes = nodesByFile[fileNode.FilePath] fileEdges = edgesByFrom[fileNode.ID] } else { fileNodes = idx.graph.GetFileNodes(fileNode.FilePath) fileEdges = idx.graph.GetOutEdges(fileNode.ID) } // Language-filtered dispatch: skip extractors that don't list // this file's language in SupportedLanguages(). On big repos // with lots of .css / .svg / .json etc. this cuts a lot of // no-op extractor calls. exts := byLang[fileNode.Language] // Re-parse for AST overlay — the language extractor's tree // from the original index pass was closed when the file was // first added. Cheap (~milliseconds per file) and cleanly // scoped to this contract-pass invocation. tree := contracts.ParseTreeForLang(fileNode.Language, src) fileContracts := idx.runContractExtractorsForFile( fileNode.FilePath, src, fileNodes, fileEdges, exts, tree) tree.Release() for _, c := range fileContracts { reg.Add(c) } idx.contractCacheMu.Lock() idx.contractCache[fileNode.FilePath] = &contractCacheEntry{ mtimeNano: currentMtime, contracts: fileContracts, } idx.contractCacheMu.Unlock() } // Prune cache entries for files that are no longer in the graph // (deleted, or repo untracked). Otherwise the cache grows unboundedly // across the lifetime of the daemon. idx.contractCacheMu.Lock() for path := range idx.contractCache { if _, ok := seenFiles[path]; !ok { delete(idx.contractCache, path) } } idx.contractCacheMu.Unlock() idx.extractGoModContracts(reg) idx.extractDIContracts(reg) idx.commitContracts(reg) } // IsStale returns true if the file at relPath has been modified on disk since // it was last indexed, based on comparing stored mtime against current disk mtime. // // relPath is folded to the canonical key (slash form, Unicode NFC) // before lookup so a caller passing a non-ASCII path in a different // Unicode form than fileMtimes was keyed with still resolves — without // the fold the lookup would miss and the file be reported permanently // stale, re-indexing it under a second key on every pass. // HasChangesSinceMtimes reports whether any indexable file under root // changed (mtime differs or is new) or was deleted, relative to the // indexer's currently-loaded fileMtimes. It runs the SAME walk + // staleness + deletion logic as IncrementalReindex but writes nothing. // // The daemon warmup uses it to choose a reconcile strategy for a // reopened repo: a repo with zero changes takes the fast no-op // IncrementalReindex path, while a repo that changed while the daemon // was down is routed through the shadow/bulk re-track path instead. // That routing matters because IncrementalReindex re-resolves changed // files through per-edge graph.ReindexEdges, and the per-edge write // path against a freshly reopened disk store is slow and unreliable. // The shadow path resolves entirely in an in-memory graph and commits // the result in one bulk load, so it never issues a per-edge write to // the reopened store. It re-indexes the whole repo (more work than a // true incremental pass), but it is reliable, and only repos that // actually changed during downtime pay the cost. // // Conservative on error: anything it can't determine (bad root, walk // error) returns true so the caller re-indexes rather than silently // serving a stale graph. func (idx *Indexer) HasChangesSinceMtimes(root string) bool { absRoot, err := filepath.Abs(root) if err != nil { return true } idx.storeRootPath(absRoot) diskFiles := make(map[string]bool) errStop := errors.New("stop-walk") walkErr := filepath.WalkDir(absRoot, func(path string, d os.DirEntry, werr error) error { if werr != nil { return nil } if d.IsDir() { if idx.shouldPruneDir(path, absRoot) { return filepath.SkipDir } return nil } if _, ok := idx.effectiveLanguage(path, nil); !ok { return nil } if idx.shouldExclude(path, absRoot, false) { return nil } rel := idx.relKey(path) diskFiles[rel] = true if idx.IsStale(rel) { return errStop // a single changed/new file is enough } return nil }) if errors.Is(walkErr, errStop) { return true } if walkErr != nil { return true } // Deletion check: a previously-indexed file absent from the walk and // confirmed gone from disk counts as a change (its edges must drop). idx.mtimeMu.RLock() var candidates []string for rel := range idx.fileMtimes { if !diskFiles[rel] { candidates = append(candidates, rel) } } idx.mtimeMu.RUnlock() for _, rel := range candidates { if _, err := os.Stat(filepath.Join(absRoot, filepath.FromSlash(rel))); errors.Is(err, os.ErrNotExist) { return true } } return false } // ChangedSinceMtimes is the accumulating variant of HasChangesSinceMtimes: // it runs the SAME walk + IsStale staleness predicate + deletion-confirm // logic but, instead of short-circuiting on the first stale file, returns // the full changed and deleted sets as in-repo relative slash-paths. // // The warm-restart reconcile router uses the census to size the reconcile: // an empty result takes the fast no-op path, a small delta is scoped to // exactly those files (re-index the changed, evict the deleted), and only a // large-fraction churn falls back to a whole-repo re-track. // // changed holds files that are new or whose mtime drifted since the last // pass; deleted holds previously-indexed files confirmed gone from disk. A // walk (or abs-path) error returns a non-nil err with nil slices — the // caller treats that as "unknown, do a full re-track", exactly as // HasChangesSinceMtimes conservatively returns true on the same condition. func (idx *Indexer) ChangedSinceMtimes(root string) (changed []string, deleted []string, err error) { absRoot, absErr := filepath.Abs(root) if absErr != nil { return nil, nil, absErr } idx.storeRootPath(absRoot) diskFiles := make(map[string]bool) walkErr := filepath.WalkDir(absRoot, func(path string, d os.DirEntry, werr error) error { if werr != nil { return nil } if d.IsDir() { if idx.shouldPruneDir(path, absRoot) { return filepath.SkipDir } return nil } if _, ok := idx.effectiveLanguage(path, nil); !ok { return nil } if idx.shouldExclude(path, absRoot, false) { return nil } rel := idx.relKey(path) diskFiles[rel] = true if idx.IsStale(rel) { changed = append(changed, rel) } return nil }) if walkErr != nil { return nil, nil, walkErr } // Deletion check: a previously-indexed file absent from the walk and // confirmed gone from disk counts as a change (its edges must drop). idx.mtimeMu.RLock() var candidates []string for rel := range idx.fileMtimes { if !diskFiles[rel] { candidates = append(candidates, rel) } } idx.mtimeMu.RUnlock() for _, rel := range candidates { if _, statErr := os.Stat(filepath.Join(absRoot, filepath.FromSlash(rel))); errors.Is(statErr, os.ErrNotExist) { deleted = append(deleted, rel) } } return changed, deleted, nil } func (idx *Indexer) IsStale(relPath string) bool { relPath = pathkey.Normalize(filepath.ToSlash(relPath)) idx.mtimeMu.RLock() storedMtime, ok := idx.fileMtimes[relPath] idx.mtimeMu.RUnlock() if !ok { // Unknown file — treat as stale. return true } absPath := filepath.Join(idx.rootPath, filepath.FromSlash(relPath)) info, err := os.Stat(absPath) if err != nil { // Can't stat — treat as stale. return true } return info.ModTime().UnixNano() != storedMtime } // IsTrackedStale reports whether a file that IS in the index has changed // on disk since it was indexed. Unlike IsStale it returns false for an // untracked path (a new file, a non-source path, or a path-form // mismatch), so a freshness signal never false-positives on a file the // index legitimately does not cover. func (idx *Indexer) IsTrackedStale(relPath string) bool { return idx.TrackedFileState(relPath) == FileStale } // FileFreshness is the verdict TrackedFileState returns for a tracked file: // its indexed view is current, has drifted, or the file is gone from disk. type FileFreshness string const ( // FileFresh means the file is not tracked (so no claim is made) or its // on-disk mtime still matches what was indexed. FileFresh FileFreshness = "fresh" // FileStale means the file is tracked and its on-disk mtime differs from // the indexed mtime — the graph view lags the working tree. FileStale FileFreshness = "stale" // FileMissing means the file is tracked in the graph but no longer exists // on disk — it was deleted or moved since indexing. A plain staleness // check folds this into "not stale"; the distinct verdict lets a list // result flag hits that point at vanished files. FileMissing FileFreshness = "missing" ) // TrackedFileState classifies one repo-relative file against the indexer's // recorded mtimes: fresh (untracked or unchanged), stale (mtime drift), or // missing (tracked but absent on disk). Splitting the stat-failure branch out // of IsTrackedStale is what lets the freshness rider distinguish a stale hit // from one whose underlying file has been deleted. func (idx *Indexer) TrackedFileState(relPath string) FileFreshness { relPath = pathkey.Normalize(filepath.ToSlash(relPath)) idx.mtimeMu.RLock() storedMtime, ok := idx.fileMtimes[relPath] idx.mtimeMu.RUnlock() if !ok { return FileFresh } absPath := filepath.Join(idx.rootPath, filepath.FromSlash(relPath)) info, err := os.Stat(absPath) if err != nil { if os.IsNotExist(err) { return FileMissing } // A transient / permission stat error is not evidence of drift — // don't cry wolf for a file we simply could not read. return FileFresh } if info.ModTime().UnixNano() != storedMtime { return FileStale } return FileFresh }