package indexer import ( "os/exec" "path/filepath" "strings" "testing" "github.com/stretchr/testify/require" "go.uber.org/zap" "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" ) // newAssetTestIndexer registers the code + asset extractors the // content-admission gate keys off, so a temp repo of documents / data // files exercises the real walk-time gate. func newAssetTestIndexer(g graph.Store) *Indexer { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) reg.Register(languages.NewTextExtractor()) reg.Register(languages.NewPDFExtractor()) reg.Register(languages.NewPptxExtractor()) reg.Register(languages.NewXlsxExtractor()) reg.Register(languages.NewDataAssetExtractor()) cfg := config.Default().Index cfg.Workers = 2 return New(g, reg, cfg, zap.NewNop()) } func TestContentAdmissionConfig_EffectiveCaps(t *testing.T) { // 0 (absent) → built-in default, capped. lim, capped := config.ContentAdmissionConfig{MaxDocumentBytes: 0}.EffectiveMaxDocumentBytes() require.True(t, capped) require.Equal(t, int64(10<<20), lim) // >0 → explicit cap. lim, capped = config.ContentAdmissionConfig{MaxDocumentBytes: 4096}.EffectiveMaxDocumentBytes() require.True(t, capped) require.Equal(t, int64(4096), lim) // <0 → no cap. _, capped = config.ContentAdmissionConfig{MaxDocumentBytes: -1}.EffectiveMaxDocumentBytes() require.False(t, capped) // Same tri-state for data. _, capped = config.ContentAdmissionConfig{MaxDataBytes: -1}.EffectiveMaxDataBytes() require.False(t, capped) lim, capped = config.ContentAdmissionConfig{MaxDataBytes: 7}.EffectiveMaxDataBytes() require.True(t, capped) require.Equal(t, int64(7), lim) } func TestContentAdmissionGate_Skip(t *testing.T) { idx := newAssetTestIndexer(graph.New()) // Default policy: documents capped at 10 MiB, data skipped. idx.config.Content = config.Default().Index.Content gate := idx.newContentAdmissionGate() require.NotNil(t, gate) // Code is never gated. _, skip := gate.skip("go", 1<<30) require.False(t, skip) // A small document is admitted; a huge one is dropped. _, skip = gate.skip("text", 1024) require.False(t, skip) reason, skip := gate.skip("text", 11<<20) require.True(t, skip) require.Equal(t, skipReasonLargeDocument, reason) // Data assets are skipped by default at any size. reason, skip = gate.skip("data", 1) require.True(t, skip) require.Equal(t, skipReasonVectorData, reason) // Opting data in admits it, subject to its own cap. idx.config.Content.IndexData = true idx.config.Content.MaxDataBytes = 1024 gate = idx.newContentAdmissionGate() _, skip = gate.skip("data", 512) require.False(t, skip) reason, skip = gate.skip("data", 4096) require.True(t, skip) require.Equal(t, skipReasonLargeData, reason) // A negative document cap disables document gating entirely. idx.config.Content = config.ContentAdmissionConfig{MaxDocumentBytes: -1} gate = idx.newContentAdmissionGate() _, skip = gate.skip("text", 1<<30) require.False(t, skip) } // TestContentAdmissionGate_NilInert verifies a nil gate (no asset extractors // registered) never skips — the all-code-repo fast path. func TestContentAdmissionGate_NilInert(t *testing.T) { var g *contentAdmissionGate _, skip := g.skip("text", 1<<30) require.False(t, skip) } func TestContentSkipNode(t *testing.T) { n := contentSkipNode(skippedFile{ relPath: "data/deck.pptx", lang: "pptx", size: 50 << 20, reason: skipReasonLargeDocument, }) require.Equal(t, graph.KindFile, n.Kind) require.Equal(t, "data/deck.pptx", n.ID) require.Equal(t, "deck.pptx", n.Name) require.Equal(t, "pptx", n.Language) require.Equal(t, skipReasonLargeDocument, n.Meta["skip_reason"]) require.Equal(t, true, n.Meta["skipped_due_to_content"]) require.Equal(t, int64(50<<20), n.Meta["file_size_bytes"]) } // TestIndex_DocumentCapSkip verifies a document over the cap becomes a // large_document skip node while a small one is still ingested as content. func TestIndex_DocumentCapSkip(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "small.txt"), "a short note") writeFile(t, filepath.Join(dir, "big.txt"), strings.Repeat("filler text ", 400)) // ~4.8 KiB writeFile(t, filepath.Join(dir, "code.go"), "package main\n\nfunc A() {}\n") g := graph.New() idx := newAssetTestIndexer(g) idx.config.Content.MaxDocumentBytes = 1024 // big.txt is over the cap result, err := idx.Index(dir) require.NoError(t, err) require.Equal(t, 1, result.SkippedFiles) big := g.GetNode("big.txt") require.NotNil(t, big, "an over-cap document must still leave a skip node") require.Equal(t, skipReasonLargeDocument, big.Meta["skip_reason"]) require.Equal(t, true, big.Meta["skipped_due_to_content"]) // The small document was admitted (no skip reason). small := g.GetNode("small.txt") require.NotNil(t, small) require.Nil(t, small.Meta["skip_reason"]) } // TestIndex_DataAssetSkippedByDefault verifies a vector/data artifact is // dropped by default and admitted as a metadata node when opted in. func TestIndex_DataAssetSkippedByDefault(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "embeddings.npy"), "NUMPY-array-placeholder-bytes") // Default: skipped with vector_data. g := graph.New() idx := newAssetTestIndexer(g) result, err := idx.Index(dir) require.NoError(t, err) require.Equal(t, 1, result.SkippedFiles) n := g.GetNode("embeddings.npy") require.NotNil(t, n) require.Equal(t, skipReasonVectorData, n.Meta["skip_reason"]) // Opt in: admitted as a data metadata node, no skip reason. g2 := graph.New() idx2 := newAssetTestIndexer(g2) idx2.config.Content.IndexData = true _, err = idx2.Index(dir) require.NoError(t, err) n2 := g2.GetNode("embeddings.npy") require.NotNil(t, n2) require.Nil(t, n2.Meta["skip_reason"]) require.Equal(t, "data", n2.Meta["data_class"]) } // TestIndex_SkipUntrackedAssets verifies that, with the opt-in flag on, // untracked document/data assets are dropped while tracked assets and // untracked CODE are still indexed; and that the default (flag off) admits // untracked documents. func TestIndex_SkipUntrackedAssets(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git binary not available in PATH") } dir := t.TempDir() runGit(t, dir, "init", "-q", "-b", "main") runGit(t, dir, "config", "user.email", "t@example.com") runGit(t, dir, "config", "user.name", "T") runGit(t, dir, "config", "commit.gpgsign", "false") // Tracked: a code file and a small document. writeFile(t, filepath.Join(dir, "code.go"), "package main\n\nfunc A() {}\n") writeFile(t, filepath.Join(dir, "tracked.txt"), "committed note") runGit(t, dir, "add", ".") runGit(t, dir, "commit", "-q", "-m", "init") // Untracked working-tree files: a document, a data asset, and code. writeFile(t, filepath.Join(dir, "untracked.txt"), "scratch rag asset") writeFile(t, filepath.Join(dir, "vectors.npy"), "NUMPY-placeholder-bytes") writeFile(t, filepath.Join(dir, "new.go"), "package main\n\nfunc B() {}\n") // Flag ON: untracked assets dropped, tracked asset + untracked code kept. g := graph.New() idx := newAssetTestIndexer(g) idx.config.SkipUntrackedAssets = true if _, err := idx.IndexCtx(testCtx(), dir); err != nil { t.Fatalf("index: %v", err) } for _, p := range []string{"untracked.txt", "vectors.npy"} { n := g.GetNode(p) require.NotNil(t, n, p) require.Equal(t, skipReasonUntrackedAsset, n.Meta["skip_reason"], p) } tracked := g.GetNode("tracked.txt") require.NotNil(t, tracked) require.Nil(t, tracked.Meta["skip_reason"], "tracked document must be admitted") newGo := g.GetNode("new.go") require.NotNil(t, newGo, "untracked CODE must still be indexed") require.Nil(t, newGo.Meta["skip_reason"]) // Flag OFF (default): the untracked document is admitted as content. g2 := graph.New() idx2 := newAssetTestIndexer(g2) if _, err := idx2.IndexCtx(testCtx(), dir); err != nil { t.Fatalf("index: %v", err) } un := g2.GetNode("untracked.txt") require.NotNil(t, un) require.Nil(t, un.Meta["skip_reason"], "with the flag off, an untracked document is admitted") } // bucketByKey finds an intake bucket by its extension key. func bucketByKey(buckets []IntakeBucket, key string) (IntakeBucket, bool) { for _, b := range buckets { if b.Key == key { return b, true } } return IntakeBucket{}, false } // TestDryRunIntake_ReflectsContentGate verifies the dry-run manifest reports // oversized documents and data assets as skipped with the same reasons the // real index walk uses — so the preflight matches the indexer (#120). func TestDryRunIntake_ReflectsContentGate(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "code.go"), "package main\n\nfunc A() {}\n") writeFile(t, filepath.Join(dir, "small.txt"), "short") writeFile(t, filepath.Join(dir, "big.txt"), strings.Repeat("filler text ", 400)) writeFile(t, filepath.Join(dir, "vectors.npy"), "NUMPY-placeholder-bytes") idx := newAssetTestIndexer(graph.New()) idx.config.Content.MaxDocumentBytes = 1024 m, err := idx.DryRunIntake(testCtx(), dir) require.NoError(t, err) txt, ok := bucketByKey(m.TopSkippedExtensionsByBytes, "txt") require.True(t, ok, "oversized document must be reported skipped") require.Equal(t, skipReasonLargeDocument, txt.Reason) npy, ok := bucketByKey(m.TopSkippedExtensionsByBytes, "npy") require.True(t, ok, "data asset must be reported skipped") require.Equal(t, skipReasonVectorData, npy.Reason) // Code and the small document stay admitted. _, ok = bucketByKey(m.TopAdmittedExtensionsByBytes, "go") require.True(t, ok) smallTxt, ok := bucketByKey(m.TopAdmittedExtensionsByBytes, "txt") require.True(t, ok) require.Equal(t, 1, smallTxt.Files, "only the small document is admitted") } // TestDryRunIntake_ReflectsUntrackedGate verifies the dry-run manifest applies // the opt-in untracked-asset gate too. func TestDryRunIntake_ReflectsUntrackedGate(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git binary not available in PATH") } dir := t.TempDir() runGit(t, dir, "init", "-q", "-b", "main") runGit(t, dir, "config", "user.email", "t@example.com") runGit(t, dir, "config", "user.name", "T") runGit(t, dir, "config", "commit.gpgsign", "false") writeFile(t, filepath.Join(dir, "code.go"), "package main\n\nfunc A() {}\n") runGit(t, dir, "add", ".") runGit(t, dir, "commit", "-q", "-m", "init") writeFile(t, filepath.Join(dir, "untracked.txt"), "scratch asset") idx := newAssetTestIndexer(graph.New()) idx.config.SkipUntrackedAssets = true m, err := idx.DryRunIntake(testCtx(), dir) require.NoError(t, err) txt, ok := bucketByKey(m.TopSkippedExtensionsByBytes, "txt") require.True(t, ok) require.Equal(t, skipReasonUntrackedAsset, txt.Reason) } // TestGitTrackedSet_NonGitDirInert verifies gitTrackedSet reports failure // (gate inert) outside a git repo. func TestGitTrackedSet_NonGitDirInert(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git binary not available in PATH") } _, ok := gitTrackedSet(testCtx(), t.TempDir()) require.False(t, ok) } // TestIndexFile_ContentSkip verifies the incremental (watcher) path applies // the same gate, so a document the cold walk would skip doesn't get parsed // back in on re-index. func TestIndexFile_ContentSkip(t *testing.T) { dir := t.TempDir() g := graph.New() idx := newAssetTestIndexer(g) idx.config.Content.MaxDocumentBytes = 1024 if _, err := idx.Index(dir); err != nil { t.Fatalf("index: %v", err) } big := filepath.Join(dir, "report.txt") writeFile(t, big, strings.Repeat("report line ", 400)) require.NoError(t, idx.indexFile(big, false)) n := g.GetNode("report.txt") require.NotNil(t, n) require.Equal(t, skipReasonLargeDocument, n.Meta["skip_reason"]) }