From 0d9ff2ef6cea8af11a36184f50e61c3151f0aaff Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 8 May 2026 01:21:32 -0700 Subject: [PATCH 1/5] Add walkSourceFiles utility and SLACK_WEBHOOK_URL config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit walkSourceFiles enumerates matching files for a source config without reading content — used by the post-reindex health audit. SLACK_WEBHOOK_URL env var for operational alerts. --- .env.example | 3 + src/__tests__/analytics-endpoints.test.ts | 6 ++ src/__tests__/analytics-server.test.ts | 6 ++ src/config.ts | 3 + src/indexing/utils.ts | 89 +++++++++++++++++++++++ 5 files changed, 107 insertions(+) diff --git a/.env.example b/.env.example index 32c27ab..a509a78 100644 --- a/.env.example +++ b/.env.example @@ -35,3 +35,6 @@ LOG_LEVEL=debug # Git clone directory (default: /tmp/mcp-repos) CLONE_DIR=/tmp/mcp-repos + +# Slack webhook URL for operational alerts (optional — reindex audit, deploy health) +SLACK_WEBHOOK_URL= diff --git a/src/__tests__/analytics-endpoints.test.ts b/src/__tests__/analytics-endpoints.test.ts index 2f2083b..171007e 100644 --- a/src/__tests__/analytics-endpoints.test.ts +++ b/src/__tests__/analytics-endpoints.test.ts @@ -38,6 +38,7 @@ vi.mock("../config.js", () => ({ p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }), hasSearchTools: vi.fn().mockReturnValue(false), hasKnowledgeTools: vi.fn().mockReturnValue(false), @@ -74,6 +75,7 @@ const DEFAULT_TEST_CONFIG = { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }; function mockRes() { @@ -358,6 +360,7 @@ describe("analyticsAuth middleware", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); const res = mockRes(); const next = vi.fn(); @@ -398,6 +401,7 @@ describe("analyticsAuth middleware", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); const res = mockRes(); const next = vi.fn(); @@ -440,6 +444,7 @@ describe("analyticsAuth middleware", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); const consoleErrSpy = vi .spyOn(console, "error") @@ -484,6 +489,7 @@ describe("analyticsAuth middleware", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); const res = mockRes(); const next = vi.fn(); diff --git a/src/__tests__/analytics-server.test.ts b/src/__tests__/analytics-server.test.ts index 530cc72..2bed827 100644 --- a/src/__tests__/analytics-server.test.ts +++ b/src/__tests__/analytics-server.test.ts @@ -36,6 +36,7 @@ vi.mock("../config.js", () => ({ p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }), hasSearchTools: vi.fn().mockReturnValue(false), hasKnowledgeTools: vi.fn().mockReturnValue(false), @@ -955,6 +956,7 @@ describe("Analytics server routes (HTTP-level)", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); mockGetAnalyticsConfigFn.mockReturnValue({ enabled: true, @@ -995,6 +997,7 @@ describe("Analytics server routes (HTTP-level)", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); const fakeReq = { socket: { remoteAddress: "203.0.113.7" }, @@ -1021,6 +1024,7 @@ describe("Analytics server routes (HTTP-level)", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); mockGetAnalyticsConfigFn.mockReturnValue({ enabled: true, @@ -1060,6 +1064,7 @@ describe("Analytics server routes (HTTP-level)", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); mockGetAnalyticsConfigFn.mockReturnValue({ enabled: true, @@ -1113,6 +1118,7 @@ describe("Analytics server routes (HTTP-level)", () => { p2pTelemetryUrl: undefined, p2pTelemetryDisabled: false, packageVersion: "test", + slackWebhookUrl: "", }); } diff --git a/src/config.ts b/src/config.ts index f1e306b..01c14b6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -70,6 +70,8 @@ export interface Config { p2pTelemetryDisabled: boolean; /** Pathfinder package version, read from package.json at startup. */ packageVersion: string; + /** Slack webhook URL for operational alerts (reindex audit, deploy health). */ + slackWebhookUrl: string; } /** @@ -219,6 +221,7 @@ function parseConfig(): Config { p2pTelemetryUrl, p2pTelemetryDisabled, packageVersion: readPackageVersion(), + slackWebhookUrl: process.env.SLACK_WEBHOOK_URL ?? "", }; } diff --git a/src/indexing/utils.ts b/src/indexing/utils.ts index 285fd00..a493e09 100644 --- a/src/indexing/utils.ts +++ b/src/indexing/utils.ts @@ -1,5 +1,7 @@ // Shared indexing utilities — used by providers, bash-fs, and test scripts. +import fs from "node:fs"; +import path from "node:path"; import type { FileSourceConfig } from "../types.js"; /** @@ -75,3 +77,90 @@ export function matchesPatterns( return false; } + +// ── Default constants (mirrored from FileDataProvider) ─────────────────────── + +const DEFAULT_SKIP_DIRS = new Set(["node_modules", "dist", "build", ".git"]); +const DEFAULT_MAX_FILE_SIZE = 102400; // 100KB + +/** + * Derive a repository directory name from a git URL. + * Example: "https://github.com/org/repo.git" → "repo" + */ +function repoNameFromUrl(url: string): string { + return ( + url + .split("/") + .pop() + ?.replace(/\.git$/, "") ?? "repo" + ); +} + +/** + * Enumerate all source files that match a FileSourceConfig's patterns, + * WITHOUT reading their contents. Useful for auditing which files would + * be indexed vs. which are actually present in the index. + * + * Returns a Set of relative paths (relative to the repo root for git + * sources, or the resolved local path for local sources). + */ +export async function walkSourceFiles( + sourceConfig: FileSourceConfig, + cloneDir: string, + _githubToken?: string, +): Promise> { + // Determine repo root directory + const repoDir = sourceConfig.repo + ? path.join(cloneDir, repoNameFromUrl(sourceConfig.repo)) + : path.resolve(sourceConfig.path); + + // Determine walk starting point + const walkRoot = sourceConfig.repo + ? path.join(repoDir, sourceConfig.path) + : repoDir; + + if (!fs.existsSync(walkRoot)) { + return new Set(); + } + + const skipDirs = new Set([ + ...DEFAULT_SKIP_DIRS, + ...(sourceConfig.skip_dirs ?? []), + ]); + const maxFileSize = sourceConfig.max_file_size ?? DEFAULT_MAX_FILE_SIZE; + + const result = new Set(); + + async function walk(dir: string): Promise { + let entries: fs.Dirent[]; + try { + entries = await fs.promises.readdir(dir, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + if (skipDirs.has(entry.name)) continue; + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + await walk(fullPath); + } else if (entry.isFile()) { + try { + const stat = await fs.promises.stat(fullPath); + if (stat.size > maxFileSize) continue; + } catch { + continue; + } + + const relPath = path.relative(repoDir, fullPath); + if (matchesPatterns(relPath, sourceConfig)) { + result.add(relPath); + } + } + } + } + + await walk(walkRoot); + return result; +} From b1eb4d6d923a273cee842dd9202b94723071c6a4 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 8 May 2026 01:21:41 -0700 Subject: [PATCH 2/5] Add post-reindex health audit with Slack alerting Three read-only checks after each reindex: stale files (DB entries with no disk file), scope leaks (DB paths outside config.path), and count divergence (DB vs disk file count mismatch). Fires Slack webhook when issues detected. Wired into onReindexComplete, fire-and-forget. --- src/indexing/reindex-audit.ts | 115 ++++++++++++++++++++++++++++++++++ src/server.ts | 4 ++ 2 files changed, 119 insertions(+) create mode 100644 src/indexing/reindex-audit.ts diff --git a/src/indexing/reindex-audit.ts b/src/indexing/reindex-audit.ts new file mode 100644 index 0000000..2275688 --- /dev/null +++ b/src/indexing/reindex-audit.ts @@ -0,0 +1,115 @@ +import { getConfig, getServerConfig } from "../config.js"; +import { getIndexedItemIds } from "../db/queries.js"; +import { walkSourceFiles } from "./utils.js"; +import { isFileSourceConfig } from "../types.js"; +import type { FileSourceConfig } from "../types.js"; + +export interface AuditFinding { + source: string; + check: "stale_files" | "scope_leak" | "count_divergence"; + count: number; + samples: string[]; + direction?: "db_has_more" | "db_has_fewer"; +} + +export async function runReindexAudit( + sourceNames: string[], +): Promise { + try { + const serverCfg = getServerConfig(); + const cfg = getConfig(); + const findings: AuditFinding[] = []; + + const sourceNameSet = new Set(sourceNames); + const fileSources = serverCfg.sources.filter( + (s): s is FileSourceConfig => + isFileSourceConfig(s) && sourceNameSet.has(s.name), + ); + + for (const sourceConfig of fileSources) { + const diskFiles = await walkSourceFiles( + sourceConfig, + cfg.cloneDir, + cfg.githubToken, + ); + const dbFiles = await getIndexedItemIds(sourceConfig.name); + + // Check 1 — Stale files: in DB but not on disk + const stale = [...dbFiles].filter((p) => !diskFiles.has(p)); + if (stale.length > 0) { + findings.push({ + source: sourceConfig.name, + check: "stale_files", + count: stale.length, + samples: stale.slice(0, 10), + }); + } + + // Check 2 — Scope leaks (git sources only, skip when path is "." or "") + if (sourceConfig.repo && sourceConfig.path && sourceConfig.path !== ".") { + const prefix = sourceConfig.path.replace(/\/$/, "") + "/"; + const leaks = [...dbFiles].filter((p) => !p.startsWith(prefix)); + if (leaks.length > 0) { + findings.push({ + source: sourceConfig.name, + check: "scope_leak", + count: leaks.length, + samples: leaks.slice(0, 10), + }); + } + } + + // Check 3 — Count divergence + const dbCount = dbFiles.size; + const diskCount = diskFiles.size; + if (dbCount !== diskCount) { + findings.push({ + source: sourceConfig.name, + check: "count_divergence", + count: Math.abs(dbCount - diskCount), + samples: [], + direction: dbCount > diskCount ? "db_has_more" : "db_has_fewer", + }); + } + } + + if (findings.length > 0 && cfg.slackWebhookUrl) { + await sendSlackAlert(findings, cfg.slackWebhookUrl); + } + + return findings; + } catch (err) { + console.error( + "[reindex-audit] Audit failed:", + err instanceof Error ? err.message : err, + ); + return []; + } +} + +async function sendSlackAlert( + findings: AuditFinding[], + webhookUrl: string, +): Promise { + const lines = findings.map((f) => { + let msg = `*${f.source}* — ${f.check}: ${f.count} issues`; + if (f.direction) msg += ` (${f.direction})`; + if (f.samples.length > 0) { + msg += `\n ${f.samples.join("\n ")}`; + } + return msg; + }); + const text = `🔍 *Reindex Audit Alert*\n${lines.join("\n\n")}`; + try { + await fetch(webhookUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text }), + }); + } catch (err) { + console.error( + "[reindex-audit] Failed to send Slack alert:", + err instanceof Error ? err.message : err, + ); + } +} diff --git a/src/server.ts b/src/server.ts index 51b18e5..04bc597 100644 --- a/src/server.ts +++ b/src/server.ts @@ -36,6 +36,7 @@ import { type FaqChunkResult, } from "./types.js"; import { IndexingOrchestrator } from "./indexing/orchestrator.js"; +import { runReindexAudit } from "./indexing/reindex-audit.js"; import { createWebhookHandler } from "./webhooks/github.js"; import { createSlackWebhookHandler } from "./webhooks/slack.js"; @@ -3318,6 +3319,9 @@ async function startServerInner(options?: ServerOptions): Promise { refreshBashInstances(sourceNames, "reindex").catch((err) => { console.error("[reindex] Bash refresh failed:", err); }); + runReindexAudit(sourceNames).catch((err) => { + console.error("[reindex-audit] Audit failed:", err); + }); }; // R3 #5: use two distinct catches so an index-check failure and a From a22330c6cdc4e026008dee9dbbc82640b0301aaa Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 8 May 2026 01:21:57 -0700 Subject: [PATCH 3/5] Add tests for reindex audit and walkSourceFiles 18 audit tests (stale files, scope leaks, count divergence, Slack alerting, error handling, multi-source). 7 walkSourceFiles tests (matching, exclusion, skipDirs, maxFileSize, missing dir). --- src/__tests__/reindex-audit.test.ts | 459 ++++++++++++++++++++++++ src/__tests__/walk-source-files.test.ts | 153 ++++++++ 2 files changed, 612 insertions(+) create mode 100644 src/__tests__/reindex-audit.test.ts create mode 100644 src/__tests__/walk-source-files.test.ts diff --git a/src/__tests__/reindex-audit.test.ts b/src/__tests__/reindex-audit.test.ts new file mode 100644 index 0000000..0bad84f --- /dev/null +++ b/src/__tests__/reindex-audit.test.ts @@ -0,0 +1,459 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// --------------------------------------------------------------------------- +// Hoisted mocks — variables created before vi.mock hoisting +// --------------------------------------------------------------------------- + +const { + mockGetConfig, + mockGetServerConfig, + mockGetIndexedItemIds, + mockWalkSourceFiles, +} = vi.hoisted(() => ({ + mockGetConfig: vi.fn(), + mockGetServerConfig: vi.fn(), + mockGetIndexedItemIds: vi.fn(), + mockWalkSourceFiles: vi.fn(), +})); + +// --------------------------------------------------------------------------- +// Module mocks +// --------------------------------------------------------------------------- + +vi.mock("../config.js", () => ({ + getConfig: (...args: unknown[]) => mockGetConfig(...args), + getServerConfig: (...args: unknown[]) => mockGetServerConfig(...args), +})); + +vi.mock("../db/queries.js", () => ({ + getIndexedItemIds: (...args: unknown[]) => mockGetIndexedItemIds(...args), +})); + +vi.mock("../indexing/utils.js", () => ({ + walkSourceFiles: (...args: unknown[]) => mockWalkSourceFiles(...args), +})); + +// --------------------------------------------------------------------------- +// Import under test (after mocks are wired) +// --------------------------------------------------------------------------- + +import { + runReindexAudit, + type AuditFinding, +} from "../indexing/reindex-audit.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Build a minimal file-source config for tests. */ +function fileSource(name: string, overrides: Record = {}) { + return { + name, + type: "markdown", + path: "/repo/docs", + file_patterns: ["**/*.md"], + chunk: {}, + ...overrides, + }; +} + +/** Build a minimal server config with the given sources. */ +function serverConfig(sources: Record[]) { + return { + server: { name: "test", version: "1.0" }, + sources, + tools: [], + embedding: { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }, + indexing: { + auto_reindex: false, + reindex_hour_utc: 3, + stale_threshold_hours: 24, + }, + }; +} + +/** Build a minimal app config. */ +function appConfig( + overrides: Partial<{ slackWebhookUrl: string; cloneDir: string }> = {}, +) { + return { + databaseUrl: "postgresql://test", + openaiApiKey: "test-key", + githubToken: "", + githubWebhookSecret: "", + port: 3001, + nodeEnv: "test", + logLevel: "info", + cloneDir: "/tmp/test", + slackBotToken: "", + slackSigningSecret: "", + discordBotToken: "", + discordPublicKey: "", + notionToken: "", + mcpJwtSecret: "", + p2pTelemetryUrl: undefined, + p2pTelemetryDisabled: false, + packageVersion: "1.0.0", + slackWebhookUrl: "", + ...overrides, + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("runReindexAudit", () => { + let mockFetch: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + + // Default: single markdown source, DB and disk in sync + const src = fileSource("docs"); + mockGetServerConfig.mockReturnValue(serverConfig([src])); + mockGetConfig.mockReturnValue(appConfig()); + mockGetIndexedItemIds.mockResolvedValue(new Set(["README.md"])); + mockWalkSourceFiles.mockResolvedValue(new Set(["README.md"])); + + // Stub global fetch for Slack alert tests + mockFetch = vi.fn().mockResolvedValue({ ok: true }); + vi.stubGlobal("fetch", mockFetch); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + // ── Check 1: Stale files ──────────────────────────────────────────────── + + describe("stale_files check", () => { + it("returns stale_files finding when DB has paths not on disk", async () => { + mockGetIndexedItemIds.mockResolvedValue( + new Set(["README.md", "old-file.md", "removed.md"]), + ); + mockWalkSourceFiles.mockResolvedValue(new Set(["README.md"])); + + const findings = await runReindexAudit(["docs"]); + + const stale = findings.find((f) => f.check === "stale_files"); + expect(stale).toBeDefined(); + expect(stale!.source).toBe("docs"); + expect(stale!.count).toBe(2); + expect(stale!.samples).toEqual( + expect.arrayContaining(["old-file.md", "removed.md"]), + ); + }); + + it("returns no findings when DB and disk match exactly", async () => { + const paths = new Set(["a.md", "b.md"]); + mockGetIndexedItemIds.mockResolvedValue(new Set(paths)); + mockWalkSourceFiles.mockResolvedValue(new Set(paths)); + + const findings = await runReindexAudit(["docs"]); + + const stale = findings.find((f) => f.check === "stale_files"); + expect(stale).toBeUndefined(); + }); + + it("limits samples to 10 paths max", async () => { + const dbPaths = new Set( + Array.from({ length: 15 }, (_, i) => `stale-${i}.md`), + ); + mockGetIndexedItemIds.mockResolvedValue(dbPaths); + mockWalkSourceFiles.mockResolvedValue(new Set()); + + const findings = await runReindexAudit(["docs"]); + + const stale = findings.find((f) => f.check === "stale_files"); + expect(stale).toBeDefined(); + expect(stale!.count).toBe(15); + expect(stale!.samples.length).toBeLessThanOrEqual(10); + }); + + it("skips non-file sources (e.g., slack, notion)", async () => { + mockGetServerConfig.mockReturnValue( + serverConfig([ + { + name: "slack-support", + type: "slack", + channels: ["C001"], + confidence_threshold: 0.7, + trigger_emoji: "pathfinder", + min_thread_replies: 2, + chunk: {}, + }, + ]), + ); + + const findings = await runReindexAudit(["slack-support"]); + + expect(findings).toEqual([]); + expect(mockGetIndexedItemIds).not.toHaveBeenCalled(); + expect(mockWalkSourceFiles).not.toHaveBeenCalled(); + }); + }); + + // ── Check 2: Scope leaks ────────────────────────────────────────────── + + describe("scope_leak check", () => { + it("returns scope_leak finding when DB has paths outside config.path", async () => { + const src = fileSource("docs", { + repo: "https://github.com/org/repo.git", + path: "docs", + }); + mockGetServerConfig.mockReturnValue(serverConfig([src])); + + // DB has a file outside the "docs" prefix + mockGetIndexedItemIds.mockResolvedValue( + new Set(["docs/README.md", "src/index.ts", "lib/util.ts"]), + ); + mockWalkSourceFiles.mockResolvedValue(new Set(["docs/README.md"])); + + const findings = await runReindexAudit(["docs"]); + + const leak = findings.find((f) => f.check === "scope_leak"); + expect(leak).toBeDefined(); + expect(leak!.source).toBe("docs"); + expect(leak!.count).toBe(2); + expect(leak!.samples).toEqual( + expect.arrayContaining(["src/index.ts", "lib/util.ts"]), + ); + }); + + it("skips scope check for local sources (no repo field)", async () => { + const src = fileSource("local-docs", { path: "/absolute/path/docs" }); + mockGetServerConfig.mockReturnValue(serverConfig([src])); + + mockGetIndexedItemIds.mockResolvedValue( + new Set(["README.md", "outside/file.md"]), + ); + mockWalkSourceFiles.mockResolvedValue( + new Set(["README.md", "outside/file.md"]), + ); + + const findings = await runReindexAudit(["local-docs"]); + + const leak = findings.find((f) => f.check === "scope_leak"); + expect(leak).toBeUndefined(); + }); + + it("skips scope check when config.path is '.' or empty", async () => { + for (const pathVal of [".", ""]) { + const src = fileSource("docs", { + repo: "https://github.com/org/repo.git", + path: pathVal || ".", + }); + mockGetServerConfig.mockReturnValue(serverConfig([src])); + + mockGetIndexedItemIds.mockResolvedValue(new Set(["anywhere/file.md"])); + mockWalkSourceFiles.mockResolvedValue(new Set(["anywhere/file.md"])); + + const findings = await runReindexAudit(["docs"]); + const leak = findings.find((f) => f.check === "scope_leak"); + expect(leak).toBeUndefined(); + } + }); + }); + + // ── Check 3: Count divergence ───────────────────────────────────────── + + describe("count_divergence check", () => { + it("returns count_divergence with direction db_has_more when DB > disk", async () => { + mockGetIndexedItemIds.mockResolvedValue( + new Set(["a.md", "b.md", "c.md"]), + ); + mockWalkSourceFiles.mockResolvedValue(new Set(["a.md"])); + + const findings = await runReindexAudit(["docs"]); + + const divergence = findings.find((f) => f.check === "count_divergence"); + expect(divergence).toBeDefined(); + expect(divergence!.direction).toBe("db_has_more"); + expect(divergence!.count).toBe(2); // difference: 3 - 1 + }); + + it("returns count_divergence with direction db_has_fewer when disk > DB", async () => { + mockGetIndexedItemIds.mockResolvedValue(new Set(["a.md"])); + mockWalkSourceFiles.mockResolvedValue( + new Set(["a.md", "b.md", "c.md", "d.md"]), + ); + + const findings = await runReindexAudit(["docs"]); + + const divergence = findings.find((f) => f.check === "count_divergence"); + expect(divergence).toBeDefined(); + expect(divergence!.direction).toBe("db_has_fewer"); + expect(divergence!.count).toBe(3); // difference: 4 - 1 + }); + + it("does not report divergence when counts match", async () => { + const paths = new Set(["a.md", "b.md"]); + mockGetIndexedItemIds.mockResolvedValue(new Set(paths)); + mockWalkSourceFiles.mockResolvedValue(new Set(paths)); + + const findings = await runReindexAudit(["docs"]); + + const divergence = findings.find((f) => f.check === "count_divergence"); + expect(divergence).toBeUndefined(); + }); + }); + + // ── Slack alerting ──────────────────────────────────────────────────── + + describe("Slack alerting", () => { + it("sends Slack alert when findings exist and webhookUrl is set", async () => { + mockGetConfig.mockReturnValue( + appConfig({ slackWebhookUrl: "https://hooks.slack.com/test" }), + ); + // Create a stale-file scenario so findings are non-empty + mockGetIndexedItemIds.mockResolvedValue( + new Set(["README.md", "gone.md"]), + ); + mockWalkSourceFiles.mockResolvedValue(new Set(["README.md"])); + + await runReindexAudit(["docs"]); + + expect(mockFetch).toHaveBeenCalledWith( + "https://hooks.slack.com/test", + expect.objectContaining({ + method: "POST", + headers: expect.objectContaining({ + "Content-Type": "application/json", + }), + }), + ); + }); + + it("does NOT send Slack alert when webhookUrl is empty", async () => { + mockGetConfig.mockReturnValue(appConfig({ slackWebhookUrl: "" })); + mockGetIndexedItemIds.mockResolvedValue( + new Set(["README.md", "gone.md"]), + ); + mockWalkSourceFiles.mockResolvedValue(new Set(["README.md"])); + + await runReindexAudit(["docs"]); + + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("does NOT throw when Slack fetch fails", async () => { + mockGetConfig.mockReturnValue( + appConfig({ slackWebhookUrl: "https://hooks.slack.com/test" }), + ); + mockFetch.mockRejectedValue(new Error("network error")); + mockGetIndexedItemIds.mockResolvedValue( + new Set(["README.md", "gone.md"]), + ); + mockWalkSourceFiles.mockResolvedValue(new Set(["README.md"])); + + // Should not throw + const findings = await runReindexAudit(["docs"]); + expect(findings).toBeInstanceOf(Array); + }); + + it("does NOT send Slack alert when no findings", async () => { + mockGetConfig.mockReturnValue( + appConfig({ slackWebhookUrl: "https://hooks.slack.com/test" }), + ); + // DB and disk match — no findings + const paths = new Set(["README.md"]); + mockGetIndexedItemIds.mockResolvedValue(new Set(paths)); + mockWalkSourceFiles.mockResolvedValue(new Set(paths)); + + await runReindexAudit(["docs"]); + + expect(mockFetch).not.toHaveBeenCalled(); + }); + }); + + // ── Error handling ──────────────────────────────────────────────────── + + describe("error handling", () => { + it("returns empty array and does not throw when getIndexedItemIds fails", async () => { + mockGetIndexedItemIds.mockRejectedValue(new Error("DB connection lost")); + + const findings = await runReindexAudit(["docs"]); + + expect(findings).toEqual([]); + }); + + it("returns empty array and does not throw when walkSourceFiles fails", async () => { + mockWalkSourceFiles.mockRejectedValue(new Error("ENOENT: no such file")); + + const findings = await runReindexAudit(["docs"]); + + expect(findings).toEqual([]); + }); + + it("returns empty array for empty sourceNames", async () => { + const findings = await runReindexAudit([]); + + expect(findings).toEqual([]); + expect(mockGetIndexedItemIds).not.toHaveBeenCalled(); + expect(mockWalkSourceFiles).not.toHaveBeenCalled(); + }); + }); + + // ── Integration: multiple sources ───────────────────────────────────── + + describe("multiple sources", () => { + it("returns findings for each source independently", async () => { + const docsSrc = fileSource("docs", { path: "/repo/docs" }); + const codeSrc = fileSource("code", { + type: "code", + path: "/repo/src", + file_patterns: ["**/*.ts"], + }); + mockGetServerConfig.mockReturnValue(serverConfig([docsSrc, codeSrc])); + + // docs: stale file + mockGetIndexedItemIds.mockImplementation((sourceName: string) => { + if (sourceName === "docs") { + return Promise.resolve(new Set(["README.md", "old.md"])); + } + if (sourceName === "code") { + return Promise.resolve(new Set(["index.ts"])); + } + return Promise.resolve(new Set()); + }); + + mockWalkSourceFiles.mockImplementation( + (sourceConfig: { name: string }) => { + if (sourceConfig.name === "docs") { + return Promise.resolve(new Set(["README.md"])); + } + if (sourceConfig.name === "code") { + // disk has more files than DB + return Promise.resolve( + new Set(["index.ts", "util.ts", "types.ts"]), + ); + } + return Promise.resolve(new Set()); + }, + ); + + const findings = await runReindexAudit(["docs", "code"]); + + // docs should have a stale_files finding + const docsStale = findings.find( + (f) => f.source === "docs" && f.check === "stale_files", + ); + expect(docsStale).toBeDefined(); + expect(docsStale!.count).toBe(1); + expect(docsStale!.samples).toContain("old.md"); + + // code should have a count_divergence finding (db_has_fewer) + const codeDivergence = findings.find( + (f) => f.source === "code" && f.check === "count_divergence", + ); + expect(codeDivergence).toBeDefined(); + expect(codeDivergence!.direction).toBe("db_has_fewer"); + }); + }); +}); diff --git a/src/__tests__/walk-source-files.test.ts b/src/__tests__/walk-source-files.test.ts new file mode 100644 index 0000000..803bde4 --- /dev/null +++ b/src/__tests__/walk-source-files.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { walkSourceFiles } from "../indexing/utils.js"; +import type { FileSourceConfig } from "../types.js"; + +describe("walkSourceFiles", () => { + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), "walk-source-test-"), + ); + }); + + afterEach(async () => { + await fs.promises.rm(tmpDir, { recursive: true, force: true }); + }); + + function makeSourceConfig( + overrides: Partial = {}, + ): FileSourceConfig { + return { + name: "test-source", + type: "markdown", + path: tmpDir, + file_patterns: ["**/*.md"], + chunk: { target_tokens: 500, overlap_tokens: 50 }, + ...overrides, + } as FileSourceConfig; + } + + async function writeFile(relPath: string, content: string): Promise { + const absPath = path.join(tmpDir, relPath); + await fs.promises.mkdir(path.dirname(absPath), { recursive: true }); + await fs.promises.writeFile(absPath, content); + } + + it("returns matching files from a local source directory", async () => { + await writeFile("README.md", "# Hello"); + await writeFile("docs/guide.md", "# Guide"); + await writeFile("docs/nested/deep.md", "# Deep"); + + const config = makeSourceConfig(); + const result = await walkSourceFiles(config, "/unused"); + + expect(result).toBeInstanceOf(Set); + expect(result.size).toBe(3); + expect(result.has("README.md")).toBe(true); + expect(result.has(path.join("docs", "guide.md"))).toBe(true); + expect(result.has(path.join("docs", "nested", "deep.md"))).toBe(true); + }); + + it("excludes files that don't match file_patterns", async () => { + await writeFile("README.md", "# Hello"); + await writeFile("index.ts", "export const x = 1;"); + await writeFile("styles.css", "body {}"); + + const config = makeSourceConfig({ file_patterns: ["**/*.md"] }); + const result = await walkSourceFiles(config, "/unused"); + + expect(result.size).toBe(1); + expect(result.has("README.md")).toBe(true); + expect(result.has("index.ts")).toBe(false); + expect(result.has("styles.css")).toBe(false); + }); + + it("excludes files in skip_dirs", async () => { + await writeFile("README.md", "# Hello"); + await writeFile("node_modules/pkg/README.md", "# Pkg"); + await writeFile("vendor/lib/README.md", "# Lib"); + await writeFile(".git/objects/README.md", "# Git"); + + // node_modules and .git are in the default skip list; add vendor + const config = makeSourceConfig({ skip_dirs: ["vendor"] }); + const result = await walkSourceFiles(config, "/unused"); + + expect(result.size).toBe(1); + expect(result.has("README.md")).toBe(true); + expect(result.has(path.join("node_modules", "pkg", "README.md"))).toBe( + false, + ); + expect(result.has(path.join("vendor", "lib", "README.md"))).toBe(false); + expect(result.has(path.join(".git", "objects", "README.md"))).toBe(false); + }); + + it("returns empty set when directory doesn't exist", async () => { + const config = makeSourceConfig({ + path: path.join(tmpDir, "nonexistent"), + }); + const result = await walkSourceFiles(config, "/unused"); + + expect(result).toBeInstanceOf(Set); + expect(result.size).toBe(0); + }); + + it("respects max_file_size", async () => { + const smallContent = "# Small file"; + const largeContent = "x".repeat(200_000); // 200KB > default 100KB + + await writeFile("small.md", smallContent); + await writeFile("large.md", largeContent); + + const config = makeSourceConfig(); + const result = await walkSourceFiles(config, "/unused"); + + expect(result.size).toBe(1); + expect(result.has("small.md")).toBe(true); + expect(result.has("large.md")).toBe(false); + }); + + it("respects custom max_file_size", async () => { + const content = "x".repeat(200_000); // 200KB + + await writeFile("big.md", content); + + // With a higher limit, the file should be included + const config = makeSourceConfig({ max_file_size: 300_000 }); + const result = await walkSourceFiles(config, "/unused"); + + expect(result.size).toBe(1); + expect(result.has("big.md")).toBe(true); + }); + + it("handles git source path resolution", async () => { + // Simulate a cloned repo structure: cloneDir/repoName/path/files + const cloneDir = path.join(tmpDir, "clones"); + const repoDir = path.join(cloneDir, "my-repo"); + const docsDir = path.join(repoDir, "docs"); + + await fs.promises.mkdir(docsDir, { recursive: true }); + await fs.promises.writeFile( + path.join(docsDir, "guide.md"), + "# Guide content", + ); + await fs.promises.writeFile( + path.join(docsDir, "api.md"), + "# API reference", + ); + + const config = makeSourceConfig({ + repo: "https://github.com/org/my-repo.git", + path: "docs", + }); + const result = await walkSourceFiles(config, cloneDir); + + expect(result.size).toBe(2); + // Paths should be relative to repoDir, not docsDir + expect(result.has(path.join("docs", "guide.md"))).toBe(true); + expect(result.has(path.join("docs", "api.md"))).toBe(true); + }); +}); From 85fce8a79fa38079a8d85d8978f3bf480a5a085a Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 8 May 2026 01:22:01 -0700 Subject: [PATCH 4/5] Bump version to 1.13.2 Post-reindex health audit, response compression, stale chunk cleanup. --- CHANGELOG.md | 8 ++++++++ package-lock.json | 4 ++-- package.json | 2 +- src/cli.ts | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4974725..c998650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # @copilotkit/pathfinder +## 1.13.2 + +### Patch Changes + +- **Post-Reindex Health Audit**: Automated integrity checks run after each reindex cycle — detects stale files in the index, scope leaks (files outside `config.path`), and chunk count divergence between DB and disk. Alerts via Slack webhook (`SLACK_WEBHOOK_URL` env var) when issues are found. Read-only — no automatic deletions. +- **Response Compression**: Added gzip/deflate/brotli compression via `compression` middleware for all HTTP responses. Text typically compresses 70-85%, especially beneficial for `/llms-full.txt`. +- **Stale Chunk Cleanup**: `fullAcquire` now detects and removes orphaned chunks from deleted files. Handles walkRoot-missing and git-rename cases. `incrementalAcquire` hardened with `config.path` scoping, `skipDirs` enforcement, and error handling on `--name-status` diff. + ## 1.13.1 ### Patch Changes diff --git a/package-lock.json b/package-lock.json index 22e7313..80f676a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@copilotkit/pathfinder", - "version": "1.13.1", + "version": "1.13.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@copilotkit/pathfinder", - "version": "1.13.1", + "version": "1.13.2", "license": "Elastic-2.0", "dependencies": { "@discordjs/rest": "^2.6.1", diff --git a/package.json b/package.json index 7b45ec2..959429c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@copilotkit/pathfinder", - "version": "1.13.1", + "version": "1.13.2", "description": "The knowledge server for AI agents — index docs, code, Notion, Slack, and Discord into searchable, agent-accessible knowledge via MCP. Supports OpenAI, Ollama, and local transformers.js embeddings.", "type": "module", "repository": { diff --git a/src/cli.ts b/src/cli.ts index 19431b8..998d1b0 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -11,7 +11,7 @@ const program = new Command(); program .name("pathfinder") .description("The knowledge server for AI agents") - .version("1.13.1"); + .version("1.13.2"); program .command("init") From 7a43007ac34db69b469ed22704c321610d859a74 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 8 May 2026 01:31:44 -0700 Subject: [PATCH 5/5] Fix audit false positives, Slack response checking, and walkRoot handling --- src/__tests__/reindex-audit.test.ts | 84 ++++++++++++++++++------- src/__tests__/walk-source-files.test.ts | 54 ++++++++-------- src/indexing/reindex-audit.ts | 22 +++++-- src/indexing/utils.ts | 10 ++- 4 files changed, 115 insertions(+), 55 deletions(-) diff --git a/src/__tests__/reindex-audit.test.ts b/src/__tests__/reindex-audit.test.ts index 0bad84f..63c3910 100644 --- a/src/__tests__/reindex-audit.test.ts +++ b/src/__tests__/reindex-audit.test.ts @@ -176,6 +176,24 @@ describe("runReindexAudit", () => { expect(stale!.samples.length).toBeLessThanOrEqual(10); }); + it("skips audit when walk root does not exist (walkSourceFiles returns null)", async () => { + mockWalkSourceFiles.mockResolvedValue(null); + mockGetIndexedItemIds.mockResolvedValue( + new Set(["README.md", "old.md", "stale.md"]), + ); + + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const findings = await runReindexAudit(["docs"]); + + expect(findings).toEqual([]); + expect(mockGetIndexedItemIds).not.toHaveBeenCalled(); + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("walk root not found"), + ); + consoleSpy.mockRestore(); + }); + it("skips non-file sources (e.g., slack, notion)", async () => { mockGetServerConfig.mockReturnValue( serverConfig([ @@ -243,21 +261,19 @@ describe("runReindexAudit", () => { expect(leak).toBeUndefined(); }); - it("skips scope check when config.path is '.' or empty", async () => { - for (const pathVal of [".", ""]) { - const src = fileSource("docs", { - repo: "https://github.com/org/repo.git", - path: pathVal || ".", - }); - mockGetServerConfig.mockReturnValue(serverConfig([src])); - - mockGetIndexedItemIds.mockResolvedValue(new Set(["anywhere/file.md"])); - mockWalkSourceFiles.mockResolvedValue(new Set(["anywhere/file.md"])); - - const findings = await runReindexAudit(["docs"]); - const leak = findings.find((f) => f.check === "scope_leak"); - expect(leak).toBeUndefined(); - } + it("skips scope check when config.path is '.'", async () => { + const src = fileSource("docs", { + repo: "https://github.com/org/repo.git", + path: ".", + }); + mockGetServerConfig.mockReturnValue(serverConfig([src])); + + mockGetIndexedItemIds.mockResolvedValue(new Set(["anywhere/file.md"])); + mockWalkSourceFiles.mockResolvedValue(new Set(["anywhere/file.md"])); + + const findings = await runReindexAudit(["docs"]); + const leak = findings.find((f) => f.check === "scope_leak"); + expect(leak).toBeUndefined(); }); }); @@ -278,7 +294,7 @@ describe("runReindexAudit", () => { expect(divergence!.count).toBe(2); // difference: 3 - 1 }); - it("returns count_divergence with direction db_has_fewer when disk > DB", async () => { + it("does not report divergence when disk > DB (db_has_fewer is expected from content filtering)", async () => { mockGetIndexedItemIds.mockResolvedValue(new Set(["a.md"])); mockWalkSourceFiles.mockResolvedValue( new Set(["a.md", "b.md", "c.md", "d.md"]), @@ -287,9 +303,7 @@ describe("runReindexAudit", () => { const findings = await runReindexAudit(["docs"]); const divergence = findings.find((f) => f.check === "count_divergence"); - expect(divergence).toBeDefined(); - expect(divergence!.direction).toBe("db_has_fewer"); - expect(divergence!.count).toBe(3); // difference: 4 - 1 + expect(divergence).toBeUndefined(); }); it("does not report divergence when counts match", async () => { @@ -342,6 +356,33 @@ describe("runReindexAudit", () => { expect(mockFetch).not.toHaveBeenCalled(); }); + it("logs error when Slack webhook returns non-ok response", async () => { + mockGetConfig.mockReturnValue( + appConfig({ slackWebhookUrl: "https://hooks.slack.com/test" }), + ); + mockFetch.mockResolvedValue({ + ok: false, + status: 403, + text: () => Promise.resolve("invalid_token"), + }); + mockGetIndexedItemIds.mockResolvedValue( + new Set(["README.md", "gone.md"]), + ); + mockWalkSourceFiles.mockResolvedValue(new Set(["README.md"])); + + const consoleSpy = vi + .spyOn(console, "error") + .mockImplementation(() => {}); + + const findings = await runReindexAudit(["docs"]); + + expect(findings).toBeInstanceOf(Array); + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("Slack webhook returned 403"), + ); + consoleSpy.mockRestore(); + }); + it("does NOT throw when Slack fetch fails", async () => { mockGetConfig.mockReturnValue( appConfig({ slackWebhookUrl: "https://hooks.slack.com/test" }), @@ -448,12 +489,11 @@ describe("runReindexAudit", () => { expect(docsStale!.count).toBe(1); expect(docsStale!.samples).toContain("old.md"); - // code should have a count_divergence finding (db_has_fewer) + // code: disk > DB (db_has_fewer) is no longer reported — expected from content filtering const codeDivergence = findings.find( (f) => f.source === "code" && f.check === "count_divergence", ); - expect(codeDivergence).toBeDefined(); - expect(codeDivergence!.direction).toBe("db_has_fewer"); + expect(codeDivergence).toBeUndefined(); }); }); }); diff --git a/src/__tests__/walk-source-files.test.ts b/src/__tests__/walk-source-files.test.ts index 803bde4..b5e6718 100644 --- a/src/__tests__/walk-source-files.test.ts +++ b/src/__tests__/walk-source-files.test.ts @@ -45,11 +45,11 @@ describe("walkSourceFiles", () => { const config = makeSourceConfig(); const result = await walkSourceFiles(config, "/unused"); - expect(result).toBeInstanceOf(Set); - expect(result.size).toBe(3); - expect(result.has("README.md")).toBe(true); - expect(result.has(path.join("docs", "guide.md"))).toBe(true); - expect(result.has(path.join("docs", "nested", "deep.md"))).toBe(true); + expect(result).not.toBeNull(); + expect(result!.size).toBe(3); + expect(result!.has("README.md")).toBe(true); + expect(result!.has(path.join("docs", "guide.md"))).toBe(true); + expect(result!.has(path.join("docs", "nested", "deep.md"))).toBe(true); }); it("excludes files that don't match file_patterns", async () => { @@ -60,10 +60,11 @@ describe("walkSourceFiles", () => { const config = makeSourceConfig({ file_patterns: ["**/*.md"] }); const result = await walkSourceFiles(config, "/unused"); - expect(result.size).toBe(1); - expect(result.has("README.md")).toBe(true); - expect(result.has("index.ts")).toBe(false); - expect(result.has("styles.css")).toBe(false); + expect(result).not.toBeNull(); + expect(result!.size).toBe(1); + expect(result!.has("README.md")).toBe(true); + expect(result!.has("index.ts")).toBe(false); + expect(result!.has("styles.css")).toBe(false); }); it("excludes files in skip_dirs", async () => { @@ -76,23 +77,23 @@ describe("walkSourceFiles", () => { const config = makeSourceConfig({ skip_dirs: ["vendor"] }); const result = await walkSourceFiles(config, "/unused"); - expect(result.size).toBe(1); - expect(result.has("README.md")).toBe(true); - expect(result.has(path.join("node_modules", "pkg", "README.md"))).toBe( + expect(result).not.toBeNull(); + expect(result!.size).toBe(1); + expect(result!.has("README.md")).toBe(true); + expect(result!.has(path.join("node_modules", "pkg", "README.md"))).toBe( false, ); - expect(result.has(path.join("vendor", "lib", "README.md"))).toBe(false); - expect(result.has(path.join(".git", "objects", "README.md"))).toBe(false); + expect(result!.has(path.join("vendor", "lib", "README.md"))).toBe(false); + expect(result!.has(path.join(".git", "objects", "README.md"))).toBe(false); }); - it("returns empty set when directory doesn't exist", async () => { + it("returns null when directory doesn't exist", async () => { const config = makeSourceConfig({ path: path.join(tmpDir, "nonexistent"), }); const result = await walkSourceFiles(config, "/unused"); - expect(result).toBeInstanceOf(Set); - expect(result.size).toBe(0); + expect(result).toBeNull(); }); it("respects max_file_size", async () => { @@ -105,9 +106,10 @@ describe("walkSourceFiles", () => { const config = makeSourceConfig(); const result = await walkSourceFiles(config, "/unused"); - expect(result.size).toBe(1); - expect(result.has("small.md")).toBe(true); - expect(result.has("large.md")).toBe(false); + expect(result).not.toBeNull(); + expect(result!.size).toBe(1); + expect(result!.has("small.md")).toBe(true); + expect(result!.has("large.md")).toBe(false); }); it("respects custom max_file_size", async () => { @@ -119,8 +121,9 @@ describe("walkSourceFiles", () => { const config = makeSourceConfig({ max_file_size: 300_000 }); const result = await walkSourceFiles(config, "/unused"); - expect(result.size).toBe(1); - expect(result.has("big.md")).toBe(true); + expect(result).not.toBeNull(); + expect(result!.size).toBe(1); + expect(result!.has("big.md")).toBe(true); }); it("handles git source path resolution", async () => { @@ -145,9 +148,10 @@ describe("walkSourceFiles", () => { }); const result = await walkSourceFiles(config, cloneDir); - expect(result.size).toBe(2); + expect(result).not.toBeNull(); + expect(result!.size).toBe(2); // Paths should be relative to repoDir, not docsDir - expect(result.has(path.join("docs", "guide.md"))).toBe(true); - expect(result.has(path.join("docs", "api.md"))).toBe(true); + expect(result!.has(path.join("docs", "guide.md"))).toBe(true); + expect(result!.has(path.join("docs", "api.md"))).toBe(true); }); }); diff --git a/src/indexing/reindex-audit.ts b/src/indexing/reindex-audit.ts index 2275688..4020714 100644 --- a/src/indexing/reindex-audit.ts +++ b/src/indexing/reindex-audit.ts @@ -32,6 +32,12 @@ export async function runReindexAudit( cfg.cloneDir, cfg.githubToken, ); + if (diskFiles === null) { + console.warn( + `[reindex-audit] Source "${sourceConfig.name}" walk root not found, skipping audit`, + ); + continue; + } const dbFiles = await getIndexedItemIds(sourceConfig.name); // Check 1 — Stale files: in DB but not on disk @@ -59,16 +65,17 @@ export async function runReindexAudit( } } - // Check 3 — Count divergence + // Check 3 — Count divergence (db_has_more only; db_has_fewer is expected + // when the indexer filters low-semantic-value files like SVGs, base64, etc.) const dbCount = dbFiles.size; const diskCount = diskFiles.size; - if (dbCount !== diskCount) { + if (dbCount > diskCount) { findings.push({ source: sourceConfig.name, check: "count_divergence", - count: Math.abs(dbCount - diskCount), + count: dbCount - diskCount, samples: [], - direction: dbCount > diskCount ? "db_has_more" : "db_has_fewer", + direction: "db_has_more", }); } } @@ -101,11 +108,16 @@ async function sendSlackAlert( }); const text = `🔍 *Reindex Audit Alert*\n${lines.join("\n\n")}`; try { - await fetch(webhookUrl, { + const response = await fetch(webhookUrl, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }), }); + if (!response.ok) { + console.error( + `[reindex-audit] Slack webhook returned ${response.status}: ${await response.text().catch(() => "(no body)")}`, + ); + } } catch (err) { console.error( "[reindex-audit] Failed to send Slack alert:", diff --git a/src/indexing/utils.ts b/src/indexing/utils.ts index a493e09..705bc02 100644 --- a/src/indexing/utils.ts +++ b/src/indexing/utils.ts @@ -108,7 +108,7 @@ export async function walkSourceFiles( sourceConfig: FileSourceConfig, cloneDir: string, _githubToken?: string, -): Promise> { +): Promise | null> { // Determine repo root directory const repoDir = sourceConfig.repo ? path.join(cloneDir, repoNameFromUrl(sourceConfig.repo)) @@ -120,7 +120,7 @@ export async function walkSourceFiles( : repoDir; if (!fs.existsSync(walkRoot)) { - return new Set(); + return null; } const skipDirs = new Set([ @@ -135,7 +135,11 @@ export async function walkSourceFiles( let entries: fs.Dirent[]; try { entries = await fs.promises.readdir(dir, { withFileTypes: true }); - } catch { + } catch (err) { + console.warn( + `[walkSourceFiles] Failed to read directory ${dir}:`, + err instanceof Error ? err.message : err, + ); return; }