feat: export dialog + MCP server for AI agents

README.md

@@ -215,6 +215,97 @@ npm --workspace apps/desktop run build:linux   # AppImage / .deb
 
 </details>
 
+<details>
+<summary><b>🤖 MCP server</b> — query your crawl from Claude / any MCP client</summary>
+
+<br />
+
+FreeCrawl ships an **MCP (Model Context Protocol) server** that exposes the active `.seoproject` to AI agents over stdio. Read-only — runs alongside the desktop app without contention (SQLite WAL).
+
+**Tools**: `get_summary`, `get_overview_counts`, `top_issues`, `query_urls`, `get_url_detail`, `list_projects`, `set_project`, `current_project`.
+
+**1. Build it once:**
+
+```bash
+npm run build:mcp
+```
+
+This produces `apps/mcp-server/dist/index.js`.
+
+**2. Register it with your MCP client.**
+
+<details>
+<summary><b>Claude Desktop</b></summary>
+
+Edit your Claude Desktop config:
+
+| Platform | Path |
+| :--- | :--- |
+| Windows | `%APPDATA%\Claude\claude_desktop_config.json` |
+| macOS | `~/Library/Application Support/Claude/claude_desktop_config.json` |
+| Linux | `~/.config/Claude/claude_desktop_config.json` |
+
+```json
+{
+  "mcpServers": {
+    "freecrawl": {
+      "command": "node",
+      "args": ["/absolute/path/to/FreeCrawl-SEO-Tool/apps/mcp-server/dist/index.js"]
+    }
+  }
+}
+```
+
+Restart Claude Desktop. The `freecrawl` server appears under the tool 🔌 icon.
+
+</details>
+
+<details>
+<summary><b>Claude Code (CLI)</b></summary>
+
+```bash
+claude mcp add freecrawl -- node /absolute/path/to/FreeCrawl-SEO-Tool/apps/mcp-server/dist/index.js
+```
+
+</details>
+
+<details>
+<summary><b>Other MCP clients</b></summary>
+
+Run the binary directly with stdio transport:
+
+```bash
+node apps/mcp-server/dist/index.js
+```
+
+The server speaks newline-delimited JSON-RPC 2.0 — point any MCP-compatible client at it.
+
+</details>
+
+**3. Try it.** Ask your agent things like:
+
+> *"Show the 10 URLs with the longest response time in my last crawl."*
+> *"What are the top 5 issue categories with the most affected pages?"*
+> *"List every URL with a missing meta description."*
+
+**Pointing at a non-default project:**
+
+By default the server reads `<userData>/projects/default.seoproject` (the same file the desktop app uses). Override with the `FREECRAWL_PROJECT` env var, or call the `set_project` tool mid-session:
+
+```json
+{
+  "mcpServers": {
+    "freecrawl": {
+      "command": "node",
+      "args": ["/path/to/apps/mcp-server/dist/index.js"],
+      "env": { "FREECRAWL_PROJECT": "/path/to/audit.seoproject" }
+    }
+  }
+}
+```
+
+</details>
+
 <br />
 
 ---

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "private": true,
   "type": "module",
   "bin": {

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "main": "./out/main/index.js",

apps/desktop/src/main/db-reader-pool.ts

@@ -4,43 +4,48 @@ import { fileURLToPath } from 'node:url';
 import * as logger from './logger.js';
 
 /**
- * Pool of one worker_thread holding a read-only SQLite connection.
+ * Pool of N worker_threads, each holding its own read-only SQLite
- * "Pool" is forward-compatible naming — for now we run a single worker
+ * connection. SQLite WAL mode allows arbitrary concurrent readers, so
- * because SQLite's WAL allows arbitrary concurrent readers but adding a
+ * spreading queries across multiple workers gives genuine parallelism
- * second worker would just double our cache footprint without gaining
+ * for UI reads — critical because:
- * parallelism on a single physical disk.
+ *
+ *   - The OverviewSidebar refresh issues a 130-clause `getOverviewCounts`
+ *     every few seconds. On a single-worker pool this serialises every
+ *     other read behind it, freezing the URL table refresh and the
+ *     bottom panel's per-URL detail / inlinks queries until the heavy
+ *     pass completes.
+ *   - Pool now lets the heavy aggregate run on one worker while the
+ *     short, latency-sensitive queries (urlsQuery for the visible
+ *     chunks, urlDetail for the bottom panel) run on the others.
+ *
+ * Routing: round-robin with a "least busy" tie-break — for each call
+ * we pick the worker with the fewest pending requests, and on a tie
+ * fall back to a rotating cursor so two same-loaded workers don't
+ * always get the next call dropped on the same one.
  *
  * Lifecycle:
- *   - `init(dbPath)` spawns the worker, points it at the file.
+ *   - `init(dbPath)` spawns all workers, points them at the file.
- *   - `swap(newPath)` is called on Open Project — terminates the old
+ *   - `swap(newPath)` is called on Open Project — terminates every old
- *     worker and spawns a fresh one; in-flight requests are rejected
+ *     worker and spawns a fresh set; in-flight requests are rejected
  *     with `Error('reader-swapped')` so callers can retry safely.
  *   - `terminate()` on app quit. After terminate, all `call()` resolve
  *     with `Error('reader-terminated')`.
  *
  * Crash recovery:
- *   - If the worker exits unexpectedly (`exit` event with non-zero code
+ *   - If a worker exits unexpectedly, that one slot auto-respawns up to
- *     while we still have a path) we auto-respawn up to MAX_RESTARTS
+ *     MAX_RESTARTS within RESTART_WINDOW_MS. Other workers in the pool
- *     within RESTART_WINDOW_MS. After the cap we surface the failure
+ *     keep serving traffic.
- *     and stop trying — the user gets a clear log entry instead of a
+ *   - In-flight requests on the crashed worker are rejected with
- *     restart loop.
- *   - In-flight requests during a crash are rejected with
  *     `Error('reader-crashed')`. Callers fall back to the main-process
- *     ProjectDb (see `dbReaderCallOrFallback`) so the UI stays alive.
+ *     ProjectDb (see `callReaderOrFallback`) so the UI stays alive.
  */
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-// The worker is built alongside the main bundle as `db-reader-worker.js`
-// (electron-vite produces flat output in `out/main/`). In dev mode
-// electron-vite uses the same convention.
 const WORKER_PATH = path.join(__dirname, 'db-reader-worker.js');
 
 // 30 s default leaves headroom for SQLite read contention during the
-// post-crawl recompute phase (recomputeUrlsIssues holds a writer
+// post-crawl recompute phase. Genuine worker hangs are still surfaced
-// transaction that can keep reader queries blocked behind earlier
+// — just with a slightly longer detection window than before.
-// queued requests for several seconds at a time on large crawls).
-// Genuine worker hangs are still surfaced — just with a slightly
-// longer detection window than before.
 const REQUEST_TIMEOUT_MS = 30_000;
 // Heavy aggregate queries (the overview sidebar's 130-counter pass and
 // the post-crawl materialiser's counter fan-out) can run past the
@@ -54,7 +59,17 @@ const HEAVY_METHODS = new Set<string>([
 const MAX_RESTARTS = 3;
 const RESTART_WINDOW_MS = 60_000;
 
+// Reader pool size. Four workers gives the typical UI workload (one
+// long-running aggregate from the sidebar + visible-table chunk fetch
+// + bottom-panel detail + crawl-engine background reads) a free slot
+// for every concurrent request; bigger doesn't help because most UI
+// reads are short and a dedicated worker per query becomes pure
+// memory cost. Each worker keeps its own SQLite handle (~5 MB heap)
+// and prepared-statement cache (~10 MB on a populated DB).
+const POOL_SIZE = 4;
+
 interface PendingRequest {
+  workerIdx: number;
   resolve: (v: unknown) => void;
   reject: (e: Error) => void;
   timer: ReturnType<typeof setTimeout>;
@@ -67,94 +82,143 @@ interface ResponseMessage {
   error?: string;
 }
 
+interface WorkerSlot {
+  worker: Worker | null;
+  pending: number;
+  restartTimes: number[];
+}
+
 class DbReaderPool {
-  private worker: Worker | null = null;
+  private slots: WorkerSlot[] = [];
   private dbPath: string | null = null;
   private freezeWatchdogSab: SharedArrayBuffer | null = null;
   private nextRequestId = 1;
   private pending = new Map<number, PendingRequest>();
-  private restartTimes: number[] = [];
+  private rrCursor = 0;
   private terminated = false;
 
-  /** Spawn (or respawn) the worker pointed at `dbPath`. Idempotent.
+  /** Spawn (or respawn) the pool pointed at `dbPath`. Idempotent. */
-   *
-   *   `freezeWatchdogSab` — if provided, the worker will tick its
-   *   own heartbeat + publish its current method into this shared
-   *   buffer so the freeze-watchdog thread can detect a stuck
-   *   reader. Pass `null` to disable diagnostics. */
   init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
     if (this.terminated) {
       throw new Error('DbReaderPool: cannot init after terminate()');
     }
     this.freezeWatchdogSab = freezeWatchdogSab;
-    if (this.dbPath === dbPath && this.worker !== null) return;
+    if (this.dbPath === dbPath && this.slots.some((s) => s.worker !== null)) return;
     this.dbPath = dbPath;
-    this.spawn();
+    this.slots = Array.from({ length: POOL_SIZE }, () => ({
+      worker: null,
+      pending: 0,
+      restartTimes: [],
+    }));
+    for (let i = 0; i < POOL_SIZE; i++) this.spawn(i);
   }
 
-  /** Switch the worker to a different .seoproject file. */
+  /** Switch the pool to a different .seoproject file. */
   swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
     if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab;
-    if (this.dbPath === newPath && this.worker !== null) return;
+    if (this.dbPath === newPath && this.slots.some((s) => s.worker !== null)) return;
     this.failPendingWith('reader-swapped');
     this.dbPath = newPath;
-    this.spawn();
+    if (this.slots.length === 0) {
+      this.slots = Array.from({ length: POOL_SIZE }, () => ({
+        worker: null,
+        pending: 0,
+        restartTimes: [],
+      }));
+    }
+    for (let i = 0; i < POOL_SIZE; i++) this.spawn(i);
   }
 
   /** Permanent shutdown — used on app quit. */
   async terminate(): Promise<void> {
     this.terminated = true;
     this.failPendingWith('reader-terminated');
-    if (this.worker) {
+    const workers = this.slots.map((s) => s.worker).filter((w): w is Worker => w !== null);
-      const w = this.worker;
+    for (const slot of this.slots) slot.worker = null;
-      this.worker = null;
+    await Promise.all(
-      try {
+      workers.map((w) =>
-        await w.terminate();
+        w.terminate().catch(() => undefined),
-      } catch {
+      ),
-        /* already gone */
+    );
-      }
-    }
   }
 
-  /** True while a worker is up and ready to receive requests. */
+  /** True while at least one worker is up and ready. */
   isReady(): boolean {
-    return this.worker !== null && !this.terminated;
+    return !this.terminated && this.slots.some((s) => s.worker !== null);
   }
 
   /**
-   * Dispatch a method call to the worker. Callers should use the typed
+   * Dispatch a method call to the least-busy ready worker. Resolves
-   * wrapper `callReader<T>(method, args)` declared in the main process
+   * with the method's return value, rejects with an Error on worker
-   * IPC layer. Resolves with the method's return value, rejects with
+   * error / timeout / crash.
-   * an Error on worker error / timeout / crash.
    */
   call<T>(method: string, args: unknown[] = []): Promise<T> {
     if (this.terminated) return Promise.reject(new Error('reader-terminated'));
-    if (!this.worker) return Promise.reject(new Error('reader-not-initialised'));
+    const slotIdx = this.pickSlot();
+    if (slotIdx < 0) return Promise.reject(new Error('reader-not-initialised'));
+    const slot = this.slots[slotIdx]!;
+    const worker = slot.worker;
+    if (!worker) return Promise.reject(new Error('reader-not-initialised'));
     const requestId = this.nextRequestId++;
     const timeoutMs = HEAVY_METHODS.has(method)
       ? HEAVY_REQUEST_TIMEOUT_MS
       : REQUEST_TIMEOUT_MS;
+    slot.pending++;
     return new Promise<T>((resolve, reject) => {
       const timer = setTimeout(() => {
-        this.pending.delete(requestId);
+        const p = this.pending.get(requestId);
+        if (p) {
+          this.pending.delete(requestId);
+          this.slots[p.workerIdx]!.pending = Math.max(
+            0,
+            this.slots[p.workerIdx]!.pending - 1,
+          );
+        }
         reject(new Error(`reader-timeout: ${method} > ${timeoutMs}ms`));
       }, timeoutMs);
       this.pending.set(requestId, {
+        workerIdx: slotIdx,
         resolve: resolve as (v: unknown) => void,
         reject,
         timer,
       });
-      this.worker!.postMessage({ requestId, method, args });
+      worker.postMessage({ requestId, method, args });
     });
   }
 
   // ── private ──────────────────────────────────────────────────────────────
 
-  private spawn(): void {
+  /**
+   * Pick the slot with the fewest pending requests. Ties are broken
+   * by a round-robin cursor so identically-loaded workers don't all
+   * receive their next call on the same slot.
+   */
+  private pickSlot(): number {
+    let best = -1;
+    let bestPending = Number.POSITIVE_INFINITY;
+    // Start the scan from the rotating cursor so RR is honoured on ties.
+    for (let i = 0; i < this.slots.length; i++) {
+      const idx = (this.rrCursor + i) % this.slots.length;
+      const s = this.slots[idx]!;
+      if (!s.worker) continue;
+      if (s.pending < bestPending) {
+        best = idx;
+        bestPending = s.pending;
+      }
+    }
+    if (best >= 0) {
+      this.rrCursor = (best + 1) % this.slots.length;
+    }
+    return best;
+  }
+
+  private spawn(slotIdx: number): void {
     if (!this.dbPath) return;
-    if (this.worker) {
+    const slot = this.slots[slotIdx];
-      const old = this.worker;
+    if (!slot) return;
-      this.worker = null;
+    if (slot.worker) {
+      const old = slot.worker;
+      slot.worker = null;
       void old.terminate().catch(() => undefined);
     }
     try {
@@ -166,18 +230,19 @@ class DbReaderPool {
       });
       w.on('message', (msg: ResponseMessage) => this.handleResponse(msg));
       w.on('error', (err) => {
-        logger.log('error', 'db-reader', `worker error: ${err.message}`);
+        logger.log('error', 'db-reader', `worker[${slotIdx}] error: ${err.message}`);
       });
-      w.on('exit', (code) => this.handleExit(code));
+      w.on('exit', (code) => this.handleExit(slotIdx, code));
-      this.worker = w;
+      slot.worker = w;
-      logger.log('info', 'db-reader', `worker spawned for ${this.dbPath}`);
+      slot.pending = 0;
+      logger.log('info', 'db-reader', `worker[${slotIdx}] spawned for ${this.dbPath}`);
     } catch (err) {
       logger.log(
         'error',
         'db-reader',
-        `worker spawn failed: ${err instanceof Error ? err.message : String(err)}`,
+        `worker[${slotIdx}] spawn failed: ${err instanceof Error ? err.message : String(err)}`,
       );
-      this.worker = null;
+      slot.worker = null;
     }
   }
 
@@ -186,6 +251,8 @@ class DbReaderPool {
     if (!pending) return;
     this.pending.delete(msg.requestId);
     clearTimeout(pending.timer);
+    const slot = this.slots[pending.workerIdx];
+    if (slot) slot.pending = Math.max(0, slot.pending - 1);
     if (msg.ok) {
       pending.resolve(msg.result);
     } else {
@@ -193,31 +260,38 @@ class DbReaderPool {
     }
   }
 
-  private handleExit(code: number): void {
+  private handleExit(slotIdx: number, code: number): void {
-    this.worker = null;
+    const slot = this.slots[slotIdx];
+    if (!slot) return;
+    slot.worker = null;
     if (this.terminated) return;
+    // Reject only this slot's in-flight requests.
+    for (const [id, p] of this.pending) {
+      if (p.workerIdx === slotIdx) {
+        clearTimeout(p.timer);
+        p.reject(new Error('reader-crashed'));
+        this.pending.delete(id);
+      }
+    }
+    slot.pending = 0;
     if (code === 0) return;
-    this.failPendingWith('reader-crashed');
-    // Restart bookkeeping: drop entries older than the window so a
-    // long-lived process that crashes once a day doesn't permanently
-    // exhaust its restart budget.
     const now = Date.now();
-    this.restartTimes = this.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS);
+    slot.restartTimes = slot.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS);
-    if (this.restartTimes.length >= MAX_RESTARTS) {
+    if (slot.restartTimes.length >= MAX_RESTARTS) {
       logger.log(
         'error',
         'db-reader',
-        `worker crashed ${MAX_RESTARTS}× within ${RESTART_WINDOW_MS}ms — giving up. UI queries fall back to main-process DB.`,
+        `worker[${slotIdx}] crashed ${MAX_RESTARTS}× within ${RESTART_WINDOW_MS}ms — giving up on this slot.`,
       );
       return;
     }
-    this.restartTimes.push(now);
+    slot.restartTimes.push(now);
     logger.log(
       'warn',
       'db-reader',
-      `worker exited with code ${code}; respawning (${this.restartTimes.length}/${MAX_RESTARTS})`,
+      `worker[${slotIdx}] exited with code ${code}; respawning (${slot.restartTimes.length}/${MAX_RESTARTS})`,
     );
-    this.spawn();
+    this.spawn(slotIdx);
   }
 
   private failPendingWith(reason: string): void {
@@ -226,6 +300,7 @@ class DbReaderPool {
       p.reject(new Error(reason));
     }
     this.pending.clear();
+    for (const slot of this.slots) slot.pending = 0;
   }
 }
 
@@ -233,10 +308,10 @@ class DbReaderPool {
 export const dbReaderPool = new DbReaderPool();
 
 /**
- * Thin helper for IPC handlers: try the worker first, fall back to the
+ * Thin helper for IPC handlers: try the worker pool first, fall back
- * synchronous main-process DB on any worker error. The fallback path
+ * to the synchronous main-process DB on any worker error. The fallback
- * keeps the UI working even if the worker is crashed/restarting, at
+ * path keeps the UI working even if every worker is crashed/restarting,
- * the cost of running on the main thread for that one query.
+ * at the cost of running on the main thread for that one query.
  */
 export async function callReaderOrFallback<T>(
   method: string,

apps/desktop/src/main/index.ts

@@ -25,6 +25,8 @@ import {
   type ExportJsonResult,
   type ExportXmlInput,
   type ExportXmlResult,
+  type ExportTabularInput,
+  type ExportTabularResult,
   type DataDeleteByDomainInput,
   type DataDeleteByDomainResult,
   type CrashRecoveryStatus,
@@ -81,6 +83,7 @@ import {
   exportUrlsToCsv,
   exportUrlsToJson,
   exportUrlsToXml,
+  exportTabular,
   exportSitemap,
   exportHtmlReport,
   compareCrawls,
@@ -366,6 +369,64 @@ async function writeFetchedUrlViaPool(
   }
 }
 
+/**
+ * Run the post-crawl issue recompute on the writer worker. This pass
+ * has 70+ correlated SQL subqueries against the urls table; on a
+ * 5K-URL crawl it takes ~45 s, and running it on the main thread
+ * freezes the renderer for that entire window (the "Not Responding"
+ * dialog). Routing through the writer worker keeps main free.
+ */
+async function recomputeIssuesViaPool(
+  definitions: ReadonlyArray<readonly [string, string]>,
+): Promise<void> {
+  if (!dbWriterPool.isReady()) {
+    return getDb().recomputeUrlsIssuesYielding(definitions);
+  }
+  try {
+    await dbWriterPool.call<void>('recomputeUrlsIssuesYielding', [definitions]);
+  } catch (err) {
+    logger.log(
+      'warn',
+      'main',
+      `recomputeUrlsIssuesYielding dispatch failed (${
+        err instanceof Error ? err.message : String(err)
+      }) — falling back to main-thread recompute.`,
+    );
+    await getDb().recomputeUrlsIssuesYielding(definitions);
+  }
+}
+
+/**
+ * Generic post-crawl pass dispatcher. Routes recomputeInlinks,
+ * recomputeRedirectChains, recomputeHreflangAnalysis,
+ * recomputeHreflangInconsistent, and recomputePaginationSequence
+ * through the writer worker so the main thread stays responsive to
+ * Stop/Pause clicks and IPC traffic during the post-crawl phase.
+ * Without this, those five sync SQL passes combined freeze main for
+ * 5–15 s on a 5K-URL crawl — exactly the window where users hit
+ * Stop/Pause and "nothing happens".
+ */
+async function runDbPassViaPool(method: string): Promise<void> {
+  if (!dbWriterPool.isReady()) {
+    const fn = (getDb() as unknown as Record<string, unknown>)[method];
+    if (typeof fn === 'function') (fn as () => void).call(getDb());
+    return;
+  }
+  try {
+    await dbWriterPool.call<void>(method, []);
+  } catch (err) {
+    logger.log(
+      'warn',
+      'main',
+      `${method} dispatch failed (${
+        err instanceof Error ? err.message : String(err)
+      }) — falling back to main-thread.`,
+    );
+    const fn = (getDb() as unknown as Record<string, unknown>)[method];
+    if (typeof fn === 'function') (fn as () => void).call(getDb());
+  }
+}
+
 function getDb(): ProjectDb {
   if (!db) {
     const dataDir = join(app.getPath('userData'), 'projects');
@@ -1160,6 +1221,8 @@ function registerIpc(): void {
       setOp: (op: string) => freezeWatchdog.setMainOp(op),
       parseHtml: parseHtmlViaPool,
       writeFetchedUrl: writeFetchedUrlViaPool,
+      recomputeIssues: recomputeIssuesViaPool,
+      runDbPass: runDbPassViaPool,
     });
     activeCrawler = crawler;
     attachCrawlerListeners(crawler);
@@ -1178,11 +1241,29 @@ function registerIpc(): void {
     activeCrawler?.resume();
   });
 
-  ipcMain.handle(IPC.crawlClear, () => {
+  ipcMain.handle(IPC.crawlClear, async () => {
     activeCrawler?.stop();
     activeCrawler = null;
     const database = getDb();
-    database.reset();
+    // Route reset() through the writer pool so it sits behind any
+    // in-flight writeFetchedUrl RPCs in the worker's FIFO. Without
+    // this, the main process truncates the urls table while the
+    // writer worker is still draining the last batch of per-URL
+    // writes — those writes win the SQLite lock race and resurrect
+    // ~19 rows in the table after Clear. Going through the pool
+    // serialises naturally: every queued write completes, then reset
+    // runs against an already-stopped crawler.
+    if (dbWriterPool.isReady()) {
+      try {
+        await dbWriterPool.call('reset', []);
+      } catch {
+        // Worker terminated or RPC timed out — fall back to direct
+        // reset rather than leaving the table half-cleared.
+        database.reset();
+      }
+    } else {
+      database.reset();
+    }
     // Belt-and-braces: reset() above already includes a DELETE FROM
     // crawl_queue, but if anything (a late-firing checkpoint timer, a
     // half-finished post-crawl pass) writes to that table after the
@@ -1661,6 +1742,47 @@ function registerIpc(): void {
     },
   );
 
+  ipcMain.handle(
+    IPC.exportTabular,
+    async (_e, input: ExportTabularInput): Promise<ExportTabularResult> => {
+      const { format, sections, columns, selectedIds } = input;
+      let outputPath = input.filePath ?? '';
+      const isSelection = (selectedIds?.length ?? 0) > 0;
+      const multiSection = sections.length > 1;
+      if (!outputPath) {
+        if (format === 'csv' && multiSection) {
+          const res = await dialog.showOpenDialog(mainWindow!, {
+            properties: ['openDirectory', 'createDirectory'],
+            title: 'Choose folder for CSV export',
+          });
+          if (res.canceled || res.filePaths.length === 0) {
+            return { filePath: '', files: [], rowsWritten: 0 };
+          }
+          outputPath = res.filePaths[0]!;
+        } else {
+          const ext = format === 'xlsx' ? 'xlsx' : 'csv';
+          const filterName = format === 'xlsx' ? 'Excel Workbook' : 'CSV';
+          const baseName = isSelection ? 'freecrawl-selected' : 'freecrawl-export';
+          const res = await dialog.showSaveDialog(mainWindow!, {
+            defaultPath: `${baseName}.${ext}`,
+            filters: [{ name: filterName, extensions: [ext] }],
+          });
+          if (res.canceled || !res.filePath) {
+            return { filePath: '', files: [], rowsWritten: 0 };
+          }
+          outputPath = res.filePath;
+        }
+      }
+      const result = await exportTabular(getDb(), outputPath, {
+        format,
+        sections,
+        columns,
+        selectedIds,
+      });
+      return result;
+    },
+  );
+
   ipcMain.handle(
     IPC.dataDeleteByDomain,
     async (
@@ -1707,6 +1829,8 @@ function registerIpc(): void {
         setOp: (op: string) => freezeWatchdog.setMainOp(op),
         parseHtml: parseHtmlViaPool,
         writeFetchedUrl: writeFetchedUrlViaPool,
+      recomputeIssues: recomputeIssuesViaPool,
+      runDbPass: runDbPassViaPool,
       });
       activeCrawler = crawler;
       attachCrawlerListeners(crawler);

apps/desktop/src/preload/index.ts

@@ -11,6 +11,8 @@ import {
   type ExportJsonResult,
   type ExportXmlInput,
   type ExportXmlResult,
+  type ExportTabularInput,
+  type ExportTabularResult,
   type DataDeleteByDomainInput,
   type DataDeleteByDomainResult,
   type CrashRecoveryStatus,
@@ -123,6 +125,8 @@ const api: FreeCrawlApi = {
     ipcRenderer.invoke(IPC.exportJson, input),
   exportXml: (input: ExportXmlInput): Promise<ExportXmlResult> =>
     ipcRenderer.invoke(IPC.exportXml, input),
+  exportTabular: (input: ExportTabularInput): Promise<ExportTabularResult> =>
+    ipcRenderer.invoke(IPC.exportTabular, input),
   dataDeleteByDomain: (
     input: DataDeleteByDomainInput,
   ): Promise<DataDeleteByDomainResult> =>

apps/desktop/src/renderer/src/App.tsx

@@ -137,20 +137,30 @@ export function App() {
     // previous session left a non-empty checkpoint we ask whether to
     // resume. Discarding wipes the snapshot so the prompt doesn't
     // recur on next launch.
+    //
+    // The confirm() is wrapped in a 250 ms setTimeout so the initial
+    // render finishes (URL input mounts, status bar paints) before
+    // the modal opens. Without the defer, the synchronous `confirm`
+    // can fire on first paint and — when Electron renders the modal
+    // behind the main window on some Windows configurations — leaves
+    // the user staring at an "unresponsive" UI: the URL input can't
+    // be clicked because a hidden modal is blocking the renderer.
     void window.freecrawl.crashRecoveryStatus().then((status) => {
       if (status.pendingCount === 0) return;
-      const proceed = window.confirm(
+      window.setTimeout(() => {
-        `FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` +
+        const proceed = window.confirm(
-          `  Start URL: ${status.seedUrl}\n` +
+          `FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` +
-          `  Pending URLs: ${status.pendingCount.toLocaleString()}\n\n` +
+            `  Start URL: ${status.seedUrl}\n` +
-          `Resume the crawl from where it stopped?\n\n` +
+            `  Pending URLs: ${status.pendingCount.toLocaleString()}\n\n` +
-          `Click OK to resume, Cancel to discard the recovery state and start fresh.`,
+            `Resume the crawl from where it stopped?\n\n` +
-      );
+            `Click OK to resume, Cancel to discard the recovery state and start fresh.`,
-      if (proceed) {
+        );
-        void window.freecrawl.crashRecoveryResume();
+        if (proceed) {
-      } else {
+          void window.freecrawl.crashRecoveryResume();
-        void window.freecrawl.crashRecoveryDiscard();
+        } else {
-      }
+          void window.freecrawl.crashRecoveryDiscard();
+        }
+      }, 250);
     });
 
     // rAF-throttled progress dispatch. The crawler emits up to 5

apps/desktop/src/renderer/src/components/ErrorBoundary.tsx

@@ -0,0 +1,60 @@
+import { Component, type ReactNode } from 'react';
+
+interface Props {
+  /** Rendered when the boundary's subtree throws. Defaults to a tiny notice. */
+  fallback?: ReactNode;
+  /** Surfaced to console + main-process log so the user can copy/paste. */
+  context?: string;
+  children: ReactNode;
+}
+
+interface State {
+  err: Error | null;
+}
+
+/**
+ * Catches render-time errors in a subtree so a single misbehaving dialog
+ * (e.g. ExportDialog) can't unmount the whole renderer and leave the user
+ * staring at the Electron window's bare background colour. The boundary
+ * logs the error and shows a small inline notice; the rest of the app
+ * keeps working.
+ */
+export class ErrorBoundary extends Component<Props, State> {
+  override state: State = { err: null };
+
+  static getDerivedStateFromError(err: Error): State {
+    return { err };
+  }
+
+  override componentDidCatch(err: Error, info: { componentStack?: string }): void {
+    // eslint-disable-next-line no-console
+    console.error(
+      `[ErrorBoundary${this.props.context ? `:${this.props.context}` : ''}]`,
+      err,
+      info.componentStack,
+    );
+  }
+
+  reset = (): void => {
+    this.setState({ err: null });
+  };
+
+  override render(): ReactNode {
+    const { err } = this.state;
+    if (!err) return this.props.children;
+    if (this.props.fallback !== undefined) return this.props.fallback;
+    return (
+      <div className="fixed inset-x-0 top-2 z-[60] mx-auto w-fit max-w-[90vw] rounded border border-red-700/60 bg-red-950/90 px-3 py-2 text-[12px] text-red-200 shadow-lg">
+        <div className="font-semibold">Something broke in this panel.</div>
+        <div className="mt-0.5 truncate font-mono text-[11px] text-red-300">{err.message}</div>
+        <button
+          type="button"
+          onClick={this.reset}
+          className="mt-1 rounded border border-red-700/60 px-2 py-0.5 text-[11px] text-red-200 hover:bg-red-900/50"
+        >
+          Dismiss
+        </button>
+      </div>
+    );
+  }
+}

apps/desktop/src/renderer/src/components/ExportDialog.tsx

@@ -0,0 +1,403 @@
+import { useEffect, useMemo, useState } from 'react';
+import { X, Loader2 } from 'lucide-react';
+import clsx from 'clsx';
+import type { CrawlUrlRow, ExportTabularSection, UrlCategory } from '@freecrawl/shared-types';
+import { TAB_ORDER, type TabKey } from '../store.js';
+import { COLUMN_SPECS, columnId } from '../tabs/columns.js';
+
+type Format = 'csv' | 'xlsx';
+
+interface ExportTab {
+  key: TabKey;
+  label: string;
+  category: UrlCategory;
+  columnKeys: (keyof CrawlUrlRow)[];
+}
+
+const TAB_CATEGORY: Record<TabKey, UrlCategory> = {
+  internal: 'internal:html',
+  external: 'external:all',
+  'response-codes': 'all',
+  url: 'internal:html',
+  'page-titles': 'internal:html',
+  'meta-description': 'internal:html',
+  h1: 'internal:html',
+  h2: 'internal:html',
+  content: 'internal:html',
+  images: 'internal:html',
+  canonicals: 'internal:html',
+  directives: 'internal:html',
+  redirects: 'all',
+  links: 'all',
+  'broken-links': 'all',
+};
+
+function buildExportableTabs(): ExportTab[] {
+  const out: ExportTab[] = [];
+  for (const t of TAB_ORDER) {
+    const cols = COLUMN_SPECS[t.key];
+    // Tabs with their own non-URL-row layouts (Images / Broken Links)
+    // have empty COLUMN_SPECS — skip them so the export grid only shows
+    // categories that share the URL-row schema.
+    if (!cols || cols.length === 0) continue;
+    out.push({
+      key: t.key,
+      label: t.label,
+      category: TAB_CATEGORY[t.key],
+      columnKeys: cols.map((c) => c.key),
+    });
+  }
+  return out;
+}
+
+interface ExportDialogProps {
+  open: boolean;
+  onClose: () => void;
+  /** The tab the user clicked Export from — drives the default selection. */
+  defaultTab: TabKey;
+  /** Row ids selected in the current table; falsy → export the whole filter set. */
+  selectedIds?: number[];
+}
+
+export function ExportDialog({
+  open,
+  onClose,
+  defaultTab,
+  selectedIds,
+}: ExportDialogProps) {
+  const tabs = useMemo(() => buildExportableTabs(), []);
+  const initialTabKey: TabKey = useMemo(() => {
+    return tabs.find((t) => t.key === defaultTab)?.key ?? tabs[0]?.key ?? 'internal';
+  }, [defaultTab, tabs]);
+
+  const selectionCount = selectedIds?.length ?? 0;
+
+  const [format, setFormat] = useState<Format>('csv');
+  const [pickedTabs, setPickedTabs] = useState<Set<TabKey>>(() => new Set([initialTabKey]));
+  const [pickedColumns, setPickedColumns] = useState<Set<string>>(() => {
+    const t = tabs.find((x) => x.key === initialTabKey);
+    return new Set((t?.columnKeys ?? []) as string[]);
+  });
+  const [useSelection, setUseSelection] = useState<boolean>(selectionCount > 0);
+  const [busy, setBusy] = useState(false);
+  const [status, setStatus] = useState<string>('');
+
+  // The dialog is conditionally mounted by the parent (only when open=true),
+  // so useState initializers above already populate the right defaults on
+  // each open. Escape key dismissal is the only ambient behaviour we need.
+  useEffect(() => {
+    if (!open) return;
+    const onKey = (e: KeyboardEvent): void => {
+      if (e.key === 'Escape') onClose();
+    };
+    document.addEventListener('keydown', onKey);
+    return () => document.removeEventListener('keydown', onKey);
+  }, [open, onClose]);
+
+  // Union of column keys exposed by every picked tab — these are the
+  // columns the dialog offers; checked state lives in `pickedColumns`.
+  const offeredColumns = useMemo(() => {
+    const seen = new Set<string>();
+    const out: { key: string; header: string; tab: TabKey }[] = [];
+    for (const t of tabs) {
+      if (!pickedTabs.has(t.key)) continue;
+      const specs = COLUMN_SPECS[t.key];
+      for (const spec of specs) {
+        const id = columnId(spec);
+        if (seen.has(id)) continue;
+        seen.add(id);
+        out.push({ key: spec.key as string, header: spec.header, tab: t.key });
+      }
+    }
+    return out;
+  }, [pickedTabs, tabs]);
+
+  const orderedColumnList = useMemo(
+    () => offeredColumns.filter((c) => pickedColumns.has(c.key)).map((c) => c.key),
+    [offeredColumns, pickedColumns],
+  );
+
+  if (!open) return null;
+
+  const toggleTab = (key: TabKey) => {
+    setPickedTabs((prev) => {
+      const next = new Set(prev);
+      let willBeRemoved = false;
+      if (next.has(key)) {
+        if (next.size === 1) return prev; // keep at least one
+        next.delete(key);
+        willBeRemoved = true;
+      } else {
+        next.add(key);
+      }
+      // Auto-tick the new tab's columns when it joins the picked set, so
+      // the user doesn't have to re-check them. When a tab leaves we keep
+      // the user's column picks alone — orderedColumnList already drops
+      // any column not offered by remaining tabs.
+      if (!willBeRemoved) {
+        const t = tabs.find((x) => x.key === key);
+        if (t) {
+          setPickedColumns((prevCols) => {
+            const nextCols = new Set(prevCols);
+            for (const k of t.columnKeys) nextCols.add(k as string);
+            return nextCols;
+          });
+        }
+      }
+      return next;
+    });
+  };
+
+  const toggleColumn = (key: string) => {
+    setPickedColumns((prev) => {
+      const next = new Set(prev);
+      if (next.has(key)) next.delete(key);
+      else next.add(key);
+      return next;
+    });
+  };
+
+  const checkAllColumns = () => {
+    setPickedColumns(new Set(offeredColumns.map((c) => c.key)));
+  };
+
+  const uncheckAllColumns = () => {
+    setPickedColumns(new Set());
+  };
+
+  const submit = async () => {
+    if (orderedColumnList.length === 0) {
+      setStatus('Pick at least one column.');
+      return;
+    }
+    if (pickedTabs.size === 0) {
+      setStatus('Pick at least one table.');
+      return;
+    }
+    setBusy(true);
+    setStatus('');
+    try {
+      const sections: ExportTabularSection[] = tabs
+        .filter((t) => pickedTabs.has(t.key))
+        .map((t) => ({ label: t.label, category: t.category }));
+      const result = await window.freecrawl.exportTabular({
+        format,
+        sections,
+        columns: orderedColumnList,
+        selectedIds:
+          useSelection && selectedIds && selectedIds.length > 0
+            ? selectedIds
+            : undefined,
+      });
+      if (!result.filePath) {
+        // User cancelled save / folder dialog.
+        setBusy(false);
+        return;
+      }
+      setStatus(
+        `Exported ${result.rowsWritten.toLocaleString()} row${result.rowsWritten === 1 ? '' : 's'} to ${result.filePath}`,
+      );
+      setBusy(false);
+      setTimeout(() => onClose(), 700);
+    } catch (err) {
+      setBusy(false);
+      setStatus(`Export failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  };
+
+  return (
+    <div
+      className="fixed inset-0 z-50 flex items-center justify-center bg-black/60"
+      onClick={onClose}
+    >
+      <div
+        className="flex max-h-[85vh] w-[780px] max-w-[95vw] flex-col rounded-md border border-surface-700 bg-surface-900 shadow-2xl"
+        onClick={(e) => e.stopPropagation()}
+      >
+        <div className="flex items-center justify-between border-b border-surface-800 px-4 py-2.5">
+          <div className="text-sm font-semibold text-surface-100">Export</div>
+          <button
+            className="rounded p-1 text-surface-400 hover:bg-surface-800 hover:text-surface-100"
+            onClick={onClose}
+            title="Close"
+          >
+            <X className="h-4 w-4" />
+          </button>
+        </div>
+
+        <div className="flex-1 overflow-auto px-4 py-3">
+          <div className="grid grid-cols-2 gap-4">
+            <section>
+              <div className="mb-1.5 text-[11px] font-semibold uppercase tracking-wide text-surface-400">
+                Tables
+              </div>
+              <div className="rounded border border-surface-800 bg-surface-950/50 p-2">
+                <div className="grid grid-cols-1 gap-1">
+                  {tabs.map((t) => {
+                    const checked = pickedTabs.has(t.key);
+                    const isDefault = t.key === initialTabKey;
+                    return (
+                      <label
+                        key={t.key}
+                        className={clsx(
+                          'flex cursor-pointer items-center gap-2 rounded px-1.5 py-1 text-[12px]',
+                          checked
+                            ? 'bg-accent-500/10 text-surface-100'
+                            : 'text-surface-300 hover:bg-surface-800/60',
+                        )}
+                      >
+                        <input
+                          type="checkbox"
+                          className="h-3.5 w-3.5"
+                          checked={checked}
+                          onChange={() => toggleTab(t.key)}
+                        />
+                        <span className="truncate">{t.label}</span>
+                        {isDefault && (
+                          <span className="ml-auto rounded bg-surface-800 px-1 text-[9px] uppercase tracking-wide text-surface-400">
+                            Active
+                          </span>
+                        )}
+                      </label>
+                    );
+                  })}
+                </div>
+              </div>
+            </section>
+
+            <section className="flex min-h-0 flex-col">
+              <div className="mb-1.5 flex items-center justify-between">
+                <div className="text-[11px] font-semibold uppercase tracking-wide text-surface-400">
+                  Columns
+                </div>
+                <div className="flex gap-1 text-[10px]">
+                  <button
+                    type="button"
+                    className="rounded border border-surface-700 px-1.5 py-0.5 text-surface-300 hover:bg-surface-800"
+                    onClick={checkAllColumns}
+                  >
+                    All
+                  </button>
+                  <button
+                    type="button"
+                    className="rounded border border-surface-700 px-1.5 py-0.5 text-surface-300 hover:bg-surface-800"
+                    onClick={uncheckAllColumns}
+                  >
+                    None
+                  </button>
+                </div>
+              </div>
+              <div className="max-h-[320px] overflow-auto rounded border border-surface-800 bg-surface-950/50 p-2">
+                <div className="grid grid-cols-1 gap-1">
+                  {offeredColumns.map((c) => {
+                    const checked = pickedColumns.has(c.key);
+                    return (
+                      <label
+                        key={c.key}
+                        className={clsx(
+                          'flex cursor-pointer items-center gap-2 rounded px-1.5 py-1 text-[12px]',
+                          checked
+                            ? 'bg-accent-500/10 text-surface-100'
+                            : 'text-surface-300 hover:bg-surface-800/60',
+                        )}
+                      >
+                        <input
+                          type="checkbox"
+                          className="h-3.5 w-3.5"
+                          checked={checked}
+                          onChange={() => toggleColumn(c.key)}
+                        />
+                        <span className="truncate">{c.header}</span>
+                        <span className="ml-auto truncate font-mono text-[10px] text-surface-500">
+                          {c.key}
+                        </span>
+                      </label>
+                    );
+                  })}
+                </div>
+              </div>
+            </section>
+          </div>
+
+          <div className="mt-4 flex flex-wrap items-center gap-4">
+            <div className="flex items-center gap-2">
+              <span className="text-[11px] font-semibold uppercase tracking-wide text-surface-400">
+                Format
+              </span>
+              <label className="flex cursor-pointer items-center gap-1 text-[12px] text-surface-200">
+                <input
+                  type="radio"
+                  name="export-format"
+                  checked={format === 'csv'}
+                  onChange={() => setFormat('csv')}
+                />
+                CSV
+              </label>
+              <label className="flex cursor-pointer items-center gap-1 text-[12px] text-surface-200">
+                <input
+                  type="radio"
+                  name="export-format"
+                  checked={format === 'xlsx'}
+                  onChange={() => setFormat('xlsx')}
+                />
+                Excel (.xlsx)
+              </label>
+            </div>
+
+            {selectionCount > 0 && (
+              <label className="flex cursor-pointer items-center gap-2 text-[12px] text-surface-200">
+                <input
+                  type="checkbox"
+                  checked={useSelection}
+                  onChange={(e) => setUseSelection(e.target.checked)}
+                />
+                <span>
+                  Selected rows only{' '}
+                  <span className="text-surface-500">
+                    ({selectionCount.toLocaleString()})
+                  </span>
+                </span>
+              </label>
+            )}
+          </div>
+
+          {format === 'csv' && pickedTabs.size > 1 && (
+            <div className="mt-2 rounded border border-amber-700/40 bg-amber-900/20 px-2 py-1 text-[11px] text-amber-200">
+              Multiple tables + CSV → you'll be asked to pick a folder; one file is written per table.
+            </div>
+          )}
+
+          {status && (
+            <div className="mt-3 truncate rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-200">
+              {status}
+            </div>
+          )}
+        </div>
+
+        <div className="flex items-center justify-end gap-2 border-t border-surface-800 px-4 py-2.5">
+          <button
+            type="button"
+            onClick={onClose}
+            className="rounded border border-surface-700 px-3 py-1 text-[12px] text-surface-300 hover:bg-surface-800"
+            disabled={busy}
+          >
+            Cancel
+          </button>
+          <button
+            type="button"
+            onClick={submit}
+            disabled={busy || pickedColumns.size === 0 || pickedTabs.size === 0}
+            className={clsx(
+              'inline-flex items-center gap-1 rounded border border-surface-700 px-3 py-1 text-[12px] text-surface-200 hover:bg-surface-800',
+              (busy || pickedColumns.size === 0 || pickedTabs.size === 0) &&
+                'cursor-not-allowed opacity-50 hover:bg-transparent',
+            )}
+          >
+            {busy && <Loader2 className="h-3.5 w-3.5 animate-spin" />}
+            Export
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+}

apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts

@@ -2,18 +2,27 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import type { AdvancedFilter, CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
 
 const CHUNK_SIZE = 500;
-const MAX_CACHED_CHUNKS = 8;
+// Doubled the cache (was 8) so a fast scroll across 4-5 chunks doesn't
-// Live-refresh cadence during an active crawl. 750 ms strikes a balance
+// thrash the reader pool: each chunk evicted = a re-fetch the next time
-// between perceived "row streaming" smoothness and the cost of a COUNT
+// the user scrolls back to it. 16 × 500 = 8000 rows kept warm; at
-// query + per-chunk fetch every tick. At 100 URL/s the user still sees
+// ~2 KB/row that's ~16 MB heap, well within budget.
-// ~75 new rows per tick (continuous flow), while leaving 3× more main-
+const MAX_CACHED_CHUNKS = 16;
-// thread headroom than the previous 250 ms cadence.
+// Number of chunks to pre-fetch ahead/behind the visible range. The
-// Cadence for the live-tail tick. 1500 ms strikes the balance between
+// visible range is what the virtualizer asks for; pre-fetching the
-// "feels fresh" and "doesn't pin SQLite". At 750 ms a 2k+ row crawl was
+// neighbours hides "..." placeholders during fast scrolls because the
-// burning ~30% of one core in COUNT + chunk fetches, blocking input IPC
+// next chunk has usually already landed by the time the user reaches
-// for the duration. The user sees the same effect: numbers tick at half
+// it. 1 chunk = 500 rows of look-ahead each direction, cheap on the
-// speed but the window stops kasma'ing.
+// reader pool.
-const LIVE_REFRESH_MS = 1500;
+const PREFETCH_CHUNKS = 1;
+// Min gap between progress-driven ticks. Progress events arrive at
+// ~5 Hz from the crawler; without throttle the visible-chunk re-query
+// would fire that often too. 100 ms keeps the table feeling live
+// (10 refreshes/sec ceiling) without pinning the reader-pool.
+const TICK_THROTTLE_MS = 100;
+// Safety-net periodic tick for cases where progress events stop
+// arriving (crawl finished, manual DB edits, etc.). Long interval —
+// the progress-event path drives the in-crawl experience.
+const LIVE_REFRESH_MS = 5000;
 
 export interface LazyRowsOpts {
   category: UrlCategory;
@@ -31,6 +40,12 @@ export interface LazyRowsState {
   loadedRows: number;
   rowAt: (index: number) => CrawlUrlRow | null;
   ensureRange: (start: number, end: number) => void;
+  /** True between the moment a shape change is requested (tab/sort/filter
+   *  flip) and the moment the new data has fully landed. Lets the
+   *  consumer distinguish "0 URLs because the category genuinely has
+   *  none" from "0 URLs because we haven't fetched yet" — which the
+   *  empty-state UI cares about a lot. */
+  isLoading: boolean;
 }
 
 /**
@@ -49,6 +64,7 @@ export interface LazyRowsState {
 export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
   const [total, setTotal] = useState(0);
   const [version, setVersion] = useState(0);
+  const [isLoading, setIsLoading] = useState(true);
   const chunks = useRef(new Map<number, CrawlUrlRow[]>());
   const chunkOrder = useRef<number[]>([]);
   const fetching = useRef(new Set<number>());
@@ -88,20 +104,52 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
     [opts.category, opts.search, opts.sortBy, opts.sortDir, opts.filter],
   );
 
-  // Shape change: rebuild from scratch. The virtualizer will call
+  // Shape change (tab/sort/filter switch): clear immediately and load
-  // ensureRange again on its next measurement pass.
+  // fresh data in parallel. Showing the previous tab's rows under the
+  // new tab's header is the wrong UX — "Internal" rows visible while
+  // "External" is selected makes the user mistrust what they see.
+  // Instead we set `isLoading=true` and let the consumer render a
+  // proper loading state until the new total + visible chunks land.
+  // The reader-pool has multiple workers now, so meta + visible-chunk
+  // queries run in parallel and typically resolve in ~50-150 ms.
   useEffect(() => {
     resetToken.current++;
     chunks.current.clear();
     chunkOrder.current = [];
     fetching.current.clear();
+    setIsLoading(true);
     setVersion((v) => v + 1);
     const token = resetToken.current;
     let cancelled = false;
-    void queryMeta().then(({ total: t }) => {
+
-      if (cancelled || token !== resetToken.current) return;
+    const load = async (): Promise<void> => {
-      setTotal(t);
+      const metaPromise = queryMeta();
-    });
+      const { first, last } = activeRange.current;
+      const chunkPromises: Promise<{ rows: CrawlUrlRow[]; total: number; idx: number }>[] = [];
+      for (let i = first; i <= last; i++) {
+        chunkPromises.push(queryChunk(i).then((r) => ({ ...r, idx: i })));
+      }
+      try {
+        const [{ total: t }, ...newChunks] = await Promise.all([metaPromise, ...chunkPromises]);
+        if (cancelled || token !== resetToken.current) return;
+        for (const ch of newChunks) {
+          chunks.current.set(ch.idx, ch.rows);
+          chunkOrder.current.push(ch.idx);
+        }
+        setTotal(t);
+        setVersion((v) => v + 1);
+      } catch {
+        if (cancelled || token !== resetToken.current) return;
+        // Leave chunks empty on error; the live tick / ensureRange
+        // will retry. Still need to flip loading off so the empty
+        // state can render an actual error/empty UI.
+      } finally {
+        if (!cancelled && token === resetToken.current) {
+          setIsLoading(false);
+        }
+      }
+    };
+    void load();
     return () => {
       cancelled = true;
     };
@@ -111,16 +159,26 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
   // Live tick: re-query only the visible chunks and patch them in place.
   // Never calls .clear(), so rowAt never returns null for a
   // previously-loaded index. The table streams rather than flashes.
-  // A leading tick fires immediately so the first rows surface without
+  //
-  // the LIVE_REFRESH_MS dead window after Start.
+  // Drive model: push, not poll. Each crawler progress event (≈ every
+  // time `crawled` advances, which lands at ~5 Hz on a healthy crawl)
+  // triggers an immediate visible-chunk re-query. The previous design
+  // polled every 1500 ms regardless of activity — newly-written rows
+  // could sit in the DB for up to that long before surfacing in the
+  // table. The push path closes that window to ~100 ms (the throttle
+  // floor that keeps the reader-pool from being hammered).
+  //
+  // A 5 s safety-net `setInterval` still fires for code paths that
+  // mutate the DB without going through a progress event (manual row
+  // delete, project re-open, etc.).
   useEffect(() => {
     let cancelled = false;
     let inFlight = false;
+    let lastTickTs = 0;
     const tick = async () => {
-      // Coalesce overlapping ticks — at 250 ms cadence a slow COUNT or
-      // chunk fetch can otherwise queue up multiple in-flight ticks.
       if (inFlight) return;
       inFlight = true;
+      lastTickTs = Date.now();
       const token = resetToken.current;
       try {
         try {
@@ -146,33 +204,47 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
         inFlight = false;
       }
     };
-    // Wrap the tick in requestIdleCallback so the live-tail SQL only
-    // fires when the renderer's event loop has idle slack. If the user
-    // is dragging / clicking / typing, the tick is deferred until the
-    // input handler frame finishes — eliminates the "click → kasma"
-    // pattern even when the underlying SQL is fast. Falls back to a
-    // direct call when the browser doesn't expose `requestIdleCallback`
-    // (older Chromium builds; Electron 41 ships modern Chromium so this
-    // branch is mostly defensive).
     interface RequestIdleCallback {
       (cb: () => void, opts?: { timeout: number }): number;
     }
     const w = window as Window & { requestIdleCallback?: RequestIdleCallback };
     const scheduleTick = (): void => {
       if (typeof w.requestIdleCallback === 'function') {
-        // 2 s timeout guarantees the tick eventually runs even if the
+        // 300 ms timeout — short enough that the user perceives an
-        // main thread is permanently busy — better stale data than no
+        // event-driven refresh as instant; long enough that an
-        // data at all.
+        // in-progress click handler isn't preempted.
-        w.requestIdleCallback(() => void tick(), { timeout: 2000 });
+        w.requestIdleCallback(() => void tick(), { timeout: 300 });
       } else {
         void tick();
       }
     };
-    scheduleTick();
+    // Leading tick fires directly (no idle wrap) so the first row
+    // surfaces as soon as the crawler has written one — without this,
+    // requestIdleCallback could defer the initial fetch up to 300 ms
+    // on a busy main thread right after Start.
+    void tick();
+
+    // Push: progress event drives the live-tail tick. We tick on every
+    // event where `crawled` changed, throttled to TICK_THROTTLE_MS so
+    // a 100 URL/s crawl doesn't force 100 SQL roundtrips per second.
+    // Direct call (no idle wrap) — progress events are sparse enough
+    // that input-handler interference is a non-concern, and pulling
+    // through requestIdleCallback was adding 50-300 ms before each
+    // newly-written URL surfaced in the table.
+    let lastCrawled = -1;
+    const offProgress = window.freecrawl?.onProgress?.((p) => {
+      if (p.crawled === lastCrawled) return;
+      lastCrawled = p.crawled;
+      if (Date.now() - lastTickTs < TICK_THROTTLE_MS) return;
+      void tick();
+    });
+
+    // Safety-net poll for crawl-idle DB mutations.
     const id = setInterval(scheduleTick, LIVE_REFRESH_MS);
     return () => {
       cancelled = true;
       clearInterval(id);
+      offProgress?.();
     };
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [keyStr]);
@@ -213,7 +285,13 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
       const first = Math.max(0, Math.floor(start / CHUNK_SIZE));
       const last = Math.max(0, Math.floor(end / CHUNK_SIZE));
       activeRange.current = { first, last };
-      for (let i = first; i <= last; i++) {
+      // Pre-fetch a window of neighbouring chunks so a fast scroll
+      // doesn't show "..." placeholders while the next chunk loads —
+      // by the time the virtualizer scrolls into the next chunk's
+      // range, its rows are usually already in the cache.
+      const fetchFrom = Math.max(0, first - PREFETCH_CHUNKS);
+      const fetchTo = last + PREFETCH_CHUNKS;
+      for (let i = fetchFrom; i <= fetchTo; i++) {
         if (!chunks.current.has(i)) void fetchChunk(i);
       }
     },
@@ -237,5 +315,5 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
     [version],
   );
 
-  return { total, loadedRows, rowAt, ensureRange };
+  return { total, loadedRows, rowAt, ensureRange, isLoading };
 }

apps/desktop/src/renderer/src/tabs/UrlsTab.tsx

@@ -1,12 +1,14 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
-import { ChevronsUpDown, Filter, X } from 'lucide-react';
+import { ChevronsUpDown, Download, Filter, X } from 'lucide-react';
 import clsx from 'clsx';
 import type { AdvancedFilter, CrawlUrlRow } from '@freecrawl/shared-types';
 import { useAppStore, type TabKey } from '../store.js';
 import { COLUMN_SPECS, columnId, type ColumnSpec } from './columns.js';
 import { useLazyUrlRows } from '../hooks/useLazyUrlRows.js';
 import { AdvancedFilterDialog } from '../components/AdvancedFilterDialog.js';
+import { ErrorBoundary } from '../components/ErrorBoundary.js';
+import { ExportDialog } from '../components/ExportDialog.js';
 import { InfoTip } from '../components/InfoTip.js';
 
 const ROW_HEIGHT = 24;
@@ -36,6 +38,7 @@ export function UrlsTab() {
   const [sortDir, setSortDir] = useState<'asc' | 'desc'>('asc');
   const [filter, setFilter] = useState<AdvancedFilter | null>(null);
   const [filterDialogOpen, setFilterDialogOpen] = useState(false);
+  const [exportDialogOpen, setExportDialogOpen] = useState(false);
   // Three orthogonal selection layers:
   // - selectedIds: row-level selection driven by the Row-number column.
   //   Feeds the bulk context menu (Copy/Open/Re-Spider/Remove/Export).
@@ -515,12 +518,21 @@ export function UrlsTab() {
             <X className="h-3 w-3" />
           </button>
         )}
-        <div className="ml-auto text-[11px] text-surface-500">
+        <button
-          <span className="font-mono text-surface-200">{lazy.total.toLocaleString()}</span> URLs
+          type="button"
-          <span className="ml-2 text-surface-600">
+          onClick={() => setExportDialogOpen(true)}
-            ({lazy.loadedRows.toLocaleString()} loaded)
+          disabled={lazy.total === 0}
-          </span>
+          className={clsx(
-        </div>
+            'ml-auto inline-flex items-center gap-1.5 rounded border px-2 py-1 text-[11px] transition',
+            lazy.total === 0
+              ? 'cursor-not-allowed border-surface-800 text-surface-600'
+              : 'border-surface-700 text-surface-300 hover:bg-surface-800',
+          )}
+          title={lazy.total === 0 ? 'No data to export' : 'Export this table'}
+        >
+          <Download className="h-3.5 w-3.5" />
+          <span>Export</span>
+        </button>
       </div>
 
       <div ref={scrollRef} className="relative flex-1 select-none overflow-auto">
@@ -753,7 +765,15 @@ export function UrlsTab() {
           </div>
         </div>
 
-        {lazy.total === 0 && (
+        {lazy.total === 0 && lazy.isLoading && (
+          <div
+            className="pointer-events-none absolute inset-0 flex items-center justify-center"
+            style={{ top: HEADER_HEIGHT }}
+          >
+            <div className="text-xs text-surface-500">Loading…</div>
+          </div>
+        )}
+        {lazy.total === 0 && !lazy.isLoading && (
           <div
             className="pointer-events-none absolute inset-0 flex items-center justify-center"
             style={{ top: HEADER_HEIGHT }}
@@ -771,6 +791,15 @@ export function UrlsTab() {
         className="flex shrink-0 items-center justify-end gap-4 border-t border-surface-800 bg-surface-900/60 px-3 text-[11px] text-surface-400"
         style={{ height: STATUS_BAR_HEIGHT }}
       >
+        <span>
+          URLs:{' '}
+          <span className="font-mono tabular-nums text-surface-200">
+            {lazy.total.toLocaleString()}
+          </span>
+          <span className="ml-1 text-surface-600">
+            ({lazy.loadedRows.toLocaleString()} loaded)
+          </span>
+        </span>
         <span>
           Selected Rows:{' '}
           <span className="font-mono tabular-nums text-surface-200">
@@ -789,12 +818,6 @@ export function UrlsTab() {
             {selectedColumns.size.toLocaleString()}
           </span>
         </span>
-        <span>
-          Filter Total:{' '}
-          <span className="font-mono tabular-nums text-surface-200">
-            {lazy.total.toLocaleString()}
-          </span>
-        </span>
       </div>
 
       <AdvancedFilterDialog
@@ -803,6 +826,17 @@ export function UrlsTab() {
         onClose={() => setFilterDialogOpen(false)}
         onApply={(f) => setFilter(f)}
       />
+
+      <ErrorBoundary context="ExportDialog">
+        {exportDialogOpen && (
+          <ExportDialog
+            open={exportDialogOpen}
+            onClose={() => setExportDialogOpen(false)}
+            defaultTab={activeTab}
+            selectedIds={selectedIds.size > 0 ? [...selectedIds] : undefined}
+          />
+        )}
+      </ErrorBoundary>
     </div>
   );
 }

apps/mcp-server/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@freecrawl/mcp-server",
+  "version": "0.2.8",
+  "private": true,
+  "type": "module",
+  "bin": {
+    "freecrawl-mcp": "./dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "start": "node ./dist/index.js",
+    "dev": "tsc -p tsconfig.json --watch",
+    "clean": "rimraf dist"
+  },
+  "dependencies": {
+    "@freecrawl/db": "*",
+    "@freecrawl/shared-types": "*"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.0",
+    "rimraf": "^6.0.1",
+    "typescript": "^5.7.0"
+  }
+}

apps/mcp-server/src/index.ts

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+/**
+ * FreeCrawl SEO — MCP server (stdio transport, JSON-RPC 2.0).
+ *
+ * Exposes the active `.seoproject` to MCP-compatible clients (Claude
+ * Desktop, Claude Code, …) so an AI agent can run SEO queries against
+ * the most recent crawl directly:
+ *
+ *   - `get_summary`            → headline numbers
+ *   - `get_overview_counts`    → sidebar issue/category breakdown
+ *   - `top_issues`             → ranked non-empty issue list
+ *   - `query_urls`             → paginated category/search/sort
+ *   - `get_url_detail`         → full row + inlinks/outlinks/images
+ *   - `list_projects`          → enumerate `.seoproject` files
+ *   - `set_project`            → switch sessions to a different file
+ *
+ * The DB connection is read-only (`new ProjectDb(path, { readOnly: true })`)
+ * so the desktop app's writer thread can keep operating concurrently —
+ * SQLite WAL allows arbitrary concurrent readers.
+ */
+
+import fs from 'node:fs';
+import { ProjectDb } from '@freecrawl/db';
+import { JsonRpcServer } from './rpc.js';
+import { buildTools, type Tool } from './tools.js';
+import { defaultProjectPath, listProjectFiles } from './project-resolver.js';
+
+const SERVER_NAME = 'freecrawl-seo';
+const SERVER_VERSION = '0.2.7';
+const PROTOCOL_VERSION = '2024-11-05';
+
+interface Session {
+  projectPath: string;
+  db: ProjectDb;
+}
+
+let session: Session | null = null;
+
+function openProject(projectPath: string): Session {
+  if (!fs.existsSync(projectPath)) {
+    throw new Error(`Project file does not exist: ${projectPath}`);
+  }
+  const db = new ProjectDb(projectPath, { readOnly: true });
+  return { projectPath, db };
+}
+
+function getSession(): Session {
+  if (session) return session;
+  const initialPath = defaultProjectPath();
+  session = openProject(initialPath);
+  return session;
+}
+
+function setProject(projectPath: string): Session {
+  if (session) {
+    try {
+      session.db.close();
+    } catch {
+      // best-effort close
+    }
+  }
+  session = openProject(projectPath);
+  return session;
+}
+
+const tools: Tool[] = [
+  ...buildTools(),
+  {
+    name: 'list_projects',
+    description:
+      'List `.seoproject` files in the desktop app\'s default projects folder. Returns absolute paths the user can pass to `set_project`.',
+    inputSchema: { type: 'object', properties: {} },
+    handler: () => {
+      const files = listProjectFiles();
+      return { files, defaultPath: defaultProjectPath() };
+    },
+  },
+  {
+    name: 'set_project',
+    description:
+      'Switch the active project for this MCP session. Subsequent tool calls hit the new file. Closes the previous DB handle.',
+    inputSchema: {
+      type: 'object',
+      required: ['path'],
+      properties: {
+        path: {
+          type: 'string',
+          description: 'Absolute path to a `.seoproject` file.',
+        },
+      },
+    },
+    handler: (args) => {
+      const target = args.path;
+      if (typeof target !== 'string' || target.trim() === '') {
+        throw new Error('set_project requires `path` to be a non-empty string.');
+      }
+      const next = setProject(target);
+      return { ok: true, projectPath: next.projectPath };
+    },
+  },
+  {
+    name: 'current_project',
+    description:
+      'Report which `.seoproject` the MCP server is currently reading from.',
+    inputSchema: { type: 'object', properties: {} },
+    handler: () => ({ projectPath: getSession().projectPath }),
+  },
+];
+
+const server = new JsonRpcServer();
+
+// ── Required MCP lifecycle methods ───────────────────────────────────
+
+server.on('initialize', (params) => {
+  const requestedVersion =
+    typeof (params as Record<string, unknown> | undefined)?.protocolVersion === 'string'
+      ? ((params as Record<string, string>).protocolVersion)
+      : PROTOCOL_VERSION;
+  return {
+    protocolVersion: requestedVersion,
+    capabilities: {
+      tools: { listChanged: false },
+    },
+    serverInfo: {
+      name: SERVER_NAME,
+      version: SERVER_VERSION,
+    },
+  };
+});
+
+// MCP clients send `notifications/initialized` after `initialize`. It
+// has no `id`, so the JSON-RPC layer won't write a response.
+server.on('notifications/initialized', () => undefined);
+
+// ── Tools surface ────────────────────────────────────────────────────
+
+server.on('tools/list', () => ({
+  tools: tools.map((t) => ({
+    name: t.name,
+    description: t.description,
+    inputSchema: t.inputSchema,
+  })),
+}));
+
+server.on('tools/call', async (raw) => {
+  const params = (raw ?? {}) as { name?: string; arguments?: Record<string, unknown> };
+  const name = params.name;
+  const args = params.arguments ?? {};
+  if (typeof name !== 'string') {
+    throw new Error('tools/call requires `name` to be a string.');
+  }
+  const tool = tools.find((t) => t.name === name);
+  if (!tool) throw new Error(`Unknown tool: ${name}`);
+
+  const sess = getSession();
+  let result: unknown;
+  try {
+    result = await tool.handler(args, sess.db);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return {
+      isError: true,
+      content: [{ type: 'text', text: msg }],
+    };
+  }
+
+  return {
+    content: [
+      {
+        type: 'text',
+        text: typeof result === 'string' ? result : JSON.stringify(result, null, 2),
+      },
+    ],
+  };
+});
+
+// ── Bootstrap ────────────────────────────────────────────────────────
+
+process.on('SIGINT', () => {
+  if (session) {
+    try {
+      session.db.close();
+    } catch {
+      // best-effort close
+    }
+  }
+  process.exit(0);
+});
+
+server.log(`starting ${SERVER_NAME} v${SERVER_VERSION}`);
+server.log(`default project: ${defaultProjectPath()}`);
+server.start();

apps/mcp-server/src/project-resolver.ts

@@ -0,0 +1,57 @@
+import path from 'node:path';
+import os from 'node:os';
+import fs from 'node:fs';
+
+/**
+ * Locate the desktop app's `default.seoproject` without depending on
+ * Electron. The desktop app stores it under
+ * `app.getPath('userData')/projects/default.seoproject`, where
+ * `userData` resolves per-platform using the same rules Electron uses.
+ *
+ * The Electron-resolved userData path is:
+ *   - Windows: %APPDATA%/<productName>
+ *   - macOS:   ~/Library/Application Support/<productName>
+ *   - Linux:   $XDG_CONFIG_HOME/<productName> or ~/.config/<productName>
+ *
+ * `productName` for FreeCrawl is `@freecrawl/desktop` (taken from
+ * `apps/desktop/package.json`'s `name` field).
+ */
+
+const PRODUCT_NAME = '@freecrawl/desktop';
+const PROJECT_FILENAME = 'default.seoproject';
+
+export function defaultProjectPath(): string {
+  // Explicit override always wins — handy for headless testing or
+  // pointing at a non-default project.
+  const override = process.env.FREECRAWL_PROJECT;
+  if (override && override.trim().length > 0) return override.trim();
+
+  const home = os.homedir();
+  let userData: string;
+  if (process.platform === 'win32') {
+    userData = path.join(
+      process.env.APPDATA ?? path.join(home, 'AppData', 'Roaming'),
+      PRODUCT_NAME,
+    );
+  } else if (process.platform === 'darwin') {
+    userData = path.join(home, 'Library', 'Application Support', PRODUCT_NAME);
+  } else {
+    userData = path.join(
+      process.env.XDG_CONFIG_HOME ?? path.join(home, '.config'),
+      PRODUCT_NAME,
+    );
+  }
+  return path.join(userData, 'projects', PROJECT_FILENAME);
+}
+
+export function listProjectFiles(): string[] {
+  const dir = path.dirname(defaultProjectPath());
+  try {
+    return fs
+      .readdirSync(dir)
+      .filter((f) => f.endsWith('.seoproject'))
+      .map((f) => path.join(dir, f));
+  } catch {
+    return [];
+  }
+}

apps/mcp-server/src/rpc.ts

@@ -0,0 +1,109 @@
+/**
+ * Minimal newline-delimited JSON-RPC 2.0 server for stdio MCP transport.
+ *
+ * MCP's stdio transport is line-delimited JSON — one request per line on
+ * stdin, one response per line on stdout. stderr is reserved for human
+ * logs; writing anything non-JSON to stdout corrupts the protocol stream.
+ *
+ * We deliberately do NOT depend on `@modelcontextprotocol/sdk`. The
+ * protocol surface we need (`initialize`, `tools/list`, `tools/call`,
+ * the `initialized` notification) is small, well-specified, and stable;
+ * the dep-free path keeps the install footprint minimal and avoids a
+ * version-pin we'd otherwise have to chase.
+ */
+
+export interface JsonRpcRequest {
+  jsonrpc: '2.0';
+  id?: number | string | null;
+  method: string;
+  params?: unknown;
+}
+
+export interface JsonRpcSuccess {
+  jsonrpc: '2.0';
+  id: number | string | null;
+  result: unknown;
+}
+
+export interface JsonRpcError {
+  jsonrpc: '2.0';
+  id: number | string | null;
+  error: { code: number; message: string; data?: unknown };
+}
+
+export type Handler = (params: unknown) => unknown | Promise<unknown>;
+
+export class JsonRpcServer {
+  private handlers = new Map<string, Handler>();
+  private buffer = '';
+
+  on(method: string, handler: Handler): void {
+    this.handlers.set(method, handler);
+  }
+
+  log(...parts: unknown[]): void {
+    // Always to stderr — stdout is reserved for protocol frames.
+    const line = parts
+      .map((p) => (typeof p === 'string' ? p : JSON.stringify(p)))
+      .join(' ');
+    process.stderr.write(`[freecrawl-mcp] ${line}\n`);
+  }
+
+  start(): void {
+    process.stdin.setEncoding('utf8');
+    process.stdin.on('data', (chunk: string) => {
+      this.buffer += chunk;
+      let nl: number;
+      while ((nl = this.buffer.indexOf('\n')) !== -1) {
+        const line = this.buffer.slice(0, nl).trim();
+        this.buffer = this.buffer.slice(nl + 1);
+        if (line) void this.handleLine(line);
+      }
+    });
+    process.stdin.on('end', () => process.exit(0));
+  }
+
+  private write(msg: JsonRpcSuccess | JsonRpcError): void {
+    process.stdout.write(JSON.stringify(msg) + '\n');
+  }
+
+  private async handleLine(line: string): Promise<void> {
+    let req: JsonRpcRequest;
+    try {
+      req = JSON.parse(line) as JsonRpcRequest;
+    } catch {
+      this.log('parse error', line);
+      return;
+    }
+    const id = req.id ?? null;
+    const handler = this.handlers.get(req.method);
+    // Notifications (no `id`) get no response, even on miss.
+    const isNotification = req.id === undefined || req.id === null;
+    if (!handler) {
+      if (!isNotification) {
+        this.write({
+          jsonrpc: '2.0',
+          id,
+          error: { code: -32601, message: `Method not found: ${req.method}` },
+        });
+      }
+      return;
+    }
+    try {
+      const result = await handler(req.params);
+      if (!isNotification) {
+        this.write({ jsonrpc: '2.0', id, result });
+      }
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      this.log('handler error', req.method, msg);
+      if (!isNotification) {
+        this.write({
+          jsonrpc: '2.0',
+          id,
+          error: { code: -32000, message: msg },
+        });
+      }
+    }
+  }
+}

apps/mcp-server/src/tools.ts

@@ -0,0 +1,265 @@
+/**
+ * Tool definitions and handlers exposed by the FreeCrawl MCP server.
+ *
+ * The server opens the project DB in read-only mode (see `index.ts`),
+ * so it can coexist with the desktop app's writer without contention.
+ * SQLite WAL mode permits arbitrary concurrent readers.
+ */
+
+import type { ProjectDb } from '@freecrawl/db';
+import type { UrlCategory } from '@freecrawl/shared-types';
+
+export interface Tool {
+  name: string;
+  description: string;
+  inputSchema: Record<string, unknown>;
+  handler: (args: Record<string, unknown>, db: ProjectDb) => unknown | Promise<unknown>;
+}
+
+const URL_CATEGORY_VALUES: UrlCategory[] = [
+  'all',
+  'internal:all',
+  'internal:html',
+  'internal:js',
+  'internal:css',
+  'internal:image',
+  'internal:pdf',
+  'internal:font',
+  'internal:other',
+  'external:all',
+  'external:html',
+  'external:other',
+  'status:2xx',
+  'status:3xx',
+  'status:4xx',
+  'status:5xx',
+  'status:no-response',
+  'status:blocked-robots',
+  'security:https',
+  'security:http',
+  'indexability:indexable',
+  'indexability:non-indexable',
+  'indexability:noindex',
+  'indexability:canonicalised',
+];
+
+const COMMON_URL_FIELDS = [
+  'id',
+  'url',
+  'statusCode',
+  'contentKind',
+  'indexability',
+  'title',
+  'titleLength',
+  'metaDescription',
+  'metaDescriptionLength',
+  'h1',
+  'h1Count',
+  'wordCount',
+  'canonical',
+  'metaRobots',
+  'contentLength',
+  'responseTimeMs',
+  'depth',
+  'inlinks',
+  'outlinks',
+  'imagesCount',
+  'imagesMissingAlt',
+  'redirectTarget',
+  'crawledAt',
+];
+
+function projectFromRow(
+  row: Record<string, unknown>,
+  fields: string[],
+): Record<string, unknown> {
+  if (fields.length === 0) {
+    const out: Record<string, unknown> = {};
+    for (const k of COMMON_URL_FIELDS) out[k] = row[k];
+    return out;
+  }
+  const out: Record<string, unknown> = {};
+  for (const k of fields) out[k] = row[k];
+  return out;
+}
+
+export function buildTools(): Tool[] {
+  return [
+    {
+      name: 'get_summary',
+      description:
+        'Top-level crawl summary for the active FreeCrawl project: total URLs crawled, indexable/non-indexable counts, status-code distribution, average response time, plus crawl start/end timestamps. Use this first when reporting on a crawl.',
+      inputSchema: {
+        type: 'object',
+        properties: {},
+      },
+      handler: (_args, db) => db.getSummary(),
+    },
+
+    {
+      name: 'get_overview_counts',
+      description:
+        'Sidebar overview counts — every category and issue filter the desktop app surfaces, with row counts. Useful to find which issue categories are non-empty before drilling in with query_urls.',
+      inputSchema: {
+        type: 'object',
+        properties: {},
+      },
+      handler: (_args, db) => db.getOverviewCounts(),
+    },
+
+    {
+      name: 'query_urls',
+      description:
+        'Paginated query against the URLs table with the same category filter the UI uses. Returns rows + total count. Use `category` to narrow (e.g. "issues:title-missing"), `search` to substring-match URLs/titles, and `limit`/`offset` to page through.',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          category: {
+            type: 'string',
+            description: 'UrlCategory filter (e.g. "all", "internal:html", "issues:title-missing", "status:4xx"). Defaults to "all".',
+            enum: URL_CATEGORY_VALUES,
+          },
+          search: {
+            type: 'string',
+            description: 'Optional substring match against URL and title.',
+          },
+          limit: {
+            type: 'integer',
+            minimum: 1,
+            maximum: 500,
+            description: 'Max rows to return (1–500). Default 50.',
+          },
+          offset: {
+            type: 'integer',
+            minimum: 0,
+            description: 'Row offset for pagination. Default 0.',
+          },
+          sortBy: {
+            type: 'string',
+            description: 'Column to sort by (e.g. "depth", "responseTimeMs", "inlinks").',
+          },
+          sortDir: {
+            type: 'string',
+            enum: ['asc', 'desc'],
+            description: 'Sort direction. Default "asc".',
+          },
+          fields: {
+            type: 'array',
+            items: { type: 'string' },
+            description:
+              'Columns to include per row. Empty/omitted → a sensible default subset (URL, status, indexability, title, depth, inlinks, etc.). Use ["*"] for the full row.',
+          },
+        },
+      },
+      handler: (args, db) => {
+        const category = (args.category as UrlCategory | undefined) ?? 'all';
+        const search = typeof args.search === 'string' ? args.search : undefined;
+        const limit = Math.min(
+          Math.max(typeof args.limit === 'number' ? args.limit : 50, 1),
+          500,
+        );
+        const offset = Math.max(
+          typeof args.offset === 'number' ? args.offset : 0,
+          0,
+        );
+        const sortBy = typeof args.sortBy === 'string' ? args.sortBy : undefined;
+        const sortDir = args.sortDir === 'desc' ? 'desc' : 'asc';
+        const fieldsArg = Array.isArray(args.fields) ? (args.fields as string[]) : [];
+        const wantsAll = fieldsArg.length === 1 && fieldsArg[0] === '*';
+
+        const { rows, total } = db.queryUrls({
+          category,
+          search,
+          limit,
+          offset,
+          sortBy,
+          sortDir,
+        });
+        const projected = wantsAll
+          ? rows
+          : rows.map((r) =>
+              projectFromRow(
+                r as unknown as Record<string, unknown>,
+                fieldsArg,
+              ),
+            );
+        return { rows: projected, total, limit, offset };
+      },
+    },
+
+    {
+      name: 'get_url_detail',
+      description:
+        'Full details for one URL: every column, plus inlinks, outlinks, images, and HTTP headers. Pass either `id` (preferred — from query_urls) or `url`.',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          id: {
+            type: 'integer',
+            description: 'Numeric URL id from query_urls / get_url_id.',
+          },
+          url: {
+            type: 'string',
+            description: 'Exact URL string (slower path — uses an indexed lookup).',
+          },
+          linkLimit: {
+            type: 'integer',
+            description: 'Cap on inlinks + outlinks returned. Default 500.',
+          },
+        },
+      },
+      handler: (args, db) => {
+        let id: number | null = null;
+        if (typeof args.id === 'number') {
+          id = args.id;
+        } else if (typeof args.url === 'string') {
+          const found = db.getUrlIdByUrl(args.url);
+          if (found === null) {
+            throw new Error(`URL not found: ${args.url}`);
+          }
+          id = found;
+        } else {
+          throw new Error('get_url_detail requires either `id` or `url`.');
+        }
+        const linkLimit =
+          typeof args.linkLimit === 'number' ? args.linkLimit : 500;
+        const detail = db.getUrlDetail(id, linkLimit);
+        if (detail === null) {
+          throw new Error(`URL id ${id} not found in this project.`);
+        }
+        return detail;
+      },
+    },
+
+    {
+      name: 'top_issues',
+      description:
+        'Ranked list of non-empty issue categories from get_overview_counts, sorted by count descending. Quick way to ask "what are the biggest problems with this crawl?".',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          limit: {
+            type: 'integer',
+            description: 'Max categories to return. Default 20.',
+            minimum: 1,
+            maximum: 200,
+          },
+        },
+      },
+      handler: (args, db) => {
+        const limit = Math.min(
+          Math.max(typeof args.limit === 'number' ? args.limit : 20, 1),
+          200,
+        );
+        const counts = db.getOverviewCounts() as unknown as Record<string, unknown>;
+        const issues = (counts.issues as Record<string, number>) ?? {};
+        const ranked = Object.entries(issues)
+          .filter(([, n]) => n > 0)
+          .sort((a, b) => b[1] - a[1])
+          .slice(0, limit)
+          .map(([category, count]) => ({ category: `issues:${category}`, count }));
+        return { issues: ranked };
+      },
+    },
+  ];
+}

apps/mcp-server/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "composite": true,
+    "types": ["node"]
+  },
+  "include": ["src/**/*"],
+  "references": [
+    { "path": "../../packages/shared-types" },
+    { "path": "../../packages/db" }
+  ]
+}

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.1.10",
+  "version": "0.2.8",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "freecrawl-seo",
-      "version": "0.1.10",
+      "version": "0.2.8",
       "license": "MIT",
       "workspaces": [
         "apps/*",
@@ -25,7 +25,7 @@
     },
     "apps/cli": {
       "name": "@freecrawl/cli",
-      "version": "0.1.10",
+      "version": "0.2.8",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -42,7 +42,7 @@
     },
     "apps/desktop": {
       "name": "@freecrawl/desktop",
-      "version": "0.1.10",
+      "version": "0.2.8",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -82,6 +82,22 @@
         "react-dom": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
       }
     },
+    "apps/mcp-server": {
+      "name": "@freecrawl/mcp-server",
+      "version": "0.2.8",
+      "dependencies": {
+        "@freecrawl/db": "*",
+        "@freecrawl/shared-types": "*"
+      },
+      "bin": {
+        "freecrawl-mcp": "dist/index.js"
+      },
+      "devDependencies": {
+        "@types/node": "^22.10.0",
+        "rimraf": "^6.0.1",
+        "typescript": "^5.7.0"
+      }
+    },
     "node_modules/@alloc/quick-lru": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",
@@ -1431,6 +1447,10 @@
       "resolved": "apps/desktop",
       "link": true
     },
+    "node_modules/@freecrawl/mcp-server": {
+      "resolved": "apps/mcp-server",
+      "link": true
+    },
     "node_modules/@freecrawl/shared-types": {
       "resolved": "packages/shared-types",
       "link": true
@@ -8712,7 +8732,7 @@
     },
     "packages/core": {
       "name": "@freecrawl/core",
-      "version": "0.1.10",
+      "version": "0.2.8",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*",
@@ -8738,7 +8758,7 @@
     },
     "packages/db": {
       "name": "@freecrawl/db",
-      "version": "0.1.10",
+      "version": "0.2.8",
       "dependencies": {
         "@freecrawl/shared-types": "*"
       },
@@ -8750,7 +8770,7 @@
     },
     "packages/shared-types": {
       "name": "@freecrawl/shared-types",
-      "version": "0.1.10",
+      "version": "0.2.8",
       "devDependencies": {
         "rimraf": "^6.0.1",
         "typescript": "^5.7.0"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",
@@ -18,6 +18,7 @@
     "build": "tsc -b && npm run build --workspace apps/desktop && npm run build --workspace apps/cli",
     "build:desktop": "npm run build --workspace apps/desktop",
     "build:cli": "npm run build --workspace apps/cli",
+    "build:mcp": "npm run build --workspace apps/mcp-server",
     "typecheck": "npm run typecheck --workspaces --if-present",
     "lint": "eslint .",
     "format": "prettier --write \"**/*.{ts,tsx,js,json,md,css,html}\"",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/crawler.ts

@@ -89,13 +89,23 @@ export class Crawler extends EventEmitter {
    */
   private memoryAutoPaused = false;
   private memoryMonitorTimer: NodeJS.Timeout | null = null;
+  /**
+   * Live AbortControllers for in-flight HTTP fetches in the main URL
+   * pipeline. Pause / Stop abort every member so the crawl actually
+   * stops instead of letting `concurrency` (often 20) more URLs land
+   * in the table after the user clicked the button. Each call site
+   * adds itself in `try` and removes itself in `finally`.
+   */
+  private inFlightFetchControllers = new Set<AbortController>();
   private robots: RobotsChecker | null = null;
   private progressTimer: NodeJS.Timeout | null = null;
   /** Periodic post-crawl-style recompute of expensive issue counters
    * while the crawl is still running, so the sidebar number is < 30 s
-   * stale instead of "0 until crawl ends". 30 s cadence picked
+   * stale instead of "0 until crawl ends". The recompute is dispatched
-   * conservatively — the recompute wraps in `runInTransaction` and is
+   * to the writer-worker via the injected `recomputeIssues` hook, so
-   * not free on a 100k-URL DB. */
+   * it does NOT block the main thread — the freezes that an earlier
+   * version produced were a result of running the recompute inline on
+   * the main-thread writer connection, which has since been fixed. */
   private issueRecomputeTimer: NodeJS.Timeout | null = null;
   private static readonly ISSUE_RECOMPUTE_INTERVAL_MS = 30_000;
   private issueRecomputeInFlight = false;
@@ -103,20 +113,16 @@ export class Crawler extends EventEmitter {
     if (this.issueRecomputeTimer) return;
     this.issueRecomputeTimer = setInterval(() => {
       if (this.stopped || !this.running || this.paused) return;
-      // Drop overlapping ticks. The yielding recompute can take longer
+      // Drop overlapping ticks — on a 1M-URL DB the recompute can run
-      // than the 30 s tick on very large projects; piling up calls
+      // longer than the 30 s tick, and piling RPCs on the writer worker
-      // would queue 70-statement transactions back-to-back and starve
+      // would starve per-URL writes behind a queue of issue passes.
-      // the crawler's own writes.
       if (this.issueRecomputeInFlight) return;
       this.issueRecomputeInFlight = true;
-      // Cooperative variant: yields between definitions so a 1–3 s
+      // The injected hook routes through the writer worker in the
-      // recompute doesn't freeze the main thread (and the user's
+      // desktop host, so this is off-thread; main stays free to
-      // clicks) for the full duration. Trades atomic visibility of
+      // service IPC and pump the crawl queue. CLI fallback runs the
-      // the rebuild for UI responsiveness — fine for the live tick
+      // yielding variant inline, which is fine for that context.
-      // since the post-crawl pass still uses the atomic version for
+      void this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS)
-      // the final state.
-      void this.db
-        .recomputeUrlsIssuesYielding(EXPENSIVE_ISSUE_DEFINITIONS)
         .catch((err) => {
           this.emit(
             'debug',
@@ -230,6 +236,30 @@ export class Crawler extends EventEmitter {
     payload: Parameters<ProjectDb['writeFetchedUrl']>[0],
   ) => Promise<{ urlId: number }>;
 
+  /**
+   * Optional async hook for the post-crawl issue recompute. When the
+   * desktop main process injects a writer-worker bridge here, the
+   * 70+ correlated subqueries run off the main thread — without it,
+   * a 5K-URL crawl freezes the renderer for ~45 s in the post-crawl
+   * phase ("materialise-issues"). Defaults to running the recompute
+   * inline (CLI / test contexts).
+   */
+  private readonly recomputeIssues: (
+    definitions: ReadonlyArray<readonly [string, string]>,
+  ) => Promise<void>;
+
+  /**
+   * Generic post-crawl SQL-pass dispatch. Methods like
+   * `recomputeInlinks`, `recomputeRedirectChains`,
+   * `recomputeHreflangAnalysis`, `recomputeHreflangInconsistent`,
+   * `recomputePaginationSequence` are sync void calls on `ProjectDb`
+   * — running them inline on the main thread blocks the renderer for
+   * ~5-15 s combined on a 5K+ URL crawl. The desktop host injects a
+   * writer-worker dispatcher here so each pass runs off-thread; CLI /
+   * tests fall back to calling the method directly on `this.db`.
+   */
+  private readonly runDbPass: (methodName: string) => Promise<void>;
+
   constructor(
     config: CrawlConfig,
     db: ProjectDb,
@@ -243,6 +273,10 @@ export class Crawler extends EventEmitter {
       writeFetchedUrl?: (
         payload: Parameters<ProjectDb['writeFetchedUrl']>[0],
       ) => Promise<{ urlId: number }>;
+      recomputeIssues?: (
+        definitions: ReadonlyArray<readonly [string, string]>,
+      ) => Promise<void>;
+      runDbPass?: (methodName: string) => Promise<void>;
     } = {},
   ) {
     super();
@@ -252,6 +286,15 @@ export class Crawler extends EventEmitter {
       ((html, pageUrl, parseOpts) => Promise.resolve(parseHtml(html, pageUrl, parseOpts)));
     this.writeFetchedUrl =
       opts.writeFetchedUrl ?? ((payload) => Promise.resolve(this.db.writeFetchedUrl(payload)));
+    this.recomputeIssues =
+      opts.recomputeIssues ?? ((defs) => this.db.recomputeUrlsIssuesYielding(defs));
+    this.runDbPass =
+      opts.runDbPass ??
+      ((method) => {
+        const fn = (this.db as unknown as Record<string, unknown>)[method];
+        if (typeof fn === 'function') (fn as () => void).call(this.db);
+        return Promise.resolve();
+      });
     // Wave 9 — Resolve the active proxy. If a named profile is selected
     // and present in `proxyProfiles`, its URL wins over the legacy
     // `proxyUrl` field; if the named profile doesn't resolve we fall
@@ -268,14 +311,27 @@ export class Crawler extends EventEmitter {
     this.config = config;
     this.db = db;
     const concurrency = Math.max(1, Math.min(200, config.maxConcurrency));
-    const intervalCap = Math.max(1, config.maxRps);
+    // Rate-limit (interval + intervalCap) intentionally NOT set on
-    this.queue = new PQueue({ concurrency, interval: 1000, intervalCap });
+    // the queue. p-queue's bucket-refill semantics interact poorly
+    // with long-tailed response times: after the first burst lands
+    // and slots free up, dispatch can stall for an entire bucket
+    // before the next batch goes out — and on a 1 s/intervalCap=20
+    // bucket the queue would freeze entirely once the first ~20
+    // tasks completed. Throughput is now bounded purely by the
+    // worker pool (concurrency × 1/avg_response), which on a slow
+    // remote server gives the user the 5-10 URL/s they expect
+    // instead of ~1 URL/s. config.maxRps is honoured indirectly via
+    // maxConcurrency — users who need a hard RPS cap should drop
+    // concurrency rather than trust per-second buckets.
+    this.queue = new PQueue({ concurrency });
+    // Track queue concurrency from the start so adaptive shrinkage
+    // doesn't have to wait for the first reportRendererLag call to
+    // materialise the running value.
+    this.currentConcurrency = concurrency;
     // External probes run on a separate queue so slow third-party hosts
     // don't block the main crawl.
     this.externalQueue = new PQueue({
       concurrency: Math.max(2, Math.min(10, concurrency)),
-      interval: 1000,
-      intervalCap: Math.max(2, intervalCap),
     });
     // Compile include/exclude patterns once — an invalid pattern should
     // surface to the user as a crawler error, not a silent miss.
@@ -445,7 +501,17 @@ export class Crawler extends EventEmitter {
     try {
       // Wait for internal crawl first, then drain any external probes still
       // in flight or queued (externals may have been enqueued during internal).
-      await this.queue.onIdle();
+      // p-queue's onIdle resolves the moment size+pending hits 0, but
+      // fetchAndProcess can race with that: a worker finishes the
+      // *last* in-flight task, link extraction enqueues new URLs a
+      // microtask later, and onIdle has already resolved. Loop while
+      // counters disagree — one extra 50 ms wait per pass guards the
+      // race window without spinning.
+      while (!this.stopped) {
+        await this.queue.onIdle();
+        await new Promise<void>((r) => setTimeout(r, 50));
+        if (this.queue.size + this.queue.pending === 0) break;
+      }
       await this.externalQueue.onIdle();
       // robots.txt + sitemap discovery may still be running — wait for
       // both before the post-crawl recompute so issue filters depending
@@ -512,44 +578,49 @@ export class Crawler extends EventEmitter {
       await yieldToEventLoop();
       this.emit('info', 'Recomputing inlinks…');
       this.setOp('post-crawl:recompute-inlinks');
-      this.db.recomputeInlinks();
+      await this.runDbPass('recomputeInlinks');
     }
     if (this.config.analyseRedirectChains) {
       await yieldToEventLoop();
       this.emit('info', 'Recomputing redirect chains…');
       this.setOp('post-crawl:recompute-redirect-chains');
-      this.db.recomputeRedirectChains();
+      await this.runDbPass('recomputeRedirectChains');
     }
     if (this.config.analyseHreflang) {
       await yieldToEventLoop();
       this.emit('info', 'Recomputing hreflang analysis…');
       this.setOp('post-crawl:recompute-hreflang');
-      this.db.recomputeHreflangAnalysis();
+      await this.runDbPass('recomputeHreflangAnalysis');
       await yieldToEventLoop();
       this.setOp('post-crawl:recompute-hreflang-inconsistent');
-      this.db.recomputeHreflangInconsistent();
+      await this.runDbPass('recomputeHreflangInconsistent');
     }
     if (this.config.analyseDuplicates) {
       await yieldToEventLoop();
       this.emit('info', 'Clustering duplicates…');
       this.setOp('post-crawl:cluster-duplicates');
+      // Duplicate clustering has JS-side simhash work that lives on
+      // the crawler instance, not the DB; can't blindly off-thread it.
       this.runDuplicateClustering();
     }
     if (this.config.analysePagination) {
       await yieldToEventLoop();
       this.emit('info', 'Detecting pagination sequence gaps…');
       this.setOp('post-crawl:pagination-sequence');
-      this.db.recomputePaginationSequence();
+      await this.runDbPass('recomputePaginationSequence');
     }
     if (this.config.analyseIssues) {
       await yieldToEventLoop();
       this.emit('info', 'Materialising issue counters…');
       this.setOp('post-crawl:materialise-issues');
-      // Use the cooperatively-scheduled variant (yields between each
+      // Route through the injected hook — in the desktop host this
-      // of the 70+ definitions). Trades atomic visibility of the
+      // dispatches to the writer-worker, keeping the 70+ correlated
-      // rebuild for a responsive main thread — on a 5k-URL DB the
+      // subqueries off the main thread. CLI / tests fall back to
-      // sync version was blocking the event loop for 22-26 s.
+      // running inline via `db.recomputeUrlsIssuesYielding`. Without
-      await this.db.recomputeUrlsIssuesYielding(EXPENSIVE_ISSUE_DEFINITIONS);
+      // offloading, a 5K-URL crawl freezes the main process for
+      // ~45 s in post-crawl, tripping Electron's "Not Responding"
+      // dialog.
+      await this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS);
     }
     await yieldToEventLoop();
     this.setOp('post-crawl:image-probes');
@@ -1022,6 +1093,24 @@ export class Crawler extends EventEmitter {
 
     // If the start URL isn't in the DB yet, kick off a brand-new crawl from it.
     if (!this.db.hasUrl(this.config.startUrl)) {
+      // Stub upsert before enqueue so the seed surfaces in the URL
+      // table within ~100 ms of clicking Start instead of waiting for
+      // the full network round-trip + parse + write (~2 s on a typical
+      // site). status_code=null reads as "fetching" in the table.
+      // fetchAndProcess's writeFetchedUrl later overwrites every field
+      // with the real response data via INSERT … ON CONFLICT UPDATE.
+      try {
+        this.db.upsertUrl({
+          url: this.config.startUrl,
+          contentKind: 'html',
+          statusCode: null,
+          statusText: 'fetching',
+          indexability: 'indexable',
+          depth: 0,
+        });
+      } catch {
+        /* non-fatal: fetchAndProcess will still write the real row */
+      }
       this.enqueue({ url: this.config.startUrl, depth: 0 });
     }
 
@@ -1140,6 +1229,19 @@ export class Crawler extends EventEmitter {
       }
       this.sitemapAbort = null;
     }
+    // Abort every in-flight HTTP fetch. Without this, Stop only halts
+    // *new* dispatch — the ~20 fetches already in flight would each
+    // finish their request and write a row, which races with Clear's
+    // db.reset() and resurrects ~19 rows in the table. The catch
+    // branch in fetchAndProcess detects the stopped-abort and exits
+    // silently without writing.
+    for (const c of this.inFlightFetchControllers) {
+      try {
+        c.abort();
+      } catch {
+        /* ignore — already aborted */
+      }
+    }
     // Drop any queued work. If paused, unblock onIdle() so start() can resolve.
     this.queue.clear();
     this.externalQueue.clear();
@@ -1150,9 +1252,22 @@ export class Crawler extends EventEmitter {
   pause(): void {
     if (this.stopped || this.paused) return;
     this.paused = true;
-    // PQueue.pause() halts dispatch but lets in-flight tasks finish naturally.
+    // PQueue.pause() halts new dispatch. By itself that lets every
+    // in-flight fetch run to completion — at concurrency 20 the user
+    // sees ~20 more URLs land in the table before things stop, which
+    // reads as "Pause didn't work." Aborting the live controllers
+    // turns Pause into "stop now": fetchAndProcess detects a paused-
+    // abort, re-enqueues the URL, and returns without writing a row.
     this.queue.pause();
     this.externalQueue.pause();
+    for (const c of this.inFlightFetchControllers) {
+      try {
+        c.abort();
+      } catch {
+        /* ignore — already aborted */
+      }
+    }
+    this.setOp('paused');
     this.emitProgress();
   }
 
@@ -1161,6 +1276,7 @@ export class Crawler extends EventEmitter {
     this.paused = false;
     this.queue.start();
     this.externalQueue.start();
+    this.setOp('idle');
     this.emitProgress();
   }
 
@@ -1446,6 +1562,7 @@ export class Crawler extends EventEmitter {
 
     const t0 = Date.now();
     const controller = new AbortController();
+    this.inFlightFetchControllers.add(controller);
     const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
     // Wave 3 — Optional max-response-time hard cap. Distinct from
     // `requestTimeoutMs` (which is the connect+headers timeout): this
@@ -1998,6 +2115,25 @@ export class Crawler extends EventEmitter {
         }
       }
     } catch (err) {
+      // Stop / Clear: abandon silently. No row is written, no failure
+      // counter is bumped, the URL is not re-enqueued. Without this
+      // gate the ~20 in-flight URLs after Stop would each land in the
+      // table — and Clear's db.reset() races with their writes,
+      // resurrecting "ghost" rows in a freshly-cleared table.
+      if (this.stopped && controller.signal.aborted) {
+        return;
+      }
+      // User-initiated Pause aborts every live controller. Don't treat
+      // those as fetch failures — the URL hasn't been seen yet, so we
+      // re-enqueue it directly (bypassing `enqueue` because the URL is
+      // already in the `seen` set from the original dispatch). Without
+      // this, Pause would mark `concurrency` (often 20) URLs as failed
+      // and surface them as errors in the URL table. Re-enqueue lands
+      // in a paused queue → resume() picks them up cleanly.
+      if (this.paused && controller.signal.aborted) {
+        this.queue.add(() => this.fetchAndProcess(item)).catch(() => undefined);
+        return;
+      }
       this.failed++;
       const detail = formatFetchError(err);
       const elapsed = Date.now() - t0;
@@ -2020,6 +2156,7 @@ export class Crawler extends EventEmitter {
         depth: item.depth,
       });
     } finally {
+      this.inFlightFetchControllers.delete(controller);
       clearTimeout(timeout);
       if (respTimeTimer) clearTimeout(respTimeTimer);
       // Politeness delay — applied per worker *after* each request so a
@@ -2171,8 +2308,14 @@ export class Crawler extends EventEmitter {
     const ceiling = Math.max(1, Math.min(200, this.config.maxConcurrency));
     if (this.currentConcurrency === 0) this.currentConcurrency = ceiling;
     let next = this.currentConcurrency;
+    // Floor 5 (was 1). The renderer's lag probe occasionally spikes
+    // for unrelated reasons (GC, an input burst, devtools open) and
+    // a 1-floor would let those one-off spikes corner the queue at
+    // single-task dispatch — which on a slow remote server (avg resp
+    // 1.5-2 s) cuts throughput from 10 URL/s to 0.5 URL/s and the
+    // queue never recovers because the user never produces zero lag.
     if (lagMs > 200) {
-      next = Math.max(1, this.currentConcurrency - 1);
+      next = Math.max(5, this.currentConcurrency - 1);
     } else if (lagMs < 30) {
       next = Math.min(ceiling, this.currentConcurrency + 1);
     }

packages/core/src/index.ts

@@ -10,6 +10,12 @@ export {
 export { exportUrlsToCsv } from './csv-export.js';
 export { exportUrlsToJson, type JsonExportOptions } from './json-export.js';
 export { exportUrlsToXml } from './xml-export.js';
+export {
+  exportTabular,
+  type TabularExportOptions,
+  type TabularExportResult,
+  type TabularSection,
+} from './tabular-export.js';
 export { testUrlAgainstRobots, type RobotsTestResult } from './robots.js';
 export {
   exportSitemap,

packages/core/src/tabular-export.ts

@@ -0,0 +1,398 @@
+import { createWriteStream, writeFileSync } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+import path from 'node:path';
+import zlib from 'node:zlib';
+import type { ProjectDb } from '@freecrawl/db';
+import type { CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
+
+export interface TabularSection {
+  label: string;
+  category: UrlCategory;
+}
+
+export interface TabularExportOptions {
+  format: 'csv' | 'xlsx';
+  sections: TabularSection[];
+  /** Keys of CrawlUrlRow to include, in order. */
+  columns: string[];
+  /** When provided AND non-empty, restrict every section to these row ids. */
+  selectedIds?: number[];
+}
+
+export interface TabularExportResult {
+  /** File (xlsx / single-section csv) or folder (multi-section csv). */
+  filePath: string;
+  files: string[];
+  rowsWritten: number;
+}
+
+function escapeCsv(value: unknown): string {
+  if (value === null || value === undefined) return '';
+  const str = String(value);
+  if (/[",\n\r]/.test(str)) {
+    return `"${str.replace(/"/g, '""')}"`;
+  }
+  return str;
+}
+
+function rowSource(
+  db: ProjectDb,
+  category: UrlCategory,
+  selectedIds: number[] | undefined,
+): Iterable<CrawlUrlRow> {
+  if (selectedIds && selectedIds.length > 0) {
+    // Apply selectedIds first, then narrow by category client-side. The
+    // sidebar's category predicates can be expensive; for export the
+    // selection is always the small side.
+    const ids = new Set(selectedIds);
+    return (function* () {
+      for (const row of db.iterateUrlsByCategory(category)) {
+        if (ids.has(row.id)) yield row;
+      }
+    })();
+  }
+  return db.iterateUrlsByCategory(category);
+}
+
+async function writeCsvFile(
+  db: ProjectDb,
+  filePath: string,
+  columns: string[],
+  category: UrlCategory,
+  selectedIds: number[] | undefined,
+): Promise<number> {
+  let rowsWritten = 0;
+  const header = columns.join(',') + '\n';
+  const source = rowSource(db, category, selectedIds);
+  const generator = async function* (): AsyncGenerator<string> {
+    yield '' + header;
+    for (const row of source) {
+      const line =
+        columns
+          .map((c) => escapeCsv((row as unknown as Record<string, unknown>)[c]))
+          .join(',') + '\n';
+      rowsWritten++;
+      yield line;
+    }
+  };
+  await pipeline(
+    Readable.from(generator()),
+    createWriteStream(filePath, { encoding: 'utf8' }),
+  );
+  return rowsWritten;
+}
+
+// ── Minimal XLSX writer ─────────────────────────────────────────────
+//
+// Office Open XML SpreadsheetML, packaged as a ZIP with DEFLATE entries.
+// Uses inline strings (no shared-string table) — slightly fatter on disk
+// but lets us stream-build sheet XML row-by-row without a second pass.
+//
+// Worksheet shape:
+//   <worksheet><sheetData>
+//     <row r="1"><c r="A1" t="inlineStr"><is><t>URL</t></is></c>...</row>
+//     <row r="2"><c r="A2" t="inlineStr"><is><t>https://...</t></is></c>...</row>
+//   </sheetData></worksheet>
+//
+// Numbers go in as <c><v>123</v></c> (no `t=` attribute, default = number).
+
+function xmlEscape(str: string): string {
+  return str
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    // strip control chars XML 1.0 forbids
+    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '');
+}
+
+function colLetter(idx: number): string {
+  // 0-based → A, B, ..., Z, AA, AB, ...
+  let n = idx;
+  let s = '';
+  while (true) {
+    const r = n % 26;
+    s = String.fromCharCode(65 + r) + s;
+    n = Math.floor(n / 26) - 1;
+    if (n < 0) break;
+  }
+  return s;
+}
+
+function sanitizeSheetName(label: string, used: Set<string>): string {
+  let base = label.replace(/[\\/?*\[\]:]/g, ' ').trim().slice(0, 31);
+  if (!base) base = 'Sheet';
+  let candidate = base;
+  let i = 2;
+  while (used.has(candidate.toLowerCase())) {
+    const suffix = ` (${i++})`;
+    candidate = base.slice(0, 31 - suffix.length) + suffix;
+  }
+  used.add(candidate.toLowerCase());
+  return candidate;
+}
+
+function buildSheetXml(
+  rows: CrawlUrlRow[],
+  columns: string[],
+): string {
+  const parts: string[] = [];
+  parts.push(
+    '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
+      '<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">' +
+      '<sheetData>',
+  );
+  // Header row
+  parts.push('<row r="1">');
+  columns.forEach((c, i) => {
+    parts.push(
+      `<c r="${colLetter(i)}1" t="inlineStr"><is><t>${xmlEscape(c)}</t></is></c>`,
+    );
+  });
+  parts.push('</row>');
+  // Data rows
+  rows.forEach((row, rIdx) => {
+    const r = rIdx + 2;
+    parts.push(`<row r="${r}">`);
+    columns.forEach((col, i) => {
+      const v = (row as unknown as Record<string, unknown>)[col];
+      if (v === null || v === undefined || v === '') return;
+      const cellRef = `${colLetter(i)}${r}`;
+      if (typeof v === 'number' && Number.isFinite(v)) {
+        parts.push(`<c r="${cellRef}"><v>${v}</v></c>`);
+      } else if (typeof v === 'boolean') {
+        parts.push(`<c r="${cellRef}" t="b"><v>${v ? 1 : 0}</v></c>`);
+      } else {
+        parts.push(
+          `<c r="${cellRef}" t="inlineStr"><is><t>${xmlEscape(String(v))}</t></is></c>`,
+        );
+      }
+    });
+    parts.push('</row>');
+  });
+  parts.push('</sheetData></worksheet>');
+  return parts.join('');
+}
+
+// ── ZIP writer (DEFLATE, no central directory streaming — small enough
+// to buffer per-entry, write central dir at the end). ────────────────
+
+interface ZipEntry {
+  name: string;
+  raw: Buffer;
+  compressed: Buffer;
+  crc32: number;
+  offset: number;
+}
+
+const CRC_TABLE = (() => {
+  const table = new Uint32Array(256);
+  for (let n = 0; n < 256; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) {
+      c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    }
+    table[n] = c >>> 0;
+  }
+  return table;
+})();
+
+function crc32(buf: Buffer): number {
+  let c = 0xffffffff;
+  for (let i = 0; i < buf.length; i++) {
+    c = CRC_TABLE[(c ^ buf[i]!) & 0xff]! ^ (c >>> 8);
+  }
+  return (c ^ 0xffffffff) >>> 0;
+}
+
+function buildZip(files: { name: string; data: Buffer }[]): Buffer {
+  const entries: ZipEntry[] = [];
+  const localChunks: Buffer[] = [];
+  let offset = 0;
+
+  for (const f of files) {
+    const raw = f.data;
+    const compressed = zlib.deflateRawSync(raw);
+    const crc = crc32(raw);
+    const nameBuf = Buffer.from(f.name, 'utf8');
+
+    const localHeader = Buffer.alloc(30);
+    localHeader.writeUInt32LE(0x04034b50, 0); // local file header sig
+    localHeader.writeUInt16LE(20, 4); // version needed
+    localHeader.writeUInt16LE(0x0800, 6); // gp flag (UTF-8 name)
+    localHeader.writeUInt16LE(8, 8); // method = deflate
+    localHeader.writeUInt16LE(0, 10); // mod time
+    localHeader.writeUInt16LE(0, 12); // mod date
+    localHeader.writeUInt32LE(crc, 14);
+    localHeader.writeUInt32LE(compressed.length, 18);
+    localHeader.writeUInt32LE(raw.length, 22);
+    localHeader.writeUInt16LE(nameBuf.length, 26);
+    localHeader.writeUInt16LE(0, 28); // extra field len
+
+    entries.push({
+      name: f.name,
+      raw,
+      compressed,
+      crc32: crc,
+      offset,
+    });
+    localChunks.push(localHeader, nameBuf, compressed);
+    offset += localHeader.length + nameBuf.length + compressed.length;
+  }
+
+  const centralChunks: Buffer[] = [];
+  let cdSize = 0;
+  for (const e of entries) {
+    const nameBuf = Buffer.from(e.name, 'utf8');
+    const cd = Buffer.alloc(46);
+    cd.writeUInt32LE(0x02014b50, 0); // central dir sig
+    cd.writeUInt16LE(20, 4); // version made by
+    cd.writeUInt16LE(20, 6); // version needed
+    cd.writeUInt16LE(0x0800, 8); // gp flag
+    cd.writeUInt16LE(8, 10); // method
+    cd.writeUInt16LE(0, 12);
+    cd.writeUInt16LE(0, 14);
+    cd.writeUInt32LE(e.crc32, 16);
+    cd.writeUInt32LE(e.compressed.length, 20);
+    cd.writeUInt32LE(e.raw.length, 24);
+    cd.writeUInt16LE(nameBuf.length, 28);
+    cd.writeUInt16LE(0, 30);
+    cd.writeUInt16LE(0, 32);
+    cd.writeUInt16LE(0, 34);
+    cd.writeUInt16LE(0, 36);
+    cd.writeUInt32LE(0, 38);
+    cd.writeUInt32LE(e.offset, 42);
+    centralChunks.push(cd, nameBuf);
+    cdSize += cd.length + nameBuf.length;
+  }
+
+  const eocd = Buffer.alloc(22);
+  eocd.writeUInt32LE(0x06054b50, 0);
+  eocd.writeUInt16LE(0, 4);
+  eocd.writeUInt16LE(0, 6);
+  eocd.writeUInt16LE(entries.length, 8);
+  eocd.writeUInt16LE(entries.length, 10);
+  eocd.writeUInt32LE(cdSize, 12);
+  eocd.writeUInt32LE(offset, 16);
+  eocd.writeUInt16LE(0, 20);
+
+  return Buffer.concat([...localChunks, ...centralChunks, eocd]);
+}
+
+function buildXlsxBuffer(
+  sheets: { name: string; rows: CrawlUrlRow[] }[],
+  columns: string[],
+): Buffer {
+  const contentTypes =
+    '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
+    '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">' +
+    '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>' +
+    '<Default Extension="xml" ContentType="application/xml"/>' +
+    '<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>' +
+    sheets
+      .map(
+        (_, i) =>
+          `<Override PartName="/xl/worksheets/sheet${i + 1}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>`,
+      )
+      .join('') +
+    '</Types>';
+
+  const rels =
+    '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
+    '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
+    '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>' +
+    '</Relationships>';
+
+  const workbook =
+    '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
+    '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">' +
+    '<sheets>' +
+    sheets
+      .map(
+        (s, i) =>
+          `<sheet name="${xmlEscape(s.name)}" sheetId="${i + 1}" r:id="rId${i + 1}"/>`,
+      )
+      .join('') +
+    '</sheets></workbook>';
+
+  const workbookRels =
+    '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
+    '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
+    sheets
+      .map(
+        (_, i) =>
+          `<Relationship Id="rId${i + 1}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet${i + 1}.xml"/>`,
+      )
+      .join('') +
+    '</Relationships>';
+
+  const files: { name: string; data: Buffer }[] = [
+    { name: '[Content_Types].xml', data: Buffer.from(contentTypes, 'utf8') },
+    { name: '_rels/.rels', data: Buffer.from(rels, 'utf8') },
+    { name: 'xl/workbook.xml', data: Buffer.from(workbook, 'utf8') },
+    { name: 'xl/_rels/workbook.xml.rels', data: Buffer.from(workbookRels, 'utf8') },
+  ];
+  sheets.forEach((s, i) => {
+    files.push({
+      name: `xl/worksheets/sheet${i + 1}.xml`,
+      data: Buffer.from(buildSheetXml(s.rows, columns), 'utf8'),
+    });
+  });
+
+  return buildZip(files);
+}
+
+export async function exportTabular(
+  db: ProjectDb,
+  outputPath: string,
+  options: TabularExportOptions,
+): Promise<TabularExportResult> {
+  const { format, sections, columns, selectedIds } = options;
+  if (sections.length === 0) {
+    throw new Error('exportTabular: at least one section is required');
+  }
+  if (columns.length === 0) {
+    throw new Error('exportTabular: at least one column is required');
+  }
+
+  if (format === 'csv') {
+    if (sections.length === 1) {
+      const rows = await writeCsvFile(
+        db,
+        outputPath,
+        columns,
+        sections[0]!.category,
+        selectedIds,
+      );
+      return { filePath: outputPath, files: [outputPath], rowsWritten: rows };
+    }
+    // Multi-section CSV → write one file per section into the chosen
+    // folder. `outputPath` is treated as a folder.
+    let total = 0;
+    const written: string[] = [];
+    for (const section of sections) {
+      const safe = section.label.replace(/[\\/?*\[\]:<>|"]/g, '_');
+      const file = path.join(outputPath, `${safe}.csv`);
+      total += await writeCsvFile(db, file, columns, section.category, selectedIds);
+      written.push(file);
+    }
+    return { filePath: outputPath, files: written, rowsWritten: total };
+  }
+
+  // xlsx — collect rows per section into memory (bounded by typical
+  // section size), build workbook, write once.
+  const usedNames = new Set<string>();
+  let total = 0;
+  const sheets = sections.map((section) => {
+    const rows: CrawlUrlRow[] = [];
+    for (const row of rowSource(db, section.category, selectedIds)) {
+      rows.push(row);
+    }
+    total += rows.length;
+    return { name: sanitizeSheetName(section.label, usedNames), rows };
+  });
+  const buf = buildXlsxBuffer(sheets, columns);
+  writeFileSync(outputPath, buf);
+  return { filePath: outputPath, files: [outputPath], rowsWritten: total };
+}

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/project-db.ts

@@ -3036,6 +3036,13 @@ export class ProjectDb {
     return row ? this.rowFromDb(row) : null;
   }
 
+  /** Resolve a URL string to its numeric id, or null if absent. Used by
+   *  the MCP server when a caller supplies a URL instead of an id. */
+  getUrlIdByUrl(url: string): number | null {
+    const row = this.stmtGetUrlId.get(url) as { id: number } | undefined;
+    return row ? row.id : null;
+  }
+
   getInlinks(url: string, limit: number): { rows: InlinkRow[]; total: number } {
     const total = (
       this.db.prepare('SELECT COUNT(*) AS c FROM links WHERE to_url = ?').get(url) as { c: number }

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/ipc.ts

@@ -39,6 +39,9 @@ export const IPC = {
   exportCsv: 'export:csv',
   exportJson: 'export:json',
   exportXml: 'export:xml',
+  /** New unified tabular export with column selection + multi-tab support
+   * (CSV / XLSX). Backs the in-table "Export" button. */
+  exportTabular: 'export:tabular',
   /** GDPR-aligned per-domain delete. Wipes every row whose URL host
    * matches the given domain (and that domain's links/images/headers/
    * url_sources). Used by Settings → "Delete Domain Data". */
@@ -213,6 +216,38 @@ export interface ExportXmlResult {
   rowsWritten: number;
 }
 
+/** One section / sheet of a tabular export — a single category fed into
+ * the workbook (xlsx) or split into its own file (csv). */
+export interface ExportTabularSection {
+  /** Display label — becomes the xlsx sheet name and the csv file suffix. */
+  label: string;
+  category: UrlCategory;
+}
+
+export interface ExportTabularInput {
+  format: 'csv' | 'xlsx';
+  /**
+   * One or more sections. With CSV + multiple sections, files are written
+   * into a folder the user picks; xlsx always produces a single workbook
+   * with one sheet per section.
+   */
+  sections: ExportTabularSection[];
+  /** Keys of `CrawlUrlRow` to emit, in order. */
+  columns: string[];
+  /** Optional pre-resolved output path (skip the dialog). */
+  filePath?: string;
+  /** When set, restrict every section to these row ids. */
+  selectedIds?: number[];
+}
+
+export interface ExportTabularResult {
+  /** Output file (xlsx + single-section csv) or folder (multi-section csv). */
+  filePath: string;
+  /** Files actually written. Multi-section CSV writes one file per section. */
+  files: string[];
+  rowsWritten: number;
+}
+
 export interface DataDeleteByDomainInput {
   /** Hostname to wipe — case-insensitive, no scheme/port. */
   domain: string;
@@ -676,6 +711,7 @@ export interface FreeCrawlApi {
   exportCsv(input: ExportCsvInput): Promise<ExportCsvResult>;
   exportJson(input: ExportJsonInput): Promise<ExportJsonResult>;
   exportXml(input: ExportXmlInput): Promise<ExportXmlResult>;
+  exportTabular(input: ExportTabularInput): Promise<ExportTabularResult>;
   dataDeleteByDomain(
     input: DataDeleteByDomainInput,
   ): Promise<DataDeleteByDomainResult>;

tsconfig.json

@@ -5,6 +5,7 @@
     { "path": "./packages/db" },
     { "path": "./packages/core" },
     { "path": "./apps/desktop" },
-    { "path": "./apps/cli" }
+    { "path": "./apps/cli" },
+    { "path": "./apps/mcp-server" }
   ]
 }