diff --git a/README.md b/README.md
index f9df488..490000b 100644
--- a/README.md
+++ b/README.md
@@ -215,6 +215,97 @@ npm --workspace apps/desktop run build:linux # AppImage / .deb
+
+π€ MCP server β query your crawl from Claude / any MCP client
+
+
+
+FreeCrawl ships an **MCP (Model Context Protocol) server** that exposes the active `.seoproject` to AI agents over stdio. Read-only β runs alongside the desktop app without contention (SQLite WAL).
+
+**Tools**: `get_summary`, `get_overview_counts`, `top_issues`, `query_urls`, `get_url_detail`, `list_projects`, `set_project`, `current_project`.
+
+**1. Build it once:**
+
+```bash
+npm run build:mcp
+```
+
+This produces `apps/mcp-server/dist/index.js`.
+
+**2. Register it with your MCP client.**
+
+
+Claude Desktop
+
+Edit your Claude Desktop config:
+
+| Platform | Path |
+| :--- | :--- |
+| Windows | `%APPDATA%\Claude\claude_desktop_config.json` |
+| macOS | `~/Library/Application Support/Claude/claude_desktop_config.json` |
+| Linux | `~/.config/Claude/claude_desktop_config.json` |
+
+```json
+{
+ "mcpServers": {
+ "freecrawl": {
+ "command": "node",
+ "args": ["/absolute/path/to/FreeCrawl-SEO-Tool/apps/mcp-server/dist/index.js"]
+ }
+ }
+}
+```
+
+Restart Claude Desktop. The `freecrawl` server appears under the tool π icon.
+
+
+
+
+Claude Code (CLI)
+
+```bash
+claude mcp add freecrawl -- node /absolute/path/to/FreeCrawl-SEO-Tool/apps/mcp-server/dist/index.js
+```
+
+
+
+
+Other MCP clients
+
+Run the binary directly with stdio transport:
+
+```bash
+node apps/mcp-server/dist/index.js
+```
+
+The server speaks newline-delimited JSON-RPC 2.0 β point any MCP-compatible client at it.
+
+
+
+**3. Try it.** Ask your agent things like:
+
+> *"Show the 10 URLs with the longest response time in my last crawl."*
+> *"What are the top 5 issue categories with the most affected pages?"*
+> *"List every URL with a missing meta description."*
+
+**Pointing at a non-default project:**
+
+By default the server reads `/projects/default.seoproject` (the same file the desktop app uses). Override with the `FREECRAWL_PROJECT` env var, or call the `set_project` tool mid-session:
+
+```json
+{
+ "mcpServers": {
+ "freecrawl": {
+ "command": "node",
+ "args": ["/path/to/apps/mcp-server/dist/index.js"],
+ "env": { "FREECRAWL_PROJECT": "/path/to/audit.seoproject" }
+ }
+ }
+}
+```
+
+
+
---
diff --git a/apps/cli/package.json b/apps/cli/package.json
index 0e19cdf..c366cd3 100644
--- a/apps/cli/package.json
+++ b/apps/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/cli",
- "version": "0.2.7",
+ "version": "0.2.8",
"private": true,
"type": "module",
"bin": {
diff --git a/apps/desktop/package.json b/apps/desktop/package.json
index 8da8ed7..5d67858 100644
--- a/apps/desktop/package.json
+++ b/apps/desktop/package.json
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/desktop",
- "version": "0.2.7",
+ "version": "0.2.8",
"private": true,
"description": "FreeCrawl SEO Γ’β¬β desktop application",
"main": "./out/main/index.js",
diff --git a/apps/desktop/src/main/db-reader-pool.ts b/apps/desktop/src/main/db-reader-pool.ts
index fd3cfb9..ebda2db 100644
--- a/apps/desktop/src/main/db-reader-pool.ts
+++ b/apps/desktop/src/main/db-reader-pool.ts
@@ -4,43 +4,48 @@ import { fileURLToPath } from 'node:url';
import * as logger from './logger.js';
/**
- * Pool of one worker_thread holding a read-only SQLite connection.
- * "Pool" is forward-compatible naming β for now we run a single worker
- * because SQLite's WAL allows arbitrary concurrent readers but adding a
- * second worker would just double our cache footprint without gaining
- * parallelism on a single physical disk.
+ * Pool of N worker_threads, each holding its own read-only SQLite
+ * connection. SQLite WAL mode allows arbitrary concurrent readers, so
+ * spreading queries across multiple workers gives genuine parallelism
+ * for UI reads β critical because:
+ *
+ * - The OverviewSidebar refresh issues a 130-clause `getOverviewCounts`
+ * every few seconds. On a single-worker pool this serialises every
+ * other read behind it, freezing the URL table refresh and the
+ * bottom panel's per-URL detail / inlinks queries until the heavy
+ * pass completes.
+ * - Pool now lets the heavy aggregate run on one worker while the
+ * short, latency-sensitive queries (urlsQuery for the visible
+ * chunks, urlDetail for the bottom panel) run on the others.
+ *
+ * Routing: round-robin with a "least busy" tie-break β for each call
+ * we pick the worker with the fewest pending requests, and on a tie
+ * fall back to a rotating cursor so two same-loaded workers don't
+ * always get the next call dropped on the same one.
*
* Lifecycle:
- * - `init(dbPath)` spawns the worker, points it at the file.
- * - `swap(newPath)` is called on Open Project β terminates the old
- * worker and spawns a fresh one; in-flight requests are rejected
+ * - `init(dbPath)` spawns all workers, points them at the file.
+ * - `swap(newPath)` is called on Open Project β terminates every old
+ * worker and spawns a fresh set; in-flight requests are rejected
* with `Error('reader-swapped')` so callers can retry safely.
* - `terminate()` on app quit. After terminate, all `call()` resolve
* with `Error('reader-terminated')`.
*
* Crash recovery:
- * - If the worker exits unexpectedly (`exit` event with non-zero code
- * while we still have a path) we auto-respawn up to MAX_RESTARTS
- * within RESTART_WINDOW_MS. After the cap we surface the failure
- * and stop trying β the user gets a clear log entry instead of a
- * restart loop.
- * - In-flight requests during a crash are rejected with
+ * - If a worker exits unexpectedly, that one slot auto-respawns up to
+ * MAX_RESTARTS within RESTART_WINDOW_MS. Other workers in the pool
+ * keep serving traffic.
+ * - In-flight requests on the crashed worker are rejected with
* `Error('reader-crashed')`. Callers fall back to the main-process
- * ProjectDb (see `dbReaderCallOrFallback`) so the UI stays alive.
+ * ProjectDb (see `callReaderOrFallback`) so the UI stays alive.
*/
const __dirname = path.dirname(fileURLToPath(import.meta.url));
-// The worker is built alongside the main bundle as `db-reader-worker.js`
-// (electron-vite produces flat output in `out/main/`). In dev mode
-// electron-vite uses the same convention.
const WORKER_PATH = path.join(__dirname, 'db-reader-worker.js');
// 30 s default leaves headroom for SQLite read contention during the
-// post-crawl recompute phase (recomputeUrlsIssues holds a writer
-// transaction that can keep reader queries blocked behind earlier
-// queued requests for several seconds at a time on large crawls).
-// Genuine worker hangs are still surfaced β just with a slightly
-// longer detection window than before.
+// post-crawl recompute phase. Genuine worker hangs are still surfaced
+// β just with a slightly longer detection window than before.
const REQUEST_TIMEOUT_MS = 30_000;
// Heavy aggregate queries (the overview sidebar's 130-counter pass and
// the post-crawl materialiser's counter fan-out) can run past the
@@ -54,7 +59,17 @@ const HEAVY_METHODS = new Set([
const MAX_RESTARTS = 3;
const RESTART_WINDOW_MS = 60_000;
+// Reader pool size. Four workers gives the typical UI workload (one
+// long-running aggregate from the sidebar + visible-table chunk fetch
+// + bottom-panel detail + crawl-engine background reads) a free slot
+// for every concurrent request; bigger doesn't help because most UI
+// reads are short and a dedicated worker per query becomes pure
+// memory cost. Each worker keeps its own SQLite handle (~5 MB heap)
+// and prepared-statement cache (~10 MB on a populated DB).
+const POOL_SIZE = 4;
+
interface PendingRequest {
+ workerIdx: number;
resolve: (v: unknown) => void;
reject: (e: Error) => void;
timer: ReturnType;
@@ -67,94 +82,143 @@ interface ResponseMessage {
error?: string;
}
+interface WorkerSlot {
+ worker: Worker | null;
+ pending: number;
+ restartTimes: number[];
+}
+
class DbReaderPool {
- private worker: Worker | null = null;
+ private slots: WorkerSlot[] = [];
private dbPath: string | null = null;
private freezeWatchdogSab: SharedArrayBuffer | null = null;
private nextRequestId = 1;
private pending = new Map();
- private restartTimes: number[] = [];
+ private rrCursor = 0;
private terminated = false;
- /** Spawn (or respawn) the worker pointed at `dbPath`. Idempotent.
- *
- * `freezeWatchdogSab` β if provided, the worker will tick its
- * own heartbeat + publish its current method into this shared
- * buffer so the freeze-watchdog thread can detect a stuck
- * reader. Pass `null` to disable diagnostics. */
+ /** Spawn (or respawn) the pool pointed at `dbPath`. Idempotent. */
init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
if (this.terminated) {
throw new Error('DbReaderPool: cannot init after terminate()');
}
this.freezeWatchdogSab = freezeWatchdogSab;
- if (this.dbPath === dbPath && this.worker !== null) return;
+ if (this.dbPath === dbPath && this.slots.some((s) => s.worker !== null)) return;
this.dbPath = dbPath;
- this.spawn();
+ this.slots = Array.from({ length: POOL_SIZE }, () => ({
+ worker: null,
+ pending: 0,
+ restartTimes: [],
+ }));
+ for (let i = 0; i < POOL_SIZE; i++) this.spawn(i);
}
- /** Switch the worker to a different .seoproject file. */
+ /** Switch the pool to a different .seoproject file. */
swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab;
- if (this.dbPath === newPath && this.worker !== null) return;
+ if (this.dbPath === newPath && this.slots.some((s) => s.worker !== null)) return;
this.failPendingWith('reader-swapped');
this.dbPath = newPath;
- this.spawn();
+ if (this.slots.length === 0) {
+ this.slots = Array.from({ length: POOL_SIZE }, () => ({
+ worker: null,
+ pending: 0,
+ restartTimes: [],
+ }));
+ }
+ for (let i = 0; i < POOL_SIZE; i++) this.spawn(i);
}
/** Permanent shutdown β used on app quit. */
async terminate(): Promise {
this.terminated = true;
this.failPendingWith('reader-terminated');
- if (this.worker) {
- const w = this.worker;
- this.worker = null;
- try {
- await w.terminate();
- } catch {
- /* already gone */
- }
- }
+ const workers = this.slots.map((s) => s.worker).filter((w): w is Worker => w !== null);
+ for (const slot of this.slots) slot.worker = null;
+ await Promise.all(
+ workers.map((w) =>
+ w.terminate().catch(() => undefined),
+ ),
+ );
}
- /** True while a worker is up and ready to receive requests. */
+ /** True while at least one worker is up and ready. */
isReady(): boolean {
- return this.worker !== null && !this.terminated;
+ return !this.terminated && this.slots.some((s) => s.worker !== null);
}
/**
- * Dispatch a method call to the worker. Callers should use the typed
- * wrapper `callReader(method, args)` declared in the main process
- * IPC layer. Resolves with the method's return value, rejects with
- * an Error on worker error / timeout / crash.
+ * Dispatch a method call to the least-busy ready worker. Resolves
+ * with the method's return value, rejects with an Error on worker
+ * error / timeout / crash.
*/
call(method: string, args: unknown[] = []): Promise {
if (this.terminated) return Promise.reject(new Error('reader-terminated'));
- if (!this.worker) return Promise.reject(new Error('reader-not-initialised'));
+ const slotIdx = this.pickSlot();
+ if (slotIdx < 0) return Promise.reject(new Error('reader-not-initialised'));
+ const slot = this.slots[slotIdx]!;
+ const worker = slot.worker;
+ if (!worker) return Promise.reject(new Error('reader-not-initialised'));
const requestId = this.nextRequestId++;
const timeoutMs = HEAVY_METHODS.has(method)
? HEAVY_REQUEST_TIMEOUT_MS
: REQUEST_TIMEOUT_MS;
+ slot.pending++;
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
- this.pending.delete(requestId);
+ const p = this.pending.get(requestId);
+ if (p) {
+ this.pending.delete(requestId);
+ this.slots[p.workerIdx]!.pending = Math.max(
+ 0,
+ this.slots[p.workerIdx]!.pending - 1,
+ );
+ }
reject(new Error(`reader-timeout: ${method} > ${timeoutMs}ms`));
}, timeoutMs);
this.pending.set(requestId, {
+ workerIdx: slotIdx,
resolve: resolve as (v: unknown) => void,
reject,
timer,
});
- this.worker!.postMessage({ requestId, method, args });
+ worker.postMessage({ requestId, method, args });
});
}
// ββ private ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
- private spawn(): void {
+ /**
+ * Pick the slot with the fewest pending requests. Ties are broken
+ * by a round-robin cursor so identically-loaded workers don't all
+ * receive their next call on the same slot.
+ */
+ private pickSlot(): number {
+ let best = -1;
+ let bestPending = Number.POSITIVE_INFINITY;
+ // Start the scan from the rotating cursor so RR is honoured on ties.
+ for (let i = 0; i < this.slots.length; i++) {
+ const idx = (this.rrCursor + i) % this.slots.length;
+ const s = this.slots[idx]!;
+ if (!s.worker) continue;
+ if (s.pending < bestPending) {
+ best = idx;
+ bestPending = s.pending;
+ }
+ }
+ if (best >= 0) {
+ this.rrCursor = (best + 1) % this.slots.length;
+ }
+ return best;
+ }
+
+ private spawn(slotIdx: number): void {
if (!this.dbPath) return;
- if (this.worker) {
- const old = this.worker;
- this.worker = null;
+ const slot = this.slots[slotIdx];
+ if (!slot) return;
+ if (slot.worker) {
+ const old = slot.worker;
+ slot.worker = null;
void old.terminate().catch(() => undefined);
}
try {
@@ -166,18 +230,19 @@ class DbReaderPool {
});
w.on('message', (msg: ResponseMessage) => this.handleResponse(msg));
w.on('error', (err) => {
- logger.log('error', 'db-reader', `worker error: ${err.message}`);
+ logger.log('error', 'db-reader', `worker[${slotIdx}] error: ${err.message}`);
});
- w.on('exit', (code) => this.handleExit(code));
- this.worker = w;
- logger.log('info', 'db-reader', `worker spawned for ${this.dbPath}`);
+ w.on('exit', (code) => this.handleExit(slotIdx, code));
+ slot.worker = w;
+ slot.pending = 0;
+ logger.log('info', 'db-reader', `worker[${slotIdx}] spawned for ${this.dbPath}`);
} catch (err) {
logger.log(
'error',
'db-reader',
- `worker spawn failed: ${err instanceof Error ? err.message : String(err)}`,
+ `worker[${slotIdx}] spawn failed: ${err instanceof Error ? err.message : String(err)}`,
);
- this.worker = null;
+ slot.worker = null;
}
}
@@ -186,6 +251,8 @@ class DbReaderPool {
if (!pending) return;
this.pending.delete(msg.requestId);
clearTimeout(pending.timer);
+ const slot = this.slots[pending.workerIdx];
+ if (slot) slot.pending = Math.max(0, slot.pending - 1);
if (msg.ok) {
pending.resolve(msg.result);
} else {
@@ -193,31 +260,38 @@ class DbReaderPool {
}
}
- private handleExit(code: number): void {
- this.worker = null;
+ private handleExit(slotIdx: number, code: number): void {
+ const slot = this.slots[slotIdx];
+ if (!slot) return;
+ slot.worker = null;
if (this.terminated) return;
+ // Reject only this slot's in-flight requests.
+ for (const [id, p] of this.pending) {
+ if (p.workerIdx === slotIdx) {
+ clearTimeout(p.timer);
+ p.reject(new Error('reader-crashed'));
+ this.pending.delete(id);
+ }
+ }
+ slot.pending = 0;
if (code === 0) return;
- this.failPendingWith('reader-crashed');
- // Restart bookkeeping: drop entries older than the window so a
- // long-lived process that crashes once a day doesn't permanently
- // exhaust its restart budget.
const now = Date.now();
- this.restartTimes = this.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS);
- if (this.restartTimes.length >= MAX_RESTARTS) {
+ slot.restartTimes = slot.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS);
+ if (slot.restartTimes.length >= MAX_RESTARTS) {
logger.log(
'error',
'db-reader',
- `worker crashed ${MAX_RESTARTS}Γ within ${RESTART_WINDOW_MS}ms β giving up. UI queries fall back to main-process DB.`,
+ `worker[${slotIdx}] crashed ${MAX_RESTARTS}Γ within ${RESTART_WINDOW_MS}ms β giving up on this slot.`,
);
return;
}
- this.restartTimes.push(now);
+ slot.restartTimes.push(now);
logger.log(
'warn',
'db-reader',
- `worker exited with code ${code}; respawning (${this.restartTimes.length}/${MAX_RESTARTS})`,
+ `worker[${slotIdx}] exited with code ${code}; respawning (${slot.restartTimes.length}/${MAX_RESTARTS})`,
);
- this.spawn();
+ this.spawn(slotIdx);
}
private failPendingWith(reason: string): void {
@@ -226,6 +300,7 @@ class DbReaderPool {
p.reject(new Error(reason));
}
this.pending.clear();
+ for (const slot of this.slots) slot.pending = 0;
}
}
@@ -233,10 +308,10 @@ class DbReaderPool {
export const dbReaderPool = new DbReaderPool();
/**
- * Thin helper for IPC handlers: try the worker first, fall back to the
- * synchronous main-process DB on any worker error. The fallback path
- * keeps the UI working even if the worker is crashed/restarting, at
- * the cost of running on the main thread for that one query.
+ * Thin helper for IPC handlers: try the worker pool first, fall back
+ * to the synchronous main-process DB on any worker error. The fallback
+ * path keeps the UI working even if every worker is crashed/restarting,
+ * at the cost of running on the main thread for that one query.
*/
export async function callReaderOrFallback(
method: string,
diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts
index f3da26b..7c8c77e 100644
--- a/apps/desktop/src/main/index.ts
+++ b/apps/desktop/src/main/index.ts
@@ -25,6 +25,8 @@ import {
type ExportJsonResult,
type ExportXmlInput,
type ExportXmlResult,
+ type ExportTabularInput,
+ type ExportTabularResult,
type DataDeleteByDomainInput,
type DataDeleteByDomainResult,
type CrashRecoveryStatus,
@@ -81,6 +83,7 @@ import {
exportUrlsToCsv,
exportUrlsToJson,
exportUrlsToXml,
+ exportTabular,
exportSitemap,
exportHtmlReport,
compareCrawls,
@@ -366,6 +369,64 @@ async function writeFetchedUrlViaPool(
}
}
+/**
+ * Run the post-crawl issue recompute on the writer worker. This pass
+ * has 70+ correlated SQL subqueries against the urls table; on a
+ * 5K-URL crawl it takes ~45 s, and running it on the main thread
+ * freezes the renderer for that entire window (the "Not Responding"
+ * dialog). Routing through the writer worker keeps main free.
+ */
+async function recomputeIssuesViaPool(
+ definitions: ReadonlyArray,
+): Promise {
+ if (!dbWriterPool.isReady()) {
+ return getDb().recomputeUrlsIssuesYielding(definitions);
+ }
+ try {
+ await dbWriterPool.call('recomputeUrlsIssuesYielding', [definitions]);
+ } catch (err) {
+ logger.log(
+ 'warn',
+ 'main',
+ `recomputeUrlsIssuesYielding dispatch failed (${
+ err instanceof Error ? err.message : String(err)
+ }) β falling back to main-thread recompute.`,
+ );
+ await getDb().recomputeUrlsIssuesYielding(definitions);
+ }
+}
+
+/**
+ * Generic post-crawl pass dispatcher. Routes recomputeInlinks,
+ * recomputeRedirectChains, recomputeHreflangAnalysis,
+ * recomputeHreflangInconsistent, and recomputePaginationSequence
+ * through the writer worker so the main thread stays responsive to
+ * Stop/Pause clicks and IPC traffic during the post-crawl phase.
+ * Without this, those five sync SQL passes combined freeze main for
+ * 5β15 s on a 5K-URL crawl β exactly the window where users hit
+ * Stop/Pause and "nothing happens".
+ */
+async function runDbPassViaPool(method: string): Promise {
+ if (!dbWriterPool.isReady()) {
+ const fn = (getDb() as unknown as Record)[method];
+ if (typeof fn === 'function') (fn as () => void).call(getDb());
+ return;
+ }
+ try {
+ await dbWriterPool.call(method, []);
+ } catch (err) {
+ logger.log(
+ 'warn',
+ 'main',
+ `${method} dispatch failed (${
+ err instanceof Error ? err.message : String(err)
+ }) β falling back to main-thread.`,
+ );
+ const fn = (getDb() as unknown as Record)[method];
+ if (typeof fn === 'function') (fn as () => void).call(getDb());
+ }
+}
+
function getDb(): ProjectDb {
if (!db) {
const dataDir = join(app.getPath('userData'), 'projects');
@@ -1160,6 +1221,8 @@ function registerIpc(): void {
setOp: (op: string) => freezeWatchdog.setMainOp(op),
parseHtml: parseHtmlViaPool,
writeFetchedUrl: writeFetchedUrlViaPool,
+ recomputeIssues: recomputeIssuesViaPool,
+ runDbPass: runDbPassViaPool,
});
activeCrawler = crawler;
attachCrawlerListeners(crawler);
@@ -1178,11 +1241,29 @@ function registerIpc(): void {
activeCrawler?.resume();
});
- ipcMain.handle(IPC.crawlClear, () => {
+ ipcMain.handle(IPC.crawlClear, async () => {
activeCrawler?.stop();
activeCrawler = null;
const database = getDb();
- database.reset();
+ // Route reset() through the writer pool so it sits behind any
+ // in-flight writeFetchedUrl RPCs in the worker's FIFO. Without
+ // this, the main process truncates the urls table while the
+ // writer worker is still draining the last batch of per-URL
+ // writes β those writes win the SQLite lock race and resurrect
+ // ~19 rows in the table after Clear. Going through the pool
+ // serialises naturally: every queued write completes, then reset
+ // runs against an already-stopped crawler.
+ if (dbWriterPool.isReady()) {
+ try {
+ await dbWriterPool.call('reset', []);
+ } catch {
+ // Worker terminated or RPC timed out β fall back to direct
+ // reset rather than leaving the table half-cleared.
+ database.reset();
+ }
+ } else {
+ database.reset();
+ }
// Belt-and-braces: reset() above already includes a DELETE FROM
// crawl_queue, but if anything (a late-firing checkpoint timer, a
// half-finished post-crawl pass) writes to that table after the
@@ -1661,6 +1742,47 @@ function registerIpc(): void {
},
);
+ ipcMain.handle(
+ IPC.exportTabular,
+ async (_e, input: ExportTabularInput): Promise => {
+ const { format, sections, columns, selectedIds } = input;
+ let outputPath = input.filePath ?? '';
+ const isSelection = (selectedIds?.length ?? 0) > 0;
+ const multiSection = sections.length > 1;
+ if (!outputPath) {
+ if (format === 'csv' && multiSection) {
+ const res = await dialog.showOpenDialog(mainWindow!, {
+ properties: ['openDirectory', 'createDirectory'],
+ title: 'Choose folder for CSV export',
+ });
+ if (res.canceled || res.filePaths.length === 0) {
+ return { filePath: '', files: [], rowsWritten: 0 };
+ }
+ outputPath = res.filePaths[0]!;
+ } else {
+ const ext = format === 'xlsx' ? 'xlsx' : 'csv';
+ const filterName = format === 'xlsx' ? 'Excel Workbook' : 'CSV';
+ const baseName = isSelection ? 'freecrawl-selected' : 'freecrawl-export';
+ const res = await dialog.showSaveDialog(mainWindow!, {
+ defaultPath: `${baseName}.${ext}`,
+ filters: [{ name: filterName, extensions: [ext] }],
+ });
+ if (res.canceled || !res.filePath) {
+ return { filePath: '', files: [], rowsWritten: 0 };
+ }
+ outputPath = res.filePath;
+ }
+ }
+ const result = await exportTabular(getDb(), outputPath, {
+ format,
+ sections,
+ columns,
+ selectedIds,
+ });
+ return result;
+ },
+ );
+
ipcMain.handle(
IPC.dataDeleteByDomain,
async (
@@ -1707,6 +1829,8 @@ function registerIpc(): void {
setOp: (op: string) => freezeWatchdog.setMainOp(op),
parseHtml: parseHtmlViaPool,
writeFetchedUrl: writeFetchedUrlViaPool,
+ recomputeIssues: recomputeIssuesViaPool,
+ runDbPass: runDbPassViaPool,
});
activeCrawler = crawler;
attachCrawlerListeners(crawler);
diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts
index 564cb66..f0df037 100644
--- a/apps/desktop/src/preload/index.ts
+++ b/apps/desktop/src/preload/index.ts
@@ -11,6 +11,8 @@ import {
type ExportJsonResult,
type ExportXmlInput,
type ExportXmlResult,
+ type ExportTabularInput,
+ type ExportTabularResult,
type DataDeleteByDomainInput,
type DataDeleteByDomainResult,
type CrashRecoveryStatus,
@@ -123,6 +125,8 @@ const api: FreeCrawlApi = {
ipcRenderer.invoke(IPC.exportJson, input),
exportXml: (input: ExportXmlInput): Promise =>
ipcRenderer.invoke(IPC.exportXml, input),
+ exportTabular: (input: ExportTabularInput): Promise =>
+ ipcRenderer.invoke(IPC.exportTabular, input),
dataDeleteByDomain: (
input: DataDeleteByDomainInput,
): Promise =>
diff --git a/apps/desktop/src/renderer/src/App.tsx b/apps/desktop/src/renderer/src/App.tsx
index 0e2573a..c6429a5 100644
--- a/apps/desktop/src/renderer/src/App.tsx
+++ b/apps/desktop/src/renderer/src/App.tsx
@@ -137,20 +137,30 @@ export function App() {
// previous session left a non-empty checkpoint we ask whether to
// resume. Discarding wipes the snapshot so the prompt doesn't
// recur on next launch.
+ //
+ // The confirm() is wrapped in a 250 ms setTimeout so the initial
+ // render finishes (URL input mounts, status bar paints) before
+ // the modal opens. Without the defer, the synchronous `confirm`
+ // can fire on first paint and β when Electron renders the modal
+ // behind the main window on some Windows configurations β leaves
+ // the user staring at an "unresponsive" UI: the URL input can't
+ // be clicked because a hidden modal is blocking the renderer.
void window.freecrawl.crashRecoveryStatus().then((status) => {
if (status.pendingCount === 0) return;
- const proceed = window.confirm(
- `FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` +
- ` Start URL: ${status.seedUrl}\n` +
- ` Pending URLs: ${status.pendingCount.toLocaleString()}\n\n` +
- `Resume the crawl from where it stopped?\n\n` +
- `Click OK to resume, Cancel to discard the recovery state and start fresh.`,
- );
- if (proceed) {
- void window.freecrawl.crashRecoveryResume();
- } else {
- void window.freecrawl.crashRecoveryDiscard();
- }
+ window.setTimeout(() => {
+ const proceed = window.confirm(
+ `FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` +
+ ` Start URL: ${status.seedUrl}\n` +
+ ` Pending URLs: ${status.pendingCount.toLocaleString()}\n\n` +
+ `Resume the crawl from where it stopped?\n\n` +
+ `Click OK to resume, Cancel to discard the recovery state and start fresh.`,
+ );
+ if (proceed) {
+ void window.freecrawl.crashRecoveryResume();
+ } else {
+ void window.freecrawl.crashRecoveryDiscard();
+ }
+ }, 250);
});
// rAF-throttled progress dispatch. The crawler emits up to 5
diff --git a/apps/desktop/src/renderer/src/components/ErrorBoundary.tsx b/apps/desktop/src/renderer/src/components/ErrorBoundary.tsx
new file mode 100644
index 0000000..931973d
--- /dev/null
+++ b/apps/desktop/src/renderer/src/components/ErrorBoundary.tsx
@@ -0,0 +1,60 @@
+import { Component, type ReactNode } from 'react';
+
+interface Props {
+ /** Rendered when the boundary's subtree throws. Defaults to a tiny notice. */
+ fallback?: ReactNode;
+ /** Surfaced to console + main-process log so the user can copy/paste. */
+ context?: string;
+ children: ReactNode;
+}
+
+interface State {
+ err: Error | null;
+}
+
+/**
+ * Catches render-time errors in a subtree so a single misbehaving dialog
+ * (e.g. ExportDialog) can't unmount the whole renderer and leave the user
+ * staring at the Electron window's bare background colour. The boundary
+ * logs the error and shows a small inline notice; the rest of the app
+ * keeps working.
+ */
+export class ErrorBoundary extends Component {
+ override state: State = { err: null };
+
+ static getDerivedStateFromError(err: Error): State {
+ return { err };
+ }
+
+ override componentDidCatch(err: Error, info: { componentStack?: string }): void {
+ // eslint-disable-next-line no-console
+ console.error(
+ `[ErrorBoundary${this.props.context ? `:${this.props.context}` : ''}]`,
+ err,
+ info.componentStack,
+ );
+ }
+
+ reset = (): void => {
+ this.setState({ err: null });
+ };
+
+ override render(): ReactNode {
+ const { err } = this.state;
+ if (!err) return this.props.children;
+ if (this.props.fallback !== undefined) return this.props.fallback;
+ return (
+
+
Something broke in this panel.
+
{err.message}
+
+
+ );
+ }
+}
diff --git a/apps/desktop/src/renderer/src/components/ExportDialog.tsx b/apps/desktop/src/renderer/src/components/ExportDialog.tsx
new file mode 100644
index 0000000..03c3961
--- /dev/null
+++ b/apps/desktop/src/renderer/src/components/ExportDialog.tsx
@@ -0,0 +1,403 @@
+import { useEffect, useMemo, useState } from 'react';
+import { X, Loader2 } from 'lucide-react';
+import clsx from 'clsx';
+import type { CrawlUrlRow, ExportTabularSection, UrlCategory } from '@freecrawl/shared-types';
+import { TAB_ORDER, type TabKey } from '../store.js';
+import { COLUMN_SPECS, columnId } from '../tabs/columns.js';
+
+type Format = 'csv' | 'xlsx';
+
+interface ExportTab {
+ key: TabKey;
+ label: string;
+ category: UrlCategory;
+ columnKeys: (keyof CrawlUrlRow)[];
+}
+
+const TAB_CATEGORY: Record = {
+ internal: 'internal:html',
+ external: 'external:all',
+ 'response-codes': 'all',
+ url: 'internal:html',
+ 'page-titles': 'internal:html',
+ 'meta-description': 'internal:html',
+ h1: 'internal:html',
+ h2: 'internal:html',
+ content: 'internal:html',
+ images: 'internal:html',
+ canonicals: 'internal:html',
+ directives: 'internal:html',
+ redirects: 'all',
+ links: 'all',
+ 'broken-links': 'all',
+};
+
+function buildExportableTabs(): ExportTab[] {
+ const out: ExportTab[] = [];
+ for (const t of TAB_ORDER) {
+ const cols = COLUMN_SPECS[t.key];
+ // Tabs with their own non-URL-row layouts (Images / Broken Links)
+ // have empty COLUMN_SPECS β skip them so the export grid only shows
+ // categories that share the URL-row schema.
+ if (!cols || cols.length === 0) continue;
+ out.push({
+ key: t.key,
+ label: t.label,
+ category: TAB_CATEGORY[t.key],
+ columnKeys: cols.map((c) => c.key),
+ });
+ }
+ return out;
+}
+
+interface ExportDialogProps {
+ open: boolean;
+ onClose: () => void;
+ /** The tab the user clicked Export from β drives the default selection. */
+ defaultTab: TabKey;
+ /** Row ids selected in the current table; falsy β export the whole filter set. */
+ selectedIds?: number[];
+}
+
+export function ExportDialog({
+ open,
+ onClose,
+ defaultTab,
+ selectedIds,
+}: ExportDialogProps) {
+ const tabs = useMemo(() => buildExportableTabs(), []);
+ const initialTabKey: TabKey = useMemo(() => {
+ return tabs.find((t) => t.key === defaultTab)?.key ?? tabs[0]?.key ?? 'internal';
+ }, [defaultTab, tabs]);
+
+ const selectionCount = selectedIds?.length ?? 0;
+
+ const [format, setFormat] = useState('csv');
+ const [pickedTabs, setPickedTabs] = useState>(() => new Set([initialTabKey]));
+ const [pickedColumns, setPickedColumns] = useState>(() => {
+ const t = tabs.find((x) => x.key === initialTabKey);
+ return new Set((t?.columnKeys ?? []) as string[]);
+ });
+ const [useSelection, setUseSelection] = useState(selectionCount > 0);
+ const [busy, setBusy] = useState(false);
+ const [status, setStatus] = useState('');
+
+ // The dialog is conditionally mounted by the parent (only when open=true),
+ // so useState initializers above already populate the right defaults on
+ // each open. Escape key dismissal is the only ambient behaviour we need.
+ useEffect(() => {
+ if (!open) return;
+ const onKey = (e: KeyboardEvent): void => {
+ if (e.key === 'Escape') onClose();
+ };
+ document.addEventListener('keydown', onKey);
+ return () => document.removeEventListener('keydown', onKey);
+ }, [open, onClose]);
+
+ // Union of column keys exposed by every picked tab β these are the
+ // columns the dialog offers; checked state lives in `pickedColumns`.
+ const offeredColumns = useMemo(() => {
+ const seen = new Set();
+ const out: { key: string; header: string; tab: TabKey }[] = [];
+ for (const t of tabs) {
+ if (!pickedTabs.has(t.key)) continue;
+ const specs = COLUMN_SPECS[t.key];
+ for (const spec of specs) {
+ const id = columnId(spec);
+ if (seen.has(id)) continue;
+ seen.add(id);
+ out.push({ key: spec.key as string, header: spec.header, tab: t.key });
+ }
+ }
+ return out;
+ }, [pickedTabs, tabs]);
+
+ const orderedColumnList = useMemo(
+ () => offeredColumns.filter((c) => pickedColumns.has(c.key)).map((c) => c.key),
+ [offeredColumns, pickedColumns],
+ );
+
+ if (!open) return null;
+
+ const toggleTab = (key: TabKey) => {
+ setPickedTabs((prev) => {
+ const next = new Set(prev);
+ let willBeRemoved = false;
+ if (next.has(key)) {
+ if (next.size === 1) return prev; // keep at least one
+ next.delete(key);
+ willBeRemoved = true;
+ } else {
+ next.add(key);
+ }
+ // Auto-tick the new tab's columns when it joins the picked set, so
+ // the user doesn't have to re-check them. When a tab leaves we keep
+ // the user's column picks alone β orderedColumnList already drops
+ // any column not offered by remaining tabs.
+ if (!willBeRemoved) {
+ const t = tabs.find((x) => x.key === key);
+ if (t) {
+ setPickedColumns((prevCols) => {
+ const nextCols = new Set(prevCols);
+ for (const k of t.columnKeys) nextCols.add(k as string);
+ return nextCols;
+ });
+ }
+ }
+ return next;
+ });
+ };
+
+ const toggleColumn = (key: string) => {
+ setPickedColumns((prev) => {
+ const next = new Set(prev);
+ if (next.has(key)) next.delete(key);
+ else next.add(key);
+ return next;
+ });
+ };
+
+ const checkAllColumns = () => {
+ setPickedColumns(new Set(offeredColumns.map((c) => c.key)));
+ };
+
+ const uncheckAllColumns = () => {
+ setPickedColumns(new Set());
+ };
+
+ const submit = async () => {
+ if (orderedColumnList.length === 0) {
+ setStatus('Pick at least one column.');
+ return;
+ }
+ if (pickedTabs.size === 0) {
+ setStatus('Pick at least one table.');
+ return;
+ }
+ setBusy(true);
+ setStatus('');
+ try {
+ const sections: ExportTabularSection[] = tabs
+ .filter((t) => pickedTabs.has(t.key))
+ .map((t) => ({ label: t.label, category: t.category }));
+ const result = await window.freecrawl.exportTabular({
+ format,
+ sections,
+ columns: orderedColumnList,
+ selectedIds:
+ useSelection && selectedIds && selectedIds.length > 0
+ ? selectedIds
+ : undefined,
+ });
+ if (!result.filePath) {
+ // User cancelled save / folder dialog.
+ setBusy(false);
+ return;
+ }
+ setStatus(
+ `Exported ${result.rowsWritten.toLocaleString()} row${result.rowsWritten === 1 ? '' : 's'} to ${result.filePath}`,
+ );
+ setBusy(false);
+ setTimeout(() => onClose(), 700);
+ } catch (err) {
+ setBusy(false);
+ setStatus(`Export failed: ${err instanceof Error ? err.message : String(err)}`);
+ }
+ };
+
+ return (
+
+ Multiple tables + CSV β you'll be asked to pick a folder; one file is written per table.
+
+ )}
+
+ {status && (
+
+ {status}
+
+ )}
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts b/apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts
index db620b4..a01e2ec 100644
--- a/apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts
+++ b/apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts
@@ -2,18 +2,27 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import type { AdvancedFilter, CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
const CHUNK_SIZE = 500;
-const MAX_CACHED_CHUNKS = 8;
-// Live-refresh cadence during an active crawl. 750 ms strikes a balance
-// between perceived "row streaming" smoothness and the cost of a COUNT
-// query + per-chunk fetch every tick. At 100 URL/s the user still sees
-// ~75 new rows per tick (continuous flow), while leaving 3Γ more main-
-// thread headroom than the previous 250 ms cadence.
-// Cadence for the live-tail tick. 1500 ms strikes the balance between
-// "feels fresh" and "doesn't pin SQLite". At 750 ms a 2k+ row crawl was
-// burning ~30% of one core in COUNT + chunk fetches, blocking input IPC
-// for the duration. The user sees the same effect: numbers tick at half
-// speed but the window stops kasma'ing.
-const LIVE_REFRESH_MS = 1500;
+// Doubled the cache (was 8) so a fast scroll across 4-5 chunks doesn't
+// thrash the reader pool: each chunk evicted = a re-fetch the next time
+// the user scrolls back to it. 16 Γ 500 = 8000 rows kept warm; at
+// ~2 KB/row that's ~16 MB heap, well within budget.
+const MAX_CACHED_CHUNKS = 16;
+// Number of chunks to pre-fetch ahead/behind the visible range. The
+// visible range is what the virtualizer asks for; pre-fetching the
+// neighbours hides "..." placeholders during fast scrolls because the
+// next chunk has usually already landed by the time the user reaches
+// it. 1 chunk = 500 rows of look-ahead each direction, cheap on the
+// reader pool.
+const PREFETCH_CHUNKS = 1;
+// Min gap between progress-driven ticks. Progress events arrive at
+// ~5 Hz from the crawler; without throttle the visible-chunk re-query
+// would fire that often too. 100 ms keeps the table feeling live
+// (10 refreshes/sec ceiling) without pinning the reader-pool.
+const TICK_THROTTLE_MS = 100;
+// Safety-net periodic tick for cases where progress events stop
+// arriving (crawl finished, manual DB edits, etc.). Long interval β
+// the progress-event path drives the in-crawl experience.
+const LIVE_REFRESH_MS = 5000;
export interface LazyRowsOpts {
category: UrlCategory;
@@ -31,6 +40,12 @@ export interface LazyRowsState {
loadedRows: number;
rowAt: (index: number) => CrawlUrlRow | null;
ensureRange: (start: number, end: number) => void;
+ /** True between the moment a shape change is requested (tab/sort/filter
+ * flip) and the moment the new data has fully landed. Lets the
+ * consumer distinguish "0 URLs because the category genuinely has
+ * none" from "0 URLs because we haven't fetched yet" β which the
+ * empty-state UI cares about a lot. */
+ isLoading: boolean;
}
/**
@@ -49,6 +64,7 @@ export interface LazyRowsState {
export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
const [total, setTotal] = useState(0);
const [version, setVersion] = useState(0);
+ const [isLoading, setIsLoading] = useState(true);
const chunks = useRef(new Map());
const chunkOrder = useRef([]);
const fetching = useRef(new Set());
@@ -88,20 +104,52 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
[opts.category, opts.search, opts.sortBy, opts.sortDir, opts.filter],
);
- // Shape change: rebuild from scratch. The virtualizer will call
- // ensureRange again on its next measurement pass.
+ // Shape change (tab/sort/filter switch): clear immediately and load
+ // fresh data in parallel. Showing the previous tab's rows under the
+ // new tab's header is the wrong UX β "Internal" rows visible while
+ // "External" is selected makes the user mistrust what they see.
+ // Instead we set `isLoading=true` and let the consumer render a
+ // proper loading state until the new total + visible chunks land.
+ // The reader-pool has multiple workers now, so meta + visible-chunk
+ // queries run in parallel and typically resolve in ~50-150 ms.
useEffect(() => {
resetToken.current++;
chunks.current.clear();
chunkOrder.current = [];
fetching.current.clear();
+ setIsLoading(true);
setVersion((v) => v + 1);
const token = resetToken.current;
let cancelled = false;
- void queryMeta().then(({ total: t }) => {
- if (cancelled || token !== resetToken.current) return;
- setTotal(t);
- });
+
+ const load = async (): Promise => {
+ const metaPromise = queryMeta();
+ const { first, last } = activeRange.current;
+ const chunkPromises: Promise<{ rows: CrawlUrlRow[]; total: number; idx: number }>[] = [];
+ for (let i = first; i <= last; i++) {
+ chunkPromises.push(queryChunk(i).then((r) => ({ ...r, idx: i })));
+ }
+ try {
+ const [{ total: t }, ...newChunks] = await Promise.all([metaPromise, ...chunkPromises]);
+ if (cancelled || token !== resetToken.current) return;
+ for (const ch of newChunks) {
+ chunks.current.set(ch.idx, ch.rows);
+ chunkOrder.current.push(ch.idx);
+ }
+ setTotal(t);
+ setVersion((v) => v + 1);
+ } catch {
+ if (cancelled || token !== resetToken.current) return;
+ // Leave chunks empty on error; the live tick / ensureRange
+ // will retry. Still need to flip loading off so the empty
+ // state can render an actual error/empty UI.
+ } finally {
+ if (!cancelled && token === resetToken.current) {
+ setIsLoading(false);
+ }
+ }
+ };
+ void load();
return () => {
cancelled = true;
};
@@ -111,16 +159,26 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
// Live tick: re-query only the visible chunks and patch them in place.
// Never calls .clear(), so rowAt never returns null for a
// previously-loaded index. The table streams rather than flashes.
- // A leading tick fires immediately so the first rows surface without
- // the LIVE_REFRESH_MS dead window after Start.
+ //
+ // Drive model: push, not poll. Each crawler progress event (β every
+ // time `crawled` advances, which lands at ~5 Hz on a healthy crawl)
+ // triggers an immediate visible-chunk re-query. The previous design
+ // polled every 1500 ms regardless of activity β newly-written rows
+ // could sit in the DB for up to that long before surfacing in the
+ // table. The push path closes that window to ~100 ms (the throttle
+ // floor that keeps the reader-pool from being hammered).
+ //
+ // A 5 s safety-net `setInterval` still fires for code paths that
+ // mutate the DB without going through a progress event (manual row
+ // delete, project re-open, etc.).
useEffect(() => {
let cancelled = false;
let inFlight = false;
+ let lastTickTs = 0;
const tick = async () => {
- // Coalesce overlapping ticks β at 250 ms cadence a slow COUNT or
- // chunk fetch can otherwise queue up multiple in-flight ticks.
if (inFlight) return;
inFlight = true;
+ lastTickTs = Date.now();
const token = resetToken.current;
try {
try {
@@ -146,33 +204,47 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
inFlight = false;
}
};
- // Wrap the tick in requestIdleCallback so the live-tail SQL only
- // fires when the renderer's event loop has idle slack. If the user
- // is dragging / clicking / typing, the tick is deferred until the
- // input handler frame finishes β eliminates the "click β kasma"
- // pattern even when the underlying SQL is fast. Falls back to a
- // direct call when the browser doesn't expose `requestIdleCallback`
- // (older Chromium builds; Electron 41 ships modern Chromium so this
- // branch is mostly defensive).
interface RequestIdleCallback {
(cb: () => void, opts?: { timeout: number }): number;
}
const w = window as Window & { requestIdleCallback?: RequestIdleCallback };
const scheduleTick = (): void => {
if (typeof w.requestIdleCallback === 'function') {
- // 2 s timeout guarantees the tick eventually runs even if the
- // main thread is permanently busy β better stale data than no
- // data at all.
- w.requestIdleCallback(() => void tick(), { timeout: 2000 });
+ // 300 ms timeout β short enough that the user perceives an
+ // event-driven refresh as instant; long enough that an
+ // in-progress click handler isn't preempted.
+ w.requestIdleCallback(() => void tick(), { timeout: 300 });
} else {
void tick();
}
};
- scheduleTick();
+ // Leading tick fires directly (no idle wrap) so the first row
+ // surfaces as soon as the crawler has written one β without this,
+ // requestIdleCallback could defer the initial fetch up to 300 ms
+ // on a busy main thread right after Start.
+ void tick();
+
+ // Push: progress event drives the live-tail tick. We tick on every
+ // event where `crawled` changed, throttled to TICK_THROTTLE_MS so
+ // a 100 URL/s crawl doesn't force 100 SQL roundtrips per second.
+ // Direct call (no idle wrap) β progress events are sparse enough
+ // that input-handler interference is a non-concern, and pulling
+ // through requestIdleCallback was adding 50-300 ms before each
+ // newly-written URL surfaced in the table.
+ let lastCrawled = -1;
+ const offProgress = window.freecrawl?.onProgress?.((p) => {
+ if (p.crawled === lastCrawled) return;
+ lastCrawled = p.crawled;
+ if (Date.now() - lastTickTs < TICK_THROTTLE_MS) return;
+ void tick();
+ });
+
+ // Safety-net poll for crawl-idle DB mutations.
const id = setInterval(scheduleTick, LIVE_REFRESH_MS);
return () => {
cancelled = true;
clearInterval(id);
+ offProgress?.();
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [keyStr]);
@@ -213,7 +285,13 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
const first = Math.max(0, Math.floor(start / CHUNK_SIZE));
const last = Math.max(0, Math.floor(end / CHUNK_SIZE));
activeRange.current = { first, last };
- for (let i = first; i <= last; i++) {
+ // Pre-fetch a window of neighbouring chunks so a fast scroll
+ // doesn't show "..." placeholders while the next chunk loads β
+ // by the time the virtualizer scrolls into the next chunk's
+ // range, its rows are usually already in the cache.
+ const fetchFrom = Math.max(0, first - PREFETCH_CHUNKS);
+ const fetchTo = last + PREFETCH_CHUNKS;
+ for (let i = fetchFrom; i <= fetchTo; i++) {
if (!chunks.current.has(i)) void fetchChunk(i);
}
},
@@ -237,5 +315,5 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
[version],
);
- return { total, loadedRows, rowAt, ensureRange };
+ return { total, loadedRows, rowAt, ensureRange, isLoading };
}
diff --git a/apps/desktop/src/renderer/src/tabs/UrlsTab.tsx b/apps/desktop/src/renderer/src/tabs/UrlsTab.tsx
index 6bd8c5f..ad8bd5c 100644
--- a/apps/desktop/src/renderer/src/tabs/UrlsTab.tsx
+++ b/apps/desktop/src/renderer/src/tabs/UrlsTab.tsx
@@ -1,12 +1,14 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useVirtualizer } from '@tanstack/react-virtual';
-import { ChevronsUpDown, Filter, X } from 'lucide-react';
+import { ChevronsUpDown, Download, Filter, X } from 'lucide-react';
import clsx from 'clsx';
import type { AdvancedFilter, CrawlUrlRow } from '@freecrawl/shared-types';
import { useAppStore, type TabKey } from '../store.js';
import { COLUMN_SPECS, columnId, type ColumnSpec } from './columns.js';
import { useLazyUrlRows } from '../hooks/useLazyUrlRows.js';
import { AdvancedFilterDialog } from '../components/AdvancedFilterDialog.js';
+import { ErrorBoundary } from '../components/ErrorBoundary.js';
+import { ExportDialog } from '../components/ExportDialog.js';
import { InfoTip } from '../components/InfoTip.js';
const ROW_HEIGHT = 24;
@@ -36,6 +38,7 @@ export function UrlsTab() {
const [sortDir, setSortDir] = useState<'asc' | 'desc'>('asc');
const [filter, setFilter] = useState(null);
const [filterDialogOpen, setFilterDialogOpen] = useState(false);
+ const [exportDialogOpen, setExportDialogOpen] = useState(false);
// Three orthogonal selection layers:
// - selectedIds: row-level selection driven by the Row-number column.
// Feeds the bulk context menu (Copy/Open/Re-Spider/Remove/Export).
@@ -515,12 +518,21 @@ export function UrlsTab() {
)}
-