From cec358be9ff67b23ee38a6d2ad3e5efe892858e7 Mon Sep 17 00:00:00 2001 From: kemalai Date: Sun, 3 May 2026 02:43:24 +0300 Subject: [PATCH] feat: export dialog + MCP server for AI agents --- README.md | 91 ++++ apps/cli/package.json | 2 +- apps/desktop/package.json | 2 +- apps/desktop/src/main/db-reader-pool.ts | 235 ++++++---- apps/desktop/src/main/index.ts | 128 +++++- apps/desktop/src/preload/index.ts | 4 + apps/desktop/src/renderer/src/App.tsx | 34 +- .../renderer/src/components/ErrorBoundary.tsx | 60 +++ .../renderer/src/components/ExportDialog.tsx | 403 ++++++++++++++++++ .../src/renderer/src/hooks/useLazyUrlRows.ts | 152 +++++-- .../desktop/src/renderer/src/tabs/UrlsTab.tsx | 62 ++- apps/mcp-server/package.json | 25 ++ apps/mcp-server/src/index.ts | 192 +++++++++ apps/mcp-server/src/project-resolver.ts | 57 +++ apps/mcp-server/src/rpc.ts | 109 +++++ apps/mcp-server/src/tools.ts | 265 ++++++++++++ apps/mcp-server/tsconfig.json | 14 + package-lock.json | 34 +- package.json | 3 +- packages/core/package.json | 2 +- packages/core/src/crawler.ts | 207 +++++++-- packages/core/src/index.ts | 6 + packages/core/src/tabular-export.ts | 398 +++++++++++++++++ packages/db/package.json | 2 +- packages/db/src/project-db.ts | 7 + packages/shared-types/package.json | 2 +- packages/shared-types/src/ipc.ts | 36 ++ tsconfig.json | 3 +- 28 files changed, 2344 insertions(+), 191 deletions(-) create mode 100644 apps/desktop/src/renderer/src/components/ErrorBoundary.tsx create mode 100644 apps/desktop/src/renderer/src/components/ExportDialog.tsx create mode 100644 apps/mcp-server/package.json create mode 100644 apps/mcp-server/src/index.ts create mode 100644 apps/mcp-server/src/project-resolver.ts create mode 100644 apps/mcp-server/src/rpc.ts create mode 100644 apps/mcp-server/src/tools.ts create mode 100644 apps/mcp-server/tsconfig.json create mode 100644 packages/core/src/tabular-export.ts diff --git a/README.md b/README.md index f9df488..490000b 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,97 @@ npm --workspace apps/desktop run build:linux # AppImage / .deb +
+πŸ€– MCP server β€” query your crawl from Claude / any MCP client + +
+ +FreeCrawl ships an **MCP (Model Context Protocol) server** that exposes the active `.seoproject` to AI agents over stdio. Read-only β€” runs alongside the desktop app without contention (SQLite WAL). + +**Tools**: `get_summary`, `get_overview_counts`, `top_issues`, `query_urls`, `get_url_detail`, `list_projects`, `set_project`, `current_project`. + +**1. Build it once:** + +```bash +npm run build:mcp +``` + +This produces `apps/mcp-server/dist/index.js`. + +**2. Register it with your MCP client.** + +
+Claude Desktop + +Edit your Claude Desktop config: + +| Platform | Path | +| :--- | :--- | +| Windows | `%APPDATA%\Claude\claude_desktop_config.json` | +| macOS | `~/Library/Application Support/Claude/claude_desktop_config.json` | +| Linux | `~/.config/Claude/claude_desktop_config.json` | + +```json +{ + "mcpServers": { + "freecrawl": { + "command": "node", + "args": ["/absolute/path/to/FreeCrawl-SEO-Tool/apps/mcp-server/dist/index.js"] + } + } +} +``` + +Restart Claude Desktop. The `freecrawl` server appears under the tool πŸ”Œ icon. + +
+ +
+Claude Code (CLI) + +```bash +claude mcp add freecrawl -- node /absolute/path/to/FreeCrawl-SEO-Tool/apps/mcp-server/dist/index.js +``` + +
+ +
+Other MCP clients + +Run the binary directly with stdio transport: + +```bash +node apps/mcp-server/dist/index.js +``` + +The server speaks newline-delimited JSON-RPC 2.0 β€” point any MCP-compatible client at it. + +
+ +**3. Try it.** Ask your agent things like: + +> *"Show the 10 URLs with the longest response time in my last crawl."* +> *"What are the top 5 issue categories with the most affected pages?"* +> *"List every URL with a missing meta description."* + +**Pointing at a non-default project:** + +By default the server reads `/projects/default.seoproject` (the same file the desktop app uses). Override with the `FREECRAWL_PROJECT` env var, or call the `set_project` tool mid-session: + +```json +{ + "mcpServers": { + "freecrawl": { + "command": "node", + "args": ["/path/to/apps/mcp-server/dist/index.js"], + "env": { "FREECRAWL_PROJECT": "/path/to/audit.seoproject" } + } + } +} +``` + +
+
--- diff --git a/apps/cli/package.json b/apps/cli/package.json index 0e19cdf..c366cd3 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/cli", - "version": "0.2.7", + "version": "0.2.8", "private": true, "type": "module", "bin": { diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 8da8ed7..5d67858 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/desktop", - "version": "0.2.7", + "version": "0.2.8", "private": true, "description": "FreeCrawl SEO Ò€” desktop application", "main": "./out/main/index.js", diff --git a/apps/desktop/src/main/db-reader-pool.ts b/apps/desktop/src/main/db-reader-pool.ts index fd3cfb9..ebda2db 100644 --- a/apps/desktop/src/main/db-reader-pool.ts +++ b/apps/desktop/src/main/db-reader-pool.ts @@ -4,43 +4,48 @@ import { fileURLToPath } from 'node:url'; import * as logger from './logger.js'; /** - * Pool of one worker_thread holding a read-only SQLite connection. - * "Pool" is forward-compatible naming β€” for now we run a single worker - * because SQLite's WAL allows arbitrary concurrent readers but adding a - * second worker would just double our cache footprint without gaining - * parallelism on a single physical disk. + * Pool of N worker_threads, each holding its own read-only SQLite + * connection. SQLite WAL mode allows arbitrary concurrent readers, so + * spreading queries across multiple workers gives genuine parallelism + * for UI reads β€” critical because: + * + * - The OverviewSidebar refresh issues a 130-clause `getOverviewCounts` + * every few seconds. On a single-worker pool this serialises every + * other read behind it, freezing the URL table refresh and the + * bottom panel's per-URL detail / inlinks queries until the heavy + * pass completes. + * - Pool now lets the heavy aggregate run on one worker while the + * short, latency-sensitive queries (urlsQuery for the visible + * chunks, urlDetail for the bottom panel) run on the others. + * + * Routing: round-robin with a "least busy" tie-break β€” for each call + * we pick the worker with the fewest pending requests, and on a tie + * fall back to a rotating cursor so two same-loaded workers don't + * always get the next call dropped on the same one. * * Lifecycle: - * - `init(dbPath)` spawns the worker, points it at the file. - * - `swap(newPath)` is called on Open Project β€” terminates the old - * worker and spawns a fresh one; in-flight requests are rejected + * - `init(dbPath)` spawns all workers, points them at the file. + * - `swap(newPath)` is called on Open Project β€” terminates every old + * worker and spawns a fresh set; in-flight requests are rejected * with `Error('reader-swapped')` so callers can retry safely. * - `terminate()` on app quit. After terminate, all `call()` resolve * with `Error('reader-terminated')`. * * Crash recovery: - * - If the worker exits unexpectedly (`exit` event with non-zero code - * while we still have a path) we auto-respawn up to MAX_RESTARTS - * within RESTART_WINDOW_MS. After the cap we surface the failure - * and stop trying β€” the user gets a clear log entry instead of a - * restart loop. - * - In-flight requests during a crash are rejected with + * - If a worker exits unexpectedly, that one slot auto-respawns up to + * MAX_RESTARTS within RESTART_WINDOW_MS. Other workers in the pool + * keep serving traffic. + * - In-flight requests on the crashed worker are rejected with * `Error('reader-crashed')`. Callers fall back to the main-process - * ProjectDb (see `dbReaderCallOrFallback`) so the UI stays alive. + * ProjectDb (see `callReaderOrFallback`) so the UI stays alive. */ const __dirname = path.dirname(fileURLToPath(import.meta.url)); -// The worker is built alongside the main bundle as `db-reader-worker.js` -// (electron-vite produces flat output in `out/main/`). In dev mode -// electron-vite uses the same convention. const WORKER_PATH = path.join(__dirname, 'db-reader-worker.js'); // 30 s default leaves headroom for SQLite read contention during the -// post-crawl recompute phase (recomputeUrlsIssues holds a writer -// transaction that can keep reader queries blocked behind earlier -// queued requests for several seconds at a time on large crawls). -// Genuine worker hangs are still surfaced β€” just with a slightly -// longer detection window than before. +// post-crawl recompute phase. Genuine worker hangs are still surfaced +// β€” just with a slightly longer detection window than before. const REQUEST_TIMEOUT_MS = 30_000; // Heavy aggregate queries (the overview sidebar's 130-counter pass and // the post-crawl materialiser's counter fan-out) can run past the @@ -54,7 +59,17 @@ const HEAVY_METHODS = new Set([ const MAX_RESTARTS = 3; const RESTART_WINDOW_MS = 60_000; +// Reader pool size. Four workers gives the typical UI workload (one +// long-running aggregate from the sidebar + visible-table chunk fetch +// + bottom-panel detail + crawl-engine background reads) a free slot +// for every concurrent request; bigger doesn't help because most UI +// reads are short and a dedicated worker per query becomes pure +// memory cost. Each worker keeps its own SQLite handle (~5 MB heap) +// and prepared-statement cache (~10 MB on a populated DB). +const POOL_SIZE = 4; + interface PendingRequest { + workerIdx: number; resolve: (v: unknown) => void; reject: (e: Error) => void; timer: ReturnType; @@ -67,94 +82,143 @@ interface ResponseMessage { error?: string; } +interface WorkerSlot { + worker: Worker | null; + pending: number; + restartTimes: number[]; +} + class DbReaderPool { - private worker: Worker | null = null; + private slots: WorkerSlot[] = []; private dbPath: string | null = null; private freezeWatchdogSab: SharedArrayBuffer | null = null; private nextRequestId = 1; private pending = new Map(); - private restartTimes: number[] = []; + private rrCursor = 0; private terminated = false; - /** Spawn (or respawn) the worker pointed at `dbPath`. Idempotent. - * - * `freezeWatchdogSab` β€” if provided, the worker will tick its - * own heartbeat + publish its current method into this shared - * buffer so the freeze-watchdog thread can detect a stuck - * reader. Pass `null` to disable diagnostics. */ + /** Spawn (or respawn) the pool pointed at `dbPath`. Idempotent. */ init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void { if (this.terminated) { throw new Error('DbReaderPool: cannot init after terminate()'); } this.freezeWatchdogSab = freezeWatchdogSab; - if (this.dbPath === dbPath && this.worker !== null) return; + if (this.dbPath === dbPath && this.slots.some((s) => s.worker !== null)) return; this.dbPath = dbPath; - this.spawn(); + this.slots = Array.from({ length: POOL_SIZE }, () => ({ + worker: null, + pending: 0, + restartTimes: [], + })); + for (let i = 0; i < POOL_SIZE; i++) this.spawn(i); } - /** Switch the worker to a different .seoproject file. */ + /** Switch the pool to a different .seoproject file. */ swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void { if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab; - if (this.dbPath === newPath && this.worker !== null) return; + if (this.dbPath === newPath && this.slots.some((s) => s.worker !== null)) return; this.failPendingWith('reader-swapped'); this.dbPath = newPath; - this.spawn(); + if (this.slots.length === 0) { + this.slots = Array.from({ length: POOL_SIZE }, () => ({ + worker: null, + pending: 0, + restartTimes: [], + })); + } + for (let i = 0; i < POOL_SIZE; i++) this.spawn(i); } /** Permanent shutdown β€” used on app quit. */ async terminate(): Promise { this.terminated = true; this.failPendingWith('reader-terminated'); - if (this.worker) { - const w = this.worker; - this.worker = null; - try { - await w.terminate(); - } catch { - /* already gone */ - } - } + const workers = this.slots.map((s) => s.worker).filter((w): w is Worker => w !== null); + for (const slot of this.slots) slot.worker = null; + await Promise.all( + workers.map((w) => + w.terminate().catch(() => undefined), + ), + ); } - /** True while a worker is up and ready to receive requests. */ + /** True while at least one worker is up and ready. */ isReady(): boolean { - return this.worker !== null && !this.terminated; + return !this.terminated && this.slots.some((s) => s.worker !== null); } /** - * Dispatch a method call to the worker. Callers should use the typed - * wrapper `callReader(method, args)` declared in the main process - * IPC layer. Resolves with the method's return value, rejects with - * an Error on worker error / timeout / crash. + * Dispatch a method call to the least-busy ready worker. Resolves + * with the method's return value, rejects with an Error on worker + * error / timeout / crash. */ call(method: string, args: unknown[] = []): Promise { if (this.terminated) return Promise.reject(new Error('reader-terminated')); - if (!this.worker) return Promise.reject(new Error('reader-not-initialised')); + const slotIdx = this.pickSlot(); + if (slotIdx < 0) return Promise.reject(new Error('reader-not-initialised')); + const slot = this.slots[slotIdx]!; + const worker = slot.worker; + if (!worker) return Promise.reject(new Error('reader-not-initialised')); const requestId = this.nextRequestId++; const timeoutMs = HEAVY_METHODS.has(method) ? HEAVY_REQUEST_TIMEOUT_MS : REQUEST_TIMEOUT_MS; + slot.pending++; return new Promise((resolve, reject) => { const timer = setTimeout(() => { - this.pending.delete(requestId); + const p = this.pending.get(requestId); + if (p) { + this.pending.delete(requestId); + this.slots[p.workerIdx]!.pending = Math.max( + 0, + this.slots[p.workerIdx]!.pending - 1, + ); + } reject(new Error(`reader-timeout: ${method} > ${timeoutMs}ms`)); }, timeoutMs); this.pending.set(requestId, { + workerIdx: slotIdx, resolve: resolve as (v: unknown) => void, reject, timer, }); - this.worker!.postMessage({ requestId, method, args }); + worker.postMessage({ requestId, method, args }); }); } // ── private ────────────────────────────────────────────────────────────── - private spawn(): void { + /** + * Pick the slot with the fewest pending requests. Ties are broken + * by a round-robin cursor so identically-loaded workers don't all + * receive their next call on the same slot. + */ + private pickSlot(): number { + let best = -1; + let bestPending = Number.POSITIVE_INFINITY; + // Start the scan from the rotating cursor so RR is honoured on ties. + for (let i = 0; i < this.slots.length; i++) { + const idx = (this.rrCursor + i) % this.slots.length; + const s = this.slots[idx]!; + if (!s.worker) continue; + if (s.pending < bestPending) { + best = idx; + bestPending = s.pending; + } + } + if (best >= 0) { + this.rrCursor = (best + 1) % this.slots.length; + } + return best; + } + + private spawn(slotIdx: number): void { if (!this.dbPath) return; - if (this.worker) { - const old = this.worker; - this.worker = null; + const slot = this.slots[slotIdx]; + if (!slot) return; + if (slot.worker) { + const old = slot.worker; + slot.worker = null; void old.terminate().catch(() => undefined); } try { @@ -166,18 +230,19 @@ class DbReaderPool { }); w.on('message', (msg: ResponseMessage) => this.handleResponse(msg)); w.on('error', (err) => { - logger.log('error', 'db-reader', `worker error: ${err.message}`); + logger.log('error', 'db-reader', `worker[${slotIdx}] error: ${err.message}`); }); - w.on('exit', (code) => this.handleExit(code)); - this.worker = w; - logger.log('info', 'db-reader', `worker spawned for ${this.dbPath}`); + w.on('exit', (code) => this.handleExit(slotIdx, code)); + slot.worker = w; + slot.pending = 0; + logger.log('info', 'db-reader', `worker[${slotIdx}] spawned for ${this.dbPath}`); } catch (err) { logger.log( 'error', 'db-reader', - `worker spawn failed: ${err instanceof Error ? err.message : String(err)}`, + `worker[${slotIdx}] spawn failed: ${err instanceof Error ? err.message : String(err)}`, ); - this.worker = null; + slot.worker = null; } } @@ -186,6 +251,8 @@ class DbReaderPool { if (!pending) return; this.pending.delete(msg.requestId); clearTimeout(pending.timer); + const slot = this.slots[pending.workerIdx]; + if (slot) slot.pending = Math.max(0, slot.pending - 1); if (msg.ok) { pending.resolve(msg.result); } else { @@ -193,31 +260,38 @@ class DbReaderPool { } } - private handleExit(code: number): void { - this.worker = null; + private handleExit(slotIdx: number, code: number): void { + const slot = this.slots[slotIdx]; + if (!slot) return; + slot.worker = null; if (this.terminated) return; + // Reject only this slot's in-flight requests. + for (const [id, p] of this.pending) { + if (p.workerIdx === slotIdx) { + clearTimeout(p.timer); + p.reject(new Error('reader-crashed')); + this.pending.delete(id); + } + } + slot.pending = 0; if (code === 0) return; - this.failPendingWith('reader-crashed'); - // Restart bookkeeping: drop entries older than the window so a - // long-lived process that crashes once a day doesn't permanently - // exhaust its restart budget. const now = Date.now(); - this.restartTimes = this.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS); - if (this.restartTimes.length >= MAX_RESTARTS) { + slot.restartTimes = slot.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS); + if (slot.restartTimes.length >= MAX_RESTARTS) { logger.log( 'error', 'db-reader', - `worker crashed ${MAX_RESTARTS}Γ— within ${RESTART_WINDOW_MS}ms β€” giving up. UI queries fall back to main-process DB.`, + `worker[${slotIdx}] crashed ${MAX_RESTARTS}Γ— within ${RESTART_WINDOW_MS}ms β€” giving up on this slot.`, ); return; } - this.restartTimes.push(now); + slot.restartTimes.push(now); logger.log( 'warn', 'db-reader', - `worker exited with code ${code}; respawning (${this.restartTimes.length}/${MAX_RESTARTS})`, + `worker[${slotIdx}] exited with code ${code}; respawning (${slot.restartTimes.length}/${MAX_RESTARTS})`, ); - this.spawn(); + this.spawn(slotIdx); } private failPendingWith(reason: string): void { @@ -226,6 +300,7 @@ class DbReaderPool { p.reject(new Error(reason)); } this.pending.clear(); + for (const slot of this.slots) slot.pending = 0; } } @@ -233,10 +308,10 @@ class DbReaderPool { export const dbReaderPool = new DbReaderPool(); /** - * Thin helper for IPC handlers: try the worker first, fall back to the - * synchronous main-process DB on any worker error. The fallback path - * keeps the UI working even if the worker is crashed/restarting, at - * the cost of running on the main thread for that one query. + * Thin helper for IPC handlers: try the worker pool first, fall back + * to the synchronous main-process DB on any worker error. The fallback + * path keeps the UI working even if every worker is crashed/restarting, + * at the cost of running on the main thread for that one query. */ export async function callReaderOrFallback( method: string, diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index f3da26b..7c8c77e 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -25,6 +25,8 @@ import { type ExportJsonResult, type ExportXmlInput, type ExportXmlResult, + type ExportTabularInput, + type ExportTabularResult, type DataDeleteByDomainInput, type DataDeleteByDomainResult, type CrashRecoveryStatus, @@ -81,6 +83,7 @@ import { exportUrlsToCsv, exportUrlsToJson, exportUrlsToXml, + exportTabular, exportSitemap, exportHtmlReport, compareCrawls, @@ -366,6 +369,64 @@ async function writeFetchedUrlViaPool( } } +/** + * Run the post-crawl issue recompute on the writer worker. This pass + * has 70+ correlated SQL subqueries against the urls table; on a + * 5K-URL crawl it takes ~45 s, and running it on the main thread + * freezes the renderer for that entire window (the "Not Responding" + * dialog). Routing through the writer worker keeps main free. + */ +async function recomputeIssuesViaPool( + definitions: ReadonlyArray, +): Promise { + if (!dbWriterPool.isReady()) { + return getDb().recomputeUrlsIssuesYielding(definitions); + } + try { + await dbWriterPool.call('recomputeUrlsIssuesYielding', [definitions]); + } catch (err) { + logger.log( + 'warn', + 'main', + `recomputeUrlsIssuesYielding dispatch failed (${ + err instanceof Error ? err.message : String(err) + }) β€” falling back to main-thread recompute.`, + ); + await getDb().recomputeUrlsIssuesYielding(definitions); + } +} + +/** + * Generic post-crawl pass dispatcher. Routes recomputeInlinks, + * recomputeRedirectChains, recomputeHreflangAnalysis, + * recomputeHreflangInconsistent, and recomputePaginationSequence + * through the writer worker so the main thread stays responsive to + * Stop/Pause clicks and IPC traffic during the post-crawl phase. + * Without this, those five sync SQL passes combined freeze main for + * 5–15 s on a 5K-URL crawl β€” exactly the window where users hit + * Stop/Pause and "nothing happens". + */ +async function runDbPassViaPool(method: string): Promise { + if (!dbWriterPool.isReady()) { + const fn = (getDb() as unknown as Record)[method]; + if (typeof fn === 'function') (fn as () => void).call(getDb()); + return; + } + try { + await dbWriterPool.call(method, []); + } catch (err) { + logger.log( + 'warn', + 'main', + `${method} dispatch failed (${ + err instanceof Error ? err.message : String(err) + }) β€” falling back to main-thread.`, + ); + const fn = (getDb() as unknown as Record)[method]; + if (typeof fn === 'function') (fn as () => void).call(getDb()); + } +} + function getDb(): ProjectDb { if (!db) { const dataDir = join(app.getPath('userData'), 'projects'); @@ -1160,6 +1221,8 @@ function registerIpc(): void { setOp: (op: string) => freezeWatchdog.setMainOp(op), parseHtml: parseHtmlViaPool, writeFetchedUrl: writeFetchedUrlViaPool, + recomputeIssues: recomputeIssuesViaPool, + runDbPass: runDbPassViaPool, }); activeCrawler = crawler; attachCrawlerListeners(crawler); @@ -1178,11 +1241,29 @@ function registerIpc(): void { activeCrawler?.resume(); }); - ipcMain.handle(IPC.crawlClear, () => { + ipcMain.handle(IPC.crawlClear, async () => { activeCrawler?.stop(); activeCrawler = null; const database = getDb(); - database.reset(); + // Route reset() through the writer pool so it sits behind any + // in-flight writeFetchedUrl RPCs in the worker's FIFO. Without + // this, the main process truncates the urls table while the + // writer worker is still draining the last batch of per-URL + // writes β€” those writes win the SQLite lock race and resurrect + // ~19 rows in the table after Clear. Going through the pool + // serialises naturally: every queued write completes, then reset + // runs against an already-stopped crawler. + if (dbWriterPool.isReady()) { + try { + await dbWriterPool.call('reset', []); + } catch { + // Worker terminated or RPC timed out β€” fall back to direct + // reset rather than leaving the table half-cleared. + database.reset(); + } + } else { + database.reset(); + } // Belt-and-braces: reset() above already includes a DELETE FROM // crawl_queue, but if anything (a late-firing checkpoint timer, a // half-finished post-crawl pass) writes to that table after the @@ -1661,6 +1742,47 @@ function registerIpc(): void { }, ); + ipcMain.handle( + IPC.exportTabular, + async (_e, input: ExportTabularInput): Promise => { + const { format, sections, columns, selectedIds } = input; + let outputPath = input.filePath ?? ''; + const isSelection = (selectedIds?.length ?? 0) > 0; + const multiSection = sections.length > 1; + if (!outputPath) { + if (format === 'csv' && multiSection) { + const res = await dialog.showOpenDialog(mainWindow!, { + properties: ['openDirectory', 'createDirectory'], + title: 'Choose folder for CSV export', + }); + if (res.canceled || res.filePaths.length === 0) { + return { filePath: '', files: [], rowsWritten: 0 }; + } + outputPath = res.filePaths[0]!; + } else { + const ext = format === 'xlsx' ? 'xlsx' : 'csv'; + const filterName = format === 'xlsx' ? 'Excel Workbook' : 'CSV'; + const baseName = isSelection ? 'freecrawl-selected' : 'freecrawl-export'; + const res = await dialog.showSaveDialog(mainWindow!, { + defaultPath: `${baseName}.${ext}`, + filters: [{ name: filterName, extensions: [ext] }], + }); + if (res.canceled || !res.filePath) { + return { filePath: '', files: [], rowsWritten: 0 }; + } + outputPath = res.filePath; + } + } + const result = await exportTabular(getDb(), outputPath, { + format, + sections, + columns, + selectedIds, + }); + return result; + }, + ); + ipcMain.handle( IPC.dataDeleteByDomain, async ( @@ -1707,6 +1829,8 @@ function registerIpc(): void { setOp: (op: string) => freezeWatchdog.setMainOp(op), parseHtml: parseHtmlViaPool, writeFetchedUrl: writeFetchedUrlViaPool, + recomputeIssues: recomputeIssuesViaPool, + runDbPass: runDbPassViaPool, }); activeCrawler = crawler; attachCrawlerListeners(crawler); diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 564cb66..f0df037 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -11,6 +11,8 @@ import { type ExportJsonResult, type ExportXmlInput, type ExportXmlResult, + type ExportTabularInput, + type ExportTabularResult, type DataDeleteByDomainInput, type DataDeleteByDomainResult, type CrashRecoveryStatus, @@ -123,6 +125,8 @@ const api: FreeCrawlApi = { ipcRenderer.invoke(IPC.exportJson, input), exportXml: (input: ExportXmlInput): Promise => ipcRenderer.invoke(IPC.exportXml, input), + exportTabular: (input: ExportTabularInput): Promise => + ipcRenderer.invoke(IPC.exportTabular, input), dataDeleteByDomain: ( input: DataDeleteByDomainInput, ): Promise => diff --git a/apps/desktop/src/renderer/src/App.tsx b/apps/desktop/src/renderer/src/App.tsx index 0e2573a..c6429a5 100644 --- a/apps/desktop/src/renderer/src/App.tsx +++ b/apps/desktop/src/renderer/src/App.tsx @@ -137,20 +137,30 @@ export function App() { // previous session left a non-empty checkpoint we ask whether to // resume. Discarding wipes the snapshot so the prompt doesn't // recur on next launch. + // + // The confirm() is wrapped in a 250 ms setTimeout so the initial + // render finishes (URL input mounts, status bar paints) before + // the modal opens. Without the defer, the synchronous `confirm` + // can fire on first paint and β€” when Electron renders the modal + // behind the main window on some Windows configurations β€” leaves + // the user staring at an "unresponsive" UI: the URL input can't + // be clicked because a hidden modal is blocking the renderer. void window.freecrawl.crashRecoveryStatus().then((status) => { if (status.pendingCount === 0) return; - const proceed = window.confirm( - `FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` + - ` Start URL: ${status.seedUrl}\n` + - ` Pending URLs: ${status.pendingCount.toLocaleString()}\n\n` + - `Resume the crawl from where it stopped?\n\n` + - `Click OK to resume, Cancel to discard the recovery state and start fresh.`, - ); - if (proceed) { - void window.freecrawl.crashRecoveryResume(); - } else { - void window.freecrawl.crashRecoveryDiscard(); - } + window.setTimeout(() => { + const proceed = window.confirm( + `FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` + + ` Start URL: ${status.seedUrl}\n` + + ` Pending URLs: ${status.pendingCount.toLocaleString()}\n\n` + + `Resume the crawl from where it stopped?\n\n` + + `Click OK to resume, Cancel to discard the recovery state and start fresh.`, + ); + if (proceed) { + void window.freecrawl.crashRecoveryResume(); + } else { + void window.freecrawl.crashRecoveryDiscard(); + } + }, 250); }); // rAF-throttled progress dispatch. The crawler emits up to 5 diff --git a/apps/desktop/src/renderer/src/components/ErrorBoundary.tsx b/apps/desktop/src/renderer/src/components/ErrorBoundary.tsx new file mode 100644 index 0000000..931973d --- /dev/null +++ b/apps/desktop/src/renderer/src/components/ErrorBoundary.tsx @@ -0,0 +1,60 @@ +import { Component, type ReactNode } from 'react'; + +interface Props { + /** Rendered when the boundary's subtree throws. Defaults to a tiny notice. */ + fallback?: ReactNode; + /** Surfaced to console + main-process log so the user can copy/paste. */ + context?: string; + children: ReactNode; +} + +interface State { + err: Error | null; +} + +/** + * Catches render-time errors in a subtree so a single misbehaving dialog + * (e.g. ExportDialog) can't unmount the whole renderer and leave the user + * staring at the Electron window's bare background colour. The boundary + * logs the error and shows a small inline notice; the rest of the app + * keeps working. + */ +export class ErrorBoundary extends Component { + override state: State = { err: null }; + + static getDerivedStateFromError(err: Error): State { + return { err }; + } + + override componentDidCatch(err: Error, info: { componentStack?: string }): void { + // eslint-disable-next-line no-console + console.error( + `[ErrorBoundary${this.props.context ? `:${this.props.context}` : ''}]`, + err, + info.componentStack, + ); + } + + reset = (): void => { + this.setState({ err: null }); + }; + + override render(): ReactNode { + const { err } = this.state; + if (!err) return this.props.children; + if (this.props.fallback !== undefined) return this.props.fallback; + return ( +
+
Something broke in this panel.
+
{err.message}
+ +
+ ); + } +} diff --git a/apps/desktop/src/renderer/src/components/ExportDialog.tsx b/apps/desktop/src/renderer/src/components/ExportDialog.tsx new file mode 100644 index 0000000..03c3961 --- /dev/null +++ b/apps/desktop/src/renderer/src/components/ExportDialog.tsx @@ -0,0 +1,403 @@ +import { useEffect, useMemo, useState } from 'react'; +import { X, Loader2 } from 'lucide-react'; +import clsx from 'clsx'; +import type { CrawlUrlRow, ExportTabularSection, UrlCategory } from '@freecrawl/shared-types'; +import { TAB_ORDER, type TabKey } from '../store.js'; +import { COLUMN_SPECS, columnId } from '../tabs/columns.js'; + +type Format = 'csv' | 'xlsx'; + +interface ExportTab { + key: TabKey; + label: string; + category: UrlCategory; + columnKeys: (keyof CrawlUrlRow)[]; +} + +const TAB_CATEGORY: Record = { + internal: 'internal:html', + external: 'external:all', + 'response-codes': 'all', + url: 'internal:html', + 'page-titles': 'internal:html', + 'meta-description': 'internal:html', + h1: 'internal:html', + h2: 'internal:html', + content: 'internal:html', + images: 'internal:html', + canonicals: 'internal:html', + directives: 'internal:html', + redirects: 'all', + links: 'all', + 'broken-links': 'all', +}; + +function buildExportableTabs(): ExportTab[] { + const out: ExportTab[] = []; + for (const t of TAB_ORDER) { + const cols = COLUMN_SPECS[t.key]; + // Tabs with their own non-URL-row layouts (Images / Broken Links) + // have empty COLUMN_SPECS β€” skip them so the export grid only shows + // categories that share the URL-row schema. + if (!cols || cols.length === 0) continue; + out.push({ + key: t.key, + label: t.label, + category: TAB_CATEGORY[t.key], + columnKeys: cols.map((c) => c.key), + }); + } + return out; +} + +interface ExportDialogProps { + open: boolean; + onClose: () => void; + /** The tab the user clicked Export from β€” drives the default selection. */ + defaultTab: TabKey; + /** Row ids selected in the current table; falsy β†’ export the whole filter set. */ + selectedIds?: number[]; +} + +export function ExportDialog({ + open, + onClose, + defaultTab, + selectedIds, +}: ExportDialogProps) { + const tabs = useMemo(() => buildExportableTabs(), []); + const initialTabKey: TabKey = useMemo(() => { + return tabs.find((t) => t.key === defaultTab)?.key ?? tabs[0]?.key ?? 'internal'; + }, [defaultTab, tabs]); + + const selectionCount = selectedIds?.length ?? 0; + + const [format, setFormat] = useState('csv'); + const [pickedTabs, setPickedTabs] = useState>(() => new Set([initialTabKey])); + const [pickedColumns, setPickedColumns] = useState>(() => { + const t = tabs.find((x) => x.key === initialTabKey); + return new Set((t?.columnKeys ?? []) as string[]); + }); + const [useSelection, setUseSelection] = useState(selectionCount > 0); + const [busy, setBusy] = useState(false); + const [status, setStatus] = useState(''); + + // The dialog is conditionally mounted by the parent (only when open=true), + // so useState initializers above already populate the right defaults on + // each open. Escape key dismissal is the only ambient behaviour we need. + useEffect(() => { + if (!open) return; + const onKey = (e: KeyboardEvent): void => { + if (e.key === 'Escape') onClose(); + }; + document.addEventListener('keydown', onKey); + return () => document.removeEventListener('keydown', onKey); + }, [open, onClose]); + + // Union of column keys exposed by every picked tab β€” these are the + // columns the dialog offers; checked state lives in `pickedColumns`. + const offeredColumns = useMemo(() => { + const seen = new Set(); + const out: { key: string; header: string; tab: TabKey }[] = []; + for (const t of tabs) { + if (!pickedTabs.has(t.key)) continue; + const specs = COLUMN_SPECS[t.key]; + for (const spec of specs) { + const id = columnId(spec); + if (seen.has(id)) continue; + seen.add(id); + out.push({ key: spec.key as string, header: spec.header, tab: t.key }); + } + } + return out; + }, [pickedTabs, tabs]); + + const orderedColumnList = useMemo( + () => offeredColumns.filter((c) => pickedColumns.has(c.key)).map((c) => c.key), + [offeredColumns, pickedColumns], + ); + + if (!open) return null; + + const toggleTab = (key: TabKey) => { + setPickedTabs((prev) => { + const next = new Set(prev); + let willBeRemoved = false; + if (next.has(key)) { + if (next.size === 1) return prev; // keep at least one + next.delete(key); + willBeRemoved = true; + } else { + next.add(key); + } + // Auto-tick the new tab's columns when it joins the picked set, so + // the user doesn't have to re-check them. When a tab leaves we keep + // the user's column picks alone β€” orderedColumnList already drops + // any column not offered by remaining tabs. + if (!willBeRemoved) { + const t = tabs.find((x) => x.key === key); + if (t) { + setPickedColumns((prevCols) => { + const nextCols = new Set(prevCols); + for (const k of t.columnKeys) nextCols.add(k as string); + return nextCols; + }); + } + } + return next; + }); + }; + + const toggleColumn = (key: string) => { + setPickedColumns((prev) => { + const next = new Set(prev); + if (next.has(key)) next.delete(key); + else next.add(key); + return next; + }); + }; + + const checkAllColumns = () => { + setPickedColumns(new Set(offeredColumns.map((c) => c.key))); + }; + + const uncheckAllColumns = () => { + setPickedColumns(new Set()); + }; + + const submit = async () => { + if (orderedColumnList.length === 0) { + setStatus('Pick at least one column.'); + return; + } + if (pickedTabs.size === 0) { + setStatus('Pick at least one table.'); + return; + } + setBusy(true); + setStatus(''); + try { + const sections: ExportTabularSection[] = tabs + .filter((t) => pickedTabs.has(t.key)) + .map((t) => ({ label: t.label, category: t.category })); + const result = await window.freecrawl.exportTabular({ + format, + sections, + columns: orderedColumnList, + selectedIds: + useSelection && selectedIds && selectedIds.length > 0 + ? selectedIds + : undefined, + }); + if (!result.filePath) { + // User cancelled save / folder dialog. + setBusy(false); + return; + } + setStatus( + `Exported ${result.rowsWritten.toLocaleString()} row${result.rowsWritten === 1 ? '' : 's'} to ${result.filePath}`, + ); + setBusy(false); + setTimeout(() => onClose(), 700); + } catch (err) { + setBusy(false); + setStatus(`Export failed: ${err instanceof Error ? err.message : String(err)}`); + } + }; + + return ( +
+
e.stopPropagation()} + > +
+
Export
+ +
+ +
+
+
+
+ Tables +
+
+
+ {tabs.map((t) => { + const checked = pickedTabs.has(t.key); + const isDefault = t.key === initialTabKey; + return ( + + ); + })} +
+
+
+ +
+
+
+ Columns +
+
+ + +
+
+
+
+ {offeredColumns.map((c) => { + const checked = pickedColumns.has(c.key); + return ( + + ); + })} +
+
+
+
+ +
+
+ + Format + + + +
+ + {selectionCount > 0 && ( + + )} +
+ + {format === 'csv' && pickedTabs.size > 1 && ( +
+ Multiple tables + CSV β†’ you'll be asked to pick a folder; one file is written per table. +
+ )} + + {status && ( +
+ {status} +
+ )} +
+ +
+ + +
+
+
+ ); +} diff --git a/apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts b/apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts index db620b4..a01e2ec 100644 --- a/apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts +++ b/apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts @@ -2,18 +2,27 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import type { AdvancedFilter, CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types'; const CHUNK_SIZE = 500; -const MAX_CACHED_CHUNKS = 8; -// Live-refresh cadence during an active crawl. 750 ms strikes a balance -// between perceived "row streaming" smoothness and the cost of a COUNT -// query + per-chunk fetch every tick. At 100 URL/s the user still sees -// ~75 new rows per tick (continuous flow), while leaving 3Γ— more main- -// thread headroom than the previous 250 ms cadence. -// Cadence for the live-tail tick. 1500 ms strikes the balance between -// "feels fresh" and "doesn't pin SQLite". At 750 ms a 2k+ row crawl was -// burning ~30% of one core in COUNT + chunk fetches, blocking input IPC -// for the duration. The user sees the same effect: numbers tick at half -// speed but the window stops kasma'ing. -const LIVE_REFRESH_MS = 1500; +// Doubled the cache (was 8) so a fast scroll across 4-5 chunks doesn't +// thrash the reader pool: each chunk evicted = a re-fetch the next time +// the user scrolls back to it. 16 Γ— 500 = 8000 rows kept warm; at +// ~2 KB/row that's ~16 MB heap, well within budget. +const MAX_CACHED_CHUNKS = 16; +// Number of chunks to pre-fetch ahead/behind the visible range. The +// visible range is what the virtualizer asks for; pre-fetching the +// neighbours hides "..." placeholders during fast scrolls because the +// next chunk has usually already landed by the time the user reaches +// it. 1 chunk = 500 rows of look-ahead each direction, cheap on the +// reader pool. +const PREFETCH_CHUNKS = 1; +// Min gap between progress-driven ticks. Progress events arrive at +// ~5 Hz from the crawler; without throttle the visible-chunk re-query +// would fire that often too. 100 ms keeps the table feeling live +// (10 refreshes/sec ceiling) without pinning the reader-pool. +const TICK_THROTTLE_MS = 100; +// Safety-net periodic tick for cases where progress events stop +// arriving (crawl finished, manual DB edits, etc.). Long interval β€” +// the progress-event path drives the in-crawl experience. +const LIVE_REFRESH_MS = 5000; export interface LazyRowsOpts { category: UrlCategory; @@ -31,6 +40,12 @@ export interface LazyRowsState { loadedRows: number; rowAt: (index: number) => CrawlUrlRow | null; ensureRange: (start: number, end: number) => void; + /** True between the moment a shape change is requested (tab/sort/filter + * flip) and the moment the new data has fully landed. Lets the + * consumer distinguish "0 URLs because the category genuinely has + * none" from "0 URLs because we haven't fetched yet" β€” which the + * empty-state UI cares about a lot. */ + isLoading: boolean; } /** @@ -49,6 +64,7 @@ export interface LazyRowsState { export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState { const [total, setTotal] = useState(0); const [version, setVersion] = useState(0); + const [isLoading, setIsLoading] = useState(true); const chunks = useRef(new Map()); const chunkOrder = useRef([]); const fetching = useRef(new Set()); @@ -88,20 +104,52 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState { [opts.category, opts.search, opts.sortBy, opts.sortDir, opts.filter], ); - // Shape change: rebuild from scratch. The virtualizer will call - // ensureRange again on its next measurement pass. + // Shape change (tab/sort/filter switch): clear immediately and load + // fresh data in parallel. Showing the previous tab's rows under the + // new tab's header is the wrong UX β€” "Internal" rows visible while + // "External" is selected makes the user mistrust what they see. + // Instead we set `isLoading=true` and let the consumer render a + // proper loading state until the new total + visible chunks land. + // The reader-pool has multiple workers now, so meta + visible-chunk + // queries run in parallel and typically resolve in ~50-150 ms. useEffect(() => { resetToken.current++; chunks.current.clear(); chunkOrder.current = []; fetching.current.clear(); + setIsLoading(true); setVersion((v) => v + 1); const token = resetToken.current; let cancelled = false; - void queryMeta().then(({ total: t }) => { - if (cancelled || token !== resetToken.current) return; - setTotal(t); - }); + + const load = async (): Promise => { + const metaPromise = queryMeta(); + const { first, last } = activeRange.current; + const chunkPromises: Promise<{ rows: CrawlUrlRow[]; total: number; idx: number }>[] = []; + for (let i = first; i <= last; i++) { + chunkPromises.push(queryChunk(i).then((r) => ({ ...r, idx: i }))); + } + try { + const [{ total: t }, ...newChunks] = await Promise.all([metaPromise, ...chunkPromises]); + if (cancelled || token !== resetToken.current) return; + for (const ch of newChunks) { + chunks.current.set(ch.idx, ch.rows); + chunkOrder.current.push(ch.idx); + } + setTotal(t); + setVersion((v) => v + 1); + } catch { + if (cancelled || token !== resetToken.current) return; + // Leave chunks empty on error; the live tick / ensureRange + // will retry. Still need to flip loading off so the empty + // state can render an actual error/empty UI. + } finally { + if (!cancelled && token === resetToken.current) { + setIsLoading(false); + } + } + }; + void load(); return () => { cancelled = true; }; @@ -111,16 +159,26 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState { // Live tick: re-query only the visible chunks and patch them in place. // Never calls .clear(), so rowAt never returns null for a // previously-loaded index. The table streams rather than flashes. - // A leading tick fires immediately so the first rows surface without - // the LIVE_REFRESH_MS dead window after Start. + // + // Drive model: push, not poll. Each crawler progress event (β‰ˆ every + // time `crawled` advances, which lands at ~5 Hz on a healthy crawl) + // triggers an immediate visible-chunk re-query. The previous design + // polled every 1500 ms regardless of activity β€” newly-written rows + // could sit in the DB for up to that long before surfacing in the + // table. The push path closes that window to ~100 ms (the throttle + // floor that keeps the reader-pool from being hammered). + // + // A 5 s safety-net `setInterval` still fires for code paths that + // mutate the DB without going through a progress event (manual row + // delete, project re-open, etc.). useEffect(() => { let cancelled = false; let inFlight = false; + let lastTickTs = 0; const tick = async () => { - // Coalesce overlapping ticks β€” at 250 ms cadence a slow COUNT or - // chunk fetch can otherwise queue up multiple in-flight ticks. if (inFlight) return; inFlight = true; + lastTickTs = Date.now(); const token = resetToken.current; try { try { @@ -146,33 +204,47 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState { inFlight = false; } }; - // Wrap the tick in requestIdleCallback so the live-tail SQL only - // fires when the renderer's event loop has idle slack. If the user - // is dragging / clicking / typing, the tick is deferred until the - // input handler frame finishes β€” eliminates the "click β†’ kasma" - // pattern even when the underlying SQL is fast. Falls back to a - // direct call when the browser doesn't expose `requestIdleCallback` - // (older Chromium builds; Electron 41 ships modern Chromium so this - // branch is mostly defensive). interface RequestIdleCallback { (cb: () => void, opts?: { timeout: number }): number; } const w = window as Window & { requestIdleCallback?: RequestIdleCallback }; const scheduleTick = (): void => { if (typeof w.requestIdleCallback === 'function') { - // 2 s timeout guarantees the tick eventually runs even if the - // main thread is permanently busy β€” better stale data than no - // data at all. - w.requestIdleCallback(() => void tick(), { timeout: 2000 }); + // 300 ms timeout β€” short enough that the user perceives an + // event-driven refresh as instant; long enough that an + // in-progress click handler isn't preempted. + w.requestIdleCallback(() => void tick(), { timeout: 300 }); } else { void tick(); } }; - scheduleTick(); + // Leading tick fires directly (no idle wrap) so the first row + // surfaces as soon as the crawler has written one β€” without this, + // requestIdleCallback could defer the initial fetch up to 300 ms + // on a busy main thread right after Start. + void tick(); + + // Push: progress event drives the live-tail tick. We tick on every + // event where `crawled` changed, throttled to TICK_THROTTLE_MS so + // a 100 URL/s crawl doesn't force 100 SQL roundtrips per second. + // Direct call (no idle wrap) β€” progress events are sparse enough + // that input-handler interference is a non-concern, and pulling + // through requestIdleCallback was adding 50-300 ms before each + // newly-written URL surfaced in the table. + let lastCrawled = -1; + const offProgress = window.freecrawl?.onProgress?.((p) => { + if (p.crawled === lastCrawled) return; + lastCrawled = p.crawled; + if (Date.now() - lastTickTs < TICK_THROTTLE_MS) return; + void tick(); + }); + + // Safety-net poll for crawl-idle DB mutations. const id = setInterval(scheduleTick, LIVE_REFRESH_MS); return () => { cancelled = true; clearInterval(id); + offProgress?.(); }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [keyStr]); @@ -213,7 +285,13 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState { const first = Math.max(0, Math.floor(start / CHUNK_SIZE)); const last = Math.max(0, Math.floor(end / CHUNK_SIZE)); activeRange.current = { first, last }; - for (let i = first; i <= last; i++) { + // Pre-fetch a window of neighbouring chunks so a fast scroll + // doesn't show "..." placeholders while the next chunk loads β€” + // by the time the virtualizer scrolls into the next chunk's + // range, its rows are usually already in the cache. + const fetchFrom = Math.max(0, first - PREFETCH_CHUNKS); + const fetchTo = last + PREFETCH_CHUNKS; + for (let i = fetchFrom; i <= fetchTo; i++) { if (!chunks.current.has(i)) void fetchChunk(i); } }, @@ -237,5 +315,5 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState { [version], ); - return { total, loadedRows, rowAt, ensureRange }; + return { total, loadedRows, rowAt, ensureRange, isLoading }; } diff --git a/apps/desktop/src/renderer/src/tabs/UrlsTab.tsx b/apps/desktop/src/renderer/src/tabs/UrlsTab.tsx index 6bd8c5f..ad8bd5c 100644 --- a/apps/desktop/src/renderer/src/tabs/UrlsTab.tsx +++ b/apps/desktop/src/renderer/src/tabs/UrlsTab.tsx @@ -1,12 +1,14 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useVirtualizer } from '@tanstack/react-virtual'; -import { ChevronsUpDown, Filter, X } from 'lucide-react'; +import { ChevronsUpDown, Download, Filter, X } from 'lucide-react'; import clsx from 'clsx'; import type { AdvancedFilter, CrawlUrlRow } from '@freecrawl/shared-types'; import { useAppStore, type TabKey } from '../store.js'; import { COLUMN_SPECS, columnId, type ColumnSpec } from './columns.js'; import { useLazyUrlRows } from '../hooks/useLazyUrlRows.js'; import { AdvancedFilterDialog } from '../components/AdvancedFilterDialog.js'; +import { ErrorBoundary } from '../components/ErrorBoundary.js'; +import { ExportDialog } from '../components/ExportDialog.js'; import { InfoTip } from '../components/InfoTip.js'; const ROW_HEIGHT = 24; @@ -36,6 +38,7 @@ export function UrlsTab() { const [sortDir, setSortDir] = useState<'asc' | 'desc'>('asc'); const [filter, setFilter] = useState(null); const [filterDialogOpen, setFilterDialogOpen] = useState(false); + const [exportDialogOpen, setExportDialogOpen] = useState(false); // Three orthogonal selection layers: // - selectedIds: row-level selection driven by the Row-number column. // Feeds the bulk context menu (Copy/Open/Re-Spider/Remove/Export). @@ -515,12 +518,21 @@ export function UrlsTab() { )} -
- {lazy.total.toLocaleString()} URLs - - ({lazy.loadedRows.toLocaleString()} loaded) - -
+
@@ -753,7 +765,15 @@ export function UrlsTab() {
- {lazy.total === 0 && ( + {lazy.total === 0 && lazy.isLoading && ( +
+
Loading…
+
+ )} + {lazy.total === 0 && !lazy.isLoading && (
+ + URLs:{' '} + + {lazy.total.toLocaleString()} + + + ({lazy.loadedRows.toLocaleString()} loaded) + + Selected Rows:{' '} @@ -789,12 +818,6 @@ export function UrlsTab() { {selectedColumns.size.toLocaleString()} - - Filter Total:{' '} - - {lazy.total.toLocaleString()} - -
setFilterDialogOpen(false)} onApply={(f) => setFilter(f)} /> + + + {exportDialogOpen && ( + setExportDialogOpen(false)} + defaultTab={activeTab} + selectedIds={selectedIds.size > 0 ? [...selectedIds] : undefined} + /> + )} + ); } diff --git a/apps/mcp-server/package.json b/apps/mcp-server/package.json new file mode 100644 index 0000000..07baf3e --- /dev/null +++ b/apps/mcp-server/package.json @@ -0,0 +1,25 @@ +{ + "name": "@freecrawl/mcp-server", + "version": "0.2.8", + "private": true, + "type": "module", + "bin": { + "freecrawl-mcp": "./dist/index.js" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "start": "node ./dist/index.js", + "dev": "tsc -p tsconfig.json --watch", + "clean": "rimraf dist" + }, + "dependencies": { + "@freecrawl/db": "*", + "@freecrawl/shared-types": "*" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "rimraf": "^6.0.1", + "typescript": "^5.7.0" + } +} diff --git a/apps/mcp-server/src/index.ts b/apps/mcp-server/src/index.ts new file mode 100644 index 0000000..9f84493 --- /dev/null +++ b/apps/mcp-server/src/index.ts @@ -0,0 +1,192 @@ +#!/usr/bin/env node +/** + * FreeCrawl SEO β€” MCP server (stdio transport, JSON-RPC 2.0). + * + * Exposes the active `.seoproject` to MCP-compatible clients (Claude + * Desktop, Claude Code, …) so an AI agent can run SEO queries against + * the most recent crawl directly: + * + * - `get_summary` β†’ headline numbers + * - `get_overview_counts` β†’ sidebar issue/category breakdown + * - `top_issues` β†’ ranked non-empty issue list + * - `query_urls` β†’ paginated category/search/sort + * - `get_url_detail` β†’ full row + inlinks/outlinks/images + * - `list_projects` β†’ enumerate `.seoproject` files + * - `set_project` β†’ switch sessions to a different file + * + * The DB connection is read-only (`new ProjectDb(path, { readOnly: true })`) + * so the desktop app's writer thread can keep operating concurrently β€” + * SQLite WAL allows arbitrary concurrent readers. + */ + +import fs from 'node:fs'; +import { ProjectDb } from '@freecrawl/db'; +import { JsonRpcServer } from './rpc.js'; +import { buildTools, type Tool } from './tools.js'; +import { defaultProjectPath, listProjectFiles } from './project-resolver.js'; + +const SERVER_NAME = 'freecrawl-seo'; +const SERVER_VERSION = '0.2.7'; +const PROTOCOL_VERSION = '2024-11-05'; + +interface Session { + projectPath: string; + db: ProjectDb; +} + +let session: Session | null = null; + +function openProject(projectPath: string): Session { + if (!fs.existsSync(projectPath)) { + throw new Error(`Project file does not exist: ${projectPath}`); + } + const db = new ProjectDb(projectPath, { readOnly: true }); + return { projectPath, db }; +} + +function getSession(): Session { + if (session) return session; + const initialPath = defaultProjectPath(); + session = openProject(initialPath); + return session; +} + +function setProject(projectPath: string): Session { + if (session) { + try { + session.db.close(); + } catch { + // best-effort close + } + } + session = openProject(projectPath); + return session; +} + +const tools: Tool[] = [ + ...buildTools(), + { + name: 'list_projects', + description: + 'List `.seoproject` files in the desktop app\'s default projects folder. Returns absolute paths the user can pass to `set_project`.', + inputSchema: { type: 'object', properties: {} }, + handler: () => { + const files = listProjectFiles(); + return { files, defaultPath: defaultProjectPath() }; + }, + }, + { + name: 'set_project', + description: + 'Switch the active project for this MCP session. Subsequent tool calls hit the new file. Closes the previous DB handle.', + inputSchema: { + type: 'object', + required: ['path'], + properties: { + path: { + type: 'string', + description: 'Absolute path to a `.seoproject` file.', + }, + }, + }, + handler: (args) => { + const target = args.path; + if (typeof target !== 'string' || target.trim() === '') { + throw new Error('set_project requires `path` to be a non-empty string.'); + } + const next = setProject(target); + return { ok: true, projectPath: next.projectPath }; + }, + }, + { + name: 'current_project', + description: + 'Report which `.seoproject` the MCP server is currently reading from.', + inputSchema: { type: 'object', properties: {} }, + handler: () => ({ projectPath: getSession().projectPath }), + }, +]; + +const server = new JsonRpcServer(); + +// ── Required MCP lifecycle methods ─────────────────────────────────── + +server.on('initialize', (params) => { + const requestedVersion = + typeof (params as Record | undefined)?.protocolVersion === 'string' + ? ((params as Record).protocolVersion) + : PROTOCOL_VERSION; + return { + protocolVersion: requestedVersion, + capabilities: { + tools: { listChanged: false }, + }, + serverInfo: { + name: SERVER_NAME, + version: SERVER_VERSION, + }, + }; +}); + +// MCP clients send `notifications/initialized` after `initialize`. It +// has no `id`, so the JSON-RPC layer won't write a response. +server.on('notifications/initialized', () => undefined); + +// ── Tools surface ──────────────────────────────────────────────────── + +server.on('tools/list', () => ({ + tools: tools.map((t) => ({ + name: t.name, + description: t.description, + inputSchema: t.inputSchema, + })), +})); + +server.on('tools/call', async (raw) => { + const params = (raw ?? {}) as { name?: string; arguments?: Record }; + const name = params.name; + const args = params.arguments ?? {}; + if (typeof name !== 'string') { + throw new Error('tools/call requires `name` to be a string.'); + } + const tool = tools.find((t) => t.name === name); + if (!tool) throw new Error(`Unknown tool: ${name}`); + + const sess = getSession(); + let result: unknown; + try { + result = await tool.handler(args, sess.db); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { + isError: true, + content: [{ type: 'text', text: msg }], + }; + } + + return { + content: [ + { + type: 'text', + text: typeof result === 'string' ? result : JSON.stringify(result, null, 2), + }, + ], + }; +}); + +// ── Bootstrap ──────────────────────────────────────────────────────── + +process.on('SIGINT', () => { + if (session) { + try { + session.db.close(); + } catch { + // best-effort close + } + } + process.exit(0); +}); + +server.log(`starting ${SERVER_NAME} v${SERVER_VERSION}`); +server.log(`default project: ${defaultProjectPath()}`); +server.start(); diff --git a/apps/mcp-server/src/project-resolver.ts b/apps/mcp-server/src/project-resolver.ts new file mode 100644 index 0000000..ff23583 --- /dev/null +++ b/apps/mcp-server/src/project-resolver.ts @@ -0,0 +1,57 @@ +import path from 'node:path'; +import os from 'node:os'; +import fs from 'node:fs'; + +/** + * Locate the desktop app's `default.seoproject` without depending on + * Electron. The desktop app stores it under + * `app.getPath('userData')/projects/default.seoproject`, where + * `userData` resolves per-platform using the same rules Electron uses. + * + * The Electron-resolved userData path is: + * - Windows: %APPDATA%/ + * - macOS: ~/Library/Application Support/ + * - Linux: $XDG_CONFIG_HOME/ or ~/.config/ + * + * `productName` for FreeCrawl is `@freecrawl/desktop` (taken from + * `apps/desktop/package.json`'s `name` field). + */ + +const PRODUCT_NAME = '@freecrawl/desktop'; +const PROJECT_FILENAME = 'default.seoproject'; + +export function defaultProjectPath(): string { + // Explicit override always wins β€” handy for headless testing or + // pointing at a non-default project. + const override = process.env.FREECRAWL_PROJECT; + if (override && override.trim().length > 0) return override.trim(); + + const home = os.homedir(); + let userData: string; + if (process.platform === 'win32') { + userData = path.join( + process.env.APPDATA ?? path.join(home, 'AppData', 'Roaming'), + PRODUCT_NAME, + ); + } else if (process.platform === 'darwin') { + userData = path.join(home, 'Library', 'Application Support', PRODUCT_NAME); + } else { + userData = path.join( + process.env.XDG_CONFIG_HOME ?? path.join(home, '.config'), + PRODUCT_NAME, + ); + } + return path.join(userData, 'projects', PROJECT_FILENAME); +} + +export function listProjectFiles(): string[] { + const dir = path.dirname(defaultProjectPath()); + try { + return fs + .readdirSync(dir) + .filter((f) => f.endsWith('.seoproject')) + .map((f) => path.join(dir, f)); + } catch { + return []; + } +} diff --git a/apps/mcp-server/src/rpc.ts b/apps/mcp-server/src/rpc.ts new file mode 100644 index 0000000..6b22281 --- /dev/null +++ b/apps/mcp-server/src/rpc.ts @@ -0,0 +1,109 @@ +/** + * Minimal newline-delimited JSON-RPC 2.0 server for stdio MCP transport. + * + * MCP's stdio transport is line-delimited JSON β€” one request per line on + * stdin, one response per line on stdout. stderr is reserved for human + * logs; writing anything non-JSON to stdout corrupts the protocol stream. + * + * We deliberately do NOT depend on `@modelcontextprotocol/sdk`. The + * protocol surface we need (`initialize`, `tools/list`, `tools/call`, + * the `initialized` notification) is small, well-specified, and stable; + * the dep-free path keeps the install footprint minimal and avoids a + * version-pin we'd otherwise have to chase. + */ + +export interface JsonRpcRequest { + jsonrpc: '2.0'; + id?: number | string | null; + method: string; + params?: unknown; +} + +export interface JsonRpcSuccess { + jsonrpc: '2.0'; + id: number | string | null; + result: unknown; +} + +export interface JsonRpcError { + jsonrpc: '2.0'; + id: number | string | null; + error: { code: number; message: string; data?: unknown }; +} + +export type Handler = (params: unknown) => unknown | Promise; + +export class JsonRpcServer { + private handlers = new Map(); + private buffer = ''; + + on(method: string, handler: Handler): void { + this.handlers.set(method, handler); + } + + log(...parts: unknown[]): void { + // Always to stderr β€” stdout is reserved for protocol frames. + const line = parts + .map((p) => (typeof p === 'string' ? p : JSON.stringify(p))) + .join(' '); + process.stderr.write(`[freecrawl-mcp] ${line}\n`); + } + + start(): void { + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk: string) => { + this.buffer += chunk; + let nl: number; + while ((nl = this.buffer.indexOf('\n')) !== -1) { + const line = this.buffer.slice(0, nl).trim(); + this.buffer = this.buffer.slice(nl + 1); + if (line) void this.handleLine(line); + } + }); + process.stdin.on('end', () => process.exit(0)); + } + + private write(msg: JsonRpcSuccess | JsonRpcError): void { + process.stdout.write(JSON.stringify(msg) + '\n'); + } + + private async handleLine(line: string): Promise { + let req: JsonRpcRequest; + try { + req = JSON.parse(line) as JsonRpcRequest; + } catch { + this.log('parse error', line); + return; + } + const id = req.id ?? null; + const handler = this.handlers.get(req.method); + // Notifications (no `id`) get no response, even on miss. + const isNotification = req.id === undefined || req.id === null; + if (!handler) { + if (!isNotification) { + this.write({ + jsonrpc: '2.0', + id, + error: { code: -32601, message: `Method not found: ${req.method}` }, + }); + } + return; + } + try { + const result = await handler(req.params); + if (!isNotification) { + this.write({ jsonrpc: '2.0', id, result }); + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + this.log('handler error', req.method, msg); + if (!isNotification) { + this.write({ + jsonrpc: '2.0', + id, + error: { code: -32000, message: msg }, + }); + } + } + } +} diff --git a/apps/mcp-server/src/tools.ts b/apps/mcp-server/src/tools.ts new file mode 100644 index 0000000..0a89628 --- /dev/null +++ b/apps/mcp-server/src/tools.ts @@ -0,0 +1,265 @@ +/** + * Tool definitions and handlers exposed by the FreeCrawl MCP server. + * + * The server opens the project DB in read-only mode (see `index.ts`), + * so it can coexist with the desktop app's writer without contention. + * SQLite WAL mode permits arbitrary concurrent readers. + */ + +import type { ProjectDb } from '@freecrawl/db'; +import type { UrlCategory } from '@freecrawl/shared-types'; + +export interface Tool { + name: string; + description: string; + inputSchema: Record; + handler: (args: Record, db: ProjectDb) => unknown | Promise; +} + +const URL_CATEGORY_VALUES: UrlCategory[] = [ + 'all', + 'internal:all', + 'internal:html', + 'internal:js', + 'internal:css', + 'internal:image', + 'internal:pdf', + 'internal:font', + 'internal:other', + 'external:all', + 'external:html', + 'external:other', + 'status:2xx', + 'status:3xx', + 'status:4xx', + 'status:5xx', + 'status:no-response', + 'status:blocked-robots', + 'security:https', + 'security:http', + 'indexability:indexable', + 'indexability:non-indexable', + 'indexability:noindex', + 'indexability:canonicalised', +]; + +const COMMON_URL_FIELDS = [ + 'id', + 'url', + 'statusCode', + 'contentKind', + 'indexability', + 'title', + 'titleLength', + 'metaDescription', + 'metaDescriptionLength', + 'h1', + 'h1Count', + 'wordCount', + 'canonical', + 'metaRobots', + 'contentLength', + 'responseTimeMs', + 'depth', + 'inlinks', + 'outlinks', + 'imagesCount', + 'imagesMissingAlt', + 'redirectTarget', + 'crawledAt', +]; + +function projectFromRow( + row: Record, + fields: string[], +): Record { + if (fields.length === 0) { + const out: Record = {}; + for (const k of COMMON_URL_FIELDS) out[k] = row[k]; + return out; + } + const out: Record = {}; + for (const k of fields) out[k] = row[k]; + return out; +} + +export function buildTools(): Tool[] { + return [ + { + name: 'get_summary', + description: + 'Top-level crawl summary for the active FreeCrawl project: total URLs crawled, indexable/non-indexable counts, status-code distribution, average response time, plus crawl start/end timestamps. Use this first when reporting on a crawl.', + inputSchema: { + type: 'object', + properties: {}, + }, + handler: (_args, db) => db.getSummary(), + }, + + { + name: 'get_overview_counts', + description: + 'Sidebar overview counts β€” every category and issue filter the desktop app surfaces, with row counts. Useful to find which issue categories are non-empty before drilling in with query_urls.', + inputSchema: { + type: 'object', + properties: {}, + }, + handler: (_args, db) => db.getOverviewCounts(), + }, + + { + name: 'query_urls', + description: + 'Paginated query against the URLs table with the same category filter the UI uses. Returns rows + total count. Use `category` to narrow (e.g. "issues:title-missing"), `search` to substring-match URLs/titles, and `limit`/`offset` to page through.', + inputSchema: { + type: 'object', + properties: { + category: { + type: 'string', + description: 'UrlCategory filter (e.g. "all", "internal:html", "issues:title-missing", "status:4xx"). Defaults to "all".', + enum: URL_CATEGORY_VALUES, + }, + search: { + type: 'string', + description: 'Optional substring match against URL and title.', + }, + limit: { + type: 'integer', + minimum: 1, + maximum: 500, + description: 'Max rows to return (1–500). Default 50.', + }, + offset: { + type: 'integer', + minimum: 0, + description: 'Row offset for pagination. Default 0.', + }, + sortBy: { + type: 'string', + description: 'Column to sort by (e.g. "depth", "responseTimeMs", "inlinks").', + }, + sortDir: { + type: 'string', + enum: ['asc', 'desc'], + description: 'Sort direction. Default "asc".', + }, + fields: { + type: 'array', + items: { type: 'string' }, + description: + 'Columns to include per row. Empty/omitted β†’ a sensible default subset (URL, status, indexability, title, depth, inlinks, etc.). Use ["*"] for the full row.', + }, + }, + }, + handler: (args, db) => { + const category = (args.category as UrlCategory | undefined) ?? 'all'; + const search = typeof args.search === 'string' ? args.search : undefined; + const limit = Math.min( + Math.max(typeof args.limit === 'number' ? args.limit : 50, 1), + 500, + ); + const offset = Math.max( + typeof args.offset === 'number' ? args.offset : 0, + 0, + ); + const sortBy = typeof args.sortBy === 'string' ? args.sortBy : undefined; + const sortDir = args.sortDir === 'desc' ? 'desc' : 'asc'; + const fieldsArg = Array.isArray(args.fields) ? (args.fields as string[]) : []; + const wantsAll = fieldsArg.length === 1 && fieldsArg[0] === '*'; + + const { rows, total } = db.queryUrls({ + category, + search, + limit, + offset, + sortBy, + sortDir, + }); + const projected = wantsAll + ? rows + : rows.map((r) => + projectFromRow( + r as unknown as Record, + fieldsArg, + ), + ); + return { rows: projected, total, limit, offset }; + }, + }, + + { + name: 'get_url_detail', + description: + 'Full details for one URL: every column, plus inlinks, outlinks, images, and HTTP headers. Pass either `id` (preferred β€” from query_urls) or `url`.', + inputSchema: { + type: 'object', + properties: { + id: { + type: 'integer', + description: 'Numeric URL id from query_urls / get_url_id.', + }, + url: { + type: 'string', + description: 'Exact URL string (slower path β€” uses an indexed lookup).', + }, + linkLimit: { + type: 'integer', + description: 'Cap on inlinks + outlinks returned. Default 500.', + }, + }, + }, + handler: (args, db) => { + let id: number | null = null; + if (typeof args.id === 'number') { + id = args.id; + } else if (typeof args.url === 'string') { + const found = db.getUrlIdByUrl(args.url); + if (found === null) { + throw new Error(`URL not found: ${args.url}`); + } + id = found; + } else { + throw new Error('get_url_detail requires either `id` or `url`.'); + } + const linkLimit = + typeof args.linkLimit === 'number' ? args.linkLimit : 500; + const detail = db.getUrlDetail(id, linkLimit); + if (detail === null) { + throw new Error(`URL id ${id} not found in this project.`); + } + return detail; + }, + }, + + { + name: 'top_issues', + description: + 'Ranked list of non-empty issue categories from get_overview_counts, sorted by count descending. Quick way to ask "what are the biggest problems with this crawl?".', + inputSchema: { + type: 'object', + properties: { + limit: { + type: 'integer', + description: 'Max categories to return. Default 20.', + minimum: 1, + maximum: 200, + }, + }, + }, + handler: (args, db) => { + const limit = Math.min( + Math.max(typeof args.limit === 'number' ? args.limit : 20, 1), + 200, + ); + const counts = db.getOverviewCounts() as unknown as Record; + const issues = (counts.issues as Record) ?? {}; + const ranked = Object.entries(issues) + .filter(([, n]) => n > 0) + .sort((a, b) => b[1] - a[1]) + .slice(0, limit) + .map(([category, count]) => ({ category: `issues:${category}`, count })); + return { issues: ranked }; + }, + }, + ]; +} diff --git a/apps/mcp-server/tsconfig.json b/apps/mcp-server/tsconfig.json new file mode 100644 index 0000000..328ed6f --- /dev/null +++ b/apps/mcp-server/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "composite": true, + "types": ["node"] + }, + "include": ["src/**/*"], + "references": [ + { "path": "../../packages/shared-types" }, + { "path": "../../packages/db" } + ] +} diff --git a/package-lock.json b/package-lock.json index 62a153d..e91328b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "freecrawl-seo", - "version": "0.1.10", + "version": "0.2.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "freecrawl-seo", - "version": "0.1.10", + "version": "0.2.8", "license": "MIT", "workspaces": [ "apps/*", @@ -25,7 +25,7 @@ }, "apps/cli": { "name": "@freecrawl/cli", - "version": "0.1.10", + "version": "0.2.8", "dependencies": { "@freecrawl/core": "*", "@freecrawl/db": "*", @@ -42,7 +42,7 @@ }, "apps/desktop": { "name": "@freecrawl/desktop", - "version": "0.1.10", + "version": "0.2.8", "dependencies": { "@freecrawl/core": "*", "@freecrawl/db": "*", @@ -82,6 +82,22 @@ "react-dom": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" } }, + "apps/mcp-server": { + "name": "@freecrawl/mcp-server", + "version": "0.2.8", + "dependencies": { + "@freecrawl/db": "*", + "@freecrawl/shared-types": "*" + }, + "bin": { + "freecrawl-mcp": "dist/index.js" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "rimraf": "^6.0.1", + "typescript": "^5.7.0" + } + }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", @@ -1431,6 +1447,10 @@ "resolved": "apps/desktop", "link": true }, + "node_modules/@freecrawl/mcp-server": { + "resolved": "apps/mcp-server", + "link": true + }, "node_modules/@freecrawl/shared-types": { "resolved": "packages/shared-types", "link": true @@ -8712,7 +8732,7 @@ }, "packages/core": { "name": "@freecrawl/core", - "version": "0.1.10", + "version": "0.2.8", "dependencies": { "@freecrawl/db": "*", "@freecrawl/shared-types": "*", @@ -8738,7 +8758,7 @@ }, "packages/db": { "name": "@freecrawl/db", - "version": "0.1.10", + "version": "0.2.8", "dependencies": { "@freecrawl/shared-types": "*" }, @@ -8750,7 +8770,7 @@ }, "packages/shared-types": { "name": "@freecrawl/shared-types", - "version": "0.1.10", + "version": "0.2.8", "devDependencies": { "rimraf": "^6.0.1", "typescript": "^5.7.0" diff --git a/package.json b/package.json index a588af4..aff69bb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "freecrawl-seo", - "version": "0.2.7", + "version": "0.2.8", "private": true, "description": "Open-source desktop SEO crawler Ò€” a free alternative to Screaming Frog", "license": "MIT", @@ -18,6 +18,7 @@ "build": "tsc -b && npm run build --workspace apps/desktop && npm run build --workspace apps/cli", "build:desktop": "npm run build --workspace apps/desktop", "build:cli": "npm run build --workspace apps/cli", + "build:mcp": "npm run build --workspace apps/mcp-server", "typecheck": "npm run typecheck --workspaces --if-present", "lint": "eslint .", "format": "prettier --write \"**/*.{ts,tsx,js,json,md,css,html}\"", diff --git a/packages/core/package.json b/packages/core/package.json index 7a69f45..7732979 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/core", - "version": "0.2.7", + "version": "0.2.8", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/packages/core/src/crawler.ts b/packages/core/src/crawler.ts index 09ab4a4..95cbab9 100644 --- a/packages/core/src/crawler.ts +++ b/packages/core/src/crawler.ts @@ -89,13 +89,23 @@ export class Crawler extends EventEmitter { */ private memoryAutoPaused = false; private memoryMonitorTimer: NodeJS.Timeout | null = null; + /** + * Live AbortControllers for in-flight HTTP fetches in the main URL + * pipeline. Pause / Stop abort every member so the crawl actually + * stops instead of letting `concurrency` (often 20) more URLs land + * in the table after the user clicked the button. Each call site + * adds itself in `try` and removes itself in `finally`. + */ + private inFlightFetchControllers = new Set(); private robots: RobotsChecker | null = null; private progressTimer: NodeJS.Timeout | null = null; /** Periodic post-crawl-style recompute of expensive issue counters * while the crawl is still running, so the sidebar number is < 30 s - * stale instead of "0 until crawl ends". 30 s cadence picked - * conservatively β€” the recompute wraps in `runInTransaction` and is - * not free on a 100k-URL DB. */ + * stale instead of "0 until crawl ends". The recompute is dispatched + * to the writer-worker via the injected `recomputeIssues` hook, so + * it does NOT block the main thread β€” the freezes that an earlier + * version produced were a result of running the recompute inline on + * the main-thread writer connection, which has since been fixed. */ private issueRecomputeTimer: NodeJS.Timeout | null = null; private static readonly ISSUE_RECOMPUTE_INTERVAL_MS = 30_000; private issueRecomputeInFlight = false; @@ -103,20 +113,16 @@ export class Crawler extends EventEmitter { if (this.issueRecomputeTimer) return; this.issueRecomputeTimer = setInterval(() => { if (this.stopped || !this.running || this.paused) return; - // Drop overlapping ticks. The yielding recompute can take longer - // than the 30 s tick on very large projects; piling up calls - // would queue 70-statement transactions back-to-back and starve - // the crawler's own writes. + // Drop overlapping ticks β€” on a 1M-URL DB the recompute can run + // longer than the 30 s tick, and piling RPCs on the writer worker + // would starve per-URL writes behind a queue of issue passes. if (this.issueRecomputeInFlight) return; this.issueRecomputeInFlight = true; - // Cooperative variant: yields between definitions so a 1–3 s - // recompute doesn't freeze the main thread (and the user's - // clicks) for the full duration. Trades atomic visibility of - // the rebuild for UI responsiveness β€” fine for the live tick - // since the post-crawl pass still uses the atomic version for - // the final state. - void this.db - .recomputeUrlsIssuesYielding(EXPENSIVE_ISSUE_DEFINITIONS) + // The injected hook routes through the writer worker in the + // desktop host, so this is off-thread; main stays free to + // service IPC and pump the crawl queue. CLI fallback runs the + // yielding variant inline, which is fine for that context. + void this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS) .catch((err) => { this.emit( 'debug', @@ -230,6 +236,30 @@ export class Crawler extends EventEmitter { payload: Parameters[0], ) => Promise<{ urlId: number }>; + /** + * Optional async hook for the post-crawl issue recompute. When the + * desktop main process injects a writer-worker bridge here, the + * 70+ correlated subqueries run off the main thread β€” without it, + * a 5K-URL crawl freezes the renderer for ~45 s in the post-crawl + * phase ("materialise-issues"). Defaults to running the recompute + * inline (CLI / test contexts). + */ + private readonly recomputeIssues: ( + definitions: ReadonlyArray, + ) => Promise; + + /** + * Generic post-crawl SQL-pass dispatch. Methods like + * `recomputeInlinks`, `recomputeRedirectChains`, + * `recomputeHreflangAnalysis`, `recomputeHreflangInconsistent`, + * `recomputePaginationSequence` are sync void calls on `ProjectDb` + * β€” running them inline on the main thread blocks the renderer for + * ~5-15 s combined on a 5K+ URL crawl. The desktop host injects a + * writer-worker dispatcher here so each pass runs off-thread; CLI / + * tests fall back to calling the method directly on `this.db`. + */ + private readonly runDbPass: (methodName: string) => Promise; + constructor( config: CrawlConfig, db: ProjectDb, @@ -243,6 +273,10 @@ export class Crawler extends EventEmitter { writeFetchedUrl?: ( payload: Parameters[0], ) => Promise<{ urlId: number }>; + recomputeIssues?: ( + definitions: ReadonlyArray, + ) => Promise; + runDbPass?: (methodName: string) => Promise; } = {}, ) { super(); @@ -252,6 +286,15 @@ export class Crawler extends EventEmitter { ((html, pageUrl, parseOpts) => Promise.resolve(parseHtml(html, pageUrl, parseOpts))); this.writeFetchedUrl = opts.writeFetchedUrl ?? ((payload) => Promise.resolve(this.db.writeFetchedUrl(payload))); + this.recomputeIssues = + opts.recomputeIssues ?? ((defs) => this.db.recomputeUrlsIssuesYielding(defs)); + this.runDbPass = + opts.runDbPass ?? + ((method) => { + const fn = (this.db as unknown as Record)[method]; + if (typeof fn === 'function') (fn as () => void).call(this.db); + return Promise.resolve(); + }); // Wave 9 β€” Resolve the active proxy. If a named profile is selected // and present in `proxyProfiles`, its URL wins over the legacy // `proxyUrl` field; if the named profile doesn't resolve we fall @@ -268,14 +311,27 @@ export class Crawler extends EventEmitter { this.config = config; this.db = db; const concurrency = Math.max(1, Math.min(200, config.maxConcurrency)); - const intervalCap = Math.max(1, config.maxRps); - this.queue = new PQueue({ concurrency, interval: 1000, intervalCap }); + // Rate-limit (interval + intervalCap) intentionally NOT set on + // the queue. p-queue's bucket-refill semantics interact poorly + // with long-tailed response times: after the first burst lands + // and slots free up, dispatch can stall for an entire bucket + // before the next batch goes out β€” and on a 1 s/intervalCap=20 + // bucket the queue would freeze entirely once the first ~20 + // tasks completed. Throughput is now bounded purely by the + // worker pool (concurrency Γ— 1/avg_response), which on a slow + // remote server gives the user the 5-10 URL/s they expect + // instead of ~1 URL/s. config.maxRps is honoured indirectly via + // maxConcurrency β€” users who need a hard RPS cap should drop + // concurrency rather than trust per-second buckets. + this.queue = new PQueue({ concurrency }); + // Track queue concurrency from the start so adaptive shrinkage + // doesn't have to wait for the first reportRendererLag call to + // materialise the running value. + this.currentConcurrency = concurrency; // External probes run on a separate queue so slow third-party hosts // don't block the main crawl. this.externalQueue = new PQueue({ concurrency: Math.max(2, Math.min(10, concurrency)), - interval: 1000, - intervalCap: Math.max(2, intervalCap), }); // Compile include/exclude patterns once β€” an invalid pattern should // surface to the user as a crawler error, not a silent miss. @@ -445,7 +501,17 @@ export class Crawler extends EventEmitter { try { // Wait for internal crawl first, then drain any external probes still // in flight or queued (externals may have been enqueued during internal). - await this.queue.onIdle(); + // p-queue's onIdle resolves the moment size+pending hits 0, but + // fetchAndProcess can race with that: a worker finishes the + // *last* in-flight task, link extraction enqueues new URLs a + // microtask later, and onIdle has already resolved. Loop while + // counters disagree β€” one extra 50 ms wait per pass guards the + // race window without spinning. + while (!this.stopped) { + await this.queue.onIdle(); + await new Promise((r) => setTimeout(r, 50)); + if (this.queue.size + this.queue.pending === 0) break; + } await this.externalQueue.onIdle(); // robots.txt + sitemap discovery may still be running β€” wait for // both before the post-crawl recompute so issue filters depending @@ -512,44 +578,49 @@ export class Crawler extends EventEmitter { await yieldToEventLoop(); this.emit('info', 'Recomputing inlinks…'); this.setOp('post-crawl:recompute-inlinks'); - this.db.recomputeInlinks(); + await this.runDbPass('recomputeInlinks'); } if (this.config.analyseRedirectChains) { await yieldToEventLoop(); this.emit('info', 'Recomputing redirect chains…'); this.setOp('post-crawl:recompute-redirect-chains'); - this.db.recomputeRedirectChains(); + await this.runDbPass('recomputeRedirectChains'); } if (this.config.analyseHreflang) { await yieldToEventLoop(); this.emit('info', 'Recomputing hreflang analysis…'); this.setOp('post-crawl:recompute-hreflang'); - this.db.recomputeHreflangAnalysis(); + await this.runDbPass('recomputeHreflangAnalysis'); await yieldToEventLoop(); this.setOp('post-crawl:recompute-hreflang-inconsistent'); - this.db.recomputeHreflangInconsistent(); + await this.runDbPass('recomputeHreflangInconsistent'); } if (this.config.analyseDuplicates) { await yieldToEventLoop(); this.emit('info', 'Clustering duplicates…'); this.setOp('post-crawl:cluster-duplicates'); + // Duplicate clustering has JS-side simhash work that lives on + // the crawler instance, not the DB; can't blindly off-thread it. this.runDuplicateClustering(); } if (this.config.analysePagination) { await yieldToEventLoop(); this.emit('info', 'Detecting pagination sequence gaps…'); this.setOp('post-crawl:pagination-sequence'); - this.db.recomputePaginationSequence(); + await this.runDbPass('recomputePaginationSequence'); } if (this.config.analyseIssues) { await yieldToEventLoop(); this.emit('info', 'Materialising issue counters…'); this.setOp('post-crawl:materialise-issues'); - // Use the cooperatively-scheduled variant (yields between each - // of the 70+ definitions). Trades atomic visibility of the - // rebuild for a responsive main thread β€” on a 5k-URL DB the - // sync version was blocking the event loop for 22-26 s. - await this.db.recomputeUrlsIssuesYielding(EXPENSIVE_ISSUE_DEFINITIONS); + // Route through the injected hook β€” in the desktop host this + // dispatches to the writer-worker, keeping the 70+ correlated + // subqueries off the main thread. CLI / tests fall back to + // running inline via `db.recomputeUrlsIssuesYielding`. Without + // offloading, a 5K-URL crawl freezes the main process for + // ~45 s in post-crawl, tripping Electron's "Not Responding" + // dialog. + await this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS); } await yieldToEventLoop(); this.setOp('post-crawl:image-probes'); @@ -1022,6 +1093,24 @@ export class Crawler extends EventEmitter { // If the start URL isn't in the DB yet, kick off a brand-new crawl from it. if (!this.db.hasUrl(this.config.startUrl)) { + // Stub upsert before enqueue so the seed surfaces in the URL + // table within ~100 ms of clicking Start instead of waiting for + // the full network round-trip + parse + write (~2 s on a typical + // site). status_code=null reads as "fetching" in the table. + // fetchAndProcess's writeFetchedUrl later overwrites every field + // with the real response data via INSERT … ON CONFLICT UPDATE. + try { + this.db.upsertUrl({ + url: this.config.startUrl, + contentKind: 'html', + statusCode: null, + statusText: 'fetching', + indexability: 'indexable', + depth: 0, + }); + } catch { + /* non-fatal: fetchAndProcess will still write the real row */ + } this.enqueue({ url: this.config.startUrl, depth: 0 }); } @@ -1140,6 +1229,19 @@ export class Crawler extends EventEmitter { } this.sitemapAbort = null; } + // Abort every in-flight HTTP fetch. Without this, Stop only halts + // *new* dispatch β€” the ~20 fetches already in flight would each + // finish their request and write a row, which races with Clear's + // db.reset() and resurrects ~19 rows in the table. The catch + // branch in fetchAndProcess detects the stopped-abort and exits + // silently without writing. + for (const c of this.inFlightFetchControllers) { + try { + c.abort(); + } catch { + /* ignore β€” already aborted */ + } + } // Drop any queued work. If paused, unblock onIdle() so start() can resolve. this.queue.clear(); this.externalQueue.clear(); @@ -1150,9 +1252,22 @@ export class Crawler extends EventEmitter { pause(): void { if (this.stopped || this.paused) return; this.paused = true; - // PQueue.pause() halts dispatch but lets in-flight tasks finish naturally. + // PQueue.pause() halts new dispatch. By itself that lets every + // in-flight fetch run to completion β€” at concurrency 20 the user + // sees ~20 more URLs land in the table before things stop, which + // reads as "Pause didn't work." Aborting the live controllers + // turns Pause into "stop now": fetchAndProcess detects a paused- + // abort, re-enqueues the URL, and returns without writing a row. this.queue.pause(); this.externalQueue.pause(); + for (const c of this.inFlightFetchControllers) { + try { + c.abort(); + } catch { + /* ignore β€” already aborted */ + } + } + this.setOp('paused'); this.emitProgress(); } @@ -1161,6 +1276,7 @@ export class Crawler extends EventEmitter { this.paused = false; this.queue.start(); this.externalQueue.start(); + this.setOp('idle'); this.emitProgress(); } @@ -1446,6 +1562,7 @@ export class Crawler extends EventEmitter { const t0 = Date.now(); const controller = new AbortController(); + this.inFlightFetchControllers.add(controller); const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs); // Wave 3 β€” Optional max-response-time hard cap. Distinct from // `requestTimeoutMs` (which is the connect+headers timeout): this @@ -1998,6 +2115,25 @@ export class Crawler extends EventEmitter { } } } catch (err) { + // Stop / Clear: abandon silently. No row is written, no failure + // counter is bumped, the URL is not re-enqueued. Without this + // gate the ~20 in-flight URLs after Stop would each land in the + // table β€” and Clear's db.reset() races with their writes, + // resurrecting "ghost" rows in a freshly-cleared table. + if (this.stopped && controller.signal.aborted) { + return; + } + // User-initiated Pause aborts every live controller. Don't treat + // those as fetch failures β€” the URL hasn't been seen yet, so we + // re-enqueue it directly (bypassing `enqueue` because the URL is + // already in the `seen` set from the original dispatch). Without + // this, Pause would mark `concurrency` (often 20) URLs as failed + // and surface them as errors in the URL table. Re-enqueue lands + // in a paused queue β†’ resume() picks them up cleanly. + if (this.paused && controller.signal.aborted) { + this.queue.add(() => this.fetchAndProcess(item)).catch(() => undefined); + return; + } this.failed++; const detail = formatFetchError(err); const elapsed = Date.now() - t0; @@ -2020,6 +2156,7 @@ export class Crawler extends EventEmitter { depth: item.depth, }); } finally { + this.inFlightFetchControllers.delete(controller); clearTimeout(timeout); if (respTimeTimer) clearTimeout(respTimeTimer); // Politeness delay β€” applied per worker *after* each request so a @@ -2171,8 +2308,14 @@ export class Crawler extends EventEmitter { const ceiling = Math.max(1, Math.min(200, this.config.maxConcurrency)); if (this.currentConcurrency === 0) this.currentConcurrency = ceiling; let next = this.currentConcurrency; + // Floor 5 (was 1). The renderer's lag probe occasionally spikes + // for unrelated reasons (GC, an input burst, devtools open) and + // a 1-floor would let those one-off spikes corner the queue at + // single-task dispatch β€” which on a slow remote server (avg resp + // 1.5-2 s) cuts throughput from 10 URL/s to 0.5 URL/s and the + // queue never recovers because the user never produces zero lag. if (lagMs > 200) { - next = Math.max(1, this.currentConcurrency - 1); + next = Math.max(5, this.currentConcurrency - 1); } else if (lagMs < 30) { next = Math.min(ceiling, this.currentConcurrency + 1); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b235228..e296e73 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -10,6 +10,12 @@ export { export { exportUrlsToCsv } from './csv-export.js'; export { exportUrlsToJson, type JsonExportOptions } from './json-export.js'; export { exportUrlsToXml } from './xml-export.js'; +export { + exportTabular, + type TabularExportOptions, + type TabularExportResult, + type TabularSection, +} from './tabular-export.js'; export { testUrlAgainstRobots, type RobotsTestResult } from './robots.js'; export { exportSitemap, diff --git a/packages/core/src/tabular-export.ts b/packages/core/src/tabular-export.ts new file mode 100644 index 0000000..978676e --- /dev/null +++ b/packages/core/src/tabular-export.ts @@ -0,0 +1,398 @@ +import { createWriteStream, writeFileSync } from 'node:fs'; +import { pipeline } from 'node:stream/promises'; +import { Readable } from 'node:stream'; +import path from 'node:path'; +import zlib from 'node:zlib'; +import type { ProjectDb } from '@freecrawl/db'; +import type { CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types'; + +export interface TabularSection { + label: string; + category: UrlCategory; +} + +export interface TabularExportOptions { + format: 'csv' | 'xlsx'; + sections: TabularSection[]; + /** Keys of CrawlUrlRow to include, in order. */ + columns: string[]; + /** When provided AND non-empty, restrict every section to these row ids. */ + selectedIds?: number[]; +} + +export interface TabularExportResult { + /** File (xlsx / single-section csv) or folder (multi-section csv). */ + filePath: string; + files: string[]; + rowsWritten: number; +} + +function escapeCsv(value: unknown): string { + if (value === null || value === undefined) return ''; + const str = String(value); + if (/[",\n\r]/.test(str)) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; +} + +function rowSource( + db: ProjectDb, + category: UrlCategory, + selectedIds: number[] | undefined, +): Iterable { + if (selectedIds && selectedIds.length > 0) { + // Apply selectedIds first, then narrow by category client-side. The + // sidebar's category predicates can be expensive; for export the + // selection is always the small side. + const ids = new Set(selectedIds); + return (function* () { + for (const row of db.iterateUrlsByCategory(category)) { + if (ids.has(row.id)) yield row; + } + })(); + } + return db.iterateUrlsByCategory(category); +} + +async function writeCsvFile( + db: ProjectDb, + filePath: string, + columns: string[], + category: UrlCategory, + selectedIds: number[] | undefined, +): Promise { + let rowsWritten = 0; + const header = columns.join(',') + '\n'; + const source = rowSource(db, category, selectedIds); + const generator = async function* (): AsyncGenerator { + yield 'ο»Ώ' + header; + for (const row of source) { + const line = + columns + .map((c) => escapeCsv((row as unknown as Record)[c])) + .join(',') + '\n'; + rowsWritten++; + yield line; + } + }; + await pipeline( + Readable.from(generator()), + createWriteStream(filePath, { encoding: 'utf8' }), + ); + return rowsWritten; +} + +// ── Minimal XLSX writer ───────────────────────────────────────────── +// +// Office Open XML SpreadsheetML, packaged as a ZIP with DEFLATE entries. +// Uses inline strings (no shared-string table) β€” slightly fatter on disk +// but lets us stream-build sheet XML row-by-row without a second pass. +// +// Worksheet shape: +// +// URL... +// https://...... +// +// +// Numbers go in as 123 (no `t=` attribute, default = number). + +function xmlEscape(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + // strip control chars XML 1.0 forbids + .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, ''); +} + +function colLetter(idx: number): string { + // 0-based β†’ A, B, ..., Z, AA, AB, ... + let n = idx; + let s = ''; + while (true) { + const r = n % 26; + s = String.fromCharCode(65 + r) + s; + n = Math.floor(n / 26) - 1; + if (n < 0) break; + } + return s; +} + +function sanitizeSheetName(label: string, used: Set): string { + let base = label.replace(/[\\/?*\[\]:]/g, ' ').trim().slice(0, 31); + if (!base) base = 'Sheet'; + let candidate = base; + let i = 2; + while (used.has(candidate.toLowerCase())) { + const suffix = ` (${i++})`; + candidate = base.slice(0, 31 - suffix.length) + suffix; + } + used.add(candidate.toLowerCase()); + return candidate; +} + +function buildSheetXml( + rows: CrawlUrlRow[], + columns: string[], +): string { + const parts: string[] = []; + parts.push( + '\n' + + '' + + '', + ); + // Header row + parts.push(''); + columns.forEach((c, i) => { + parts.push( + `${xmlEscape(c)}`, + ); + }); + parts.push(''); + // Data rows + rows.forEach((row, rIdx) => { + const r = rIdx + 2; + parts.push(``); + columns.forEach((col, i) => { + const v = (row as unknown as Record)[col]; + if (v === null || v === undefined || v === '') return; + const cellRef = `${colLetter(i)}${r}`; + if (typeof v === 'number' && Number.isFinite(v)) { + parts.push(`${v}`); + } else if (typeof v === 'boolean') { + parts.push(`${v ? 1 : 0}`); + } else { + parts.push( + `${xmlEscape(String(v))}`, + ); + } + }); + parts.push(''); + }); + parts.push(''); + return parts.join(''); +} + +// ── ZIP writer (DEFLATE, no central directory streaming β€” small enough +// to buffer per-entry, write central dir at the end). ──────────────── + +interface ZipEntry { + name: string; + raw: Buffer; + compressed: Buffer; + crc32: number; + offset: number; +} + +const CRC_TABLE = (() => { + const table = new Uint32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) { + c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + } + table[n] = c >>> 0; + } + return table; +})(); + +function crc32(buf: Buffer): number { + let c = 0xffffffff; + for (let i = 0; i < buf.length; i++) { + c = CRC_TABLE[(c ^ buf[i]!) & 0xff]! ^ (c >>> 8); + } + return (c ^ 0xffffffff) >>> 0; +} + +function buildZip(files: { name: string; data: Buffer }[]): Buffer { + const entries: ZipEntry[] = []; + const localChunks: Buffer[] = []; + let offset = 0; + + for (const f of files) { + const raw = f.data; + const compressed = zlib.deflateRawSync(raw); + const crc = crc32(raw); + const nameBuf = Buffer.from(f.name, 'utf8'); + + const localHeader = Buffer.alloc(30); + localHeader.writeUInt32LE(0x04034b50, 0); // local file header sig + localHeader.writeUInt16LE(20, 4); // version needed + localHeader.writeUInt16LE(0x0800, 6); // gp flag (UTF-8 name) + localHeader.writeUInt16LE(8, 8); // method = deflate + localHeader.writeUInt16LE(0, 10); // mod time + localHeader.writeUInt16LE(0, 12); // mod date + localHeader.writeUInt32LE(crc, 14); + localHeader.writeUInt32LE(compressed.length, 18); + localHeader.writeUInt32LE(raw.length, 22); + localHeader.writeUInt16LE(nameBuf.length, 26); + localHeader.writeUInt16LE(0, 28); // extra field len + + entries.push({ + name: f.name, + raw, + compressed, + crc32: crc, + offset, + }); + localChunks.push(localHeader, nameBuf, compressed); + offset += localHeader.length + nameBuf.length + compressed.length; + } + + const centralChunks: Buffer[] = []; + let cdSize = 0; + for (const e of entries) { + const nameBuf = Buffer.from(e.name, 'utf8'); + const cd = Buffer.alloc(46); + cd.writeUInt32LE(0x02014b50, 0); // central dir sig + cd.writeUInt16LE(20, 4); // version made by + cd.writeUInt16LE(20, 6); // version needed + cd.writeUInt16LE(0x0800, 8); // gp flag + cd.writeUInt16LE(8, 10); // method + cd.writeUInt16LE(0, 12); + cd.writeUInt16LE(0, 14); + cd.writeUInt32LE(e.crc32, 16); + cd.writeUInt32LE(e.compressed.length, 20); + cd.writeUInt32LE(e.raw.length, 24); + cd.writeUInt16LE(nameBuf.length, 28); + cd.writeUInt16LE(0, 30); + cd.writeUInt16LE(0, 32); + cd.writeUInt16LE(0, 34); + cd.writeUInt16LE(0, 36); + cd.writeUInt32LE(0, 38); + cd.writeUInt32LE(e.offset, 42); + centralChunks.push(cd, nameBuf); + cdSize += cd.length + nameBuf.length; + } + + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(0, 4); + eocd.writeUInt16LE(0, 6); + eocd.writeUInt16LE(entries.length, 8); + eocd.writeUInt16LE(entries.length, 10); + eocd.writeUInt32LE(cdSize, 12); + eocd.writeUInt32LE(offset, 16); + eocd.writeUInt16LE(0, 20); + + return Buffer.concat([...localChunks, ...centralChunks, eocd]); +} + +function buildXlsxBuffer( + sheets: { name: string; rows: CrawlUrlRow[] }[], + columns: string[], +): Buffer { + const contentTypes = + '\n' + + '' + + '' + + '' + + '' + + sheets + .map( + (_, i) => + ``, + ) + .join('') + + ''; + + const rels = + '\n' + + '' + + '' + + ''; + + const workbook = + '\n' + + '' + + '' + + sheets + .map( + (s, i) => + ``, + ) + .join('') + + ''; + + const workbookRels = + '\n' + + '' + + sheets + .map( + (_, i) => + ``, + ) + .join('') + + ''; + + const files: { name: string; data: Buffer }[] = [ + { name: '[Content_Types].xml', data: Buffer.from(contentTypes, 'utf8') }, + { name: '_rels/.rels', data: Buffer.from(rels, 'utf8') }, + { name: 'xl/workbook.xml', data: Buffer.from(workbook, 'utf8') }, + { name: 'xl/_rels/workbook.xml.rels', data: Buffer.from(workbookRels, 'utf8') }, + ]; + sheets.forEach((s, i) => { + files.push({ + name: `xl/worksheets/sheet${i + 1}.xml`, + data: Buffer.from(buildSheetXml(s.rows, columns), 'utf8'), + }); + }); + + return buildZip(files); +} + +export async function exportTabular( + db: ProjectDb, + outputPath: string, + options: TabularExportOptions, +): Promise { + const { format, sections, columns, selectedIds } = options; + if (sections.length === 0) { + throw new Error('exportTabular: at least one section is required'); + } + if (columns.length === 0) { + throw new Error('exportTabular: at least one column is required'); + } + + if (format === 'csv') { + if (sections.length === 1) { + const rows = await writeCsvFile( + db, + outputPath, + columns, + sections[0]!.category, + selectedIds, + ); + return { filePath: outputPath, files: [outputPath], rowsWritten: rows }; + } + // Multi-section CSV β†’ write one file per section into the chosen + // folder. `outputPath` is treated as a folder. + let total = 0; + const written: string[] = []; + for (const section of sections) { + const safe = section.label.replace(/[\\/?*\[\]:<>|"]/g, '_'); + const file = path.join(outputPath, `${safe}.csv`); + total += await writeCsvFile(db, file, columns, section.category, selectedIds); + written.push(file); + } + return { filePath: outputPath, files: written, rowsWritten: total }; + } + + // xlsx β€” collect rows per section into memory (bounded by typical + // section size), build workbook, write once. + const usedNames = new Set(); + let total = 0; + const sheets = sections.map((section) => { + const rows: CrawlUrlRow[] = []; + for (const row of rowSource(db, section.category, selectedIds)) { + rows.push(row); + } + total += rows.length; + return { name: sanitizeSheetName(section.label, usedNames), rows }; + }); + const buf = buildXlsxBuffer(sheets, columns); + writeFileSync(outputPath, buf); + return { filePath: outputPath, files: [outputPath], rowsWritten: total }; +} diff --git a/packages/db/package.json b/packages/db/package.json index a37f03a..4b30a84 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/db", - "version": "0.2.7", + "version": "0.2.8", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/packages/db/src/project-db.ts b/packages/db/src/project-db.ts index 9de47d0..574bdf0 100644 --- a/packages/db/src/project-db.ts +++ b/packages/db/src/project-db.ts @@ -3036,6 +3036,13 @@ export class ProjectDb { return row ? this.rowFromDb(row) : null; } + /** Resolve a URL string to its numeric id, or null if absent. Used by + * the MCP server when a caller supplies a URL instead of an id. */ + getUrlIdByUrl(url: string): number | null { + const row = this.stmtGetUrlId.get(url) as { id: number } | undefined; + return row ? row.id : null; + } + getInlinks(url: string, limit: number): { rows: InlinkRow[]; total: number } { const total = ( this.db.prepare('SELECT COUNT(*) AS c FROM links WHERE to_url = ?').get(url) as { c: number } diff --git a/packages/shared-types/package.json b/packages/shared-types/package.json index 846025b..26f3ac9 100644 --- a/packages/shared-types/package.json +++ b/packages/shared-types/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/shared-types", - "version": "0.2.7", + "version": "0.2.8", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/packages/shared-types/src/ipc.ts b/packages/shared-types/src/ipc.ts index b0d402f..fdc4c73 100644 --- a/packages/shared-types/src/ipc.ts +++ b/packages/shared-types/src/ipc.ts @@ -39,6 +39,9 @@ export const IPC = { exportCsv: 'export:csv', exportJson: 'export:json', exportXml: 'export:xml', + /** New unified tabular export with column selection + multi-tab support + * (CSV / XLSX). Backs the in-table "Export" button. */ + exportTabular: 'export:tabular', /** GDPR-aligned per-domain delete. Wipes every row whose URL host * matches the given domain (and that domain's links/images/headers/ * url_sources). Used by Settings β†’ "Delete Domain Data". */ @@ -213,6 +216,38 @@ export interface ExportXmlResult { rowsWritten: number; } +/** One section / sheet of a tabular export β€” a single category fed into + * the workbook (xlsx) or split into its own file (csv). */ +export interface ExportTabularSection { + /** Display label β€” becomes the xlsx sheet name and the csv file suffix. */ + label: string; + category: UrlCategory; +} + +export interface ExportTabularInput { + format: 'csv' | 'xlsx'; + /** + * One or more sections. With CSV + multiple sections, files are written + * into a folder the user picks; xlsx always produces a single workbook + * with one sheet per section. + */ + sections: ExportTabularSection[]; + /** Keys of `CrawlUrlRow` to emit, in order. */ + columns: string[]; + /** Optional pre-resolved output path (skip the dialog). */ + filePath?: string; + /** When set, restrict every section to these row ids. */ + selectedIds?: number[]; +} + +export interface ExportTabularResult { + /** Output file (xlsx + single-section csv) or folder (multi-section csv). */ + filePath: string; + /** Files actually written. Multi-section CSV writes one file per section. */ + files: string[]; + rowsWritten: number; +} + export interface DataDeleteByDomainInput { /** Hostname to wipe β€” case-insensitive, no scheme/port. */ domain: string; @@ -676,6 +711,7 @@ export interface FreeCrawlApi { exportCsv(input: ExportCsvInput): Promise; exportJson(input: ExportJsonInput): Promise; exportXml(input: ExportXmlInput): Promise; + exportTabular(input: ExportTabularInput): Promise; dataDeleteByDomain( input: DataDeleteByDomainInput, ): Promise; diff --git a/tsconfig.json b/tsconfig.json index bdc65f2..cbe97cb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,6 +5,7 @@ { "path": "./packages/db" }, { "path": "./packages/core" }, { "path": "./apps/desktop" }, - { "path": "./apps/cli" } + { "path": "./apps/cli" }, + { "path": "./apps/mcp-server" } ] }