diff --git a/FreeCrawl-SEO-Tool-Start.bat b/FreeCrawl-SEO-Tool-Start.bat index a575448..8da4dc1 100644 --- a/FreeCrawl-SEO-Tool-Start.bat +++ b/FreeCrawl-SEO-Tool-Start.bat @@ -71,7 +71,7 @@ if not exist "packages\core\dist\index.js" set NEED_BUILD=1 if "!NEED_BUILD!"=="1" goto DO_BUILD echo [OK] Paylasilan paketler hazir. echo. -goto LAUNCH +goto CHECK_DESKTOP_BUILD :DO_BUILD echo ------------------------------------------------------------ @@ -82,15 +82,47 @@ if errorlevel 1 goto ERR_BUILD echo [OK] Paketler hazir. echo. +REM ---- 4b) Desktop uygulamasini production build et ------------------------- +REM Dev mod (electron-vite dev) ilk acilista 1700+ modulu on-demand transform +REM ediyor ve 20-30 sn cold start suruyor. Production build'de tum moduller +REM tek bir bundle'a derlenmis oldugu icin acilis 1-2 sn'ye iniyor. +:CHECK_DESKTOP_BUILD +set NEED_DESKTOP_BUILD=0 +if not exist "apps\desktop\out\main\index.js" set NEED_DESKTOP_BUILD=1 +if not exist "apps\desktop\out\preload\index.js" set NEED_DESKTOP_BUILD=1 +if not exist "apps\desktop\out\renderer\index.html" set NEED_DESKTOP_BUILD=1 + +if "!NEED_DESKTOP_BUILD!"=="1" goto DO_DESKTOP_BUILD +echo [OK] Desktop bundle hazir. +echo. +goto LAUNCH + +:DO_DESKTOP_BUILD +echo ------------------------------------------------------------ +echo Desktop uygulamasi build ediliyor (production)... +echo Bu islem yaklasik 10-15 sn surer ve sadece ilk calistirmada +echo (veya kod degisikligi sonrasi) yeniden yapilir. +echo ------------------------------------------------------------ +call npm --workspace apps/desktop run build +if errorlevel 1 goto ERR_BUILD +echo [OK] Desktop bundle hazir. +echo. + REM ---- 5) Uygulamayi baslat ------------------------------------------------- :LAUNCH echo ============================================================ -echo FreeCrawl SEO Tool baslatiliyor (npm run dev)... +echo FreeCrawl SEO Tool baslatiliyor (production)... echo Bu pencereyi KAPATMAYIN - uygulamanin yasam dongusu buna bagli. echo ============================================================ echo. -call npm run dev +REM `node:sqlite` Node 24'te kararli ama hala "experimental" bayragi tasiyor +REM ve modul yuklenirken ExperimentalWarning basiyor. NODE_NO_WARNINGS=1 +REM ile bu kozmetik uyariyi sustururuz; gercek hatalar (TypeError vs.) +REM yine konsola yazilir. +set NODE_NO_WARNINGS=1 + +call npm --workspace apps/desktop run start echo. echo ------------------------------------------------------------ diff --git a/apps/desktop/electron.vite.config.ts b/apps/desktop/electron.vite.config.ts index 294d245..eedcc1b 100644 --- a/apps/desktop/electron.vite.config.ts +++ b/apps/desktop/electron.vite.config.ts @@ -7,14 +7,20 @@ export default defineConfig({ plugins: [externalizeDepsPlugin()], build: { rollupOptions: { - // Two entry points: the main process bootstrap, and the - // read-only DB worker that runs in a `worker_threads.Worker`. - // Both compile into `out/main/`; the Worker constructor in - // `db-reader-pool.ts` resolves the worker file relative to - // `__dirname` so it picks up the same bundled module. + // Five entry points: the main process bootstrap and four + // worker-thread bundles (DB reader, DB writer, HTML parser, + // freeze-watchdog). All compile into `out/main/`; each Worker + // constructor resolves its peer file relative to `__dirname` + // so they pick up the same bundle directory. input: { index: resolve(__dirname, 'src/main/index.ts'), 'db-reader-worker': resolve(__dirname, 'src/main/db-reader-worker.ts'), + 'db-writer-worker': resolve(__dirname, 'src/main/db-writer-worker.ts'), + 'parser-worker': resolve(__dirname, 'src/main/parser-worker.ts'), + 'freeze-watchdog-worker': resolve( + __dirname, + 'src/main/freeze-watchdog-worker.ts', + ), }, }, }, diff --git a/apps/desktop/src/main/db-reader-pool.ts b/apps/desktop/src/main/db-reader-pool.ts index edf6902..fd3cfb9 100644 --- a/apps/desktop/src/main/db-reader-pool.ts +++ b/apps/desktop/src/main/db-reader-pool.ts @@ -35,7 +35,22 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); // electron-vite uses the same convention. const WORKER_PATH = path.join(__dirname, 'db-reader-worker.js'); -const REQUEST_TIMEOUT_MS = 15_000; +// 30 s default leaves headroom for SQLite read contention during the +// post-crawl recompute phase (recomputeUrlsIssues holds a writer +// transaction that can keep reader queries blocked behind earlier +// queued requests for several seconds at a time on large crawls). +// Genuine worker hangs are still surfaced — just with a slightly +// longer detection window than before. +const REQUEST_TIMEOUT_MS = 30_000; +// Heavy aggregate queries (the overview sidebar's 130-counter pass and +// the post-crawl materialiser's counter fan-out) can run past the +// default budget on million-URL projects, so they get their own +// bigger ceiling. +const HEAVY_REQUEST_TIMEOUT_MS = 60_000; +const HEAVY_METHODS = new Set([ + 'getOverviewCounts', + 'getOverviewCountsAsync', +]); const MAX_RESTARTS = 3; const RESTART_WINDOW_MS = 60_000; @@ -55,23 +70,31 @@ interface ResponseMessage { class DbReaderPool { private worker: Worker | null = null; private dbPath: string | null = null; + private freezeWatchdogSab: SharedArrayBuffer | null = null; private nextRequestId = 1; private pending = new Map(); private restartTimes: number[] = []; private terminated = false; - /** Spawn (or respawn) the worker pointed at `dbPath`. Idempotent. */ - init(dbPath: string): void { + /** Spawn (or respawn) the worker pointed at `dbPath`. Idempotent. + * + * `freezeWatchdogSab` — if provided, the worker will tick its + * own heartbeat + publish its current method into this shared + * buffer so the freeze-watchdog thread can detect a stuck + * reader. Pass `null` to disable diagnostics. */ + init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void { if (this.terminated) { throw new Error('DbReaderPool: cannot init after terminate()'); } + this.freezeWatchdogSab = freezeWatchdogSab; if (this.dbPath === dbPath && this.worker !== null) return; this.dbPath = dbPath; this.spawn(); } /** Switch the worker to a different .seoproject file. */ - swap(newPath: string): void { + swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void { + if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab; if (this.dbPath === newPath && this.worker !== null) return; this.failPendingWith('reader-swapped'); this.dbPath = newPath; @@ -108,11 +131,14 @@ class DbReaderPool { if (this.terminated) return Promise.reject(new Error('reader-terminated')); if (!this.worker) return Promise.reject(new Error('reader-not-initialised')); const requestId = this.nextRequestId++; + const timeoutMs = HEAVY_METHODS.has(method) + ? HEAVY_REQUEST_TIMEOUT_MS + : REQUEST_TIMEOUT_MS; return new Promise((resolve, reject) => { const timer = setTimeout(() => { this.pending.delete(requestId); - reject(new Error(`reader-timeout: ${method} > ${REQUEST_TIMEOUT_MS}ms`)); - }, REQUEST_TIMEOUT_MS); + reject(new Error(`reader-timeout: ${method} > ${timeoutMs}ms`)); + }, timeoutMs); this.pending.set(requestId, { resolve: resolve as (v: unknown) => void, reject, @@ -133,7 +159,10 @@ class DbReaderPool { } try { const w = new Worker(WORKER_PATH, { - workerData: { dbPath: this.dbPath }, + workerData: { + dbPath: this.dbPath, + freezeWatchdogSab: this.freezeWatchdogSab, + }, }); w.on('message', (msg: ResponseMessage) => this.handleResponse(msg)); w.on('error', (err) => { diff --git a/apps/desktop/src/main/db-reader-worker.ts b/apps/desktop/src/main/db-reader-worker.ts index 983592c..d350584 100644 --- a/apps/desktop/src/main/db-reader-worker.ts +++ b/apps/desktop/src/main/db-reader-worker.ts @@ -1,5 +1,6 @@ import { parentPort, workerData } from 'node:worker_threads'; import { ProjectDb } from '@freecrawl/db'; +import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js'; /** * Read-only SQLite worker. @@ -29,6 +30,7 @@ if (!parentPort) { interface InitData { dbPath: string; + freezeWatchdogSab?: SharedArrayBuffer | null; } const init = workerData as InitData; @@ -38,6 +40,22 @@ if (!init?.dbPath) { const db = new ProjectDb(init.dbPath, { readOnly: true }); +// Freeze-watchdog plumbing. The reader thread publishes a heartbeat +// every 250 ms and updates `readerOp` to the currently-dispatching +// method name so the watchdog can spot a stuck SQLite query and +// surface "[STALL:READER ... op=getUrlDetail]" in debug.txt. +const watchdog = init.freezeWatchdogSab + ? new FreezeWatchdogSharedState(init.freezeWatchdogSab) + : null; +let watchdogHeartbeatTimer: NodeJS.Timeout | null = null; +if (watchdog) { + watchdog.tickReaderHeartbeat(); + watchdog.setReaderOp('idle'); + watchdogHeartbeatTimer = setInterval(() => { + watchdog.tickReaderHeartbeat(); + }, 250); +} + interface RequestMessage { requestId: number; method: string; @@ -81,6 +99,7 @@ const ALLOWED_METHODS = new Set([ parentPort.on('message', async (msg: RequestMessage) => { if (!msg || typeof msg.requestId !== 'number') return; const { requestId, method, args } = msg; + watchdog?.setReaderOp(method); try { if (!ALLOWED_METHODS.has(method)) { throw new Error(`db-reader-worker: method '${method}' is not whitelisted`); @@ -99,6 +118,8 @@ parentPort.on('message', async (msg: RequestMessage) => { ok: false, error: err instanceof Error ? `${err.name}: ${err.message}` : String(err), }); + } finally { + watchdog?.setReaderOp('idle'); } }); @@ -107,6 +128,7 @@ parentPort.on('message', async (msg: RequestMessage) => { // file handle (which on Windows would block the project from being // reopened until full process exit). parentPort.on('close', () => { + if (watchdogHeartbeatTimer) clearInterval(watchdogHeartbeatTimer); try { db.close(); } catch { diff --git a/apps/desktop/src/main/db-writer-pool.ts b/apps/desktop/src/main/db-writer-pool.ts new file mode 100644 index 0000000..2083436 --- /dev/null +++ b/apps/desktop/src/main/db-writer-pool.ts @@ -0,0 +1,207 @@ +import { Worker } from 'node:worker_threads'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as logger from './logger.js'; + +/** + * Single-worker pool for SQLite writes. Mirrors the structure of + * `db-reader-pool.ts` so the rest of the main process talks to both + * via the same `call(method, args)` shape. + * + * The pool is intentionally a "pool of one": SQLite's writer model + * is single-writer-at-a-time. A second writer worker would just + * serialise behind the first via `busy_timeout`, gaining no + * parallelism. The wrapper exists to: + * 1. Move the synchronous `.run()` calls off the Electron main + * thread so IPC + queue scheduling stay live during writes. + * 2. Give us one centralised place to add batching / rate-limiting + * later (Phase 2 — micro-batching). + */ + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const WORKER_PATH = path.join(__dirname, 'db-writer-worker.js'); + +const REQUEST_TIMEOUT_MS = 60_000; +const MAX_RESTARTS = 3; +const RESTART_WINDOW_MS = 60_000; + +interface PendingRequest { + resolve: (v: unknown) => void; + reject: (e: Error) => void; + timer: ReturnType; +} + +interface ResponseMessage { + requestId: number; + ok: boolean; + result?: unknown; + error?: string; +} + +class DbWriterPool { + private worker: Worker | null = null; + private dbPath: string | null = null; + private freezeWatchdogSab: SharedArrayBuffer | null = null; + private nextRequestId = 1; + private pending = new Map(); + private restartTimes: number[] = []; + private terminated = false; + + init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void { + if (this.terminated) { + throw new Error('DbWriterPool: cannot init after terminate()'); + } + this.freezeWatchdogSab = freezeWatchdogSab; + if (this.dbPath === dbPath && this.worker !== null) return; + this.dbPath = dbPath; + this.spawn(); + } + + swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void { + if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab; + if (this.dbPath === newPath && this.worker !== null) return; + this.failPendingWith('writer-swapped'); + this.dbPath = newPath; + this.spawn(); + } + + isReady(): boolean { + return this.worker !== null && !this.terminated; + } + + call(method: string, args: unknown[] = []): Promise { + if (this.terminated) return Promise.reject(new Error('writer-terminated')); + if (!this.worker) return Promise.reject(new Error('writer-not-initialised')); + const requestId = this.nextRequestId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(requestId); + reject(new Error(`writer-timeout: ${method} > ${REQUEST_TIMEOUT_MS}ms`)); + }, REQUEST_TIMEOUT_MS); + this.pending.set(requestId, { + resolve: resolve as (v: unknown) => void, + reject, + timer, + }); + this.worker!.postMessage({ requestId, method, args }); + }); + } + + async terminate(): Promise { + this.terminated = true; + this.failPendingWith('writer-terminated'); + if (this.worker) { + const w = this.worker; + this.worker = null; + try { + await w.terminate(); + } catch { + /* already gone */ + } + } + } + + private spawn(): void { + if (!this.dbPath) return; + if (this.worker) { + const old = this.worker; + this.worker = null; + void old.terminate().catch(() => undefined); + } + try { + const w = new Worker(WORKER_PATH, { + workerData: { + dbPath: this.dbPath, + freezeWatchdogSab: this.freezeWatchdogSab, + }, + }); + w.on('message', (msg: ResponseMessage) => this.handleResponse(msg)); + w.on('error', (err) => { + logger.log('error', 'main', `db-writer worker error: ${err.message}`); + }); + w.on('exit', (code) => this.handleExit(code)); + this.worker = w; + logger.log('info', 'main', `db-writer worker spawned for ${this.dbPath}`); + } catch (err) { + logger.log( + 'error', + 'main', + `db-writer worker spawn failed: ${err instanceof Error ? err.message : String(err)}`, + ); + this.worker = null; + } + } + + private handleResponse(msg: ResponseMessage): void { + const pending = this.pending.get(msg.requestId); + if (!pending) return; + this.pending.delete(msg.requestId); + clearTimeout(pending.timer); + if (msg.ok) { + pending.resolve(msg.result); + } else { + pending.reject(new Error(msg.error ?? 'writer-unknown-error')); + } + } + + private handleExit(code: number): void { + this.worker = null; + if (this.terminated) return; + if (code === 0) return; + this.failPendingWith('writer-crashed'); + const now = Date.now(); + this.restartTimes = this.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS); + if (this.restartTimes.length >= MAX_RESTARTS) { + logger.log( + 'error', + 'main', + `db-writer worker crashed ${MAX_RESTARTS}× within ${RESTART_WINDOW_MS}ms — giving up. Writes fall back to main-thread DB.`, + ); + return; + } + this.restartTimes.push(now); + logger.log( + 'warn', + 'main', + `db-writer worker exited with code ${code}; respawning (${this.restartTimes.length}/${MAX_RESTARTS})`, + ); + this.spawn(); + } + + private failPendingWith(reason: string): void { + for (const p of this.pending.values()) { + clearTimeout(p.timer); + p.reject(new Error(reason)); + } + this.pending.clear(); + } +} + +export const dbWriterPool = new DbWriterPool(); + +/** + * Helper: try the worker first, fall back to a synchronous main-thread + * call on any worker error. The fallback keeps writes happening even + * when the worker is crashed/restarting, at the cost of running on + * main for that one operation. + */ +export async function callWriterOrFallback( + method: string, + args: unknown[], + fallback: () => T | Promise, +): Promise { + if (!dbWriterPool.isReady()) { + return fallback(); + } + try { + return await dbWriterPool.call(method, args); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.log( + 'warn', + 'main', + `'${method}' fell back to main-thread writer: ${msg}`, + ); + return fallback(); + } +} diff --git a/apps/desktop/src/main/db-writer-worker.ts b/apps/desktop/src/main/db-writer-worker.ts new file mode 100644 index 0000000..91910cd --- /dev/null +++ b/apps/desktop/src/main/db-writer-worker.ts @@ -0,0 +1,129 @@ +import { parentPort, workerData } from 'node:worker_threads'; +import { ProjectDb } from '@freecrawl/db'; +import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js'; + +/** + * Writer worker — owns a SQLite writer connection and runs every + * write the desktop main process used to run synchronously on its + * own event loop. + * + * Why a dedicated thread: + * - `node:sqlite` is synchronous; each `.run()` blocks the JS + * thread until SQLite returns. With 20 concurrent fetches all + * inserting per-URL data, the main thread used to block ~250 ms + * per second on writes alone — visible as IPC stalls. + * - Moving the writer into a worker frees the main loop for IPC + * dispatch and queue scheduling. The worker still serialises + * writes one at a time (SQLite's single-writer model is + * unchanged), but main never feels it. + * + * Two writer connections (this worker + the main process's + * ProjectDb instance, which is still used for some methods that + * weren't worth refactoring) share the same DB file. WAL + + * `busy_timeout` makes SQLite serialise the two cleanly — at the + * small risk of a SQLITE_BUSY error if both fight for the lock + * for longer than the timeout, which we treat as best-effort and + * surface to the caller. + * + * Protocol: + * main → worker: `{ requestId, method, args }` + * worker → main: `{ requestId, ok: true, result }` + * or `{ requestId, ok: false, error: string }` + */ + +if (!parentPort) { + throw new Error('db-writer-worker: must be loaded via worker_threads'); +} + +interface InitData { + dbPath: string; + freezeWatchdogSab?: SharedArrayBuffer | null; +} + +const init = workerData as InitData; +if (!init?.dbPath) { + throw new Error('db-writer-worker: workerData.dbPath required'); +} + +const db = new ProjectDb(init.dbPath); +// `busy_timeout` is set inside ProjectDb's constructor — no per-worker +// override needed. We rely on it to serialise this writer connection +// with the main process's writer instead of throwing SQLITE_BUSY when +// both happen to want the lock at the same moment. + +const watchdog = init.freezeWatchdogSab + ? new FreezeWatchdogSharedState(init.freezeWatchdogSab) + : null; + +interface RequestMessage { + requestId: number; + method: string; + args: unknown[]; +} + +// Whitelist mirrors ProjectDb's write surface. Reading methods stay +// off this list — those go to the read-only worker. +const ALLOWED_METHODS = new Set([ + 'writeFetchedUrl', + 'upsertUrl', + 'insertLinks', + 'insertImages', + 'setUrlHeaders', + 'setUrlSource', + 'setSitemapUrls', + 'setHostCert', + 'setImageSize', + 'setMeta', + 'updateExternalProbe', + 'recomputeInlinks', + 'recomputeRedirectChains', + 'recomputeHreflangAnalysis', + 'recomputeHreflangInconsistent', + 'recomputeDuplicateClusters', + 'recomputePaginationSequence', + 'recomputeUrlsIssues', + 'recomputeUrlsIssuesYielding', + 'checkpointQueue', + 'clearQueueCheckpoint', + 'markUrlForRecrawl', + 'markUrlsForRecrawl', + 'deleteUrl', + 'deleteUrls', + 'reset', + 'deleteByDomain', +]); + +parentPort.on('message', async (msg: RequestMessage) => { + if (!msg || typeof msg.requestId !== 'number') return; + const { requestId, method, args } = msg; + if (watchdog) watchdog.setReaderOp('writer:' + method); + try { + if (!ALLOWED_METHODS.has(method)) { + throw new Error(`db-writer-worker: method '${method}' is not whitelisted`); + } + const fn = (db as unknown as Record)[method]; + if (typeof fn !== 'function') { + throw new Error(`db-writer-worker: method '${method}' missing on ProjectDb`); + } + const out = (fn as (...a: unknown[]) => unknown).apply(db, args ?? []); + const result = out instanceof Promise ? await out : out; + parentPort!.postMessage({ requestId, ok: true, result }); + } catch (err) { + parentPort!.postMessage({ + requestId, + ok: false, + error: err instanceof Error ? `${err.name}: ${err.message}` : String(err), + }); + } finally { + if (watchdog) watchdog.setReaderOp('idle'); + } +}); + +parentPort.on('close', () => { + try { + db.close(); + } catch { + /* already closed */ + } + process.exit(0); +}); diff --git a/apps/desktop/src/main/freeze-watchdog-shared.ts b/apps/desktop/src/main/freeze-watchdog-shared.ts new file mode 100644 index 0000000..13b3681 --- /dev/null +++ b/apps/desktop/src/main/freeze-watchdog-shared.ts @@ -0,0 +1,196 @@ +/** + * Shared SharedArrayBuffer layout for the freeze watchdog. + * + * Three execution contexts read/write to the same SAB: + * 1. The Electron main process (the writer for `mainHB`, counters, + * `mainOp`, and rendererLag forwarded from IPC). + * 2. The db-reader worker thread (writes its own `readerHB` and + * `readerOp`). + * 3. The freeze-watchdog worker thread (reads everything; writes + * nothing — its job is purely observational). + * + * All scalar reads/writes use `Atomics.{load,store}` so the watchdog + * is guaranteed to see the latest value even if the writer is mid- + * function. The op-string regions are written non-atomically (a 256- + * byte memcpy isn't atomic on any CPU) — we accept torn reads there + * because the op string is best-effort context, not a correctness + * boundary; the heartbeat timestamp is what tells us whether a + * thread is alive, and that IS atomic. + * + * Layout (1024 bytes total, generous for future fields): + * [0..32) BigInt64Array(4) — heartbeat timestamps + spare + * [0] main heartbeat (ms since epoch) + * [1] db-reader heartbeat + * [2] last renderer lag-report timestamp + * [3] spare + * [32..96) Int32Array(16) — counters + op-string lengths + * [0] renderer lag (ms) + * [1] crawled + * [2] discovered + * [3] pending + * [4] failed + * [5] main op string byte length + * [6] reader op string byte length + * [7..15] spare + * [96..) Uint8Array — op string bytes (UTF-8) + * [0..256) main op + * [256..512) reader op + */ + +const I64_BYTE_OFFSET = 0; +const I64_LEN = 4; +const I32_BYTE_OFFSET = 32; +const I32_LEN = 16; +const U8_BYTE_OFFSET = 96; + +const MAIN_OP_OFFSET_IN_U8 = 0; +const READER_OP_OFFSET_IN_U8 = 256; +const OP_MAX_BYTES = 255; + +const I64_IDX_MAIN_HB = 0; +const I64_IDX_READER_HB = 1; +const I64_IDX_RENDERER_REPORT_TS = 2; + +const I32_IDX_RENDERER_LAG = 0; +const I32_IDX_CRAWLED = 1; +const I32_IDX_DISCOVERED = 2; +const I32_IDX_PENDING = 3; +const I32_IDX_FAILED = 4; +const I32_IDX_MAIN_OP_LEN = 5; +const I32_IDX_READER_OP_LEN = 6; + +export const SAB_BYTE_LENGTH = 1024; + +export interface CounterPatch { + crawled?: number; + discovered?: number; + pending?: number; + failed?: number; +} + +export interface CounterSnapshot { + crawled: number; + discovered: number; + pending: number; + failed: number; +} + +/** + * Wraps a `SharedArrayBuffer` with typed accessors. Construct one in + * the main process via `FreezeWatchdogSharedState.create()`, then pass + * `state.sab` to each worker via `workerData` so all sides share the + * same memory. + */ +export class FreezeWatchdogSharedState { + private readonly i64: BigInt64Array; + private readonly i32: Int32Array; + private readonly u8: Uint8Array; + + static create(): FreezeWatchdogSharedState { + return new FreezeWatchdogSharedState(new SharedArrayBuffer(SAB_BYTE_LENGTH)); + } + + constructor(public readonly sab: SharedArrayBuffer) { + this.i64 = new BigInt64Array(sab, I64_BYTE_OFFSET, I64_LEN); + this.i32 = new Int32Array(sab, I32_BYTE_OFFSET, I32_LEN); + this.u8 = new Uint8Array(sab, U8_BYTE_OFFSET, sab.byteLength - U8_BYTE_OFFSET); + } + + // ── Writers ─────────────────────────────────────────────────────── + + tickMainHeartbeat(): void { + Atomics.store(this.i64, I64_IDX_MAIN_HB, BigInt(Date.now())); + } + + tickReaderHeartbeat(): void { + Atomics.store(this.i64, I64_IDX_READER_HB, BigInt(Date.now())); + } + + reportRendererLag(lagMs: number): void { + Atomics.store(this.i32, I32_IDX_RENDERER_LAG, lagMs | 0); + Atomics.store(this.i64, I64_IDX_RENDERER_REPORT_TS, BigInt(Date.now())); + } + + setMainOp(op: string): void { + this.writeOpString(op, MAIN_OP_OFFSET_IN_U8, I32_IDX_MAIN_OP_LEN); + } + + setReaderOp(op: string): void { + this.writeOpString(op, READER_OP_OFFSET_IN_U8, I32_IDX_READER_OP_LEN); + } + + updateCounters(c: CounterPatch): void { + if (c.crawled !== undefined) { + Atomics.store(this.i32, I32_IDX_CRAWLED, c.crawled | 0); + } + if (c.discovered !== undefined) { + Atomics.store(this.i32, I32_IDX_DISCOVERED, c.discovered | 0); + } + if (c.pending !== undefined) { + Atomics.store(this.i32, I32_IDX_PENDING, c.pending | 0); + } + if (c.failed !== undefined) { + Atomics.store(this.i32, I32_IDX_FAILED, c.failed | 0); + } + } + + // ── Readers (used by the watchdog worker) ───────────────────────── + + readMainHeartbeatMs(): number { + return Number(Atomics.load(this.i64, I64_IDX_MAIN_HB)); + } + + readReaderHeartbeatMs(): number { + return Number(Atomics.load(this.i64, I64_IDX_READER_HB)); + } + + readRendererReportTsMs(): number { + return Number(Atomics.load(this.i64, I64_IDX_RENDERER_REPORT_TS)); + } + + readRendererLagMs(): number { + return Atomics.load(this.i32, I32_IDX_RENDERER_LAG); + } + + readCounters(): CounterSnapshot { + return { + crawled: Atomics.load(this.i32, I32_IDX_CRAWLED), + discovered: Atomics.load(this.i32, I32_IDX_DISCOVERED), + pending: Atomics.load(this.i32, I32_IDX_PENDING), + failed: Atomics.load(this.i32, I32_IDX_FAILED), + }; + } + + readMainOp(): string { + return this.readOpString(MAIN_OP_OFFSET_IN_U8, I32_IDX_MAIN_OP_LEN); + } + + readReaderOp(): string { + return this.readOpString(READER_OP_OFFSET_IN_U8, I32_IDX_READER_OP_LEN); + } + + // ── Internals ───────────────────────────────────────────────────── + + private writeOpString(op: string, offset: number, lenIdx: number): void { + const bytes = new TextEncoder().encode(op); + const len = Math.min(bytes.length, OP_MAX_BYTES); + // Order matters: write the bytes first, then the length last. + // A reader that races could see the new length before the bytes + // land, so reading old bytes with new length would corrupt; the + // reverse (old length, partial new bytes) is safe — the reader + // reads only `len` bytes from the head, ignoring the tail. + this.u8.set(bytes.subarray(0, len), offset); + Atomics.store(this.i32, lenIdx, len); + } + + private readOpString(offset: number, lenIdx: number): string { + const len = Atomics.load(this.i32, lenIdx); + if (len <= 0) return ''; + const slice = this.u8.slice(offset, offset + Math.min(len, OP_MAX_BYTES)); + try { + return new TextDecoder('utf-8', { fatal: false }).decode(slice); + } catch { + return ''; + } + } +} diff --git a/apps/desktop/src/main/freeze-watchdog-worker.ts b/apps/desktop/src/main/freeze-watchdog-worker.ts new file mode 100644 index 0000000..9ea4536 --- /dev/null +++ b/apps/desktop/src/main/freeze-watchdog-worker.ts @@ -0,0 +1,197 @@ +import { parentPort, workerData } from 'node:worker_threads'; +import * as fs from 'node:fs'; +import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js'; + +/** + * Freeze-watchdog worker. + * + * Lives in its own `worker_thread`, so it has an independent V8 + * isolate and OS thread. Even when the Electron main thread, the + * db-reader thread, or the renderer is fully blocked, this worker + * keeps running and writes evidence to `debug.txt`. + * + * Protocol: + * - The main process owns a `SharedArrayBuffer` that this worker + * reads. There is no two-way IPC between main and watchdog + * during normal operation — IPC would itself stall when main + * stalls, defeating the purpose. + * - We poll the shared state every CHECK_INTERVAL_MS and append + * stall events to `debug.txt` synchronously. + * + * The worker writes nothing else — no console output, no events + * back to main — so its only side effect is the debug file. + */ + +const CHECK_INTERVAL_MS = 250; +const STALL_THRESHOLD_MAIN_MS = 500; +const STALL_THRESHOLD_READER_MS = 1000; +const STALL_THRESHOLD_RENDERER_LAG_MS = 500; +const STALL_THRESHOLD_RENDERER_SILENCE_MS = 1500; +const HEARTBEAT_LOG_INTERVAL_MS = 5000; + +interface InitData { + sab: SharedArrayBuffer; + debugFilePath: string; +} + +if (!parentPort) { + throw new Error('freeze-watchdog-worker: must be loaded via worker_threads'); +} + +const init = workerData as InitData; +if (!init?.sab || !init?.debugFilePath) { + throw new Error('freeze-watchdog-worker: workerData must include sab + debugFilePath'); +} + +const state = new FreezeWatchdogSharedState(init.sab); +const debugPath = init.debugFilePath; + +// Silently swallow append failures — if the disk is full or the +// file is locked by AV, we can't recover, and writing would just +// thrash retries. The watchdog stays useful for the next stall. +function appendLine(line: string): void { + try { + fs.appendFileSync(debugPath, line + '\n'); + } catch { + /* best-effort */ + } +} + +function isoNow(): string { + return new Date().toISOString(); +} + +function sanitizeOp(op: string): string { + // Keep newlines/tabs out of the log line — they'd break grep. + if (!op) return ''; + return op.replace(/[\r\n\t]+/g, ' ').slice(0, 240); +} + +interface StallTracker { + active: boolean; + startTs: number; + startOp: string; +} + +const mainStall: StallTracker = { active: false, startTs: 0, startOp: '' }; +const readerStall: StallTracker = { active: false, startTs: 0, startOp: '' }; +const rendererStall: StallTracker = { active: false, startTs: 0, startOp: '' }; + +let lastHeartbeatLogTs = 0; +let bootLogged = false; + +function check(): void { + const now = Date.now(); + const counters = state.readCounters(); + const mainHb = state.readMainHeartbeatMs(); + const readerHb = state.readReaderHeartbeatMs(); + const rendererTs = state.readRendererReportTsMs(); + const rendererLag = state.readRendererLagMs(); + const mainOp = sanitizeOp(state.readMainOp()); + const readerOp = sanitizeOp(state.readReaderOp()); + + const mainGap = mainHb > 0 ? now - mainHb : 0; + const readerGap = readerHb > 0 ? now - readerHb : 0; + const rendererSilence = rendererTs > 0 ? now - rendererTs : 0; + + // ── Main thread stall ── + if (mainHb > 0 && mainGap > STALL_THRESHOLD_MAIN_MS) { + if (!mainStall.active) { + mainStall.active = true; + mainStall.startTs = now; + mainStall.startOp = mainOp; + appendLine( + `${isoNow()} [STALL:MAIN start gap=${mainGap}ms] op="${mainOp || ''}" ` + + `crawled=${counters.crawled} discovered=${counters.discovered} ` + + `pending=${counters.pending} failed=${counters.failed}`, + ); + } + } else if (mainStall.active) { + const dur = now - mainStall.startTs; + appendLine( + `${isoNow()} [STALL:MAIN end after ${dur}ms] startOp="${mainStall.startOp}" endOp="${mainOp}"`, + ); + mainStall.active = false; + } + + // ── DB reader thread stall ── + if (readerHb > 0 && readerGap > STALL_THRESHOLD_READER_MS) { + if (!readerStall.active) { + readerStall.active = true; + readerStall.startTs = now; + readerStall.startOp = readerOp; + appendLine( + `${isoNow()} [STALL:READER start gap=${readerGap}ms] op="${readerOp || ''}"`, + ); + } + } else if (readerStall.active) { + const dur = now - readerStall.startTs; + appendLine( + `${isoNow()} [STALL:READER end after ${dur}ms] startOp="${readerStall.startOp}" endOp="${readerOp}"`, + ); + readerStall.active = false; + } + + // ── Renderer stall ── + // We treat both an explicit high-lag report AND a long silence + // (renderer hasn't pinged back) as stall signals. The silence + // case catches a frozen renderer that can't even fire its + // setTimeout-based lag probe. + let rendererStalled = false; + let rendererReason = ''; + if (rendererLag > STALL_THRESHOLD_RENDERER_LAG_MS) { + rendererStalled = true; + rendererReason = `lag=${rendererLag}ms`; + } else if (rendererTs > 0 && rendererSilence > STALL_THRESHOLD_RENDERER_SILENCE_MS) { + rendererStalled = true; + rendererReason = `silence=${rendererSilence}ms`; + } + if (rendererStalled) { + if (!rendererStall.active) { + rendererStall.active = true; + rendererStall.startTs = now; + rendererStall.startOp = mainOp; + appendLine( + `${isoNow()} [STALL:RENDERER start ${rendererReason}] last_main_op="${mainOp || ''}" ` + + `crawled=${counters.crawled} pending=${counters.pending}`, + ); + } + } else if (rendererStall.active) { + const dur = now - rendererStall.startTs; + appendLine( + `${isoNow()} [STALL:RENDERER end after ${dur}ms] last_lag=${rendererLag}ms`, + ); + rendererStall.active = false; + } + + // ── Periodic heartbeat (so the file shows the app is alive even + // when nothing has stalled) ── + if (!bootLogged) { + bootLogged = true; + appendLine( + `${isoNow()} [BOOT] freeze-watchdog started ` + + `thresholds: main>${STALL_THRESHOLD_MAIN_MS}ms reader>${STALL_THRESHOLD_READER_MS}ms ` + + `renderer_lag>${STALL_THRESHOLD_RENDERER_LAG_MS}ms renderer_silence>${STALL_THRESHOLD_RENDERER_SILENCE_MS}ms`, + ); + } + if (now - lastHeartbeatLogTs >= HEARTBEAT_LOG_INTERVAL_MS) { + lastHeartbeatLogTs = now; + appendLine( + `${isoNow()} [HEARTBEAT] main_op="${mainOp}" reader_op="${readerOp}" ` + + `crawled=${counters.crawled} discovered=${counters.discovered} ` + + `pending=${counters.pending} renderer_lag=${rendererLag}ms`, + ); + } +} + +setInterval(check, CHECK_INTERVAL_MS); + +// Accept a graceful-shutdown ping from the parent. We don't strictly +// need it (the worker exits with the parent), but logging the close +// lets the user see "watchdog stopped" instead of a silent truncation. +parentPort.on('message', (msg: { type?: string }) => { + if (msg?.type === 'shutdown') { + appendLine(`${isoNow()} [SHUTDOWN] watchdog received shutdown signal`); + process.exit(0); + } +}); diff --git a/apps/desktop/src/main/freeze-watchdog.ts b/apps/desktop/src/main/freeze-watchdog.ts new file mode 100644 index 0000000..9e5def2 --- /dev/null +++ b/apps/desktop/src/main/freeze-watchdog.ts @@ -0,0 +1,139 @@ +import { Worker } from 'node:worker_threads'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + FreezeWatchdogSharedState, + type CounterPatch, +} from './freeze-watchdog-shared.js'; +import * as logger from './logger.js'; + +/** + * Main-process façade for the freeze watchdog. + * + * Owns the SharedArrayBuffer and the worker thread. Exposes a tiny + * imperative API the rest of the main process (and the crawler via + * dependency injection) calls during normal operation: + * + * - `setMainOp("crawl:fetch:" + url)` whenever a top-level operation + * starts on the main thread, so the watchdog has context if the + * thread blocks immediately afterwards. + * - `updateCounters({ crawled, pending })` for the snapshot fields. + * - `reportRendererLag(ms)` from the IPC handler that already + * receives renderer lag samples. + * + * The watchdog itself does the heartbeat tick on a 100 ms timer + * inside `init()` — callers don't have to remember to ping. + */ + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const WORKER_PATH = path.join(__dirname, 'freeze-watchdog-worker.js'); + +const MAIN_HEARTBEAT_INTERVAL_MS = 100; + +class FreezeWatchdog { + private worker: Worker | null = null; + private state: FreezeWatchdogSharedState | null = null; + private heartbeatTimer: NodeJS.Timeout | null = null; + private terminated = false; + + /** + * Spawn the watchdog worker, allocate the SharedArrayBuffer, and + * start the main-thread heartbeat ticker. Idempotent. + */ + init(debugFilePath: string): void { + if (this.terminated) return; + if (this.worker) return; + let state: FreezeWatchdogSharedState; + try { + state = FreezeWatchdogSharedState.create(); + } catch (err) { + logger.log( + 'warn', + 'main', + `freeze-watchdog: SharedArrayBuffer allocation failed (${ + err instanceof Error ? err.message : String(err) + }) — diagnostics disabled.`, + ); + return; + } + try { + this.worker = new Worker(WORKER_PATH, { + workerData: { + sab: state.sab, + debugFilePath, + }, + }); + } catch (err) { + logger.log( + 'warn', + 'main', + `freeze-watchdog worker spawn failed: ${ + err instanceof Error ? err.message : String(err) + } — diagnostics disabled.`, + ); + return; + } + this.state = state; + this.worker.on('error', (err) => { + logger.log('warn', 'main', `freeze-watchdog worker error: ${err.message}`); + }); + this.worker.on('exit', (code) => { + this.worker = null; + if (!this.terminated && code !== 0) { + logger.log( + 'warn', + 'main', + `freeze-watchdog worker exited unexpectedly (code=${code})`, + ); + } + }); + // Heartbeat tick — independent of the rest of the app's setIntervals + // so a crashed timer elsewhere doesn't silently disable detection. + this.heartbeatTimer = setInterval(() => { + this.state?.tickMainHeartbeat(); + }, MAIN_HEARTBEAT_INTERVAL_MS); + logger.log('info', 'main', `freeze-watchdog started → ${debugFilePath}`); + } + + /** Pass-through to the shared buffer so workers can attach. */ + get sharedBuffer(): SharedArrayBuffer | null { + return this.state?.sab ?? null; + } + + setMainOp(op: string): void { + this.state?.setMainOp(op); + } + + reportRendererLag(lagMs: number): void { + this.state?.reportRendererLag(lagMs); + } + + updateCounters(c: CounterPatch): void { + this.state?.updateCounters(c); + } + + /** Graceful shutdown — flushes the heartbeat timer and signals the + * worker to exit cleanly so it can write a `[SHUTDOWN]` line. */ + async terminate(): Promise { + this.terminated = true; + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + if (this.worker) { + const w = this.worker; + this.worker = null; + try { + w.postMessage({ type: 'shutdown' }); + // Give the worker up to 250 ms to write its shutdown line + // before we forcibly terminate. + await new Promise((resolve) => setTimeout(resolve, 250)); + await w.terminate(); + } catch { + /* already exited */ + } + } + } +} + +export const freezeWatchdog = new FreezeWatchdog(); diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index e1fa35a..f3da26b 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -23,6 +23,11 @@ import { type ExportCsvResult, type ExportJsonInput, type ExportJsonResult, + type ExportXmlInput, + type ExportXmlResult, + type DataDeleteByDomainInput, + type DataDeleteByDomainResult, + type CrashRecoveryStatus, type ExportHtmlReportInput, type ExportHtmlReportResult, type BulkExportFile, @@ -75,6 +80,7 @@ import { Crawler, exportUrlsToCsv, exportUrlsToJson, + exportUrlsToXml, exportSitemap, exportHtmlReport, compareCrawls, @@ -86,6 +92,11 @@ import { ProjectDb } from '@freecrawl/db'; import { buildAppMenu } from './menu.js'; import * as logger from './logger.js'; import { dbReaderPool, callReaderOrFallback } from './db-reader-pool.js'; +import { dbWriterPool } from './db-writer-pool.js'; +import { freezeWatchdog } from './freeze-watchdog.js'; +import { parserPool } from './parser-pool.js'; +import { parseHtml as inlineParseHtml } from '@freecrawl/core'; +import type { ProjectDb as ProjectDbType } from '@freecrawl/db'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -93,6 +104,15 @@ let mainWindow: BrowserWindow | null = null; let logsWindow: BrowserWindow | null = null; let db: ProjectDb | null = null; let activeCrawler: Crawler | null = null; +/** Most-recent CrawlConfig — needed by the crash-recovery resume + * flow so we can re-create the Crawler with the same knobs the user + * had set, without re-prompting. Hydrated lazily from `project_meta` + * on first DB open. */ +let lastCrawlConfig: CrawlConfig | null = null; +/** In-memory snapshot of the previous session's checkpoint, captured + * before `db.reset()` wipes it. Cleared once the user accepts or + * discards the recovery prompt. */ +let pendingRecoveryCheckpoint: { url: string; depth: number; seedUrl: string }[] = []; // UI preferences (column widths, panel sizes, etc.) live in a JSON file // under userData — separate from the crawl DB so "Clear" wipes crawl data @@ -291,6 +311,61 @@ function maybeShowDiagnosticDialog(msg: string): void { /** Currently-open project file path (empty when using the default scratch DB). */ let currentProjectPath = ''; +/** + * Dispatch HTML parsing to the worker pool when ready, otherwise + * fall back to the inline parser on the main thread. This wrapper is + * passed into every `new Crawler(...)` call so the same code path + * gets pool acceleration in the desktop app and stays simple in + * tests / headless contexts. + */ +async function parseHtmlViaPool( + html: string, + pageUrl: string, + opts: Parameters[2], +): Promise> { + if (!parserPool.isReady()) { + return inlineParseHtml(html, pageUrl, opts); + } + try { + return await parserPool.parse(html, pageUrl, opts); + } catch (err) { + logger.log( + 'warn', + 'main', + `parser-pool dispatch failed (${ + err instanceof Error ? err.message : String(err) + }) — using inline parseHtml as fallback.`, + ); + return inlineParseHtml(html, pageUrl, opts); + } +} + +/** + * Dispatch a per-URL write batch to the writer worker. Same fall-back + * pattern as the parser pool: when the worker is unhealthy we run + * the write on the main-thread `ProjectDb` instance so the crawl + * never silently drops data. + */ +async function writeFetchedUrlViaPool( + payload: Parameters[0], +): Promise<{ urlId: number }> { + if (!dbWriterPool.isReady()) { + return getDb().writeFetchedUrl(payload); + } + try { + return await dbWriterPool.call<{ urlId: number }>('writeFetchedUrl', [payload]); + } catch (err) { + logger.log( + 'warn', + 'main', + `db-writer dispatch failed (${ + err instanceof Error ? err.message : String(err) + }) — falling back to main-thread writeFetchedUrl.`, + ); + return getDb().writeFetchedUrl(payload); + } +} + function getDb(): ProjectDb { if (!db) { const dataDir = join(app.getPath('userData'), 'projects'); @@ -298,6 +373,25 @@ function getDb(): ProjectDb { const defaultPath = join(dataDir, 'default.seoproject'); db = new ProjectDb(defaultPath); currentProjectPath = ''; + // Wave 6 — Snapshot the previous session's crash-recovery state + // BEFORE the reset wipes the project tables. We capture the + // checkpointed pending queue + the cached crawl config so the + // user can be offered a resume on app start. The DB tables go + // back to empty after `reset()`; the in-memory snapshot stays + // until the user accepts/discards the recovery prompt. + try { + const raw = db.getMeta('lastCrawlConfig'); + if (raw) lastCrawlConfig = JSON.parse(raw) as CrawlConfig; + } catch { + /* malformed JSON or missing key — recovery just won't fire */ + } + pendingRecoveryCheckpoint = (() => { + try { + return db.loadQueueCheckpoint(); + } catch { + return []; + } + })(); // Fresh start on every app launch — clear any data carried over from // the previous session. Explicit Save Project will be added later. db.reset(); @@ -305,7 +399,9 @@ function getDb(): ProjectDb { // already ran above (the writer owns them) so the worker just // attaches to the WAL and starts serving SELECTs concurrently. try { - dbReaderPool.init(defaultPath); + // Pass the freeze-watchdog SAB so the reader worker can publish + // its own heartbeat + current op into the same diagnostic file. + dbReaderPool.init(defaultPath, freezeWatchdog.sharedBuffer); } catch (err) { logger.log( 'warn', @@ -313,6 +409,15 @@ function getDb(): ProjectDb { `db-reader pool init failed: ${err instanceof Error ? err.message : String(err)} — falling back to main-thread queries.`, ); } + try { + dbWriterPool.init(defaultPath, freezeWatchdog.sharedBuffer); + } catch (err) { + logger.log( + 'warn', + 'main', + `db-writer pool init failed: ${err instanceof Error ? err.message : String(err)} — writes fall back to main thread.`, + ); + } } return db; } @@ -341,7 +446,7 @@ function openProjectAtPath(filePath: string): void { // in-flight requests with `reader-swapped`; the next IPC call will // hit the freshly opened worker. try { - dbReaderPool.swap(filePath); + dbReaderPool.swap(filePath, freezeWatchdog.sharedBuffer); } catch (err) { logger.log( 'warn', @@ -349,6 +454,15 @@ function openProjectAtPath(filePath: string): void { `db-reader pool swap failed: ${err instanceof Error ? err.message : String(err)}`, ); } + try { + dbWriterPool.swap(filePath, freezeWatchdog.sharedBuffer); + } catch (err) { + logger.log( + 'warn', + 'main', + `db-writer pool swap failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } pushRecentProject(filePath); rebuildMenu(); if (mainWindow) { @@ -661,6 +775,9 @@ function registerIpc(): void { ipcMain.on(IPC.rendererLagReport, (_e, lagMs: number) => { if (typeof lagMs === 'number' && Number.isFinite(lagMs)) { activeCrawler?.reportRendererLag(lagMs); + // Forward the same sample to the freeze-watchdog so a frozen + // renderer surfaces in `debug.txt` even when no crawl is active. + freezeWatchdog.reportRendererLag(lagMs); } }); @@ -958,6 +1075,63 @@ function registerIpc(): void { schedulePrefsWrite(); }); + /** Wave 6 — Cached most-recent crawl config so the crash-recovery + * resume can re-create a Crawler without forcing the user to fill + * in the Start dialog again. Persisted across restarts via DB + * project_meta so it survives the very crash we're recovering from. */ + function attachCrawlerListeners(crawler: Crawler): void { + crawler.on('progress', (p: CrawlProgress) => { + if (activeCrawler !== crawler) return; + mainWindow?.webContents.send(IPC.crawlProgress, p); + freezeWatchdog.updateCounters({ + crawled: p.crawled, + discovered: p.discovered, + pending: p.pending, + failed: p.failed, + }); + }); + crawler.on('done', (summary: CrawlSummary) => { + if (activeCrawler !== crawler) return; + logger.log( + 'info', + 'crawler', + `Crawl done: total=${summary.total} avgResp=${summary.avgResponseTimeMs}ms totalBytes=${summary.totalBytes}`, + ); + mainWindow?.webContents.send(IPC.crawlDone, summary); + activeCrawler = null; + freezeWatchdog.setMainOp('idle'); + if (Notification.isSupported() && !mainWindow?.isFocused()) { + try { + new Notification({ + title: 'FreeCrawl SEO Tool', + body: `Crawl finished: ${summary.total.toLocaleString()} URLs · avg ${Math.round(summary.avgResponseTimeMs)} ms`, + silent: false, + }).show(); + } catch { + /* Linux without notification daemon — swallow */ + } + } + }); + crawler.on('error', (msg: string) => { + if (activeCrawler !== crawler) return; + logger.log('error', 'crawler', msg); + mainWindow?.webContents.send(IPC.crawlError, msg); + maybeShowDiagnosticDialog(msg); + }); + crawler.on('warn', (msg: string) => { + if (activeCrawler !== crawler) return; + logger.log('warn', 'crawler', msg); + }); + crawler.on('info', (msg: string) => { + if (activeCrawler !== crawler) return; + logger.log('info', 'crawler', msg); + }); + crawler.on('debug', (msg: string) => { + if (activeCrawler !== crawler) return; + logger.log('debug', 'crawler', msg); + }); + } + ipcMain.handle(IPC.crawlStart, (_e, config: CrawlConfig) => { if (activeCrawler) { activeCrawler.stop(); @@ -973,69 +1147,22 @@ function registerIpc(): void { ); const database = getDb(); database.setMeta('lastStartUrl', config.startUrl); - const crawler = new Crawler(config, database); + // Persist the config so a crash-then-resume can rehydrate it + // without the user re-entering anything. Stored as JSON in + // `project_meta` since the table is keyed by string. + try { + database.setMeta('lastCrawlConfig', JSON.stringify(config)); + } catch { + /* never fatal */ + } + lastCrawlConfig = config; + const crawler = new Crawler(config, database, { + setOp: (op: string) => freezeWatchdog.setMainOp(op), + parseHtml: parseHtmlViaPool, + writeFetchedUrl: writeFetchedUrlViaPool, + }); activeCrawler = crawler; - - // Every event handler is gated by `activeCrawler === crawler`. A - // stopped crawler can still emit late events (e.g. an in-flight - // sitemap fetch resolving after Stop, or a queued done-event) — if - // we forwarded those to the UI they'd clobber the new crawl's state - // (the "pır pır" effect: rapid Running ↔ Done flicker). - crawler.on('progress', (p: CrawlProgress) => { - if (activeCrawler !== crawler) return; - mainWindow?.webContents.send(IPC.crawlProgress, p); - }); - crawler.on('done', (summary: CrawlSummary) => { - if (activeCrawler !== crawler) return; - logger.log( - 'info', - 'crawler', - `Crawl done: total=${summary.total} avgResp=${summary.avgResponseTimeMs}ms totalBytes=${summary.totalBytes}`, - ); - mainWindow?.webContents.send(IPC.crawlDone, summary); - activeCrawler = null; - // OS-level toast — only when the window isn't focused, so the - // user actually benefits (Electron suppresses the notification - // sound on focused windows on most platforms anyway, but we - // gate explicitly to avoid distracting users who're watching). - if (Notification.isSupported() && !mainWindow?.isFocused()) { - try { - new Notification({ - title: 'FreeCrawl SEO Tool', - body: `Crawl finished: ${summary.total.toLocaleString()} URLs · avg ${Math.round(summary.avgResponseTimeMs)} ms`, - silent: false, - }).show(); - } catch { - // Notification can throw on some Linux distros without a - // notification daemon. Swallow — the in-app done banner - // already surfaces completion. - } - } - }); - crawler.on('error', (msg: string) => { - if (activeCrawler !== crawler) return; - logger.log('error', 'crawler', msg); - mainWindow?.webContents.send(IPC.crawlError, msg); - // Match against environment-specific failure patterns (DNS / TLS - // inspection / unreachable seed) and surface a modal popup the - // first time each category fires per crawl session. Site-specific - // 4xx/WAF/timeout errors are NOT promoted to popups — they belong - // in the log panel, not in the user's face. - maybeShowDiagnosticDialog(msg); - }); - crawler.on('warn', (msg: string) => { - if (activeCrawler !== crawler) return; - logger.log('warn', 'crawler', msg); - }); - crawler.on('info', (msg: string) => { - if (activeCrawler !== crawler) return; - logger.log('info', 'crawler', msg); - }); - crawler.on('debug', (msg: string) => { - if (activeCrawler !== crawler) return; - logger.log('debug', 'crawler', msg); - }); - + attachCrawlerListeners(crawler); void crawler.start(); }); @@ -1054,7 +1181,25 @@ function registerIpc(): void { ipcMain.handle(IPC.crawlClear, () => { activeCrawler?.stop(); activeCrawler = null; - getDb().reset(); + const database = getDb(); + database.reset(); + // Belt-and-braces: reset() above already includes a DELETE FROM + // crawl_queue, but if anything (a late-firing checkpoint timer, a + // half-finished post-crawl pass) writes to that table after the + // exec() returns, the user would see a stale recovery prompt next + // launch. An explicit second sweep here is cheap and guarantees + // the queue is empty when the IPC handler resolves. + try { + database.clearQueueCheckpoint(); + } catch { + /* table missing on very old DBs — recovery just won't fire */ + } + // The in-memory recovery snapshot is captured at app boot from the + // queue-on-disk; reset() now wipes that data, but the snapshot + // mirror still holds the pre-reset entries. Drop it so a stale + // dialog can't fire after Clear if the user cancelled the boot + // prompt earlier. + pendingRecoveryCheckpoint = []; }); ipcMain.handle(IPC.crawlAddUrl, (_e, url: string): { accepted: boolean } => { @@ -1493,6 +1638,106 @@ function registerIpc(): void { }, ); + ipcMain.handle( + IPC.exportXml, + async (_e, input: ExportXmlInput): Promise => { + let filePath = input.filePath; + const isSelection = (input.selectedIds?.length ?? 0) > 0; + if (!filePath) { + const res = await dialog.showSaveDialog(mainWindow!, { + defaultPath: isSelection ? 'freecrawl-selected.xml' : 'freecrawl-export.xml', + filters: [{ name: 'XML', extensions: ['xml'] }], + }); + if (res.canceled || !res.filePath) { + return { filePath: '', rowsWritten: 0 }; + } + filePath = res.filePath; + } + const { rowsWritten } = await exportUrlsToXml(getDb(), filePath, { + selectedIds: input.selectedIds, + category: input.category, + }); + return { filePath, rowsWritten }; + }, + ); + + ipcMain.handle( + IPC.dataDeleteByDomain, + async ( + _e, + input: DataDeleteByDomainInput, + ): Promise => { + const { urlsDeleted, linksDeleted } = getDb().deleteByDomain(input.domain); + // Force-invalidate UI caches: row counts, sidebar issues, etc. + fireDataChanged(); + return { urlsDeleted, linksDeleted }; + }, + ); + + // Wave 6 — Crash recovery handlers. The renderer asks `…Status` on + // mount; if `pendingCount > 0` it shows a confirmation dialog and + // routes the user's choice to `…Resume` or `…Discard`. + ipcMain.handle( + IPC.crashRecoveryStatus, + async (): Promise => { + // Read from the in-memory snapshot we captured before + // `db.reset()` wiped the previous session's data. + if (pendingRecoveryCheckpoint.length === 0) { + return { pendingCount: 0, seedUrl: '' }; + } + return { + pendingCount: pendingRecoveryCheckpoint.length, + seedUrl: pendingRecoveryCheckpoint[0]?.seedUrl ?? '', + }; + }, + ); + + ipcMain.handle( + IPC.crashRecoveryResume, + async (): Promise<{ accepted: boolean }> => { + if (activeCrawler) return { accepted: false }; + if (pendingRecoveryCheckpoint.length === 0) return { accepted: false }; + const seedUrl = pendingRecoveryCheckpoint[0]?.seedUrl ?? ''; + if (!seedUrl) return { accepted: false }; + const cfg = lastCrawlConfig + ? { ...lastCrawlConfig, startUrl: seedUrl } + : null; + if (!cfg) return { accepted: false }; + const crawler = new Crawler(cfg, getDb(), { + setOp: (op: string) => freezeWatchdog.setMainOp(op), + parseHtml: parseHtmlViaPool, + writeFetchedUrl: writeFetchedUrlViaPool, + }); + activeCrawler = crawler; + attachCrawlerListeners(crawler); + const items = pendingRecoveryCheckpoint.map((c) => ({ + url: c.url, + depth: c.depth, + })); + // Snapshot consumed — clear so a second click can't double-fire. + pendingRecoveryCheckpoint = []; + crawler.enqueueCheckpointed(items); + void crawler.start(); + return { accepted: true }; + }, + ); + + ipcMain.handle(IPC.crashRecoveryDiscard, async (): Promise => { + pendingRecoveryCheckpoint = []; + // Wipe the on-disk crawl_queue too — without this, anything that + // re-populates the queue between now and the next app launch would + // resurrect the recovery prompt the user just dismissed. + try { + getDb().clearQueueCheckpoint(); + } catch (err) { + logger.log( + 'warn', + 'main', + `clearQueueCheckpoint failed on discard: ${err instanceof Error ? err.message : String(err)}`, + ); + } + }); + ipcMain.handle(IPC.exportBulk, async (): Promise => { if (!mainWindow) { return { outputDir: '', files: [], errors: [] }; @@ -1836,11 +2081,36 @@ void app.whenReady().then(() => { } catch (err) { logger.log('warn', 'main', `Disk logging unavailable: ${(err as Error).message}`); } + // Boot the freeze-watchdog before anything else CPU-heavy. The + // worker writes to `/debug.txt` (separate from the + // regular structured log so users + dev can grep stalls without + // noise from normal app events). + try { + const debugPath = join(app.getPath('userData'), 'debug.txt'); + freezeWatchdog.init(debugPath); + } catch (err) { + logger.log('warn', 'main', `freeze-watchdog init failed: ${(err as Error).message}`); + } + freezeWatchdog.setMainOp('boot'); + // Spawn the parser worker pool so cheerio runs off the main thread. + // Falls back to inline parseHtml automatically when init fails on a + // constrained host (the pool's `parse()` rejects, the dispatch + // helper below catches and falls back). + try { + parserPool.init(); + } catch (err) { + logger.log( + 'warn', + 'main', + `parser-pool init failed: ${(err as Error).message} — falling back to inline parseHtml.`, + ); + } loadPrefs(); rebuildMenu(); registerIpc(); createWindow(); logger.log('info', 'main', `App ready — version ${app.getVersion()}`); + freezeWatchdog.setMainOp('idle'); app.on('activate', () => { if (BrowserWindow.getAllWindows().length === 0) createWindow(); @@ -1849,11 +2119,15 @@ void app.whenReady().then(() => { app.on('window-all-closed', () => { activeCrawler?.stop(); + freezeWatchdog.setMainOp('shutdown'); + void freezeWatchdog.terminate(); + void parserPool.terminate(); // Terminate the read-only worker BEFORE closing the writer connection. // Order matters on Windows: SQLite holds a file lock per connection // and the writer's WAL checkpoint at close-time can stall if a // reader is still mid-query. void dbReaderPool.terminate(); + void dbWriterPool.terminate(); db?.close(); db = null; flushPrefs(); diff --git a/apps/desktop/src/main/menu.ts b/apps/desktop/src/main/menu.ts index a3aeccc..4d23d5a 100644 --- a/apps/desktop/src/main/menu.ts +++ b/apps/desktop/src/main/menu.ts @@ -78,6 +78,10 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu { accelerator: 'CmdOrCtrl+Shift+E', click: () => send('export-json'), }, + { + label: 'Export Current View as XML…', + click: () => send('export-xml'), + }, { label: 'Generate XML Sitemap…', click: () => send('generate-sitemap'), @@ -182,6 +186,19 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu { click: () => handlers.onResetDiagnosticDialogs(), }, { type: 'separator' }, + { + label: 'Delete Domain Data…', + toolTip: + 'GDPR-aligned per-domain wipe. Removes every URL row whose host matches the entered domain plus every dependent record (links, headers, images, source snapshots).', + click: () => send('delete-domain-data'), + }, + { + label: 'Clear All Data…', + toolTip: + 'Wipe the entire active project (URLs, links, images, headers, source snapshots, sitemaps). Cannot be undone — Save Project As… first if you want a backup.', + click: () => send('clear-all-data'), + }, + { type: 'separator' }, { label: 'About FreeCrawl SEO', click: () => send('about') }, ], }, diff --git a/apps/desktop/src/main/parser-pool.ts b/apps/desktop/src/main/parser-pool.ts new file mode 100644 index 0000000..45a595a --- /dev/null +++ b/apps/desktop/src/main/parser-pool.ts @@ -0,0 +1,192 @@ +import { Worker } from 'node:worker_threads'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import os from 'node:os'; +import type { parseHtml } from '@freecrawl/core'; +import * as logger from './logger.js'; + +/** + * Round-robin pool of parser worker threads. + * + * Why a pool, not a single worker: + * - Cheerio is single-threaded inside the worker. One worker can + * parse one document at a time. + * - The crawler runs at concurrency 20 by default, so up to 20 + * pages return their HTML body simultaneously. With one worker + * they would serialise behind it, defeating the point. + * - 4-8 workers (capped to physical cores - 2) saturate the CPU + * while leaving the main thread headroom for IPC and queue + * scheduling. + * + * Round-robin dispatch is sufficient: parse durations are roughly + * proportional to body size, and the crawler's URL mix makes it + * unlikely that long-tail pages all land on the same worker. A + * least-loaded scheduler would add complexity for marginal gain. + */ + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const WORKER_PATH = path.join(__dirname, 'parser-worker.js'); +const REQUEST_TIMEOUT_MS = 60_000; + +type ParseHtmlArgs = Parameters; +type ParseOpts = ParseHtmlArgs[2]; +type ParseResult = ReturnType; + +interface PendingRequest { + resolve: (v: ParseResult) => void; + reject: (e: Error) => void; + timer: ReturnType; +} + +interface ResponseMessage { + requestId: number; + ok: boolean; + result?: ParseResult; + error?: string; +} + +class ParserPool { + private workers: Worker[] = []; + private pending = new Map(); + private nextRequestId = 1; + private nextWorkerIdx = 0; + private terminated = false; + + /** + * Spawn the worker pool. Idempotent — calling init() twice is a + * no-op once workers are already running. + * + * `size` — number of worker threads. Defaults to a sensible + * value based on CPU count (cores - 2, clamped to [2, 8]). + * Leave room for the main thread + db-reader + db-writer + + * freeze-watchdog so the crawler isn't competing with its own + * plumbing for cores. + */ + init(size?: number): void { + if (this.terminated) { + throw new Error('ParserPool: cannot init after terminate()'); + } + if (this.workers.length > 0) return; + const cpuCount = os.cpus().length; + const target = + size ?? Math.max(2, Math.min(8, cpuCount - 2)); + for (let i = 0; i < target; i++) { + try { + this.workers.push(this.spawnWorker(i)); + } catch (err) { + logger.log( + 'warn', + 'main', + `parser-pool: worker ${i} spawn failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } + } + if (this.workers.length === 0) { + logger.log( + 'warn', + 'main', + 'parser-pool: no workers spawned — falling back to inline parseHtml.', + ); + } else { + logger.log( + 'info', + 'main', + `parser-pool: spawned ${this.workers.length} parser worker(s)`, + ); + } + } + + /** True when at least one worker is running and accepting requests. */ + isReady(): boolean { + return this.workers.length > 0 && !this.terminated; + } + + /** + * Dispatch a parse request to the next available worker. + * Resolves with the parsed page; rejects on timeout or worker + * crash. Caller is expected to fall back to inline `parseHtml` + * if the pool isn't ready. + */ + parse(html: string, pageUrl: string, opts: ParseOpts = {}): Promise { + if (this.terminated || this.workers.length === 0) { + return Promise.reject(new Error('parser-pool: not ready')); + } + const requestId = this.nextRequestId++; + const worker = this.workers[this.nextWorkerIdx % this.workers.length]!; + this.nextWorkerIdx++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(requestId); + reject(new Error(`parser-pool: timeout > ${REQUEST_TIMEOUT_MS}ms`)); + }, REQUEST_TIMEOUT_MS); + this.pending.set(requestId, { resolve, reject, timer }); + worker.postMessage({ requestId, html, pageUrl, opts }); + }); + } + + async terminate(): Promise { + this.terminated = true; + for (const p of this.pending.values()) { + clearTimeout(p.timer); + p.reject(new Error('parser-pool: terminated')); + } + this.pending.clear(); + const workers = this.workers; + this.workers = []; + await Promise.allSettled(workers.map((w) => w.terminate())); + } + + // ── private ────────────────────────────────────────────────────── + + private spawnWorker(index: number): Worker { + const w = new Worker(WORKER_PATH); + w.on('message', (msg: ResponseMessage) => this.handleResponse(msg)); + w.on('error', (err) => { + logger.log('error', 'main', `parser-worker[${index}] error: ${err.message}`); + }); + w.on('exit', (code) => { + if (this.terminated) return; + if (code === 0) return; + logger.log( + 'warn', + 'main', + `parser-worker[${index}] exited (code=${code}) — respawning`, + ); + const respawn = this.tryRespawn(index); + if (respawn) this.workers[index] = respawn; + }); + return w; + } + + private tryRespawn(index: number): Worker | null { + if (this.terminated) return null; + try { + return this.spawnWorker(index); + } catch (err) { + logger.log( + 'error', + 'main', + `parser-worker[${index}] respawn failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return null; + } + } + + private handleResponse(msg: ResponseMessage): void { + const pending = this.pending.get(msg.requestId); + if (!pending) return; + this.pending.delete(msg.requestId); + clearTimeout(pending.timer); + if (msg.ok && msg.result !== undefined) { + pending.resolve(msg.result); + } else { + pending.reject(new Error(msg.error ?? 'parser-pool: unknown error')); + } + } +} + +export const parserPool = new ParserPool(); diff --git a/apps/desktop/src/main/parser-worker.ts b/apps/desktop/src/main/parser-worker.ts new file mode 100644 index 0000000..efec68b --- /dev/null +++ b/apps/desktop/src/main/parser-worker.ts @@ -0,0 +1,46 @@ +import { parentPort } from 'node:worker_threads'; +import { parseHtml } from '@freecrawl/core'; + +/** + * HTML-parser worker thread. + * + * Cheerio + the link/image/structured-data extractor is the hottest + * CPU path in `fetchAndProcess`. On large mağaza listing / blog + * pages it can spend 10-25 s of pure JS on a single document while + * the main thread holds the Electron event loop. Running the parse + * here means the main process only waits for the result via + * postMessage — IPC pumps and crawler scheduling stay live. + * + * Protocol: + * main → worker: `{ requestId, html, pageUrl, opts }` + * worker → main: `{ requestId, ok: true, result }` + * or `{ requestId, ok: false, error }` + * + * One worker handles one request at a time; the pool spreads requests + * across workers via round-robin (see `parser-pool.ts`). + */ + +if (!parentPort) { + throw new Error('parser-worker: must be loaded via worker_threads'); +} + +interface ParseRequest { + requestId: number; + html: string; + pageUrl: string; + opts: Parameters[2]; +} + +parentPort.on('message', (msg: ParseRequest) => { + if (!msg || typeof msg.requestId !== 'number') return; + try { + const result = parseHtml(msg.html, msg.pageUrl, msg.opts); + parentPort!.postMessage({ requestId: msg.requestId, ok: true, result }); + } catch (err) { + parentPort!.postMessage({ + requestId: msg.requestId, + ok: false, + error: err instanceof Error ? `${err.name}: ${err.message}` : String(err), + }); + } +}); diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index c6f2466..564cb66 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -9,6 +9,11 @@ import { type ExportCsvResult, type ExportJsonInput, type ExportJsonResult, + type ExportXmlInput, + type ExportXmlResult, + type DataDeleteByDomainInput, + type DataDeleteByDomainResult, + type CrashRecoveryStatus, type ExportHtmlReportInput, type ExportHtmlReportResult, type BulkExportResult, @@ -116,6 +121,18 @@ const api: FreeCrawlApi = { ipcRenderer.invoke(IPC.exportCsv, input), exportJson: (input: ExportJsonInput): Promise => ipcRenderer.invoke(IPC.exportJson, input), + exportXml: (input: ExportXmlInput): Promise => + ipcRenderer.invoke(IPC.exportXml, input), + dataDeleteByDomain: ( + input: DataDeleteByDomainInput, + ): Promise => + ipcRenderer.invoke(IPC.dataDeleteByDomain, input), + crashRecoveryStatus: (): Promise => + ipcRenderer.invoke(IPC.crashRecoveryStatus), + crashRecoveryResume: (): Promise<{ accepted: boolean }> => + ipcRenderer.invoke(IPC.crashRecoveryResume), + crashRecoveryDiscard: (): Promise => + ipcRenderer.invoke(IPC.crashRecoveryDiscard), exportHtmlReport: (input: ExportHtmlReportInput): Promise => ipcRenderer.invoke(IPC.exportHtmlReport, input), exportBulk: (): Promise => ipcRenderer.invoke(IPC.exportBulk), diff --git a/apps/desktop/src/renderer/src/App.tsx b/apps/desktop/src/renderer/src/App.tsx index e727a10..0e2573a 100644 --- a/apps/desktop/src/renderer/src/App.tsx +++ b/apps/desktop/src/renderer/src/App.tsx @@ -57,6 +57,30 @@ export function App() { const file = files[0]; if (!file) return; const name = file.name.toLowerCase(); + // Wave 5 — `.seoproject` drop opens the project. Electron exposes + // `file.path` (not part of the standard File API) when files come + // from the OS file system; we route to `projectOpen` which swaps + // the active DB and refreshes the UI. + if (name.endsWith('.seoproject')) { + e.preventDefault(); + const fileWithPath = file as File & { path?: string }; + const filePath = fileWithPath.path; + if (!filePath) { + setDropFlash( + `Couldn't read the dropped path for ${file.name}. Use File → Open Project instead.`, + ); + setTimeout(() => setDropFlash(null), 4000); + return; + } + const result = await window.freecrawl.projectOpen(filePath); + if (result) { + setDropFlash(`Opened project: ${result.filePath}`); + } else { + setDropFlash(`Failed to open project: ${file.name}`); + } + setTimeout(() => setDropFlash(null), 4000); + return; + } if (!/\.(txt|csv|list)$/.test(name)) return; e.preventDefault(); const text = await file.text(); @@ -109,7 +133,48 @@ export function App() { ); useEffect(() => { - const off1 = window.freecrawl.onProgress(setProgress); + // Wave 6 — Crash-recovery prompt. Fires once on mount; if the + // previous session left a non-empty checkpoint we ask whether to + // resume. Discarding wipes the snapshot so the prompt doesn't + // recur on next launch. + void window.freecrawl.crashRecoveryStatus().then((status) => { + if (status.pendingCount === 0) return; + const proceed = window.confirm( + `FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` + + ` Start URL: ${status.seedUrl}\n` + + ` Pending URLs: ${status.pendingCount.toLocaleString()}\n\n` + + `Resume the crawl from where it stopped?\n\n` + + `Click OK to resume, Cancel to discard the recovery state and start fresh.`, + ); + if (proceed) { + void window.freecrawl.crashRecoveryResume(); + } else { + void window.freecrawl.crashRecoveryDiscard(); + } + }); + + // rAF-throttled progress dispatch. The crawler emits up to 5 + // progress events/sec and each one triggers a Zustand store + // update — TopBar, StatsBar and App.tsx all subscribe to the + // `progress` object reference, so every event fans out to a full + // React reconciliation. Coalescing into one update per animation + // frame caps the React work at 60 Hz worst-case (and at 5 Hz in + // the common case where the crawler's throttle dominates), so a + // user click never queues behind a stack of pending re-renders. + let pendingProgress: import('@freecrawl/shared-types').CrawlProgress | null = null; + let progressRafScheduled = false; + const off1 = window.freecrawl.onProgress((p) => { + pendingProgress = p; + if (progressRafScheduled) return; + progressRafScheduled = true; + requestAnimationFrame(() => { + progressRafScheduled = false; + if (pendingProgress) { + setProgress(pendingProgress); + pendingProgress = null; + } + }); + }); const off2 = window.freecrawl.onDone((s) => setSummary(s)); const off3 = window.freecrawl.onError(setError); const offData = window.freecrawl.onDataChanged(() => bumpDataVersion()); @@ -133,6 +198,45 @@ export function App() { case 'export-json': void window.freecrawl.exportJson({ filePath: '', pretty: true }); break; + case 'export-xml': + void window.freecrawl.exportXml({ filePath: '' }); + break; + case 'delete-domain-data': { + const domain = window.prompt( + 'GDPR Domain Wipe — enter the host to delete (case-insensitive, exact match).\n\nExamples: example.com · blog.example.com\n\nThis cannot be undone. Save Project As… first to keep a backup.', + '', + ); + if (!domain || !domain.trim()) break; + const target = domain.trim(); + if ( + !window.confirm( + `Delete every URL hosted on "${target}", along with its links, headers, images, and source snapshots?\n\nThis action cannot be undone.`, + ) + ) { + break; + } + void window.freecrawl.dataDeleteByDomain({ domain: target }).then((res) => { + window.alert( + `Deleted ${res.urlsDeleted.toLocaleString()} URL row${ + res.urlsDeleted === 1 ? '' : 's' + } from ${target} (and ${res.linksDeleted.toLocaleString()} associated link${ + res.linksDeleted === 1 ? '' : 's' + }).`, + ); + }); + break; + } + case 'clear-all-data': { + if ( + !window.confirm( + 'Clear ALL data in the active project?\n\nEvery URL, link, image, header, source snapshot, sitemap, and issue will be wiped. This cannot be undone — Save Project As… first if you want a backup.', + ) + ) { + break; + } + void window.freecrawl.crawlClear(); + break; + } case 'open-robots-tester': setRobotsTesterOpen(true); break; diff --git a/apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx b/apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx index 03e6b11..e63a197 100644 --- a/apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx +++ b/apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState, type ReactNode } from 'react'; +import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react'; import clsx from 'clsx'; import type { CrawlUrlRow, @@ -42,23 +42,81 @@ const SUB_TABS: { key: SubTab; label: string; disabled?: boolean }[] = [ { key: 'view-source', label: 'View Source' }, ]; +// Maximum number of URLs we aggregate over in one go. Anything larger is +// treated as a "too many" hint so the user can narrow the selection +// before we burn N parallel reader-pool requests on it. +const MULTI_DETAIL_LIMIT = 50; + +// Tabs that pivot on a single page (HTTP response, snippet, source, etc.) +// — when the user has multi-selected rows we keep these scoped to the +// primary URL and surface a banner so the scope is clear. +const SINGLE_URL_ONLY_TABS: ReadonlySet = new Set([ + 'url-details', + 'outline', + 'extracted-data', + 'serp-snippet', + 'http-headers', + 'cookies', + 'structured-data', + 'view-source', +]); + export function BottomDetailPanel() { const selectedUrlId = useAppStore((s) => s.selectedUrlId); + const selectedUrlIds = useAppStore((s) => s.selectedUrlIds); const [detail, setDetail] = useState(null); + const [details, setDetails] = useState([]); const [loading, setLoading] = useState(false); const [subTab, setSubTab] = useState('url-details'); + // Effective scope: if the user has 2+ rows selected we aggregate; one + // row (or none) keeps the existing single-URL behaviour intact. + const effectiveIds = useMemo(() => { + if (selectedUrlIds.length > 1) return selectedUrlIds; + if (selectedUrlId !== null) return [selectedUrlId]; + return []; + }, [selectedUrlId, selectedUrlIds]); + + const isMulti = effectiveIds.length > 1; + const truncated = isMulti && effectiveIds.length > MULTI_DETAIL_LIMIT; + const fetchIds = useMemo( + () => (truncated ? effectiveIds.slice(0, MULTI_DETAIL_LIMIT) : effectiveIds), + [effectiveIds, truncated], + ); + + // Stable cache key so we don't re-fetch on every render of the same set. + const fetchKey = fetchIds.join(','); + useEffect(() => { - if (selectedUrlId === null) { + if (fetchIds.length === 0) { setDetail(null); + setDetails([]); return; } let cancelled = false; const load = async () => { setLoading(true); try { - const d = await window.freecrawl.urlDetailGet({ id: selectedUrlId }); - if (!cancelled) setDetail(d); + if (fetchIds.length === 1) { + const d = await window.freecrawl.urlDetailGet({ id: fetchIds[0]! }); + if (!cancelled) { + setDetail(d); + setDetails(d ? [d] : []); + } + } else { + const results = await Promise.all( + fetchIds.map((id) => window.freecrawl.urlDetailGet({ id })), + ); + if (!cancelled) { + const list = results.filter((r): r is UrlDetail => r !== null); + setDetails(list); + // The "primary" detail is the one matching `selectedUrlId` + // (last clicked); fall back to the first if it's missing. + const primary = + list.find((r) => r.row.id === selectedUrlId) ?? list[0] ?? null; + setDetail(primary); + } + } } finally { if (!cancelled) setLoading(false); } @@ -67,7 +125,101 @@ export function BottomDetailPanel() { return () => { cancelled = true; }; - }, [selectedUrlId]); + // selectedUrlId is intentionally excluded from deps — it only steers + // which entry of `details` becomes "primary" and we resolve that + // inside the load() body. fetchKey covers multi-set changes. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [fetchKey]); + + // When only the primary id changes (e.g. user clicked a different cell + // within the same multi-selection set), re-pick the primary detail + // without re-fetching the whole batch. + useEffect(() => { + if (!isMulti || details.length === 0) return; + const primary = + details.find((r) => r.row.id === selectedUrlId) ?? details[0] ?? null; + setDetail(primary); + }, [selectedUrlId, details, isMulti]); + + // Aggregated link rows: each detail's inlinks/outlinks already carry + // their fromUrl / toUrl so concatenation is enough — the From / To + // columns in the table naturally distinguish which page each row + // belongs to. + const aggregatedInlinks = useMemo(() => { + if (details.length === 0) return []; + const rows: string[][] = []; + for (const d of details) { + for (const l of d.inlinks) { + rows.push( + buildLinkRow({ + fromUrl: l.fromUrl, + toUrl: d.row.url, + toStatusCode: l.toStatusCode, + toSize: l.toSize, + type: l.type, + anchor: l.anchor, + altText: l.altText, + rel: l.rel, + target: l.target, + pathType: l.pathType, + linkPath: l.linkPath, + linkPosition: l.linkPosition, + linkOrigin: l.linkOrigin, + }), + ); + } + } + return rows; + }, [details]); + + const aggregatedOutlinks = useMemo(() => { + if (details.length === 0) return []; + const rows: string[][] = []; + for (const d of details) { + for (const l of d.outlinks) { + rows.push( + buildLinkRow({ + fromUrl: d.row.url, + toUrl: l.toUrl, + toStatusCode: l.toStatusCode, + toSize: l.toSize, + type: l.type, + anchor: l.anchor, + altText: l.altText, + rel: l.rel, + target: l.target, + pathType: l.pathType, + linkPath: l.linkPath, + linkPosition: l.linkPosition, + linkOrigin: l.linkOrigin, + }), + ); + } + } + return rows; + }, [details]); + + const aggregatedInlinksTotal = useMemo( + () => details.reduce((n, d) => n + d.inlinksTotal, 0), + [details], + ); + const aggregatedOutlinksTotal = useMemo( + () => details.reduce((n, d) => n + d.outlinksTotal, 0), + [details], + ); + + // For Images / Resources we collect (urlId, row) pairs so the dedicated + // multi-views can spread their fetches across the same set the user + // picked in the main table. + const multiPages = useMemo( + () => details.map((d) => ({ id: d.row.id, row: d.row })), + [details], + ); + + const inlinksCountLabel = isMulti ? aggregatedInlinksTotal : detail?.inlinksTotal; + const outlinksCountLabel = isMulti ? aggregatedOutlinksTotal : detail?.outlinksTotal; + + const showSingleScopeBanner = isMulti && SINGLE_URL_ONLY_TABS.has(subTab); return (
@@ -85,85 +237,144 @@ export function BottomDetailPanel() { title={t.disabled ? 'Coming soon' : undefined} > {t.label} - {t.key === 'inlinks' && detail && ( - ({detail.inlinksTotal}) + {t.key === 'inlinks' && inlinksCountLabel !== undefined && ( + + ({inlinksCountLabel.toLocaleString()}) + )} - {t.key === 'outlinks' && detail && ( - ({detail.outlinksTotal}) + {t.key === 'outlinks' && outlinksCountLabel !== undefined && ( + + ({outlinksCountLabel.toLocaleString()}) + )} ))} + {isMulti && ( +
+ + {effectiveIds.length.toLocaleString()} + {' '} + URLs selected + {truncated && ( + + · aggregating first {MULTI_DETAIL_LIMIT} + + )} +
+ )}
+ {showSingleScopeBanner && detail && ( +
+ This tab is per-page — showing data for{' '} + {detail.row.url}{' '} + (primary URL of the selection). +
+ )} +
- {selectedUrlId === null && ( + {effectiveIds.length === 0 && (
Select a URL from the table to see details.
)} - {selectedUrlId !== null && !detail && loading && ( + {effectiveIds.length > 0 && !detail && loading && (
Loading…
)} {detail && subTab === 'url-details' && } - {detail && subTab === 'inlinks' && ( - - buildLinkRow({ - fromUrl: l.fromUrl, - toUrl: detail.row.url, - toStatusCode: l.toStatusCode, - toSize: l.toSize, - type: l.type, - anchor: l.anchor, - altText: l.altText, - rel: l.rel, - target: l.target, - pathType: l.pathType, - linkPath: l.linkPath, - linkPosition: l.linkPosition, - linkOrigin: l.linkOrigin, - }), - )} - /> - )} - {detail && subTab === 'outlinks' && ( - - buildLinkRow({ - fromUrl: detail.row.url, - toUrl: l.toUrl, - toStatusCode: l.toStatusCode, - toSize: l.toSize, - type: l.type, - anchor: l.anchor, - altText: l.altText, - rel: l.rel, - target: l.target, - pathType: l.pathType, - linkPath: l.linkPath, - linkPosition: l.linkPosition, - linkOrigin: l.linkOrigin, - }), - )} - /> - )} + {subTab === 'inlinks' && + (isMulti ? ( + details.length > 0 && ( + + ) + ) : ( + detail && ( + + buildLinkRow({ + fromUrl: l.fromUrl, + toUrl: detail.row.url, + toStatusCode: l.toStatusCode, + toSize: l.toSize, + type: l.type, + anchor: l.anchor, + altText: l.altText, + rel: l.rel, + target: l.target, + pathType: l.pathType, + linkPath: l.linkPath, + linkPosition: l.linkPosition, + linkOrigin: l.linkOrigin, + }), + )} + /> + ) + ))} + {subTab === 'outlinks' && + (isMulti ? ( + details.length > 0 && ( + + ) + ) : ( + detail && ( + + buildLinkRow({ + fromUrl: detail.row.url, + toUrl: l.toUrl, + toStatusCode: l.toStatusCode, + toSize: l.toSize, + type: l.type, + anchor: l.anchor, + altText: l.altText, + rel: l.rel, + target: l.target, + pathType: l.pathType, + linkPath: l.linkPath, + linkPosition: l.linkPosition, + linkOrigin: l.linkOrigin, + }), + )} + /> + ) + ))} {detail && subTab === 'outline' && } - {detail && subTab === 'images' && ( - - )} - {detail && subTab === 'resources' && ( - - )} + {subTab === 'images' && + (isMulti ? ( + + ) : ( + detail && + ))} + {subTab === 'resources' && + (isMulti ? ( + + ) : ( + detail && + ))} {detail && subTab === 'extracted-data' && ( )} @@ -173,10 +384,10 @@ export function BottomDetailPanel() { )} {detail && subTab === 'structured-data' && ( - + )} {detail && subTab === 'view-source' && ( - + )}
@@ -735,11 +946,27 @@ function LinksView({ | { kind: 'column'; aC: number; c: number; additive: boolean; base: Set } | null >(null); + // Right-click context-menu position. Null = menu hidden. The menu is a + // small in-page popover (not the native Electron menu) so we can wire + // it up without round-tripping through IPC for every click. + const [menu, setMenu] = useState< + | { x: number; y: number; row: number; col: number } + | null + >(null); + const rootRef = useRef(null); + // `selected` and `rows` change every render; keep the latest in refs + // so the document-level keydown listener can read them without being + // re-attached on every state change. + const selectedRef = useRef(selected); + const rowsRef = useRef(rows); + selectedRef.current = selected; + rowsRef.current = rows; // Reset selection when the detail target or table switches — otherwise // stale cells from a previous URL would remain highlighted. useEffect(() => { setSelected(new Set()); + setMenu(null); anchor.current = null; dragRef.current = null; }, [selectedUrlId, tableId]); @@ -755,6 +982,48 @@ function LinksView({ return () => document.removeEventListener('mouseup', onUp); }, []); + // Ctrl/Cmd+C: copy the current cell selection to the clipboard as TSV. + // Listening at the document level means the user doesn't have to focus + // the table first — pressing the shortcut while there's at least one + // selected cell in this LinksView is enough. + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + if (!(e.ctrlKey || e.metaKey)) return; + if (e.key !== 'c' && e.key !== 'C') return; + const sel = selectedRef.current; + if (sel.size === 0) return; + // Don't override copy when the user is in an input/textarea — they + // probably want the input's selection, not ours. + const target = e.target as HTMLElement | null; + const tag = target?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || target?.isContentEditable) { + return; + } + e.preventDefault(); + void copyCellsToClipboard(sel, rowsRef.current); + }; + document.addEventListener('keydown', onKey); + return () => document.removeEventListener('keydown', onKey); + }, []); + + // Close the context menu on outside click / scroll / escape so it + // doesn't get left dangling when the user clicks elsewhere. + useEffect(() => { + if (!menu) return; + const close = () => setMenu(null); + const onKey = (e: KeyboardEvent) => { + if (e.key === 'Escape') setMenu(null); + }; + document.addEventListener('mousedown', close); + document.addEventListener('scroll', close, true); + document.addEventListener('keydown', onKey); + return () => { + document.removeEventListener('mousedown', close); + document.removeEventListener('scroll', close, true); + document.removeEventListener('keydown', onKey); + }; + }, [menu]); + const getWidth = (c: LinksColumn): number => colWidths[c.id] ?? c.width; const totalWidth = columns.reduce((n, c) => n + getWidth(c), 0); @@ -929,8 +1198,12 @@ function LinksView({ applyColumnDrag(c); }; + const menuClickedCell = + menu !== null ? rows[menu.row]?.[menu.col] ?? '' : ''; + const menuClickedIsUrl = isUrlLike(menuClickedCell); + return ( -
+
Showing {shown.toLocaleString()} of{' '} {total.toLocaleString()} @@ -1013,6 +1286,18 @@ function LinksView({ onMouseEnter={() => { if (dragRef.current?.kind === 'cell') applyCellDrag(ri, ci); }} + onContextMenu={(e) => { + e.preventDefault(); + // If this cell wasn't already in the selection, + // promote it so the menu's "Copy" scope matches + // the right-clicked cell — matches main-table + // behaviour. + if (!selected.has(cellKey(ri, ci))) { + setSelected(new Set([cellKey(ri, ci)])); + anchor.current = { r: ri, c: ci }; + } + setMenu({ x: e.clientX, y: e.clientY, row: ri, col: ci }); + }} title={cell} > @@ -1027,10 +1312,239 @@ function LinksView({
)} + {menu && ( + { + void copyCellsToClipboard(selected, rows); + setMenu(null); + }} + onCopyValue={() => { + void navigator.clipboard.writeText(menuClickedCell); + setMenu(null); + }} + onCopyUrls={() => { + const urls = collectUrlsFromSelection(selected, rows); + if (urls.length > 0) { + void writeTextToClipboard(urls.join('\n')); + } + setMenu(null); + }} + onOpen={() => { + void window.open(menuClickedCell, '_blank'); + setMenu(null); + }} + onClose={() => setMenu(null)} + /> + )}
); } +/** + * In-page context menu for the Inlinks/Outlinks tables. Lives inside the + * LinksView so it can read the same `selected` set without prop-drilling + * an entire menu component. + */ +function CellContextMenu({ + x, + y, + selectionSize, + clickedValue, + clickedIsUrl, + urlCountInSelection, + onCopy, + onCopyValue, + onCopyUrls, + onOpen, + onClose, +}: { + x: number; + y: number; + selectionSize: number; + clickedValue: string; + clickedIsUrl: boolean; + urlCountInSelection: number; + onCopy: () => void; + onCopyValue: () => void; + onCopyUrls: () => void; + onOpen: () => void; + onClose: () => void; +}) { + const items: { label: string; action: () => void; disabled?: boolean }[] = [ + { + label: + selectionSize > 1 + ? `Copy ${selectionSize.toLocaleString()} Cells` + : 'Copy Cell', + action: onCopy, + disabled: selectionSize === 0, + }, + ]; + if (clickedIsUrl) { + // When the right-clicked URL is part of a multi-cell selection + // that spans multiple URL-looking cells, "Copy URL" copies ALL of + // them (one per line). Single-cell selection collapses to the + // legacy "copy this one URL" behaviour. + if (urlCountInSelection > 1) { + items.push({ + label: `Copy ${urlCountInSelection.toLocaleString()} URLs`, + action: onCopyUrls, + }); + } else { + items.push({ label: 'Copy URL', action: onCopyValue }); + } + items.push({ label: 'Open in Browser', action: onOpen }); + } else if (clickedValue) { + items.push({ label: 'Copy Value', action: onCopyValue }); + } + + return ( +
e.stopPropagation()} + onContextMenu={(e) => { + e.preventDefault(); + onClose(); + }} + > + {items.map((it, i) => ( + + ))} +
+ ); +} + +function isUrlLike(s: string): boolean { + return /^https?:\/\//i.test(s.trim()); +} + +/** + * Pull every URL-looking cell value out of the current selection, + * de-duplicate while preserving first-seen order (row → column), and + * return them as an array. Used by the right-click "Copy N URLs" + * action so a multi-cell selection in the From / To columns produces + * one URL per line on the clipboard. + */ +function collectUrlsFromSelection( + selected: Set, + rows: string[][], +): string[] { + if (selected.size === 0) return []; + const keys = [...selected].sort((a, b) => { + const [ra, ca] = a.split(':').map((n) => Number(n)); + const [rb, cb] = b.split(':').map((n) => Number(n)); + return (ra ?? 0) - (rb ?? 0) || (ca ?? 0) - (cb ?? 0); + }); + const seen = new Set(); + const out: string[] = []; + for (const k of keys) { + const [rs, cs] = k.split(':'); + const r = Number(rs); + const c = Number(cs); + if (!Number.isFinite(r) || !Number.isFinite(c)) continue; + const value = rows[r]?.[c] ?? ''; + if (isUrlLike(value) && !seen.has(value)) { + seen.add(value); + out.push(value); + } + } + return out; +} + +/** + * Write `text` to the OS clipboard with the same fallback chain as + * `copyCellsToClipboard` — async API first, hidden textarea + + * execCommand if the API is unavailable or permission-denied. + */ +async function writeTextToClipboard(text: string): Promise { + try { + await navigator.clipboard.writeText(text); + } catch { + const ta = document.createElement('textarea'); + ta.value = text; + ta.style.position = 'fixed'; + ta.style.opacity = '0'; + document.body.appendChild(ta); + ta.select(); + try { + document.execCommand('copy'); + } finally { + document.body.removeChild(ta); + } + } +} + +/** + * Build a TSV string from the selected cells of a 2-D table and write it + * to the OS clipboard. Cells are grouped by row (preserving the row + * order shown in the UI) and within a row by ascending column index, so + * paste-ing into a spreadsheet drops cells into matching grid positions. + */ +async function copyCellsToClipboard( + selected: Set, + rows: string[][], +): Promise { + if (selected.size === 0) return; + // Group selected cells by row index → list of column indexes. + const byRow = new Map(); + for (const k of selected) { + const [rs, cs] = k.split(':'); + const r = Number(rs); + const c = Number(cs); + if (!Number.isFinite(r) || !Number.isFinite(c)) continue; + const list = byRow.get(r); + if (list) list.push(c); + else byRow.set(r, [c]); + } + const sortedRows = [...byRow.keys()].sort((a, b) => a - b); + const lines: string[] = []; + for (const r of sortedRows) { + const row = rows[r]; + if (!row) continue; + const cols = (byRow.get(r) ?? []).sort((a, b) => a - b); + lines.push(cols.map((c) => row[c] ?? '').join('\t')); + } + const text = lines.join('\n'); + try { + await navigator.clipboard.writeText(text); + } catch { + // Clipboard API can fail (e.g. when window not focused); fall back + // to a hidden textarea + execCommand which works in Electron even + // when the clipboard permission is unset. + const ta = document.createElement('textarea'); + ta.value = text; + ta.style.position = 'fixed'; + ta.style.opacity = '0'; + document.body.appendChild(ta); + ta.select(); + try { + document.execCommand('copy'); + } finally { + document.body.removeChild(ta); + } + } +} + function SerpSnippet({ row }: { row: CrawlUrlRow }) { const title = row.title ?? '(no title)'; const desc = row.metaDescription ?? '(no meta description)'; @@ -1952,6 +2466,349 @@ function ResourcesView({ ); } +interface MultiPage { + id: number; + row: CrawlUrlRow; +} + +/** + * Multi-URL Images view — used when the user has 2+ rows selected in the + * main table. Fetches `` references for each page in parallel and + * shows them in a single flat table with a leading Page column so the + * source URL of each image is unambiguous. + */ +function MultiImagesView({ pages }: { pages: MultiPage[] }) { + const [byPage, setByPage] = useState>(new Map()); + const [loading, setLoading] = useState(false); + const key = pages.map((p) => p.id).join(','); + + useEffect(() => { + if (pages.length === 0) { + setByPage(new Map()); + return; + } + let cancelled = false; + setLoading(true); + const load = async () => { + const results = await Promise.all( + pages.map((p) => + window.freecrawl + .urlPageImages({ id: p.id }) + .then((r): [number, UrlPageImageRow[]] => [p.id, r.rows]) + .catch((): [number, UrlPageImageRow[]] => [p.id, []]), + ), + ); + if (cancelled) return; + setByPage(new Map(results)); + setLoading(false); + }; + void load(); + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [key]); + + if (loading && byPage.size === 0) { + return
Loading images…
; + } + let total = 0; + let missingAlt = 0; + let externalCount = 0; + let largeCount = 0; + const LARGE_BYTES = 102_400; + for (const rows of byPage.values()) { + total += rows.length; + for (const r of rows) { + if (r.alt === null) missingAlt++; + if (!r.isInternal) externalCount++; + if (r.byteSize !== null && r.byteSize > LARGE_BYTES) largeCount++; + } + } + + if (total === 0) { + return ( +
+ No <img> tags discovered across the {pages.length} selected pages. +
+ ); + } + + return ( +
+
+ + {total.toLocaleString()}{' '} + images across {pages.length}{' '} + pages + + {missingAlt > 0 && {missingAlt} missing alt} + {externalCount > 0 && ( + + {externalCount} external + + )} + {largeCount > 0 && ( + {largeCount} > 100 KB + )} +
+
+ + + + + + + + + + + + + + {pages.map((p) => { + const rows = byPage.get(p.id) ?? []; + return rows.map((r, i) => ( + + + + + + + + + + )); + })} + +
PageSourceAltWHSizeScope
+ {p.row.url} + + { + e.preventDefault(); + void window.open(r.src, '_blank'); + }} + className="text-blue-300 hover:text-blue-200" + > + {r.src} + + + {r.alt === null ? ( + + missing + + ) : r.alt === '' ? ( + + empty + + ) : ( + r.alt + )} + + {r.width ?? '—'} + + {r.height ?? '—'} + LARGE_BYTES + ? 'text-amber-400' + : 'text-surface-400', + )} + > + {r.byteSize === null ? '—' : formatBytesShort(r.byteSize)} + + {r.isInternal ? 'internal' : 'external'} +
+
+
+ ); +} + +/** + * Multi-URL Resources view — fetches each page's stored body snapshot in + * parallel, runs the same `extractResources` parser per page, and merges + * everything into one flat table with a leading Page column. + */ +function MultiResourcesView({ pages }: { pages: MultiPage[] }) { + const [byPage, setByPage] = useState>(new Map()); + const [missingSnapshots, setMissingSnapshots] = useState(0); + const [loading, setLoading] = useState(false); + const [filter, setFilter] = useState< + 'all' | 'script' | 'stylesheet' | 'font' | 'iframe' | 'external' + >('all'); + const key = pages.map((p) => p.id).join(','); + + useEffect(() => { + if (pages.length === 0) { + setByPage(new Map()); + return; + } + let cancelled = false; + setLoading(true); + const load = async () => { + const results = await Promise.all( + pages.map((p) => + window.freecrawl + .urlSourceGet({ id: p.id }) + .then((r): [number, ResourceEntry[] | null] => { + if (!r || r.body === null) return [p.id, null]; + return [p.id, extractResources(r.body, p.row.url)]; + }) + .catch((): [number, ResourceEntry[] | null] => [p.id, null]), + ), + ); + if (cancelled) return; + const map = new Map(); + let missing = 0; + for (const [id, entries] of results) { + if (entries === null) missing++; + else map.set(id, entries); + } + setByPage(map); + setMissingSnapshots(missing); + setLoading(false); + }; + void load(); + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [key]); + + if (loading && byPage.size === 0 && missingSnapshots === 0) { + return
Loading resources…
; + } + + const allEntries: { page: MultiPage; entry: ResourceEntry }[] = []; + for (const p of pages) { + const list = byPage.get(p.id); + if (!list) continue; + for (const entry of list) allEntries.push({ page: p, entry }); + } + + const filtered = + filter === 'all' + ? allEntries + : filter === 'external' + ? allEntries.filter((x) => x.entry.isExternal) + : allEntries.filter((x) => x.entry.type === filter); + const counts = { + all: allEntries.length, + script: allEntries.filter((x) => x.entry.type === 'script').length, + stylesheet: allEntries.filter((x) => x.entry.type === 'stylesheet').length, + font: allEntries.filter((x) => x.entry.type === 'font').length, + iframe: allEntries.filter((x) => x.entry.type === 'iframe').length, + external: allEntries.filter((x) => x.entry.isExternal).length, + }; + const FILTERS: { key: typeof filter; label: string; count: number }[] = [ + { key: 'all', label: 'All', count: counts.all }, + { key: 'script', label: 'Scripts', count: counts.script }, + { key: 'stylesheet', label: 'Stylesheets', count: counts.stylesheet }, + { key: 'font', label: 'Fonts', count: counts.font }, + { key: 'iframe', label: 'Iframes', count: counts.iframe }, + { key: 'external', label: 'External (3rd-party)', count: counts.external }, + ]; + + if (allEntries.length === 0 && missingSnapshots === pages.length) { + return ( +
+ Resources view requires a stored HTML body snapshot for each page. +
+ Re-crawl with the storeBodySnapshots setting enabled. +
+
+ ); + } + + return ( +
+
+ {FILTERS.map((f) => ( + + ))} + {missingSnapshots > 0 && ( + + {missingSnapshots} of {pages.length} pages have no stored body + + )} +
+
+ {filtered.length === 0 ? ( +
No resources match this filter.
+ ) : ( + + + + + + + + + + + {filtered.map(({ page, entry }, i) => ( + + + + + + + ))} + +
PageTypeURL3rd-party
+ {page.row.url} + + {entry.type} + + { + e.preventDefault(); + void window.open(entry.url, '_blank'); + }} + className="text-blue-300 hover:text-blue-200" + > + {entry.url} + + + {entry.isExternal ? ( + ✓ + ) : ( + — + )} +
+ )} +
+
+ ); +} + function ExtractedDataView({ row }: { row: CrawlUrlRow }) { const extraction = row.extractionResults ? safeJsonParse(row.extractionResults) diff --git a/apps/desktop/src/renderer/src/components/OverviewSidebar.tsx b/apps/desktop/src/renderer/src/components/OverviewSidebar.tsx index 061d7cd..4c013de 100644 --- a/apps/desktop/src/renderer/src/components/OverviewSidebar.tsx +++ b/apps/desktop/src/renderer/src/components/OverviewSidebar.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from 'react'; +import { memo, useEffect, useMemo, useRef, useState } from 'react'; import { ChevronDown, ChevronRight } from 'lucide-react'; import clsx from 'clsx'; import type { OverviewCounts, UrlCategory } from '@freecrawl/shared-types'; @@ -16,7 +16,13 @@ export function OverviewSidebar() { const overview = useAppStore((s) => s.overview); const setOverview = useAppStore((s) => s.setOverview); const dataVersion = useAppStore((s) => s.dataVersion); - const progress = useAppStore((s) => s.progress); + // Subscribe to scalar fields of `progress` rather than the whole + // object. Zustand re-renders this component only when these specific + // values change — so the 5/s progress events that don't move the + // crawled count (the in-flight pending fluctuates every poll) no + // longer trigger a full sidebar re-render. + const progressRunning = useAppStore((s) => s.progress?.running ?? false); + const progressCrawled = useAppStore((s) => s.progress?.crawled ?? 0); const activeCategory = useAppStore((s) => s.activeCategory); const navigateToCategory = useAppStore((s) => s.navigateToCategory); const [expanded, setExpanded] = useState>( @@ -84,13 +90,13 @@ export function OverviewSidebar() { // The crawler's `progress` event also bumps `progress.crawled` // which triggers a separate progress-driven refetch below, so // this interval is just a safety net. - const intervalMs = progress?.running ? 3000 : 30_000; + const intervalMs = progressRunning ? 3000 : 30_000; const id = setInterval(scheduleLoad, intervalMs); return () => { cancelled = true; clearInterval(id); }; - }, [dataVersion, setOverview, progress?.running]); + }, [dataVersion, setOverview, progressRunning]); // Progress-driven refetch. When the crawler reports a meaningful // change (every 50 URLs crawled) we trigger an immediate sidebar @@ -99,12 +105,12 @@ export function OverviewSidebar() { // twice if React re-renders for an unrelated reason. const lastRefetchAtRef = useRef(0); useEffect(() => { - if (!progress?.running) return; - const bucket = Math.floor((progress.crawled ?? 0) / 50); + if (!progressRunning) return; + const bucket = Math.floor(progressCrawled / 50); if (bucket === lastRefetchAtRef.current) return; lastRefetchAtRef.current = bucket; void window.freecrawl.overviewGet().then((o) => setOverview(o)); - }, [progress?.crawled, progress?.running, setOverview]); + }, [progressCrawled, progressRunning, setOverview]); const toggle = (k: string) => { setExpanded((prev) => { @@ -115,7 +121,9 @@ export function OverviewSidebar() { }); }; - const tree = buildTree(overview); + // The tree itself only depends on `overview`. Memoising it skips the + // 150-node object construction on every progress-driven render. + const tree = useMemo(() => buildTree(overview), [overview]); const totalForPercent = overview?.summary.totalInternalUrls ?? 0; return ( @@ -149,7 +157,7 @@ export function OverviewSidebar() { ); } -function TreeNode({ +const TreeNode = memo(function TreeNode({ node, depth, expanded, @@ -219,7 +227,7 @@ function TreeNode({ ))} ); -} +}); function buildTree(o: OverviewCounts | null): Node[] { if (!o) return []; @@ -1106,6 +1114,12 @@ function buildTree(o: OverviewCounts | null): Node[] { count: o.issues.paginationBroken, category: 'issues:pagination-broken', }, + { + key: 'issues-pagination-sequence-break', + label: 'Sequence Break (gap in numbering)', + count: o.issues.paginationSequenceBreak, + category: 'issues:pagination-sequence-break', + }, ], }, { @@ -1142,6 +1156,12 @@ function buildTree(o: OverviewCounts | null): Node[] { count: o.issues.hreflangTargetIssues, category: 'issues:hreflang-target-issues', }, + { + key: 'issues-hreflang-inconsistent-lang', + label: 'Inconsistent Lang (same lang, two hrefs)', + count: o.issues.hreflangInconsistentLang, + category: 'issues:hreflang-inconsistent-lang', + }, ], }, { @@ -1232,6 +1252,12 @@ function buildTree(o: OverviewCounts | null): Node[] { count: o.issues.externalLinksTooMany, category: 'issues:external-links-too-many', }, + { + key: 'issues-links-per-page-too-many', + label: 'Total Links per Page > 100', + count: o.issues.linksPerPageTooMany, + category: 'issues:links-per-page-too-many', + }, { key: 'issues-outlinks-zero', label: 'No Outlinks (Dead-End)', diff --git a/apps/desktop/src/renderer/src/components/SettingsDialog.tsx b/apps/desktop/src/renderer/src/components/SettingsDialog.tsx index 844fabd..6e011d4 100644 --- a/apps/desktop/src/renderer/src/components/SettingsDialog.tsx +++ b/apps/desktop/src/renderer/src/components/SettingsDialog.tsx @@ -10,6 +10,7 @@ import { Cpu, Copy, Code2, + Cookie, Webhook, Plus, Trash2, @@ -17,6 +18,9 @@ import { Network, Sparkles, Gauge, + FileText, + AlertTriangle, + Wrench, type LucideIcon, } from 'lucide-react'; import clsx from 'clsx'; @@ -87,6 +91,33 @@ interface FormState { proxyUrl: string; excludeExtensionsText: string; maxRedirects: string; + // Wave 8 additions + // crawl analysis (per-pass post-crawl toggles, Wave 6) + analyseInlinks: boolean; + analyseRedirectChains: boolean; + analyseHreflang: boolean; + analyseDuplicates: boolean; + analysePagination: boolean; + analyseIssues: boolean; + // content (body snapshot) + storeBodySnapshots: boolean; + bodySnapshotMaxBytes: string; + // advanced (Wave 3 caps + URL/query thresholds) + maxLinksPerPage: string; + maxResponseTimeMs: string; + maxFileSizeBytes: string; + maxUrlLength: string; + maxQueryStringLength: string; + maxFolderDepth: string; + followCanonicals: boolean; + followPaginationLinks: boolean; + followNofollow: boolean; + followJsRedirects: boolean; + // Wave 9 — Cookies / per-host UA / proxy profiles + cookiePolicy: 'reject-all' | 'accept-all' | 'block-third-party'; + perHostUserAgents: { hostPattern: string; userAgent: string }[]; + proxyProfiles: { name: string; url: string }[]; + proxyProfileActive: string; } type SectionKey = @@ -96,6 +127,7 @@ type SectionKey = | 'speed' | 'requests' | 'filters' + | 'content' | 'custom-search' | 'custom-extraction' | 'url-rewriting' @@ -103,7 +135,12 @@ type SectionKey = | 'auth' | 'network' | 'hardware' - | 'webhook'; + | 'webhook' + | 'crawl-analysis' + | 'issues' + | 'advanced' + | 'cookies' + | 'per-host-ua'; interface SectionDef { key: SectionKey; @@ -201,6 +238,45 @@ const SECTIONS: SectionDef[] = [ icon: Webhook, keywords: 'webhook notify slack discord zapier post crawl complete', }, + { + key: 'content', + label: 'Content', + icon: FileText, + keywords: + 'content body source snapshot store html size cap thin word count text view source', + }, + { + key: 'crawl-analysis', + label: 'Crawl Analysis', + icon: ListChecks, + keywords: + 'analysis post crawl inlinks redirect hreflang duplicate pagination issues materialise pass toggle', + }, + { + key: 'issues', + label: 'Issues', + icon: AlertTriangle, + keywords: 'issues check filter enable disable severity false positive', + }, + { + key: 'advanced', + label: 'Advanced', + icon: Wrench, + keywords: + 'advanced max links per page response time file size url length query folder depth follow canonical pagination nofollow js redirect', + }, + { + key: 'cookies', + label: 'Cookies', + icon: Cookie, + keywords: 'cookie session reject accept block third party set-cookie policy', + }, + { + key: 'per-host-ua', + label: 'Per-Host UA', + icon: Send, + keywords: 'per host user agent subdomain mobile desktop pattern wildcard', + }, ]; function configToForm(c: CrawlConfig): FormState { @@ -243,6 +319,28 @@ function configToForm(c: CrawlConfig): FormState { proxyUrl: c.proxyUrl ?? '', excludeExtensionsText: (c.excludeExtensions ?? []).join(', '), maxRedirects: String(c.maxRedirects ?? 10), + analyseInlinks: c.analyseInlinks ?? true, + analyseRedirectChains: c.analyseRedirectChains ?? true, + analyseHreflang: c.analyseHreflang ?? true, + analyseDuplicates: c.analyseDuplicates ?? true, + analysePagination: c.analysePagination ?? true, + analyseIssues: c.analyseIssues ?? true, + storeBodySnapshots: c.storeBodySnapshots ?? true, + bodySnapshotMaxBytes: String(c.bodySnapshotMaxBytes ?? 1_048_576), + maxLinksPerPage: String(c.maxLinksPerPage ?? 100), + maxResponseTimeMs: String(c.maxResponseTimeMs ?? 0), + maxFileSizeBytes: String(c.maxFileSizeBytes ?? 0), + maxUrlLength: String(c.maxUrlLength ?? 2048), + maxQueryStringLength: String(c.maxQueryStringLength ?? 0), + maxFolderDepth: String(c.maxFolderDepth ?? 0), + followCanonicals: c.followCanonicals ?? false, + followPaginationLinks: c.followPaginationLinks ?? true, + followNofollow: c.followNofollow ?? false, + followJsRedirects: c.followJsRedirects ?? false, + cookiePolicy: c.cookiePolicy ?? 'reject-all', + perHostUserAgents: (c.perHostUserAgents ?? []).map((r) => ({ ...r })), + proxyProfiles: (c.proxyProfiles ?? []).map((p) => ({ ...p })), + proxyProfileActive: c.proxyProfileActive ?? '', }; } @@ -371,6 +469,47 @@ export function SettingsDialog({ open, onClose }: Props) { .map((s) => s.trim().toLowerCase().replace(/^\./, '')) .filter(Boolean), maxRedirects: Math.max(0, num(form.maxRedirects, config.maxRedirects)), + analyseInlinks: form.analyseInlinks, + analyseRedirectChains: form.analyseRedirectChains, + analyseHreflang: form.analyseHreflang, + analyseDuplicates: form.analyseDuplicates, + analysePagination: form.analysePagination, + analyseIssues: form.analyseIssues, + storeBodySnapshots: form.storeBodySnapshots, + bodySnapshotMaxBytes: Math.max( + 0, + num(form.bodySnapshotMaxBytes, config.bodySnapshotMaxBytes), + ), + maxLinksPerPage: Math.max(0, num(form.maxLinksPerPage, config.maxLinksPerPage)), + maxResponseTimeMs: Math.max( + 0, + num(form.maxResponseTimeMs, config.maxResponseTimeMs), + ), + maxFileSizeBytes: Math.max( + 0, + num(form.maxFileSizeBytes, config.maxFileSizeBytes), + ), + maxUrlLength: Math.max(0, num(form.maxUrlLength, config.maxUrlLength)), + maxQueryStringLength: Math.max( + 0, + num(form.maxQueryStringLength, config.maxQueryStringLength), + ), + maxFolderDepth: Math.max(0, num(form.maxFolderDepth, config.maxFolderDepth)), + followCanonicals: form.followCanonicals, + followPaginationLinks: form.followPaginationLinks, + followNofollow: form.followNofollow, + followJsRedirects: form.followJsRedirects, + cookiePolicy: form.cookiePolicy, + perHostUserAgents: form.perHostUserAgents + .map((r) => ({ + hostPattern: r.hostPattern.trim(), + userAgent: r.userAgent.trim(), + })) + .filter((r) => r.hostPattern && r.userAgent), + proxyProfiles: form.proxyProfiles + .map((p) => ({ name: p.name.trim(), url: p.url.trim() })) + .filter((p) => p.name && p.url), + proxyProfileActive: form.proxyProfileActive.trim(), }); onClose(); } @@ -511,6 +650,22 @@ export function SettingsDialog({ open, onClose }: Props) { {active === 'webhook' && ( )} + {active === 'content' && ( + + )} + {active === 'crawl-analysis' && ( + + )} + {active === 'issues' && } + {active === 'advanced' && ( + + )} + {active === 'cookies' && ( + + )} + {active === 'per-host-ua' && ( + + )} @@ -1415,6 +1570,88 @@ function NetworkPanel({ form, update }: PanelProps) { /> +
+
+ Saved Proxy Profiles +
+

+ Save multiple `(name, URL)` pairs and switch between them via + the dropdown below. The active profile overrides the Proxy URL + field and the HTTPS_PROXY env var. Empty selection falls back + to the Proxy URL above. +

+ +
+ {form.proxyProfiles.length === 0 && ( +
+ No saved profiles — add one below. +
+ )} + {form.proxyProfiles.map((p, i) => ( +
+ { + const next = [...form.proxyProfiles]; + next[i] = { ...p, name: e.target.value }; + update('proxyProfiles', next); + }} + /> + { + const next = [...form.proxyProfiles]; + next[i] = { ...p, url: e.target.value }; + update('proxyProfiles', next); + }} + /> + +
+ ))} +
+ +
+
+

+ How crawled HTML is stored on disk. Disable body snapshots to keep + the project file small when you don't need the View Source detail + tab; tighten the cap for sites with adversarially-large pages. +

+ +
+
+ Body Snapshots +
+ update('storeBodySnapshots', v)} + info="Drives the View Source detail tab. ~30–200 KB on disk per HTML page; turn off if you only need metadata and not full source viewing." + example="On for SEO audits where View Source matters; off for 1M-URL crawls where disk is tight." + /> +
+ update('bodySnapshotMaxBytes', v)} + info="Bodies over this are truncated and flagged. 1 MB covers the 99.9th percentile of HTML pages without letting one adversarial 50 MB page bloat the project file." + example="1048576 (1 MB) default; 524288 (512 KB) on tight disks; 0 to disable truncation entirely." + /> +
+
+ + ); +} + +function CrawlAnalysisPanel({ form, update }: PanelProps) { + return ( + <> +

+ Per-pass post-crawl analysis toggles. Each pass runs after the + HTTP fetch phase finishes and feeds different issue filters. Skip + passes you don't need to shave wall-clock on large crawls; the + related issue counters quietly read as 0 until the pass runs. +

+ +
+ update('analyseInlinks', v)} + info="Counts how many internal pages link to each URL. Drives the Most-Linked URLs report and the per-row Inlinks column." + /> + update('analyseRedirectChains', v)} + info="Walks 3xx redirect chains, fills `redirect_chain_length` / `redirect_loop`. Drives the 'Long Chain' and 'Redirect Loop' issues + the Redirects tab." + /> + update('analyseHreflang', v)} + info="Page A→B declared but B→A absent flags 'Reciprocity Missing'; same lang on two hrefs flags 'Inconsistent Lang'." + /> + update('analyseDuplicates', v)} + info="64-bit SimHash + LSH bucketing + Union-Find clustering on body shingles. Most expensive pass — typical 5–10 s on a 100k crawl." + /> + update('analysePagination', v)} + info="?page=1 / ?page=2 / ?page=4 → flags 'Sequence Break' on every member of the broken cluster." + /> + update('analyseIssues', v)} + info="Pre-computes Dead External Domain, Duplicate URL post-norm, Canonical Chain Multi-hop. Without this the sidebar shows 0 for those three." + /> +
+ + ); +} + +function IssuesPanel() { + return ( + <> +

+ Per-issue check on/off toggles. +

+
+ Coming in V2. Today every issue check runs + unconditionally and surfaces in the sidebar. The plan is to let + you silence specific checks per-project (e.g. disable + "Description = Title" on a CMS that's known to do it + intentionally). Until that ships, hide rows you don't care about + by collapsing the sidebar group, or filter them out via the + Advanced filter on each tab. +
+ + ); +} + +function AdvancedPanel({ form, update }: PanelProps) { + return ( + <> +

+ Lower-level caps and link-follow toggles. Defaults are tuned for + typical SEO audits — only touch these if you know why. +

+ +
+
+ Link & Response Caps +
+ update('maxLinksPerPage', v)} + info="Pages with > this many outgoing links (internal + external) trip the 'Total Links per Page' issue. Google's historic recommendation is 100; mega-menus/hub-pages routinely blow past this." + example="100 default; 50 for tight on-page link discipline; 0 to disable the issue." + /> + update('maxResponseTimeMs', v)} + info="Aborts requests whose total lifetime (connect + headers + body) exceeds this. Distinct from `requestTimeoutMs` which is the headers timeout. Useful for capping individual slow pages without lowering the overall fetch timeout." + example="60000 (1 minute) for huge resources; 0 to rely solely on the fetch timeout." + /> + update('maxFileSizeBytes', v)} + info="Skips body parsing for pages whose Content-Length header exceeds this. The page row is still created so links to it aren't lost; only body parsing and source snapshot capture are skipped." + example="10485760 (10 MB) on bandwidth-tight crawls; 0 to download anything." + /> +
+ +
+
+ URL Structure Thresholds +
+ update('maxUrlLength', v)} + info="Trips the 'URL Too Long' issue when LENGTH(url) > this. RFC 7230 doesn't mandate a max but most servers + middleboxes fail above ~2 KB; Chrome itself caps at ~32 KB." + example="2048 default (RFC-suggested practical ceiling)." + /> + update('maxQueryStringLength', v)} + info="Trips 'Long Query String' when LENGTH(query) > this. Typical session-id sprawl + UTM tracking hits 100+ chars; over 200 starts to look like a bug." + example="100 default for most audits; 0 to disable the check." + /> + update('maxFolderDepth', v)} + info="Trips 'Folder Depth Too Deep' when the URL path's `/`-segment count exceeds this. Useful for spotting over-nested URL structures that bury content from crawlers." + example="4 default; 6 on documentation sites with deep TOC trees; 0 to disable." + /> +
+ +
+
+ Link Follow Behaviour +
+ update('followCanonicals', v)} + info="When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint." + /> + update('followPaginationLinks', v)} + info="When on (default), pagination_next + pagination_prev URLs are post-fetch enqueued. Off only to debug pagination-only loops without disabling all link follow." + /> + update('followNofollow', v)} + info="When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default." + /> + update('followJsRedirects', v)} + info="When on, `` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope." + /> +
+ + ); +} + +function CookiesPanel({ form, update }: PanelProps) { + return ( + <> +

+ Cookie policy applied to every fetch. The crawler is otherwise + stateless across requests; this setting controls whether + Set-Cookie response headers are recorded for the cookie-flag + issue checks (Missing Secure / HttpOnly / SameSite). +

+ +
+ +

+ Cookie values themselves are never stored in + the project file regardless of this setting — only the security + flag (Secure / HttpOnly / SameSite) counts are kept. +

+
+ + ); +} + +function PerHostUaPanel({ form, update }: PanelProps) { + const rules = form.perHostUserAgents; + return ( + <> +

+ Override the User-Agent on a per-host basis. Useful when crawling + a mobile subdomain with the mobile-Googlebot UA in the same run + as the desktop site, or when a CDN serves a different page based + on the requester's UA. The first matching pattern wins; the + global User-Agent (Requests tab) is the fallback. +

+
+

+ Pattern syntax: exact host (m.example.com) or + leading wildcard (*.example.com) — the wildcard + form matches any subdomain but not the apex. +

+
+ {rules.length === 0 && ( +
+ No per-host overrides yet — add one below. +
+ )} + {rules.map((r, i) => ( +
+ { + const next = [...rules]; + next[i] = { ...r, hostPattern: e.target.value }; + update('perHostUserAgents', next); + }} + /> + { + const next = [...rules]; + next[i] = { ...r, userAgent: e.target.value }; + update('perHostUserAgents', next); + }} + /> + +
+ ))} +
+ +
+ + ); +} + function FieldLabel({ label, info, diff --git a/apps/desktop/src/renderer/src/components/TopBar.tsx b/apps/desktop/src/renderer/src/components/TopBar.tsx index 165a61a..9e1285e 100644 --- a/apps/desktop/src/renderer/src/components/TopBar.tsx +++ b/apps/desktop/src/renderer/src/components/TopBar.tsx @@ -15,10 +15,23 @@ const SCOPE_OPTIONS: { value: CrawlScope; label: string; hint: string }[] = [ export function TopBar() { const config = useAppStore((s) => s.config); const setConfig = useAppStore((s) => s.setConfig); - const progress = useAppStore((s) => s.progress); + // Scalar subscriptions instead of the full `progress` object — TopBar + // only needs a handful of fields, and Zustand bails out of the + // re-render when the SCALAR values are unchanged. With the full + // object subscription a new reference on every emitProgress() pinned + // this component to the renderer's hot path. + const running = useAppStore((s) => s.progress?.running === true); + const paused = useAppStore((s) => s.progress?.paused === true); + const progressDiscovered = useAppStore((s) => s.progress?.discovered ?? 0); + const progressCrawled = useAppStore((s) => s.progress?.crawled ?? 0); const setProgress = useAppStore((s) => s.setProgress); - const summary = useAppStore((s) => s.summary); - const overview = useAppStore((s) => s.overview); + const summaryTotal = useAppStore((s) => s.summary?.total ?? 0); + const overviewInternalTotal = useAppStore( + (s) => s.overview?.summary.totalInternalUrls ?? 0, + ); + const overviewExternalTotal = useAppStore( + (s) => s.overview?.summary.totalExternalUrls ?? 0, + ); const reset = useAppStore((s) => s.reset); const setError = useAppStore((s) => s.setError); const setSettingsOpen = useAppStore((s) => s.setSettingsOpen); @@ -27,24 +40,15 @@ export function TopBar() { const [scopeOpen, setScopeOpen] = useState(false); const [recentOpen, setRecentOpen] = useState(false); - const running = progress?.running === true; - const paused = progress?.paused === true; - // Clear is enabled whenever there's anything to wipe. We check four - // signals because each one alone is incomplete: - // - `progress.discovered/crawled` covers a fresh run - // - `summary.total` covers post-`done` state (progress may have - // been wiped to its idle baseline by then) - // - `overview.summary.totalInternalUrls` covers projects opened - // from disk via File → Open Recent (no progress event ever - // fired for those rows) - // - `paused` is a "running" sub-state where the table already has - // rows that the user might want to dump + // Clear is enabled whenever there's anything to wipe. Four signals + // because each one alone is incomplete (fresh run vs. post-`done` + // state vs. project opened from disk vs. paused crawl with rows). const hasData = - (progress?.discovered ?? 0) > 0 || - (progress?.crawled ?? 0) > 0 || - (summary?.total ?? 0) > 0 || - (overview?.summary.totalInternalUrls ?? 0) > 0 || - (overview?.summary.totalExternalUrls ?? 0) > 0; + progressDiscovered > 0 || + progressCrawled > 0 || + summaryTotal > 0 || + overviewInternalTotal > 0 || + overviewExternalTotal > 0; const activeScope = SCOPE_OPTIONS.find((o) => o.value === config.scope)!; async function start() { diff --git a/apps/desktop/src/renderer/src/components/VisualizationDialog.tsx b/apps/desktop/src/renderer/src/components/VisualizationDialog.tsx index e50c104..aa4910d 100644 --- a/apps/desktop/src/renderer/src/components/VisualizationDialog.tsx +++ b/apps/desktop/src/renderer/src/components/VisualizationDialog.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState } from 'react'; -import { X, RefreshCw, Sparkles, Settings2, RotateCcw } from 'lucide-react'; +import { X, RefreshCw, Sparkles, Settings2, RotateCcw, Download } from 'lucide-react'; import cytoscape, { type Core } from 'cytoscape'; /** @@ -42,6 +42,152 @@ function loadTuning(): VisTuning { } return DEFAULT_TUNING; } + +/** + * Trigger a download of `data` as `filename`. Anchor-click pattern + * works in Electron's renderer because chromium honours the + * `download` attribute and the in-app save flow doesn't need shell + * permissions for a Blob URL. URL is revoked after a short delay so + * Chromium has time to attach the download. + */ +function downloadBlob(data: Blob, filename: string): void { + const url = URL.createObjectURL(data); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + a.style.display = 'none'; + document.body.appendChild(a); + a.click(); + setTimeout(() => { + document.body.removeChild(a); + URL.revokeObjectURL(url); + }, 5_000); +} + +/** + * Export the current Cytoscape canvas as a high-DPI PNG. Cytoscape's + * `cy.png()` rasterises into the requested pixel dimensions (we use 2× + * the visible size for retina-quality print) and returns a base64 data + * URL which we convert to a Blob for the file download. + */ +function exportPng(cy: Core, filename = 'freecrawl-graph.png'): void { + const dataUrl = cy.png({ + output: 'base64uri', + full: true, // include nodes outside the current viewport + bg: '#0a0a0f', + scale: 2, + }); + // base64uri → Blob via fetch — cleaner than manual atob() decoding + // and supported in Chromium since forever. + void fetch(dataUrl) + .then((r) => r.blob()) + .then((b) => downloadBlob(b, filename)); +} + +/** + * Export the current canvas as SVG. We don't pull in cytoscape-svg as + * a dependency — Cytoscape's `jpg`/`png` API has no native SVG output, + * so we emit a minimal hand-rolled SVG: `` per node + `` + * per edge, using the runtime renderedPosition coordinates so the + * output matches what the user sees on screen. + * + * The result is editable in Illustrator / Inkscape / Figma — the + * primary use case for "give me an SVG of my graph". + */ +function exportSvg(cy: Core, filename = 'freecrawl-graph.svg'): void { + const bbox = cy.elements().boundingBox({}); + const pad = 40; + const width = Math.ceil(bbox.w + pad * 2); + const height = Math.ceil(bbox.h + pad * 2); + const tx = -bbox.x1 + pad; + const ty = -bbox.y1 + pad; + const parts: string[] = []; + parts.push( + ``, + ); + parts.push(``); + // Edges first so they sit beneath nodes. + cy.edges().forEach((e) => { + const src = e.source().position(); + const tgt = e.target().position(); + const c = String((e.style('line-color') as unknown) ?? '#475569'); + parts.push( + ``, + ); + }); + cy.nodes().forEach((n) => { + const p = n.position(); + const r = Number(n.style('width') ?? 12) / 2; + const fill = String((n.style('background-color') as unknown) ?? '#3b82f6'); + parts.push( + ``, + ); + }); + parts.push(``); + downloadBlob(new Blob([parts.join('\n')], { type: 'image/svg+xml' }), filename); +} + +function escapeXml(s: string): string { + return s + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +/** + * Export the graph as a self-contained HTML file. Embeds Cytoscape + * from a CDN + the snapshot data inline; the user can open it + * directly in any browser, share via email/Slack, or attach to a + * client report. No FreeCrawl runtime needed for the recipient. + * + * Trade-off: the resulting file requires internet to load Cytoscape + * the first time (CDN fetch). We could inline the library too at the + * cost of ~250 KB per export, which is reasonable for a one-shot + * deliverable but adds 5–10× to file size on most graphs. + */ +function exportStandaloneHtml( + cy: Core, + filename = 'freecrawl-graph.html', +): void { + const elements = cy.json() as { elements: unknown }; + const dataJson = JSON.stringify(elements.elements ?? []); + const html = ` + + + +FreeCrawl Graph + + + +
+
+

FreeCrawl SEO — Site Graph

+
Exported: ${new Date().toISOString()}
+
Nodes: ${cy.nodes().length} · Edges: ${cy.edges().length}
+
+ + + +`; + downloadBlob(new Blob([html], { type: 'text/html' }), filename); +} import type { AnchorTextRow, GraphSnapshotResult, @@ -117,6 +263,7 @@ export function VisualizationDialog({ open, onClose }: Props) { const [selectedUrl, setSelectedUrl] = useState(null); const [tuning, setTuning] = useState(() => loadTuning()); const [tunerOpen, setTunerOpen] = useState(false); + const [exportMenuOpen, setExportMenuOpen] = useState(false); function patchTuning(patch: Partial) { setTuning((prev) => { @@ -632,6 +779,55 @@ export function VisualizationDialog({ open, onClose }: Props) { /> )}
+
+ + {exportMenuOpen && ( +
setExportMenuOpen(false)} + > + + + +
+ )} +