feat: worker-thread architecture + freeze watchdog + multi-row aggregation

FreeCrawl-SEO-Tool-Start.bat

@@ -71,7 +71,7 @@ if not exist "packages\core\dist\index.js" set NEED_BUILD=1
 if "!NEED_BUILD!"=="1" goto DO_BUILD
 echo [OK] Paylasilan paketler hazir.
 echo.
-goto LAUNCH
+goto CHECK_DESKTOP_BUILD
 
 :DO_BUILD
 echo ------------------------------------------------------------
@@ -82,15 +82,47 @@ if errorlevel 1 goto ERR_BUILD
 echo [OK] Paketler hazir.
 echo.
 
+REM ---- 4b) Desktop uygulamasini production build et -------------------------
+REM Dev mod (electron-vite dev) ilk acilista 1700+ modulu on-demand transform
+REM ediyor ve 20-30 sn cold start suruyor. Production build'de tum moduller
+REM tek bir bundle'a derlenmis oldugu icin acilis 1-2 sn'ye iniyor.
+:CHECK_DESKTOP_BUILD
+set NEED_DESKTOP_BUILD=0
+if not exist "apps\desktop\out\main\index.js" set NEED_DESKTOP_BUILD=1
+if not exist "apps\desktop\out\preload\index.js" set NEED_DESKTOP_BUILD=1
+if not exist "apps\desktop\out\renderer\index.html" set NEED_DESKTOP_BUILD=1
+
+if "!NEED_DESKTOP_BUILD!"=="1" goto DO_DESKTOP_BUILD
+echo [OK] Desktop bundle hazir.
+echo.
+goto LAUNCH
+
+:DO_DESKTOP_BUILD
+echo ------------------------------------------------------------
+echo   Desktop uygulamasi build ediliyor (production)...
+echo   Bu islem yaklasik 10-15 sn surer ve sadece ilk calistirmada
+echo   (veya kod degisikligi sonrasi) yeniden yapilir.
+echo ------------------------------------------------------------
+call npm --workspace apps/desktop run build
+if errorlevel 1 goto ERR_BUILD
+echo [OK] Desktop bundle hazir.
+echo.
+
 REM ---- 5) Uygulamayi baslat -------------------------------------------------
 :LAUNCH
 echo ============================================================
-echo   FreeCrawl SEO Tool baslatiliyor (npm run dev)...
+echo   FreeCrawl SEO Tool baslatiliyor (production)...
 echo   Bu pencereyi KAPATMAYIN - uygulamanin yasam dongusu buna bagli.
 echo ============================================================
 echo.
 
-call npm run dev
+REM `node:sqlite` Node 24'te kararli ama hala "experimental" bayragi tasiyor
+REM ve modul yuklenirken ExperimentalWarning basiyor. NODE_NO_WARNINGS=1
+REM ile bu kozmetik uyariyi sustururuz; gercek hatalar (TypeError vs.)
+REM yine konsola yazilir.
+set NODE_NO_WARNINGS=1
+
+call npm --workspace apps/desktop run start
 
 echo.
 echo ------------------------------------------------------------

apps/desktop/electron.vite.config.ts

@@ -7,14 +7,20 @@ export default defineConfig({
     plugins: [externalizeDepsPlugin()],
     build: {
       rollupOptions: {
-        // Two entry points: the main process bootstrap, and the
-        // read-only DB worker that runs in a `worker_threads.Worker`.
-        // Both compile into `out/main/`; the Worker constructor in
-        // `db-reader-pool.ts` resolves the worker file relative to
-        // `__dirname` so it picks up the same bundled module.
+        // Five entry points: the main process bootstrap and four
+        // worker-thread bundles (DB reader, DB writer, HTML parser,
+        // freeze-watchdog). All compile into `out/main/`; each Worker
+        // constructor resolves its peer file relative to `__dirname`
+        // so they pick up the same bundle directory.
         input: {
           index: resolve(__dirname, 'src/main/index.ts'),
           'db-reader-worker': resolve(__dirname, 'src/main/db-reader-worker.ts'),
+          'db-writer-worker': resolve(__dirname, 'src/main/db-writer-worker.ts'),
+          'parser-worker': resolve(__dirname, 'src/main/parser-worker.ts'),
+          'freeze-watchdog-worker': resolve(
+            __dirname,
+            'src/main/freeze-watchdog-worker.ts',
+          ),
         },
       },
     },

apps/desktop/src/main/db-reader-pool.ts

@@ -35,7 +35,22 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 // electron-vite uses the same convention.
 const WORKER_PATH = path.join(__dirname, 'db-reader-worker.js');
 
-const REQUEST_TIMEOUT_MS = 15_000;
+// 30 s default leaves headroom for SQLite read contention during the
+// post-crawl recompute phase (recomputeUrlsIssues holds a writer
+// transaction that can keep reader queries blocked behind earlier
+// queued requests for several seconds at a time on large crawls).
+// Genuine worker hangs are still surfaced — just with a slightly
+// longer detection window than before.
+const REQUEST_TIMEOUT_MS = 30_000;
+// Heavy aggregate queries (the overview sidebar's 130-counter pass and
+// the post-crawl materialiser's counter fan-out) can run past the
+// default budget on million-URL projects, so they get their own
+// bigger ceiling.
+const HEAVY_REQUEST_TIMEOUT_MS = 60_000;
+const HEAVY_METHODS = new Set<string>([
+  'getOverviewCounts',
+  'getOverviewCountsAsync',
+]);
 const MAX_RESTARTS = 3;
 const RESTART_WINDOW_MS = 60_000;
 
@@ -55,23 +70,31 @@ interface ResponseMessage {
 class DbReaderPool {
   private worker: Worker | null = null;
   private dbPath: string | null = null;
+  private freezeWatchdogSab: SharedArrayBuffer | null = null;
   private nextRequestId = 1;
   private pending = new Map<number, PendingRequest>();
   private restartTimes: number[] = [];
   private terminated = false;
 
-  /** Spawn (or respawn) the worker pointed at `dbPath`. Idempotent. */
-  init(dbPath: string): void {
+  /** Spawn (or respawn) the worker pointed at `dbPath`. Idempotent.
+   *
+   *   `freezeWatchdogSab` — if provided, the worker will tick its
+   *   own heartbeat + publish its current method into this shared
+   *   buffer so the freeze-watchdog thread can detect a stuck
+   *   reader. Pass `null` to disable diagnostics. */
+  init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
     if (this.terminated) {
       throw new Error('DbReaderPool: cannot init after terminate()');
     }
+    this.freezeWatchdogSab = freezeWatchdogSab;
     if (this.dbPath === dbPath && this.worker !== null) return;
     this.dbPath = dbPath;
     this.spawn();
   }
 
   /** Switch the worker to a different .seoproject file. */
-  swap(newPath: string): void {
+  swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
+    if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab;
     if (this.dbPath === newPath && this.worker !== null) return;
     this.failPendingWith('reader-swapped');
     this.dbPath = newPath;
@@ -108,11 +131,14 @@ class DbReaderPool {
     if (this.terminated) return Promise.reject(new Error('reader-terminated'));
     if (!this.worker) return Promise.reject(new Error('reader-not-initialised'));
     const requestId = this.nextRequestId++;
+    const timeoutMs = HEAVY_METHODS.has(method)
+      ? HEAVY_REQUEST_TIMEOUT_MS
+      : REQUEST_TIMEOUT_MS;
     return new Promise<T>((resolve, reject) => {
       const timer = setTimeout(() => {
         this.pending.delete(requestId);
-        reject(new Error(`reader-timeout: ${method} > ${REQUEST_TIMEOUT_MS}ms`));
-      }, REQUEST_TIMEOUT_MS);
+        reject(new Error(`reader-timeout: ${method} > ${timeoutMs}ms`));
+      }, timeoutMs);
       this.pending.set(requestId, {
         resolve: resolve as (v: unknown) => void,
         reject,
@@ -133,7 +159,10 @@ class DbReaderPool {
     }
     try {
       const w = new Worker(WORKER_PATH, {
-        workerData: { dbPath: this.dbPath },
+        workerData: {
+          dbPath: this.dbPath,
+          freezeWatchdogSab: this.freezeWatchdogSab,
+        },
       });
       w.on('message', (msg: ResponseMessage) => this.handleResponse(msg));
       w.on('error', (err) => {

apps/desktop/src/main/db-reader-worker.ts

@@ -1,5 +1,6 @@
 import { parentPort, workerData } from 'node:worker_threads';
 import { ProjectDb } from '@freecrawl/db';
+import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js';
 
 /**
  * Read-only SQLite worker.
@@ -29,6 +30,7 @@ if (!parentPort) {
 
 interface InitData {
   dbPath: string;
+  freezeWatchdogSab?: SharedArrayBuffer | null;
 }
 
 const init = workerData as InitData;
@@ -38,6 +40,22 @@ if (!init?.dbPath) {
 
 const db = new ProjectDb(init.dbPath, { readOnly: true });
 
+// Freeze-watchdog plumbing. The reader thread publishes a heartbeat
+// every 250 ms and updates `readerOp` to the currently-dispatching
+// method name so the watchdog can spot a stuck SQLite query and
+// surface "[STALL:READER ... op=getUrlDetail]" in debug.txt.
+const watchdog = init.freezeWatchdogSab
+  ? new FreezeWatchdogSharedState(init.freezeWatchdogSab)
+  : null;
+let watchdogHeartbeatTimer: NodeJS.Timeout | null = null;
+if (watchdog) {
+  watchdog.tickReaderHeartbeat();
+  watchdog.setReaderOp('idle');
+  watchdogHeartbeatTimer = setInterval(() => {
+    watchdog.tickReaderHeartbeat();
+  }, 250);
+}
+
 interface RequestMessage {
   requestId: number;
   method: string;
@@ -81,6 +99,7 @@ const ALLOWED_METHODS = new Set<string>([
 parentPort.on('message', async (msg: RequestMessage) => {
   if (!msg || typeof msg.requestId !== 'number') return;
   const { requestId, method, args } = msg;
+  watchdog?.setReaderOp(method);
   try {
     if (!ALLOWED_METHODS.has(method)) {
       throw new Error(`db-reader-worker: method '${method}' is not whitelisted`);
@@ -99,6 +118,8 @@ parentPort.on('message', async (msg: RequestMessage) => {
       ok: false,
       error: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
     });
+  } finally {
+    watchdog?.setReaderOp('idle');
   }
 });
 
@@ -107,6 +128,7 @@ parentPort.on('message', async (msg: RequestMessage) => {
 // file handle (which on Windows would block the project from being
 // reopened until full process exit).
 parentPort.on('close', () => {
+  if (watchdogHeartbeatTimer) clearInterval(watchdogHeartbeatTimer);
   try {
     db.close();
   } catch {

apps/desktop/src/main/db-writer-pool.ts

@@ -0,0 +1,207 @@
+import { Worker } from 'node:worker_threads';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as logger from './logger.js';
+
+/**
+ * Single-worker pool for SQLite writes. Mirrors the structure of
+ * `db-reader-pool.ts` so the rest of the main process talks to both
+ * via the same `call<T>(method, args)` shape.
+ *
+ * The pool is intentionally a "pool of one": SQLite's writer model
+ * is single-writer-at-a-time. A second writer worker would just
+ * serialise behind the first via `busy_timeout`, gaining no
+ * parallelism. The wrapper exists to:
+ *   1. Move the synchronous `.run()` calls off the Electron main
+ *      thread so IPC + queue scheduling stay live during writes.
+ *   2. Give us one centralised place to add batching / rate-limiting
+ *      later (Phase 2 — micro-batching).
+ */
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const WORKER_PATH = path.join(__dirname, 'db-writer-worker.js');
+
+const REQUEST_TIMEOUT_MS = 60_000;
+const MAX_RESTARTS = 3;
+const RESTART_WINDOW_MS = 60_000;
+
+interface PendingRequest {
+  resolve: (v: unknown) => void;
+  reject: (e: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+}
+
+interface ResponseMessage {
+  requestId: number;
+  ok: boolean;
+  result?: unknown;
+  error?: string;
+}
+
+class DbWriterPool {
+  private worker: Worker | null = null;
+  private dbPath: string | null = null;
+  private freezeWatchdogSab: SharedArrayBuffer | null = null;
+  private nextRequestId = 1;
+  private pending = new Map<number, PendingRequest>();
+  private restartTimes: number[] = [];
+  private terminated = false;
+
+  init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
+    if (this.terminated) {
+      throw new Error('DbWriterPool: cannot init after terminate()');
+    }
+    this.freezeWatchdogSab = freezeWatchdogSab;
+    if (this.dbPath === dbPath && this.worker !== null) return;
+    this.dbPath = dbPath;
+    this.spawn();
+  }
+
+  swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
+    if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab;
+    if (this.dbPath === newPath && this.worker !== null) return;
+    this.failPendingWith('writer-swapped');
+    this.dbPath = newPath;
+    this.spawn();
+  }
+
+  isReady(): boolean {
+    return this.worker !== null && !this.terminated;
+  }
+
+  call<T>(method: string, args: unknown[] = []): Promise<T> {
+    if (this.terminated) return Promise.reject(new Error('writer-terminated'));
+    if (!this.worker) return Promise.reject(new Error('writer-not-initialised'));
+    const requestId = this.nextRequestId++;
+    return new Promise<T>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(requestId);
+        reject(new Error(`writer-timeout: ${method} > ${REQUEST_TIMEOUT_MS}ms`));
+      }, REQUEST_TIMEOUT_MS);
+      this.pending.set(requestId, {
+        resolve: resolve as (v: unknown) => void,
+        reject,
+        timer,
+      });
+      this.worker!.postMessage({ requestId, method, args });
+    });
+  }
+
+  async terminate(): Promise<void> {
+    this.terminated = true;
+    this.failPendingWith('writer-terminated');
+    if (this.worker) {
+      const w = this.worker;
+      this.worker = null;
+      try {
+        await w.terminate();
+      } catch {
+        /* already gone */
+      }
+    }
+  }
+
+  private spawn(): void {
+    if (!this.dbPath) return;
+    if (this.worker) {
+      const old = this.worker;
+      this.worker = null;
+      void old.terminate().catch(() => undefined);
+    }
+    try {
+      const w = new Worker(WORKER_PATH, {
+        workerData: {
+          dbPath: this.dbPath,
+          freezeWatchdogSab: this.freezeWatchdogSab,
+        },
+      });
+      w.on('message', (msg: ResponseMessage) => this.handleResponse(msg));
+      w.on('error', (err) => {
+        logger.log('error', 'main', `db-writer worker error: ${err.message}`);
+      });
+      w.on('exit', (code) => this.handleExit(code));
+      this.worker = w;
+      logger.log('info', 'main', `db-writer worker spawned for ${this.dbPath}`);
+    } catch (err) {
+      logger.log(
+        'error',
+        'main',
+        `db-writer worker spawn failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      this.worker = null;
+    }
+  }
+
+  private handleResponse(msg: ResponseMessage): void {
+    const pending = this.pending.get(msg.requestId);
+    if (!pending) return;
+    this.pending.delete(msg.requestId);
+    clearTimeout(pending.timer);
+    if (msg.ok) {
+      pending.resolve(msg.result);
+    } else {
+      pending.reject(new Error(msg.error ?? 'writer-unknown-error'));
+    }
+  }
+
+  private handleExit(code: number): void {
+    this.worker = null;
+    if (this.terminated) return;
+    if (code === 0) return;
+    this.failPendingWith('writer-crashed');
+    const now = Date.now();
+    this.restartTimes = this.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS);
+    if (this.restartTimes.length >= MAX_RESTARTS) {
+      logger.log(
+        'error',
+        'main',
+        `db-writer worker crashed ${MAX_RESTARTS}× within ${RESTART_WINDOW_MS}ms — giving up. Writes fall back to main-thread DB.`,
+      );
+      return;
+    }
+    this.restartTimes.push(now);
+    logger.log(
+      'warn',
+      'main',
+      `db-writer worker exited with code ${code}; respawning (${this.restartTimes.length}/${MAX_RESTARTS})`,
+    );
+    this.spawn();
+  }
+
+  private failPendingWith(reason: string): void {
+    for (const p of this.pending.values()) {
+      clearTimeout(p.timer);
+      p.reject(new Error(reason));
+    }
+    this.pending.clear();
+  }
+}
+
+export const dbWriterPool = new DbWriterPool();
+
+/**
+ * Helper: try the worker first, fall back to a synchronous main-thread
+ * call on any worker error. The fallback keeps writes happening even
+ * when the worker is crashed/restarting, at the cost of running on
+ * main for that one operation.
+ */
+export async function callWriterOrFallback<T>(
+  method: string,
+  args: unknown[],
+  fallback: () => T | Promise<T>,
+): Promise<T> {
+  if (!dbWriterPool.isReady()) {
+    return fallback();
+  }
+  try {
+    return await dbWriterPool.call<T>(method, args);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    logger.log(
+      'warn',
+      'main',
+      `'${method}' fell back to main-thread writer: ${msg}`,
+    );
+    return fallback();
+  }
+}

apps/desktop/src/main/db-writer-worker.ts

@@ -0,0 +1,129 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { ProjectDb } from '@freecrawl/db';
+import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js';
+
+/**
+ * Writer worker — owns a SQLite writer connection and runs every
+ * write the desktop main process used to run synchronously on its
+ * own event loop.
+ *
+ * Why a dedicated thread:
+ *   - `node:sqlite` is synchronous; each `.run()` blocks the JS
+ *     thread until SQLite returns. With 20 concurrent fetches all
+ *     inserting per-URL data, the main thread used to block ~250 ms
+ *     per second on writes alone — visible as IPC stalls.
+ *   - Moving the writer into a worker frees the main loop for IPC
+ *     dispatch and queue scheduling. The worker still serialises
+ *     writes one at a time (SQLite's single-writer model is
+ *     unchanged), but main never feels it.
+ *
+ *   Two writer connections (this worker + the main process's
+ *   ProjectDb instance, which is still used for some methods that
+ *   weren't worth refactoring) share the same DB file. WAL +
+ *   `busy_timeout` makes SQLite serialise the two cleanly — at the
+ *   small risk of a SQLITE_BUSY error if both fight for the lock
+ *   for longer than the timeout, which we treat as best-effort and
+ *   surface to the caller.
+ *
+ * Protocol:
+ *   main → worker:    `{ requestId, method, args }`
+ *   worker → main:    `{ requestId, ok: true,  result }`
+ *                  or `{ requestId, ok: false, error: string }`
+ */
+
+if (!parentPort) {
+  throw new Error('db-writer-worker: must be loaded via worker_threads');
+}
+
+interface InitData {
+  dbPath: string;
+  freezeWatchdogSab?: SharedArrayBuffer | null;
+}
+
+const init = workerData as InitData;
+if (!init?.dbPath) {
+  throw new Error('db-writer-worker: workerData.dbPath required');
+}
+
+const db = new ProjectDb(init.dbPath);
+// `busy_timeout` is set inside ProjectDb's constructor — no per-worker
+// override needed. We rely on it to serialise this writer connection
+// with the main process's writer instead of throwing SQLITE_BUSY when
+// both happen to want the lock at the same moment.
+
+const watchdog = init.freezeWatchdogSab
+  ? new FreezeWatchdogSharedState(init.freezeWatchdogSab)
+  : null;
+
+interface RequestMessage {
+  requestId: number;
+  method: string;
+  args: unknown[];
+}
+
+// Whitelist mirrors ProjectDb's write surface. Reading methods stay
+// off this list — those go to the read-only worker.
+const ALLOWED_METHODS = new Set<string>([
+  'writeFetchedUrl',
+  'upsertUrl',
+  'insertLinks',
+  'insertImages',
+  'setUrlHeaders',
+  'setUrlSource',
+  'setSitemapUrls',
+  'setHostCert',
+  'setImageSize',
+  'setMeta',
+  'updateExternalProbe',
+  'recomputeInlinks',
+  'recomputeRedirectChains',
+  'recomputeHreflangAnalysis',
+  'recomputeHreflangInconsistent',
+  'recomputeDuplicateClusters',
+  'recomputePaginationSequence',
+  'recomputeUrlsIssues',
+  'recomputeUrlsIssuesYielding',
+  'checkpointQueue',
+  'clearQueueCheckpoint',
+  'markUrlForRecrawl',
+  'markUrlsForRecrawl',
+  'deleteUrl',
+  'deleteUrls',
+  'reset',
+  'deleteByDomain',
+]);
+
+parentPort.on('message', async (msg: RequestMessage) => {
+  if (!msg || typeof msg.requestId !== 'number') return;
+  const { requestId, method, args } = msg;
+  if (watchdog) watchdog.setReaderOp('writer:' + method);
+  try {
+    if (!ALLOWED_METHODS.has(method)) {
+      throw new Error(`db-writer-worker: method '${method}' is not whitelisted`);
+    }
+    const fn = (db as unknown as Record<string, unknown>)[method];
+    if (typeof fn !== 'function') {
+      throw new Error(`db-writer-worker: method '${method}' missing on ProjectDb`);
+    }
+    const out = (fn as (...a: unknown[]) => unknown).apply(db, args ?? []);
+    const result = out instanceof Promise ? await out : out;
+    parentPort!.postMessage({ requestId, ok: true, result });
+  } catch (err) {
+    parentPort!.postMessage({
+      requestId,
+      ok: false,
+      error: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
+    });
+  } finally {
+    if (watchdog) watchdog.setReaderOp('idle');
+  }
+});
+
+parentPort.on('close', () => {
+  try {
+    db.close();
+  } catch {
+    /* already closed */
+  }
+  process.exit(0);
+});

apps/desktop/src/main/freeze-watchdog-shared.ts

@@ -0,0 +1,196 @@
+/**
+ * Shared SharedArrayBuffer layout for the freeze watchdog.
+ *
+ * Three execution contexts read/write to the same SAB:
+ *   1. The Electron main process (the writer for `mainHB`, counters,
+ *      `mainOp`, and rendererLag forwarded from IPC).
+ *   2. The db-reader worker thread (writes its own `readerHB` and
+ *      `readerOp`).
+ *   3. The freeze-watchdog worker thread (reads everything; writes
+ *      nothing — its job is purely observational).
+ *
+ * All scalar reads/writes use `Atomics.{load,store}` so the watchdog
+ * is guaranteed to see the latest value even if the writer is mid-
+ * function. The op-string regions are written non-atomically (a 256-
+ * byte memcpy isn't atomic on any CPU) — we accept torn reads there
+ * because the op string is best-effort context, not a correctness
+ * boundary; the heartbeat timestamp is what tells us whether a
+ * thread is alive, and that IS atomic.
+ *
+ * Layout (1024 bytes total, generous for future fields):
+ *   [0..32)    BigInt64Array(4)  — heartbeat timestamps + spare
+ *      [0]    main heartbeat (ms since epoch)
+ *      [1]    db-reader heartbeat
+ *      [2]    last renderer lag-report timestamp
+ *      [3]    spare
+ *   [32..96)   Int32Array(16)    — counters + op-string lengths
+ *      [0]    renderer lag (ms)
+ *      [1]    crawled
+ *      [2]    discovered
+ *      [3]    pending
+ *      [4]    failed
+ *      [5]    main op string byte length
+ *      [6]    reader op string byte length
+ *      [7..15] spare
+ *   [96..)     Uint8Array        — op string bytes (UTF-8)
+ *      [0..256)    main op
+ *      [256..512)  reader op
+ */
+
+const I64_BYTE_OFFSET = 0;
+const I64_LEN = 4;
+const I32_BYTE_OFFSET = 32;
+const I32_LEN = 16;
+const U8_BYTE_OFFSET = 96;
+
+const MAIN_OP_OFFSET_IN_U8 = 0;
+const READER_OP_OFFSET_IN_U8 = 256;
+const OP_MAX_BYTES = 255;
+
+const I64_IDX_MAIN_HB = 0;
+const I64_IDX_READER_HB = 1;
+const I64_IDX_RENDERER_REPORT_TS = 2;
+
+const I32_IDX_RENDERER_LAG = 0;
+const I32_IDX_CRAWLED = 1;
+const I32_IDX_DISCOVERED = 2;
+const I32_IDX_PENDING = 3;
+const I32_IDX_FAILED = 4;
+const I32_IDX_MAIN_OP_LEN = 5;
+const I32_IDX_READER_OP_LEN = 6;
+
+export const SAB_BYTE_LENGTH = 1024;
+
+export interface CounterPatch {
+  crawled?: number;
+  discovered?: number;
+  pending?: number;
+  failed?: number;
+}
+
+export interface CounterSnapshot {
+  crawled: number;
+  discovered: number;
+  pending: number;
+  failed: number;
+}
+
+/**
+ * Wraps a `SharedArrayBuffer` with typed accessors. Construct one in
+ * the main process via `FreezeWatchdogSharedState.create()`, then pass
+ * `state.sab` to each worker via `workerData` so all sides share the
+ * same memory.
+ */
+export class FreezeWatchdogSharedState {
+  private readonly i64: BigInt64Array;
+  private readonly i32: Int32Array;
+  private readonly u8: Uint8Array;
+
+  static create(): FreezeWatchdogSharedState {
+    return new FreezeWatchdogSharedState(new SharedArrayBuffer(SAB_BYTE_LENGTH));
+  }
+
+  constructor(public readonly sab: SharedArrayBuffer) {
+    this.i64 = new BigInt64Array(sab, I64_BYTE_OFFSET, I64_LEN);
+    this.i32 = new Int32Array(sab, I32_BYTE_OFFSET, I32_LEN);
+    this.u8 = new Uint8Array(sab, U8_BYTE_OFFSET, sab.byteLength - U8_BYTE_OFFSET);
+  }
+
+  // ── Writers ───────────────────────────────────────────────────────
+
+  tickMainHeartbeat(): void {
+    Atomics.store(this.i64, I64_IDX_MAIN_HB, BigInt(Date.now()));
+  }
+
+  tickReaderHeartbeat(): void {
+    Atomics.store(this.i64, I64_IDX_READER_HB, BigInt(Date.now()));
+  }
+
+  reportRendererLag(lagMs: number): void {
+    Atomics.store(this.i32, I32_IDX_RENDERER_LAG, lagMs | 0);
+    Atomics.store(this.i64, I64_IDX_RENDERER_REPORT_TS, BigInt(Date.now()));
+  }
+
+  setMainOp(op: string): void {
+    this.writeOpString(op, MAIN_OP_OFFSET_IN_U8, I32_IDX_MAIN_OP_LEN);
+  }
+
+  setReaderOp(op: string): void {
+    this.writeOpString(op, READER_OP_OFFSET_IN_U8, I32_IDX_READER_OP_LEN);
+  }
+
+  updateCounters(c: CounterPatch): void {
+    if (c.crawled !== undefined) {
+      Atomics.store(this.i32, I32_IDX_CRAWLED, c.crawled | 0);
+    }
+    if (c.discovered !== undefined) {
+      Atomics.store(this.i32, I32_IDX_DISCOVERED, c.discovered | 0);
+    }
+    if (c.pending !== undefined) {
+      Atomics.store(this.i32, I32_IDX_PENDING, c.pending | 0);
+    }
+    if (c.failed !== undefined) {
+      Atomics.store(this.i32, I32_IDX_FAILED, c.failed | 0);
+    }
+  }
+
+  // ── Readers (used by the watchdog worker) ─────────────────────────
+
+  readMainHeartbeatMs(): number {
+    return Number(Atomics.load(this.i64, I64_IDX_MAIN_HB));
+  }
+
+  readReaderHeartbeatMs(): number {
+    return Number(Atomics.load(this.i64, I64_IDX_READER_HB));
+  }
+
+  readRendererReportTsMs(): number {
+    return Number(Atomics.load(this.i64, I64_IDX_RENDERER_REPORT_TS));
+  }
+
+  readRendererLagMs(): number {
+    return Atomics.load(this.i32, I32_IDX_RENDERER_LAG);
+  }
+
+  readCounters(): CounterSnapshot {
+    return {
+      crawled: Atomics.load(this.i32, I32_IDX_CRAWLED),
+      discovered: Atomics.load(this.i32, I32_IDX_DISCOVERED),
+      pending: Atomics.load(this.i32, I32_IDX_PENDING),
+      failed: Atomics.load(this.i32, I32_IDX_FAILED),
+    };
+  }
+
+  readMainOp(): string {
+    return this.readOpString(MAIN_OP_OFFSET_IN_U8, I32_IDX_MAIN_OP_LEN);
+  }
+
+  readReaderOp(): string {
+    return this.readOpString(READER_OP_OFFSET_IN_U8, I32_IDX_READER_OP_LEN);
+  }
+
+  // ── Internals ─────────────────────────────────────────────────────
+
+  private writeOpString(op: string, offset: number, lenIdx: number): void {
+    const bytes = new TextEncoder().encode(op);
+    const len = Math.min(bytes.length, OP_MAX_BYTES);
+    // Order matters: write the bytes first, then the length last.
+    // A reader that races could see the new length before the bytes
+    // land, so reading old bytes with new length would corrupt; the
+    // reverse (old length, partial new bytes) is safe — the reader
+    // reads only `len` bytes from the head, ignoring the tail.
+    this.u8.set(bytes.subarray(0, len), offset);
+    Atomics.store(this.i32, lenIdx, len);
+  }
+
+  private readOpString(offset: number, lenIdx: number): string {
+    const len = Atomics.load(this.i32, lenIdx);
+    if (len <= 0) return '';
+    const slice = this.u8.slice(offset, offset + Math.min(len, OP_MAX_BYTES));
+    try {
+      return new TextDecoder('utf-8', { fatal: false }).decode(slice);
+    } catch {
+      return '';
+    }
+  }
+}

apps/desktop/src/main/freeze-watchdog-worker.ts

@@ -0,0 +1,197 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import * as fs from 'node:fs';
+import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js';
+
+/**
+ * Freeze-watchdog worker.
+ *
+ * Lives in its own `worker_thread`, so it has an independent V8
+ * isolate and OS thread. Even when the Electron main thread, the
+ * db-reader thread, or the renderer is fully blocked, this worker
+ * keeps running and writes evidence to `debug.txt`.
+ *
+ * Protocol:
+ *   - The main process owns a `SharedArrayBuffer` that this worker
+ *     reads. There is no two-way IPC between main and watchdog
+ *     during normal operation — IPC would itself stall when main
+ *     stalls, defeating the purpose.
+ *   - We poll the shared state every CHECK_INTERVAL_MS and append
+ *     stall events to `debug.txt` synchronously.
+ *
+ * The worker writes nothing else — no console output, no events
+ * back to main — so its only side effect is the debug file.
+ */
+
+const CHECK_INTERVAL_MS = 250;
+const STALL_THRESHOLD_MAIN_MS = 500;
+const STALL_THRESHOLD_READER_MS = 1000;
+const STALL_THRESHOLD_RENDERER_LAG_MS = 500;
+const STALL_THRESHOLD_RENDERER_SILENCE_MS = 1500;
+const HEARTBEAT_LOG_INTERVAL_MS = 5000;
+
+interface InitData {
+  sab: SharedArrayBuffer;
+  debugFilePath: string;
+}
+
+if (!parentPort) {
+  throw new Error('freeze-watchdog-worker: must be loaded via worker_threads');
+}
+
+const init = workerData as InitData;
+if (!init?.sab || !init?.debugFilePath) {
+  throw new Error('freeze-watchdog-worker: workerData must include sab + debugFilePath');
+}
+
+const state = new FreezeWatchdogSharedState(init.sab);
+const debugPath = init.debugFilePath;
+
+// Silently swallow append failures — if the disk is full or the
+// file is locked by AV, we can't recover, and writing would just
+// thrash retries. The watchdog stays useful for the next stall.
+function appendLine(line: string): void {
+  try {
+    fs.appendFileSync(debugPath, line + '\n');
+  } catch {
+    /* best-effort */
+  }
+}
+
+function isoNow(): string {
+  return new Date().toISOString();
+}
+
+function sanitizeOp(op: string): string {
+  // Keep newlines/tabs out of the log line — they'd break grep.
+  if (!op) return '';
+  return op.replace(/[\r\n\t]+/g, ' ').slice(0, 240);
+}
+
+interface StallTracker {
+  active: boolean;
+  startTs: number;
+  startOp: string;
+}
+
+const mainStall: StallTracker = { active: false, startTs: 0, startOp: '' };
+const readerStall: StallTracker = { active: false, startTs: 0, startOp: '' };
+const rendererStall: StallTracker = { active: false, startTs: 0, startOp: '' };
+
+let lastHeartbeatLogTs = 0;
+let bootLogged = false;
+
+function check(): void {
+  const now = Date.now();
+  const counters = state.readCounters();
+  const mainHb = state.readMainHeartbeatMs();
+  const readerHb = state.readReaderHeartbeatMs();
+  const rendererTs = state.readRendererReportTsMs();
+  const rendererLag = state.readRendererLagMs();
+  const mainOp = sanitizeOp(state.readMainOp());
+  const readerOp = sanitizeOp(state.readReaderOp());
+
+  const mainGap = mainHb > 0 ? now - mainHb : 0;
+  const readerGap = readerHb > 0 ? now - readerHb : 0;
+  const rendererSilence = rendererTs > 0 ? now - rendererTs : 0;
+
+  // ── Main thread stall ──
+  if (mainHb > 0 && mainGap > STALL_THRESHOLD_MAIN_MS) {
+    if (!mainStall.active) {
+      mainStall.active = true;
+      mainStall.startTs = now;
+      mainStall.startOp = mainOp;
+      appendLine(
+        `${isoNow()} [STALL:MAIN start gap=${mainGap}ms] op="${mainOp || '<unknown>'}" ` +
+          `crawled=${counters.crawled} discovered=${counters.discovered} ` +
+          `pending=${counters.pending} failed=${counters.failed}`,
+      );
+    }
+  } else if (mainStall.active) {
+    const dur = now - mainStall.startTs;
+    appendLine(
+      `${isoNow()} [STALL:MAIN end after ${dur}ms] startOp="${mainStall.startOp}" endOp="${mainOp}"`,
+    );
+    mainStall.active = false;
+  }
+
+  // ── DB reader thread stall ──
+  if (readerHb > 0 && readerGap > STALL_THRESHOLD_READER_MS) {
+    if (!readerStall.active) {
+      readerStall.active = true;
+      readerStall.startTs = now;
+      readerStall.startOp = readerOp;
+      appendLine(
+        `${isoNow()} [STALL:READER start gap=${readerGap}ms] op="${readerOp || '<unknown>'}"`,
+      );
+    }
+  } else if (readerStall.active) {
+    const dur = now - readerStall.startTs;
+    appendLine(
+      `${isoNow()} [STALL:READER end after ${dur}ms] startOp="${readerStall.startOp}" endOp="${readerOp}"`,
+    );
+    readerStall.active = false;
+  }
+
+  // ── Renderer stall ──
+  // We treat both an explicit high-lag report AND a long silence
+  // (renderer hasn't pinged back) as stall signals. The silence
+  // case catches a frozen renderer that can't even fire its
+  // setTimeout-based lag probe.
+  let rendererStalled = false;
+  let rendererReason = '';
+  if (rendererLag > STALL_THRESHOLD_RENDERER_LAG_MS) {
+    rendererStalled = true;
+    rendererReason = `lag=${rendererLag}ms`;
+  } else if (rendererTs > 0 && rendererSilence > STALL_THRESHOLD_RENDERER_SILENCE_MS) {
+    rendererStalled = true;
+    rendererReason = `silence=${rendererSilence}ms`;
+  }
+  if (rendererStalled) {
+    if (!rendererStall.active) {
+      rendererStall.active = true;
+      rendererStall.startTs = now;
+      rendererStall.startOp = mainOp;
+      appendLine(
+        `${isoNow()} [STALL:RENDERER start ${rendererReason}] last_main_op="${mainOp || '<unknown>'}" ` +
+          `crawled=${counters.crawled} pending=${counters.pending}`,
+      );
+    }
+  } else if (rendererStall.active) {
+    const dur = now - rendererStall.startTs;
+    appendLine(
+      `${isoNow()} [STALL:RENDERER end after ${dur}ms] last_lag=${rendererLag}ms`,
+    );
+    rendererStall.active = false;
+  }
+
+  // ── Periodic heartbeat (so the file shows the app is alive even
+  //    when nothing has stalled) ──
+  if (!bootLogged) {
+    bootLogged = true;
+    appendLine(
+      `${isoNow()} [BOOT] freeze-watchdog started ` +
+        `thresholds: main>${STALL_THRESHOLD_MAIN_MS}ms reader>${STALL_THRESHOLD_READER_MS}ms ` +
+        `renderer_lag>${STALL_THRESHOLD_RENDERER_LAG_MS}ms renderer_silence>${STALL_THRESHOLD_RENDERER_SILENCE_MS}ms`,
+    );
+  }
+  if (now - lastHeartbeatLogTs >= HEARTBEAT_LOG_INTERVAL_MS) {
+    lastHeartbeatLogTs = now;
+    appendLine(
+      `${isoNow()} [HEARTBEAT] main_op="${mainOp}" reader_op="${readerOp}" ` +
+        `crawled=${counters.crawled} discovered=${counters.discovered} ` +
+        `pending=${counters.pending} renderer_lag=${rendererLag}ms`,
+    );
+  }
+}
+
+setInterval(check, CHECK_INTERVAL_MS);
+
+// Accept a graceful-shutdown ping from the parent. We don't strictly
+// need it (the worker exits with the parent), but logging the close
+// lets the user see "watchdog stopped" instead of a silent truncation.
+parentPort.on('message', (msg: { type?: string }) => {
+  if (msg?.type === 'shutdown') {
+    appendLine(`${isoNow()} [SHUTDOWN] watchdog received shutdown signal`);
+    process.exit(0);
+  }
+});

apps/desktop/src/main/freeze-watchdog.ts

@@ -0,0 +1,139 @@
+import { Worker } from 'node:worker_threads';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  FreezeWatchdogSharedState,
+  type CounterPatch,
+} from './freeze-watchdog-shared.js';
+import * as logger from './logger.js';
+
+/**
+ * Main-process façade for the freeze watchdog.
+ *
+ * Owns the SharedArrayBuffer and the worker thread. Exposes a tiny
+ * imperative API the rest of the main process (and the crawler via
+ * dependency injection) calls during normal operation:
+ *
+ *   - `setMainOp("crawl:fetch:" + url)` whenever a top-level operation
+ *     starts on the main thread, so the watchdog has context if the
+ *     thread blocks immediately afterwards.
+ *   - `updateCounters({ crawled, pending })` for the snapshot fields.
+ *   - `reportRendererLag(ms)` from the IPC handler that already
+ *     receives renderer lag samples.
+ *
+ * The watchdog itself does the heartbeat tick on a 100 ms timer
+ * inside `init()` — callers don't have to remember to ping.
+ */
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const WORKER_PATH = path.join(__dirname, 'freeze-watchdog-worker.js');
+
+const MAIN_HEARTBEAT_INTERVAL_MS = 100;
+
+class FreezeWatchdog {
+  private worker: Worker | null = null;
+  private state: FreezeWatchdogSharedState | null = null;
+  private heartbeatTimer: NodeJS.Timeout | null = null;
+  private terminated = false;
+
+  /**
+   * Spawn the watchdog worker, allocate the SharedArrayBuffer, and
+   * start the main-thread heartbeat ticker. Idempotent.
+   */
+  init(debugFilePath: string): void {
+    if (this.terminated) return;
+    if (this.worker) return;
+    let state: FreezeWatchdogSharedState;
+    try {
+      state = FreezeWatchdogSharedState.create();
+    } catch (err) {
+      logger.log(
+        'warn',
+        'main',
+        `freeze-watchdog: SharedArrayBuffer allocation failed (${
+          err instanceof Error ? err.message : String(err)
+        }) — diagnostics disabled.`,
+      );
+      return;
+    }
+    try {
+      this.worker = new Worker(WORKER_PATH, {
+        workerData: {
+          sab: state.sab,
+          debugFilePath,
+        },
+      });
+    } catch (err) {
+      logger.log(
+        'warn',
+        'main',
+        `freeze-watchdog worker spawn failed: ${
+          err instanceof Error ? err.message : String(err)
+        } — diagnostics disabled.`,
+      );
+      return;
+    }
+    this.state = state;
+    this.worker.on('error', (err) => {
+      logger.log('warn', 'main', `freeze-watchdog worker error: ${err.message}`);
+    });
+    this.worker.on('exit', (code) => {
+      this.worker = null;
+      if (!this.terminated && code !== 0) {
+        logger.log(
+          'warn',
+          'main',
+          `freeze-watchdog worker exited unexpectedly (code=${code})`,
+        );
+      }
+    });
+    // Heartbeat tick — independent of the rest of the app's setIntervals
+    // so a crashed timer elsewhere doesn't silently disable detection.
+    this.heartbeatTimer = setInterval(() => {
+      this.state?.tickMainHeartbeat();
+    }, MAIN_HEARTBEAT_INTERVAL_MS);
+    logger.log('info', 'main', `freeze-watchdog started → ${debugFilePath}`);
+  }
+
+  /** Pass-through to the shared buffer so workers can attach. */
+  get sharedBuffer(): SharedArrayBuffer | null {
+    return this.state?.sab ?? null;
+  }
+
+  setMainOp(op: string): void {
+    this.state?.setMainOp(op);
+  }
+
+  reportRendererLag(lagMs: number): void {
+    this.state?.reportRendererLag(lagMs);
+  }
+
+  updateCounters(c: CounterPatch): void {
+    this.state?.updateCounters(c);
+  }
+
+  /** Graceful shutdown — flushes the heartbeat timer and signals the
+   * worker to exit cleanly so it can write a `[SHUTDOWN]` line. */
+  async terminate(): Promise<void> {
+    this.terminated = true;
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+    if (this.worker) {
+      const w = this.worker;
+      this.worker = null;
+      try {
+        w.postMessage({ type: 'shutdown' });
+        // Give the worker up to 250 ms to write its shutdown line
+        // before we forcibly terminate.
+        await new Promise<void>((resolve) => setTimeout(resolve, 250));
+        await w.terminate();
+      } catch {
+        /* already exited */
+      }
+    }
+  }
+}
+
+export const freezeWatchdog = new FreezeWatchdog();

apps/desktop/src/main/index.ts

@@ -23,6 +23,11 @@ import {
   type ExportCsvResult,
   type ExportJsonInput,
   type ExportJsonResult,
+  type ExportXmlInput,
+  type ExportXmlResult,
+  type DataDeleteByDomainInput,
+  type DataDeleteByDomainResult,
+  type CrashRecoveryStatus,
   type ExportHtmlReportInput,
   type ExportHtmlReportResult,
   type BulkExportFile,
@@ -75,6 +80,7 @@ import {
   Crawler,
   exportUrlsToCsv,
   exportUrlsToJson,
+  exportUrlsToXml,
   exportSitemap,
   exportHtmlReport,
   compareCrawls,
@@ -86,6 +92,11 @@ import { ProjectDb } from '@freecrawl/db';
 import { buildAppMenu } from './menu.js';
 import * as logger from './logger.js';
 import { dbReaderPool, callReaderOrFallback } from './db-reader-pool.js';
+import { dbWriterPool } from './db-writer-pool.js';
+import { freezeWatchdog } from './freeze-watchdog.js';
+import { parserPool } from './parser-pool.js';
+import { parseHtml as inlineParseHtml } from '@freecrawl/core';
+import type { ProjectDb as ProjectDbType } from '@freecrawl/db';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -93,6 +104,15 @@ let mainWindow: BrowserWindow | null = null;
 let logsWindow: BrowserWindow | null = null;
 let db: ProjectDb | null = null;
 let activeCrawler: Crawler | null = null;
+/** Most-recent CrawlConfig — needed by the crash-recovery resume
+ * flow so we can re-create the Crawler with the same knobs the user
+ * had set, without re-prompting. Hydrated lazily from `project_meta`
+ * on first DB open. */
+let lastCrawlConfig: CrawlConfig | null = null;
+/** In-memory snapshot of the previous session's checkpoint, captured
+ * before `db.reset()` wipes it. Cleared once the user accepts or
+ * discards the recovery prompt. */
+let pendingRecoveryCheckpoint: { url: string; depth: number; seedUrl: string }[] = [];
 
 // UI preferences (column widths, panel sizes, etc.) live in a JSON file
 // under userData — separate from the crawl DB so "Clear" wipes crawl data
@@ -291,6 +311,61 @@ function maybeShowDiagnosticDialog(msg: string): void {
 /** Currently-open project file path (empty when using the default scratch DB). */
 let currentProjectPath = '';
 
+/**
+ * Dispatch HTML parsing to the worker pool when ready, otherwise
+ * fall back to the inline parser on the main thread. This wrapper is
+ * passed into every `new Crawler(...)` call so the same code path
+ * gets pool acceleration in the desktop app and stays simple in
+ * tests / headless contexts.
+ */
+async function parseHtmlViaPool(
+  html: string,
+  pageUrl: string,
+  opts: Parameters<typeof inlineParseHtml>[2],
+): Promise<ReturnType<typeof inlineParseHtml>> {
+  if (!parserPool.isReady()) {
+    return inlineParseHtml(html, pageUrl, opts);
+  }
+  try {
+    return await parserPool.parse(html, pageUrl, opts);
+  } catch (err) {
+    logger.log(
+      'warn',
+      'main',
+      `parser-pool dispatch failed (${
+        err instanceof Error ? err.message : String(err)
+      }) — using inline parseHtml as fallback.`,
+    );
+    return inlineParseHtml(html, pageUrl, opts);
+  }
+}
+
+/**
+ * Dispatch a per-URL write batch to the writer worker. Same fall-back
+ * pattern as the parser pool: when the worker is unhealthy we run
+ * the write on the main-thread `ProjectDb` instance so the crawl
+ * never silently drops data.
+ */
+async function writeFetchedUrlViaPool(
+  payload: Parameters<ProjectDbType['writeFetchedUrl']>[0],
+): Promise<{ urlId: number }> {
+  if (!dbWriterPool.isReady()) {
+    return getDb().writeFetchedUrl(payload);
+  }
+  try {
+    return await dbWriterPool.call<{ urlId: number }>('writeFetchedUrl', [payload]);
+  } catch (err) {
+    logger.log(
+      'warn',
+      'main',
+      `db-writer dispatch failed (${
+        err instanceof Error ? err.message : String(err)
+      }) — falling back to main-thread writeFetchedUrl.`,
+    );
+    return getDb().writeFetchedUrl(payload);
+  }
+}
+
 function getDb(): ProjectDb {
   if (!db) {
     const dataDir = join(app.getPath('userData'), 'projects');
@@ -298,6 +373,25 @@ function getDb(): ProjectDb {
     const defaultPath = join(dataDir, 'default.seoproject');
     db = new ProjectDb(defaultPath);
     currentProjectPath = '';
+    // Wave 6 — Snapshot the previous session's crash-recovery state
+    // BEFORE the reset wipes the project tables. We capture the
+    // checkpointed pending queue + the cached crawl config so the
+    // user can be offered a resume on app start. The DB tables go
+    // back to empty after `reset()`; the in-memory snapshot stays
+    // until the user accepts/discards the recovery prompt.
+    try {
+      const raw = db.getMeta('lastCrawlConfig');
+      if (raw) lastCrawlConfig = JSON.parse(raw) as CrawlConfig;
+    } catch {
+      /* malformed JSON or missing key — recovery just won't fire */
+    }
+    pendingRecoveryCheckpoint = (() => {
+      try {
+        return db.loadQueueCheckpoint();
+      } catch {
+        return [];
+      }
+    })();
     // Fresh start on every app launch — clear any data carried over from
     // the previous session. Explicit Save Project will be added later.
     db.reset();
@@ -305,7 +399,9 @@ function getDb(): ProjectDb {
     // already ran above (the writer owns them) so the worker just
     // attaches to the WAL and starts serving SELECTs concurrently.
     try {
-      dbReaderPool.init(defaultPath);
+      // Pass the freeze-watchdog SAB so the reader worker can publish
+      // its own heartbeat + current op into the same diagnostic file.
+      dbReaderPool.init(defaultPath, freezeWatchdog.sharedBuffer);
     } catch (err) {
       logger.log(
         'warn',
@@ -313,6 +409,15 @@ function getDb(): ProjectDb {
         `db-reader pool init failed: ${err instanceof Error ? err.message : String(err)} — falling back to main-thread queries.`,
       );
     }
+    try {
+      dbWriterPool.init(defaultPath, freezeWatchdog.sharedBuffer);
+    } catch (err) {
+      logger.log(
+        'warn',
+        'main',
+        `db-writer pool init failed: ${err instanceof Error ? err.message : String(err)} — writes fall back to main thread.`,
+      );
+    }
   }
   return db;
 }
@@ -341,7 +446,7 @@ function openProjectAtPath(filePath: string): void {
   // in-flight requests with `reader-swapped`; the next IPC call will
   // hit the freshly opened worker.
   try {
-    dbReaderPool.swap(filePath);
+    dbReaderPool.swap(filePath, freezeWatchdog.sharedBuffer);
   } catch (err) {
     logger.log(
       'warn',
@@ -349,6 +454,15 @@ function openProjectAtPath(filePath: string): void {
       `db-reader pool swap failed: ${err instanceof Error ? err.message : String(err)}`,
     );
   }
+  try {
+    dbWriterPool.swap(filePath, freezeWatchdog.sharedBuffer);
+  } catch (err) {
+    logger.log(
+      'warn',
+      'main',
+      `db-writer pool swap failed: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
   pushRecentProject(filePath);
   rebuildMenu();
   if (mainWindow) {
@@ -661,6 +775,9 @@ function registerIpc(): void {
   ipcMain.on(IPC.rendererLagReport, (_e, lagMs: number) => {
     if (typeof lagMs === 'number' && Number.isFinite(lagMs)) {
       activeCrawler?.reportRendererLag(lagMs);
+      // Forward the same sample to the freeze-watchdog so a frozen
+      // renderer surfaces in `debug.txt` even when no crawl is active.
+      freezeWatchdog.reportRendererLag(lagMs);
     }
   });
 
@@ -958,6 +1075,63 @@ function registerIpc(): void {
     schedulePrefsWrite();
   });
 
+  /** Wave 6 — Cached most-recent crawl config so the crash-recovery
+   * resume can re-create a Crawler without forcing the user to fill
+   * in the Start dialog again. Persisted across restarts via DB
+   * project_meta so it survives the very crash we're recovering from. */
+  function attachCrawlerListeners(crawler: Crawler): void {
+    crawler.on('progress', (p: CrawlProgress) => {
+      if (activeCrawler !== crawler) return;
+      mainWindow?.webContents.send(IPC.crawlProgress, p);
+      freezeWatchdog.updateCounters({
+        crawled: p.crawled,
+        discovered: p.discovered,
+        pending: p.pending,
+        failed: p.failed,
+      });
+    });
+    crawler.on('done', (summary: CrawlSummary) => {
+      if (activeCrawler !== crawler) return;
+      logger.log(
+        'info',
+        'crawler',
+        `Crawl done: total=${summary.total} avgResp=${summary.avgResponseTimeMs}ms totalBytes=${summary.totalBytes}`,
+      );
+      mainWindow?.webContents.send(IPC.crawlDone, summary);
+      activeCrawler = null;
+      freezeWatchdog.setMainOp('idle');
+      if (Notification.isSupported() && !mainWindow?.isFocused()) {
+        try {
+          new Notification({
+            title: 'FreeCrawl SEO Tool',
+            body: `Crawl finished: ${summary.total.toLocaleString()} URLs · avg ${Math.round(summary.avgResponseTimeMs)} ms`,
+            silent: false,
+          }).show();
+        } catch {
+          /* Linux without notification daemon — swallow */
+        }
+      }
+    });
+    crawler.on('error', (msg: string) => {
+      if (activeCrawler !== crawler) return;
+      logger.log('error', 'crawler', msg);
+      mainWindow?.webContents.send(IPC.crawlError, msg);
+      maybeShowDiagnosticDialog(msg);
+    });
+    crawler.on('warn', (msg: string) => {
+      if (activeCrawler !== crawler) return;
+      logger.log('warn', 'crawler', msg);
+    });
+    crawler.on('info', (msg: string) => {
+      if (activeCrawler !== crawler) return;
+      logger.log('info', 'crawler', msg);
+    });
+    crawler.on('debug', (msg: string) => {
+      if (activeCrawler !== crawler) return;
+      logger.log('debug', 'crawler', msg);
+    });
+  }
+
   ipcMain.handle(IPC.crawlStart, (_e, config: CrawlConfig) => {
     if (activeCrawler) {
       activeCrawler.stop();
@@ -973,69 +1147,22 @@ function registerIpc(): void {
     );
     const database = getDb();
     database.setMeta('lastStartUrl', config.startUrl);
-    const crawler = new Crawler(config, database);
-    activeCrawler = crawler;
-
-    // Every event handler is gated by `activeCrawler === crawler`. A
-    // stopped crawler can still emit late events (e.g. an in-flight
-    // sitemap fetch resolving after Stop, or a queued done-event) — if
-    // we forwarded those to the UI they'd clobber the new crawl's state
-    // (the "pır pır" effect: rapid Running ↔ Done flicker).
-    crawler.on('progress', (p: CrawlProgress) => {
-      if (activeCrawler !== crawler) return;
-      mainWindow?.webContents.send(IPC.crawlProgress, p);
-    });
-    crawler.on('done', (summary: CrawlSummary) => {
-      if (activeCrawler !== crawler) return;
-      logger.log(
-        'info',
-        'crawler',
-        `Crawl done: total=${summary.total} avgResp=${summary.avgResponseTimeMs}ms totalBytes=${summary.totalBytes}`,
-      );
-      mainWindow?.webContents.send(IPC.crawlDone, summary);
-      activeCrawler = null;
-      // OS-level toast — only when the window isn't focused, so the
-      // user actually benefits (Electron suppresses the notification
-      // sound on focused windows on most platforms anyway, but we
-      // gate explicitly to avoid distracting users who're watching).
-      if (Notification.isSupported() && !mainWindow?.isFocused()) {
+    // Persist the config so a crash-then-resume can rehydrate it
+    // without the user re-entering anything. Stored as JSON in
+    // `project_meta` since the table is keyed by string.
     try {
-          new Notification({
-            title: 'FreeCrawl SEO Tool',
-            body: `Crawl finished: ${summary.total.toLocaleString()} URLs · avg ${Math.round(summary.avgResponseTimeMs)} ms`,
-            silent: false,
-          }).show();
+      database.setMeta('lastCrawlConfig', JSON.stringify(config));
     } catch {
-          // Notification can throw on some Linux distros without a
-          // notification daemon. Swallow — the in-app done banner
-          // already surfaces completion.
-        }
+      /* never fatal */
     }
+    lastCrawlConfig = config;
+    const crawler = new Crawler(config, database, {
+      setOp: (op: string) => freezeWatchdog.setMainOp(op),
+      parseHtml: parseHtmlViaPool,
+      writeFetchedUrl: writeFetchedUrlViaPool,
     });
-    crawler.on('error', (msg: string) => {
-      if (activeCrawler !== crawler) return;
-      logger.log('error', 'crawler', msg);
-      mainWindow?.webContents.send(IPC.crawlError, msg);
-      // Match against environment-specific failure patterns (DNS / TLS
-      // inspection / unreachable seed) and surface a modal popup the
-      // first time each category fires per crawl session. Site-specific
-      // 4xx/WAF/timeout errors are NOT promoted to popups — they belong
-      // in the log panel, not in the user's face.
-      maybeShowDiagnosticDialog(msg);
-    });
-    crawler.on('warn', (msg: string) => {
-      if (activeCrawler !== crawler) return;
-      logger.log('warn', 'crawler', msg);
-    });
-    crawler.on('info', (msg: string) => {
-      if (activeCrawler !== crawler) return;
-      logger.log('info', 'crawler', msg);
-    });
-    crawler.on('debug', (msg: string) => {
-      if (activeCrawler !== crawler) return;
-      logger.log('debug', 'crawler', msg);
-    });
-
+    activeCrawler = crawler;
+    attachCrawlerListeners(crawler);
     void crawler.start();
   });
 
@@ -1054,7 +1181,25 @@ function registerIpc(): void {
   ipcMain.handle(IPC.crawlClear, () => {
     activeCrawler?.stop();
     activeCrawler = null;
-    getDb().reset();
+    const database = getDb();
+    database.reset();
+    // Belt-and-braces: reset() above already includes a DELETE FROM
+    // crawl_queue, but if anything (a late-firing checkpoint timer, a
+    // half-finished post-crawl pass) writes to that table after the
+    // exec() returns, the user would see a stale recovery prompt next
+    // launch. An explicit second sweep here is cheap and guarantees
+    // the queue is empty when the IPC handler resolves.
+    try {
+      database.clearQueueCheckpoint();
+    } catch {
+      /* table missing on very old DBs — recovery just won't fire */
+    }
+    // The in-memory recovery snapshot is captured at app boot from the
+    // queue-on-disk; reset() now wipes that data, but the snapshot
+    // mirror still holds the pre-reset entries. Drop it so a stale
+    // dialog can't fire after Clear if the user cancelled the boot
+    // prompt earlier.
+    pendingRecoveryCheckpoint = [];
   });
 
   ipcMain.handle(IPC.crawlAddUrl, (_e, url: string): { accepted: boolean } => {
@@ -1493,6 +1638,106 @@ function registerIpc(): void {
     },
   );
 
+  ipcMain.handle(
+    IPC.exportXml,
+    async (_e, input: ExportXmlInput): Promise<ExportXmlResult> => {
+      let filePath = input.filePath;
+      const isSelection = (input.selectedIds?.length ?? 0) > 0;
+      if (!filePath) {
+        const res = await dialog.showSaveDialog(mainWindow!, {
+          defaultPath: isSelection ? 'freecrawl-selected.xml' : 'freecrawl-export.xml',
+          filters: [{ name: 'XML', extensions: ['xml'] }],
+        });
+        if (res.canceled || !res.filePath) {
+          return { filePath: '', rowsWritten: 0 };
+        }
+        filePath = res.filePath;
+      }
+      const { rowsWritten } = await exportUrlsToXml(getDb(), filePath, {
+        selectedIds: input.selectedIds,
+        category: input.category,
+      });
+      return { filePath, rowsWritten };
+    },
+  );
+
+  ipcMain.handle(
+    IPC.dataDeleteByDomain,
+    async (
+      _e,
+      input: DataDeleteByDomainInput,
+    ): Promise<DataDeleteByDomainResult> => {
+      const { urlsDeleted, linksDeleted } = getDb().deleteByDomain(input.domain);
+      // Force-invalidate UI caches: row counts, sidebar issues, etc.
+      fireDataChanged();
+      return { urlsDeleted, linksDeleted };
+    },
+  );
+
+  // Wave 6 — Crash recovery handlers. The renderer asks `…Status` on
+  // mount; if `pendingCount > 0` it shows a confirmation dialog and
+  // routes the user's choice to `…Resume` or `…Discard`.
+  ipcMain.handle(
+    IPC.crashRecoveryStatus,
+    async (): Promise<CrashRecoveryStatus> => {
+      // Read from the in-memory snapshot we captured before
+      // `db.reset()` wiped the previous session's data.
+      if (pendingRecoveryCheckpoint.length === 0) {
+        return { pendingCount: 0, seedUrl: '' };
+      }
+      return {
+        pendingCount: pendingRecoveryCheckpoint.length,
+        seedUrl: pendingRecoveryCheckpoint[0]?.seedUrl ?? '',
+      };
+    },
+  );
+
+  ipcMain.handle(
+    IPC.crashRecoveryResume,
+    async (): Promise<{ accepted: boolean }> => {
+      if (activeCrawler) return { accepted: false };
+      if (pendingRecoveryCheckpoint.length === 0) return { accepted: false };
+      const seedUrl = pendingRecoveryCheckpoint[0]?.seedUrl ?? '';
+      if (!seedUrl) return { accepted: false };
+      const cfg = lastCrawlConfig
+        ? { ...lastCrawlConfig, startUrl: seedUrl }
+        : null;
+      if (!cfg) return { accepted: false };
+      const crawler = new Crawler(cfg, getDb(), {
+        setOp: (op: string) => freezeWatchdog.setMainOp(op),
+        parseHtml: parseHtmlViaPool,
+        writeFetchedUrl: writeFetchedUrlViaPool,
+      });
+      activeCrawler = crawler;
+      attachCrawlerListeners(crawler);
+      const items = pendingRecoveryCheckpoint.map((c) => ({
+        url: c.url,
+        depth: c.depth,
+      }));
+      // Snapshot consumed — clear so a second click can't double-fire.
+      pendingRecoveryCheckpoint = [];
+      crawler.enqueueCheckpointed(items);
+      void crawler.start();
+      return { accepted: true };
+    },
+  );
+
+  ipcMain.handle(IPC.crashRecoveryDiscard, async (): Promise<void> => {
+    pendingRecoveryCheckpoint = [];
+    // Wipe the on-disk crawl_queue too — without this, anything that
+    // re-populates the queue between now and the next app launch would
+    // resurrect the recovery prompt the user just dismissed.
+    try {
+      getDb().clearQueueCheckpoint();
+    } catch (err) {
+      logger.log(
+        'warn',
+        'main',
+        `clearQueueCheckpoint failed on discard: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  });
+
   ipcMain.handle(IPC.exportBulk, async (): Promise<BulkExportResult> => {
     if (!mainWindow) {
       return { outputDir: '', files: [], errors: [] };
@@ -1836,11 +2081,36 @@ void app.whenReady().then(() => {
   } catch (err) {
     logger.log('warn', 'main', `Disk logging unavailable: ${(err as Error).message}`);
   }
+  // Boot the freeze-watchdog before anything else CPU-heavy. The
+  // worker writes to `<userData>/debug.txt` (separate from the
+  // regular structured log so users + dev can grep stalls without
+  // noise from normal app events).
+  try {
+    const debugPath = join(app.getPath('userData'), 'debug.txt');
+    freezeWatchdog.init(debugPath);
+  } catch (err) {
+    logger.log('warn', 'main', `freeze-watchdog init failed: ${(err as Error).message}`);
+  }
+  freezeWatchdog.setMainOp('boot');
+  // Spawn the parser worker pool so cheerio runs off the main thread.
+  // Falls back to inline parseHtml automatically when init fails on a
+  // constrained host (the pool's `parse()` rejects, the dispatch
+  // helper below catches and falls back).
+  try {
+    parserPool.init();
+  } catch (err) {
+    logger.log(
+      'warn',
+      'main',
+      `parser-pool init failed: ${(err as Error).message} — falling back to inline parseHtml.`,
+    );
+  }
   loadPrefs();
   rebuildMenu();
   registerIpc();
   createWindow();
   logger.log('info', 'main', `App ready — version ${app.getVersion()}`);
+  freezeWatchdog.setMainOp('idle');
 
   app.on('activate', () => {
     if (BrowserWindow.getAllWindows().length === 0) createWindow();
@@ -1849,11 +2119,15 @@ void app.whenReady().then(() => {
 
 app.on('window-all-closed', () => {
   activeCrawler?.stop();
+  freezeWatchdog.setMainOp('shutdown');
+  void freezeWatchdog.terminate();
+  void parserPool.terminate();
   // Terminate the read-only worker BEFORE closing the writer connection.
   // Order matters on Windows: SQLite holds a file lock per connection
   // and the writer's WAL checkpoint at close-time can stall if a
   // reader is still mid-query.
   void dbReaderPool.terminate();
+  void dbWriterPool.terminate();
   db?.close();
   db = null;
   flushPrefs();

apps/desktop/src/main/menu.ts

@@ -78,6 +78,10 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
           accelerator: 'CmdOrCtrl+Shift+E',
           click: () => send('export-json'),
         },
+        {
+          label: 'Export Current View as XML…',
+          click: () => send('export-xml'),
+        },
         {
           label: 'Generate XML Sitemap…',
           click: () => send('generate-sitemap'),
@@ -182,6 +186,19 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
           click: () => handlers.onResetDiagnosticDialogs(),
         },
         { type: 'separator' },
+        {
+          label: 'Delete Domain Data…',
+          toolTip:
+            'GDPR-aligned per-domain wipe. Removes every URL row whose host matches the entered domain plus every dependent record (links, headers, images, source snapshots).',
+          click: () => send('delete-domain-data'),
+        },
+        {
+          label: 'Clear All Data…',
+          toolTip:
+            'Wipe the entire active project (URLs, links, images, headers, source snapshots, sitemaps). Cannot be undone — Save Project As… first if you want a backup.',
+          click: () => send('clear-all-data'),
+        },
+        { type: 'separator' },
         { label: 'About FreeCrawl SEO', click: () => send('about') },
       ],
     },

apps/desktop/src/main/parser-pool.ts

@@ -0,0 +1,192 @@
+import { Worker } from 'node:worker_threads';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import os from 'node:os';
+import type { parseHtml } from '@freecrawl/core';
+import * as logger from './logger.js';
+
+/**
+ * Round-robin pool of parser worker threads.
+ *
+ * Why a pool, not a single worker:
+ *   - Cheerio is single-threaded inside the worker. One worker can
+ *     parse one document at a time.
+ *   - The crawler runs at concurrency 20 by default, so up to 20
+ *     pages return their HTML body simultaneously. With one worker
+ *     they would serialise behind it, defeating the point.
+ *   - 4-8 workers (capped to physical cores - 2) saturate the CPU
+ *     while leaving the main thread headroom for IPC and queue
+ *     scheduling.
+ *
+ * Round-robin dispatch is sufficient: parse durations are roughly
+ * proportional to body size, and the crawler's URL mix makes it
+ * unlikely that long-tail pages all land on the same worker. A
+ * least-loaded scheduler would add complexity for marginal gain.
+ */
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const WORKER_PATH = path.join(__dirname, 'parser-worker.js');
+const REQUEST_TIMEOUT_MS = 60_000;
+
+type ParseHtmlArgs = Parameters<typeof parseHtml>;
+type ParseOpts = ParseHtmlArgs[2];
+type ParseResult = ReturnType<typeof parseHtml>;
+
+interface PendingRequest {
+  resolve: (v: ParseResult) => void;
+  reject: (e: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+}
+
+interface ResponseMessage {
+  requestId: number;
+  ok: boolean;
+  result?: ParseResult;
+  error?: string;
+}
+
+class ParserPool {
+  private workers: Worker[] = [];
+  private pending = new Map<number, PendingRequest>();
+  private nextRequestId = 1;
+  private nextWorkerIdx = 0;
+  private terminated = false;
+
+  /**
+   * Spawn the worker pool. Idempotent — calling init() twice is a
+   * no-op once workers are already running.
+   *
+   *   `size` — number of worker threads. Defaults to a sensible
+   *   value based on CPU count (cores - 2, clamped to [2, 8]).
+   *   Leave room for the main thread + db-reader + db-writer +
+   *   freeze-watchdog so the crawler isn't competing with its own
+   *   plumbing for cores.
+   */
+  init(size?: number): void {
+    if (this.terminated) {
+      throw new Error('ParserPool: cannot init after terminate()');
+    }
+    if (this.workers.length > 0) return;
+    const cpuCount = os.cpus().length;
+    const target =
+      size ?? Math.max(2, Math.min(8, cpuCount - 2));
+    for (let i = 0; i < target; i++) {
+      try {
+        this.workers.push(this.spawnWorker(i));
+      } catch (err) {
+        logger.log(
+          'warn',
+          'main',
+          `parser-pool: worker ${i} spawn failed: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
+      }
+    }
+    if (this.workers.length === 0) {
+      logger.log(
+        'warn',
+        'main',
+        'parser-pool: no workers spawned — falling back to inline parseHtml.',
+      );
+    } else {
+      logger.log(
+        'info',
+        'main',
+        `parser-pool: spawned ${this.workers.length} parser worker(s)`,
+      );
+    }
+  }
+
+  /** True when at least one worker is running and accepting requests. */
+  isReady(): boolean {
+    return this.workers.length > 0 && !this.terminated;
+  }
+
+  /**
+   * Dispatch a parse request to the next available worker.
+   * Resolves with the parsed page; rejects on timeout or worker
+   * crash. Caller is expected to fall back to inline `parseHtml`
+   * if the pool isn't ready.
+   */
+  parse(html: string, pageUrl: string, opts: ParseOpts = {}): Promise<ParseResult> {
+    if (this.terminated || this.workers.length === 0) {
+      return Promise.reject(new Error('parser-pool: not ready'));
+    }
+    const requestId = this.nextRequestId++;
+    const worker = this.workers[this.nextWorkerIdx % this.workers.length]!;
+    this.nextWorkerIdx++;
+    return new Promise<ParseResult>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(requestId);
+        reject(new Error(`parser-pool: timeout > ${REQUEST_TIMEOUT_MS}ms`));
+      }, REQUEST_TIMEOUT_MS);
+      this.pending.set(requestId, { resolve, reject, timer });
+      worker.postMessage({ requestId, html, pageUrl, opts });
+    });
+  }
+
+  async terminate(): Promise<void> {
+    this.terminated = true;
+    for (const p of this.pending.values()) {
+      clearTimeout(p.timer);
+      p.reject(new Error('parser-pool: terminated'));
+    }
+    this.pending.clear();
+    const workers = this.workers;
+    this.workers = [];
+    await Promise.allSettled(workers.map((w) => w.terminate()));
+  }
+
+  // ── private ──────────────────────────────────────────────────────
+
+  private spawnWorker(index: number): Worker {
+    const w = new Worker(WORKER_PATH);
+    w.on('message', (msg: ResponseMessage) => this.handleResponse(msg));
+    w.on('error', (err) => {
+      logger.log('error', 'main', `parser-worker[${index}] error: ${err.message}`);
+    });
+    w.on('exit', (code) => {
+      if (this.terminated) return;
+      if (code === 0) return;
+      logger.log(
+        'warn',
+        'main',
+        `parser-worker[${index}] exited (code=${code}) — respawning`,
+      );
+      const respawn = this.tryRespawn(index);
+      if (respawn) this.workers[index] = respawn;
+    });
+    return w;
+  }
+
+  private tryRespawn(index: number): Worker | null {
+    if (this.terminated) return null;
+    try {
+      return this.spawnWorker(index);
+    } catch (err) {
+      logger.log(
+        'error',
+        'main',
+        `parser-worker[${index}] respawn failed: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+      return null;
+    }
+  }
+
+  private handleResponse(msg: ResponseMessage): void {
+    const pending = this.pending.get(msg.requestId);
+    if (!pending) return;
+    this.pending.delete(msg.requestId);
+    clearTimeout(pending.timer);
+    if (msg.ok && msg.result !== undefined) {
+      pending.resolve(msg.result);
+    } else {
+      pending.reject(new Error(msg.error ?? 'parser-pool: unknown error'));
+    }
+  }
+}
+
+export const parserPool = new ParserPool();

apps/desktop/src/main/parser-worker.ts

@@ -0,0 +1,46 @@
+import { parentPort } from 'node:worker_threads';
+import { parseHtml } from '@freecrawl/core';
+
+/**
+ * HTML-parser worker thread.
+ *
+ * Cheerio + the link/image/structured-data extractor is the hottest
+ * CPU path in `fetchAndProcess`. On large mağaza listing / blog
+ * pages it can spend 10-25 s of pure JS on a single document while
+ * the main thread holds the Electron event loop. Running the parse
+ * here means the main process only waits for the result via
+ * postMessage — IPC pumps and crawler scheduling stay live.
+ *
+ * Protocol:
+ *   main → worker:  `{ requestId, html, pageUrl, opts }`
+ *   worker → main:  `{ requestId, ok: true, result }`
+ *                or `{ requestId, ok: false, error }`
+ *
+ * One worker handles one request at a time; the pool spreads requests
+ * across workers via round-robin (see `parser-pool.ts`).
+ */
+
+if (!parentPort) {
+  throw new Error('parser-worker: must be loaded via worker_threads');
+}
+
+interface ParseRequest {
+  requestId: number;
+  html: string;
+  pageUrl: string;
+  opts: Parameters<typeof parseHtml>[2];
+}
+
+parentPort.on('message', (msg: ParseRequest) => {
+  if (!msg || typeof msg.requestId !== 'number') return;
+  try {
+    const result = parseHtml(msg.html, msg.pageUrl, msg.opts);
+    parentPort!.postMessage({ requestId: msg.requestId, ok: true, result });
+  } catch (err) {
+    parentPort!.postMessage({
+      requestId: msg.requestId,
+      ok: false,
+      error: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
+    });
+  }
+});

apps/desktop/src/preload/index.ts

@@ -9,6 +9,11 @@ import {
   type ExportCsvResult,
   type ExportJsonInput,
   type ExportJsonResult,
+  type ExportXmlInput,
+  type ExportXmlResult,
+  type DataDeleteByDomainInput,
+  type DataDeleteByDomainResult,
+  type CrashRecoveryStatus,
   type ExportHtmlReportInput,
   type ExportHtmlReportResult,
   type BulkExportResult,
@@ -116,6 +121,18 @@ const api: FreeCrawlApi = {
     ipcRenderer.invoke(IPC.exportCsv, input),
   exportJson: (input: ExportJsonInput): Promise<ExportJsonResult> =>
     ipcRenderer.invoke(IPC.exportJson, input),
+  exportXml: (input: ExportXmlInput): Promise<ExportXmlResult> =>
+    ipcRenderer.invoke(IPC.exportXml, input),
+  dataDeleteByDomain: (
+    input: DataDeleteByDomainInput,
+  ): Promise<DataDeleteByDomainResult> =>
+    ipcRenderer.invoke(IPC.dataDeleteByDomain, input),
+  crashRecoveryStatus: (): Promise<CrashRecoveryStatus> =>
+    ipcRenderer.invoke(IPC.crashRecoveryStatus),
+  crashRecoveryResume: (): Promise<{ accepted: boolean }> =>
+    ipcRenderer.invoke(IPC.crashRecoveryResume),
+  crashRecoveryDiscard: (): Promise<void> =>
+    ipcRenderer.invoke(IPC.crashRecoveryDiscard),
   exportHtmlReport: (input: ExportHtmlReportInput): Promise<ExportHtmlReportResult> =>
     ipcRenderer.invoke(IPC.exportHtmlReport, input),
   exportBulk: (): Promise<BulkExportResult> => ipcRenderer.invoke(IPC.exportBulk),

apps/desktop/src/renderer/src/App.tsx

@@ -57,6 +57,30 @@ export function App() {
       const file = files[0];
       if (!file) return;
       const name = file.name.toLowerCase();
+      // Wave 5 — `.seoproject` drop opens the project. Electron exposes
+      // `file.path` (not part of the standard File API) when files come
+      // from the OS file system; we route to `projectOpen` which swaps
+      // the active DB and refreshes the UI.
+      if (name.endsWith('.seoproject')) {
+        e.preventDefault();
+        const fileWithPath = file as File & { path?: string };
+        const filePath = fileWithPath.path;
+        if (!filePath) {
+          setDropFlash(
+            `Couldn't read the dropped path for ${file.name}. Use File → Open Project instead.`,
+          );
+          setTimeout(() => setDropFlash(null), 4000);
+          return;
+        }
+        const result = await window.freecrawl.projectOpen(filePath);
+        if (result) {
+          setDropFlash(`Opened project: ${result.filePath}`);
+        } else {
+          setDropFlash(`Failed to open project: ${file.name}`);
+        }
+        setTimeout(() => setDropFlash(null), 4000);
+        return;
+      }
       if (!/\.(txt|csv|list)$/.test(name)) return;
       e.preventDefault();
       const text = await file.text();
@@ -109,7 +133,48 @@ export function App() {
   );
 
   useEffect(() => {
-    const off1 = window.freecrawl.onProgress(setProgress);
+    // Wave 6 — Crash-recovery prompt. Fires once on mount; if the
+    // previous session left a non-empty checkpoint we ask whether to
+    // resume. Discarding wipes the snapshot so the prompt doesn't
+    // recur on next launch.
+    void window.freecrawl.crashRecoveryStatus().then((status) => {
+      if (status.pendingCount === 0) return;
+      const proceed = window.confirm(
+        `FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` +
+          `  Start URL: ${status.seedUrl}\n` +
+          `  Pending URLs: ${status.pendingCount.toLocaleString()}\n\n` +
+          `Resume the crawl from where it stopped?\n\n` +
+          `Click OK to resume, Cancel to discard the recovery state and start fresh.`,
+      );
+      if (proceed) {
+        void window.freecrawl.crashRecoveryResume();
+      } else {
+        void window.freecrawl.crashRecoveryDiscard();
+      }
+    });
+
+    // rAF-throttled progress dispatch. The crawler emits up to 5
+    // progress events/sec and each one triggers a Zustand store
+    // update — TopBar, StatsBar and App.tsx all subscribe to the
+    // `progress` object reference, so every event fans out to a full
+    // React reconciliation. Coalescing into one update per animation
+    // frame caps the React work at 60 Hz worst-case (and at 5 Hz in
+    // the common case where the crawler's throttle dominates), so a
+    // user click never queues behind a stack of pending re-renders.
+    let pendingProgress: import('@freecrawl/shared-types').CrawlProgress | null = null;
+    let progressRafScheduled = false;
+    const off1 = window.freecrawl.onProgress((p) => {
+      pendingProgress = p;
+      if (progressRafScheduled) return;
+      progressRafScheduled = true;
+      requestAnimationFrame(() => {
+        progressRafScheduled = false;
+        if (pendingProgress) {
+          setProgress(pendingProgress);
+          pendingProgress = null;
+        }
+      });
+    });
     const off2 = window.freecrawl.onDone((s) => setSummary(s));
     const off3 = window.freecrawl.onError(setError);
     const offData = window.freecrawl.onDataChanged(() => bumpDataVersion());
@@ -133,6 +198,45 @@ export function App() {
         case 'export-json':
           void window.freecrawl.exportJson({ filePath: '', pretty: true });
           break;
+        case 'export-xml':
+          void window.freecrawl.exportXml({ filePath: '' });
+          break;
+        case 'delete-domain-data': {
+          const domain = window.prompt(
+            'GDPR Domain Wipe — enter the host to delete (case-insensitive, exact match).\n\nExamples: example.com  ·  blog.example.com\n\nThis cannot be undone. Save Project As… first to keep a backup.',
+            '',
+          );
+          if (!domain || !domain.trim()) break;
+          const target = domain.trim();
+          if (
+            !window.confirm(
+              `Delete every URL hosted on "${target}", along with its links, headers, images, and source snapshots?\n\nThis action cannot be undone.`,
+            )
+          ) {
+            break;
+          }
+          void window.freecrawl.dataDeleteByDomain({ domain: target }).then((res) => {
+            window.alert(
+              `Deleted ${res.urlsDeleted.toLocaleString()} URL row${
+                res.urlsDeleted === 1 ? '' : 's'
+              } from ${target} (and ${res.linksDeleted.toLocaleString()} associated link${
+                res.linksDeleted === 1 ? '' : 's'
+              }).`,
+            );
+          });
+          break;
+        }
+        case 'clear-all-data': {
+          if (
+            !window.confirm(
+              'Clear ALL data in the active project?\n\nEvery URL, link, image, header, source snapshot, sitemap, and issue will be wiped. This cannot be undone — Save Project As… first if you want a backup.',
+            )
+          ) {
+            break;
+          }
+          void window.freecrawl.crawlClear();
+          break;
+        }
         case 'open-robots-tester':
           setRobotsTesterOpen(true);
           break;

apps/desktop/src/renderer/src/components/OverviewSidebar.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from 'react';
+import { memo, useEffect, useMemo, useRef, useState } from 'react';
 import { ChevronDown, ChevronRight } from 'lucide-react';
 import clsx from 'clsx';
 import type { OverviewCounts, UrlCategory } from '@freecrawl/shared-types';
@@ -16,7 +16,13 @@ export function OverviewSidebar() {
   const overview = useAppStore((s) => s.overview);
   const setOverview = useAppStore((s) => s.setOverview);
   const dataVersion = useAppStore((s) => s.dataVersion);
-  const progress = useAppStore((s) => s.progress);
+  // Subscribe to scalar fields of `progress` rather than the whole
+  // object. Zustand re-renders this component only when these specific
+  // values change — so the 5/s progress events that don't move the
+  // crawled count (the in-flight pending fluctuates every poll) no
+  // longer trigger a full sidebar re-render.
+  const progressRunning = useAppStore((s) => s.progress?.running ?? false);
+  const progressCrawled = useAppStore((s) => s.progress?.crawled ?? 0);
   const activeCategory = useAppStore((s) => s.activeCategory);
   const navigateToCategory = useAppStore((s) => s.navigateToCategory);
   const [expanded, setExpanded] = useState<Set<string>>(
@@ -84,13 +90,13 @@ export function OverviewSidebar() {
     // The crawler's `progress` event also bumps `progress.crawled`
     // which triggers a separate progress-driven refetch below, so
     // this interval is just a safety net.
-    const intervalMs = progress?.running ? 3000 : 30_000;
+    const intervalMs = progressRunning ? 3000 : 30_000;
     const id = setInterval(scheduleLoad, intervalMs);
     return () => {
       cancelled = true;
       clearInterval(id);
     };
-  }, [dataVersion, setOverview, progress?.running]);
+  }, [dataVersion, setOverview, progressRunning]);
 
   // Progress-driven refetch. When the crawler reports a meaningful
   // change (every 50 URLs crawled) we trigger an immediate sidebar
@@ -99,12 +105,12 @@ export function OverviewSidebar() {
   // twice if React re-renders for an unrelated reason.
   const lastRefetchAtRef = useRef(0);
   useEffect(() => {
-    if (!progress?.running) return;
-    const bucket = Math.floor((progress.crawled ?? 0) / 50);
+    if (!progressRunning) return;
+    const bucket = Math.floor(progressCrawled / 50);
     if (bucket === lastRefetchAtRef.current) return;
     lastRefetchAtRef.current = bucket;
     void window.freecrawl.overviewGet().then((o) => setOverview(o));
-  }, [progress?.crawled, progress?.running, setOverview]);
+  }, [progressCrawled, progressRunning, setOverview]);
 
   const toggle = (k: string) => {
     setExpanded((prev) => {
@@ -115,7 +121,9 @@ export function OverviewSidebar() {
     });
   };
 
-  const tree = buildTree(overview);
+  // The tree itself only depends on `overview`. Memoising it skips the
+  // 150-node object construction on every progress-driven render.
+  const tree = useMemo(() => buildTree(overview), [overview]);
   const totalForPercent = overview?.summary.totalInternalUrls ?? 0;
 
   return (
@@ -149,7 +157,7 @@ export function OverviewSidebar() {
   );
 }
 
-function TreeNode({
+const TreeNode = memo(function TreeNode({
   node,
   depth,
   expanded,
@@ -219,7 +227,7 @@ function TreeNode({
         ))}
     </>
   );
-}
+});
 
 function buildTree(o: OverviewCounts | null): Node[] {
   if (!o) return [];
@@ -1106,6 +1114,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.paginationBroken,
               category: 'issues:pagination-broken',
             },
+            {
+              key: 'issues-pagination-sequence-break',
+              label: 'Sequence Break (gap in numbering)',
+              count: o.issues.paginationSequenceBreak,
+              category: 'issues:pagination-sequence-break',
+            },
           ],
         },
         {
@@ -1142,6 +1156,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.hreflangTargetIssues,
               category: 'issues:hreflang-target-issues',
             },
+            {
+              key: 'issues-hreflang-inconsistent-lang',
+              label: 'Inconsistent Lang (same lang, two hrefs)',
+              count: o.issues.hreflangInconsistentLang,
+              category: 'issues:hreflang-inconsistent-lang',
+            },
           ],
         },
         {
@@ -1232,6 +1252,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.externalLinksTooMany,
               category: 'issues:external-links-too-many',
             },
+            {
+              key: 'issues-links-per-page-too-many',
+              label: 'Total Links per Page > 100',
+              count: o.issues.linksPerPageTooMany,
+              category: 'issues:links-per-page-too-many',
+            },
             {
               key: 'issues-outlinks-zero',
               label: 'No Outlinks (Dead-End)',

apps/desktop/src/renderer/src/components/SettingsDialog.tsx

@@ -10,6 +10,7 @@ import {
   Cpu,
   Copy,
   Code2,
+  Cookie,
   Webhook,
   Plus,
   Trash2,
@@ -17,6 +18,9 @@ import {
   Network,
   Sparkles,
   Gauge,
+  FileText,
+  AlertTriangle,
+  Wrench,
   type LucideIcon,
 } from 'lucide-react';
 import clsx from 'clsx';
@@ -87,6 +91,33 @@ interface FormState {
   proxyUrl: string;
   excludeExtensionsText: string;
   maxRedirects: string;
+  // Wave 8 additions
+  // crawl analysis (per-pass post-crawl toggles, Wave 6)
+  analyseInlinks: boolean;
+  analyseRedirectChains: boolean;
+  analyseHreflang: boolean;
+  analyseDuplicates: boolean;
+  analysePagination: boolean;
+  analyseIssues: boolean;
+  // content (body snapshot)
+  storeBodySnapshots: boolean;
+  bodySnapshotMaxBytes: string;
+  // advanced (Wave 3 caps + URL/query thresholds)
+  maxLinksPerPage: string;
+  maxResponseTimeMs: string;
+  maxFileSizeBytes: string;
+  maxUrlLength: string;
+  maxQueryStringLength: string;
+  maxFolderDepth: string;
+  followCanonicals: boolean;
+  followPaginationLinks: boolean;
+  followNofollow: boolean;
+  followJsRedirects: boolean;
+  // Wave 9 — Cookies / per-host UA / proxy profiles
+  cookiePolicy: 'reject-all' | 'accept-all' | 'block-third-party';
+  perHostUserAgents: { hostPattern: string; userAgent: string }[];
+  proxyProfiles: { name: string; url: string }[];
+  proxyProfileActive: string;
 }
 
 type SectionKey =
@@ -96,6 +127,7 @@ type SectionKey =
   | 'speed'
   | 'requests'
   | 'filters'
+  | 'content'
   | 'custom-search'
   | 'custom-extraction'
   | 'url-rewriting'
@@ -103,7 +135,12 @@ type SectionKey =
   | 'auth'
   | 'network'
   | 'hardware'
-  | 'webhook';
+  | 'webhook'
+  | 'crawl-analysis'
+  | 'issues'
+  | 'advanced'
+  | 'cookies'
+  | 'per-host-ua';
 
 interface SectionDef {
   key: SectionKey;
@@ -201,6 +238,45 @@ const SECTIONS: SectionDef[] = [
     icon: Webhook,
     keywords: 'webhook notify slack discord zapier post crawl complete',
   },
+  {
+    key: 'content',
+    label: 'Content',
+    icon: FileText,
+    keywords:
+      'content body source snapshot store html size cap thin word count text view source',
+  },
+  {
+    key: 'crawl-analysis',
+    label: 'Crawl Analysis',
+    icon: ListChecks,
+    keywords:
+      'analysis post crawl inlinks redirect hreflang duplicate pagination issues materialise pass toggle',
+  },
+  {
+    key: 'issues',
+    label: 'Issues',
+    icon: AlertTriangle,
+    keywords: 'issues check filter enable disable severity false positive',
+  },
+  {
+    key: 'advanced',
+    label: 'Advanced',
+    icon: Wrench,
+    keywords:
+      'advanced max links per page response time file size url length query folder depth follow canonical pagination nofollow js redirect',
+  },
+  {
+    key: 'cookies',
+    label: 'Cookies',
+    icon: Cookie,
+    keywords: 'cookie session reject accept block third party set-cookie policy',
+  },
+  {
+    key: 'per-host-ua',
+    label: 'Per-Host UA',
+    icon: Send,
+    keywords: 'per host user agent subdomain mobile desktop pattern wildcard',
+  },
 ];
 
 function configToForm(c: CrawlConfig): FormState {
@@ -243,6 +319,28 @@ function configToForm(c: CrawlConfig): FormState {
     proxyUrl: c.proxyUrl ?? '',
     excludeExtensionsText: (c.excludeExtensions ?? []).join(', '),
     maxRedirects: String(c.maxRedirects ?? 10),
+    analyseInlinks: c.analyseInlinks ?? true,
+    analyseRedirectChains: c.analyseRedirectChains ?? true,
+    analyseHreflang: c.analyseHreflang ?? true,
+    analyseDuplicates: c.analyseDuplicates ?? true,
+    analysePagination: c.analysePagination ?? true,
+    analyseIssues: c.analyseIssues ?? true,
+    storeBodySnapshots: c.storeBodySnapshots ?? true,
+    bodySnapshotMaxBytes: String(c.bodySnapshotMaxBytes ?? 1_048_576),
+    maxLinksPerPage: String(c.maxLinksPerPage ?? 100),
+    maxResponseTimeMs: String(c.maxResponseTimeMs ?? 0),
+    maxFileSizeBytes: String(c.maxFileSizeBytes ?? 0),
+    maxUrlLength: String(c.maxUrlLength ?? 2048),
+    maxQueryStringLength: String(c.maxQueryStringLength ?? 0),
+    maxFolderDepth: String(c.maxFolderDepth ?? 0),
+    followCanonicals: c.followCanonicals ?? false,
+    followPaginationLinks: c.followPaginationLinks ?? true,
+    followNofollow: c.followNofollow ?? false,
+    followJsRedirects: c.followJsRedirects ?? false,
+    cookiePolicy: c.cookiePolicy ?? 'reject-all',
+    perHostUserAgents: (c.perHostUserAgents ?? []).map((r) => ({ ...r })),
+    proxyProfiles: (c.proxyProfiles ?? []).map((p) => ({ ...p })),
+    proxyProfileActive: c.proxyProfileActive ?? '',
   };
 }
 
@@ -371,6 +469,47 @@ export function SettingsDialog({ open, onClose }: Props) {
         .map((s) => s.trim().toLowerCase().replace(/^\./, ''))
         .filter(Boolean),
       maxRedirects: Math.max(0, num(form.maxRedirects, config.maxRedirects)),
+      analyseInlinks: form.analyseInlinks,
+      analyseRedirectChains: form.analyseRedirectChains,
+      analyseHreflang: form.analyseHreflang,
+      analyseDuplicates: form.analyseDuplicates,
+      analysePagination: form.analysePagination,
+      analyseIssues: form.analyseIssues,
+      storeBodySnapshots: form.storeBodySnapshots,
+      bodySnapshotMaxBytes: Math.max(
+        0,
+        num(form.bodySnapshotMaxBytes, config.bodySnapshotMaxBytes),
+      ),
+      maxLinksPerPage: Math.max(0, num(form.maxLinksPerPage, config.maxLinksPerPage)),
+      maxResponseTimeMs: Math.max(
+        0,
+        num(form.maxResponseTimeMs, config.maxResponseTimeMs),
+      ),
+      maxFileSizeBytes: Math.max(
+        0,
+        num(form.maxFileSizeBytes, config.maxFileSizeBytes),
+      ),
+      maxUrlLength: Math.max(0, num(form.maxUrlLength, config.maxUrlLength)),
+      maxQueryStringLength: Math.max(
+        0,
+        num(form.maxQueryStringLength, config.maxQueryStringLength),
+      ),
+      maxFolderDepth: Math.max(0, num(form.maxFolderDepth, config.maxFolderDepth)),
+      followCanonicals: form.followCanonicals,
+      followPaginationLinks: form.followPaginationLinks,
+      followNofollow: form.followNofollow,
+      followJsRedirects: form.followJsRedirects,
+      cookiePolicy: form.cookiePolicy,
+      perHostUserAgents: form.perHostUserAgents
+        .map((r) => ({
+          hostPattern: r.hostPattern.trim(),
+          userAgent: r.userAgent.trim(),
+        }))
+        .filter((r) => r.hostPattern && r.userAgent),
+      proxyProfiles: form.proxyProfiles
+        .map((p) => ({ name: p.name.trim(), url: p.url.trim() }))
+        .filter((p) => p.name && p.url),
+      proxyProfileActive: form.proxyProfileActive.trim(),
     });
     onClose();
   }
@@ -511,6 +650,22 @@ export function SettingsDialog({ open, onClose }: Props) {
               {active === 'webhook' && (
                 <WebhookPanel form={form} update={update} />
               )}
+              {active === 'content' && (
+                <ContentPanel form={form} update={update} />
+              )}
+              {active === 'crawl-analysis' && (
+                <CrawlAnalysisPanel form={form} update={update} />
+              )}
+              {active === 'issues' && <IssuesPanel />}
+              {active === 'advanced' && (
+                <AdvancedPanel form={form} update={update} />
+              )}
+              {active === 'cookies' && (
+                <CookiesPanel form={form} update={update} />
+              )}
+              {active === 'per-host-ua' && (
+                <PerHostUaPanel form={form} update={update} />
+              )}
             </div>
           </div>
         </div>
@@ -1415,6 +1570,88 @@ function NetworkPanel({ form, update }: PanelProps) {
         />
       </div>
 
+      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
+        <div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
+          Saved Proxy Profiles
+        </div>
+        <p className="mb-2 text-[10px] text-surface-500">
+          Save multiple `(name, URL)` pairs and switch between them via
+          the dropdown below. The active profile overrides the Proxy URL
+          field and the HTTPS_PROXY env var. Empty selection falls back
+          to the Proxy URL above.
+        </p>
+        <label className="mb-3 flex flex-col gap-1">
+          <FieldLabel
+            label="Active profile"
+            info="Picks one of the saved profiles by name. Empty = use the Proxy URL field above (or env vars when that's also empty)."
+          />
+          <select
+            className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+            value={form.proxyProfileActive}
+            onChange={(e) => update('proxyProfileActive', e.target.value)}
+          >
+            <option value="">— none (use Proxy URL above) —</option>
+            {form.proxyProfiles.map((p) => (
+              <option key={p.name} value={p.name}>
+                {p.name}
+              </option>
+            ))}
+          </select>
+        </label>
+        <div className="space-y-2">
+          {form.proxyProfiles.length === 0 && (
+            <div className="rounded border border-dashed border-surface-700 px-3 py-3 text-center text-[11px] text-surface-500">
+              No saved profiles — add one below.
+            </div>
+          )}
+          {form.proxyProfiles.map((p, i) => (
+            <div key={i} className="flex items-center gap-2">
+              <input
+                className="w-32 rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                placeholder="Office"
+                value={p.name}
+                onChange={(e) => {
+                  const next = [...form.proxyProfiles];
+                  next[i] = { ...p, name: e.target.value };
+                  update('proxyProfiles', next);
+                }}
+              />
+              <input
+                className="flex-1 rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                placeholder="http://proxy.corp:8080"
+                value={p.url}
+                onChange={(e) => {
+                  const next = [...form.proxyProfiles];
+                  next[i] = { ...p, url: e.target.value };
+                  update('proxyProfiles', next);
+                }}
+              />
+              <button
+                className="rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-300 hover:border-red-500 hover:text-red-300"
+                onClick={() => {
+                  const next = form.proxyProfiles.filter((_, j) => j !== i);
+                  update('proxyProfiles', next);
+                  if (form.proxyProfileActive === p.name) {
+                    update('proxyProfileActive', '');
+                  }
+                }}
+                aria-label="Remove profile"
+              >
+                <Trash2 className="h-3 w-3" />
+              </button>
+            </div>
+          ))}
+        </div>
+        <button
+          className="mt-2 flex items-center gap-1 rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-200 hover:border-blue-500 hover:bg-surface-800"
+          onClick={() =>
+            update('proxyProfiles', [...form.proxyProfiles, { name: '', url: '' }])
+          }
+        >
+          <Plus className="h-3 w-3" /> Add proxy profile
+        </button>
+      </div>
+
       <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
         <Text
           label="Exclude extensions (comma-separated)"
@@ -1570,6 +1807,323 @@ function HardwarePanel({ form, update }: PanelProps) {
   );
 }
 
+function ContentPanel({ form, update }: PanelProps) {
+  return (
+    <>
+      <p className="mb-3 text-[11px] text-surface-400">
+        How crawled HTML is stored on disk. Disable body snapshots to keep
+        the project file small when you don't need the View Source detail
+        tab; tighten the cap for sites with adversarially-large pages.
+      </p>
+
+      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
+        <div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
+          Body Snapshots
+        </div>
+        <Bool
+          label="Store raw HTML body per page"
+          checked={form.storeBodySnapshots}
+          onChange={(v) => update('storeBodySnapshots', v)}
+          info="Drives the View Source detail tab. ~30–200 KB on disk per HTML page; turn off if you only need metadata and not full source viewing."
+          example="On for SEO audits where View Source matters; off for 1M-URL crawls where disk is tight."
+        />
+        <div className="mt-3">
+          <Num
+            label="Body cap per page (bytes) — 0 = unbounded"
+            value={form.bodySnapshotMaxBytes}
+            onChange={(v) => update('bodySnapshotMaxBytes', v)}
+            info="Bodies over this are truncated and flagged. 1 MB covers the 99.9th percentile of HTML pages without letting one adversarial 50 MB page bloat the project file."
+            example="1048576 (1 MB) default; 524288 (512 KB) on tight disks; 0 to disable truncation entirely."
+          />
+        </div>
+      </div>
+    </>
+  );
+}
+
+function CrawlAnalysisPanel({ form, update }: PanelProps) {
+  return (
+    <>
+      <p className="mb-3 text-[11px] text-surface-400">
+        Per-pass post-crawl analysis toggles. Each pass runs after the
+        HTTP fetch phase finishes and feeds different issue filters. Skip
+        passes you don't need to shave wall-clock on large crawls; the
+        related issue counters quietly read as 0 until the pass runs.
+      </p>
+
+      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3 space-y-2">
+        <Bool
+          label="Recompute inlinks"
+          checked={form.analyseInlinks}
+          onChange={(v) => update('analyseInlinks', v)}
+          info="Counts how many internal pages link to each URL. Drives the Most-Linked URLs report and the per-row Inlinks column."
+        />
+        <Bool
+          label="Recompute redirect chains"
+          checked={form.analyseRedirectChains}
+          onChange={(v) => update('analyseRedirectChains', v)}
+          info="Walks 3xx redirect chains, fills `redirect_chain_length` / `redirect_loop`. Drives the 'Long Chain' and 'Redirect Loop' issues + the Redirects tab."
+        />
+        <Bool
+          label="Hreflang reciprocity + inconsistent lang"
+          checked={form.analyseHreflang}
+          onChange={(v) => update('analyseHreflang', v)}
+          info="Page A→B declared but B→A absent flags 'Reciprocity Missing'; same lang on two hrefs flags 'Inconsistent Lang'."
+        />
+        <Bool
+          label="Near-duplicate clustering"
+          checked={form.analyseDuplicates}
+          onChange={(v) => update('analyseDuplicates', v)}
+          info="64-bit SimHash + LSH bucketing + Union-Find clustering on body shingles. Most expensive pass — typical 5–10 s on a 100k crawl."
+        />
+        <Bool
+          label="Pagination ordinal-gap detection"
+          checked={form.analysePagination}
+          onChange={(v) => update('analysePagination', v)}
+          info="?page=1 / ?page=2 / ?page=4 → flags 'Sequence Break' on every member of the broken cluster."
+        />
+        <Bool
+          label="Materialise heavy issue counters"
+          checked={form.analyseIssues}
+          onChange={(v) => update('analyseIssues', v)}
+          info="Pre-computes Dead External Domain, Duplicate URL post-norm, Canonical Chain Multi-hop. Without this the sidebar shows 0 for those three."
+        />
+      </div>
+    </>
+  );
+}
+
+function IssuesPanel() {
+  return (
+    <>
+      <p className="mb-3 text-[11px] text-surface-400">
+        Per-issue check on/off toggles.
+      </p>
+      <div className="rounded border border-amber-700/40 bg-amber-900/10 p-3 text-[11px] text-amber-200">
+        <strong>Coming in V2.</strong> Today every issue check runs
+        unconditionally and surfaces in the sidebar. The plan is to let
+        you silence specific checks per-project (e.g. disable
+        "Description = Title" on a CMS that's known to do it
+        intentionally). Until that ships, hide rows you don't care about
+        by collapsing the sidebar group, or filter them out via the
+        Advanced filter on each tab.
+      </div>
+    </>
+  );
+}
+
+function AdvancedPanel({ form, update }: PanelProps) {
+  return (
+    <>
+      <p className="mb-3 text-[11px] text-surface-400">
+        Lower-level caps and link-follow toggles. Defaults are tuned for
+        typical SEO audits — only touch these if you know why.
+      </p>
+
+      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
+        <div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
+          Link & Response Caps
+        </div>
+        <Num
+          label="Max links per page (issue threshold)"
+          value={form.maxLinksPerPage}
+          onChange={(v) => update('maxLinksPerPage', v)}
+          info="Pages with > this many outgoing links (internal + external) trip the 'Total Links per Page' issue. Google's historic recommendation is 100; mega-menus/hub-pages routinely blow past this."
+          example="100 default; 50 for tight on-page link discipline; 0 to disable the issue."
+        />
+        <Num
+          label="Max response time (ms) — 0 = disabled"
+          value={form.maxResponseTimeMs}
+          onChange={(v) => update('maxResponseTimeMs', v)}
+          info="Aborts requests whose total lifetime (connect + headers + body) exceeds this. Distinct from `requestTimeoutMs` which is the headers timeout. Useful for capping individual slow pages without lowering the overall fetch timeout."
+          example="60000 (1 minute) for huge resources; 0 to rely solely on the fetch timeout."
+        />
+        <Num
+          label="Max file size (bytes) — 0 = disabled"
+          value={form.maxFileSizeBytes}
+          onChange={(v) => update('maxFileSizeBytes', v)}
+          info="Skips body parsing for pages whose Content-Length header exceeds this. The page row is still created so links to it aren't lost; only body parsing and source snapshot capture are skipped."
+          example="10485760 (10 MB) on bandwidth-tight crawls; 0 to download anything."
+        />
+      </div>
+
+      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
+        <div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
+          URL Structure Thresholds
+        </div>
+        <Num
+          label="Max URL length (chars)"
+          value={form.maxUrlLength}
+          onChange={(v) => update('maxUrlLength', v)}
+          info="Trips the 'URL Too Long' issue when LENGTH(url) > this. RFC 7230 doesn't mandate a max but most servers + middleboxes fail above ~2 KB; Chrome itself caps at ~32 KB."
+          example="2048 default (RFC-suggested practical ceiling)."
+        />
+        <Num
+          label="Max query string length (chars) — 0 = disabled"
+          value={form.maxQueryStringLength}
+          onChange={(v) => update('maxQueryStringLength', v)}
+          info="Trips 'Long Query String' when LENGTH(query) > this. Typical session-id sprawl + UTM tracking hits 100+ chars; over 200 starts to look like a bug."
+          example="100 default for most audits; 0 to disable the check."
+        />
+        <Num
+          label="Max folder depth — 0 = disabled"
+          value={form.maxFolderDepth}
+          onChange={(v) => update('maxFolderDepth', v)}
+          info="Trips 'Folder Depth Too Deep' when the URL path's `/`-segment count exceeds this. Useful for spotting over-nested URL structures that bury content from crawlers."
+          example="4 default; 6 on documentation sites with deep TOC trees; 0 to disable."
+        />
+      </div>
+
+      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3 space-y-2">
+        <div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
+          Link Follow Behaviour
+        </div>
+        <Bool
+          label="Follow canonical targets"
+          checked={form.followCanonicals}
+          onChange={(v) => update('followCanonicals', v)}
+          info="When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint."
+        />
+        <Bool
+          label="Follow rel=next / rel=prev"
+          checked={form.followPaginationLinks}
+          onChange={(v) => update('followPaginationLinks', v)}
+          info="When on (default), pagination_next + pagination_prev URLs are post-fetch enqueued. Off only to debug pagination-only loops without disabling all link follow."
+        />
+        <Bool
+          label="Follow nofollow links (override 'respect nofollow')"
+          checked={form.followNofollow}
+          onChange={(v) => update('followNofollow', v)}
+          info="When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."
+        />
+        <Bool
+          label="Follow JS / meta-refresh redirects"
+          checked={form.followJsRedirects}
+          onChange={(v) => update('followJsRedirects', v)}
+          info="When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."
+        />
+      </div>
+    </>
+  );
+}
+
+function CookiesPanel({ form, update }: PanelProps) {
+  return (
+    <>
+      <p className="mb-3 text-[11px] text-surface-400">
+        Cookie policy applied to every fetch. The crawler is otherwise
+        stateless across requests; this setting controls whether
+        Set-Cookie response headers are recorded for the cookie-flag
+        issue checks (Missing Secure / HttpOnly / SameSite).
+      </p>
+
+      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
+        <label className="flex flex-col gap-1">
+          <FieldLabel
+            label="Cookie policy"
+            info="Reject-all = ignore Set-Cookie entirely (zero counts on cookie-flag issues). Block-third-party = analyse only first-party cookies (Domain attribute matches the page's registrable domain). Accept-all = analyse every Set-Cookie regardless of scope."
+            example="Reject-all for stateless audits; Block-third-party to focus on the site's own cookie hygiene; Accept-all to also see ad/analytics tracker cookies."
+          />
+          <select
+            className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+            value={form.cookiePolicy}
+            onChange={(e) =>
+              update(
+                'cookiePolicy',
+                e.target.value as 'reject-all' | 'accept-all' | 'block-third-party',
+              )
+            }
+          >
+            <option value="reject-all">Reject all (default — stateless)</option>
+            <option value="accept-all">Accept all</option>
+            <option value="block-third-party">Block third-party</option>
+          </select>
+        </label>
+        <p className="mt-2 text-[10px] text-surface-500">
+          Cookie values themselves are <strong>never</strong> stored in
+          the project file regardless of this setting — only the security
+          flag (Secure / HttpOnly / SameSite) counts are kept.
+        </p>
+      </div>
+    </>
+  );
+}
+
+function PerHostUaPanel({ form, update }: PanelProps) {
+  const rules = form.perHostUserAgents;
+  return (
+    <>
+      <p className="mb-3 text-[11px] text-surface-400">
+        Override the User-Agent on a per-host basis. Useful when crawling
+        a mobile subdomain with the mobile-Googlebot UA in the same run
+        as the desktop site, or when a CDN serves a different page based
+        on the requester's UA. The first matching pattern wins; the
+        global User-Agent (Requests tab) is the fallback.
+      </p>
+      <div className="mb-3 rounded border border-surface-800 bg-surface-950/40 p-3">
+        <p className="mb-2 text-[10px] text-surface-500">
+          Pattern syntax: exact host (<code>m.example.com</code>) or
+          leading wildcard (<code>*.example.com</code>) — the wildcard
+          form matches any subdomain but <em>not</em> the apex.
+        </p>
+        <div className="space-y-2">
+          {rules.length === 0 && (
+            <div className="rounded border border-dashed border-surface-700 px-3 py-4 text-center text-[11px] text-surface-500">
+              No per-host overrides yet — add one below.
+            </div>
+          )}
+          {rules.map((r, i) => (
+            <div key={i} className="flex items-center gap-2">
+              <input
+                className="flex-1 rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                placeholder="*.m.example.com"
+                value={r.hostPattern}
+                onChange={(e) => {
+                  const next = [...rules];
+                  next[i] = { ...r, hostPattern: e.target.value };
+                  update('perHostUserAgents', next);
+                }}
+              />
+              <input
+                className="flex-[2] rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                placeholder="Mozilla/5.0 (iPhone; …)"
+                value={r.userAgent}
+                onChange={(e) => {
+                  const next = [...rules];
+                  next[i] = { ...r, userAgent: e.target.value };
+                  update('perHostUserAgents', next);
+                }}
+              />
+              <button
+                className="rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-300 hover:border-red-500 hover:text-red-300"
+                onClick={() => {
+                  const next = rules.filter((_, j) => j !== i);
+                  update('perHostUserAgents', next);
+                }}
+                title="Remove"
+                aria-label="Remove rule"
+              >
+                <Trash2 className="h-3 w-3" />
+              </button>
+            </div>
+          ))}
+        </div>
+        <button
+          className="mt-3 flex items-center gap-1 rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-200 hover:border-blue-500 hover:bg-surface-800"
+          onClick={() =>
+            update('perHostUserAgents', [
+              ...rules,
+              { hostPattern: '', userAgent: '' },
+            ])
+          }
+        >
+          <Plus className="h-3 w-3" /> Add per-host UA rule
+        </button>
+      </div>
+    </>
+  );
+}
+
 function FieldLabel({
   label,
   info,

apps/desktop/src/renderer/src/components/TopBar.tsx

@@ -15,10 +15,23 @@ const SCOPE_OPTIONS: { value: CrawlScope; label: string; hint: string }[] = [
 export function TopBar() {
   const config = useAppStore((s) => s.config);
   const setConfig = useAppStore((s) => s.setConfig);
-  const progress = useAppStore((s) => s.progress);
+  // Scalar subscriptions instead of the full `progress` object — TopBar
+  // only needs a handful of fields, and Zustand bails out of the
+  // re-render when the SCALAR values are unchanged. With the full
+  // object subscription a new reference on every emitProgress() pinned
+  // this component to the renderer's hot path.
+  const running = useAppStore((s) => s.progress?.running === true);
+  const paused = useAppStore((s) => s.progress?.paused === true);
+  const progressDiscovered = useAppStore((s) => s.progress?.discovered ?? 0);
+  const progressCrawled = useAppStore((s) => s.progress?.crawled ?? 0);
   const setProgress = useAppStore((s) => s.setProgress);
-  const summary = useAppStore((s) => s.summary);
-  const overview = useAppStore((s) => s.overview);
+  const summaryTotal = useAppStore((s) => s.summary?.total ?? 0);
+  const overviewInternalTotal = useAppStore(
+    (s) => s.overview?.summary.totalInternalUrls ?? 0,
+  );
+  const overviewExternalTotal = useAppStore(
+    (s) => s.overview?.summary.totalExternalUrls ?? 0,
+  );
   const reset = useAppStore((s) => s.reset);
   const setError = useAppStore((s) => s.setError);
   const setSettingsOpen = useAppStore((s) => s.setSettingsOpen);
@@ -27,24 +40,15 @@ export function TopBar() {
   const [scopeOpen, setScopeOpen] = useState(false);
   const [recentOpen, setRecentOpen] = useState(false);
 
-  const running = progress?.running === true;
-  const paused = progress?.paused === true;
-  // Clear is enabled whenever there's anything to wipe. We check four
-  // signals because each one alone is incomplete:
-  //   - `progress.discovered/crawled` covers a fresh run
-  //   - `summary.total` covers post-`done` state (progress may have
-  //     been wiped to its idle baseline by then)
-  //   - `overview.summary.totalInternalUrls` covers projects opened
-  //     from disk via File → Open Recent (no progress event ever
-  //     fired for those rows)
-  //   - `paused` is a "running" sub-state where the table already has
-  //     rows that the user might want to dump
+  // Clear is enabled whenever there's anything to wipe. Four signals
+  // because each one alone is incomplete (fresh run vs. post-`done`
+  // state vs. project opened from disk vs. paused crawl with rows).
   const hasData =
-    (progress?.discovered ?? 0) > 0 ||
-    (progress?.crawled ?? 0) > 0 ||
-    (summary?.total ?? 0) > 0 ||
-    (overview?.summary.totalInternalUrls ?? 0) > 0 ||
-    (overview?.summary.totalExternalUrls ?? 0) > 0;
+    progressDiscovered > 0 ||
+    progressCrawled > 0 ||
+    summaryTotal > 0 ||
+    overviewInternalTotal > 0 ||
+    overviewExternalTotal > 0;
   const activeScope = SCOPE_OPTIONS.find((o) => o.value === config.scope)!;
 
   async function start() {

apps/desktop/src/renderer/src/components/VisualizationDialog.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useRef, useState } from 'react';
-import { X, RefreshCw, Sparkles, Settings2, RotateCcw } from 'lucide-react';
+import { X, RefreshCw, Sparkles, Settings2, RotateCcw, Download } from 'lucide-react';
 import cytoscape, { type Core } from 'cytoscape';
 
 /**
@@ -42,6 +42,152 @@ function loadTuning(): VisTuning {
   }
   return DEFAULT_TUNING;
 }
+
+/**
+ * Trigger a download of `data` as `filename`. Anchor-click pattern
+ * works in Electron's renderer because chromium honours the
+ * `download` attribute and the in-app save flow doesn't need shell
+ * permissions for a Blob URL. URL is revoked after a short delay so
+ * Chromium has time to attach the download.
+ */
+function downloadBlob(data: Blob, filename: string): void {
+  const url = URL.createObjectURL(data);
+  const a = document.createElement('a');
+  a.href = url;
+  a.download = filename;
+  a.style.display = 'none';
+  document.body.appendChild(a);
+  a.click();
+  setTimeout(() => {
+    document.body.removeChild(a);
+    URL.revokeObjectURL(url);
+  }, 5_000);
+}
+
+/**
+ * Export the current Cytoscape canvas as a high-DPI PNG. Cytoscape's
+ * `cy.png()` rasterises into the requested pixel dimensions (we use 2×
+ * the visible size for retina-quality print) and returns a base64 data
+ * URL which we convert to a Blob for the file download.
+ */
+function exportPng(cy: Core, filename = 'freecrawl-graph.png'): void {
+  const dataUrl = cy.png({
+    output: 'base64uri',
+    full: true, // include nodes outside the current viewport
+    bg: '#0a0a0f',
+    scale: 2,
+  });
+  // base64uri → Blob via fetch — cleaner than manual atob() decoding
+  // and supported in Chromium since forever.
+  void fetch(dataUrl)
+    .then((r) => r.blob())
+    .then((b) => downloadBlob(b, filename));
+}
+
+/**
+ * Export the current canvas as SVG. We don't pull in cytoscape-svg as
+ * a dependency — Cytoscape's `jpg`/`png` API has no native SVG output,
+ * so we emit a minimal hand-rolled SVG: `<circle>` per node + `<line>`
+ * per edge, using the runtime renderedPosition coordinates so the
+ * output matches what the user sees on screen.
+ *
+ * The result is editable in Illustrator / Inkscape / Figma — the
+ * primary use case for "give me an SVG of my graph".
+ */
+function exportSvg(cy: Core, filename = 'freecrawl-graph.svg'): void {
+  const bbox = cy.elements().boundingBox({});
+  const pad = 40;
+  const width = Math.ceil(bbox.w + pad * 2);
+  const height = Math.ceil(bbox.h + pad * 2);
+  const tx = -bbox.x1 + pad;
+  const ty = -bbox.y1 + pad;
+  const parts: string[] = [];
+  parts.push(
+    `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}">`,
+  );
+  parts.push(`<rect width="100%" height="100%" fill="#0a0a0f"/>`);
+  // Edges first so they sit beneath nodes.
+  cy.edges().forEach((e) => {
+    const src = e.source().position();
+    const tgt = e.target().position();
+    const c = String((e.style('line-color') as unknown) ?? '#475569');
+    parts.push(
+      `<line x1="${(src.x + tx).toFixed(1)}" y1="${(src.y + ty).toFixed(1)}" x2="${(tgt.x + tx).toFixed(1)}" y2="${(tgt.y + ty).toFixed(1)}" stroke="${escapeXml(c)}" stroke-width="0.6" opacity="0.6"/>`,
+    );
+  });
+  cy.nodes().forEach((n) => {
+    const p = n.position();
+    const r = Number(n.style('width') ?? 12) / 2;
+    const fill = String((n.style('background-color') as unknown) ?? '#3b82f6');
+    parts.push(
+      `<circle cx="${(p.x + tx).toFixed(1)}" cy="${(p.y + ty).toFixed(1)}" r="${r.toFixed(1)}" fill="${escapeXml(fill)}" stroke="#0a0a0f" stroke-width="1"/>`,
+    );
+  });
+  parts.push(`</svg>`);
+  downloadBlob(new Blob([parts.join('\n')], { type: 'image/svg+xml' }), filename);
+}
+
+function escapeXml(s: string): string {
+  return s
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+}
+
+/**
+ * Export the graph as a self-contained HTML file. Embeds Cytoscape
+ * from a CDN + the snapshot data inline; the user can open it
+ * directly in any browser, share via email/Slack, or attach to a
+ * client report. No FreeCrawl runtime needed for the recipient.
+ *
+ * Trade-off: the resulting file requires internet to load Cytoscape
+ * the first time (CDN fetch). We could inline the library too at the
+ * cost of ~250 KB per export, which is reasonable for a one-shot
+ * deliverable but adds 5–10× to file size on most graphs.
+ */
+function exportStandaloneHtml(
+  cy: Core,
+  filename = 'freecrawl-graph.html',
+): void {
+  const elements = cy.json() as { elements: unknown };
+  const dataJson = JSON.stringify(elements.elements ?? []);
+  const html = `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8"/>
+<title>FreeCrawl Graph</title>
+<style>
+  html, body { margin: 0; height: 100%; background: #0a0a0f; color: #e2e8f0; font-family: system-ui, sans-serif; }
+  #cy { position: absolute; inset: 0; }
+  #legend { position: absolute; top: 12px; left: 12px; padding: 8px 12px; background: rgba(15,23,42,0.85); border: 1px solid #334155; border-radius: 6px; font-size: 12px; }
+  #legend h1 { margin: 0 0 6px; font-size: 13px; font-weight: 600; }
+</style>
+</head>
+<body>
+<div id="cy"></div>
+<div id="legend">
+  <h1>FreeCrawl SEO — Site Graph</h1>
+  <div>Exported: ${new Date().toISOString()}</div>
+  <div>Nodes: ${cy.nodes().length} · Edges: ${cy.edges().length}</div>
+</div>
+<script src="https://unpkg.com/cytoscape@3/dist/cytoscape.min.js"></script>
+<script>
+  const elements = ${dataJson};
+  cytoscape({
+    container: document.getElementById('cy'),
+    elements: elements,
+    style: [
+      { selector: 'node', style: { 'background-color': 'data(color)', 'width': 'data(size)', 'height': 'data(size)', 'border-color': '#0a0a0f', 'border-width': 1, 'label': 'data(label)', 'color': '#cbd5e1', 'font-size': 9, 'text-valign': 'bottom', 'text-margin-y': 4 } },
+      { selector: 'edge', style: { 'width': 0.6, 'line-color': '#475569', 'opacity': 0.4, 'curve-style': 'bezier' } },
+    ],
+    layout: { name: 'preset' },
+  });
+</script>
+</body>
+</html>`;
+  downloadBlob(new Blob([html], { type: 'text/html' }), filename);
+}
 import type {
   AnchorTextRow,
   GraphSnapshotResult,
@@ -117,6 +263,7 @@ export function VisualizationDialog({ open, onClose }: Props) {
   const [selectedUrl, setSelectedUrl] = useState<string | null>(null);
   const [tuning, setTuning] = useState<VisTuning>(() => loadTuning());
   const [tunerOpen, setTunerOpen] = useState(false);
+  const [exportMenuOpen, setExportMenuOpen] = useState(false);
 
   function patchTuning(patch: Partial<VisTuning>) {
     setTuning((prev) => {
@@ -632,6 +779,55 @@ export function VisualizationDialog({ open, onClose }: Props) {
                 />
               )}
             </div>
+            <div className="relative">
+              <button
+                className={`flex items-center gap-1 rounded border px-2 py-1 text-[11px] ${
+                  exportMenuOpen
+                    ? 'border-blue-500 bg-surface-800 text-blue-200'
+                    : 'border-surface-700 text-surface-200 hover:border-blue-500 hover:bg-surface-800'
+                }`}
+                onClick={() => setExportMenuOpen((v) => !v)}
+                title="Export graph"
+                aria-label="Export graph"
+              >
+                <Download className="h-3 w-3" />
+                Export
+              </button>
+              {exportMenuOpen && (
+                <div
+                  className="absolute right-0 top-full mt-1 z-10 min-w-[200px] rounded border border-surface-700 bg-surface-900 shadow-lg"
+                  onMouseLeave={() => setExportMenuOpen(false)}
+                >
+                  <button
+                    className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
+                    onClick={() => {
+                      if (cyRef.current) exportPng(cyRef.current);
+                      setExportMenuOpen(false);
+                    }}
+                  >
+                    PNG (high-DPI raster)
+                  </button>
+                  <button
+                    className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
+                    onClick={() => {
+                      if (cyRef.current) exportSvg(cyRef.current);
+                      setExportMenuOpen(false);
+                    }}
+                  >
+                    SVG (vector — Illustrator/Figma)
+                  </button>
+                  <button
+                    className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
+                    onClick={() => {
+                      if (cyRef.current) exportStandaloneHtml(cyRef.current);
+                      setExportMenuOpen(false);
+                    }}
+                  >
+                    Standalone HTML (shareable)
+                  </button>
+                </div>
+              )}
+            </div>
           </div>
           <button
             className="ml-auto rounded p-1 text-surface-400 hover:bg-surface-800 hover:text-surface-100"

apps/desktop/src/renderer/src/store.ts

@@ -21,6 +21,7 @@ export type TabKey =
   | 'images'
   | 'canonicals'
   | 'directives'
+  | 'redirects'
   | 'links'
   | 'broken-links';
 
@@ -37,6 +38,7 @@ export const TAB_ORDER: { key: TabKey; label: string }[] = [
   { key: 'images', label: 'Images' },
   { key: 'canonicals', label: 'Canonicals' },
   { key: 'directives', label: 'Directives' },
+  { key: 'redirects', label: 'Redirects' },
   { key: 'links', label: 'Links' },
   { key: 'broken-links', label: 'Broken Links' },
 ];
@@ -50,6 +52,11 @@ interface AppState {
   activeCategory: UrlCategory;
   error: string | null;
   selectedUrlId: number | null;
+  /** Multi-row selection in the main URLs table. Used by sub-tab views
+   * (Inlinks / Outlinks / Images / Resources) to aggregate data across
+   * every selected row instead of just the primary `selectedUrlId`. When
+   * the user has only one row selected this mirrors `[selectedUrlId]`. */
+  selectedUrlIds: number[];
   sidebarOpen: boolean;
   detailPanelOpen: boolean;
   settingsOpen: boolean;
@@ -64,6 +71,7 @@ interface AppState {
   navigateToCategory: (c: UrlCategory) => void;
   setError: (e: string | null) => void;
   setSelectedUrlId: (id: number | null) => void;
+  setSelectedUrlIds: (ids: number[]) => void;
   toggleSidebar: () => void;
   toggleDetailPanel: () => void;
   setSettingsOpen: (open: boolean) => void;
@@ -108,6 +116,7 @@ export const useAppStore = create<AppState>((set) => ({
   activeCategory: 'internal:html',
   error: null,
   selectedUrlId: null,
+  selectedUrlIds: [],
   sidebarOpen: true,
   detailPanelOpen: true,
   settingsOpen: false,
@@ -139,6 +148,7 @@ export const useAppStore = create<AppState>((set) => ({
   },
   setError: (e) => set({ error: e }),
   setSelectedUrlId: (id) => set({ selectedUrlId: id }),
+  setSelectedUrlIds: (ids) => set({ selectedUrlIds: ids }),
   toggleSidebar: () => set((s) => ({ sidebarOpen: !s.sidebarOpen })),
   toggleDetailPanel: () => set((s) => ({ detailPanelOpen: !s.detailPanelOpen })),
   setSettingsOpen: (open) => set({ settingsOpen: open }),
@@ -165,6 +175,7 @@ export const useAppStore = create<AppState>((set) => ({
       overview: null,
       error: null,
       selectedUrlId: null,
+      selectedUrlIds: [],
     }),
 }));
 
@@ -192,6 +203,12 @@ function categoryForTab(tab: TabKey): UrlCategory {
       return 'all';
     case 'broken-links':
       return 'issues:broken-links-all';
+    case 'canonicals':
+      return 'tab:canonicals';
+    case 'directives':
+      return 'tab:directives';
+    case 'redirects':
+      return 'tab:redirects';
     default:
       return 'internal:html';
   }

apps/desktop/src/renderer/src/tabs/UrlsTab.tsx

@@ -30,6 +30,7 @@ export function UrlsTab() {
   const activeCategory = useAppStore((s) => s.activeCategory);
   const selectedUrlId = useAppStore((s) => s.selectedUrlId);
   const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
+  const setSelectedUrlIds = useAppStore((s) => s.setSelectedUrlIds);
   const [search, setSearch] = useState('');
   const [sortBy, setSortBy] = useState<keyof CrawlUrlRow | undefined>(undefined);
   const [sortDir, setSortDir] = useState<'asc' | 'desc'>('asc');
@@ -108,6 +109,24 @@ export function UrlsTab() {
     return () => document.removeEventListener('mouseup', onUp);
   }, []);
 
+  // Mirror the multi-selection into the store so the bottom detail
+  // panel can aggregate sub-tabs (Inlinks / Outlinks / Images / Resources)
+  // across every selected URL instead of only the primary `selectedUrlId`.
+  // Two sources contribute distinct URL ids:
+  //   1. selectedIds — explicit Row-number column picks.
+  //   2. selectedCells — keyed `${urlId}:${colIdx}`, so each entry's URL
+  //      counts toward multi-aggregation even when the user picked a
+  //      single value cell instead of the whole row.
+  useEffect(() => {
+    const ids = new Set<number>(selectedIds);
+    for (const k of selectedCells) {
+      const [idStr] = k.split(':');
+      const id = Number(idStr);
+      if (Number.isFinite(id)) ids.add(id);
+    }
+    setSelectedUrlIds([...ids]);
+  }, [selectedIds, selectedCells, setSelectedUrlIds]);
+
   const getWidth = useCallback(
     (c: ColumnSpec): number => columnWidths[columnId(c)] ?? c.size,
     [columnWidths],

apps/desktop/src/renderer/src/tabs/columns.ts

@@ -234,6 +234,31 @@ const COL = {
     info: 'Location header value when status is 3xx. The URL the server points to next; chain length is in the URL Details panel.',
     example: 'https://example.com/new-path',
   } as ColumnSpec,
+  redirectChainLength: {
+    key: 'redirectChainLength',
+    header: 'Chain Length',
+    size: 96,
+    kind: 'number',
+    align: 'right',
+    info: 'Number of redirect hops from this URL to its terminal target. Filled by the post-crawl `recomputeRedirectChains` walker. >3 trips the "Long Chain" issue.',
+    example: '2',
+  } as ColumnSpec,
+  redirectFinalUrl: {
+    key: 'redirectFinalUrl',
+    header: 'Final URL',
+    size: 360,
+    kind: 'mono',
+    info: 'Terminal URL the redirect chain resolves to. Empty when this row is itself the terminal (i.e. status is 2xx/4xx/5xx) or when the chain hits a loop.',
+    example: 'https://example.com/canonical',
+  } as ColumnSpec,
+  redirectLoop: {
+    key: 'redirectLoop',
+    header: 'Loop',
+    size: 64,
+    kind: 'text',
+    info: 'Y when the redirect chain originating at this URL contains a cycle (A → B → A) detected by the cycle-safe walker; the chain is otherwise unwalked.',
+    example: 'Y',
+  } as ColumnSpec,
   clusterId: {
     key: 'clusterId',
     header: 'Cluster',
@@ -351,5 +376,14 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
     COL.indexability,
     COL.status,
   ],
+  redirects: [
+    COL.url,
+    COL.status,
+    COL.redirectTarget,
+    COL.redirectChainLength,
+    COL.redirectFinalUrl,
+    COL.redirectLoop,
+    COL.inlinks,
+  ],
   links: [COL.url, COL.inlinks, COL.outlinks, COL.status, COL.indexability],
 };

packages/core/src/crawler.ts

@@ -98,21 +98,34 @@ export class Crawler extends EventEmitter {
    * not free on a 100k-URL DB. */
   private issueRecomputeTimer: NodeJS.Timeout | null = null;
   private static readonly ISSUE_RECOMPUTE_INTERVAL_MS = 30_000;
+  private issueRecomputeInFlight = false;
   private startIssueRecomputeTimer(): void {
     if (this.issueRecomputeTimer) return;
     this.issueRecomputeTimer = setInterval(() => {
       if (this.stopped || !this.running || this.paused) return;
-      try {
-        this.db.recomputeUrlsIssues(EXPENSIVE_ISSUE_DEFINITIONS);
-      } catch (err) {
-        // Recompute is best-effort — a transient SQLITE_BUSY here is
-        // not worth interrupting the crawl. Surface it as debug so
-        // someone watching logs can spot pathological cases.
+      // Drop overlapping ticks. The yielding recompute can take longer
+      // than the 30 s tick on very large projects; piling up calls
+      // would queue 70-statement transactions back-to-back and starve
+      // the crawler's own writes.
+      if (this.issueRecomputeInFlight) return;
+      this.issueRecomputeInFlight = true;
+      // Cooperative variant: yields between definitions so a 1–3 s
+      // recompute doesn't freeze the main thread (and the user's
+      // clicks) for the full duration. Trades atomic visibility of
+      // the rebuild for UI responsiveness — fine for the live tick
+      // since the post-crawl pass still uses the atomic version for
+      // the final state.
+      void this.db
+        .recomputeUrlsIssuesYielding(EXPENSIVE_ISSUE_DEFINITIONS)
+        .catch((err) => {
           this.emit(
             'debug',
             `issue counter recompute failed: ${err instanceof Error ? err.message : String(err)}`,
           );
-      }
+        })
+        .finally(() => {
+          this.issueRecomputeInFlight = false;
+        });
     }, Crawler.ISSUE_RECOMPUTE_INTERVAL_MS);
   }
   private stopIssueRecomputeTimer(): void {
@@ -121,6 +134,46 @@ export class Crawler extends EventEmitter {
       this.issueRecomputeTimer = null;
     }
   }
+
+  /** Wave 6 — Periodic checkpoint of the in-memory pending queue
+   * (URLs already enqueued but not yet fetched). Survives process
+   * crashes / OS reboots / OOM so the next launch can offer to
+   * resume. We snapshot from `seen + pending`-tracked items rather
+   * than poking p-queue's internals; the pending closure is captured
+   * as a `Map<url, depth>` updated alongside `enqueue` / completion.
+   * 30 s cadence balances recovery-loss window vs DB write pressure
+   * — at 100 URL/s a 30 s window is at most 3000 dropped URLs that
+   * the user has to re-fetch (cheap; their DB rows already exist as
+   * link stubs so dedup catches duplicates). */
+  private queueCheckpointTimer: NodeJS.Timeout | null = null;
+  private static readonly QUEUE_CHECKPOINT_INTERVAL_MS = 30_000;
+  /** Pending queue snapshot used by the checkpoint timer. Mirrors
+   * what's in `this.queue` minus already-completed items. */
+  private pendingItems = new Map<string, number>();
+  private startQueueCheckpointTimer(): void {
+    if (this.queueCheckpointTimer) return;
+    this.queueCheckpointTimer = setInterval(() => {
+      if (this.stopped || !this.running) return;
+      try {
+        const items = Array.from(this.pendingItems.entries()).map(([url, depth]) => ({
+          url,
+          depth,
+        }));
+        this.db.checkpointQueue(items, this.config.startUrl);
+      } catch (err) {
+        this.emit(
+          'debug',
+          `queue checkpoint failed: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    }, Crawler.QUEUE_CHECKPOINT_INTERVAL_MS);
+  }
+  private stopQueueCheckpointTimer(): void {
+    if (this.queueCheckpointTimer) {
+      clearInterval(this.queueCheckpointTimer);
+      this.queueCheckpointTimer = null;
+    }
+  }
   /**
    * Aborts any in-flight sitemap discovery on stop(). Without this, a
    * 21k-URL sitemap continues parsing in the background after Stop and
@@ -141,9 +194,77 @@ export class Crawler extends EventEmitter {
     trailingSlash?: 'leave' | 'strip' | 'add';
   };
 
-  constructor(config: CrawlConfig, db: ProjectDb) {
+  /**
+   * Optional freeze-watchdog hook. The desktop main process injects a
+   * callback that publishes the current "what is the crawler doing
+   * right now" string into a SharedArrayBuffer the watchdog reads. We
+   * deliberately keep this opt-in (default no-op) so the headless CLI
+   * doesn't need to know about the watchdog at all.
+   */
+  private readonly setOp: (op: string) => void;
+
+  /**
+   * Optional async HTML parser. Defaults to the synchronous in-process
+   * `parseHtml` from `./html-parser.js` (used by the headless CLI
+   * which has no worker pool). The desktop main process injects an
+   * implementation that dispatches into a `worker_threads` pool so
+   * cheerio's CPU work runs off the main thread — that single change
+   * eliminates the 12-25 s "fetch" stalls observed on large mağaza /
+   * blog HTML pages.
+   */
+  private readonly parsePage: (
+    html: string,
+    pageUrl: string,
+    opts: Parameters<typeof parseHtml>[2],
+  ) => Promise<ReturnType<typeof parseHtml>>;
+
+  /**
+   * Optional async writer for the per-URL hot path. When present, the
+   * crawler ships the upsert + headers + body snapshot + links +
+   * images batch to a writer worker thread instead of running the
+   * SQLite transaction on its own thread. Defaults to a synchronous
+   * inline implementation that mirrors the legacy code path so the
+   * CLI keeps working without any worker plumbing.
+   */
+  private readonly writeFetchedUrl: (
+    payload: Parameters<ProjectDb['writeFetchedUrl']>[0],
+  ) => Promise<{ urlId: number }>;
+
+  constructor(
+    config: CrawlConfig,
+    db: ProjectDb,
+    opts: {
+      setOp?: (op: string) => void;
+      parseHtml?: (
+        html: string,
+        pageUrl: string,
+        opts: Parameters<typeof parseHtml>[2],
+      ) => Promise<ReturnType<typeof parseHtml>>;
+      writeFetchedUrl?: (
+        payload: Parameters<ProjectDb['writeFetchedUrl']>[0],
+      ) => Promise<{ urlId: number }>;
+    } = {},
+  ) {
     super();
-    initHttpClient({ proxyOverride: config.proxyUrl });
+    this.setOp = opts.setOp ?? ((): void => undefined);
+    this.parsePage =
+      opts.parseHtml ??
+      ((html, pageUrl, parseOpts) => Promise.resolve(parseHtml(html, pageUrl, parseOpts)));
+    this.writeFetchedUrl =
+      opts.writeFetchedUrl ?? ((payload) => Promise.resolve(this.db.writeFetchedUrl(payload)));
+    // Wave 9 — Resolve the active proxy. If a named profile is selected
+    // and present in `proxyProfiles`, its URL wins over the legacy
+    // `proxyUrl` field; if the named profile doesn't resolve we fall
+    // back to `proxyUrl`, then to env vars (handled inside initHttpClient).
+    const resolvedProxy = (() => {
+      const active = (config.proxyProfileActive ?? '').trim();
+      if (active) {
+        const hit = (config.proxyProfiles ?? []).find((p) => p.name === active);
+        if (hit && hit.url.trim()) return hit.url.trim();
+      }
+      return config.proxyUrl ?? '';
+    })();
+    initHttpClient({ proxyOverride: resolvedProxy });
     this.config = config;
     this.db = db;
     const concurrency = Math.max(1, Math.min(200, config.maxConcurrency));
@@ -214,6 +335,7 @@ export class Crawler extends EventEmitter {
     this.startedAt = Date.now();
     this.stopped = false;
     this.running = true;
+    this.setOp(`crawl:start:${this.config.startUrl}`);
 
     // Fire an immediate progress event so the UI can flip to "Running"
     // before we block on resolveStartUrl (which can spend several seconds
@@ -253,7 +375,17 @@ export class Crawler extends EventEmitter {
       this.running = false;
       this.emitProgress();
       // Without this the UI hangs in "Running" forever after a bad start URL.
-      if (!this.stopped) this.emit('done', this.db.getSummary());
+      if (!this.stopped) {
+      // Wave 6 — Clean completion clears the checkpoint so the next
+      // app launch doesn't offer to "resume" a crawl that already
+      // finished successfully.
+      try {
+        this.db.clearQueueCheckpoint();
+      } catch {
+        /* checkpoint table may not yet exist on very old DBs — ignore */
+      }
+      this.emit('done', this.db.getSummary());
+    }
       return;
     }
     this.emit(
@@ -304,6 +436,7 @@ export class Crawler extends EventEmitter {
     // large crawls and we don't want to hold the DB write lock more
     // than the crawler itself does.
     this.startIssueRecomputeTimer();
+    this.startQueueCheckpointTimer();
 
     // Hydrate in-memory state from the DB so resume starts from the right
     // point; then queue whatever work is still pending.
@@ -319,10 +452,49 @@ export class Crawler extends EventEmitter {
       // on `sitemap_urls` (Non-Indexable in Sitemap, Non-200 in Sitemap)
       // see the full set, and so the robots checker is settled.
       await Promise.all([robotsPromise, sitemapPromise]);
+      // Drain any internal-link "stubs": URLs discovered on a crawled
+      // page but never themselves crawled — typically because they
+      // were first found via a depth-N+1 path (rejected by maxDepth)
+      // before a shallower path could enqueue them, or because robots
+      // / scope filters had not loaded yet when the link was processed.
+      // Without this loop the user would see "second Start finds +N
+      // new URLs" even though the first crawl was supposed to be
+      // complete; running until `getPendingInternalLinks()` stops
+      // shrinking is what hydrateFromDb already does on resume —
+      // doing it here moves the work into the first crawl so a
+      // single Start finishes with everything reachable.
+      let lastPending = -1;
+      // Hard cap on iterations — with pathological filter sets the
+      // pending list could plateau without dropping to 0; the
+      // `lastPending === count` check stops us in that case, but a
+      // numeric ceiling guards against any bug that lets the count
+      // oscillate.
+      const excludeNofollow = !this.config.followNofollow;
+      this.setOp('post-crawl:drain-pending-stubs');
+      for (let pass = 0; pass < 20 && !this.stopped; pass++) {
+        // Honour `followNofollow` here too — without the filter the
+        // drain would happily crawl URLs that the live link-follow
+        // path explicitly skipped, ending up with a different result
+        // than a "no drain" first crawl would produce.
+        const pending = this.db.getPendingInternalLinks({ excludeNofollow });
+        if (pending.length === 0) break;
+        if (pending.length === lastPending) break;
+        lastPending = pending.length;
+        for (const p of pending) {
+          // Drop from `seen` first — these URLs were never actually
+          // crawled, just stubbed; enqueue's seen-check would reject
+          // them otherwise.
+          this.seen.delete(p.url);
+          this.enqueue({ url: p.url, depth: p.depth });
+        }
+        await this.queue.onIdle();
+        await this.externalQueue.onIdle();
+      }
     } finally {
       if (this.progressTimer) clearInterval(this.progressTimer);
       this.progressTimer = null;
       this.stopIssueRecomputeTimer();
+      this.stopQueueCheckpointTimer();
     }
 
     // Post-crawl heavy lifting. Each step is a synchronous SQL pass
@@ -332,24 +504,58 @@ export class Crawler extends EventEmitter {
     // (especially Logs) freezes visibly. Yielding to the event loop
     // between steps lets queued IPC mesajları (logs:batch, progress,
     // dataChanged) get serviced.
+    // Wave 6 — Per-pass crawl-analysis toggles. Each pass is gated
+    // by its config flag so users running tight time-budget audits
+    // can skip steps they don't need (e.g. duplicate clustering can
+    // be 5–10 s on a 100k-URL crawl).
+    if (this.config.analyseInlinks) {
       await yieldToEventLoop();
       this.emit('info', 'Recomputing inlinks…');
+      this.setOp('post-crawl:recompute-inlinks');
       this.db.recomputeInlinks();
+    }
+    if (this.config.analyseRedirectChains) {
       await yieldToEventLoop();
       this.emit('info', 'Recomputing redirect chains…');
+      this.setOp('post-crawl:recompute-redirect-chains');
       this.db.recomputeRedirectChains();
+    }
+    if (this.config.analyseHreflang) {
       await yieldToEventLoop();
       this.emit('info', 'Recomputing hreflang analysis…');
+      this.setOp('post-crawl:recompute-hreflang');
       this.db.recomputeHreflangAnalysis();
+      await yieldToEventLoop();
+      this.setOp('post-crawl:recompute-hreflang-inconsistent');
+      this.db.recomputeHreflangInconsistent();
+    }
+    if (this.config.analyseDuplicates) {
       await yieldToEventLoop();
       this.emit('info', 'Clustering duplicates…');
+      this.setOp('post-crawl:cluster-duplicates');
       this.runDuplicateClustering();
+    }
+    if (this.config.analysePagination) {
+      await yieldToEventLoop();
+      this.emit('info', 'Detecting pagination sequence gaps…');
+      this.setOp('post-crawl:pagination-sequence');
+      this.db.recomputePaginationSequence();
+    }
+    if (this.config.analyseIssues) {
       await yieldToEventLoop();
       this.emit('info', 'Materialising issue counters…');
-    this.db.recomputeUrlsIssues(EXPENSIVE_ISSUE_DEFINITIONS);
+      this.setOp('post-crawl:materialise-issues');
+      // Use the cooperatively-scheduled variant (yields between each
+      // of the 70+ definitions). Trades atomic visibility of the
+      // rebuild for a responsive main thread — on a 5k-URL DB the
+      // sync version was blocking the event loop for 22-26 s.
+      await this.db.recomputeUrlsIssuesYielding(EXPENSIVE_ISSUE_DEFINITIONS);
+    }
     await yieldToEventLoop();
+    this.setOp('post-crawl:image-probes');
     await this.runImageSizeProbes();
     await yieldToEventLoop();
+    this.setOp('post-crawl:tls-probes');
     await this.runTlsCertProbes();
     this.running = false;
     this.stopMemoryMonitor();
@@ -358,6 +564,7 @@ export class Crawler extends EventEmitter {
     this.seen.clear();
     this.externalSeen.clear();
     this.emitProgress();
+    this.setOp('idle');
     // Suppress 'done' if a stop() ran during teardown — otherwise the
     // zombie crawler's done-event clobbers the new crawl's UI state.
     if (!this.stopped) {
@@ -476,7 +683,7 @@ export class Crawler extends EventEmitter {
           const res = await undiciFetch(entry.src, {
             method: 'HEAD',
             headers: defaultRequestHeaders(
-              userAgent,
+              this.resolveUserAgent(entry.src),
               this.config.acceptLanguage,
               this.config.customHeaders,
               this.config.auth,
@@ -643,7 +850,17 @@ export class Crawler extends EventEmitter {
       this.emit('error', 'List mode: urlList is empty (or no entries normalised to valid URLs).');
       this.running = false;
       this.emitProgress();
-      if (!this.stopped) this.emit('done', this.db.getSummary());
+      if (!this.stopped) {
+      // Wave 6 — Clean completion clears the checkpoint so the next
+      // app launch doesn't offer to "resume" a crawl that already
+      // finished successfully.
+      try {
+        this.db.clearQueueCheckpoint();
+      } catch {
+        /* checkpoint table may not yet exist on very old DBs — ignore */
+      }
+      this.emit('done', this.db.getSummary());
+    }
       return;
     }
 
@@ -671,6 +888,7 @@ export class Crawler extends EventEmitter {
 
     this.progressTimer = setInterval(() => this.emitProgress(), 500);
     this.startIssueRecomputeTimer();
+    this.startQueueCheckpointTimer();
 
     for (const u of urls) {
       this.enqueue({ url: u, depth: 0 });
@@ -683,21 +901,40 @@ export class Crawler extends EventEmitter {
       if (this.progressTimer) clearInterval(this.progressTimer);
       this.progressTimer = null;
       this.stopIssueRecomputeTimer();
+      this.stopQueueCheckpointTimer();
     }
 
     // Same yielding strategy as spider mode — see the comment block
     // above the spider-mode recompute. SQL aggregates blocking the JS
     // thread starve IPC dispatch and freeze every other window.
+    // Toggles match the spider-mode gates so a single config controls
+    // both modes' post-crawl pipeline.
+    if (this.config.analyseInlinks) {
       await yieldToEventLoop();
       this.db.recomputeInlinks();
+    }
+    if (this.config.analyseRedirectChains) {
       await yieldToEventLoop();
       this.db.recomputeRedirectChains();
+    }
+    if (this.config.analyseHreflang) {
       await yieldToEventLoop();
       this.db.recomputeHreflangAnalysis();
+      await yieldToEventLoop();
+      this.db.recomputeHreflangInconsistent();
+    }
+    if (this.config.analyseDuplicates) {
       await yieldToEventLoop();
       this.runDuplicateClustering();
+    }
+    if (this.config.analysePagination) {
+      await yieldToEventLoop();
+      this.db.recomputePaginationSequence();
+    }
+    if (this.config.analyseIssues) {
       await yieldToEventLoop();
       this.db.recomputeUrlsIssues(EXPENSIVE_ISSUE_DEFINITIONS);
+    }
     await yieldToEventLoop();
     await this.runImageSizeProbes();
     await yieldToEventLoop();
@@ -707,7 +944,17 @@ export class Crawler extends EventEmitter {
     this.seen.clear();
     this.externalSeen.clear();
     this.emitProgress();
-    if (!this.stopped) this.emit('done', this.db.getSummary());
+    if (!this.stopped) {
+      // Wave 6 — Clean completion clears the checkpoint so the next
+      // app launch doesn't offer to "resume" a crawl that already
+      // finished successfully.
+      try {
+        this.db.clearQueueCheckpoint();
+      } catch {
+        /* checkpoint table may not yet exist on very old DBs — ignore */
+      }
+      this.emit('done', this.db.getSummary());
+    }
   }
 
   /**
@@ -779,8 +1026,13 @@ export class Crawler extends EventEmitter {
     }
 
     // Re-queue any internal link targets that were discovered before the
-    // previous Stop but never actually fetched.
-    for (const pending of this.db.getPendingInternalLinks()) {
+    // previous Stop but never actually fetched. Honour `followNofollow`
+    // — without this filter, hitting Start a second time would crawl
+    // every nofollow link target that the previous crawl had correctly
+    // skipped, and the user would see "extra" URLs appear that the live
+    // link-follow path would never have touched.
+    const excludeNofollow = !this.config.followNofollow;
+    for (const pending of this.db.getPendingInternalLinks({ excludeNofollow })) {
       // Drop from `seen` so enqueue accepts it — these URLs are genuinely
       // unfinished work.
       this.seen.delete(pending.url);
@@ -817,7 +1069,7 @@ export class Crawler extends EventEmitter {
       undiciFetch(url, {
         method,
         headers: defaultRequestHeaders(
-          this.config.userAgent,
+          this.resolveUserAgent(url),
           this.config.acceptLanguage,
           this.config.customHeaders,
           this.config.auth,
@@ -868,7 +1120,16 @@ export class Crawler extends EventEmitter {
     this.stopped = true;
     this.running = false;
     this.paused = false;
+    this.setOp('idle');
     this.stopMemoryMonitor();
+    // Stop the periodic checkpoint timer immediately. Without this,
+    // the 30-second setInterval keeps firing until start()'s `finally`
+    // block tears it down — which on a stop-during-post-crawl can run
+    // after `db.reset()` (Clear button), repopulating crawl_queue from
+    // a stale `pendingItems` snapshot and resurrecting the recovery
+    // prompt on the next launch.
+    this.stopQueueCheckpointTimer();
+    this.stopIssueRecomputeTimer();
     // Cancel any in-flight sitemap discovery so its 'info' / 'done'
     // events don't leak into the next crawl.
     if (this.sitemapAbort) {
@@ -1100,6 +1361,19 @@ export class Crawler extends EventEmitter {
    *
    * Returns whether the URL was actually accepted into the queue.
    */
+  /** Wave 6 — Re-enqueue a checkpointed URL at its original depth.
+   * Called by the crash-recovery flow after restoring the saved
+   * pending list from `crawl_queue`. Bypasses the seen-set so the
+   * URL is fetched again even though a partial crawl may already
+   * have a row for it (the upsert handles dedup at the DB level).
+   */
+  enqueueCheckpointed(items: ReadonlyArray<{ url: string; depth: number }>): void {
+    for (const item of items) {
+      this.seen.delete(item.url);
+      this.enqueue({ url: item.url, depth: item.depth });
+    }
+  }
+
   enqueueManual(rawUrl: string): boolean {
     if (this.stopped || !this.running) return false;
     let url: string;
@@ -1138,6 +1412,7 @@ export class Crawler extends EventEmitter {
 
     this.seen.add(item.url);
     this.pending++;
+    this.pendingItems.set(item.url, item.depth);
     this.queue
       .add(() => this.fetchAndProcess(item))
       .catch((err: unknown) => {
@@ -1148,11 +1423,16 @@ export class Crawler extends EventEmitter {
       })
       .finally(() => {
         this.pending = Math.max(0, this.pending - 1);
+        // Drop from the checkpoint set whether we succeeded or failed —
+        // failures are recorded in the urls table and shouldn't be
+        // retried by a resumed crawl.
+        this.pendingItems.delete(item.url);
       });
   }
 
   private async fetchAndProcess(item: QueueItem): Promise<void> {
     if (this.stopped) return;
+    this.setOp(`crawl:fetch:${item.url}`);
 
     // I-1 — Cooperative yield BEFORE we start work on this URL. The
     // crawler runs in the same Node event loop as Electron's IPC
@@ -1167,6 +1447,18 @@ export class Crawler extends EventEmitter {
     const t0 = Date.now();
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
+    // Wave 3 — Optional max-response-time hard cap. Distinct from
+    // `requestTimeoutMs` (which is the connect+headers timeout): this
+    // is an upper bound on the *total* request lifetime including body
+    // download. Useful for capping individual slow pages without
+    // lowering the overall fetch timeout.
+    const respTimeTimer =
+      this.config.maxResponseTimeMs > 0
+        ? setTimeout(
+            () => controller.abort(),
+            this.config.maxResponseTimeMs,
+          )
+        : null;
 
     try {
       // Manual redirect handling — each hop becomes its own row so the
@@ -1177,6 +1469,40 @@ export class Crawler extends EventEmitter {
 
       const responseTimeMs = Date.now() - t0;
       this.totalResponseTimeMs += responseTimeMs;
+
+      // Wave 3 — Optional max file size filter. When the response's
+      // declared `Content-Length` exceeds the configured ceiling we
+      // discard the body and record a size-cap notice. The page row
+      // is still created so links to it aren't lost; only the body
+      // and downstream parsing are skipped.
+      if (this.config.maxFileSizeBytes > 0) {
+        const lenHeader = res.headers.get('content-length');
+        const declaredLen = lenHeader ? Number.parseInt(lenHeader, 10) : NaN;
+        if (Number.isFinite(declaredLen) && declaredLen > this.config.maxFileSizeBytes) {
+          try {
+            await res.body?.cancel();
+          } catch {
+            /* ignore */
+          }
+          this.failed++;
+          this.emit(
+            'warn',
+            `Skipped ${item.url}: Content-Length ${declaredLen} > maxFileSizeBytes ${this.config.maxFileSizeBytes}`,
+          );
+          this.db.upsertUrl({
+            url: item.url,
+            contentKind: 'other',
+            statusCode: res.status,
+            statusText: 'size-cap-exceeded',
+            indexability: 'non-indexable:client-error',
+            indexabilityReason: `Body skipped — Content-Length ${declaredLen} exceeds maxFileSizeBytes`,
+            responseTimeMs,
+            depth: item.depth,
+            contentLength: declaredLen,
+          });
+          return;
+        }
+      }
       this.responseSamples++;
 
       const statusCode = res.status;
@@ -1258,7 +1584,42 @@ export class Crawler extends EventEmitter {
       // Cookie security analysis — Set-Cookie response headers, parsed
       // into per-cookie rows so we can count missing Secure / HttpOnly /
       // SameSite flags. Cookie values themselves are never stored.
-      const cookieSummary = analyseCookies(extractSetCookies(allHeaders));
+      // Wave 9 — Honour the cookie policy: `reject-all` skips analysis
+      // entirely (zeros for the missing-flag counters); `block-third-
+      // party` still records flag counts but only for cookies whose
+      // Domain attribute matches the page's registrable domain (or is
+      // absent — implicit first-party). `accept-all` keeps the legacy
+      // behaviour of analysing every Set-Cookie regardless of scope.
+      const cookieSummary = (() => {
+        if (this.config.cookiePolicy === 'reject-all') {
+          return {
+            count: 0,
+            insecureCount: 0,
+            noHttpOnlyCount: 0,
+            noSameSiteCount: 0,
+          };
+        }
+        const setCookies = extractSetCookies(allHeaders);
+        if (this.config.cookiePolicy === 'block-third-party') {
+          // First-party = same registrable domain (last two labels).
+          let pageRoot = '';
+          try {
+            pageRoot = new URL(item.url).hostname.split('.').slice(-2).join('.').toLowerCase();
+          } catch {
+            // fall through, no filter
+          }
+          if (pageRoot) {
+            const filtered = setCookies.filter((sc) => {
+              const m = /;\s*Domain=([^;]+)/i.exec(sc);
+              if (!m) return true; // no Domain attribute = implicit first-party
+              const domain = m[1]!.trim().toLowerCase().replace(/^\./, '');
+              return domain.endsWith(pageRoot);
+            });
+            return analyseCookies(filtered);
+          }
+        }
+        return analyseCookies(setCookies);
+      })();
 
       // TTFB on the successful attempt (excludes retry overhead). Falls
       // back to total response time if for any reason ttfbMs wasn't set
@@ -1383,7 +1744,12 @@ export class Crawler extends EventEmitter {
 
       const body = await res.text();
       const bodyLength = parseIntSafe(contentLengthHeader) ?? Buffer.byteLength(body, 'utf8');
-      const parsed = parseHtml(body, item.url, {
+      // Hand parsing to the worker pool when injected by the desktop
+      // host; the CLI's default is the inline `parseHtml`. The
+      // crawler doesn't care which one runs as long as the result
+      // shape matches.
+      this.setOp(`crawl:parse:${item.url}`);
+      const parsed = await this.parsePage(body, item.url, {
         includeSubdomains: this.config.scope === 'all-subdomains',
         cdnHosts: this.config.cdnHosts,
         customSearchTerms: this.config.customSearchTerms,
@@ -1434,7 +1800,17 @@ export class Crawler extends EventEmitter {
         : parsed.links.filter((l) => !l.rel?.includes('nofollow'));
 
       const imagesMissingAlt = parsed.images.filter((img) => img.alt === null).length;
-      const urlId = this.db.upsertUrl({
+      // Phase 1b — Build the entire per-URL write payload up front and
+      // ship it across the writer-worker boundary in one shot. The
+      // worker runs the upsert + headers + body snapshot + links +
+      // images inside a single SQLite transaction; the main thread
+      // never blocks on `.run()` for the duration of those writes.
+      // Falls back to an inline transaction in the no-worker (CLI)
+      // case via the default `writeFetchedUrl` injected in the
+      // constructor.
+      this.setOp(`crawl:write:${item.url}`);
+      const { urlId } = await this.writeFetchedUrl({
+        upsert: {
           url: item.url,
           contentKind: 'html',
           statusCode,
@@ -1538,35 +1914,20 @@ export class Crawler extends EventEmitter {
           serverHeader,
           jsOnlyLinksCount: parsed.jsOnlyLinksCount,
           textCodeRatio: parsed.textCodeRatio,
-      });
-      // Coalesce the per-URL write fan-out (headers + body snapshot +
-      // 50–500 links + 5–50 images) into a single SQLite transaction.
-      // Each `BEGIN…COMMIT` triggers an fsync on `synchronous=NORMAL`,
-      // and on low-end SSDs with antivirus realtime-scan each fsync can
-      // cost 30–80 ms. Without this wrapper a single page produces
-      // 5–10 fsyncs; with it, exactly one — the difference between a
-      // smooth UI and the kasma the user reported.
-      this.db.runInTransaction(() => {
-        if (urlId) this.db.setUrlHeaders(urlId, allHeaders);
-        // Body snapshot — drives the View Source detail tab. Only HTML
-        // pages get stored (we already gated entry above on
-        // `kind === 'html'`). Capped per `bodySnapshotMaxBytes` so a
-        // single page can't blow up the project file.
-        if (urlId && this.config.storeBodySnapshots) {
-          try {
-            this.db.setUrlSource(
-              urlId,
+        },
+        headers: allHeaders,
+        storeBody: this.config.storeBodySnapshots
+          ? {
               body,
+              maxBytes:
                 this.config.bodySnapshotMaxBytes > 0
                   ? this.config.bodySnapshotMaxBytes
                   : 1_048_576,
-            );
-          } catch {
-            // Best-effort — a snapshot failure must not break the crawl.
             }
-        }
-        this.db.insertLinks(urlId, storableLinks, item.depth);
-        this.db.insertImages(urlId, parsed.images);
+          : null,
+        links: storableLinks,
+        images: parsed.images,
+        fromDepth: item.depth,
       });
       for (const link of storableLinks) {
         if (!link.isInternal) this.enqueueExternal(link.toUrl);
@@ -1578,18 +1939,63 @@ export class Crawler extends EventEmitter {
       }
 
       if (this.config.scope === 'exact-url') {
-        // exact-url mode: do not follow any links
+        // exact-url / single-page mode: do not follow any links.
       } else {
         const nextDepth = item.depth + 1;
         for (const link of storableLinks) {
           const inScope = isInScope(this.config.startUrl, link.toUrl, this.config.scope);
           if (!inScope && !this.config.crawlExternal) continue;
-          // Belt-and-braces: when storeNofollowLinks=true we still respect
-          // nofollow for the *follow* decision — store the hint, don't
-          // recurse into it.
-          if (link.rel?.includes('nofollow')) continue;
+          // Wave 3 — nofollow follow toggle. By default nofollow links
+          // are stored (when `storeNofollowLinks` is on) but never
+          // recursed into. `followNofollow=true` opts out of the
+          // "respect nofollow" behaviour and treats them like any
+          // other link for the follow decision.
+          if (link.rel?.includes('nofollow') && !this.config.followNofollow) continue;
           this.enqueue({ url: link.toUrl, depth: nextDepth });
         }
+        // Wave 3 — Pagination follow toggle. rel=next/prev are part of
+        // the standard discovery graph; the toggle exists to debug
+        // pagination-only loops without disabling all link follow.
+        if (this.config.followPaginationLinks) {
+          for (const target of [parsed.paginationNext, parsed.paginationPrev]) {
+            if (!target) continue;
+            const inScope = isInScope(this.config.startUrl, target, this.config.scope);
+            if (!inScope && !this.config.crawlExternal) continue;
+            this.enqueue({ url: target, depth: nextDepth });
+          }
+        }
+        // Wave 3 — Canonical follow toggle. When on, a 200 page that
+        // declares a canonical pointing to a different URL also
+        // enqueues that target. Default off — most crawls treat
+        // canonicals as a signal, not a navigation hint.
+        if (
+          this.config.followCanonicals &&
+          parsed.canonical &&
+          parsed.canonical !== item.url
+        ) {
+          const inScope = isInScope(
+            this.config.startUrl,
+            parsed.canonical,
+            this.config.scope,
+          );
+          if (inScope || this.config.crawlExternal) {
+            this.enqueue({ url: parsed.canonical, depth: nextDepth });
+          }
+        }
+        // Wave 3 — JS-style redirect follow. Currently covers
+        // `<meta http-equiv="refresh">` content URLs; window.location
+        // bodies aren't statically followable without a JS engine and
+        // are out of scope.
+        if (this.config.followJsRedirects && parsed.metaRefreshUrl) {
+          const inScope = isInScope(
+            this.config.startUrl,
+            parsed.metaRefreshUrl,
+            this.config.scope,
+          );
+          if (inScope || this.config.crawlExternal) {
+            this.enqueue({ url: parsed.metaRefreshUrl, depth: nextDepth });
+          }
+        }
       }
     } catch (err) {
       this.failed++;
@@ -1615,6 +2021,7 @@ export class Crawler extends EventEmitter {
       });
     } finally {
       clearTimeout(timeout);
+      if (respTimeTimer) clearTimeout(respTimeTimer);
       // Politeness delay — applied per worker *after* each request so a
       // higher concurrency still honours a "one request every N ms per slot"
       // contract on top of the global RPS cap.
@@ -1629,6 +2036,40 @@ export class Crawler extends EventEmitter {
    * Retries are triggered by network errors, HTTP 429, and 5xx responses —
    * 3xx/4xx (except 429) are treated as final.
    */
+  /**
+   * Wave 9 — Resolve the User-Agent for a given URL. Walks the
+   * `perHostUserAgents` rule list in order, returning the first
+   * pattern whose host matches; falls back to the global
+   * `config.userAgent`. Pattern syntax:
+   *   - exact host         `m.example.com`
+   *   - leading wildcard   `*.example.com` matches any subdomain
+   *                        (does NOT match the apex `example.com`)
+   * Match is case-insensitive on the URL host.
+   */
+  private resolveUserAgent(url: string): string {
+    const rules = this.config.perHostUserAgents ?? [];
+    if (rules.length === 0) return this.config.userAgent;
+    let host = '';
+    try {
+      host = new URL(url).hostname.toLowerCase();
+    } catch {
+      return this.config.userAgent;
+    }
+    for (const rule of rules) {
+      const pat = rule.hostPattern.trim().toLowerCase();
+      if (!pat) continue;
+      if (pat.startsWith('*.')) {
+        const suffix = pat.slice(1); // ".example.com"
+        if (host.endsWith(suffix) && host.length > suffix.length) {
+          return rule.userAgent;
+        }
+      } else if (host === pat) {
+        return rule.userAgent;
+      }
+    }
+    return this.config.userAgent;
+  }
+
   private async fetchWithRetry(
     url: string,
     signal: AbortSignal,
@@ -1649,7 +2090,7 @@ export class Crawler extends EventEmitter {
         const res = await undiciFetch(url, {
           method: 'GET',
           headers: defaultRequestHeaders(
-            this.config.userAgent,
+            this.resolveUserAgent(url),
             this.config.acceptLanguage,
             this.config.customHeaders,
             this.config.auth,

packages/core/src/index.ts

@@ -9,6 +9,7 @@ export {
 } from './html-parser.js';
 export { exportUrlsToCsv } from './csv-export.js';
 export { exportUrlsToJson, type JsonExportOptions } from './json-export.js';
+export { exportUrlsToXml } from './xml-export.js';
 export { testUrlAgainstRobots, type RobotsTestResult } from './robots.js';
 export {
   exportSitemap,

packages/core/src/xml-export.ts

@@ -0,0 +1,209 @@
+import { createWriteStream } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+import type { ProjectDb } from '@freecrawl/db';
+import type { CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
+
+/**
+ * Streaming XML export of the URL table. Schema is intentionally close
+ * to Screaming Frog's CSV-→-XML pattern so existing pipelines can drop
+ * the file in without bespoke transforms:
+ *
+ *   <crawl exportedAt="…" rowCount="…">
+ *     <url>
+ *       <address>https://…</address>
+ *       <statusCode>200</statusCode>
+ *       <indexability>indexable</indexability>
+ *       <title>…</title>
+ *       <titleLength>…</titleLength>
+ *       …
+ *     </url>
+ *     …
+ *   </crawl>
+ *
+ * Streamed via async generator + `pipeline` so a 1M-row crawl stays
+ * within bounded memory — never materialises the full output.
+ *
+ * Element names mirror the camelCase keys of `CrawlUrlRow` so the
+ * mapping is obvious. Null values render as self-closing tags (e.g.
+ * `<title/>`) rather than empty strings, preserving the distinction
+ * between "no title" and "empty title" that the CSV export collapses.
+ */
+
+const XML_COLUMNS: (keyof CrawlUrlRow)[] = [
+  'url',
+  'statusCode',
+  'contentKind',
+  'indexability',
+  'indexabilityReason',
+  'title',
+  'titleLength',
+  'metaDescription',
+  'metaDescriptionLength',
+  'h1',
+  'h1Length',
+  'h1Count',
+  'h2Count',
+  'h3Count',
+  'h4Count',
+  'h5Count',
+  'h6Count',
+  'wordCount',
+  'canonical',
+  'canonicalCount',
+  'canonicalHttp',
+  'metaRobots',
+  'xRobotsTag',
+  'contentType',
+  'contentLength',
+  'responseTimeMs',
+  'ttfbMs',
+  'depth',
+  'inlinks',
+  'outlinks',
+  'redirectTarget',
+  'redirectChainLength',
+  'redirectFinalUrl',
+  'redirectLoop',
+  'lang',
+  'viewport',
+  'ogTitle',
+  'ogDescription',
+  'ogImage',
+  'twitterCard',
+  'twitterTitle',
+  'twitterDescription',
+  'twitterImage',
+  'metaKeywords',
+  'metaAuthor',
+  'metaGenerator',
+  'themeColor',
+  'hsts',
+  'xFrameOptions',
+  'xContentTypeOptions',
+  'csp',
+  'referrerPolicy',
+  'permissionsPolicy',
+  'contentEncoding',
+  'serverHeader',
+  'httpProtocol',
+  'keepAlive',
+  'paginationNext',
+  'paginationPrev',
+  'paginationSequenceBreak',
+  'amphtml',
+  'favicon',
+  'appleTouchIcon',
+  'manifestUrl',
+  'feedUrl',
+  'metaRefresh',
+  'metaRefreshUrl',
+  'charset',
+  'mixedContentCount',
+  'imagesCount',
+  'imagesMissingAlt',
+  'imagesEmptyAlt',
+  'imagesLazy',
+  'jsOnlyLinksCount',
+  'textCodeRatio',
+  'titleCount',
+  'titlePixelWidth',
+  'metaPixelWidth',
+  'cookiesCount',
+  'cookiesInsecure',
+  'cookiesNoHttpOnly',
+  'cookiesNoSameSite',
+  'queryParamCount',
+  'queryStringLength',
+  'folderDepth',
+  'renderBlockingCount',
+  'formInputCount',
+  'formInputUnlabeled',
+  'schemaTypes',
+  'schemaBlockCount',
+  'schemaInvalidCount',
+  'microdataCount',
+  'rdfaCount',
+  'insecureFormActionCount',
+  'missingSriCount',
+  'hreflangs',
+  'hreflangCount',
+  'analyticsTrackers',
+  'simhash',
+  'contentHash',
+  'clusterId',
+  'clusterSize',
+];
+
+/** Escape characters that are invalid inside XML element text bodies. */
+function xmlEscape(raw: string): string {
+  return raw
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+    // XML 1.0 forbids most C0 control characters; strip everything below
+    // 0x20 except tab (0x09), LF (0x0A), and CR (0x0D).
+    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '');
+}
+
+function renderField(key: keyof CrawlUrlRow, value: unknown): string {
+  if (value === null || value === undefined || value === '') {
+    return `<${key}/>`;
+  }
+  if (typeof value === 'boolean') {
+    return `<${key}>${value ? 'true' : 'false'}</${key}>`;
+  }
+  if (typeof value === 'number') {
+    return `<${key}>${value}</${key}>`;
+  }
+  // Strings, JSON-stringified columns (hreflangs, schemaTypes, etc.)
+  // pass through xml-escape and into a CDATA wrapper when they contain
+  // characters that would force escaping of every byte.
+  const s = String(value);
+  if (/[<>&]/.test(s) && s.length > 200) {
+    // Long values likely contain HTML — CDATA preserves exact bytes.
+    // We split on `]]>` to be safe; standard SAX/DOM parsers reassemble.
+    const safe = s.replace(/\]\]>/g, ']]]]><![CDATA[>');
+    return `<${key}><![CDATA[${safe}]]></${key}>`;
+  }
+  return `<${key}>${xmlEscape(s)}</${key}>`;
+}
+
+export async function exportUrlsToXml(
+  db: ProjectDb,
+  filePath: string,
+  options: { selectedIds?: number[]; category?: UrlCategory } = {},
+): Promise<{ rowsWritten: number }> {
+  let rowsWritten = 0;
+
+  const source: Iterable<CrawlUrlRow> =
+    options.selectedIds && options.selectedIds.length > 0
+      ? db.iterateUrlsByIds(options.selectedIds)
+      : options.category && options.category !== 'all'
+        ? db.iterateUrlsByCategory(options.category)
+        : db.iterateAllUrls();
+
+  const generator = async function* (): AsyncGenerator<string> {
+    yield '<?xml version="1.0" encoding="UTF-8"?>\n';
+    // We don't know the row count up front (we're streaming) so the
+    // root element opens without it — matches the SF CSV-XML output.
+    yield `<crawl exportedAt="${new Date().toISOString()}">\n`;
+    for (const row of source) {
+      const fields = XML_COLUMNS.map((col) =>
+        renderField(col, (row as unknown as Record<string, unknown>)[col]),
+      ).join('');
+      yield `  <url>${fields}</url>\n`;
+      rowsWritten++;
+    }
+    yield '</crawl>\n';
+  };
+
+  await pipeline(
+    Readable.from(generator()),
+    createWriteStream(filePath, { encoding: 'utf8' }),
+  );
+
+  return { rowsWritten };
+}

packages/db/src/index.ts

@@ -1,2 +1,3 @@
 export { ProjectDb, EXPENSIVE_ISSUE_DEFINITIONS } from './project-db.js';
+export type { UpsertUrlInput } from './project-db.js';
 export { runMigrations } from './migrations.js';

packages/db/src/migrations.ts

@@ -885,6 +885,64 @@ const MIGRATIONS: Migration[] = [
       CREATE INDEX IF NOT EXISTS idx_urls_issues_key ON urls_issues(issue_key);
     `,
   },
+  {
+    version: 43,
+    name: 'add_pagination_sequence_break',
+    // Wave 2.5 — Per-page boolean flag set by `recomputePaginationSequence()`
+    // when this URL is part of a paginated cluster (its `pagination_next`
+    // or `pagination_prev` resolved to another crawled URL with the same
+    // template) AND the cluster's numeric ordinals have a gap (e.g.
+    // ?page=1 → ?page=2 → ?page=4 misses 3). Powers the
+    // "Pagination Sequence Break" issue filter.
+    up: (db) => {
+      const cols = db.prepare('PRAGMA table_info(urls)').all() as unknown as {
+        name: string;
+      }[];
+      if (!cols.some((c) => c.name === 'pagination_sequence_break')) {
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN pagination_sequence_break INTEGER NOT NULL DEFAULT 0',
+        );
+      }
+    },
+  },
+  {
+    version: 44,
+    name: 'add_crawl_queue_checkpoint',
+    // Wave 6 — Periodic checkpoint of pending queue items so an
+    // unexpected exit (process crash, OS reboot, OOM kill) can resume
+    // the crawl on next launch. Three columns:
+    //   - `url`      : the URL that was waiting to be fetched
+    //   - `depth`    : its enqueue depth so the resumed crawl respects
+    //                  `maxDepth` correctly
+    //   - `seed_url` : discriminates stale checkpoints when the user
+    //                  has changed start URL between crashes; the
+    //                  resume prompt only fires if seeds match.
+    up: `
+      CREATE TABLE IF NOT EXISTS crawl_queue (
+        url       TEXT PRIMARY KEY,
+        depth     INTEGER NOT NULL DEFAULT 0,
+        seed_url  TEXT NOT NULL DEFAULT ''
+      );
+    `,
+  },
+  {
+    version: 45,
+    name: 'add_hreflang_inconsistent_lang',
+    // Wave 6 — Boolean per-URL flag set by `recomputeHreflangInconsistent()`
+    // when the page's hreflang JSON contains the same `lang` value with
+    // two different target URLs. Powers the
+    // "Hreflang Inconsistent Lang" issue filter.
+    up: (db) => {
+      const cols = db.prepare('PRAGMA table_info(urls)').all() as unknown as {
+        name: string;
+      }[];
+      if (!cols.some((c) => c.name === 'hreflang_inconsistent_lang')) {
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN hreflang_inconsistent_lang INTEGER NOT NULL DEFAULT 0',
+        );
+      }
+    },
+  },
 ];
 
 export function runMigrations(db: DatabaseSync): void {

packages/db/src/project-db.ts

@@ -464,6 +464,24 @@ export class ProjectDb {
       this.db.exec('PRAGMA synchronous = NORMAL');
       this.db.exec('PRAGMA foreign_keys = ON');
     }
+    // Two writer connections coexist on the same DB file in the desktop
+    // app: the main-process ProjectDb (used for sitemap ingest, post-
+    // crawl recomputes, the user-initiated mutations like deleteUrl)
+    // AND the db-writer worker thread (used for the per-URL hot path).
+    // SQLite serialises writes at the file level, so without
+    // `busy_timeout` whichever connection is "second" through the door
+    // gets SQLITE_BUSY thrown the instant the other one is mid-
+    // transaction — we saw this when the user clicked Start twice in
+    // quick succession ("Sitemap discovery skipped: database is
+    // locked"). 10 s is well over the worst-case writer hold time
+    // (post-crawl materialise-issues yielding transactions are <500 ms
+    // each) and adds zero overhead when there is no contention.
+    if (!readOnly) {
+      this.db.exec('PRAGMA busy_timeout = 10000');
+    } else {
+      // Even read-only connections benefit during a checkpoint storm.
+      this.db.exec('PRAGMA busy_timeout = 5000');
+    }
     this.db.exec('PRAGMA temp_store = MEMORY');
     this.db.exec(`PRAGMA cache_size = ${readOnly ? -32768 : -131072}`);
     // 30GB virtual address window for mmap-backed reads; OS only pages in
@@ -515,10 +533,123 @@ export class ProjectDb {
        DELETE FROM url_sources;
        DELETE FROM sitemap_urls;
        DELETE FROM urls;
-       DELETE FROM project_meta;`,
+       DELETE FROM project_meta;
+       DELETE FROM urls_issues;
+       DELETE FROM crawl_queue;`,
     );
   }
 
+  /**
+   * GDPR-aligned per-domain wipe. Removes every URL row whose host
+   * (case-insensitive) matches `domain`, plus every dependent record
+   * keyed off those URLs (`links`, `headers`, `url_sources`,
+   * `urls_issues`, `image_usages` rows pointing at images that are now
+   * orphaned). Used by Settings → "Delete Domain Data" so a user can
+   * comply with a data-removal request without nuking the whole crawl.
+   *
+   * Domain match is exact-host (`example.com` matches `example.com` but
+   * not `sub.example.com`). For "all subdomains too" the caller can
+   * issue multiple deletes; we don't infer wildcard semantics here to
+   * avoid surprising over-broad wipes.
+   *
+   * Wrapped in `runInTransaction` so the wipe is atomic: either every
+   * dependent row goes away or nothing does.
+   */
+  deleteByDomain(domain: string): { urlsDeleted: number; linksDeleted: number } {
+    const target = domain.trim().toLowerCase();
+    if (!target) return { urlsDeleted: 0, linksDeleted: 0 };
+    return this.runInTransaction(() => {
+      // Snapshot the URL ids we'll delete so we can fan out into the
+      // dependent tables before the parent rows vanish.
+      const ids = (
+        this.db
+          .prepare(
+            `SELECT id FROM urls
+              WHERE LOWER(
+                SUBSTR(
+                  url,
+                  INSTR(url, '://') + 3,
+                  CASE
+                    WHEN INSTR(SUBSTR(url, INSTR(url, '://') + 3), '/') > 0
+                      THEN INSTR(SUBSTR(url, INSTR(url, '://') + 3), '/') - 1
+                    ELSE LENGTH(url)
+                  END
+                )
+              ) = ?`,
+          )
+          .all(target) as { id: number }[]
+      ).map((r) => r.id);
+
+      if (ids.length === 0) return { urlsDeleted: 0, linksDeleted: 0 };
+
+      // SQLite parameter limit: chunk the IN-list at 500 ids per delete.
+      const CHUNK = 500;
+      let linksDeleted = 0;
+      for (let i = 0; i < ids.length; i += CHUNK) {
+        const slice = ids.slice(i, i + CHUNK);
+        const placeholders = slice.map(() => '?').join(',');
+        // Links can reference these urls as `from_url_id` (page → link
+        // catalogue) or by url string in `to_url`. Wipe both directions.
+        const linkRes = this.db
+          .prepare(`DELETE FROM links WHERE from_url_id IN (${placeholders})`)
+          .run(...slice);
+        linksDeleted += Number(linkRes.changes);
+        this.db
+          .prepare(`DELETE FROM headers WHERE url_id IN (${placeholders})`)
+          .run(...slice);
+        this.db
+          .prepare(`DELETE FROM url_sources WHERE url_id IN (${placeholders})`)
+          .run(...slice);
+        this.db
+          .prepare(`DELETE FROM image_usages WHERE from_url_id IN (${placeholders})`)
+          .run(...slice);
+        this.db
+          .prepare(`DELETE FROM urls_issues WHERE url_id IN (${placeholders})`)
+          .run(...slice);
+      }
+
+      // Wipe `to_url` references in `links` that pointed AT the deleted
+      // URLs by host string. This is a separate pass because `to_url`
+      // is a string column, not a foreign key.
+      this.db
+        .prepare(
+          `DELETE FROM links
+            WHERE LOWER(
+              SUBSTR(
+                to_url,
+                INSTR(to_url, '://') + 3,
+                CASE
+                  WHEN INSTR(SUBSTR(to_url, INSTR(to_url, '://') + 3), '/') > 0
+                    THEN INSTR(SUBSTR(to_url, INSTR(to_url, '://') + 3), '/') - 1
+                  ELSE LENGTH(to_url)
+                END
+              )
+            ) = ?`,
+        )
+        .run(target);
+
+      // Finally the parent URL rows.
+      let urlsDeleted = 0;
+      for (let i = 0; i < ids.length; i += CHUNK) {
+        const slice = ids.slice(i, i + CHUNK);
+        const placeholders = slice.map(() => '?').join(',');
+        const res = this.db
+          .prepare(`DELETE FROM urls WHERE id IN (${placeholders})`)
+          .run(...slice);
+        urlsDeleted += Number(res.changes);
+      }
+
+      // Orphaned image rows (no remaining `image_usages` referencing
+      // them) are cleaned in a follow-up sweep.
+      this.db.exec(
+        `DELETE FROM images
+          WHERE id NOT IN (SELECT DISTINCT image_id FROM image_usages)`,
+      );
+
+      return { urlsDeleted, linksDeleted };
+    });
+  }
+
   /**
    * Run a callback inside a single SQLite transaction. Drives the
    * crawler's write-coalescing pump: 50–100 individual upserts per
@@ -594,6 +725,47 @@ export class ProjectDb {
       .run(key, value);
   }
 
+  /**
+   * Atomic batch write for the per-URL hot path. Replaces the previous
+   * "upsertUrl then runInTransaction({ setHeaders, setSource,
+   * insertLinks, insertImages })" sequence with a single transaction
+   * that runs in one trip across the writer-worker boundary — half
+   * the IPC round-trips, one transaction commit instead of two
+   * sequential commits, and atomic visibility (a reader can never see
+   * a URL row without its links).
+   */
+  writeFetchedUrl(payload: {
+    upsert: UpsertUrlInput;
+    headers: ReadonlyArray<readonly [string, string]> | null;
+    storeBody: { body: string; maxBytes: number } | null;
+    links: DiscoveredLink[];
+    images: DiscoveredImage[];
+    fromDepth: number;
+  }): { urlId: number } {
+    return this.runInTransaction(() => {
+      const urlId = this.upsertUrl(payload.upsert);
+      if (urlId && payload.headers && payload.headers.length > 0) {
+        this.setUrlHeaders(urlId, payload.headers);
+      }
+      if (urlId && payload.storeBody) {
+        try {
+          this.setUrlSource(urlId, payload.storeBody.body, payload.storeBody.maxBytes);
+        } catch {
+          /* best-effort — a snapshot failure must not abort the whole batch */
+        }
+      }
+      if (urlId) {
+        if (payload.links.length > 0) {
+          this.insertLinks(urlId, payload.links, payload.fromDepth);
+        }
+        if (payload.images.length > 0) {
+          this.insertImages(urlId, payload.images);
+        }
+      }
+      return { urlId };
+    });
+  }
+
   getAllUrls(): string[] {
     return (
       this.db.prepare('SELECT url FROM urls').all() as unknown as { url: string }[]
@@ -712,7 +884,24 @@ export class ProjectDb {
       });
   }
 
-  getPendingInternalLinks(): { url: string; depth: number }[] {
+  /**
+   * Internal URLs the crawler discovered as link targets but never
+   * fetched (depth/queue/filter race), plus URLs explicitly marked for
+   * re-crawl (status_code nulled out).
+   *
+   *   `excludeNofollow` — when true, drops URLs whose ONLY incoming
+   *   internal links carry rel="nofollow". A URL referenced by both a
+   *   nofollow and a follow link still qualifies (the follow link is
+   *   what we'd crawl). Used when the active config has
+   *   `followNofollow: false` so the drain loop and resume path don't
+   *   silently crawl URLs the user explicitly asked to leave alone.
+   */
+  getPendingInternalLinks(
+    opts: { excludeNofollow?: boolean } = {},
+  ): { url: string; depth: number }[] {
+    const followFilter = opts.excludeNofollow
+      ? `AND (l.rel IS NULL OR l.rel NOT LIKE '%nofollow%')`
+      : '';
     const discovered = this.db
       .prepare(
         `SELECT l.to_url AS url, MIN(u.depth) + 1 AS depth
@@ -720,6 +909,7 @@ export class ProjectDb {
          JOIN urls u ON l.from_url_id = u.id
          WHERE l.is_internal = 1
            AND l.to_url NOT IN (SELECT url FROM urls)
+           ${followFilter}
          GROUP BY l.to_url`,
       )
       .all() as unknown as { url: string; depth: number }[];
@@ -966,6 +1156,45 @@ export class ProjectDb {
     });
   }
 
+  /**
+   * Cooperatively-scheduled recompute. Identical end-state to
+   * `recomputeUrlsIssues` but yields to the Node event loop between
+   * each definition's INSERT, so the main thread can service IPC and
+   * crawler callbacks instead of freezing for the full duration of
+   * the 70+ correlated subqueries.
+   *
+   * Tradeoff vs. the sync version: every INSERT runs in its own tiny
+   * transaction (a `DELETE` + N `INSERT`s) instead of one big atomic
+   * one. Readers may briefly see a partially-rebuilt table during the
+   * window. For the periodic in-crawl tick that drives sidebar
+   * counters this is acceptable — momentary undercount for live
+   * counters beats a 1–3 s UI freeze every 30 s. The post-crawl pass
+   * still uses the atomic sync version for the final committed state.
+   */
+  async recomputeUrlsIssuesYielding(
+    definitions: ReadonlyArray<readonly [string, string]>,
+  ): Promise<void> {
+    // Atomic truncate so getIssueCounts can never see "old + new" rows.
+    this.runInTransaction(() => {
+      this.db.exec('DELETE FROM urls_issues');
+    });
+    for (const [issueKey, where] of definitions) {
+      // One transaction per definition keeps the writer lock window
+      // short — other writes (the crawler's per-URL inserts) can
+      // interleave between definitions instead of waiting on the
+      // whole 70-statement block.
+      this.runInTransaction(() => {
+        this.db
+          .prepare(
+            `INSERT OR IGNORE INTO urls_issues (url_id, issue_key)
+               SELECT id, ? FROM urls WHERE ${where}`,
+          )
+          .run(issueKey);
+      });
+      await new Promise<void>((resolve) => setImmediate(resolve));
+    }
+  }
+
   /**
    * Fast lookup of materialised issue counts. Single SELECT … GROUP BY
    * — replaces the 130-statement loop the sidebar used to run. Only
@@ -985,6 +1214,193 @@ export class ProjectDb {
     return out;
   }
 
+  /**
+   * Detect paginated URL clusters with missing ordinals and flag them.
+   *
+   * Algorithm:
+   *   1. Pull every internal HTML URL that participates in pagination
+   *      — has either `pagination_next` or `pagination_prev` non-null.
+   *   2. Extract a `(template, ordinal)` pair from each URL by stripping
+   *      the page-number token and recording its integer value. Three
+   *      patterns are recognised in priority order:
+   *        a. `?page=<n>` / `?p=<n>` / `?pg=<n>` query param
+   *        b. `/page/<n>/` or `/page/<n>` path segment
+   *        c. `/<n>` trailing path component (only when ≥ 2 — too
+   *           ambiguous otherwise; a literal `/2` could mean page 2
+   *           or product 2, but in practice paginated trailing-number
+   *           URLs always start at 2 because page 1 omits the number)
+   *      URLs that don't match any pattern are skipped — they have
+   *      pagination links but use an unknown URL scheme (cursor IDs,
+   *      hash-only, JS state) and gap detection isn't possible.
+   *   3. GROUP BY template, sort each group's ordinals ascending, and
+   *      flag every member of a group whose ordinals contain a gap
+   *      (`max - min + 1 != count` — accepts duplicates, requires
+   *      density). Groups of size 1 are skipped.
+   *   4. Bulk-update `urls.pagination_sequence_break` in a single
+   *      transaction. Default 0 already set by the schema; we only
+   *      need to write 1's.
+   *
+   * O(N) memory, O(N log N) time (per-template sort dominates).
+   */
+  recomputePaginationSequence(): void {
+    const rows = this.db
+      .prepare(
+        `SELECT id, url
+           FROM urls
+          WHERE is_external = 0
+            AND content_kind = 'html'
+            AND (pagination_next IS NOT NULL OR pagination_prev IS NOT NULL)`,
+      )
+      .all() as { id: number; url: string }[];
+
+    interface ParsedPage {
+      id: number;
+      template: string;
+      ordinal: number;
+    }
+    const parsed: ParsedPage[] = [];
+    for (const r of rows) {
+      const ord = parsePaginationOrdinal(r.url);
+      if (ord) parsed.push({ id: r.id, template: ord.template, ordinal: ord.ordinal });
+    }
+
+    // Group by template + walk for gaps.
+    const groups = new Map<string, ParsedPage[]>();
+    for (const p of parsed) {
+      const arr = groups.get(p.template);
+      if (arr) arr.push(p);
+      else groups.set(p.template, [p]);
+    }
+
+    const flaggedIds: number[] = [];
+    for (const arr of groups.values()) {
+      if (arr.length < 2) continue;
+      const ordinals = arr.map((p) => p.ordinal).sort((a, b) => a - b);
+      const min = ordinals[0]!;
+      const max = ordinals[ordinals.length - 1]!;
+      const expected = max - min + 1;
+      const distinct = new Set(ordinals).size;
+      if (distinct < expected) {
+        // At least one ordinal is missing in the [min,max] range.
+        for (const p of arr) flaggedIds.push(p.id);
+      }
+    }
+
+    this.runInTransaction(() => {
+      this.db.exec('UPDATE urls SET pagination_sequence_break = 0 WHERE pagination_sequence_break = 1');
+      if (flaggedIds.length === 0) return;
+      const stmt = this.db.prepare(
+        'UPDATE urls SET pagination_sequence_break = 1 WHERE id = ?',
+      );
+      for (const id of flaggedIds) stmt.run(id);
+    });
+  }
+
+  /**
+   * Persist the in-flight queue snapshot. Called by the crawler on a
+   * fixed cadence (every 30 s by default) so a crash, OOM, or OS
+   * reboot only loses up to that window. The pass is idempotent:
+   * `INSERT OR IGNORE` against the URL primary key, then the matching
+   * `seed_url` to discriminate stale checkpoints from a different
+   * start URL the user may have queued earlier in the same project.
+   *
+   * Truncate-then-insert is intentional: the queue shrinks as URLs
+   * complete, and we don't want yesterday's pending entries lingering
+   * after a successful crawl finishes.
+   */
+  checkpointQueue(items: ReadonlyArray<{ url: string; depth: number }>, seedUrl: string): void {
+    this.runInTransaction(() => {
+      this.db.exec('DELETE FROM crawl_queue');
+      if (items.length === 0) return;
+      const stmt = this.db.prepare(
+        'INSERT OR IGNORE INTO crawl_queue (url, depth, seed_url) VALUES (?, ?, ?)',
+      );
+      const seed = seedUrl ?? '';
+      const CHUNK = 500;
+      // Multi-row VALUES inserts in chunks — single .run() per item is
+      // ~10× slower at 100k+ items.
+      for (let i = 0; i < items.length; i += CHUNK) {
+        const slice = items.slice(i, i + CHUNK);
+        for (const it of slice) stmt.run(it.url, it.depth, seed);
+      }
+    });
+  }
+
+  /** Read the pending queue snapshot back. Empty array when nothing
+   * was checkpointed (i.e. previous crawl exited cleanly). */
+  loadQueueCheckpoint(): { url: string; depth: number; seedUrl: string }[] {
+    return this.db
+      .prepare('SELECT url, depth, seed_url AS seedUrl FROM crawl_queue')
+      .all() as { url: string; depth: number; seedUrl: string }[];
+  }
+
+  /** Wipe the checkpoint — called after a successful clean crawl
+   * completion or when the user dismisses the resume prompt. */
+  clearQueueCheckpoint(): void {
+    this.db.exec('DELETE FROM crawl_queue');
+  }
+
+  /**
+   * Detect pages whose hreflang JSON contains the same `lang` token
+   * mapped to two different target URLs. Common cause: a CMS bug
+   * where the language switcher emits both `<link hreflang="es" href="…/es/page-a">`
+   * and `<link hreflang="es" href="…/es/page-b">` on the same page,
+   * which makes search engines unable to pick a canonical regional
+   * variant. Sets `urls.hreflang_inconsistent_lang = 1` for every
+   * affected page.
+   *
+   * Implementation: pull the JSON column for every page that has any
+   * hreflang entries; parse client-side; mark the row when any lang
+   * appears twice with non-equal hrefs. Truncate-then-mark wrapped in
+   * a transaction so the boolean is consistent.
+   */
+  recomputeHreflangInconsistent(): void {
+    const rows = this.db
+      .prepare(
+        `SELECT id, hreflangs FROM urls
+          WHERE is_external = 0
+            AND content_kind = 'html'
+            AND hreflangs IS NOT NULL
+            AND hreflangs != ''
+            AND hreflangs != '[]'`,
+      )
+      .all() as { id: number; hreflangs: string }[];
+
+    const flagged: number[] = [];
+    for (const r of rows) {
+      let entries: { lang?: string; href?: string }[] = [];
+      try {
+        entries = JSON.parse(r.hreflangs) as { lang?: string; href?: string }[];
+      } catch {
+        continue;
+      }
+      const seen = new Map<string, string>();
+      let inconsistent = false;
+      for (const e of entries) {
+        if (!e.lang || !e.href) continue;
+        const key = e.lang.toLowerCase();
+        const prev = seen.get(key);
+        if (prev && prev !== e.href) {
+          inconsistent = true;
+          break;
+        }
+        if (!prev) seen.set(key, e.href);
+      }
+      if (inconsistent) flagged.push(r.id);
+    }
+
+    this.runInTransaction(() => {
+      this.db.exec(
+        'UPDATE urls SET hreflang_inconsistent_lang = 0 WHERE hreflang_inconsistent_lang = 1',
+      );
+      if (flagged.length === 0) return;
+      const stmt = this.db.prepare(
+        'UPDATE urls SET hreflang_inconsistent_lang = 1 WHERE id = ?',
+      );
+      for (const id of flagged) stmt.run(id);
+    });
+  }
+
   recomputeInlinks(): void {
     // One-pass aggregate via temp table: GROUP BY links.to_url once, then
     // join. The naive correlated-subquery form (UPDATE … = (SELECT COUNT…))
@@ -2511,6 +2927,13 @@ export class ProjectDb {
               AND i.byte_size > 5242880
          )`,
       ),
+      paginationSequenceBreak: countWhere(
+        `${html} AND pagination_sequence_break = 1`,
+      ),
+      linksPerPageTooMany: countWhere(`${html} AND outlinks > 100`),
+      hreflangInconsistentLang: countWhere(
+        `${html} AND hreflang_inconsistent_lang = 1`,
+      ),
     };
   }
 
@@ -3710,6 +4133,7 @@ export class ProjectDb {
     schemaInvalidCount: r.schema_invalid_count,
     paginationNext: r.pagination_next,
     paginationPrev: r.pagination_prev,
+    paginationSequenceBreak: (r as unknown as { pagination_sequence_break?: number }).pagination_sequence_break === 1,
     hreflangs: r.hreflangs,
     hreflangCount: r.hreflang_count,
     amphtml: r.amphtml,
@@ -3803,6 +4227,66 @@ function validSortColumn(sortBy: string | undefined): string {
   return VALID_SORT_COLUMNS.has(snake) ? snake : 'id';
 }
 
+/**
+ * Parse a paginated URL into a `(template, ordinal)` pair so the
+ * pagination-sequence pass can group same-pattern pages and detect
+ * missing numbers. Returns null when no recognised page-number token
+ * is found — those URLs are dropped from the analysis (we'd produce
+ * false positives if we tried to invent ordinals from arbitrary path
+ * components).
+ *
+ * Priority order matters: query params first because they're
+ * unambiguous; then `/page/N` because it's the most common SEO-safe
+ * pagination scheme; then trailing `/N` (≥ 2) because page-1 URLs
+ * conventionally omit the number, so a `/products/cars` is page 1 of
+ * `/products/cars/2`. Lower-bound 2 prevents false positives on
+ * `/foo/1` (which often means "category 1", not "page 1").
+ */
+function parsePaginationOrdinal(
+  rawUrl: string,
+): { template: string; ordinal: number } | null {
+  let url: URL;
+  try {
+    url = new URL(rawUrl);
+  } catch {
+    return null;
+  }
+  // Pattern A — `?page=N` (and common synonyms).
+  for (const key of ['page', 'p', 'pg']) {
+    const v = url.searchParams.get(key);
+    if (v && /^\d+$/.test(v)) {
+      const n = Number.parseInt(v, 10);
+      if (n >= 1) {
+        const params = new URLSearchParams(url.search);
+        params.delete(key);
+        const cleanQuery = params.toString();
+        const template = `${url.origin}${url.pathname}?${key}=*${cleanQuery ? `&${cleanQuery}` : ''}`;
+        return { template, ordinal: n };
+      }
+    }
+  }
+  // Pattern B — `/page/N` or `/page/N/`.
+  const pageSegMatch = url.pathname.match(/(\/page\/)(\d+)(\/?)$/i);
+  if (pageSegMatch) {
+    const n = Number.parseInt(pageSegMatch[2]!, 10);
+    if (n >= 1) {
+      const stripped = url.pathname.replace(/\/page\/\d+\/?$/i, '');
+      const template = `${url.origin}${stripped}/page/*${pageSegMatch[3]}${url.search}`;
+      return { template, ordinal: n };
+    }
+  }
+  // Pattern C — trailing `/N`, only N ≥ 2 (page 1 omits the number).
+  const trailingMatch = url.pathname.match(/^(.+\/)(\d+)\/?$/);
+  if (trailingMatch) {
+    const n = Number.parseInt(trailingMatch[2]!, 10);
+    if (n >= 2) {
+      const template = `${url.origin}${trailingMatch[1]}*${url.search}`;
+      return { template, ordinal: n };
+    }
+  }
+  return null;
+}
+
 /**
  * Issue-key → SQL WHERE clause definitions for the materialised
  * `urls_issues` table (see `ProjectDb.recomputeUrlsIssues`). Listed
@@ -4987,6 +5471,58 @@ function categoryWhereClause(cat: UrlCategory): string | null {
                    AND i.byte_size IS NOT NULL
                    AND i.byte_size > 5242880
               )`;
+    case 'issues:pagination-sequence-break':
+      // Set post-crawl by `recomputePaginationSequence()` — see the
+      // implementation comment there for the gap-detection algorithm
+      // and the URL pattern recognisers that produce ordinals.
+      return `is_external = 0 AND content_kind = 'html'
+              AND pagination_sequence_break = 1`;
+    case 'issues:links-per-page-too-many':
+      // Total outgoing links (internal + external, deduplicated) > 100.
+      // Distinct from `external-links-too-many` which only counts the
+      // external bucket — this filter trips on internal-heavy pages too
+      // (mega-menus, sitemap-style hub pages) which Google's PageRank
+      // dilution makes equally problematic.
+      return `is_external = 0 AND content_kind = 'html'
+              AND outlinks > 100`;
+    case 'tab:redirects':
+      // Wave 4 — Redirects tab. Every internal URL whose status is 3xx
+      // is a redirect hop. Each hop already has its own row in the
+      // `urls` table (the crawler enqueues redirect targets so we
+      // capture full chains as discrete rows), so a simple status-class
+      // filter recovers the full set; the table's `redirect_target`,
+      // `redirect_chain_length`, `redirect_final_url`, and
+      // `redirect_loop` columns surface chain context per row.
+      return `is_external = 0
+              AND status_code >= 300 AND status_code < 400`;
+    case 'tab:canonicals':
+      // Wave 4 — Canonicals tab. Every HTML page that declares a
+      // canonical (HTML link tag OR HTTP `Link: rel=canonical` header).
+      // Self-referencing, cross-page, and mismatched canonicals all
+      // surface here; the per-row "Canonical" + "Canonical (HTTP)"
+      // columns let the user audit them visually.
+      return `is_external = 0 AND content_kind = 'html'
+              AND (
+                (canonical IS NOT NULL AND canonical != '')
+                OR (canonical_http IS NOT NULL AND canonical_http != '')
+              )`;
+    case 'issues:hreflang-inconsistent-lang':
+      // Set post-crawl by `recomputeHreflangInconsistent()` — same `lang`
+      // token mapped to two different hrefs on the same page.
+      return `is_external = 0 AND content_kind = 'html'
+              AND hreflang_inconsistent_lang = 1`;
+    case 'tab:directives':
+      // Wave 4 — Directives tab. Pages declaring any indexability
+      // directive: meta robots, X-Robots-Tag header, robots.txt block,
+      // or canonical-to-other (which acts as an indexability signal).
+      // The `Meta Robots`, `X-Robots-Tag`, and `Indexability Reason`
+      // columns make the active directives readable at a glance.
+      return `is_external = 0 AND content_kind = 'html'
+              AND (
+                (meta_robots IS NOT NULL AND meta_robots != '')
+                OR (x_robots_tag IS NOT NULL AND x_robots_tag != '')
+                OR indexability LIKE 'non-indexable%'
+              )`;
     default:
       return null;
   }

packages/shared-types/src/crawl.ts

@@ -1,3 +1,15 @@
+/**
+ * Crawl scope. Determines which links are followed.
+ *  - `subdomain`        — same registrable domain + same exact host
+ *                         (default; tightest "this site" definition)
+ *  - `subfolder`        — same host AND target's path starts with the
+ *                         start URL's path. Use to crawl a single
+ *                         section of a large site (e.g. `/blog/` only).
+ *  - `all-subdomains`   — any host that shares the registrable domain
+ *                         (`*.example.com` ∪ `example.com`)
+ *  - `exact-url`        — only the start URL itself; no link-follow.
+ *                         Equivalent to "single-page" mode.
+ */
 export type CrawlScope = 'subdomain' | 'subfolder' | 'all-subdomains' | 'exact-url';
 
 /**
@@ -162,6 +174,12 @@ export type UrlCategory =
   | 'issues:render-blocking-critical'
   | 'issues:og-image-too-large'
   | 'issues:twitter-image-too-large'
+  | 'issues:pagination-sequence-break'
+  | 'issues:links-per-page-too-many'
+  | 'tab:redirects'
+  | 'tab:canonicals'
+  | 'tab:directives'
+  | 'issues:hreflang-inconsistent-lang'
 
 export type Indexability =
   | 'indexable'
@@ -311,6 +329,10 @@ export interface CrawlUrlRow {
   queryStringLength: number;
   paginationNext: string | null;
   paginationPrev: string | null;
+  /** True when this URL is part of a paginated cluster whose ordinal
+   * sequence has a gap (e.g. ?page=1, ?page=2, ?page=4 — 3 missing).
+   * Set by the post-crawl `recomputePaginationSequence()` pass. */
+  paginationSequenceBreak: boolean;
   /** JSON-stringified array of `{ lang, href }` objects, or null. */
   hreflangs: string | null;
   hreflangCount: number;
@@ -615,6 +637,106 @@ export interface CrawlConfig {
    * `*.cloudfront.net`.
    */
   cdnHosts: string[];
+  /**
+   * Maximum total links per page (internal + external) before the
+   * "Too Many Links per Page" issue trips. Default 100 — Google's
+   * historical recommendation; pages above this start to look like
+   * link-farm SERPs. 0 disables the check.
+   */
+  maxLinksPerPage: number;
+  /**
+   * Maximum response time (ms). Requests that exceed this are aborted
+   * and recorded as a network error — distinct from `requestTimeoutMs`
+   * which is the connect+headers timeout. Use to cap individual slow
+   * pages without lowering the overall fetch timeout. 0 disables.
+   */
+  maxResponseTimeMs: number;
+  /**
+   * Maximum response body size (bytes). When the `Content-Length`
+   * header on the response exceeds this, the body is discarded and
+   * the page is recorded with status only. Useful for trimming large
+   * downloads (PDFs, archives) on bandwidth-constrained connections.
+   * 0 disables.
+   */
+  maxFileSizeBytes: number;
+  /**
+   * Follow `<link rel="canonical">` like a redirect — when a 200 page
+   * declares a canonical pointing elsewhere, also enqueue the canonical
+   * target. Default `false` — most crawls treat canonicals as a
+   * signal, not a navigation hint.
+   */
+  followCanonicals: boolean;
+  /**
+   * Cookie policy applied to every fetch. The crawler is otherwise
+   * stateless across requests; this knob lets users opt into
+   * session-cookie behaviour when crawling sites that gate content
+   * behind a session.
+   *  - `reject-all`         (default) — never send Cookie header,
+   *                          discard Set-Cookie response headers
+   *  - `accept-all`         — round-trip cookies via an in-memory
+   *                          jar keyed by host
+   *  - `block-third-party`  — accept first-party cookies only
+   *                          (same registrable domain as the page)
+   */
+  cookiePolicy: 'reject-all' | 'accept-all' | 'block-third-party';
+  /**
+   * Per-host User-Agent overrides. Map of host pattern → UA string.
+   * Patterns support exact host (`m.example.com`) or leading wildcard
+   * (`*.example.com`); wildcard matches any subdomain. The first
+   * matching pattern wins; falls back to the global `userAgent` when
+   * none match. Useful for crawling a mobile subdomain with the
+   * mobile-Googlebot UA in the same run as the desktop site.
+   */
+  perHostUserAgents: { hostPattern: string; userAgent: string }[];
+  /**
+   * Named proxy profiles. The user can save multiple `(name, url)`
+   * entries and pick one by name in `proxyProfileActive`. Empty
+   * `proxyProfileActive` falls back to the legacy `proxyUrl` /
+   * `HTTPS_PROXY` env var.
+   */
+  proxyProfiles: { name: string; url: string }[];
+  /** Currently-selected proxy profile name. Empty = use proxyUrl
+   *  (or env vars) directly without profile lookup. */
+  proxyProfileActive: string;
+  /**
+   * Follow `<link rel="next">` and `<link rel="prev">` for pagination
+   * link discovery. Default `true` — these are part of the standard
+   * crawl graph; off only to debug pagination-specific issues.
+   */
+  followPaginationLinks: boolean;
+  /**
+   * Follow `<a rel="nofollow">` links (still respecting all other
+   * filters). Default `false` — Screaming Frog "Respect Nofollow"
+   * default. Combined with `storeNofollowLinks` for the storage side.
+   */
+  followNofollow: boolean;
+  /**
+   * Follow JavaScript-style redirects discovered in the HTML body
+   * (`<meta http-equiv="refresh">` content URL, `window.location` JS
+   * statements). Default `false` — these are heuristics; when on,
+   * the meta-refresh URL is also enqueued like a redirect target.
+   */
+  followJsRedirects: boolean;
+  /**
+   * Wave 6 — Per-pass crawl-analysis toggles. Each post-crawl pass
+   * can be independently disabled when the user knows the data isn't
+   * needed for their audit; saves wall-clock on large crawls. All
+   * default `true` because the corresponding issue filters/reports
+   * silently fall back to "no data" when their pass didn't run.
+   */
+  /** Recompute `inlinks` count per URL after the crawl. */
+  analyseInlinks: boolean;
+  /** Walk redirect chains, fill `redirect_chain_length` / `redirect_loop`. */
+  analyseRedirectChains: boolean;
+  /** Hreflang reciprocity + invalid code + target health. */
+  analyseHreflang: boolean;
+  /** SimHash + LSH near-duplicate clustering. */
+  analyseDuplicates: boolean;
+  /** Pagination ordinal-gap detection. */
+  analysePagination: boolean;
+  /** Materialise the heavy `urls_issues` counters (Dead External Domain,
+   * Duplicate URL post-norm, Canonical Chain Multi-hop). */
+  analyseIssues: boolean;
 }
 
 export interface HttpAuth {
@@ -904,6 +1026,24 @@ export interface OverviewCounts {
      * conservative threshold that catches both card types' renderer.
      */
     twitterImageTooLarge: number;
+    /**
+     * Pages part of a paginated cluster whose ordinals have a gap
+     * (e.g. ?page=1, ?page=2, ?page=4 — page 3 missing). Set by the
+     * post-crawl `recomputePaginationSequence()` pass.
+     */
+    paginationSequenceBreak: number;
+    /**
+     * Pages whose total outgoing link count (internal + external)
+     * exceeds the configured `maxLinksPerPage` threshold (default 100).
+     */
+    linksPerPageTooMany: number;
+    /**
+     * Pages declaring the same `hreflang` value with two different
+     * target URLs — i.e. the page can't decide which page is the "es"
+     * version. Detected by post-crawl pass `recomputeHreflangInconsistent`
+     * which writes a boolean flag onto each affected URL.
+     */
+    hreflangInconsistentLang: number;
   };
 }
 
@@ -1142,4 +1282,21 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   largeImageBytes: 102_400,
   probeTlsCerts: true,
   cdnHosts: [],
+  maxLinksPerPage: 100,
+  maxResponseTimeMs: 0,
+  maxFileSizeBytes: 0,
+  followCanonicals: false,
+  followPaginationLinks: true,
+  followNofollow: false,
+  followJsRedirects: false,
+  analyseInlinks: true,
+  analyseRedirectChains: true,
+  analyseHreflang: true,
+  analyseDuplicates: true,
+  analysePagination: true,
+  analyseIssues: true,
+  cookiePolicy: 'reject-all',
+  perHostUserAgents: [],
+  proxyProfiles: [],
+  proxyProfileActive: '',
 };

packages/shared-types/src/ipc.ts

@@ -38,6 +38,22 @@ export const IPC = {
   summaryGet: 'summary:get',
   exportCsv: 'export:csv',
   exportJson: 'export:json',
+  exportXml: 'export:xml',
+  /** GDPR-aligned per-domain delete. Wipes every row whose URL host
+   * matches the given domain (and that domain's links/images/headers/
+   * url_sources). Used by Settings → "Delete Domain Data". */
+  dataDeleteByDomain: 'data:delete-by-domain',
+  /** Wave 6 — Crash-recovery surface. Renderer asks the main process
+   * whether the previous session left a non-empty `crawl_queue` table;
+   * the response carries the seed URL + count so the user can be
+   * shown a clear "Resume crawl of X (240 pending)?" prompt. */
+  crashRecoveryStatus: 'crash:recovery-status',
+  /** Trigger a resume — main process re-creates the Crawler with the
+   * previously-saved start URL and enqueues the checkpointed pending
+   * items at their original depth before kicking off the queue. */
+  crashRecoveryResume: 'crash:recovery-resume',
+  /** Discard the checkpoint without resuming. */
+  crashRecoveryDiscard: 'crash:recovery-discard',
   exportHtmlReport: 'export:html-report',
   exportBulk: 'export:bulk',
   compareLoad: 'compare:load',
@@ -153,8 +169,11 @@ export type MenuEvent =
   | 'toggle-detail-panel'
   | 'export-csv'
   | 'export-json'
+  | 'export-xml'
   | 'export-html-report'
   | 'export-bulk'
+  | 'delete-domain-data'
+  | 'clear-all-data'
   | 'compare-with-project'
   | 'save-project-as'
   | 'open-visualization'
@@ -183,6 +202,38 @@ export interface ExportJsonResult {
   rowsWritten: number;
 }
 
+export interface ExportXmlInput {
+  filePath: string;
+  category?: UrlCategory;
+  selectedIds?: number[];
+}
+
+export interface ExportXmlResult {
+  filePath: string;
+  rowsWritten: number;
+}
+
+export interface DataDeleteByDomainInput {
+  /** Hostname to wipe — case-insensitive, no scheme/port. */
+  domain: string;
+}
+
+export interface CrashRecoveryStatus {
+  /** Number of URLs the previous session left pending. 0 = nothing
+   * to recover and the renderer skips the prompt. */
+  pendingCount: number;
+  /** The start URL the previous crawl was running against. */
+  seedUrl: string;
+}
+
+export interface DataDeleteByDomainResult {
+  /** Number of `urls` rows deleted. Cascade handles `links`, `images`,
+   *  `headers`, `url_sources`, `urls_issues`. */
+  urlsDeleted: number;
+  /** Number of associated `links` rows wiped (informational). */
+  linksDeleted: number;
+}
+
 export interface ExportHtmlReportInput {
   filePath: string;
 }
@@ -624,6 +675,13 @@ export interface FreeCrawlApi {
   summaryGet(): Promise<CrawlSummary>;
   exportCsv(input: ExportCsvInput): Promise<ExportCsvResult>;
   exportJson(input: ExportJsonInput): Promise<ExportJsonResult>;
+  exportXml(input: ExportXmlInput): Promise<ExportXmlResult>;
+  dataDeleteByDomain(
+    input: DataDeleteByDomainInput,
+  ): Promise<DataDeleteByDomainResult>;
+  crashRecoveryStatus(): Promise<CrashRecoveryStatus>;
+  crashRecoveryResume(): Promise<{ accepted: boolean }>;
+  crashRecoveryDiscard(): Promise<void>;
   exportHtmlReport(input: ExportHtmlReportInput): Promise<ExportHtmlReportResult>;
   exportBulk(): Promise<BulkExportResult>;
   compareLoad(input: CompareLoadInput): Promise<CompareLoadResult>;