feat: log threat detection, integration exports, adaptive crawl speed

apps/cli/src/index.ts

@@ -525,6 +525,8 @@ function flattenLogResult(filePath: string, result: LogAnalysisResult): LogInges
     urlBots: Array.from(result.urlStats.values()).flatMap((u) =>
       Array.from(u.botCounts.entries()).map(([bot, hits]) => ({ path: u.path, bot, hits })),
     ),
+    threats: result.threats,
+    threatsDropped: result.threatsDropped,
   };
 }
 

apps/desktop/src/main/db-writer-pool.ts

@@ -145,6 +145,20 @@ export class DbWriterPool {
     });
   }
 
+  /**
+   * Calls outstanding and the age of the oldest. The watchdog heartbeat
+   * carries both, so a crawl whose counters stopped moving shows at a
+   * glance whether it is waiting on the writer.
+   */
+  pendingStats(): { count: number; oldestMs: number } {
+    const now = Date.now();
+    let oldestMs = 0;
+    for (const p of this.pending.values()) {
+      oldestMs = Math.max(oldestMs, now - p.startedAt);
+    }
+    return { count: this.pending.size, oldestMs };
+  }
+
   async terminate(): Promise<void> {
     this.terminated = true;
     if (this.monitorTimer) {

apps/desktop/src/main/db-writer-worker.ts

@@ -69,6 +69,23 @@ if (watchdog) {
   }, WATCHDOG_HEARTBEAT_INTERVAL_MS).unref();
 }
 
+/**
+ * Requests interleave: the handler below is async, and a yielding pass
+ * (`recomputeUrlsIssuesYielding`) hands the loop back between
+ * definitions, so a per-URL write can run — and finish — while the pass
+ * is still open. Its `finally` used to overwrite the op slot with
+ * `idle`, and the pass's next multi-second statement then reached
+ * debug.txt as an `idle → idle` writer stall: real time lost, wrong
+ * label. The slot now names the oldest request still open (the one a
+ * long statement belongs to) and only reads `idle` once none is.
+ */
+const inFlight = new Map<number, string>();
+function publishOp(): void {
+  if (!watchdog) return;
+  const oldest = inFlight.values().next();
+  watchdog.setWriterOp(oldest.done ? 'idle' : oldest.value);
+}
+
 interface RequestMessage {
   requestId: number;
   method: string;
@@ -92,6 +109,7 @@ const ALLOWED_METHODS = new Set<string>([
   'setSitemapUrls',
   'setHostCert',
   'setImageSize',
+  'applyProbeWrites',
   'setMeta',
   'updateExternalProbe',
   'recomputeInlinks',
@@ -119,7 +137,8 @@ const ALLOWED_METHODS = new Set<string>([
 parentPort.on('message', async (msg: RequestMessage) => {
   if (!msg || typeof msg.requestId !== 'number') return;
   const { requestId, method, args } = msg;
-  if (watchdog) watchdog.setWriterOp(method);
+  inFlight.set(requestId, method);
+  publishOp();
   try {
     if (!ALLOWED_METHODS.has(method)) {
       throw new Error(`db-writer-worker: method '${method}' is not whitelisted`);
@@ -135,7 +154,8 @@ parentPort.on('message', async (msg: RequestMessage) => {
       // "long pass, still advancing" from a genuinely wedged worker.
       const defs = (args?.[0] ?? []) as ReadonlyArray<readonly [string, string]>;
       result = await db.recomputeUrlsIssuesYielding(defs, (done, total) => {
-        watchdog.setWriterOp(`recomputeUrlsIssuesYielding ${done}/${total}`);
+        inFlight.set(requestId, `recomputeUrlsIssuesYielding ${done}/${total}`);
+        publishOp();
       });
     } else {
       const out = (fn as (...a: unknown[]) => unknown).apply(db, args ?? []);
@@ -149,7 +169,8 @@ parentPort.on('message', async (msg: RequestMessage) => {
       error: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
     });
   } finally {
-    if (watchdog) watchdog.setWriterOp('idle');
+    inFlight.delete(requestId);
+    publishOp();
   }
 });
 

apps/desktop/src/main/freeze-watchdog-report.ts

@@ -0,0 +1,113 @@
+/**
+ * Pure helpers behind the lines the freeze watchdog writes to
+ * `debug.txt`. Kept free of Electron / worker imports so the logic that
+ * decides *what* gets recorded can be unit-tested, and so `logger.ts` can
+ * use the mirror filter without importing the watchdog (which imports the
+ * logger — a cycle).
+ *
+ * `debug.txt` is the one file a remote user is asked to send. Every
+ * helper here exists because a real investigation of that file had to
+ * guess at something it should have simply stated: which storage mode
+ * the session ran in, what the crawl was configured to do, whether the
+ * app was killed or exited, how long each post-crawl phase took, and
+ * which errors the main log had recorded meanwhile.
+ */
+
+import type { LogLevel } from '@freecrawl/shared-types';
+
+/** Cap on the message part of a mirrored log line. Stacks and SQL are
+ *  in the main log; the debug file wants the headline. */
+export const MIRROR_MAX_CHARS = 300;
+
+/**
+ * Which logger entries are worth a copy in `debug.txt`.
+ *
+ * Errors always — a `database is locked`, a worker exit, an unhandled
+ * rejection are exactly what a stall needs next to it. Warnings too,
+ * except the crawler's: per-URL fetch failures arrive by the thousand on
+ * a slow host and would drown the timeline (the heartbeat's `failed`
+ * counter already carries their number).
+ */
+export function shouldMirrorToDebug(level: LogLevel, source: string): boolean {
+  if (level === 'error') return true;
+  if (level === 'warn') return source !== 'crawler';
+  return false;
+}
+
+/** First line of a message, whitespace collapsed, capped — one log line. */
+export function compactForDebug(message: string): string {
+  const firstLine = message.split(/\r?\n/, 1)[0] ?? '';
+  const flat = firstLine.replace(/\s+/g, ' ').trim();
+  return flat.length > MIRROR_MAX_CHARS ? `${flat.slice(0, MIRROR_MAX_CHARS - 1)}…` : flat;
+}
+
+/** Render one mirrored entry as a debug line (without the timestamp). */
+export function formatMirroredLog(level: LogLevel, source: string, message: string): string {
+  return `[${level.toUpperCase()}] ${source}: ${compactForDebug(message)}`;
+}
+
+/**
+ * What the tail of the previous debug file says about how that session
+ * ended. A clean exit always leaves `[SHUTDOWN]` as its last line; any
+ * other last line means the process was killed, crashed, or the machine
+ * went down — and the line itself says what the app was doing at the
+ * time, which is usually the whole answer to "why did it stop".
+ *
+ * Returns null when there is nothing to report (fresh file, or a clean
+ * shutdown).
+ */
+export function previousSessionVerdict(tail: string): string | null {
+  const lines = tail.split(/\r?\n/).map((l) => l.trim()).filter((l) => l.length > 0);
+  const last = lines[lines.length - 1];
+  if (!last) return null;
+  if (last.includes('[SHUTDOWN]')) return null;
+  const ts = last.match(/^(\d{4}-\d{2}-\d{2}T[\d:.]+Z)/)?.[1] ?? 'unknown time';
+  const body = last.replace(/^\S+\s*/, '');
+  const clipped = body.length > 220 ? `${body.slice(0, 219)}…` : body;
+  const doing = /main_op="(crawl|post-crawl):/.test(last)
+    ? ' (a crawl was in progress)'
+    : '';
+  return `[PREV] previous session ended without a clean shutdown at ${ts}${doing} — last line: ${clipped}`;
+}
+
+/**
+ * Turns the main-thread op stream into phase durations.
+ *
+ * Ops arrive as `post-crawl:image-probes`, `post-crawl:tls-probes`, …;
+ * the tracker reports when one such phase gives way to the next, with
+ * how long it ran. A phase that swallows half an hour (16 k image
+ * probes) is then a single line, instead of something reconstructed from
+ * 360 identical heartbeats.
+ */
+export class PhaseTracker {
+  private current: string | null = null;
+  private since = 0;
+
+  /** Feed the current main op; returns a report line when a phase ended. */
+  observe(mainOp: string, nowMs: number): string | null {
+    const phase = PhaseTracker.phaseOf(mainOp);
+    if (phase === this.current) return null;
+    let line: string | null = null;
+    if (this.current !== null) {
+      const secs = Math.max(0, Math.round((nowMs - this.since) / 1000));
+      line = `[PHASE] ${this.current} took ${secs}s`;
+    }
+    this.current = phase;
+    this.since = nowMs;
+    return line;
+  }
+
+  /** Only the post-crawl passes are phases; everything else is `null`. */
+  static phaseOf(mainOp: string): string | null {
+    if (!mainOp.startsWith('post-crawl:')) return null;
+    // `post-crawl:image-probes` stays whole; a URL-suffixed op would be
+    // cut at its second colon so the phase name never carries a URL.
+    const second = mainOp.indexOf(':', 'post-crawl:'.length);
+    return second < 0 ? mainOp : mainOp.slice(0, second);
+  }
+}
+
+/** Kilobytes → whole megabytes, for the heartbeat's memory fields. */
+export function toMb(bytes: number): number {
+  return Math.round(bytes / (1024 * 1024));
+}

apps/desktop/src/main/freeze-watchdog-shared.ts

@@ -37,7 +37,12 @@
  *      [5]    main op string byte length
  *      [6]    reader op string byte length
  *      [7]    writer op string byte length
- *      [8..15] spare
+ *      [8]    main-process RSS (MB)
+ *      [9]    main-process V8 heap in use (MB)
+ *      [10]   main event-loop max lag over the last ~5 s (ms)
+ *      [11]   db-writer calls outstanding
+ *      [12]   age of the oldest outstanding db-writer call (ms)
+ *      [13..15] spare
  *   [96..)     Uint8Array        — op string bytes (UTF-8)
  *      [0..256)    main op
  *      [256..512)  reader op
@@ -68,6 +73,11 @@ const I32_IDX_FAILED = 4;
 const I32_IDX_MAIN_OP_LEN = 5;
 const I32_IDX_READER_OP_LEN = 6;
 const I32_IDX_WRITER_OP_LEN = 7;
+const I32_IDX_RSS_MB = 8;
+const I32_IDX_HEAP_MB = 9;
+const I32_IDX_MAIN_LOOP_LAG_MS = 10;
+const I32_IDX_WRITER_PENDING = 11;
+const I32_IDX_WRITER_OLDEST_MS = 12;
 
 export const SAB_BYTE_LENGTH = 1024;
 
@@ -85,6 +95,19 @@ export interface CounterSnapshot {
   failed: number;
 }
 
+/**
+ * Main-process health the heartbeat line carries next to the counters.
+ * Memory and loop lag answer "was the process under pressure" for a
+ * stall; the writer queue answers "was the crawl waiting on the DB".
+ */
+export interface HostStats {
+  rssMb: number;
+  heapMb: number;
+  mainLoopLagMs: number;
+  writerPending: number;
+  writerOldestMs: number;
+}
+
 /**
  * Wraps a `SharedArrayBuffer` with typed accessors. Construct one in
  * the main process via `FreezeWatchdogSharedState.create()`, then pass
@@ -152,8 +175,32 @@ export class FreezeWatchdogSharedState {
     }
   }
 
+  updateHostStats(s: Partial<HostStats>): void {
+    if (s.rssMb !== undefined) Atomics.store(this.i32, I32_IDX_RSS_MB, s.rssMb | 0);
+    if (s.heapMb !== undefined) Atomics.store(this.i32, I32_IDX_HEAP_MB, s.heapMb | 0);
+    if (s.mainLoopLagMs !== undefined) {
+      Atomics.store(this.i32, I32_IDX_MAIN_LOOP_LAG_MS, s.mainLoopLagMs | 0);
+    }
+    if (s.writerPending !== undefined) {
+      Atomics.store(this.i32, I32_IDX_WRITER_PENDING, s.writerPending | 0);
+    }
+    if (s.writerOldestMs !== undefined) {
+      Atomics.store(this.i32, I32_IDX_WRITER_OLDEST_MS, s.writerOldestMs | 0);
+    }
+  }
+
   // ── Readers (used by the watchdog worker) ─────────────────────────
 
+  readHostStats(): HostStats {
+    return {
+      rssMb: Atomics.load(this.i32, I32_IDX_RSS_MB),
+      heapMb: Atomics.load(this.i32, I32_IDX_HEAP_MB),
+      mainLoopLagMs: Atomics.load(this.i32, I32_IDX_MAIN_LOOP_LAG_MS),
+      writerPending: Atomics.load(this.i32, I32_IDX_WRITER_PENDING),
+      writerOldestMs: Atomics.load(this.i32, I32_IDX_WRITER_OLDEST_MS),
+    };
+  }
+
   readMainHeartbeatMs(): number {
     return Number(Atomics.load(this.i64, I64_IDX_MAIN_HB));
   }

apps/desktop/src/main/freeze-watchdog-worker.ts

@@ -1,6 +1,7 @@
 import { parentPort, workerData } from 'node:worker_threads';
 import * as fs from 'node:fs';
 import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js';
+import { PhaseTracker, previousSessionVerdict } from './freeze-watchdog-report.js';
 
 /**
  * Freeze-watchdog worker.
@@ -47,8 +48,13 @@ const HEARTBEAT_LOG_INTERVAL_MS = 5000;
 interface InitData {
   sab: SharedArrayBuffer;
   debugFilePath: string;
+  /** One-line host description (app / Electron / OS / CPU / RAM). */
+  envLine?: string;
 }
 
+/** Roll the file over past this size — keeps one predecessor. */
+const DEBUG_MAX_BYTES = 25 * 1024 * 1024;
+
 if (!parentPort) {
   throw new Error('freeze-watchdog-worker: must be loaded via worker_threads');
 }
@@ -61,6 +67,37 @@ if (!init?.sab || !init?.debugFilePath) {
 const state = new FreezeWatchdogSharedState(init.sab);
 const debugPath = init.debugFilePath;
 
+/** Last ~4 KB of the existing file — enough for its final line. */
+function readTail(): string {
+  try {
+    const fd = fs.openSync(debugPath, 'r');
+    try {
+      const size = fs.fstatSync(fd).size;
+      const len = Math.min(size, 4096);
+      if (len === 0) return '';
+      const buf = Buffer.alloc(len);
+      fs.readSync(fd, buf, 0, len, size - len);
+      return buf.toString('utf8');
+    } finally {
+      fs.closeSync(fd);
+    }
+  } catch {
+    return '';
+  }
+}
+
+// Read before rotating: the verdict is about the file we are about to
+// push aside. Rotation keeps one predecessor so a report that spans the
+// boundary is still recoverable.
+const prevSessionLine = previousSessionVerdict(readTail());
+try {
+  if (fs.statSync(debugPath).size > DEBUG_MAX_BYTES) {
+    fs.renameSync(debugPath, debugPath.replace(/\.txt$/i, '') + '.1.txt');
+  }
+} catch {
+  /* no file yet, or locked — append below either way */
+}
+
 // Silently swallow append failures — if the disk is full or the
 // file is locked by AV, we can't recover, and writing would just
 // thrash retries. The watchdog stays useful for the next stall.
@@ -82,6 +119,12 @@ function sanitizeOp(op: string): string {
   return op.replace(/[\r\n\t]+/g, ' ').slice(0, 240);
 }
 
+/** Main-process notes ([CRAWL] / [STORAGE] / mirrored log lines) — same
+ *  sanitising, longer cap: a mirrored error is already compacted. */
+function sanitizeNote(line: string): string {
+  return line.replace(/[\r\n\t]+/g, ' ').slice(0, 600);
+}
+
 interface StallTracker {
   /** Gap exceeded the threshold and we're timing it. */
   active: boolean;
@@ -91,6 +134,8 @@ interface StallTracker {
   startOp: string;
   /** `gap=612ms` / `silence=1709ms`, captured at onset. */
   detail: string;
+  /** `crawled` at onset — the end line reports how far it moved. */
+  startCrawled: number;
 }
 
 const newTracker = (): StallTracker => ({
@@ -99,6 +144,7 @@ const newTracker = (): StallTracker => ({
   startTs: 0,
   startOp: '',
   detail: '',
+  startCrawled: 0,
 });
 
 const mainStall = newTracker();
@@ -118,6 +164,7 @@ function trackStall(
   now: number,
   op: string,
   detail: string,
+  crawledNow: number,
   onStart: (t: StallTracker) => string,
   onEnd: (t: StallTracker, durationMs: number) => string,
 ): void {
@@ -128,6 +175,7 @@ function trackStall(
       t.startTs = now;
       t.startOp = op;
       t.detail = detail;
+      t.startCrawled = crawledNow;
     } else if (!t.reported && now - t.startTs >= STALL_REPORT_FLOOR_MS) {
       t.reported = true;
       appendLine(onStart(t));
@@ -142,6 +190,7 @@ function trackStall(
 
 let lastHeartbeatLogTs = 0;
 let bootLogged = false;
+const phases = new PhaseTracker();
 
 function check(): void {
   const now = Date.now();
@@ -154,6 +203,13 @@ function check(): void {
   const mainOp = sanitizeOp(state.readMainOp());
   const readerOp = sanitizeOp(state.readReaderOp());
   const writerOp = sanitizeOp(state.readWriterOp());
+  const host = state.readHostStats();
+  // Signed delta of the crawled counter across a stall: "+0" over two
+  // minutes is a wedge, "+400" is a slow patch the crawl worked through.
+  const crawledDelta = (t: StallTracker): string => {
+    const d = counters.crawled - t.startCrawled;
+    return `${d >= 0 ? '+' : ''}${d}`;
+  };
 
   const mainGap = mainHb > 0 ? now - mainHb : 0;
   const readerGap = readerHb > 0 ? now - readerHb : 0;
@@ -167,12 +223,14 @@ function check(): void {
     now,
     mainOp,
     `gap=${mainGap}ms`,
+    counters.crawled,
     (t) =>
       `${isoNow()} [STALL:MAIN start ${t.detail}] op="${t.startOp || '<unknown>'}" ` +
       `crawled=${counters.crawled} discovered=${counters.discovered} ` +
       `pending=${counters.pending} failed=${counters.failed}`,
     (t, dur) =>
-      `${isoNow()} [STALL:MAIN end after ${dur}ms] startOp="${t.startOp}" endOp="${mainOp}"`,
+      `${isoNow()} [STALL:MAIN end after ${dur}ms] startOp="${t.startOp}" endOp="${mainOp}" ` +
+      `crawled_delta=${crawledDelta(t)} writer_op="${writerOp}" rss=${host.rssMb}MB heap=${host.heapMb}MB`,
   );
 
   // ── DB reader pool stall ──
@@ -184,6 +242,7 @@ function check(): void {
     now,
     readerOp,
     `gap=${readerGap}ms`,
+    counters.crawled,
     (t) => `${isoNow()} [STALL:READER start ${t.detail}] op="${t.startOp || '<unknown>'}"`,
     (t, dur) =>
       `${isoNow()} [STALL:READER end after ${dur}ms] startOp="${t.startOp}" endOp="${readerOp}"`,
@@ -200,11 +259,13 @@ function check(): void {
     now,
     writerOp,
     `gap=${writerGap}ms`,
+    counters.crawled,
     (t) =>
       `${isoNow()} [STALL:WRITER start ${t.detail}] op="${t.startOp || '<unknown>'}" ` +
       `crawled=${counters.crawled} pending=${counters.pending}`,
     (t, dur) =>
-      `${isoNow()} [STALL:WRITER end after ${dur}ms] startOp="${t.startOp}" endOp="${writerOp}"`,
+      `${isoNow()} [STALL:WRITER end after ${dur}ms] startOp="${t.startOp}" endOp="${writerOp}" ` +
+      `crawled_delta=${crawledDelta(t)} writer_q=${host.writerPending} main_op="${mainOp}"`,
   );
 
   // ── Renderer stall ──
@@ -227,6 +288,7 @@ function check(): void {
     now,
     mainOp,
     rendererReason,
+    counters.crawled,
     (t) =>
       `${isoNow()} [STALL:RENDERER start ${t.detail}] last_main_op="${t.startOp || '<unknown>'}" ` +
       `crawled=${counters.crawled} pending=${counters.pending}`,
@@ -244,14 +306,20 @@ function check(): void {
         `renderer_lag>${STALL_THRESHOLD_RENDERER_LAG_MS}ms renderer_silence>${STALL_THRESHOLD_RENDERER_SILENCE_MS}ms ` +
         `report_floor=${STALL_REPORT_FLOOR_MS}ms`,
     );
+    if (init.envLine) appendLine(`${isoNow()} [ENV] ${sanitizeNote(init.envLine)}`);
+    if (prevSessionLine) appendLine(`${isoNow()} ${prevSessionLine}`);
   }
+  const phaseLine = phases.observe(mainOp, now);
+  if (phaseLine) appendLine(`${isoNow()} ${phaseLine}`);
   if (now - lastHeartbeatLogTs >= HEARTBEAT_LOG_INTERVAL_MS) {
     lastHeartbeatLogTs = now;
     appendLine(
       `${isoNow()} [HEARTBEAT] main_op="${mainOp}" reader_op="${readerOp}" ` +
         `writer_op="${writerOp}" ` +
         `crawled=${counters.crawled} discovered=${counters.discovered} ` +
-        `pending=${counters.pending} renderer_lag=${rendererLag}ms`,
+        `pending=${counters.pending} failed=${counters.failed} renderer_lag=${rendererLag}ms ` +
+        `rss=${host.rssMb}MB heap=${host.heapMb}MB loop_lag=${host.mainLoopLagMs}ms ` +
+        `writer_q=${host.writerPending} writer_oldest=${host.writerOldestMs}ms`,
     );
   }
 }
@@ -261,7 +329,14 @@ setInterval(check, CHECK_INTERVAL_MS);
 // Accept a graceful-shutdown ping from the parent. We don't strictly
 // need it (the worker exits with the parent), but logging the close
 // lets the user see "watchdog stopped" instead of a silent truncation.
-parentPort.on('message', (msg: { type?: string }) => {
+parentPort.on('message', (msg: { type?: string; line?: unknown }) => {
+  // Main-process annotations — crawl start/stop, storage mode, mirrored
+  // error lines. They queue behind a frozen main thread and land here in
+  // order once it resumes, timestamped at write so the delay is visible.
+  if (msg?.type === 'note' && typeof msg.line === 'string') {
+    appendLine(`${isoNow()} ${sanitizeNote(msg.line)}`);
+    return;
+  }
   if (msg?.type === 'shutdown') {
     appendLine(`${isoNow()} [SHUTDOWN] watchdog received shutdown signal`);
     process.exit(0);

apps/desktop/src/main/freeze-watchdog.ts

@@ -1,10 +1,13 @@
 import { Worker } from 'node:worker_threads';
 import path from 'node:path';
+import v8 from 'node:v8';
 import { fileURLToPath } from 'node:url';
 import {
   FreezeWatchdogSharedState,
   type CounterPatch,
+  type HostStats,
 } from './freeze-watchdog-shared.js';
+import { toMb } from './freeze-watchdog-report.js';
 import * as logger from './logger.js';
 
 /**
@@ -29,18 +32,24 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const WORKER_PATH = path.join(__dirname, 'freeze-watchdog-worker.js');
 
 const MAIN_HEARTBEAT_INTERVAL_MS = 100;
+/** Sample memory + writer queue every N heartbeat ticks (1 s). */
+const HOST_STATS_EVERY_TICKS = 10;
+/** Reset the loop-lag high-water mark every N ticks (5 s) — one
+ *  heartbeat log line's worth, so a spike is not reported forever. */
+const LOOP_LAG_WINDOW_TICKS = 50;
 
 class FreezeWatchdog {
   private worker: Worker | null = null;
   private state: FreezeWatchdogSharedState | null = null;
   private heartbeatTimer: NodeJS.Timeout | null = null;
   private terminated = false;
+  private statsProvider: (() => Partial<HostStats>) | null = null;
 
   /**
    * Spawn the watchdog worker, allocate the SharedArrayBuffer, and
    * start the main-thread heartbeat ticker. Idempotent.
    */
-  init(debugFilePath: string): void {
+  init(debugFilePath: string, envLine = ''): void {
     if (this.terminated) return;
     if (this.worker) return;
     let state: FreezeWatchdogSharedState;
@@ -61,6 +70,7 @@ class FreezeWatchdog {
         workerData: {
           sab: state.sab,
           debugFilePath,
+          envLine,
         },
       });
     } catch (err) {
@@ -89,8 +99,34 @@ class FreezeWatchdog {
     });
     // Heartbeat tick — independent of the rest of the app's setIntervals
     // so a crashed timer elsewhere doesn't silently disable detection.
+    let expectedAt = Date.now() + MAIN_HEARTBEAT_INTERVAL_MS;
+    let maxLag = 0;
+    let tick = 0;
     this.heartbeatTimer = setInterval(() => {
+      const now = Date.now();
+      // How late this tick fired is how long the loop was busy. The
+      // watchdog's own gap detector only speaks up past 500 ms; this
+      // catches the 80–400 ms stutters that make the UI feel sticky
+      // without ever tripping it.
+      maxLag = Math.max(maxLag, now - expectedAt);
+      expectedAt = now + MAIN_HEARTBEAT_INTERVAL_MS;
       this.state?.tickMainHeartbeat();
+      tick++;
+      if (tick % HOST_STATS_EVERY_TICKS === 0) {
+        let extra: Partial<HostStats> = {};
+        try {
+          extra = this.statsProvider?.() ?? {};
+        } catch {
+          /* a stats provider must never break the heartbeat */
+        }
+        this.state?.updateHostStats({
+          rssMb: toMb(process.memoryUsage.rss()),
+          heapMb: toMb(v8.getHeapStatistics().used_heap_size),
+          mainLoopLagMs: Math.max(0, maxLag),
+          ...extra,
+        });
+      }
+      if (tick % LOOP_LAG_WINDOW_TICKS === 0) maxLag = 0;
     }, MAIN_HEARTBEAT_INTERVAL_MS);
     logger.log('info', 'main', `freeze-watchdog started → ${debugFilePath}`);
   }
@@ -112,6 +148,27 @@ class FreezeWatchdog {
     this.state?.updateCounters(c);
   }
 
+  /** Source of the per-second writer-queue figures in the heartbeat. */
+  setStatsProvider(fn: (() => Partial<HostStats>) | null): void {
+    this.statsProvider = fn;
+  }
+
+  /**
+   * Append one annotation line to debug.txt — `[CRAWL] start …`,
+   * `[STORAGE] mode=ram`, a mirrored `[ERROR] …`. Goes through the
+   * worker so the file has a single writer and the note lands in
+   * sequence with the heartbeats around it. Dropped silently when the
+   * watchdog never started; diagnostics must not become a failure mode.
+   */
+  note(line: string): void {
+    if (!this.worker) return;
+    try {
+      this.worker.postMessage({ type: 'note', line });
+    } catch {
+      /* worker gone — nothing to record into */
+    }
+  }
+
   /** Graceful shutdown — flushes the heartbeat timer and signals the
    * worker to exit cleanly so it can write a `[SHUTDOWN]` line. */
   async terminate(): Promise<void> {

apps/desktop/src/main/index.ts

@@ -38,7 +38,7 @@ import {
 } from 'node:fs';
 import { copyFile } from 'node:fs/promises';
 import { createHash, randomBytes } from 'node:crypto';
-import { totalmem, freemem, cpus } from 'node:os';
+import { totalmem, freemem, cpus, release } from 'node:os';
 // Lives under src/main/assets (not resources/) on purpose: electron-vite
 // treats resources/ as the public dir and references it in place, which the
 // packaged app doesn't ship — assets/ files get copied into out/main/chunks.
@@ -200,6 +200,10 @@ import {
   type LogImportSummary,
   type LogExportInput,
   type LogExportResult,
+  type LogThreatsInput,
+  type LogThreatsResult,
+  type LogThreatSummary,
+  type LogThreatIpRow,
   type LogEntry,
   type BrowserInstallState,
   type McpStartCrawlInput,
@@ -246,6 +250,7 @@ import {
   exportUrlsToJson,
   exportUrlsToXml,
   exportTabular,
+  type DatasetExportContext,
   exportGrid,
   exportSitemap,
   exportHtmlReport,
@@ -598,7 +603,11 @@ class ProjectSession {
         'main',
         'Storage mode: RAM-only — project held in an in-memory database; worker pools disabled. Save As writes it to disk.',
       );
+      freezeWatchdog.note(
+        '[STORAGE] mode=ram (no worker pools — DB reads, writes and issue passes run on the main thread)',
+      );
     } else {
+      freezeWatchdog.note(`[STORAGE] mode=disk file=${path.split(/[\\/]/).pop() ?? path}`);
       try {
         this.readerPool.init(path, freezeWatchdog.sharedBuffer);
       } catch (err) {
@@ -4135,6 +4144,19 @@ function openLogAnalyzerWindow(): void {
   logger.log('info', 'main', 'Log Analyzer window opened');
 }
 
+/** Every flagged request, oldest first, paged under the query's 1000-row
+ *  clamp — the export wants the whole table, not one page of it. */
+function allLogThreats(db: ProjectDb): LogThreatsResult['rows'] {
+  const PAGE = 1000;
+  const rows: LogThreatsResult['rows'] = [];
+  for (let offset = 0; ; offset += PAGE) {
+    const page = db.queryLogThreats({ limit: PAGE, offset, sortBy: 'ts' });
+    rows.push(...page.rows);
+    if (page.rows.length < PAGE || rows.length >= page.total) break;
+  }
+  return rows.reverse();
+}
+
 /**
  * V2 Faz 2 — flatten the core analyzer's Map aggregates into the plain
  * arrays the DB ingest expects, then persist. Returns the import summary.
@@ -4199,6 +4221,8 @@ function ingestLogResult(filePath: string, result: LogAnalysisResult): LogImport
     urlBots: urlAggregates.flatMap((u) =>
       Array.from(u.botCounts.entries()).map(([bot, hits]) => ({ path: u.path, bot, hits })),
     ),
+    threats: result.threats,
+    threatsDropped: result.threatsDropped,
   };
   return getDb().ingestLogAnalysis(payload);
 }
@@ -4434,6 +4458,21 @@ function registerIpc(): void {
     if (typeof ownerId !== 'number') return all;
     return all.filter((e) => e.windowId === undefined || e.windowId === ownerId);
   });
+  registerHandle(IPC.clipboardWriteText, (_e, text: unknown): boolean => {
+    if (typeof text !== 'string') return false;
+    try {
+      clipboard.writeText(text);
+      return true;
+    } catch (err) {
+      logger.log(
+        'warn',
+        'main',
+        `clipboard write failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      return false;
+    }
+  });
+
   registerHandle(IPC.logsClear, () => {
     logger.clearAll();
     logger.log('info', 'main', 'Log buffer cleared');
@@ -4496,7 +4535,7 @@ function registerIpc(): void {
           logger.log(
             'info',
             'main',
-            `Log Analyzer ingested ${basename(filePath)} (${result.format}): ${result.parsedLines}/${result.totalLines} lines, ${result.bots.size} bot type(s).`,
+            `Log Analyzer ingested ${basename(filePath)} (${result.format}): ${result.parsedLines}/${result.totalLines} lines, ${result.bots.size} bot type(s), ${result.threats.length} suspicious request(s)${result.threatsDropped > 0 ? ` (+${result.threatsDropped} beyond the cap)` : ''}.`,
           );
         } catch (err) {
           const error = err instanceof Error ? err.message : String(err);
@@ -4586,6 +4625,8 @@ function registerIpc(): void {
         trend: db.getLogTrend(),
         crawlBudget: db.getLogCrawlBudget(5000),
         discovery: db.getLogDiscovery(50_000),
+        threats: allLogThreats(db),
+        threatIps: db.getLogThreatIps(100_000),
       };
       const { writeFileSync } = await import('node:fs');
       if (format === 'csv') {
@@ -4603,6 +4644,15 @@ function registerIpc(): void {
     fireDataChanged();
     logger.log('info', 'main', 'Log Analyzer data cleared');
   });
+  registerHandle(
+    IPC.logThreats,
+    (_e, input: LogThreatsInput): LogThreatsResult => getDb().queryLogThreats(input),
+  );
+  registerHandle(IPC.logThreatSummary, (): LogThreatSummary => getDb().getLogThreatSummary());
+  registerHandle(
+    IPC.logThreatIps,
+    (_e, limit?: number): LogThreatIpRow[] => getDb().getLogThreatIps(limit ?? 25),
+  );
 
   registerHandle(
     IPC.screenshotRead,
@@ -6779,6 +6829,10 @@ function registerIpc(): void {
         'crawler',
         `Crawl done: total=${summary.total} avgResp=${summary.avgResponseTimeMs}ms totalBytes=${summary.totalBytes}`,
       );
+      freezeWatchdog.note(
+        `[CRAWL] done total=${summary.total} failed=${sess.latestProgress?.failed ?? '?'} ` +
+          `avgResp=${summary.avgResponseTimeMs}ms elapsed=${Math.round((sess.latestProgress?.elapsedMs ?? 0) / 1000)}s`,
+      );
       sess.send(IPC.crawlDone, summary);
       sess.activeCrawler = null;
       // Free the MCP crawl lease so a queued agent start_crawl can proceed.
@@ -6827,6 +6881,10 @@ function registerIpc(): void {
     // and skips, leaving the NEW crawl's freshly-granted lease intact.
     crawler.on('stopped', () => {
       if (sess.activeCrawler !== crawler) return;
+      freezeWatchdog.note(
+        `[CRAWL] stopped crawled=${sess.latestProgress?.crawled ?? '?'} ` +
+          `pending=${sess.latestProgress?.pending ?? '?'} failed=${sess.latestProgress?.failed ?? '?'}`,
+      );
       releaseSessionLease(sess);
       rebalanceGlobalConcurrency();
     });
@@ -6879,6 +6937,16 @@ function registerIpc(): void {
       undefined,
       sess.logTag,
     );
+    freezeWatchdog.note(
+      `[CRAWL] start ${opts.respiderSeeds ? `respider=${opts.respiderSeeds.length} ` : ''}` +
+        `url=${config.startUrl} mode=${config.mode} scope=${config.scope} depth=${config.maxDepth} ` +
+        `maxUrls=${config.maxUrls} concurrency=${config.maxConcurrency} rps=${config.maxRps} ` +
+        `adaptiveSpeed=${config.adaptiveSpeed} ` +
+        `timeout=${config.requestTimeoutMs}ms render=${config.renderingMode} ` +
+        `robots=${config.respectRobotsTxt} crawlExternal=${config.crawlExternal} ` +
+        `storeLinks=${config.storeInternalLinks}/${config.storeExternalLinks} ` +
+        `imageProbe=${config.probeImageSizes}`,
+    );
     const database = sess.getDb();
     database.setMeta('lastStartUrl', config.startUrl);
     // Persist the config so a crash-then-resume can rehydrate it
@@ -7244,6 +7312,19 @@ function registerIpc(): void {
    * — written before multi-account existed — are relabelled to it, so an
    * upgraded project keeps the data it already pulled.
    */
+  /** Account + URL-matching context the dataset exports (Search Console,
+   *  GA4) read with — the same resolution the tabs' query handlers use,
+   *  so an exported sheet matches what the tab shows. */
+  function datasetExportContext(): DatasetExportContext {
+    const gsc = resolveGscSettings();
+    return {
+      gscAccountId: resolveGoogleAccount('gsc'),
+      ga4AccountId: resolveGoogleAccount('ga4'),
+      gscMatchSlash: gsc.matchSlash,
+      gscMatchCase: gsc.matchCase,
+    };
+  }
+
   function resolveGoogleAccount(
     integrationId: 'gsc' | 'ga4',
     requested?: string,
@@ -7373,6 +7454,7 @@ function registerIpc(): void {
   });
 
   registerHandle(IPC.crawlStop, () => {
+    freezeWatchdog.note('[CRAWL] stop requested by user');
     const sess = currentSession();
     if (sess.activeCrawler) {
       sess.activeCrawler.stop();
@@ -7393,10 +7475,12 @@ function registerIpc(): void {
   });
 
   registerHandle(IPC.crawlPause, () => {
+    freezeWatchdog.note('[CRAWL] pause requested by user');
     currentSession().activeCrawler?.pause();
   });
 
   registerHandle(IPC.crawlResume, () => {
+    freezeWatchdog.note('[CRAWL] resume requested by user');
     currentSession().activeCrawler?.resume();
   });
 
@@ -7688,6 +7772,7 @@ function registerIpc(): void {
         selectedIds: i.selectedIds,
         search: i.search,
         filter: i.filter,
+        datasetContext: datasetExportContext(),
       });
     },
 
@@ -9343,6 +9428,7 @@ function registerIpc(): void {
         search,
         filter,
         csvBom,
+        datasetContext: datasetExportContext(),
       });
       return result;
     },
@@ -9800,6 +9886,21 @@ try {
   browserCacheDecision = `resolution failed: ${(err as Error).message} — using Playwright defaults`;
 }
 
+/**
+ * One line of host facts for the top of debug.txt. Half of "why is it
+ * slow for this user" is answered by RAM, core count and Electron
+ * version — none of which a remote user thinks to mention.
+ */
+function describeHostEnvironment(): string {
+  const v = process.versions;
+  return (
+    `app=${app.getVersion()} electron=${v.electron ?? '?'} node=${v.node} ` +
+    `chrome=${v.chrome ?? '?'} os=${process.platform} ${release()} ${process.arch} ` +
+    `cpus=${cpus().length} mem=${Math.round(totalmem() / 1073741824)}GB ` +
+    `locale=${app.getLocale()} packaged=${app.isPackaged}`
+  );
+}
+
 void app.whenReady().then(async () => {
   // Disk logging is bootstrapped after `app.whenReady()` — `getPath('userData')`
   // is only valid post-ready. Prior log lines are still in the ring buffer
@@ -9817,7 +9918,15 @@ void app.whenReady().then(async () => {
   // noise from normal app events).
   try {
     const debugPath = join(app.getPath('userData'), 'debug.txt');
-    freezeWatchdog.init(debugPath);
+    freezeWatchdog.init(debugPath, describeHostEnvironment());
+    // Errors and non-crawler warnings get a copy in debug.txt, so the one
+    // file a user sends carries the stall AND the `database is locked` or
+    // worker exit that sat next to it in the main log.
+    logger.setDebugMirror((line) => freezeWatchdog.note(line));
+    freezeWatchdog.setStatsProvider(() => {
+      const s = dbWriterPool.pendingStats();
+      return { writerPending: s.count, writerOldestMs: s.oldestMs };
+    });
   } catch (err) {
     logger.log('warn', 'main', `freeze-watchdog init failed: ${(err as Error).message}`);
   }

apps/desktop/src/preload/index.ts

@@ -165,6 +165,10 @@ import {
   type LogSeedDiscoveryResult,
   type LogExportInput,
   type LogExportResult,
+  type LogThreatsInput,
+  type LogThreatsResult,
+  type LogThreatSummary,
+  type LogThreatIpRow,
 } from '@freecrawl/shared-types';
 
 function subscribe<T>(
@@ -308,6 +312,8 @@ const api: FreeCrawlApi = {
   logsGetAll: (ownerId?: number): Promise<LogEntry[]> =>
     ipcRenderer.invoke(IPC.logsGetAll, ownerId),
   logsClear: (): Promise<void> => ipcRenderer.invoke(IPC.logsClear),
+  clipboardWriteText: (text: string): Promise<boolean> =>
+    ipcRenderer.invoke(IPC.clipboardWriteText, text),
   logsOpenWindow: (): Promise<void> => ipcRenderer.invoke(IPC.logsOpenWindow),
   openVisualizationWindow: (): Promise<void> =>
     ipcRenderer.invoke(IPC.visualizationOpenWindow),
@@ -507,6 +513,11 @@ const api: FreeCrawlApi = {
   logExport: (input: LogExportInput): Promise<LogExportResult> =>
     ipcRenderer.invoke(IPC.logExport, input),
   logClear: (): Promise<void> => ipcRenderer.invoke(IPC.logClear),
+  logThreats: (input: LogThreatsInput): Promise<LogThreatsResult> =>
+    ipcRenderer.invoke(IPC.logThreats, input),
+  logThreatSummary: (): Promise<LogThreatSummary> => ipcRenderer.invoke(IPC.logThreatSummary),
+  logThreatIps: (limit?: number): Promise<LogThreatIpRow[]> =>
+    ipcRenderer.invoke(IPC.logThreatIps, limit),
   onLogEntry: (cb) => subscribe<LogEntry>(IPC.logsEntry, cb),
   onLogsBatch: (cb) => subscribe<LogEntry[]>(IPC.logsBatch, cb),
   onLogsBusy: (cb) => subscribe<boolean>(IPC.logsBusy, cb),

apps/desktop/src/renderer/src/LogAnalyzerView.tsx

@@ -8,10 +8,15 @@ import type {
   LogFormatChoice,
   LogOverview,
   LogStatusRow,
+  LogThreatCategory,
+  LogThreatRow,
+  LogThreatsInput,
+  LogThreatSummary,
   LogTrendRow,
   LogUrlStatRow,
   LogUrlStatsInput,
 } from '@freecrawl/shared-types';
+import { writeTextToClipboard } from './utils/clipboard.js';
 
 /**
  * V2 Faz 2 — standalone Log File Analyzer window (renderer `?loganalyzer=1`).
@@ -20,7 +25,7 @@ import type {
  * orphan detection, and seeding log-discovered URLs into the active crawl.
  */
 
-type Tab = 'urls' | 'bots' | 'status' | 'trend' | 'budget' | 'orphans' | 'discovery';
+type Tab = 'urls' | 'bots' | 'status' | 'trend' | 'budget' | 'orphans' | 'discovery' | 'threats';
 
 const FAMILY_COLORS: Record<string, string> = {
   googlebot: 'text-blue-300 bg-blue-500/15',
@@ -254,6 +259,7 @@ export function LogAnalyzerView() {
             {tab === 'trend' && <TrendTab version={version} />}
             {tab === 'budget' && <BudgetTab version={version} />}
             {tab === 'discovery' && <DiscoveryTab version={version} onToast={showToast} />}
+            {tab === 'threats' && <ThreatsTab version={version} onToast={showToast} />}
           </div>
         </>
       )}
@@ -332,6 +338,13 @@ function SummaryCards({ overview }: { overview: LogOverview }) {
     { label: t('logAnalyzer.cardHumanHits', { defaultValue: 'Human Hits' }), value: fmtNum(overview.humanHits) },
     { label: t('logAnalyzer.cardVerified', { defaultValue: 'Verified Bot Hits' }), value: fmtNum(overview.verifiedBotHits) },
     { label: t('logAnalyzer.cardUrls', { defaultValue: 'Distinct URLs' }), value: fmtNum(overview.distinctUrls) },
+    {
+      label: t('logAnalyzer.cardThreats', { defaultValue: 'Suspicious Requests' }),
+      value:
+        overview.threatHits > 0
+          ? `${fmtNum(overview.threatHits)} · ${fmtNum(overview.threatIps)} IP`
+          : '0',
+    },
     {
       label: t('logAnalyzer.cardRange', { defaultValue: 'Date Range' }),
       value: overview.minTs ? `${fmtTs(overview.minTs)} → ${fmtTs(overview.maxTs)}` : '—',
@@ -367,6 +380,7 @@ function TabStrip({ tab, setTab }: { tab: Tab; setTab: (t: Tab) => void }) {
     { id: 'budget', label: t('logAnalyzer.tabBudget', { defaultValue: 'Crawl Budget' }) },
     { id: 'orphans', label: t('logAnalyzer.tabOrphans', { defaultValue: 'Orphans' }) },
     { id: 'discovery', label: t('logAnalyzer.tabDiscovery', { defaultValue: 'Discovery' }) },
+    { id: 'threats', label: t('logAnalyzer.tabThreats', { defaultValue: 'Suspicious Requests' }) },
   ];
   return (
     <div className="flex shrink-0 gap-1 border-b border-surface-800 bg-surface-900/60 px-2 text-[12px]">
@@ -794,6 +808,465 @@ function DiscoveryTab({ version, onToast }: { version: number; onToast: (m: stri
   );
 }
 
+const THREAT_CATEGORIES: LogThreatCategory[] = [
+  'sqli',
+  'xss',
+  'traversal',
+  'cmdi',
+  'scanner',
+  'sensitive-file',
+  'anomaly',
+];
+
+const THREAT_COLORS: Record<LogThreatCategory, string> = {
+  sqli: 'text-red-300 bg-red-500/15',
+  xss: 'text-orange-300 bg-orange-500/15',
+  traversal: 'text-amber-300 bg-amber-500/15',
+  cmdi: 'text-rose-300 bg-rose-500/15',
+  scanner: 'text-purple-300 bg-purple-500/15',
+  'sensitive-file': 'text-pink-300 bg-pink-500/15',
+  anomaly: 'text-surface-300 bg-surface-700/40',
+};
+
+function useThreatCategoryLabel(): (cat: LogThreatCategory) => string {
+  const { t } = useTranslation();
+  return useCallback(
+    (cat: LogThreatCategory): string => {
+      switch (cat) {
+        case 'sqli':
+          return t('logAnalyzer.threatCatSqli', { defaultValue: 'SQL injection' });
+        case 'xss':
+          return t('logAnalyzer.threatCatXss', { defaultValue: 'XSS' });
+        case 'traversal':
+          return t('logAnalyzer.threatCatTraversal', { defaultValue: 'Path traversal' });
+        case 'cmdi':
+          return t('logAnalyzer.threatCatCmdi', { defaultValue: 'Command injection' });
+        case 'scanner':
+          return t('logAnalyzer.threatCatScanner', { defaultValue: 'Scanner probe' });
+        case 'sensitive-file':
+          return t('logAnalyzer.threatCatSensitiveFile', { defaultValue: 'Sensitive file' });
+        default:
+          return t('logAnalyzer.threatCatAnomaly', { defaultValue: 'Anomaly' });
+      }
+    },
+    [t],
+  );
+}
+
+function ThreatBadge({ category, label }: { category: LogThreatCategory; label: string }) {
+  return (
+    <span className={`whitespace-nowrap rounded px-1.5 py-0.5 text-[10px] ${THREAT_COLORS[category]}`}>
+      {label}
+    </span>
+  );
+}
+
+function fmtTsFull(ts: number | null): string {
+  if (ts === null) return '—';
+  try {
+    return new Date(ts).toISOString().replace('T', ' ').slice(0, 19);
+  } catch {
+    return '—';
+  }
+}
+
+/** Decoded payload with the rule evidence wrapped in <mark>. The evidence
+ *  was matched on a `+`→space normalised copy, so fall back to that. */
+function HighlightedPayload({ text, evidence }: { text: string; evidence: string }) {
+  const needle = evidence.replace(/…$/, '');
+  let hay = text;
+  let idx = needle ? hay.toLowerCase().indexOf(needle.toLowerCase()) : -1;
+  if (idx < 0 && needle) {
+    hay = text.replace(/\+/g, ' ');
+    idx = hay.toLowerCase().indexOf(needle.toLowerCase());
+  }
+  if (idx < 0) return <>{text}</>;
+  return (
+    <>
+      {hay.slice(0, idx)}
+      <mark className="rounded bg-red-500/30 px-0.5 text-red-100">{hay.slice(idx, idx + needle.length)}</mark>
+      {hay.slice(idx + needle.length)}
+    </>
+  );
+}
+
+type BlocklistFormat = 'plain' | 'nginx' | 'apache';
+
+function renderBlocklist(ips: string[], format: BlocklistFormat): string {
+  if (format === 'nginx') return ips.map((ip) => `deny ${ip};`).join('\n') + '\n';
+  if (format === 'apache') {
+    return ['<RequireAll>', '  Require all granted', ...ips.map((ip) => `  Require not ip ${ip}`), '</RequireAll>'].join('\n') + '\n';
+  }
+  return ips.join('\n') + '\n';
+}
+
+function BlocklistMenu({ onCopy }: { onCopy: (format: BlocklistFormat) => void }) {
+  const { t } = useTranslation();
+  const [open, setOpen] = useState(false);
+  const choose = (format: BlocklistFormat): void => {
+    setOpen(false);
+    onCopy(format);
+  };
+  return (
+    <div className="relative">
+      <button
+        onClick={() => setOpen((o) => !o)}
+        className="flex items-center gap-1 rounded border border-surface-700 px-2 py-1 text-surface-200 hover:bg-surface-800"
+      >
+        {t('logAnalyzer.threatCopyMenu', { defaultValue: 'Copy blocklist' })}
+        <span className="text-[9px] text-surface-400">▼</span>
+      </button>
+      {open && (
+        <>
+          <div className="fixed inset-0 z-40" onClick={() => setOpen(false)} />
+          <div className="absolute right-0 z-50 mt-1 w-52 overflow-hidden rounded border border-surface-700 bg-surface-800 shadow-lg">
+            <button onClick={() => choose('plain')} className="block w-full px-3 py-1.5 text-left text-surface-200 hover:bg-surface-700">
+              {t('logAnalyzer.threatCopyPlain', { defaultValue: 'Plain IP list' })}
+            </button>
+            <button onClick={() => choose('nginx')} className="block w-full px-3 py-1.5 text-left text-surface-200 hover:bg-surface-700">
+              {t('logAnalyzer.threatCopyNginx', { defaultValue: 'nginx deny rules' })}
+            </button>
+            <button onClick={() => choose('apache')} className="block w-full px-3 py-1.5 text-left text-surface-200 hover:bg-surface-700">
+              {t('logAnalyzer.threatCopyApache', { defaultValue: 'Apache Require not ip' })}
+            </button>
+          </div>
+        </>
+      )}
+    </div>
+  );
+}
+
+function ThreatsTab({ version, onToast }: { version: number; onToast: (m: string) => void }) {
+  const { t } = useTranslation();
+  const catLabel = useThreatCategoryLabel();
+  const [summary, setSummary] = useState<LogThreatSummary | null>(null);
+  const [rows, setRows] = useState<LogThreatRow[]>([]);
+  const [total, setTotal] = useState(0);
+  const [offset, setOffset] = useState(0);
+  const [search, setSearch] = useState('');
+  const [category, setCategory] = useState<NonNullable<LogThreatsInput['category']>>('all');
+  const [status, setStatus] = useState<NonNullable<LogThreatsInput['status']>>('all');
+  const [ip, setIp] = useState('');
+  const [sortBy, setSortBy] = useState<NonNullable<LogThreatsInput['sortBy']>>('ts');
+  const [selected, setSelected] = useState<LogThreatRow | null>(null);
+
+  useEffect(() => {
+    void window.freecrawl.logThreatSummary().then(setSummary);
+  }, [version]);
+
+  useEffect(() => {
+    setOffset(0);
+    setSelected(null);
+  }, [search, category, status, ip, sortBy, version]);
+
+  useEffect(() => {
+    let cancelled = false;
+    const input: LogThreatsInput = {
+      limit: PAGE,
+      offset,
+      search,
+      category,
+      status,
+      ip: ip || undefined,
+      sortBy,
+    };
+    void window.freecrawl.logThreats(input).then((r) => {
+      if (cancelled) return;
+      setRows(r.rows);
+      setTotal(r.total);
+    });
+    return () => {
+      cancelled = true;
+    };
+  }, [offset, search, category, status, ip, sortBy, version]);
+
+  const copyBlocklist = useCallback(
+    async (format: BlocklistFormat) => {
+      const senders = await window.freecrawl.logThreatIps(100_000);
+      // Search-engine infrastructure is never a blocklist candidate.
+      const ips = senders.filter((s) => !s.ipOwner).map((s) => s.ip);
+      if (ips.length === 0) {
+        onToast(t('logAnalyzer.threatCopyEmpty', { defaultValue: 'No IPs to copy' }));
+        return;
+      }
+      const ok = await writeTextToClipboard(renderBlocklist(ips, format));
+      onToast(
+        ok
+          ? t('logAnalyzer.threatCopied', {
+              defaultValue: 'Copied {{n}} IP(s) to the clipboard (search-engine infrastructure excluded)',
+              n: ips.length,
+            })
+          : t('logAnalyzer.threatCopyFailed', { defaultValue: 'Could not copy to the clipboard' }),
+      );
+    },
+    [onToast, t],
+  );
+
+  const copyIp = useCallback(
+    async (value: string) => {
+      const ok = await writeTextToClipboard(value);
+      if (!ok) onToast(t('logAnalyzer.threatCopyFailed', { defaultValue: 'Could not copy to the clipboard' }));
+    },
+    [onToast, t],
+  );
+
+  const hasThreats = (summary?.totalHits ?? 0) > 0;
+
+  return (
+    <div className="flex h-full flex-col">
+      {/* Summary strip */}
+      <div className="shrink-0 border-b border-surface-800 bg-surface-900/40 px-3 py-2 text-[11px]">
+        <div className="flex flex-wrap items-center gap-2">
+          <span className="text-surface-400">
+            {t('logAnalyzer.threatIntro', {
+              defaultValue:
+                'Requests that look like attacks or vulnerability probes — SQL injection, XSS, path traversal, command injection, scanner paths, sensitive-file fetches — with the sender IP and the decoded payload.',
+            })}
+          </span>
+          {hasThreats && (
+            <div className="ml-auto">
+              <BlocklistMenu onCopy={(f) => void copyBlocklist(f)} />
+            </div>
+          )}
+        </div>
+        {summary && hasThreats && (
+          <div className="mt-2 flex flex-wrap items-center gap-2">
+            <span className="font-mono text-surface-200">
+              {t('logAnalyzer.threatStats', {
+                defaultValue: '{{hits}} flagged request(s) from {{ips}} IP(s)',
+                hits: summary.totalHits.toLocaleString(),
+                ips: summary.distinctIps.toLocaleString(),
+              })}
+            </span>
+            <span className="text-surface-600">·</span>
+            {summary.servedHits > 0 ? (
+              <span className="text-amber-300">
+                {t('logAnalyzer.threatServed', {
+                  defaultValue:
+                    '{{n}} answered 2xx — the server processed those payloads; check the application, not just the log',
+                  n: summary.servedHits.toLocaleString(),
+                })}
+              </span>
+            ) : (
+              <span className="text-green-400">
+                {t('logAnalyzer.threatServedNone', { defaultValue: 'none answered 2xx' })}
+              </span>
+            )}
+            {summary.capHit && (
+              <span className="text-amber-400">
+                {t('logAnalyzer.threatCapHit', { defaultValue: 'The per-file cap was reached — this list is partial.' })}
+              </span>
+            )}
+          </div>
+        )}
+        {summary && hasThreats && (
+          <div className="mt-2 flex flex-wrap items-center gap-1.5">
+            <button
+              onClick={() => setCategory('all')}
+              className={`rounded border px-2 py-0.5 ${category === 'all' ? 'border-blue-500 text-surface-100' : 'border-surface-700 text-surface-400 hover:text-surface-200'}`}
+            >
+              {t('logAnalyzer.threatCatAll', { defaultValue: 'All categories' })} ({summary.totalHits.toLocaleString()})
+            </button>
+            {summary.byCategory.map((c) => (
+              <button
+                key={c.category}
+                onClick={() => setCategory(category === c.category ? 'all' : c.category)}
+                className={`rounded border px-2 py-0.5 ${category === c.category ? 'border-blue-500' : 'border-surface-800 hover:border-surface-600'}`}
+                title={`${c.ips.toLocaleString()} IP`}
+              >
+                <ThreatBadge category={c.category} label={`${catLabel(c.category)} ${c.hits.toLocaleString()}`} />
+              </button>
+            ))}
+          </div>
+        )}
+        {summary && summary.topIps.length > 0 && (
+          <div className="mt-2 flex flex-wrap items-center gap-1.5">
+            <span className="text-surface-500">{t('logAnalyzer.threatTopIps', { defaultValue: 'Top senders' })}:</span>
+            {summary.topIps.slice(0, 10).map((s) => (
+              <button
+                key={s.ip}
+                onClick={() => setIp(ip === s.ip ? '' : s.ip)}
+                className={`rounded border px-1.5 py-0.5 font-mono ${ip === s.ip ? 'border-blue-500 text-surface-100' : 'border-surface-800 text-surface-300 hover:border-surface-600'}`}
+                title={s.categories.map(catLabel).join(', ')}
+              >
+                {s.ip} <span className="text-surface-500">×{s.hits.toLocaleString()}</span>
+                {s.servedHits > 0 && <span className="ml-1 text-amber-400">{s.servedHits}·2xx</span>}
+                {s.ipOwner && <span className="ml-1 text-blue-300">{s.ipOwner}</span>}
+              </button>
+            ))}
+          </div>
+        )}
+      </div>
+
+      {/* Filter bar */}
+      <div className="flex shrink-0 flex-wrap items-center gap-2 border-b border-surface-800 px-3 py-1.5 text-[11px]">
+        <input
+          value={search}
+          onChange={(e) => setSearch(e.target.value)}
+          placeholder={t('logAnalyzer.threatSearch', { defaultValue: 'Filter by payload, IP or user-agent…' })}
+          className="w-64 rounded border border-surface-700 bg-surface-800 px-2 py-1 text-surface-100"
+        />
+        <select
+          value={category}
+          onChange={(e) => setCategory(e.target.value as NonNullable<LogThreatsInput['category']>)}
+          className="rounded border border-surface-700 bg-surface-800 px-2 py-1 text-surface-100"
+        >
+          <option value="all">{t('logAnalyzer.threatCatAll', { defaultValue: 'All categories' })}</option>
+          {THREAT_CATEGORIES.map((c) => (
+            <option key={c} value={c}>
+              {catLabel(c)}
+            </option>
+          ))}
+        </select>
+        <select
+          value={status}
+          onChange={(e) => setStatus(e.target.value as NonNullable<LogThreatsInput['status']>)}
+          className="rounded border border-surface-700 bg-surface-800 px-2 py-1 text-surface-100"
+        >
+          <option value="all">{t('logAnalyzer.threatStatusAll', { defaultValue: 'Any status' })}</option>
+          <option value="2xx">{t('logAnalyzer.threatStatus2xx', { defaultValue: '2xx (served)' })}</option>
+          <option value="3xx">3xx</option>
+          <option value="4xx">4xx</option>
+          <option value="5xx">5xx</option>
+        </select>
+        <select
+          value={sortBy}
+          onChange={(e) => setSortBy(e.target.value as NonNullable<LogThreatsInput['sortBy']>)}
+          className="rounded border border-surface-700 bg-surface-800 px-2 py-1 text-surface-100"
+        >
+          <option value="ts">{t('logAnalyzer.threatSortRecent', { defaultValue: 'Sort: Most recent' })}</option>
+          <option value="score">{t('logAnalyzer.threatSortScore', { defaultValue: 'Sort: Score' })}</option>
+          <option value="status">{t('logAnalyzer.threatSortStatus', { defaultValue: 'Sort: Status' })}</option>
+        </select>
+        {ip && (
+          <button
+            onClick={() => setIp('')}
+            className="rounded border border-blue-500/60 px-2 py-0.5 font-mono text-blue-200 hover:bg-surface-800"
+            title={t('logAnalyzer.threatClearIp', { defaultValue: 'Clear IP filter' })}
+          >
+            {t('logAnalyzer.threatIpFilter', { defaultValue: 'IP: {{ip}}', ip })} ✕
+          </button>
+        )}
+        <Pager offset={offset} total={total} onPrev={() => setOffset(Math.max(0, offset - PAGE))} onNext={() => setOffset(offset + PAGE)} />
+      </div>
+
+      {/* Table */}
+      <div className="min-h-0 flex-1 overflow-auto">
+        <table className="w-full border-collapse text-[11px]">
+          <thead>
+            <tr>
+              <Th>{t('logAnalyzer.colTime', { defaultValue: 'Time' })}</Th>
+              <Th>IP</Th>
+              <Th>{t('logAnalyzer.colMethod', { defaultValue: 'Method' })}</Th>
+              <Th>{t('logAnalyzer.colRequest', { defaultValue: 'Request (decoded)' })}</Th>
+              <Th className="text-right">{t('logAnalyzer.colStatus', { defaultValue: 'Status' })}</Th>
+              <Th>{t('logAnalyzer.colCategory', { defaultValue: 'Category' })}</Th>
+              <Th className="text-right">{t('logAnalyzer.colScore', { defaultValue: 'Score' })}</Th>
+              <Th>{t('logAnalyzer.colEvidence', { defaultValue: 'Evidence' })}</Th>
+              <Th>{t('logAnalyzer.colDeclaredBot', { defaultValue: 'Declared Bot' })}</Th>
+            </tr>
+          </thead>
+          <tbody>
+            {rows.map((r) => {
+              const served = r.status !== null && r.status >= 200 && r.status < 300;
+              return (
+                <tr
+                  key={r.id}
+                  onClick={() => setSelected(selected?.id === r.id ? null : r)}
+                  className={`cursor-pointer border-b border-surface-800/60 hover:bg-surface-900/40 ${selected?.id === r.id ? 'bg-surface-900/70' : ''}`}
+                >
+                  <td className="whitespace-nowrap px-2 py-1 font-mono text-surface-400">{fmtTsFull(r.ts)}</td>
+                  <td className="whitespace-nowrap px-2 py-1 font-mono text-surface-200">
+                    {r.ip ?? '—'}
+                    {r.ipOwner && <span className="ml-1 text-blue-300">{r.ipOwner}</span>}
+                  </td>
+                  <td className="px-2 py-1 font-mono text-surface-300">{r.method ?? '—'}</td>
+                  <td className="max-w-[420px] truncate px-2 py-1 font-mono text-surface-200" title={r.decoded}>
+                    {r.decoded}
+                  </td>
+                  <td className={`px-2 py-1 text-right font-mono ${served ? 'text-amber-300' : 'text-surface-300'}`}>
+                    {r.status ?? '—'}
+                    {served && (
+                      <span className="ml-1 text-[9px] uppercase text-amber-400">
+                        {t('logAnalyzer.threatServedBadge', { defaultValue: 'served' })}
+                      </span>
+                    )}
+                  </td>
+                  <td className="px-2 py-1"><ThreatBadge category={r.category} label={catLabel(r.category)} /></td>
+                  <td className="px-2 py-1 text-right font-mono">{r.score}</td>
+                  <td className="max-w-[240px] truncate px-2 py-1 font-mono text-red-200/80" title={r.evidence}>{r.evidence}</td>
+                  <td className="px-2 py-1 text-surface-400">{r.bot ?? '—'}</td>
+                </tr>
+              );
+            })}
+            {rows.length === 0 && (
+              <tr>
+                <td colSpan={9} className="px-3 py-6 text-center text-surface-500">
+                  {t('logAnalyzer.threatNone', { defaultValue: 'No suspicious requests in the ingested logs.' })}
+                </td>
+              </tr>
+            )}
+          </tbody>
+        </table>
+      </div>
+
+      {/* Detail panel */}
+      {selected && (
+        <div className="max-h-[40%] shrink-0 overflow-auto border-t border-surface-700 bg-surface-900/80 px-3 py-2 text-[11px]">
+          <div className="flex flex-wrap items-center gap-2">
+            <ThreatBadge category={selected.category} label={catLabel(selected.category)} />
+            <span className="font-mono text-surface-200">{selected.ip ?? '—'}</span>
+            <span className="text-surface-500">{fmtTsFull(selected.ts)}</span>
+            <span className="font-mono text-surface-400">{selected.method ?? '—'} · {selected.status ?? '—'}</span>
+            <span className={selected.targetInCrawl ? 'text-blue-300' : 'text-surface-500'}>
+              {selected.targetInCrawl
+                ? t('logAnalyzer.threatTargetInCrawl', { defaultValue: 'Target exists in the crawl' })
+                : t('logAnalyzer.threatTargetProbe', { defaultValue: 'Blind probe — target not in the crawl' })}
+            </span>
+            <div className="ml-auto flex items-center gap-1">
+              {selected.ip && (
+                <>
+                  <button onClick={() => setIp(selected.ip!)} className="rounded border border-surface-700 px-2 py-0.5 text-surface-300 hover:bg-surface-800">
+                    {t('logAnalyzer.threatFilterIp', { defaultValue: 'Only this IP' })}
+                  </button>
+                  <button onClick={() => void copyIp(selected.ip!)} className="rounded border border-surface-700 px-2 py-0.5 text-surface-300 hover:bg-surface-800">
+                    {t('logAnalyzer.threatCopyIp', { defaultValue: 'Copy IP' })}
+                  </button>
+                </>
+              )}
+              <button onClick={() => setSelected(null)} className="rounded border border-surface-700 px-2 py-0.5 text-surface-300 hover:bg-surface-800">
+                {t('logAnalyzer.threatClose', { defaultValue: 'Close' })}
+              </button>
+            </div>
+          </div>
+          {selected.ipOwner && (
+            <div className="mt-2 rounded border border-blue-500/40 bg-blue-500/10 px-2 py-1 text-blue-200">
+              {t('logAnalyzer.threatOwnerWarning', {
+                defaultValue:
+                  "This IP resolves to {{owner}}'s own infrastructure — the payload was relayed through the search engine (Lens, Translate, a fetch of a crafted URL). Blocking the IP would block {{owner}}; fix the application instead.",
+                owner: selected.ipOwner,
+              })}
+            </div>
+          )}
+          <pre className="mt-2 whitespace-pre-wrap break-all rounded bg-surface-950 px-2 py-1.5 font-mono text-surface-100">
+            <HighlightedPayload text={selected.decoded} evidence={selected.evidence} />
+          </pre>
+          <div className="mt-1.5 grid grid-cols-[auto_1fr] gap-x-3 gap-y-0.5 text-surface-400">
+            <span>{t('logAnalyzer.threatDetailRaw', { defaultValue: 'Raw request target' })}</span>
+            <code className="break-all text-surface-300">{selected.path}</code>
+            <span>{t('logAnalyzer.threatDetailRules', { defaultValue: 'Matched rules' })}</span>
+            <code className="text-surface-300">{selected.rules.join(', ')}</code>
+            <span>User-Agent</span>
+            <code className="break-all text-surface-300">{selected.userAgent ?? '—'}</code>
+            <span>Referer</span>
+            <code className="break-all text-surface-300">{selected.referer ?? '—'}</code>
+          </div>
+        </div>
+      )}
+    </div>
+  );
+}
+
 function EmptyRow({ cols }: { cols: number }) {
   const { t } = useTranslation();
   return (

apps/desktop/src/renderer/src/LogsView.tsx

@@ -11,6 +11,7 @@ import clsx from 'clsx';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
 import type { LogEntry, LogLevel } from '@freecrawl/shared-types';
+import { writeTextToClipboard } from './utils/clipboard.js';
 
 const LEVEL_ORDER: LogLevel[] = ['debug', 'info', 'warn', 'error'];
 
@@ -93,6 +94,7 @@ export function LogsView() {
   const [minLevel, setMinLevel] = useState<LogLevel>('info');
   const [filter, setFilter] = useState('');
   const [autoScroll, setAutoScroll] = useState(true);
+  const [copyState, setCopyState] = useState<'idle' | 'copied' | 'failed'>('idle');
 
   // Pending batches — main process already coalesces entries on a
   // 100 ms window, so the renderer's only job is to apply the batch.
@@ -216,11 +218,11 @@ export function LogsView() {
     const text = visible
       .map((e) => `${e.ts}  ${e.level.toUpperCase().padEnd(5)} [${e.source}] ${e.message}`)
       .join('\n');
-    try {
-      await navigator.clipboard.writeText(text);
-    } catch {
-      /* clipboard may be unavailable in locked-down renderer contexts */
-    }
+    // Confirm on the button itself. Copy is silent by nature, so without
+    // this the user cannot tell a working button from a broken one — which
+    // is exactly how the old swallowed failure went unnoticed.
+    setCopyState((await writeTextToClipboard(text)) ? 'copied' : 'failed');
+    window.setTimeout(() => setCopyState('idle'), 1600);
   }
 
   const totalHeight = rowVirtualizer.getTotalSize();
@@ -263,11 +265,23 @@ export function LogsView() {
           {t('logs.autoScroll', { defaultValue: 'Auto-scroll' })}
         </label>
         <button
-          className="rounded border border-surface-700 px-2 py-1 text-[11px] hover:bg-surface-800"
-          onClick={copyAll}
+          className={clsx(
+            'rounded border px-2 py-1 text-[11px]',
+            copyState === 'copied'
+              ? 'border-emerald-700/60 text-emerald-300'
+              : copyState === 'failed'
+                ? 'border-red-800/60 text-red-300'
+                : 'border-surface-700 hover:bg-surface-800',
+          )}
+          onClick={() => void copyAll()}
+          disabled={visible.length === 0}
           title={t('logs.copyTitle', { defaultValue: 'Copy visible entries to clipboard' })}
         >
-          {t('logs.copy', { defaultValue: 'Copy' })}
+          {copyState === 'copied'
+            ? t('logs.copied', { defaultValue: 'Copied' })
+            : copyState === 'failed'
+              ? t('logs.copyFailed', { defaultValue: 'Copy failed' })
+              : t('logs.copy', { defaultValue: 'Copy' })}
         </button>
         <button
           className="rounded border border-red-800/60 px-2 py-1 text-[11px] text-red-300 hover:bg-red-900/30"

apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx

@@ -335,6 +335,20 @@ export function BottomDetailPanel() {
 
   const showSingleScopeBanner = isMulti && SINGLE_URL_ONLY_TABS.has(subTab);
 
+  // "(0)" on Inlinks / Outlinks looks like a crawl result; when it is a
+  // Settings choice — Store switched off for one or both hyperlink rows,
+  // so the link graph was never recorded — say so, in the tab itself.
+  const storeInternalLinks = useAppStore((s) => s.config.storeInternalLinks);
+  const storeExternalLinks = useAppStore((s) => s.config.storeExternalLinks);
+  const linkStoreOffLabels = useMemo(() => {
+    const off: string[] = [];
+    if (!storeInternalLinks) off.push(translateLabel('Internal Hyperlinks', lang));
+    if (!storeExternalLinks) off.push(translateLabel('External Links', lang));
+    return off;
+  }, [storeInternalLinks, storeExternalLinks, lang]);
+  const showLinkStoreOffBanner =
+    (subTab === 'inlinks' || subTab === 'outlinks') && linkStoreOffLabels.length > 0;
+
   // Shared file-name stem for every sub-tab's Export button, so a saved
   // file says which page it came from ("example.com-blog-post-1-inlinks")
   // instead of a generic name the user has to rename.
@@ -397,6 +411,16 @@ export function BottomDetailPanel() {
         </div>
       )}
 
+      {showLinkStoreOffBanner && (
+        <div className="shrink-0 border-b border-surface-800 bg-surface-900/40 px-3 py-1 text-[10.5px] text-amber-400/90">
+          {tx('detail.linkStoreOff', {
+            defaultValue:
+              'Store is off for {{what}} (Settings → Spider → Crawl), so those links are not recorded — turn Store on and re-crawl to fill this tab.',
+            what: linkStoreOffLabels.join(' + '),
+          })}
+        </div>
+      )}
+
       <div className="flex-1 overflow-auto">
         {effectiveIds.length === 0 && (
           <div className="flex h-full items-center justify-center text-xs text-surface-500">
@@ -1726,8 +1750,8 @@ function HttpHeadersView({
   // 5xx-under-load) into a plain-English explanation enriched with
   // signals read off the response headers (Retry-After, cf-ray, …).
   const diag = useMemo(
-    () => diagnoseStatus(row.statusCode, row.statusText, headers),
-    [row.statusCode, row.statusText, headers],
+    () => diagnoseStatus(row.statusCode, row.statusText, headers, row.indexability),
+    [row.statusCode, row.statusText, headers, row.indexability],
   );
   const rows = side === 'response' ? headers : requestHeaders;
   // Only surface the diagnosis banner when there's something worth

apps/desktop/src/renderer/src/components/ExportDialog.tsx

@@ -4,10 +4,10 @@ import clsx from 'clsx';
 import { useTranslation } from 'react-i18next';
 import type {
   AdvancedFilter,
-  CrawlUrlRow,
   ExportTabularSection,
   UrlCategory,
 } from '@freecrawl/shared-types';
+import { EXPORT_DATASET_COLUMNS, type ExportDatasetKey } from '@freecrawl/shared-types';
 import { TAB_ORDER, type TabKey } from '../store.js';
 import { COLUMN_SPECS, columnId, type ColumnSpec } from '../tabs/columns.js';
 
@@ -28,7 +28,10 @@ type Format = 'csv' | 'xlsx' | 'json' | 'xml';
 interface SubCategory {
   /** Child label shown beneath the parent (translated via labels.ts). */
   label: string;
-  category: UrlCategory;
+  /** URL category this leaf streams — unset for dataset leaves. */
+  category?: UrlCategory;
+  /** Non-URL-row table this leaf exports (one AI / SEO provider). */
+  dataset?: ExportDatasetKey;
   /** Filename stem written under the parent's subdir. */
   filename: string;
 }
@@ -43,7 +46,9 @@ interface ExportNode {
   /** Folder name when this node has children; '' when no nesting. */
   subdir: string;
   /** Default category when the parent itself is exported (no children selected). */
-  category: UrlCategory;
+  category?: UrlCategory;
+  /** Set when the whole tab is one non-URL-row table (Images, Search Console, …). */
+  dataset?: ExportDatasetKey;
   /** Optional structural sub-categories. Empty array = leaf node. */
   children: SubCategory[];
 }
@@ -106,9 +111,65 @@ const TAB_CHILDREN: Partial<Record<TabKey, { subdir: string; children: SubCatego
   },
 };
 
-/** Tabs that aren't exportable via the URL-row schema (Images / Broken
- *  Links use their own per-tab CSV exporter; Visualization is a graph). */
-const SKIPPED_TABS = new Set<TabKey>(['images', 'broken-links']);
+/**
+ * Tabs whose rows are not `CrawlUrlRow`s. Each maps to one dataset query
+ * — or, for the per-provider AI / SEO Authority tabs, to one leaf per
+ * provider, so a workbook gets an "AI — OpenAI" sheet holding exactly that
+ * provider's rows. See `ExportDatasetKey`.
+ */
+type DatasetTab =
+  | { dataset: ExportDatasetKey }
+  | { subdir: string; providers: { id: string; label: string; dataset: ExportDatasetKey }[] };
+const DATASET_TABS: Partial<Record<TabKey, DatasetTab>> = {
+  images: { dataset: 'images' },
+  'broken-links': { dataset: 'broken-links' },
+  pagespeed: { dataset: 'pagespeed' },
+  crux: { dataset: 'crux' },
+  spelling: { dataset: 'spelling' },
+  'search-console': { dataset: 'search-console' },
+  analytics: { dataset: 'analytics' },
+  ai: {
+    subdir: 'ai',
+    providers: [
+      { id: 'openai', label: 'OpenAI', dataset: 'ai:openai' },
+      { id: 'anthropic', label: 'Anthropic Claude', dataset: 'ai:anthropic' },
+      { id: 'ollama', label: 'Ollama (local LLM)', dataset: 'ai:ollama' },
+    ],
+  },
+  seo: {
+    subdir: 'seo-authority',
+    providers: [
+      { id: 'ahrefs', label: 'Ahrefs', dataset: 'seo:ahrefs' },
+      { id: 'majestic', label: 'Majestic', dataset: 'seo:majestic' },
+      { id: 'moz', label: 'Moz', dataset: 'seo:moz' },
+      { id: 'semrush', label: 'Semrush', dataset: 'seo:semrush' },
+    ],
+  },
+};
+
+/** Column-picker id for a dataset column — prefixed so `url` on Search
+ *  Console and `url` on Internal toggle independently. */
+function datasetColumnId(dataset: ExportDatasetKey, key: string): string {
+  return `${dataset}:${key}`;
+}
+
+/** Every dataset a node can export (its own, or one per provider leaf). */
+function nodeDatasets(node: ExportNode): ExportDatasetKey[] {
+  if (node.dataset) return [node.dataset];
+  return node.children.flatMap((c) => (c.dataset ? [c.dataset] : []));
+}
+
+/** Column-picker ids a node contributes — URL tabs via their column
+ *  specs, dataset tabs via the dataset catalogue. */
+function nodeColumnIds(node: ExportNode): string[] {
+  const datasets = nodeDatasets(node);
+  if (datasets.length > 0) {
+    return datasets.flatMap((d) =>
+      EXPORT_DATASET_COLUMNS[d].map((c) => datasetColumnId(d, c.key)),
+    );
+  }
+  return (COLUMN_SPECS[node.tabKey] ?? []).map((s) => columnId(s));
+}
 
 /** Default category for top-level tabs that don't have structural
  *  sub-categories. Issues / filter content is intentionally NOT broken
@@ -142,7 +203,32 @@ const TAB_CATEGORY: Partial<Record<TabKey, UrlCategory>> = {
 function buildTree(): ExportNode[] {
   const out: ExportNode[] = [];
   for (const t of TAB_ORDER) {
-    if (SKIPPED_TABS.has(t.key)) continue;
+    const ds = DATASET_TABS[t.key];
+    if (ds) {
+      if ('dataset' in ds) {
+        out.push({
+          key: t.key,
+          tabKey: t.key,
+          label: t.label,
+          subdir: '',
+          dataset: ds.dataset,
+          children: [],
+        });
+      } else {
+        out.push({
+          key: t.key,
+          tabKey: t.key,
+          label: t.label,
+          subdir: ds.subdir,
+          children: ds.providers.map((p) => ({
+            label: p.label,
+            filename: p.id,
+            dataset: p.dataset,
+          })),
+        });
+      }
+      continue;
+    }
     const cols = COLUMN_SPECS[t.key];
     if (!cols || cols.length === 0) continue;
     const sub = TAB_CHILDREN[t.key];
@@ -178,6 +264,9 @@ interface ExportDialogProps {
   search?: string;
   /** Active advanced filter — forwarded so the export matches the grid. */
   filter?: AdvancedFilter;
+  /** Leaf to pre-select under `defaultTab` — a provider tab passes the
+   *  provider it is showing (`seo:ahrefs`) so Export means "this table". */
+  defaultLeaf?: string;
 }
 
 export function ExportDialog({
@@ -187,6 +276,7 @@ export function ExportDialog({
   selectedIds,
   search,
   filter,
+  defaultLeaf,
 }: ExportDialogProps) {
   const { t, i18n } = useTranslation();
   const lang = i18n.language;
@@ -211,7 +301,8 @@ export function ExportDialog({
     const node = tree.find((n) => n.tabKey === initialTabKey);
     if (!node) return new Set();
     if (node.children.length === 0) return new Set([leafId(node)]);
-    return new Set([leafId(node, node.children[0])]);
+    const preferred = node.children.find((c) => leafId(node, c) === defaultLeaf);
+    return new Set([leafId(node, preferred ?? node.children[0])]);
   });
 
   // Expanded parent rows in the tree — closed nodes still let their
@@ -226,8 +317,7 @@ export function ExportDialog({
   // key (indexability / indexability-status) and must toggle independently.
   const [pickedColumns, setPickedColumns] = useState<Set<string>>(() => {
     const node = tree.find((n) => n.tabKey === initialTabKey);
-    const specs = node ? COLUMN_SPECS[node.tabKey] : [];
-    return new Set(specs.map((s) => columnId(s)));
+    return new Set(node ? nodeColumnIds(node) : []);
   });
   const [useSelection, setUseSelection] = useState<boolean>(selectionCount > 0);
   const [busy, setBusy] = useState(false);
@@ -266,9 +356,33 @@ export function ExportDialog({
 
   const offeredColumns = useMemo(() => {
     const seen = new Set<string>();
-    const out: { id: string; key: string; header: string; tab: TabKey }[] = [];
+    const out: {
+      id: string;
+      key: string;
+      header: string;
+      tab: TabKey;
+      dataset?: ExportDatasetKey;
+    }[] = [];
     for (const node of tree) {
       if (!pickedTabKeys.has(node.tabKey)) continue;
+      if (nodeDatasets(node).length > 0) {
+        // Only the datasets whose leaves are picked — an Ahrefs export
+        // must not offer Semrush's columns.
+        const picked = new Set<ExportDatasetKey>();
+        if (node.dataset && pickedLeaves.has(leafId(node))) picked.add(node.dataset);
+        for (const child of node.children) {
+          if (child.dataset && pickedLeaves.has(leafId(node, child))) picked.add(child.dataset);
+        }
+        for (const d of picked) {
+          for (const col of EXPORT_DATASET_COLUMNS[d]) {
+            const id = datasetColumnId(d, col.key);
+            if (seen.has(id)) continue;
+            seen.add(id);
+            out.push({ id, key: col.key, header: col.header, tab: node.tabKey, dataset: d });
+          }
+        }
+        continue;
+      }
       const specs = COLUMN_SPECS[node.tabKey];
       for (const spec of specs) {
         const id = columnId(spec);
@@ -278,12 +392,23 @@ export function ExportDialog({
       }
     }
     return out;
-  }, [pickedTabKeys, tree]);
+  }, [pickedTabKeys, pickedLeaves, tree]);
 
+  /** CrawlUrlRow keys for the URL-category sections, in picker order. */
   const orderedColumnList = useMemo(
-    () => offeredColumns.filter((c) => pickedColumns.has(c.id)).map((c) => c.key),
+    () =>
+      offeredColumns.filter((c) => !c.dataset && pickedColumns.has(c.id)).map((c) => c.key),
     [offeredColumns, pickedColumns],
   );
+  const pickedColumnCount = useMemo(
+    () => offeredColumns.filter((c) => pickedColumns.has(c.id)).length,
+    [offeredColumns, pickedColumns],
+  );
+  /** Picked dotted keys for one dataset section, in catalogue order. */
+  const datasetColumnKeys = (dataset: ExportDatasetKey): string[] =>
+    EXPORT_DATASET_COLUMNS[dataset]
+      .filter((c) => pickedColumns.has(datasetColumnId(dataset, c.key)))
+      .map((c) => c.key);
 
   if (!open) return null;
 
@@ -325,11 +450,10 @@ export function ExportDialog({
     // currently in pickedColumns.
     if (state !== 'all') {
       setPickedColumns((prev) => {
-        const specs = COLUMN_SPECS[node.tabKey] ?? [];
-        const hasAny = specs.some((spec) => prev.has(columnId(spec)));
-        if (hasAny) return prev;
+        const ids = nodeColumnIds(node);
+        if (ids.some((id) => prev.has(id))) return prev;
         const next = new Set(prev);
-        for (const spec of specs) next.add(columnId(spec));
+        for (const id of ids) next.add(id);
         return next;
       });
     }
@@ -345,11 +469,10 @@ export function ExportDialog({
     });
     // Bug #12 — same conditional auto-add as toggleParent.
     setPickedColumns((prev) => {
-      const specs = COLUMN_SPECS[node.tabKey] ?? [];
-      const hasAny = specs.some((spec) => prev.has(columnId(spec)));
-      if (hasAny) return prev;
+      const ids = nodeColumnIds(node);
+      if (ids.some((id) => prev.has(id))) return prev;
       const next = new Set(prev);
-      for (const spec of specs) next.add(columnId(spec));
+      for (const id of ids) next.add(id);
       return next;
     });
   };
@@ -415,10 +538,6 @@ export function ExportDialog({
   };
 
   const submit = async () => {
-    if (orderedColumnList.length === 0) {
-      setStatus(t('export.errorNoColumn', { defaultValue: 'Pick at least one column.' }));
-      return;
-    }
     if (!splitBySeverity && totalLeafSelections === 0) {
       setStatus(t('export.errorNoTable', { defaultValue: 'Pick at least one table.' }));
       return;
@@ -452,7 +571,15 @@ export function ExportDialog({
         for (const node of tree) {
           if (node.children.length === 0) {
             if (pickedLeaves.has(leafId(node))) {
-              sections.push({ label: node.label, category: node.category });
+              sections.push(
+                node.dataset
+                  ? {
+                      label: node.label,
+                      dataset: node.dataset,
+                      columns: datasetColumnKeys(node.dataset),
+                    }
+                  : { label: node.label, category: node.category },
+              );
             }
             continue;
           }
@@ -460,13 +587,26 @@ export function ExportDialog({
             if (!pickedLeaves.has(leafId(node, child))) continue;
             sections.push({
               label: `${node.label} — ${child.label}`,
-              category: child.category,
               subdir: node.subdir,
               filename: child.filename,
+              ...(child.dataset
+                ? { dataset: child.dataset, columns: datasetColumnKeys(child.dataset) }
+                : { category: child.category }),
             });
           }
         }
       }
+      // URL sections share the CrawlUrlRow column list; each dataset
+      // section carries its own keys — every section needs at least one.
+      const needsUrlColumns = splitBySeverity || sections.some((s) => !s.dataset);
+      if (
+        (needsUrlColumns && orderedColumnList.length === 0) ||
+        sections.some((s) => s.dataset && (s.columns?.length ?? 0) === 0)
+      ) {
+        setStatus(t('export.errorNoColumn', { defaultValue: 'Pick at least one column.' }));
+        setBusy(false);
+        return;
+      }
       const usingSelection = useSelection && !!selectedIds && selectedIds.length > 0;
       const result = await window.freecrawl.exportTabular({
         format,
@@ -830,7 +970,7 @@ export function ExportDialog({
             {t('export.summary', {
               defaultValue: '{{n}} section(s) · {{c}} column(s)',
               n: totalLeafSelections,
-              c: orderedColumnList.length,
+              c: pickedColumnCount,
             })}
           </div>
           <div className="flex gap-2">

apps/desktop/src/renderer/src/components/SettingsDialog.tsx

@@ -33,7 +33,6 @@ import {
   Wrench,
   FolderOpen,
   SpellCheck,
-  Lock,
   Languages,
   Plug,
   ExternalLink,
@@ -89,6 +88,7 @@ interface FormState {
   maxUrls: string;
   maxConcurrency: string;
   maxRps: string;
+  adaptiveSpeed: boolean;
   requestTimeoutMs: string;
   crawlDelayMs: string;
   retryAttempts: string;
@@ -260,7 +260,6 @@ export type SettingsSectionKey =
   | 'performance-budget'
   | 'storage'
   | 'spelling'
-  | 'privacy'
   | 'language'
   /** Per-integration sub-page, e.g. `integration:gsc`. Each integration
    *  gets its own page under the "Integrations" group header, so a
@@ -487,12 +486,6 @@ const SECTIONS: SectionDef[] = [
     icon: SpellCheck,
     keywords: 'spelling grammar languagetool dictionary ignore words picky rule level yazım dilbilgisi sözlük',
   },
-  {
-    key: 'privacy',
-    label: 'Privacy',
-    icon: Lock,
-    keywords: 'privacy telemetry analytics anonymous opt in out tracking data',
-  },
   {
     key: 'language',
     label: 'Language',
@@ -510,6 +503,7 @@ function configToForm(c: CrawlConfig): FormState {
     maxUrls: String(c.maxUrls),
     maxConcurrency: String(c.maxConcurrency),
     maxRps: String(c.maxRps),
+    adaptiveSpeed: c.adaptiveSpeed,
     requestTimeoutMs: String(c.requestTimeoutMs),
     crawlDelayMs: String(c.crawlDelayMs),
     retryAttempts: String(c.retryAttempts),
@@ -810,6 +804,7 @@ export function SettingsDialog({ open, onClose }: Props) {
       maxUrls: Math.max(1, num(form.maxUrls, config.maxUrls)),
       maxConcurrency: Math.max(1, Math.min(200, num(form.maxConcurrency, config.maxConcurrency))),
       maxRps: Math.max(1, num(form.maxRps, config.maxRps)),
+      adaptiveSpeed: form.adaptiveSpeed,
       requestTimeoutMs: Math.max(1000, num(form.requestTimeoutMs, config.requestTimeoutMs)),
       crawlDelayMs: Math.max(0, num(form.crawlDelayMs, config.crawlDelayMs)),
       retryAttempts: Math.max(0, num(form.retryAttempts, config.retryAttempts)),
@@ -1233,7 +1228,6 @@ export function SettingsDialog({ open, onClose }: Props) {
               )}
               {active === 'storage' && <StoragePanel />}
               {active === 'spelling' && <SpellingPanel />}
-              {active === 'privacy' && <PrivacyPanel />}
               {active === 'language' && <LanguagePanel />}
             </div>
           </div>
@@ -2220,6 +2214,21 @@ function SpeedPanel({ form, update }: PanelProps) {
         info="Hard ceiling on requests per second across all workers combined. Equivalent to Screaming Frog's 'Max URL/s'. Acts as a token bucket — even with high concurrency the crawler waits between bursts to stay below this rate."
         example="20 for typical sites; 5 to be polite on shared hosting; 60+ when crawling your own infra."
       />
+      <div className="mt-2">
+        <Bool
+          label={t('settingsPanels.speed.adaptiveSpeed', { defaultValue: 'Adaptive speed (auto-slow on 429 / Retry-After)' })}
+          checked={form.adaptiveSpeed}
+          onChange={(v) => update('adaptiveSpeed', v)}
+          info="Treat the concurrency and RPS above as a ceiling and let the target server set the real pace. On a 429/503 (or a Retry-After header) the crawler pauses for the penalty window and steps the rate + concurrency down; after a sustained run of clean responses it grows them back toward the ceiling. Off = hold the configured rate no matter how the server responds."
+          example="Turn on for sites behind Cloudflare / a WAF that returns 429s; leave off for your own infrastructure where the fixed rate is safe."
+        />
+        <p className="mt-1 text-[10px] text-surface-500">
+          {t('settingsPanels.speed.adaptiveSpeedHint', {
+            defaultValue:
+              'Only affects behaviour when the server pushes back — a healthy site is crawled at the full ceiling either way.',
+          })}
+        </p>
+      </div>
       <Num
         label={t('settingsPanels.speed.crawlDelayMs', { defaultValue: 'Per-Worker Delay (ms after each request)' })}
         value={form.crawlDelayMs}
@@ -5239,58 +5248,6 @@ function StoragePanel() {
   );
 }
 
-/**
- * V1 Faz 9 — Privacy panel. Single toggle for anonymous telemetry opt-in.
- * No telemetry backend is wired up yet (default off, no events fire);
- * this preference is kept so future opt-in metrics ship with consent
- * already gathered and the default remains opt-out.
- */
-function PrivacyPanel() {
-  const { t } = useTranslation();
-  const [telemetryOptIn, setTelemetryOptIn] = useState<boolean>(() => {
-    return window.freecrawl.prefsGet('telemetryOptIn') === true;
-  });
-
-  function toggle(v: boolean) {
-    setTelemetryOptIn(v);
-    window.freecrawl.prefsSet('telemetryOptIn', v);
-  }
-
-  return (
-    <>
-      <p className="mb-3 text-[11px] text-surface-400">
-        {t('settings.privacy.intro', { defaultValue: "FreeCrawl is local-first — crawl results, project files, and URLs you visit never leave your machine." })}
-      </p>
-
-      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
-        <div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
-          {t('settings.privacy.telemetryHeading', { defaultValue: "Anonymous Telemetry" })}
-        </div>
-        <label className="flex items-start gap-2">
-          <input
-            type="checkbox"
-            checked={telemetryOptIn}
-            onChange={(e) => toggle(e.target.checked)}
-            className="mt-0.5"
-          />
-          <div className="flex flex-col gap-0.5">
-            <span className="text-[12px] text-surface-100">
-              {t('settings.privacy.telemetryLabel', { defaultValue: "Send anonymous usage telemetry (opt-in)" })}
-            </span>
-            <span className="text-[10px] text-surface-500">
-              {t('settings.privacy.telemetryHelp', { defaultValue: "Off by default. When on, aggregated counts of feature use (e.g. \"export ran\", \"report opened\") may be sent — never URLs, project contents, or crawl results." })}
-            </span>
-          </div>
-        </label>
-      </div>
-
-      <div className="rounded border border-amber-700/40 bg-amber-900/10 p-3 text-[11px] text-amber-200">
-        {t('settings.privacy.statusNote', { defaultValue: "No telemetry endpoint is currently wired up. Toggling this preference is recorded but nothing is transmitted in the current build. The toggle is exposed now so consent state persists across upgrades when a future build adds an endpoint." })}
-      </div>
-    </>
-  );
-}
-
 /**
  * V1 Faz 8 — UI language switcher. Persists the choice under the
  * `uiLanguage` app pref via `changeLanguage` and applies it instantly

apps/desktop/src/renderer/src/components/TabExportButton.tsx

@@ -0,0 +1,65 @@
+import { useState } from 'react';
+import { Download } from 'lucide-react';
+import clsx from 'clsx';
+import { useTranslation } from 'react-i18next';
+import type { TabKey } from '../store.js';
+import { ErrorBoundary } from './ErrorBoundary.js';
+import { ExportDialog } from './ExportDialog.js';
+
+/**
+ * Toolbar "Export" for the integration + SERP tabs — the same control the
+ * main URL table has, opening the Export dialog with this tab's table
+ * pre-selected. The dialog owns format (Excel / CSV / JSON / XML) and the
+ * column picker, so every tab exports through one path and the file a
+ * user gets from Search Console looks like the one they get from Internal.
+ */
+export function TabExportButton({
+  tab,
+  defaultLeaf,
+  disabled,
+  className,
+}: {
+  tab: TabKey;
+  /** Leaf to pre-select for tabs with provider children (`seo:ahrefs`). */
+  defaultLeaf?: string;
+  /** True when the tab has nothing loaded — keeps the button honest. */
+  disabled?: boolean;
+  className?: string;
+}) {
+  const { t } = useTranslation();
+  const [open, setOpen] = useState(false);
+  return (
+    <>
+      <button
+        type="button"
+        onClick={() => setOpen(true)}
+        disabled={disabled}
+        className={clsx(
+          'inline-flex h-6 shrink-0 items-center gap-1.5 rounded border px-2 text-[11px] transition',
+          disabled
+            ? 'cursor-not-allowed border-surface-800 text-surface-600'
+            : 'border-surface-700 text-surface-300 hover:bg-surface-800',
+          className,
+        )}
+        title={
+          disabled
+            ? t('urlsTab.noDataToExport', { defaultValue: 'No data to export' })
+            : t('urlsTab.exportThisTable', { defaultValue: 'Export this table' })
+        }
+      >
+        <Download className="h-3.5 w-3.5" />
+        <span>{t('urlsTab.export', { defaultValue: 'Export' })}</span>
+      </button>
+      <ErrorBoundary context="ExportDialog">
+        {open && (
+          <ExportDialog
+            open={open}
+            onClose={() => setOpen(false)}
+            defaultTab={tab}
+            defaultLeaf={defaultLeaf}
+          />
+        )}
+      </ErrorBoundary>
+    </>
+  );
+}

apps/desktop/src/renderer/src/hooks/useLazyUrlRows.ts

@@ -23,6 +23,13 @@ const TICK_THROTTLE_MS = 100;
 // arriving (crawl finished, manual DB edits, etc.). Long interval —
 // the progress-event path drives the in-crawl experience.
 const LIVE_REFRESH_MS = 5000;
+// After a query fails (a reader timeout, most likely), hold every
+// automatic re-issue — live tick, scroll-driven chunk fetch — for this
+// long. Each of those used to fire again immediately, and against a
+// query that had just spent 30 s timing out that stacked one abandoned
+// copy per tick onto the reader pool until nothing else could get
+// through. The user's explicit Retry bypasses the hold.
+const FAILURE_BACKOFF_MS = 5000;
 
 export interface LazyRowsOpts {
   category: UrlCategory;
@@ -46,6 +53,17 @@ export interface LazyRowsState {
    *  none" from "0 URLs because we haven't fetched yet" — which the
    *  empty-state UI cares about a lot. */
   isLoading: boolean;
+  /** Why the last query for this shape failed (reader timeout, SQL
+   *  error), or null. A silent failure looked exactly like "the filter
+   *  did nothing"; the consumer shows this instead. */
+  error: string | null;
+  /** Re-run the current shape now, ignoring the failure back-off. */
+  retry: () => void;
+}
+
+function describeError(err: unknown): string {
+  const raw = err instanceof Error ? err.message : String(err);
+  return raw.replace(/^Error invoking remote method '[^']+': /, '').replace(/^Error: /, '');
 }
 
 /**
@@ -67,6 +85,14 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
   const [total, setTotal] = useState(0);
   const [version, setVersion] = useState(0);
   const [isLoading, setIsLoading] = useState(true);
+  const [error, setError] = useState<string | null>(null);
+  const [retryNonce, setRetryNonce] = useState(0);
+  const failedAt = useRef(0);
+  const inBackoff = (): boolean => Date.now() - failedAt.current < FAILURE_BACKOFF_MS;
+  const noteFailure = (err: unknown): void => {
+    failedAt.current = Date.now();
+    setError(describeError(err));
+  };
   const chunks = useRef(new Map<number, CrawlUrlRow[]>());
   const chunkOrder = useRef<number[]>([]);
   const fetching = useRef(new Set<number>());
@@ -76,7 +102,7 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
   // Serialising the filter keeps it part of the cache-key string so
   // changes invalidate chunks the same way as sort/search.
   const filterKey = opts.filter ? JSON.stringify(opts.filter) : '';
-  const keyStr = `${opts.category}|${opts.search}|${opts.sortBy ?? ''}|${opts.sortDir}|${filterKey}|${String(opts.refreshKey ?? '')}`;
+  const keyStr = `${opts.category}|${opts.search}|${opts.sortBy ?? ''}|${opts.sortDir}|${filterKey}|${String(opts.refreshKey ?? '')}|${retryNonce}`;
 
   const queryChunk = useCallback(
     (chunkIdx: number) =>
@@ -152,6 +178,8 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
     chunkOrder.current = [];
     fetching.current.clear();
     setIsLoading(true);
+    setError(null);
+    failedAt.current = 0;
     setVersion((v) => v + 1);
     const token = resetToken.current;
     let cancelled = false;
@@ -173,11 +201,11 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
         }
         setTotal(t);
         setVersion((v) => v + 1);
-      } catch {
+      } catch (err) {
         if (cancelled || token !== resetToken.current) return;
-        // Leave chunks empty on error; the live tick / ensureRange
-        // will retry. Still need to flip loading off so the empty
-        // state can render an actual error/empty UI.
+        // Chunks stay empty; the consumer renders the error and the
+        // back-off keeps the live tick from re-issuing the same query.
+        noteFailure(err);
       } finally {
         if (!cancelled && token === resetToken.current) {
           setIsLoading(false);
@@ -211,7 +239,7 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
     let inFlight = false;
     let lastTickTs = 0;
     const tick = async () => {
-      if (inFlight) return;
+      if (inFlight || inBackoff()) return;
       inFlight = true;
       lastTickTs = Date.now();
       const token = resetToken.current;
@@ -231,8 +259,10 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
           for (let c = 0; c < spanCount; c++) {
             storeChunk(first + c, rows.slice(c * CHUNK_SIZE, (c + 1) * CHUNK_SIZE));
           }
-        } catch {
-          /* ignore transient errors */
+          setError(null);
+        } catch (err) {
+          if (cancelled || token !== resetToken.current) return;
+          noteFailure(err);
         }
         if (cancelled || token !== resetToken.current) return;
         setVersion((v) => v + 1);
@@ -289,6 +319,7 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
     async (chunkIdx: number) => {
       if (chunks.current.has(chunkIdx)) return;
       if (fetching.current.has(chunkIdx)) return;
+      if (inBackoff()) return;
       fetching.current.add(chunkIdx);
       const token = resetToken.current;
       try {
@@ -296,7 +327,13 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
         if (token !== resetToken.current) return;
         storeChunk(chunkIdx, rows);
         setTotal(t);
+        setError(null);
         setVersion((v) => v + 1);
+      } catch (err) {
+        // `ensureRange` fires this on every render; a rejection here
+        // used to surface as an unhandled promise, and the next render
+        // re-issued the same failing query.
+        if (token === resetToken.current) noteFailure(err);
       } finally {
         fetching.current.delete(chunkIdx);
       }
@@ -339,5 +376,10 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
     [version],
   );
 
-  return { total, loadedRows, rowAt, ensureRange, isLoading };
+  const retry = useCallback(() => {
+    failedAt.current = 0;
+    setRetryNonce((n) => n + 1);
+  }, []);
+
+  return { total, loadedRows, rowAt, ensureRange, isLoading, error, retry };
 }

apps/desktop/src/renderer/src/i18n/labels.ts

@@ -672,6 +672,98 @@ const TR_LABELS: Record<string, string> = {
   Storage: 'Depolama',
   Privacy: 'Gizlilik',
   Language: 'Dil',
+  // Status column — robots-blocked badge + its tooltip.
+  Robots: 'Robots',
+  'Not requested — disallowed by robots.txt':
+    'İstek atılmadı — robots.txt izin vermiyor',
+  // Export dialog — dataset column headers (Images, Broken Links,
+  // PageSpeed, CrUX, Spelling, Search Console, GA4, AI, SEO Authority).
+  'Page URL': 'Sayfa URL',
+  'Occurrences': 'Kullanım Sayısı',
+  'Source Status': 'Kaynak Durumu',
+  'Destination': 'Hedef',
+  'Destination Status': 'Hedef Durumu',
+  'Anchor': 'Bağlantı Metni',
+  'Result Status': 'Sonuç Durumu',
+  'Error': 'Hata',
+  'Fetched At': 'Çekilme Zamanı',
+  'Check Language': 'Kontrol Dili',
+  'Detected Language': 'Algılanan Dil',
+  'Matches': 'Eşleşme',
+  'Check Status': 'Kontrol Durumu',
+  'Engine': 'Motor',
+  'Checked At': 'Kontrol Zamanı',
+  'Position': 'Konum',
+  'Index Verdict': 'İndeks Kararı',
+  'Coverage State': 'Kapsam Durumu',
+  'Robots.txt State': 'Robots.txt Durumu',
+  'Indexing State': 'İndeksleme Durumu',
+  'Last Crawl Time': 'Son Tarama Zamanı',
+  'Google Canonical': 'Google Canonical',
+  'User Canonical': 'Kullanıcı Canonical',
+  'Mobile Usability': 'Mobil Kullanılabilirlik',
+  'AMP Verdict': 'AMP Kararı',
+  'Rich Results Verdict': 'Zengin Sonuç Kararı',
+  'Engagement Rate': 'Etkileşim Oranı',
+  'Avg. Session Duration': 'Ort. Oturum Süresi',
+  'Model': 'Model',
+  'Tokens In': 'Giriş Token',
+  'Tokens Out': 'Çıkış Token',
+  'Domain Rating': 'Domain Rating',
+  'URL Rating': 'URL Rating',
+  'Backlinks': 'Backlink',
+  'Referring Domains': 'Referans Domain',
+  'Trust Flow': 'Trust Flow',
+  'Citation Flow': 'Citation Flow',
+  'External Backlinks': 'Harici Backlink',
+  'Domain Authority': 'Domain Authority',
+  'Page Authority': 'Page Authority',
+  'Spam Score': 'Spam Skoru',
+  'Linking Domains': 'Bağlantı Veren Domain',
+  'Organic Keywords': 'Organik Anahtar Kelime',
+  'Organic Traffic': 'Organik Trafik',
+  'Organic Cost': 'Organik Maliyet',
+  'AdWords Keywords': 'AdWords Anahtar Kelime',
+  'Ollama (local LLM)': 'Ollama (yerel LLM)',
+  'Mobile Performance': 'Mobil Performans',
+  'Mobile LCP': 'Mobil LCP',
+  'Mobile CLS': 'Mobil CLS',
+  'Mobile FCP': 'Mobil FCP',
+  'Mobile TBT': 'Mobil TBT',
+  'Mobile TTI': 'Mobil TTI',
+  'Mobile INP': 'Mobil INP',
+  'Mobile TTFB': 'Mobil TTFB',
+  'Mobile Speed Index': 'Mobil Hız İndeksi',
+  'Mobile Result Status': 'Mobil Sonuç Durumu',
+  'Mobile Error': 'Mobil Hata',
+  'Mobile Fetched At': 'Mobil Çekilme Zamanı',
+  'Mobile Collection Period': 'Mobil Toplama Dönemi',
+  'Desktop Performance': 'Masaüstü Performans',
+  'Desktop LCP': 'Masaüstü LCP',
+  'Desktop CLS': 'Masaüstü CLS',
+  'Desktop FCP': 'Masaüstü FCP',
+  'Desktop TBT': 'Masaüstü TBT',
+  'Desktop TTI': 'Masaüstü TTI',
+  'Desktop INP': 'Masaüstü INP',
+  'Desktop TTFB': 'Masaüstü TTFB',
+  'Desktop Speed Index': 'Masaüstü Hız İndeksi',
+  'Desktop Result Status': 'Masaüstü Sonuç Durumu',
+  'Desktop Error': 'Masaüstü Hata',
+  'Desktop Fetched At': 'Masaüstü Çekilme Zamanı',
+  'Desktop Collection Period': 'Masaüstü Toplama Dönemi',
+  'Phone Performance': 'Telefon Performans',
+  'Phone LCP': 'Telefon LCP',
+  'Phone CLS': 'Telefon CLS',
+  'Phone FCP': 'Telefon FCP',
+  'Phone TBT': 'Telefon TBT',
+  'Phone TTI': 'Telefon TTI',
+  'Phone INP': 'Telefon INP',
+  'Phone TTFB': 'Telefon TTFB',
+  'Phone Speed Index': 'Telefon Hız İndeksi',
+  'Phone Result Status': 'Telefon Sonuç Durumu',
+  'Phone Error': 'Telefon Hata',
+  'Phone Fetched At': 'Telefon Çekilme Zamanı',
+  'Phone Collection Period': 'Telefon Toplama Dönemi',
 };
 
 /**

apps/desktop/src/renderer/src/i18n/locales/en.json

@@ -297,6 +297,7 @@
     "perPagePrefix": "This tab is per-page — showing data for",
     "perPageSuffix": "(primary URL of the selection).",
     "selectUrl": "Select a URL from the table to see details.",
+    "linkStoreOff": "Store is off for {{what}} (Settings → Spider → Crawl), so those links are not recorded — turn Store on and re-crawl to fill this tab.",
     "httpResponseTab": "Response ({{count}})",
     "httpRequestTab": "Request ({{count}})",
     "headersReconstructedNote": "Reconstructed from active crawl config — actual values may have differed if config changed since crawl.",
@@ -500,6 +501,8 @@
       "effectiveCeiling": "Effective ceiling",
       "maxConcurrency": "Max Concurrency (parallel workers)",
       "maxRps": "Max URL/s (rate limit)",
+      "adaptiveSpeed": "Adaptive speed (auto-slow on 429 / Retry-After)",
+      "adaptiveSpeedHint": "Only affects behaviour when the server pushes back — a healthy site is crawled at the full ceiling either way.",
       "crawlDelayMs": "Per-Worker Delay (ms after each request)",
       "retriesHeading": "Retries",
       "retryAttempts": "Retry Attempts (per URL on transient errors)",
@@ -1003,6 +1006,8 @@
     "filterPlaceholder": "Filter messages / sources…",
     "autoScroll": "Auto-scroll",
     "copy": "Copy",
+    "copied": "Copied",
+    "copyFailed": "Copy failed",
     "copyTitle": "Copy visible entries to clipboard",
     "clear": "Clear",
     "empty": "No log entries.",
@@ -1528,6 +1533,8 @@
     "treeEmpty": "No URLs to display — crawl a site or change the filter.",
     "expandAll": "Expand all",
     "collapseAll": "Collapse all",
+    "loadFailed": "Could not load this filter: {{detail}}",
+    "retry": "Retry",
     "noUrlsHint": "Start a crawl or choose a different category.",
     "statusUrls": "URLs:",
     "statusLoaded": "{{count}} loaded",
@@ -1699,7 +1706,55 @@
     "seedNoOrigin": "No crawled URLs yet to derive a base origin from.",
     "seedNone": "Nothing new to seed.",
     "clearFailed": "Could not clear log data: {{detail}}",
-    "exportFailed": "Export failed: {{detail}}"
+    "exportFailed": "Export failed: {{detail}}",
+    "tabThreats": "Suspicious Requests",
+    "cardThreats": "Suspicious Requests",
+    "threatIntro": "Requests that look like attacks or vulnerability probes — SQL injection, XSS, path traversal, command injection, scanner paths, sensitive-file fetches — with the sender IP and the decoded payload.",
+    "threatStats": "{{hits}} flagged request(s) from {{ips}} IP(s)",
+    "threatServed": "{{n}} answered 2xx — the server processed those payloads; check the application, not just the log",
+    "threatServedNone": "none answered 2xx",
+    "threatCapHit": "The per-file cap was reached — this list is partial.",
+    "threatNone": "No suspicious requests in the ingested logs.",
+    "threatCatAll": "All categories",
+    "threatCatSqli": "SQL injection",
+    "threatCatXss": "XSS",
+    "threatCatTraversal": "Path traversal",
+    "threatCatCmdi": "Command injection",
+    "threatCatScanner": "Scanner probe",
+    "threatCatSensitiveFile": "Sensitive file",
+    "threatCatAnomaly": "Anomaly",
+    "threatStatusAll": "Any status",
+    "threatStatus2xx": "2xx (served)",
+    "threatSortRecent": "Sort: Most recent",
+    "threatSortScore": "Sort: Score",
+    "threatSortStatus": "Sort: Status",
+    "threatSearch": "Filter by payload, IP or user-agent…",
+    "threatIpFilter": "IP: {{ip}}",
+    "threatClearIp": "Clear IP filter",
+    "threatTopIps": "Top senders",
+    "threatCopyMenu": "Copy blocklist",
+    "threatCopyPlain": "Plain IP list",
+    "threatCopyNginx": "nginx deny rules",
+    "threatCopyApache": "Apache Require not ip",
+    "threatCopied": "Copied {{n}} IP(s) to the clipboard (search-engine infrastructure excluded)",
+    "threatCopyFailed": "Could not copy to the clipboard",
+    "threatCopyEmpty": "No IPs to copy",
+    "colTime": "Time",
+    "colMethod": "Method",
+    "colRequest": "Request (decoded)",
+    "colCategory": "Category",
+    "colScore": "Score",
+    "colEvidence": "Evidence",
+    "colDeclaredBot": "Declared Bot",
+    "threatTargetInCrawl": "Target exists in the crawl",
+    "threatTargetProbe": "Blind probe — target not in the crawl",
+    "threatDetailRaw": "Raw request target",
+    "threatDetailRules": "Matched rules",
+    "threatOwnerWarning": "This IP resolves to {{owner}}'s own infrastructure — the payload was relayed through the search engine (Lens, Translate, a fetch of a crafted URL). Blocking the IP would block {{owner}}; fix the application instead.",
+    "threatFilterIp": "Only this IP",
+    "threatCopyIp": "Copy IP",
+    "threatServedBadge": "served",
+    "threatClose": "Close"
   },
   "detailExport": {
     "label": "Export",

apps/desktop/src/renderer/src/i18n/locales/tr.json

@@ -297,6 +297,7 @@
     "perPagePrefix": "Bu sekme sayfa başınadır — şu URL için veri gösteriliyor:",
     "perPageSuffix": "(seçimin birincil URL'i).",
     "selectUrl": "Detayları görmek için tablodan bir URL seç.",
+    "linkStoreOff": "{{what}} için Kaydet kapalı (Ayarlar → Spider → Crawl), bu linkler kaydedilmiyor — Kaydet'i açıp yeniden tarayınca bu sekme dolar.",
     "httpResponseTab": "Yanıt ({{count}})",
     "httpRequestTab": "İstek ({{count}})",
     "headersReconstructedNote": "Aktif tarama yapılandırmasından yeniden oluşturuldu — taramadan sonra yapılandırma değiştiyse gerçek değerler farklı olabilir.",
@@ -500,6 +501,8 @@
       "effectiveCeiling": "Etkin tavan",
       "maxConcurrency": "Maks Paralellik (paralel worker)",
       "maxRps": "Maks URL/sn (rate limit)",
+      "adaptiveSpeed": "Adaptif hız (429 / Retry-After'da otomatik yavaşla)",
+      "adaptiveSpeedHint": "Yalnızca sunucu geri baskı yaptığında etkilidir — sağlıklı bir site her durumda tam tavan hızda taranır.",
       "crawlDelayMs": "Worker Başına Gecikme (her istekten sonra, ms)",
       "retriesHeading": "Yeniden Denemeler",
       "retryAttempts": "Yeniden Deneme Sayısı (geçici hatada URL başına)",
@@ -1003,6 +1006,8 @@
     "filterPlaceholder": "Mesaj / kaynak filtrele…",
     "autoScroll": "Otomatik kaydır",
     "copy": "Kopyala",
+    "copied": "Kopyalandı",
+    "copyFailed": "Kopyalanamadı",
     "copyTitle": "Görünen kayıtları panoya kopyala",
     "clear": "Temizle",
     "empty": "Log kaydı yok.",
@@ -1528,6 +1533,8 @@
     "treeEmpty": "Gösterilecek URL yok — bir site tara veya filtreyi değiştir.",
     "expandAll": "Tümünü genişlet",
     "collapseAll": "Tümünü daralt",
+    "loadFailed": "Bu filtre yüklenemedi: {{detail}}",
+    "retry": "Tekrar dene",
     "noUrlsHint": "Bir crawl başlatın veya farklı bir kategori seçin.",
     "statusUrls": "URL:",
     "statusLoaded": "{{count}} yüklendi",
@@ -1698,6 +1705,54 @@
     "seedNoActive": "Aktif crawl yok — önce bir crawl başlatın, sonra ekleyin.",
     "seedNoOrigin": "Temel origin türetmek için henüz taranmış URL yok.",
     "seedNone": "Eklenecek yeni bir şey yok.",
+    "tabThreats": "Şüpheli İstekler",
+    "cardThreats": "Şüpheli İstek",
+    "threatIntro": "Saldırı ya da zafiyet taraması gibi görünen istekler — SQL enjeksiyonu, XSS, dizin geçişi, komut enjeksiyonu, tarayıcı yolları, hassas dosya çekme — gönderen IP ve çözülmüş yük ile birlikte.",
+    "threatStats": "{{ips}} IP'den {{hits}} işaretli istek",
+    "threatServed": "{{n}} tanesi 2xx aldı — sunucu bu yükleri işledi; sadece log'a değil uygulamaya bakın",
+    "threatServedNone": "hiçbiri 2xx almadı",
+    "threatCapHit": "Dosya başına üst sınıra ulaşıldı — bu liste eksik.",
+    "threatNone": "İçe aktarılan loglarda şüpheli istek yok.",
+    "threatCatAll": "Tüm kategoriler",
+    "threatCatSqli": "SQL enjeksiyonu",
+    "threatCatXss": "XSS",
+    "threatCatTraversal": "Dizin geçişi",
+    "threatCatCmdi": "Komut enjeksiyonu",
+    "threatCatScanner": "Tarayıcı sondası",
+    "threatCatSensitiveFile": "Hassas dosya",
+    "threatCatAnomaly": "Anomali",
+    "threatStatusAll": "Tüm durumlar",
+    "threatStatus2xx": "2xx (işlendi)",
+    "threatSortRecent": "Sırala: En yeni",
+    "threatSortScore": "Sırala: Skor",
+    "threatSortStatus": "Sırala: Durum",
+    "threatSearch": "Yük, IP veya user-agent ile filtrele…",
+    "threatIpFilter": "IP: {{ip}}",
+    "threatClearIp": "IP filtresini kaldır",
+    "threatTopIps": "En aktif göndericiler",
+    "threatCopyMenu": "Engel listesini kopyala",
+    "threatCopyPlain": "Düz IP listesi",
+    "threatCopyNginx": "nginx deny kuralları",
+    "threatCopyApache": "Apache Require not ip",
+    "threatCopied": "{{n}} IP panoya kopyalandı (arama motoru altyapısı hariç)",
+    "threatCopyFailed": "Panoya kopyalanamadı",
+    "threatCopyEmpty": "Kopyalanacak IP yok",
+    "colTime": "Zaman",
+    "colMethod": "Metot",
+    "colRequest": "İstek (çözülmüş)",
+    "colCategory": "Kategori",
+    "colScore": "Skor",
+    "colEvidence": "Kanıt",
+    "colDeclaredBot": "Bildirilen Bot",
+    "threatTargetInCrawl": "Hedef crawl'da mevcut",
+    "threatTargetProbe": "Kör sonda — hedef crawl'da yok",
+    "threatDetailRaw": "Ham istek hedefi",
+    "threatDetailRules": "Eşleşen kurallar",
+    "threatOwnerWarning": "Bu IP {{owner}}'ın kendi altyapısına çözümleniyor — yük arama motoru üzerinden iletilmiş (Lens, Translate, hazırlanmış bir URL'nin çekilmesi). IP'yi engellemek {{owner}}'ı engeller; bunun yerine uygulamayı düzeltin.",
+    "threatFilterIp": "Sadece bu IP",
+    "threatCopyIp": "IP'yi kopyala",
+    "threatServedBadge": "işlendi",
+    "threatClose": "Kapat",
     "clearFailed": "Log verisi temizlenemedi: {{detail}}",
     "exportFailed": "Dışa aktarma başarısız: {{detail}}"
   },

apps/desktop/src/renderer/src/tabs/AiTab.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
+import { TabExportButton } from '../components/TabExportButton.js';
 import { Loader2, Search, Play, AlertTriangle } from 'lucide-react';
 import type {
   AiProgress,
@@ -309,6 +310,7 @@ export function AiTab() {
               })}
             </button>
           )}
+          <TabExportButton tab="ai" disabled={total === 0} defaultLeaf={`ai:${provider}`} />
         </div>
       </div>
 

apps/desktop/src/renderer/src/tabs/AnalyticsTab.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
+import { TabExportButton } from '../components/TabExportButton.js';
 import { Loader2, Search, RefreshCw } from 'lucide-react';
 import {
   defaultGa4Settings,
@@ -367,6 +368,7 @@ export function AnalyticsTab() {
         </select>
 
         <div className="ml-auto flex items-center gap-2">
+          <TabExportButton tab="analytics" disabled={total === 0} />
           {accounts.length > 1 && (
             <select
               className="h-6 max-w-[170px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"

apps/desktop/src/renderer/src/tabs/CruxTab.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
+import { TabExportButton } from '../components/TabExportButton.js';
 import { Smartphone, Monitor, Loader2 } from 'lucide-react';
 import type {
   CruxMetrics,
@@ -295,6 +296,7 @@ export function CruxTab() {
               })}
             </button>
           )}
+          <TabExportButton tab="crux" disabled={total === 0} />
         </div>
       </div>
 

apps/desktop/src/renderer/src/tabs/PageSpeedTab.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
+import { TabExportButton } from '../components/TabExportButton.js';
 import { Smartphone, Monitor, Loader2 } from 'lucide-react';
 import type {
   PagespeedMetrics,
@@ -318,6 +319,7 @@ export function PageSpeedTab() {
               })}
             </button>
           )}
+          <TabExportButton tab="pagespeed" disabled={total === 0} />
         </div>
       </div>
 

apps/desktop/src/renderer/src/tabs/SearchConsoleTab.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
+import { TabExportButton } from '../components/TabExportButton.js';
 import { Loader2, Search, RefreshCw, Plus } from 'lucide-react';
 import { InfoTip } from '../components/InfoTip.js';
 import {
@@ -454,6 +455,7 @@ export function SearchConsoleTab() {
         </select>
 
         <div className="ml-auto flex items-center gap-2">
+          <TabExportButton tab="search-console" disabled={total === 0} />
           {accounts.length > 1 && (
             <select
               className="h-6 max-w-[170px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"

apps/desktop/src/renderer/src/tabs/SeoTab.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
+import { TabExportButton } from '../components/TabExportButton.js';
 import { Loader2, Search, Play, AlertTriangle } from 'lucide-react';
 import type {
   AhrefsMetrics,
@@ -308,6 +309,7 @@ export function SeoTab() {
               })}
             </button>
           )}
+          <TabExportButton tab="seo" disabled={total === 0} defaultLeaf={`seo:${provider}`} />
         </div>
       </div>
 

apps/desktop/src/renderer/src/tabs/SerpTab.tsx

@@ -1,6 +1,7 @@
 import { useEffect, useRef, useState } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
+import { TabExportButton } from '../components/TabExportButton.js';
 import type { CrawlUrlRow } from '@freecrawl/shared-types';
 import { useAppStore } from '../store.js';
 
@@ -117,7 +118,8 @@ export function SerpTab() {
             <option value="desc-length-desc">{t('serpTab.sortDescLen', { defaultValue: 'Longest description first' })}</option>
           </select>
         </label>
-        <div className="ml-auto text-[11px] text-surface-500">
+        <TabExportButton tab="serp" disabled={total === 0} className="ml-auto" />
+        <div className="text-[11px] text-surface-500">
           {/* Rows are on screen, so the list itself isn't misleading — but
               the counts are stale, and saying so beats silence. */}
           {loadError && rows.length > 0 && (

apps/desktop/src/renderer/src/tabs/SpellingTab.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { useTranslation } from 'react-i18next';
+import { TabExportButton } from '../components/TabExportButton.js';
 import { Loader2 } from 'lucide-react';
 import type { SpellingProgress, SpellingRow } from '@freecrawl/shared-types';
 import { useAppStore } from '../store.js';
@@ -269,6 +270,7 @@ export function SpellingTab() {
               })}
             </button>
           )}
+          <TabExportButton tab="spelling" disabled={total === 0} />
         </div>
       </div>
 

apps/desktop/src/renderer/src/tabs/UrlsTab.tsx

@@ -1598,6 +1598,26 @@ export function UrlsTab() {
           </div>
         </div>
 
+        {lazy.error && (
+          <div
+            className="absolute inset-x-0 z-20 flex items-center gap-2 border-b border-red-900/60 bg-red-950/70 px-3 py-1 text-[11px] text-red-200"
+            style={{ top: HEADER_HEIGHT }}
+          >
+            <span className="min-w-0 flex-1 truncate" title={lazy.error}>
+              {t('urlsTab.loadFailed', {
+                defaultValue: 'Could not load this filter: {{detail}}',
+                detail: lazy.error,
+              })}
+            </span>
+            <button
+              type="button"
+              onClick={() => lazy.retry()}
+              className="shrink-0 rounded border border-red-700/60 px-2 py-0.5 text-red-100 hover:bg-red-900/50"
+            >
+              {t('urlsTab.retry', { defaultValue: 'Retry' })}
+            </button>
+          </div>
+        )}
         {lazy.total === 0 && lazy.isLoading && (
           <div
             className="pointer-events-none absolute inset-0 flex items-center justify-center"
@@ -1606,7 +1626,7 @@ export function UrlsTab() {
             <div className="text-xs text-surface-500">{t('common.loading', { defaultValue: 'Loading…' })}</div>
           </div>
         )}
-        {lazy.total === 0 && !lazy.isLoading && (
+        {lazy.total === 0 && !lazy.isLoading && !lazy.error && (
           <div
             className="pointer-events-none absolute inset-0 flex items-center justify-center"
             style={{ top: HEADER_HEIGHT }}
@@ -2086,6 +2106,23 @@ function Cell({
 
   if (spec.kind === 'status') {
     const code = row.statusCode;
+    // A robots-blocked URL also has no status — but because the crawler
+    // chose not to request it, not because the request failed. Painting
+    // it in the failure red said "this broke" about a crawl the status
+    // bar reports as Failed 0.
+    if (
+      (code === null || code === undefined) &&
+      row.indexability === 'non-indexable:robots-blocked'
+    ) {
+      return (
+        <span
+          className="inline-block rounded bg-surface-800 px-1.5 font-mono text-[10px] text-amber-400"
+          title={translateLabel(ROBOTS_STATUS_TITLE, lang)}
+        >
+          {translateLabel('Robots', lang)}
+        </span>
+      );
+    }
     // No HTTP status (DNS/TLS/connect/timeout failure): a bare "—" hides the
     // reason the crawler already captured. Show a compact token (DNS / TLS /
     // Timeout / …) with the full error text on hover.
@@ -2317,6 +2354,9 @@ function cellText(row: CrawlUrlRow, spec: ColumnSpec, lang: string): string {
   if (spec.kind === 'status') {
     const code = row.statusCode;
     if (code !== null && code !== undefined) return String(code);
+    if (row.indexability === 'non-indexable:robots-blocked') {
+      return translateLabel('Robots', lang);
+    }
     const label = shortFailureLabel(row.statusText);
     return label === '—' ? '' : label;
   }
@@ -2407,6 +2447,10 @@ function indexabilityStatusLabel(v: CrawlUrlRow['indexability']): string {
  * localized, same convention as the numeric status codes themselves. The full
  * diagnostic is on hover (title) and in the URL Details → HTTP Headers tab.
  */
+/** Tooltip for the Robots status badge — dictionary key, so the grid
+ *  cell stays free of a `useTranslation` hook. */
+const ROBOTS_STATUS_TITLE = 'Not requested — disallowed by robots.txt';
+
 function shortFailureLabel(statusText: string | null): string {
   if (!statusText) return '—';
   if (/ENOTFOUND|EAI_AGAIN|ENODATA|ESERVFAIL|getaddrinfo|DNS/i.test(statusText)) return 'DNS';

apps/desktop/src/renderer/src/tabs/columns.ts

@@ -621,7 +621,14 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
   // schema. Entries kept so Record<TabKey, ...> is total.
   images: [],
   'broken-links': [],
-  serp: [],
+  serp: [
+    COL.url,
+    COL.title,
+    COL.titleLength,
+    COL.metaDescription,
+    COL.metaDescriptionLength,
+    COL.inlinks,
+  ],
   pagespeed: [],
   crux: [],
   spelling: [],

apps/desktop/src/renderer/src/utils/clipboard.ts

@@ -106,29 +106,43 @@ export async function copySelection(
 }
 
 /**
- * Write `text` to the OS clipboard.
+ * Write `text` to the OS clipboard. Resolves false when every path failed,
+ * so a caller can tell the user instead of leaving a dead button.
  *
- * The async Clipboard API is tried first and covers the normal case. It
- * rejects when the renderer isn't focused or the clipboard permission is
- * unset — situations that do occur in Electron — so a hidden textarea +
- * `execCommand('copy')` backs it up. The deprecated call is deliberate:
- * it is the only path that works without focus.
+ * Electron's main-process `clipboard` module goes first: it needs neither
+ * a focused document nor a clipboard permission, which is why the async
+ * Clipboard API alone left the Logs window's Copy button doing nothing at
+ * all. The two DOM paths stay as fallbacks for any renderer without the
+ * preload bridge (and `execCommand` is deprecated but is still the only
+ * one that works unfocused).
  */
-export async function writeTextToClipboard(text: string): Promise<void> {
+export async function writeTextToClipboard(text: string): Promise<boolean> {
+  const bridge = window.freecrawl?.clipboardWriteText;
+  if (bridge) {
+    try {
+      if (await bridge(text)) return true;
+    } catch {
+      /* fall through to the DOM paths */
+    }
+  }
   try {
     await navigator.clipboard.writeText(text);
+    return true;
   } catch {
-    const ta = document.createElement('textarea');
-    ta.value = text;
-    ta.style.position = 'fixed';
-    ta.style.opacity = '0';
-    document.body.appendChild(ta);
-    ta.select();
-    try {
-      document.execCommand('copy');
-    } finally {
-      document.body.removeChild(ta);
-    }
+    /* fall through */
+  }
+  const ta = document.createElement('textarea');
+  ta.value = text;
+  ta.style.position = 'fixed';
+  ta.style.opacity = '0';
+  document.body.appendChild(ta);
+  ta.select();
+  try {
+    return document.execCommand('copy');
+  } catch {
+    return false;
+  } finally {
+    document.body.removeChild(ta);
   }
 }
 

apps/desktop/src/renderer/src/utils/statusDiagnosis.ts

@@ -349,12 +349,14 @@ const GENERIC_CODES: Record<number, CodeInfo> = {
 /**
  * Build the diagnosis for a (statusCode, statusText, headers) triple.
  * `statusCode` may be null/0 for pre-response network failures, in
- * which case the explanation focuses on connection-level causes.
+ * which case the explanation focuses on connection-level causes —
+ * unless `indexability` says the URL was never requested at all.
  */
 export function diagnoseStatus(
   statusCode: number | null,
   statusText: string | null,
   headers: HeaderList,
+  indexability?: string | null,
 ): StatusDiagnosis {
   const h = headerMap(headers);
   const front = detectFront(h);
@@ -383,6 +385,34 @@ export function diagnoseStatus(
   const xRobots = h.get('x-robots-tag');
   if (xRobots) signals.push(`X-Robots-Tag: ${xRobots} — affects indexability regardless of the status.`);
 
+  // ----- Deliberately not requested (robots.txt) -----
+  // Shares the "no status code" shape with a network failure, so without
+  // this branch a disallowed URL was diagnosed as a DNS/TLS/timeout
+  // error and sent the user hunting a connection problem that isn't there.
+  if (
+    (statusCode === null || statusCode === 0) &&
+    indexability === 'non-indexable:robots-blocked'
+  ) {
+    return {
+      title: 'Not Requested — Disallowed by robots.txt',
+      severity: 'info',
+      explanation:
+        'The crawler never sent a request for this URL: a robots.txt rule for the ' +
+        'configured user-agent disallows it. There is no status code because no ' +
+        'response was ever asked for — this is not a connection failure.',
+      causes: [
+        'A Disallow rule in the site robots.txt matches this path for the crawl user-agent',
+        'The URL is linked from crawled pages, so it is recorded and counted, but never fetched',
+      ],
+      whatToDo: [
+        'Nothing, if the exclusion is intended — Google will not crawl it either',
+        'Check whether the page should be indexable: a disallowed URL cannot pass on link equity and its content is never seen',
+        'Use Reports → Robots Tester to see which rule matches, or turn off "Respect robots.txt" in Settings → Crawler to audit it anyway',
+      ],
+      signals,
+    };
+  }
+
   // ----- Network error (no response) -----
   if (statusCode === null || statusCode === 0) {
     return {

packages/core/src/crawler.ts

@@ -12,7 +12,7 @@ import type {
   DiscoveredLink,
   Indexability,
 } from '@freecrawl/shared-types';
-import { type ProjectDb, EXPENSIVE_ISSUE_DEFINITIONS } from '@freecrawl/db';
+import { type ProjectDb, type ProbeWrite, EXPENSIVE_ISSUE_DEFINITIONS } from '@freecrawl/db';
 import {
   normalizeUrl,
   isSameHost,
@@ -234,6 +234,53 @@ function isHardTransportError(err: unknown): boolean {
   return HARD_TRANSPORT_ERROR_RE.test(formatFetchError(err));
 }
 
+/**
+ * Collects post-crawl probe results and hands them to the writer in
+ * batches — see `ProjectDb.applyProbeWrites` for why they are not
+ * written one row at a time on the crawler's own connection.
+ *
+ * Flushes are chained so batches reach the writer in order and never
+ * overlap; the final `flush()` awaits everything still in flight, which
+ * is what lets a probe phase report "complete" only once its rows are on
+ * disk. A rejected batch is reported through `onError` and dropped
+ * rather than left to poison the chain for every batch after it.
+ */
+export class ProbeWriteBuffer {
+  private pending: ProbeWrite[] = [];
+  private chain: Promise<void> = Promise.resolve();
+
+  constructor(
+    private readonly write: (writes: ProbeWrite[]) => Promise<void>,
+    private readonly onError: (err: unknown) => void,
+    private readonly batchSize = 200,
+  ) {}
+
+  push(write: ProbeWrite): void {
+    this.pending.push(write);
+    if (this.pending.length >= this.batchSize) void this.flush();
+  }
+
+  flush(): Promise<void> {
+    if (this.pending.length > 0) {
+      const batch = this.pending;
+      this.pending = [];
+      this.chain = this.chain.then(() => this.write(batch)).catch(this.onError);
+    }
+    return this.chain;
+  }
+}
+
+/** Pacing + progress bookkeeping for one post-crawl probe phase. */
+interface ProbeProgress {
+  /** Human-readable phase name, used verbatim in the progress line. */
+  label: string;
+  total: number;
+  /** Items dispatched so far — slightly ahead of items finished, by at
+   *  most the worker count, which is what a progress line wants. */
+  started: number;
+  lastEmitAt: number;
+}
+
 export class Crawler extends EventEmitter {
   private config: CrawlConfig;
   private readonly db: ProjectDb;
@@ -288,27 +335,37 @@ export class Crawler extends EventEmitter {
   private memoryWatchdogTimer: NodeJS.Timeout | null = null;
   /** Periodic post-crawl-style recompute of expensive issue counters
    * while the crawl is still running, so the sidebar number is < 30 s
-   * stale instead of "0 until crawl ends". The recompute is dispatched
-   * to the writer-worker via the injected `recomputeIssues` hook, so
-   * it does NOT block the main thread — the freezes that an earlier
-   * version produced were a result of running the recompute inline on
-   * the main-thread writer connection, which has since been fixed. */
+   * stale instead of "0 until crawl ends". The desktop host routes the
+   * pass to the writer worker through the injected `recomputeIssues`
+   * hook; in RAM-only storage mode there is no worker and it runs on
+   * this thread. Either way a slow pass must not be re-issued every
+   * 30 s regardless: the next tick waits at least
+   * `ISSUE_RECOMPUTE_DUTY_DIVISOR` × the previous pass's duration, so
+   * the recompute can never occupy more than a fixed share of wall
+   * time — on the main thread that share is the freeze budget. */
   private issueRecomputeTimer: NodeJS.Timeout | null = null;
   private static readonly ISSUE_RECOMPUTE_INTERVAL_MS = 30_000;
+  private static readonly ISSUE_RECOMPUTE_MAX_INTERVAL_MS = 5 * 60_000;
+  private static readonly ISSUE_RECOMPUTE_DUTY_DIVISOR = 4;
   private issueRecomputeInFlight = false;
   private startIssueRecomputeTimer(): void {
+    this.scheduleIssueRecompute(Crawler.ISSUE_RECOMPUTE_INTERVAL_MS);
+  }
+  private scheduleIssueRecompute(delayMs: number): void {
     if (this.issueRecomputeTimer) return;
-    this.issueRecomputeTimer = setInterval(() => {
-      if (this.stopped || !this.running || this.paused) return;
-      // Drop overlapping ticks — on a 1M-URL DB the recompute can run
-      // longer than the 30 s tick, and piling RPCs on the writer worker
-      // would starve per-URL writes behind a queue of issue passes.
-      if (this.issueRecomputeInFlight) return;
+    this.issueRecomputeTimer = setTimeout(() => {
+      this.issueRecomputeTimer = null;
+      if (this.stopped || !this.running) return;
+      // Skip while paused or while a previous pass is still running —
+      // on a 1M-URL DB the recompute can outlive its own interval, and
+      // piling RPCs on the writer worker would starve per-URL writes
+      // behind a queue of issue passes.
+      if (this.paused || this.issueRecomputeInFlight) {
+        this.scheduleIssueRecompute(Crawler.ISSUE_RECOMPUTE_INTERVAL_MS);
+        return;
+      }
       this.issueRecomputeInFlight = true;
-      // The injected hook routes through the writer worker in the
-      // desktop host, so this is off-thread; main stays free to
-      // service IPC and pump the crawl queue. CLI fallback runs the
-      // yielding variant inline, which is fine for that context.
+      const startedAt = Date.now();
       void this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS)
         .catch((err) => {
           this.emit(
@@ -318,12 +375,27 @@ export class Crawler extends EventEmitter {
         })
         .finally(() => {
           this.issueRecomputeInFlight = false;
+          const tookMs = Date.now() - startedAt;
+          const nextMs = Math.min(
+            Crawler.ISSUE_RECOMPUTE_MAX_INTERVAL_MS,
+            Math.max(
+              Crawler.ISSUE_RECOMPUTE_INTERVAL_MS,
+              tookMs * Crawler.ISSUE_RECOMPUTE_DUTY_DIVISOR,
+            ),
+          );
+          if (nextMs > Crawler.ISSUE_RECOMPUTE_INTERVAL_MS) {
+            this.emit(
+              'debug',
+              `issue counter recompute took ${Math.round(tookMs / 1000)} s — next pass in ${Math.round(nextMs / 1000)} s`,
+            );
+          }
+          if (!this.stopped && this.running) this.scheduleIssueRecompute(nextMs);
         });
-    }, Crawler.ISSUE_RECOMPUTE_INTERVAL_MS);
+    }, delayMs);
   }
   private stopIssueRecomputeTimer(): void {
     if (this.issueRecomputeTimer) {
-      clearInterval(this.issueRecomputeTimer);
+      clearTimeout(this.issueRecomputeTimer);
       this.issueRecomputeTimer = null;
     }
   }
@@ -1397,6 +1469,50 @@ export class Crawler extends EventEmitter {
    * bounded by the same setting as the main crawl; failures are silent
    * (probe_status stays null and the issue check skips them).
    */
+  /**
+   * Shared pacing + progress for the post-crawl probe phases.
+   *
+   * Each of these loops fires one network request per row over a fixed
+   * worker pool, and every one of them used to do so with no spacing and
+   * no output between "Probing N…" and "complete". On a site with 16 k
+   * images that is a half-hour of silence during which the URL counters
+   * cannot move — the crawl reads as wedged — and the burst (up to 20
+   * concurrent with zero delay, ignoring the crawl's own `maxRps`) is
+   * what makes the host start timing those requests out in the first
+   * place, which is what made the phase take half an hour.
+   *
+   * Call once per item, right after the bounds check: it honours the
+   * configured rate limit and emits a progress line on a fixed cadence.
+   */
+  private static readonly PROBE_PROGRESS_INTERVAL_MS = 15_000;
+
+  private async probeTick(state: ProbeProgress): Promise<void> {
+    state.started++;
+    const now = Date.now();
+    if (now - state.lastEmitAt >= Crawler.PROBE_PROGRESS_INTERVAL_MS) {
+      state.lastEmitAt = now;
+      this.emit('info', `${state.label}: ${state.started}/${state.total}…`);
+    }
+    // Same gate the crawl loop uses. `maxRps` is a politeness setting the
+    // user configured for this host; a probe is a request like any other.
+    await this.acquireRateSlot();
+  }
+
+  private newProbeWriteBuffer(): ProbeWriteBuffer {
+    return new ProbeWriteBuffer(
+      (writes) => this.dbCall<void>('applyProbeWrites', [writes]),
+      (err) =>
+        this.emit(
+          'debug',
+          `probe result batch not stored: ${err instanceof Error ? err.message : String(err)}`,
+        ),
+    );
+  }
+
+  private newProbeProgress(label: string, total: number): ProbeProgress {
+    return { label, total, started: 0, lastEmitAt: Date.now() };
+  }
+
   private async runImageSizeProbes(): Promise<void> {
     if (!this.config.probeImageSizes) return;
     if (this.stopped) return;
@@ -1421,16 +1537,19 @@ export class Crawler extends EventEmitter {
     let large = 0;
     const threshold = Math.max(1, this.config.largeImageBytes);
 
+    const writes = this.newProbeWriteBuffer();
+    const progress = this.newProbeProgress('Image probe', unprobed.length);
     const worker = async (): Promise<void> => {
       while (!this.stopped) {
         const idx = cursor++;
         if (idx >= unprobed.length) return;
         const entry = unprobed[idx];
         if (!entry) return;
+        await this.probeTick(progress);
         // SSRF guard — same as probeExternal: an image `src` can point at an
         // internal address just as a link can.
         if (isPrivateHost(entry.src) && !isPrivateHost(this.config.startUrl)) {
-          this.db.setImageSize(entry.id, null, 0);
+          writes.push({ method: 'setImageSize', args: [entry.id, null, 0] });
           continue;
         }
         // Only send the crawl's auth/custom headers to in-scope image hosts;
@@ -1455,11 +1574,14 @@ export class Crawler extends EventEmitter {
           });
           const lenStr = res.headers.get('content-length');
           const len = lenStr !== null ? Number.parseInt(lenStr, 10) : null;
-          this.db.setImageSize(
-            entry.id,
-            Number.isFinite(len) && len !== null && len >= 0 ? len : null,
-            res.status,
-          );
+          writes.push({
+            method: 'setImageSize',
+            args: [
+              entry.id,
+              Number.isFinite(len) && len !== null && len >= 0 ? len : null,
+              res.status,
+            ],
+          });
           if (Number.isFinite(len) && len !== null && len > threshold) {
             large++;
           }
@@ -1472,11 +1594,7 @@ export class Crawler extends EventEmitter {
           }
         } catch {
           // Mark with status 0 so we don't re-probe on the next crawl.
-          try {
-            this.db.setImageSize(entry.id, null, 0);
-          } catch {
-            /* ignore */
-          }
+          writes.push({ method: 'setImageSize', args: [entry.id, null, 0] });
         } finally {
           clearTimeout(timeout);
         }
@@ -1486,6 +1604,7 @@ export class Crawler extends EventEmitter {
     const workers: Promise<void>[] = [];
     for (let i = 0; i < concurrency; i++) workers.push(worker());
     await Promise.all(workers);
+    await writes.flush();
     this.emit(
       'info',
       `Image probe complete: ${probed} sized, ${large} > ${(threshold / 1024).toFixed(0)} KB`,
@@ -1530,12 +1649,15 @@ export class Crawler extends EventEmitter {
     let probed = 0;
     let failed = 0;
 
+    const writes = this.newProbeWriteBuffer();
+    const progress = this.newProbeProgress('Manifest probe', urls.length);
     const worker = async (): Promise<void> => {
       while (!this.stopped) {
         const idx = cursor++;
         if (idx >= urls.length) return;
         const url = urls[idx];
         if (!url) return;
+        await this.probeTick(progress);
         try {
           const result = await probeManifest(url, {
             userAgent: this.resolveUserAgent(url),
@@ -1543,14 +1665,19 @@ export class Crawler extends EventEmitter {
             customHeaders: this.config.customHeaders,
             auth: this.config.auth,
           });
-          this.db.setManifestForReferrers({
-            manifestUrl: url,
-            rawJson: result.error ? null : result.rawJson,
-            themeColor: result.themeColor,
-            shortName: result.shortName,
-            display: result.display,
-            scope: result.scope,
-            iconCount: result.iconCount,
+          writes.push({
+            method: 'setManifestForReferrers',
+            args: [
+              {
+                manifestUrl: url,
+                rawJson: result.error ? null : result.rawJson,
+                themeColor: result.themeColor,
+                shortName: result.shortName,
+                display: result.display,
+                scope: result.scope,
+                iconCount: result.iconCount,
+              },
+            ],
           });
           if (result.error) failed++;
           probed++;
@@ -1563,6 +1690,7 @@ export class Crawler extends EventEmitter {
     const workers: Promise<void>[] = [];
     for (let i = 0; i < concurrency; i++) workers.push(worker());
     await Promise.all(workers);
+    await writes.flush();
     this.emit(
       'info',
       `Manifest probe complete: ${probed - failed}/${probed} parsed, ${failed} failed`,
@@ -1600,12 +1728,15 @@ export class Crawler extends EventEmitter {
     let cursor = 0;
     let decoded = 0;
 
+    const writes = this.newProbeWriteBuffer();
+    const progress = this.newProbeProgress('Social image probe', urls.length);
     const worker = async (): Promise<void> => {
       while (!this.stopped) {
         const idx = cursor++;
         if (idx >= urls.length) return;
         const src = urls[idx];
         if (!src) return;
+        await this.probeTick(progress);
         let dims: { width: number; height: number } | null = null;
         try {
           dims = await probeSocialImageDimensions(src, {
@@ -1617,17 +1748,12 @@ export class Crawler extends EventEmitter {
         } catch {
           dims = null;
         }
-        try {
-          // 0×0 marks "probed but undecodable" so the row leaves the
-          // unprobed set and we don't re-fetch a broken image next crawl.
-          this.db.setSocialImageDimsForReferrers({
-            src,
-            width: dims?.width ?? 0,
-            height: dims?.height ?? 0,
-          });
-        } catch {
-          /* ignore — best-effort */
-        }
+        // 0×0 marks "probed but undecodable" so the row leaves the
+        // unprobed set and we don't re-fetch a broken image next crawl.
+        writes.push({
+          method: 'setSocialImageDimsForReferrers',
+          args: [{ src, width: dims?.width ?? 0, height: dims?.height ?? 0 }],
+        });
         if (dims) decoded++;
       }
     };
@@ -1635,6 +1761,7 @@ export class Crawler extends EventEmitter {
     const workers: Promise<void>[] = [];
     for (let i = 0; i < concurrency; i++) workers.push(worker());
     await Promise.all(workers);
+    await writes.flush();
     this.emit(
       'info',
       `Social-image probe complete: ${decoded}/${urls.length} dimensions read`,
@@ -1674,12 +1801,15 @@ export class Crawler extends EventEmitter {
     let withMeta = 0;
     let failed = 0;
 
+    const writes = this.newProbeWriteBuffer();
+    const progress = this.newProbeProgress('PDF probe', urls.length);
     const worker = async (): Promise<void> => {
       while (!this.stopped) {
         const idx = cursor++;
         if (idx >= urls.length) return;
         const url = urls[idx];
         if (!url) return;
+        await this.probeTick(progress);
         try {
           const result = await probePdfMetadata(url, {
             userAgent: this.resolveUserAgent(url),
@@ -1695,32 +1825,38 @@ export class Crawler extends EventEmitter {
             result.producer !== null;
           // 1 = metadata found, 0 = probed-but-empty, -1 = fetch/parse error.
           const status = result.error ? -1 : hasMeta ? 1 : 0;
-          this.db.setPdfMetadata({
-            url,
-            title: result.title,
-            author: result.author,
-            pageCount: result.pageCount,
-            creationDate: result.creationDate,
-            producer: result.producer,
-            status,
+          writes.push({
+            method: 'setPdfMetadata',
+            args: [
+              {
+                url,
+                title: result.title,
+                author: result.author,
+                pageCount: result.pageCount,
+                creationDate: result.creationDate,
+                producer: result.producer,
+                status,
+              },
+            ],
           });
           if (result.error) failed++;
           else if (hasMeta) withMeta++;
         } catch {
-          // Couldn't even fetch/record — mark errored so we don't loop on it.
-          try {
-            this.db.setPdfMetadata({
-              url,
-              title: null,
-              author: null,
-              pageCount: null,
-              creationDate: null,
-              producer: null,
-              status: -1,
-            });
-          } catch {
-            /* best-effort */
-          }
+          // Couldn't even fetch — mark errored so we don't loop on it.
+          writes.push({
+            method: 'setPdfMetadata',
+            args: [
+              {
+                url,
+                title: null,
+                author: null,
+                pageCount: null,
+                creationDate: null,
+                producer: null,
+                status: -1,
+              },
+            ],
+          });
           failed++;
         }
       }
@@ -1729,6 +1865,7 @@ export class Crawler extends EventEmitter {
     const workers: Promise<void>[] = [];
     for (let i = 0; i < concurrency; i++) workers.push(worker());
     await Promise.all(workers);
+    await writes.flush();
     this.emit(
       'info',
       `PDF metadata probe complete: ${withMeta}/${urls.length} with metadata, ${failed} failed`,
@@ -1795,32 +1932,40 @@ export class Crawler extends EventEmitter {
     let expired = 0;
     let expiringSoon = 0;
 
+    const writes = this.newProbeWriteBuffer();
+    const progress = this.newProbeProgress('TLS probe', hosts.length);
     const worker = async (): Promise<void> => {
       while (!this.stopped) {
         const idx = cursor++;
         if (idx >= hosts.length) return;
         const host = hosts[idx];
         if (!host) return;
+        await this.probeTick(progress);
         try {
           const info = await probeTlsCert(
             host,
             443,
             Math.max(2_000, this.config.requestTimeoutMs / 2),
           );
-          this.db.setHostCert({
-            host,
-            port: 443,
-            validFrom: info.validFrom,
-            validTo: info.validTo,
-            daysUntilExpiry: info.daysUntilExpiry,
-            issuer: info.issuer,
-            subject: info.subject,
-            signatureAlgorithm: info.signatureAlgorithm,
-            protocol: info.protocol,
-            chainLength: info.chainLength,
-            chainSubjects: info.chainSubjects,
-            probeStatus: info.error ? 0 : 200,
-            probeError: info.error,
+          writes.push({
+            method: 'setHostCert',
+            args: [
+              {
+                host,
+                port: 443,
+                validFrom: info.validFrom,
+                validTo: info.validTo,
+                daysUntilExpiry: info.daysUntilExpiry,
+                issuer: info.issuer,
+                subject: info.subject,
+                signatureAlgorithm: info.signatureAlgorithm,
+                protocol: info.protocol,
+                chainLength: info.chainLength,
+                chainSubjects: info.chainSubjects,
+                probeStatus: info.error ? 0 : 200,
+                probeError: info.error,
+              },
+            ],
           });
           if (info.daysUntilExpiry !== null) {
             if (info.daysUntilExpiry < 0) expired++;
@@ -1828,25 +1973,26 @@ export class Crawler extends EventEmitter {
           }
           probed++;
         } catch (err) {
-          try {
-            this.db.setHostCert({
-              host,
-              port: 443,
-              validFrom: null,
-              validTo: null,
-              daysUntilExpiry: null,
-              issuer: null,
-              subject: null,
-              signatureAlgorithm: null,
-              protocol: null,
-              chainLength: null,
-              chainSubjects: null,
-              probeStatus: 0,
-              probeError: err instanceof Error ? err.message : String(err),
-            });
-          } catch {
-            /* ignore — DB failure is non-fatal for the probe pass */
-          }
+          writes.push({
+            method: 'setHostCert',
+            args: [
+              {
+                host,
+                port: 443,
+                validFrom: null,
+                validTo: null,
+                daysUntilExpiry: null,
+                issuer: null,
+                subject: null,
+                signatureAlgorithm: null,
+                protocol: null,
+                chainLength: null,
+                chainSubjects: null,
+                probeStatus: 0,
+                probeError: err instanceof Error ? err.message : String(err),
+              },
+            ],
+          });
         }
       }
     };
@@ -1854,6 +2000,7 @@ export class Crawler extends EventEmitter {
     const workers: Promise<void>[] = [];
     for (let i = 0; i < concurrency; i++) workers.push(worker());
     await Promise.all(workers);
+    await writes.flush();
     this.emit(
       'info',
       `TLS probe complete: ${probed} hosts, ${expired} expired, ${expiringSoon} expiring ≤30d`,
@@ -3105,6 +3252,14 @@ export class Crawler extends EventEmitter {
    * evenly-spaced slots. `maxRps <= 0` disables the gate (unlimited).
    */
   private async acquireRateSlot(): Promise<void> {
+    // Adaptive speed (opt-in): honour a hard cooldown set by a 429/503 or a
+    // `Retry-After` header before this dispatch even reserves a slot. The
+    // cooldown is a *global* pause — every worker waits it out — which is
+    // exactly how a server's rate-limit window works.
+    if (this.config.adaptiveSpeed && this.adaptiveCooldownUntil > 0) {
+      const waitCooldown = this.adaptiveCooldownUntil - Date.now();
+      if (waitCooldown > 0 && !this.stopped) await sleep(waitCooldown);
+    }
     // A robots.txt `Crawl-delay` means "one request every N seconds to this
     // origin" — a *global* spacing, not a per-worker one. Applying it as a
     // per-worker post-request sleep (as this used to) both violated the
@@ -3113,7 +3268,16 @@ export class Crawler extends EventEmitter {
     // literally when the user opts in. Only reached when `respectCrawlDelay`
     // is on — `robotsCrawlDelayMs` stays 0 otherwise.
     const rpsInterval = this.config.maxRps > 0 ? 1000 / this.config.maxRps : 0;
-    const minInterval = Math.max(rpsInterval, this.robotsCrawlDelayMs);
+    let minInterval = Math.max(rpsInterval, this.robotsCrawlDelayMs);
+    // Adaptive slow-down: widen the dispatch interval by the current scale
+    // (0.1–1). When the user left RPS unlimited (0), a base derived from the
+    // concurrency ceiling gives the controller a rate to shrink from once a
+    // 429 arrives — otherwise there is nothing to slow.
+    if (this.config.adaptiveSpeed && this.adaptiveRateScale < 1) {
+      const baseRps =
+        this.config.maxRps > 0 ? this.config.maxRps : Math.max(1, this.config.maxConcurrency);
+      minInterval = Math.max(minInterval, 1000 / (baseRps * this.adaptiveRateScale));
+    }
     if (minInterval <= 0) return;
     const now = Date.now();
     const scheduled = Math.max(now, this.nextDispatchAt);
@@ -3122,6 +3286,100 @@ export class Crawler extends EventEmitter {
     if (waitMs > 0 && !this.stopped) await sleep(waitMs);
   }
 
+  // ==================================================================
+  // Adaptive speed controller (opt-in via `config.adaptiveSpeed`).
+  //
+  // The fixed-rate crawler holds `maxRps` / `maxConcurrency` no matter how
+  // the origin responds, so a site whose Cloudflare rate-limits (thinpo.com)
+  // gets 429s the moment the first burst lands — and, because nothing backs
+  // off, restarting keeps re-tripping the same penalty window. With adaptive
+  // speed on, a 429/503 (or a `Retry-After`) both pauses dispatch for the
+  // penalty window and permanently trims the rate a notch; a long stretch of
+  // clean responses grows it back toward the configured ceiling. Every field
+  // and branch here is dead unless the opt-in is set.
+  // ==================================================================
+
+  /** Current fraction (0.1–1) of the configured rate/concurrency in effect. */
+  private adaptiveRateScale = 1;
+  /** Dispatch is paused until this epoch-ms (Retry-After / 429 breather). */
+  private adaptiveCooldownUntil = 0;
+  /** Consecutive clean (non-limited) responses, for gradual recovery. */
+  private adaptiveCleanStreak = 0;
+  /** Last time the rate was cut — dedups a burst of 429s into one step. */
+  private lastAdaptiveCutTs = 0;
+  private static readonly ADAPTIVE_MIN_SCALE = 0.1;
+  private static readonly ADAPTIVE_CUT_COOLDOWN_MS = 1_000;
+  private static readonly ADAPTIVE_RECOVERY_STREAK = 25;
+  private static readonly ADAPTIVE_MAX_COOLDOWN_MS = 120_000;
+
+  /** Base rate the scale is applied to, for user-facing rate messages. */
+  private adaptiveBaseRps(): number {
+    return this.config.maxRps > 0 ? this.config.maxRps : Math.max(1, this.config.maxConcurrency);
+  }
+
+  /**
+   * React to a rate-limit signal (HTTP 429/503, optionally with a
+   * `Retry-After`). Sets the cooldown pause and, at most once per
+   * {@link ADAPTIVE_CUT_COOLDOWN_MS}, halves the effective rate + concurrency.
+   */
+  private noteRateLimited(retryAfterMs: number | null): void {
+    if (!this.config.adaptiveSpeed || !this.running) return;
+    const now = Date.now();
+    const cooldown =
+      retryAfterMs && retryAfterMs > 0
+        ? Math.min(retryAfterMs, Crawler.ADAPTIVE_MAX_COOLDOWN_MS)
+        : 2_000;
+    this.adaptiveCooldownUntil = Math.max(this.adaptiveCooldownUntil, now + cooldown);
+    this.adaptiveCleanStreak = 0;
+    if (now - this.lastAdaptiveCutTs < Crawler.ADAPTIVE_CUT_COOLDOWN_MS) return;
+    this.lastAdaptiveCutTs = now;
+    const prev = this.adaptiveRateScale;
+    this.adaptiveRateScale = Math.max(Crawler.ADAPTIVE_MIN_SCALE, this.adaptiveRateScale * 0.5);
+    // Concurrency is trimmed through the shared throttle map (composes with
+    // thermal / multi-session sources via minimum), so this never fights the
+    // OS-pressure throttle.
+    this.setThrottleScale(this.adaptiveRateScale, 'adaptive');
+    if (this.adaptiveRateScale !== prev) {
+      this.emit(
+        'info',
+        `Adaptive speed: rate limited — slowing to ~${(
+          this.adaptiveBaseRps() * this.adaptiveRateScale
+        ).toFixed(1)} URL/s${
+          retryAfterMs ? `, honouring Retry-After ${Math.round(cooldown / 1000)}s` : ''
+        }`,
+      );
+    }
+  }
+
+  /**
+   * React to a clean response. After {@link ADAPTIVE_RECOVERY_STREAK}
+   * consecutive non-limited responses past the cooldown window, nudge the
+   * rate + concurrency one step back toward the configured ceiling.
+   */
+  private noteCleanResponse(): void {
+    if (!this.config.adaptiveSpeed || !this.running) return;
+    if (this.adaptiveRateScale >= 1) return;
+    if (Date.now() < this.adaptiveCooldownUntil) return;
+    if (++this.adaptiveCleanStreak < Crawler.ADAPTIVE_RECOVERY_STREAK) return;
+    this.adaptiveCleanStreak = 0;
+    this.adaptiveRateScale = Math.min(1, this.adaptiveRateScale + 0.1);
+    this.setThrottleScale(this.adaptiveRateScale, 'adaptive');
+    // setThrottleScale raises the ceiling but never grows the live worker
+    // count on its own; step it up by one so recovery is visible even when
+    // no renderer-lag samples are arriving to drive the lag-adaptive loop.
+    const ceiling = this.effectiveCeiling();
+    if (this.currentConcurrency < ceiling) {
+      this.currentConcurrency = Math.min(ceiling, this.currentConcurrency + 1);
+      this.queue.concurrency = this.currentConcurrency;
+    }
+    this.emit(
+      'debug',
+      `Adaptive speed: recovering → ~${(this.adaptiveBaseRps() * this.adaptiveRateScale).toFixed(
+        1,
+      )} URL/s (concurrency ${this.currentConcurrency}/${ceiling})`,
+    );
+  }
+
   private async fetchAndProcess(item: QueueItem): Promise<void> {
     if (this.stopped) return;
     this.setOp(`crawl:fetch:${item.url}`);
@@ -3240,6 +3498,16 @@ export class Crawler extends EventEmitter {
           }`,
         );
       }
+      // Adaptive speed: let the final response of every URL steer the rate —
+      // a 429/503 slows it (and honours any Retry-After), a clean response
+      // counts toward recovering it. No-op unless the opt-in is set.
+      if (this.config.adaptiveSpeed) {
+        if (statusCode === 429 || statusCode === 503) {
+          this.noteRateLimited(parseRetryAfterMs(res.headers.get('retry-after')));
+        } else if (statusCode < 400) {
+          this.noteCleanResponse();
+        }
+      }
       const contentLengthHeader = res.headers.get('content-length');
       const xRobotsTag = res.headers.get('x-robots-tag');
       // Security / performance headers — captured per URL for the Security
@@ -3710,12 +3978,15 @@ export class Crawler extends EventEmitter {
       // itself stays whole: traversal below and the page's outlink count
       // both describe what the page declared, which doesn't change just
       // because the user stopped recording one side of the graph.
+      // A missing flag (a config assembled outside the Settings dialog —
+      // an older saved project, an MCP override) means "store", never
+      // "drop": a silently empty link graph is the worse failure.
+      const storeInternalLinks = this.config.storeInternalLinks ?? true;
+      const storeExternalLinks = this.config.storeExternalLinks ?? true;
       const persistedLinks: DiscoveredLink[] = [];
-      if (this.config.storeInternalLinks || this.config.storeExternalLinks) {
+      if (storeInternalLinks || storeExternalLinks) {
         for (const l of storableLinks) {
-          const keep = l.isInternal
-            ? this.config.storeInternalLinks
-            : this.config.storeExternalLinks;
+          const keep = l.isInternal ? storeInternalLinks : storeExternalLinks;
           if (keep) persistedLinks.push(l);
         }
       }
@@ -3940,6 +4211,15 @@ export class Crawler extends EventEmitter {
           : null,
         links: persistedLinks,
         images: this.config.storeImages ? parsed.images : [],
+        // Whether the empty lists above mean "this page has none"
+        // (storage on → the writer replaces the page's rows) or "not
+        // recording" (storage off → it keeps what an earlier crawl stored).
+        storeLinks:
+          storeInternalLinks ||
+          storeExternalLinks ||
+          this.config.storeIframes ||
+          this.config.crawlInvalidLinks,
+        storeImages: this.config.storeImages,
         fromDepth: item.depth,
       });
       // V2 Faz 1 — Persist render artefacts inline so they finish before
@@ -4584,10 +4864,28 @@ export class Crawler extends EventEmitter {
           /* ignore */
         }
         lastError = new Error(`HTTP ${res.status}`);
+        // Adaptive speed: a retryable 429/503 feeds the controller so the
+        // *global* dispatch rate widens for every worker, and its
+        // `Retry-After` drives this URL's own backoff instead of the fixed
+        // exponential curve — retrying faster than the server allows is what
+        // deepens a Cloudflare penalty window.
+        let adaptiveRetryMs: number | null = null;
+        if (this.config.adaptiveSpeed && (res.status === 429 || res.status === 503)) {
+          adaptiveRetryMs = parseRetryAfterMs(res.headers.get('retry-after'));
+          this.noteRateLimited(adaptiveRetryMs);
+        }
         this.emit(
           'warn',
           `Retry ${attempt + 1}/${maxAttempts - 1} for ${url}: HTTP ${res.status} after ${ttfbMs}ms`,
         );
+        if (adaptiveRetryMs !== null) {
+          const wait = Math.min(
+            Math.max(adaptiveRetryMs, baseDelay * 2 ** attempt),
+            Crawler.ADAPTIVE_MAX_COOLDOWN_MS,
+          );
+          if (!this.stopped) await sleep(wait);
+          continue;
+        }
       } catch (err) {
         lastError = err;
         const elapsedMs = Date.now() - (this.startedAt > 0 ? this.startedAt : Date.now());
@@ -4814,6 +5112,20 @@ function yieldToEventLoop(): Promise<void> {
   return new Promise((resolve) => setImmediate(resolve));
 }
 
+/**
+ * Parse a `Retry-After` header to milliseconds. Accepts the two RFC forms:
+ * a delta-seconds integer (`120`) and an HTTP-date (`Wed, 21 Oct 2026 …`).
+ * Returns null when absent/unparseable; a past date clamps to 0.
+ */
+export function parseRetryAfterMs(header: string | null): number | null {
+  if (!header) return null;
+  const s = header.trim();
+  if (/^\d+$/.test(s)) return Number.parseInt(s, 10) * 1000;
+  const when = Date.parse(s);
+  if (Number.isFinite(when)) return Math.max(0, when - Date.now());
+  return null;
+}
+
 function isRetryableStatus(status: number): boolean {
   return status === 408 || status === 425 || status === 429 || status >= 500;
 }

packages/core/src/dataset-export.ts

@@ -0,0 +1,118 @@
+import type { ProjectDb } from '@freecrawl/db';
+import type { ExportDatasetKey } from '@freecrawl/shared-types';
+
+/**
+ * Row sources for the non-URL-row export tables (see `ExportDatasetKey`).
+ *
+ * Every dataset already has a paged query on `ProjectDb` that the matching
+ * tab reads through IPC; the exporter walks the same query to its end so
+ * the file holds the whole table, not the first screenful the tab happened
+ * to load. Rows are yielded as the query returns them — nested (`gsc`,
+ * `mobile`, `seo.metrics`) — and the writers read the dotted column keys
+ * from `EXPORT_DATASET_COLUMNS` straight off that shape.
+ */
+
+/** Per-host context the datasets that are keyed by account need. */
+export interface DatasetExportContext {
+  /** Linked Google account whose Search Console pull to export. */
+  gscAccountId?: string;
+  /** Linked Google account whose GA4 pull to export. */
+  ga4AccountId?: string;
+  /** Search Console URL-matching options (Settings → Integrations → GSC). */
+  gscMatchSlash?: boolean;
+  gscMatchCase?: boolean;
+}
+
+const PAGE_SIZE = 2000;
+
+function* paged<T>(
+  fetchPage: (limit: number, offset: number) => { rows: T[] },
+): Generator<T> {
+  let offset = 0;
+  for (;;) {
+    const { rows } = fetchPage(PAGE_SIZE, offset);
+    for (const row of rows) yield row;
+    if (rows.length < PAGE_SIZE) return;
+    offset += PAGE_SIZE;
+  }
+}
+
+export function datasetRows(
+  db: ProjectDb,
+  dataset: ExportDatasetKey,
+  ctx: DatasetExportContext = {},
+): Iterable<Record<string, unknown>> {
+  const asRecords = <T>(gen: Generator<T>): Iterable<Record<string, unknown>> =>
+    gen as Iterable<unknown> as Iterable<Record<string, unknown>>;
+  switch (dataset) {
+    case 'images':
+      return asRecords(paged((limit, offset) => db.queryImages({ limit, offset })));
+    case 'broken-links':
+      return asRecords(
+        paged((limit, offset) => db.queryBrokenLinks({ limit, offset, internal: 'all' })),
+      );
+    case 'pagespeed':
+      return asRecords(paged((limit, offset) => db.queryPagespeed({ limit, offset })));
+    case 'crux':
+      return asRecords(paged((limit, offset) => db.queryCrux({ limit, offset })));
+    case 'spelling':
+      return asRecords(paged((limit, offset) => db.querySpelling({ limit, offset })));
+    case 'search-console':
+      return asRecords(
+        paged((limit, offset) =>
+          db.queryGsc({
+            limit,
+            offset,
+            accountId: ctx.gscAccountId,
+            matchSlash: ctx.gscMatchSlash,
+            matchCase: ctx.gscMatchCase,
+          }),
+        ),
+      );
+    case 'analytics':
+      return asRecords(
+        paged((limit, offset) => db.queryGa4({ limit, offset, accountId: ctx.ga4AccountId })),
+      );
+    default: {
+      // Provider-keyed datasets: `ai:<provider>` / `seo:<provider>`.
+      const sep = dataset.indexOf(':');
+      const family = dataset.slice(0, sep);
+      const provider = dataset.slice(sep + 1);
+      if (family === 'ai') {
+        return asRecords(
+          paged((limit, offset) =>
+            db.queryAi({ limit, offset, provider: provider as 'openai' | 'anthropic' | 'ollama' }),
+          ),
+        );
+      }
+      if (family === 'seo') {
+        return asRecords(
+          paged((limit, offset) =>
+            db.querySeo({
+              limit,
+              offset,
+              provider: provider as 'ahrefs' | 'majestic' | 'moz' | 'semrush',
+            }),
+          ),
+        );
+      }
+      throw new Error(`exportTabular: unknown dataset '${dataset as string}'`);
+    }
+  }
+}
+
+/**
+ * Read one export cell. Column keys are dotted paths (`gsc.clicks`) so a
+ * dataset row's nested result objects export as flat columns; a missing
+ * intermediate (`gsc: null` — no Search Console data for the page) reads
+ * as an empty cell instead of throwing on the row.
+ */
+export function readCell(row: Record<string, unknown>, key: string): unknown {
+  if (key.indexOf('.') < 0) return row[key];
+  let cur: unknown = row;
+  for (const part of key.split('.')) {
+    if (cur === null || cur === undefined || typeof cur !== 'object') return null;
+    cur = (cur as Record<string, unknown>)[part];
+  }
+  return cur ?? null;
+}

packages/core/src/index.ts

@@ -58,6 +58,7 @@ export {
   type GridExportOptions,
   type GridExportResult,
 } from './tabular-export.js';
+export { datasetRows, readCell, type DatasetExportContext } from './dataset-export.js';
 export { ensureHeapHeadroom, heapHeadroomBytes } from './heap-guard.js';
 export {
   testUrlAgainstRobots,
@@ -135,6 +136,16 @@ export {
   buildLogExportCsv,
   type LogExportTables,
 } from './log-export.js';
+export {
+  classifyRequest,
+  classifyIpOwner,
+  decodeTarget,
+  ownerOfHostname,
+  THREAT_RULES,
+  THREAT_THRESHOLD,
+  type ThreatRule,
+  type ThreatMatch,
+} from './log-threats.js';
 export {
   buildXlsx,
   buildCsvTable,

packages/core/src/log-analyzer.ts

@@ -12,13 +12,16 @@
  *   - status     : response-code distribution from the log (item 8)
  *   - daily      : per-day, per-bot-family hit trend (item 10)
  *   - bots       : per-bot hit + distinct-IP + verified-IP counts (item 5)
+ *   - threats    : per-line rows for requests the threat classifier
+ *                  flagged (SQLi / XSS / scanner probes …) — the one
+ *                  place the analyzer keeps IP + full target, capped
  */
 
 import { createReadStream } from 'node:fs';
 import { createInterface } from 'node:readline';
 import type { Readable } from 'node:stream';
 import { createGunzip } from 'node:zlib';
-import type { LogFormat, LogFormatChoice } from '@freecrawl/shared-types';
+import type { LogFormat, LogFormatChoice, LogThreatHit } from '@freecrawl/shared-types';
 import {
   createLogParser,
   detectLogFormat,
@@ -26,6 +29,7 @@ import {
   type ParsedLogLine,
 } from './log-parser.js';
 import { detectBot, verifyBotByRdns, type BotFamily } from './bot-detect.js';
+import { classifyIpOwner, classifyRequest } from './log-threats.js';
 
 export interface LogUrlStat {
   path: string;
@@ -69,20 +73,29 @@ export interface LogAnalysisResult {
   bots: Map<string, LogBotAgg>;
   /** True when the distinct-URL cap was reached and new URLs were dropped. */
   urlCapHit: boolean;
+  /** Requests the threat classifier flagged, in file order. */
+  threats: LogThreatHit[];
+  /** Flagged lines dropped after `maxThreatRows` was reached. */
+  threatsDropped: number;
 }
 
 export interface LogAnalyzeOptions {
   format?: LogFormatChoice;
   customRegex?: string;
+  /** Reverse-DNS verify sampled bot IPs — and, for flagged requests, whether
+   *  the sender is a search engine's own infrastructure. */
   verifyBots?: boolean;
   /** Distinct-URL cap to bound memory on pathological logs. Default 500k. */
   maxDistinctUrls?: number;
   /** Per-bot distinct-IP sample cap (also the verification cap). Default 200. */
   maxIpsPerBot?: number;
+  /** Cap on flagged-request rows kept per file. Default 50k. */
+  maxThreatRows?: number;
 }
 
 const DEFAULT_MAX_URLS = 500_000;
 const DEFAULT_MAX_IPS = 200;
+const DEFAULT_MAX_THREATS = 50_000;
 
 /**
  * Open a log file as a UTF-8 text stream, transparently decompressing
@@ -119,6 +132,8 @@ function emptyResult(format: LogFormat): LogAnalysisResult {
     status: new Map(),
     bots: new Map(),
     urlCapHit: false,
+    threats: [],
+    threatsDropped: 0,
   };
 }
 
@@ -143,6 +158,7 @@ export async function analyzeLogFile(
   const result = emptyResult(format);
   const maxUrls = opts.maxDistinctUrls ?? DEFAULT_MAX_URLS;
   const maxIps = opts.maxIpsPerBot ?? DEFAULT_MAX_IPS;
+  const maxThreats = opts.maxThreatRows ?? DEFAULT_MAX_THREATS;
 
   const rl = createInterface({
     input: openLogStream(filePath),
@@ -162,11 +178,12 @@ export async function analyzeLogFile(
       continue;
     }
     result.parsedLines++;
-    accumulate(result, parsed, maxUrls, maxIps);
+    accumulate(result, parsed, maxUrls, maxIps, maxThreats);
   }
 
   if (opts.verifyBots) {
     await verifyBots(result, maxIps);
+    await verifyThreatIps(result, maxIps);
   }
   return result;
 }
@@ -191,10 +208,39 @@ function accumulate(
   p: ParsedLogLine,
   maxUrls: number,
   maxIps: number,
+  maxThreats: number,
 ): void {
   const bot = detectBot(p.userAgent);
   const isBot = bot !== null;
 
+  // Threat classification — the only per-line data kept. Runs before the
+  // URL cap check below so a scanner burst of unique targets (each one a
+  // new "URL") is still recorded once the URL aggregate is full.
+  const threat = classifyRequest(p);
+  if (threat) {
+    if (result.threats.length >= maxThreats) {
+      result.threatsDropped++;
+    } else {
+      result.threats.push({
+        ts: p.ts,
+        ip: p.ip,
+        method: p.method,
+        path: p.path!,
+        decoded: threat.decoded,
+        status: p.status,
+        bytes: p.bytes,
+        userAgent: p.userAgent,
+        referer: p.referer,
+        bot: bot ? bot.name : null,
+        category: threat.category,
+        rules: threat.rules,
+        score: threat.score,
+        evidence: threat.evidence,
+        ipOwner: null,
+      });
+    }
+  }
+
   // Timestamp bounds + daily trend.
   if (p.ts !== null) {
     if (result.minTs === null || p.ts < result.minTs) result.minTs = p.ts;
@@ -287,6 +333,28 @@ async function verifyBots(result: LogAnalysisResult, maxIps: number): Promise<vo
   }
 }
 
+/**
+ * Reverse-DNS the senders of flagged requests (distinct IPs, capped) and
+ * mark the ones that are a search engine's own infrastructure. That is
+ * the difference between "block this IP" and "this payload was relayed
+ * through Google — fix the application instead".
+ */
+async function verifyThreatIps(result: LogAnalysisResult, maxIps: number): Promise<void> {
+  const ips = new Set<string>();
+  for (const t of result.threats) {
+    if (t.ip) ips.add(t.ip);
+    if (ips.size >= maxIps) break;
+  }
+  if (ips.size === 0) return;
+  const list = Array.from(ips);
+  const owners = await mapLimit(list, 8, (ip) => classifyIpOwner(ip));
+  const byIp = new Map<string, string | null>();
+  list.forEach((ip, i) => byIp.set(ip, owners[i] ?? null));
+  for (const t of result.threats) {
+    if (t.ip) t.ipOwner = byIp.get(t.ip) ?? null;
+  }
+}
+
 async function mapLimit<T, R>(
   items: T[],
   limit: number,

packages/core/src/log-export.ts

@@ -13,6 +13,8 @@ import type {
   LogDiscoveryRow,
   LogOverview,
   LogStatusRow,
+  LogThreatIpRow,
+  LogThreatRow,
   LogTrendRow,
   LogUrlStatRow,
 } from '@freecrawl/shared-types';
@@ -32,6 +34,10 @@ export interface LogExportTables {
   trend: LogTrendRow[];
   crawlBudget: LogCrawlBudgetRow[];
   discovery: LogDiscoveryRow[];
+  /** Flagged requests + their senders (Suspicious Requests tab). Optional
+   *  so older callers that never ran the classifier still export. */
+  threats?: LogThreatRow[];
+  threatIps?: LogThreatIpRow[];
 }
 
 interface Section {
@@ -176,6 +182,51 @@ function buildSections(tables: LogExportTables): Section[] {
       ],
       rows: tables.discovery.map((r) => ({ ...r })),
     },
+    {
+      name: 'Suspicious Requests',
+      columns: [
+        { key: 'time', label: 'Time' },
+        { key: 'ip', label: 'IP' },
+        { key: 'ipOwner', label: 'IP Owner' },
+        { key: 'method', label: 'Method' },
+        { key: 'decoded', label: 'Request (decoded)' },
+        { key: 'path', label: 'Request (raw)' },
+        { key: 'status', label: 'Status' },
+        { key: 'category', label: 'Category' },
+        { key: 'score', label: 'Score' },
+        { key: 'evidence', label: 'Evidence' },
+        { key: 'rules', label: 'Rules' },
+        { key: 'bot', label: 'Declared Bot' },
+        { key: 'userAgent', label: 'User-Agent' },
+        { key: 'referer', label: 'Referer' },
+        { key: 'targetInCrawl', label: 'Target In Crawl' },
+      ],
+      rows: (tables.threats ?? []).map((r) => ({
+        ...r,
+        time: fmtTs(r.ts),
+        rules: r.rules.join(' '),
+        targetInCrawl: r.targetInCrawl ? 'yes' : 'no',
+      })),
+    },
+    {
+      name: 'Suspicious IPs',
+      columns: [
+        { key: 'ip', label: 'IP' },
+        { key: 'hits', label: 'Flagged Requests' },
+        { key: 'servedHits', label: 'Answered 2xx' },
+        { key: 'categories', label: 'Categories' },
+        { key: 'firstSeen', label: 'First Seen' },
+        { key: 'lastSeen', label: 'Last Seen' },
+        { key: 'bot', label: 'Declared Bot' },
+        { key: 'ipOwner', label: 'IP Owner' },
+      ],
+      rows: (tables.threatIps ?? []).map((r) => ({
+        ...r,
+        categories: r.categories.join(' '),
+        firstSeen: fmtTs(r.firstTs),
+        lastSeen: fmtTs(r.lastTs),
+      })),
+    },
   ];
 }
 

packages/core/src/log-threats.ts

@@ -0,0 +1,360 @@
+/**
+ * Access-log threat classifier — the engine behind the Log Analyzer's
+ * "Suspicious Requests" tab.
+ *
+ * A site owner reading raw logs spots SQL-injection payloads in the 404s;
+ * this module does the same thing per line during the single ingest
+ * pass, so the analyzer can show *which IP sent what* without keeping a
+ * copy of every hit. It is deliberately a signature scorer, not a WAF:
+ *
+ *   - The request target is percent-decoded (twice, for double-encoded
+ *     payloads), `+` is read as a space, and inline SQL comments used
+ *     for evasion (`union` / `select` split by a comment) are collapsed
+ *     before matching.
+ *   - Rules are multi-token patterns with weights. A line is flagged
+ *     when its summed weight reaches {@link THREAT_THRESHOLD}; lone weak
+ *     hints (a stray quote, a `select` in a search term, `{{utm}}` from a
+ *     broken mail template) stay below it. Naive keyword matching would
+ *     flag the owner's own `wp-admin/update.php?action=update-selected`.
+ *   - Everything is offline and pure; the optional reverse-DNS owner
+ *     check is the only I/O and is opt-in.
+ */
+
+import { promises as dns } from 'node:dns';
+import type { LogThreatCategory } from '@freecrawl/shared-types';
+
+export interface ThreatRule {
+  id: string;
+  category: LogThreatCategory;
+  weight: number;
+  re: RegExp;
+  /** Only counts for this HTTP method (e.g. a POST to xmlrpc.php). */
+  method?: string;
+  /** Match the raw (still-encoded) target instead of the decoded one. */
+  raw?: boolean;
+}
+
+export interface ThreatMatch {
+  category: LogThreatCategory;
+  /** Matched rule ids, strongest first. */
+  rules: string[];
+  score: number;
+  /** Fragment the strongest rule matched (≤ 120 chars). */
+  evidence: string;
+  /** Decoded target (what the UI shows as the payload). */
+  decoded: string;
+}
+
+/** Minimum summed rule weight for a line to count as a threat. */
+export const THREAT_THRESHOLD = 5;
+
+const EVIDENCE_MAX = 120;
+
+function rule(
+  id: string,
+  category: LogThreatCategory,
+  weight: number,
+  re: RegExp,
+  extra: Pick<ThreatRule, 'method' | 'raw'> = {},
+): ThreatRule {
+  return { id, category, weight, re, ...extra };
+}
+
+/**
+ * Cheap gate run on the raw target before the rule set. Any line without
+ * one of these characters / path fragments cannot match a rule, and on a
+ * WordPress log that is the vast majority of lines.
+ */
+const PREFILTER =
+  /['"<>;|`(){}\\$*]|%2[27]|%3[ce]|%2[89]|%00|%0[ad]|%25|union|select|sleep|benchmark|updatexml|extractvalue|information_schema|waitfor|concat|order.by|script|onerror|onload|javascript|alert|document\.|fromcharcode|etc\/|system32|\.\.|\.env|\.git|\.svn|\.hg|\.ssh|\.aws|\.htaccess|\.htpasswd|\.ds_store|\.sql|\.bak|\.old|\.orig|\.swp|\.zip|\.tar|\.tgz|\.gz|\.rar|\.7z|\.pem|\.key|\.log|wp-config|xmlrpc|wp-login|phpmyadmin|adminer|phpunit|eval-stdin|cgi-bin|actuator|console|manager\/html|solr|jenkins|hudson|wlwmanifest|autodiscover|\/owa\/|\/ecp\/|\/ews\/|id_rsa|shell|c99|r57|wso|backup|dump|passwd|win\.ini|boot\.ini|proc\/|server-status|server-info|phpinfo|jndi|debug\.log|config\.|credentials|secrets|invoker|druid|ignition|telescope|typo3|joomla|drupal|magento|administrator/i;
+
+const RULES: ThreatRule[] = [
+  // ---- SQL injection -------------------------------------------------
+  rule('sqli-union-select', 'sqli', 9, /union\s+(all\s+|distinct\s+)?select\b/i),
+  rule(
+    'sqli-error-fn',
+    'sqli',
+    9,
+    /\b(updatexml|extractvalue|benchmark|pg_sleep|load_file|xp_cmdshell|sp_executesql)\s*\(/i,
+  ),
+  rule('sqli-sleep', 'sqli', 8, /\bsleep\s*\(\s*\d/i),
+  rule('sqli-waitfor', 'sqli', 8, /\bwaitfor\s+delay\b/i),
+  rule('sqli-schema', 'sqli', 8, /\binformation_schema\b|\bsysobjects\b|\bpg_catalog\b/i),
+  rule(
+    'sqli-stacked',
+    'sqli',
+    8,
+    /;\s*(drop|insert|update|delete|alter|truncate|create|exec|execute)\b/i,
+  ),
+  rule('sqli-quote-logic', 'sqli', 7, /'\s*(or|and)\s+['"\d(]/i),
+  rule('sqli-quote-tautology', 'sqli', 6, /'\s*=\s*'|"\s*=\s*"/),
+  rule('sqli-tautology', 'sqli', 5, /\b(or|and)\s+\d+\s*=\s*\d+/i),
+  rule('sqli-select-from', 'sqli', 4, /\bselect\b[\s\S]{0,120}?\bfrom\b/i),
+  rule('sqli-comment', 'sqli', 4, /'\s*(--|#|\/\*)|\)\s*--|--\s*-?$/),
+  rule('sqli-concat', 'sqli', 4, /\b(concat|group_concat|concat_ws|char|chr|cast|convert)\s*\(/i),
+  rule('sqli-order-by', 'sqli', 3, /\border\s+by\s+\d+/i),
+  rule('sqli-hex', 'sqli', 2, /0x[0-9a-f]{6,}/i),
+  // ---- Cross-site scripting -----------------------------------------
+  rule('xss-script-tag', 'xss', 8, /<\s*\/?\s*script\b/i),
+  rule(
+    'xss-event-handler',
+    'xss',
+    6,
+    /\bon(error|load|mouseover|mouseenter|focus|click|input|animationstart|toggle|pointerover)\s*=/i,
+  ),
+  rule(
+    'xss-dom',
+    'xss',
+    6,
+    /document\s*\.\s*(cookie|location|write|domain)|window\s*\.\s*location|string\.fromcharcode/i,
+  ),
+  rule(
+    'xss-tag',
+    'xss',
+    5,
+    /<\s*(img|svg|iframe|body|object|embed|video|audio|details|math|marquee)\b/i,
+  ),
+  rule('xss-js-uri', 'xss', 5, /javascript\s*:/i),
+  rule('xss-alert', 'xss', 4, /\b(alert|prompt|confirm)\s*\(/i),
+  // ---- Path traversal / local file inclusion --------------------------
+  rule('traversal-etc', 'traversal', 9, /\/etc\/(passwd|shadow|hosts|group|issue)\b/i),
+  rule('traversal-deep', 'traversal', 8, /(\.\.[\\/]){2,}/),
+  rule('traversal-windows', 'traversal', 8, /\b(win\.ini|boot\.ini|system32)\b|c:\\windows/i),
+  rule('traversal-proc', 'traversal', 8, /\/proc\/(self|version|cpuinfo)\b/i),
+  rule('traversal-wrapper', 'traversal', 8, /\b(php|file|data|expect|zip|phar):\/\//i),
+  rule('traversal-null', 'traversal', 6, /\0/),
+  rule('traversal-single', 'traversal', 4, /\.\.[\\/]/),
+  // ---- Command injection / template injection / log4shell ------------
+  rule('cmdi-jndi', 'cmdi', 10, /\$\{\s*jndi\s*:|jndi:(ldap|rmi|dns|iiop)/i),
+  rule(
+    'cmdi-shell',
+    'cmdi',
+    8,
+    /[;|&`]\s*(cat|ls|id|whoami|wget|curl|bash|sh|nc|ncat|python3?|perl|php|ruby|echo|uname|ping|nslookup|dig|rm|chmod|powershell|cmd\.exe|certutil)\b/i,
+  ),
+  rule(
+    'cmdi-runtime',
+    'cmdi',
+    8,
+    /\b(runtime\.exec|processbuilder|passthru|shell_exec|system|popen|proc_open|eval|assert|base64_decode)\s*\(/i,
+  ),
+  rule('cmdi-subshell', 'cmdi', 5, /\$\(/),
+  // One rule for every unrendered-template shape: a mail template that
+  // leaked `?utm={{campaign}}&src=${source}` is one weak hint, not two.
+  rule('cmdi-template', 'cmdi', 3, /\$\{[^}]*\}|\{\{[^}]*\}\}|\{%[^%]*%\}|<%[^%]*%>/),
+  // ---- Scanner / vulnerability probes ---------------------------------
+  rule('scan-phpunit', 'scanner', 9, /vendor\/phpunit|eval-stdin\.php/i),
+  rule('scan-wp-config', 'scanner', 8, /wp-config\.php|wp-config\.(bak|old|txt|save|orig|swp)/i),
+  rule(
+    'scan-dotfile',
+    'scanner',
+    7,
+    /(^|\/)\.(env|git|svn|hg|ssh|aws|docker|npmrc|bash_history|htpasswd|htaccess|ds_store)\b/i,
+  ),
+  rule(
+    'scan-webshell',
+    'scanner',
+    7,
+    /\/(shell|cmd|c99|r57|wso|alfa|webshell|b374k|filemanager)\.php\b/i,
+  ),
+  rule(
+    'scan-phpmyadmin',
+    'scanner',
+    6,
+    /phpmyadmin|\/pma\/|\/myadmin|\/adminer|\/mysqladmin|\/dbadmin/i,
+  ),
+  rule('scan-debug-log', 'scanner', 6, /\/wp-content\/debug\.log\b/i),
+  rule(
+    'scan-admin-consoles',
+    'scanner',
+    6,
+    /\/actuator\/|\/manager\/html|\/solr\/|\/jenkins\b|\/hudson\b|\/jmx-console|\/web-console|\/invoker\/|\/druid\/|\/_ignition\/|\/telescope\/|\/debug\/default\/view|\/api\/v1\/pods|\/console\/?(\?|$)/i,
+  ),
+  rule('scan-wlw', 'scanner', 5, /wlwmanifest\.xml/i),
+  rule('scan-phpinfo', 'scanner', 5, /\/(phpinfo|info)\.php\b/i),
+  rule('scan-xmlrpc', 'scanner', 4, /\/xmlrpc\.php\b/i),
+  rule('scan-xmlrpc-post', 'scanner', 3, /\/xmlrpc\.php\b/i, { method: 'POST' }),
+  rule('scan-wp-login', 'scanner', 2, /\/wp-login\.php\b/i),
+  rule('scan-wp-login-post', 'scanner', 3, /\/wp-login\.php\b/i, { method: 'POST' }),
+  rule(
+    'scan-exchange',
+    'scanner',
+    4,
+    /\/autodiscover\/autodiscover\.(xml|json)|\/owa\/|\/ecp\/|\/ews\//i,
+  ),
+  rule(
+    'scan-cms-probe',
+    'scanner',
+    4,
+    /\/administrator\/(index\.php|manifests)|\/magento_version|\/typo3\/|\/joomla|\/drupal|\/wp-content\/plugins\/[^/]+\/readme\.txt/i,
+  ),
+  rule(
+    'scan-config-file',
+    'scanner',
+    4,
+    /\/(config|configuration|settings|database|db|credentials|secrets|application)\.(php|json|yml|yaml|xml|ini|env|txt)(\.|~|\b)/i,
+  ),
+  rule('scan-server-status', 'scanner', 3, /\/server-status|\/server-info|\/nginx_status/i),
+  rule('scan-cgi', 'scanner', 3, /\/cgi-bin\//i),
+  // ---- Sensitive files --------------------------------------------------
+  rule(
+    'file-keys',
+    'sensitive-file',
+    8,
+    /\/id_rsa\b|\/id_dsa\b|\/\.ssh\/|\/\.aws\/credentials|\/\.docker\/config\.json|\/\.git-credentials/i,
+  ),
+  rule(
+    'file-backup',
+    'sensitive-file',
+    6,
+    /\/[\w.-]*(backup|bak|dump|db|database|site|www|html|public_html|wwwroot)[\w.-]*\.(zip|tar|tar\.gz|tgz|gz|rar|7z|sql|sql\.gz|bak)\b/i,
+  ),
+  rule(
+    'file-ext',
+    'sensitive-file',
+    4,
+    /\.(sql|bak|old|orig|swp|save|log|env|pem|key|p12|pfx|kdbx)(\?|$)/i,
+  ),
+  // ---- Anomalies ----------------------------------------------------------
+  rule('anomaly-long', 'anomaly', 4, /^[\s\S]{1500,}$/),
+  rule('anomaly-encoded', 'anomaly', 3, /(?:%[0-9a-f]{2}[^%]*){60,}/i, { raw: true }),
+  rule('anomaly-double-encoded', 'anomaly', 3, /%25[0-9a-f]{2}/i, { raw: true }),
+];
+
+/** HTTP methods no browser or crawler sends to a public site. */
+const ANOMALOUS_METHODS = new Set(['TRACE', 'TRACK', 'DEBUG', 'CONNECT']);
+
+/** Rules exported for tests / documentation — not for mutation. */
+export const THREAT_RULES: readonly ThreatRule[] = RULES;
+
+/** `decodeURIComponent` that never throws: malformed runs are left as-is. */
+function safeDecode(s: string): string {
+  try {
+    return decodeURIComponent(s);
+  } catch {
+    return s.replace(/(?:%[0-9a-fA-F]{2})+/g, (run) => {
+      try {
+        return decodeURIComponent(run);
+      } catch {
+        return run;
+      }
+    });
+  }
+}
+
+/**
+ * Percent-decode a request target up to two rounds, so a double-encoded
+ * payload (`%2527` → `%27` → `'`) reads the same as a plain one.
+ */
+export function decodeTarget(raw: string): string {
+  let cur = raw;
+  for (let round = 0; round < 2; round++) {
+    if (!/%[0-9a-fA-F]{2}/.test(cur)) break;
+    const next = safeDecode(cur);
+    if (next === cur) break;
+    cur = next;
+  }
+  return cur;
+}
+
+/** Decoded target normalised for matching: `+` → space, inline comments → space. */
+function normaliseForMatch(decoded: string): string {
+  return decoded.replace(/\+/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
+}
+
+/**
+ * Score one request. Returns null when nothing matched or the summed
+ * weight stays under {@link THREAT_THRESHOLD}.
+ */
+export function classifyRequest(req: {
+  path: string | null;
+  method: string | null;
+}): ThreatMatch | null {
+  const raw = req.path;
+  if (!raw) return null;
+  const method = req.method ? req.method.toUpperCase() : null;
+  const methodAnomaly = method !== null && ANOMALOUS_METHODS.has(method);
+  if (!methodAnomaly && !PREFILTER.test(raw)) return null;
+
+  const decoded = decodeTarget(raw);
+  const text = normaliseForMatch(decoded);
+
+  const hits: Array<{ rule: ThreatRule; evidence: string }> = [];
+  for (const r of RULES) {
+    if (r.method && r.method !== method) continue;
+    const m = r.re.exec(r.raw ? raw : text);
+    if (m) hits.push({ rule: r, evidence: m[0] });
+  }
+  if (methodAnomaly) {
+    hits.push({
+      rule: { id: 'anomaly-method', category: 'anomaly', weight: 5, re: /$^/ },
+      evidence: method!,
+    });
+  }
+  if (hits.length === 0) return null;
+
+  hits.sort((a, b) => b.rule.weight - a.rule.weight);
+  const score = hits.reduce((s, h) => s + h.rule.weight, 0);
+  if (score < THREAT_THRESHOLD) return null;
+  const top = hits[0]!;
+  const evidence = top.evidence.trim();
+  return {
+    category: top.rule.category,
+    rules: hits.map((h) => h.rule.id),
+    score,
+    evidence:
+      evidence.length > EVIDENCE_MAX ? `${evidence.slice(0, EVIDENCE_MAX - 1)}…` : evidence,
+    decoded,
+  };
+}
+
+/**
+ * Search-engine infrastructure whose reverse-DNS names prove the IP is the
+ * engine's own — not a rentable cloud VM (so `googleusercontent.com` is
+ * deliberately absent). A flagged request from one of these was relayed
+ * through the engine (Google Lens / Translate / a fetch of a crafted URL);
+ * blocking the IP would block the engine.
+ */
+const IP_OWNERS: ReadonlyArray<{ owner: string; suffixes: string[] }> = [
+  { owner: 'google', suffixes: ['.googlebot.com', '.google.com', '.1e100.net'] },
+  { owner: 'bing', suffixes: ['.search.msn.com'] },
+  { owner: 'yandex', suffixes: ['.yandex.ru', '.yandex.net', '.yandex.com'] },
+  { owner: 'apple', suffixes: ['.applebot.apple.com'] },
+  { owner: 'duckduckgo', suffixes: ['.duckduckgo.com'] },
+];
+
+/** Map a reverse-DNS hostname to its owner label, or null. Pure — tested. */
+export function ownerOfHostname(hostname: string): string | null {
+  const h = hostname.toLowerCase();
+  for (const o of IP_OWNERS) {
+    if (o.suffixes.some((s) => h.endsWith(s))) return o.owner;
+  }
+  return null;
+}
+
+/**
+ * Forward-confirmed reverse-DNS owner lookup: PTR must land in a known
+ * engine domain AND resolve back to the same IP. Any DNS failure → null.
+ */
+export async function classifyIpOwner(ip: string, timeoutMs = 3000): Promise<string | null> {
+  if (!ip) return null;
+  try {
+    const hostnames = await withTimeout(dns.reverse(ip), timeoutMs);
+    for (const h of hostnames) {
+      const owner = ownerOfHostname(h);
+      if (!owner) continue;
+      const addrs = await withTimeout(dns.resolve(h), timeoutMs).catch(() => [] as string[]);
+      if (addrs.includes(ip)) return owner;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+function withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
+  return Promise.race([
+    p,
+    new Promise<T>((_, reject) => setTimeout(() => reject(new Error('dns-timeout')), ms)),
+  ]);
+}

packages/core/src/tabular-export.ts

@@ -4,9 +4,15 @@ import { Readable } from 'node:stream';
 import path from 'node:path';
 import zlib from 'node:zlib';
 import type { ProjectDb } from '@freecrawl/db';
-import type { CrawlUrlRow, UrlCategory, AdvancedFilter } from '@freecrawl/shared-types';
+import type {
+  CrawlUrlRow,
+  UrlCategory,
+  AdvancedFilter,
+  ExportDatasetKey,
+} from '@freecrawl/shared-types';
 import { ensureHeapHeadroom } from './heap-guard.js';
 import { escapeCsv } from './spreadsheet.js';
+import { datasetRows, readCell, type DatasetExportContext } from './dataset-export.js';
 
 /** How a section selects its rows. `selectedIds` wins; otherwise category is
  *  narrowed by the active search/filter so the export matches the grid. */
@@ -19,7 +25,12 @@ interface RowQuery {
 
 export interface TabularSection {
   label: string;
-  category: UrlCategory;
+  /** URL category to stream `CrawlUrlRow`s from. Required unless `dataset` is set. */
+  category?: UrlCategory;
+  /** Non-URL-row table (Search Console, Broken Links, …) — see `datasetRows`. */
+  dataset?: ExportDatasetKey;
+  /** Column keys for this section only; defaults to the export-wide `columns`. */
+  columns?: string[];
   /** Optional sub-directory under the export root (for hierarchical
    *  tree exports — e.g. `Internal` → HTML / JS / CSS land under
    *  `internal/`). Ignored for xlsx output. */
@@ -44,6 +55,8 @@ export interface TabularExportOptions {
   /** Prefix CSV files with a UTF-8 BOM so Excel-for-Windows opens them
    *  in the correct charset. Ignored for non-CSV formats. Default true. */
   csvBom?: boolean;
+  /** Account / matching context for the dataset sections that need it. */
+  datasetContext?: DatasetExportContext;
 }
 
 export interface TabularExportResult {
@@ -120,10 +133,9 @@ function rowSource(db: ProjectDb, q: RowQuery): Iterable<CrawlUrlRow> {
 }
 
 async function writeCsvFile(
-  db: ProjectDb,
+  source: Iterable<Record<string, unknown>>,
   filePath: string,
   columns: string[],
-  query: RowQuery,
   withBom: boolean,
 ): Promise<number> {
   let rowsWritten = 0;
@@ -131,14 +143,13 @@ async function writeCsvFile(
   // leading `=`/`+`/`-`/`@` would otherwise break the CSV or inject a
   // formula. The data rows below already go through escapeCsv.
   const header = columns.map((c) => escapeCsv(c)).join(',') + '\n';
-  const source = rowSource(db, query);
   const generator = async function* (): AsyncGenerator<string> {
     if (withBom) yield '' + header;
     else yield header;
     for (const row of source) {
       const line =
         columns
-          .map((c) => escapeCsv((row as unknown as Record<string, unknown>)[c]))
+          .map((c) => escapeCsv(readCell(row, c)))
           .join(',') + '\n';
       rowsWritten++;
       yield line;
@@ -180,20 +191,18 @@ function xmlSafeTag(name: string): string {
 }
 
 async function writeJsonFile(
-  db: ProjectDb,
+  source: Iterable<Record<string, unknown>>,
   filePath: string,
   columns: string[],
-  query: RowQuery,
 ): Promise<number> {
   let rowsWritten = 0;
-  const source = rowSource(db, query);
   const generator = async function* (): AsyncGenerator<string> {
     yield '[';
     let first = true;
     for (const row of source) {
       const obj: Record<string, unknown> = {};
       for (const c of columns) {
-        obj[c] = (row as unknown as Record<string, unknown>)[c] ?? null;
+        obj[c] = readCell(row, c) ?? null;
       }
       yield (first ? '\n  ' : ',\n  ') + JSON.stringify(obj);
       first = false;
@@ -209,14 +218,12 @@ async function writeJsonFile(
 }
 
 async function writeXmlFile(
-  db: ProjectDb,
+  source: Iterable<Record<string, unknown>>,
   filePath: string,
   columns: string[],
-  query: RowQuery,
   sectionLabel: string,
 ): Promise<number> {
   let rowsWritten = 0;
-  const source = rowSource(db, query);
   const rootTag = xmlSafeTag(sectionLabel) || 'export';
   const cols = columns.map((c) => ({ key: c, tag: xmlSafeTag(c) }));
   const generator = async function* (): AsyncGenerator<string> {
@@ -224,7 +231,7 @@ async function writeXmlFile(
     for (const row of source) {
       yield '  <row>\n';
       for (const { key, tag } of cols) {
-        const v = (row as unknown as Record<string, unknown>)[key];
+        const v = readCell(row, key);
         if (v === null || v === undefined) continue;
         yield `    <${tag}>${escapeXml(v)}</${tag}>\n`;
       }
@@ -368,7 +375,7 @@ function buildSheetXmlFromCells(
 }
 
 function buildSheetXml(
-  rows: CrawlUrlRow[],
+  rows: Record<string, unknown>[],
   columns: string[],
 ): string {
   const parts: string[] = [];
@@ -390,7 +397,7 @@ function buildSheetXml(
     const r = rIdx + 2;
     parts.push(`<row r="${r}">`);
     columns.forEach((col, i) => {
-      const v = (row as unknown as Record<string, unknown>)[col];
+      const v = readCell(row, col);
       if (v === null || v === undefined || v === '') return;
       const cellRef = `${colLetter(i)}${r}`;
       if (typeof v === 'number' && Number.isFinite(v)) {
@@ -513,8 +520,7 @@ function buildZip(files: { name: string; data: Buffer }[]): Buffer {
 }
 
 function buildXlsxBuffer(
-  sheets: { name: string; rows: CrawlUrlRow[] }[],
-  columns: string[],
+  sheets: { name: string; rows: Record<string, unknown>[]; columns: string[] }[],
 ): Buffer {
   const contentTypes =
     '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
@@ -568,7 +574,7 @@ function buildXlsxBuffer(
   sheets.forEach((s, i) => {
     files.push({
       name: `xl/worksheets/sheet${i + 1}.xml`,
-      data: Buffer.from(buildSheetXml(s.rows, columns), 'utf8'),
+      data: Buffer.from(buildSheetXml(s.rows, s.columns), 'utf8'),
     });
   });
 
@@ -606,8 +612,22 @@ export async function exportTabular(
   if (sections.length === 0) {
     throw new Error('exportTabular: at least one section is required');
   }
-  if (columns.length === 0) {
-    throw new Error('exportTabular: at least one column is required');
+  // Every section resolves its own row source and column list: URL
+  // categories stream CrawlUrlRows over the shared `columns`; datasets
+  // (Search Console, Broken Links, …) walk their own query with the
+  // per-section keys the dialog picked for them.
+  const columnsFor = (section: TabularSection): string[] => section.columns ?? columns;
+  const sourceFor = (section: TabularSection): Iterable<Record<string, unknown>> => {
+    if (section.dataset) return datasetRows(db, section.dataset, options.datasetContext);
+    if (!section.category) {
+      throw new Error(`exportTabular: section '${section.label}' needs a category or a dataset`);
+    }
+    return rowSource(db, queryFor(section.category)) as Iterable<Record<string, unknown>>;
+  };
+  for (const section of sections) {
+    if (columnsFor(section).length === 0) {
+      throw new Error(`exportTabular: section '${section.label}' has no columns`);
+    }
   }
 
   if (format === 'xlsx') {
@@ -619,20 +639,23 @@ export async function exportTabular(
     // datasets this big" error than a V8 OOM abort of the whole app.
     const usedNames = new Set<string>();
     let total = 0;
+    let cells = 0;
     const sheets = sections.map((section) => {
-      const rows: CrawlUrlRow[] = [];
-      for (const row of rowSource(db, queryFor(section.category))) {
+      const rows: Record<string, unknown>[] = [];
+      const sheetColumns = columnsFor(section);
+      for (const row of sourceFor(section)) {
         rows.push(row);
         if (rows.length % 50_000 === 0) {
           ensureHeapHeadroom('XLSX export', 128 * 1024 * 1024);
         }
       }
       total += rows.length;
-      return { name: sanitizeSheetName(section.label, usedNames), rows };
+      cells += rows.length * sheetColumns.length;
+      return { name: sanitizeSheetName(section.label, usedNames), rows, columns: sheetColumns };
     });
-    // Workbook buffer is roughly rows × columns × ~24 B of XML.
-    ensureHeapHeadroom('XLSX export', total * columns.length * 24);
-    const buf = buildXlsxBuffer(sheets, columns);
+    // Workbook buffer is roughly cells × ~24 B of XML.
+    ensureHeapHeadroom('XLSX export', cells * 24);
+    const buf = buildXlsxBuffer(sheets);
     writeFileSync(outputPath, buf);
     return { filePath: outputPath, files: [outputPath], rowsWritten: total };
   }
@@ -655,14 +678,15 @@ export async function exportTabular(
     : outputPath;
 
   const writeOne = async (filePath: string, section: TabularSection): Promise<number> => {
-    const query = queryFor(section.category);
+    const source = sourceFor(section);
+    const sectionColumns = columnsFor(section);
     if (format === 'csv') {
-      return writeCsvFile(db, filePath, columns, query, csvBom);
+      return writeCsvFile(source, filePath, sectionColumns, csvBom);
     }
     if (format === 'json') {
-      return writeJsonFile(db, filePath, columns, query);
+      return writeJsonFile(source, filePath, sectionColumns);
     }
-    return writeXmlFile(db, filePath, columns, query, section.label);
+    return writeXmlFile(source, filePath, sectionColumns, section.label);
   };
 
   if (singleFlat) {

packages/db/src/index.ts

@@ -1,3 +1,3 @@
 export { ProjectDb, EXPENSIVE_ISSUE_DEFINITIONS } from './project-db.js';
-export type { UpsertUrlInput } from './project-db.js';
+export type { UpsertUrlInput, ProbeWrite } from './project-db.js';
 export { runMigrations } from './migrations.js';

packages/db/src/migrations.ts

@@ -2184,6 +2184,53 @@ const MIGRATIONS: Migration[] = [
       );
     },
   },
+  {
+    version: 87,
+    name: 'log_threats',
+    // Log Analyzer — Suspicious Requests. The one per-line log table: a
+    // row for every request the threat classifier flagged (SQLi / XSS /
+    // path traversal / command injection / scanner probes / sensitive
+    // files / anomalies), with the sender IP, the decoded payload and
+    // the rule evidence. Flagged lines are a tiny fraction of a log and
+    // the analyzer caps them per file, so unlike hits this can be kept
+    // row-by-row. `ip_owner` is set only when rDNS verification ran and
+    // proved the IP is a search engine's own infrastructure.
+    // `log_ingests.threats_dropped` records how many flagged lines the
+    // per-file cap discarded, so the tab can say its list is partial.
+    up: (db) => {
+      const cols = db.prepare('PRAGMA table_info(log_ingests)').all() as unknown as {
+        name: string;
+      }[];
+      if (!cols.some((c) => c.name === 'threats_dropped')) {
+        db.exec('ALTER TABLE log_ingests ADD COLUMN threats_dropped INTEGER NOT NULL DEFAULT 0');
+      }
+      db.exec(`
+      CREATE TABLE IF NOT EXISTS log_threats (
+        id         INTEGER PRIMARY KEY AUTOINCREMENT,
+        ingest_id  INTEGER NOT NULL,
+        ts         INTEGER,
+        ip         TEXT,
+        method     TEXT,
+        path       TEXT NOT NULL,
+        decoded    TEXT NOT NULL,
+        status     INTEGER,
+        bytes      INTEGER,
+        user_agent TEXT,
+        referer    TEXT,
+        bot        TEXT,
+        category   TEXT NOT NULL,
+        rules      TEXT NOT NULL,
+        score      INTEGER NOT NULL,
+        evidence   TEXT NOT NULL,
+        ip_owner   TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_log_threats_ts ON log_threats(ts);
+      CREATE INDEX IF NOT EXISTS idx_log_threats_ip ON log_threats(ip);
+      CREATE INDEX IF NOT EXISTS idx_log_threats_category ON log_threats(category);
+      CREATE INDEX IF NOT EXISTS idx_log_threats_score ON log_threats(score);
+      `);
+    },
+  },
 ];
 
 export function runMigrations(db: DatabaseSync): void {

packages/db/src/project-db.ts

@@ -52,6 +52,12 @@ import type {
   LogTrendRow,
   LogCrawlBudgetRow,
   LogDiscoveryRow,
+  LogThreatCategory,
+  LogThreatIpRow,
+  LogThreatRow,
+  LogThreatsInput,
+  LogThreatsResult,
+  LogThreatSummary,
 } from '@freecrawl/shared-types';
 import { issueCategoriesBySeverity } from '@freecrawl/shared-types';
 import { runMigrations } from './migrations.js';
@@ -869,6 +875,23 @@ const CANONICAL_CMP = "COALESCE(NULLIF(canonical_resolved, ''), canonical)";
 const CANONICAL_CMP_Q =
   "COALESCE(NULLIF(urls.canonical_resolved, ''), urls.canonical)";
 
+/**
+ * One post-crawl probe result, addressed to the setter that stores it.
+ * The crawler buffers these and ships them through `applyProbeWrites`.
+ */
+export type ProbeWrite =
+  | { method: 'setImageSize'; args: Parameters<ProjectDb['setImageSize']> }
+  | {
+      method: 'setSocialImageDimsForReferrers';
+      args: Parameters<ProjectDb['setSocialImageDimsForReferrers']>;
+    }
+  | { method: 'setHostCert'; args: Parameters<ProjectDb['setHostCert']> }
+  | {
+      method: 'setManifestForReferrers';
+      args: Parameters<ProjectDb['setManifestForReferrers']>;
+    }
+  | { method: 'setPdfMetadata'; args: Parameters<ProjectDb['setPdfMetadata']> };
+
 export class ProjectDb {
   private readonly db: DatabaseSync;
   /** Read-only connections can't write `sqlite_stat1`; see `optimize()`. */
@@ -1040,6 +1063,12 @@ export class ProjectDb {
          DELETE FROM seo_results;
          DELETE FROM gsc_inspection_results;`,
         );
+        // Log Analyzer aggregates live in this project too. Left out of
+        // the wipe they survived every cold start and Clear inside the
+        // disk-mode scratch project, so a brand-new "Untitled project"
+        // opened the analyzer onto whatever access log an earlier
+        // session had imported — with nothing on screen to say so.
+        this.clearLogData();
       });
     } finally {
       this.db.exec('PRAGMA foreign_keys = ON');
@@ -1365,6 +1394,16 @@ export class ProjectDb {
     storeBody: { body: string; maxBytes: number } | null;
     links: DiscoveredLink[];
     images: DiscoveredImage[];
+    /**
+     * Whether the crawl is recording this page's link graph at all
+     * (Spider → Crawl → Store for hyperlinks / iframes / invalid links).
+     * When false, `links` is empty because nothing is being recorded —
+     * not because the page has none — so the rows an earlier crawl
+     * stored are left in place. Defaults to true for CLI / test callers.
+     */
+    storeLinks?: boolean;
+    /** Same contract for `image_usages` (Spider → Crawl → Store → Images). */
+    storeImages?: boolean;
     fromDepth: number;
   }): { urlId: number } {
     return this.runInTransaction(() => {
@@ -1389,28 +1428,38 @@ export class ProjectDb {
         // metric. Delete unconditionally (a no-op on a first crawl, both
         // DELETEs are index-backed) so the page's link + image set
         // always reflects the latest fetch.
-        this.db.prepare('DELETE FROM links WHERE from_url_id = ?').run(urlId);
-        // `images.occurrences` is a running tally of how many pages use each
-        // image. Deleting this page's usage rows must first decrement that
-        // tally for the affected images, otherwise the re-insert via
-        // `insertImages` (which does `occurrences + 1`) drifts the count
-        // upward without bound on every re-crawl / Re-Spider / resume.
-        // Decrement-then-reinsert nets zero for images still on the page and
-        // correctly drops images that vanished from it.
-        this.db
-          .prepare(
-            `UPDATE images SET occurrences = occurrences - 1
-             WHERE id IN (SELECT image_id FROM image_usages WHERE from_url_id = ?)`,
-          )
-          .run(urlId);
-        this.db
-          .prepare('DELETE FROM image_usages WHERE from_url_id = ?')
-          .run(urlId);
-        if (payload.links.length > 0) {
-          this.insertLinks(urlId, payload.links, payload.fromDepth);
+        //
+        // Each DELETE is gated on its Store flag. With storage switched
+        // off the crawler ships an empty list, and wiping first would
+        // erase what a previous crawl (storage on) had recorded — every
+        // resume / Re-Spider silently zeroed the page's Inlinks and
+        // Outlinks and dropped it from its neighbours' inlink counts.
+        if (payload.storeLinks ?? true) {
+          this.db.prepare('DELETE FROM links WHERE from_url_id = ?').run(urlId);
+          if (payload.links.length > 0) {
+            this.insertLinks(urlId, payload.links, payload.fromDepth);
+          }
         }
-        if (payload.images.length > 0) {
-          this.insertImages(urlId, payload.images);
+        if (payload.storeImages ?? true) {
+          // `images.occurrences` is a running tally of how many pages use each
+          // image. Deleting this page's usage rows must first decrement that
+          // tally for the affected images, otherwise the re-insert via
+          // `insertImages` (which does `occurrences + 1`) drifts the count
+          // upward without bound on every re-crawl / Re-Spider / resume.
+          // Decrement-then-reinsert nets zero for images still on the page and
+          // correctly drops images that vanished from it.
+          this.db
+            .prepare(
+              `UPDATE images SET occurrences = occurrences - 1
+               WHERE id IN (SELECT image_id FROM image_usages WHERE from_url_id = ?)`,
+            )
+            .run(urlId);
+          this.db
+            .prepare('DELETE FROM image_usages WHERE from_url_id = ?')
+            .run(urlId);
+          if (payload.images.length > 0) {
+            this.insertImages(urlId, payload.images);
+          }
         }
       }
       return { urlId };
@@ -7435,6 +7484,39 @@ export class ProjectDb {
       );
       for (const ub of input.urlBots) upUrlBot.run(ub.path, ub.bot, ub.hits);
 
+      const insThreat = this.db.prepare(
+        `INSERT INTO log_threats
+           (ingest_id, ts, ip, method, path, decoded, status, bytes, user_agent, referer, bot,
+            category, rules, score, evidence, ip_owner)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      );
+      const ingestId = ins.lastInsertRowid as number;
+      for (const t of input.threats) {
+        insThreat.run(
+          ingestId,
+          t.ts,
+          t.ip,
+          t.method,
+          t.path,
+          t.decoded,
+          t.status,
+          t.bytes,
+          t.userAgent,
+          t.referer,
+          t.bot,
+          t.category,
+          t.rules.join(','),
+          t.score,
+          t.evidence,
+          t.ipOwner,
+        );
+      }
+      if (input.threatsDropped && input.threatsDropped > 0) {
+        this.db
+          .prepare('UPDATE log_ingests SET threats_dropped = ? WHERE id = ?')
+          .run(input.threatsDropped, ingestId);
+      }
+
       this.db.exec('COMMIT');
       const row = this.db
         .prepare('SELECT ingested_at FROM log_ingests WHERE id = ?')
@@ -7518,6 +7600,10 @@ export class ProjectDb {
       }
     }
 
+    const threats = this.db
+      .prepare('SELECT COUNT(*) AS hits, COUNT(DISTINCT ip) AS ips FROM log_threats')
+      .get() as { hits: number; ips: number };
+
     return {
       hasData: files.length > 0,
       files,
@@ -7528,6 +7614,8 @@ export class ProjectDb {
       distinctUrls: agg.urls,
       minTs: span.lo,
       maxTs: span.hi,
+      threatHits: threats.hits,
+      threatIps: threats.ips,
     };
   }
 
@@ -7716,21 +7804,152 @@ export class ProjectDb {
     return out;
   }
 
-  /** Wipe every ingested log aggregate. */
-  clearLogData(): void {
-    this.db.exec('BEGIN');
-    try {
-      this.db.exec('DELETE FROM log_url_stats');
-      this.db.exec('DELETE FROM log_url_bot');
-      this.db.exec('DELETE FROM log_daily');
-      this.db.exec('DELETE FROM log_status');
-      this.db.exec('DELETE FROM log_bots');
-      this.db.exec('DELETE FROM log_ingests');
-      this.db.exec('COMMIT');
-    } catch (err) {
-      this.db.exec('ROLLBACK');
-      throw err;
+  /** Crawled internal pathnames (query stripped) for the threat-target join. */
+  private crawlInternalPathnameSet(): Set<string> {
+    const set = new Set<string>();
+    for (const p of this.crawlInternalPathSet()) {
+      const q = p.indexOf('?');
+      set.add(q >= 0 ? p.slice(0, q) : p);
     }
+    return set;
+  }
+
+  /**
+   * Suspicious Requests — paginated flagged log lines. Filters compose;
+   * `search` covers the decoded target, the IP and the user-agent. The
+   * crawl join (`targetInCrawl`) compares the target's pathname with the
+   * crawled internal URLs, so a payload aimed at a real page reads
+   * differently from a blind probe for a file that never existed.
+   */
+  queryLogThreats(input: LogThreatsInput): LogThreatsResult {
+    const limit = Math.max(1, Math.min(input.limit, 1000));
+    const offset = Math.max(0, input.offset);
+    const where: string[] = [];
+    const args: (string | number)[] = [];
+    const search = input.search?.trim();
+    if (search) {
+      where.push('(decoded LIKE ? OR ip LIKE ? OR user_agent LIKE ?)');
+      const like = `%${search}%`;
+      args.push(like, like, like);
+    }
+    if (input.category && input.category !== 'all') {
+      where.push('category = ?');
+      args.push(input.category);
+    }
+    if (input.ip && input.ip.trim()) {
+      where.push('ip = ?');
+      args.push(input.ip.trim());
+    }
+    const cls = input.status ?? 'all';
+    if (cls !== 'all') {
+      const lo = Number(cls[0]) * 100;
+      where.push('status >= ? AND status < ?');
+      args.push(lo, lo + 100);
+    }
+    const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '';
+    const orderSql =
+      input.sortBy === 'score'
+        ? 'ORDER BY score DESC, ts DESC, id DESC'
+        : input.sortBy === 'status'
+          ? 'ORDER BY status ASC, ts DESC, id DESC'
+          : 'ORDER BY ts DESC, id DESC';
+    const total = (
+      this.db.prepare(`SELECT COUNT(*) AS n FROM log_threats ${whereSql}`).get(...args) as {
+        n: number;
+      }
+    ).n;
+    const rows = this.db
+      .prepare(`SELECT * FROM log_threats ${whereSql} ${orderSql} LIMIT ? OFFSET ?`)
+      .all(...args, limit, offset) as LogThreatDbRow[];
+    const crawled = this.crawlInternalPathnameSet();
+    return { rows: rows.map((r) => toLogThreatRow(r, crawled)), total };
+  }
+
+  /** Roll-up above the Suspicious Requests table: totals, per-category
+   *  counts, the busiest senders, and whether any ingest hit the cap. */
+  getLogThreatSummary(topIps = 25): LogThreatSummary {
+    const tot = this.db
+      .prepare(
+        `SELECT COUNT(*) AS hits,
+                COUNT(DISTINCT ip) AS ips,
+                COALESCE(SUM(CASE WHEN status >= 200 AND status < 300 THEN 1 ELSE 0 END), 0) AS served
+           FROM log_threats`,
+      )
+      .get() as { hits: number; ips: number; served: number };
+    const byCategory = this.db
+      .prepare(
+        `SELECT category, COUNT(*) AS hits, COUNT(DISTINCT ip) AS ips
+           FROM log_threats GROUP BY category ORDER BY hits DESC, category ASC`,
+      )
+      .all() as { category: LogThreatCategory; hits: number; ips: number }[];
+    const dropped = this.db
+      .prepare('SELECT COALESCE(SUM(threats_dropped), 0) AS n FROM log_ingests')
+      .get() as { n: number };
+    return {
+      totalHits: tot.hits,
+      distinctIps: tot.ips,
+      servedHits: tot.served,
+      byCategory,
+      topIps: this.getLogThreatIps(topIps),
+      capHit: dropped.n > 0,
+    };
+  }
+
+  /** Senders of flagged requests, busiest first. A large `limit` serves the
+   *  blocklist export; the tab's IP panel asks for the top few dozen. */
+  getLogThreatIps(limit = 25): LogThreatIpRow[] {
+    const rows = this.db
+      .prepare(
+        `SELECT ip,
+                COUNT(*) AS hits,
+                GROUP_CONCAT(DISTINCT category) AS categories,
+                MIN(ts) AS first_ts,
+                MAX(ts) AS last_ts,
+                COALESCE(SUM(CASE WHEN status >= 200 AND status < 300 THEN 1 ELSE 0 END), 0) AS served,
+                MAX(bot) AS bot,
+                MAX(ip_owner) AS ip_owner
+           FROM log_threats
+          WHERE ip IS NOT NULL
+          GROUP BY ip
+          ORDER BY hits DESC, last_ts DESC
+          LIMIT ?`,
+      )
+      .all(Math.max(1, Math.min(limit, 100_000))) as {
+      ip: string;
+      hits: number;
+      categories: string;
+      first_ts: number | null;
+      last_ts: number | null;
+      served: number;
+      bot: string | null;
+      ip_owner: string | null;
+    }[];
+    return rows.map((r) => ({
+      ip: r.ip,
+      hits: r.hits,
+      categories: r.categories.split(',').filter(Boolean) as LogThreatCategory[],
+      firstTs: r.first_ts,
+      lastTs: r.last_ts,
+      servedHits: r.served,
+      bot: r.bot,
+      ipOwner: r.ip_owner,
+    }));
+  }
+
+  /** Wipe every ingested log aggregate. Nests inside `reset()`'s
+   *  transaction, so it must not open its own with a bare BEGIN. */
+  clearLogData(): void {
+    this.runInTransaction(() => {
+      this.db.exec(
+        `DELETE FROM log_threats;
+         DELETE FROM log_url_stats;
+         DELETE FROM log_url_bot;
+         DELETE FROM log_daily;
+         DELETE FROM log_status;
+         DELETE FROM log_bots;
+         DELETE FROM log_ingests;`,
+      );
+    });
   }
 
   /**
@@ -8216,6 +8435,45 @@ export class ProjectDb {
    * is null when the server didn't return Content-Length, status records
    * the HTTP code (or 0 when the request errored entirely).
    */
+  /**
+   * Store a batch of post-crawl probe results in one transaction.
+   *
+   * The probe loops (image size, social image, TLS, manifest, PDF) run on
+   * the main thread and used to call the per-row setters straight on its
+   * own connection — the one place in a crawl that wrote around the
+   * writer worker, and one autocommit transaction per row on top of
+   * that. This is the entry point the writer exposes for them instead:
+   * the crawler buffers results and ships a few hundred at a time, so a
+   * probe phase costs one round-trip and one commit per batch rather
+   * than per image, and the "one writer connection" rule holds end to
+   * end. A batch is atomic; a failing row rolls the batch back and the
+   * caller reports it, which the old per-row `catch {}` never did.
+   */
+  applyProbeWrites(writes: readonly ProbeWrite[]): void {
+    if (writes.length === 0) return;
+    this.runInTransaction(() => {
+      for (const w of writes) {
+        switch (w.method) {
+          case 'setImageSize':
+            this.setImageSize(...w.args);
+            break;
+          case 'setSocialImageDimsForReferrers':
+            this.setSocialImageDimsForReferrers(...w.args);
+            break;
+          case 'setHostCert':
+            this.setHostCert(...w.args);
+            break;
+          case 'setManifestForReferrers':
+            this.setManifestForReferrers(...w.args);
+            break;
+          case 'setPdfMetadata':
+            this.setPdfMetadata(...w.args);
+            break;
+        }
+      }
+    });
+  }
+
   setImageSize(imageId: number, byteSize: number | null, status: number): void {
     this.db
       .prepare(
@@ -9021,77 +9279,84 @@ function parsePaginationOrdinal(
  * `getOverviewCounts`. The materialised table picks it up on the next
  * recompute with no further plumbing.
  */
+/** Lowercased host of a URL column: `https://Ex.com/a?b` → `ex.com`. */
+function urlHostSql(col: string): string {
+  return `LOWER(
+    SUBSTR(
+      ${col},
+      INSTR(${col}, '://') + 3,
+      CASE
+        WHEN INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/') > 0
+          THEN INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/') - 1
+        ELSE LENGTH(${col})
+      END
+    )
+  )`;
+}
+
+/** Post-normalisation form of a URL column — query string stripped,
+ *  lowercased, trailing slash trimmed — the identity two "duplicate
+ *  URL" rows share. */
+function urlPostNormSql(col: string): string {
+  return `RTRIM(
+    LOWER(
+      CASE
+        WHEN INSTR(${col}, '?') > 0 THEN SUBSTR(${col}, 1, INSTR(${col}, '?') - 1)
+        ELSE ${col}
+      END
+    ),
+    '/'
+  )`;
+}
+
 export const EXPENSIVE_ISSUE_DEFINITIONS: ReadonlyArray<readonly [string, string]> = [
   [
     'issues:dead-external-domain',
+    // Set-based on purpose. Every sub-select here is uncorrelated, so
+    // SQLite materialises each once and probes it through an ephemeral
+    // index — one pass over `links` and one over the external rows. The
+    // previous correlated `EXISTS (… JOIN …)` re-ran the host grouping
+    // per page and scaled with pages × outlinks.
     `is_external = 0 AND content_kind = 'html'
-     AND EXISTS (
-       SELECT 1 FROM links l
-         JOIN urls eu ON l.to_url = eu.url
-        WHERE l.from_url_id = urls.id
-          AND l.is_internal = 0
-          AND eu.is_external = 1
-          AND LOWER(
-            SUBSTR(
-              eu.url,
-              INSTR(eu.url, '://') + 3,
-              CASE
-                WHEN INSTR(SUBSTR(eu.url, INSTR(eu.url, '://') + 3), '/') > 0
-                  THEN INSTR(SUBSTR(eu.url, INSTR(eu.url, '://') + 3), '/') - 1
-                ELSE LENGTH(eu.url)
-              END
-            )
-          ) IN (
-            SELECT host_grouped FROM (
-              SELECT
-                LOWER(
-                  SUBSTR(
-                    url,
-                    INSTR(url, '://') + 3,
-                    CASE
-                      WHEN INSTR(SUBSTR(url, INSTR(url, '://') + 3), '/') > 0
-                        THEN INSTR(SUBSTR(url, INSTR(url, '://') + 3), '/') - 1
-                      ELSE LENGTH(url)
-                    END
-                  )
-                ) AS host_grouped,
-                COUNT(*) AS total,
-                SUM(CASE WHEN status_code >= 400 THEN 1 ELSE 0 END) AS errors
-              FROM urls
-              WHERE is_external = 1 AND status_code IS NOT NULL
-              GROUP BY host_grouped
-              HAVING total >= 3 AND CAST(errors AS REAL) / total >= 0.8
-            )
+     AND id IN (
+       SELECT l.from_url_id
+         FROM links l
+        WHERE l.is_internal = 0
+          AND l.to_url IN (
+            SELECT eu.url
+              FROM urls eu
+             WHERE eu.is_external = 1
+               AND ${urlHostSql('eu.url')} IN (
+                 SELECT host_grouped FROM (
+                   SELECT
+                     ${urlHostSql('url')} AS host_grouped,
+                     COUNT(*) AS total,
+                     SUM(CASE WHEN status_code >= 400 THEN 1 ELSE 0 END) AS errors
+                   FROM urls
+                   WHERE is_external = 1 AND status_code IS NOT NULL
+                   GROUP BY host_grouped
+                   HAVING total >= 3 AND CAST(errors AS REAL) / total >= 0.8
+                 )
+               )
           )
      )`,
   ],
   [
     'issues:duplicate-url-post-norm',
-    `is_external = 0 AND content_kind = 'html' AND EXISTS (
-       SELECT 1 FROM urls u2
-        WHERE u2.id <> urls.id
-          AND u2.is_external = 0
-          AND u2.content_kind = 'html'
-          AND RTRIM(
-                LOWER(
-                  CASE
-                    WHEN INSTR(u2.url, '?') > 0
-                      THEN SUBSTR(u2.url, 1, INSTR(u2.url, '?') - 1)
-                    ELSE u2.url
-                  END
-                ),
-                '/'
-              ) =
-              RTRIM(
-                LOWER(
-                  CASE
-                    WHEN INSTR(urls.url, '?') > 0
-                      THEN SUBSTR(urls.url, 1, INSTR(urls.url, '?') - 1)
-                    ELSE urls.url
-                  END
-                ),
-                '/'
-              )
+    // "Another page normalises to the same URL" as a GROUP BY over the
+    // normalised form, probed once per row. The self-join it replaces
+    // compared computed strings, which no index can serve: a full scan
+    // of `urls` for every row — 20k pages took ~150 s and froze the app
+    // for that long on every mid-crawl recompute; this form takes ~20 ms.
+    `is_external = 0 AND content_kind = 'html'
+     AND ${urlPostNormSql('url')} IN (
+       SELECT norm_url FROM (
+         SELECT ${urlPostNormSql('url')} AS norm_url
+           FROM urls
+          WHERE is_external = 0 AND content_kind = 'html'
+          GROUP BY norm_url
+         HAVING COUNT(*) > 1
+       )
      )`,
   ],
   [
@@ -9249,7 +9514,7 @@ function buildClauseSql(c: FilterClause, args: (string | number)[]): string | nu
   }
 }
 
-function categoryWhereClause(cat: UrlCategory): string | null {
+export function categoryWhereClause(cat: UrlCategory): string | null {
   switch (cat) {
     case 'all':
       return null;
@@ -9278,7 +9543,13 @@ function categoryWhereClause(cat: UrlCategory): string | null {
     case 'status:blocked-robots':
       return "indexability = 'non-indexable:robots-blocked'";
     case 'status:no-response':
-      return 'status_code IS NULL';
+      // Robots-blocked URLs are recorded with a NULL status because the
+      // crawler deliberately never requested them — they are not pages
+      // that failed to answer. Counting them here made "No Response"
+      // read as N failures on a crawl whose failure count was zero, and
+      // double-counted them against the sibling Blocked by Robots row.
+      return `status_code IS NULL
+              AND indexability != 'non-indexable:robots-blocked'`;
     case 'status:2xx':
       return 'status_code >= 200 AND status_code < 300';
     case 'status:3xx':
@@ -10262,13 +10533,19 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       // A page links to ≥1 internal URL whose status is 3xx. Each hop
       // burns crawl budget and weakens link equity — best-practice is
       // to update every link to the redirect's final destination.
+      // Set-based: the redirecting URLs are resolved once (status index),
+      // then the links pointing at them once (to_url index). The
+      // correlated EXISTS it replaces re-joined every page's outlinks to
+      // `urls` by string — pages × outlinks lookups, ~20× slower.
       return `is_external = 0 AND content_kind = 'html'
-              AND EXISTS (
-                SELECT 1 FROM links l
-                  JOIN urls t ON l.to_url = t.url
-                 WHERE l.from_url_id = urls.id
-                   AND l.is_internal = 1
-                   AND t.status_code >= 300 AND t.status_code < 400
+              AND id IN (
+                SELECT l.from_url_id
+                  FROM links l
+                 WHERE l.is_internal = 1
+                   AND l.to_url IN (
+                     SELECT url FROM urls
+                      WHERE status_code >= 300 AND status_code < 400
+                   )
               )`;
     case 'issues:h1-equals-title':
       // Title and H1 are *almost* always supposed to differ —
@@ -10280,108 +10557,19 @@ function categoryWhereClause(cat: UrlCategory): string | null {
               AND h1 IS NOT NULL AND h1 != ''
               AND TRIM(LOWER(title)) = TRIM(LOWER(h1))`;
     case 'issues:dead-external-domain':
-      // Page links to an external domain whose own crawled pages are
-      // mostly broken: ≥3 attempts (so a single 404 doesn't poison a
-      // whole site) AND ≥80% error rate. Both thresholds match the
-      // External Domain Health report's "BAD" classification so the
-      // sidebar count and the report agree on what counts as dead.
-      // The host extraction is inlined below; SQLite's substring
-      // arithmetic is verbose but stays in the query planner so a
-      // 100K-URL crawl resolves this in a single pass.
-      return `is_external = 0 AND content_kind = 'html'
-              AND EXISTS (
-                SELECT 1 FROM links l
-                  JOIN urls eu ON l.to_url = eu.url
-                 WHERE l.from_url_id = urls.id
-                   AND l.is_internal = 0
-                   AND eu.is_external = 1
-                   AND LOWER(
-                     SUBSTR(
-                       eu.url,
-                       INSTR(eu.url, '://') + 3,
-                       CASE
-                         WHEN INSTR(SUBSTR(eu.url, INSTR(eu.url, '://') + 3), '/') > 0
-                           THEN INSTR(SUBSTR(eu.url, INSTR(eu.url, '://') + 3), '/') - 1
-                         ELSE LENGTH(eu.url)
-                       END
-                     )
-                   ) IN (
-                     SELECT host_grouped FROM (
-                       SELECT
-                         LOWER(
-                           SUBSTR(
-                             url,
-                             INSTR(url, '://') + 3,
-                             CASE
-                               WHEN INSTR(SUBSTR(url, INSTR(url, '://') + 3), '/') > 0
-                                 THEN INSTR(SUBSTR(url, INSTR(url, '://') + 3), '/') - 1
-                               ELSE LENGTH(url)
-                             END
-                           )
-                         ) AS host_grouped,
-                         COUNT(*) AS total,
-                         SUM(CASE WHEN status_code >= 400 THEN 1 ELSE 0 END) AS errors
-                       FROM urls
-                       WHERE is_external = 1 AND status_code IS NOT NULL
-                       GROUP BY host_grouped
-                       HAVING total >= 3 AND CAST(errors AS REAL) / total >= 0.8
-                     )
-                   )
-              )`;
     case 'issues:duplicate-url-post-norm':
-      // Two distinct URL rows that collapse to the same value once
-      // common normalisations are applied: lowercase the host, strip a
-      // trailing slash, drop the query string. We don't strip the
-      // fragment because we already have a separate "Fragment in URL"
-      // issue and a fragment-only difference is rare in real corpora.
-      // EXISTS pattern matches at least one OTHER row with the same
-      // normalised key; current row excluded by `urls.id <> u2.id`.
-      return `is_external = 0 AND content_kind = 'html'
-              AND EXISTS (
-                SELECT 1 FROM urls u2
-                 WHERE u2.id <> urls.id
-                   AND u2.is_external = 0
-                   AND u2.content_kind = 'html'
-                   AND RTRIM(
-                         LOWER(
-                           CASE
-                             WHEN INSTR(u2.url, '?') > 0
-                               THEN SUBSTR(u2.url, 1, INSTR(u2.url, '?') - 1)
-                             ELSE u2.url
-                           END
-                         ),
-                         '/'
-                       ) =
-                       RTRIM(
-                         LOWER(
-                           CASE
-                             WHEN INSTR(urls.url, '?') > 0
-                               THEN SUBSTR(urls.url, 1, INSTR(urls.url, '?') - 1)
-                             ELSE urls.url
-                           END
-                         ),
-                         '/'
-                       )
-              )`;
     case 'issues:canonical-chain-multi-hop':
-      // Page has a canonical, AND that canonical's target is also a
-      // crawled URL with its own *different* canonical. Two-hop
-      // detection is enough for the bulk of misconfigurations — deeper
-      // chains cascade through this filter too because the second hop
-      // would also surface as a multi-hop chain on its own row.
-      return `is_external = 0 AND content_kind = 'html'
-              AND canonical IS NOT NULL AND canonical != ''
-              AND ${CANONICAL_CMP} != url
-              AND EXISTS (
-                SELECT 1 FROM urls c2
-                 WHERE c2.url = ${CANONICAL_CMP_Q}
-                   AND c2.canonical IS NOT NULL
-                   AND c2.canonical != ''
-                   AND COALESCE(NULLIF(c2.canonical_resolved, ''), c2.canonical)
-                       != c2.url
-                   AND COALESCE(NULLIF(c2.canonical_resolved, ''), c2.canonical)
-                       != ${CANONICAL_CMP_Q}
-              )`;
+      // These three are materialised into `urls_issues` by the issue
+      // recompute (EXPENSIVE_ISSUE_DEFINITIONS, every 30 s mid-crawl and
+      // once more post-crawl), and the sidebar count already reads that
+      // table via `getIssueCounts`. The list filter carried its own
+      // inline copy of each predicate — the pre-rewrite O(n²) self-join
+      // in the duplicate case — so one sidebar click ran a 150 s query
+      // (23k URLs) against a 30 s reader timeout, and the abandoned
+      // query then held a reader worker for minutes, timing out every
+      // filter clicked after it. Count and list now come from the same
+      // rows, in one indexed lookup.
+      return `id IN (SELECT url_id FROM urls_issues WHERE issue_key = '${cat}')`;
     case 'issues:image-slow-loading':
       // Page loads at least one image > 200 KB AND the page hasn't
       // applied lazy-loading to every image. Big un-lazy images are
@@ -10777,6 +10965,51 @@ function toLogUrlStatRow(r: LogUrlStatDbRow, crawlPaths: Set<string>): LogUrlSta
   };
 }
 
+type LogThreatDbRow = {
+  id: number;
+  ts: number | null;
+  ip: string | null;
+  method: string | null;
+  path: string;
+  decoded: string;
+  status: number | null;
+  bytes: number | null;
+  user_agent: string | null;
+  referer: string | null;
+  bot: string | null;
+  category: LogThreatCategory;
+  rules: string;
+  score: number;
+  evidence: string;
+  ip_owner: string | null;
+};
+
+/** Map a raw log_threats row to the IPC row; `crawled` holds crawled
+ *  internal pathnames (query stripped). */
+function toLogThreatRow(r: LogThreatDbRow, crawled: Set<string>): LogThreatRow {
+  const q = r.path.indexOf('?');
+  const pathname = q >= 0 ? r.path.slice(0, q) : r.path;
+  return {
+    id: r.id,
+    ts: r.ts,
+    ip: r.ip,
+    method: r.method,
+    path: r.path,
+    decoded: r.decoded,
+    status: r.status,
+    bytes: r.bytes,
+    userAgent: r.user_agent,
+    referer: r.referer,
+    bot: r.bot,
+    category: r.category,
+    rules: r.rules.split(',').filter(Boolean),
+    score: r.score,
+    evidence: r.evidence,
+    ipOwner: r.ip_owner,
+    targetInCrawl: crawled.has(pathname),
+  };
+}
+
 /** Extract `pathname + search` from a full crawl URL for log path joins. */
 function urlToPath(rawUrl: string): string | null {
   try {

packages/shared-types/src/crawl.ts

@@ -712,6 +712,16 @@ export interface CrawlConfig {
   maxUrls: number;
   maxConcurrency: number;
   maxRps: number;
+  /**
+   * Adaptive speed. Off by default. When on, the crawler treats
+   * `maxConcurrency` / `maxRps` as a *ceiling* and automatically slows
+   * down when the target pushes back — a 429/503 (or a `Retry-After`
+   * header) shrinks the dispatch rate and concurrency, and a sustained
+   * run of clean responses gradually restores them toward the ceiling.
+   * When off, the crawler holds exactly the configured rate regardless
+   * of how the server responds (the previous, only, behaviour).
+   */
+  adaptiveSpeed: boolean;
   requestTimeoutMs: number;
   userAgent: string;
   /**
@@ -2209,6 +2219,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   maxUrls: 1_000_000,
   maxConcurrency: 20,
   maxRps: 20,
+  adaptiveSpeed: false,
   requestTimeoutMs: 20_000,
   userAgent: 'FreeCrawlSEO/0.1 (+https://github.com/kemalai/FreeCrawl-SEO-Tool)',
   deviceMode: 'desktop',

packages/shared-types/src/export-datasets.ts

@@ -0,0 +1,197 @@
+import type { AiProvider } from './ai.js';
+import type { SeoProvider } from './seo.js';
+
+/**
+ * Tables the Export dialog can emit that are NOT views over `CrawlUrlRow`.
+ *
+ * The main URL tabs (Internal, Page Titles, Directives, …) all project the
+ * same `urls` row, so the tabular exporter streams `CrawlUrlRow`s and the
+ * dialog offers `CrawlUrlRow` keys as columns. The integration and
+ * link-graph tabs have their own row shapes — a Search Console row is a
+ * page joined with its clicks / impressions / inspection verdict, a Broken
+ * Links row is one edge of the link graph — and each has its own DB query.
+ * A dataset key names one of those queries; its column catalogue below
+ * tells the dialog what to offer and the exporter what to read.
+ *
+ * AI and SEO Authority results are stored per provider, so those datasets
+ * are keyed by provider — one sheet per provider, each with the metric
+ * columns that provider actually returns.
+ */
+export type ExportDatasetKey =
+  | 'images'
+  | 'broken-links'
+  | 'pagespeed'
+  | 'crux'
+  | 'spelling'
+  | 'search-console'
+  | 'analytics'
+  | `ai:${AiProvider}`
+  | `seo:${SeoProvider}`;
+
+export interface ExportDatasetColumn {
+  /**
+   * Dotted path into the dataset's row object — `gsc.clicks` reads
+   * `row.gsc?.clicks`, so a page with no Search Console data exports an
+   * empty cell rather than failing the row.
+   */
+  key: string;
+  /** Column header, in English; translated through `labels.ts` in the UI. */
+  header: string;
+}
+
+const PAGE: ExportDatasetColumn[] = [
+  { key: 'url', header: 'URL' },
+  { key: 'statusCode', header: 'Status' },
+];
+
+function pagespeedColumns(device: 'mobile' | 'desktop', label: string): ExportDatasetColumn[] {
+  return [
+    { key: `${device}.performance`, header: `${label} Performance` },
+    { key: `${device}.lcp`, header: `${label} LCP` },
+    { key: `${device}.cls`, header: `${label} CLS` },
+    { key: `${device}.fcp`, header: `${label} FCP` },
+    { key: `${device}.tbt`, header: `${label} TBT` },
+    { key: `${device}.speedIndex`, header: `${label} Speed Index` },
+    { key: `${device}.tti`, header: `${label} TTI` },
+    { key: `${device}.inp`, header: `${label} INP` },
+    { key: `${device}.status`, header: `${label} Result Status` },
+    { key: `${device}.error`, header: `${label} Error` },
+    { key: `${device}.fetchedAt`, header: `${label} Fetched At` },
+  ];
+}
+
+function cruxColumns(device: 'phone' | 'desktop', label: string): ExportDatasetColumn[] {
+  return [
+    { key: `${device}.lcp`, header: `${label} LCP` },
+    { key: `${device}.cls`, header: `${label} CLS` },
+    { key: `${device}.inp`, header: `${label} INP` },
+    { key: `${device}.fcp`, header: `${label} FCP` },
+    { key: `${device}.ttfb`, header: `${label} TTFB` },
+    { key: `${device}.status`, header: `${label} Result Status` },
+    { key: `${device}.collectionPeriod`, header: `${label} Collection Period` },
+    { key: `${device}.error`, header: `${label} Error` },
+    { key: `${device}.fetchedAt`, header: `${label} Fetched At` },
+  ];
+}
+
+const AI: ExportDatasetColumn[] = [
+  ...PAGE,
+  { key: 'ai.model', header: 'Model' },
+  { key: 'ai.response', header: 'Response' },
+  { key: 'ai.tokensIn', header: 'Tokens In' },
+  { key: 'ai.tokensOut', header: 'Tokens Out' },
+  { key: 'ai.status', header: 'Result Status' },
+  { key: 'ai.error', header: 'Error' },
+  { key: 'ai.fetchedAt', header: 'Fetched At' },
+];
+
+const SEO_TAIL: ExportDatasetColumn[] = [
+  { key: 'seo.status', header: 'Result Status' },
+  { key: 'seo.error', header: 'Error' },
+  { key: 'seo.fetchedAt', header: 'Fetched At' },
+];
+
+export const EXPORT_DATASET_COLUMNS: Record<ExportDatasetKey, ExportDatasetColumn[]> = {
+  images: [
+    { key: 'src', header: 'Image URL' },
+    { key: 'fromUrl', header: 'Page URL' },
+    { key: 'alt', header: 'Alt Text' },
+    { key: 'width', header: 'Width' },
+    { key: 'height', header: 'Height' },
+    { key: 'isInternal', header: 'Internal' },
+    { key: 'occurrences', header: 'Occurrences' },
+  ],
+  'broken-links': [
+    { key: 'fromUrl', header: 'Source' },
+    { key: 'fromStatusCode', header: 'Source Status' },
+    { key: 'toUrl', header: 'Destination' },
+    { key: 'toStatusCode', header: 'Destination Status' },
+    { key: 'anchor', header: 'Anchor' },
+    { key: 'rel', header: 'Rel' },
+    { key: 'isInternal', header: 'Internal' },
+  ],
+  pagespeed: [
+    ...PAGE,
+    ...pagespeedColumns('mobile', 'Mobile'),
+    ...pagespeedColumns('desktop', 'Desktop'),
+  ],
+  crux: [...PAGE, ...cruxColumns('phone', 'Phone'), ...cruxColumns('desktop', 'Desktop')],
+  spelling: [
+    { key: 'url', header: 'URL' },
+    { key: 'lang', header: 'HTML Lang' },
+    { key: 'wordCount', header: 'Word Count' },
+    { key: 'language', header: 'Check Language' },
+    { key: 'detectedLanguage', header: 'Detected Language' },
+    { key: 'matchCount', header: 'Matches' },
+    { key: 'status', header: 'Check Status' },
+    { key: 'engine', header: 'Engine' },
+    { key: 'error', header: 'Error' },
+    { key: 'fetchedAt', header: 'Checked At' },
+  ],
+  'search-console': [
+    ...PAGE,
+    { key: 'gsc.clicks', header: 'Clicks' },
+    { key: 'gsc.impressions', header: 'Impressions' },
+    { key: 'gsc.ctr', header: 'CTR' },
+    { key: 'gsc.position', header: 'Position' },
+    { key: 'gsc.fetchedAt', header: 'Fetched At' },
+    { key: 'inspection.verdict', header: 'Index Verdict' },
+    { key: 'inspection.coverageState', header: 'Coverage State' },
+    { key: 'inspection.robotsTxtState', header: 'Robots.txt State' },
+    { key: 'inspection.indexingState', header: 'Indexing State' },
+    { key: 'inspection.lastCrawlTime', header: 'Last Crawl Time' },
+    { key: 'inspection.googleCanonical', header: 'Google Canonical' },
+    { key: 'inspection.userCanonical', header: 'User Canonical' },
+    { key: 'inspection.mobileVerdict', header: 'Mobile Usability' },
+    { key: 'inspection.ampVerdict', header: 'AMP Verdict' },
+    { key: 'inspection.richResultsVerdict', header: 'Rich Results Verdict' },
+  ],
+  analytics: [
+    ...PAGE,
+    { key: 'ga4.sessions', header: 'Sessions' },
+    { key: 'ga4.users', header: 'Users' },
+    { key: 'ga4.pageviews', header: 'Pageviews' },
+    { key: 'ga4.engagementRate', header: 'Engagement Rate' },
+    { key: 'ga4.avgSessionDuration', header: 'Avg. Session Duration' },
+    { key: 'ga4.fetchedAt', header: 'Fetched At' },
+  ],
+  'ai:openai': AI,
+  'ai:anthropic': AI,
+  'ai:ollama': AI,
+  'seo:ahrefs': [
+    ...PAGE,
+    { key: 'seo.metrics.domainRating', header: 'Domain Rating' },
+    { key: 'seo.metrics.urlRating', header: 'URL Rating' },
+    { key: 'seo.metrics.backlinks', header: 'Backlinks' },
+    { key: 'seo.metrics.refDomains', header: 'Referring Domains' },
+    ...SEO_TAIL,
+  ],
+  'seo:majestic': [
+    ...PAGE,
+    { key: 'seo.metrics.trustFlow', header: 'Trust Flow' },
+    { key: 'seo.metrics.citationFlow', header: 'Citation Flow' },
+    { key: 'seo.metrics.externalBacklinks', header: 'External Backlinks' },
+    { key: 'seo.metrics.refDomains', header: 'Referring Domains' },
+    ...SEO_TAIL,
+  ],
+  'seo:moz': [
+    ...PAGE,
+    { key: 'seo.metrics.domainAuthority', header: 'Domain Authority' },
+    { key: 'seo.metrics.pageAuthority', header: 'Page Authority' },
+    { key: 'seo.metrics.spamScore', header: 'Spam Score' },
+    { key: 'seo.metrics.linkingDomains', header: 'Linking Domains' },
+    ...SEO_TAIL,
+  ],
+  'seo:semrush': [
+    ...PAGE,
+    { key: 'seo.metrics.organicKeywords', header: 'Organic Keywords' },
+    { key: 'seo.metrics.organicTraffic', header: 'Organic Traffic' },
+    { key: 'seo.metrics.organicCost', header: 'Organic Cost' },
+    { key: 'seo.metrics.adwordsKeywords', header: 'AdWords Keywords' },
+    ...SEO_TAIL,
+  ],
+};
+
+export function isExportDatasetKey(value: string): value is ExportDatasetKey {
+  return Object.prototype.hasOwnProperty.call(EXPORT_DATASET_COLUMNS, value);
+}

packages/shared-types/src/index.ts

@@ -11,3 +11,4 @@ export * from './google.js';
 export * from './ai.js';
 export * from './seo.js';
 export * from './loganalyzer.js';
+export * from './export-datasets.js';

packages/shared-types/src/ipc.ts

@@ -1,3 +1,4 @@
+import type { ExportDatasetKey } from './export-datasets.js';
 import type {
   AdvancedFilter,
   BrokenLinkRow,
@@ -64,6 +65,10 @@ import type {
   LogSeedDiscoveryResult,
   LogExportInput,
   LogExportResult,
+  LogThreatsInput,
+  LogThreatsResult,
+  LogThreatSummary,
+  LogThreatIpRow,
 } from './loganalyzer.js';
 
 export const IPC = {
@@ -189,6 +194,11 @@ export const IPC = {
   confirmClear: 'confirm:clear',
   logsGetAll: 'logs:get-all',
   logsClear: 'logs:clear',
+  /** Write text to the OS clipboard from the main process. The renderer's
+   *  own `navigator.clipboard` needs a focused document and a granted
+   *  permission, neither of which holds in every window — Electron's
+   *  `clipboard` module has no such preconditions. */
+  clipboardWriteText: 'clipboard:write-text',
   /** Single log entry — kept for compatibility but not used for the
    * live tail anymore; high-volume crawls would saturate the IPC
    * channel. The renderer receives entries via `logsBatch` instead. */
@@ -407,6 +417,12 @@ export const IPC = {
   logSeedDiscovery: 'loganalyzer:seed-discovery',
   logExport: 'loganalyzer:export',
   logClear: 'loganalyzer:clear',
+  /** Suspicious Requests tab — flagged log lines (SQLi / XSS / scanner
+   *  probes …) with sender IP + decoded payload, their roll-up, and the
+   *  sender list that backs the blocklist copy. */
+  logThreats: 'loganalyzer:threats',
+  logThreatSummary: 'loganalyzer:threat-summary',
+  logThreatIps: 'loganalyzer:threat-ips',
 } as const;
 
 export type IpcChannel = (typeof IPC)[keyof typeof IPC];
@@ -609,7 +625,17 @@ export interface ExportImagesResult {
 export interface ExportTabularSection {
   /** Display label — becomes the xlsx sheet name and the per-file label. */
   label: string;
-  category: UrlCategory;
+  /** URL category to stream `CrawlUrlRow`s from. Required unless `dataset` is set. */
+  category?: UrlCategory;
+  /**
+   * Non-URL-row table to export instead of a category — Images, Broken
+   * Links, Search Console, GA4, PageSpeed, CrUX, Spelling, AI, SEO
+   * Authority. Rows come from the dataset's own query; `columns` names its
+   * dotted column keys (see `EXPORT_DATASET_COLUMNS`).
+   */
+  dataset?: ExportDatasetKey;
+  /** Column keys for this section only; overrides the input-level `columns`. */
+  columns?: string[];
   /** Optional subdirectory under the chosen folder. When set, the section
    *  file lands at `<root>/<subdir>/<filename>.<ext>` instead of the flat
    *  `<root>/<filename>.<ext>`. Lets a hierarchical tree pick (Internal →
@@ -1854,6 +1880,9 @@ export interface FreeCrawlApi {
    *  app-global entries; omitted returns everything (primary/legacy view). */
   logsGetAll(ownerId?: number): Promise<LogEntry[]>;
   logsClear(): Promise<void>;
+  /** Copy `text` to the OS clipboard via the main process. Resolves true
+   *  on success; the renderer falls back to the DOM paths on false. */
+  clipboardWriteText(text: string): Promise<boolean>;
   logsOpenWindow(): Promise<void>;
   openVisualizationWindow(): Promise<void>;
   readScreenshot(absolutePath: string): Promise<string | null>;
@@ -2062,6 +2091,12 @@ export interface FreeCrawlApi {
   logExport(input: LogExportInput): Promise<LogExportResult>;
   /** Wipe every ingested log aggregate from the project. */
   logClear(): Promise<void>;
+  /** Paginated flagged requests (Suspicious Requests tab). */
+  logThreats(input: LogThreatsInput): Promise<LogThreatsResult>;
+  /** Totals, per-category counts and busiest senders of flagged requests. */
+  logThreatSummary(): Promise<LogThreatSummary>;
+  /** Senders of flagged requests, busiest first — backs the blocklist copy. */
+  logThreatIps(limit?: number): Promise<LogThreatIpRow[]>;
   onProgress(cb: (p: CrawlProgress) => void): () => void;
   onDone(cb: (summary: CrawlSummary) => void): () => void;
   onError(cb: (message: string) => void): () => void;

packages/shared-types/src/loganalyzer.ts

@@ -86,6 +86,110 @@ export interface LogIngestInput {
   }>;
   /** Per-(path, bot) hit counts — backs the URL Hits per-bot filter. */
   urlBots: Array<{ path: string; bot: string; hits: number }>;
+  /** Requests the threat classifier flagged (SQLi / XSS / scanner probes …),
+   *  one row per log line — the only per-line data the analyzer keeps. */
+  threats: LogThreatHit[];
+  /** Flagged lines dropped once the per-file threat cap was reached. */
+  threatsDropped?: number;
+}
+
+/** What a flagged request was trying to do. */
+export type LogThreatCategory =
+  | 'sqli'
+  | 'xss'
+  | 'traversal'
+  | 'cmdi'
+  | 'scanner'
+  | 'sensitive-file'
+  | 'anomaly';
+
+/**
+ * One access-log line the threat classifier flagged. Unlike every other
+ * analyzer aggregate this is per-line: the point is to show the exact IP,
+ * timestamp and decoded parameter payload, so nothing is rolled up.
+ */
+export interface LogThreatHit {
+  /** Epoch ms, or null when the line's timestamp could not be parsed. */
+  ts: number | null;
+  ip: string | null;
+  method: string | null;
+  /** Request target exactly as logged (path + query, still percent-encoded). */
+  path: string;
+  /** Same target percent-decoded (up to two rounds) — the human-readable payload. */
+  decoded: string;
+  status: number | null;
+  bytes: number | null;
+  userAgent: string | null;
+  referer: string | null;
+  /** Declared bot name when the UA matched the bot library, else null. */
+  bot: string | null;
+  /** Category of the strongest matching rule. */
+  category: LogThreatCategory;
+  /** Ids of every rule that matched, strongest first. */
+  rules: string[];
+  /** Sum of matched rule weights — higher = more certain. */
+  score: number;
+  /** The decoded fragment the strongest rule matched — what to highlight. */
+  evidence: string;
+  /**
+   * Set when reverse-DNS verification ran and the IP belongs to a search
+   * engine's own infrastructure (`google` / `bing` / `yandex` / `apple`).
+   * Such a payload was relayed through the engine (Lens, Translate, a
+   * fetch of a crafted URL) — blocking the IP would block the engine.
+   */
+  ipOwner: string | null;
+}
+
+/** A persisted flagged request — `LogThreatHit` plus its row id and the
+ *  crawl join. */
+export interface LogThreatRow extends LogThreatHit {
+  id: number;
+  /** True when the target's path (query stripped) is a crawled internal URL. */
+  targetInCrawl: boolean;
+}
+
+export interface LogThreatsInput {
+  limit: number;
+  offset: number;
+  /** Substring match over the decoded target, IP and user-agent. */
+  search?: string;
+  category?: LogThreatCategory | 'all';
+  /** Exact IP filter (drill-down from the IP panel). */
+  ip?: string;
+  /** Response class of the flagged request; `2xx` = the payload was served. */
+  status?: 'all' | '2xx' | '3xx' | '4xx' | '5xx';
+  sortBy?: 'ts' | 'score' | 'status';
+}
+
+export interface LogThreatsResult {
+  rows: LogThreatRow[];
+  total: number;
+}
+
+export interface LogThreatIpRow {
+  ip: string;
+  hits: number;
+  /** Distinct categories this IP was flagged under. */
+  categories: LogThreatCategory[];
+  firstTs: number | null;
+  lastTs: number | null;
+  /** How many of its flagged requests were answered 2xx. */
+  servedHits: number;
+  /** Declared bot name seen on this IP's flagged requests (first wins). */
+  bot: string | null;
+  ipOwner: string | null;
+}
+
+/** Roll-up the Suspicious Requests tab shows above its table. */
+export interface LogThreatSummary {
+  totalHits: number;
+  distinctIps: number;
+  /** Flagged requests the server answered 2xx — the ones that matter. */
+  servedHits: number;
+  byCategory: Array<{ category: LogThreatCategory; hits: number; ips: number }>;
+  topIps: LogThreatIpRow[];
+  /** True when at least one ingest hit the per-file threat cap. */
+  capHit: boolean;
 }
 
 /** One ingested file's metadata — mirrors a `log_ingests` row. */
@@ -129,6 +233,9 @@ export interface LogOverview {
   distinctUrls: number;
   minTs: number | null;
   maxTs: number | null;
+  /** Requests the threat classifier flagged, and how many IPs sent them. */
+  threatHits: number;
+  threatIps: number;
 }
 
 /** One URL's aggregated log activity (item 6 — bot hits per URL). */