feat: canonical resolution, crawl-trap filter, gzip log + sitemap reads

README.md

@@ -177,6 +177,7 @@ node apps/cli/dist/index.js --list urls.txt --out out.json     # list mode + JSO
 # Log file analysis — parse an access log into the project for crawl × log reporting
 node apps/cli/dist/index.js analyze-logs samples/apache-access.txt --project crawl.seoproject
 node apps/cli/dist/index.js analyze-logs access.log --format iis-w3c --verify-bots --json
+node apps/cli/dist/index.js analyze-logs access_log-Jul.gz              # gzipped logs work as-is
 ```
 
 > **Sample logs** ship in [`samples/`](samples/) (`apache-access.txt`, `iis-access.txt`) so you can try the Log File Analyzer immediately — in the desktop app, open **Log Analyzer → Open Log Analyzer Window…** and import one.

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.9.5",
+  "version": "0.9.6",
   "private": true,
   "type": "module",
   "bin": {

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.9.5",
+  "version": "0.9.6",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",

apps/desktop/src/main/db-writer-pool.ts

@@ -2,6 +2,7 @@ import { Worker } from 'node:worker_threads';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import * as logger from './logger.js';
+import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js';
 
 /**
  * Single-worker pool for SQLite writes. Mirrors the structure of
@@ -16,19 +17,65 @@ import * as logger from './logger.js';
  *      thread so IPC + queue scheduling stay live during writes.
  *   2. Give us one centralised place to add batching / rate-limiting
  *      later (Phase 2 — micro-batching).
+ *
+ * ── Why there is deliberately NO per-request timeout ──────────────
+ * The pool used to reject any call still pending after 60 s. That
+ * reject did NOT cancel the job — the worker kept executing it — so
+ * every "timeout → main-thread fallback" ran the SAME write twice on
+ * TWO writer connections. On big projects the post-crawl issue
+ * recompute legitimately exceeds 60 s, at which point the fallback
+ * collided with the still-running worker pass: `database is locked`
+ * (SQLITE_BUSY after `busy_timeout`), a torn `urls_issues` table, and
+ * an unhandledRejection that aborted crawl teardown.
+ *
+ * The invariant is: while the worker is ALIVE, it is the only writer.
+ * A pending call therefore waits until the worker answers or dies.
+ * Diagnostics are preserved by the slow-call warner below, and a
+ * genuinely wedged worker (zero JS activity for `WEDGE_TIMEOUT_MS`
+ * with work pending — an infinite-loop class bug, since SQLite
+ * statements always terminate) is recovered by terminating the worker
+ * FIRST: only once its connection is dead do pending calls reject
+ * (`WriterUnavailableError`) and the main-thread fallback run, so the
+ * single-writer invariant holds on every path.
  */
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const WORKER_PATH = path.join(__dirname, 'db-writer-worker.js');
 
-const REQUEST_TIMEOUT_MS = 60_000;
 const MAX_RESTARTS = 3;
 const RESTART_WINDOW_MS = 60_000;
+/** Log a warn for any call still pending after this long (and again
+ *  every interval after) — visibility without control-flow impact. */
+const SLOW_CALL_WARN_MS = 30_000;
+const MONITOR_INTERVAL_MS = 15_000;
+/** Worker heartbeat ticks every 100 ms whenever its event loop is
+ *  free; long sync SQL blocks it between yields. Zero ticks for this
+ *  long WITH a request pending means the thread has executed no JS at
+ *  all for 10 minutes — treat as wedged and recycle the worker. */
+const WEDGE_TIMEOUT_MS = 10 * 60_000;
+
+/**
+ * Rejection type for "the worker cannot run this job at all" —
+ * crashed, terminated, swapped away, or never spawned. This is the
+ * ONLY error class on which callers may fall back to the main-thread
+ * `ProjectDb`: in every one of these states the worker's SQLite
+ * connection is gone, so a main-thread run cannot collide with it.
+ * Any other rejection means the worker is alive (the job itself
+ * threw) — falling back there would double-write.
+ */
+export class WriterUnavailableError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'WriterUnavailableError';
+  }
+}
 
 interface PendingRequest {
   resolve: (v: unknown) => void;
   reject: (e: Error) => void;
-  timer: ReturnType<typeof setTimeout>;
+  method: string;
+  startedAt: number;
+  lastWarnAt: number;
 }
 
 interface ResponseMessage {
@@ -42,23 +89,25 @@ export class DbWriterPool {
   private worker: Worker | null = null;
   private dbPath: string | null = null;
   private freezeWatchdogSab: SharedArrayBuffer | null = null;
+  private watchdogState: FreezeWatchdogSharedState | null = null;
   private nextRequestId = 1;
   private pending = new Map<number, PendingRequest>();
   private restartTimes: number[] = [];
   private terminated = false;
+  private monitorTimer: ReturnType<typeof setInterval> | null = null;
 
   init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
     if (this.terminated) {
       throw new Error('DbWriterPool: cannot init after terminate()');
     }
-    this.freezeWatchdogSab = freezeWatchdogSab;
+    this.setWatchdogSab(freezeWatchdogSab);
     if (this.dbPath === dbPath && this.worker !== null) return;
     this.dbPath = dbPath;
     this.spawn();
   }
 
   swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
-    if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab;
+    if (freezeWatchdogSab !== null) this.setWatchdogSab(freezeWatchdogSab);
     if (this.dbPath === newPath && this.worker !== null) return;
     this.failPendingWith('writer-swapped');
     this.dbPath = newPath;
@@ -70,18 +119,21 @@ export class DbWriterPool {
   }
 
   call<T>(method: string, args: unknown[] = []): Promise<T> {
-    if (this.terminated) return Promise.reject(new Error('writer-terminated'));
-    if (!this.worker) return Promise.reject(new Error('writer-not-initialised'));
+    if (this.terminated) {
+      return Promise.reject(new WriterUnavailableError('writer-terminated'));
+    }
+    if (!this.worker) {
+      return Promise.reject(new WriterUnavailableError('writer-not-initialised'));
+    }
     const requestId = this.nextRequestId++;
     return new Promise<T>((resolve, reject) => {
-      const timer = setTimeout(() => {
-        this.pending.delete(requestId);
-        reject(new Error(`writer-timeout: ${method} > ${REQUEST_TIMEOUT_MS}ms`));
-      }, REQUEST_TIMEOUT_MS);
+      const now = Date.now();
       this.pending.set(requestId, {
         resolve: resolve as (v: unknown) => void,
         reject,
-        timer,
+        method,
+        startedAt: now,
+        lastWarnAt: now,
       });
       this.worker!.postMessage({ requestId, method, args });
     });
@@ -89,6 +141,10 @@ export class DbWriterPool {
 
   async terminate(): Promise<void> {
     this.terminated = true;
+    if (this.monitorTimer) {
+      clearInterval(this.monitorTimer);
+      this.monitorTimer = null;
+    }
     this.failPendingWith('writer-terminated');
     if (this.worker) {
       const w = this.worker;
@@ -101,6 +157,13 @@ export class DbWriterPool {
     }
   }
 
+  private setWatchdogSab(sab: SharedArrayBuffer | null): void {
+    if (sab === null) return;
+    if (this.freezeWatchdogSab === sab && this.watchdogState !== null) return;
+    this.freezeWatchdogSab = sab;
+    this.watchdogState = new FreezeWatchdogSharedState(sab);
+  }
+
   private spawn(): void {
     if (!this.dbPath) return;
     if (this.worker) {
@@ -121,6 +184,7 @@ export class DbWriterPool {
       });
       w.on('exit', (code) => this.handleExit(code));
       this.worker = w;
+      this.startMonitor();
       logger.log('info', 'main', `db-writer worker spawned for ${this.dbPath}`);
     } catch (err) {
       logger.log(
@@ -132,14 +196,66 @@ export class DbWriterPool {
     }
   }
 
+  /**
+   * Periodic health sweep over pending calls. Two duties:
+   *   1. Slow-call visibility — a warn line (with the worker's current
+   *      op from the freeze-watchdog SAB) for every call running past
+   *      `SLOW_CALL_WARN_MS`, repeated so long stalls stay visible in
+   *      the log without ever rejecting the call.
+   *   2. Wedge recovery — if calls are pending but the worker thread
+   *      has executed no JS whatsoever for `WEDGE_TIMEOUT_MS`
+   *      (heartbeat frozen), terminate it. `handleExit` then rejects
+   *      the pending calls with `WriterUnavailableError` and respawns,
+   *      so fallbacks only ever run against a dead worker connection.
+   */
+  private startMonitor(): void {
+    if (this.monitorTimer) return;
+    this.monitorTimer = setInterval(() => this.monitorTick(), MONITOR_INTERVAL_MS);
+    this.monitorTimer.unref();
+  }
+
+  private monitorTick(): void {
+    if (this.pending.size === 0) return;
+    const now = Date.now();
+    let oldestStartedAt = now;
+    for (const p of this.pending.values()) {
+      oldestStartedAt = Math.min(oldestStartedAt, p.startedAt);
+      const elapsed = now - p.startedAt;
+      if (elapsed >= SLOW_CALL_WARN_MS && now - p.lastWarnAt >= SLOW_CALL_WARN_MS) {
+        p.lastWarnAt = now;
+        const op = this.watchdogState?.readWriterOp() ?? 'unknown';
+        logger.log(
+          'warn',
+          'main',
+          `db-writer '${p.method}' still running after ${Math.round(elapsed / 1000)}s (worker op: ${op}) — waiting, not falling back`,
+        );
+      }
+    }
+    if (!this.worker || !this.watchdogState) return;
+    const heartbeatAge = now - this.watchdogState.readWriterHeartbeatMs();
+    if (heartbeatAge >= WEDGE_TIMEOUT_MS && now - oldestStartedAt >= WEDGE_TIMEOUT_MS) {
+      const op = this.watchdogState.readWriterOp();
+      logger.log(
+        'error',
+        'main',
+        `db-writer worker wedged: no JS activity for ${Math.round(heartbeatAge / 1000)}s with ${this.pending.size} call(s) pending (worker op: ${op}) — terminating and respawning`,
+      );
+      // terminate() fires 'exit' → handleExit rejects pending calls
+      // (WriterUnavailableError) and respawns. Fallbacks stay safe:
+      // the wedged connection is dead before any main-thread write.
+      void this.worker.terminate().catch(() => undefined);
+    }
+  }
+
   private handleResponse(msg: ResponseMessage): void {
     const pending = this.pending.get(msg.requestId);
     if (!pending) return;
     this.pending.delete(msg.requestId);
-    clearTimeout(pending.timer);
     if (msg.ok) {
       pending.resolve(msg.result);
     } else {
+      // Worker is alive and the job itself threw — NOT a
+      // WriterUnavailableError, so callers must not fall back.
       pending.reject(new Error(msg.error ?? 'writer-unknown-error'));
     }
   }
@@ -170,8 +286,7 @@ export class DbWriterPool {
 
   private failPendingWith(reason: string): void {
     for (const p of this.pending.values()) {
-      clearTimeout(p.timer);
-      p.reject(new Error(reason));
+      p.reject(new WriterUnavailableError(reason));
     }
     this.pending.clear();
   }
@@ -195,9 +310,13 @@ export function setWriterPoolResolver(fn: () => DbWriterPool): void {
 
 /**
  * Helper: try the worker first, fall back to a synchronous main-thread
- * call on any worker error. The fallback keeps writes happening even
- * when the worker is crashed/restarting, at the cost of running on
- * main for that one operation.
+ * call ONLY when the worker cannot run the job at all
+ * (`WriterUnavailableError`: crashed / terminated / swapped / never
+ * spawned — its SQLite connection is gone, so a main-thread run can't
+ * collide with it). Errors thrown by the job itself propagate: the
+ * worker is alive, and re-running the same write on a second
+ * connection is exactly the double-writer bug this pool exists to
+ * prevent.
  */
 export async function callWriterOrFallback<T>(
   method: string,
@@ -211,11 +330,11 @@ export async function callWriterOrFallback<T>(
   try {
     return await pool.call<T>(method, args);
   } catch (err) {
-    const msg = err instanceof Error ? err.message : String(err);
+    if (!(err instanceof WriterUnavailableError)) throw err;
     logger.log(
       'warn',
       'main',
-      `'${method}' fell back to main-thread writer: ${msg}`,
+      `'${method}' fell back to main-thread writer: ${err.message}`,
     );
     return fallback();
   }

apps/desktop/src/main/db-writer-worker.ts

@@ -128,8 +128,19 @@ parentPort.on('message', async (msg: RequestMessage) => {
     if (typeof fn !== 'function') {
       throw new Error(`db-writer-worker: method '${method}' missing on ProjectDb`);
     }
-    const out = (fn as (...a: unknown[]) => unknown).apply(db, args ?? []);
-    const result = out instanceof Promise ? await out : out;
+    let result: unknown;
+    if (method === 'recomputeUrlsIssuesYielding' && watchdog) {
+      // Publish per-definition progress into the watchdog op slot so
+      // the pool's slow-call warner (and debug.txt) can distinguish
+      // "long pass, still advancing" from a genuinely wedged worker.
+      const defs = (args?.[0] ?? []) as ReadonlyArray<readonly [string, string]>;
+      result = await db.recomputeUrlsIssuesYielding(defs, (done, total) => {
+        watchdog.setWriterOp(`recomputeUrlsIssuesYielding ${done}/${total}`);
+      });
+    } else {
+      const out = (fn as (...a: unknown[]) => unknown).apply(db, args ?? []);
+      result = out instanceof Promise ? await out : out;
+    }
     parentPort!.postMessage({ requestId, ok: true, result });
   } catch (err) {
     parentPort!.postMessage({

apps/desktop/src/main/index.ts

@@ -320,6 +320,7 @@ import {
   dbWriterPool,
   DbWriterPool,
   setWriterPoolResolver,
+  WriterUnavailableError,
 } from './db-writer-pool.js';
 import { freezeWatchdog } from './freeze-watchdog.js';
 import { parserPool } from './parser-pool.js';
@@ -1605,10 +1606,13 @@ async function parseHtmlViaPool(
 }
 
 /**
- * Dispatch a per-URL write batch to the writer worker. Same fall-back
- * pattern as the parser pool: when the worker is unhealthy we run
- * the write on the main-thread `ProjectDb` instance so the crawl
- * never silently drops data.
+ * Dispatch a per-URL write batch to the writer worker. Falls back to
+ * the main-thread `ProjectDb` ONLY on `WriterUnavailableError` — the
+ * worker's connection is dead, so the crawl never silently drops data
+ * and the fallback can't collide with a live worker. Any other error
+ * means the worker is alive and the write itself failed; re-running it
+ * on a second connection is the double-writer bug (`database is
+ * locked`) this pool exists to prevent, so it propagates instead.
  */
 async function writeFetchedUrlViaPool(
   session: ProjectSession,
@@ -1620,12 +1624,11 @@ async function writeFetchedUrlViaPool(
   try {
     return await session.writerPool.call<{ urlId: number }>('writeFetchedUrl', [payload]);
   } catch (err) {
+    if (!(err instanceof WriterUnavailableError)) throw err;
     logger.log(
       'warn',
       'main',
-      `db-writer dispatch failed (${
-        err instanceof Error ? err.message : String(err)
-      }) — falling back to main-thread writeFetchedUrl.`,
+      `db-writer unavailable (${err.message}) — falling back to main-thread writeFetchedUrl.`,
     );
     return session.getDb().writeFetchedUrl(payload);
   }
@@ -1648,12 +1651,17 @@ async function recomputeIssuesViaPool(
   try {
     await session.writerPool.call<void>('recomputeUrlsIssuesYielding', [definitions]);
   } catch (err) {
+    // Fall back ONLY when the worker is gone (crashed/terminated/
+    // swapped). The old any-error fallback fired on the 60 s dispatch
+    // timeout while the worker was STILL running the pass — two writer
+    // connections re-running the same DELETE+INSERT pipeline, which is
+    // exactly the "database is locked" + torn urls_issues failure this
+    // fix removes. A slow pass now simply finishes on the worker.
+    if (!(err instanceof WriterUnavailableError)) throw err;
     logger.log(
       'warn',
       'main',
-      `recomputeUrlsIssuesYielding dispatch failed (${
-        err instanceof Error ? err.message : String(err)
-      }) — falling back to main-thread recompute.`,
+      `db-writer unavailable (${err.message}) — falling back to main-thread recompute.`,
     );
     await session.getDb().recomputeUrlsIssuesYielding(definitions);
   }
@@ -1772,12 +1780,12 @@ async function runDbPassViaPool(
   try {
     await session.writerPool.call<void>(method, []);
   } catch (err) {
+    // WriterUnavailableError only — see recomputeIssuesViaPool.
+    if (!(err instanceof WriterUnavailableError)) throw err;
     logger.log(
       'warn',
       'main',
-      `${method} dispatch failed (${
-        err instanceof Error ? err.message : String(err)
-      }) — falling back to main-thread.`,
+      `db-writer unavailable (${err.message}) — running ${method} on main thread.`,
     );
     const db = session.getDb();
     const fn = (db as unknown as Record<string, unknown>)[method];
@@ -1811,12 +1819,12 @@ async function dbCallViaPool<T>(
   try {
     return await session.writerPool.call<T>(method, args);
   } catch (err) {
+    // WriterUnavailableError only — see recomputeIssuesViaPool.
+    if (!(err instanceof WriterUnavailableError)) throw err;
     logger.log(
       'warn',
       'main',
-      `dbCall(${method}) fell back to main-thread: ${
-        err instanceof Error ? err.message : String(err)
-      }`,
+      `dbCall(${method}) fell back to main-thread: ${err.message}`,
     );
     const db = session.getDb();
     const fn = (db as unknown as Record<string, unknown>)[method];
@@ -4236,6 +4244,8 @@ function registerIpc(): void {
           stripWww: input.stripWww,
           forceHttps: input.forceHttps,
           lowercasePath: input.lowercasePath,
+          sortQueryParams: input.sortQueryParams,
+          collapseDuplicateSlashes: input.collapseDuplicateSlashes,
           trailingSlash: input.trailingSlash,
           keepQueryParams: input.keepQueryParams,
           regexRewrites: compiled,
@@ -6730,7 +6740,15 @@ function registerIpc(): void {
     crawler.on('error', () => {
       void disposeBrowserPool(sess);
     });
-    void crawler.start();
+    // Safety net for anything start() might still throw: surface it
+    // through the normal 'error' listener path (log + crawlError IPC +
+    // lease release) and drop the power-save blocker, instead of dying
+    // as an unhandledRejection that leaves the UI pinned to "Running".
+    crawler.start().catch((err) => {
+      const msg = err instanceof Error ? err.message : String(err);
+      crawler.emit('error', `Crawl aborted by unexpected error: ${msg}`);
+      releaseCrawlPowerBlocker(crawler);
+    });
     // A new crawl joined the pool — reshare the global concurrency budget
     // across every live crawl so parallel agents don't oversubscribe.
     rebalanceGlobalConcurrency();
@@ -8984,7 +9002,13 @@ function registerIpc(): void {
       // Snapshot consumed — clear so a second click can't double-fire.
       sess.pendingRecoveryCheckpoint = [];
       crawler.enqueueCheckpointed(items);
-      void crawler.start();
+      // Same safety net as launchCrawl — never let a start() throw
+      // escape as an unhandledRejection.
+      crawler.start().catch((err) => {
+        const msg = err instanceof Error ? err.message : String(err);
+        crawler.emit('error', `Crawl aborted by unexpected error: ${msg}`);
+        releaseCrawlPowerBlocker(crawler);
+      });
       // Hand the config back so the toolbar can adopt the crawl that is
       // now actually running. Without this the renderer keeps the blank
       // Spider defaults it booted with, and a Sitemap/List resume reads

apps/desktop/src/renderer/src/components/OverviewSidebar.tsx

@@ -619,6 +619,24 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.multipleCanonicals,
               category: 'issues:multiple-canonicals',
             },
+            {
+              key: 'issues-canonicals-conflicting',
+              label: 'Conflicting Canonicals',
+              count: o.issues.canonicalConflicting,
+              category: 'issues:canonical-conflicting',
+            },
+            {
+              key: 'issues-canonicals-cross-domain',
+              label: 'Canonical → Cross-Domain',
+              count: o.issues.canonicalCrossDomain,
+              category: 'issues:canonical-cross-domain',
+            },
+            {
+              key: 'issues-canonicals-noindex-conflict',
+              label: 'Noindex + Canonical Conflict',
+              count: o.issues.noindexCanonicalConflict,
+              category: 'issues:noindex-canonical-conflict',
+            },
             {
               key: 'issues-canonicals-non-200',
               label: 'Canonical → Non-200',
@@ -871,6 +889,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.urlMalformed,
               category: 'issues:url-malformed',
             },
+            {
+              key: 'issues-url-crawl-trap',
+              label: 'Crawl Trap',
+              count: o.issues.crawlTrap,
+              category: 'issues:crawl-trap',
+            },
             {
               key: 'issues-url-query-too-long',
               label: 'Long Query String (>100 chars)',
@@ -1177,6 +1201,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.redirectSelf,
               category: 'issues:redirect-self',
             },
+            {
+              key: 'issues-redirect-to-noindex',
+              label: 'Redirect → Noindex',
+              count: o.issues.redirectToNoindex,
+              category: 'issues:redirect-to-noindex',
+            },
           ],
         },
         {

apps/desktop/src/renderer/src/components/SettingsDialog.tsx

@@ -140,6 +140,10 @@ interface FormState {
   stripWww: boolean;
   forceHttps: boolean;
   lowercasePath: boolean;
+  sortQueryParams: boolean;
+  collapseDuplicateSlashes: boolean;
+  maxRepeatedPathSegments: string;
+  maxQueryParams: string;
   trailingSlash: 'leave' | 'strip' | 'add';
   keepQueryParamsText: string;
   urlRegexRewrites: Array<{ pattern: string; replacement: string; flags?: string }>;
@@ -555,6 +559,10 @@ function configToForm(c: CrawlConfig): FormState {
     stripWww: c.stripWww,
     forceHttps: c.forceHttps,
     lowercasePath: c.lowercasePath,
+    sortQueryParams: c.sortQueryParams,
+    collapseDuplicateSlashes: c.collapseDuplicateSlashes,
+    maxRepeatedPathSegments: String(c.maxRepeatedPathSegments),
+    maxQueryParams: String(c.maxQueryParams),
     trailingSlash: c.trailingSlash,
     keepQueryParamsText: (c.keepQueryParams ?? []).join('\n'),
     urlRegexRewrites: (c.urlRegexRewrites ?? []).map((r) => ({ ...r })),
@@ -848,6 +856,10 @@ export function SettingsDialog({ open, onClose }: Props) {
       stripWww: form.stripWww,
       forceHttps: form.forceHttps,
       lowercasePath: form.lowercasePath,
+      sortQueryParams: form.sortQueryParams,
+      collapseDuplicateSlashes: form.collapseDuplicateSlashes,
+      maxRepeatedPathSegments: Math.max(0, num(form.maxRepeatedPathSegments, config.maxRepeatedPathSegments)),
+      maxQueryParams: Math.max(0, num(form.maxQueryParams, config.maxQueryParams)),
       trailingSlash: form.trailingSlash,
       keepQueryParams: parseLines(form.keepQueryParamsText)
         .map((s) => s.trim())
@@ -1627,6 +1639,20 @@ function CrawlerPanel({ form, update }: PanelProps) {
           info="Per-request abort threshold. Pages that take longer than this are recorded as network errors. Screaming Frog: 'Response Timeout (secs)' (Configuration → Spider → Advanced) — that one's in seconds, this is in milliseconds."
           example="20000 (20 s) for typical use; 5000 for fast spot checks; 60000 for slow APIs."
         />
+        <Num
+          label={t('settingsPanels.crawler.maxRepeatedPathSegments', { defaultValue: 'Crawl trap: repeated path segments — 0 = off' })}
+          value={form.maxRepeatedPathSegments}
+          onChange={(v) => update('maxRepeatedPathSegments', v)}
+          info="A URL whose path repeats the same segment this many times or more (/shop/shop/shop/…) is treated as a link loop and skipped. This shape comes from a relative-href bug and has no legitimate counterpart. Skipped counts are reported when the crawl finishes."
+          example="3 is safe for every site; raise to 4–5 only if a real path legitimately repeats a segment; 0 disables the guard."
+        />
+        <Num
+          label={t('settingsPanels.crawler.maxQueryParams', { defaultValue: 'Crawl trap: max query parameters — 0 = off' })}
+          value={form.maxQueryParams}
+          onChange={(v) => update('maxQueryParams', v)}
+          info="URLs with more query parameters than this are flagged as faceted-navigation traps under Issues → URL → Crawl Trap. Detection only — the URLs are still crawled, because legitimate filter pages look the same."
+          example="4 surfaces most faceted-nav explosions; 0 disables the check."
+        />
       </div>
 
       <div className="mt-4 grid grid-cols-2 gap-2">
@@ -2422,6 +2448,8 @@ function UrlRewritingPanel({ form, update }: PanelProps) {
         stripWww: form.stripWww,
         forceHttps: form.forceHttps,
         lowercasePath: form.lowercasePath,
+        sortQueryParams: form.sortQueryParams,
+        collapseDuplicateSlashes: form.collapseDuplicateSlashes,
         trailingSlash: form.trailingSlash,
         keepQueryParams: keep,
         urlRegexRewrites: rewrites,
@@ -2470,6 +2498,22 @@ function UrlRewritingPanel({ form, update }: PanelProps) {
           info="Lowercases the URL path component. Host is already case-insensitive per the URL spec, so this only affects the path."
           example="On if your CMS serves the same page at mixed casing (/Foo and /foo)."
         />
+        <Bool
+          label={t('settingsPanels.urlRewriting.sortQueryParams', { defaultValue: 'Sort query parameters' })}
+          checked={form.sortQueryParams}
+          onChange={(v) => update('sortQueryParams', v)}
+          hint={t('settingsPanels.urlRewriting.sortQueryParamsHint', { defaultValue: 'Treat ?b=2&a=1 and ?a=1&b=2 as the same URL' })}
+          info="Sorts query parameters alphabetically at normalisation time. Repeated keys keep their relative order, so ?tag=a&tag=b is preserved. Without this the two orderings occupy separate rows and read as duplicates."
+          example="On for most sites; off if your server routes on positional parameter order."
+        />
+        <Bool
+          label={t('settingsPanels.urlRewriting.collapseDuplicateSlashes', { defaultValue: 'Collapse duplicate slashes' })}
+          checked={form.collapseDuplicateSlashes}
+          onChange={(v) => update('collapseDuplicateSlashes', v)}
+          hint={t('settingsPanels.urlRewriting.collapseDuplicateSlashesHint', { defaultValue: 'Treat /a//b and /a/b as the same URL' })}
+          info="Collapses runs of slashes in the path to a single slash. Applied before the trailing-slash policy. Web servers serve these identically, so the duplicate-slash variant is normally a false duplicate."
+          example="On if a template bug emits //  in links; off if your framework uses empty path segments as data."
+        />
         <label className="flex flex-col gap-1">
           <FieldLabel
             label={t('settingsPanels.urlRewriting.trailingSlashPolicy', { defaultValue: 'Trailing slash policy' })}

apps/desktop/src/renderer/src/i18n/labels.ts

@@ -259,6 +259,11 @@ const TR_LABELS: Record<string, string> = {
   'Canonical → Noindex': 'Canonical → Noindex',
   'Canonical → Redirect': 'Canonical → Yönlendirme',
   'Multiple Canonicals': 'Birden Çok Canonical',
+  'Conflicting Canonicals': 'Çelişen Canonical’ler',
+  'Canonical → Cross-Domain': 'Canonical → Farklı Alan Adı',
+  'Noindex + Canonical Conflict': 'Noindex + Canonical Çatışması',
+  'Redirect → Noindex': 'Yönlendirme → Noindex',
+  'Crawl Trap': 'Tarama Tuzağı',
   'Canonical Conflict': 'Canonical Çatışması',
 
   // Hreflang

apps/desktop/src/renderer/src/i18n/locales/en.json

@@ -466,6 +466,8 @@
       "maxDepth": "Crawl Depth Limit (Max Depth)",
       "maxUrls": "Crawl Total Limit (Max URLs)",
       "requestTimeoutMs": "Response Timeout (ms)",
+      "maxRepeatedPathSegments": "Crawl trap: repeated path segments — 0 = off",
+      "maxQueryParams": "Crawl trap: max query parameters — 0 = off",
       "followRedirects": "Always Follow Redirects",
       "respectRobotsTxt": "Respect robots.txt",
       "respectCrawlDelay": "Respect robots.txt Crawl-delay",
@@ -573,6 +575,10 @@
       "forceHttpsHint": "Upgrade http:// → https:// before fetching",
       "lowercasePath": "Lowercase path",
       "lowercasePathHint": "Treat /Foo and /foo as the same URL",
+      "sortQueryParams": "Sort query parameters",
+      "sortQueryParamsHint": "Treat ?b=2&a=1 and ?a=1&b=2 as the same URL",
+      "collapseDuplicateSlashes": "Collapse duplicate slashes",
+      "collapseDuplicateSlashesHint": "Treat /a//b and /a/b as the same URL",
       "trailingSlashPolicy": "Trailing slash policy",
       "trailingSlashLeave": "Leave as-is",
       "trailingSlashStrip": "Strip (/foo/ → /foo)",

apps/desktop/src/renderer/src/i18n/locales/tr.json

@@ -466,6 +466,8 @@
       "maxDepth": "Crawl Derinlik Limiti (Max Depth)",
       "maxUrls": "Toplam URL Limiti (Max URLs)",
       "requestTimeoutMs": "Yanıt Zaman Aşımı (ms)",
+      "maxRepeatedPathSegments": "Tarama tuzağı: tekrarlanan yol segmenti — 0 = kapalı",
+      "maxQueryParams": "Tarama tuzağı: max sorgu parametresi — 0 = kapalı",
       "followRedirects": "Yönlendirmeleri Daima Takip Et",
       "respectRobotsTxt": "robots.txt'ye Uy",
       "respectCrawlDelay": "robots.txt Crawl-delay'e Uy",
@@ -573,6 +575,10 @@
       "forceHttpsHint": "Fetch öncesi http:// → https:// yükselt",
       "lowercasePath": "Yolu küçük harfe çevir",
       "lowercasePathHint": "/Foo ve /foo'yu aynı URL olarak değerlendir",
+      "sortQueryParams": "Sorgu parametrelerini sırala",
+      "sortQueryParamsHint": "?b=2&a=1 ve ?a=1&b=2’yi aynı URL olarak değerlendir",
+      "collapseDuplicateSlashes": "Çift eğik çizgileri birleştir",
+      "collapseDuplicateSlashesHint": "/a//b ve /a/b’yi aynı URL olarak değerlendir",
       "trailingSlashPolicy": "Sondaki slash politikası",
       "trailingSlashLeave": "Olduğu gibi bırak",
       "trailingSlashStrip": "Kaldır (/foo/ → /foo)",

apps/desktop/src/renderer/src/store.ts

@@ -188,6 +188,7 @@ export const TAB_QUICK_FILTERS: Partial<Record<TabKey, TabQuickFilter[]>> = {
     { label: 'Fragment (#) in URL', category: 'issues:url-fragment' },
     { label: 'Spaces in URL', category: 'issues:url-spaces' },
     { label: 'Malformed URL', category: 'issues:url-malformed' },
+    { label: 'Crawl Trap', category: 'issues:crawl-trap' },
     { label: 'Long Query String (>100 chars)', category: 'issues:query-string-too-long' },
     { label: 'Folder Depth >4', category: 'issues:folder-depth-too-deep' },
     { label: 'Duplicate URL (post-norm)', category: 'issues:duplicate-url-post-norm' },
@@ -256,6 +257,9 @@ export const TAB_QUICK_FILTERS: Partial<Record<TabKey, TabQuickFilter[]>> = {
     { label: 'Canonicalised (→ other)', category: 'issues:canonical-non-self' },
     { label: 'HTTP vs HTML Mismatch', category: 'issues:canonical-mismatch' },
     { label: 'Multiple Canonicals', category: 'issues:multiple-canonicals' },
+    { label: 'Conflicting Canonicals', category: 'issues:canonical-conflicting' },
+    { label: 'Canonical → Cross-Domain', category: 'issues:canonical-cross-domain' },
+    { label: 'Noindex + Canonical Conflict', category: 'issues:noindex-canonical-conflict' },
     { label: 'Canonical → Non-200', category: 'issues:canonical-to-non-200' },
     { label: 'Canonical → Redirect', category: 'issues:canonical-to-redirect' },
     { label: 'Canonical → Noindex', category: 'issues:canonical-to-noindex' },

apps/mcp-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/mcp-server",
-  "version": "0.9.5",
+  "version": "0.9.6",
   "private": true,
   "type": "module",
   "bin": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.9.5",
+  "version": "0.9.6",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.9.5",
+  "version": "0.9.6",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/crawler.ts

@@ -19,6 +19,7 @@ import {
   extractExtension,
   isInScope,
   isUrlMalformed,
+  detectUrlTrap,
   resolveStartUrl,
   compileUrlRegexRewrites,
   toEscapedFragmentUrl,
@@ -625,6 +626,8 @@ export class Crawler extends EventEmitter {
       lowercasePath: config.lowercasePath,
       trailingSlash: config.trailingSlash,
       keepQueryParams: config.keepQueryParams,
+      sortQueryParams: config.sortQueryParams,
+      collapseDuplicateSlashes: config.collapseDuplicateSlashes,
       regexRewrites: compiledRegexRewrites,
     };
   }
@@ -644,6 +647,37 @@ export class Crawler extends EventEmitter {
     return this.includeRegexes.some((re) => re.test(url));
   }
 
+  /**
+   * Classify a URL against the crawl-trap shapes, for storage on its row.
+   * Returns the trap kind or null. Detection is independent of whether the
+   * crawler acts on it - a faceted or calendar URL is still crawled, and
+   * the flag is what makes it visible in the Crawl Trap filter.
+   */
+  private trapKind(url: string): string | null {
+    return detectUrlTrap(url, {
+      maxRepeatedSegments: this.config.maxRepeatedPathSegments,
+      maxQueryParams: this.config.maxQueryParams,
+    });
+  }
+
+  /**
+   * The only trap acted on at enqueue: a path segment repeated N+ times.
+   * That shape is produced by a relative-href bug walking `/a/a/a/...` and
+   * has no legitimate counterpart, so following it only burns budget on a
+   * loop. Every other kind is crawled and merely flagged, because real
+   * archive and filter pages are indistinguishable from the trap version.
+   *
+   * Drops are counted and reported once the crawl ends - never silent.
+   */
+  private isLoopTrap(url: string): boolean {
+    const max = this.config.maxRepeatedPathSegments;
+    if (!max || max <= 0) return false;
+    return detectUrlTrap(url, { maxRepeatedSegments: max }) === 'repeated-segment';
+  }
+
+  /** URLs dropped by `isLoopTrap`, reported at the end of the crawl. */
+  private loopTrapsDropped = 0;
+
   /**
    * Drop URLs whose path extension matches any user-configured exclude
    * (e.g. `pdf`, `jpg`). The start URL is exempt — even if it ends in
@@ -979,6 +1013,16 @@ export class Crawler extends EventEmitter {
     // so IPC dispatch (logs:batch, progress, dataChanged) never starves.
     await this.runPostCrawlPasses();
     this.running = false;
+    // Say what the loop guard dropped. Silently shrinking a crawl is exactly
+    // the kind of invisible behaviour that makes a missing page impossible
+    // to explain afterwards.
+    if (this.loopTrapsDropped > 0) {
+      this.emit(
+        'info',
+        `Crawl trap guard: skipped ${this.loopTrapsDropped.toLocaleString()} URL(s) whose path repeated a ` +
+          `segment ${this.config.maxRepeatedPathSegments}+ times (link loop).`,
+      );
+    }
     this.stopMemoryMonitor();
     // Release per-URL dedup sets — at 1M URLs this is ~80–120 MB of string
     // heap that's no longer needed once the queue is drained.
@@ -2104,6 +2148,7 @@ export class Crawler extends EventEmitter {
         this.db.upsertUrl({
           url: this.config.startUrl,
           urlMalformed: isUrlMalformed(this.config.startUrl) ? 1 : 0,
+          urlTrap: this.trapKind(this.config.startUrl),
           contentKind: 'html',
           statusCode: null,
           statusText: 'fetching',
@@ -2639,6 +2684,13 @@ export class Crawler extends EventEmitter {
       return;
     }
     if (item.depth > this.config.maxDepth) return;
+    // Link-loop guard, before robots/filters so a runaway `/a/a/a/...` fan-out
+    // costs one cheap string check rather than a robots lookup per hop.
+    // The start URL is exempt - the user asked for it explicitly.
+    if (item.url !== this.config.startUrl && this.isLoopTrap(item.url)) {
+      this.loopTrapsDropped++;
+      return;
+    }
     if (this.robots && !this.robots.isAllowed(item.url)) return;
     if (!this.passesUrlFilter(item.url)) return;
     if (!this.passesExtensionFilter(item.url)) return;
@@ -2889,6 +2941,7 @@ export class Crawler extends EventEmitter {
             {
               url: item.url,
               urlMalformed: isUrlMalformed(item.url) ? 1 : 0,
+              urlTrap: this.trapKind(item.url),
               contentKind: 'other',
               statusCode: res.status,
               statusText: 'size-cap-exceeded',
@@ -3059,6 +3112,7 @@ export class Crawler extends EventEmitter {
           {
             url: item.url,
             urlMalformed: isUrlMalformed(item.url) ? 1 : 0,
+            urlTrap: this.trapKind(item.url),
             contentKind: kind,
             statusCode,
             statusText: null,
@@ -3192,6 +3246,7 @@ export class Crawler extends EventEmitter {
           {
             url: item.url,
             urlMalformed: isUrlMalformed(item.url) ? 1 : 0,
+            urlTrap: this.trapKind(item.url),
             contentKind: kind,
             statusCode,
             statusText: null,
@@ -3248,6 +3303,7 @@ export class Crawler extends EventEmitter {
           {
             url: item.url,
             urlMalformed: isUrlMalformed(item.url) ? 1 : 0,
+            urlTrap: this.trapKind(item.url),
             contentKind: 'other',
             statusCode,
             statusText: 'size-cap-exceeded',
@@ -3376,10 +3432,7 @@ export class Crawler extends EventEmitter {
       } else if (headerNoindex) {
         indexability = 'non-indexable:noindex';
         reason = 'X-Robots-Tag: noindex';
-      } else if (
-        parsed.canonical &&
-        normalizeUrl(parsed.canonical, item.url, this.urlRewrites) !== item.url
-      ) {
+      } else if (parsed.canonicalResolved && parsed.canonicalResolved !== item.url) {
         indexability = 'non-indexable:canonical';
         reason = `canonical points to ${parsed.canonical}`;
       } else if (!parsed.canonical && canonicalHttp && canonicalHttp !== item.url) {
@@ -3468,6 +3521,7 @@ export class Crawler extends EventEmitter {
         upsert: {
           url: item.url,
           urlMalformed: isUrlMalformed(item.url) ? 1 : 0,
+          urlTrap: this.trapKind(item.url),
           contentKind: 'html',
           statusCode,
           statusText: null,
@@ -3488,7 +3542,13 @@ export class Crawler extends EventEmitter {
           // empties its tab without silently re-labelling pages as
           // indexable — the verdict and the evidence stay in agreement.
           canonical: this.config.storeCanonicals ? parsed.canonical : null,
+          canonicalResolved: this.config.storeCanonicals ? parsed.canonicalResolved : null,
           canonicalCount: this.config.storeCanonicals ? parsed.canonicalCount : 0,
+          canonicalDistinctCount: this.config.storeCanonicals
+            ? parsed.canonicalDistinctCount
+            : 0,
+          canonicalCrossDomain:
+            this.config.storeCanonicals && parsed.canonicalCrossDomain ? 1 : 0,
           canonicalHttp: this.config.storeCanonicals ? canonicalHttp : null,
           metaRobots: parsed.metaRobots,
           xRobotsTag,
@@ -3760,16 +3820,19 @@ export class Crawler extends EventEmitter {
         // canonicals as a signal, not a navigation hint.
         if (
           this.config.followCanonicals &&
-          parsed.canonical &&
-          parsed.canonical !== item.url
+          parsed.canonicalResolved &&
+          parsed.canonicalResolved !== item.url
         ) {
+          // Resolved, not raw: a relative canonical passed verbatim to
+          // isInScope/enqueue is not a URL and would either be dropped or
+          // fetched against the wrong base.
           const inScope = isInScope(
             this.config.startUrl,
-            parsed.canonical,
+            parsed.canonicalResolved,
             this.config.scope,
           );
           if (inScope || this.config.crawlExternal) {
-            this.enqueue({ url: parsed.canonical, depth: nextDepth });
+            this.enqueue({ url: parsed.canonicalResolved, depth: nextDepth });
           }
         }
         // Wave 3 — JS-style redirect follow. Currently covers
@@ -3839,6 +3902,7 @@ export class Crawler extends EventEmitter {
         {
           url: item.url,
           urlMalformed: isUrlMalformed(item.url) ? 1 : 0,
+          urlTrap: this.trapKind(item.url),
           contentKind: 'html',
           statusCode: null,
           statusText: detail,

packages/core/src/html-parser.ts

@@ -8,7 +8,12 @@ import type {
   LinkPathType,
   LinkPosition,
 } from '@freecrawl/shared-types';
-import { normalizeUrl, isSameHost, type UrlRewriteOptions } from './url-utils.js';
+import {
+  normalizeUrl,
+  isSameHost,
+  registrableDomain,
+  type UrlRewriteOptions,
+} from './url-utils.js';
 import { computeContentFingerprint } from './simhash.js';
 import { runExtractionRules } from './extraction.js';
 
@@ -55,8 +60,22 @@ export interface ParsedPage {
   h6Count: number;
   wordCount: number;
   canonical: string | null;
+  /**
+   * `canonical` resolved against the page URL and normalized - the form a
+   * search engine actually compares. Null when absent or unresolvable.
+   * Use this for every comparison; use `canonical` only to show the author's
+   * literal value.
+   */
+  canonicalResolved: string | null;
   /** Number of `<link rel="canonical">` elements declared. >1 is a confusion signal. */
   canonicalCount: number;
+  /**
+   * Distinct resolved canonical targets. >1 means the page's canonicals
+   * disagree with each other, which makes search engines discard all of them.
+   */
+  canonicalDistinctCount: number;
+  /** True when the resolved canonical points at a different registrable domain. */
+  canonicalCrossDomain: boolean;
   /**
    * Every `<meta name="robots">` content value on the page, lowercased
    * and joined with `, ` in document order. Multiple declarations are
@@ -530,6 +549,41 @@ export function parseHtml(
   const canonicalEls = $('link[rel="canonical"]');
   const canonical = (canonicalEls.first().attr('href') ?? '').trim() || null;
   const canonicalCount = canonicalEls.length;
+  // `canonical` above stays verbatim - the "Canonical Not Absolute" filter
+  // and the URL detail panel both want to show what the page actually
+  // shipped. Everything that *compares* a canonical (self-reference,
+  // canonical->redirect/noindex joins, cross-domain) needs it resolved
+  // against the page URL first, exactly as a search engine resolves it:
+  // `href="/x/"` on `https://a.com/y` is a self-canonical only after
+  // resolution. Same treatment `amphtml` already gets further down.
+  const canonicalResolved = canonical
+    ? normalizeUrl(canonical, pageUrl, opts.urlRewrites)
+    : null;
+  // Every declared canonical, resolved and deduped. `canonicalCount > 1`
+  // only says a page repeated the tag - often harmlessly identical (a CMS
+  // plus a plugin emitting the same href). More than one *distinct* target
+  // is the real defect: Google discards all of them and picks its own.
+  const canonicalTargets = new Set<string>();
+  canonicalEls.each((_, el) => {
+    const href = ($(el).attr('href') ?? '').trim();
+    if (!href) return;
+    const abs = normalizeUrl(href, pageUrl, opts.urlRewrites);
+    if (abs) canonicalTargets.add(abs);
+  });
+  const canonicalDistinctCount = canonicalTargets.size;
+  // Cross-domain canonical - legitimate for syndicated content, but far
+  // more often a migration or staging leak that deindexes the page, so it
+  // is worth surfacing on its own rather than hiding inside "non-self".
+  let canonicalCrossDomain = false;
+  if (canonicalResolved) {
+    try {
+      const target = registrableDomain(new URL(canonicalResolved).hostname);
+      const self = registrableDomain(new URL(pageUrl).hostname);
+      canonicalCrossDomain = target !== self;
+    } catch {
+      canonicalCrossDomain = false;
+    }
+  }
   // Robots directives — collected from EVERY `<meta name="robots">` in
   // the document, not just the first. Two details that a plain
   // `$('meta[name="robots"]').attr('content')` gets wrong on real sites:
@@ -1509,7 +1563,10 @@ export function parseHtml(
     h6Count,
     wordCount,
     canonical,
+    canonicalResolved,
     canonicalCount,
+    canonicalDistinctCount,
+    canonicalCrossDomain,
     metaRobots,
     lang,
     viewport,

packages/core/src/log-analyzer.ts

@@ -16,6 +16,8 @@
 
 import { createReadStream } from 'node:fs';
 import { createInterface } from 'node:readline';
+import type { Readable } from 'node:stream';
+import { createGunzip } from 'node:zlib';
 import type { LogFormat, LogFormatChoice } from '@freecrawl/shared-types';
 import {
   createLogParser,
@@ -82,6 +84,28 @@ export interface LogAnalyzeOptions {
 const DEFAULT_MAX_URLS = 500_000;
 const DEFAULT_MAX_IPS = 200;
 
+/**
+ * Open a log file as a UTF-8 text stream, transparently decompressing
+ * `.gz` files on the fly. Access logs are very commonly shipped gzipped
+ * (`access_log-Jul-2026.gz`); without this the raw DEFLATE bytes would be
+ * decoded as text and every line would fail to parse (0 lines accepted).
+ *
+ * Errors on the underlying file read are forwarded onto the returned
+ * stream so the `for await` consumer rejects instead of hanging, and a
+ * corrupt gzip surfaces as a stream error caught by the caller.
+ */
+function openLogStream(filePath: string): Readable {
+  if (/\.gz$/i.test(filePath)) {
+    const source = createReadStream(filePath);
+    const gunzip = createGunzip();
+    source.on('error', (err) => gunzip.destroy(err));
+    source.pipe(gunzip);
+    gunzip.setEncoding('utf8');
+    return gunzip;
+  }
+  return createReadStream(filePath, { encoding: 'utf8' });
+}
+
 function emptyResult(format: LogFormat): LogAnalysisResult {
   return {
     format,
@@ -121,7 +145,7 @@ export async function analyzeLogFile(
   const maxIps = opts.maxIpsPerBot ?? DEFAULT_MAX_IPS;
 
   const rl = createInterface({
-    input: createReadStream(filePath, { encoding: 'utf8' }),
+    input: openLogStream(filePath),
     crlfDelay: Infinity,
   });
 
@@ -149,15 +173,16 @@ export async function analyzeLogFile(
 
 async function sniffFormat(filePath: string): Promise<LogFormat> {
   const sample: string[] = [];
-  const rl = createInterface({
-    input: createReadStream(filePath, { encoding: 'utf8' }),
-    crlfDelay: Infinity,
-  });
+  const input = openLogStream(filePath);
+  const rl = createInterface({ input, crlfDelay: Infinity });
   for await (const line of rl) {
     sample.push(line);
     if (sample.length >= 50) break;
   }
   rl.close();
+  // Break-early stops readline but leaves the file/gunzip stream open;
+  // destroy it so we don't leak a descriptor (and a paused gunzip) per sniff.
+  input.destroy();
   return detectLogFormat(sample);
 }
 

packages/core/src/sitemap.ts

@@ -1,8 +1,36 @@
 import * as cheerio from 'cheerio';
+import { gunzipSync } from 'node:zlib';
 import { fetch as undiciFetch } from 'undici';
 import { normalizeUrl } from './url-utils.js';
 import { defaultRequestHeaders, formatFetchError } from './http-client.js';
 
+/**
+ * sitemaps.org protocol limits. A sitemap breaching either is still parsed
+ * (the data is useful and Google reads what it can), but the breach is
+ * reported as a warning so the user can fix the generator.
+ */
+const SITEMAP_MAX_URLS = 50_000;
+const SITEMAP_MAX_BYTES = 50 * 1024 * 1024;
+
+/**
+ * Content types a sitemap may legitimately arrive as. Servers are sloppy
+ * here — `text/plain` and `application/octet-stream` are common for `.xml`
+ * files on misconfigured hosts — so a mismatch is a warning, never a
+ * rejection: the body is still parsed and only flagged.
+ */
+const SITEMAP_OK_CONTENT_TYPES = [
+  'xml', // application/xml, text/xml, application/rss+xml, atom+xml
+  'gzip',
+  'x-gzip',
+  'octet-stream',
+  'text/plain',
+];
+
+/** Magic bytes every gzip member starts with (RFC 1952 §2.3.1). */
+function looksGzipped(buf: Uint8Array): boolean {
+  return buf.length >= 2 && buf[0] === 0x1f && buf[1] === 0x8b;
+}
+
 export interface SitemapEntry {
   /** Absolute URL of the actual page (the sitemap's `<loc>` value). */
   url: string;
@@ -33,6 +61,12 @@ export interface SitemapDiscoveryResult {
   entries: SitemapEntry[];
   /** True if `maxUrls` cap was hit and additional entries were dropped. */
   truncated: boolean;
+  /**
+   * Non-fatal protocol breaches found while reading (sitemaps.org 50,000-URL
+   * / 50 MB caps, unexpected content types). The sitemap was still parsed —
+   * these are reported so the user can fix the generator.
+   */
+  warnings: { sitemap: string; warning: string }[];
 }
 
 /**
@@ -154,6 +188,7 @@ export async function fetchSitemaps(
     errors: [],
     entries: [],
     truncated: false,
+    warnings: [],
   };
   const visited = new Set<string>();
   type QueueItem = { url: string; depth: number };
@@ -188,27 +223,76 @@ export async function fetchSitemaps(
         continue;
       }
       const ct = (res.headers.get('content-type') ?? '').toLowerCase();
-      // Skip gzipped sitemaps for V1 — supporting them needs zlib stream
-      // handling which we'd rather not pull in until users actually need it.
-      if (ct.includes('gzip') || item.url.toLowerCase().endsWith('.gz')) {
-        result.errors.push({
+      // Read as bytes, not text: a `.xml.gz` sitemap arrives as raw gzip
+      // (Content-Type: application/gzip and NO Content-Encoding, so undici
+      // does not decompress it for us). Decoding those bytes as UTF-8 would
+      // yield mojibake that parses to zero URLs.
+      const raw = new Uint8Array(await res.arrayBuffer());
+
+      // Protocol size cap. Measured on the bytes actually served; the spec's
+      // 50 MB is the *uncompressed* limit, so for gzip we re-check after
+      // inflating below.
+      if (raw.byteLength > SITEMAP_MAX_BYTES) {
+        result.warnings.push({
           sitemap: item.url,
-          error: 'Gzipped sitemap — not yet supported',
+          warning: `Sitemap is ${(raw.byteLength / (1024 * 1024)).toFixed(1)} MB, over the sitemaps.org 50 MB limit.`,
         });
-        try {
-          await res.body?.cancel();
-        } catch {
-          /* ignore */
-        }
-        continue;
       }
-      const xml = await res.text();
+
+      // Gzip detection by magic bytes rather than by header or extension:
+      // plenty of servers mislabel `.xml.gz` as `text/xml`, and some serve a
+      // plain XML file from a `.gz` URL. The bytes are the ground truth.
+      let xml: string;
+      if (looksGzipped(raw)) {
+        try {
+          const inflated = gunzipSync(raw);
+          if (inflated.byteLength > SITEMAP_MAX_BYTES) {
+            result.warnings.push({
+              sitemap: item.url,
+              warning: `Sitemap is ${(inflated.byteLength / (1024 * 1024)).toFixed(1)} MB uncompressed, over the sitemaps.org 50 MB limit.`,
+            });
+          }
+          xml = inflated.toString('utf8');
+        } catch (err) {
+          result.errors.push({
+            sitemap: item.url,
+            error: `Corrupt gzip sitemap: ${err instanceof Error ? err.message : String(err)}`,
+          });
+          continue;
+        }
+      } else {
+        xml = Buffer.from(raw).toString('utf8');
+        // Only meaningful for non-gzip bodies — a gzipped sitemap that
+        // inflated cleanly is self-evidently a sitemap whatever the header
+        // claimed.
+        if (ct && !SITEMAP_OK_CONTENT_TYPES.some((ok) => ct.includes(ok))) {
+          result.warnings.push({
+            sitemap: item.url,
+            warning: `Unexpected Content-Type "${ct}" (expected an XML type).`,
+          });
+        }
+      }
+
       const parsed = parseSitemap(xml, item.url);
       if (parsed.type === 'unknown') {
         result.errors.push({ sitemap: item.url, error: 'Unrecognized sitemap format' });
         continue;
       }
       result.sitemapsParsed.push(item.url);
+      // Per-file URL cap (sitemapindex children are counted individually,
+      // which is exactly what the protocol limits).
+      if (parsed.entries.length > SITEMAP_MAX_URLS) {
+        result.warnings.push({
+          sitemap: item.url,
+          warning: `Sitemap declares ${parsed.entries.length.toLocaleString()} URLs, over the sitemaps.org 50,000 limit.`,
+        });
+      }
+      if (parsed.childSitemaps.length > SITEMAP_MAX_URLS) {
+        result.warnings.push({
+          sitemap: item.url,
+          warning: `Sitemap index references ${parsed.childSitemaps.length.toLocaleString()} sitemaps, over the sitemaps.org 50,000 limit.`,
+        });
+      }
       for (const entry of parsed.entries) {
         if (result.entries.length >= opts.maxUrls) {
           result.truncated = true;

packages/core/src/url-utils.ts

@@ -33,6 +33,42 @@ export interface UrlRewriteOptions {
    * up-front instead of on every URL.
    */
   regexRewrites?: readonly CompiledUrlRegexRewrite[];
+  /**
+   * Sort query parameters alphabetically by name. `?b=2&a=1` and
+   * `?a=1&b=2` are the same resource on virtually every stack, but the
+   * WHATWG parser preserves author order, so without this they occupy two
+   * rows and register as a false duplicate pair. Repeated keys keep their
+   * relative order (`sort` is stable), which matters for array-style
+   * params like `?tag=a&tag=b`.
+   *
+   * Opt-in: a small number of servers route on positional parameter
+   * order, and reordering those would fetch the wrong page.
+   */
+  sortQueryParams?: boolean;
+  /**
+   * Collapse runs of `/` in the path to a single slash (`/a//b` -> `/a/b`).
+   * Web servers serve these identically, so the duplicate-slash variant is
+   * a false duplicate - but it is not universally true (a few frameworks
+   * carry empty path segments as real data), so this is opt-in too.
+   */
+  collapseDuplicateSlashes?: boolean;
+}
+
+/**
+ * Uppercase the hex digits of every percent-escape (`%2f` -> `%2F`).
+ *
+ * RFC 3986 6.2.2.1 makes percent-encoding hex digits case-insensitive and
+ * names the uppercase form canonical, but the WHATWG URL parser preserves
+ * whatever case the author wrote. Left alone, `/a%2fb` and `/a%2Fb` are the
+ * same resource stored as two separate rows - a false duplicate that also
+ * splits the link graph and inflates crawl budget.
+ *
+ * Only the two hex digits are touched; the decoded value is unchanged, so
+ * `%2F` still stays distinct from a literal `/` (which is the whole point
+ * of encoding it). A bare `%` not followed by two hex digits is left as-is.
+ */
+function upperPercentEscapes(value: string): string {
+  return value.replace(/%([0-9a-fA-F]{2})/g, (_m, hex: string) => `%${hex.toUpperCase()}`);
 }
 
 const DEFAULT_TRACKING_PARAMS = [
@@ -109,6 +145,11 @@ export function normalizeUrl(
     if (rewrites.lowercasePath && u.pathname) {
       u.pathname = u.pathname.toLowerCase();
     }
+    // Before the trailing-slash policy, so `add`/`strip` sees the collapsed
+    // path and `/a//` doesn't survive as `/a//` after a strip.
+    if (rewrites.collapseDuplicateSlashes && u.pathname.includes('//')) {
+      u.pathname = u.pathname.replace(/\/{2,}/g, '/');
+    }
     if (rewrites.trailingSlash === 'strip' && u.pathname.length > 1 && u.pathname.endsWith('/')) {
       u.pathname = u.pathname.slice(0, -1);
     } else if (
@@ -120,6 +161,18 @@ export function normalizeUrl(
       if (!last.includes('.')) u.pathname += '/';
     }
 
+    // Sort last among the query mutations so it also orders whatever the
+    // keep-list / tracking strip above left behind.
+    if (rewrites.sortQueryParams && u.search !== '') {
+      u.searchParams.sort();
+    }
+
+    // RFC 3986 6.2.2.1 percent-escape case folding - always on. Applied to
+    // the path and query only: the host has no percent-escapes (IDN is
+    // punycoded by the parser) and the fragment is already stripped.
+    if (u.pathname.includes('%')) u.pathname = upperPercentEscapes(u.pathname);
+    if (u.search.includes('%')) u.search = upperPercentEscapes(u.search);
+
     let result = u.toString();
     if (rewrites.regexRewrites && rewrites.regexRewrites.length > 0) {
       for (const { re, replacement } of rewrites.regexRewrites) {
@@ -324,6 +377,122 @@ export function isUrlMalformed(url: string): boolean {
   return false;
 }
 
+/**
+ * Query parameters that carry a per-visit session identifier. Every request
+ * mints a new value, so a crawler following them sees an unbounded stream of
+ * URLs that all serve the same page - the classic infinite-duplicate trap.
+ * Matched case-insensitively on the parameter name.
+ */
+const SESSION_ID_PARAMS: ReadonlySet<string> = new Set([
+  'phpsessid',
+  'jsessionid',
+  'aspsessionid',
+  'sid',
+  'sessionid',
+  'session_id',
+  'cfid',
+  'cftoken',
+  'zenid',
+  'oscsid',
+]);
+
+/** Query parameters that drive date navigation on calendar widgets. */
+const CALENDAR_PARAMS: ReadonlySet<string> = new Set([
+  'year',
+  'month',
+  'day',
+  'date',
+  'cal_date',
+  'calendar',
+  'mini',
+  'from',
+  'to',
+  'start_date',
+  'end_date',
+]);
+
+/** Why a URL was judged a crawl trap. */
+export type UrlTrapKind = 'repeated-segment' | 'query-params' | 'session-id' | 'calendar';
+
+export interface UrlTrapOptions {
+  /**
+   * A single path segment appearing this many times or more marks the URL
+   * as a link loop (`/shop/cat/shop/cat/shop/cat`). 0 disables the check.
+   */
+  maxRepeatedSegments?: number;
+  /**
+   * Query-parameter count above which the URL is treated as faceted-
+   * navigation explosion. 0 disables the check.
+   */
+  maxQueryParams?: number;
+}
+
+/**
+ * Classify a URL against the four crawl-trap shapes that generate unbounded
+ * URL spaces on real sites. Returns the matched kind, or `null` when the URL
+ * looks ordinary.
+ *
+ * Ordered most- to least-certain, because only the first is safe to act on
+ * automatically:
+ *
+ *  1. `repeated-segment` - the same path segment repeated N+ times. Produced
+ *     by a relative-link bug (`href="shop/"` inside `/shop/`), which walks
+ *     `/shop/shop/shop/...` forever. Legitimate URLs essentially never do this.
+ *  2. `query-params`     - more parameters than the configured cap: faceted
+ *     navigation, whose URL count is the product of every filter combination.
+ *  3. `session-id`       - carries a per-visit session token, so every crawl
+ *     of the same page yields a brand-new URL.
+ *  4. `calendar`         - two or more date-navigation parameters, the shape
+ *     of a "next month" widget that recedes infinitely into the future.
+ *
+ * Kinds 2-4 routinely appear on legitimate archive and filter pages, so they
+ * are reported for the user to see rather than acted on by default.
+ */
+export function detectUrlTrap(
+  url: string,
+  opts: UrlTrapOptions = {},
+): UrlTrapKind | null {
+  let u: URL;
+  try {
+    u = new URL(url);
+  } catch {
+    return null;
+  }
+
+  const maxRepeat = opts.maxRepeatedSegments ?? 0;
+  if (maxRepeat > 0) {
+    const segments = u.pathname.split('/').filter((s) => s !== '');
+    if (segments.length >= maxRepeat) {
+      const seen = new Map<string, number>();
+      for (const seg of segments) {
+        // Case-fold: `/Shop/shop/SHOP/` is the same loop.
+        const key = seg.toLowerCase();
+        const n = (seen.get(key) ?? 0) + 1;
+        if (n >= maxRepeat) return 'repeated-segment';
+        seen.set(key, n);
+      }
+    }
+  }
+
+  const names: string[] = [];
+  for (const k of u.searchParams.keys()) names.push(k.toLowerCase());
+
+  if (names.some((n) => SESSION_ID_PARAMS.has(n))) return 'session-id';
+
+  const maxParams = opts.maxQueryParams ?? 0;
+  if (maxParams > 0 && names.length > maxParams) return 'query-params';
+
+  // Two or more calendar parameters together - one alone (`?year=2026`) is
+  // an ordinary archive page, but `?year=2026&month=03` is a date navigator.
+  let calendarHits = 0;
+  for (const n of names) {
+    if (CALENDAR_PARAMS.has(n)) calendarHits++;
+  }
+  if (calendarHits >= 2) return 'calendar';
+
+  return null;
+}
+
 /**
  * Resolve a user-typed start URL to a full URL with protocol, following
  * any initial redirect chain so the crawler's scope calculation (e.g.

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.9.5",
+  "version": "0.9.6",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/migrations.ts

@@ -2110,6 +2110,58 @@ const MIGRATIONS: Migration[] = [
       }
     },
   },
+  {
+    version: 85,
+    name: 'add_canonical_resolved',
+    // `urls.canonical` stores the href exactly as authored, which is what
+    // the "Canonical Not Absolute" filter and the detail panel want. But
+    // roughly twenty predicates compare that raw value against `urls.url`
+    // (self-reference, canonical→redirect/noindex joins, chain walking),
+    // and a relative `href="/x/"` never equals an absolute URL — so a
+    // correctly self-canonicalising page was reported as canonicalised
+    // away, and canonical→target joins silently matched nothing.
+    //
+    // `canonical_resolved` holds the same href resolved against the page
+    // URL and normalised — the form a search engine actually compares.
+    // Comparisons read COALESCE(NULLIF(canonical_resolved,''), canonical)
+    // so projects crawled before this column existed keep their old
+    // behaviour instead of silently reporting zero.
+    //
+    // `canonical_distinct_count` counts *distinct* resolved targets, which
+    // separates a harmlessly repeated tag from canonicals that disagree;
+    // `canonical_cross_domain` flags a canonical pointing off-domain.
+    // Both default 0 — "nothing declared" for rows crawled earlier.
+    up: (db) => {
+      const cols = db
+        .prepare('PRAGMA table_info(urls)')
+        .all() as unknown as { name: string }[];
+      const has = (n: string): boolean => cols.some((c) => c.name === n);
+      if (!has('canonical_resolved')) {
+        db.exec('ALTER TABLE urls ADD COLUMN canonical_resolved TEXT');
+      }
+      if (!has('canonical_distinct_count')) {
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN canonical_distinct_count INTEGER NOT NULL DEFAULT 0',
+        );
+      }
+      if (!has('canonical_cross_domain')) {
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN canonical_cross_domain INTEGER NOT NULL DEFAULT 0',
+        );
+      }
+      // Crawl-trap classification (`detectUrlTrap`): NULL/'' for ordinary
+      // URLs, otherwise the trap kind so the filter can explain *why* a URL
+      // was flagged rather than just that it was.
+      if (!has('url_trap')) {
+        db.exec('ALTER TABLE urls ADD COLUMN url_trap TEXT');
+      }
+      // The canonical→target EXISTS joins match on this column, so without
+      // an index they degrade to a scan per row on large projects.
+      db.exec(
+        'CREATE INDEX IF NOT EXISTS idx_urls_canonical_resolved ON urls(canonical_resolved)',
+      );
+    },
+  },
 ];
 
 export function runMigrations(db: DatabaseSync): void {

packages/db/src/project-db.ts

@@ -124,6 +124,9 @@ interface UrlRowDb {
   canonical_count: number;
   word_count: number | null;
   canonical: string | null;
+  canonical_resolved: string | null;
+  canonical_distinct_count: number;
+  canonical_cross_domain: number;
   canonical_http: string | null;
   meta_robots: string | null;
   x_robots_tag: string | null;
@@ -259,6 +262,7 @@ interface UrlRowDb {
   images_responsive: number;
   picture_count: number;
   url_malformed: number;
+  url_trap: string | null;
   og_type: string | null;
   og_url: string | null;
   og_site_name: string | null;
@@ -331,6 +335,9 @@ export interface UpsertUrlInput {
   canonicalCount?: number;
   wordCount?: number | null;
   canonical?: string | null;
+  canonicalResolved?: string | null;
+  canonicalDistinctCount?: number;
+  canonicalCrossDomain?: number;
   canonicalHttp?: string | null;
   metaRobots?: string | null;
   xRobotsTag?: string | null;
@@ -423,6 +430,7 @@ export interface UpsertUrlInput {
   pictureCount?: number;
   /** 0/1 — set by the crawler via `isUrlMalformed(url)` from `@freecrawl/core/url-utils`. */
   urlMalformed?: number;
+  urlTrap?: string | null;
   ogType?: string | null;
   ogUrl?: string | null;
   ogSiteName?: string | null;
@@ -515,7 +523,8 @@ const UPSERT_URL_SQL = `
     url, content_kind, status_code, status_text, indexability, indexability_reason,
     title, title_length, meta_description, meta_description_length,
     h1, h1_length, h1_count, h2_count, h3_count, h4_count, h5_count, h6_count,
-    word_count, canonical, canonical_count, canonical_http, meta_robots, x_robots_tag,
+    word_count, canonical, canonical_resolved, canonical_distinct_count,
+    canonical_cross_domain, canonical_count, canonical_http, meta_robots, x_robots_tag,
     content_type, content_length, response_time_ms, depth, outlinks, redirect_target,
     images_count, images_missing_alt,
     lang, viewport, og_title, og_description, og_image,
@@ -546,7 +555,7 @@ const UPSERT_URL_SQL = `
     flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, sentence_count, complex_word_count,
     cors_allow_origin, cors_allow_credentials, cors_allow_methods, cors_allow_headers,
     images_responsive, picture_count,
-    url_malformed, og_type, og_url, og_site_name, og_locale, android_icon,
+    url_malformed, url_trap, og_type, og_url, og_site_name, og_locale, android_icon,
     landmark_main, skip_link_present, aria_invalid_roles,
     schema_duplicate_ids, schema_unknown_types, schema_missing_required,
     schema_missing_recommended,
@@ -556,7 +565,8 @@ const UPSERT_URL_SQL = `
     :url, :content_kind, :status_code, :status_text, :indexability, :indexability_reason,
     :title, :title_length, :meta_description, :meta_description_length,
     :h1, :h1_length, :h1_count, :h2_count, :h3_count, :h4_count, :h5_count, :h6_count,
-    :word_count, :canonical, :canonical_count, :canonical_http, :meta_robots, :x_robots_tag,
+    :word_count, :canonical, :canonical_resolved, :canonical_distinct_count,
+    :canonical_cross_domain, :canonical_count, :canonical_http, :meta_robots, :x_robots_tag,
     :content_type, :content_length, :response_time_ms, :depth, :outlinks, :redirect_target,
     :images_count, :images_missing_alt,
     :lang, :viewport, :og_title, :og_description, :og_image,
@@ -587,7 +597,7 @@ const UPSERT_URL_SQL = `
     :flesch_reading_ease, :flesch_kincaid_grade, :gunning_fog_index, :sentence_count, :complex_word_count,
     :cors_allow_origin, :cors_allow_credentials, :cors_allow_methods, :cors_allow_headers,
     :images_responsive, :picture_count,
-    :url_malformed, :og_type, :og_url, :og_site_name, :og_locale, :android_icon,
+    :url_malformed, :url_trap, :og_type, :og_url, :og_site_name, :og_locale, :android_icon,
     :landmark_main, :skip_link_present, :aria_invalid_roles,
     :schema_duplicate_ids, :schema_unknown_types, :schema_missing_required,
     :schema_missing_recommended,
@@ -614,6 +624,9 @@ const UPSERT_URL_SQL = `
     h6_count = excluded.h6_count,
     word_count = excluded.word_count,
     canonical = excluded.canonical,
+    canonical_resolved = excluded.canonical_resolved,
+    canonical_distinct_count = excluded.canonical_distinct_count,
+    canonical_cross_domain = excluded.canonical_cross_domain,
     canonical_count = excluded.canonical_count,
     canonical_http = excluded.canonical_http,
     meta_robots = excluded.meta_robots,
@@ -709,6 +722,7 @@ const UPSERT_URL_SQL = `
     images_responsive = excluded.images_responsive,
     picture_count = excluded.picture_count,
     url_malformed = excluded.url_malformed,
+    url_trap = excluded.url_trap,
     og_type = excluded.og_type,
     og_url = excluded.og_url,
     og_site_name = excluded.og_site_name,
@@ -788,6 +802,26 @@ const INTERNAL_HTML_SCOPE = "is_external = 0 AND content_kind = 'html'";
 const INDEXABLE_HTML_SCOPE =
   INTERNAL_HTML_SCOPE + ' AND status_code >= 200 AND status_code < 300';
 
+/**
+ * The canonical value to *compare* with — resolved to an absolute URL at
+ * crawl time (migration 85), falling back to the raw href for rows crawled
+ * before that column existed.
+ *
+ * `urls.canonical` deliberately keeps the href exactly as the page authored
+ * it (the "Canonical Not Absolute" filter and the detail panel both want the
+ * literal value), but a relative `href="/x/"` never equals an absolute
+ * `urls.url`. Comparing raw made a correctly self-canonicalising page report
+ * as canonicalised away, and made every canonical→target join match nothing.
+ *
+ * Use this anywhere a canonical is matched against `url`, `canonical_http`,
+ * or another row's canonical. Use bare `canonical` only to test whether the
+ * page declared one at all, or to inspect its literal form.
+ */
+const CANONICAL_CMP = "COALESCE(NULLIF(canonical_resolved, ''), canonical)";
+/** Same value, qualified — for use inside a correlated subquery. */
+const CANONICAL_CMP_Q =
+  "COALESCE(NULLIF(urls.canonical_resolved, ''), urls.canonical)";
+
 export class ProjectDb {
   private readonly db: DatabaseSync;
   /** Read-only connections can't write `sqlite_stat1`; see `optimize()`. */
@@ -1753,6 +1787,9 @@ export class ProjectDb {
       h6_count: input.h6Count ?? 0,
       word_count: input.wordCount ?? null,
       canonical: input.canonical ?? null,
+      canonical_resolved: input.canonicalResolved ?? null,
+      canonical_distinct_count: input.canonicalDistinctCount ?? 0,
+      canonical_cross_domain: input.canonicalCrossDomain ?? 0,
       canonical_count: input.canonicalCount ?? 0,
       canonical_http: input.canonicalHttp ?? null,
       meta_robots: input.metaRobots ?? null,
@@ -1851,6 +1888,7 @@ export class ProjectDb {
       images_responsive: input.imagesResponsive ?? 0,
       picture_count: input.pictureCount ?? 0,
       url_malformed: input.urlMalformed ?? 0,
+      url_trap: input.urlTrap ?? null,
       og_type: input.ogType ?? null,
       og_url: input.ogUrl ?? null,
       og_site_name: input.ogSiteName ?? null,
@@ -2001,14 +2039,41 @@ export class ProjectDb {
    * counters this is acceptable — momentary undercount for live
    * counters beats a 1–3 s UI freeze every 30 s. The post-crawl pass
    * still uses the atomic sync version for the final committed state.
+   *
+   * Runs are serialised per instance: because the pass yields between
+   * definitions, two concurrent invocations (the 30 s in-crawl tick
+   * still in flight when the post-crawl pass fires) would interleave —
+   * one's DELETE wiping the other's half-written INSERTs, leaving
+   * silently wrong issue counters. The chain makes the second caller
+   * wait for the first instead.
+   *
+   * `onProgress` fires after each definition — the desktop writer
+   * worker uses it to publish "n/total" into the freeze-watchdog op
+   * slot so a long pass is observably progressing, not wedged.
    */
   async recomputeUrlsIssuesYielding(
     definitions: ReadonlyArray<readonly [string, string]>,
+    onProgress?: (done: number, total: number) => void,
+  ): Promise<void> {
+    const run = this.issueRecomputeChain.then(() =>
+      this.recomputeUrlsIssuesYieldingInner(definitions, onProgress),
+    );
+    // Keep the chain alive on failure — the next caller must still run.
+    this.issueRecomputeChain = run.catch(() => undefined);
+    return run;
+  }
+
+  private issueRecomputeChain: Promise<void> = Promise.resolve();
+
+  private async recomputeUrlsIssuesYieldingInner(
+    definitions: ReadonlyArray<readonly [string, string]>,
+    onProgress?: (done: number, total: number) => void,
   ): Promise<void> {
     // Atomic truncate so getIssueCounts can never see "old + new" rows.
     this.runInTransaction(() => {
       this.db.exec('DELETE FROM urls_issues');
     });
+    let done = 0;
     for (const [issueKey, where] of definitions) {
       // One transaction per definition keeps the writer lock window
       // short — other writes (the crawler's per-URL inserts) can
@@ -2022,6 +2087,8 @@ export class ProjectDb {
           )
           .run(issueKey);
       });
+      done++;
+      onProgress?.(done, definitions.length);
       await new Promise<void>((resolve) => setImmediate(resolve));
     }
   }
@@ -3193,7 +3260,8 @@ export class ProjectDb {
         const placeholders = slice.map(() => '?').join(',');
         const metaRows = this.db
           .prepare(
-            `SELECT url, status_code, indexability, canonical FROM urls
+            `SELECT url, status_code, indexability,
+                    ${CANONICAL_CMP} AS canonical FROM urls
               WHERE url IN (${placeholders})`,
           )
           .all(...slice) as {
@@ -5476,22 +5544,41 @@ export class ProjectDb {
          AND (canonical_http IS NULL OR canonical_http = '')`,
       ),
       canonicalSelfReferencing: countWhere(
-        `${html} AND canonical IS NOT NULL AND canonical = url`,
+        `${html} AND canonical IS NOT NULL AND ${CANONICAL_CMP} = url`,
       ),
       canonicalNonSelf: countWhere(
         `${html} AND canonical IS NOT NULL AND canonical != ''
-         AND canonical != url`,
+         AND ${CANONICAL_CMP} != url`,
       ),
       canonicalMismatch: countWhere(
         `${html}
          AND canonical IS NOT NULL AND canonical != ''
          AND canonical_http IS NOT NULL AND canonical_http != ''
-         AND canonical != canonical_http`,
+         AND ${CANONICAL_CMP} != canonical_http`,
+      ),
+      canonicalConflicting: countWhere(
+        `${html} AND canonical_distinct_count > 1`,
+      ),
+      canonicalCrossDomain: countWhere(
+        `${html} AND canonical_cross_domain = 1`,
+      ),
+      noindexCanonicalConflict: countWhere(
+        `${html} AND canonical IS NOT NULL AND canonical != ''
+         AND ${CANONICAL_CMP} != url
+         AND (meta_robots LIKE '%noindex%' OR x_robots_tag LIKE '%noindex%')`,
+      ),
+      redirectToNoindex: countWhere(
+        `${html} AND status_code >= 300 AND status_code < 400
+         AND redirect_final_url IS NOT NULL AND redirect_final_url != ''
+         AND EXISTS (
+           SELECT 1 FROM urls t WHERE t.url = urls.redirect_final_url
+             AND t.indexability = 'non-indexable:noindex'
+         )`,
       ),
       canonicalToNon200: countWhere(
         `${html} AND canonical IS NOT NULL AND canonical != ''
          AND EXISTS (
-           SELECT 1 FROM urls t WHERE t.url = urls.canonical
+           SELECT 1 FROM urls t WHERE t.url = ${CANONICAL_CMP_Q}
              AND t.status_code IS NOT NULL
              AND (t.status_code < 200 OR t.status_code >= 400)
          )`,
@@ -5499,14 +5586,14 @@ export class ProjectDb {
       canonicalToRedirect: countWhere(
         `${html} AND canonical IS NOT NULL AND canonical != ''
          AND EXISTS (
-           SELECT 1 FROM urls t WHERE t.url = urls.canonical
+           SELECT 1 FROM urls t WHERE t.url = ${CANONICAL_CMP_Q}
              AND t.status_code >= 300 AND t.status_code < 400
          )`,
       ),
       canonicalToNoindex: countWhere(
         `${html} AND canonical IS NOT NULL AND canonical != ''
          AND EXISTS (
-           SELECT 1 FROM urls t WHERE t.url = urls.canonical
+           SELECT 1 FROM urls t WHERE t.url = ${CANONICAL_CMP_Q}
              AND t.indexability = 'non-indexable:noindex'
          )`,
       ),
@@ -5635,7 +5722,8 @@ export class ProjectDb {
            WHERE u2.cluster_id = urls.cluster_id
              AND u2.is_external = 0 AND u2.content_kind = 'html'
              AND u2.canonical IS NOT NULL AND u2.canonical != ''
-             AND u2.canonical != urls.canonical
+             AND COALESCE(NULLIF(u2.canonical_resolved, ''), u2.canonical)
+                 != ${CANONICAL_CMP_Q}
              AND u2.url != urls.url
          )`,
       ),
@@ -5675,6 +5763,7 @@ export class ProjectDb {
         `${html} AND (INSTR(url, ' ') > 0 OR INSTR(url, '%20') > 0)`,
       ),
       urlMalformed: countWhere(`${html} AND url_malformed = 1`),
+      crawlTrap: countWhere(`${html} AND url_trap IS NOT NULL AND url_trap != ''`),
       // Image-level (was page-level `images_empty_alt > 0`): count distinct
       // images with an explicit `alt=""` so it's the same unit as Missing
       // Alt and matches the Images tab drill-down.
@@ -5875,7 +5964,7 @@ export class ProjectDb {
         `${html} AND ((pagination_next IS NOT NULL AND pagination_next != '')
                   OR (pagination_prev IS NOT NULL AND pagination_prev != ''))
          AND canonical IS NOT NULL AND canonical != ''
-         AND canonical != url`,
+         AND ${CANONICAL_CMP} != url`,
       ),
       schemaDuplicateId: countWhere(`${html} AND schema_duplicate_ids > 0`),
       schemaUnknownType: countWhere(`${html} AND schema_unknown_types > 0`),
@@ -8423,7 +8512,10 @@ export class ProjectDb {
     h6Count: r.h6_count,
     wordCount: r.word_count,
     canonical: r.canonical,
+    canonicalResolved: r.canonical_resolved,
     canonicalCount: r.canonical_count,
+    canonicalDistinctCount: r.canonical_distinct_count,
+    canonicalCrossDomain: r.canonical_cross_domain === 1,
     canonicalHttp: r.canonical_http,
     metaRobots: r.meta_robots,
     xRobotsTag: r.x_robots_tag,
@@ -8543,6 +8635,7 @@ export class ProjectDb {
     imagesResponsive: r.images_responsive ?? 0,
     pictureCount: r.picture_count ?? 0,
     urlMalformed: r.url_malformed ?? 0,
+    urlTrap: r.url_trap ?? null,
     ogType: r.og_type ?? null,
     ogUrl: r.og_url ?? null,
     ogSiteName: r.og_site_name ?? null,
@@ -8763,14 +8856,15 @@ export const EXPENSIVE_ISSUE_DEFINITIONS: ReadonlyArray<readonly [string, string
     'issues:canonical-chain-multi-hop',
     `is_external = 0 AND content_kind = 'html'
      AND canonical IS NOT NULL AND canonical != ''
-     AND canonical != url
+     AND ${CANONICAL_CMP} != url
      AND EXISTS (
        SELECT 1 FROM urls c2
-        WHERE c2.url = urls.canonical
+        WHERE c2.url = ${CANONICAL_CMP_Q}
           AND c2.canonical IS NOT NULL
           AND c2.canonical != ''
-          AND c2.canonical != c2.url
-          AND c2.canonical != urls.canonical
+          AND COALESCE(NULLIF(c2.canonical_resolved, ''), c2.canonical) != c2.url
+          AND COALESCE(NULLIF(c2.canonical_resolved, ''), c2.canonical)
+              != ${CANONICAL_CMP_Q}
      )`,
   ],
 ];
@@ -9049,21 +9143,58 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       // Informational filter: this page's canonical points back to
       // itself — the typical "good" state for a primary URL.
       return `is_external = 0 AND content_kind = 'html'
-              AND canonical IS NOT NULL AND canonical = url`;
+              AND canonical IS NOT NULL AND ${CANONICAL_CMP} = url`;
     case 'issues:canonical-non-self':
       // Page canonical points to a different URL — this page is
       // canonicalised to another. Often intentional (paginated /
       // duplicates) but always worth surfacing.
       return `is_external = 0 AND content_kind = 'html'
               AND canonical IS NOT NULL AND canonical != ''
-              AND canonical != url`;
+              AND ${CANONICAL_CMP} != url`;
     case 'issues:canonical-mismatch':
       // HTML and HTTP-header canonicals both exist but disagree —
       // Google picks one unpredictably. Always a misconfiguration.
+      // `canonical_http` is already absolute, so the HTML side has to be
+      // resolved too or every relative canonical reads as a mismatch.
       return `is_external = 0 AND content_kind = 'html'
               AND canonical IS NOT NULL AND canonical != ''
               AND canonical_http IS NOT NULL AND canonical_http != ''
-              AND canonical != canonical_http`;
+              AND ${CANONICAL_CMP} != canonical_http`;
+    case 'issues:canonical-conflicting':
+      // Two or more `<link rel="canonical">` elements resolving to
+      // DIFFERENT targets. `multiple-canonicals` only counts repeated
+      // tags — often a CMS and a plugin emitting the same href, which is
+      // harmless. Disagreeing targets are the real defect: search engines
+      // discard all of them and choose a canonical themselves.
+      return `is_external = 0 AND content_kind = 'html'
+              AND canonical_distinct_count > 1`;
+    case 'issues:canonical-cross-domain':
+      // Canonical points at a different registrable domain. Legitimate
+      // for syndicated content, but far more often a migration or staging
+      // leak that hands the page's ranking signals to another site.
+      return `is_external = 0 AND content_kind = 'html'
+              AND canonical_cross_domain = 1`;
+    case 'issues:noindex-canonical-conflict':
+      // The page says "don't index me" AND "the authoritative version is
+      // over there". Contradictory: the noindex wins, so the canonical's
+      // consolidation never happens and the target gains nothing — while
+      // a crawler following the canonical may propagate the noindex.
+      return `is_external = 0 AND content_kind = 'html'
+              AND canonical IS NOT NULL AND canonical != ''
+              AND ${CANONICAL_CMP} != url
+              AND (meta_robots LIKE '%noindex%' OR x_robots_tag LIKE '%noindex%')`;
+    case 'issues:redirect-to-noindex':
+      // A redirect whose final destination is noindexed — the redirect
+      // spends its link equity on a page that will never be indexed.
+      // Uses `redirect_final_url` (written by the post-crawl chain walk)
+      // so multi-hop chains are judged on their real terminus.
+      return `is_external = 0 AND content_kind = 'html'
+              AND status_code >= 300 AND status_code < 400
+              AND redirect_final_url IS NOT NULL AND redirect_final_url != ''
+              AND EXISTS (
+                SELECT 1 FROM urls t WHERE t.url = urls.redirect_final_url
+                  AND t.indexability = 'non-indexable:noindex'
+              )`;
     case 'issues:canonical-to-non-200':
       // Canonical points to a URL we crawled and it returned 4xx/5xx —
       // major SEO bug, the canonical is broken. 3xx is excluded here
@@ -9072,7 +9203,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       return `is_external = 0 AND content_kind = 'html'
               AND canonical IS NOT NULL AND canonical != ''
               AND EXISTS (
-                SELECT 1 FROM urls t WHERE t.url = urls.canonical
+                SELECT 1 FROM urls t WHERE t.url = ${CANONICAL_CMP_Q}
                   AND t.status_code IS NOT NULL
                   AND (t.status_code < 200 OR t.status_code >= 400)
               )`;
@@ -9083,7 +9214,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       return `is_external = 0 AND content_kind = 'html'
               AND canonical IS NOT NULL AND canonical != ''
               AND EXISTS (
-                SELECT 1 FROM urls t WHERE t.url = urls.canonical
+                SELECT 1 FROM urls t WHERE t.url = ${CANONICAL_CMP_Q}
                   AND t.status_code >= 300 AND t.status_code < 400
               )`;
     case 'issues:canonical-to-noindex':
@@ -9093,7 +9224,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       return `is_external = 0 AND content_kind = 'html'
               AND canonical IS NOT NULL AND canonical != ''
               AND EXISTS (
-                SELECT 1 FROM urls t WHERE t.url = urls.canonical
+                SELECT 1 FROM urls t WHERE t.url = ${CANONICAL_CMP_Q}
                   AND t.indexability = 'non-indexable:noindex'
               )`;
     case 'issues:high-boilerplate':
@@ -9125,7 +9256,8 @@ function categoryWhereClause(cat: UrlCategory): string | null {
                 WHERE u2.cluster_id = urls.cluster_id
                   AND u2.is_external = 0 AND u2.content_kind = 'html'
                   AND u2.canonical IS NOT NULL AND u2.canonical != ''
-                  AND u2.canonical != urls.canonical
+                  AND COALESCE(NULLIF(u2.canonical_resolved, ''), u2.canonical)
+                      != ${CANONICAL_CMP_Q}
                   AND u2.url != urls.url
               )`;
     case 'issues:content-thin':
@@ -9385,6 +9517,16 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       // multiple `?`/`#`, control chars, unescaped reserved chars,
       // double-encoding sequences (`%2520` etc).
       return "is_external = 0 AND content_kind = 'html' AND url_malformed = 1";
+    case 'issues:crawl-trap':
+      // Classified by `detectUrlTrap` at upsert time: faceted navigation
+      // (query-param count over the cap), a session-ID parameter, a
+      // calendar date navigator, or a repeated path segment. These URL
+      // spaces are unbounded, so they burn crawl budget on near-identical
+      // pages. Reported rather than blocked — legitimate archive and
+      // filter pages have the same shape — except the repeated-segment
+      // loop, which the crawler drops at enqueue.
+      return `is_external = 0 AND content_kind = 'html'
+              AND url_trap IS NOT NULL AND url_trap != ''`;
     case 'issues:image-empty-alt':
       // alt="" specifically. Decorative images use this intentionally, but
       // many sites apply it to content images by mistake.
@@ -9736,7 +9878,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
               AND ((pagination_next IS NOT NULL AND pagination_next != '')
                 OR (pagination_prev IS NOT NULL AND pagination_prev != ''))
               AND canonical IS NOT NULL AND canonical != ''
-              AND canonical != url`;
+              AND ${CANONICAL_CMP} != url`;
     case 'issues:schema-duplicate-id':
       // Page has at least one `@id` appearing in two or more JSON-LD
       // entities. Two entities sharing an `@id` collide in Google's
@@ -9977,14 +10119,16 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       // would also surface as a multi-hop chain on its own row.
       return `is_external = 0 AND content_kind = 'html'
               AND canonical IS NOT NULL AND canonical != ''
-              AND canonical != url
+              AND ${CANONICAL_CMP} != url
               AND EXISTS (
                 SELECT 1 FROM urls c2
-                 WHERE c2.url = urls.canonical
+                 WHERE c2.url = ${CANONICAL_CMP_Q}
                    AND c2.canonical IS NOT NULL
                    AND c2.canonical != ''
-                   AND c2.canonical != c2.url
-                   AND c2.canonical != urls.canonical
+                   AND COALESCE(NULLIF(c2.canonical_resolved, ''), c2.canonical)
+                       != c2.url
+                   AND COALESCE(NULLIF(c2.canonical_resolved, ''), c2.canonical)
+                       != ${CANONICAL_CMP_Q}
               )`;
     case 'issues:image-slow-loading':
       // Page loads at least one image > 200 KB AND the page hasn't

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.9.5",
+  "version": "0.9.6",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/crawl.ts

@@ -72,6 +72,11 @@ export type UrlCategory =
   | 'issues:canonical-to-non-200'
   | 'issues:canonical-to-redirect'
   | 'issues:canonical-to-noindex'
+  | 'issues:crawl-trap'
+  | 'issues:canonical-conflicting'
+  | 'issues:canonical-cross-domain'
+  | 'issues:noindex-canonical-conflict'
+  | 'issues:redirect-to-noindex'
   | 'issues:content-thin'
   | 'issues:spelling-grammar'
   | 'issues:response-slow'
@@ -268,7 +273,16 @@ export interface CrawlUrlRow {
   h6Count: number;
   wordCount: number | null;
   canonical: string | null;
+  /**
+   * `canonical` resolved against the page URL and normalized — the form
+   * search engines compare. `canonical` keeps the literal authored href.
+   */
+  canonicalResolved: string | null;
   canonicalCount: number;
+  /** Distinct resolved canonical targets; >1 means the page's canonicals disagree. */
+  canonicalDistinctCount: number;
+  /** True when the canonical points at a different registrable domain. */
+  canonicalCrossDomain: boolean;
   /** `<URL>; rel="canonical"` parsed out of the `Link:` HTTP response header. */
   canonicalHttp: string | null;
   metaRobots: string | null;
@@ -305,6 +319,11 @@ export interface CrawlUrlRow {
   pictureCount: number;
   /** 0/1 — set when the canonical URL string is structurally suspect (multiple `?`/`#`, control chars, unescaped reserved chars, double-encoding). */
   urlMalformed: number;
+  /**
+   * Crawl-trap classification, or null for an ordinary URL:
+   * `repeated-segment` | `query-params` | `session-id` | `calendar`.
+   */
+  urlTrap: string | null;
   /** `og:type` (e.g. `website`, `article`, `product`), lowercased. */
   ogType: string | null;
   /** Raw `og:url` content, verbatim. */
@@ -772,6 +791,25 @@ export interface CrawlConfig {
   includePatterns: string[];
   /** URLs matching any of these regexes are skipped during enqueue. */
   excludePatterns: string[];
+  /**
+   * Crawl-trap guard. A single path segment repeated this many times or
+   * more (`/shop/shop/shop/…`) is a link loop — almost always a relative-
+   * href bug — and those URLs are dropped at enqueue so the crawl cannot
+   * run away. Every drop is counted and reported at the end of the crawl,
+   * never silently. 0 disables the guard.
+   *
+   * Only this shape is acted on automatically: it has no legitimate
+   * counterpart. The other trap kinds (faceted navigation, session IDs,
+   * calendar widgets) are detected and flagged as issues but still
+   * crawled, because real archive and filter pages look identical.
+   */
+  maxRepeatedPathSegments: number;
+  /**
+   * Flag URLs carrying more than this many query parameters as faceted-
+   * navigation traps. Detection only — the URLs are still crawled. 0
+   * disables the check.
+   */
+  maxQueryParams: number;
   /**
    * On crawl start, discover sitemap.xml URLs from robots.txt + default
    * paths and persist their entries into `sitemap_urls`. Used for the
@@ -812,6 +850,19 @@ export interface CrawlConfig {
    * the legacy behaviour (strip only utm_* / fbclid / gclid / mc_*).
    */
   keepQueryParams: string[];
+  /**
+   * Sort query parameters alphabetically so `?b=2&a=1` and `?a=1&b=2`
+   * collapse to one URL instead of two rows that read as duplicates.
+   * Repeated keys keep their relative order. Off by default because a
+   * few servers route on positional parameter order.
+   */
+  sortQueryParams: boolean;
+  /**
+   * Collapse repeated slashes in the path (`/a//b` → `/a/b`). Servers
+   * serve these identically, so the variant is a false duplicate. Off by
+   * default — a few frameworks treat empty path segments as real data.
+   */
+  collapseDuplicateSlashes: boolean;
   /**
    * Regex rewrites applied to the fully-normalized URL string in order.
    * Each entry is `{pattern, replacement, flags?}` where `flags` defaults
@@ -1580,6 +1631,20 @@ export interface OverviewCounts {
     canonicalToNon200: number;
     canonicalToRedirect: number;
     canonicalToNoindex: number;
+    /**
+     * URLs matching a crawl-trap shape (faceted navigation, session ID,
+     * calendar navigation, repeated path segment). Detected and crawled —
+     * only the repeated-segment loop is dropped at enqueue.
+     */
+    crawlTrap: number;
+    /** Pages whose multiple canonicals resolve to *different* targets. */
+    canonicalConflicting: number;
+    /** Pages whose canonical points at another registrable domain. */
+    canonicalCrossDomain: number;
+    /** Pages declaring both noindex and a canonical to a different URL. */
+    noindexCanonicalConflict: number;
+    /** Redirects whose final destination is noindexed. */
+    redirectToNoindex: number;
     contentThin: number;
     /** Pages whose LanguageTool check returned ≥1 spelling/grammar match. */
     spellingGrammar: number;
@@ -2149,6 +2214,8 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   customHeaders: {},
   includePatterns: [],
   excludePatterns: [],
+  maxRepeatedPathSegments: 3,
+  maxQueryParams: 0,
   discoverSitemaps: true,
   customSearchTerms: [],
   stripWww: false,
@@ -2156,6 +2223,8 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   lowercasePath: false,
   trailingSlash: 'leave',
   keepQueryParams: [],
+  sortQueryParams: false,
+  collapseDuplicateSlashes: false,
   urlRegexRewrites: [],
   memoryLimitMb: 0,
   maxQueueSize: 0,

packages/shared-types/src/ipc.ts

@@ -1305,6 +1305,8 @@ export interface UrlRewritePreviewInput {
   stripWww: boolean;
   forceHttps: boolean;
   lowercasePath: boolean;
+  sortQueryParams: boolean;
+  collapseDuplicateSlashes: boolean;
   trailingSlash: 'leave' | 'strip' | 'add';
   keepQueryParams: string[];
   urlRegexRewrites: Array<{ pattern: string; replacement: string; flags?: string }>;

packages/shared-types/src/issue-severity.ts

@@ -33,6 +33,11 @@ const CRITICAL: UrlCategory[] = [
   'issues:canonical-mismatch',
   'issues:canonical-to-non-200',
   'issues:canonical-to-noindex',
+  // Canonicals that disagree with each other are discarded wholesale by
+  // search engines, and a noindex sitting on a canonicalised page cancels
+  // the consolidation outright — both break indexing, not just degrade it.
+  'issues:canonical-conflicting',
+  'issues:noindex-canonical-conflict',
   'issues:response-very-slow',
   'issues:mixed-content',
   'issues:redirect-loop',
@@ -117,6 +122,13 @@ const WARNING: UrlCategory[] = [
   'issues:og-image-not-absolute',
   'issues:twitter-image-not-absolute',
   'issues:canonical-not-absolute',
+  // Both are legitimate in some setups (syndication; a redirect that
+  // deliberately lands on a noindexed page), so they warn rather than fail.
+  'issues:canonical-cross-domain',
+  'issues:redirect-to-noindex',
+  // Faceted / calendar / session-ID URL spaces waste crawl budget on
+  // near-identical pages, but plenty of them are legitimate archive pages.
+  'issues:crawl-trap',
   'issues:title-single-word',
   'issues:internal-link-to-redirect',
   'issues:dead-external-domain',

tests/canonical-resolution.test.ts

@@ -0,0 +1,339 @@
+import { describe, it, expect } from 'vitest';
+import { ProjectDb } from '../packages/db/src/project-db.js';
+import { normalizeUrl, detectUrlTrap } from '../packages/core/src/url-utils.js';
+import { parseHtml } from '../packages/core/src/html-parser.js';
+
+// FEATURES.md gaps 1–5: canonical comparisons were done against the raw
+// authored href, so a relative canonical never matched any absolute URL.
+// These lock in the resolved-canonical behaviour end-to-end: parse → store
+// → the SQL predicates the sidebar and drill-down both read.
+
+function makeDb(): ProjectDb {
+  return new ProjectDb(':memory:');
+}
+
+/** Upsert a crawled HTML page the way the crawler does. */
+function page(
+  db: ProjectDb,
+  url: string,
+  extra: Record<string, unknown> = {},
+): void {
+  db.upsertUrl({
+    url,
+    depth: 0,
+    statusCode: 200,
+    contentKind: 'html',
+    indexability: 'indexable',
+    ...extra,
+  } as Parameters<ProjectDb['upsertUrl']>[0]);
+}
+
+describe('gap #3 — relative canonical resolves before comparison', () => {
+  it('parses a relative canonical into an absolute resolved form', () => {
+    const parsed = parseHtml(
+      '<html><head><link rel="canonical" href="/page/"></head><body>hi</body></html>',
+      'https://ex.com/page/',
+    );
+    expect(parsed.canonical).toBe('/page/'); // raw kept verbatim
+    expect(parsed.canonicalResolved).toBe('https://ex.com/page/');
+  });
+
+  it('counts a relative self-canonical as self-referencing, not canonicalised away', () => {
+    const db = makeDb();
+    page(db, 'https://ex.com/page/', {
+      canonical: '/page/',
+      canonicalResolved: 'https://ex.com/page/',
+      canonicalCount: 1,
+      canonicalDistinctCount: 1,
+    });
+    const c = db.getOverviewCounts().issues;
+    expect(c.canonicalSelfReferencing).toBe(1);
+    // The bug: this used to be 1 because '/page/' !== the absolute url.
+    expect(c.canonicalNonSelf).toBe(0);
+  });
+
+  it('still reports a genuinely non-self canonical', () => {
+    const db = makeDb();
+    page(db, 'https://ex.com/a', {
+      canonical: '/b',
+      canonicalResolved: 'https://ex.com/b',
+      canonicalCount: 1,
+      canonicalDistinctCount: 1,
+    });
+    const c = db.getOverviewCounts().issues;
+    expect(c.canonicalNonSelf).toBe(1);
+    expect(c.canonicalSelfReferencing).toBe(0);
+  });
+
+  it('joins canonical → target on the resolved value', () => {
+    const db = makeDb();
+    // /a canonicalises to /gone (relative href), which 404s.
+    page(db, 'https://ex.com/a', {
+      canonical: '/gone',
+      canonicalResolved: 'https://ex.com/gone',
+      canonicalCount: 1,
+      canonicalDistinctCount: 1,
+    });
+    page(db, 'https://ex.com/gone', { statusCode: 404 });
+    // Previously 0 — 't.url = urls.canonical' never matched '/gone'.
+    expect(db.getOverviewCounts().issues.canonicalToNon200).toBe(1);
+  });
+
+  it('does not flag a relative canonical as an HTTP/HTML mismatch', () => {
+    const db = makeDb();
+    page(db, 'https://ex.com/a', {
+      canonical: '/a',
+      canonicalResolved: 'https://ex.com/a',
+      canonicalHttp: 'https://ex.com/a',
+      canonicalCount: 1,
+      canonicalDistinctCount: 1,
+    });
+    expect(db.getOverviewCounts().issues.canonicalMismatch).toBe(0);
+  });
+
+  it('keeps working on legacy rows that predate canonical_resolved', () => {
+    const db = makeDb();
+    // No canonicalResolved written — the COALESCE must fall back to raw.
+    page(db, 'https://ex.com/a', {
+      canonical: 'https://ex.com/a',
+      canonicalCount: 1,
+    });
+    expect(db.getOverviewCounts().issues.canonicalSelfReferencing).toBe(1);
+  });
+});
+
+describe('gap #1 — unusual canonical combinations', () => {
+  it('separates repeated-but-identical canonicals from disagreeing ones', () => {
+    const html = (hrefs: string[]): string =>
+      `<html><head>${hrefs
+        .map((h) => `<link rel="canonical" href="${h}">`)
+        .join('')}</head><body>x</body></html>`;
+
+    // Same target declared twice — noisy, but not a conflict.
+    const same = parseHtml(html(['/a', 'https://ex.com/a']), 'https://ex.com/a');
+    expect(same.canonicalCount).toBe(2);
+    expect(same.canonicalDistinctCount).toBe(1);
+
+    // Two different targets — search engines discard both.
+    const diff = parseHtml(html(['/a', '/b']), 'https://ex.com/a');
+    expect(diff.canonicalCount).toBe(2);
+    expect(diff.canonicalDistinctCount).toBe(2);
+  });
+
+  it('counts conflicting canonicals as an issue', () => {
+    const db = makeDb();
+    page(db, 'https://ex.com/a', {
+      canonical: '/a',
+      canonicalResolved: 'https://ex.com/a',
+      canonicalCount: 2,
+      canonicalDistinctCount: 2,
+    });
+    page(db, 'https://ex.com/b', {
+      canonical: '/b',
+      canonicalResolved: 'https://ex.com/b',
+      canonicalCount: 2,
+      canonicalDistinctCount: 1, // repeated but identical
+    });
+    const c = db.getOverviewCounts().issues;
+    expect(c.canonicalConflicting).toBe(1);
+    expect(c.multipleCanonicals).toBe(2); // both still count as "multiple"
+  });
+
+  it('detects a cross-domain canonical at parse time and counts it', () => {
+    const off = parseHtml(
+      '<html><head><link rel="canonical" href="https://other.com/x"></head></html>',
+      'https://ex.com/a',
+    );
+    expect(off.canonicalCrossDomain).toBe(true);
+
+    // A subdomain of the same registrable domain is NOT cross-domain.
+    const sub = parseHtml(
+      '<html><head><link rel="canonical" href="https://www.ex.com/a"></head></html>',
+      'https://ex.com/a',
+    );
+    expect(sub.canonicalCrossDomain).toBe(false);
+
+    const db = makeDb();
+    page(db, 'https://ex.com/a', {
+      canonical: 'https://other.com/x',
+      canonicalResolved: 'https://other.com/x',
+      canonicalCount: 1,
+      canonicalDistinctCount: 1,
+      canonicalCrossDomain: 1,
+    });
+    expect(db.getOverviewCounts().issues.canonicalCrossDomain).toBe(1);
+  });
+});
+
+describe('gap #2 — redirect / canonical / noindex relationships', () => {
+  it('flags noindex sitting on a page canonicalised elsewhere', () => {
+    const db = makeDb();
+    page(db, 'https://ex.com/a', {
+      canonical: '/b',
+      canonicalResolved: 'https://ex.com/b',
+      canonicalCount: 1,
+      canonicalDistinctCount: 1,
+      metaRobots: 'noindex, follow',
+      indexability: 'non-indexable:noindex',
+    });
+    // A self-canonical + noindex is not a conflict — nothing is being
+    // consolidated, so there is no contradiction to report.
+    page(db, 'https://ex.com/c', {
+      canonical: '/c',
+      canonicalResolved: 'https://ex.com/c',
+      canonicalCount: 1,
+      canonicalDistinctCount: 1,
+      metaRobots: 'noindex',
+      indexability: 'non-indexable:noindex',
+    });
+    expect(db.getOverviewCounts().issues.noindexCanonicalConflict).toBe(1);
+  });
+
+  it('flags a redirect whose final destination is noindexed', () => {
+    const db = makeDb();
+    db.upsertUrl({
+      url: 'https://ex.com/old',
+      depth: 0,
+      statusCode: 301,
+      contentKind: 'html',
+      indexability: 'non-indexable:redirect',
+      redirectTarget: 'https://ex.com/new',
+    } as Parameters<ProjectDb['upsertUrl']>[0]);
+    page(db, 'https://ex.com/new', {
+      metaRobots: 'noindex',
+      indexability: 'non-indexable:noindex',
+    });
+    // `redirect_final_url` is written by the post-crawl chain walk, not by
+    // upsert — the filter reads the terminus, so multi-hop chains resolve.
+    db.recomputeRedirectChains();
+    expect(db.getOverviewCounts().issues.redirectToNoindex).toBe(1);
+  });
+
+  it('does not flag a redirect landing on an indexable page', () => {
+    const db = makeDb();
+    db.upsertUrl({
+      url: 'https://ex.com/old',
+      depth: 0,
+      statusCode: 301,
+      contentKind: 'html',
+      indexability: 'non-indexable:redirect',
+      redirectTarget: 'https://ex.com/new',
+    } as Parameters<ProjectDb['upsertUrl']>[0]);
+    page(db, 'https://ex.com/new');
+    db.recomputeRedirectChains();
+    expect(db.getOverviewCounts().issues.redirectToNoindex).toBe(0);
+  });
+});
+
+describe('gaps #4 / #5 — URL normalisation completeness', () => {
+  it('folds percent-escape case (RFC 3986) by default', () => {
+    expect(normalizeUrl('https://x.com/a%2fb')).toBe(
+      normalizeUrl('https://x.com/a%2Fb'),
+    );
+    expect(normalizeUrl('https://x.com/p?q=%2fc')).toBe(
+      normalizeUrl('https://x.com/p?q=%2Fc'),
+    );
+  });
+
+  it('keeps an encoded slash distinct from a literal one', () => {
+    expect(normalizeUrl('https://x.com/a%2Fb')).not.toBe(
+      normalizeUrl('https://x.com/a/b'),
+    );
+  });
+
+  it('leaves a bare percent sign alone', () => {
+    expect(normalizeUrl('https://x.com/100%pure')).toBe('https://x.com/100%pure');
+  });
+
+  it('sorts query params only when asked', () => {
+    const a = 'https://x.com/p?b=2&a=1';
+    const b = 'https://x.com/p?a=1&b=2';
+    expect(normalizeUrl(a)).not.toBe(normalizeUrl(b));
+    expect(normalizeUrl(a, undefined, { sortQueryParams: true })).toBe(
+      normalizeUrl(b, undefined, { sortQueryParams: true }),
+    );
+  });
+
+  it('preserves repeated-key order when sorting', () => {
+    expect(
+      normalizeUrl('https://x.com/p?tag=z&a=1&tag=b', undefined, {
+        sortQueryParams: true,
+      }),
+    ).toBe('https://x.com/p?a=1&tag=z&tag=b');
+  });
+
+  it('collapses duplicate slashes only when asked', () => {
+    const dup = 'https://x.com//a//b';
+    expect(normalizeUrl(dup)).not.toBe(normalizeUrl('https://x.com/a/b'));
+    expect(normalizeUrl(dup, undefined, { collapseDuplicateSlashes: true })).toBe(
+      'https://x.com/a/b',
+    );
+  });
+
+  it('collapses before applying the trailing-slash policy', () => {
+    expect(
+      normalizeUrl('https://x.com/a//b//', undefined, {
+        collapseDuplicateSlashes: true,
+        trailingSlash: 'strip',
+      }),
+    ).toBe('https://x.com/a/b');
+  });
+});
+
+describe('gap #8 — crawl-trap heuristics', () => {
+  const REPEAT = { maxRepeatedSegments: 3 };
+
+  it('detects a repeated-segment link loop', () => {
+    expect(detectUrlTrap('https://x.com/shop/shop/shop/', REPEAT)).toBe(
+      'repeated-segment',
+    );
+    expect(detectUrlTrap('https://x.com/a/b/a/b/a/b', REPEAT)).toBe(
+      'repeated-segment',
+    );
+    // Case-folded: /Shop/shop/SHOP is the same loop.
+    expect(detectUrlTrap('https://x.com/Shop/shop/SHOP', REPEAT)).toBe(
+      'repeated-segment',
+    );
+  });
+
+  it('leaves ordinary deep paths alone', () => {
+    expect(
+      detectUrlTrap('https://x.com/blog/2026/03/how-to-crawl', REPEAT),
+    ).toBeNull();
+    // Twice is not a loop — only the configured threshold is.
+    expect(detectUrlTrap('https://x.com/shop/cat/shop', REPEAT)).toBeNull();
+  });
+
+  it('honours the threshold and can be disabled', () => {
+    const url = 'https://x.com/a/a/a';
+    expect(detectUrlTrap(url, { maxRepeatedSegments: 4 })).toBeNull();
+    expect(detectUrlTrap(url, { maxRepeatedSegments: 0 })).toBeNull();
+  });
+
+  it('detects session-id parameters', () => {
+    expect(detectUrlTrap('https://x.com/p?PHPSESSID=abc123')).toBe('session-id');
+    expect(detectUrlTrap('https://x.com/p?jsessionid=zz')).toBe('session-id');
+  });
+
+  it('detects a calendar navigator but not a plain archive page', () => {
+    expect(detectUrlTrap('https://x.com/events?year=2026&month=03')).toBe(
+      'calendar',
+    );
+    expect(detectUrlTrap('https://x.com/events?year=2026')).toBeNull();
+  });
+
+  it('detects faceted navigation only when a cap is set', () => {
+    const faceted = 'https://x.com/p?a=1&b=2&c=3&d=4&e=5';
+    expect(detectUrlTrap(faceted)).toBeNull();
+    expect(detectUrlTrap(faceted, { maxQueryParams: 4 })).toBe('query-params');
+    expect(detectUrlTrap(faceted, { maxQueryParams: 5 })).toBeNull();
+  });
+
+  it('surfaces flagged traps through the issue filter', () => {
+    const db = makeDb();
+    page(db, 'https://x.com/p?PHPSESSID=abc', { urlTrap: 'session-id' });
+    page(db, 'https://x.com/events?year=2026&month=3', { urlTrap: 'calendar' });
+    page(db, 'https://x.com/normal');
+    expect(db.getOverviewCounts().issues.crawlTrap).toBe(2);
+  });
+});