feat: status-code diagnosis, top words report, request headers, settings UX

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": true,
   "type": "module",
   "bin": {

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "main": "./out/main/index.js",

apps/desktop/src/main/index.ts

@@ -56,6 +56,8 @@ import {
   type ImageWeightRow,
   type BucketHistogramRow,
   type ServerHeaderRow,
+  type TopWordsInput,
+  type TopWordsRow,
   type WordCountPerDirectoryInput,
   type WordCountPerDirectoryRow,
   type SitemapOrphanRow,
@@ -94,6 +96,7 @@ import {
   compareCrawls,
   testUrlAgainstRobots,
   validateRobotsTxt,
+  aggregateTopWords,
   fetchSitemaps,
   validateSitemap,
   normalizeUrl,
@@ -1548,6 +1551,23 @@ function registerIpc(): void {
     ),
   );
 
+  // Top Words — pull the title+meta+H1 corpus from the active DB then
+  // aggregate via core's `aggregateTopWords` (kept in core because the
+  // db package isn't allowed to depend on core per the monorepo
+  // dependency graph). Aggregation is sync + cheap so we don't need
+  // the reader pool's worker offload here.
+  ipcMain.handle(
+    IPC.reportsTopWords,
+    (_e, input: TopWordsInput): TopWordsRow[] => {
+      const corpus = getDb().seoTextCorpus();
+      return aggregateTopWords(corpus, {
+        limit: input.limit,
+        minLength: input.minLength,
+        locale: input.locale,
+      });
+    },
+  );
+
   ipcMain.handle(
     IPC.prefsExportSettings,
     async (_e, input: SettingsExportInput): Promise<SettingsExportResult> => {

apps/desktop/src/preload/index.ts

@@ -52,6 +52,8 @@ import {
   type ImageWeightRow,
   type BucketHistogramRow,
   type ServerHeaderRow,
+  type TopWordsInput,
+  type TopWordsRow,
   type WordCountPerDirectoryInput,
   type WordCountPerDirectoryRow,
   type SitemapOrphanRow,
@@ -220,6 +222,8 @@ const api: FreeCrawlApi = {
   },
   reportsServerHeaders: (): Promise<ServerHeaderRow[]> =>
     ipcRenderer.invoke(IPC.reportsServerHeaders),
+  reportsTopWords: (input: TopWordsInput): Promise<TopWordsRow[]> =>
+    ipcRenderer.invoke(IPC.reportsTopWords, input),
   prefsExportSettings: (input: SettingsExportInput): Promise<SettingsExportResult> =>
     ipcRenderer.invoke(IPC.prefsExportSettings, input),
   prefsImportSettings: (): Promise<SettingsImportResult> =>

apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx

@@ -1,6 +1,7 @@
 import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react';
 import clsx from 'clsx';
 import type {
+  CrawlConfig,
   CrawlUrlRow,
   LinkOrigin,
   LinkPathType,
@@ -13,6 +14,7 @@ import type {
   UrlSourceResult,
 } from '@freecrawl/shared-types';
 import { useAppStore } from '../store.js';
+import { diagnoseStatus, type StatusDiagnosis } from '../utils/statusDiagnosis.js';
 
 type SubTab =
   | 'url-details'
@@ -383,7 +385,9 @@ export function BottomDetailPanel() {
           <ExtractedDataView row={detail.row} />
         )}
         {detail && subTab === 'serp-snippet' && <SerpSnippet row={detail.row} />}
-        {detail && subTab === 'http-headers' && <HttpHeadersView headers={detail.headers} />}
+        {detail && subTab === 'http-headers' && (
+          <HttpHeadersView headers={detail.headers} row={detail.row} />
+        )}
         {detail && subTab === 'cookies' && (
           <CookiesView row={detail.row} headers={detail.headers} />
         )}
@@ -796,38 +800,232 @@ function renderHighlighted(
   return out;
 }
 
-function HttpHeadersView({ headers }: { headers: { name: string; value: string }[] }) {
-  if (headers.length === 0) {
-    return (
-      <div className="p-4 text-[11px] text-surface-500">
-        No response headers captured for this URL.
-      </div>
-    );
-  }
+function HttpHeadersView({
+  headers,
+  row,
+}: {
+  headers: { name: string; value: string }[];
+  row: CrawlUrlRow;
+}) {
+  const config = useAppStore((s) => s.config);
+  const [side, setSide] = useState<'response' | 'request'>('response');
+  // Request headers reconstructed from the active crawl config — the
+  // same defaults the crawler dispatches (`defaultRequestHeaders` in
+  // `@freecrawl/core/http-client`) using a renderer-safe basic-auth
+  // builder. This is reconstructed-not-stored because per-URL request
+  // headers are deterministic from (config, URL) and storing them
+  // would multiply the row size by ~5x for zero diagnostic gain.
+  const requestHeaders = useMemo(
+    () => buildPreviewRequestHeaders(config, row.url),
+    [config, row.url],
+  );
+  // Decode the status code (esp. Cloudflare 52x / 530, 4xx bot blocks,
+  // 5xx-under-load) into a plain-English explanation enriched with
+  // signals read off the response headers (Retry-After, cf-ray, …).
+  const diag = useMemo(
+    () => diagnoseStatus(row.statusCode, row.statusText, headers),
+    [row.statusCode, row.statusText, headers],
+  );
+  const rows = side === 'response' ? headers : requestHeaders;
+  // Only surface the diagnosis banner when there's something worth
+  // saying — i.e. not on a clean 2xx.
+  const showDiag = diag.severity !== 'ok';
+
   return (
-    <div className="p-3">
-      <table className="w-full text-[11px]">
-        <thead className="sticky top-0 bg-surface-900">
-          <tr className="text-surface-400">
-            <th className="w-64 py-1 pr-3 text-left font-medium">Header</th>
-            <th className="py-1 text-left font-medium">Value</th>
-          </tr>
-        </thead>
-        <tbody>
-          {headers.map((h) => (
-            <tr key={h.name} className="border-b border-surface-900 last:border-0">
-              <td className="py-1.5 pr-3 align-top font-mono text-surface-400">{h.name}</td>
-              <td className="break-all py-1.5 align-top font-mono text-surface-100">
-                {h.value}
-              </td>
-            </tr>
-          ))}
-        </tbody>
-      </table>
+    <div className="flex h-full flex-col">
+      <div className="sticky top-0 flex shrink-0 items-center gap-1 border-b border-surface-800 bg-surface-900 px-3 py-1.5 text-[11px]">
+        <button
+          className={clsx(
+            'rounded px-2 py-0.5',
+            side === 'response'
+              ? 'bg-surface-800 text-surface-100'
+              : 'text-surface-400 hover:bg-surface-800/60 hover:text-surface-200',
+          )}
+          onClick={() => setSide('response')}
+        >
+          Response ({headers.length})
+        </button>
+        <button
+          className={clsx(
+            'rounded px-2 py-0.5',
+            side === 'request'
+              ? 'bg-surface-800 text-surface-100'
+              : 'text-surface-400 hover:bg-surface-800/60 hover:text-surface-200',
+          )}
+          onClick={() => setSide('request')}
+        >
+          Request ({requestHeaders.length})
+        </button>
+        <span className="ml-auto text-surface-500">
+          {side === 'request'
+            ? 'Reconstructed from active crawl config — actual values may have differed if config changed since crawl.'
+            : 'Captured at crawl time.'}
+        </span>
+      </div>
+      <div className="flex-1 overflow-auto">
+        {showDiag && <StatusDiagnosisBanner diag={diag} />}
+        {rows.length === 0 ? (
+          <div className="p-4 text-[11px] text-surface-500">
+            {side === 'response'
+              ? 'No response headers captured for this URL.'
+              : 'No request headers — config may be empty.'}
+          </div>
+        ) : (
+          <div className="p-3">
+            <table className="w-full text-[11px]">
+              <thead className="sticky top-0 bg-surface-900">
+                <tr className="text-surface-400">
+                  <th className="w-64 py-1 pr-3 text-left font-medium">Header</th>
+                  <th className="py-1 text-left font-medium">Value</th>
+                </tr>
+              </thead>
+              <tbody>
+                {rows.map((h) => (
+                  <tr key={h.name} className="border-b border-surface-900 last:border-0">
+                    <td className="py-1.5 pr-3 align-top font-mono text-surface-400">{h.name}</td>
+                    <td className="break-all py-1.5 align-top font-mono text-surface-100">
+                      {h.value}
+                    </td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </div>
+        )}
+      </div>
     </div>
   );
 }
 
+function StatusDiagnosisBanner({ diag }: { diag: StatusDiagnosis }) {
+  const tone =
+    diag.severity === 'error'
+      ? { border: 'border-red-700/50', bg: 'bg-red-900/15', text: 'text-red-200', head: 'text-red-300' }
+      : diag.severity === 'warning'
+        ? { border: 'border-amber-700/50', bg: 'bg-amber-900/15', text: 'text-amber-200', head: 'text-amber-300' }
+        : { border: 'border-blue-700/40', bg: 'bg-blue-900/15', text: 'text-blue-200', head: 'text-blue-300' };
+  return (
+    <div className={clsx('m-3 mb-2 rounded border px-3 py-2.5 text-[11px]', tone.border, tone.bg, tone.text)}>
+      <div className={clsx('mb-1 text-[12px] font-semibold', tone.head)}>{diag.title}</div>
+      <p className="leading-relaxed text-surface-200">{diag.explanation}</p>
+      {diag.signals.length > 0 && (
+        <div className="mt-2">
+          <div className="mb-0.5 text-[10px] font-medium uppercase tracking-wider text-surface-400">
+            From the response headers
+          </div>
+          <ul className="list-disc space-y-0.5 pl-4 text-surface-200">
+            {diag.signals.map((s, i) => (
+              <li key={i}>{s}</li>
+            ))}
+          </ul>
+        </div>
+      )}
+      {diag.causes.length > 0 && (
+        <div className="mt-2">
+          <div className="mb-0.5 text-[10px] font-medium uppercase tracking-wider text-surface-400">
+            Likely causes
+          </div>
+          <ul className="list-disc space-y-0.5 pl-4 text-surface-300">
+            {diag.causes.map((c, i) => (
+              <li key={i}>{c}</li>
+            ))}
+          </ul>
+        </div>
+      )}
+      {diag.whatToDo.length > 0 && (
+        <div className="mt-2">
+          <div className="mb-0.5 text-[10px] font-medium uppercase tracking-wider text-surface-400">
+            What to do
+          </div>
+          <ul className="list-disc space-y-0.5 pl-4 text-surface-200">
+            {diag.whatToDo.map((w, i) => (
+              <li key={i}>{w}</li>
+            ))}
+          </ul>
+        </div>
+      )}
+    </div>
+  );
+}
+
+/**
+ * Renderer-side counterpart to `defaultRequestHeaders` in
+ * `@freecrawl/core/http-client`. Kept inline (not imported from core)
+ * because (a) the basic-auth path uses Node's `Buffer` which isn't
+ * available in the browser, and (b) the renderer doesn't otherwise
+ * pull core, so dragging it in would bloat the renderer bundle for
+ * one display-only helper.
+ *
+ * Per-host UA matching mirrors `Crawler.resolveUserAgent`. Auth /
+ * custom headers / accept-language come straight from the active
+ * config. This is reconstructed at render time, so if the user
+ * changed the config since the crawl ran the values reflect the
+ * NEW config — that's noted to the user via the panel header banner.
+ */
+function buildPreviewRequestHeaders(
+  config: CrawlConfig,
+  url: string,
+): { name: string; value: string }[] {
+  const userAgent = resolvePreviewUserAgent(config, url);
+  const map = new Map<string, string>();
+  map.set('user-agent', userAgent);
+  map.set('accept-language', config.acceptLanguage || 'en');
+  map.set('accept-encoding', 'gzip, deflate, br');
+  map.set(
+    'accept',
+    'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+  );
+  if (config.auth.type === 'basic' && config.auth.username) {
+    // btoa is browser-safe; the crawler uses Buffer.from(...).toString('base64')
+    // which produces the same encoding.
+    const creds = btoa(`${config.auth.username}:${config.auth.password ?? ''}`);
+    map.set('authorization', `Basic ${creds}`);
+  } else if (config.auth.type === 'bearer' && config.auth.token) {
+    map.set('authorization', `Bearer ${config.auth.token}`);
+  }
+  // Custom headers override defaults (case-insensitive). Same logic
+  // as `defaultRequestHeaders` so the preview matches reality.
+  for (const [rawKey, value] of Object.entries(config.customHeaders ?? {})) {
+    const key = rawKey.trim();
+    if (!key) continue;
+    const lower = key.toLowerCase();
+    for (const existing of [...map.keys()]) {
+      if (existing.toLowerCase() === lower) map.delete(existing);
+    }
+    map.set(key, value);
+  }
+  return [...map.entries()].map(([name, value]) => ({ name, value }));
+}
+
+/**
+ * Renderer counterpart to `Crawler.resolveUserAgent`. Walks
+ * `perHostUserAgents` in order; first match wins. Pattern syntax:
+ * exact host (`m.example.com`) or leading wildcard (`*.example.com`).
+ */
+function resolvePreviewUserAgent(config: CrawlConfig, url: string): string {
+  const rules = config.perHostUserAgents ?? [];
+  if (rules.length === 0) return config.userAgent;
+  let host = '';
+  try {
+    host = new URL(url).hostname.toLowerCase();
+  } catch {
+    return config.userAgent;
+  }
+  for (const rule of rules) {
+    const pat = rule.hostPattern.trim().toLowerCase();
+    if (!pat) continue;
+    if (pat.startsWith('*.')) {
+      const suffix = pat.slice(1);
+      if (host.endsWith(suffix) && host.length > suffix.length) {
+        return rule.userAgent;
+      }
+    } else if (host === pat) {
+      return rule.userAgent;
+    }
+  }
+  return config.userAgent;
+}
+
 function fleschBand(score: number): string {
   if (score >= 90) return 'very easy';
   if (score >= 80) return 'easy';

apps/desktop/src/renderer/src/components/ReportsDialog.tsx

@@ -13,6 +13,7 @@ import type {
   ImageWeightRow,
   BucketHistogramRow,
   ServerHeaderRow,
+  TopWordsRow,
   WordCountPerDirectoryRow,
   SitemapOrphanRow,
 } from '@freecrawl/shared-types';
@@ -42,7 +43,8 @@ type ReportKind =
   | 'url-length-histogram'
   | 'word-count-per-dir'
   | 'sitemap-orphans'
-  | 'server-headers';
+  | 'server-headers'
+  | 'top-words';
 
 interface ReportRow {
   /** Display key (directory path / status code / depth label). */
@@ -79,6 +81,7 @@ const REPORT_LABELS: Record<ReportKind, string> = {
   'word-count-per-dir': 'Word Count per Directory',
   'sitemap-orphans': 'Sitemap Orphans (Top 1000)',
   'server-headers': 'Server Stack (Server Header)',
+  'top-words': 'Top Words (Title + Meta + H1, Top 100)',
 };
 
 const KEY_LABELS: Record<ReportKind, string> = {
@@ -102,6 +105,7 @@ const KEY_LABELS: Record<ReportKind, string> = {
   'word-count-per-dir': 'Directory',
   'sitemap-orphans': 'URL',
   'server-headers': 'Server',
+  'top-words': 'Word',
 };
 
 const TOP_URL_METRIC: Record<ReportKind, TopUrlMetric | null> = {
@@ -125,6 +129,7 @@ const TOP_URL_METRIC: Record<ReportKind, TopUrlMetric | null> = {
   'word-count-per-dir': null,
   'sitemap-orphans': null,
   'server-headers': null,
+  'top-words': null,
 };
 
 const VALUE_FORMAT: Record<ReportKind, (v: number | null) => string> = {
@@ -153,6 +158,7 @@ const VALUE_FORMAT: Record<ReportKind, (v: number | null) => string> = {
   'word-count-per-dir': (v) => (v ?? 0).toLocaleString(),
   'sitemap-orphans': (v) => (v ?? 0).toLocaleString(),
   'server-headers': (v) => (v ?? 0).toLocaleString(),
+  'top-words': (v) => (v ?? 0).toLocaleString(),
 };
 
 export function ReportsDialog({ open, onClose }: Props) {
@@ -301,6 +307,22 @@ export function ReportsDialog({ open, onClose }: Props) {
           const r = await window.freecrawl.reportsServerHeaders();
           if (!cancelled)
             setRows(r.map((x: ServerHeaderRow) => ({ key: x.server, count: x.count })));
+        } else if (kind === 'top-words') {
+          const r = await window.freecrawl.reportsTopWords({ limit: 100, minLength: 3 });
+          if (!cancelled)
+            setRows(
+              r.map((x: TopWordsRow) => ({
+                key: x.word,
+                // Bar metric is total occurrence count so the loudest
+                // terms spike visually. Page-coverage is shown beside it
+                // so the user can spot "high count, narrow coverage" =
+                // a single page repeating a word.
+                count: x.count,
+                valueLabel: `${x.count.toLocaleString()} hits · ${x.pages.toLocaleString()} page${
+                  x.pages === 1 ? '' : 's'
+                }`,
+              })),
+            );
         } else {
           const metric = TOP_URL_METRIC[kind];
           if (metric) {
@@ -395,6 +417,7 @@ export function ReportsDialog({ open, onClose }: Props) {
               <option value="word-count-per-dir">Word Count per Directory</option>
               <option value="sitemap-orphans">Sitemap Orphans (Top 1000)</option>
               <option value="server-headers">Server Stack</option>
+              <option value="top-words">Top Words (Title + Meta + H1)</option>
             </select>
           </label>
           {(kind === 'pages-per-dir' || kind === 'word-count-per-dir') && (

apps/desktop/src/renderer/src/components/SettingsDialog.tsx

@@ -30,6 +30,7 @@ import type {
   CustomExtractionRule,
   HttpAuth,
 } from '@freecrawl/shared-types';
+import { DEFAULT_CRAWL_CONFIG } from '@freecrawl/shared-types';
 import { useAppStore } from '../store.js';
 import { InfoTip, type FieldInfo } from './InfoTip.js';
 
@@ -428,6 +429,21 @@ export function SettingsDialog({ open, onClose }: Props) {
     setForm((s) => ({ ...s, [key]: value }));
   }
 
+  /**
+   * Reset the in-progress form to the factory `DEFAULT_CRAWL_CONFIG`.
+   * Only touches the local form — the user still has to click Save to
+   * persist, and Cancel discards. We confirm first because this wipes
+   * include/exclude patterns, custom-extraction rules, auth, proxy
+   * profiles, etc. — all of which live in the same form.
+   */
+  function resetToDefaults() {
+    const ok = window.confirm(
+      'Reset ALL crawl settings to their defaults?\n\nThis clears include/exclude patterns, custom search & extraction rules, URL rewrites, auth, proxy profiles, per-host user-agents, and webhook URL. Nothing is saved until you click Save — Cancel still discards.',
+    );
+    if (!ok) return;
+    setForm(configToForm(DEFAULT_CRAWL_CONFIG));
+  }
+
   function save() {
     setConfig({
       mode: form.mode,
@@ -689,19 +705,28 @@ export function SettingsDialog({ open, onClose }: Props) {
           </div>
         </div>
 
-        <div className="flex items-center justify-end gap-2 border-t border-surface-800 px-4 py-2.5">
+        <div className="flex items-center justify-between gap-2 border-t border-surface-800 px-4 py-2.5">
           <button
-            className="rounded border border-surface-700 px-3 py-1 text-[11px] hover:bg-surface-800"
-            onClick={onClose}
+            className="rounded border border-surface-700 px-3 py-1 text-[11px] text-surface-300 hover:bg-surface-800 hover:text-surface-100"
+            onClick={resetToDefaults}
+            title="Reset every crawl setting to its factory default (you still have to Save)"
           >
-            Cancel
-          </button>
-          <button
-            className="rounded bg-blue-600 px-3 py-1 text-[11px] font-medium text-white hover:bg-blue-500"
-            onClick={save}
-          >
-            Save
+            Reset to Defaults
           </button>
+          <div className="flex items-center gap-2">
+            <button
+              className="rounded border border-surface-700 px-3 py-1 text-[11px] hover:bg-surface-800"
+              onClick={onClose}
+            >
+              Cancel
+            </button>
+            <button
+              className="rounded bg-blue-600 px-3 py-1 text-[11px] font-medium text-white hover:bg-blue-500"
+              onClick={save}
+            >
+              Save
+            </button>
+          </div>
         </div>
       </div>
     </div>
@@ -947,102 +972,77 @@ function CrawlerPanel({ form, update }: PanelProps) {
   return (
     <>
       <p className="mb-3 text-[11px] text-surface-400">
-        Throughput, concurrency, and traversal limits.
+        Traversal limits and crawl-scope toggles. Screaming Frog equivalents shown in
+        parentheses.
       </p>
+
+      <div className="mb-4 rounded border border-blue-700/40 bg-blue-900/15 px-3 py-2 text-[11px] text-blue-200">
+        <span className="font-medium">Looking for crawl speed?</span> Thread count, requests
+        per second, per-request delay, and retries live in the{' '}
+        <span className="font-medium">Speed</span> section — that&apos;s where you cap how fast
+        the crawler hits a server.
+      </div>
+
       <div className="grid grid-cols-2 gap-3">
         <Num
-          label="Max Depth"
+          label="Crawl Depth Limit (Max Depth)"
           value={form.maxDepth}
           onChange={(v) => update('maxDepth', v)}
-          info="Hop count from the start URL. Start URL is depth 0; its outlinks are depth 1, theirs depth 2, and so on."
+          info="Hop count from the start URL. Start URL is depth 0; its outlinks are depth 1, theirs depth 2, and so on. Screaming Frog: 'Limit Crawl Depth' (Configuration → Spider → Limits)."
           example="10 covers most sites; 3 limits crawls to top-of-funnel pages only."
         />
         <Num
-          label="Max URLs"
+          label="Crawl Total Limit (Max URLs)"
           value={form.maxUrls}
           onChange={(v) => update('maxUrls', v)}
-          info="Hard cap on total URLs crawled. The crawl stops as soon as this is reached."
+          info="Hard cap on total URLs crawled. The crawl stops as soon as this is reached. Screaming Frog: 'Limit Crawl Total'."
           example="1000000 (1M) for a full site audit; 5000 for spot checks."
         />
         <Num
-          label="Max Concurrency"
-          value={form.maxConcurrency}
-          onChange={(v) => update('maxConcurrency', v)}
-          info="Number of parallel HTTP workers. Higher = faster crawl but more load on the target server."
-          example="20 is a safe default; bump to 50 on fast servers, drop to 5 if the site rate-limits."
-        />
-        <Num
-          label="Max RPS"
-          value={form.maxRps}
-          onChange={(v) => update('maxRps', v)}
-          info="Requests per second cap across all workers combined. Hard rate limit independent of concurrency."
-          example="20 for typical sites; 5 to be polite on shared hosting."
-        />
-        <Num
-          label="Request Timeout (ms)"
+          label="Response Timeout (ms)"
           value={form.requestTimeoutMs}
           onChange={(v) => update('requestTimeoutMs', v)}
-          info="Per-request abort threshold. Pages that take longer than this are recorded as network errors."
+          info="Per-request abort threshold. Pages that take longer than this are recorded as network errors. Screaming Frog: 'Response Timeout (secs)' (Configuration → Spider → Advanced) — that one's in seconds, this is in milliseconds."
           example="20000 (20 s) for typical use; 5000 for fast spot checks; 60000 for slow APIs."
         />
-        <Num
-          label="Crawl Delay (ms, per worker)"
-          value={form.crawlDelayMs}
-          onChange={(v) => update('crawlDelayMs', v)}
-          info="Sleep inserted after each request, applied per worker. Stacks with robots.txt's own crawl-delay if larger."
-          example="0 disables; 250 for very polite crawling; 1000 to throttle aggressively."
-        />
-        <Num
-          label="Retry Attempts"
-          value={form.retryAttempts}
-          onChange={(v) => update('retryAttempts', v)}
-          info="How many times to retry on network errors / 5xx / 429. The original request is not counted."
-          example="2 (default) means original + 2 retries (3 total). 0 disables retry."
-        />
-        <Num
-          label="Retry Initial Delay (ms)"
-          value={form.retryInitialDelayMs}
-          onChange={(v) => update('retryInitialDelayMs', v)}
-          info="Backoff before the first retry. Doubles each subsequent attempt (exponential backoff)."
-          example="500 → next attempts wait 500 ms, then 1000 ms, then 2000 ms…"
-        />
       </div>
+
       <div className="mt-4 grid grid-cols-2 gap-2">
         <Bool
-          label="Follow redirects"
+          label="Always Follow Redirects"
           checked={form.followRedirects}
           onChange={(v) => update('followRedirects', v)}
-          info="Crawl 3xx redirect targets. Each hop is its own row; the chain is reconstructed in the Response Codes view."
+          info="Crawl 3xx redirect targets. Each hop is its own row; the chain is reconstructed in the Response Codes view. Screaming Frog: 'Always Follow Redirects' (Configuration → Spider → Advanced)."
           example="On for normal audits; off when you only want to inspect raw 3xx behaviour."
         />
         <Bool
           label="Respect robots.txt"
           checked={form.respectRobotsTxt}
           onChange={(v) => update('respectRobotsTxt', v)}
-          info="Honor Disallow rules + crawl-delay declared in /robots.txt for the configured User-Agent."
+          info="Honor Disallow rules + crawl-delay declared in /robots.txt for the configured User-Agent. Screaming Frog: 'Respect robots.txt' (Configuration → robots.txt)."
           example="On (default). Off only when crawling sites you own and need to bypass."
         />
         <Bool
-          label="Crawl external links"
+          label="Check External Links"
           checked={form.crawlExternal}
           onChange={(v) => update('crawlExternal', v)}
-          info="Probe outbound links to other hosts (HEAD only) so the Broken Links view catches dead externals."
+          info="Probe outbound links to other hosts (HEAD only) so the Broken Links view catches dead externals. Screaming Frog: 'External Links' (Configuration → Spider → Crawl)."
           example="On for outbound link audits; off for fast internal-only crawls."
         />
         <Bool
-          label="Store nofollow links"
+          label="Follow / Store Nofollow Links"
           checked={form.storeNofollowLinks}
           onChange={(v) => update('storeNofollowLinks', v)}
-          hint="Default off (Screaming-Frog style 'Respect Nofollow')"
-          info='Persist rel="nofollow" links in the link graph. When off, nofollow links are dropped entirely (not counted in outlinks, not probed as externals).'
+          hint="Default off — Screaming Frog 'Respect Nofollow' behaviour"
+          info='Persist rel="nofollow" links in the link graph. When off, nofollow links are dropped entirely (not counted in outlinks, not probed as externals). Screaming Frog inverse: turning this ON ≈ unchecking "Follow Internal/External Nofollow".'
           example="On if you need nofollow attribute audits; off keeps the link graph cleaner."
         />
         <Bool
-          label="Discover sitemaps"
+          label="Auto-Discover XML Sitemaps"
           checked={form.discoverSitemaps}
           onChange={(v) => update('discoverSitemaps', v)}
-          hint="Read sitemap.xml from robots.txt + default paths at crawl start"
-          info="Fetches /robots.txt sitemap directives + /sitemap.xml fallbacks. Powers the 'Non-Indexable in Sitemap' issue filter."
+          hint="Reads sitemap.xml from robots.txt + default paths at crawl start"
+          info="Fetches /robots.txt sitemap directives + /sitemap.xml fallbacks. Powers the 'Non-Indexable in Sitemap' issue filter. Screaming Frog: 'Auto Discover XML Sitemaps via robots.txt' (Configuration → Spider → Crawl)."
           example="On (default) — cheap I/O, high SEO value."
         />
       </div>

apps/desktop/src/renderer/src/utils/statusDiagnosis.ts

@@ -0,0 +1,511 @@
+/**
+ * Human-readable diagnosis for an HTTP status code, enriched with
+ * signals read off the response headers. Drives the "what does this
+ * mean / why did it happen" banner in the URL Details → HTTP Headers
+ * sub-tab — the place users land when a crawl returns a status they
+ * don't recognise (520-class Cloudflare codes are the canonical
+ * example: "Server Error" tells you nothing about the cause).
+ *
+ * Pure / sync / renderer-only — everything it needs is already in the
+ * `CrawlUrlRow` (statusCode, statusText) plus the captured response
+ * headers, so there's no IPC.
+ */
+
+export interface StatusDiagnosis {
+  /** "520 — Web Server Returned an Unknown Error (Cloudflare)" */
+  title: string;
+  severity: 'ok' | 'info' | 'warning' | 'error';
+  /** One-paragraph plain-English explanation of what the code means. */
+  explanation: string;
+  /** Bullet list of the most likely causes in a crawl context. */
+  causes: string[];
+  /**
+   * Signals derived from the response headers — e.g. "Behind Cloudflare
+   * (cf-ray: 9faa…)", "Server asked to retry after 60 s". Empty when no
+   * recognised hint headers were present.
+   */
+  signals: string[];
+  /**
+   * Actionable next steps tuned to a crawler workflow (lower the rate,
+   * change the User-Agent, etc.). Empty for codes where there's nothing
+   * the crawler can do.
+   */
+  whatToDo: string[];
+}
+
+type HeaderList = ReadonlyArray<{ name: string; value: string }>;
+
+function headerMap(headers: HeaderList): Map<string, string> {
+  const m = new Map<string, string>();
+  for (const h of headers) m.set(h.name.toLowerCase(), h.value);
+  return m;
+}
+
+/** Detect the fronting CDN / proxy from response headers, if any. */
+function detectFront(h: Map<string, string>): string | null {
+  const server = (h.get('server') ?? '').toLowerCase();
+  if (h.has('cf-ray') || server.includes('cloudflare')) return 'Cloudflare';
+  if (h.has('x-amz-cf-id') || server.includes('cloudfront')) return 'Amazon CloudFront';
+  if (h.has('x-served-by') && (h.get('x-served-by') ?? '').includes('cache')) return 'Fastly';
+  if (server.includes('akamai') || h.has('x-akamai-transformed')) return 'Akamai';
+  if (server.includes('vercel') || h.has('x-vercel-id')) return 'Vercel';
+  if (server.includes('netlify')) return 'Netlify';
+  if (h.has('x-sucuri-id') || h.has('x-sucuri-cache')) return 'Sucuri WAF';
+  if (server.includes('varnish') || h.has('x-varnish')) return 'Varnish cache';
+  if (server.includes('nginx')) return 'nginx';
+  if (server.includes('apache')) return 'Apache';
+  if (server.includes('litespeed')) return 'LiteSpeed';
+  if (server.includes('microsoft-iis') || server.includes('iis')) return 'IIS';
+  return null;
+}
+
+interface CodeInfo {
+  title: string;
+  severity: StatusDiagnosis['severity'];
+  explanation: string;
+  causes: string[];
+  whatToDo: string[];
+}
+
+/**
+ * Cloudflare's 5xx extension codes (520-530). These are NOT origin
+ * errors in the usual sense — they describe a failure between Cloudflare
+ * and the customer's origin server, OR (530) a Cloudflare-side block.
+ * `statusText` for these is usually the unhelpful generic "Server Error",
+ * which is exactly why this table exists.
+ */
+const CLOUDFLARE_CODES: Record<number, CodeInfo> = {
+  520: {
+    title: '520 — Web Server Returned an Unknown Error (Cloudflare)',
+    severity: 'error',
+    explanation:
+      "Cloudflare connected to the origin server but got back an empty, malformed, or unexpected response (e.g. the connection was reset mid-response, the origin returned headers that are too large, or the origin process crashed). It is a catch-all 'something went wrong at the origin' code.",
+    causes: [
+      'Origin server crashed, restarted, or ran out of workers/memory while serving the request',
+      'Origin closed the TCP connection before sending a complete response (often an app-level timeout or OOM kill)',
+      "The crawl's request rate overwhelmed the origin's process pool — 520s that appear in bursts under load almost always mean this",
+      'Origin returned response headers larger than Cloudflare allows (rare; usually a misconfigured app)',
+    ],
+    whatToDo: [
+      'If 520s appear in bursts across many URLs, lower Max Threads / Max URL/s (Settings → Speed) and add a Crawl Delay — the origin is being overloaded by the crawl',
+      'Re-crawl the affected URLs after the crawl finishes (transient 520s often clear on retry — bump Retry Attempts in Settings → Speed)',
+      'If only specific URLs 520 consistently, those pages have a server-side bug — flag them to the site owner',
+    ],
+  },
+  521: {
+    title: '521 — Web Server Is Down (Cloudflare)',
+    severity: 'error',
+    explanation:
+      'Cloudflare could not open a TCP connection to the origin server — the origin actively refused the connection or is not listening on the expected port.',
+    causes: [
+      'Origin web server (nginx/Apache/Node/etc.) is stopped or has crashed',
+      "Origin's firewall is blocking Cloudflare's IP ranges (a misconfiguration — the origin should allow Cloudflare)",
+      'Origin server is overloaded to the point of not accepting new connections',
+    ],
+    whatToDo: [
+      'This is an origin-availability problem, not something the crawler can work around — the site is effectively down for these URLs',
+      'Re-crawl later; if it persists, the site is broken',
+    ],
+  },
+  522: {
+    title: '522 — Connection Timed Out (Cloudflare)',
+    severity: 'error',
+    explanation:
+      'Cloudflare started a TCP handshake with the origin but it never completed within the timeout — the origin accepted the SYN but never replied, or a network device between them dropped the packets.',
+    causes: [
+      'Origin is severely overloaded (TCP backlog full)',
+      "Origin's firewall is silently dropping (not rejecting) Cloudflare's packets",
+      'Network/routing issue between Cloudflare and the origin',
+      "Under crawl load: the origin's connection table is exhausted by the crawl's parallel requests",
+    ],
+    whatToDo: [
+      'Lower concurrency (Settings → Speed → Max Threads) and re-crawl — connection-table exhaustion is the usual cause when this shows up mid-crawl',
+      'Retry the affected URLs after the crawl',
+    ],
+  },
+  523: {
+    title: '523 — Origin Is Unreachable (Cloudflare)',
+    severity: 'error',
+    explanation:
+      "Cloudflare could not route to the origin server at all — typically a DNS problem in Cloudflare's origin record, or the origin's network is down.",
+    causes: [
+      "The site's DNS record in Cloudflare points to a wrong / dead IP",
+      "Origin's hosting provider has a network outage",
+      'BGP / routing failure between Cloudflare and the origin',
+    ],
+    whatToDo: [
+      'Nothing the crawler can do — this is an origin DNS/network misconfiguration',
+    ],
+  },
+  524: {
+    title: '524 — A Timeout Occurred (Cloudflare)',
+    severity: 'error',
+    explanation:
+      'Cloudflare made a TCP connection to the origin successfully, but the origin did not send a complete HTTP response within Cloudflare\'s timeout (100 s by default). The origin is alive but too slow on this request.',
+    causes: [
+      'A slow database query, external API call, or heavy computation on the page',
+      'Origin is overloaded — requests are queuing behind a saturated worker pool',
+      "Under crawl load: the crawl filled the origin's request queue and this URL waited too long behind it",
+    ],
+    whatToDo: [
+      'Lower Max Threads / Max URL/s and add a Crawl Delay (Settings → Speed) so the origin isn\'t saturated',
+      'Increase Response Timeout (Settings → Crawler) if you want to give genuinely slow pages more time',
+      'Retry affected URLs after the crawl',
+    ],
+  },
+  525: {
+    title: '525 — SSL Handshake Failed (Cloudflare)',
+    severity: 'error',
+    explanation:
+      "The TLS handshake between Cloudflare and the origin server failed. This is the Cloudflare↔origin leg, not the browser↔Cloudflare leg you see in the address bar — so the site can look fine to a normal visitor and still 525.",
+    causes: [
+      "Origin's TLS certificate is expired, self-signed, or for the wrong hostname (with Cloudflare in 'Full (strict)' mode)",
+      'Origin only supports old TLS versions / cipher suites that Cloudflare rejects',
+      'Origin server is overloaded and dropping TLS handshakes',
+    ],
+    whatToDo: [
+      "Origin TLS misconfiguration — not something the crawler can change. Check the host's certificate (URL Details → TLS Cert if HTTPS)",
+    ],
+  },
+  526: {
+    title: '526 — Invalid SSL Certificate (Cloudflare)',
+    severity: 'error',
+    explanation:
+      "Cloudflare (in 'Full (strict)' SSL mode) could not validate the origin server's TLS certificate.",
+    causes: [
+      "Origin certificate is expired, self-signed, or issued for a different hostname",
+      'Certificate chain on the origin is incomplete (missing intermediates)',
+    ],
+    whatToDo: ['Origin certificate problem — nothing the crawler can do'],
+  },
+  527: {
+    title: '527 — Railgun Error (Cloudflare, deprecated)',
+    severity: 'error',
+    explanation:
+      'A connection error in Cloudflare Railgun (a now-retired origin-acceleration product). Effectively obsolete; if you see it the site is on a very old Cloudflare setup.',
+    causes: ['Legacy Cloudflare Railgun connection failure'],
+    whatToDo: ['Nothing actionable from the crawler side'],
+  },
+  530: {
+    title: '530 — Cloudflare Error (usually a firewall / WAF block — error 1020)',
+    severity: 'error',
+    explanation:
+      'A 530 is Cloudflare\'s way of saying "an error happened, see the error 1xxx code in the body". In a crawl context it is almost always 1020 "Access Denied" — a Cloudflare Firewall / WAF / Bot-Fight rule blocked the request. Less commonly it pairs with 1016 (origin DNS error) or other 1xxx codes.',
+    causes: [
+      "Cloudflare WAF / Firewall Rules / Bot Fight Mode flagged the crawler — by IP, by User-Agent, by request rate, or by missing browser fingerprints",
+      'The site has a "managed challenge" the crawler can\'t solve',
+      '(Rarely) an origin DNS error (1016) rather than a block',
+    ],
+    whatToDo: [
+      'Set a browser-like User-Agent (Settings → Requests) — the default crawler UA is the #1 trigger',
+      'Drop Max Threads / Max URL/s and add a Crawl Delay (Settings → Speed) — rapid requests from one IP trip rate-based rules',
+      'Respect robots.txt and any declared Crawl-Delay (Settings → Crawler)',
+      "If the site owner is you, allowlist the crawler in Cloudflare; otherwise this site is intentionally blocking automated access",
+    ],
+  },
+};
+
+/** Non-Cloudflare codes worth a richer explanation than `statusText`. */
+const GENERIC_CODES: Record<number, CodeInfo> = {
+  401: {
+    title: '401 — Unauthorized',
+    severity: 'warning',
+    explanation: 'The resource requires authentication and none (or invalid) credentials were sent.',
+    causes: ['Page is behind HTTP Basic/Bearer auth', 'A session cookie is required and none was sent'],
+    whatToDo: [
+      'Configure credentials in Settings → Authentication if you have them',
+      'Use Settings → Cookies → "Accept all" if the site gates content behind a session',
+    ],
+  },
+  403: {
+    title: '403 — Forbidden',
+    severity: 'warning',
+    explanation:
+      'The server understood the request but refuses to serve it. In a crawl this is most often a bot block or a WAF rule rather than a genuine permission issue.',
+    causes: [
+      'WAF / bot-protection rule triggered by the crawler User-Agent, IP, or request rate',
+      'Directory or file genuinely not accessible to anonymous users',
+      'Hotlink / referer protection',
+    ],
+    whatToDo: [
+      'Try a browser-like User-Agent (Settings → Requests)',
+      'Lower the crawl rate (Settings → Speed)',
+      "If it's site-wide, the site is actively blocking automated access",
+    ],
+  },
+  404: {
+    title: '404 — Not Found',
+    severity: 'warning',
+    explanation: 'The server has no resource at this URL.',
+    causes: ['Broken internal/external link', 'Page was removed without a redirect', 'Typo in a link href'],
+    whatToDo: ['Fix or remove the linking pages — see the Inlinks tab for who links here'],
+  },
+  408: {
+    title: '408 — Request Timeout',
+    severity: 'warning',
+    explanation: "The server timed out waiting for the request to complete.",
+    causes: ['Slow network between crawler and server', 'Server-side connection timeout set very low'],
+    whatToDo: ['Usually transient — bump Retry Attempts (Settings → Speed)'],
+  },
+  410: {
+    title: '410 — Gone',
+    severity: 'info',
+    explanation: 'The resource was intentionally and permanently removed — stronger than a 404.',
+    causes: ['Page deliberately retired; the server is telling crawlers to drop it'],
+    whatToDo: ['Remove links to this URL — Google will de-index it faster than a 404'],
+  },
+  429: {
+    title: '429 — Too Many Requests (rate limited)',
+    severity: 'warning',
+    explanation:
+      'The server is rate-limiting the crawler — too many requests in too short a window from this IP/User-Agent.',
+    causes: ['Crawl rate exceeds the server\'s per-client limit', 'Shared hosting with aggressive throttling'],
+    whatToDo: [
+      'Lower Max Threads / Max URL/s and add a Crawl Delay (Settings → Speed)',
+      'Honour the Retry-After header — see the Signals below if the server sent one',
+      'Enable robots.txt Crawl-Delay respect (Settings → Crawler)',
+    ],
+  },
+  451: {
+    title: '451 — Unavailable for Legal Reasons',
+    severity: 'info',
+    explanation: 'The resource is blocked for legal reasons (court order, GDPR geoblock, DMCA, etc.).',
+    causes: ['Geo-restricted content', 'Legally-mandated takedown'],
+    whatToDo: ['Nothing — the block is intentional'],
+  },
+  500: {
+    title: '500 — Internal Server Error',
+    severity: 'error',
+    explanation: 'An unhandled error occurred in the server-side application while building the page.',
+    causes: ['Bug in the page template / backend code', 'Database unavailable', 'Misconfiguration on the server'],
+    whatToDo: [
+      'Retry after the crawl (transient 500s clear; persistent ones are a real bug)',
+      'If 500s appear under load, the crawl may be overwhelming a database — slow the crawl (Settings → Speed)',
+    ],
+  },
+  502: {
+    title: '502 — Bad Gateway',
+    severity: 'error',
+    explanation:
+      'A reverse proxy / load balancer in front of the app got an invalid response from the upstream application server.',
+    causes: [
+      'Upstream app server crashed or returned garbage',
+      'Upstream is overloaded — common under crawl load',
+      'Proxy↔upstream timeout',
+    ],
+    whatToDo: [
+      'Lower the crawl rate (Settings → Speed) if 502s appear in bursts',
+      'Retry affected URLs after the crawl',
+    ],
+  },
+  503: {
+    title: '503 — Service Unavailable',
+    severity: 'error',
+    explanation:
+      'The server is temporarily unable to handle the request — usually overload or maintenance. Often comes with a Retry-After header.',
+    causes: [
+      'Server overloaded — the crawl itself can cause this',
+      'Scheduled maintenance window',
+      'Auto-scaling lag (new instances spinning up)',
+    ],
+    whatToDo: [
+      'Lower Max Threads / Max URL/s and add a Crawl Delay (Settings → Speed)',
+      'Honour Retry-After (see Signals below)',
+      'Retry affected URLs after the crawl',
+    ],
+  },
+  504: {
+    title: '504 — Gateway Timeout',
+    severity: 'error',
+    explanation:
+      "A reverse proxy / load balancer didn't get a response from the upstream app server in time.",
+    causes: [
+      'Slow database query / external API call on the page',
+      'Upstream overloaded — requests queueing',
+      'Under crawl load: the crawl saturated the upstream worker pool',
+    ],
+    whatToDo: [
+      'Lower the crawl rate (Settings → Speed)',
+      'Increase Response Timeout (Settings → Crawler) for genuinely slow pages',
+      'Retry affected URLs after the crawl',
+    ],
+  },
+  508: {
+    title: '508 — Loop Detected',
+    severity: 'error',
+    explanation: 'The server detected an infinite loop while processing the request (often WebDAV or a redirect loop server-side).',
+    causes: ['Server-side redirect loop', 'Recursive include / symlink loop'],
+    whatToDo: ['Server bug — nothing the crawler can do'],
+  },
+  509: {
+    title: '509 — Bandwidth Limit Exceeded',
+    severity: 'error',
+    explanation: 'A non-standard code (cPanel/Apache) meaning the hosting account hit its monthly bandwidth cap.',
+    causes: ['Shared hosting bandwidth quota exhausted'],
+    whatToDo: ['Nothing the crawler can do — the host is throttling the whole account'],
+  },
+};
+
+/**
+ * Build the diagnosis for a (statusCode, statusText, headers) triple.
+ * `statusCode` may be null/0 for pre-response network failures, in
+ * which case the explanation focuses on connection-level causes.
+ */
+export function diagnoseStatus(
+  statusCode: number | null,
+  statusText: string | null,
+  headers: HeaderList,
+): StatusDiagnosis {
+  const h = headerMap(headers);
+  const front = detectFront(h);
+
+  // Header-derived signals — independent of the status code, but most
+  // useful on the error codes.
+  const signals: string[] = [];
+  if (h.has('cf-ray')) {
+    signals.push(`Behind Cloudflare — cf-ray ${h.get('cf-ray')}. The status code describes the Cloudflare↔origin link, not what a browser sees.`);
+  } else if (front && front !== 'nginx' && front !== 'Apache' && front !== 'IIS' && front !== 'LiteSpeed') {
+    signals.push(`Fronted by ${front}.`);
+  } else if (front) {
+    signals.push(`Origin server: ${front}${h.get('server') ? ` (${h.get('server')})` : ''}.`);
+  }
+  const retryAfter = h.get('retry-after');
+  if (retryAfter) {
+    const secs = Number(retryAfter);
+    signals.push(
+      Number.isFinite(secs)
+        ? `Server sent Retry-After: ${secs} s — it is explicitly asking clients to back off for that long. A polite crawl should not re-request this URL sooner.`
+        : `Server sent Retry-After: ${retryAfter}.`,
+    );
+  }
+  const cfCache = h.get('cf-cache-status');
+  if (cfCache) signals.push(`Cloudflare cache status: ${cfCache}.`);
+  const xRobots = h.get('x-robots-tag');
+  if (xRobots) signals.push(`X-Robots-Tag: ${xRobots} — affects indexability regardless of the status.`);
+
+  // ----- Network error (no response) -----
+  if (statusCode === null || statusCode === 0) {
+    return {
+      title: 'No HTTP Response — Network / Connection Error',
+      severity: 'error',
+      explanation:
+        statusText && statusText.trim()
+          ? `The request never received an HTTP response. The crawler recorded: "${statusText}".`
+          : 'The request never received an HTTP response — DNS lookup, TCP connection, TLS handshake, or the response read failed before any status line arrived.',
+      causes: [
+        'DNS could not resolve the hostname (typo, dead domain, or a name only resolvable from inside a private network)',
+        'TCP connection refused or timed out (host down, firewall, wrong port)',
+        'TLS handshake failed (expired/invalid certificate, protocol mismatch)',
+        'Connection reset or read timed out mid-response',
+        'Request aborted by the crawler because it exceeded Response Timeout (Settings → Crawler) or Max Response Time',
+      ],
+      signals,
+      whatToDo: [
+        'Check the hostname resolves and the site is reachable from a normal browser on this machine',
+        'If it is reachable but slow, raise Response Timeout (Settings → Crawler)',
+        'If it only fails under load, lower Max Threads / Max URL/s (Settings → Speed) — connection-table exhaustion on the origin looks like a network error to the client',
+        'Retry the affected URLs after the crawl (transient resets clear on retry)',
+      ],
+    };
+  }
+
+  // ----- 1xx / 2xx / 3xx — not errors -----
+  if (statusCode >= 100 && statusCode < 200) {
+    return {
+      title: `${statusCode} — Informational`,
+      severity: 'info',
+      explanation: `A 1xx response (${statusText || 'Informational'}). These are interim responses; the crawler treats them as transient and waits for the final status.`,
+      causes: [],
+      signals,
+      whatToDo: [],
+    };
+  }
+  if (statusCode >= 200 && statusCode < 300) {
+    return {
+      title: `${statusCode} — ${statusText || 'Success'}`,
+      severity: 'ok',
+      explanation: 'The request succeeded. No diagnosis needed — this section only has extra detail for error and redirect codes.',
+      causes: [],
+      signals,
+      whatToDo: [],
+    };
+  }
+  if (statusCode >= 300 && statusCode < 400) {
+    const loc = h.get('location');
+    return {
+      title: `${statusCode} — ${statusText || 'Redirect'}`,
+      severity: 'info',
+      explanation:
+        statusCode === 301
+          ? 'Permanent redirect. Search engines transfer ranking signals to the target and update their index to the new URL.'
+          : statusCode === 302 || statusCode === 307
+            ? 'Temporary redirect. Search engines keep the original URL indexed. A 302 used long-term is a common SEO mistake — it should usually be a 301.'
+            : statusCode === 308
+              ? 'Permanent redirect (like 301 but the request method is preserved).'
+              : statusCode === 304
+                ? 'Not Modified — the cached copy is still fresh. The crawler normally requests without conditional headers, so this is unusual.'
+                : `A ${statusCode} redirect (${statusText || ''}).`,
+      causes: loc ? [`Location header points to: ${loc}`] : ['No Location header was captured — unusual for a redirect.'],
+      signals,
+      whatToDo:
+        statusCode === 302 || statusCode === 307
+          ? ['If this redirect is permanent, change it to a 301/308 so ranking signals consolidate', 'Check the Response Codes view to see the full chain']
+          : ['Check the Response Codes view to see the full redirect chain and its final target'],
+    };
+  }
+
+  // ----- 4xx / 5xx — look up the knowledge base -----
+  const isCfCode = statusCode >= 520 && statusCode <= 530;
+  const info =
+    (front === 'Cloudflare' && CLOUDFLARE_CODES[statusCode]) ||
+    CLOUDFLARE_CODES[statusCode] || // 52x are Cloudflare-only even if we didn't detect the front (e.g. cf-ray stripped)
+    GENERIC_CODES[statusCode] ||
+    null;
+
+  if (info) {
+    // For 530 specifically, if there's NO cf-ray we shouldn't claim Cloudflare.
+    const adjusted: CodeInfo =
+      isCfCode && front !== 'Cloudflare' && !h.has('cf-ray')
+        ? {
+            ...info,
+            title: info.title.replace(' (Cloudflare)', ' (Cloudflare-style edge code)'),
+          }
+        : info;
+    return {
+      title: adjusted.title,
+      severity: adjusted.severity,
+      explanation: adjusted.explanation,
+      causes: adjusted.causes,
+      signals,
+      whatToDo: adjusted.whatToDo,
+    };
+  }
+
+  // ----- 4xx / 5xx with no specific entry -----
+  if (statusCode >= 400 && statusCode < 500) {
+    return {
+      title: `${statusCode} — ${statusText || 'Client Error'}`,
+      severity: 'warning',
+      explanation:
+        'A 4xx code means the server rejected the request as the client\'s fault (bad request, not allowed, not found, etc.). In a crawl, persistent 4xx on linked pages usually means broken links; sporadic 4xx can be bot-protection.',
+      causes: ['Broken or malformed link', 'Bot-protection / WAF rule', 'Method or content the server refuses'],
+      signals,
+      whatToDo: [
+        'Check the Inlinks tab to see what links to this URL',
+        'If it looks like a block, try a browser-like User-Agent and a slower rate (Settings → Requests / Speed)',
+      ],
+    };
+  }
+  return {
+    title: `${statusCode} — ${statusText || 'Server Error'}`,
+    severity: 'error',
+    explanation:
+      'A 5xx code means the server failed to fulfil a request it accepted as valid. Under crawl load these are frequently the crawl overwhelming the origin rather than a genuine page bug.',
+    causes: ['Server-side error or crash', 'Origin overloaded — possibly by the crawl itself', 'Upstream/proxy failure'],
+    signals,
+    whatToDo: [
+      'Lower Max Threads / Max URL/s and add a Crawl Delay (Settings → Speed) if 5xx appear in bursts',
+      'Bump Retry Attempts (Settings → Speed) and re-crawl affected URLs',
+    ],
+  };
+}

apps/mcp-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/mcp-server",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": true,
   "type": "module",
   "bin": {

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "freecrawl-seo",
-      "version": "0.3.5",
+      "version": "0.3.6",
       "license": "MIT",
       "workspaces": [
         "apps/*",
@@ -25,7 +25,7 @@
     },
     "apps/cli": {
       "name": "@freecrawl/cli",
-      "version": "0.3.5",
+      "version": "0.3.6",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -42,7 +42,7 @@
     },
     "apps/desktop": {
       "name": "@freecrawl/desktop",
-      "version": "0.3.5",
+      "version": "0.3.6",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -84,7 +84,7 @@
     },
     "apps/mcp-server": {
       "name": "@freecrawl/mcp-server",
-      "version": "0.3.5",
+      "version": "0.3.6",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*"
@@ -8732,7 +8732,7 @@
     },
     "packages/core": {
       "name": "@freecrawl/core",
-      "version": "0.3.5",
+      "version": "0.3.6",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*",
@@ -8758,7 +8758,7 @@
     },
     "packages/db": {
       "name": "@freecrawl/db",
-      "version": "0.3.5",
+      "version": "0.3.6",
       "dependencies": {
         "@freecrawl/shared-types": "*"
       },
@@ -8770,7 +8770,7 @@
     },
     "packages/shared-types": {
       "name": "@freecrawl/shared-types",
-      "version": "0.3.5",
+      "version": "0.3.6",
       "devDependencies": {
         "rimraf": "^6.0.1",
         "typescript": "^5.7.0"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/index.ts

@@ -32,6 +32,13 @@ export {
   type RobotsTestResult,
   type RobotsValidationIssue,
 } from './robots.js';
+export {
+  aggregateTopWords,
+  tokenizeForTopWords,
+  type TopWordsRow,
+  type TopWordsOptions,
+  type TopWordsLocale,
+} from './top-words.js';
 export {
   exportSitemap,
   validateSitemap,

packages/core/src/top-words.ts

@@ -0,0 +1,154 @@
+/**
+ * Word-frequency aggregation for the "Top Words" report. Pure / sync /
+ * data-only — the DB layer feeds raw text strings (titles, descriptions,
+ * H1s, optionally body text) and gets back a deduped frequency map.
+ *
+ * Stopwords are bundled English + Turkish (the two languages the user
+ * crawls most). Adding more locales is a matter of dropping another set
+ * into `STOPWORDS_BY_LOCALE`. Detection is intentionally absent: if the
+ * site mixes languages we just union all enabled stopword sets.
+ *
+ * Tokenisation uses Unicode-aware `\p{L}` so Turkish (ç, ğ, ı, ş, ü, ö),
+ * accented Western European, and Cyrillic all produce clean tokens
+ * without us having to maintain a per-language regex.
+ */
+
+const STOPWORDS_EN = new Set<string>([
+  // articles, conjunctions, common prepositions
+  'the', 'and', 'but', 'for', 'are', 'was', 'were', 'with', 'that', 'this',
+  'these', 'those', 'from', 'have', 'has', 'had', 'will', 'would', 'should',
+  'can', 'could', 'may', 'might', 'must', 'into', 'onto', 'upon', 'over',
+  'under', 'about', 'after', 'before', 'between', 'through', 'during',
+  'where', 'when', 'while', 'because', 'although', 'though', 'than', 'then',
+  'too', 'very', 'just', 'only', 'also', 'still', 'such', 'some', 'any',
+  'each', 'every', 'all', 'most', 'more', 'less', 'much', 'many', 'few',
+  'other', 'another', 'same', 'own', 'here', 'there', 'what', 'which',
+  'who', 'whom', 'whose', 'why', 'how', 'not', 'nor', 'either', 'neither',
+  'both', 'not', 'yet', 'without', 'within', 'against', 'across', 'among',
+  // pronouns
+  'they', 'them', 'their', 'his', 'her', 'hers', 'its', 'our', 'ours',
+  'your', 'yours', 'mine', 'myself', 'yourself', 'himself', 'herself',
+  'itself', 'ourselves', 'themselves', 'you', 'she', 'him', 'one',
+  // auxiliary / very common verbs
+  'been', 'being', 'doing', 'does', 'done', 'goes', 'gone', 'come', 'came',
+  'take', 'taken', 'took', 'make', 'made', 'get', 'got', 'gets',
+  'know', 'knew', 'known', 'see', 'saw', 'seen', 'say', 'said',
+  // common short
+  'use', 'used', 'using', 'two', 'three', 'four', 'five',
+  'new', 'now', 'way', 'day', 'year', 'time', 'thing',
+]);
+
+const STOPWORDS_TR = new Set<string>([
+  // bağlaçlar, yardımcı fiiller, çok geçen kısa kelimeler
+  've', 'veya', 'ile', 'ama', 'fakat', 'ancak', 'çünkü', 'için', 'gibi',
+  'kadar', 'değil', 'daha', 'çok', 'tüm', 'tüm', 'hepsi', 'bazı', 'her',
+  'hiç', 'şey', 'yine', 'hep', 'hep', 'eğer', 'ise', 'iken', 'fakat',
+  'lakin', 'oysa', 'oysaki', 'üzere', 'önce', 'sonra', 'arada',
+  // zamirler
+  'ben', 'sen', 'biz', 'siz', 'onlar', 'kim', 'ne', 'nasıl', 'neden',
+  'niçin', 'kendi', 'kendisi', 'kendileri', 'şu', 'şunu', 'şunlar',
+  'bu', 'bunu', 'bunlar', 'bunların',
+  // yapılar
+  'olan', 'olan', 'olduğu', 'olmak', 'oldu', 'olur', 'olabilir',
+  'oldu', 'eden', 'edilen', 'eden',
+  // bağlaç + edat varyantları
+  'göre', 'rağmen', 'doğru', 'karşı', 'beri', 'kez', 'kere', 'defa',
+  'an', 'biri', 'birisi', 'birkaç', 'birçok', 'kimse', 'herkes',
+  // sayılar / zaman
+  'bir', 'iki', 'üç', 'dört', 'beş', 'yıl', 'gün', 'ay', 'saat',
+  'şimdi', 'sonra', 'önce', 'bugün', 'yarın', 'dün',
+  // vs
+  'var', 'yok', 'mı', 'mi', 'mu', 'mü', 'da', 'de', 'ki',
+]);
+
+export type TopWordsLocale = 'en' | 'tr' | 'all';
+
+const STOPWORDS_BY_LOCALE: Record<TopWordsLocale, Set<string>> = {
+  en: STOPWORDS_EN,
+  tr: STOPWORDS_TR,
+  // Default — most crawls hit either English or Turkish text, often
+  // mixed (Turkish e-commerce sites sprinkle English brand names). The
+  // union catches the most common stopwords regardless of language.
+  all: new Set<string>([...STOPWORDS_EN, ...STOPWORDS_TR]),
+};
+
+export interface TopWordsRow {
+  word: string;
+  /** Total occurrences across all input strings. */
+  count: number;
+  /** Number of distinct input strings (≈ pages) that contained the word. */
+  pages: number;
+}
+
+export interface TopWordsOptions {
+  /** Max rows to return (default 100, hard cap 1000). */
+  limit?: number;
+  /** Minimum word length (default 3 — drops 1–2 char noise). */
+  minLength?: number;
+  /** Stopword set to apply (default `all`). */
+  locale?: TopWordsLocale;
+}
+
+/**
+ * Tokenise a string into lowercase words, dropping numbers + punctuation
+ * + tokens shorter than `minLength`. Unicode `\p{L}` covers Latin,
+ * Turkish, Cyrillic, Greek, etc. without us needing per-locale regex.
+ */
+export function tokenizeForTopWords(text: string, minLength: number): string[] {
+  if (!text) return [];
+  const out: string[] = [];
+  const re = /[\p{L}]+/gu;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(text)) !== null) {
+    if (m[0].length >= minLength) out.push(m[0].toLowerCase());
+  }
+  return out;
+}
+
+/**
+ * Aggregate word frequencies from a stream of input strings (one per
+ * page, typically `title + " " + metaDescription + " " + h1`). Returns
+ * the top-N most frequent non-stopword tokens with both raw count and
+ * page-coverage count (how many distinct strings contained the word).
+ *
+ * The page-coverage signal lets the UI distinguish "popular topic
+ * across the site" from "one page repeats a word a hundred times".
+ */
+export function aggregateTopWords(
+  texts: Iterable<string>,
+  opts: TopWordsOptions = {},
+): TopWordsRow[] {
+  const limit = Math.max(1, Math.min(1000, opts.limit ?? 100));
+  const minLength = Math.max(1, Math.min(10, opts.minLength ?? 3));
+  const stopwords = STOPWORDS_BY_LOCALE[opts.locale ?? 'all'];
+
+  // Two parallel maps: total frequency, and per-page frequency
+  // (incremented at most once per source string). Keeping them
+  // separate avoids re-tokenising or threading a Set through the
+  // hot path.
+  const total = new Map<string, number>();
+  const pages = new Map<string, number>();
+  const seenInThisPage = new Set<string>();
+
+  for (const text of texts) {
+    const tokens = tokenizeForTopWords(text, minLength);
+    seenInThisPage.clear();
+    for (const tok of tokens) {
+      if (stopwords.has(tok)) continue;
+      total.set(tok, (total.get(tok) ?? 0) + 1);
+      if (!seenInThisPage.has(tok)) {
+        pages.set(tok, (pages.get(tok) ?? 0) + 1);
+        seenInThisPage.add(tok);
+      }
+    }
+  }
+
+  return [...total.entries()]
+    .map(([word, count]) => ({
+      word,
+      count,
+      pages: pages.get(word) ?? 0,
+    }))
+    .sort((a, b) => b.count - a.count || a.word.localeCompare(b.word))
+    .slice(0, limit);
+}

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/project-db.ts

@@ -4284,6 +4284,38 @@ export class ProjectDb {
       .slice(0, limit);
   }
 
+  /**
+   * Concatenated SEO text for the Top Words report — title, meta
+   * description, and H1 joined with spaces, one row per indexable HTML
+   * page. Body text is intentionally NOT included: at 100K URLs the
+   * `url_sources.body` aggregate would be hundreds of MB and the SEO
+   * signal is well concentrated in the title-meta-h1 trio anyway.
+   *
+   * Returns an array because the consumer (main process) feeds it into
+   * `aggregateTopWords` which expects an iterable.
+   */
+  seoTextCorpus(): string[] {
+    const rows = this.db
+      .prepare(
+        `SELECT title, meta_description, h1
+           FROM urls
+          WHERE is_external = 0
+            AND content_kind = 'html'
+            AND status_code BETWEEN 200 AND 299
+            AND indexability = 'indexable'`,
+      )
+      .all() as { title: string | null; meta_description: string | null; h1: string | null }[];
+    const out: string[] = [];
+    for (const r of rows) {
+      const parts: string[] = [];
+      if (r.title) parts.push(r.title);
+      if (r.meta_description) parts.push(r.meta_description);
+      if (r.h1) parts.push(r.h1);
+      if (parts.length > 0) out.push(parts.join(' '));
+    }
+    return out;
+  }
+
   /**
    * URLs declared in the sitemap but never reached during the crawl —
    * the canonical "orphan" definition. Surfaces sitemap URLs that the

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/ipc.ts

@@ -110,6 +110,7 @@ export const IPC = {
   reportsWordCountPerDirectory: 'reports:word-count-per-directory',
   reportsSitemapOrphans: 'reports:sitemap-orphans',
   reportsServerHeaders: 'reports:server-headers',
+  reportsTopWords: 'reports:top-words',
   /**
    * Renderer → main heartbeat carrying the live input-lag estimate (ms).
    * The crawler subscribes to this so it can adaptively shrink its
@@ -657,6 +658,31 @@ export interface ServerHeaderRow {
   count: number;
 }
 
+/**
+ * One row in the Top Words report — body-text frequency aggregation
+ * over the title + meta description + H1 corpus of every indexable
+ * 2xx HTML page. Stopwords (en/tr) and tokens shorter than the
+ * configured `minLength` are filtered out before counting.
+ *
+ * `count` is total occurrences, `pages` is how many distinct pages
+ * contained the word. The latter discriminates "site-wide topic"
+ * (high pages) from "one page repeats this term" (high count, low pages).
+ */
+export interface TopWordsRow {
+  word: string;
+  count: number;
+  pages: number;
+}
+
+export interface TopWordsInput {
+  /** Max rows to return. Default 100, hard cap 1000. */
+  limit?: number;
+  /** Minimum word length. Default 3. */
+  minLength?: number;
+  /** Stopword set: `en`, `tr`, or `all` (union). Default `all`. */
+  locale?: 'en' | 'tr' | 'all';
+}
+
 /**
  * One row in the Analytics Coverage report. Counts how many indexable HTML
  * pages declare a given tracker — useful to spot incomplete rollouts
@@ -855,6 +881,7 @@ export interface FreeCrawlApi {
   ): Promise<WordCountPerDirectoryRow[]>;
   reportsSitemapOrphans(limit?: number): Promise<SitemapOrphanRow[]>;
   reportsServerHeaders(): Promise<ServerHeaderRow[]>;
+  reportsTopWords(input: TopWordsInput): Promise<TopWordsRow[]>;
   /** Heartbeat: renderer reports its latest input-lag sample (ms). */
   reportRendererLag(lagMs: number): void;
   prefsExportSettings(input: SettingsExportInput): Promise<SettingsExportResult>;