feat: memory monitor, robots.txt lint, security + duplicates tabs

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "type": "module",
   "bin": {

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "main": "./out/main/index.js",

apps/desktop/src/main/index.ts

@@ -14,6 +14,7 @@ import {
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
 import { mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { totalmem, freemem } from 'node:os';
 import {
   DEFAULT_CRAWL_CONFIG,
   IPC,
@@ -92,6 +93,7 @@ import {
   exportHtmlReport,
   compareCrawls,
   testUrlAgainstRobots,
+  validateRobotsTxt,
   fetchSitemaps,
   validateSitemap,
   normalizeUrl,
@@ -934,8 +936,34 @@ async function downloadAndRevealInstaller(asset: GitHubReleaseAsset): Promise<vo
  * Downloads folder and reveals it in Explorer / Finder; user double-
  * clicks to upgrade.
  */
+/**
+ * Re-entrancy guard for `checkForUpdates`. Prevents stacked dialogs
+ * when (a) the user clicks Help → Check for Updates while the silent
+ * startup auto-check is still resolving its GitHub API request, or
+ * (b) the user double-clicks the menu item before the first dialog
+ * paints. Cleared in the `finally` block of `checkForUpdates`.
+ */
+let updateCheckInFlight = false;
+
 async function checkForUpdates(opts: { silent?: boolean } = {}): Promise<void> {
   const silent = opts.silent === true;
+  if (updateCheckInFlight) {
+    logger.log(
+      'debug',
+      'main',
+      `checkForUpdates skipped — another check is in progress (silent=${silent}).`,
+    );
+    return;
+  }
+  updateCheckInFlight = true;
+  try {
+    await runUpdateCheck(silent);
+  } finally {
+    updateCheckInFlight = false;
+  }
+}
+
+async function runUpdateCheck(silent: boolean): Promise<void> {
   const win = mainWindow;
   if (!win) return;
   const installed = app.getVersion();
@@ -1269,6 +1297,32 @@ function openLogsWindow(): void {
 function registerIpc(): void {
   ipcMain.handle(IPC.appVersion, () => app.getVersion());
 
+  // Live memory snapshot for the in-app monitor (status bar). Returns
+  // process RSS / heap + system total/free + the active project's URL
+  // count so the renderer can derive per-URL cost + capacity ceilings
+  // (1M / 5M / 10M projection) without separate round-trips.
+  // `urlsCrawled = 0` when no project is open — the renderer suppresses
+  // the per-URL cost UI in that case.
+  ipcMain.handle(IPC.memoryStats, () => {
+    const mu = process.memoryUsage();
+    let urlsCrawled = 0;
+    try {
+      urlsCrawled = getDb().countUrls();
+    } catch {
+      // DB may be mid-truncate or closed — treat as 0 rather than throwing.
+    }
+    return {
+      rss: mu.rss,
+      heapUsed: mu.heapUsed,
+      heapTotal: mu.heapTotal,
+      external: mu.external,
+      arrayBuffers: mu.arrayBuffers,
+      systemTotal: totalmem(),
+      systemFree: freemem(),
+      urlsCrawled,
+    };
+  });
+
   // Renderer → main heartbeat carrying live input-lag (ms). Forwarded
   // to the active crawler so it can adaptively shrink its concurrency
   // when the renderer's main thread is starved. `ipcMain.on` (not
@@ -1290,6 +1344,10 @@ function registerIpc(): void {
   });
   ipcMain.handle(IPC.logsOpenWindow, () => openLogsWindow());
 
+  ipcMain.handle(IPC.robotsValidate, (_e, text: string) =>
+    validateRobotsTxt(typeof text === 'string' ? text : ''),
+  );
+
   ipcMain.handle(IPC.robotsTest, (_e, input: RobotsTestInput) =>
     testUrlAgainstRobots(input.url, input.userAgent, input.customRobots),
   );

apps/desktop/src/preload/index.ts

@@ -33,6 +33,7 @@ import {
   type MenuEvent,
   type RobotsTestInput,
   type RobotsTestResult,
+  type RobotsValidationIssue,
   type SitemapValidateInput,
   type SitemapValidateResult,
   type UrlRewritePreviewInput,
@@ -72,6 +73,7 @@ import {
   type UrlCertInfoResult,
   type UrlsQueryInput,
   type UrlsQueryResult,
+  type MemoryStats,
 } from '@freecrawl/shared-types';
 
 function subscribe<T>(
@@ -152,6 +154,7 @@ const api: FreeCrawlApi = {
   sitemapGenerate: (input: SitemapGenerateInput): Promise<SitemapGenerateResult> =>
     ipcRenderer.invoke(IPC.sitemapGenerate, input),
   appVersion: (): Promise<string> => ipcRenderer.invoke(IPC.appVersion),
+  memoryStats: (): Promise<MemoryStats> => ipcRenderer.invoke(IPC.memoryStats),
   prefsGetAll: () => ({ ...prefsCache }),
   prefsGet: (key) => prefsCache[key],
   prefsSet: (key, value) => {
@@ -168,6 +171,8 @@ const api: FreeCrawlApi = {
   logsOpenWindow: (): Promise<void> => ipcRenderer.invoke(IPC.logsOpenWindow),
   robotsTest: (input: RobotsTestInput): Promise<RobotsTestResult> =>
     ipcRenderer.invoke(IPC.robotsTest, input),
+  robotsValidate: (text: string): Promise<RobotsValidationIssue[]> =>
+    ipcRenderer.invoke(IPC.robotsValidate, text),
   sitemapValidate: (input: SitemapValidateInput): Promise<SitemapValidateResult> =>
     ipcRenderer.invoke(IPC.sitemapValidate, input),
   urlRewritePreview: (

apps/desktop/src/renderer/src/components/ExportDialog.tsx

@@ -35,6 +35,8 @@ const TAB_CATEGORY: Record<TabKey, UrlCategory> = {
   'meta-refresh': 'tab:meta-refresh',
   'custom-extraction': 'tab:custom-extraction',
   'custom-search': 'tab:custom-search',
+  security: 'tab:security',
+  duplicates: 'tab:duplicates',
   links: 'all',
   'broken-links': 'all',
 };

apps/desktop/src/renderer/src/components/OverviewSidebar.tsx

@@ -1078,6 +1078,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.renderBlockingCritical,
               category: 'issues:render-blocking-critical',
             },
+            {
+              key: 'issues-page-many-requests',
+              label: 'Too Many Requests (>100 subresources)',
+              count: o.issues.pageManyRequests,
+              category: 'issues:page-many-requests',
+            },
             {
               key: 'issues-text-code-ratio-low',
               label: 'Low Text/Code Ratio (<10%)',

apps/desktop/src/renderer/src/components/RobotsTesterDialog.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useState } from 'react';
 import { X } from 'lucide-react';
 import clsx from 'clsx';
-import type { RobotsTestResult } from '@freecrawl/shared-types';
+import type { RobotsTestResult, RobotsValidationIssue } from '@freecrawl/shared-types';
 import { useAppStore } from '../store.js';
 
 interface Props {
@@ -20,6 +20,32 @@ export function RobotsTesterDialog({ open, onClose }: Props) {
   // (probe live robots.txt) still works in one click.
   const [useCustom, setUseCustom] = useState(false);
   const [customRobots, setCustomRobots] = useState('');
+  // Live syntax-validation results for the custom robots.txt body.
+  // Recomputed in main process via `robots:validate` whenever the body
+  // changes (debounced 250 ms — local lint is cheap but pure-IPC churn
+  // on every keystroke would still be noisy).
+  const [validation, setValidation] = useState<RobotsValidationIssue[]>([]);
+
+  // Re-lint the custom-robots body whenever it changes. Validation is
+  // sync + pure, so we don't need cancellation; the debounce is only
+  // about smoothing keystrokes.
+  useEffect(() => {
+    if (!useCustom) {
+      setValidation([]);
+      return;
+    }
+    if (!customRobots.trim()) {
+      setValidation([]);
+      return;
+    }
+    const handle = window.setTimeout(() => {
+      void window.freecrawl
+        .robotsValidate(customRobots)
+        .then(setValidation)
+        .catch(() => setValidation([]));
+    }, 250);
+    return () => window.clearTimeout(handle);
+  }, [useCustom, customRobots]);
 
   // Re-seed inputs whenever the dialog opens — pre-fills the URL with the
   // current crawl's start URL (handy: most "why is this blocked?" checks
@@ -144,18 +170,21 @@ export function RobotsTesterDialog({ open, onClose }: Props) {
           </div>
 
           {useCustom && (
-            <label className="mb-3 flex flex-col gap-1">
-              <span className="text-[10px] text-surface-400">
-                Custom robots.txt body (parsed — no fetch)
-              </span>
-              <textarea
-                className="h-40 w-full resize-y rounded border border-surface-700 bg-surface-950 px-2 py-1.5 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
-                value={customRobots}
-                onChange={(e) => setCustomRobots(e.target.value)}
-                placeholder={`User-agent: *\nDisallow: /admin/\nDisallow: /private/\nSitemap: https://example.com/sitemap.xml`}
-                spellCheck={false}
-              />
-            </label>
+            <>
+              <label className="mb-3 flex flex-col gap-1">
+                <span className="text-[10px] text-surface-400">
+                  Custom robots.txt body (parsed — no fetch)
+                </span>
+                <textarea
+                  className="h-40 w-full resize-y rounded border border-surface-700 bg-surface-950 px-2 py-1.5 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                  value={customRobots}
+                  onChange={(e) => setCustomRobots(e.target.value)}
+                  placeholder={`User-agent: *\nDisallow: /admin/\nDisallow: /private/\nSitemap: https://example.com/sitemap.xml`}
+                  spellCheck={false}
+                />
+              </label>
+              <ValidationPanel issues={validation} bodyEmpty={!customRobots.trim()} />
+            </>
           )}
 
           <div className="mb-4 flex items-center gap-2">
@@ -260,3 +289,67 @@ function Stat({ label, value }: { label: string; value: string }) {
     </div>
   );
 }
+
+function ValidationPanel({
+  issues,
+  bodyEmpty,
+}: {
+  issues: RobotsValidationIssue[];
+  bodyEmpty: boolean;
+}) {
+  if (bodyEmpty) {
+    return (
+      <div className="mb-3 rounded border border-surface-800 bg-surface-950 px-2 py-1.5 text-[10px] text-surface-500">
+        Paste a draft robots.txt above to lint syntax.
+      </div>
+    );
+  }
+  if (issues.length === 0) {
+    return (
+      <div className="mb-3 rounded border border-emerald-700/60 bg-emerald-900/20 px-2 py-1.5 text-[11px] text-emerald-200">
+        ✓ No syntax issues detected.
+      </div>
+    );
+  }
+  const errors = issues.filter((i) => i.severity === 'error').length;
+  const warnings = issues.length - errors;
+  return (
+    <div className="mb-3 rounded border border-amber-700/60 bg-amber-900/15">
+      <div className="flex items-center gap-3 border-b border-amber-700/30 px-2 py-1 text-[10px] text-amber-300">
+        <span>
+          {errors > 0 && <span className="font-medium">{errors} error{errors === 1 ? '' : 's'}</span>}
+          {errors > 0 && warnings > 0 && <span> · </span>}
+          {warnings > 0 && (
+            <span className="font-medium text-amber-200/80">
+              {warnings} warning{warnings === 1 ? '' : 's'}
+            </span>
+          )}
+        </span>
+      </div>
+      <ul className="max-h-40 overflow-auto px-2 py-1 text-[10px]">
+        {issues.map((issue, idx) => (
+          <li
+            key={idx}
+            className={clsx(
+              'flex items-start gap-2 py-0.5 font-mono',
+              issue.severity === 'error' ? 'text-red-300' : 'text-amber-200/80',
+            )}
+          >
+            <span className="w-12 shrink-0 text-right text-surface-500">L{issue.line}</span>
+            <span
+              className={clsx(
+                'inline-block w-12 shrink-0 rounded px-1 text-center text-[9px] uppercase tracking-wider',
+                issue.severity === 'error'
+                  ? 'bg-red-900/40 text-red-200'
+                  : 'bg-amber-900/30 text-amber-200',
+              )}
+            >
+              {issue.severity}
+            </span>
+            <span className="text-[10px] leading-snug text-surface-200">{issue.message}</span>
+          </li>
+        ))}
+      </ul>
+    </div>
+  );
+}

apps/desktop/src/renderer/src/components/StatsBar.tsx

@@ -1,6 +1,7 @@
 import clsx from 'clsx';
 import { useAppStore } from '../store.js';
 import { usePerfMeter } from '../hooks/usePerfMeter.js';
+import { useMemoryMonitor } from '../hooks/useMemoryMonitor.js';
 
 function Stat({
   label,
@@ -44,6 +45,50 @@ function heapClass(heapMb: number | null): string {
   return 'text-surface-100';
 }
 
+/** Format bytes for the status bar. Uses MB / GB depending on
+ * magnitude — RSS routinely sits at 250-1500 MB on a busy crawl, so
+ * we render in MB until the value crosses 1 GB to keep the column
+ * width stable. */
+function formatBytes(b: number): string {
+  if (b >= 1024 * 1024 * 1024) return `${(b / (1024 * 1024 * 1024)).toFixed(2)} GB`;
+  if (b >= 1024 * 1024) return `${Math.round(b / (1024 * 1024))} MB`;
+  if (b >= 1024) return `${Math.round(b / 1024)} KB`;
+  return `${b} B`;
+}
+
+/** Compact integer (1.2K / 850K / 5.3M / 12.4B) — used for the
+ *  "Capacity" projection so a 1 234 567 estimate doesn't blow out
+ *  the StatsBar layout. */
+function formatCount(n: number): string {
+  if (!Number.isFinite(n) || n <= 0) return '—';
+  if (n >= 1_000_000_000) return `${(n / 1_000_000_000).toFixed(1)}B`;
+  if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
+  if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
+  return Math.round(n).toString();
+}
+
+/** RSS colour. Electron base + main + workers idle around 250-400 MB;
+ *  during heavy crawls 800-1500 MB is normal; > 2 GB on a 4 GB box is
+ *  a memory-leak red flag. We tier against the system total so a 32 GB
+ *  workstation isn't flagged at 1.5 GB. */
+function rssClass(rss: number, systemTotal: number): string {
+  if (systemTotal === 0) return 'text-surface-100';
+  const ratio = rss / systemTotal;
+  if (ratio >= 0.5) return 'text-red-300';
+  if (ratio >= 0.25) return 'text-amber-300';
+  return 'text-surface-100';
+}
+
+/** System-free colour. Lower than 10% remaining = OS will start
+ *  swapping; 10-25% = pause-soon territory; > 25% = fine. */
+function systemFreeClass(freeBytes: number, totalBytes: number): string {
+  if (totalBytes === 0) return 'text-surface-100';
+  const ratio = freeBytes / totalBytes;
+  if (ratio < 0.1) return 'text-red-300';
+  if (ratio < 0.25) return 'text-amber-300';
+  return 'text-emerald-300';
+}
+
 /** Input lag colour. The same numbers a user "feels":
  *   < 16 ms = one frame at 60 Hz (input feels instant)
  *   16–50 ms = a couple of frames late (subtle drag stutter)
@@ -60,10 +105,22 @@ export function StatsBar() {
   const error = useAppStore((s) => s.error);
   const setError = useAppStore((s) => s.setError);
   const perf = usePerfMeter();
+  const mem = useMemoryMonitor();
 
   const elapsed = progress?.elapsedMs ?? 0;
   const elapsedStr = formatElapsed(elapsed);
 
+  // Per-URL memory cost = RSS / urlsCrawled. Suppressed below 100 URLs
+  // because the Electron base footprint dominates the calculation and
+  // the projection becomes meaningless (e.g. one URL would imply a
+  // 400 MB/page cost). Capacity = systemFree / perUrlCost — projects
+  // how many MORE URLs the current process can fit in remaining
+  // system memory at the observed cost.
+  const perUrlCost =
+    mem && mem.urlsCrawled >= 100 ? mem.rss / mem.urlsCrawled : null;
+  const capacity =
+    perUrlCost && perUrlCost > 0 ? mem!.systemFree / perUrlCost : null;
+
   return (
     <div className="flex shrink-0 items-center gap-5 border-t border-surface-800 bg-surface-900/50 px-3 py-1.5 text-[11px]">
       <Stat label="Discovered" value={progress?.discovered ?? 0} />
@@ -93,6 +150,36 @@ export function StatsBar() {
           title="Renderer JS heap. >500 MB = warm, >1 GB = likely a listener / cache leak"
         />
       )}
+      {mem && (
+        <>
+          <Stat
+            label="RSS"
+            value={formatBytes(mem.rss)}
+            valueClassName={rssClass(mem.rss, mem.systemTotal)}
+            title={`Main-process resident set size (Electron + workers). System total: ${formatBytes(mem.systemTotal)}.`}
+          />
+          <Stat
+            label="Sys Free"
+            value={formatBytes(mem.systemFree)}
+            valueClassName={systemFreeClass(mem.systemFree, mem.systemTotal)}
+            title={`OS-reported free memory. < 10% of system total triggers swap; pause the crawl before that. System total: ${formatBytes(mem.systemTotal)}.`}
+          />
+          {perUrlCost !== null && (
+            <Stat
+              label="Per URL"
+              value={formatBytes(perUrlCost)}
+              title={`Average bytes of RSS per crawled URL (${mem.urlsCrawled.toLocaleString()} URLs). Includes Electron overhead, so the marginal cost on real-world large crawls is typically lower.`}
+            />
+          )}
+          {capacity !== null && (
+            <Stat
+              label="Capacity"
+              value={formatCount(capacity)}
+              title={`Estimated additional URLs that fit in remaining system memory at the current per-URL cost. Calc: systemFree / perUrlCost. Treat as an upper bound — sustained throughput is usually CPU-bound first.`}
+            />
+          )}
+        </>
+      )}
       <Stat
         label="Lag"
         value={`${perf.inputLagMs}ms`}

apps/desktop/src/renderer/src/hooks/useMemoryMonitor.ts

@@ -0,0 +1,41 @@
+import { useEffect, useState } from 'react';
+import type { MemoryStats } from '@freecrawl/shared-types';
+
+/**
+ * Polls the main process for live memory stats every ~2 seconds.
+ *
+ * Returns `null` until the first sample arrives so consumers can render
+ * a placeholder. Polling is deliberately slow (2 s) — RSS doesn't change
+ * faster than that meaningfully, and `process.memoryUsage()` walks every
+ * V8 heap segment, so we don't want to call it on every animation frame.
+ *
+ * The poll continues regardless of crawl state because the user's RSS
+ * curiosity outlasts the active fetch loop (e.g. wanting to confirm
+ * memory was reclaimed after Clear).
+ */
+export function useMemoryMonitor(): MemoryStats | null {
+  const [stats, setStats] = useState<MemoryStats | null>(null);
+
+  useEffect(() => {
+    let cancelled = false;
+    async function tick(): Promise<void> {
+      try {
+        const next = await window.freecrawl.memoryStats();
+        if (!cancelled) setStats(next);
+      } catch {
+        // IPC may be unavailable during app shutdown — swallow and let
+        // the next tick retry; stale state is fine for a status bar.
+      }
+    }
+    void tick();
+    const handle = window.setInterval(() => {
+      void tick();
+    }, 2000);
+    return () => {
+      cancelled = true;
+      window.clearInterval(handle);
+    };
+  }, []);
+
+  return stats;
+}

apps/desktop/src/renderer/src/store.ts

@@ -29,6 +29,8 @@ export type TabKey =
   | 'meta-refresh'
   | 'custom-extraction'
   | 'custom-search'
+  | 'security'
+  | 'duplicates'
   | 'links'
   | 'broken-links';
 
@@ -53,6 +55,8 @@ export const TAB_ORDER: { key: TabKey; label: string }[] = [
   { key: 'meta-refresh', label: 'Meta Refresh' },
   { key: 'custom-extraction', label: 'Custom Extraction' },
   { key: 'custom-search', label: 'Custom Search' },
+  { key: 'security', label: 'Security' },
+  { key: 'duplicates', label: 'Duplicates' },
   { key: 'links', label: 'Links' },
   { key: 'broken-links', label: 'Broken Links' },
 ];
@@ -237,6 +241,10 @@ function categoryForTab(tab: TabKey): UrlCategory {
       return 'tab:custom-extraction';
     case 'custom-search':
       return 'tab:custom-search';
+    case 'security':
+      return 'tab:security';
+    case 'duplicates':
+      return 'tab:duplicates';
     default:
       return 'internal:html';
   }

apps/desktop/src/renderer/src/tabs/UrlsTab.tsx

@@ -846,7 +846,15 @@ function Cell({ row, spec }: { row: CrawlUrlRow | null; spec: ColumnSpec }) {
     return <span className="text-surface-700">…</span>;
   }
   const raw = row[spec.key];
-  const value = raw === null || raw === undefined ? '' : String(raw);
+  // Booleans render as "Y" / "—" rather than the JS-default "true"/"false".
+  // Matches Screaming Frog's compact convention for flag columns
+  // (Loop, Sequence Break, Self-Ref Missing, …).
+  const value =
+    raw === null || raw === undefined
+      ? ''
+      : typeof raw === 'boolean'
+        ? raw ? 'Y' : ''
+        : String(raw);
 
   if (spec.kind === 'status') {
     const code = row.statusCode;

apps/desktop/src/renderer/src/tabs/columns.ts

@@ -277,6 +277,250 @@ const COL = {
     info: 'Number of pages in this URL\'s near-duplicate cluster (1 = no duplicates, ≥2 = part of a duplicate group). Tunable via Settings → Duplicates.',
     example: '1 = unique, 5 = part of a 5-page near-duplicate group',
   } as ColumnSpec,
+  contentHash: {
+    key: 'contentHash',
+    header: 'Content Hash',
+    size: 144,
+    kind: 'mono',
+    info: 'FNV-1a 64-bit hash of the normalised body token stream. Two pages sharing this hash are byte-identical post-tokenisation — the basis of the Exact Duplicate filter.',
+    example: '7c3e9f1a8b2d4c5e',
+  } as ColumnSpec,
+  simhash: {
+    key: 'simhash',
+    header: 'SimHash',
+    size: 144,
+    kind: 'mono',
+    info: 'Charikar 64-bit SimHash of body shingles. Used by the post-crawl near-duplicate clustering pass. Two SimHashes within the configured Hamming threshold are considered similar.',
+    example: '9a4f23bcd1e8077f',
+  } as ColumnSpec,
+  hsts: {
+    key: 'hsts',
+    header: 'HSTS',
+    size: 220,
+    kind: 'mono',
+    info: 'Raw `Strict-Transport-Security` header. Empty when missing — for HTTPS pages this is a security regression.',
+    example: 'max-age=31536000; includeSubDomains; preload',
+  } as ColumnSpec,
+  csp: {
+    key: 'csp',
+    header: 'CSP',
+    size: 220,
+    kind: 'mono',
+    info: 'Raw `Content-Security-Policy` response header. Empty when missing.',
+    example: "default-src 'self'; script-src 'self' cdn.example.com",
+  } as ColumnSpec,
+  xFrameOptions: {
+    key: 'xFrameOptions',
+    header: 'X-Frame-Options',
+    size: 140,
+    kind: 'mono',
+    info: 'Raw `X-Frame-Options` header. SAMEORIGIN / DENY / ALLOW-FROM. Clickjacking defence.',
+    example: 'SAMEORIGIN',
+  } as ColumnSpec,
+  xContentTypeOptions: {
+    key: 'xContentTypeOptions',
+    header: 'X-Content-Type-Options',
+    size: 160,
+    kind: 'mono',
+    info: 'Raw `X-Content-Type-Options` header. `nosniff` blocks MIME sniffing — prevents some XSS via content-type confusion.',
+    example: 'nosniff',
+  } as ColumnSpec,
+  mixedContentActive: {
+    key: 'mixedContentActive',
+    header: 'Mixed Active',
+    size: 100,
+    kind: 'number',
+    align: 'right',
+    info: 'HTTP `<script>` / `<link rel=stylesheet>` / `<iframe>` / `<object>` / `<embed>` references on an HTTPS page — browsers BLOCK these silently.',
+    example: '0',
+  } as ColumnSpec,
+  mixedContentPassive: {
+    key: 'mixedContentPassive',
+    header: 'Mixed Passive',
+    size: 100,
+    kind: 'number',
+    align: 'right',
+    info: 'HTTP `<img>` / `<video>` / `<audio>` / `<source>` references on an HTTPS page — rendered but the URL bar reads "Not Secure".',
+    example: '0',
+  } as ColumnSpec,
+  missingSri: {
+    key: 'missingSriCount',
+    header: 'Missing SRI',
+    size: 90,
+    kind: 'number',
+    align: 'right',
+    info: 'Third-party `<script>` / `<link rel=stylesheet>` references without an `integrity=` attribute. SRI is recommended for any cross-origin subresource.',
+    example: '0',
+  } as ColumnSpec,
+  insecureFormAction: {
+    key: 'insecureFormActionCount',
+    header: 'Insecure Form Action',
+    size: 130,
+    kind: 'number',
+    align: 'right',
+    info: 'Number of `<form action="http://…">` declarations on an HTTPS page. Submitting one downgrades the connection.',
+    example: '0',
+  } as ColumnSpec,
+  paginationNext: {
+    key: 'paginationNext',
+    header: 'rel=next',
+    size: 320,
+    kind: 'mono',
+    info: '`<link rel="next" href="…">` value resolved to absolute. Empty when the page is not paginated forward.',
+    example: 'https://example.com/blog/page/2',
+  } as ColumnSpec,
+  paginationPrev: {
+    key: 'paginationPrev',
+    header: 'rel=prev',
+    size: 320,
+    kind: 'mono',
+    info: '`<link rel="prev" href="…">` value resolved to absolute. Empty when the page is the first in its pagination cluster.',
+    example: 'https://example.com/blog/page/1',
+  } as ColumnSpec,
+  paginationSequenceBreak: {
+    key: 'paginationSequenceBreak',
+    header: 'Sequence Break',
+    size: 110,
+    kind: 'text',
+    info: 'Y when this URL belongs to a paginated cluster whose ordinal sequence has a gap (e.g. ?page=1, 2, 4 — page 3 missing). Set by the post-crawl `recomputePaginationSequence` pass.',
+    example: 'Y',
+  } as ColumnSpec,
+  hreflangCount: {
+    key: 'hreflangCount',
+    header: 'Hreflang Count',
+    size: 110,
+    kind: 'number',
+    align: 'right',
+    info: 'Number of `<link rel="alternate" hreflang>` entries declared on this page. 0 = no alternates declared.',
+    example: '8',
+  } as ColumnSpec,
+  hreflangs: {
+    key: 'hreflangs',
+    header: 'Hreflang Targets',
+    size: 360,
+    kind: 'mono',
+    info: 'JSON-stringified array of `{ lang, href }` pairs. Heavy column — better consumed via the URL Details panel.',
+    example: '[{"lang":"en","href":"…"},{"lang":"x-default","href":"…"}]',
+  } as ColumnSpec,
+  hreflangSelfRefMissing: {
+    key: 'hreflangSelfRefMissing',
+    header: 'Self-Ref Missing',
+    size: 120,
+    kind: 'text',
+    info: 'Y when the page declares hreflang alternates but no entry whose `href` matches the page URL. Google requires a self-reference.',
+    example: 'Y',
+  } as ColumnSpec,
+  hreflangTargetIssues: {
+    key: 'hreflangTargetIssues',
+    header: 'Target Issues',
+    size: 110,
+    kind: 'number',
+    align: 'right',
+    info: 'Number of hreflang targets that are non-200, noindex, or canonicalised away. Aggregated by the post-crawl pass.',
+    example: '0',
+  } as ColumnSpec,
+  amphtml: {
+    key: 'amphtml',
+    header: 'AMP HTML',
+    size: 360,
+    kind: 'mono',
+    info: '`<link rel="amphtml" href="…">` value — the AMP version of this page. Empty when the page does not declare an AMP alternate.',
+    example: 'https://example.com/amp/post-1',
+  } as ColumnSpec,
+  schemaTypes: {
+    key: 'schemaTypes',
+    header: 'Schema Types',
+    size: 280,
+    kind: 'mono',
+    info: 'Comma-joined sorted unique JSON-LD `@type` values declared on the page (Article, BreadcrumbList, Product, …).',
+    example: 'Article, BreadcrumbList, Organization',
+  } as ColumnSpec,
+  schemaBlockCount: {
+    key: 'schemaBlockCount',
+    header: 'JSON-LD Blocks',
+    size: 110,
+    kind: 'number',
+    align: 'right',
+    info: 'Number of `<script type="application/ld+json">` blocks parsed successfully on the page.',
+    example: '3',
+  } as ColumnSpec,
+  schemaInvalidCount: {
+    key: 'schemaInvalidCount',
+    header: 'Invalid JSON-LD',
+    size: 120,
+    kind: 'number',
+    align: 'right',
+    info: 'Number of `<script type="application/ld+json">` blocks that failed to parse as JSON.',
+    example: '0',
+  } as ColumnSpec,
+  schemaDuplicateIds: {
+    key: 'schemaDuplicateIds',
+    header: 'Duplicate @id',
+    size: 110,
+    kind: 'number',
+    align: 'right',
+    info: 'Surplus `@id` occurrences across all JSON-LD blocks (page declares the same `@id` more than once).',
+    example: '0',
+  } as ColumnSpec,
+  metaRefresh: {
+    key: 'metaRefresh',
+    header: 'Meta Refresh',
+    size: 220,
+    kind: 'mono',
+    info: 'Raw `content` attribute of `<meta http-equiv="refresh">`, e.g. "5; url=/foo".',
+    example: '0; url=/new-path',
+  } as ColumnSpec,
+  metaRefreshUrl: {
+    key: 'metaRefreshUrl',
+    header: 'Meta Refresh URL',
+    size: 320,
+    kind: 'mono',
+    info: 'Absolute redirect target parsed from the meta-refresh content. Empty when meta-refresh sets only a delay.',
+    example: 'https://example.com/new-path',
+  } as ColumnSpec,
+  extractionResults: {
+    key: 'extractionResults',
+    header: 'Extracted Data',
+    size: 360,
+    kind: 'mono',
+    info: 'JSON-stringified custom-extraction results map. Heavy column — render verbatim, easier to read in the URL Details panel.',
+    example: '{"price":"$19.99","sku":"AB-123"}',
+  } as ColumnSpec,
+  customSearchHits: {
+    key: 'customSearchHits',
+    header: 'Custom Search Hits',
+    size: 220,
+    kind: 'mono',
+    info: 'JSON map of `{ term: count }` literal-substring hits from the configured Custom Search terms.',
+    example: '{"pricing":3,"beta":1}',
+  } as ColumnSpec,
+  emptyAnchorCount: {
+    key: 'emptyAnchorCount',
+    header: 'Empty Anchors',
+    size: 110,
+    kind: 'number',
+    align: 'right',
+    info: 'Number of internal `<a>` elements with no usable anchor text or alt — accessibility / SEO regression.',
+    example: '0',
+  } as ColumnSpec,
+  jsOnlyLinksCount: {
+    key: 'jsOnlyLinksCount',
+    header: 'JS-Only Links',
+    size: 110,
+    kind: 'number',
+    align: 'right',
+    info: '`<a>` elements that look clickable but aren\'t crawlable (no href + onclick, `href="javascript:…"`, or `href="#"` with onclick).',
+    example: '0',
+  } as ColumnSpec,
+  canonicalCount: {
+    key: 'canonicalCount',
+    header: 'Canonical Count',
+    size: 110,
+    kind: 'number',
+    align: 'right',
+    info: 'Number of `<link rel="canonical">` tags on the page. >1 is a "Multiple Canonicals" issue.',
+    example: '1',
+  } as ColumnSpec,
 };
 
 export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
@@ -363,6 +607,7 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
   canonicals: [
     COL.url,
     COL.canonical,
+    COL.canonicalCount,
     COL.indexability,
     COL.indexabilityStatus,
     COL.status,
@@ -392,7 +637,9 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
     COL.url,
     COL.status,
     COL.indexability,
-    COL.title,
+    COL.paginationNext,
+    COL.paginationPrev,
+    COL.paginationSequenceBreak,
     COL.canonical,
     COL.inlinks,
   ],
@@ -400,29 +647,43 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
     COL.url,
     COL.status,
     COL.indexability,
+    COL.hreflangCount,
+    COL.hreflangSelfRefMissing,
+    COL.hreflangTargetIssues,
+    COL.hreflangs,
+    COL.inlinks,
+  ],
+  amp: [
+    COL.url,
+    COL.status,
+    COL.indexability,
+    COL.amphtml,
     COL.title,
-    COL.canonical,
     COL.inlinks,
   ],
-  amp: [COL.url, COL.status, COL.indexability, COL.title, COL.inlinks],
   'structured-data': [
     COL.url,
     COL.status,
     COL.indexability,
-    COL.title,
+    COL.schemaTypes,
+    COL.schemaBlockCount,
+    COL.schemaInvalidCount,
+    COL.schemaDuplicateIds,
     COL.inlinks,
   ],
   'meta-refresh': [
     COL.url,
     COL.status,
     COL.indexability,
-    COL.title,
+    COL.metaRefresh,
+    COL.metaRefreshUrl,
     COL.inlinks,
   ],
   'custom-extraction': [
     COL.url,
     COL.status,
     COL.indexability,
+    COL.extractionResults,
     COL.title,
     COL.inlinks,
   ],
@@ -430,8 +691,41 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
     COL.url,
     COL.status,
     COL.indexability,
+    COL.customSearchHits,
     COL.title,
     COL.inlinks,
   ],
-  links: [COL.url, COL.inlinks, COL.outlinks, COL.status, COL.indexability],
+  security: [
+    COL.url,
+    COL.status,
+    COL.indexability,
+    COL.hsts,
+    COL.csp,
+    COL.xFrameOptions,
+    COL.xContentTypeOptions,
+    COL.mixedContentActive,
+    COL.mixedContentPassive,
+    COL.missingSri,
+    COL.insecureFormAction,
+  ],
+  duplicates: [
+    COL.url,
+    COL.status,
+    COL.indexability,
+    COL.title,
+    COL.clusterId,
+    COL.clusterSize,
+    COL.contentHash,
+    COL.simhash,
+    COL.inlinks,
+  ],
+  links: [
+    COL.url,
+    COL.status,
+    COL.indexability,
+    COL.inlinks,
+    COL.outlinks,
+    COL.emptyAnchorCount,
+    COL.jsOnlyLinksCount,
+  ],
 };

apps/mcp-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/mcp-server",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "type": "module",
   "bin": {

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "freecrawl-seo",
-      "version": "0.3.4",
+      "version": "0.3.5",
       "license": "MIT",
       "workspaces": [
         "apps/*",
@@ -25,7 +25,7 @@
     },
     "apps/cli": {
       "name": "@freecrawl/cli",
-      "version": "0.3.4",
+      "version": "0.3.5",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -42,7 +42,7 @@
     },
     "apps/desktop": {
       "name": "@freecrawl/desktop",
-      "version": "0.3.4",
+      "version": "0.3.5",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -84,7 +84,7 @@
     },
     "apps/mcp-server": {
       "name": "@freecrawl/mcp-server",
-      "version": "0.3.4",
+      "version": "0.3.5",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*"
@@ -8732,7 +8732,7 @@
     },
     "packages/core": {
       "name": "@freecrawl/core",
-      "version": "0.3.4",
+      "version": "0.3.5",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*",
@@ -8758,7 +8758,7 @@
     },
     "packages/db": {
       "name": "@freecrawl/db",
-      "version": "0.3.4",
+      "version": "0.3.5",
       "dependencies": {
         "@freecrawl/shared-types": "*"
       },
@@ -8770,7 +8770,7 @@
     },
     "packages/shared-types": {
       "name": "@freecrawl/shared-types",
-      "version": "0.3.4",
+      "version": "0.3.5",
       "devDependencies": {
         "rimraf": "^6.0.1",
         "typescript": "^5.7.0"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/crawler.ts

@@ -2192,6 +2192,8 @@ export class Crawler extends EventEmitter {
           schemaDuplicateIds: parsed.schemaDuplicateIds,
           schemaUnknownTypes: parsed.schemaUnknownTypes,
           schemaMissingRequired: parsed.schemaMissingRequired,
+          headingOrderViolations: parsed.headingOrderViolations,
+          subresourceRequestCount: parsed.subresourceRequestCount,
           headings:
             parsed.headings.length > 0 ? JSON.stringify(parsed.headings) : null,
           serverHeader,

packages/core/src/html-parser.ts

@@ -221,6 +221,20 @@ export interface ParsedPage {
    * 200 chars per heading to keep the JSON payload small.
    */
   headings: { level: number; text: string }[];
+  /**
+   * Number of times the source-order heading sequence skips levels
+   * (e.g. h1 → h3 counts 1, h1 → h4 counts 1). Going back UP the
+   * hierarchy (h3 → h2) is allowed. WCAG 1.3.1 violation when > 0.
+   */
+  headingOrderViolations: number;
+  /**
+   * Total subresource references the page declares — `<img>` +
+   * `<script src>` + `<link rel="stylesheet">` + `<iframe>` +
+   * `<video src>` + `<audio src>`. Each entry counted once even if
+   * referenced multiple times via the same DOM node. High counts
+   * (>100) regress LCP on slower connections.
+   */
+  subresourceRequestCount: number;
   /**
    * Number of `http://` subresources (img, script, stylesheet, iframe, …)
    * referenced from a HTTPS page — i.e. mixed-content findings. Always 0
@@ -333,11 +347,21 @@ export function parseHtml(
   // page's structural hierarchy and flag skipped levels (h1 → h3 with
   // no h2 in between).
   const headings: { level: number; text: string }[] = [];
+  // Skipped-level events accumulate across the WHOLE document, not only
+  // the first 200 captured for the outline. We track via a separate
+  // counter so the outline cap doesn't suppress the issue signal on
+  // page templates that emit thousands of headings.
+  let headingOrderViolations = 0;
+  let prevHeadingLevel = 0;
   $('h1, h2, h3, h4, h5, h6').each((_, el) => {
-    if (headings.length >= 200) return;
     const tag = (el as { name?: string }).name?.toLowerCase() ?? '';
     const level = parseInt(tag.slice(1), 10);
     if (!Number.isFinite(level) || level < 1 || level > 6) return;
+    if (prevHeadingLevel > 0 && level > prevHeadingLevel + 1) {
+      headingOrderViolations++;
+    }
+    prevHeadingLevel = level;
+    if (headings.length >= 200) return;
     const raw = $(el).text().replace(/\s+/g, ' ').trim();
     const text = decodeEntities(raw).slice(0, 200);
     headings.push({ level, text });
@@ -502,6 +526,24 @@ export function parseHtml(
     renderBlockingCount++;
   });
 
+  // Total subresource request count — the headline LCP-relevant signal.
+  // Counts only nodes the browser will actually fetch: `<img>` (skips
+  // `data:` URIs because those don't hit the network), `<script src>`,
+  // `<link rel="stylesheet">` (anywhere on the page), `<iframe src>`,
+  // `<video src>`, `<audio src>`. Each DOM node counts once; we don't
+  // dedupe by URL because two `<img>` referencing the same src still
+  // generate two render reservations even if the second is cache-hit.
+  let subresourceRequestCount = 0;
+  $('img[src]').each((_, el) => {
+    const src = ($(el).attr('src') ?? '').trim();
+    if (src && !src.startsWith('data:')) subresourceRequestCount++;
+  });
+  subresourceRequestCount += $('script[src]').length;
+  subresourceRequestCount += $('link[rel="stylesheet"]').length;
+  subresourceRequestCount += $('iframe[src]').length;
+  subresourceRequestCount += $('video[src]').length;
+  subresourceRequestCount += $('audio[src]').length;
+
   // Pagination — `<link rel="next">` / `<link rel="prev">`. Resolved to
   // absolute via normalizeUrl so the values are comparable to the URLs
   // we crawl and store.
@@ -1178,6 +1220,8 @@ export function parseHtml(
     skipLinkPresent,
     ariaInvalidRolesCount,
     headings,
+    headingOrderViolations,
+    subresourceRequestCount,
     mixedContentCount,
     mixedContentActive,
     mixedContentPassive,

packages/core/src/index.ts

@@ -26,7 +26,12 @@ export {
   type TabularExportResult,
   type TabularSection,
 } from './tabular-export.js';
-export { testUrlAgainstRobots, type RobotsTestResult } from './robots.js';
+export {
+  testUrlAgainstRobots,
+  validateRobotsTxt,
+  type RobotsTestResult,
+  type RobotsValidationIssue,
+} from './robots.js';
 export {
   exportSitemap,
   validateSitemap,

packages/core/src/robots.ts

@@ -186,6 +186,226 @@ export async function testUrlAgainstRobots(
   };
 }
 
+/**
+ * One issue surfaced by `validateRobotsTxt`. Lines are 1-indexed
+ * (matching what the user sees in the editor gutter); `severity`
+ * separates "this will be ignored" (warning) from "this changes
+ * crawler behaviour" (error). `directive` carries the lowercased
+ * directive name when one was recognised, else null.
+ */
+export interface RobotsValidationIssue {
+  line: number;
+  severity: 'error' | 'warning';
+  message: string;
+  directive: string | null;
+}
+
+const KNOWN_DIRECTIVES = new Set([
+  // Core RFC 9309 directives.
+  'user-agent',
+  'disallow',
+  'allow',
+  // Widely-supported extensions Googlebot and most major crawlers honour.
+  'sitemap',
+  'crawl-delay',
+  // Yandex-specific but commonly seen — not an error, only an info-level
+  // reminder isn't useful at this granularity, so we accept silently.
+  'host',
+  'clean-param',
+  // Very common typos / case-folded: 'noindex' was historically used
+  // here but Google formally dropped it in 2019. We warn rather than
+  // error because it's not actively harmful.
+  'noindex',
+]);
+
+const TYPO_DIRECTIVES: Record<string, string> = {
+  disalow: 'disallow',
+  dissalow: 'disallow',
+  dissallow: 'disallow',
+  alow: 'allow',
+  allwo: 'allow',
+  'user-agnet': 'user-agent',
+  useragent: 'user-agent',
+  'crawldelay': 'crawl-delay',
+  sitmap: 'sitemap',
+  sitemaps: 'sitemap',
+};
+
+/**
+ * Lint a robots.txt body and return per-line issues. Pure / network-
+ * free / sync — safe to call from IPC handlers without timeout
+ * worries. Validates:
+ *   - Unknown directives (with a near-miss suggestion when one matches
+ *     a common typo).
+ *   - Missing `:` separator (e.g. `Disallow /admin` instead of
+ *     `Disallow: /admin`).
+ *   - `Disallow` / `Allow` rules appearing before any `User-agent`
+ *     group is opened — these rules are silently dropped by parsers.
+ *   - Sitemap references that are not absolute URLs (RFC 9309 §2.2.4
+ *     requires absolute URIs).
+ *   - `Crawl-delay` values that are not valid non-negative numbers.
+ *   - Wildcards (`*` / `$`) used anywhere other than path patterns
+ *     where browsers honour them — `User-agent: foo*` is fine but
+ *     `User-agent: foo*bar` is non-portable.
+ *
+ * Returns an empty array when the body is well-formed.
+ */
+export function validateRobotsTxt(text: string): RobotsValidationIssue[] {
+  const issues: RobotsValidationIssue[] = [];
+  const lines = text.split(/\r?\n/);
+  let groupOpen = false; // any `User-agent:` line seen yet?
+  for (let i = 0; i < lines.length; i++) {
+    const lineNo = i + 1;
+    const raw = lines[i] ?? '';
+    // Strip BOM on first line — some Windows editors prepend U+FEFF.
+    const noBom = i === 0 ? raw.replace(/^/, '') : raw;
+    // Strip trailing CR (handled by split, but defensive) + comments.
+    // Comments start at `#` and consume the rest of the line (including
+    // `#` mid-line); per RFC 9309 §2.2 leading whitespace is allowed.
+    const noComment = noBom.replace(/#.*$/, '');
+    const trimmed = noComment.trim();
+    if (trimmed === '') continue;
+
+    // RFC 9309 line shape: `<directive>:<value>` with optional whitespace.
+    // The colon is mandatory — without it the line is silently skipped
+    // by every major parser, so flag it.
+    const colonIdx = trimmed.indexOf(':');
+    if (colonIdx <= 0) {
+      issues.push({
+        line: lineNo,
+        severity: 'error',
+        message:
+          'Missing `:` separator. Robots.txt directives must be `<name>: <value>`.',
+        directive: null,
+      });
+      continue;
+    }
+
+    const directive = trimmed.slice(0, colonIdx).trim().toLowerCase();
+    const value = trimmed.slice(colonIdx + 1).trim();
+
+    if (directive === '') {
+      issues.push({
+        line: lineNo,
+        severity: 'error',
+        message: 'Empty directive name before `:`.',
+        directive: null,
+      });
+      continue;
+    }
+
+    // Unknown directive — but if it looks like a typo of a known one,
+    // surface the suggestion so the user can fix faster.
+    if (!KNOWN_DIRECTIVES.has(directive)) {
+      const suggestion = TYPO_DIRECTIVES[directive];
+      issues.push({
+        line: lineNo,
+        severity: 'warning',
+        message: suggestion
+          ? `Unknown directive "${directive}" — did you mean "${suggestion}"?`
+          : `Unknown directive "${directive}". Crawlers will ignore this line.`,
+        directive,
+      });
+      // Still allow the rest of the validators to run on this line so
+      // the user sees every issue, not just the first.
+    }
+
+    // Directive-specific checks.
+    switch (directive) {
+      case 'user-agent':
+        if (value === '') {
+          issues.push({
+            line: lineNo,
+            severity: 'error',
+            message: '`User-agent:` value is empty.',
+            directive,
+          });
+        } else if (/[*$].*[^*$]/.test(value) && value !== '*') {
+          // Wildcards mid-token (e.g. `foo*bar`) are non-portable.
+          issues.push({
+            line: lineNo,
+            severity: 'warning',
+            message:
+              'User-agent wildcards are only widely supported as a single `*`. Mid-token wildcards may not match.',
+            directive,
+          });
+        }
+        groupOpen = true;
+        break;
+
+      case 'disallow':
+      case 'allow':
+        if (!groupOpen) {
+          issues.push({
+            line: lineNo,
+            severity: 'error',
+            message:
+              '`Disallow` / `Allow` must come after a `User-agent:` line — orphan rules are dropped.',
+            directive,
+          });
+        }
+        // Empty `Disallow:` is the canonical "allow everything" idiom — accept.
+        break;
+
+      case 'sitemap': {
+        if (value === '') {
+          issues.push({
+            line: lineNo,
+            severity: 'error',
+            message: '`Sitemap:` value is empty.',
+            directive,
+          });
+          break;
+        }
+        try {
+          const u = new URL(value);
+          if (u.protocol !== 'http:' && u.protocol !== 'https:') {
+            issues.push({
+              line: lineNo,
+              severity: 'warning',
+              message: `Sitemap URL uses scheme "${u.protocol}" — most crawlers only accept http/https.`,
+              directive,
+            });
+          }
+        } catch {
+          issues.push({
+            line: lineNo,
+            severity: 'error',
+            message:
+              'Sitemap value must be an absolute URL (e.g. `https://example.com/sitemap.xml`).',
+            directive,
+          });
+        }
+        break;
+      }
+
+      case 'crawl-delay': {
+        const n = Number(value);
+        if (!Number.isFinite(n) || n < 0) {
+          issues.push({
+            line: lineNo,
+            severity: 'error',
+            message: '`Crawl-delay:` must be a non-negative number (seconds).',
+            directive,
+          });
+        }
+        break;
+      }
+
+      case 'noindex':
+        issues.push({
+          line: lineNo,
+          severity: 'warning',
+          message:
+            '`Noindex` in robots.txt was deprecated by Google in 2019 — use a `<meta name="robots">` or `X-Robots-Tag` header instead.',
+          directive,
+        });
+        break;
+    }
+  }
+  return issues;
+}
+
 /**
  * Coerce flexible user input into a fully-qualified URL.
  *  - `gamesatis.com`        → `https://gamesatis.com/`

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/migrations.ts

@@ -1211,6 +1211,37 @@ const MIGRATIONS: Migration[] = [
         );
     },
   },
+  {
+    version: 54,
+    name: 'add_heading_order_and_subresource_count',
+    // V1 Faz 2 closeout — final two HTML-only issue filters:
+    //   - `heading_order_violations` count of times a heading skips
+    //     levels (e.g. h1→h3 = 1, h1→h4 = 1) computed from the source-
+    //     order outline. Replaces the earlier coarse h2-missing-but-h3-
+    //     present SQL approximation in `headingSkippedLevel` with a
+    //     precise per-page count. Old rows default to 0 (no violation
+    //     counted) until they're recrawled.
+    //   - `subresource_request_count` total number of fetched
+    //     subresources the page declares: `<img>` + `<script src>` +
+    //     `<link rel="stylesheet">` + `<iframe>` + `<video src>` +
+    //     `<audio src>`. Powers the "Too Many Requests" issue filter
+    //     (>100 typical threshold) — high request count is a known LCP
+    //     regression on slower connections.
+    up: (db) => {
+      const cols = db.prepare('PRAGMA table_info(urls)').all() as unknown as {
+        name: string;
+      }[];
+      const has = (n: string) => cols.some((c) => c.name === n);
+      if (!has('heading_order_violations'))
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN heading_order_violations INTEGER NOT NULL DEFAULT 0',
+        );
+      if (!has('subresource_request_count'))
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN subresource_request_count INTEGER NOT NULL DEFAULT 0',
+        );
+    },
+  },
 ];
 
 export function runMigrations(db: DatabaseSync): void {

packages/db/src/project-db.ts

@@ -165,6 +165,8 @@ interface UrlRowDb {
   schema_duplicate_ids: number;
   schema_unknown_types: number;
   schema_missing_required: number;
+  heading_order_violations: number;
+  subresource_request_count: number;
 }
 
 interface ImageRowDb {
@@ -292,6 +294,10 @@ export interface UpsertUrlInput {
   schemaDuplicateIds?: number;
   schemaUnknownTypes?: number;
   schemaMissingRequired?: number;
+  /** Skipped-level heading events on this page (h1→h3 etc.). */
+  headingOrderViolations?: number;
+  /** Total subresource requests declared (img/script/stylesheet/iframe/video/audio). */
+  subresourceRequestCount?: number;
   /** JSON-stringified outline array, or null. */
   headings?: string | null;
   /** Raw `Server` response header, or null when absent. */
@@ -358,7 +364,8 @@ const UPSERT_URL_SQL = `
     images_responsive, picture_count,
     url_malformed, og_type, og_url, og_site_name, og_locale, android_icon,
     landmark_main, skip_link_present, aria_invalid_roles,
-    schema_duplicate_ids, schema_unknown_types, schema_missing_required
+    schema_duplicate_ids, schema_unknown_types, schema_missing_required,
+    heading_order_violations, subresource_request_count
   ) VALUES (
     :url, :content_kind, :status_code, :status_text, :indexability, :indexability_reason,
     :title, :title_length, :meta_description, :meta_description_length,
@@ -396,7 +403,8 @@ const UPSERT_URL_SQL = `
     :images_responsive, :picture_count,
     :url_malformed, :og_type, :og_url, :og_site_name, :og_locale, :android_icon,
     :landmark_main, :skip_link_present, :aria_invalid_roles,
-    :schema_duplicate_ids, :schema_unknown_types, :schema_missing_required
+    :schema_duplicate_ids, :schema_unknown_types, :schema_missing_required,
+    :heading_order_violations, :subresource_request_count
   )
   ON CONFLICT(url) DO UPDATE SET
     content_kind = excluded.content_kind,
@@ -523,6 +531,8 @@ const UPSERT_URL_SQL = `
     schema_duplicate_ids = excluded.schema_duplicate_ids,
     schema_unknown_types = excluded.schema_unknown_types,
     schema_missing_required = excluded.schema_missing_required,
+    heading_order_violations = excluded.heading_order_violations,
+    subresource_request_count = excluded.subresource_request_count,
     crawled_at = CURRENT_TIMESTAMP
   RETURNING id
 `;
@@ -1195,6 +1205,8 @@ export class ProjectDb {
       schema_duplicate_ids: input.schemaDuplicateIds ?? 0,
       schema_unknown_types: input.schemaUnknownTypes ?? 0,
       schema_missing_required: input.schemaMissingRequired ?? 0,
+      heading_order_violations: input.headingOrderViolations ?? 0,
+      subresource_request_count: input.subresourceRequestCount ?? 0,
     };
 
     const row = this.stmtUpsertUrl.get(params) as { id: number } | undefined;
@@ -2674,12 +2686,7 @@ export class ProjectDb {
       h1Duplicate: dup('h1'),
       h1Multiple: countWhere(`${html} AND h1_count > 1`),
       headingSkippedLevel: countWhere(
-        `${html} AND (
-           (h2_count = 0 AND h3_count > 0)
-           OR (h3_count = 0 AND h4_count > 0)
-           OR (h4_count = 0 AND h5_count > 0)
-           OR (h5_count = 0 AND h6_count > 0)
-         )`,
+        `${html} AND heading_order_violations > 0`,
       ),
       multipleCanonicals: countWhere(`${html} AND canonical_count > 1`),
       canonicalMissing: countWhere(
@@ -3212,6 +3219,9 @@ export class ProjectDb {
       hreflangInconsistentLang: countWhere(
         `${html} AND hreflang_inconsistent_lang = 1`,
       ),
+      pageManyRequests: countWhere(
+        `${html} AND subresource_request_count > 100`,
+      ),
     };
   }
 
@@ -4599,6 +4609,8 @@ export class ProjectDb {
     schemaDuplicateIds: r.schema_duplicate_ids ?? 0,
     schemaUnknownTypes: r.schema_unknown_types ?? 0,
     schemaMissingRequired: r.schema_missing_required ?? 0,
+    headingOrderViolations: r.heading_order_violations ?? 0,
+    subresourceRequestCount: r.subresource_request_count ?? 0,
     crawledAt: r.crawled_at,
   });
 
@@ -5016,15 +5028,11 @@ function categoryWhereClause(cat: UrlCategory): string | null {
     case 'issues:h1-multiple':
       return "is_external = 0 AND content_kind = 'html' AND h1_count > 1";
     case 'issues:heading-skipped-level':
-      // A "skipped" heading level is when level N+ exists but level N
-      // is missing — the page jumps over a tier (e.g. H1 → H3 with no
-      // H2). Each rung is checked independently.
-      return `is_external = 0 AND content_kind = 'html' AND (
-                (h2_count = 0 AND h3_count > 0)
-                OR (h3_count = 0 AND h4_count > 0)
-                OR (h4_count = 0 AND h5_count > 0)
-                OR (h5_count = 0 AND h6_count > 0)
-              )`;
+      // Skipped-level heading detection — counts source-order events
+      // where two consecutive headings differ by more than one tier
+      // (e.g. h1 → h3, h2 → h5). Computed during HTML parse and
+      // stored as `heading_order_violations`.
+      return `is_external = 0 AND content_kind = 'html' AND heading_order_violations > 0`;
     case 'issues:multiple-canonicals':
       // More than one `<link rel="canonical">` on a page is a confusion
       // signal — Google may pick any of them, defeating the canonical's
@@ -6045,6 +6053,14 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       // token mapped to two different hrefs on the same page.
       return `is_external = 0 AND content_kind = 'html'
               AND hreflang_inconsistent_lang = 1`;
+    case 'issues:page-many-requests':
+      // V1 Faz 2 closeout — pages whose total subresource count
+      // (img + script + stylesheet + iframe + video + audio) exceeds
+      // 100. High request count is a known LCP regression on slower
+      // connections and a typical consequence of unbounded third-
+      // party tag injection.
+      return `is_external = 0 AND content_kind = 'html'
+              AND subresource_request_count > 100`;
     case 'tab:directives':
       // Wave 4 — Directives tab. Pages declaring any indexability
       // directive: meta robots, X-Robots-Tag header, robots.txt block,
@@ -6115,6 +6131,33 @@ function categoryWhereClause(cat: UrlCategory): string | null {
               AND custom_search_hits IS NOT NULL
               AND custom_search_hits != ''
               AND custom_search_hits != '{}'`;
+    case 'tab:security':
+      // V1 Faz 3 — Security tab. Every internal HTML page so the user
+      // can audit security posture wholesale (HSTS, CSP, X-Frame-
+      // Options, X-Content-Type-Options, mixed content active/passive,
+      // missing-SRI, insecure form actions). The COL spec surfaces
+      // these as the visible columns; the row set itself is the same
+      // universe as Internal HTML.
+      return `is_external = 0 AND content_kind = 'html'`;
+    case 'tab:duplicates':
+      // V1 Faz 3 — Duplicates tab. Pages that participate in either
+      // a near-duplicate cluster (cluster_size > 1, post-crawl SimHash
+      // pass) or share a content_hash with another page (exact
+      // duplicate). Both signals come from the existing SimHash + FNV
+      // recompute pass, no new column required.
+      return `is_external = 0 AND content_kind = 'html'
+              AND (
+                cluster_size > 1
+                OR (
+                  content_hash IS NOT NULL AND content_hash != ''
+                  AND content_hash IN (
+                    SELECT content_hash FROM urls
+                    WHERE is_external = 0 AND content_kind = 'html'
+                      AND content_hash IS NOT NULL AND content_hash != ''
+                    GROUP BY content_hash HAVING COUNT(*) > 1
+                  )
+                )
+              )`;
     default:
       return null;
   }

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/crawl.ts

@@ -203,7 +203,10 @@ export type UrlCategory =
   | 'tab:meta-refresh'
   | 'tab:custom-extraction'
   | 'tab:custom-search'
+  | 'tab:security'
+  | 'tab:duplicates'
   | 'issues:hreflang-inconsistent-lang'
+  | 'issues:page-many-requests'
 
 export type Indexability =
   | 'indexable'
@@ -317,6 +320,18 @@ export interface CrawlUrlRow {
    * headings. Capped at 200 entries server-side.
    */
   headings: string | null;
+  /**
+   * Number of times the heading sequence skips a level (e.g. h1 → h3
+   * counts 1, h1 → h4 counts 1). 0 = well-formed outline. Computed
+   * across the entire page, independent of the 200-entry outline cap.
+   */
+  headingOrderViolations: number;
+  /**
+   * Total subresource references the page declares — `<img>` +
+   * `<script src>` + `<link rel="stylesheet">` + `<iframe>` +
+   * `<video src>` + `<audio src>`. Each DOM node counted once.
+   */
+  subresourceRequestCount: number;
   /**
    * Raw `Server` response header (e.g. `"nginx/1.25.0"`, `"cloudflare"`,
    * `"Apache/2.4.41 (Ubuntu)"`). Useful for stack auditing — surfaces
@@ -1245,6 +1260,13 @@ export interface OverviewCounts {
      * which writes a boolean flag onto each affected URL.
      */
     hreflangInconsistentLang: number;
+    /**
+     * Pages whose total subresource count (img + script + stylesheet +
+     * iframe + video + audio) exceeds 100 — a known LCP regression on
+     * slower connections and a common consequence of unbounded third-
+     * party tag injection.
+     */
+    pageManyRequests: number;
   };
 }
 

packages/shared-types/src/ipc.ts

@@ -66,6 +66,9 @@ export const IPC = {
   menuEvent: 'menu:event',
   dataChanged: 'data:changed',
   appVersion: 'app:version',
+  /** Returns live process + system memory stats for the in-app
+   * memory monitor (status bar). One-shot pull; renderer polls. */
+  memoryStats: 'app:memory-stats',
   prefsGetAllSync: 'prefs:get-all-sync',
   prefsSet: 'prefs:set',
   prefsDelete: 'prefs:delete',
@@ -88,6 +91,7 @@ export const IPC = {
    * drag. */
   logsBusy: 'logs:busy',
   robotsTest: 'robots:test',
+  robotsValidate: 'robots:validate',
   sitemapValidate: 'sitemap:validate',
   urlRewritePreview: 'url:rewrite-preview',
   urlClusterMembers: 'urls:cluster-members',
@@ -263,6 +267,33 @@ export interface CrashRecoveryStatus {
   seedUrl: string;
 }
 
+/**
+ * Snapshot of process + system memory used by the in-app memory
+ * monitor. All values in bytes; the renderer formats for display.
+ *
+ * `urlsCrawled` lets the monitor compute the per-URL cost
+ * (`rss / urlsCrawled`) and the projected ceiling for 1M / 5M / 10M
+ * URLs without a separate IPC.
+ */
+export interface MemoryStats {
+  /** `process.memoryUsage().rss` — resident set size in bytes. */
+  rss: number;
+  /** V8 heap currently in use. */
+  heapUsed: number;
+  /** V8 heap total reservation (committed). */
+  heapTotal: number;
+  /** Off-heap native allocations (Buffer, n-api, etc.). */
+  external: number;
+  /** ArrayBuffer-backed allocations counted toward `external`. */
+  arrayBuffers: number;
+  /** `os.totalmem()` — installed system memory in bytes. */
+  systemTotal: number;
+  /** `os.freemem()` — currently free system memory in bytes. */
+  systemFree: number;
+  /** Number of URLs in the active project DB (used for per-URL cost). */
+  urlsCrawled: number;
+}
+
 export interface DataDeleteByDomainResult {
   /** Number of `urls` rows deleted. Cascade handles `links`, `images`,
    *  `headers`, `url_sources`, `urls_issues`. */
@@ -482,6 +513,19 @@ export interface RobotsTestInput {
   customRobots?: string;
 }
 
+/**
+ * One issue surfaced by `robots:validate`. Lines are 1-indexed
+ * (matching what the user sees in the editor gutter). Re-exported
+ * from `@freecrawl/core` so the renderer can type the IPC result
+ * without depending on core directly.
+ */
+export interface RobotsValidationIssue {
+  line: number;
+  severity: 'error' | 'warning';
+  message: string;
+  directive: string | null;
+}
+
 export interface PagesPerDirectoryInput {
   /** Path-segment depth to group at (1 = top-level only). Default 1. */
   depth?: number;
@@ -780,6 +824,7 @@ export interface FreeCrawlApi {
   topAnchorTexts(limit?: number): Promise<AnchorTextRow[]>;
   sitemapGenerate(input: SitemapGenerateInput): Promise<SitemapGenerateResult>;
   appVersion(): Promise<string>;
+  memoryStats(): Promise<MemoryStats>;
   prefsGetAll(): Record<string, unknown>;
   prefsGet(key: string): unknown;
   prefsSet(key: string, value: unknown): void;
@@ -789,6 +834,7 @@ export interface FreeCrawlApi {
   logsClear(): Promise<void>;
   logsOpenWindow(): Promise<void>;
   robotsTest(input: RobotsTestInput): Promise<RobotsTestResult>;
+  robotsValidate(text: string): Promise<RobotsValidationIssue[]>;
   sitemapValidate(input: SitemapValidateInput): Promise<SitemapValidateResult>;
   urlRewritePreview(input: UrlRewritePreviewInput): Promise<UrlRewritePreviewResult>;
   urlClusterMembers(urlId: number): Promise<UrlClusterMember[]>;