feat: readability + CORS audit + URL rewriting + duplicates view

README.md

@@ -17,7 +17,7 @@
 
 <br />
 
-A high-performance website crawler built for serious SEO audits. Targets <kbd>1M+ URLs</kbd> on a single machine, with a dense Screaming Frog–style UI, ~70 SEO issue checks, and zero native dependencies.
+A high-performance website crawler built for serious SEO audits. Targets <kbd>1M+ URLs</kbd> on a single machine, with a dense Screaming Frog–style UI, ~80 SEO issue checks, and zero native dependencies.
 
 </div>
 
@@ -43,12 +43,15 @@ A high-performance website crawler built for serious SEO audits. Targets <kbd>1M
 <td width="50%" valign="top">
 
 ### 🔍 Analysis
-- **~70 SEO issue checks** across 14 tabs
+- **~80 SEO issue checks** across 14 tabs
 - Near-duplicate clustering (SimHash + LSH)
 - Full hreflang validation, sitemap diff
 - OpenGraph / Twitter Card / JSON-LD / AMP
-- SSL/TLS cert audit, security headers
+- SSL/TLS cert audit, security headers, **CORS audit**
+- **Active vs passive mixed content** split
+- Readability (Flesch, Flesch–Kincaid, Gunning Fog)
 - Custom CSS + Regex extraction (10 rules)
+- **Custom search** with `/regex/` or literal mode
 
 </td>
 </tr>
@@ -59,7 +62,9 @@ A high-performance website crawler built for serious SEO audits. Targets <kbd>1M
 - Dense dark theme, virtualized 1M+ row tables
 - Live-streaming rows during crawl
 - Advanced AND/OR search (24 fields × 12 ops)
-- Detail panel with 9 sub-tabs per URL
+- Detail panel with **13 sub-tabs** per URL
+- Per-URL **Duplicates** view (cluster siblings)
+- URL rewriting (regex + whitelist + live preview)
 - Cytoscape graph + anchor-text word cloud
 - Diagnostic popups for DNS/TLS/proxy issues
 
@@ -67,11 +72,13 @@ A high-performance website crawler built for serious SEO audits. Targets <kbd>1M
 <td valign="top">
 
 ### 📤 Export & Reports
-- 16 reports (histograms, top URLs, link positions)
+- **20 reports** (histograms, top/bottom URLs, link positions)
+- **Export Dialog** with column picker — CSV or **XLSX** (multi-sheet)
 - Bulk export (22 categorised CSVs)
 - Standalone HTML audit report
 - Sitemap generator (image / hreflang / sharded / gz)
 - Project-vs-project compare diff
+- **MCP server** for AI agents (Claude / any MCP client)
 - Webhook on completion + OS notifications
 
 </td>
@@ -394,7 +401,7 @@ graph LR
 ## 📈 Status
 
 > [!NOTE]
-> **Active development.** All 14 analysis tabs, advanced search, ~70 issue categories, sitemap export variants, JSON / CSV / HTML reports, list mode, custom extraction, near-duplicate detection, hreflang validation, project compare, Cytoscape visualization, auth, proxy, webhook, OS notifications, robots.txt tester, in-app logs, and diagnostic popups are working. Live-streaming UX with **first row in ~1 s**.
+> **Active development.** All 14 analysis tabs, advanced search, ~80 issue categories, sitemap export variants, JSON / CSV / XLSX / HTML reports, list mode, custom extraction, near-duplicate detection (per-URL Duplicates view), hreflang validation, project compare, Cytoscape visualization, auth, proxy, webhook, MCP server, OS notifications, robots.txt tester, URL rewriting + preview, in-app logs, and diagnostic popups are working. Live-streaming UX with **first row in ~1 s**.
 >
 > **Upcoming:** plugin system, JavaScript rendering, log analyzer, PageSpeed API integration.
 

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "private": true,
   "type": "module",
   "bin": {

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "main": "./out/main/index.js",

apps/desktop/src/main/index.ts

@@ -43,6 +43,8 @@ import {
   type RobotsTestInput,
   type SitemapValidateInput,
   type SitemapValidateResult,
+  type UrlRewritePreviewInput,
+  type UrlRewritePreviewResult,
   type TopUrlsInput,
   type TopUrlsRow,
   type ExternalDomainHealthRow,
@@ -90,6 +92,8 @@ import {
   testUrlAgainstRobots,
   fetchSitemaps,
   validateSitemap,
+  normalizeUrl,
+  compileUrlRegexRewrites,
 } from '@freecrawl/core';
 import { ProjectDb } from '@freecrawl/db';
 import { buildAppMenu } from './menu.js';
@@ -853,6 +857,32 @@ function registerIpc(): void {
     testUrlAgainstRobots(input.url, input.userAgent, input.customRobots),
   );
 
+  ipcMain.handle(
+    IPC.urlRewritePreview,
+    (_e, input: UrlRewritePreviewInput): UrlRewritePreviewResult => {
+      const errors: Array<{ pattern: string; error: string }> = [];
+      const compiled = compileUrlRegexRewrites(input.urlRegexRewrites, (p, err) =>
+        errors.push({ pattern: p, error: err }),
+      );
+      let result: string | null = null;
+      let parseError: string | undefined;
+      try {
+        result = normalizeUrl(input.url, undefined, {
+          stripWww: input.stripWww,
+          forceHttps: input.forceHttps,
+          lowercasePath: input.lowercasePath,
+          trailingSlash: input.trailingSlash,
+          keepQueryParams: input.keepQueryParams,
+          regexRewrites: compiled,
+        });
+        if (result === null) parseError = 'URL failed to parse or rewrite produced an invalid URL';
+      } catch (err) {
+        parseError = err instanceof Error ? err.message : String(err);
+      }
+      return { result, regexErrors: errors, parseError };
+    },
+  );
+
   ipcMain.handle(
     IPC.sitemapValidate,
     async (_e, input: SitemapValidateInput): Promise<SitemapValidateResult> => {
@@ -937,10 +967,11 @@ function registerIpc(): void {
               : input.metric === 'depth'
                 ? 'depth'
                 : 'content_length';
+      const direction = input.direction === 'asc' ? 'asc' : 'desc';
       return callReaderOrFallback<TopUrlsRow[]>(
         'topUrlsBy',
-        [column, limit],
-        () => getDb().topUrlsBy(column, limit),
+        [column, limit, direction],
+        () => getDb().topUrlsBy(column, limit, direction),
       );
     },
   );
@@ -1647,6 +1678,10 @@ function registerIpc(): void {
     },
   );
 
+  ipcMain.handle(IPC.urlClusterMembers, (_e, urlId: number) =>
+    getDb().urlClusterMembers(urlId),
+  );
+
   ipcMain.handle(
     IPC.urlCertInfo,
     (_e, input: UrlCertInfoInput): UrlCertInfoResult => {

apps/desktop/src/preload/index.ts

@@ -35,6 +35,9 @@ import {
   type RobotsTestResult,
   type SitemapValidateInput,
   type SitemapValidateResult,
+  type UrlRewritePreviewInput,
+  type UrlRewritePreviewResult,
+  type UrlClusterMember,
   type PagesPerDirectoryInput,
   type PagesPerDirectoryRow,
   type StatusCodeHistogramRow,
@@ -167,6 +170,11 @@ const api: FreeCrawlApi = {
     ipcRenderer.invoke(IPC.robotsTest, input),
   sitemapValidate: (input: SitemapValidateInput): Promise<SitemapValidateResult> =>
     ipcRenderer.invoke(IPC.sitemapValidate, input),
+  urlRewritePreview: (
+    input: UrlRewritePreviewInput,
+  ): Promise<UrlRewritePreviewResult> => ipcRenderer.invoke(IPC.urlRewritePreview, input),
+  urlClusterMembers: (urlId: number): Promise<UrlClusterMember[]> =>
+    ipcRenderer.invoke(IPC.urlClusterMembers, urlId),
   reportsPagesPerDirectory: (
     input: PagesPerDirectoryInput,
   ): Promise<PagesPerDirectoryRow[]> => ipcRenderer.invoke(IPC.reportsPagesPerDirectory, input),

apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx

@@ -7,6 +7,7 @@ import type {
   LinkPosition,
   LinkType,
   UrlCertInfoResult,
+  UrlClusterMember,
   UrlDetail,
   UrlPageImageRow,
   UrlSourceResult,
@@ -25,7 +26,8 @@ type SubTab =
   | 'http-headers'
   | 'cookies'
   | 'structured-data'
-  | 'view-source';
+  | 'view-source'
+  | 'duplicates';
 
 const SUB_TABS: { key: SubTab; label: string; disabled?: boolean }[] = [
   { key: 'url-details', label: 'URL Details' },
@@ -40,6 +42,7 @@ const SUB_TABS: { key: SubTab; label: string; disabled?: boolean }[] = [
   { key: 'cookies', label: 'Cookies' },
   { key: 'structured-data', label: 'Structured Data' },
   { key: 'view-source', label: 'View Source' },
+  { key: 'duplicates', label: 'Duplicates' },
 ];
 
 // Maximum number of URLs we aggregate over in one go. Anything larger is
@@ -59,6 +62,7 @@ const SINGLE_URL_ONLY_TABS: ReadonlySet<SubTab> = new Set([
   'cookies',
   'structured-data',
   'view-source',
+  'duplicates',
 ]);
 
 export function BottomDetailPanel() {
@@ -389,6 +393,9 @@ export function BottomDetailPanel() {
         {detail && subTab === 'view-source' && (
           <ViewSourceView urlId={detail.row.id} pageUrl={detail.row.url} />
         )}
+        {detail && subTab === 'duplicates' && (
+          <DuplicatesView urlId={detail.row.id} row={detail.row} />
+        )}
       </div>
     </div>
   );
@@ -530,6 +537,113 @@ function ViewSourceView({
   );
 }
 
+function DuplicatesView({ urlId, row }: { urlId: number; row: CrawlUrlRow }) {
+  const [members, setMembers] = useState<UrlClusterMember[] | null>(null);
+  const [loading, setLoading] = useState(false);
+
+  useEffect(() => {
+    let cancelled = false;
+    setLoading(true);
+    setMembers(null);
+    window.freecrawl
+      .urlClusterMembers(urlId)
+      .then((rows) => {
+        if (!cancelled) setMembers(rows);
+      })
+      .finally(() => {
+        if (!cancelled) setLoading(false);
+      });
+    return () => {
+      cancelled = true;
+    };
+  }, [urlId]);
+
+  if (loading) {
+    return <div className="p-3 text-[11px] text-surface-400">Loading…</div>;
+  }
+  if (row.clusterId === 0 || (row.clusterSize ?? 1) <= 1) {
+    return (
+      <div className="p-3 text-[11px] text-surface-500">
+        This URL is not part of a near-duplicate cluster. Run Crawl Analysis on
+        a finished crawl, or lower the Hamming threshold in Settings →
+        Duplicates, to widen what counts as a near-duplicate.
+      </div>
+    );
+  }
+  if (!members || members.length === 0) {
+    return (
+      <div className="p-3 text-[11px] text-surface-500">
+        No other members in this cluster. (Cluster size {row.clusterSize}.)
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex h-full flex-col">
+      <div className="flex items-center gap-3 border-b border-surface-800 bg-surface-900 px-3 py-1.5 text-[10px] text-surface-400">
+        <span>
+          Cluster <span className="text-surface-200">#{row.clusterId}</span>
+        </span>
+        <span>
+          {members.length + 1} member{members.length + 1 === 1 ? '' : 's'} total
+        </span>
+        <span className="text-surface-500">
+          (Hamming distance from this URL — lower means more similar)
+        </span>
+      </div>
+      <div className="flex-1 overflow-auto">
+        <table className="w-full text-[11px]">
+          <thead className="sticky top-0 bg-surface-900 text-left text-[10px] text-surface-400">
+            <tr>
+              <th className="px-3 py-1 font-normal">Hamming</th>
+              <th className="px-3 py-1 font-normal">URL</th>
+              <th className="px-3 py-1 font-normal">Status</th>
+              <th className="px-3 py-1 font-normal">Indexability</th>
+              <th className="px-3 py-1 font-normal">Words</th>
+              <th className="px-3 py-1 font-normal">Inlinks</th>
+              <th className="px-3 py-1 font-normal">Title</th>
+            </tr>
+          </thead>
+          <tbody>
+            {members.map((m) => (
+              <tr
+                key={m.url}
+                className="border-t border-surface-800 hover:bg-surface-900"
+              >
+                <td className="px-3 py-1 font-mono text-emerald-300">
+                  {m.hammingDistance}
+                </td>
+                <td
+                  className="px-3 py-1 text-surface-200"
+                  title={m.url}
+                >
+                  <a
+                    href={m.url}
+                    onClick={(e) => {
+                      e.preventDefault();
+                      void window.open(m.url, '_blank');
+                    }}
+                    className="hover:text-blue-400 hover:underline"
+                  >
+                    {m.url}
+                  </a>
+                </td>
+                <td className="px-3 py-1 text-surface-300">{m.statusCode ?? '—'}</td>
+                <td className="px-3 py-1 text-surface-300">{m.indexability}</td>
+                <td className="px-3 py-1 text-surface-300">{m.wordCount ?? '—'}</td>
+                <td className="px-3 py-1 text-surface-300">{m.inlinks}</td>
+                <td className="px-3 py-1 text-surface-400 truncate max-w-[280px]" title={m.title ?? ''}>
+                  {m.title ?? '—'}
+                </td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </div>
+    </div>
+  );
+}
+
 function countMatches(haystack: string, needle: string): number {
   if (!needle) return 0;
   const h = haystack.toLowerCase();
@@ -609,6 +723,16 @@ function HttpHeadersView({ headers }: { headers: { name: string; value: string }
   );
 }
 
+function fleschBand(score: number): string {
+  if (score >= 90) return 'very easy';
+  if (score >= 80) return 'easy';
+  if (score >= 70) return 'fairly easy';
+  if (score >= 60) return 'standard';
+  if (score >= 50) return 'fairly difficult';
+  if (score >= 30) return 'difficult';
+  return 'very difficult';
+}
+
 function NameValueView({ row }: { row: CrawlUrlRow }) {
   // Server-side pixel-width is the source of truth (drives the issue
   // filters); fall back to the renderer estimate only when the column is
@@ -692,6 +816,22 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
     ['H5 Count', row.h5Count > 0 ? row.h5Count : null],
     ['H6 Count', row.h6Count > 0 ? row.h6Count : null],
     ['Word Count', row.wordCount],
+    ['Sentence Count', row.sentenceCount > 0 ? row.sentenceCount : null],
+    [
+      'Flesch Reading Ease',
+      row.fleschReadingEase !== null
+        ? `${row.fleschReadingEase} (${fleschBand(row.fleschReadingEase)})`
+        : null,
+    ],
+    [
+      'Flesch–Kincaid Grade',
+      row.fleschKincaidGrade !== null ? row.fleschKincaidGrade : null,
+    ],
+    [
+      'Gunning Fog Index',
+      row.gunningFogIndex !== null ? row.gunningFogIndex : null,
+    ],
+    ['Complex Words', row.complexWordCount > 0 ? row.complexWordCount : null],
     ['Canonical Link Element 1', row.canonical],
     ['Canonical Count', row.canonicalCount > 1 ? row.canonicalCount : null],
     ['Canonical HTTP Header', row.canonicalHttp],
@@ -726,6 +866,17 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
     ['Content-Security-Policy', row.csp],
     ['Referrer-Policy', row.referrerPolicy],
     ['Permissions-Policy', row.permissionsPolicy],
+    ['Access-Control-Allow-Origin', row.corsAllowOrigin],
+    [
+      'Access-Control-Allow-Credentials',
+      row.corsAllowCredentials === -1
+        ? null
+        : row.corsAllowCredentials === 1
+          ? 'true'
+          : 'false',
+    ],
+    ['Access-Control-Allow-Methods', row.corsAllowMethods],
+    ['Access-Control-Allow-Headers', row.corsAllowHeaders],
     ['Content-Encoding', row.contentEncoding],
     ['Analytics Tags', summarizeAnalyticsTrackers(row.analyticsTrackers)],
     ['Schema Types', row.schemaTypes],
@@ -765,6 +916,14 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
     ],
     ['Empty Anchor Links', row.emptyAnchorCount > 0 ? row.emptyAnchorCount : null],
     ['Mixed Content (subresources)', row.mixedContentCount > 0 ? row.mixedContentCount : null],
+    [
+      'Mixed Content Active (blocked)',
+      row.mixedContentActive > 0 ? row.mixedContentActive : null,
+    ],
+    [
+      'Mixed Content Passive (warning)',
+      row.mixedContentPassive > 0 ? row.mixedContentPassive : null,
+    ],
     [
       'Redirect Chain Length',
       row.redirectChainLength > 0 ? row.redirectChainLength : null,
@@ -1665,17 +1824,22 @@ function extractionRows(
 
 function customSearchRows(
   json: string | null,
-): [string, number | null][] {
+): [string, number | string | null][] {
   if (!json) return [];
   try {
     const obj = JSON.parse(json) as Record<string, unknown>;
     if (!obj || typeof obj !== 'object') return [];
-    const out: [string, number | null][] = [];
+    const out: [string, number | string | null][] = [];
     for (const [term, raw] of Object.entries(obj)) {
       const count = typeof raw === 'number' ? raw : null;
-      // null-render small zeros to keep the panel compact — users care
-      // about hits, not absences.
-      out.push([`Custom: "${term}"`, count && count > 0 ? count : null]);
+      // -1 is a sentinel for "regex pattern failed to compile" — surface
+      // it explicitly so the user can fix the typo. 0 is hidden to keep
+      // the panel compact (users care about hits, not absences).
+      if (count === -1) {
+        out.push([`Custom: "${term}"`, 'invalid regex']);
+      } else {
+        out.push([`Custom: "${term}"`, count && count > 0 ? count : null]);
+      }
     }
     return out;
   } catch {

apps/desktop/src/renderer/src/components/OverviewSidebar.tsx

@@ -616,6 +616,18 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.duplicateContentExact,
               category: 'issues:duplicate-content-exact',
             },
+            {
+              key: 'issues-flesch-very-difficult',
+              label: 'Hard to Read (Flesch <30)',
+              count: o.issues.fleschVeryDifficult,
+              category: 'issues:flesch-very-difficult',
+            },
+            {
+              key: 'issues-gunning-fog-very-high',
+              label: 'Gunning Fog >17',
+              count: o.issues.gunningFogVeryHigh,
+              category: 'issues:gunning-fog-very-high',
+            },
           ],
         },
         {
@@ -894,10 +906,22 @@ function buildTree(o: OverviewCounts | null): Node[] {
             },
             {
               key: 'issues-mixed-content',
-              label: 'Mixed Content',
+              label: 'Mixed Content (any)',
               count: o.issues.mixedContent,
               category: 'issues:mixed-content',
             },
+            {
+              key: 'issues-mixed-content-active',
+              label: 'Mixed Content Active (Blocked)',
+              count: o.issues.mixedContentActive,
+              category: 'issues:mixed-content-active',
+            },
+            {
+              key: 'issues-mixed-content-passive',
+              label: 'Mixed Content Passive (Warning)',
+              count: o.issues.mixedContentPassive,
+              category: 'issues:mixed-content-passive',
+            },
             {
               key: 'issues-insecure-form-action',
               label: 'Insecure Form Action',
@@ -910,6 +934,24 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.missingSri,
               category: 'issues:missing-sri',
             },
+            {
+              key: 'issues-cors-wildcard-with-credentials',
+              label: 'CORS Wildcard + Credentials',
+              count: o.issues.corsWildcardWithCredentials,
+              category: 'issues:cors-wildcard-with-credentials',
+            },
+            {
+              key: 'issues-cors-wildcard-origin',
+              label: 'CORS Wildcard Origin',
+              count: o.issues.corsWildcardOrigin,
+              category: 'issues:cors-wildcard-origin',
+            },
+            {
+              key: 'issues-http-not-https',
+              label: 'HTTP (not HTTPS)',
+              count: o.issues.httpNotHttps,
+              category: 'issues:http-not-https',
+            },
           ],
         },
         {

apps/desktop/src/renderer/src/components/ReportsDialog.tsx

@@ -29,6 +29,7 @@ type ReportKind =
   | 'response-time'
   | 'slowest-urls'
   | 'most-inlinks'
+  | 'least-inlinks'
   | 'most-outlinks'
   | 'biggest-pages'
   | 'deepest-urls'
@@ -64,6 +65,7 @@ const REPORT_LABELS: Record<ReportKind, string> = {
   'response-time': 'Response Time Histogram',
   'slowest-urls': 'Slowest URLs (Top 25)',
   'most-inlinks': 'Most-Linked URLs (Top 25)',
+  'least-inlinks': 'Least-Linked URLs (Bottom 25, indexable HTML)',
   'most-outlinks': 'Most-Outlinking URLs (Top 25)',
   'biggest-pages': 'Biggest Pages (Top 25)',
   'deepest-urls': 'Deepest URLs (Top 25)',
@@ -86,6 +88,7 @@ const KEY_LABELS: Record<ReportKind, string> = {
   'response-time': 'Bucket',
   'slowest-urls': 'URL',
   'most-inlinks': 'URL',
+  'least-inlinks': 'URL',
   'most-outlinks': 'URL',
   'biggest-pages': 'URL',
   'deepest-urls': 'URL',
@@ -108,6 +111,7 @@ const TOP_URL_METRIC: Record<ReportKind, TopUrlMetric | null> = {
   'response-time': null,
   'slowest-urls': 'response-time',
   'most-inlinks': 'inlinks',
+  'least-inlinks': 'inlinks',
   'most-outlinks': 'outlinks',
   'biggest-pages': 'page-size',
   'deepest-urls': 'depth',
@@ -130,6 +134,7 @@ const VALUE_FORMAT: Record<ReportKind, (v: number | null) => string> = {
   'response-time': (v) => (v ?? 0).toLocaleString(),
   'slowest-urls': (v) => (v == null ? '—' : `${v.toLocaleString()} ms`),
   'most-inlinks': (v) => (v ?? 0).toLocaleString(),
+  'least-inlinks': (v) => (v ?? 0).toLocaleString(),
   'most-outlinks': (v) => (v ?? 0).toLocaleString(),
   'biggest-pages': (v) => (v == null ? '—' : `${(v / 1024).toFixed(1)} KB`),
   'deepest-urls': (v) => (v ?? 0).toLocaleString(),
@@ -299,7 +304,12 @@ export function ReportsDialog({ open, onClose }: Props) {
         } else {
           const metric = TOP_URL_METRIC[kind];
           if (metric) {
-            const r = await window.freecrawl.reportsTopUrls({ metric, limit: 25 });
+            const direction = kind === 'least-inlinks' ? 'asc' : 'desc';
+            const r = await window.freecrawl.reportsTopUrls({
+              metric,
+              limit: 25,
+              direction,
+            });
             if (!cancelled)
               setRows(
                 r.map((x: TopUrlsRow) => ({
@@ -371,6 +381,7 @@ export function ReportsDialog({ open, onClose }: Props) {
               <option value="response-time">Response Time Histogram</option>
               <option value="slowest-urls">Slowest URLs (Top 25)</option>
               <option value="most-inlinks">Most-Linked URLs (Top 25)</option>
+              <option value="least-inlinks">Least-Linked URLs (Bottom 25, indexable)</option>
               <option value="most-outlinks">Most-Outlinking URLs (Top 25)</option>
               <option value="biggest-pages">Biggest Pages (Top 25)</option>
               <option value="deepest-urls">Deepest URLs (Top 25)</option>

apps/desktop/src/renderer/src/components/SettingsDialog.tsx

@@ -75,6 +75,9 @@ interface FormState {
   forceHttps: boolean;
   lowercasePath: boolean;
   trailingSlash: 'leave' | 'strip' | 'add';
+  keepQueryParamsText: string;
+  urlRegexRewrites: Array<{ pattern: string; replacement: string; flags?: string }>;
+  urlRewritePreview: string;
   // hardware
   memoryLimitMb: string;
   maxQueueSize: string;
@@ -308,6 +311,9 @@ function configToForm(c: CrawlConfig): FormState {
     forceHttps: c.forceHttps,
     lowercasePath: c.lowercasePath,
     trailingSlash: c.trailingSlash,
+    keepQueryParamsText: (c.keepQueryParams ?? []).join('\n'),
+    urlRegexRewrites: (c.urlRegexRewrites ?? []).map((r) => ({ ...r })),
+    urlRewritePreview: '',
     memoryLimitMb: String(c.memoryLimitMb),
     maxQueueSize: String(c.maxQueueSize),
     processPriority: c.processPriority,
@@ -447,6 +453,16 @@ export function SettingsDialog({ open, onClose }: Props) {
       forceHttps: form.forceHttps,
       lowercasePath: form.lowercasePath,
       trailingSlash: form.trailingSlash,
+      keepQueryParams: parseLines(form.keepQueryParamsText)
+        .map((s) => s.trim())
+        .filter(Boolean),
+      urlRegexRewrites: form.urlRegexRewrites
+        .map((r) => ({
+          pattern: r.pattern.trim(),
+          replacement: r.replacement,
+          flags: r.flags?.trim() || undefined,
+        }))
+        .filter((r) => r.pattern !== ''),
       memoryLimitMb: Math.max(0, num(form.memoryLimitMb, config.memoryLimitMb)),
       maxQueueSize: Math.max(0, num(form.maxQueueSize, config.maxQueueSize)),
       processPriority: form.processPriority,
@@ -1198,22 +1214,93 @@ function CustomSearchPanel({ form, update }: PanelProps) {
   return (
     <>
       <p className="mb-3 text-[11px] text-surface-400">
-        Flag pages whose body contains any of these substrings (case-insensitive).
+        Flag pages whose body contains any of these terms. Two modes per
+        line:
+        <span className="ml-1 font-mono text-surface-200">/pattern/flags</span>{' '}
+        for regex,
+        <span className="ml-1 font-mono text-surface-200">anything else</span>{' '}
+        for literal case-insensitive substring.
       </p>
       <Area
-        label="Search Terms (case-insensitive literal substring; one per line)"
+        label="Search Terms (one per line; /…/ = regex, plain = literal)"
         value={form.customSearchTermsText}
         onChange={(v) => update('customSearchTermsText', v)}
         rows={8}
-        placeholder={'pricing\nfree shipping\nlimited time'}
-        info="Case-insensitive literal substring (NOT regex). Each term's per-page hit count is shown in the URL Details panel. Empty list disables the scan entirely."
-        example={'free shipping\npricing\nbeta\ncoming soon'}
+        placeholder={'pricing\nfree shipping\n/coming\\s+soon/i\n/(call|contact)\\s+us/i'}
+        info="Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."
+        example={'free shipping\npricing\n/(?:^|\\s)beta(?:\\s|$)/i\n/coming\\s+soon/i'}
       />
     </>
   );
 }
 
 function UrlRewritingPanel({ form, update }: PanelProps) {
+  const [previewState, setPreviewState] = useState<{
+    pending: boolean;
+    result?: string | null;
+    parseError?: string;
+    regexErrors?: Array<{ pattern: string; error: string }>;
+  }>({ pending: false });
+
+  const updateRegexRow = (i: number, patch: Partial<{ pattern: string; replacement: string; flags?: string }>) => {
+    const next = form.urlRegexRewrites.slice();
+    next[i] = { ...next[i]!, ...patch };
+    update('urlRegexRewrites', next);
+  };
+  const removeRegexRow = (i: number) => {
+    const next = form.urlRegexRewrites.slice();
+    next.splice(i, 1);
+    update('urlRegexRewrites', next);
+  };
+  const addRegexRow = () => {
+    update('urlRegexRewrites', [
+      ...form.urlRegexRewrites,
+      { pattern: '', replacement: '', flags: 'g' },
+    ]);
+  };
+
+  const runPreview = async () => {
+    const sample = form.urlRewritePreview.trim();
+    if (!sample) {
+      setPreviewState({ pending: false, parseError: 'Enter a URL above to preview' });
+      return;
+    }
+    setPreviewState({ pending: true });
+    try {
+      const keep = form.keepQueryParamsText
+        .split('\n')
+        .map((s) => s.trim())
+        .filter(Boolean);
+      const rewrites = form.urlRegexRewrites
+        .map((r) => ({
+          pattern: r.pattern.trim(),
+          replacement: r.replacement,
+          flags: r.flags?.trim() || undefined,
+        }))
+        .filter((r) => r.pattern !== '');
+      const res = await window.freecrawl.urlRewritePreview({
+        url: sample,
+        stripWww: form.stripWww,
+        forceHttps: form.forceHttps,
+        lowercasePath: form.lowercasePath,
+        trailingSlash: form.trailingSlash,
+        keepQueryParams: keep,
+        urlRegexRewrites: rewrites,
+      });
+      setPreviewState({
+        pending: false,
+        result: res.result,
+        parseError: res.parseError,
+        regexErrors: res.regexErrors,
+      });
+    } catch (err) {
+      setPreviewState({
+        pending: false,
+        parseError: err instanceof Error ? err.message : String(err),
+      });
+    }
+  };
+
   return (
     <>
       <p className="mb-3 text-[11px] text-surface-400">
@@ -1263,6 +1350,124 @@ function UrlRewritingPanel({ form, update }: PanelProps) {
           </select>
         </label>
       </div>
+
+      <div className="mt-4 border-t border-surface-800 pt-3">
+        <Area
+          label="Keep query parameters (whitelist; one per line)"
+          value={form.keepQueryParamsText}
+          onChange={(v) => update('keepQueryParamsText', v)}
+          rows={4}
+          placeholder={'id\npage\nlang'}
+          info="When non-empty, ALL query parameters not on this list are dropped during normalisation (case-insensitive name match). Leave empty to keep the default behaviour, which strips just utm_*, fbclid, gclid, mc_cid, and mc_eid."
+          example={'id\npage\nlang\nproduct'}
+        />
+      </div>
+
+      <div className="mt-2">
+        <div className="mb-1 flex items-center justify-between">
+          <FieldLabel
+            label="Regex rewrites (applied to the final URL string in order)"
+            info="Each rule runs JavaScript RegExp.replace on the fully-normalised URL. Flags default to 'g'. After all rules run, the result is re-parsed as a URL — if the rewrite produces an invalid URL, the link is dropped at normalisation time."
+            example="Pattern: ^https://m\\.(.+) · Replacement: https://www.$1 · Flags: i  (collapse mobile subdomain to www)"
+          />
+          <button
+            type="button"
+            onClick={addRegexRow}
+            className="inline-flex items-center gap-1 rounded border border-surface-700 px-2 py-0.5 text-[10px] text-surface-300 hover:bg-surface-800"
+          >
+            <Plus size={11} /> Add rule
+          </button>
+        </div>
+        {form.urlRegexRewrites.length === 0 ? (
+          <div className="rounded border border-dashed border-surface-800 p-2 text-[10px] text-surface-500">
+            No regex rules. Click "Add rule" to define one.
+          </div>
+        ) : (
+          <div className="space-y-1">
+            {form.urlRegexRewrites.map((row, i) => (
+              <div key={i} className="grid grid-cols-[1fr_1fr_60px_24px] gap-1 items-center">
+                <input
+                  type="text"
+                  className="rounded border border-surface-700 bg-surface-950 px-2 py-1 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                  placeholder="Pattern (regex)"
+                  value={row.pattern}
+                  onChange={(e) => updateRegexRow(i, { pattern: e.target.value })}
+                  spellCheck={false}
+                />
+                <input
+                  type="text"
+                  className="rounded border border-surface-700 bg-surface-950 px-2 py-1 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                  placeholder="Replacement (use $1, $2 …)"
+                  value={row.replacement}
+                  onChange={(e) => updateRegexRow(i, { replacement: e.target.value })}
+                  spellCheck={false}
+                />
+                <input
+                  type="text"
+                  className="rounded border border-surface-700 bg-surface-950 px-2 py-1 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                  placeholder="g"
+                  value={row.flags ?? ''}
+                  onChange={(e) => updateRegexRow(i, { flags: e.target.value })}
+                  spellCheck={false}
+                />
+                <button
+                  type="button"
+                  onClick={() => removeRegexRow(i)}
+                  className="rounded p-1 text-surface-400 hover:bg-surface-800 hover:text-red-400"
+                  title="Remove rule"
+                >
+                  <Trash2 size={12} />
+                </button>
+              </div>
+            ))}
+          </div>
+        )}
+      </div>
+
+      <div className="mt-4 border-t border-surface-800 pt-3">
+        <FieldLabel
+          label="Preview — test a URL against current settings"
+          info="Sends the URL through the same normalisation pipeline used by the crawler, with your unsaved settings applied. Useful for verifying regex rules before kicking off a crawl."
+        />
+        <div className="mt-1 grid grid-cols-[1fr_auto] gap-1">
+          <input
+            type="text"
+            className="rounded border border-surface-700 bg-surface-950 px-2 py-1 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
+            placeholder="https://www.example.com/Path/?utm_source=x&id=42"
+            value={form.urlRewritePreview}
+            onChange={(e) => update('urlRewritePreview', e.target.value)}
+            spellCheck={false}
+          />
+          <button
+            type="button"
+            onClick={runPreview}
+            disabled={previewState.pending}
+            className="rounded border border-surface-700 px-3 py-1 text-[11px] text-surface-200 hover:bg-surface-800 disabled:opacity-50"
+          >
+            {previewState.pending ? 'Running…' : 'Preview'}
+          </button>
+        </div>
+        {(previewState.result !== undefined || previewState.parseError) && (
+          <div className="mt-2 rounded border border-surface-800 bg-surface-950 p-2 text-[11px]">
+            {previewState.parseError ? (
+              <div className="text-red-400">{previewState.parseError}</div>
+            ) : previewState.result === null ? (
+              <div className="text-red-400">URL could not be normalised.</div>
+            ) : (
+              <div className="break-all font-mono text-emerald-300">{previewState.result}</div>
+            )}
+            {previewState.regexErrors && previewState.regexErrors.length > 0 && (
+              <ul className="mt-1 list-disc pl-4 text-amber-400">
+                {previewState.regexErrors.map((e, i) => (
+                  <li key={i}>
+                    <span className="font-mono">{e.pattern}</span> — {e.error}
+                  </li>
+                ))}
+              </ul>
+            )}
+          </div>
+        )}
+      </div>
     </>
   );
 }

apps/mcp-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/mcp-server",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "private": true,
   "type": "module",
   "bin": {

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "freecrawl-seo",
-      "version": "0.2.8",
+      "version": "0.2.9",
       "license": "MIT",
       "workspaces": [
         "apps/*",
@@ -25,7 +25,7 @@
     },
     "apps/cli": {
       "name": "@freecrawl/cli",
-      "version": "0.2.8",
+      "version": "0.2.9",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -42,7 +42,7 @@
     },
     "apps/desktop": {
       "name": "@freecrawl/desktop",
-      "version": "0.2.8",
+      "version": "0.2.9",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -84,7 +84,7 @@
     },
     "apps/mcp-server": {
       "name": "@freecrawl/mcp-server",
-      "version": "0.2.8",
+      "version": "0.2.9",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*"
@@ -8732,7 +8732,7 @@
     },
     "packages/core": {
       "name": "@freecrawl/core",
-      "version": "0.2.8",
+      "version": "0.2.9",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*",
@@ -8758,7 +8758,7 @@
     },
     "packages/db": {
       "name": "@freecrawl/db",
-      "version": "0.2.8",
+      "version": "0.2.9",
       "dependencies": {
         "@freecrawl/shared-types": "*"
       },
@@ -8770,7 +8770,7 @@
     },
     "packages/shared-types": {
       "name": "@freecrawl/shared-types",
-      "version": "0.2.8",
+      "version": "0.2.9",
       "devDependencies": {
         "rimraf": "^6.0.1",
         "typescript": "^5.7.0"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/crawler.ts

@@ -18,6 +18,7 @@ import {
   extractExtension,
   isInScope,
   resolveStartUrl,
+  compileUrlRegexRewrites,
 } from './url-utils.js';
 import { parseHtml, estimatePixelWidth } from './html-parser.js';
 import { analyseCookies, extractSetCookies } from './cookies.js';
@@ -193,12 +194,7 @@ export class Crawler extends EventEmitter {
    * nothing per call (no `?:` chains, no per-link `if`s) and so changing
    * config mid-crawl can't desync the seen-set's keying.
    */
-  private readonly urlRewrites: {
-    stripWww?: boolean;
-    forceHttps?: boolean;
-    lowercasePath?: boolean;
-    trailingSlash?: 'leave' | 'strip' | 'add';
-  };
+  private readonly urlRewrites: import('./url-utils.js').UrlRewriteOptions;
 
   /**
    * Optional freeze-watchdog hook. The desktop main process injects a
@@ -341,11 +337,16 @@ export class Crawler extends EventEmitter {
     this.excludeRegexes = compilePatterns(config.excludePatterns, (p, err) => {
       this.emit('error', `Invalid exclude pattern "${p}": ${err}`);
     });
+    const compiledRegexRewrites = compileUrlRegexRewrites(config.urlRegexRewrites, (p, err) => {
+      this.emit('error', `Invalid URL regex rewrite "${p}": ${err}`);
+    });
     this.urlRewrites = {
       stripWww: config.stripWww,
       forceHttps: config.forceHttps,
       lowercasePath: config.lowercasePath,
       trailingSlash: config.trailingSlash,
+      keepQueryParams: config.keepQueryParams,
+      regexRewrites: compiledRegexRewrites,
     };
   }
 
@@ -1663,6 +1664,25 @@ export class Crawler extends EventEmitter {
       const csp = res.headers.get('content-security-policy');
       const referrerPolicy = res.headers.get('referrer-policy');
       const permissionsPolicy = res.headers.get('permissions-policy');
+      // CORS — captured raw so the issue filters can reason about wildcard
+      // + credentials misconfigurations and the URL Details panel can show
+      // the values verbatim. `Access-Control-Allow-Headers` is sometimes
+      // very long (every API field listed); truncated to 1 KB to keep the
+      // row size predictable.
+      const corsAllowOrigin = res.headers.get('access-control-allow-origin');
+      const corsAllowCredentialsHeader = res.headers.get('access-control-allow-credentials');
+      const corsAllowCredentials =
+        corsAllowCredentialsHeader === null
+          ? -1
+          : corsAllowCredentialsHeader.trim().toLowerCase() === 'true'
+            ? 1
+            : 0;
+      const corsAllowMethods = res.headers.get('access-control-allow-methods');
+      const corsAllowHeadersRaw = res.headers.get('access-control-allow-headers');
+      const corsAllowHeaders =
+        corsAllowHeadersRaw && corsAllowHeadersRaw.length > 1024
+          ? `${corsAllowHeadersRaw.slice(0, 1024)}…`
+          : corsAllowHeadersRaw;
       // Stack auditing — captured raw, no parsing. Sites typically return
       // `nginx/1.25.0`, `cloudflare`, `Apache/2.4.41`, `Microsoft-IIS/10`.
       const serverHeader = res.headers.get('server');
@@ -1775,6 +1795,10 @@ export class Crawler extends EventEmitter {
           csp,
           referrerPolicy,
           permissionsPolicy,
+          corsAllowOrigin,
+          corsAllowCredentials,
+          corsAllowMethods,
+          corsAllowHeaders,
           canonicalHttp,
           cookiesCount: cookieSummary.count,
           cookiesInsecure: cookieSummary.insecureCount,
@@ -1844,6 +1868,10 @@ export class Crawler extends EventEmitter {
           csp,
           referrerPolicy,
           permissionsPolicy,
+          corsAllowOrigin,
+          corsAllowCredentials,
+          corsAllowMethods,
+          corsAllowHeaders,
           canonicalHttp,
           cookiesCount: cookieSummary.count,
           cookiesInsecure: cookieSummary.insecureCount,
@@ -1994,6 +2022,8 @@ export class Crawler extends EventEmitter {
           manifestUrl: parsed.manifestUrl,
           feedUrl: parsed.feedUrl,
           mixedContentCount: parsed.mixedContentCount,
+          mixedContentActive: parsed.mixedContentActive,
+          mixedContentPassive: parsed.mixedContentPassive,
           metaRefresh: parsed.metaRefresh,
           metaRefreshUrl: parsed.metaRefreshUrl,
           charset,
@@ -2031,6 +2061,11 @@ export class Crawler extends EventEmitter {
           serverHeader,
           jsOnlyLinksCount: parsed.jsOnlyLinksCount,
           textCodeRatio: parsed.textCodeRatio,
+          fleschReadingEase: parsed.fleschReadingEase,
+          fleschKincaidGrade: parsed.fleschKincaidGrade,
+          gunningFogIndex: parsed.gunningFogIndex,
+          sentenceCount: parsed.sentenceCount,
+          complexWordCount: parsed.complexWordCount,
         },
         headers: allHeaders,
         storeBody: this.config.storeBodySnapshots

packages/core/src/html-parser.ts

@@ -122,6 +122,18 @@ export interface ParsedPage {
    * high ratio (>30%) is content-rich. Null on non-HTML or empty pages.
    */
   textCodeRatio: number | null;
+  /**
+   * Readability scores derived from the same body-text token stream used
+   * for `wordCount`. All `null` when the page has too little prose
+   * (<50 words or zero sentences) to produce a stable score — that
+   * threshold matches the Hemingway/Yoast convention so we don't surface
+   * `0.0 grade` on a navigation-only landing page.
+   */
+  fleschReadingEase: number | null;
+  fleschKincaidGrade: number | null;
+  gunningFogIndex: number | null;
+  sentenceCount: number;
+  complexWordCount: number;
   /** Number of `<img>` with explicit `alt=""` (decorative — distinct from missing alt). */
   imagesEmptyAlt: number;
   /**
@@ -157,6 +169,17 @@ export interface ParsedPage {
    * when the page itself is served over plain HTTP.
    */
   mixedContentCount: number;
+  /**
+   * Active mixed content — script/iframe/object/embed/stylesheet served
+   * over HTTP from an HTTPS page. Browsers BLOCK these silently, so the
+   * page is missing JS/CSS the user cannot see in the rendered DOM.
+   */
+  mixedContentActive: number;
+  /**
+   * Passive mixed content — img/video/audio/source over HTTP from an
+   * HTTPS page. Browsers display it but mark the URL bar "Not Secure".
+   */
+  mixedContentPassive: number;
   /**
    * `{ "term1": count, "term2": count, ... }` — case-insensitive literal
    * substring match counts. Empty if no terms requested.
@@ -495,19 +518,34 @@ export function parseHtml(
   // Mixed content — only relevant on HTTPS pages. We scan the standard
   // subresource elements (Google's mixed-content audit list); plain
   // `<a href>` doesn't count because anchor links aren't subresources.
-  let mixedContentCount = 0;
+  //
+  // Split into the two browser-policy tiers:
+  //   - active   (script / iframe / object / embed / link rel=stylesheet)
+  //              — Chrome/Firefox/Safari BLOCK these on HTTPS, so the
+  //              page silently misses script/CSS that breaks features.
+  //   - passive  (img / video / audio / source)
+  //              — browsers downgrade UI to "Not Secure" but still load.
+  // The legacy `mixedContentCount` is the sum, kept for back-compat.
+  let mixedContentActive = 0;
+  let mixedContentPassive = 0;
   if (pageUrl.startsWith('https://')) {
+    const passiveTags = new Set(['img', 'video', 'audio', 'source']);
     $(
-      'img[src], script[src], iframe[src], video[src], audio[src], source[src], embed[src], link[rel="stylesheet"][href]',
+      'img[src], script[src], iframe[src], video[src], audio[src], source[src], embed[src], object[data], link[rel="stylesheet"][href]',
     ).each((_, el) => {
       const $el = $(el);
-      const ref = ($el.attr('src') ?? $el.attr('href') ?? '').trim();
-      if (ref.startsWith('http://')) mixedContentCount++;
+      const ref = ($el.attr('src') ?? $el.attr('href') ?? $el.attr('data') ?? '').trim();
+      if (!ref.startsWith('http://')) return;
+      const tag = (el as { tagName?: string }).tagName?.toLowerCase() ?? '';
+      if (passiveTags.has(tag)) mixedContentPassive++;
+      else mixedContentActive++;
     });
   }
+  const mixedContentCount = mixedContentActive + mixedContentPassive;
 
   const text = $('body').text().replace(/\s+/g, ' ').trim();
   const wordCount = text.length > 0 ? text.split(' ').filter(Boolean).length : 0;
+  const readability = computeReadability(text, wordCount);
 
   // Analytics / marketing trackers — fingerprinted by script src + inline
   // JS substrings + meta tags. Cheaper to scan once over `$` than per-tracker.
@@ -525,20 +563,57 @@ export function parseHtml(
       ? runExtractionRules(html, $, opts.customExtractionRules)
       : null;
 
-  // Custom search — count case-insensitive literal substring occurrences
-  // in the visible body text (not raw HTML, to avoid attribute / inline-JS
-  // false positives). Lowercase haystack/needle once per page rather than
-  // per-term so cost stays linear in body size.
+  // Custom search — counts term occurrences in the visible body text
+  // (not raw HTML, to avoid attribute / inline-JS false positives).
+  //
+  // Two modes per term, switched by the entry's syntactic shape:
+  //   - `/pattern/flags`  — regex; `g` flag is forced so all matches count.
+  //                         Invalid patterns are recorded as count = -1 so
+  //                         the user can spot the typo in the detail panel
+  //                         instead of silently getting 0 hits forever.
+  //   - anything else      — literal case-insensitive substring (legacy).
+  //
+  // Lowercase haystack is computed once per page in the literal path so
+  // cost stays linear in body size regardless of how many literal terms.
   const customSearchHits: Record<string, number> = {};
   if (opts.customSearchTerms && opts.customSearchTerms.length > 0 && text.length > 0) {
-    const haystack = text.toLowerCase();
+    let haystackLower: string | null = null;
     for (const raw of opts.customSearchTerms) {
       const term = raw.trim();
       if (!term) continue;
+      // Regex mode: /pattern/flags — last `/` separates pattern from flags.
+      // Require length ≥3 so `/` and `//` aren't mistaken for regexes.
+      if (term.length >= 3 && term.startsWith('/')) {
+        const lastSlash = term.lastIndexOf('/');
+        if (lastSlash > 0) {
+          const pattern = term.slice(1, lastSlash);
+          const userFlags = term.slice(lastSlash + 1);
+          // Force `g` so `.match()` returns every hit. Strip duplicate `g`
+          // from user input and validate the rest of the flag string.
+          const flagsClean = userFlags.replace(/g/g, '');
+          if (/^[imsuy]*$/.test(flagsClean)) {
+            try {
+              const re = new RegExp(pattern, flagsClean + 'g');
+              const matches = text.match(re);
+              customSearchHits[term] = matches ? matches.length : 0;
+              continue;
+            } catch {
+              // Invalid pattern — surface as -1 so the UI can flag it.
+              customSearchHits[term] = -1;
+              continue;
+            }
+          } else {
+            customSearchHits[term] = -1;
+            continue;
+          }
+        }
+      }
+      // Literal substring (legacy path).
+      if (haystackLower === null) haystackLower = text.toLowerCase();
       const needle = term.toLowerCase();
       let count = 0;
       let pos = 0;
-      while ((pos = haystack.indexOf(needle, pos)) !== -1) {
+      while ((pos = haystackLower.indexOf(needle, pos)) !== -1) {
         count++;
         pos += needle.length;
       }
@@ -804,12 +879,19 @@ export function parseHtml(
     emptyAnchorCount,
     jsOnlyLinksCount,
     textCodeRatio,
+    fleschReadingEase: readability.fleschReadingEase,
+    fleschKincaidGrade: readability.fleschKincaidGrade,
+    gunningFogIndex: readability.gunningFogIndex,
+    sentenceCount: readability.sentenceCount,
+    complexWordCount: readability.complexWordCount,
     imagesEmptyAlt,
     imagesLazy,
     formInputCount,
     formInputUnlabeledCount,
     headings,
     mixedContentCount,
+    mixedContentActive,
+    mixedContentPassive,
     customSearchHits,
     metaRefresh,
     metaRefreshUrl,
@@ -1186,3 +1268,96 @@ function decodeEntities(s: string): string {
     return NAMED_ENTITIES[name.toLowerCase()] ?? m;
   });
 }
+
+/**
+ * Counts vowel groups as a syllable-count proxy. The dictionary-free
+ * heuristic is the same one Hemingway / textstat / Yoast use:
+ *  - lowercase, strip non-letters
+ *  - drop a silent terminal `e` (but not `le` after a consonant)
+ *  - count contiguous vowel groups (a, e, i, o, u, y)
+ *  - clamp to ≥1
+ *
+ * Accuracy is ±1 per word; over a multi-paragraph page the noise averages
+ * out into stable Flesch / Gunning Fog scores. Cheap (~2 µs/word) so we
+ * can run it inline during parsing without showing up in the profile.
+ */
+function countSyllables(word: string): number {
+  let w = word.toLowerCase().replace(/[^a-z]/g, '');
+  if (w.length === 0) return 0;
+  if (w.length <= 3) return 1;
+  if (w.endsWith('e') && !w.endsWith('le')) w = w.slice(0, -1);
+  const groups = w.match(/[aeiouy]+/g);
+  return groups ? Math.max(1, groups.length) : 1;
+}
+
+interface ReadabilityScores {
+  fleschReadingEase: number | null;
+  fleschKincaidGrade: number | null;
+  gunningFogIndex: number | null;
+  sentenceCount: number;
+  complexWordCount: number;
+}
+
+/**
+ * Flesch / Flesch–Kincaid / Gunning Fog over the body text. Returns nulls
+ * for the three scores when the page has too little prose to be stable
+ * (<50 words OR zero sentences). The intermediate `sentenceCount` and
+ * `complexWordCount` are still surfaced so the detail panel can show
+ * what the engine actually measured.
+ */
+function computeReadability(text: string, wordCount: number): ReadabilityScores {
+  if (!text || wordCount === 0) {
+    return {
+      fleschReadingEase: null,
+      fleschKincaidGrade: null,
+      gunningFogIndex: null,
+      sentenceCount: 0,
+      complexWordCount: 0,
+    };
+  }
+  // Sentence boundary heuristic: . / ! / ? / … followed by whitespace or
+  // end of text. Multiple terminators collapse into one. Newline already
+  // collapsed to single space upstream.
+  const sentenceMatches = text.match(/[^.!?…]+[.!?…]+/g);
+  const sentenceCount = sentenceMatches ? sentenceMatches.length : 1;
+
+  let totalSyllables = 0;
+  let complexWordCount = 0;
+  // Tokenise on whitespace; the upstream `text` is already normalised.
+  for (const token of text.split(' ')) {
+    if (!token) continue;
+    const syl = countSyllables(token);
+    if (syl === 0) continue;
+    totalSyllables += syl;
+    // Gunning Fog "complex" word definition: ≥3 syllables, excluding
+    // common suffixes (-es, -ed, -ing) and proper nouns (capitalised).
+    if (syl >= 3 && !/^[A-Z]/.test(token) && !/(es|ed|ing)$/i.test(token)) {
+      complexWordCount++;
+    }
+  }
+
+  if (wordCount < 50 || sentenceCount === 0) {
+    return {
+      fleschReadingEase: null,
+      fleschKincaidGrade: null,
+      gunningFogIndex: null,
+      sentenceCount,
+      complexWordCount,
+    };
+  }
+
+  const wordsPerSentence = wordCount / sentenceCount;
+  const syllablesPerWord = totalSyllables / wordCount;
+  const flesch = 206.835 - 1.015 * wordsPerSentence - 84.6 * syllablesPerWord;
+  const grade = 0.39 * wordsPerSentence + 11.8 * syllablesPerWord - 15.59;
+  const fog = 0.4 * (wordsPerSentence + (100 * complexWordCount) / wordCount);
+
+  // Clamp to the same ranges the academic formulas allow.
+  return {
+    fleschReadingEase: Math.round(Math.max(-50, Math.min(120, flesch)) * 10) / 10,
+    fleschKincaidGrade: Math.round(Math.max(-2, Math.min(30, grade)) * 10) / 10,
+    gunningFogIndex: Math.round(Math.max(0, Math.min(30, fog)) * 10) / 10,
+    sentenceCount,
+    complexWordCount,
+  };
+}

packages/core/src/index.ts

@@ -1,6 +1,16 @@
 export { Crawler } from './crawler.js';
 export type { CrawlerEvents } from './crawler.js';
-export { normalizeUrl, isSameHost, resolveStartUrl } from './url-utils.js';
+export {
+  normalizeUrl,
+  isSameHost,
+  resolveStartUrl,
+  compileUrlRegexRewrites,
+} from './url-utils.js';
+export type {
+  UrlRewriteOptions,
+  UrlRegexRewrite,
+  CompiledUrlRegexRewrite,
+} from './url-utils.js';
 export {
   parseHtml,
   estimatePixelWidth,

packages/core/src/url-utils.ts

@@ -1,3 +1,14 @@
+export interface UrlRegexRewrite {
+  pattern: string;
+  replacement: string;
+  flags?: string;
+}
+
+export interface CompiledUrlRegexRewrite {
+  re: RegExp;
+  replacement: string;
+}
+
 export interface UrlRewriteOptions {
   /** Strip a leading `www.` from the host. */
   stripWww?: boolean;
@@ -7,6 +18,51 @@ export interface UrlRewriteOptions {
   lowercasePath?: boolean;
   /** Trailing-slash policy: leave / strip / add. */
   trailingSlash?: 'leave' | 'strip' | 'add';
+  /**
+   * Query-param whitelist. When non-empty, ALL query parameters not in this
+   * list are dropped — replacing the default tracking-only strip. Match is
+   * case-insensitive on the param name. Empty list = default behaviour
+   * (strip just utm_* / fbclid / gclid / mc_* tracking params).
+   */
+  keepQueryParams?: readonly string[];
+  /**
+   * Regex rewrites applied to the final stringified URL in order. Each
+   * `pattern` is compiled with the supplied `flags` (default `g`). Invalid
+   * patterns are skipped silently when passed pre-compiled — call sites
+   * should compile via `compileUrlRegexRewrites` so errors surface once
+   * up-front instead of on every URL.
+   */
+  regexRewrites?: readonly CompiledUrlRegexRewrite[];
+}
+
+const DEFAULT_TRACKING_PARAMS = [
+  'utm_source',
+  'utm_medium',
+  'utm_campaign',
+  'utm_term',
+  'utm_content',
+  'fbclid',
+  'gclid',
+  'mc_cid',
+  'mc_eid',
+] as const;
+
+export function compileUrlRegexRewrites(
+  raw: readonly UrlRegexRewrite[] | undefined,
+  onError?: (pattern: string, err: string) => void,
+): CompiledUrlRegexRewrite[] {
+  if (!raw || raw.length === 0) return [];
+  const out: CompiledUrlRegexRewrite[] = [];
+  for (const r of raw) {
+    if (!r || typeof r.pattern !== 'string' || r.pattern === '') continue;
+    try {
+      const re = new RegExp(r.pattern, r.flags ?? 'g');
+      out.push({ re, replacement: typeof r.replacement === 'string' ? r.replacement : '' });
+    } catch (err) {
+      if (onError) onError(r.pattern, err instanceof Error ? err.message : String(err));
+    }
+  }
+  return out;
 }
 
 export function normalizeUrl(
@@ -17,26 +73,24 @@ export function normalizeUrl(
   try {
     const u = new URL(raw, base);
     u.hash = '';
-    const tracking = [
-      'utm_source',
-      'utm_medium',
-      'utm_campaign',
-      'utm_term',
-      'utm_content',
-      'fbclid',
-      'gclid',
-      'mc_cid',
-      'mc_eid',
-    ];
-    for (const p of tracking) u.searchParams.delete(p);
+
+    const keep = rewrites.keepQueryParams;
+    if (keep && keep.length > 0) {
+      const allowed = new Set(keep.map((p) => p.toLowerCase()));
+      const drop: string[] = [];
+      for (const k of u.searchParams.keys()) {
+        if (!allowed.has(k.toLowerCase())) drop.push(k);
+      }
+      for (const k of drop) u.searchParams.delete(k);
+    } else {
+      for (const p of DEFAULT_TRACKING_PARAMS) u.searchParams.delete(p);
+    }
+
     if (u.pathname === '') u.pathname = '/';
     if ((u.protocol === 'http:' && u.port === '80') || (u.protocol === 'https:' && u.port === '443')) {
       u.port = '';
     }
 
-    // User-configured rewrites — applied after the canonical pass so each
-    // branch sees a clean URL. Combinations (forceHttps + stripWww) compose
-    // naturally because each branch only touches one URL component.
     if (rewrites.forceHttps && u.protocol === 'http:') {
       u.protocol = 'https:';
     }
@@ -53,12 +107,24 @@ export function normalizeUrl(
       u.pathname.length > 0 &&
       !u.pathname.endsWith('/')
     ) {
-      // Skip file-extension paths (`.css`, `.html`, …) — adding `/` to those
-      // creates broken URLs. Detect by the final segment containing a `.`.
       const last = u.pathname.slice(u.pathname.lastIndexOf('/') + 1);
       if (!last.includes('.')) u.pathname += '/';
     }
-    return u.toString();
+
+    let result = u.toString();
+    if (rewrites.regexRewrites && rewrites.regexRewrites.length > 0) {
+      for (const { re, replacement } of rewrites.regexRewrites) {
+        re.lastIndex = 0;
+        result = result.replace(re, replacement);
+      }
+      try {
+        const reparsed = new URL(result);
+        result = reparsed.toString();
+      } catch {
+        return null;
+      }
+    }
+    return result;
   } catch {
     return null;
   }

packages/core/src/xml-export.ts

@@ -100,12 +100,23 @@ const XML_COLUMNS: (keyof CrawlUrlRow)[] = [
   'metaRefreshUrl',
   'charset',
   'mixedContentCount',
+  'mixedContentActive',
+  'mixedContentPassive',
   'imagesCount',
   'imagesMissingAlt',
   'imagesEmptyAlt',
   'imagesLazy',
   'jsOnlyLinksCount',
   'textCodeRatio',
+  'fleschReadingEase',
+  'fleschKincaidGrade',
+  'gunningFogIndex',
+  'sentenceCount',
+  'complexWordCount',
+  'corsAllowOrigin',
+  'corsAllowCredentials',
+  'corsAllowMethods',
+  'corsAllowHeaders',
   'titleCount',
   'titlePixelWidth',
   'metaPixelWidth',

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/migrations.ts

@@ -943,6 +943,87 @@ const MIGRATIONS: Migration[] = [
       }
     },
   },
+  {
+    version: 46,
+    name: 'add_readability_columns',
+    // Per-URL readability fields populated by `parseHtml()` from the body
+    // text it already tokenises for word count. Stored as REAL so the UI
+    // can render `flesch_reading_ease=58.3` without rounding.
+    //   - flesch_reading_ease     (0–100, higher = easier; <30 = "very difficult")
+    //   - flesch_kincaid_grade    (US grade level; 8 ≈ 8th grade)
+    //   - gunning_fog_index       (years of formal education needed)
+    //   - sentence_count          (used by all three formulas; surfaced in detail)
+    //   - complex_word_count      (≥3-syllable non-suffix words; Gunning Fog input)
+    up: (db) => {
+      const cols = db.prepare('PRAGMA table_info(urls)').all() as unknown as {
+        name: string;
+      }[];
+      const has = (n: string) => cols.some((c) => c.name === n);
+      if (!has('flesch_reading_ease')) db.exec('ALTER TABLE urls ADD COLUMN flesch_reading_ease REAL');
+      if (!has('flesch_kincaid_grade')) db.exec('ALTER TABLE urls ADD COLUMN flesch_kincaid_grade REAL');
+      if (!has('gunning_fog_index')) db.exec('ALTER TABLE urls ADD COLUMN gunning_fog_index REAL');
+      if (!has('sentence_count'))
+        db.exec('ALTER TABLE urls ADD COLUMN sentence_count INTEGER NOT NULL DEFAULT 0');
+      if (!has('complex_word_count'))
+        db.exec('ALTER TABLE urls ADD COLUMN complex_word_count INTEGER NOT NULL DEFAULT 0');
+    },
+  },
+  {
+    version: 47,
+    name: 'add_cors_columns',
+    // CORS response-header audit fields. Captured per URL so the
+    // "CORS Wildcard + Credentials" + "CORS Wildcard Origin" issue
+    // filters can flag misconfigured origins, and the URL Details panel
+    // can render the values for review.
+    //   - cors_allow_origin       (raw header, e.g. `*`, `https://x.com`, `null`)
+    //   - cors_allow_credentials  (-1 missing / 0 false / 1 true)
+    //   - cors_allow_methods      (raw header)
+    //   - cors_allow_headers      (raw header, often verbose; truncated upstream)
+    up: (db) => {
+      const cols = db.prepare('PRAGMA table_info(urls)').all() as unknown as {
+        name: string;
+      }[];
+      const has = (n: string) => cols.some((c) => c.name === n);
+      if (!has('cors_allow_origin'))
+        db.exec('ALTER TABLE urls ADD COLUMN cors_allow_origin TEXT');
+      if (!has('cors_allow_credentials'))
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN cors_allow_credentials INTEGER NOT NULL DEFAULT -1',
+        );
+      if (!has('cors_allow_methods'))
+        db.exec('ALTER TABLE urls ADD COLUMN cors_allow_methods TEXT');
+      if (!has('cors_allow_headers'))
+        db.exec('ALTER TABLE urls ADD COLUMN cors_allow_headers TEXT');
+    },
+  },
+  {
+    version: 48,
+    name: 'split_mixed_content_active_passive',
+    // Split the legacy `mixed_content_count` into the two browser-policy
+    // tiers:
+    //   - active   (script / iframe / object / embed / link rel=stylesheet)
+    //                — Chrome / Firefox / Safari BLOCK these on HTTPS;
+    //                  the page silently misses script/CSS subresources.
+    //   - passive  (img / video / audio / source)
+    //                — browsers downgrade to "Not Secure" UI but render.
+    // Both columns coexist with the existing `mixed_content_count` so
+    // older code paths keep working. Backfill is a no-op — the legacy
+    // column was a sum, so it doesn't tell us the split for past crawls.
+    up: (db) => {
+      const cols = db.prepare('PRAGMA table_info(urls)').all() as unknown as {
+        name: string;
+      }[];
+      const has = (n: string) => cols.some((c) => c.name === n);
+      if (!has('mixed_content_active'))
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN mixed_content_active INTEGER NOT NULL DEFAULT 0',
+        );
+      if (!has('mixed_content_passive'))
+        db.exec(
+          'ALTER TABLE urls ADD COLUMN mixed_content_passive INTEGER NOT NULL DEFAULT 0',
+        );
+    },
+  },
 ];
 
 export function runMigrations(db: DatabaseSync): void {

packages/db/src/project-db.ts

@@ -84,6 +84,8 @@ interface UrlRowDb {
   amphtml: string | null;
   favicon: string | null;
   mixed_content_count: number;
+  mixed_content_active: number;
+  mixed_content_passive: number;
   hreflang_invalid_count: number;
   hreflang_self_ref_missing: number;
   hreflang_reciprocity_missing: number;
@@ -134,6 +136,15 @@ interface UrlRowDb {
   server_header: string | null;
   js_only_links_count: number;
   text_code_ratio: number | null;
+  flesch_reading_ease: number | null;
+  flesch_kincaid_grade: number | null;
+  gunning_fog_index: number | null;
+  sentence_count: number;
+  complex_word_count: number;
+  cors_allow_origin: string | null;
+  cors_allow_credentials: number;
+  cors_allow_methods: string | null;
+  cors_allow_headers: string | null;
 }
 
 interface ImageRowDb {
@@ -212,6 +223,10 @@ export interface UpsertUrlInput {
   amphtml?: string | null;
   favicon?: string | null;
   mixedContentCount?: number;
+  /** Active mixed content (blocked by browser) — script/iframe/object/embed/stylesheet over HTTP. */
+  mixedContentActive?: number;
+  /** Passive mixed content (warned but rendered) — img/video/audio/source over HTTP. */
+  mixedContentPassive?: number;
   /** JSON-stringified custom-extraction results map. */
   extractionResults?: string | null;
   simhash?: string | null;
@@ -250,6 +265,24 @@ export interface UpsertUrlInput {
   jsOnlyLinksCount?: number;
   /** Visible-text bytes / total HTML bytes as integer percent (0–100). */
   textCodeRatio?: number | null;
+  /** Flesch Reading Ease (0–100, higher = easier). Null when too little prose. */
+  fleschReadingEase?: number | null;
+  /** Flesch–Kincaid US grade level. Null when too little prose. */
+  fleschKincaidGrade?: number | null;
+  /** Gunning Fog Index (years of formal education). Null when too little prose. */
+  gunningFogIndex?: number | null;
+  /** Sentence count over body text — used by all three readability formulas. */
+  sentenceCount?: number;
+  /** Complex word count (≥3 syllables, sans common suffixes/proper nouns). */
+  complexWordCount?: number;
+  /** Raw `Access-Control-Allow-Origin` response header (or null). */
+  corsAllowOrigin?: string | null;
+  /** -1 = header missing, 0 = `false`, 1 = `true`. */
+  corsAllowCredentials?: number;
+  /** Raw `Access-Control-Allow-Methods` response header (or null). */
+  corsAllowMethods?: string | null;
+  /** Raw `Access-Control-Allow-Headers` response header (or null), truncated to 1 KB. */
+  corsAllowHeaders?: string | null;
 }
 
 const UPSERT_URL_SQL = `
@@ -266,7 +299,7 @@ const UPSERT_URL_SQL = `
     hsts, x_frame_options, x_content_type_options, content_encoding,
     schema_types, schema_block_count, schema_invalid_count,
     pagination_next, pagination_prev, hreflangs, hreflang_count,
-    amphtml, favicon, mixed_content_count,
+    amphtml, favicon, mixed_content_count, mixed_content_active, mixed_content_passive,
     folder_depth, query_param_count,
     csp, referrer_policy, permissions_policy,
     custom_search_hits,
@@ -284,7 +317,9 @@ const UPSERT_URL_SQL = `
     form_input_count, form_input_unlabeled, images_lazy,
     headings,
     server_header,
-    js_only_links_count, text_code_ratio
+    js_only_links_count, text_code_ratio,
+    flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, sentence_count, complex_word_count,
+    cors_allow_origin, cors_allow_credentials, cors_allow_methods, cors_allow_headers
   ) VALUES (
     :url, :content_kind, :status_code, :status_text, :indexability, :indexability_reason,
     :title, :title_length, :meta_description, :meta_description_length,
@@ -298,7 +333,7 @@ const UPSERT_URL_SQL = `
     :hsts, :x_frame_options, :x_content_type_options, :content_encoding,
     :schema_types, :schema_block_count, :schema_invalid_count,
     :pagination_next, :pagination_prev, :hreflangs, :hreflang_count,
-    :amphtml, :favicon, :mixed_content_count,
+    :amphtml, :favicon, :mixed_content_count, :mixed_content_active, :mixed_content_passive,
     :folder_depth, :query_param_count,
     :csp, :referrer_policy, :permissions_policy,
     :custom_search_hits,
@@ -316,7 +351,9 @@ const UPSERT_URL_SQL = `
     :form_input_count, :form_input_unlabeled, :images_lazy,
     :headings,
     :server_header,
-    :js_only_links_count, :text_code_ratio
+    :js_only_links_count, :text_code_ratio,
+    :flesch_reading_ease, :flesch_kincaid_grade, :gunning_fog_index, :sentence_count, :complex_word_count,
+    :cors_allow_origin, :cors_allow_credentials, :cors_allow_methods, :cors_allow_headers
   )
   ON CONFLICT(url) DO UPDATE SET
     content_kind = excluded.content_kind,
@@ -377,6 +414,8 @@ const UPSERT_URL_SQL = `
     amphtml = excluded.amphtml,
     favicon = excluded.favicon,
     mixed_content_count = excluded.mixed_content_count,
+    mixed_content_active = excluded.mixed_content_active,
+    mixed_content_passive = excluded.mixed_content_passive,
     folder_depth = excluded.folder_depth,
     query_param_count = excluded.query_param_count,
     csp = excluded.csp,
@@ -418,6 +457,15 @@ const UPSERT_URL_SQL = `
     server_header = excluded.server_header,
     js_only_links_count = excluded.js_only_links_count,
     text_code_ratio = excluded.text_code_ratio,
+    flesch_reading_ease = excluded.flesch_reading_ease,
+    flesch_kincaid_grade = excluded.flesch_kincaid_grade,
+    gunning_fog_index = excluded.gunning_fog_index,
+    sentence_count = excluded.sentence_count,
+    complex_word_count = excluded.complex_word_count,
+    cors_allow_origin = excluded.cors_allow_origin,
+    cors_allow_credentials = excluded.cors_allow_credentials,
+    cors_allow_methods = excluded.cors_allow_methods,
+    cors_allow_headers = excluded.cors_allow_headers,
     crawled_at = CURRENT_TIMESTAMP
   RETURNING id
 `;
@@ -993,6 +1041,8 @@ export class ProjectDb {
       amphtml: input.amphtml ?? null,
       favicon: input.favicon ?? null,
       mixed_content_count: input.mixedContentCount ?? 0,
+      mixed_content_active: input.mixedContentActive ?? 0,
+      mixed_content_passive: input.mixedContentPassive ?? 0,
       folder_depth: computeFolderDepth(input.url),
       query_param_count: computeQueryParamCount(input.url),
       csp: input.csp ?? null,
@@ -1034,6 +1084,15 @@ export class ProjectDb {
       server_header: input.serverHeader ?? null,
       js_only_links_count: input.jsOnlyLinksCount ?? 0,
       text_code_ratio: input.textCodeRatio ?? null,
+      flesch_reading_ease: input.fleschReadingEase ?? null,
+      flesch_kincaid_grade: input.fleschKincaidGrade ?? null,
+      gunning_fog_index: input.gunningFogIndex ?? null,
+      sentence_count: input.sentenceCount ?? 0,
+      complex_word_count: input.complexWordCount ?? 0,
+      cors_allow_origin: input.corsAllowOrigin ?? null,
+      cors_allow_credentials: input.corsAllowCredentials ?? -1,
+      cors_allow_methods: input.corsAllowMethods ?? null,
+      cors_allow_headers: input.corsAllowHeaders ?? null,
     };
 
     const row = this.stmtUpsertUrl.get(params) as { id: number } | undefined;
@@ -1755,6 +1814,70 @@ export class ProjectDb {
     });
   }
 
+  /**
+   * Returns OTHER URLs in the same near-duplicate cluster as `urlId`.
+   * Powers the URL Details panel's "Duplicates" sub-tab — the user
+   * selects a URL and immediately sees which other pages SimHash + LSH
+   * grouped with it. Excludes the URL itself; sorted by hamming distance
+   * from the queried URL ascending (closest matches first), then URL
+   * alphabetically. Singleton clusters (cluster_id=0) return [].
+   */
+  urlClusterMembers(urlId: number): Array<{
+    url: string;
+    statusCode: number | null;
+    indexability: Indexability;
+    title: string | null;
+    wordCount: number | null;
+    inlinks: number;
+    hammingDistance: number;
+  }> {
+    const self = this.db
+      .prepare('SELECT cluster_id, simhash FROM urls WHERE id = ?')
+      .get(urlId) as { cluster_id: number | null; simhash: string | null } | undefined;
+    if (!self || !self.cluster_id || self.cluster_id <= 0) return [];
+
+    const rows = this.db
+      .prepare(
+        `SELECT id, url, status_code, indexability, title, word_count, inlinks, simhash
+           FROM urls
+          WHERE cluster_id = ? AND id != ?
+          ORDER BY url ASC`,
+      )
+      .all(self.cluster_id, urlId) as Array<{
+      id: number;
+      url: string;
+      status_code: number | null;
+      indexability: Indexability;
+      title: string | null;
+      word_count: number | null;
+      inlinks: number;
+      simhash: string | null;
+    }>;
+
+    const popcount = (x: bigint): number => {
+      let c = 0;
+      while (x !== 0n) {
+        x &= x - 1n;
+        c++;
+      }
+      return c;
+    };
+
+    const selfBig = self.simhash ? BigInt('0x' + self.simhash) : null;
+    return rows
+      .map((r) => ({
+        url: r.url,
+        statusCode: r.status_code,
+        indexability: r.indexability,
+        title: r.title,
+        wordCount: r.word_count,
+        inlinks: r.inlinks,
+        hammingDistance:
+          selfBig !== null && r.simhash ? popcount(selfBig ^ BigInt('0x' + r.simhash)) : 0,
+      }))
+      .sort((a, b) => a.hammingDistance - b.hammingDistance || a.url.localeCompare(b.url));
+  }
+
   countDuplicateClusterMembers(): number {
     return (
       this.db
@@ -2554,6 +2677,12 @@ export class ProjectDb {
       mixedContent: countWhere(
         `${html} AND url LIKE 'https://%' AND mixed_content_count > 0`,
       ),
+      mixedContentActive: countWhere(
+        `${html} AND url LIKE 'https://%' AND mixed_content_active > 0`,
+      ),
+      mixedContentPassive: countWhere(
+        `${html} AND url LIKE 'https://%' AND mixed_content_passive > 0`,
+      ),
       faviconMissing: countWhere(`${html} AND (favicon IS NULL OR favicon = '')`),
       redirectLoop: countWhere('is_external = 0 AND redirect_loop = 1'),
       redirectChainLong: countWhere('is_external = 0 AND redirect_chain_length > 3'),
@@ -2908,6 +3037,26 @@ export class ProjectDb {
       textCodeRatioLow: countWhere(
         `${html} AND text_code_ratio IS NOT NULL AND text_code_ratio < 10`,
       ),
+      fleschVeryDifficult: countWhere(
+        `${html} AND flesch_reading_ease IS NOT NULL AND flesch_reading_ease < 30`,
+      ),
+      gunningFogVeryHigh: countWhere(
+        `${html} AND gunning_fog_index IS NOT NULL AND gunning_fog_index > 17`,
+      ),
+      corsWildcardWithCredentials: countWhere(
+        `is_external = 0 AND TRIM(cors_allow_origin) = '*' AND cors_allow_credentials = 1`,
+      ),
+      corsWildcardOrigin: countWhere(
+        `is_external = 0 AND TRIM(cors_allow_origin) = '*' AND cors_allow_credentials != 1`,
+      ),
+      httpNotHttps: countWhere(
+        `is_external = 0
+         AND url LIKE 'http://%'
+         AND url NOT LIKE 'http://localhost%'
+         AND url NOT LIKE 'http://127.0.0.1%'
+         AND url NOT LIKE 'http://%.local/%'
+         AND url NOT LIKE 'http://%.local'`,
+      ),
       renderBlockingCritical: countWhere(`${html} AND render_blocking_count > 20`),
       ogImageTooLarge: countWhere(
         `${html} AND og_image IS NOT NULL AND og_image != ''
@@ -3426,13 +3575,20 @@ export class ProjectDb {
   topUrlsBy(
     column: 'response_time_ms' | 'depth' | 'outlinks' | 'inlinks' | 'content_length',
     limit: number,
+    direction: 'asc' | 'desc' = 'desc',
   ): { url: string; value: number | null }[] {
+    // For ASC ("least-linked") we exclude URLs with `column = 0` so the
+    // user gets a list of weakly-linked indexable pages, not unlinked
+    // ones — orphans are surfaced through their own filters/reports.
+    const minClause = direction === 'asc' ? `AND ${column} > 0` : '';
+    const order = direction === 'asc' ? 'ASC' : 'DESC';
     return this.db
       .prepare(
         `SELECT url, ${column} AS value FROM urls
           WHERE is_external = 0 AND content_kind = 'html'
             AND ${column} IS NOT NULL
-          ORDER BY ${column} DESC
+            ${minClause}
+          ORDER BY ${column} ${order}, url ASC
           LIMIT ?`,
       )
       .all(limit) as { url: string; value: number | null }[];
@@ -4146,6 +4302,8 @@ export class ProjectDb {
     amphtml: r.amphtml,
     favicon: r.favicon,
     mixedContentCount: r.mixed_content_count,
+    mixedContentActive: r.mixed_content_active ?? 0,
+    mixedContentPassive: r.mixed_content_passive ?? 0,
     hreflangInvalidCount: r.hreflang_invalid_count,
     hreflangSelfRefMissing: r.hreflang_self_ref_missing === 1,
     hreflangReciprocityMissing: r.hreflang_reciprocity_missing,
@@ -4196,6 +4354,15 @@ export class ProjectDb {
     serverHeader: r.server_header ?? null,
     jsOnlyLinksCount: r.js_only_links_count ?? 0,
     textCodeRatio: r.text_code_ratio ?? null,
+    fleschReadingEase: r.flesch_reading_ease ?? null,
+    fleschKincaidGrade: r.flesch_kincaid_grade ?? null,
+    gunningFogIndex: r.gunning_fog_index ?? null,
+    sentenceCount: r.sentence_count ?? 0,
+    complexWordCount: r.complex_word_count ?? 0,
+    corsAllowOrigin: r.cors_allow_origin ?? null,
+    corsAllowCredentials: r.cors_allow_credentials ?? -1,
+    corsAllowMethods: r.cors_allow_methods ?? null,
+    corsAllowHeaders: r.cors_allow_headers ?? null,
     crawledAt: r.crawled_at,
   });
 
@@ -5448,6 +5615,55 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       return `is_external = 0 AND content_kind = 'html'
               AND text_code_ratio IS NOT NULL
               AND text_code_ratio < 10`;
+    case 'issues:flesch-very-difficult':
+      // Flesch Reading Ease < 30 — academic / "very difficult" prose,
+      // hard for the general audience. Hemingway/Yoast threshold.
+      return `is_external = 0 AND content_kind = 'html'
+              AND flesch_reading_ease IS NOT NULL
+              AND flesch_reading_ease < 30`;
+    case 'issues:gunning-fog-very-high':
+      // Gunning Fog Index > 17 — beyond college-graduate reading level.
+      // Long sentences with dense vocabulary; readers will bounce.
+      return `is_external = 0 AND content_kind = 'html'
+              AND gunning_fog_index IS NOT NULL
+              AND gunning_fog_index > 17`;
+    case 'issues:cors-wildcard-with-credentials':
+      // `Access-Control-Allow-Origin: *` paired with
+      // `Access-Control-Allow-Credentials: true`. Browsers reject this
+      // combination per the CORS spec, but misconfigured servers reach
+      // it and any working XSS could exfiltrate cookies cross-origin.
+      return `is_external = 0
+              AND TRIM(cors_allow_origin) = '*'
+              AND cors_allow_credentials = 1`;
+    case 'issues:cors-wildcard-origin':
+      // `Access-Control-Allow-Origin: *` without credentials. Not always
+      // wrong (public APIs do this) but worth reviewing on production
+      // HTML so an audit can confirm intent.
+      return `is_external = 0
+              AND TRIM(cors_allow_origin) = '*'
+              AND cors_allow_credentials != 1`;
+    case 'issues:http-not-https':
+      // Internal URL still served over plain HTTP. Modern Google ranks
+      // HTTPS as a positive signal and Chrome marks HTTP pages as "Not
+      // Secure". Excludes the localhost / 127.0.0.1 / .local hosts a
+      // user might legitimately crawl over HTTP for local dev work.
+      return `is_external = 0
+              AND url LIKE 'http://%'
+              AND url NOT LIKE 'http://localhost%'
+              AND url NOT LIKE 'http://127.0.0.1%'
+              AND url NOT LIKE 'http://%.local/%'
+              AND url NOT LIKE 'http://%.local'`;
+    case 'issues:mixed-content-active':
+      // Active mixed content: HTTPS page references HTTP script /
+      // iframe / object / embed / stylesheet. Browsers BLOCK these so
+      // the page silently misses functionality the user can't see.
+      return `is_external = 0 AND content_kind = 'html'
+              AND url LIKE 'https://%' AND mixed_content_active > 0`;
+    case 'issues:mixed-content-passive':
+      // Passive mixed content: HTTPS page references HTTP img / video /
+      // audio / source. Browser warns "Not Secure" but renders.
+      return `is_external = 0 AND content_kind = 'html'
+              AND url LIKE 'https://%' AND mixed_content_passive > 0`;
     case 'issues:render-blocking-critical':
       // Tier above the existing "Render-Blocking Head (>5)" issue.
       // 20+ blocking head resources is almost universally third-party

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/crawl.ts

@@ -171,6 +171,13 @@ export type UrlCategory =
   | 'issues:description-equals-h1'
   | 'issues:js-only-navigation'
   | 'issues:text-code-ratio-low'
+  | 'issues:flesch-very-difficult'
+  | 'issues:gunning-fog-very-high'
+  | 'issues:cors-wildcard-with-credentials'
+  | 'issues:cors-wildcard-origin'
+  | 'issues:http-not-https'
+  | 'issues:mixed-content-active'
+  | 'issues:mixed-content-passive'
   | 'issues:render-blocking-critical'
   | 'issues:og-image-too-large'
   | 'issues:twitter-image-too-large'
@@ -267,6 +274,30 @@ export interface CrawlUrlRow {
    * little crawlable content. Null on non-HTML or empty pages.
    */
   textCodeRatio: number | null;
+  /**
+   * Readability scores derived from the body text. All `null` when the
+   * page has too little prose (<50 words or zero sentences) for a
+   * stable score. Higher Flesch = easier to read; lower grade/Fog =
+   * easier to read.
+   */
+  fleschReadingEase: number | null;
+  fleschKincaidGrade: number | null;
+  gunningFogIndex: number | null;
+  /** Sentence count over body text. Used by all three formulas. */
+  sentenceCount: number;
+  /** Complex-word count (≥3 syllables, sans common suffixes). */
+  complexWordCount: number;
+  /**
+   * CORS response headers — captured raw for the audit/issue filters and
+   * rendered verbatim in the URL Details panel. `corsAllowCredentials`
+   * is a tri-state: -1 (header absent), 0 (`false`), 1 (`true`). The
+   * "wildcard origin + credentials true" combination is a known XSS-
+   * leveraged credential-theft vector, surfaced as a critical issue.
+   */
+  corsAllowOrigin: string | null;
+  corsAllowCredentials: number;
+  corsAllowMethods: string | null;
+  corsAllowHeaders: string | null;
   redirectTarget: string | null;
   lang: string | null;
   viewport: string | null;
@@ -345,6 +376,17 @@ export interface CrawlUrlRow {
   /** Resolved RSS / Atom `<link rel="alternate" type="application/rss+xml|atom+xml">` URL, else null. */
   feedUrl: string | null;
   mixedContentCount: number;
+  /**
+   * Active mixed content — script / iframe / object / embed / stylesheet
+   * served over HTTP from an HTTPS page. Browsers BLOCK these silently
+   * so the page is missing JS/CSS the user can't see in the rendered DOM.
+   */
+  mixedContentActive: number;
+  /**
+   * Passive mixed content — img / video / audio / source over HTTP from
+   * an HTTPS page. Rendered but the URL bar shows "Not Secure".
+   */
+  mixedContentPassive: number;
   /** Number of `<title>` elements (>1 is a duplicate-tag issue). */
   titleCount: number;
   /** Number of internal hyperlinks with no anchor text or alt — accessibility/SEO issue. */
@@ -508,6 +550,21 @@ export interface CrawlConfig {
    *  - `add`    — `…/foo` → `…/foo/`  (only when path has no trailing `.ext`)
    */
   trailingSlash: 'leave' | 'strip' | 'add';
+  /**
+   * Query-param whitelist. When non-empty, the normalizer drops every
+   * query parameter not on this list (case-insensitive). Empty list keeps
+   * the legacy behaviour (strip only utm_* / fbclid / gclid / mc_*).
+   */
+  keepQueryParams: string[];
+  /**
+   * Regex rewrites applied to the fully-normalized URL string in order.
+   * Each entry is `{pattern, replacement, flags?}` where `flags` defaults
+   * to `g`. The result is re-parsed as a URL — if the rewrite produces an
+   * invalid URL the link is dropped at normalization time, so authors
+   * should test patterns in the Settings → URL Rewriting preview field
+   * before saving.
+   */
+  urlRegexRewrites: Array<{ pattern: string; replacement: string; flags?: string }>;
   /**
    * Hardware / resource caps. All `0` means unlimited.
    *
@@ -851,6 +908,16 @@ export interface OverviewCounts {
     paginationBroken: number;
     hreflangXDefaultMissing: number;
     mixedContent: number;
+    /**
+     * Active mixed content (blocked by browser) — script/iframe/object/
+     * embed/stylesheet over HTTP. Critical: subresources silently miss.
+     */
+    mixedContentActive: number;
+    /**
+     * Passive mixed content (warned but rendered) — img/video/audio/source
+     * over HTTP. Lower-priority than active but still flags "Not Secure".
+     */
+    mixedContentPassive: number;
     faviconMissing: number;
     redirectLoop: number;
     redirectChainLong: number;
@@ -1007,6 +1074,40 @@ export interface OverviewCounts {
      * JavaScript / template scaffolding with little crawlable content.
      */
     textCodeRatioLow: number;
+    /**
+     * Pages with Flesch Reading Ease < 30 — academic / "very difficult"
+     * tier. Long sentences and dense vocabulary; SEO and user comprehension
+     * suffer below this score (Hemingway uses the same threshold).
+     */
+    fleschVeryDifficult: number;
+    /**
+     * Pages with Gunning Fog Index > 17 — beyond college-graduate reading
+     * level. Combined with low Flesch Reading Ease, indicates content that
+     * is unlikely to be understood by a general audience.
+     */
+    gunningFogVeryHigh: number;
+    /**
+     * Pages whose `Access-Control-Allow-Origin: *` is paired with
+     * `Access-Control-Allow-Credentials: true`. This combination is
+     * effectively impossible per the CORS spec (browsers reject it), but
+     * misconfigured servers do reach this state and any working XSS
+     * could exfiltrate cookies cross-origin. Surfaced as a critical
+     * security issue.
+     */
+    corsWildcardWithCredentials: number;
+    /**
+     * Pages with `Access-Control-Allow-Origin: *` (without credentials).
+     * Informational — not always wrong (public APIs do this) but worth
+     * reviewing on production HTML pages.
+     */
+    corsWildcardOrigin: number;
+    /**
+     * Internal URLs still served over plain HTTP. Modern Google rewards
+     * HTTPS and Chrome marks HTTP pages "Not Secure". Excludes localhost
+     * / 127.0.0.1 / .local hosts so local-dev crawls don't trip the
+     * filter.
+     */
+    httpNotHttps: number;
     /**
      * Pages with > 20 render-blocking head resources (escalated tier
      * above the existing > 5 "Render-Blocking Head" issue) — almost
@@ -1262,6 +1363,8 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   forceHttps: false,
   lowercasePath: false,
   trailingSlash: 'leave',
+  keepQueryParams: [],
+  urlRegexRewrites: [],
   memoryLimitMb: 0,
   maxQueueSize: 0,
   processPriority: 'normal',

packages/shared-types/src/ipc.ts

@@ -89,6 +89,8 @@ export const IPC = {
   logsBusy: 'logs:busy',
   robotsTest: 'robots:test',
   sitemapValidate: 'sitemap:validate',
+  urlRewritePreview: 'url:rewrite-preview',
+  urlClusterMembers: 'urls:cluster-members',
   reportsPagesPerDirectory: 'reports:pages-per-directory',
   reportsStatusCodeHistogram: 'reports:status-code-histogram',
   reportsDepthHistogram: 'reports:depth-histogram',
@@ -515,6 +517,14 @@ export interface TopUrlsInput {
   metric: TopUrlMetric;
   /** Default 25, capped at 500. */
   limit?: number;
+  /**
+   * Sort direction. `desc` (default) gives the conventional "Top N"
+   * leaderboards (most inlinks, slowest, biggest, deepest). `asc`
+   * surfaces the bottom of each metric — Least-Linked, fastest, smallest,
+   * shallowest. ASC mode excludes rows where the metric is 0 so it
+   * doesn't degenerate into an unlinked-pages report.
+   */
+  direction?: 'asc' | 'desc';
 }
 
 export interface TopUrlsRow {
@@ -676,6 +686,47 @@ export interface SitemapValidateResult {
 
 export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
 
+/**
+ * Inputs for the URL rewrite preview IPC. The renderer sends a sample
+ * URL plus the same set of normalisation knobs the crawler uses, so the
+ * Settings → URL Rewriting tab can show users exactly what their config
+ * will produce before they save it.
+ */
+export interface UrlRewritePreviewInput {
+  url: string;
+  stripWww: boolean;
+  forceHttps: boolean;
+  lowercasePath: boolean;
+  trailingSlash: 'leave' | 'strip' | 'add';
+  keepQueryParams: string[];
+  urlRegexRewrites: Array<{ pattern: string; replacement: string; flags?: string }>;
+}
+
+/**
+ * Per-URL near-duplicate cluster member row. Surfaced in the URL Details
+ * panel's "Duplicates" sub-tab so the user can jump from one page in a
+ * cluster to its sibling pages without leaving the detail context.
+ */
+export interface UrlClusterMember {
+  url: string;
+  statusCode: number | null;
+  indexability: Indexability;
+  title: string | null;
+  wordCount: number | null;
+  inlinks: number;
+  /** SimHash hamming distance from the queried URL (0 = identical fingerprint). */
+  hammingDistance: number;
+}
+
+export interface UrlRewritePreviewResult {
+  /** Final URL string after all rewrites. `null` if normalisation failed. */
+  result: string | null;
+  /** Per-regex compile errors (if any). Empty when all patterns valid. */
+  regexErrors: Array<{ pattern: string; error: string }>;
+  /** Set when the input URL itself failed to parse. */
+  parseError?: string;
+}
+
 export interface LogEntry {
   /** Monotonic sequence id, increments on every log call this session. */
   id: number;
@@ -735,6 +786,8 @@ export interface FreeCrawlApi {
   logsOpenWindow(): Promise<void>;
   robotsTest(input: RobotsTestInput): Promise<RobotsTestResult>;
   sitemapValidate(input: SitemapValidateInput): Promise<SitemapValidateResult>;
+  urlRewritePreview(input: UrlRewritePreviewInput): Promise<UrlRewritePreviewResult>;
+  urlClusterMembers(urlId: number): Promise<UrlClusterMember[]>;
   reportsPagesPerDirectory(input: PagesPerDirectoryInput): Promise<PagesPerDirectoryRow[]>;
   reportsStatusCodeHistogram(): Promise<StatusCodeHistogramRow[]>;
   reportsDepthHistogram(): Promise<DepthHistogramRow[]>;