feat: Faz 14 content analysis polish — boilerplate, canonical conflict, sitemap mismatch

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "private": true,
   "type": "module",
   "bin": {

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",

apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx

@@ -1502,6 +1502,16 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
     ['H5 Count', row.h5Count > 0 ? row.h5Count : null],
     ['H6 Count', row.h6Count > 0 ? row.h6Count : null],
     ['Word Count', row.wordCount],
+    [
+      'Boilerplate Coverage',
+      // Post-crawl pass samples up to ~2K pages; URLs outside the
+      // sample (or pages without a body snapshot) keep null. Show "—"
+      // for null instead of "0%" so the user can tell "not computed"
+      // apart from "0% boilerplate".
+      row.boilerplateCoverage !== null && row.boilerplateCoverage !== undefined
+        ? `${row.boilerplateCoverage}%`
+        : null,
+    ],
     ['Sentence Count', row.sentenceCount > 0 ? row.sentenceCount : null],
     [
       'Flesch Reading Ease',

apps/desktop/src/renderer/src/components/OverviewSidebar.tsx

@@ -611,6 +611,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.canonicalChainMultiHop,
               category: 'issues:canonical-chain-multi-hop',
             },
+            {
+              key: 'issues-canonical-conflict-near-duplicate',
+              label: 'Canonical Conflict (Near-Duplicate)',
+              count: o.issues.canonicalConflictNearDuplicate,
+              category: 'issues:canonical-conflict-near-duplicate',
+            },
           ],
         },
         {
@@ -629,6 +635,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
               count: o.issues.pageEmpty,
               category: 'issues:page-empty',
             },
+            {
+              key: 'issues-high-boilerplate',
+              label: 'High Boilerplate (>50%)',
+              count: o.issues.highBoilerplate,
+              category: 'issues:high-boilerplate',
+            },
             {
               key: 'issues-near-duplicate',
               label: 'Near-Duplicate Content',

apps/desktop/src/renderer/src/components/SettingsDialog.tsx

@@ -108,6 +108,7 @@ interface FormState {
   nearDuplicateHammingThreshold: string;
   duplicatesOnlyIndexable: boolean;
   dedupePreNormalize: boolean;
+  contentAreaSelector: string;
   // custom extraction
   customExtractionRules: CustomExtractionRule[];
   // webhook
@@ -402,6 +403,7 @@ function configToForm(c: CrawlConfig): FormState {
     nearDuplicateHammingThreshold: String(c.nearDuplicateHammingThreshold),
     duplicatesOnlyIndexable: c.duplicatesOnlyIndexable,
     dedupePreNormalize: c.dedupePreNormalize ?? true,
+    contentAreaSelector: c.contentAreaSelector ?? '',
     customExtractionRules: (c.customExtractionRules ?? []).map((r) => ({ ...r })),
     webhookUrl: c.webhookUrl ?? '',
     auth: { ...(c.auth ?? { type: 'none' }) },
@@ -597,6 +599,7 @@ export function SettingsDialog({ open, onClose }: Props) {
       ),
       duplicatesOnlyIndexable: form.duplicatesOnlyIndexable,
       dedupePreNormalize: form.dedupePreNormalize,
+      contentAreaSelector: form.contentAreaSelector.trim(),
       customExtractionRules: form.customExtractionRules
         .filter((r) => r.name.trim() && r.selector.trim())
         .slice(0, 10),
@@ -2291,6 +2294,27 @@ function DuplicatesPanel({ form, update }: PanelProps) {
         />
       </div>
 
+      <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
+        <div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
+          {t('settingsGroups.contentArea', { defaultValue: 'Content area' })}
+        </div>
+        <Text
+          label={t('settingsPanels.duplicates.contentAreaSelector', {
+            defaultValue: 'Content area CSS selector (optional)',
+          })}
+          value={form.contentAreaSelector}
+          onChange={(v) => update('contentAreaSelector', v)}
+          info="CSS selector that pins the duplicate-fingerprint text extraction to a specific page region. When set, the heuristic (main / role=main / article / body-minus-chrome) is bypassed and the selector wins. Useful on sites where the heuristic misclassifies — e.g. CMSes that wrap navigation inside `<main>` or sites with no semantic landmarks at all. Empty = use the heuristic. Invalid selectors silently fall back to the heuristic so a typo doesn't break the crawl."
+          example="article.main-content, #post-body, .article-body, div[itemprop='articleBody']"
+        />
+        <p className="mt-1 text-[10px] text-surface-500">
+          {t('settingsPanels.duplicates.contentAreaHint', {
+            defaultValue:
+              'Applies to NEW crawls. Existing crawls keep their old fingerprints until you re-crawl.',
+          })}
+        </p>
+      </div>
+
       <div className="rounded border border-surface-800 bg-surface-950/40 p-3 text-[10px] text-surface-500">
         <strong className="text-surface-300">{t('settingsPanels.duplicates.costLabel', { defaultValue: 'Cost:' })}</strong>{' '}
         {t('settingsPanels.duplicates.costBody', { defaultValue: 'SimHash adds ~5-10 ms per page during crawl; clustering itself runs after the last URL completes (~3-10 s at 1M URLs, <500 ms at 100K).' })}

apps/desktop/src/renderer/src/i18n/labels.ts

@@ -141,6 +141,8 @@ const TR_LABELS: Record<string, string> = {
   'Lang Attribute Missing': 'Lang Niteliği Eksik',
   'Charset Missing': 'Charset Eksik',
   'AMP Validation Errors': 'AMP Doğrulama Hataları',
+  'Canonical Conflict (Near-Duplicate)': 'Canonical Çakışması (Yakın-Yinelenen)',
+  'High Boilerplate (>50%)': 'Yüksek Şablon İçerik (>%50)',
 
   // Response performance
   'Slow (>1s)': 'Yavaş (>1sn)',

apps/desktop/src/renderer/src/store.ts

@@ -523,6 +523,7 @@ function tabForCategory(cat: UrlCategory): TabKey | null {
   if (
     cat === 'issues:content-thin' ||
     cat === 'issues:page-empty' ||
+    cat === 'issues:high-boilerplate' ||
     cat === 'issues:flesch-very-difficult' ||
     cat === 'issues:gunning-fog-very-high' ||
     cat === 'issues:page-large' ||

apps/mcp-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/mcp-server",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "private": true,
   "type": "module",
   "bin": {

apps/mcp-server/src/tools.ts

@@ -968,6 +968,50 @@ export function buildTools(): Tool[] {
       handler: (_args, db) => ({ rows: db.serverHeaderBreakdown() }),
     },
 
+    {
+      name: 'report_query_string_variants',
+      description:
+        'Base URLs (path without `?…`) that appear with 2+ query-string variants in the crawl. Each row carries the variant count and a sample of the actual `?…` strings — useful for spotting session-id / faceted-nav / `utm_*` parameter bloat that the URL normaliser did not strip. Use this to decide which params to add to Settings → URL Rewriting → Keep query parameters list, or which routes to exclude.',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          limit: { type: 'integer', minimum: 1, maximum: 5000, description: 'Max base URLs returned. Default 500.' },
+          sampleLimit: { type: 'integer', minimum: 1, maximum: 20, description: 'Per-group sample size for the query-string list. Default 5.' },
+        },
+      },
+      handler: (args, db) => ({
+        rows: db.getQueryStringVariantGroups({
+          limit: clamp(args.limit, 1, 5000, 500),
+          sampleLimit: clamp(args.sampleLimit, 1, 20, 5),
+        }),
+      }),
+    },
+
+    {
+      name: 'report_sitemap_priority_mismatch',
+      description:
+        'Sitemap-declared high-priority URLs whose actual crawl outcome contradicts the declared importance — 4xx/5xx page, noindex, canonicalised to another URL, redirected, robots-blocked, or never reached. The classic "sitemap lies to Google" signal. `priorityThreshold` defaults to 0.8 (most sitemap generators use this as the "important pages" floor). Each row carries the declared priority/changefreq/lastmod plus the actual status_code/indexability and a typed `issue` discriminator (not-crawled / status-error / noindex / canonicalised / redirect / blocked-robots).',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          priorityThreshold: {
+            type: 'number',
+            minimum: 0,
+            maximum: 1,
+            description: 'Only flag URLs whose declared sitemap priority is at least this value. Default 0.8.',
+          },
+          limit: { type: 'integer', minimum: 1, maximum: 10_000, description: 'Default 1000.' },
+        },
+      },
+      handler: (args, db) => ({
+        rows: db.getSitemapPriorityMismatch({
+          priorityThreshold:
+            typeof args.priorityThreshold === 'number' ? args.priorityThreshold : 0.8,
+          limit: clamp(args.limit, 1, 10_000, 1000),
+        }),
+      }),
+    },
+
     // =================================================================
     // V2 Faz 0.5 Increment 2 — Action tools. Each one POSTs through the
     // bridge's generic `/v1/action/<name>` route; the main process

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "freecrawl-seo",
-      "version": "0.6.7",
+      "version": "0.6.8",
       "hasInstallScript": true,
       "license": "MIT",
       "workspaces": [
@@ -26,7 +26,7 @@
     },
     "apps/cli": {
       "name": "@freecrawl/cli",
-      "version": "0.6.7",
+      "version": "0.6.8",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -43,7 +43,7 @@
     },
     "apps/desktop": {
       "name": "@freecrawl/desktop",
-      "version": "0.6.7",
+      "version": "0.6.8",
       "dependencies": {
         "@dnd-kit/core": "^6.3.1",
         "@dnd-kit/sortable": "^10.0.0",
@@ -90,7 +90,7 @@
     },
     "apps/mcp-server": {
       "name": "@freecrawl/mcp-server",
-      "version": "0.6.7",
+      "version": "0.6.8",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*"
@@ -8934,7 +8934,7 @@
     },
     "packages/core": {
       "name": "@freecrawl/core",
-      "version": "0.6.7",
+      "version": "0.6.8",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*",
@@ -8961,7 +8961,7 @@
     },
     "packages/db": {
       "name": "@freecrawl/db",
-      "version": "0.6.7",
+      "version": "0.6.8",
       "dependencies": {
         "@freecrawl/shared-types": "*"
       },
@@ -8973,7 +8973,7 @@
     },
     "packages/shared-types": {
       "name": "@freecrawl/shared-types",
-      "version": "0.6.7",
+      "version": "0.6.8",
       "devDependencies": {
         "rimraf": "^6.0.1",
         "typescript": "^5.7.0"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/crawler.ts

@@ -736,6 +736,32 @@ export class Crawler extends EventEmitter {
       // Duplicate clustering has JS-side simhash work that lives on
       // the crawler instance, not the DB; can't blindly off-thread it.
       this.runDuplicateClustering();
+
+      // V2 Faz 14 #5 — Template / boilerplate detection. Runs alongside
+      // duplicate clustering because both are content-similarity passes;
+      // this one specifically surfaces the repeated chrome (nav, footer,
+      // sidebar) that bleeds into every page and dilutes thin-content
+      // detection. Memory-bounded (samples ~2K bodies), skipped when
+      // no body snapshots are stored.
+      await yieldToEventLoop();
+      this.emit('info', 'Detecting boilerplate coverage…');
+      this.setOp('post-crawl:boilerplate-coverage');
+      try {
+        const result = this.db.recomputeBoilerplateCoverage();
+        if (result.sampled > 0) {
+          this.emit(
+            'info',
+            `Boilerplate: ${result.boilerplateShingles} shared shingles across ${result.sampled} sampled pages; ${result.pagesAboveHighThreshold} page(s) > 50% template`,
+          );
+        }
+      } catch (err) {
+        this.emit(
+          'error',
+          new Error(
+            `recomputeBoilerplateCoverage failed: ${err instanceof Error ? err.message : String(err)}`,
+          ),
+        );
+      }
     }
     if (this.config.analysePagination) {
       await yieldToEventLoop();
@@ -2204,6 +2230,7 @@ export class Crawler extends EventEmitter {
         customSearchTerms: this.config.customSearchTerms,
         urlRewrites: this.urlRewrites,
         customExtractionRules: this.config.customExtractionRules,
+        contentAreaSelector: this.config.contentAreaSelector,
       });
 
       // Charset resolution — prefer the document's own declaration (HTML5

packages/core/src/html-parser.ts

@@ -331,9 +331,31 @@ export interface ParsedPage {
  * fingerprint uses this; `wordCount` / readability stay on the full
  * body so existing thin-content thresholds don't shift.
  */
-function extractMainContentText($: cheerio.CheerioAPI): string {
+function extractMainContentText(
+  $: cheerio.CheerioAPI,
+  userSelector?: string,
+): string {
   const collapse = (s: string): string => s.replace(/\s+/g, ' ').trim();
 
+  // V2 Faz 14 — user-supplied content area selector takes precedence
+  // over the heuristic. Lets duplicate fingerprinting target a specific
+  // region (e.g. `article.main-content`, `#content`) when the heuristic
+  // misclassifies — common on CMSes where the navigation sits inside
+  // `<main>` or where there's no semantic landmark at all. Invalid /
+  // unmatched selectors silently fall through to the heuristic so a
+  // typo doesn't break the crawl.
+  if (userSelector && userSelector.trim().length > 0) {
+    try {
+      const region = $(userSelector.trim()).first();
+      if (region.length > 0) {
+        const t = collapse(region.text());
+        if (t.length > 0) return t;
+      }
+    } catch {
+      // Bad selector — fall through to heuristic.
+    }
+  }
+
   const main = $('main, [role="main"]').first();
   if (main.length > 0) {
     const t = collapse(main.text());
@@ -372,6 +394,10 @@ export function parseHtml(
     urlRewrites?: UrlRewriteOptions;
     /** Custom Extraction rules to run against this page. */
     customExtractionRules?: ReadonlyArray<CustomExtractionRule>;
+    /** Optional CSS selector that pins duplicate-fingerprint text
+     *  extraction to a specific region. Empty / missing → heuristic
+     *  (main/article/body-minus-chrome) is used. */
+    contentAreaSelector?: string;
   } = {},
 ): ParsedPage {
   // Fast path: force the htmlparser2 backend and skip entity decoding.
@@ -785,7 +811,7 @@ export function parseHtml(
   // pages that share a big nav/footer template but differ in body
   // content aren't falsely clustered as near-duplicates.
   const { simhash, contentHash } = computeContentFingerprint(
-    extractMainContentText($),
+    extractMainContentText($, opts.contentAreaSelector),
   );
 
   // Custom Extraction — runs after all standard fields so the cheerio

packages/core/src/html-report.ts

@@ -127,6 +127,7 @@ const ISSUES: IssueDef[] = [
   { key: 'faviconMissing', label: 'Favicon Missing', severity: 'info' },
   { key: 'charsetMissing', label: 'Charset Missing', severity: 'warn' },
   { key: 'ampValidationErrors', label: 'AMP Validation Errors', severity: 'warn' },
+  { key: 'highBoilerplate', label: 'High Boilerplate (>50%)', severity: 'warn' },
   { key: 'nonIndexableInSitemap', label: 'Non-Indexable in Sitemap', severity: 'error' },
   { key: 'non200InSitemap', label: 'Non-200 in Sitemap', severity: 'error' },
   { key: 'redirectInSitemap', label: 'Redirect in Sitemap', severity: 'warn' },

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/migrations.ts

@@ -1526,6 +1526,26 @@ const MIGRATIONS: Migration[] = [
       }
     },
   },
+  {
+    version: 65,
+    name: 'add_boilerplate_coverage',
+    // V2 Faz 14 #5 — Template / boilerplate detection. Memory-bounded
+    // post-crawl pass samples up to ~2K stored bodies, generates
+    // 5-word shingles (FNV-1a hashed), identifies shingles appearing
+    // on > 30% of sampled pages as "boilerplate", and writes a per-URL
+    // coverage percentage (0-100) to this column. NULL = not yet
+    // computed (or page wasn't in the sample / had no body snapshot).
+    // 50+ is the "high boilerplate" threshold for the dedicated issue
+    // filter — typical templated pages with thin content land here.
+    up: (db) => {
+      const cols = db
+        .prepare('PRAGMA table_info(urls)')
+        .all() as unknown as { name: string }[];
+      if (!cols.some((c) => c.name === 'boilerplate_coverage')) {
+        db.exec('ALTER TABLE urls ADD COLUMN boilerplate_coverage INTEGER');
+      }
+    },
+  },
 ];
 
 export function runMigrations(db: DatabaseSync): void {

packages/db/src/project-db.ts

@@ -101,6 +101,10 @@ interface UrlRowDb {
   /** JSON string array of AMP smoke-validator error codes. Empty when
    *  not an AMP page or when validation passes. */
   amp_validation_errors: string | null;
+  /** V2 Faz 14 #5 — per-URL boilerplate coverage percentage (0-100)
+   *  from the sampled post-crawl pass. NULL when the page wasn't in
+   *  the sample (body snapshot missing, or sample LIMIT exceeded). */
+  boilerplate_coverage: number | null;
   favicon: string | null;
   mixed_content_count: number;
   mixed_content_active: number;
@@ -2007,6 +2011,163 @@ export class ProjectDb {
     return { clusters: rootToCluster.size, clusteredUrls };
   }
 
+  /**
+   * V2 Faz 14 #5 — Template / boilerplate detection.
+   *
+   * Memory-bounded post-crawl pass that samples up to `sampleSize` pages
+   * (default 2000), extracts plain text from each stored body, generates
+   * fixed-length word shingles (default 5-word window), hashes them with
+   * FNV-1a 32-bit, and tracks per-shingle page coverage. Shingles
+   * appearing on more than `pageThresholdPct` of the sample (default
+   * 30%) are flagged as "boilerplate" — repeated chrome (nav, footer,
+   * sidebar copy) that bleeds into every page. Each sampled page gets
+   * a `boilerplate_coverage` percentage (0-100) = ratio of its unique
+   * shingles that are boilerplate. Pages with no body snapshot stay
+   * NULL.
+   *
+   * Why sampling instead of full-crawl pass: at 100K URLs × 50KB body
+   * × 100 unique 5-word shingles, a full pass needs ~5GB of working
+   * memory for the shingle→pageSet inverted index. Sampling 2K bodies
+   * keeps it under 200MB while still surfacing the same boilerplate
+   * (templated chrome doesn't change between page 2000 and page 100000).
+   *
+   * Pages with < shingleSize words contribute nothing (no shingles
+   * generated) and get coverage = 0.
+   */
+  recomputeBoilerplateCoverage(
+    opts: {
+      sampleSize?: number;
+      shingleSize?: number;
+      pageThresholdPct?: number;
+    } = {},
+  ): {
+    sampled: number;
+    boilerplateShingles: number;
+    pagesAboveHighThreshold: number;
+  } {
+    const sampleSize = Math.max(1, Math.min(20_000, opts.sampleSize ?? 2000));
+    const shingleSize = Math.max(2, Math.min(10, opts.shingleSize ?? 5));
+    const pageThresholdPct = Math.max(
+      5,
+      Math.min(80, opts.pageThresholdPct ?? 30),
+    );
+
+    // FNV-1a 32-bit hash — compact, fast, decent distribution for
+    // text shingles. Returned as unsigned via `>>> 0`.
+    const fnv1a32 = (s: string): number => {
+      let h = 0x811c9dc5;
+      for (let i = 0; i < s.length; i++) {
+        h = (h ^ s.charCodeAt(i)) >>> 0;
+        h = Math.imul(h, 0x01000193) >>> 0;
+      }
+      return h;
+    };
+
+    // Rough HTML-to-text — strips `<script>` / `<style>` blocks, then
+    // every remaining tag, then named entities, collapses whitespace,
+    // lowercases. We don't bother with a full cheerio parse because the
+    // boilerplate-detection signal is robust to noise (repeated
+    // navigation copy is repeated whether we get tags right or not),
+    // and the 2K-page pass needs to be fast.
+    const stripHtml = (html: string): string =>
+      html
+        .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+        .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+        .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
+        .replace(/<!--[\s\S]*?-->/g, ' ')
+        .replace(/<[^>]+>/g, ' ')
+        .replace(/&[#a-z0-9]+;/gi, ' ')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .toLowerCase();
+
+    const shingleHashes = (text: string): Set<number> => {
+      const words = text.split(' ').filter((w) => w.length > 0);
+      const out = new Set<number>();
+      if (words.length < shingleSize) return out;
+      for (let i = 0; i + shingleSize <= words.length; i++) {
+        out.add(fnv1a32(words.slice(i, i + shingleSize).join(' ')));
+      }
+      return out;
+    };
+
+    // Phase 1 — collect per-page shingle sets + shingle→pageCount map.
+    // We hold one Set<number> per sampled page; each Set has ~100-2000
+    // entries depending on body length. Memory budget: 2000 pages ×
+    // 1000 shingles × ~30 bytes ≈ 60MB worst case.
+    const pageShingles = new Map<number, Set<number>>();
+    const shingleCounts = new Map<number, number>();
+
+    const rows = this.db
+      .prepare(
+        `SELECT u.id AS id, us.body AS body
+           FROM urls u
+           JOIN url_sources us ON us.url_id = u.id
+          WHERE u.is_external = 0
+            AND u.content_kind = 'html'
+            AND us.body IS NOT NULL
+            AND LENGTH(us.body) > 0
+          ORDER BY u.id
+          LIMIT ?`,
+      )
+      .all(sampleSize) as { id: number; body: string }[];
+
+    for (const row of rows) {
+      const shingles = shingleHashes(stripHtml(row.body));
+      pageShingles.set(row.id, shingles);
+      for (const h of shingles) {
+        shingleCounts.set(h, (shingleCounts.get(h) ?? 0) + 1);
+      }
+    }
+
+    // Phase 2 — identify boilerplate shingles (appear on more than
+    // pageThresholdPct% of sampled pages, AND on at least 2 pages so
+    // a single-page crawl never flags itself).
+    const totalSampled = pageShingles.size;
+    const minPages = Math.max(2, Math.ceil((totalSampled * pageThresholdPct) / 100));
+    const boilerplateSet = new Set<number>();
+    for (const [h, n] of shingleCounts) {
+      if (n >= minPages) boilerplateSet.add(h);
+    }
+
+    // Phase 3 — write per-page coverage. Wipe stale values from prior
+    // runs first so pages that dropped out of the sample (because the
+    // crawl grew and bumped them off the LIMIT) don't keep a stale
+    // coverage from an older state.
+    const update = this.db.prepare(
+      'UPDATE urls SET boilerplate_coverage = ? WHERE id = ?',
+    );
+    const wipe = this.db.prepare(
+      "UPDATE urls SET boilerplate_coverage = NULL WHERE is_external = 0 AND content_kind = 'html'",
+    );
+    let pagesAboveHighThreshold = 0;
+    this.db.exec('BEGIN');
+    try {
+      wipe.run();
+      for (const [urlId, shingles] of pageShingles) {
+        if (shingles.size === 0) {
+          update.run(0, urlId);
+          continue;
+        }
+        let boilerCount = 0;
+        for (const h of shingles) if (boilerplateSet.has(h)) boilerCount++;
+        const coverage = Math.round((boilerCount / shingles.size) * 100);
+        if (coverage >= 50) pagesAboveHighThreshold++;
+        update.run(coverage, urlId);
+      }
+      this.db.exec('COMMIT');
+    } catch (err) {
+      this.db.exec('ROLLBACK');
+      throw err;
+    }
+
+    return {
+      sampled: totalSampled,
+      boilerplateShingles: boilerplateSet.size,
+      pagesAboveHighThreshold,
+    };
+  }
+
   /**
    * Page through near-duplicate clusters for the dedicated Duplicates tab.
    * Members are returned grouped: ORDER BY cluster_size DESC, cluster_id,
@@ -2437,6 +2598,160 @@ export class ProjectDb {
       .slice(0, limit);
   }
 
+  /**
+   * V2 Faz 14 — Sitemap priority / changefreq inconsistency report.
+   * Surfaces sitemap-declared high-priority URLs whose actual crawl
+   * outcome contradicts that priority: page returned 4xx/5xx, was
+   * marked noindex, was canonicalised to another URL, redirected, or
+   * never reached. A `priority` ≥ `priorityThreshold` (default 0.8 —
+   * the "important page" floor most sitemap generators use) combined
+   * with a quality issue means the sitemap is misleading search
+   * engines. Also surfaces the changefreq mismatch when present
+   * (e.g. `daily` declared on a page whose `lastmod` is months old —
+   * a soft signal that the sitemap generator is stale).
+   */
+  getSitemapPriorityMismatch(
+    opts: { priorityThreshold?: number; limit?: number } = {},
+  ): {
+    url: string;
+    declaredPriority: number;
+    declaredChangefreq: string | null;
+    declaredLastmod: string | null;
+    sourceSitemap: string;
+    actualStatusCode: number | null;
+    actualIndexability: string | null;
+    issue:
+      | 'not-crawled'
+      | 'status-error'
+      | 'noindex'
+      | 'canonicalised'
+      | 'redirect'
+      | 'blocked-robots';
+  }[] {
+    const threshold = Math.max(0, Math.min(1, opts.priorityThreshold ?? 0.8));
+    const limit = Math.max(1, Math.min(10_000, opts.limit ?? 1000));
+    const rows = this.db
+      .prepare(
+        `SELECT
+           s.url            AS url,
+           s.priority       AS priority,
+           s.changefreq     AS changefreq,
+           s.lastmod        AS lastmod,
+           s.source_sitemap AS source_sitemap,
+           u.status_code    AS status_code,
+           u.indexability   AS indexability
+         FROM sitemap_urls s
+         LEFT JOIN urls u ON u.url = s.url
+         WHERE s.priority IS NOT NULL AND s.priority >= ?
+         ORDER BY s.priority DESC, s.url ASC`,
+      )
+      .all(threshold) as {
+      url: string;
+      priority: number;
+      changefreq: string | null;
+      lastmod: string | null;
+      source_sitemap: string;
+      status_code: number | null;
+      indexability: string | null;
+    }[];
+    const out: ReturnType<ProjectDb['getSitemapPriorityMismatch']> = [];
+    for (const r of rows) {
+      let issue: (typeof out)[number]['issue'] | null = null;
+      if (r.status_code === null || r.indexability === null) {
+        issue = 'not-crawled';
+      } else if (r.status_code >= 400) {
+        issue = 'status-error';
+      } else if (r.indexability === 'non-indexable:noindex') {
+        issue = 'noindex';
+      } else if (r.indexability === 'non-indexable:canonical') {
+        issue = 'canonicalised';
+      } else if (r.indexability === 'non-indexable:redirect') {
+        issue = 'redirect';
+      } else if (r.indexability === 'non-indexable:robots-blocked') {
+        issue = 'blocked-robots';
+      }
+      if (issue === null) continue;
+      out.push({
+        url: r.url,
+        declaredPriority: r.priority,
+        declaredChangefreq: r.changefreq,
+        declaredLastmod: r.lastmod,
+        sourceSitemap: r.source_sitemap,
+        actualStatusCode: r.status_code,
+        actualIndexability: r.indexability,
+        issue,
+      });
+      if (out.length >= limit) break;
+    }
+    return out;
+  }
+
+  /**
+   * V2 Faz 14 — Query-string variant grouping. Surface base URLs (path
+   * without `?…`) that have multiple variant URLs in the crawl —
+   * typically session IDs / faceted nav / utm_*-style trackers that
+   * the URL normaliser didn't strip (or that the user disabled
+   * stripping for). Each group carries variant count + sample query
+   * strings so the user can decide whether to add the params to the
+   * normaliser's strip-list or to exclude them via include/exclude
+   * patterns. Excludes URLs without a query string (they can't be in
+   * a multi-variant group by definition) and singletons (count = 1).
+   */
+  getQueryStringVariantGroups(
+    opts: { limit?: number; sampleLimit?: number } = {},
+  ): {
+    baseUrl: string;
+    variantCount: number;
+    sampleQueries: string[];
+  }[] {
+    const limit = Math.max(1, Math.min(5000, opts.limit ?? 500));
+    const sampleLimit = Math.max(1, Math.min(20, opts.sampleLimit ?? 5));
+    const rows = this.db
+      .prepare(
+        `SELECT url FROM urls
+          WHERE is_external = 0
+            AND content_kind = 'html'
+            AND INSTR(url, '?') > 0`,
+      )
+      .all() as { url: string }[];
+    // SQLite SUBSTR with INSTR works in principle but URL parsing in JS
+    // is more robust (handles bare `#`, malformed query, etc.). Group
+    // in memory — query-bearing pages are a small subset of the total
+    // crawl in practice (handful of K at most for million-URL crawls).
+    const groups = new Map<string, { count: number; samples: string[] }>();
+    for (const r of rows) {
+      let baseUrl: string;
+      let query: string;
+      try {
+        const u = new URL(r.url);
+        baseUrl = `${u.protocol}//${u.host}${u.pathname}`;
+        query = u.search; // includes leading '?'
+        if (!query) continue;
+      } catch {
+        continue;
+      }
+      let g = groups.get(baseUrl);
+      if (!g) {
+        g = { count: 0, samples: [] };
+        groups.set(baseUrl, g);
+      }
+      g.count += 1;
+      if (g.samples.length < sampleLimit) g.samples.push(query);
+    }
+    return [...groups.entries()]
+      .filter(([, v]) => v.count > 1)
+      .map(([baseUrl, v]) => ({
+        baseUrl,
+        variantCount: v.count,
+        sampleQueries: v.samples,
+      }))
+      .sort(
+        (a, b) =>
+          b.variantCount - a.variantCount || a.baseUrl.localeCompare(b.baseUrl),
+      )
+      .slice(0, limit);
+  }
+
   /**
    * Status-code histogram across internal URLs (every kind, not just HTML —
    * users want to see image 4xx, JS 5xx, etc.). Null status (network error)
@@ -3816,6 +4131,21 @@ export class ProjectDb {
          AND amp_validation_errors != ''
          AND amp_validation_errors != '[]'`,
       ),
+      canonicalConflictNearDuplicate: countWhere(
+        `${html} AND cluster_id > 0 AND cluster_size > 1
+         AND canonical IS NOT NULL AND canonical != ''
+         AND EXISTS (
+           SELECT 1 FROM urls u2
+           WHERE u2.cluster_id = urls.cluster_id
+             AND u2.is_external = 0 AND u2.content_kind = 'html'
+             AND u2.canonical IS NOT NULL AND u2.canonical != ''
+             AND u2.canonical != urls.canonical
+             AND u2.url != urls.url
+         )`,
+      ),
+      highBoilerplate: countWhere(
+        `${html} AND boilerplate_coverage IS NOT NULL AND boilerplate_coverage > 50`,
+      ),
       brokenLinksInternal: this.countBrokenLinks('internal'),
       brokenLinksExternal: this.countBrokenLinks('external'),
       nearDuplicate: countWhere(`${html} AND cluster_id > 0 AND cluster_size > 1`),
@@ -5849,6 +6179,7 @@ export class ProjectDb {
     amphtml: r.amphtml,
     ampPage: r.amp_page === 1,
     ampValidationErrors: r.amp_validation_errors,
+    boilerplateCoverage: r.boilerplate_coverage,
     favicon: r.favicon,
     mixedContentCount: r.mixed_content_count,
     mixedContentActive: r.mixed_content_active ?? 0,
@@ -6421,6 +6752,38 @@ function categoryWhereClause(cat: UrlCategory): string | null {
                 SELECT 1 FROM urls t WHERE t.url = urls.canonical
                   AND t.indexability = 'non-indexable:noindex'
               )`;
+    case 'issues:high-boilerplate':
+      // V2 Faz 14 #5 — Page's unique 5-word shingles are >50%
+      // boilerplate (shingles that repeat across >30% of the sampled
+      // 2K pages — typically nav / footer / sidebar copy). High
+      // coverage = templated thin content; the actual unique body is
+      // a small fraction of the rendered page. Excludes pages that
+      // weren't in the sample (NULL coverage) so the filter never
+      // false-positives on bodies we never analysed.
+      return `is_external = 0 AND content_kind = 'html'
+              AND boilerplate_coverage IS NOT NULL
+              AND boilerplate_coverage > 50`;
+    case 'issues:canonical-conflict-near-duplicate':
+      // V2 Faz 14 — two pages clustered as near-duplicates (same
+      // SimHash band / cluster_id > 0) but pointing to DIFFERENT
+      // canonicals. Classic CMS misconfig: localised variants or
+      // pagination shards drift apart on the canonical signal even
+      // though Google sees them as the same content. Distinct from
+      // `issues:canonical-mismatch` (HTML vs HTTP header on the same
+      // page); this is cross-page within a near-duplicate cluster.
+      // Excludes singletons (cluster_id = 0) and pages without a
+      // declared canonical (those are caught by `issues:canonical-missing`).
+      return `is_external = 0 AND content_kind = 'html'
+              AND cluster_id > 0 AND cluster_size > 1
+              AND canonical IS NOT NULL AND canonical != ''
+              AND EXISTS (
+                SELECT 1 FROM urls u2
+                WHERE u2.cluster_id = urls.cluster_id
+                  AND u2.is_external = 0 AND u2.content_kind = 'html'
+                  AND u2.canonical IS NOT NULL AND u2.canonical != ''
+                  AND u2.canonical != urls.canonical
+                  AND u2.url != urls.url
+              )`;
     case 'issues:content-thin':
       return "is_external = 0 AND content_kind = 'html' AND word_count IS NOT NULL AND word_count < 300";
     case 'issues:response-slow':

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/crawl.ts

@@ -102,6 +102,8 @@ export type UrlCategory =
   | 'issues:meta-refresh-used'
   | 'issues:charset-missing'
   | 'issues:amp-validation-errors'
+  | 'issues:canonical-conflict-near-duplicate'
+  | 'issues:high-boilerplate'
   | 'issues:broken-links-all'
   | 'issues:broken-links-internal'
   | 'issues:broken-links-external'
@@ -454,6 +456,12 @@ export interface CrawlUrlRow {
    *  `["missing-boilerplate","forbidden-script-tag"]`), or null when
    *  not an AMP page / clean validation. */
   ampValidationErrors: string | null;
+  /** V2 Faz 14 #5 — Boilerplate coverage percentage (0-100) from the
+   *  post-crawl sampling pass. Pages whose unique 5-word shingles are
+   *  >50% boilerplate (= repeated across the cluster of sampled pages)
+   *  are flagged as templated / thin-content. NULL when the page
+   *  wasn't in the sample (no body snapshot, or sample LIMIT exceeded). */
+  boilerplateCoverage: number | null;
   favicon: string | null;
   /** Resolved `<link rel="apple-touch-icon">` URL, else null. */
   appleTouchIcon: string | null;
@@ -691,6 +699,16 @@ export interface CrawlConfig {
    * surfaces issues that actually affect search visibility.
    */
   duplicatesOnlyIndexable: boolean;
+  /**
+   * V2 Faz 14 — Optional CSS selector that pins the duplicate-fingerprint
+   * text extraction to a specific page region (e.g. `article.main`,
+   * `#content`). When set, the heuristic (main / role=main / article /
+   * body-minus-chrome) is bypassed and the selector wins. Empty string
+   * falls back to the heuristic — typical for sites with semantic
+   * HTML5 landmarks. Invalid selectors silently degrade to the
+   * heuristic so a typo doesn't break the crawl.
+   */
+  contentAreaSelector: string;
   /**
    * Optional webhook URL that receives a single `POST` with a JSON
    * summary when a crawl finishes. Empty string disables it. Failures
@@ -1188,6 +1206,8 @@ export interface OverviewCounts {
     metaRefreshUsed: number;
     charsetMissing: number;
     ampValidationErrors: number;
+    canonicalConflictNearDuplicate: number;
+    highBoilerplate: number;
     brokenLinksInternal: number;
     brokenLinksExternal: number;
     nearDuplicate: number;
@@ -1656,6 +1676,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   processPriority: 'normal',
   nearDuplicateHammingThreshold: 3,
   duplicatesOnlyIndexable: true,
+  contentAreaSelector: '',
   webhookUrl: '',
   customExtractionRules: [],
   auth: { type: 'none' },