feat: benchmark-driven SEO checks and false-alarm fixes

apps/desktop/src/main/db-writer-worker.ts

@@ -126,6 +126,7 @@ const ALLOWED_METHODS = new Set<string>([
   'recomputeHreflangAnalysis',
   'recomputeHreflangInconsistent',
   'recomputeDuplicateClusters',
+  'markUrlTrap',
   'recomputeBoilerplateCoverage',
   'recomputePaginationSequence',
   'recomputeUrlsIssues',

apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx

@@ -1667,6 +1667,7 @@ function DuplicatesView({
           disabled={members.length === 0}
           getData={() => ({
             headers: [
+              translateLabel('Similarity %', lang),
               translateLabel('Hamming', lang),
               'URL',
               translateLabel('Status', lang),
@@ -1676,6 +1677,7 @@ function DuplicatesView({
               translateLabel('Title', lang),
             ],
             rows: members.map((m) => [
+              m.similarity ?? '',
               m.hammingDistance,
               m.url,
               m.statusCode ?? '',
@@ -1691,6 +1693,7 @@ function DuplicatesView({
         <table className="w-full text-[11px]">
           <thead className="sticky top-0 bg-surface-900 text-left text-[10px] text-surface-400">
             <tr>
+              <th className="px-3 py-1 font-normal">{translateLabel('Similarity %', lang)}</th>
               <th className="px-3 py-1 font-normal">{translateLabel('Hamming', lang)}</th>
               <th className="px-3 py-1 font-normal">URL</th>
               <th className="px-3 py-1 font-normal">{translateLabel('Status', lang)}</th>
@@ -1707,6 +1710,9 @@ function DuplicatesView({
                 className="border-t border-surface-800 hover:bg-surface-900"
               >
                 <td className="px-3 py-1 font-mono text-emerald-300">
+                  {m.similarity !== null ? `${m.similarity}%` : '—'}
+                </td>
+                <td className="px-3 py-1 font-mono text-surface-300">
                   {m.hammingDistance}
                 </td>
                 <td

apps/desktop/src/renderer/src/components/SettingsDialog.tsx

@@ -175,6 +175,7 @@ interface FormState {
   processPriority: 'normal' | 'below-normal' | 'idle';
   // duplicates
   nearDuplicateHammingThreshold: string;
+  nearDuplicateSimilarity: string;
   duplicatesOnlyIndexable: boolean;
   dedupePreNormalize: boolean;
   contentAreaSelector: string;
@@ -583,7 +584,7 @@ function configToForm(c: CrawlConfig): FormState {
     storeExternalLinks: c.storeExternalLinks ?? true,
     storeCanonicals: c.storeCanonicals ?? true,
     storePagination: c.storePagination ?? true,
-    crawlHreflang: c.crawlHreflang ?? false,
+    crawlHreflang: c.crawlHreflang ?? true,
     storeHreflang: c.storeHreflang ?? true,
     crawlAmp: c.crawlAmp ?? false,
     storeAmp: c.storeAmp ?? true,
@@ -598,7 +599,7 @@ function configToForm(c: CrawlConfig): FormState {
     crawlInvalidLinks: c.crawlInvalidLinks ?? false,
     crawlQueryStrings: c.crawlQueryStrings ?? true,
     crawlQueryStringExceptionsText: (c.crawlQueryStringExceptions ?? []).join(', '),
-    crawlLinkedSitemaps: c.crawlLinkedSitemaps ?? false,
+    crawlLinkedSitemaps: c.crawlLinkedSitemaps ?? true,
     userAgent: c.userAgent,
     acceptLanguage: c.acceptLanguage,
     customHeadersText: Object.entries(c.customHeaders ?? {})
@@ -622,6 +623,7 @@ function configToForm(c: CrawlConfig): FormState {
     maxQueueSize: String(c.maxQueueSize),
     processPriority: c.processPriority,
     nearDuplicateHammingThreshold: String(c.nearDuplicateHammingThreshold),
+    nearDuplicateSimilarity: String(c.nearDuplicateSimilarity ?? 90),
     duplicatesOnlyIndexable: c.duplicatesOnlyIndexable,
     dedupePreNormalize: c.dedupePreNormalize ?? true,
     contentAreaSelector: c.contentAreaSelector ?? '',
@@ -667,7 +669,7 @@ function configToForm(c: CrawlConfig): FormState {
     maxUrlLength: String(c.maxUrlLength ?? 2048),
     maxQueryStringLength: String(c.maxQueryStringLength ?? 0),
     maxFolderDepth: String(c.maxFolderDepth ?? 0),
-    followCanonicals: c.followCanonicals ?? false,
+    followCanonicals: c.followCanonicals ?? true,
     followPaginationLinks: c.followPaginationLinks ?? true,
     followNofollow: c.followNofollow ?? false,
     followJsRedirects: c.followJsRedirects ?? false,
@@ -956,6 +958,10 @@ export function SettingsDialog({ open, onClose }: Props) {
           num(form.nearDuplicateHammingThreshold, config.nearDuplicateHammingThreshold),
         ),
       ),
+      nearDuplicateSimilarity: Math.max(
+        0,
+        Math.min(100, num(form.nearDuplicateSimilarity, config.nearDuplicateSimilarity ?? 90)),
+      ),
       duplicatesOnlyIndexable: form.duplicatesOnlyIndexable,
       dedupePreNormalize: form.dedupePreNormalize,
       contentAreaSelector: form.contentAreaSelector.trim(),
@@ -3787,12 +3793,22 @@ function DuplicatesPanel({ form, update }: PanelProps) {
           label={t('settingsPanels.duplicates.maxHamming', { defaultValue: 'Max Hamming distance (0 = exact only, 12 = very loose, 0 disables)' })}
           value={form.nearDuplicateHammingThreshold}
           onChange={(v) => update('nearDuplicateHammingThreshold', v)}
-          info="Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."
+          info="Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."
           example="3 = recommended; 5 catches looser duplicates (templated content with light variation); 0 turns the post-crawl pass off."
         />
         <p className="mt-1 text-[10px] text-surface-500">
           {t('settingsPanels.duplicates.thresholdHint', { defaultValue: 'Lower = stricter. 3 is the SF-equivalent default. Pages with too little body content (<50 characters) are excluded from clustering regardless of threshold.' })}
         </p>
+        <Num
+          label={t('settingsPanels.duplicates.minSimilarity', { defaultValue: 'Minimum text similarity % (0 disables)' })}
+          value={form.nearDuplicateSimilarity}
+          onChange={(v) => update('nearDuplicateSimilarity', v)}
+          info="Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."
+          example="90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."
+        />
+        <p className="mt-1 text-[10px] text-surface-500">
+          {t('settingsPanels.duplicates.similarityHint', { defaultValue: 'Pages whose main-content wording is at least this similar are clustered too — this catches rewritten or lightly edited copies that SimHash misses. 90 is the SF-equivalent default.' })}
+        </p>
       </div>
 
       <div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
@@ -4228,7 +4244,7 @@ function AdvancedPanel({ form, update }: PanelProps) {
           label={t('settingsPanels.advanced.followCanonical', { defaultValue: 'Follow canonical targets' })}
           checked={form.followCanonicals}
           onChange={(v) => update('followCanonicals', v)}
-          info="When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint."
+          info="When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on."
         />
         <Bool
           label={t('settingsPanels.advanced.followNextPrev', { defaultValue: 'Follow rel=next / rel=prev' })}

apps/desktop/src/renderer/src/i18n/info-tips/az.ts

@@ -454,8 +454,8 @@ export const AZ_INFO_TIPS: Record<string, string> = {
     "LENGTH(url) > bu olduqda 'URL çox uzundur' problemi işə düşür. RFC 7230 maksimum tələb etmir, amma serverlərin və ara qurğuların əksəriyyəti ~2 KB-dan yuxarı uğursuz olur; Chrome özü ~32 KB-da məhdudlaşdırır.",
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     'Hər sətirdə iki rejim. (1) Regex üçün slaşlarla bükün: /şablon/bayraqlar — dəstəklənən bayraqlar imsuy (g məcburidir). Yanlış şablonlar detal panelində -1 sayı ilə görünür ki, yazı səhvini tapa biləsiniz. (2) Qalan hər şey hərf registrinə həssas olmayan hərfi alt-sətirdir — köhnə davranış. Hər terminin səhifə başına uyğunluq sayı URL detalları panelində göstərilir.',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    'İki səhifənin 64-bit SimHash-i ən çox bu qədər bit fərqlənirsə oxşar-təkrar kimi işarələnir. 3 ≈ gövdə mətni shingle-ları üzərində 95% oxşarlıq (Screaming Frog-un ən sıx filtri). Klasterləşdirməni tamamilə atlamaq üçün 0 təyin edin.',
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "İki səhifənin 64 bitlik SimHash dəyərləri ən çox bu qədər bit fərqlənirsə, onlar yaxın dublikat kimi işarələnir — bu, demək olar ki, eyni nüsxələri tutur. Böyük ölçüdə eyni olan mətnlər aşağıdakı oxşarlıq həddi ilə qiymətləndirilir. Bu mərhələni keçmək üçün 0 yazın.",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     'İlk <link rel="canonical"> teqi ilə bəyan edilmiş URL. Təkrarlar olduqda axtarış sistemlərinə hansı versiyanı indeksləməli olduğunu bildirir.',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -488,8 +488,8 @@ export const AZ_INFO_TIPS: Record<string, string> = {
     'AKTİV olduqda (standart) Təkrar URL filtri URL-ləri hostu kiçik hərfə çevirib, sorğu sətrini atıb və sonluq slaşını kəsib müqayisə edir — kanonik SEO davranışı. SÖNDÜRÜLÜ olduqda müqayisə bayt-dəqiqdir, ona görə filtr yalnız eyni xam URL sətrini paylaşan sətirlərdə işləyir (nadirdir, çünki URL-lər daxil edilərkən təkrarsızlaşdırılır).',
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "Aktiv olduqda `<meta http-equiv='refresh'>` content URL-ləri yönləndirmə hədəfi kimi növbəyə qoyulur. window.location gövdə yönləndirmələri yalnız evristikdir və hazırda əhatə xaricindədir.",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    'Aktiv olduqda başqa yerə kanonik bəyan edən 200 səhifə həmin hədəfi də növbəyə qoyur. Standart söndürülü — taramaların əksəriyyəti kanonikləri naviqasiya göstərişi yox, siqnal kimi qəbul edir.',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    'Aktiv olduqda başqa yerə kanonik bəyan edən 200 səhifə həmin hədəfi də növbəyə qoyur, beləliklə mövcud olmayan və ya yönləndirilən səhifəyə göstərən kanonik aşkarlanır. Standart aktivdir.',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     'Aktiv olduqda noindex / kanonikləşdirilmiş / robots ilə bloklanmış indekslənmə statusu olan səhifələr klasterləşdirmədən çıxarılır — Oxşar-təkrar hesabatı o zaman yalnız axtarış görünürlüyünə təsir edən problemləri göstərir.',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -662,4 +662,18 @@ export const AZ_INFO_TIPS: Record<string, string> = {
     'Səhifələri heç vaxt sənəd sorğusu yaratmayan React Router / Vue Router / Angular saytları üçün aktiv.',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     '`<link rel="alternate" media="only screen and (max-width: …)" href="…">` dəyəri — bu səhifənin ayrı URL-li (m-dot) mobil versiyası. Responsiv saytlarda boşdur, ki onlar əksəriyyətdir. Burada dəyər olub geriyə işarə edən qarşılıqlı kanonik yoxdursa, bu klassik qırıq m-dot qurulumudur.',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "İki səhifə əsas məzmun mətni söz-söz ən azı bu qədər oxşar olduqda yaxın dublikat kimi qruplaşdırılır. 3 sözlük parçalar üzərində MinHash ilə qiymətləndirilir, buna görə yalnız eyni nüsxələri deyil, yenidən yazılmış və ya az dəyişdirilmiş yazıları da tutur. Bu mərhələni keçmək üçün 0 yazın.",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "Səhifənin əsas məzmun mətninin yazıldığı dil; lang atributundan deyil, mətnin özündən oxunur. 'lang ≠ məzmun dili' yoxlaması üçün html[lang] ilə müqayisə edilir. Mətn çox qısa olduqda və ya dil etibarlı müəyyən edilə bilmədikdə boşdur.",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Content-Type cavab başlığının göstərdiyi simvol kodlaması, standart kodlama adı ilə. Brauzerlər səhifənin öz bəyanı əvəzinə buna əməl edir.",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "Sənədin özünün <meta charset> və ya http-equiv ilə bəyan etdiyi simvol kodlaması. HTTP charset ilə uyğun gəlmədikdə səhifə başlıqdakı kodlama ilə göstərilir.",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "JavaScript-in səhifəyə verdiyi başlıq: JavaScript render rejimində render edilmiş DOM-dan, mətn rejimində daxili document.title təyinatından oxunur. Skriptlər başlığa toxunmursa boşdur.",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "Bu URL-in yaxın dublikat qrupundakı başqa bir səhifə ilə ən yüksək təxmini mətn oxşarlığı; əsas məzmun mətni üzərindən ölçülür. Yaxın dublikat olmayan səhifələrdə boşdur.",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = tövsiyə olunan (Screaming Frog standartı); 80 daha sərbəst yenidən yazılmaları tutur; 0 bu mərhələni söndürür.",
 };

apps/desktop/src/renderer/src/i18n/info-tips/es.ts

@@ -457,8 +457,8 @@ export const ES_INFO_TIPS: Record<string, string> = {
     "Dispara el problema 'URL demasiado larga' cuando LENGTH(url) > este valor. El RFC 7230 no impone un máximo, pero la mayoría de servidores y middleboxes fallan por encima de ~2 KB; el propio Chrome limita a ~32 KB.",
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     'Dos modos por línea. (1) Envuelve entre barras para una regex: /patrón/flags — flags admitidos imsuy (g se fuerza). Los patrones no válidos aparecen con recuento -1 en el panel de detalle para que veas la errata. (2) Cualquier otra cosa es una subcadena literal sin distinguir mayúsculas — el comportamiento heredado. El recuento de coincidencias por página de cada término se muestra en el panel Detalles de URL.',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    'Dos páginas se marcan como casi duplicadas si su SimHash de 64 bits difiere como máximo en esta cantidad de bits. 3 ≈ 95 % de similitud sobre shingles del texto del cuerpo (el filtro más estricto de Screaming Frog). Pon 0 para omitir la agrupación por completo.',
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "Dos páginas se marcan como casi duplicadas si su SimHash de 64 bits difiere en como máximo estos bits: detecta copias casi idénticas. El texto que solo es mayormente igual se evalúa con el umbral de similitud de abajo. Pon 0 para omitir este paso.",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     'URL declarada por la primera etiqueta <link rel="canonical">. Indica a los buscadores qué versión indexar cuando existen duplicados.',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -491,8 +491,8 @@ export const ES_INFO_TIPS: Record<string, string> = {
     'Cuando está ACTIVADO (por defecto), el filtro URL duplicada compara las URL tras pasar el host a minúsculas, eliminar la cadena de consulta y recortar la barra final — el comportamiento SEO canónico. Cuando está DESACTIVADO, la comparación es byte a byte, así que el filtro solo salta en filas que comparten una cadena de URL idéntica sin procesar (raro, ya que las URL se deduplican al insertarse).',
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "Cuando está activado, las URL del content de `<meta http-equiv='refresh'>` se encolan como un destino de redirección. Las redirecciones window.location del cuerpo son solo heurísticas y actualmente están fuera del alcance.",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    'Cuando está activado, una página 200 que declara un canonical apuntando a otro lugar también encola ese destino. Desactivado por defecto — la mayoría de rastreos tratan los canonicals como una señal, no como una indicación de navegación.',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    'Cuando está activado, una página 200 que declara un canonical apuntando a otro lugar también encola ese destino, así se detecta un canonical que apunta a una página inexistente o redirigida. Activado por defecto.',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     'Cuando está activado, las páginas con indexabilidad noindex / canonicalizada / bloqueada por robots se excluyen de la agrupación — el informe de Casi duplicados muestra entonces solo problemas que afectan a la visibilidad en buscadores.',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -665,4 +665,18 @@ export const ES_INFO_TIPS: Record<string, string> = {
     'Activado para sitios con React Router / Vue Router / Angular cuyas páginas nunca producen una petición de documento.',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     'Valor de `<link rel="alternate" media="only screen and (max-width: …)" href="…">` — la versión móvil con URL separada (m-dot) de esta página. Vacío en sitios responsive, que son la mayoría. Un valor aquí sin canonical recíproco que apunte de vuelta es la clásica configuración m-dot rota.',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "Dos páginas se agrupan como casi duplicadas cuando su texto principal es al menos así de similar, palabra por palabra. Se estima con MinHash sobre fragmentos de 3 palabras, así que detecta textos reescritos o ligeramente editados, no solo copias idénticas. Pon 0 para omitir este paso.",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "Idioma en el que está escrito el texto principal de la página, leído de sus palabras y no del atributo lang. Se compara con html[lang] para la comprobación 'lang ≠ idioma del contenido'. Vacío si el texto es demasiado corto o el idioma no se puede determinar con seguridad.",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Codificación de caracteres que indica la cabecera de respuesta Content-Type, con su nombre canónico. Los navegadores la obedecen por encima de la declaración de la propia página.",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "Codificación de caracteres que el propio documento declara en <meta charset> o http-equiv. Si no coincide con el charset HTTP, la página se muestra con la codificación de la cabecera.",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "El título que JavaScript asigna a la página: desde el DOM renderizado en el modo de renderizado JavaScript, o desde una asignación document.title en línea en el modo texto. Vacío si los scripts no tocan el título.",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "Mayor similitud de texto estimada con otra página del grupo de casi duplicados de esta URL, medida sobre el texto principal. Vacío en páginas que no son casi duplicadas.",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = recomendado (valor predeterminado de Screaming Frog); 80 detecta reescrituras más libres; 0 desactiva este paso.",
 };

apps/desktop/src/renderer/src/i18n/info-tips/fr.ts

@@ -456,8 +456,8 @@ export const FR_INFO_TIPS: Record<string, string> = {
     "Déclenche le problème « URL trop longue » lorsque LENGTH(url) > cette valeur. La RFC 7230 n'impose pas de maximum, mais la plupart des serveurs et équipements intermédiaires échouent au-delà de ~2 Ko ; Chrome lui-même plafonne à ~32 Ko.",
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     "Deux modes par ligne. (1) Entourez de barres obliques pour une regex : /motif/flags — flags pris en charge imsuy (g est forcé). Les motifs invalides apparaissent avec un décompte de -1 dans le panneau de détail pour repérer la faute de frappe. (2) Tout le reste est une sous-chaîne littérale insensible à la casse — le comportement hérité. Le nombre d'occurrences par page de chaque terme apparaît dans le panneau Détails de l'URL.",
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    "Deux pages sont signalées comme quasi-doublons si leur SimHash 64 bits diffère d'au plus ce nombre de bits. 3 ≈ 95 % de similarité sur les shingles du texte du corps (le filtre le plus strict de Screaming Frog). Mettez 0 pour ignorer totalement le regroupement.",
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "Deux pages sont signalées comme quasi-doublons si leur SimHash 64 bits diffère d'au plus ce nombre de bits — cela détecte les copies quasi identiques. Le texte seulement en grande partie identique est jugé par le seuil de similarité ci-dessous. Mettez 0 pour ignorer cette étape.",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     'URL déclarée par la première balise <link rel="canonical">. Indique aux moteurs de recherche quelle version indexer lorsqu\'il existe des doublons.',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -491,8 +491,8 @@ export const FR_INFO_TIPS: Record<string, string> = {
     "ACTIVÉ (par défaut), le filtre URL dupliquée compare les URL après mise en minuscules de l'hôte, suppression de la chaîne de requête et retrait de la barre finale — le comportement SEO canonique. DÉSACTIVÉ, la comparaison est octet pour octet, donc le filtre ne se déclenche que sur des lignes partageant une chaîne d'URL brute identique (rare puisque les URL sont dédupliquées à l'insertion).",
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "Activé, les URL du content de `<meta http-equiv='refresh'>` sont mises en file comme une cible de redirection. Les redirections window.location dans le corps sont purement heuristiques et actuellement hors périmètre.",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    'Activé, une page 200 qui déclare une canonique pointant ailleurs met aussi cette cible en file. Désactivé par défaut — la plupart des explorations traitent les canoniques comme un signal, pas comme une indication de navigation.',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    'Activé, une page 200 qui déclare une canonique pointant ailleurs met aussi cette cible en file, ce qui détecte une canonique pointant vers une page absente ou redirigée. Activé par défaut.',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     "Activé, les pages dont l'indexabilité est noindex / canonisée / bloquée par robots sont exclues du regroupement — le rapport Quasi-doublons ne fait alors remonter que les problèmes qui affectent la visibilité dans les moteurs.",
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -665,4 +665,18 @@ export const FR_INFO_TIPS: Record<string, string> = {
     'Activé pour les sites React Router / Vue Router / Angular dont les pages ne produisent jamais de requête de document.',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     'Valeur de `<link rel="alternate" media="only screen and (max-width: …)" href="…">` — la version mobile à URL distincte (m-dot) de cette page. Vide sur les sites responsive, soit la plupart. Une valeur ici sans canonique réciproque pointant en retour est la configuration m-dot cassée classique.',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "Deux pages sont regroupées comme quasi-doublons quand leur texte principal est au moins aussi similaire, mot pour mot. Estimé par MinHash sur des fragments de 3 mots, il détecte les textes réécrits ou légèrement modifiés, pas seulement les copies identiques. Mettez 0 pour ignorer cette étape.",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "Langue dans laquelle le texte principal de la page est rédigé, déduite de ses mots et non de l'attribut lang. Comparée à html[lang] pour le contrôle « lang ≠ langue du contenu ». Vide si le texte est trop court ou si la langue ne peut pas être déterminée avec certitude.",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Encodage des caractères indiqué par l'en-tête de réponse Content-Type, sous son nom canonique. Les navigateurs le suivent plutôt que la déclaration de la page elle-même.",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "Encodage des caractères que le document déclare lui-même dans <meta charset> ou http-equiv. S'il diffère du charset HTTP, la page s'affiche avec l'encodage de l'en-tête.",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "Le titre que JavaScript donne à la page : lu dans le DOM rendu en mode de rendu JavaScript, ou dans une affectation document.title en ligne en mode texte. Vide si les scripts ne modifient pas le titre.",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "Plus forte similarité textuelle estimée avec une autre page du groupe de quasi-doublons de cette URL, mesurée sur le texte principal. Vide pour les pages qui ne sont pas des quasi-doublons.",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = recommandé (valeur par défaut de Screaming Frog) ; 80 détecte des réécritures plus libres ; 0 désactive cette étape.",
 };

apps/desktop/src/renderer/src/i18n/info-tips/hi.ts

@@ -452,8 +452,8 @@ export const HI_INFO_TIPS: Record<string, string> = {
     "जब LENGTH(url) > यह हो तो 'URL बहुत लंबा' समस्या उठाता है। RFC 7230 कोई अधिकतम अनिवार्य नहीं करता पर अधिकांश सर्वर + मिडलबॉक्स ~2 KB से ऊपर विफल होते हैं; Chrome स्वयं ~32 KB पर सीमित करता है।",
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     'प्रति पंक्ति दो मोड। (1) regex के लिए स्लैश में लपेटें: /pattern/flags — समर्थित फ़्लैग imsuy (g बाध्य)। अमान्य पैटर्न विवरण पैनल में -1 गिनती के साथ दिखते हैं ताकि आप टाइपो पकड़ सकें। (2) बाकी सब केस-असंवेदनशील शाब्दिक सबस्ट्रिंग है — पुराना व्यवहार। हर शब्द की प्रति-पेज हिट गिनती URL विवरण पैनल में दिखती है।',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    'दो पेज निकट-डुप्लिकेट चिह्नित होते हैं यदि उनका 64-बिट SimHash अधिकतम इतने बिट से भिन्न हो। 3 ≈ बॉडी-टेक्स्ट शिंगल्स पर 95% समानता (Screaming Frog का सबसे सख्त फ़िल्टर)। क्लस्टरिंग पूरी तरह छोड़ने के लिए 0 सेट करें।',
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "यदि दो पेजों का 64-बिट SimHash अधिकतम इतने बिट से भिन्न है, तो उन्हें निकट-प्रतिलिपि चिह्नित किया जाता है — इससे लगभग समान प्रतियां पकड़ी जाती हैं। जो टेक्स्ट केवल अधिकांशतः समान है, उसे नीचे दी गई समानता सीमा से आंका जाता है। इस चरण को छोड़ने के लिए 0 रखें।",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     'पहले <link rel="canonical"> टैग द्वारा घोषित URL। डुप्लिकेट होने पर सर्च इंजन को बताता है कि कौन-सा संस्करण इंडेक्स करें।',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -486,8 +486,8 @@ export const HI_INFO_TIPS: Record<string, string> = {
     'चालू (डिफ़ॉल्ट) होने पर डुप्लिकेट URL फ़िल्टर होस्ट लोअरकेस करके, क्वेरी स्ट्रिंग हटाकर और अंतिम स्लैश काटकर URL की तुलना करता है — कैनोनिकल SEO व्यवहार। बंद होने पर तुलना बाइट-सटीक है, इसलिए फ़िल्टर केवल उन पंक्तियों पर चलता है जिनकी कच्ची URL स्ट्रिंग समान हो (दुर्लभ, क्योंकि URL इन्सर्ट के समय डीडुप्लिकेट होते हैं)।',
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "चालू होने पर `<meta http-equiv='refresh'>` content URL रीडायरेक्ट लक्ष्य की तरह कतार में डाले जाते हैं। window.location बॉडी रीडायरेक्ट केवल ह्यूरिस्टिक हैं और फ़िलहाल दायरे से बाहर।",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    'चालू होने पर कहीं और कैनोनिकल घोषित करने वाला 200 पेज उस लक्ष्य को भी कतार में डालता है। डिफ़ॉल्ट बंद — अधिकांश क्रॉल कैनोनिकल को नेविगेशन संकेत नहीं, सिग्नल मानते हैं।',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    'चालू होने पर कहीं और कैनोनिकल घोषित करने वाला 200 पेज उस लक्ष्य को भी कतार में डालता है, ताकि गायब या रीडायरेक्ट हुए पेज की ओर इशारा करने वाला कैनोनिकल पकड़ा जा सके। डिफ़ॉल्ट चालू।',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     'चालू होने पर noindex / कैनोनिकल किए गए / robots-अवरुद्ध इंडेक्सेबिलिटी वाले पेज क्लस्टरिंग से बाहर रखे जाते हैं — निकट-डुप्लिकेट रिपोर्ट तब केवल खोज दृश्यता को प्रभावित करने वाली समस्याएँ दिखाती है।',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -659,4 +659,18 @@ export const HI_INFO_TIPS: Record<string, string> = {
     'React Router / Vue Router / Angular साइटों के लिए चालू जिनके पेज कभी दस्तावेज़ अनुरोध नहीं बनाते।',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     '`<link rel="alternate" media="only screen and (max-width: …)" href="…">` का मान — इस पेज का अलग-URL (m-dot) मोबाइल संस्करण। रिस्पॉन्सिव साइटों पर खाली, जो अधिकांश हैं। यहाँ मान हो पर वापस इशारा करने वाला कोई पारस्परिक कैनोनिकल न हो तो यह क्लासिक टूटा m-dot सेटअप है।',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "जब दो पेजों का मुख्य टेक्स्ट शब्द-दर-शब्द कम से कम इतना समान हो, तो उन्हें निकट-प्रतिलिपि के रूप में क्लस्टर किया जाता है। 3-शब्द टुकड़ों पर MinHash से अनुमानित, इसलिए यह केवल समान प्रतियां ही नहीं, बल्कि दोबारा लिखे या थोड़े बदले गए लेख भी पकड़ता है। इस चरण को छोड़ने के लिए 0 रखें।",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "पेज का मुख्य टेक्स्ट जिस भाषा में लिखा है, वह lang विशेषता से नहीं बल्कि शब्दों से पढ़ी जाती है। 'lang ≠ सामग्री की भाषा' जांच के लिए html[lang] से तुलना की जाती है। टेक्स्ट बहुत छोटा हो या भाषा भरोसे से तय न हो सके तो खाली रहता है।",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Content-Type रिस्पॉन्स हेडर में बताई गई कैरेक्टर एन्कोडिंग, मानक नाम के रूप में। ब्राउज़र पेज की अपनी घोषणा के बजाय इसी का पालन करते हैं।",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "दस्तावेज़ स्वयं <meta charset> या http-equiv में जो कैरेक्टर एन्कोडिंग घोषित करता है। HTTP charset से मेल न खाने पर पेज हेडर की एन्कोडिंग में दिखता है।",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "JavaScript पेज पर जो शीर्षक सेट करता है: JavaScript रेंडरिंग मोड में रेंडर किए गए DOM से, टेक्स्ट मोड में इनलाइन document.title असाइनमेंट से। स्क्रिप्ट शीर्षक न बदलें तो खाली रहता है।",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "इस URL के निकट-प्रतिलिपि क्लस्टर के किसी अन्य पेज से सबसे अधिक अनुमानित टेक्स्ट समानता, मुख्य टेक्स्ट पर मापी गई। जो पेज निकट-प्रतिलिपि नहीं हैं उनमें खाली रहता है।",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = अनुशंसित (Screaming Frog डिफ़ॉल्ट); 80 अधिक ढीले पुनर्लेखन पकड़ता है; 0 इस चरण को बंद करता है।",
 };

apps/desktop/src/renderer/src/i18n/info-tips/it.ts

@@ -457,8 +457,8 @@ export const IT_INFO_TIPS: Record<string, string> = {
     "Fa scattare il problema 'URL troppo lungo' quando LENGTH(url) > questo valore. L'RFC 7230 non impone un massimo, ma la maggior parte di server e middlebox fallisce oltre ~2 KB; Chrome stesso limita a ~32 KB.",
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     'Due modalità per riga. (1) Racchiudi tra barre per una regex: /pattern/flag — flag supportati imsuy (g è forzato). I pattern non validi compaiono con conteggio -1 nel pannello di dettaglio così individui il refuso. (2) Qualsiasi altra cosa è una sottostringa letterale senza distinzione di maiuscole — il comportamento legacy. Il conteggio delle occorrenze per pagina di ogni termine compare nel pannello Dettagli URL.',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    'Due pagine sono segnalate come quasi duplicate se il loro SimHash a 64 bit differisce al massimo di questo numero di bit. 3 ≈ 95% di somiglianza sugli shingle del testo del corpo (il filtro più stretto di Screaming Frog). Imposta 0 per saltare del tutto il raggruppamento.',
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "Due pagine vengono segnalate come quasi-duplicati se il loro SimHash a 64 bit differisce al massimo di questi bit: individua copie quasi identiche. Il testo solo in gran parte uguale viene valutato dalla soglia di similarità qui sotto. Imposta 0 per saltare questo passaggio.",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     'URL dichiarato dal primo tag <link rel="canonical">. Indica ai motori di ricerca quale versione indicizzare quando esistono duplicati.',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -491,8 +491,8 @@ export const IT_INFO_TIPS: Record<string, string> = {
     "Se ATTIVATO (predefinito), il filtro URL duplicato confronta gli URL dopo aver reso minuscolo l'host, rimosso la query string e tolto la barra finale — il comportamento SEO canonico. Se DISATTIVATO, il confronto è byte per byte, quindi il filtro scatta solo su righe che condividono una stringa URL grezza identica (raro, perché gli URL vengono deduplicati all'inserimento).",
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "Se attivato, gli URL nel content di `<meta http-equiv='refresh'>` vengono accodati come una destinazione di reindirizzamento. I reindirizzamenti window.location nel corpo sono solo euristici e attualmente fuori ambito.",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    'Se attivato, una pagina 200 che dichiara un canonical verso altrove accoda anche quella destinazione. Disattivato per impostazione predefinita — la maggior parte delle scansioni tratta i canonical come un segnale, non come un suggerimento di navigazione.',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    'Se attivato, una pagina 200 che dichiara un canonical verso altrove accoda anche quella destinazione, così un canonical che punta a una pagina inesistente o reindirizzata viene rilevato. Attivato per impostazione predefinita.',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     'Se attivato, le pagine con indicizzabilità noindex / canonicalizzata / bloccata da robots vengono escluse dal raggruppamento — il report Quasi duplicati mostra allora solo i problemi che influiscono sulla visibilità nei motori di ricerca.',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -665,4 +665,18 @@ export const IT_INFO_TIPS: Record<string, string> = {
     'Attivato per siti React Router / Vue Router / Angular le cui pagine non producono mai una richiesta di documento.',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     'Valore di `<link rel="alternate" media="only screen and (max-width: …)" href="…">` — la versione mobile con URL separato (m-dot) di questa pagina. Vuoto sui siti responsive, cioè la maggior parte. Un valore qui senza canonical reciproco che punti indietro è la classica configurazione m-dot rotta.',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "Due pagine vengono raggruppate come quasi-duplicati quando il testo principale è almeno così simile, parola per parola. Stimato con MinHash su frammenti di 3 parole, individua testi riscritti o leggermente modificati, non solo copie identiche. Imposta 0 per saltare questo passaggio.",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "Lingua in cui è scritto il testo principale della pagina, ricavata dalle parole e non dall'attributo lang. Confrontata con html[lang] per il controllo 'lang ≠ lingua del contenuto'. Vuota se il testo è troppo breve o la lingua non si può stabilire con certezza.",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Codifica dei caratteri indicata dall'intestazione di risposta Content-Type, con il nome canonico. I browser la seguono al posto della dichiarazione della pagina stessa.",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "Codifica dei caratteri che il documento dichiara in <meta charset> o http-equiv. Se non corrisponde al charset HTTP, la pagina viene mostrata con la codifica dell'intestazione.",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "Il titolo che JavaScript imposta sulla pagina: dal DOM renderizzato in modalità rendering JavaScript, o da un'assegnazione document.title inline in modalità testo. Vuoto se gli script non toccano il titolo.",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "Massima similarità testuale stimata con un'altra pagina del gruppo di quasi-duplicati di questo URL, misurata sul testo principale. Vuota per le pagine che non sono quasi-duplicati.",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = consigliato (predefinito di Screaming Frog); 80 individua riscritture più libere; 0 disattiva questo passaggio.",
 };

apps/desktop/src/renderer/src/i18n/info-tips/ko.ts

@@ -447,8 +447,8 @@ export const KO_INFO_TIPS: Record<string, string> = {
     "LENGTH(url) > 이 값이면 'URL이 너무 김' 문제가 발생합니다. RFC 7230은 최대치를 정하지 않지만 대부분의 서버와 중간 장비는 약 2 KB 이상에서 실패하며, Chrome 자체는 약 32 KB로 제한합니다.",
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     '한 줄당 두 가지 모드. (1) 정규식은 슬래시로 감쌉니다: /패턴/플래그 — 지원 플래그 imsuy(g는 강제). 잘못된 패턴은 상세 패널에 카운트 -1로 표시되어 오타를 찾을 수 있습니다. (2) 그 외는 모두 대소문자를 구분하지 않는 리터럴 부분 문자열 — 기존 동작입니다. 각 검색어의 페이지별 일치 수는 URL 상세 패널에 표시됩니다.',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    '두 페이지의 64비트 SimHash가 이 비트 수 이하로 다르면 유사 중복으로 표시됩니다. 3 ≈ 본문 텍스트 shingle 기준 95% 유사도(Screaming Frog의 가장 엄격한 필터). 클러스터링을 완전히 건너뛰려면 0으로 설정하세요.',
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "두 페이지의 64비트 SimHash 차이가 이 비트 수 이하이면 유사 중복으로 표시합니다. 거의 동일한 사본을 잡아냅니다. 대부분만 같은 텍스트는 아래 유사도 임계값으로 판단합니다. 이 단계를 건너뛰려면 0으로 설정하세요.",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     '첫 번째 <link rel="canonical"> 태그가 선언한 URL. 중복이 있을 때 검색 엔진에 어떤 버전을 색인할지 알려줍니다.',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -481,8 +481,8 @@ export const KO_INFO_TIPS: Record<string, string> = {
     '켜면(기본값) 중복 URL 필터가 호스트를 소문자로 바꾸고 쿼리 문자열을 버리고 후행 슬래시를 제거한 뒤 URL을 비교합니다 — 표준적인 SEO 동작입니다. 끄면 바이트 단위로 정확히 비교하므로 동일한 원시 URL 문자열을 공유하는 행에서만 필터가 발동합니다(URL은 삽입 시 중복 제거되므로 드뭅니다).',
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "켜면 `<meta http-equiv='refresh'>` content의 URL이 리디렉션 대상처럼 큐에 들어갑니다. 본문의 window.location 리디렉션은 휴리스틱에 불과하며 현재 범위 밖입니다.",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    '켜면 다른 곳을 가리키는 canonical을 선언한 200 페이지가 그 대상도 큐에 넣습니다. 기본값은 끔 — 대부분의 크롤링은 canonical을 탐색 힌트가 아닌 신호로 취급합니다.',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    '켜면 다른 곳을 가리키는 canonical을 선언한 200 페이지가 그 대상도 큐에 넣어, 없는 페이지나 리다이렉트되는 페이지를 가리키는 canonical을 잡아냅니다. 기본값은 켬.',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     '켜면 색인 가능성이 noindex / 표준화됨 / robots 차단인 페이지가 클러스터링에서 제외됩니다 — 유사 중복 보고서에는 검색 가시성에 영향을 주는 문제만 나타납니다.',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -652,4 +652,18 @@ export const KO_INFO_TIPS: Record<string, string> = {
     '페이지가 문서 요청을 전혀 만들지 않는 React Router / Vue Router / Angular 사이트에는 켬.',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     '`<link rel="alternate" media="only screen and (max-width: …)" href="…">` 값 — 이 페이지의 별도 URL(m-dot) 모바일 버전. 반응형 사이트(대부분)에서는 비어 있습니다. 여기 값이 있는데 되돌아오는 상호 canonical이 없으면 전형적인 깨진 m-dot 설정입니다.',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "두 페이지의 본문 텍스트가 단어 단위로 이 정도 이상 유사하면 유사 중복으로 묶습니다. 3단어 조각에 대한 MinHash로 추정하므로 동일한 사본뿐 아니라 재작성되거나 약간 수정된 글도 잡아냅니다. 이 단계를 건너뛰려면 0으로 설정하세요.",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "페이지 본문 텍스트가 쓰인 언어로, lang 속성이 아니라 텍스트 자체에서 판단합니다. 'lang ≠ 콘텐츠 언어' 검사를 위해 html[lang]과 비교합니다. 텍스트가 너무 짧거나 언어를 확실히 판단할 수 없으면 비어 있습니다.",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Content-Type 응답 헤더가 지정한 문자 인코딩(표준 이름). 브라우저는 페이지 자체 선언보다 이를 따릅니다.",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "문서 자체가 <meta charset> 또는 http-equiv로 선언한 문자 인코딩. HTTP charset과 다르면 페이지는 헤더의 인코딩으로 표시됩니다.",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "JavaScript가 페이지에 설정한 제목: JavaScript 렌더링 모드에서는 렌더링된 DOM에서, 텍스트 모드에서는 인라인 document.title 할당에서 읽습니다. 스크립트가 제목을 건드리지 않으면 비어 있습니다.",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "이 URL의 유사 중복 클러스터 안 다른 페이지와의 최고 추정 텍스트 유사도로, 본문 텍스트 기준입니다. 유사 중복이 아닌 페이지는 비어 있습니다.",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = 권장(Screaming Frog 기본값), 80은 더 느슨한 재작성까지 잡음, 0은 이 단계를 끔.",
 };

apps/desktop/src/renderer/src/i18n/info-tips/pt-BR.ts

@@ -454,8 +454,8 @@ export const PT_BR_INFO_TIPS: Record<string, string> = {
     "Dispara o problema 'URL longa demais' quando LENGTH(url) > este valor. A RFC 7230 não impõe um máximo, mas a maioria dos servidores e middleboxes falha acima de ~2 KB; o próprio Chrome limita a ~32 KB.",
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     'Dois modos por linha. (1) Envolva em barras para uma regex: /padrão/flags — flags suportadas imsuy (g é forçado). Padrões inválidos aparecem com contagem -1 no painel de detalhe para você achar o erro de digitação. (2) Qualquer outra coisa é uma substring literal sem distinção de maiúsculas — o comportamento legado. A contagem de ocorrências por página de cada termo aparece no painel Detalhes da URL.',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    'Duas páginas são marcadas como quase duplicatas se seus SimHashes de 64 bits diferem em no máximo esta quantidade de bits. 3 ≈ 95% de similaridade sobre shingles do texto do corpo (o filtro mais rigoroso do Screaming Frog). Defina 0 para pular o agrupamento por completo.',
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "Duas páginas são marcadas como quase duplicadas se o SimHash de 64 bits diferir em no máximo esta quantidade de bits — isso pega cópias quase idênticas. Texto apenas em grande parte igual é avaliado pelo limite de similaridade abaixo. Use 0 para pular esta etapa.",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     'URL declarada pela primeira tag <link rel="canonical">. Diz aos mecanismos de busca qual versão indexar quando existem duplicatas.',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -489,8 +489,8 @@ export const PT_BR_INFO_TIPS: Record<string, string> = {
     'Quando LIGADO (padrão), o filtro URL duplicada compara as URLs após converter o host para minúsculas, remover a query string e cortar a barra final — o comportamento SEO canônico. Quando DESLIGADO, a comparação é byte a byte, então o filtro só dispara em linhas que compartilham uma string de URL bruta idêntica (raro, já que as URLs são desduplicadas na inserção).',
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "Quando ligado, as URLs do content de `<meta http-equiv='refresh'>` são enfileiradas como destino de redirecionamento. Redirecionamentos window.location no corpo são apenas heurísticos e atualmente fora do escopo.",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    'Quando ligado, uma página 200 que declara um canonical apontando para outro lugar também enfileira esse destino. Desligado por padrão — a maioria dos rastreamentos trata canonicals como sinal, não como dica de navegação.',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    'Quando ligado, uma página 200 que declara um canonical apontando para outro lugar também enfileira esse destino, assim um canonical que aponta para uma página inexistente ou redirecionada é detectado. Ligado por padrão.',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     'Quando ligado, páginas com indexabilidade noindex / canonicalizada / bloqueada por robots são excluídas do agrupamento — o relatório Quase duplicatas passa a mostrar apenas problemas que afetam a visibilidade na busca.',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -662,4 +662,18 @@ export const PT_BR_INFO_TIPS: Record<string, string> = {
     'Ligado para sites com React Router / Vue Router / Angular cujas páginas nunca produzem uma requisição de documento.',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     'Valor de `<link rel="alternate" media="only screen and (max-width: …)" href="…">` — a versão mobile com URL separada (m-dot) desta página. Vazio em sites responsivos, que são a maioria. Um valor aqui sem canonical recíproco apontando de volta é a clássica configuração m-dot quebrada.',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "Duas páginas são agrupadas como quase duplicadas quando o texto principal é pelo menos tão similar, palavra por palavra. Estimado com MinHash sobre trechos de 3 palavras, pega textos reescritos ou levemente editados, não só cópias idênticas. Use 0 para pular esta etapa.",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "Idioma em que o texto principal da página está escrito, lido das próprias palavras e não do atributo lang. Comparado com html[lang] na verificação 'lang ≠ idioma do conteúdo'. Vazio quando o texto é curto demais ou o idioma não pode ser determinado com segurança.",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Codificação de caracteres indicada pelo cabeçalho de resposta Content-Type, com o nome canônico. Os navegadores obedecem a ela em vez da declaração da própria página.",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "Codificação de caracteres que o próprio documento declara em <meta charset> ou http-equiv. Quando diverge do charset HTTP, a página é exibida com a codificação do cabeçalho.",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "O título que o JavaScript define na página: do DOM renderizado no modo de renderização JavaScript, ou de uma atribuição document.title inline no modo texto. Vazio quando os scripts não mexem no título.",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "Maior similaridade de texto estimada com outra página do grupo de quase duplicadas desta URL, medida no texto principal. Vazio em páginas que não são quase duplicadas.",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = recomendado (padrão do Screaming Frog); 80 pega reescritas mais livres; 0 desativa esta etapa.",
 };

apps/desktop/src/renderer/src/i18n/info-tips/ru.ts

@@ -456,8 +456,8 @@ export const RU_INFO_TIPS: Record<string, string> = {
     'Вызывает проблему «Слишком длинный URL», когда LENGTH(url) > этого значения. RFC 7230 не задаёт максимум, но большинство серверов и промежуточных узлов отказывают выше ~2 КБ; сам Chrome ограничивает ~32 КБ.',
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     'Два режима на строку. (1) Заключите в слэши для regex: /шаблон/флаги — поддерживаются флаги imsuy (g принудительно). Некорректные шаблоны показываются со счётчиком -1 в панели деталей, чтобы заметить опечатку. (2) Всё остальное — литеральная регистронезависимая подстрока — прежнее поведение. Число совпадений каждого термина на странице видно в панели «Детали URL».',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    'Две страницы помечаются как почти дубликаты, если их 64-битные SimHash различаются не более чем на это число бит. 3 ≈ 95 % сходства по шинглам текста тела (самый строгий фильтр Screaming Frog). Установите 0, чтобы полностью пропустить кластеризацию.',
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "Две страницы отмечаются как почти дубликаты, если их 64-битный SimHash отличается не более чем на столько бит — так находятся почти идентичные копии. Текст, совпадающий лишь в основном, оценивается порогом сходства ниже. Укажите 0, чтобы пропустить этот шаг.",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     'URL, объявленный первым тегом <link rel="canonical">. Сообщает поисковым системам, какую версию индексировать при наличии дубликатов.',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -490,8 +490,8 @@ export const RU_INFO_TIPS: Record<string, string> = {
     'Если ВКЛ. (по умолчанию), фильтр «Дубликат URL» сравнивает URL после приведения хоста к нижнему регистру, отбрасывания строки запроса и удаления завершающего слэша — каноническое SEO-поведение. Если ВЫКЛ., сравнение побайтовое, поэтому фильтр срабатывает только на строках с идентичной сырой строкой URL (редко, так как URL дедуплицируются при вставке).',
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "Если включено, URL из content `<meta http-equiv='refresh'>` ставятся в очередь как цель перенаправления. Перенаправления window.location в теле — только эвристика и сейчас вне области.",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    'Если включено, страница 200, объявляющая canonical на другой адрес, также ставит эту цель в очередь. По умолчанию выключено — большинство сканирований трактуют canonical как сигнал, а не как подсказку навигации.',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    'Если включено, страница 200, объявляющая canonical на другой адрес, также ставит эту цель в очередь — так обнаруживается canonical, указывающий на отсутствующую или перенаправленную страницу. По умолчанию включено.',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     'Если включено, страницы с индексируемостью noindex / канонизирована / заблокирована robots исключаются из кластеризации — отчёт «Почти дубликаты» тогда показывает только проблемы, влияющие на видимость в поиске.',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -663,4 +663,18 @@ export const RU_INFO_TIPS: Record<string, string> = {
     'Вкл. для сайтов на React Router / Vue Router / Angular, чьи страницы никогда не порождают запрос документа.',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     'Значение `<link rel="alternate" media="only screen and (max-width: …)" href="…">` — мобильная версия этой страницы на отдельном URL (m-dot). Пусто на адаптивных сайтах, а таких большинство. Значение здесь без ответного canonical назад — классическая сломанная m-dot-настройка.',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "Две страницы объединяются как почти дубликаты, если их основной текст похож пословно как минимум на столько. Оценивается через MinHash по фрагментам из 3 слов, поэтому находит переписанные или слегка изменённые тексты, а не только идентичные копии. Укажите 0, чтобы пропустить этот шаг.",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "Язык, на котором написан основной текст страницы, — определяется по самим словам, а не по атрибуту lang. Сравнивается с html[lang] в проверке «lang ≠ язык контента». Пусто, если текст слишком короткий или язык нельзя определить уверенно.",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Кодировка символов из заголовка ответа Content-Type в каноническом виде. Браузеры следуют ей, а не объявлению в самой странице.",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "Кодировка, которую сам документ объявляет в <meta charset> или http-equiv. Если она не совпадает с charset из HTTP, страница отображается в кодировке заголовка.",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "Заголовок, который JavaScript задаёт странице: из отрисованного DOM в режиме рендеринга JavaScript или из встроенного присваивания document.title в текстовом режиме. Пусто, если скрипты не трогают заголовок.",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "Наибольшее расчётное сходство текста с другой страницей из группы почти дубликатов этого URL, по основному тексту. Пусто для страниц, не являющихся почти дубликатами.",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = рекомендуется (по умолчанию в Screaming Frog); 80 находит более свободные переписывания; 0 отключает этот шаг.",
 };

apps/desktop/src/renderer/src/i18n/info-tips/tr.ts

@@ -456,8 +456,8 @@ export const TR_INFO_TIPS: Record<string, string> = {
     "LENGTH(url) bu değeri aştığında 'URL Çok Uzun' sorununu tetikler. RFC 7230 bir üst sınır zorunlu kılmaz ama çoğu sunucu ve ara katman ~2 KB üzerinde hata verir; Chrome'un kendi sınırı ~32 KB'dir.",
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     'Satır başına iki mod. (1) Regex için eğik çizgiler arasına alın: /desen/bayraklar — desteklenen bayraklar imsuy (g zorunlu uygulanır). Geçersiz desenler detay panelinde -1 sayısıyla görünür, böylece yazım hatasını fark edebilirsiniz. (2) Diğer her şey büyük/küçük harf duyarsız birebir alt dizedir — eski davranış. Her terimin sayfa başına eşleşme sayısı URL Detayları panelinde gösterilir.',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    "İki sayfanın 64-bit SimHash değeri en fazla bu kadar bit farklıysa yakın-yinelenen olarak işaretlenir. 3 ≈ gövde metni parçaları üzerinde %95 benzerlik (Screaming Frog'un en sıkı filtresi). Kümelemeyi tamamen atlamak için 0 yapın.",
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "İki sayfanın 64 bitlik SimHash değerleri en fazla bu kadar bit farklıysa yakın yinelenen olarak işaretlenir — bu, neredeyse birebir kopyaları yakalar. Büyük ölçüde aynı olan metinler aşağıdaki benzerlik eşiğiyle değerlendirilir. Bu adımı atlamak için 0 girin.",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     'İlk <link rel="canonical"> etiketinde tanımlanan URL. Yinelenenler varken arama motorlarına hangi sürümün indeksleneceğini söyler.',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -490,8 +490,8 @@ export const TR_INFO_TIPS: Record<string, string> = {
     "AÇIK olduğunda (varsayılan), Yinelenen URL filtresi URL'leri host küçük harfe çevrilmiş, sorgu dizesi atılmış ve sondaki eğik çizgi kırpılmış halde karşılaştırır — standart SEO davranışı. KAPALI olduğunda karşılaştırma bayt bazındadır, bu yüzden filtre yalnızca birebir aynı ham URL dizesini paylaşan satırlarda tetiklenir (URL'ler ekleme sırasında zaten tekilleştirildiği için nadirdir).",
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "Açıkken `<meta http-equiv='refresh'>` içeriğindeki URL'ler bir yönlendirme hedefi gibi kuyruğa alınır. window.location ile yapılan gövde yönlendirmeleri yalnızca sezgiseldir ve şu an kapsam dışıdır.",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    "Açıkken, başka bir yeri gösteren canonical tanımlayan 200 sayfaları o hedefi de kuyruğa alır. Varsayılan kapalıdır — çoğu crawl canonical'ı gezinme ipucu değil, sinyal olarak değerlendirir.",
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    'Açıkken, başka bir yeri gösteren canonical tanımlayan 200 sayfaları o hedefi de kuyruğa alır; böylece olmayan ya da yönlenen bir sayfayı gösteren canonical yakalanır. Varsayılan açıktır.',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     'Açıkken noindex / canonical ile yönlendirilmiş / robots tarafından engellenmiş sayfalar kümelemenin dışında tutulur — Yakın Yinelenen raporu böylece yalnızca arama görünürlüğünü etkileyen sorunları gösterir.',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -663,4 +663,18 @@ export const TR_INFO_TIPS: Record<string, string> = {
     'Sayfaları hiçbir zaman bir doküman isteği üretmeyen React Router / Vue Router / Angular siteleri için açık.',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     '`<link rel="alternate" media="only screen and (max-width: …)" href="…">` değeri — bu sayfanın ayrı URL\'li (m-dot) mobil sürümü. Responsive sitelerde — ki çoğu öyledir — boştur. Burada bir değer olup geri işaret eden karşılıklı bir canonical yoksa, bu klasik bozuk m-dot kurulumudur.',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "İki sayfa, ana içerik metinleri kelime kelime en az bu oranda benzediğinde yakın yinelenen olarak kümelenir. 3 kelimelik parçalar üzerinden MinHash ile tahmin edilir; yalnızca birebir kopyaları değil, yeniden yazılmış ya da az değiştirilmiş yazıları da yakalar. Bu adımı atlamak için 0 girin.",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "Sayfanın ana içerik metninin yazıldığı dil; lang niteliğinden değil, metnin kendisinden okunur. 'Lang ≠ İçerik Dili' kontrolü için html[lang] ile karşılaştırılır. Metin çok kısaysa ya da dil güvenle belirlenemiyorsa boştur.",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Content-Type yanıt başlığının belirttiği karakter kodlaması, standart kodlama adıyla. Tarayıcılar sayfanın kendi beyanı yerine buna uyar.",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "Belgenin kendi içinde <meta charset> ya da http-equiv ile beyan ettiği karakter kodlaması. HTTP charset'iyle uyuşmazsa sayfa başlıktaki kodlamayla gösterilir.",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "JavaScript'in sayfaya verdiği başlık: JavaScript render modunda render edilmiş DOM'dan, metin modunda satır içi document.title atamasından okunur. Betikler başlığa dokunmuyorsa boştur.",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "Bu URL'nin yakın yinelenen kümesindeki başka bir sayfayla en yüksek tahmini metin benzerliği; ana içerik metni üzerinden ölçülür. Yakın yinelenen olmayan sayfalarda boştur.",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = önerilen (Screaming Frog varsayılanı); 80 daha gevşek yeniden yazımları yakalar; 0 bu adımı kapatır.",
 };

apps/desktop/src/renderer/src/i18n/info-tips/zh-CN.ts

@@ -443,8 +443,8 @@ export const ZH_CN_INFO_TIPS: Record<string, string> = {
     '当 LENGTH(url) > 此值时触发“URL 过长”问题。RFC 7230 未规定最大值，但大多数服务器和中间设备在约 2 KB 以上就会失败；Chrome 自身上限约 32 KB。',
   ["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
     '每行两种模式。(1) 用斜杠包裹表示正则：/模式/标志——支持标志 imsuy（g 强制启用）。无效模式在详情面板中显示计数 -1，便于发现拼写错误。(2) 其他任何内容都是不区分大小写的字面子串——即旧有行为。每个词的每页命中数显示在“URL 详情”面板中。',
-  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
-    '两个页面的 64 位 SimHash 相差不超过此位数时标记为近似重复。3 ≈ 正文 shingle 95% 相似度（Screaming Frog 最严格的筛选）。设为 0 则完全跳过聚类。',
+  ["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits — this catches near-identical copies. Text that is only largely the same is judged by the similarity threshold below. Set to 0 to skip this pass."]:
+    "如果两个页面的 64 位 SimHash 最多相差这么多位，则被标记为近似重复——用于发现几乎相同的副本。仅大部分相同的文本由下方的相似度阈值判断。设为 0 可跳过此步骤。",
   ['URL declared by the first <link rel="canonical"> tag. Tells search engines which version to index when duplicates exist.']:
     '第一个 <link rel="canonical"> 标签声明的 URL。存在重复时告诉搜索引擎应索引哪个版本。',
   ['URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless.']:
@@ -477,8 +477,8 @@ export const ZH_CN_INFO_TIPS: Record<string, string> = {
     '开启（默认）时，“重复 URL”筛选器在主机转小写、去掉查询字符串、去除尾部斜杠后比较 URL——这是规范的 SEO 行为。关闭时按字节精确比较，因此筛选器只对原始 URL 字符串完全相同的行生效（因为 URL 在插入时已去重，所以很少见）。',
   ["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
     "开启时，`<meta http-equiv='refresh'>` content 中的 URL 会像重定向目标一样入队。正文中的 window.location 重定向仅为启发式判断，目前不在范围内。",
-  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint.']:
-    '开启时，声明了指向他处 canonical 的 200 页面也会将该目标入队。默认关闭——大多数抓取将 canonical 视为信号而非导航提示。',
+  ['When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target, so a canonical pointing at a missing or redirected page is caught. Default on.']:
+    '开启时，声明了指向他处 canonical 的 200 页面也会将该目标入队，从而发现指向不存在或已重定向页面的 canonical。默认开启。',
   ['When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility.']:
     '开启时，可索引性为 noindex / 已规范化 / 被 robots 阻止的页面会被排除在聚类之外——“近似重复”报告随之只显示影响搜索可见性的问题。',
   ["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
@@ -646,4 +646,18 @@ export const ZH_CN_INFO_TIPS: Record<string, string> = {
     '对页面从不产生文档请求的 React Router / Vue Router / Angular 站点开启。',
   ['`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.']:
     '`<link rel="alternate" media="only screen and (max-width: …)" href="…">` 的值——本页独立 URL 的移动版本（m-dot）。在响应式站点上为空（大多数站点如此）。此处有值却没有反向指回的 canonical，是典型的失效 m-dot 配置。',
+  ["Two pages are clustered as near-duplicates when their main-content text is at least this similar, word for word. Estimated with MinHash over 3-word shingles, so it catches posts that were rewritten or lightly edited — not only identical copies. Set to 0 to skip this pass."]:
+    "当两个页面的主体文本逐词相似度至少达到该值时，会被归为近似重复。基于 3 词片段的 MinHash 估算，因此不仅能发现完全相同的副本，也能发现改写或轻微修改的文章。设为 0 可跳过此步骤。",
+  ["Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence."]:
+    "页面主体文本所用的语言，根据文本内容判断，而不是 lang 属性。用于与 html[lang] 比较以进行“lang ≠ 内容语言”检查。文本过短或无法可靠判断语言时为空。",
+  ["Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration."]:
+    "Content-Type 响应头声明的字符编码（规范名称）。浏览器优先遵循它，而不是页面自身的声明。",
+  ["Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding."]:
+    "文档自身在 <meta charset> 或 http-equiv 中声明的字符编码。与 HTTP charset 不一致时，页面按响应头的编码显示。",
+  ["The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone."]:
+    "JavaScript 为页面设置的标题：JavaScript 渲染模式下取自渲染后的 DOM，文本模式下取自内联的 document.title 赋值。脚本未修改标题时为空。",
+  ["Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates."]:
+    "与该 URL 近似重复聚类中其他页面的最高估算文本相似度，基于主体文本计算。非近似重复页面为空。",
+  ["90 = recommended (Screaming Frog's default); 80 catches looser rewrites; 0 turns this pass off."]:
+    "90 = 推荐值（Screaming Frog 默认）；80 可发现改写幅度更大的副本；0 关闭此步骤。",
 };

apps/desktop/src/renderer/src/i18n/labels/az.ts

@@ -198,6 +198,14 @@ export const AZ_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog > 17',
   'Lang Attribute Missing': 'lang atributu yoxdur',
   'Charset Missing': 'Charset yoxdur',
+  'Canonical Outside <head>': 'Canonical <head> xaricində',
+  'Soft 404': 'Soft 404',
+  'Linked URL Blocked by robots.txt': 'Keçid verilən URL robots.txt ilə bloklanıb',
+  'Charset Mismatch (HTTP vs HTML)': 'Charset uyğunsuzluğu (HTTP və HTML)',
+  'Lang ≠ Content Language': 'lang ≠ məzmun dili',
+  'Redirect Chain (2+ hops)': 'Yönləndirmə zənciri (2+ addım)',
+  'Noindex Added by JavaScript': 'JavaScript ilə əlavə edilən noindex',
+  'Title Changed by JavaScript': 'JavaScript ilə dəyişdirilən başlıq',
   'AMP Validation Errors': 'AMP yoxlama xətaları',
   'Canonical Conflict (Near-Duplicate)': 'Canonical ziddiyyəti (yaxın dublikat)',
   'High Boilerplate (>50%)': 'Çox şablon mətn (> 50 %)',
@@ -578,6 +586,11 @@ export const AZ_LABELS: Record<string, string> = {
   Term: 'İfadə',
   Hits: 'Uyğunluq',
   Hamming: 'Hamming',
+  'Similarity %': "Oxşarlıq %",
+  "Content Language": "Məzmun dili",
+  "HTTP Charset": "HTTP charset",
+  "Meta Charset": "Meta charset",
+  "JS Title": "JS başlığı",
   Words: 'Söz',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/es.ts

@@ -190,6 +190,14 @@ export const ES_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog >17',
   'Lang Attribute Missing': 'Falta el atributo lang',
   'Charset Missing': 'Falta el charset',
+  'Canonical Outside <head>': 'Canonical fuera de <head>',
+  'Soft 404': 'Soft 404',
+  'Linked URL Blocked by robots.txt': 'URL enlazada bloqueada por robots.txt',
+  'Charset Mismatch (HTTP vs HTML)': 'Charset no coincide (HTTP vs HTML)',
+  'Lang ≠ Content Language': 'lang ≠ idioma del contenido',
+  'Redirect Chain (2+ hops)': 'Cadena de redirecciones (2+ saltos)',
+  'Noindex Added by JavaScript': 'noindex añadido por JavaScript',
+  'Title Changed by JavaScript': 'Título cambiado por JavaScript',
   'AMP Validation Errors': 'Errores de validación de AMP',
   'Canonical Conflict (Near-Duplicate)': 'Conflicto de canonical (casi duplicado)',
   'High Boilerplate (>50%)': 'Mucho contenido repetido (>50 %)',
@@ -578,6 +586,11 @@ export const ES_LABELS: Record<string, string> = {
   Term: 'Término',
   Hits: 'Coincidencias',
   Hamming: 'Hamming',
+  'Similarity %': "Similitud %",
+  "Content Language": "Idioma del contenido",
+  "HTTP Charset": "Charset HTTP",
+  "Meta Charset": "Charset meta",
+  "JS Title": "Título JS",
   Words: 'Palabras',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/fr.ts

@@ -190,6 +190,14 @@ export const FR_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog >17',
   'Lang Attribute Missing': 'Attribut lang manquant',
   'Charset Missing': 'Charset manquant',
+  'Canonical Outside <head>': 'Canonical hors de <head>',
+  'Soft 404': 'Soft 404',
+  'Linked URL Blocked by robots.txt': 'URL liée bloquée par robots.txt',
+  'Charset Mismatch (HTTP vs HTML)': 'Charset incohérent (HTTP vs HTML)',
+  'Lang ≠ Content Language': 'lang ≠ langue du contenu',
+  'Redirect Chain (2+ hops)': 'Chaîne de redirections (2+ sauts)',
+  'Noindex Added by JavaScript': 'noindex ajouté par JavaScript',
+  'Title Changed by JavaScript': 'Titre modifié par JavaScript',
   'AMP Validation Errors': 'Erreurs de validation AMP',
   'Canonical Conflict (Near-Duplicate)': 'Conflit de canonical (quasi-doublon)',
   'High Boilerplate (>50%)': 'Beaucoup de contenu répété (>50 %)',
@@ -576,6 +584,11 @@ export const FR_LABELS: Record<string, string> = {
   Term: 'Terme',
   Hits: 'Correspondances',
   Hamming: 'Hamming',
+  'Similarity %': "Similarité %",
+  "Content Language": "Langue du contenu",
+  "HTTP Charset": "Charset HTTP",
+  "Meta Charset": "Charset meta",
+  "JS Title": "Titre JS",
   Words: 'Mots',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/hi.ts

@@ -190,6 +190,14 @@ export const HI_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog >17',
   'Lang Attribute Missing': 'lang विशेषता नहीं',
   'Charset Missing': 'charset नहीं',
+  'Canonical Outside <head>': 'Canonical <head> के बाहर',
+  'Soft 404': 'सॉफ़्ट 404',
+  'Linked URL Blocked by robots.txt': 'लिंक किया गया URL robots.txt से अवरुद्ध',
+  'Charset Mismatch (HTTP vs HTML)': 'charset बेमेल (HTTP बनाम HTML)',
+  'Lang ≠ Content Language': 'lang ≠ सामग्री की भाषा',
+  'Redirect Chain (2+ hops)': 'रीडायरेक्ट शृंखला (2+ चरण)',
+  'Noindex Added by JavaScript': 'JavaScript द्वारा जोड़ा गया noindex',
+  'Title Changed by JavaScript': 'JavaScript द्वारा बदला गया शीर्षक',
   'AMP Validation Errors': 'AMP सत्यापन त्रुटियाँ',
   'Canonical Conflict (Near-Duplicate)': 'Canonical टकराव (निकट-प्रतिलिपि)',
   'High Boilerplate (>50%)': 'दोहराया ढाँचा अधिक (>50%)',
@@ -571,6 +579,11 @@ export const HI_LABELS: Record<string, string> = {
   Term: 'शब्द',
   Hits: 'मिलान',
   Hamming: 'हैमिंग',
+  'Similarity %': "समानता %",
+  "Content Language": "सामग्री की भाषा",
+  "HTTP Charset": "HTTP charset",
+  "Meta Charset": "Meta charset",
+  "JS Title": "JS शीर्षक",
   Words: 'शब्द',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/it.ts

@@ -190,6 +190,14 @@ export const IT_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog >17',
   'Lang Attribute Missing': 'Attributo lang mancante',
   'Charset Missing': 'Charset mancante',
+  'Canonical Outside <head>': 'Canonical fuori da <head>',
+  'Soft 404': 'Soft 404',
+  'Linked URL Blocked by robots.txt': 'URL collegato bloccato da robots.txt',
+  'Charset Mismatch (HTTP vs HTML)': 'Charset non corrispondente (HTTP vs HTML)',
+  'Lang ≠ Content Language': 'lang ≠ lingua del contenuto',
+  'Redirect Chain (2+ hops)': 'Catena di reindirizzamenti (2+ salti)',
+  'Noindex Added by JavaScript': 'noindex aggiunto da JavaScript',
+  'Title Changed by JavaScript': 'Titolo modificato da JavaScript',
   'AMP Validation Errors': 'Errori di validazione AMP',
   'Canonical Conflict (Near-Duplicate)': 'Conflitto di canonical (quasi-duplicato)',
   'High Boilerplate (>50%)': 'Molto contenuto ripetuto (>50%)',
@@ -574,6 +582,11 @@ export const IT_LABELS: Record<string, string> = {
   Term: 'Termine',
   Hits: 'Corrispondenze',
   Hamming: 'Hamming',
+  'Similarity %': "Similarità %",
+  "Content Language": "Lingua del contenuto",
+  "HTTP Charset": "Charset HTTP",
+  "Meta Charset": "Charset meta",
+  "JS Title": "Titolo JS",
   Words: 'Parole',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/ko.ts

@@ -190,6 +190,14 @@ export const KO_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog 17 초과',
   'Lang Attribute Missing': 'lang 속성 없음',
   'Charset Missing': 'charset 없음',
+  'Canonical Outside <head>': '<head> 밖의 canonical',
+  'Soft 404': '소프트 404',
+  'Linked URL Blocked by robots.txt': 'robots.txt로 차단된 링크 URL',
+  'Charset Mismatch (HTTP vs HTML)': 'charset 불일치(HTTP vs HTML)',
+  'Lang ≠ Content Language': 'lang ≠ 콘텐츠 언어',
+  'Redirect Chain (2+ hops)': '리다이렉트 체인(2홉 이상)',
+  'Noindex Added by JavaScript': 'JavaScript가 추가한 noindex',
+  'Title Changed by JavaScript': 'JavaScript가 변경한 제목',
   'AMP Validation Errors': 'AMP 검증 오류',
   'Canonical Conflict (Near-Duplicate)': 'Canonical 충돌(유사 중복)',
   'High Boilerplate (>50%)': '반복 문구 과다(50% 초과)',
@@ -566,6 +574,11 @@ export const KO_LABELS: Record<string, string> = {
   Term: '용어',
   Hits: '일치',
   Hamming: '해밍 거리',
+  'Similarity %': "유사도 %",
+  "Content Language": "콘텐츠 언어",
+  "HTTP Charset": "HTTP charset",
+  "Meta Charset": "Meta charset",
+  "JS Title": "JS 제목",
   Words: '단어',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/pt-BR.ts

@@ -190,6 +190,14 @@ export const PT_BR_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog >17',
   'Lang Attribute Missing': 'Falta o atributo lang',
   'Charset Missing': 'Falta o charset',
+  'Canonical Outside <head>': 'Canonical fora do <head>',
+  'Soft 404': 'Soft 404',
+  'Linked URL Blocked by robots.txt': 'URL com link bloqueada pelo robots.txt',
+  'Charset Mismatch (HTTP vs HTML)': 'Charset divergente (HTTP vs HTML)',
+  'Lang ≠ Content Language': 'lang ≠ idioma do conteúdo',
+  'Redirect Chain (2+ hops)': 'Cadeia de redirecionamentos (2+ saltos)',
+  'Noindex Added by JavaScript': 'noindex adicionado por JavaScript',
+  'Title Changed by JavaScript': 'Título alterado por JavaScript',
   'AMP Validation Errors': 'Erros de validação do AMP',
   'Canonical Conflict (Near-Duplicate)': 'Conflito de canonical (quase duplicado)',
   'High Boilerplate (>50%)': 'Muito conteúdo repetido (>50%)',
@@ -576,6 +584,11 @@ export const PT_BR_LABELS: Record<string, string> = {
   Term: 'Termo',
   Hits: 'Correspondências',
   Hamming: 'Hamming',
+  'Similarity %': "Similaridade %",
+  "Content Language": "Idioma do conteúdo",
+  "HTTP Charset": "Charset HTTP",
+  "Meta Charset": "Charset meta",
+  "JS Title": "Título JS",
   Words: 'Palavras',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/ru.ts

@@ -190,6 +190,14 @@ export const RU_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog >17',
   'Lang Attribute Missing': 'Нет атрибута lang',
   'Charset Missing': 'Нет charset',
+  'Canonical Outside <head>': 'Canonical вне <head>',
+  'Soft 404': 'Soft 404',
+  'Linked URL Blocked by robots.txt': 'URL со ссылками заблокирован robots.txt',
+  'Charset Mismatch (HTTP vs HTML)': 'Несовпадение charset (HTTP и HTML)',
+  'Lang ≠ Content Language': 'lang ≠ язык контента',
+  'Redirect Chain (2+ hops)': 'Цепочка перенаправлений (2+ переходов)',
+  'Noindex Added by JavaScript': 'noindex, добавленный JavaScript',
+  'Title Changed by JavaScript': 'Заголовок, изменённый JavaScript',
   'AMP Validation Errors': 'Ошибки проверки AMP',
   'Canonical Conflict (Near-Duplicate)': 'Конфликт canonical (почти дубликат)',
   'High Boilerplate (>50%)': 'Много шаблонного текста (>50 %)',
@@ -573,6 +581,11 @@ export const RU_LABELS: Record<string, string> = {
   Term: 'Термин',
   Hits: 'Совпадения',
   Hamming: 'Хэмминг',
+  'Similarity %': "Сходство, %",
+  "Content Language": "Язык контента",
+  "HTTP Charset": "Charset HTTP",
+  "Meta Charset": "Charset в meta",
+  "JS Title": "Заголовок JS",
   Words: 'Слова',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/tr.ts

@@ -189,6 +189,14 @@ export const TR_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog >17',
   'Lang Attribute Missing': 'Lang Niteliği Eksik',
   'Charset Missing': 'Charset Eksik',
+  'Canonical Outside <head>': 'Canonical <head> Dışında',
+  'Soft 404': 'Soft 404',
+  'Linked URL Blocked by robots.txt': 'Bağlantı Verilen URL robots.txt ile Engelli',
+  'Charset Mismatch (HTTP vs HTML)': 'Charset Uyuşmazlığı (HTTP ve HTML)',
+  'Lang ≠ Content Language': 'Lang ≠ İçerik Dili',
+  'Redirect Chain (2+ hops)': 'Yönlendirme Zinciri (2+ hop)',
+  'Noindex Added by JavaScript': 'JavaScript ile Eklenen Noindex',
+  'Title Changed by JavaScript': 'JavaScript ile Değişen Başlık',
   'AMP Validation Errors': 'AMP Doğrulama Hataları',
   'Canonical Conflict (Near-Duplicate)': 'Canonical Çakışması (Yakın-Yinelenen)',
   'High Boilerplate (>50%)': 'Yüksek Şablon İçerik (>%50)',
@@ -561,6 +569,11 @@ export const TR_LABELS: Record<string, string> = {
   Term: 'Terim',
   Hits: 'Eşleşme',
   Hamming: 'Hamming',
+  'Similarity %': "Benzerlik %",
+  "Content Language": "İçerik Dili",
+  "HTTP Charset": "HTTP Charset",
+  "Meta Charset": "Meta Charset",
+  "JS Title": "JS Başlığı",
   Words: 'Kelime',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/labels/zh-CN.ts

@@ -190,6 +190,14 @@ export const ZH_CN_LABELS: Record<string, string> = {
   'Gunning Fog >17': 'Gunning Fog >17',
   'Lang Attribute Missing': '缺少 lang 属性',
   'Charset Missing': '缺少 charset',
+  'Canonical Outside <head>': 'Canonical 位于 <head> 之外',
+  'Soft 404': '软 404',
+  'Linked URL Blocked by robots.txt': '被 robots.txt 拦截的链接 URL',
+  'Charset Mismatch (HTTP vs HTML)': 'charset 不一致（HTTP 与 HTML）',
+  'Lang ≠ Content Language': 'lang ≠ 内容语言',
+  'Redirect Chain (2+ hops)': '重定向链（2 跳及以上）',
+  'Noindex Added by JavaScript': 'JavaScript 添加的 noindex',
+  'Title Changed by JavaScript': 'JavaScript 修改的标题',
   'AMP Validation Errors': 'AMP 校验错误',
   'Canonical Conflict (Near-Duplicate)': 'Canonical 冲突（近似重复）',
   'High Boilerplate (>50%)': '模板化内容过多（>50%）',
@@ -565,6 +573,11 @@ export const ZH_CN_LABELS: Record<string, string> = {
   Term: '词条',
   Hits: '命中',
   Hamming: '汉明距离',
+  'Similarity %': "相似度 %",
+  "Content Language": "内容语言",
+  "HTTP Charset": "HTTP charset",
+  "Meta Charset": "Meta charset",
+  "JS Title": "JS 标题",
   Words: '词数',
 
   // URL Details (NameValueView) field labels

apps/desktop/src/renderer/src/i18n/locales/az.json

@@ -707,6 +707,8 @@
       "path": "Problemlər → Məzmun → Yaxın dublikat",
       "maxHamming": "Maksimum Hamming məsafəsi (0 = yalnız tam eyni, 12 = çox geniş, 0 söndürür)",
       "thresholdHint": "Nə qədər aşağı olsa, o qədər sərtdir. 3 Screaming Frog-un qarşılığı olan dəyərdir. Mətni çox az olan səhifələr (50 simvoldan az) hədd nə olursa olsun qruplaşmadan kənarda qalır.",
+      "minSimilarity": "Minimum mətn oxşarlığı % (0 söndürür)",
+      "similarityHint": "Əsas məzmun mətni ən azı bu qədər oxşar olan səhifələr də qruplaşdırılır — SimHash-in qaçırdığı yenidən yazılmış və ya az dəyişdirilmiş nüsxələri tutur. 90 SF-ə ekvivalent standart dəyərdir.",
       "onlyIndexable": "Yalnız indekslənə bilən səhifələri qruplaşdır",
       "preNormalize": "Dublikat URL yoxlamasından əvvəl URL-ləri normallaşdır",
       "costLabel": "Xərc:",

apps/desktop/src/renderer/src/i18n/locales/en.json

@@ -707,6 +707,8 @@
       "path": "Issues → Content → Near-Duplicate",
       "maxHamming": "Max Hamming distance (0 = exact only, 12 = very loose, 0 disables)",
       "thresholdHint": "Lower = stricter. 3 is the SF-equivalent default. Pages with too little body content (<50 characters) are excluded from clustering regardless of threshold.",
+      "minSimilarity": "Minimum text similarity % (0 disables)",
+      "similarityHint": "Pages whose main-content wording is at least this similar are clustered too — this catches rewritten or lightly edited copies that SimHash misses. 90 is the SF-equivalent default.",
       "onlyIndexable": "Only cluster indexable pages",
       "preNormalize": "Normalise URLs before duplicate-URL check",
       "costLabel": "Cost:",

apps/desktop/src/renderer/src/i18n/locales/es.json

@@ -725,6 +725,8 @@
       "path": "Problemas → Contenido → Casi duplicado",
       "maxHamming": "Distancia de Hamming máxima (0 = solo exactos, 12 = muy laxo, 0 desactiva)",
       "thresholdHint": "Más bajo = más estricto. 3 es el valor equivalente al de Screaming Frog. Las páginas con muy poco contenido en el cuerpo (<50 caracteres) quedan excluidas del agrupamiento con independencia del umbral.",
+      "minSimilarity": "Similitud mínima de texto % (0 desactiva)",
+      "similarityHint": "También se agrupan las páginas cuyo texto principal es al menos así de similar: detecta copias reescritas o ligeramente editadas que SimHash no ve. 90 es el valor predeterminado equivalente a SF.",
       "onlyIndexable": "Agrupar solo páginas indexables",
       "preNormalize": "Normalizar las URL antes de comprobar URL duplicadas",
       "costLabel": "Coste:",

apps/desktop/src/renderer/src/i18n/locales/fr.json

@@ -725,6 +725,8 @@
       "path": "Problèmes → Contenu → Quasi-doublon",
       "maxHamming": "Distance de Hamming maximale (0 = exacts uniquement, 12 = très permissif, 0 désactive)",
       "thresholdHint": "Plus bas = plus strict. 3 est la valeur équivalente à Screaming Frog. Les pages dont le corps contient trop peu de texte (<50 caractères) sont exclues du regroupement quel que soit le seuil.",
+      "minSimilarity": "Similarité textuelle minimale % (0 désactive)",
+      "similarityHint": "Les pages dont le texte principal est au moins aussi similaire sont aussi regroupées — cela détecte les copies réécrites ou légèrement modifiées que SimHash manque. 90 est la valeur par défaut équivalente à SF.",
       "onlyIndexable": "Ne regrouper que les pages indexables",
       "preNormalize": "Normaliser les URL avant la vérification des URL dupliquées",
       "costLabel": "Coût :",

apps/desktop/src/renderer/src/i18n/locales/hi.json

@@ -707,6 +707,8 @@
       "path": "समस्याएँ → सामग्री → निकट-प्रतिलिपि",
       "maxHamming": "अधिकतम हैमिंग दूरी (0 = केवल पूर्णतः समान, 12 = बहुत ढीला, 0 से बंद)",
       "thresholdHint": "जितना कम, उतना कड़ा। 3 Screaming Frog के समकक्ष डिफ़ॉल्ट है। जिन पृष्ठों में बहुत कम पाठ है (50 अक्षर से कम), वे सीमा चाहे जो हो, समूहन से बाहर रहते हैं।",
+      "minSimilarity": "न्यूनतम टेक्स्ट समानता % (0 बंद करता है)",
+      "similarityHint": "जिन पेजों का मुख्य टेक्स्ट कम से कम इतना समान है, वे भी क्लस्टर किए जाते हैं — इससे वे दोबारा लिखी या थोड़ी बदली गई प्रतियां पकड़ी जाती हैं जिन्हें SimHash चूक जाता है। 90 SF के बराबर डिफ़ॉल्ट है।",
       "onlyIndexable": "केवल अनुक्रमणीय पृष्ठ समूहित करें",
       "preNormalize": "डुप्लिकेट URL जाँच से पहले URL सामान्यीकृत करें",
       "costLabel": "लागत:",

apps/desktop/src/renderer/src/i18n/locales/it.json

@@ -725,6 +725,8 @@
       "path": "Problemi → Contenuto → Quasi-duplicato",
       "maxHamming": "Distanza di Hamming massima (0 = solo esatti, 12 = molto permissivo, 0 disattiva)",
       "thresholdHint": "Più basso = più severo. 3 è il valore equivalente a quello di Screaming Frog. Le pagine con troppo poco contenuto nel corpo (<50 caratteri) restano escluse dal raggruppamento indipendentemente dalla soglia.",
+      "minSimilarity": "Similarità minima del testo % (0 disattiva)",
+      "similarityHint": "Vengono raggruppate anche le pagine il cui testo principale è almeno così simile: individua copie riscritte o leggermente modificate che SimHash non vede. 90 è il valore predefinito equivalente a SF.",
       "onlyIndexable": "Raggruppa solo le pagine indicizzabili",
       "preNormalize": "Normalizza gli URL prima della verifica degli URL duplicati",
       "costLabel": "Costo:",

apps/desktop/src/renderer/src/i18n/locales/ko.json

@@ -689,6 +689,8 @@
       "path": "문제 → 콘텐츠 → 유사 중복",
       "maxHamming": "최대 해밍 거리(0 = 완전 일치만, 12 = 매우 느슨, 0이면 사용 안 함)",
       "thresholdHint": "값이 작을수록 엄격합니다. 3이 Screaming Frog와 같은 기본값입니다. 본문이 너무 적은 페이지(50자 미만)는 임계값과 관계없이 묶음에서 제외됩니다.",
+      "minSimilarity": "최소 텍스트 유사도 %(0이면 끔)",
+      "similarityHint": "본문 텍스트가 이 정도 이상 유사한 페이지도 클러스터로 묶습니다. SimHash가 놓치는 재작성되거나 약간 수정된 사본을 잡아냅니다. 90이 SF와 같은 기본값입니다.",
       "onlyIndexable": "색인 가능한 페이지만 묶기",
       "preNormalize": "중복 URL 검사 전에 URL 정규화",
       "costLabel": "비용:",

apps/desktop/src/renderer/src/i18n/locales/pt-BR.json

@@ -725,6 +725,8 @@
       "path": "Problemas → Conteúdo → Quase duplicado",
       "maxHamming": "Distância de Hamming máxima (0 = apenas exatos, 12 = muito frouxo, 0 desativa)",
       "thresholdHint": "Mais baixo = mais rígido. 3 é o padrão equivalente ao do Screaming Frog. Páginas com pouquíssimo conteúdo no corpo (<50 caracteres) ficam fora do agrupamento independentemente do limite.",
+      "minSimilarity": "Similaridade mínima de texto % (0 desativa)",
+      "similarityHint": "Páginas cujo texto principal é pelo menos tão similar também são agrupadas — isso pega cópias reescritas ou levemente editadas que o SimHash não vê. 90 é o padrão equivalente ao SF.",
       "onlyIndexable": "Agrupar somente páginas indexáveis",
       "preNormalize": "Normalizar URLs antes de verificar URLs duplicadas",
       "costLabel": "Custo:",

apps/desktop/src/renderer/src/i18n/locales/ru.json

@@ -743,6 +743,8 @@
       "path": "Проблемы → Контент → Почти дубликат",
       "maxHamming": "Макс. расстояние Хэмминга (0 = только точные, 12 = очень свободно, 0 отключает)",
       "thresholdHint": "Меньше = строже. 3 — значение, эквивалентное Screaming Frog. Страницы со слишком малым текстом в теле (<50 символов) исключаются из группировки независимо от порога.",
+      "minSimilarity": "Минимальное сходство текста, % (0 — отключить)",
+      "similarityHint": "Страницы, основной текст которых похож как минимум на столько, тоже группируются — так находятся переписанные или слегка изменённые копии, которые пропускает SimHash. 90 — значение по умолчанию, как в SF.",
       "onlyIndexable": "Группировать только индексируемые страницы",
       "preNormalize": "Нормализовать URL перед проверкой дубликатов URL",
       "costLabel": "Стоимость:",

apps/desktop/src/renderer/src/i18n/locales/tr.json

@@ -707,6 +707,8 @@
       "path": "Issues → Content → Near-Duplicate",
       "maxHamming": "Maks Hamming mesafesi (0 = sadece birebir, 12 = çok gevşek, 0 devre dışı bırakır)",
       "thresholdHint": "Düşük = daha katı. 3 SF eşdeğeri varsayılan. Çok az gövde içeriği (<50 karakter) olan sayfalar eşikten bağımsız olarak kümeleme dışı kalır.",
+      "minSimilarity": "En düşük metin benzerliği % (0 kapatır)",
+      "similarityHint": "Ana içerik metni en az bu oranda benzeyen sayfalar da kümelenir — SimHash'in kaçırdığı yeniden yazılmış ya da az değiştirilmiş kopyaları yakalar. 90, SF ile eşdeğer varsayılandır.",
       "onlyIndexable": "Yalnızca indekslenebilir sayfaları kümele",
       "preNormalize": "Yinelenen URL kontrolünden önce URL'leri normalleştir",
       "costLabel": "Maliyet:",

apps/desktop/src/renderer/src/i18n/locales/zh-CN.json

@@ -689,6 +689,8 @@
       "path": "问题 → 内容 → 近似重复",
       "maxHamming": "最大汉明距离（0 = 仅完全相同，12 = 非常宽松，0 表示关闭）",
       "thresholdHint": "数值越小越严格。3 相当于 Screaming Frog 的默认值。正文内容过少（少于 50 个字符）的页面无论阈值如何都不参与聚类。",
+      "minSimilarity": "最低文本相似度 %（0 表示关闭）",
+      "similarityHint": "主体文本相似度至少达到该值的页面也会被聚类——可发现 SimHash 漏掉的改写或轻微修改的副本。90 为与 SF 等效的默认值。",
       "onlyIndexable": "仅对可索引页面聚类",
       "preNormalize": "在检查重复 URL 前先规范化 URL",
       "costLabel": "开销：",

apps/desktop/src/renderer/src/store.ts

@@ -145,6 +145,8 @@ export const TAB_QUICK_FILTERS: Partial<Record<TabKey, TabQuickFilter[]>> = {
     { label: 'Very Slow (>3s)', category: 'issues:response-very-slow' },
     { label: 'TTFB Slow (>600ms)', category: 'issues:ttfb-slow' },
     { label: 'TTFB Very Slow (>1.8s)', category: 'issues:ttfb-very-slow' },
+    { label: 'Soft 404', category: 'issues:soft-404' },
+    { label: 'Linked URL Blocked by robots.txt', category: 'issues:blocked-by-robots' },
   ],
   security: [
     { label: 'HTTPS URLs', category: 'security:https' },
@@ -181,6 +183,7 @@ export const TAB_QUICK_FILTERS: Partial<Record<TabKey, TabQuickFilter[]>> = {
     { label: 'Noindex', category: 'indexability:noindex' },
     { label: 'Canonicalised', category: 'indexability:canonicalised' },
     { label: 'Blocked by Robots', category: 'indexability:blocked-robots' },
+    { label: 'Noindex Added by JavaScript', category: 'issues:noindex-js-injected' },
   ],
 
   // ─── URL tab ────────────────────────────────────────────────────────────
@@ -212,6 +215,7 @@ export const TAB_QUICK_FILTERS: Partial<Record<TabKey, TabQuickFilter[]>> = {
     { label: 'Pixel Width Truncated (>600px)', category: 'issues:title-pixel-width-too-long' },
     { label: 'Placeholder Title', category: 'issues:title-placeholder' },
     { label: 'Single-Word Title', category: 'issues:title-single-word' },
+    { label: 'Title Changed by JavaScript', category: 'issues:title-changed-by-js' },
   ],
 
   // ─── Meta Description tab ───────────────────────────────────────────────
@@ -254,6 +258,7 @@ export const TAB_QUICK_FILTERS: Partial<Record<TabKey, TabQuickFilter[]>> = {
     { label: 'Large (>1MB)', category: 'issues:page-large' },
     { label: 'Critical Size (>3MB)', category: 'issues:page-too-large-critical' },
     { label: 'Charset Missing', category: 'issues:charset-missing' },
+    { label: 'Charset Mismatch (HTTP vs HTML)', category: 'issues:charset-mismatch' },
   ],
 
   // ─── Canonicals tab ─────────────────────────────────────────────────────
@@ -272,12 +277,14 @@ export const TAB_QUICK_FILTERS: Partial<Record<TabKey, TabQuickFilter[]>> = {
     { label: 'Canonical → Noindex', category: 'issues:canonical-to-noindex' },
     { label: 'Canonical Not Absolute', category: 'issues:canonical-not-absolute' },
     { label: 'Canonical Chain (Multi-hop)', category: 'issues:canonical-chain-multi-hop' },
+    { label: 'Canonical Outside <head>', category: 'issues:canonical-outside-head' },
   ],
 
   // ─── Redirects tab ──────────────────────────────────────────────────────
   redirects: [
     { label: 'All', category: 'tab:redirects' },
     { label: 'Redirect Loop', category: 'issues:redirect-loop' },
+    { label: 'Redirect Chain (2+ hops)', category: 'issues:redirect-chain' },
     { label: 'Long Chain (>3 hops)', category: 'issues:redirect-chain-long' },
     { label: 'Self-Redirect', category: 'issues:redirect-self' },
   ],
@@ -822,7 +829,8 @@ function tabForCategory(cat: UrlCategory): TabKey | null {
     cat === 'issues:gunning-fog-very-high' ||
     cat === 'issues:page-large' ||
     cat === 'issues:page-too-large-critical' ||
-    cat === 'issues:charset-missing'
+    cat === 'issues:charset-missing' ||
+    cat === 'issues:charset-mismatch'
   ) {
     return 'content';
   }
@@ -873,10 +881,13 @@ function tabForCategory(cat: UrlCategory): TabKey | null {
     cat === 'issues:response-slow' ||
     cat === 'issues:response-very-slow' ||
     cat === 'issues:ttfb-slow' ||
-    cat === 'issues:ttfb-very-slow'
+    cat === 'issues:ttfb-very-slow' ||
+    cat === 'issues:soft-404' ||
+    cat === 'issues:blocked-by-robots'
   ) {
     return 'response-codes';
   }
+  if (cat === 'issues:noindex-js-injected') return 'directives';
   // Page-level tag checks (no GA4 snippet, legacy UA, …) filter the URL
   // table; the 'analytics' tab is the GA4 traffic grid, which ignores the
   // category entirely and would show unrelated data (or a connect screen).

apps/desktop/src/renderer/src/tabs/columns.ts

@@ -315,6 +315,47 @@ const COL = {
     info: 'FNV-1a 64-bit hash of the normalised body token stream. Two pages sharing this hash are byte-identical post-tokenisation — the basis of the Exact Duplicate filter.',
     example: '7c3e9f1a8b2d4c5e',
   } as ColumnSpec,
+  contentLang: {
+    key: 'contentLang',
+    header: "Content Language",
+    size: 120,
+    kind: 'mono',
+    info: "Language the page's main-content text is written in, read from its wording rather than the lang attribute. Compared with html[lang] for the 'Lang ≠ Content Language' check. Empty when the text is too short or the language can't be told with confidence.",
+    example: "tr, en, de",
+  } as ColumnSpec,
+  httpCharset: {
+    key: 'httpCharset',
+    header: "HTTP Charset",
+    size: 110,
+    kind: 'mono',
+    info: "Character encoding named by the Content-Type response header, as a canonical encoding name. Browsers obey it over the page's own declaration.",
+    example: "utf-8, windows-1254",
+  } as ColumnSpec,
+  metaCharset: {
+    key: 'metaCharset',
+    header: "Meta Charset",
+    size: 110,
+    kind: 'mono',
+    info: "Character encoding the document itself declares in <meta charset> or http-equiv. When it disagrees with the HTTP charset, the page renders in the header's encoding.",
+    example: "utf-8",
+  } as ColumnSpec,
+  jsTitle: {
+    key: 'jsTitle',
+    header: "JS Title",
+    size: 260,
+    kind: 'text',
+    info: "The title JavaScript sets on the page: from the rendered DOM in JavaScript rendering mode, or from an inline document.title assignment in text mode. Empty when scripts leave the title alone.",
+    example: "Sale! 50% off | Shop",
+  } as ColumnSpec,
+  nearDupSimilarity: {
+    key: 'nearDupSimilarity',
+    header: "Similarity %",
+    size: 100,
+    kind: 'number',
+    align: 'right',
+    info: "Highest estimated text similarity to another page in this URL's near-duplicate cluster, measured on the main-content wording. Empty for pages that aren't near-duplicates.",
+    example: "96 = almost the same text",
+  } as ColumnSpec,
   simhash: {
     key: 'simhash',
     header: 'SimHash',
@@ -614,6 +655,7 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
     COL.url,
     COL.title,
     COL.titleLength,
+    COL.jsTitle,
     COL.status,
     COL.indexability,
     COL.inlinks,
@@ -640,8 +682,12 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
     COL.url,
     COL.wordCount,
     COL.contentLength,
+    COL.contentLang,
+    COL.httpCharset,
+    COL.metaCharset,
     COL.clusterId,
     COL.clusterSize,
+    COL.nearDupSimilarity,
     COL.status,
     COL.indexability,
     COL.inlinks,
@@ -785,6 +831,7 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
     COL.title,
     COL.clusterId,
     COL.clusterSize,
+    COL.nearDupSimilarity,
     COL.contentHash,
     COL.simhash,
     COL.inlinks,

packages/core/src/content-language.ts

@@ -0,0 +1,261 @@
+/**
+ * Content-language detection for the "declared `lang` ≠ content language"
+ * check and for choosing a readability formula.
+ *
+ * Deliberately not a general-purpose identifier. Its one job is to answer
+ * "is this page obviously NOT written in the language it declares?" with
+ * as close to zero false alarms as possible, so it only ever speaks up on
+ * two kinds of evidence a reviewer would accept at a glance:
+ *
+ *  1. **Writing system.** `lang="en"` over Hangul, Cyrillic or Devanagari
+ *     text is a mismatch no matter which language the text actually is.
+ *  2. **Function words.** Every language leans on a small closed set of
+ *     words (en *the / and / of*, tr *ve / için / ile*, es *el / los / y*).
+ *     Only the words distinctive to one language in the table are scored,
+ *     so the shared *de / la / a / o* of the Romance languages never vote.
+ *     A page declaring language X whose prose contains almost none of X's
+ *     function words while being dense with Y's is written in Y.
+ *
+ * Statistical identifiers (franc and friends) rank close languages within
+ * a hair of each other — Turkish prose scores Turkmen at 0.99 of Turkish —
+ * which is fine for picking a dictionary and wrong for raising an issue.
+ * Anything this module can't call with confidence is reported as unknown,
+ * never as a mismatch.
+ */
+
+/** Below this many word tokens function-word ratios are noise. */
+const MIN_TOKENS = 40;
+/** Below this many letters the script distribution is noise. */
+const MIN_LETTERS = 120;
+/** A declared language this rare in its own function words isn't the text's language. */
+const DECLARED_ABSENT_RATIO = 0.015;
+/** A language this dense in its distinctive function words is the text's language. */
+const DETECTED_PRESENT_RATIO = 0.05;
+/** Only this much of the text is scanned — plenty for a stable reading. */
+const SAMPLE_CHARS = 6_000;
+
+// Function words per language, lower-case. Words that occur in more than
+// one list are dropped from all of them when scoring (see DISTINCTIVE).
+// One-letter words that are also common elsewhere (pt *a* / *o*, it *i*
+// — English "a" and "I") are left out so English prose can't vote for them.
+const FUNCTION_WORDS: Readonly<Record<string, readonly string[]>> = {
+  en: ['the', 'and', 'of', 'to', 'is', 'that', 'for', 'it', 'with', 'as', 'was', 'on', 'are', 'be', 'this', 'by', 'at', 'from', 'or', 'an', 'have', 'not', 'but', 'which', 'they', 'you', 'we', 'their', 'has', 'were', 'can', 'will', 'been', 'more', 'there', 'all', 'would', 'its', 'also', 'about', 'into', 'how', 'what', 'when', 'your', 'our', 'these', 'than', 'other', 'every', 'very', 'until', 'some'],
+  tr: ['ve', 'bir', 'bu', 'için', 'ile', 'çok', 'gibi', 'olan', 'olarak', 'kadar', 'değil', 'ama', 'veya', 'şey', 'nasıl', 'neden', 'sonra', 'önce', 'her', 'hem', 'şu', 'bunu', 'buna', 'bunun', 'ise', 'yok', 'ancak', 'çünkü', 'göre', 'sadece', 'aynı', 'diğer', 'tüm', 'bütün', 'olduğu', 'olur', 'oldu', 'artık', 'yani', 'üzere', 'arasında', 'hakkında', 'ya', 'mı', 'mu', 'mü', 'en', 'daha', 'var'],
+  az: ['və', 'üçün', 'ilə', 'çox', 'kimi', 'olaraq', 'qədər', 'deyil', 'amma', 'nə', 'necə', 'niyə', 'əvvəl', 'hər', 'həm', 'həmin', 'isə', 'yox', 'lakin', 'çünki', 'görə', 'yalnız', 'eyni', 'digər', 'olduğu', 'artıq', 'yəni', 'haqqında', 'ən', 'də', 'bu', 'bir', 'daha', 'var'],
+  es: ['el', 'los', 'las', 'del', 'y', 'que', 'una', 'es', 'por', 'con', 'para', 'su', 'sus', 'al', 'como', 'más', 'pero', 'este', 'esta', 'son', 'entre', 'cuando', 'muy', 'sin', 'sobre', 'también', 'hasta', 'hay', 'donde', 'quien', 'desde', 'todo', 'nos', 'durante', 'porque', 'cada', 'otros', 'ese', 'eso', 'la', 'de', 'en', 'un', 'lo'],
+  fr: ['le', 'les', 'des', 'du', 'et', 'une', 'est', 'qui', 'dans', 'pas', 'sur', 'au', 'aux', 'avec', 'ce', 'cette', 'il', 'elle', 'ils', 'sont', 'plus', 'mais', 'ou', 'ne', 'son', 'sa', 'ses', 'leur', 'nous', 'vous', 'être', 'été', 'comme', 'tout', 'aussi', 'très', 'sans', 'peut', 'même', 'la', 'de', 'en', 'un', 'que', 'par', 'pour'],
+  pt: ['os', 'do', 'da', 'dos', 'das', 'em', 'no', 'na', 'nos', 'nas', 'um', 'uma', 'é', 'com', 'não', 'mais', 'como', 'mas', 'ao', 'aos', 'ou', 'seu', 'sua', 'são', 'foi', 'pelo', 'pela', 'também', 'já', 'muito', 'quando', 'isso', 'este', 'esta', 'sem', 'até', 'ser', 'há', 'e', 'de', 'que', 'para', 'por'],
+  it: ['il', 'gli', 'della', 'dei', 'delle', 'è', 'che', 'per', 'con', 'non', 'si', 'alla', 'sono', 'come', 'ma', 'più', 'anche', 'questo', 'questa', 'nel', 'nella', 'tra', 'fra', 'ha', 'essere', 'molto', 'cui', 'suo', 'sua', 'loro', 'quando', 'perché', 'di', 'del', 'un', 'una', 'la', 'le', 'lo', 'e', 'in', 'al', 'da'],
+  de: ['der', 'die', 'das', 'und', 'ist', 'nicht', 'ein', 'eine', 'zu', 'den', 'dem', 'des', 'mit', 'sich', 'auf', 'für', 'von', 'im', 'auch', 'es', 'an', 'werden', 'wird', 'aus', 'er', 'sie', 'wir', 'ihr', 'sind', 'war', 'bei', 'oder', 'nach', 'wie', 'aber', 'noch', 'nur', 'über', 'so', 'kann', 'dass', 'durch', 'hat', 'einen', 'einer'],
+  nl: ['het', 'een', 'van', 'dat', 'op', 'te', 'voor', 'niet', 'met', 'zijn', 'aan', 'er', 'ook', 'als', 'bij', 'maar', 'om', 'door', 'wordt', 'worden', 'nog', 'naar', 'kan', 'dan', 'dit', 'deze', 'wat', 'uit', 'hun', 'ze', 'we', 'je', 'wij', 'hebben', 'heeft', 'zo', 'meer', 'onze', 'de', 'en', 'is', 'in'],
+  pl: ['w', 'na', 'z', 'nie', 'się', 'do', 'to', 'że', 'jest', 'jak', 'ale', 'po', 'co', 'tak', 'za', 'od', 'przez', 'dla', 'są', 'jego', 'jej', 'ich', 'oraz', 'czy', 'lub', 'być', 'może', 'już', 'tylko', 'przy', 'tym', 'ten', 'ta', 'które', 'który', 'która', 'także', 'bardzo'],
+  sv: ['och', 'att', 'det', 'som', 'på', 'är', 'av', 'för', 'med', 'till', 'den', 'har', 'inte', 'om', 'ett', 'jag', 'men', 'så', 'från', 'kan', 'ska', 'vi', 'vid', 'eller', 'sig', 'när', 'hur', 'alla', 'man', 'efter', 'också', 'nu', 'där'],
+  fi: ['ja', 'on', 'ei', 'se', 'että', 'oli', 'ovat', 'kuin', 'myös', 'tai', 'mutta', 'kun', 'jos', 'niin', 'sekä', 'joka', 'jotka', 'hän', 'tämä', 'nämä', 'ole', 'voi', 'vain', 'mukaan', 'sen', 'sitä', 'kaikki', 'olla', 'jo', 'nyt', 'vielä'],
+  ro: ['și', 'în', 'cu', 'pe', 'care', 'este', 'nu', 'că', 'din', 'pentru', 'mai', 'sunt', 'sau', 'ale', 'au', 'fost', 'ca', 'dar', 'prin', 'lui', 'acest', 'această', 'foarte', 'fi', 'ce', 'cum'],
+  hu: ['az', 'és', 'hogy', 'nem', 'egy', 'van', 'meg', 'ez', 'csak', 'mint', 'már', 'még', 'vagy', 'volt', 'ki', 'ha', 'kell', 'nagyon', 'mert', 'azt', 'ezt', 'minden', 'lesz', 'akkor', 'után', 'között', 'pedig'],
+  id: ['yang', 'dan', 'di', 'ini', 'itu', 'dengan', 'untuk', 'dari', 'dalam', 'tidak', 'akan', 'pada', 'juga', 'ke', 'ada', 'atau', 'oleh', 'karena', 'sudah', 'bisa', 'saya', 'kami', 'kita', 'mereka', 'adalah', 'telah', 'lebih', 'seperti', 'hanya', 'harus', 'sangat', 'bahwa'],
+  ru: ['и', 'в', 'не', 'на', 'что', 'с', 'по', 'это', 'как', 'к', 'из', 'о', 'для', 'за', 'от', 'он', 'она', 'но', 'так', 'же', 'бы', 'все', 'его', 'ее', 'её', 'был', 'были', 'только', 'или', 'при', 'мы', 'вы', 'они', 'уже', 'есть', 'если'],
+  uk: ['і', 'й', 'що', 'це', 'як', 'до', 'із', 'від', 'він', 'вона', 'але', 'його', 'її', 'був', 'були', 'тільки', 'або', 'ми', 'ви', 'вони', 'вже', 'є', 'якщо', 'та', 'також', 'дуже'],
+};
+
+/** word → the single language it is distinctive for (shared words omitted). */
+const DISTINCTIVE: ReadonlyMap<string, string> = (() => {
+  const owners = new Map<string, Set<string>>();
+  for (const [lang, words] of Object.entries(FUNCTION_WORDS)) {
+    for (const w of words) {
+      let set = owners.get(w);
+      if (!set) owners.set(w, (set = new Set()));
+      set.add(lang);
+    }
+  }
+  const out = new Map<string, string>();
+  for (const [w, langs] of owners) if (langs.size === 1) out.set(w, [...langs][0] as string);
+  return out;
+})();
+
+type Script =
+  | 'latin'
+  | 'cyrillic'
+  | 'greek'
+  | 'arabic'
+  | 'hebrew'
+  | 'devanagari'
+  | 'bengali'
+  | 'gurmukhi'
+  | 'gujarati'
+  | 'tamil'
+  | 'telugu'
+  | 'kannada'
+  | 'malayalam'
+  | 'sinhala'
+  | 'thai'
+  | 'lao'
+  | 'khmer'
+  | 'myanmar'
+  | 'georgian'
+  | 'armenian'
+  | 'ethiopic'
+  | 'hangul'
+  | 'kana'
+  | 'han';
+
+const SCRIPT_RANGES: readonly [number, number, Script][] = [
+  [0x0041, 0x024f, 'latin'],
+  [0x1e00, 0x1eff, 'latin'],
+  [0x0370, 0x03ff, 'greek'],
+  [0x0400, 0x052f, 'cyrillic'],
+  [0x0530, 0x058f, 'armenian'],
+  [0x0590, 0x05ff, 'hebrew'],
+  [0x0600, 0x06ff, 'arabic'],
+  [0x0750, 0x077f, 'arabic'],
+  [0x0900, 0x097f, 'devanagari'],
+  [0x0980, 0x09ff, 'bengali'],
+  [0x0a00, 0x0a7f, 'gurmukhi'],
+  [0x0a80, 0x0aff, 'gujarati'],
+  [0x0b80, 0x0bff, 'tamil'],
+  [0x0c00, 0x0c7f, 'telugu'],
+  [0x0c80, 0x0cff, 'kannada'],
+  [0x0d00, 0x0d7f, 'malayalam'],
+  [0x0d80, 0x0dff, 'sinhala'],
+  [0x0e00, 0x0e7f, 'thai'],
+  [0x0e80, 0x0eff, 'lao'],
+  [0x1000, 0x109f, 'myanmar'],
+  [0x10a0, 0x10ff, 'georgian'],
+  [0x1100, 0x11ff, 'hangul'],
+  [0x1200, 0x137f, 'ethiopic'],
+  [0x1780, 0x17ff, 'khmer'],
+  [0x3040, 0x30ff, 'kana'],
+  [0x3400, 0x9fff, 'han'],
+  [0xac00, 0xd7af, 'hangul'],
+  [0xf900, 0xfaff, 'han'],
+];
+
+function scriptOf(cp: number): Script | null {
+  // Basic-Latin letters are the hot path.
+  if ((cp >= 0x61 && cp <= 0x7a) || (cp >= 0x41 && cp <= 0x5a)) return 'latin';
+  if (cp < 0xc0) return null;
+  for (const [lo, hi, s] of SCRIPT_RANGES) if (cp >= lo && cp <= hi) return s;
+  return null;
+}
+
+/** Scripts a page in a given primary language is legitimately written in. */
+const LANGUAGE_SCRIPTS: Readonly<Record<string, readonly Script[]>> = {
+  ru: ['cyrillic'], uk: ['cyrillic'], be: ['cyrillic'], bg: ['cyrillic'], mk: ['cyrillic'],
+  kk: ['cyrillic', 'latin'], ky: ['cyrillic'], mn: ['cyrillic'], tg: ['cyrillic'],
+  sr: ['cyrillic', 'latin'], az: ['latin', 'cyrillic', 'arabic'], uz: ['latin', 'cyrillic'],
+  el: ['greek'],
+  ar: ['arabic'], fa: ['arabic'], ur: ['arabic'], ps: ['arabic'], ku: ['arabic', 'latin'], sd: ['arabic'],
+  he: ['hebrew'], yi: ['hebrew'],
+  hi: ['devanagari'], mr: ['devanagari'], ne: ['devanagari'], sa: ['devanagari'],
+  bn: ['bengali'], as: ['bengali'], pa: ['gurmukhi', 'arabic'], gu: ['gujarati'],
+  ta: ['tamil'], te: ['telugu'], kn: ['kannada'], ml: ['malayalam'], si: ['sinhala'],
+  th: ['thai'], lo: ['lao'], km: ['khmer'], my: ['myanmar'],
+  ka: ['georgian'], hy: ['armenian'], am: ['ethiopic'], ti: ['ethiopic'],
+  zh: ['han'], ja: ['han', 'kana'], ko: ['hangul', 'han'],
+};
+
+/** The language a script implies on its own, when it implies exactly one. */
+const SCRIPT_LANGUAGE: Partial<Record<Script, string>> = {
+  greek: 'el', hebrew: 'he', hangul: 'ko', kana: 'ja', han: 'zh', thai: 'th', lao: 'lo',
+  khmer: 'km', myanmar: 'my', georgian: 'ka', armenian: 'hy', bengali: 'bn', gurmukhi: 'pa',
+  gujarati: 'gu', tamil: 'ta', telugu: 'te', kannada: 'kn', malayalam: 'ml', sinhala: 'si',
+  devanagari: 'hi', ethiopic: 'am', arabic: 'ar',
+};
+
+/** Normalise an `html[lang]` value to its primary subtag, or null. */
+export function primaryLanguage(tag: string | null | undefined): string | null {
+  const primary = (tag ?? '').trim().toLowerCase().split(/[-_]/)[0] ?? '';
+  if (!/^[a-z]{2,3}$/.test(primary) || primary === 'und' || primary === 'mul') return null;
+  // Legacy / alias primaries that name the same language.
+  if (primary === 'iw') return 'he';
+  if (primary === 'in') return 'id';
+  if (primary === 'ji') return 'yi';
+  if (primary === 'nb' || primary === 'nn') return 'no';
+  return primary;
+}
+
+export interface ContentLanguage {
+  /** Primary language the prose is written in, or null when not determinable. */
+  detected: string | null;
+  /**
+   * True only when the page declares a language and the prose is, with
+   * high confidence, written in a different one.
+   */
+  mismatch: boolean;
+}
+
+/**
+ * Read the language of `text` and compare it with the declared
+ * `html[lang]`. `text` should be the page's main-content prose (chrome
+ * stripped) so a translated page under a shared menu is judged on its
+ * own words.
+ */
+export function assessContentLanguage(text: string, declaredLang: string | null): ContentLanguage {
+  const declared = primaryLanguage(declaredLang);
+  const sample = text.length > SAMPLE_CHARS ? text.slice(0, SAMPLE_CHARS) : text;
+
+  // ── 1. Script distribution ────────────────────────────────────────
+  const scriptCounts = new Map<Script, number>();
+  let letters = 0;
+  for (const ch of sample) {
+    const s = scriptOf(ch.codePointAt(0) ?? 0);
+    if (!s) continue;
+    letters++;
+    scriptCounts.set(s, (scriptCounts.get(s) ?? 0) + 1);
+  }
+  if (letters < MIN_LETTERS) return { detected: null, mismatch: false };
+  let dominant: Script | null = null;
+  let dominantCount = 0;
+  for (const [s, n] of scriptCounts) {
+    if (n > dominantCount) {
+      dominant = s;
+      dominantCount = n;
+    }
+  }
+  if (!dominant || dominantCount / letters < 0.6) return { detected: null, mismatch: false };
+  // Japanese mixes kanji and kana; any real share of kana makes Han text Japanese.
+  if (dominant === 'han' && (scriptCounts.get('kana') ?? 0) / letters > 0.1) dominant = 'kana';
+
+  // ── 2. Function words (alphabetic scripts with word spacing) ──────
+  let detected: string | null = null;
+  let ratios: Map<string, number> | null = null;
+  if (dominant === 'latin' || dominant === 'cyrillic') {
+    const tokens = sample.toLowerCase().match(/\p{L}+/gu) ?? [];
+    if (tokens.length >= MIN_TOKENS) {
+      const hits = new Map<string, number>();
+      for (const t of tokens) {
+        const lang = DISTINCTIVE.get(t);
+        if (lang) hits.set(lang, (hits.get(lang) ?? 0) + 1);
+      }
+      ratios = new Map([...hits].map(([l, n]) => [l, n / tokens.length]));
+      let best: string | null = null;
+      let bestRatio = 0;
+      for (const [l, r] of ratios) {
+        if (r > bestRatio) {
+          best = l;
+          bestRatio = r;
+        }
+      }
+      if (best && bestRatio >= DETECTED_PRESENT_RATIO) detected = best;
+    }
+  } else {
+    detected = SCRIPT_LANGUAGE[dominant] ?? null;
+  }
+
+  if (!declared) return { detected, mismatch: false };
+
+  // ── 3. Compare with the declaration ───────────────────────────────
+  const expectedScripts = LANGUAGE_SCRIPTS[declared] ?? (FUNCTION_WORDS[declared] ? ['latin'] : null);
+  if (expectedScripts && !expectedScripts.includes(dominant)) {
+    // Kana pages are Japanese; Han-only pages are fine for ja too.
+    return { detected, mismatch: true };
+  }
+  // Same script: only the function-word evidence can tell languages
+  // apart, and only for languages it knows on both sides.
+  if (!ratios || !FUNCTION_WORDS[declared] || !detected || detected === declared) {
+    return { detected, mismatch: false };
+  }
+  const declaredRatio = ratios.get(declared) ?? 0;
+  const mismatch = declaredRatio < DECLARED_ABSENT_RATIO;
+  return { detected, mismatch };
+}

packages/core/src/crawler.ts

@@ -20,12 +20,21 @@ import {
   isPrivateHost,
   isUrlMalformed,
   detectUrlTrap,
+  calendarPatternKey,
   resolveStartUrl,
   compileUrlRegexRewrites,
   toEscapedFragmentUrl,
   type UrlRewriteOptions,
 } from './url-utils.js';
-import { parseHtml, estimatePixelWidth, type ParsedPage } from './html-parser.js';
+import { hammingDistance } from './simhash.js';
+import {
+  parseHtml,
+  estimatePixelWidth,
+  canonicalCharset,
+  readServedHeadSignals,
+  META_DESCRIPTION_FONT_PX,
+  type ParsedPage,
+} from './html-parser.js';
 import { analyseCookies, extractSetCookies } from './cookies.js';
 import { loadRobots, robotsUserAgentToken, type RobotsChecker } from './robots.js';
 import {
@@ -914,6 +923,10 @@ export class Crawler extends EventEmitter {
    * the flag is what makes it visible in the Crawl Trap filter.
    */
   private trapKind(url: string): string | null {
+    if (this.calendarTraps.size > 0) {
+      const key = calendarPatternKey(url);
+      if (key && this.calendarTraps.has(key)) return 'calendar';
+    }
     return detectUrlTrap(url, {
       maxRepeatedSegments: this.config.maxRepeatedPathSegments,
       maxQueryParams: this.config.maxQueryParams,
@@ -938,6 +951,83 @@ export class Crawler extends EventEmitter {
   /** URLs dropped by `isLoopTrap`, reported at the end of the crawl. */
   private loopTrapsDropped = 0;
 
+  /**
+   * Infinite date navigation ("next month" forever). One calendar page
+   * looks like any other page, so the trap is recognised by behaviour:
+   * pages sharing a date-URL shape (`calendarPatternKey`) that keep coming
+   * back with the same near-empty content. Once a shape has produced
+   * CALENDAR_TRAP_MIN_PAGES pages and CALENDAR_TRAP_SIMILAR_SHARE of them
+   * are near-identical, it is flagged and no longer followed. A real event
+   * calendar or a blog's monthly archive shows different content per
+   * period and never trips it.
+   */
+  private static readonly CALENDAR_TRAP_MIN_PAGES = 20;
+  private static readonly CALENDAR_TRAP_SIMILAR_SHARE = 0.8;
+  /** SimHash bits two "same template, different month" pages may differ by. */
+  private static readonly CALENDAR_TRAP_MAX_HAMMING = 6;
+  private calendarStats = new Map<
+    string,
+    { pages: number; similar: number; base: string | null; urls: string[] }
+  >();
+  private calendarTraps = new Set<string>();
+  private calendarTrapDropped = 0;
+
+  private isCalendarTrapped(url: string): boolean {
+    if (this.calendarTraps.size === 0) return false;
+    const key = calendarPatternKey(url);
+    return key !== null && this.calendarTraps.has(key);
+  }
+
+  /** Feed one fetched 2xx HTML page into the calendar-trap detector. */
+  private observeCalendarPage(url: string, simhash: string | null): void {
+    const key = calendarPatternKey(url);
+    if (!key || this.calendarTraps.has(key)) return;
+    let st = this.calendarStats.get(key);
+    if (!st) {
+      // `base` is the first page's fingerprint; empty pages (no SimHash —
+      // too little text) count as alike with each other.
+      st = { pages: 0, similar: 0, base: simhash, urls: [] };
+      this.calendarStats.set(key, st);
+    }
+    st.pages++;
+    st.urls.push(url);
+    const alike =
+      st.pages === 1 ||
+      (simhash === null && st.base === null) ||
+      (simhash !== null &&
+        st.base !== null &&
+        hammingDistance(simhash, st.base) <= Crawler.CALENDAR_TRAP_MAX_HAMMING);
+    if (alike) st.similar++;
+    if (
+      st.pages >= Crawler.CALENDAR_TRAP_MIN_PAGES &&
+      st.similar / st.pages >= Crawler.CALENDAR_TRAP_SIMILAR_SHARE
+    ) {
+      this.calendarTraps.add(key);
+      const urls = st.urls;
+      this.calendarStats.delete(key);
+      this.emit(
+        'warn',
+        `Crawl trap: ${urls.length} near-identical pages behind one date-navigation URL pattern ` +
+          `(${urls[0]} …) — flagged, and further URLs of that pattern are not followed.`,
+      );
+      void this.dbCall<void>('markUrlTrap', [urls, 'calendar']).catch(() => {
+        /* best-effort flag — the crawl itself is unaffected */
+      });
+    } else if (st.pages > Crawler.CALENDAR_TRAP_MIN_PAGES * 4 && st.similar / st.pages < 0.5) {
+      // Plainly a real archive — stop tracking it so memory stays flat.
+      this.calendarStats.delete(key);
+    }
+  }
+  /**
+   * Sitemap entries waiting to be crawled (Crawl Linked XML Sitemaps).
+   * Held back until the link crawl has drained: seeding them up front put
+   * every sitemap URL in the queue at depth 0, so pages the home page
+   * links to were recorded at depth 0 instead of 1 and the depth report
+   * read as flat. Fed after the link graph is complete, they only add
+   * what links never reached — the orphans.
+   */
+  private deferredSitemapSeeds: string[] = [];
+
   /**
    * Drop URLs whose path extension matches any user-configured exclude
    * (e.g. `pdf`, `jpg`). The start URL is exempt — even if it ends in
@@ -1301,6 +1391,29 @@ export class Crawler extends EventEmitter {
         await this.queue.onIdle();
         await this.externalQueue.onIdle();
       }
+      // Sitemap URLs no link reached (see `deferredSitemapSeeds`). The
+      // `seen` check inside enqueue drops everything already crawled, so
+      // only orphans — and 404s the sitemap still lists — are fetched.
+      if (this.deferredSitemapSeeds.length > 0 && !this.stopped) {
+        const seeds = this.deferredSitemapSeeds;
+        this.deferredSitemapSeeds = [];
+        let queued = 0;
+        for (const url of seeds) {
+          if (this.seen.has(url)) continue;
+          this.enqueue({ url, depth: 0 });
+          queued++;
+        }
+        if (queued > 0) {
+          this.emit('info', `Sitemap: ${queued} URL(s) not reached by links queued from the sitemap`);
+          this.setOp('crawl:sitemap-orphans');
+          while (!this.stopped) {
+            await this.queue.onIdle();
+            await new Promise<void>((r) => setTimeout(r, 50));
+            if (this.queue.size + this.queue.pending === 0) break;
+          }
+          await this.externalQueue.onIdle();
+        }
+      }
     } finally {
       if (this.progressTimer) clearInterval(this.progressTimer);
       this.progressTimer = null;
@@ -1322,6 +1435,13 @@ export class Crawler extends EventEmitter {
     // Say what the loop guard dropped. Silently shrinking a crawl is exactly
     // the kind of invisible behaviour that makes a missing page impossible
     // to explain afterwards.
+    if (this.calendarTrapDropped > 0) {
+      this.emit(
+        'info',
+        `Crawl trap guard: skipped ${this.calendarTrapDropped.toLocaleString()} URL(s) of a ` +
+          'date-navigation pattern that only produced near-identical pages.',
+      );
+    }
     if (this.loopTrapsDropped > 0) {
       this.emit(
         'info',
@@ -1408,9 +1528,9 @@ export class Crawler extends EventEmitter {
       await yieldToEventLoop();
       this.emit('info', 'Clustering duplicates…');
       this.setOp('post-crawl:cluster-duplicates');
-      // Duplicate clustering has JS-side simhash work that lives on
-      // the crawler instance, not the DB; can't blindly off-thread it.
-      this.runDuplicateClustering();
+      // Through the host's DB hook — the writer worker on desktop — so the
+      // SimHash + MinHash passes never run on the main thread.
+      await this.runDuplicateClustering();
 
       // Template / boilerplate detection — runs alongside duplicate
       // clustering because both are content-similarity passes; surfaces the
@@ -1451,16 +1571,6 @@ export class Crawler extends EventEmitter {
       this.setOp('post-crawl:pagination-sequence');
       await this.runDbPass('recomputePaginationSequence');
     }
-    if (this.config.analyseIssues) {
-      await yieldToEventLoop();
-      this.emit('info', 'Materialising issue counters…');
-      this.setOp('post-crawl:materialise-issues');
-      // Route through the injected hook — in the desktop host this
-      // dispatches to the writer-worker, keeping the 70+ correlated
-      // subqueries off the main thread. CLI / tests fall back to
-      // running inline via `db.recomputeUrlsIssuesYielding`.
-      await this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS);
-    }
     await yieldToEventLoop();
     this.setOp('post-crawl:image-probes');
     await this.runImageSizeProbes();
@@ -1482,6 +1592,21 @@ export class Crawler extends EventEmitter {
     await yieldToEventLoop();
     this.setOp('post-crawl:performance-budget');
     this.runBudgetPass();
+    if (this.config.analyseIssues) {
+      await yieldToEventLoop();
+      this.emit('info', 'Materialising issue counters…');
+      this.setOp('post-crawl:materialise-issues');
+      // Last of the analysis passes on purpose: Broken Image, Large Image,
+      // Slow Image and the share-card size checks read the image / social
+      // probe results above. Materialised before them, those counters
+      // stayed at zero after every fresh crawl.
+      //
+      // Route through the injected hook — in the desktop host this
+      // dispatches to the writer-worker, keeping the 70+ correlated
+      // subqueries off the main thread. CLI / tests fall back to
+      // running inline via `db.recomputeUrlsIssuesYielding`.
+      await this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS);
+    }
     // Fold the WAL back into the main DB file now that the write burst
     // is over. wal_autocheckpoint bounds WAL growth *during* the crawl;
     // this TRUNCATE pass reclaims the file afterwards so the project
@@ -1544,18 +1669,24 @@ export class Crawler extends EventEmitter {
    * cluster count as an `info` event so the user can see the post-crawl
    * pass actually fired. Threshold = 0 disables clustering.
    */
-  private runDuplicateClustering(): void {
+  private async runDuplicateClustering(): Promise<void> {
     const threshold = this.config.nearDuplicateHammingThreshold;
-    if (!threshold || threshold <= 0) return;
+    const similarity = this.config.nearDuplicateSimilarity ?? 0;
+    if ((!threshold || threshold <= 0) && similarity <= 0) return;
     try {
-      const { clusters, clusteredUrls } = this.db.recomputeDuplicateClusters(
-        threshold,
+      const { clusters, clusteredUrls } = await this.dbCall<{
+        clusters: number;
+        clusteredUrls: number;
+      }>('recomputeDuplicateClusters', [
+        threshold > 0 ? threshold : -1,
         this.config.duplicatesOnlyIndexable,
-      );
+        similarity,
+      ]);
       if (clusters > 0) {
         this.emit(
           'info',
-          `Duplicates: ${clusters} near-duplicate clusters across ${clusteredUrls} URLs (hamming ≤ ${threshold})`,
+          `Duplicates: ${clusters} near-duplicate clusters across ${clusteredUrls} URLs ` +
+            `(SimHash hamming ≤ ${threshold}, text similarity ≥ ${similarity}%)`,
         );
       }
     } catch (err) {
@@ -2504,15 +2635,9 @@ export class Crawler extends EventEmitter {
           // filters compare against. Crawling them as well is what finds
           // orphans — pages the sitemap declares but nothing links to.
           if (this.config.crawlLinkedSitemaps) {
-            let seeded = 0;
             for (const e of res.entries) {
               const norm = normalizeUrl(e.url, undefined, this.urlRewrites);
-              if (!norm) continue;
-              this.enqueue({ url: norm, depth: 0 });
-              seeded++;
-            }
-            if (seeded > 0) {
-              this.emit('info', `Sitemap: ${seeded} discovered URL(s) queued as crawl seeds`);
+              if (norm) this.deferredSitemapSeeds.push(norm);
             }
           }
         }
@@ -3285,6 +3410,10 @@ export class Crawler extends EventEmitter {
       this.loopTrapsDropped++;
       return;
     }
+    if (item.url !== this.config.startUrl && this.isCalendarTrapped(item.url)) {
+      this.calendarTrapDropped++;
+      return;
+    }
     if (this.config.respectRobotsTxt && !this.robotsAllows(item.url)) {
       // Record it instead of dropping it silently. This is the whole point
       // of the "Blocked by robots.txt" counter — which was permanently 0
@@ -3379,7 +3508,7 @@ export class Crawler extends EventEmitter {
 
   private enqueueResources(parsed: ParsedPage, depth: number): void {
     if (this.config.checkImages) {
-      for (const img of parsed.images) {
+      for (const img of [...parsed.images, ...parsed.socialImages]) {
         if (img.isInternal) this.enqueue({ url: img.src, depth, resourceRow: 'image' });
         // External images: status/size are filled by the post-crawl image
         // probe (images table), which now covers external images too — so
@@ -3626,6 +3755,11 @@ export class Crawler extends EventEmitter {
 
   private async fetchAndProcess(item: QueueItem): Promise<void> {
     if (this.stopped) return;
+    // Queued before its pattern was recognised as a calendar trap.
+    if (item.url !== this.config.startUrl && this.isCalendarTrapped(item.url)) {
+      this.calendarTrapDropped++;
+      return;
+    }
     this.setOp(`crawl:fetch:${item.url}`);
 
     // I-1 — Cooperative yield BEFORE we start work on this URL. The
@@ -4209,9 +4343,38 @@ export class Crawler extends EventEmitter {
       // HTTP Content-Type header's `charset=` parameter so older sites
       // without a meta still surface a value.
       let charset: string | null = parsed.charset;
-      if (!charset && contentType) {
-        const m = contentType.toLowerCase().match(/charset\s*=\s*([^\s;]+)/);
-        if (m && m[1]) charset = m[1];
+      const headerCharset = contentType
+        ? (contentType.toLowerCase().match(/charset\s*=\s*([^\s;]+)/)?.[1] ?? null)
+        : null;
+      if (!charset && headerCharset) charset = headerCharset;
+      // Both declarations, canonicalised, for the mismatch check. When
+      // they disagree the browser obeys the header and the page renders
+      // in the wrong encoding — the declared one in the document never
+      // gets a say.
+      const httpCharset = canonicalCharset(headerCharset);
+      const metaCharset = canonicalCharset(parsed.charset);
+
+      // What JavaScript does to <title> and robots. In rendering mode the
+      // page was parsed from the rendered DOM, so the served HTML is the
+      // reference; in text mode the inline scripts are the only evidence.
+      const sameTitle = (a: string | null, b: string | null): boolean =>
+        (a ?? '').replace(/\s+/g, ' ').trim() === (b ?? '').replace(/\s+/g, ' ').trim();
+      let jsNoindex = 0;
+      let jsTitleChanged = 0;
+      let jsTitle: string | null = null;
+      if (renderedHtml) {
+        const served = readServedHeadSignals(body);
+        if (parsed.hasNoindex && !served.noindex) jsNoindex = 2;
+        if (parsed.title && !sameTitle(parsed.title, served.title)) {
+          jsTitleChanged = 2;
+          jsTitle = parsed.title;
+        }
+      } else {
+        if (parsed.jsNoindexWrite && !parsed.hasNoindex) jsNoindex = 1;
+        if (parsed.jsTitleWrite && (parsed.jsTitle === null || !sameTitle(parsed.jsTitle, parsed.title))) {
+          jsTitleChanged = 1;
+          jsTitle = parsed.jsTitle;
+        }
       }
 
       // Honour bot-scoped directives correctly: `X-Robots-Tag: bingbot:
@@ -4307,7 +4470,6 @@ export class Crawler extends EventEmitter {
         }
       }
 
-      const imagesMissingAlt = parsed.images.filter((img) => img.alt === null).length;
       // Phase 1b — Build the entire per-URL write payload up front and
       // ship it across the writer-worker boundary in one shot. The
       // worker runs the upsert + headers + body snapshot + links +
@@ -4316,6 +4478,9 @@ export class Crawler extends EventEmitter {
       // Falls back to an inline transaction in the no-worker (CLI)
       // case via the default `writeFetchedUrl` injected in the
       // constructor.
+      // Before the write, so a page that completes a calendar trap is
+      // itself stored with the trap flag.
+      if (statusCode >= 200 && statusCode < 300) this.observeCalendarPage(item.url, parsed.simhash);
       this.setOp(`crawl:write:${item.url}`);
       const { urlId } = await this.writeFetchedUrl({
         upsert: {
@@ -4358,8 +4523,8 @@ export class Crawler extends EventEmitter {
           ttfbMs,
           depth: item.depth,
           outlinks: storableLinks.length,
-          imagesCount: parsed.images.length,
-          imagesMissingAlt,
+          imagesCount: parsed.imageCount,
+          imagesMissingAlt: parsed.imagesMissingAlt,
           lang: parsed.lang,
           viewport: parsed.viewport,
           ogTitle: parsed.ogTitle,
@@ -4420,6 +4585,16 @@ export class Crawler extends EventEmitter {
           metaRefreshUrl: this.config.storeMetaRefresh ? parsed.metaRefreshUrl : null,
           jsRedirectUrl: parsed.jsRedirectUrl,
           charset,
+          httpCharset,
+          metaCharset,
+          canonicalOutsideHead: this.config.storeCanonicals ? parsed.canonicalOutsideHead : 0,
+          contentLang: parsed.contentLang,
+          langMismatch: parsed.langMismatch ? 1 : 0,
+          soft404: parsed.soft404Signal && statusCode >= 200 && statusCode < 300 ? 1 : 0,
+          jsNoindex,
+          jsTitleChanged,
+          jsTitle,
+          minhash: parsed.minhash,
           extractionResults: parsed.extractionResults
             ? JSON.stringify(parsed.extractionResults)
             : null,
@@ -4436,7 +4611,7 @@ export class Crawler extends EventEmitter {
           insecureFormActionCount: parsed.insecureFormActionCount,
           missingSriCount: parsed.missingSriCount,
           titlePixelWidth: estimatePixelWidth(parsed.title ?? ''),
-          metaPixelWidth: estimatePixelWidth(parsed.metaDescription ?? ''),
+          metaPixelWidth: estimatePixelWidth(parsed.metaDescription ?? '', META_DESCRIPTION_FONT_PX),
           cookiesCount: cookieSummary.count,
           cookiesInsecure: cookieSummary.insecureCount,
           cookiesNoHttpOnly: cookieSummary.noHttpOnlyCount,
@@ -4490,6 +4665,7 @@ export class Crawler extends EventEmitter {
           : null,
         links: persistedLinks,
         images: this.config.storeImages ? parsed.images : [],
+        socialImages: parsed.socialImages,
         // Whether the empty lists above mean "this page has none"
         // (storage on → the writer replaces the page's rows) or "not
         // recording" (storage off → it keeps what an earlier crawl stored).

packages/core/src/html-parser.ts

@@ -17,6 +17,7 @@ import {
   type UrlRewriteOptions,
 } from './url-utils.js';
 import { computeContentFingerprint } from './simhash.js';
+import { assessContentLanguage, primaryLanguage } from './content-language.js';
 import { runExtractionRules } from './extraction.js';
 
 const GENERIC_ANCHORS: ReadonlySet<string> = new Set(GENERIC_ANCHOR_TEXTS);
@@ -217,6 +218,8 @@ export interface ParsedPage {
    * these so any navigation that depends on them is invisible to crawl.
    * Excludes `href="#"` without `onclick` (legitimate scroll anchors)
    * and `href="#section-id"` (in-page jumps — crawlable as the same URL).
+   * Also counts non-`<a>` elements whose `onclick` navigates
+   * (`location.href = …`, `location.assign(…)`, `window.open(…)`).
    */
   jsOnlyLinksCount: number;
   /**
@@ -373,7 +376,37 @@ export interface ParsedPage {
    */
   analyticsTrackers: AnalyticsTracker[];
   links: DiscoveredLink[];
+  /**
+   * Primary language the main-content prose is written in (`tr`, `en`, …),
+   * null when it can't be told with confidence. See `content-language.ts`.
+   */
+  contentLang: string | null;
+  /** `html[lang]` names one language while the prose is clearly another. */
+  langMismatch: boolean;
+  /** MinHash signature of the main content (near-duplicate similarity); null when too short. */
+  minhash: Uint8Array | null;
+  /** `<link rel="canonical">` elements inside `<body>` (not honoured — see `canonical`). */
+  canonicalOutsideHead: number;
+  /** "Not found" wording in the title/H1 of a page with almost no content. */
+  soft404Signal: boolean;
+  /** An inline script assigns `document.title`. */
+  jsTitleWrite: boolean;
+  /** The literal title the script assigns, when readable. */
+  jsTitle: string | null;
+  /** An inline script writes a robots meta carrying noindex. */
+  jsNoindexWrite: boolean;
+  /** On-page images (`<img>` + its srcset / `<picture>` variants), deduped by URL. */
   images: DiscoveredImage[];
+  /** `<img>` elements with a usable src — the page's image slots. */
+  imageCount: number;
+  /** `<img>` slots with no alt attribute that aren't marked decorative. */
+  imagesMissingAlt: number;
+  /**
+   * `og:image` / `twitter:image` targets not already used as an `<img>`.
+   * Probed for size (the 5 MB share-card limit) but never counted as
+   * on-page images.
+   */
+  socialImages: DiscoveredImage[];
   /**
    * Non-anchor subresources (`<script src>`, `<link rel="stylesheet">`) the
    * crawler can fetch so they surface in the Internal tab. Images are carried
@@ -462,6 +495,148 @@ const JS_SCRIPT_TYPES = new Set([
   'application/ecmascript',
 ]);
 
+/** An `onclick` body that navigates somewhere rather than acting in place. */
+const JS_NAVIGATION_RE =
+  /(?:^|[^\w$.])(?:window\.|document\.|top\.|self\.)?location(?:\.href)?\s*=(?!=)|location\.(?:assign|replace)\s*\(|window\.open\s*\(/;
+
+/** `document.title = …` — an assignment, never a read or a comparison. */
+const JS_TITLE_ASSIGN_RE =
+  /(?:^|[^\w$.])document\.title\s*=(?!=)\s*(?:(["'`])((?:(?!\1)[^\\\n$]|\\.)*)\1)?/;
+/** Writing the `<title>` element's text directly. */
+const JS_TITLE_ELEMENT_RE =
+  /(?:querySelector\(\s*["']title["']\s*\)|getElementsByTagName\(\s*["']title["']\s*\)\s*\[\s*0\s*\])\s*\.\s*(?:textContent|innerText|innerHTML)\s*=(?!=)/;
+/** Markup or DOM writes a script could add a robots meta with. */
+const JS_META_WRITE_RE =
+  /createElement\s*\(\s*["']meta["']|setAttribute\s*\(\s*["']content["']|\.content\s*=(?!=)|innerHTML\s*[+]?=|insertAdjacentHTML|document\.write/;
+
+export interface InlineScriptHeadWrites {
+  /** An inline script assigns the document title. */
+  titleWrite: boolean;
+  /** The literal it assigns, when it's a plain string; null otherwise. */
+  title: string | null;
+  /** An inline script writes a robots / googlebot meta carrying noindex. */
+  noindexWrite: boolean;
+}
+
+/**
+ * Static read of what inline scripts do to `<head>` — the text-mode
+ * stand-in for rendering. A crawler that doesn't run JavaScript indexes
+ * the served title and robots directives, while Google renders the page
+ * and sees whatever the scripts leave behind; a mismatch between the two
+ * is the finding. Only executable inline scripts are read (JSON-LD and
+ * other data blocks are not code), and only unambiguous writes count:
+ * `document.title = …`, or a script that both mentions `noindex` next to
+ * `robots`/`googlebot` and writes a meta tag.
+ */
+function findInlineScriptHeadWrites($: cheerio.CheerioAPI): InlineScriptHeadWrites {
+  const out: InlineScriptHeadWrites = { titleWrite: false, title: null, noindexWrite: false };
+  $('script:not([src])').each((_, el) => {
+    const type = ($(el).attr('type') ?? '').trim().toLowerCase();
+    if (!JS_SCRIPT_TYPES.has(type)) return;
+    const code = $(el).text();
+    if (!code) return;
+    if (!out.titleWrite && /title/.test(code)) {
+      const m = JS_TITLE_ASSIGN_RE.exec(code);
+      if (m) {
+        out.titleWrite = true;
+        const literal = m[2];
+        out.title =
+          literal !== undefined
+            ? decodeEntities(literal.replace(/\\(.)/g, '$1')).trim() || null
+            : null;
+      } else if (JS_TITLE_ELEMENT_RE.test(code)) {
+        out.titleWrite = true;
+      }
+    }
+    if (
+      !out.noindexWrite &&
+      /noindex/i.test(code) &&
+      /\brobots\b|googlebot/i.test(code) &&
+      JS_META_WRITE_RE.test(code)
+    ) {
+      out.noindexWrite = true;
+    }
+    return undefined;
+  });
+  return out;
+}
+
+/**
+ * "Not found" wording in a page's title or H1, in the languages the app
+ * ships plus the larger European ones. A bare "404" only counts as an
+ * error code — at the start of the title or next to an error word — so a
+ * product called "Peugeot 404" stays a product.
+ */
+const NOT_FOUND_RES: readonly RegExp[] = [
+  /\bnot\s+(?:be\s+)?found\b|\bno\s+longer\s+(?:exists?|available)\b|\bdoes\s*n[o’']t\s+exist\b|\bno\s+results\b|\bnothing\s+found\b/i,
+  /^\s*(?:error\s*)?404\b|\b(?:error|hata(?:s[ıi])?|erreur|fehler|errore|erro)\s*:?\s*404\b|ошибка\s*:?\s*404\b|\b404\s*(?:error|hata|erreur|fehler|errore|erro|not\b|-\s*page)/i,
+  /bulunamad[ıi]|mevcut\s+de[ğg]il|art[ıi]k\s+yok|tap[ıi]lmad[ıi]|m[öo]vcud\s+deyil/i,
+  /\bno\s+(?:se\s+(?:ha\s+)?)?encontr(?:ad[oa]s?|[óo])\b|\bno\s+existe\b|\bsin\s+resultados\b/i,
+  /introuvable|\b(?:non|pas)\s+trouv[ée]e?s?\b|n['’]existe\s+(?:pas|plus)|aucun\s+r[ée]sultat/i,
+  /n[ãa]o\s+(?:foi\s+)?encontrad[oa]|n[ãa]o\s+existe|nenhum\s+resultado/i,
+  /\bnon\s+(?:[èe]\s+stat[oa]\s+)?trovat[oa]\b|\bnon\s+esiste\b|nessun\s+risultato/i,
+  /nicht\s+gefunden|existiert\s+nicht|keine\s+ergebnisse|niet\s+gevonden|bestaat\s+niet|nie\s+znaleziono|nie\s+istnieje/i,
+  /не\s+найден|не\s+существует|ничего\s+не\s+найдено|не\s+знайдено/i,
+  /未找到|找不到|页面不存在|頁面不存在|見つかりません|存在しません|찾을\s*수\s*없|존재하지\s*않|नहीं\s+मिल|मौजूद\s+नहीं/,
+];
+
+/** Main-content words a genuine "not found" page stays under. */
+const SOFT_404_MAX_MAIN_WORDS = 150;
+
+function looksNotFound(text: string | null): boolean {
+  if (!text) return false;
+  return NOT_FOUND_RES.some((re) => re.test(text));
+}
+
+/**
+ * WHATWG-canonical name for a charset label (`utf8` → `utf-8`,
+ * `iso-8859-9` → `windows-1254`, `latin1` → `windows-1252`), so two
+ * spellings of the same encoding never read as a mismatch. Labels the
+ * Encoding Standard doesn't know are returned lower-cased as-is.
+ */
+export function canonicalCharset(label: string | null | undefined): string | null {
+  const raw = (label ?? '').trim().replace(/^["']|["']$/g, '').toLowerCase();
+  if (!raw) return null;
+  try {
+    return new TextDecoder(raw).encoding;
+  } catch {
+    return raw;
+  }
+}
+
+/** Served-HTML `<title>` and robots noindex, without a full parse. */
+export interface ServedHeadSignals {
+  title: string | null;
+  noindex: boolean;
+}
+
+/**
+ * Read the title and the robots/googlebot noindex out of the HTML as the
+ * server sent it. Used in JavaScript-rendering mode, where the page is
+ * parsed from the rendered DOM, to tell what the scripts changed. A
+ * regex pass over `<head>` is enough for these two fields and keeps the
+ * second look at the document off the parser's critical path.
+ */
+export function readServedHeadSignals(html: string): ServedHeadSignals {
+  const headEnd = html.search(/<\/head\s*>/i);
+  const head = headEnd >= 0 ? html.slice(0, headEnd) : html.slice(0, 200_000);
+  const t = /<title\b[^>]*>([\s\S]*?)<\/title\s*>/i.exec(head);
+  const title = t?.[1] ? decodeEntities(t[1].replace(/\s+/g, ' ').trim()) || null : null;
+  let noindex = false;
+  for (const m of head.matchAll(/<meta\b[^>]*>/gi)) {
+    const tag = m[0];
+    const name = /\bname\s*=\s*["']?([^"'\s>]+)/i.exec(tag)?.[1]?.toLowerCase();
+    if (name !== 'robots' && name !== 'googlebot') continue;
+    const content = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i.exec(tag);
+    const value = (content?.[1] ?? content?.[2] ?? content?.[3] ?? '').toLowerCase();
+    if (/\bnoindex\b|\bnone\b/.test(value)) {
+      noindex = true;
+      break;
+    }
+  }
+  return { title, noindex };
+}
+
 function findJsRedirect(
   $: cheerio.CheerioAPI,
   baseUrl: string,
@@ -499,6 +674,14 @@ function extractMainContentText(
   userSelector?: string,
 ): string {
   const collapse = (s: string): string => s.replace(/\s+/g, ' ').trim();
+  // Text of a region without its inline script / style payloads — a
+  // `<script>` sitting inside `<main>` is code, not content.
+  const prose = (region: ReturnType<typeof $>): string => {
+    if (region.find('script, style, noscript, template').length === 0) return collapse(region.text());
+    const copy = region.clone();
+    copy.find('script, style, noscript, template').remove();
+    return collapse(copy.text());
+  };
 
   // V2 Faz 14 — user-supplied content area selector takes precedence
   // over the heuristic. Lets duplicate fingerprinting target a specific
@@ -511,7 +694,7 @@ function extractMainContentText(
     try {
       const region = $(userSelector.trim()).first();
       if (region.length > 0) {
-        const t = collapse(region.text());
+        const t = prose(region);
         if (t.length > 0) return t;
       }
     } catch {
@@ -521,13 +704,13 @@ function extractMainContentText(
 
   const main = $('main, [role="main"]').first();
   if (main.length > 0) {
-    const t = collapse(main.text());
+    const t = prose(main);
     if (t.length > 0) return t;
   }
 
   const article = $('article').first();
   if (article.length > 0) {
-    const t = collapse(article.text());
+    const t = prose(article);
     if (t.length > 0) return t;
   }
 
@@ -536,7 +719,7 @@ function extractMainContentText(
   const body = $('body').clone();
   body
     .find(
-      'nav, header, footer, aside, [role="navigation"], [role="banner"], [role="contentinfo"], [role="complementary"]',
+      'nav, header, footer, aside, [role="navigation"], [role="banner"], [role="contentinfo"], [role="complementary"], script, style, noscript, template',
     )
     .remove();
   const stripped = collapse(body.text());
@@ -636,7 +819,15 @@ export function parseHtml(
     const text = decodeEntities(raw).slice(0, 200);
     headings.push({ level, text });
   });
-  const canonicalEls = $('link[rel="canonical"]');
+  // Only a canonical in `<head>` counts. Google ignores `rel=canonical`
+  // anywhere in `<body>` — honouring it here reported a page as properly
+  // canonicalised when the search engine sees no canonical at all. The
+  // body ones are counted for their own issue instead. (No explicit
+  // `<body>` element — tag omitted in the source — reads as head, which
+  // is where such a link sits in the HTML5 tree.)
+  const allCanonicalEls = $('link[rel="canonical"]');
+  const canonicalEls = allCanonicalEls.filter((_, el) => $(el).closest('body').length === 0);
+  const canonicalOutsideHead = allCanonicalEls.length - canonicalEls.length;
   const canonical = (canonicalEls.first().attr('href') ?? '').trim() || null;
   const canonicalCount = canonicalEls.length;
   // `canonical` above stays verbatim - the "Canonical Not Absolute" filter
@@ -1193,7 +1384,28 @@ export function parseHtml(
   $visibleBody.find('script, style, noscript').remove();
   const text = $visibleBody.text().replace(/\s+/g, ' ').trim();
   const wordCount = text.length > 0 ? text.split(' ').filter(Boolean).length : 0;
-  const readability = computeReadability(text, wordCount);
+
+  // Main-content prose (chrome stripped) — the duplicate fingerprint and
+  // the content-language check both judge a page on its own words, not
+  // on the menu it shares with every other page.
+  const mainText = extractMainContentText($, opts.contentAreaSelector);
+  const contentLanguage = assessContentLanguage(mainText, lang);
+  // Readability formula: the declared language when the prose agrees
+  // with it, otherwise whatever the prose actually is.
+  const declaredPrimary = primaryLanguage(lang);
+  const readabilityLang =
+    declaredPrimary && !contentLanguage.mismatch
+      ? declaredPrimary
+      : (contentLanguage.detected ?? assessContentLanguage(text, null).detected);
+  const readability = computeReadability(text, wordCount, readabilityLang);
+
+  // Soft 404: a 2xx page that says it isn't there. Needs the wording in
+  // the title or H1 AND almost no content of its own, so an article
+  // *about* 404 errors — long, on topic — never qualifies.
+  const mainWordCount = mainText ? mainText.split(' ').length : 0;
+  const soft404Signal =
+    (looksNotFound(title) || looksNotFound(h1)) && mainWordCount < SOFT_404_MAX_MAIN_WORDS;
+  const headWrites = findInlineScriptHeadWrites($);
 
   // Analytics / marketing trackers — fingerprinted by script src + inline
   // JS substrings + meta tags. Cheaper to scan once over `$` than per-tracker.
@@ -1203,9 +1415,7 @@ export function parseHtml(
   // Computed from the main-content region (boilerplate stripped) so two
   // pages that share a big nav/footer template but differ in body
   // content aren't falsely clustered as near-duplicates.
-  const { simhash, contentHash } = computeContentFingerprint(
-    extractMainContentText($, opts.contentAreaSelector),
-  );
+  const { simhash, contentHash, minhash } = computeContentFingerprint(mainText);
 
   // Custom Extraction — runs after all standard fields so the cheerio
   // tree is fully populated. Per-rule failures are isolated; the worst
@@ -1329,6 +1539,15 @@ export function parseHtml(
       jsOnlyLinksCount++;
     }
   });
+  // Clickable non-link elements that navigate in script — `<span onclick=
+  // "location.href='/x'">`, `<button onclick="window.open(…)">`. There is
+  // no URL in the markup at all, so the target is reachable only by a
+  // visitor running JavaScript. Handlers that don't navigate (add to cart,
+  // toggle a menu) are left alone.
+  $('[onclick]').each((_, el) => {
+    if ((el as { name?: string }).name?.toLowerCase() === 'a') return;
+    if (JS_NAVIGATION_RE.test($(el).attr('onclick') ?? '')) jsOnlyLinksCount++;
+  });
 
   // Hrefs that survive being an `<a href>` but not being a URL: doubled
   // schemes, stray whitespace inside the authority, bare `:::`. Kept raw
@@ -1596,6 +1815,7 @@ export function parseHtml(
       width,
       height,
       isInternal: isSameHost(pageUrl, normalized, opts),
+      variant: true,
     });
   };
 
@@ -1605,6 +1825,26 @@ export function parseHtml(
   // imagery adoption" signal alongside `images_responsive`.
   const pictureCount = $('picture').length;
 
+  // `<img>` alt as the accessibility tree sees it: the attribute's text,
+  // `''` for a decorative image, `null` when it is genuinely missing. An
+  // image explicitly taken out of the tree (`role="presentation"` /
+  // `role="none"` / `aria-hidden="true"`) is decorative even without the
+  // attribute — screen readers skip it, so there is nothing to describe.
+  const imgAlt = ($img: ReturnType<typeof $>): string | null => {
+    const altAttr = $img.attr('alt');
+    if (altAttr !== undefined) return decodeEntities(altAttr.trim());
+    const role = ($img.attr('role') ?? '').trim().toLowerCase();
+    const hidden = ($img.attr('aria-hidden') ?? '').trim().toLowerCase() === 'true';
+    return role === 'presentation' || role === 'none' || hidden ? '' : null;
+  };
+
+  // Per-`<img>` tallies. `imageMap` below is keyed by URL and also holds
+  // srcset / `<picture>` variants, so its size is not "how many images
+  // the page shows" — counting slots keeps Missing Alt and the lazy /
+  // responsive adoption ratios on the same footing.
+  let imageSlotCount = 0;
+  let imagesMissingAlt = 0;
+
   $('img[src]').each((_, el) => {
     const $el = $(el);
     const rawSrc = $el.attr('src');
@@ -1616,18 +1856,13 @@ export function parseHtml(
     if (!normalized) return;
     if (!/^https?:/.test(normalized)) return;
 
-    const altAttr = $el.attr('alt');
-    const alt =
-      altAttr === undefined
-        ? null // alt missing entirely (accessibility issue)
-        : decodeEntities(altAttr.trim()); // empty string means decorative — kept as ''
+    // `null` = alt missing entirely (accessibility issue); `''` = decorative.
+    const alt = imgAlt($el);
+    imageSlotCount++;
+    if (alt === null) imagesMissingAlt++;
     // Count `alt=""` (decorative) separately from missing alt — Screaming
     // Frog distinguishes them in its accessibility filters and so should we.
-    if (altAttr !== undefined && (alt === '' || alt === null)) {
-      // alt="" specifically; missing-alt is tallied at the DB level via
-      // image_usages rows whose alt is null.
-      if (altAttr.trim() === '') imagesEmptyAlt++;
-    }
+    if (alt === '') imagesEmptyAlt++;
     const width = parseIntAttr($el.attr('width'));
     const height = parseIntAttr($el.attr('height'));
     const isInternal = isSameHost(pageUrl, normalized, opts);
@@ -1674,24 +1909,30 @@ export function parseHtml(
   $('picture > source[srcset]').each((_, el) => {
     const srcsetAttr = ($(el).attr('srcset') ?? '').trim();
     if (!srcsetAttr) return;
+    // A `<source>` is a format/breakpoint variant of the picture's own
+    // `<img>`, which carries the alt for every variant — the source
+    // element has no alt of its own to be "missing".
+    const $img = $(el).parent('picture').find('img').first();
+    const alt = $img.length > 0 ? imgAlt($img) : null;
     for (const candidate of parseSrcset(srcsetAttr)) {
-      addImageCandidate(candidate, null, null, null);
+      addImageCandidate(candidate, alt, null, null);
     }
   });
 
-  // OG / Twitter share-card images. They rarely appear inside `<img>`
-  // tags so the loop above misses them, but we want them in the probe
-  // pipeline so the size-validation issue filters can see whether
-  // Facebook/Twitter will refuse to render the card. We stub them as
-  // image rows with no alt/dimensions; the existing size-probe pass
-  // will fill `byte_size` automatically post-crawl.
+  // OG / Twitter share-card images. They are metadata for the social
+  // preview, not images on the page: they have no alt attribute to be
+  // missing and no slot to lazy-load. Kept out of `images` (which feeds
+  // Missing Alt and the per-page image counts) and returned separately
+  // so the post-crawl size probe can still check them against the
+  // 5 MB share-card limit.
+  const socialImages: DiscoveredImage[] = [];
   for (const ogish of [ogImage, twitterImage]) {
     if (!ogish) continue;
     const normalized = normalizeUrl(ogish, baseUrl, opts.urlRewrites);
     if (!normalized) continue;
     if (!/^https?:/.test(normalized)) continue;
-    if (imageMap.has(normalized)) continue;
-    imageMap.set(normalized, {
+    if (imageMap.has(normalized) || socialImages.some((s) => s.src === normalized)) continue;
+    socialImages.push({
       src: normalized,
       alt: null,
       width: null,
@@ -1796,7 +2037,18 @@ export function parseHtml(
     contentHash,
     analyticsTrackers,
     links: [...linkMap.values()],
+    contentLang: contentLanguage.detected,
+    langMismatch: contentLanguage.mismatch,
+    minhash,
+    canonicalOutsideHead,
+    soft404Signal,
+    jsTitleWrite: headWrites.titleWrite,
+    jsTitle: headWrites.title,
+    jsNoindexWrite: headWrites.noindexWrite,
     images: [...imageMap.values()],
+    imageCount: imageSlotCount,
+    imagesMissingAlt,
+    socialImages,
     resources: [...resourceMap.values()],
     iframes: [...iframeMap.values()],
     invalidLinks: [...invalidLinkSet],
@@ -2302,15 +2554,17 @@ function detectPathType(rawHref: string): LinkPathType {
 }
 
 /**
- * Approximate pixel width of `text` rendered in Arial 18 px (Google SERP
- * title font). Uses a per-character width table derived from canvas
- * measurements — accuracy is ~±2% which is plenty for the usual "is the
- * title going to truncate?" question. Skipping HTML canvas keeps this
- * pure-Node so the same code runs in CLI and tests.
+ * Approximate pixel width of `text` in Arial, from a per-character width
+ * table measured at 18 px (Google's SERP title size) and scaled linearly
+ * to `fontPx`. Accuracy is ~±2%, plenty for the "is this going to
+ * truncate?" question. Skipping HTML canvas keeps this pure-Node so the
+ * same code runs in CLI and tests.
  *
  * Reference truncation thresholds (Google):
- *   - Title: ~600 px before "..." appears
- *   - Meta description: ~990 px (mobile) / ~920 px (desktop)
+ *   - Title: ~600 px at 18 px
+ *   - Meta description: ~990 px at 14 px — the snippet font is smaller
+ *     than the title font, so measuring it with the title table read
+ *     every ordinary 130–160 character description as truncated.
  */
 const ARIAL_18_WIDTHS: Record<string, number> = {
   // Narrow
@@ -2333,18 +2587,49 @@ const ARIAL_18_WIDTHS: Record<string, number> = {
   '8': 10, '9': 10, '-': 6, '_': 10, '/': 5, '?': 10, '(': 6, ')': 6,
   '[': 6, ']': 6, '{': 6, '}': 6, '"': 7, '*': 7, '+': 11, '=': 11, '<': 11, '>': 11,
   '#': 11, '$': 10, '%': 16, '&': 12, '@': 18,
+  // Typographic punctuation CMSes substitute for the ASCII forms
+  '‘': 4, '’': 4, '“': 7, '”': 7, '–': 10, '—': 18,
+  '…': 18, '·': 5,
+  // Turkish dotless i has no decomposition to fall back on
+  'ı': 5,
 };
 const ARIAL_18_DEFAULT = 10;
 
-export function estimatePixelWidth(text: string): number {
+function arial18Width(ch: string): number {
+  const known = ARIAL_18_WIDTHS[ch];
+  if (known !== undefined) return known;
+  // Accented Latin letters (ç ş ğ ü ö İ é ñ …) render at their base
+  // letter's width; NFD splits off the combining mark.
+  const base = ch.normalize('NFD').charAt(0);
+  if (base !== ch) {
+    const w = ARIAL_18_WIDTHS[base];
+    if (w !== undefined) return w;
+  }
+  // CJK, kana, Hangul and full-width forms occupy a full em.
+  const cp = ch.codePointAt(0) ?? 0;
+  if (
+    (cp >= 0x1100 && cp <= 0x11ff) ||
+    (cp >= 0x2e80 && cp <= 0x9fff) ||
+    (cp >= 0xac00 && cp <= 0xd7af) ||
+    (cp >= 0xf900 && cp <= 0xfaff) ||
+    (cp >= 0xff00 && cp <= 0xff60) ||
+    (cp >= 0x20000 && cp <= 0x2fa1f)
+  ) {
+    return 18;
+  }
+  return ARIAL_18_DEFAULT;
+}
+
+export function estimatePixelWidth(text: string, fontPx = 18): number {
   if (!text) return 0;
   let total = 0;
-  for (const ch of text) {
-    total += ARIAL_18_WIDTHS[ch] ?? ARIAL_18_DEFAULT;
-  }
-  return total;
+  for (const ch of text) total += arial18Width(ch);
+  return fontPx === 18 ? total : Math.round((total * fontPx) / 18);
 }
 
+/** Google renders the description snippet at 14 px. */
+export const META_DESCRIPTION_FONT_PX = 14;
+
 /**
  * Crude DOM breadcrumb for a link element — e.g. "body > main > article > a".
  * Includes up to 8 ancestors so very deep DOMs don't produce huge strings.
@@ -2586,13 +2871,58 @@ interface ReadabilityScores {
 }
 
 /**
- * Flesch / Flesch–Kincaid / Gunning Fog over the body text. Returns nulls
- * for the three scores when the page has too little prose to be stable
- * (<50 words OR zero sentences). The intermediate `sentenceCount` and
- * `complexWordCount` are still surfaced so the detail panel can show
- * what the engine actually measured.
+ * Reading-ease formula per language. The Flesch constants are calibrated
+ * on English word and syllable lengths; applied to an agglutinative
+ * language such as Turkish — long words, many syllables per word — they
+ * report every ordinary page as "very difficult". Each entry is that
+ * language's published Flesch adaptation on the same 0–100 scale, so the
+ * "Hard to Read (<30)" threshold keeps its meaning:
+ *
+ *   tr Ateşman (1997) · es Fernández Huerta · fr Kandel & Moles ·
+ *   it Franchina & Vacca · pt Martins et al. · de Amstad · nl Douma ·
+ *   ru Oborneva
+ *
+ * Syllables are counted from vowels: one per vowel in Turkish and Russian
+ * (where that is exact), one per vowel group elsewhere.
  */
-function computeReadability(text: string, wordCount: number): ReadabilityScores {
+interface ReadingEaseFormula {
+  vowels: RegExp;
+  /** Count every vowel (tr, ru) rather than every vowel group. */
+  perVowel: boolean;
+  ease(wordsPerSentence: number, syllablesPerWord: number): number;
+}
+const READING_EASE: Readonly<Record<string, ReadingEaseFormula>> = {
+  tr: { vowels: /[aeıioöuüâîû]/g, perVowel: true, ease: (w, s) => 198.825 - 40.175 * s - 2.61 * w },
+  es: { vowels: /[aeiouáéíóúü]+/g, perVowel: false, ease: (w, s) => 206.84 - 60 * s - 1.02 * w },
+  fr: { vowels: /[aeiouyàâéèêëîïôûùüœæ]+/g, perVowel: false, ease: (w, s) => 207 - 1.015 * w - 73.6 * s },
+  it: { vowels: /[aeiouàèéìíòóùú]+/g, perVowel: false, ease: (w, s) => 217 - 1.3 * w - 60 * s },
+  pt: { vowels: /[aeiouáéíóúâêôãõàü]+/g, perVowel: false, ease: (w, s) => 248.835 - 1.015 * w - 84.6 * s },
+  de: { vowels: /[aeiouyäöü]+/g, perVowel: false, ease: (w, s) => 180 - w - 58.5 * s },
+  nl: { vowels: /[aeiouyàéèëïöü]+/g, perVowel: false, ease: (w, s) => 206.835 - 0.93 * w - 77 * s },
+  ru: { vowels: /[аеёиоуыэюя]/g, perVowel: true, ease: (w, s) => 206.835 - 1.3 * w - 60.1 * s },
+};
+
+function countSyllablesIn(word: string, f: ReadingEaseFormula): number {
+  const n = word.toLocaleLowerCase().match(f.vowels)?.length ?? 0;
+  return f.perVowel ? n : Math.max(n > 0 ? 1 : 0, n);
+}
+
+/**
+ * Reading ease over the body text, in the formula for `lang` (see
+ * READING_EASE); English keeps Flesch plus Flesch–Kincaid and Gunning Fog,
+ * which have no calibrated equivalent elsewhere. A language with no
+ * published formula — or an unknown language — gets nulls rather than an
+ * English score that would misreport it. Returns nulls for the three
+ * scores when the page has too little prose to be stable (<50 words OR
+ * zero sentences). The intermediate `sentenceCount` and `complexWordCount`
+ * are still surfaced so the detail panel can show what the engine
+ * actually measured.
+ */
+function computeReadability(
+  text: string,
+  wordCount: number,
+  lang: string | null,
+): ReadabilityScores {
   if (!text || wordCount === 0) {
     return {
       fleschReadingEase: null,
@@ -2609,13 +2939,15 @@ function computeReadability(text: string, wordCount: number): ReadabilityScores
   // which backtracks quadratically on a long unterminated run of text.
   const sentenceMatches = text.match(/[^.!?…][.!?…]/g);
   const sentenceCount = sentenceMatches ? sentenceMatches.length : 1;
+  const english = lang === 'en';
+  const formula = english ? null : (lang ? READING_EASE[lang] : undefined) ?? null;
 
   let totalSyllables = 0;
   let complexWordCount = 0;
   // Tokenise on whitespace; the upstream `text` is already normalised.
   for (const token of text.split(' ')) {
     if (!token) continue;
-    const syl = countSyllables(token);
+    const syl = formula ? countSyllablesIn(token, formula) : countSyllables(token);
     if (syl === 0) continue;
     totalSyllables += syl;
     // Gunning Fog "complex" word definition: ≥3 syllables, excluding
@@ -2625,7 +2957,7 @@ function computeReadability(text: string, wordCount: number): ReadabilityScores
     }
   }
 
-  if (wordCount < 50 || sentenceCount === 0) {
+  if (wordCount < 50 || sentenceCount === 0 || (!english && !formula)) {
     return {
       fleschReadingEase: null,
       fleschKincaidGrade: null,
@@ -2637,15 +2969,17 @@ function computeReadability(text: string, wordCount: number): ReadabilityScores
 
   const wordsPerSentence = wordCount / sentenceCount;
   const syllablesPerWord = totalSyllables / wordCount;
-  const flesch = 206.835 - 1.015 * wordsPerSentence - 84.6 * syllablesPerWord;
+  const ease = formula
+    ? formula.ease(wordsPerSentence, syllablesPerWord)
+    : 206.835 - 1.015 * wordsPerSentence - 84.6 * syllablesPerWord;
   const grade = 0.39 * wordsPerSentence + 11.8 * syllablesPerWord - 15.59;
   const fog = 0.4 * (wordsPerSentence + (100 * complexWordCount) / wordCount);
 
   // Clamp to the same ranges the academic formulas allow.
   return {
-    fleschReadingEase: Math.round(Math.max(-50, Math.min(120, flesch)) * 10) / 10,
-    fleschKincaidGrade: Math.round(Math.max(-2, Math.min(30, grade)) * 10) / 10,
-    gunningFogIndex: Math.round(Math.max(0, Math.min(30, fog)) * 10) / 10,
+    fleschReadingEase: Math.round(Math.max(-50, Math.min(120, ease)) * 10) / 10,
+    fleschKincaidGrade: english ? Math.round(Math.max(-2, Math.min(30, grade)) * 10) / 10 : null,
+    gunningFogIndex: english ? Math.round(Math.max(0, Math.min(30, fog)) * 10) / 10 : null,
     sentenceCount,
     complexWordCount,
   };

packages/core/src/html-report.ts

@@ -125,13 +125,21 @@ const ISSUES: IssueDef[] = [
   { key: 'structuredDataMissing', label: 'JSON-LD Missing', severity: 'info' },
   { key: 'structuredDataInvalid', label: 'Invalid JSON-LD', severity: 'error' },
   { key: 'paginationBroken', label: 'Pagination Broken', severity: 'error' },
-  { key: 'hreflangXDefaultMissing', label: 'Hreflang x-default Missing', severity: 'warn' },
+  { key: 'hreflangXDefaultMissing', label: 'Hreflang x-default Missing', severity: 'info' },
   { key: 'hreflangInvalidCode', label: 'Hreflang Invalid Code', severity: 'error' },
   { key: 'hreflangSelfRefMissing', label: 'Hreflang Self-Ref Missing', severity: 'error' },
   { key: 'hreflangReciprocityMissing', label: 'Hreflang Reciprocity Missing', severity: 'warn' },
   { key: 'hreflangTargetIssues', label: 'Hreflang Target Issues', severity: 'error' },
   { key: 'faviconMissing', label: 'Favicon Missing', severity: 'info' },
   { key: 'charsetMissing', label: 'Charset Missing', severity: 'warn' },
+  { key: 'charsetMismatch', label: 'Charset Mismatch (HTTP vs HTML)', severity: 'warn' },
+  { key: 'canonicalOutsideHead', label: 'Canonical Outside <head>', severity: 'warn' },
+  { key: 'langContentMismatch', label: 'Lang ≠ Content Language', severity: 'warn' },
+  { key: 'soft404', label: 'Soft 404', severity: 'warn' },
+  { key: 'redirectChain', label: 'Redirect Chain (2+ hops)', severity: 'warn' },
+  { key: 'blockedByRobots', label: 'Linked URL Blocked by robots.txt', severity: 'warn' },
+  { key: 'noindexJsInjected', label: 'Noindex Added by JavaScript', severity: 'warn' },
+  { key: 'titleChangedByJs', label: 'Title Changed by JavaScript', severity: 'warn' },
   { key: 'ampValidationErrors', label: 'AMP Validation Errors', severity: 'warn' },
   { key: 'highBoilerplate', label: 'High Boilerplate (>50%)', severity: 'warn' },
   { key: 'nonIndexableInSitemap', label: 'Non-Indexable in Sitemap', severity: 'error' },
@@ -230,6 +238,14 @@ const RECOMMENDATIONS: Partial<Record<IssueDef['key'], string>> = {
   hreflangTargetIssues: 'Point hreflang at live, indexable, self-canonical pages.',
   faviconMissing: 'Add a favicon; it appears next to the site in mobile results.',
   charsetMissing: 'Declare the character encoding (meta charset or Content-Type header).',
+  charsetMismatch: 'Make the Content-Type header charset match the meta charset — browsers obey the header.',
+  canonicalOutsideHead: 'Move rel=canonical into <head>; search engines ignore it in <body>.',
+  langContentMismatch: 'Set html[lang] to the language the page is actually written in.',
+  soft404: 'Return 404/410 for pages that no longer exist instead of a 200 "not found" page.',
+  redirectChain: 'Point links and redirects straight at the final URL.',
+  blockedByRobots: 'Unblock the URL in robots.txt or stop linking to it internally.',
+  noindexJsInjected: 'Put noindex in the served HTML or X-Robots-Tag; Google may index before rendering.',
+  titleChangedByJs: 'Serve the final <title> in the HTML instead of rewriting it with JavaScript.',
   ampValidationErrors: 'Fix AMP validation errors or the AMP version will not be served.',
   nonIndexableInSitemap: 'Remove noindex / canonicalised URLs from the XML sitemap.',
   non200InSitemap: 'Remove error and redirecting URLs from the XML sitemap.',

packages/core/src/simhash.ts

@@ -10,8 +10,11 @@
  *      shingle hash, `-freq` if 0.
  *   5. Final SimHash bit = sign of the accumulator.
  *
- * Two pages with hamming distance ≤ ~3 over 64 bits are considered
- * near-duplicates (>~95% similarity at the word-shingle level).
+ * Two pages within a hamming distance of ~3 over 64 bits are near-identical
+ * copies. SimHash is not a similarity percentage: two posts sharing 96% of
+ * their wording measured ~15 bits apart. The same shingle set therefore
+ * also feeds a MinHash signature (see shared-types/minhash.ts), which is
+ * what the similarity-based near-duplicate pass compares.
  *
  * Cost: ~5–10 ms on a 1 MB page (htmlparser2 already gave us trimmed text).
  * The body text itself is NOT stored in the database — only the hash.
@@ -20,6 +23,8 @@
  * numbers because Node's BigInt is fast enough at this scale (millions of
  * shingles per crawl total) and the code stays trivially correct.
  */
+import { computeMinHash } from '@freecrawl/shared-types';
+
 const FNV_OFFSET = 0xcbf29ce484222325n;
 const FNV_PRIME = 0x100000001b3n;
 const MASK64 = (1n << 64n) - 1n;
@@ -62,14 +67,15 @@ function tokenize(text: string): string[] {
 export function computeContentFingerprint(text: string): {
   simhash: string | null;
   contentHash: string | null;
+  minhash: Uint8Array | null;
 } {
   if (!text || text.length < 50) {
-    return { simhash: null, contentHash: null };
+    return { simhash: null, contentHash: null, minhash: null };
   }
 
   const tokens = tokenize(text);
   if (tokens.length < SHINGLE_SIZE) {
-    return { simhash: null, contentHash: null };
+    return { simhash: null, contentHash: null, minhash: null };
   }
 
   // Aggregate shingle frequencies first — duplicate shingles within the
@@ -107,6 +113,7 @@ export function computeContentFingerprint(text: string): {
   return {
     simhash: toHex64(sim),
     contentHash: toHex64(exact),
+    minhash: computeMinHash(shingleFreq.keys()),
   };
 }
 

packages/core/src/url-utils.ts

@@ -487,6 +487,57 @@ const CALENDAR_PARAMS: ReadonlySet<string> = new Set([
   'end_date',
 ]);
 
+/**
+ * A query value or path segment that names a calendar period: `2026-10`,
+ * `2026/10`, `202610`, `2026-10-16`, `20261016`. Years outside 1900–2199
+ * don't qualify, so IDs and prices that happen to be six digits rarely do.
+ */
+const DATE_VALUE_RE = /^(?:19|20|21)\d{2}[-_/.]?(?:0[1-9]|1[0-2])(?:[-_/.]?(?:0[1-9]|[12]\d|3[01]))?$/;
+const YEAR_RE = /^(?:19|20|21)\d{2}$/;
+
+/**
+ * The URL "shape" behind a date-navigation widget, or null when the URL
+ * carries no date at all. Two URLs that differ only in which period they
+ * show — `/events?month=2026-10&day=5` and `/events?month=2031-02&day=9`,
+ * or `/agenda/2026/10/` and `/agenda/2027/03/` — share a key, which is
+ * what lets the crawler see a "next month" link receding forever as one
+ * growing family of pages instead of thousands of unrelated URLs.
+ *
+ * Parameter names are kept and values dropped; numeric path segments are
+ * folded. Detection is by the value's shape, so it works whatever the
+ * site calls the parameter (`ay`, `mes`, `monat`, `month`, …).
+ */
+export function calendarPatternKey(url: string): string | null {
+  let u: URL;
+  try {
+    u = new URL(url);
+  } catch {
+    return null;
+  }
+  let dated = false;
+  const names = new Set<string>();
+  for (const [k, v] of u.searchParams) {
+    const name = k.toLowerCase();
+    names.add(name);
+    const value = v.trim();
+    if (DATE_VALUE_RE.test(value) || (CALENDAR_PARAMS.has(name) && /^\d{1,8}$/.test(value))) {
+      dated = true;
+    }
+  }
+  const path = u.pathname
+    .split('/')
+    .map((seg) => {
+      if (DATE_VALUE_RE.test(seg) || YEAR_RE.test(seg)) {
+        dated = true;
+        return '{date}';
+      }
+      return /^\d+$/.test(seg) ? '{n}' : seg;
+    })
+    .join('/');
+  if (!dated) return null;
+  return `${u.host}${path}?${[...names].sort().join('&')}`;
+}
+
 /** Why a URL was judged a crawl trap. */
 export type UrlTrapKind = 'repeated-segment' | 'query-params' | 'session-id' | 'calendar';
 

packages/db/src/migrations.ts

@@ -2366,6 +2366,63 @@ const MIGRATIONS: Migration[] = [
       }
     },
   },
+  {
+    version: 95,
+    name: 'add_document_integrity_signals',
+    // Signals behind the charset-mismatch, canonical-outside-head,
+    // content-language, soft-404 and JavaScript-modification checks.
+    up: (db) => {
+      const cols = db
+        .prepare('PRAGMA table_info(urls)')
+        .all() as unknown as { name: string }[];
+      const names = new Set(cols.map((c) => c.name));
+      const add = (col: string, decl: string): void => {
+        if (!names.has(col)) db.exec(`ALTER TABLE urls ADD COLUMN ${col} ${decl}`);
+      };
+      add('http_charset', 'TEXT');
+      add('meta_charset', 'TEXT');
+      add('canonical_outside_head', 'INTEGER NOT NULL DEFAULT 0');
+      add('content_lang', 'TEXT');
+      add('lang_mismatch', 'INTEGER NOT NULL DEFAULT 0');
+      add('soft_404', 'INTEGER NOT NULL DEFAULT 0');
+      add('js_noindex', 'INTEGER NOT NULL DEFAULT 0');
+      add('js_title_changed', 'INTEGER NOT NULL DEFAULT 0');
+      add('js_title', 'TEXT');
+    },
+  },
+  {
+    version: 96,
+    name: 'add_minhash_similarity',
+    // MinHash signature per page and the best similarity found, for the
+    // similarity-based near-duplicate pass (SimHash only catches
+    // near-identical copies).
+    up: (db) => {
+      const cols = db
+        .prepare('PRAGMA table_info(urls)')
+        .all() as unknown as { name: string }[];
+      const names = new Set(cols.map((c) => c.name));
+      if (!names.has('minhash')) db.exec('ALTER TABLE urls ADD COLUMN minhash BLOB');
+      if (!names.has('near_dup_similarity')) {
+        db.exec('ALTER TABLE urls ADD COLUMN near_dup_similarity INTEGER');
+      }
+    },
+  },
+  {
+    version: 97,
+    name: 'add_image_variant_flag',
+    // srcset / <picture> candidates of an <img>: probed like any image but
+    // kept out of the alt checks (they share the slot's alt by design).
+    up: (db) => {
+      for (const table of ['images', 'image_usages']) {
+        const cols = db
+          .prepare(`PRAGMA table_info(${table})`)
+          .all() as unknown as { name: string }[];
+        if (!cols.some((c) => c.name === 'is_variant')) {
+          db.exec(`ALTER TABLE ${table} ADD COLUMN is_variant INTEGER NOT NULL DEFAULT 0`);
+        }
+      }
+    },
+  },
 ];
 
 export function runMigrations(db: DatabaseSync): void {

packages/db/src/project-db.ts

@@ -68,6 +68,12 @@ import {
   ISSUE_COUNT_KEY,
   issueCategoriesBySeverity,
   normalizeDisabledIssues,
+  isValidHreflangCode,
+  MINHASH_SLOTS,
+  MINHASH_BANDS,
+  MINHASH_BAND_ROWS,
+  minhashSlot,
+  minhashSimilarity,
 } from '@freecrawl/shared-types';
 import { runMigrations } from './migrations.js';
 
@@ -206,6 +212,28 @@ interface UrlRowDb {
   amphtml: string | null;
   /** Separate-URL mobile version from `<link rel="alternate" media=…>`. */
   mobile_alternate: string | null;
+  /** `charset=` of the HTTP Content-Type header (WHATWG-canonical name). */
+  http_charset: string | null;
+  /** `<meta charset>` / `http-equiv` declaration (WHATWG-canonical name). */
+  meta_charset: string | null;
+  /** `<link rel="canonical">` elements inside `<body>` — ignored by search engines. */
+  canonical_outside_head: number;
+  /** Primary language the main-content prose is written in. */
+  content_lang: string | null;
+  /** 1 when `html[lang]` names a language the prose clearly isn't in. */
+  lang_mismatch: number;
+  /** Soft 404 evidence: 0 none, 1 not-found wording, 2 the site's 404 template. */
+  soft_404: number;
+  /** JS-added noindex: 0 none, 1 inline-script evidence, 2 confirmed by rendering. */
+  js_noindex: number;
+  /** JS-changed `<title>`: 0 none, 1 inline-script evidence, 2 confirmed by rendering. */
+  js_title_changed: number;
+  /** The title JavaScript sets, when it can be read. */
+  js_title: string | null;
+  /** 128-byte MinHash signature of the main content (near-duplicate similarity). */
+  minhash: Uint8Array | null;
+  /** Best estimated similarity (%) to a near-duplicate cluster mate. */
+  near_dup_similarity: number | null;
   /** 1 when the page declares `<html ⚡>` / `<html amp>`, else 0. */
   amp_page: number;
   /** JSON string array of AMP smoke-validator error codes. Empty when
@@ -477,6 +505,16 @@ export interface UpsertUrlInput {
   amphtml?: string | null;
   /** Separate-URL mobile version from `<link rel="alternate" media=…>`. */
   mobileAlternate?: string | null;
+  httpCharset?: string | null;
+  metaCharset?: string | null;
+  canonicalOutsideHead?: number;
+  contentLang?: string | null;
+  langMismatch?: number;
+  soft404?: number;
+  jsNoindex?: number;
+  jsTitleChanged?: number;
+  jsTitle?: string | null;
+  minhash?: Uint8Array | null;
   ampPage?: boolean;
   /** JSON-stringified array of AMP validator error codes. */
   ampValidationErrors?: string | null;
@@ -659,7 +697,9 @@ const UPSERT_URL_SQL = `
     schema_duplicate_ids, schema_unknown_types, schema_missing_required,
     schema_missing_recommended, schema_findings,
     heading_order_violations, subresource_request_count,
-    amp_page, amp_validation_errors, mobile_alternate
+    amp_page, amp_validation_errors, mobile_alternate,
+    http_charset, meta_charset, canonical_outside_head, content_lang, lang_mismatch,
+    soft_404, js_noindex, js_title_changed, js_title, minhash
   ) VALUES (
     :url, :content_kind, :status_code, :status_text, :indexability, :indexability_reason,
     :title, :title_length, :meta_description, :meta_description_length,
@@ -702,7 +742,9 @@ const UPSERT_URL_SQL = `
     :schema_duplicate_ids, :schema_unknown_types, :schema_missing_required,
     :schema_missing_recommended, :schema_findings,
     :heading_order_violations, :subresource_request_count,
-    :amp_page, :amp_validation_errors, :mobile_alternate
+    :amp_page, :amp_validation_errors, :mobile_alternate,
+    :http_charset, :meta_charset, :canonical_outside_head, :content_lang, :lang_mismatch,
+    :soft_404, :js_noindex, :js_title_changed, :js_title, :minhash
   )
   ON CONFLICT(url) DO UPDATE SET
     content_kind = excluded.content_kind,
@@ -845,6 +887,16 @@ const UPSERT_URL_SQL = `
     amp_page = excluded.amp_page,
     amp_validation_errors = excluded.amp_validation_errors,
     mobile_alternate = excluded.mobile_alternate,
+    http_charset = excluded.http_charset,
+    meta_charset = excluded.meta_charset,
+    canonical_outside_head = excluded.canonical_outside_head,
+    content_lang = excluded.content_lang,
+    lang_mismatch = excluded.lang_mismatch,
+    soft_404 = excluded.soft_404,
+    js_noindex = excluded.js_noindex,
+    js_title_changed = excluded.js_title_changed,
+    js_title = excluded.js_title,
+    minhash = excluded.minhash,
     changed = ${CHANGED_FLAG_SQL},
     crawled_at = CURRENT_TIMESTAMP
   RETURNING id
@@ -881,14 +933,18 @@ function truncateUtf8(input: string, cap: number): string {
  * templated alt. Shared by the overview count, `queryImages`, and the CSV
  * export so all three agree on what "duplicate alt" means.
  */
-const DUPLICATE_ALT_PREDICATE = `alt IS NOT NULL AND alt != '' AND alt IN (
-  SELECT alt FROM images WHERE alt IS NOT NULL AND alt != '' GROUP BY alt HAVING COUNT(*) > 1
+// Variants (srcset / <picture> candidates) share their slot's alt by
+// design and are left out on both sides.
+const DUPLICATE_ALT_PREDICATE = `is_variant = 0 AND alt IS NOT NULL AND alt != '' AND alt IN (
+  SELECT alt FROM images WHERE is_variant = 0 AND alt IS NOT NULL AND alt != ''
+   GROUP BY alt HAVING COUNT(*) > 1
 )`;
 
 // Per-usage duplicate-alt: a usage whose alt text is shared by >1 usage.
 // Operates on `image_usages.alt` to match the per-usage Images view.
-const USAGE_DUPLICATE_ALT_PREDICATE = `iu.alt IS NOT NULL AND iu.alt != '' AND iu.alt IN (
-  SELECT alt FROM image_usages WHERE alt IS NOT NULL AND alt != '' GROUP BY alt HAVING COUNT(*) > 1
+const USAGE_DUPLICATE_ALT_PREDICATE = `iu.is_variant = 0 AND iu.alt IS NOT NULL AND iu.alt != '' AND iu.alt IN (
+  SELECT alt FROM image_usages WHERE is_variant = 0 AND alt IS NOT NULL AND alt != ''
+   GROUP BY alt HAVING COUNT(*) > 1
 )`;
 
 // Missing / empty alt are computed from PER-USAGE truth (`image_usages.alt`):
@@ -1616,6 +1672,13 @@ export class ProjectDb {
     storeBody: { body: string; maxBytes: number } | null;
     links: DiscoveredLink[];
     images: DiscoveredImage[];
+    /**
+     * `og:image` / `twitter:image` targets. Registered in `images` so the
+     * post-crawl size probe measures them for the share-card size checks,
+     * but given no `image_usages` row: they aren't on-page images, so
+     * they never count toward Missing Alt or the Images tab.
+     */
+    socialImages?: DiscoveredImage[];
     /**
      * Whether the crawl is recording this page's link graph at all
      * (Spider → Crawl → Store for hyperlinks / iframes / invalid links).
@@ -1683,6 +1746,9 @@ export class ProjectDb {
             this.insertImages(urlId, payload.images);
           }
         }
+        if (payload.socialImages && payload.socialImages.length > 0) {
+          this.registerProbeOnlyImages(payload.socialImages);
+        }
       }
       return { urlId };
     });
@@ -2153,6 +2219,16 @@ export class ProjectDb {
       amp_page: input.ampPage ? 1 : 0,
       amp_validation_errors: input.ampValidationErrors ?? null,
       mobile_alternate: input.mobileAlternate ?? null,
+      http_charset: input.httpCharset ?? null,
+      meta_charset: input.metaCharset ?? null,
+      canonical_outside_head: input.canonicalOutsideHead ?? 0,
+      content_lang: input.contentLang ?? null,
+      lang_mismatch: input.langMismatch ?? 0,
+      soft_404: input.soft404 ?? 0,
+      js_noindex: input.jsNoindex ?? 0,
+      js_title_changed: input.jsTitleChanged ?? 0,
+      js_title: input.jsTitle ?? null,
+      minhash: input.minhash ?? null,
       favicon: input.favicon ?? null,
       mixed_content_count: input.mixedContentCount ?? 0,
       mixed_content_active: input.mixedContentActive ?? 0,
@@ -3006,26 +3082,27 @@ export class ProjectDb {
   }
 
   /**
-   * Cluster pages by SimHash hamming distance ≤ `thresholdBits` and
-   * write `cluster_id` / `cluster_size` back to every URL.
+   * Cluster near-duplicate pages and write `cluster_id` / `cluster_size`
+   * (and the best similarity found, `near_dup_similarity`) back to every
+   * URL. Two independent passes feed one Union-Find:
    *
-   * Algorithm — band-based LSH + Union-Find:
+   *   1. **SimHash** (`thresholdBits`, 0–64; negative skips the pass) —
+   *      near-identical copies. Each 64-bit hash is split into 4 × 16-bit
+   *      bands; by pigeonhole any two hashes within ≤ 3 bits share a band,
+   *      so only same-band pairs are compared.
+   *   2. **MinHash** (`minSimilarityPct`, 1–100; 0 skips the pass) — pages
+   *      whose main-content text is at least that similar, word for word.
+   *      21 LSH bands of 3 slots surface candidate pairs (a 90%-similar
+   *      pair collides in some band ~99% of the time); each candidate's
+   *      similarity is then estimated from the full 64-slot signature
+   *      (see shared-types/minhash.ts) and kept only above the threshold.
    *
-   *   1. Pull `(id, simhash)` for every internal HTML row whose simhash
-   *      is non-null. (Optionally restrict to indexable rows.)
-   *   2. Split each 64-bit hash into 4 × 16-bit bands. By the pigeonhole
-   *      principle, any two hashes within hamming distance ≤ 3 must
-   *      agree on at least one band — so candidates are a strict subset
-   *      of "shares a band value". This collapses the comparison from
-   *      O(N²) to ~O(N · avg_bucket_size).
-   *   3. For every band-value bucket, do exact pairwise hamming checks.
-   *      Pairs within the threshold get unioned in a Union-Find DSU.
-   *   4. The DSU's connected components ARE the clusters. Map each root
-   *      to a sequential cluster_id and write back `(cluster_id,
-   *      cluster_size)` in a batched UPDATE.
+   * Connected components are the clusters, numbered from 1 (0 means
+   * "singleton" so the `cluster_id > 0` issue filter stays correct).
+   * Buckets are capped at BUCKET_LIMIT so a flood of empty template pages
+   * sharing one band value can't turn the pass quadratic.
    *
-   * Memory: O(N) for the SimHash list + DSU. ~80 MB at 1M URLs.
-   * Time:   ~3–10 s at 1M URLs depending on bucket distribution.
+   * Memory: O(N) — the hashes plus 128 bytes of signature per page.
    *
    * `onlyIndexable=true` skips noindex / canonicalised / blocked-robots
    * pages — the duplicate report then surfaces only issues that actually
@@ -3034,32 +3111,37 @@ export class ProjectDb {
   recomputeDuplicateClusters(
     thresholdBits: number,
     onlyIndexable: boolean,
+    minSimilarityPct = 0,
   ): { clusters: number; clusteredUrls: number } {
     // Reset all clustering state first so a re-run with different
     // thresholds doesn't leave stale partitions behind.
     this.db.exec('UPDATE urls SET cluster_id = 0, cluster_size = 1 WHERE cluster_id != 0');
+    this.db.exec('UPDATE urls SET near_dup_similarity = NULL WHERE near_dup_similarity IS NOT NULL');
 
-    if (thresholdBits < 0 || thresholdBits > 64) {
-      return { clusters: 0, clusteredUrls: 0 };
-    }
+    const useSimhash = thresholdBits >= 0 && thresholdBits <= 64;
+    const useMinhash = minSimilarityPct > 0 && minSimilarityPct <= 100;
+    if (!useSimhash && !useMinhash) return { clusters: 0, clusteredUrls: 0 };
 
     const indexClause = onlyIndexable ? "AND indexability = 'indexable'" : '';
     const rows = this.db
       .prepare(
-        `SELECT id, simhash FROM urls
+        `SELECT id, simhash, minhash FROM urls
            WHERE is_external = 0 AND content_kind = 'html'
-             AND simhash IS NOT NULL ${indexClause}`,
+             AND (simhash IS NOT NULL OR minhash IS NOT NULL) ${indexClause}`,
       )
-      .all() as { id: number; simhash: string }[];
+      .all() as { id: number; simhash: string | null; minhash: Uint8Array | null }[];
 
     if (rows.length < 2) return { clusters: 0, clusteredUrls: 0 };
 
     const N = rows.length;
     const ids = new Int32Array(N);
-    const hashes: bigint[] = new Array<bigint>(N);
+    const hashes: (bigint | null)[] = new Array<bigint | null>(N);
+    const sigs: (Uint8Array | null)[] = new Array<Uint8Array | null>(N);
     for (let i = 0; i < N; i++) {
-      ids[i] = rows[i]!.id;
-      hashes[i] = BigInt('0x' + rows[i]!.simhash);
+      const r = rows[i]!;
+      ids[i] = r.id;
+      hashes[i] = r.simhash ? BigInt('0x' + r.simhash) : null;
+      sigs[i] = r.minhash && r.minhash.length === MINHASH_SLOTS * 2 ? r.minhash : null;
     }
 
     // Union-Find with path compression + union-by-rank.
@@ -3088,6 +3170,16 @@ export class ProjectDb {
         rank[ra]!++;
       }
     };
+    // Best similarity (0–1) each page reached against a cluster mate.
+    const best = new Float64Array(N).fill(-1);
+    const noteSimilarity = (a: number, b: number): void => {
+      const sa = sigs[a];
+      const sb = sigs[b];
+      if (!sa || !sb) return;
+      const s = minhashSimilarity(sa, sb);
+      if (s > best[a]!) best[a] = s;
+      if (s > best[b]!) best[b] = s;
+    };
 
     // Hamming distance over 64-bit BigInt — popcount via Brian Kernighan.
     const hamming = (a: bigint, b: bigint): number => {
@@ -3100,36 +3192,80 @@ export class ProjectDb {
       return c;
     };
 
-    // Band buckets: 4 bands × Map<bandValue, indices[]>. We cap bucket
-    // size to BUCKET_LIMIT — pathological banner-only pages can otherwise
-    // produce a single bucket containing every page on the site, which
-    // would make the inner-loop comparison quadratic again.
     const BUCKET_LIMIT = 5000;
-    for (let band = 0; band < 4; band++) {
-      const shift = BigInt(band * 16);
-      const mask = 0xffffn;
-      const buckets = new Map<number, number[]>();
-      for (let i = 0; i < N; i++) {
-        const v = Number((hashes[i]! >> shift) & mask);
-        let bucket = buckets.get(v);
-        if (!bucket) {
-          bucket = [];
-          buckets.set(v, bucket);
+
+    // ── Pass 1: SimHash ────────────────────────────────────────────────
+    if (useSimhash) {
+      for (let band = 0; band < 4; band++) {
+        const shift = BigInt(band * 16);
+        const mask = 0xffffn;
+        const buckets = new Map<number, number[]>();
+        for (let i = 0; i < N; i++) {
+          const h = hashes[i];
+          if (h === null || h === undefined) continue;
+          const v = Number((h >> shift) & mask);
+          let bucket = buckets.get(v);
+          if (!bucket) {
+            bucket = [];
+            buckets.set(v, bucket);
+          }
+          if (bucket.length < BUCKET_LIMIT) bucket.push(i);
+        }
+        for (const bucket of buckets.values()) {
+          if (bucket.length < 2) continue;
+          for (let a = 0; a < bucket.length; a++) {
+            const ia = bucket[a]!;
+            for (let b = a + 1; b < bucket.length; b++) {
+              const ib = bucket[b]!;
+              // Skip pairs already in the same component — the same pair
+              // shows up again in the other bands and the DSU find is cheap.
+              if (find(ia) === find(ib)) continue;
+              if (hamming(hashes[ia]!, hashes[ib]!) <= thresholdBits) {
+                union(ia, ib);
+                noteSimilarity(ia, ib);
+              }
+            }
+          }
         }
-        if (bucket.length < BUCKET_LIMIT) bucket.push(i);
       }
-      for (const bucket of buckets.values()) {
-        if (bucket.length < 2) continue;
-        for (let a = 0; a < bucket.length; a++) {
-          const ia = bucket[a]!;
-          for (let b = a + 1; b < bucket.length; b++) {
-            const ib = bucket[b]!;
-            // Skip pairs already in the same component — we're going to
-            // touch the same band repeatedly across all 4 passes and the
-            // DSU find is cheap.
-            if (find(ia) === find(ib)) continue;
-            if (hamming(hashes[ia]!, hashes[ib]!) <= thresholdBits) {
+    }
+
+    // ── Pass 2: MinHash similarity ─────────────────────────────────────
+    if (useMinhash) {
+      const threshold = minSimilarityPct / 100;
+      for (let band = 0; band < MINHASH_BANDS; band++) {
+        const first = band * MINHASH_BAND_ROWS;
+        const buckets = new Map<number, number[]>();
+        for (let i = 0; i < N; i++) {
+          const sig = sigs[i];
+          if (!sig) continue;
+          // Three 16-bit slots → one exact 48-bit key.
+          const key =
+            minhashSlot(sig, first) +
+            minhashSlot(sig, first + 1) * 0x10000 +
+            minhashSlot(sig, first + 2) * 0x100000000;
+          let bucket = buckets.get(key);
+          if (!bucket) {
+            bucket = [];
+            buckets.set(key, bucket);
+          }
+          if (bucket.length < BUCKET_LIMIT) bucket.push(i);
+        }
+        for (const bucket of buckets.values()) {
+          if (bucket.length < 2) continue;
+          for (let a = 0; a < bucket.length; a++) {
+            const ia = bucket[a]!;
+            for (let b = a + 1; b < bucket.length; b++) {
+              const ib = bucket[b]!;
+              // Already clustered and both sides have a similarity on
+              // record: nothing left to learn from this pair. Keeps a
+              // flood of identical template pages linear per band.
+              if (find(ia) === find(ib) && best[ia]! >= threshold && best[ib]! >= threshold) continue;
+              const s = minhashSimilarity(sigs[ia]!, sigs[ib]!);
+              if (s < threshold) continue;
               union(ia, ib);
+              if (s > best[ia]!) best[ia] = s;
+              if (s > best[ib]!) best[ib] = s;
             }
           }
         }
@@ -3156,7 +3292,7 @@ export class ProjectDb {
     }
 
     const upd = this.db.prepare(
-      'UPDATE urls SET cluster_id = ?, cluster_size = ? WHERE id = ?',
+      'UPDATE urls SET cluster_id = ?, cluster_size = ?, near_dup_similarity = ? WHERE id = ?',
     );
     this.db.exec('BEGIN');
     try {
@@ -3164,7 +3300,8 @@ export class ProjectDb {
         const root = find(i);
         const cid = rootToCluster.get(root);
         if (cid !== undefined) {
-          upd.run(cid, clusterSize.get(root)!, ids[i]!);
+          const sim = best[i]! >= 0 ? Math.round(best[i]! * 100) : null;
+          upd.run(cid, clusterSize.get(root)!, sim, ids[i]!);
         }
       }
       this.db.exec('COMMIT');
@@ -3561,15 +3698,18 @@ export class ProjectDb {
     wordCount: number | null;
     inlinks: number;
     hammingDistance: number;
+    similarity: number | null;
   }> {
     const self = this.db
-      .prepare('SELECT cluster_id, simhash FROM urls WHERE id = ?')
-      .get(urlId) as { cluster_id: number | null; simhash: string | null } | undefined;
+      .prepare('SELECT cluster_id, simhash, minhash FROM urls WHERE id = ?')
+      .get(urlId) as
+      | { cluster_id: number | null; simhash: string | null; minhash: Uint8Array | null }
+      | undefined;
     if (!self || !self.cluster_id || self.cluster_id <= 0) return [];
 
     const rows = this.db
       .prepare(
-        `SELECT id, url, status_code, indexability, title, word_count, inlinks, simhash
+        `SELECT id, url, status_code, indexability, title, word_count, inlinks, simhash, minhash
            FROM urls
           WHERE cluster_id = ? AND id != ?
           ORDER BY url ASC`,
@@ -3583,6 +3723,7 @@ export class ProjectDb {
       word_count: number | null;
       inlinks: number;
       simhash: string | null;
+      minhash: Uint8Array | null;
     }>;
 
     const popcount = (x: bigint): number => {
@@ -3605,6 +3746,10 @@ export class ProjectDb {
         inlinks: r.inlinks,
         hammingDistance:
           selfBig !== null && r.simhash ? popcount(selfBig ^ BigInt('0x' + r.simhash)) : 0,
+        similarity:
+          self.minhash && r.minhash
+            ? Math.round(minhashSimilarity(self.minhash, r.minhash) * 100)
+            : null,
       }))
       .sort((a, b) => a.hammingDistance - b.hammingDistance || a.url.localeCompare(b.url));
   }
@@ -4174,10 +4319,12 @@ export class ProjectDb {
   insertImages(fromUrlId: number, images: DiscoveredImage[]): void {
     if (images.length === 0) return;
     const upsertImage = this.db.prepare(
-      `INSERT INTO images (src, alt, width, height, is_internal, occurrences)
-       VALUES (?, ?, ?, ?, ?, 1)
+      `INSERT INTO images (src, alt, width, height, is_internal, occurrences, is_variant)
+       VALUES (?, ?, ?, ?, ?, 1, ?)
        ON CONFLICT(src) DO UPDATE SET
          occurrences = occurrences + 1,
+         -- A variant anywhere that is also a slot somewhere is a slot.
+         is_variant = MIN(images.is_variant, excluded.is_variant),
          -- Fill in alt / dimensions from later occurrences only if the
          -- current stored row is missing them.
          alt = COALESCE(images.alt, excluded.alt),
@@ -4186,22 +4333,26 @@ export class ProjectDb {
        RETURNING id`,
     );
     const upsertUsage = this.db.prepare(
-      `INSERT INTO image_usages (from_url_id, image_id, alt)
-       VALUES (?, ?, ?)
-       ON CONFLICT(from_url_id, image_id) DO UPDATE SET alt = excluded.alt`,
+      `INSERT INTO image_usages (from_url_id, image_id, alt, is_variant)
+       VALUES (?, ?, ?, ?)
+       ON CONFLICT(from_url_id, image_id) DO UPDATE SET
+         alt = excluded.alt,
+         is_variant = MIN(image_usages.is_variant, excluded.is_variant)`,
     );
     const ownsTx = !this.isInTransaction();
     if (ownsTx) this.db.exec('BEGIN');
     try {
       for (const img of images) {
+        const variant = img.variant ? 1 : 0;
         const row = upsertImage.get(
           img.src,
           img.alt,
           img.width,
           img.height,
           img.isInternal ? 1 : 0,
+          variant,
         ) as { id: number };
-        upsertUsage.run(fromUrlId, row.id, img.alt);
+        upsertUsage.run(fromUrlId, row.id, img.alt, variant);
       }
       if (ownsTx) this.db.exec('COMMIT');
     } catch (err) {
@@ -4216,6 +4367,34 @@ export class ProjectDb {
     }
   }
 
+  /**
+   * Flag already-crawled URLs as part of a crawl trap the crawler only
+   * recognised after fetching them (a calendar's pages look ordinary one
+   * at a time). Chunked under SQLite's parameter limit.
+   */
+  markUrlTrap(urls: readonly string[], kind: string): void {
+    for (let i = 0; i < urls.length; i += 500) {
+      const slice = urls.slice(i, i + 500);
+      this.db
+        .prepare(`UPDATE urls SET url_trap = ? WHERE url IN (${slice.map(() => '?').join(',')})`)
+        .run(kind, ...slice);
+    }
+  }
+
+  /**
+   * Make sure each image has an `images` row for the post-crawl probes
+   * without recording a usage. An image that is also shown via `<img>`
+   * somewhere keeps its usage-derived alt and occurrence count.
+   */
+  registerProbeOnlyImages(images: DiscoveredImage[]): void {
+    const insert = this.db.prepare(
+      `INSERT INTO images (src, alt, width, height, is_internal, occurrences)
+       VALUES (?, NULL, NULL, NULL, ?, 0)
+       ON CONFLICT(src) DO NOTHING`,
+    );
+    for (const img of images) insert.run(img.src, img.isInternal ? 1 : 0);
+  }
+
   /**
    * Return link rows whose destination is a 4xx/5xx URL. Joins the link
    * catalogue to the URL table twice — once for the source page's status,
@@ -5876,8 +6055,8 @@ export class ProjectDb {
       const where: string[] = [];
       const args: (string | number)[] = [];
       if (params.internalOnly) where.push('i.is_internal = 1');
-      if (params.missingAltOnly) where.push('iu.alt IS NULL');
-      if (params.emptyAltOnly) where.push("iu.alt = ''");
+      if (params.missingAltOnly) where.push('iu.alt IS NULL AND iu.is_variant = 0');
+      if (params.emptyAltOnly) where.push("iu.alt = '' AND iu.is_variant = 0");
       if (params.duplicateAltOnly) where.push(USAGE_DUPLICATE_ALT_PREDICATE);
       if (params.search) {
         where.push('(i.src LIKE ? OR iu.alt LIKE ?)');
@@ -5937,7 +6116,8 @@ export class ProjectDb {
     }
 
     // Legacy fallback — per distinct image (no usage rows to expand).
-    const where: string[] = [];
+    // `occurrences = 0` rows are probe-only share-card images, not page images.
+    const where: string[] = ['i.occurrences > 0'];
     const args: (string | number)[] = [];
     if (params.internalOnly) where.push('i.is_internal = 1');
     if (params.missingAltOnly) where.push('i.alt IS NULL');
@@ -6231,7 +6411,8 @@ export class ProjectDb {
     // tab shows exactly the rows this headline counts.
     const totalIndexable = countCategory('indexability:indexable');
     const totalNonIndexable = countCategory('indexability:non-indexable');
-    const totalImages = this.scalarCount('SELECT COUNT(*) AS c FROM images');
+    // Probe-only rows (share-card images, `occurrences = 0`) aren't on any page.
+    const totalImages = this.scalarCount('SELECT COUNT(*) AS c FROM images WHERE occurrences > 0');
 
     return {
       summary: {
@@ -6379,12 +6560,15 @@ export class ProjectDb {
       ),
       canonicalToNoindex: countWhere(
         `${html} AND canonical IS NOT NULL AND canonical != ''
+         AND ${CANONICAL_CMP} != url
          AND EXISTS (
            SELECT 1 FROM urls t WHERE t.url = ${CANONICAL_CMP_Q}
              AND t.indexability = 'non-indexable:noindex'
          )`,
       ),
-      contentThin: countWhere(`${indexable} AND word_count IS NOT NULL AND word_count < 300`),
+      contentThin: countWhere(
+        `${indexable} AND indexability = 'indexable' AND word_count IS NOT NULL AND word_count < 300`,
+      ),
       // Only a completed check yields a finding count — an unsupported or
       // language-mismatched page stores zero because nothing was collected.
       // Stating `status = 'ok'` keeps that explicit rather than leaning on
@@ -6489,8 +6673,8 @@ export class ProjectDb {
         // Per-usage to match the per-usage Images tab (count = table). Legacy
         // projects with no usage rows fall back to the distinct-image count.
         `SELECT (CASE WHEN EXISTS(SELECT 1 FROM image_usages)
-           THEN (SELECT COUNT(*) FROM image_usages WHERE alt IS NULL)
-           ELSE (SELECT COUNT(*) FROM images WHERE alt IS NULL) END) AS c`,
+           THEN (SELECT COUNT(*) FROM image_usages WHERE alt IS NULL AND is_variant = 0)
+           ELSE (SELECT COUNT(*) FROM images WHERE alt IS NULL AND occurrences > 0) END) AS c`,
       ),
       // Distinct images sharing a non-empty alt with ≥1 other image —
       // non-descriptive / templated alt reuse. Counted image-level (the
@@ -6502,6 +6686,22 @@ export class ProjectDb {
       metaRefreshUsed: countWhere(
         `${html} AND meta_refresh IS NOT NULL AND meta_refresh != ''`,
       ),
+      charsetMismatch: countWhere(
+        `${html} AND http_charset IS NOT NULL AND meta_charset IS NOT NULL
+         AND http_charset != meta_charset`,
+      ),
+      canonicalOutsideHead: countWhere(`${html} AND canonical_outside_head > 0`),
+      langContentMismatch: countWhere(`${html} AND lang_mismatch = 1`),
+      soft404: countWhere(`${html} AND status_code BETWEEN 200 AND 299 AND soft_404 > 0`),
+      redirectChain: countWhere(
+        `is_external = 0 AND redirect_chain_length >= 2
+         AND NOT EXISTS (SELECT 1 FROM urls p WHERE p.redirect_target = urls.url)`,
+      ),
+      blockedByRobots: countWhere(
+        `is_external = 0 AND indexability = 'non-indexable:robots-blocked' AND inlinks > 0`,
+      ),
+      noindexJsInjected: countWhere(`${html} AND js_noindex > 0`),
+      titleChangedByJs: countWhere(`${html} AND js_title_changed > 0`),
       charsetMissing: countWhere(
         `${html} AND status_code >= 200 AND status_code < 300
          AND (charset IS NULL OR charset = '')`,
@@ -6569,7 +6769,7 @@ export class ProjectDb {
       // Alt and matches the Images tab drill-down.
       imageEmptyAlt: this.scalarCount(
         `SELECT (CASE WHEN EXISTS(SELECT 1 FROM image_usages)
-           THEN (SELECT COUNT(*) FROM image_usages WHERE alt = '')
+           THEN (SELECT COUNT(*) FROM image_usages WHERE alt = '' AND is_variant = 0)
            ELSE (SELECT COUNT(*) FROM images WHERE alt = '') END) AS c`,
       ),
       linkEmptyAnchor: countWhere(`${html} AND empty_anchor_count > 0`),
@@ -6757,7 +6957,7 @@ export class ProjectDb {
       imageBrokenSrc: issueCount('issues:image-broken-src'),
       targetBlankNoNoopener: issueCount('issues:target-blank-no-noopener'),
       pageEmpty: countWhere(
-        `${html} AND status_code BETWEEN 200 AND 299
+        `${html} AND status_code BETWEEN 200 AND 299 AND indexability = 'indexable'
          AND word_count IS NOT NULL AND word_count < 30`,
       ),
       ogImageNotAbsolute: countWhere(
@@ -7483,8 +7683,8 @@ export class ProjectDb {
     const minBytes = Math.max(0, Math.floor(options.minByteSize ?? 0));
     // Per-usage export to match the per-usage Images tab (one row per page).
     if (this.hasImageUsages()) {
-      if (options.missingAltOnly) where.push('iu.alt IS NULL');
-      if (options.emptyAltOnly) where.push("iu.alt = ''");
+      if (options.missingAltOnly) where.push('iu.alt IS NULL AND iu.is_variant = 0');
+      if (options.emptyAltOnly) where.push("iu.alt = '' AND iu.is_variant = 0");
       if (options.duplicateAltOnly) where.push(USAGE_DUPLICATE_ALT_PREDICATE);
       if (options.search) {
         where.push('(i.src LIKE ? OR iu.alt LIKE ?)');
@@ -7529,7 +7729,9 @@ export class ProjectDb {
       return;
     }
 
-    // Legacy fallback — per distinct image.
+    // Legacy fallback — per distinct image. Probe-only share-card rows
+    // (`occurrences = 0`) aren't page images.
+    where.push('i.occurrences > 0');
     if (options.missingAltOnly) where.push('i.alt IS NULL');
     if (options.emptyAltOnly) where.push("i.alt = ''");
     if (options.duplicateAltOnly) where.push(DUPLICATE_ALT_PREDICATE);
@@ -9773,6 +9975,16 @@ export class ProjectDb {
     videos: r.videos ?? null,
     amphtml: r.amphtml,
     mobileAlternate: r.mobile_alternate ?? null,
+    httpCharset: r.http_charset ?? null,
+    metaCharset: r.meta_charset ?? null,
+    canonicalOutsideHead: r.canonical_outside_head ?? 0,
+    contentLang: r.content_lang ?? null,
+    langMismatch: (r.lang_mismatch ?? 0) === 1,
+    soft404: r.soft_404 ?? 0,
+    jsNoindex: r.js_noindex ?? 0,
+    jsTitleChanged: r.js_title_changed ?? 0,
+    jsTitle: r.js_title ?? null,
+    nearDupSimilarity: r.near_dup_similarity ?? null,
     ampPage: r.amp_page === 1,
     ampValidationErrors: r.amp_validation_errors,
     boilerplateCoverage: r.boilerplate_coverage,
@@ -10635,9 +10847,12 @@ export function categoryWhereClause(
     case 'issues:canonical-to-noindex':
       // Canonical implies "use this as authoritative"; noindex says
       // "don't index". Contradictory — page sends mixed signals to
-      // search engines.
+      // search engines. Only when the canonical points elsewhere: a
+      // noindexed page naming itself as canonical (a cart, an account
+      // page) asks for nothing contradictory — the noindex simply wins.
       return `is_external = 0 AND content_kind = 'html'
               AND canonical IS NOT NULL AND canonical != ''
+              AND ${CANONICAL_CMP} != url
               AND EXISTS (
                 SELECT 1 FROM urls t WHERE t.url = ${CANONICAL_CMP_Q}
                   AND t.indexability = 'non-indexable:noindex'
@@ -10677,8 +10892,11 @@ export function categoryWhereClause(
               )`;
     case 'issues:content-thin':
       // 2xx only — a text/html 404 template is short by design and was
-      // being counted as thin content.
-      return `${INDEXABLE_HTML_SCOPE} AND word_count IS NOT NULL AND word_count < 300`;
+      // being counted as thin content. Indexable only: a noindexed cart or
+      // a page canonicalised elsewhere never competes in search, so its
+      // word count says nothing about content quality.
+      return `${INDEXABLE_HTML_SCOPE} AND indexability = 'indexable'
+              AND word_count IS NOT NULL AND word_count < 300`;
     case 'issues:spelling-grammar':
       return `is_external = 0 AND content_kind = 'html' AND EXISTS (
                 SELECT 1 FROM spelling_results s
@@ -10840,6 +11058,46 @@ export function categoryWhereClause(
       // occurrence is worth surfacing.
       return `is_external = 0 AND content_kind = 'html'
               AND meta_refresh IS NOT NULL AND meta_refresh != ''`;
+    case 'issues:charset-mismatch':
+      // Header and document name different encodings. The header wins in
+      // every browser, so the document's own declaration is ignored and
+      // the text renders as mojibake. Both sides are stored canonicalised
+      // (WHATWG names), so `utf8` vs `UTF-8` never reads as a mismatch.
+      return `is_external = 0 AND content_kind = 'html'
+              AND http_charset IS NOT NULL AND meta_charset IS NOT NULL
+              AND http_charset != meta_charset`;
+    case 'issues:canonical-outside-head':
+      // `rel=canonical` inside <body> is ignored by search engines — the
+      // page has no canonical as far as they're concerned.
+      return "is_external = 0 AND content_kind = 'html' AND canonical_outside_head > 0";
+    case 'issues:lang-content-mismatch':
+      // html[lang] names a language the main content clearly isn't in
+      // (different writing system, or none of its function words while
+      // dense with another language's). See core/content-language.ts.
+      return "is_external = 0 AND content_kind = 'html' AND lang_mismatch = 1";
+    case 'issues:soft-404':
+      // 2xx page that tells the visitor it doesn't exist: "not found"
+      // wording in the title/H1 over a near-empty page, or the site's own
+      // 404 template served with 200.
+      return `is_external = 0 AND content_kind = 'html'
+              AND status_code BETWEEN 200 AND 299 AND soft_404 > 0`;
+    case 'issues:redirect-chain':
+      // Two or more redirects before the final URL, reported once at the
+      // first URL of the chain (a URL some redirect points at is a middle
+      // hop, not a start).
+      return `is_external = 0
+              AND redirect_chain_length >= 2
+              AND NOT EXISTS (SELECT 1 FROM urls p WHERE p.redirect_target = urls.url)`;
+    case 'issues:blocked-by-robots':
+      // Internal URLs the site links to but robots.txt forbids crawling —
+      // the links spend equity on pages search engines can't fetch.
+      return `is_external = 0 AND indexability = 'non-indexable:robots-blocked' AND inlinks > 0`;
+    case 'issues:noindex-js-injected':
+      // Noindex that exists only after JavaScript runs: confirmed by the
+      // rendered DOM in JS mode, or an inline script that writes it.
+      return "is_external = 0 AND content_kind = 'html' AND js_noindex > 0";
+    case 'issues:title-changed-by-js':
+      return "is_external = 0 AND content_kind = 'html' AND js_title_changed > 0";
     case 'issues:charset-missing':
       // HTML 2xx page declares no charset anywhere — neither
       // `<meta charset>` / `<meta http-equiv="Content-Type">` nor the
@@ -11322,6 +11580,7 @@ export function categoryWhereClause(
       // wasted crawl budget and a thin-content signal.
       return `is_external = 0 AND content_kind = 'html'
               AND status_code BETWEEN 200 AND 299
+              AND indexability = 'indexable'
               AND word_count IS NOT NULL
               AND word_count < 30`;
     case 'issues:og-image-not-absolute':
@@ -11704,23 +11963,6 @@ export function categoryWhereClause(
   }
 }
 
-/**
- * Validate a hreflang `lang` token against the practical BCP-47 subset
- * that Google accepts: ISO 639-1 (2 chars), ISO 639-2/3 (3 chars),
- * optional region (`-XX` country or `-NNN` UN M.49 numeric), or the
- * literal `x-default`. Case-insensitive in the wild (`tr-TR` and
- * `TR-tr` both accepted by Google) — we lowercase before matching.
- *
- * Rejects: bare uppercase/lowercase mixed errors, spaces, country-only,
- * underscored variants, three-letter region codes that aren't M.49.
- */
-function isValidHreflangCode(raw: string): boolean {
-  const code = raw.trim().toLowerCase();
-  if (!code) return false;
-  if (code === 'x-default') return true;
-  return /^[a-z]{2,3}(-[a-z]{2}|-[0-9]{3})?$/.test(code);
-}
-
 /**
  * V2 Faz 2 — raw `log_url_stats` row shape (snake_case columns). Declared
  * as a `type` (not `interface`) so it gets an implicit index signature and

packages/shared-types/src/crawl.ts

@@ -116,6 +116,14 @@ export type UrlCategory =
   | 'issues:image-duplicate-alt'
   | 'issues:meta-refresh-used'
   | 'issues:charset-missing'
+  | 'issues:charset-mismatch'
+  | 'issues:canonical-outside-head'
+  | 'issues:lang-content-mismatch'
+  | 'issues:soft-404'
+  | 'issues:redirect-chain'
+  | 'issues:blocked-by-robots'
+  | 'issues:noindex-js-injected'
+  | 'issues:title-changed-by-js'
   | 'issues:amp-validation-errors'
   | 'issues:canonical-conflict-near-duplicate'
   | 'issues:high-boilerplate'
@@ -583,6 +591,26 @@ export interface CrawlUrlRow {
    * classic broken m-dot setup.
    */
   mobileAlternate: string | null;
+  /** `charset=` of the HTTP Content-Type header (WHATWG-canonical encoding name). */
+  httpCharset: string | null;
+  /** Encoding the document declares in `<meta charset>` / `http-equiv` (canonical name). */
+  metaCharset: string | null;
+  /** `<link rel="canonical">` elements placed inside `<body>` — search engines ignore them. */
+  canonicalOutsideHead: number;
+  /** Primary language the main-content prose is written in (`tr`, `en`, …); null when unclear. */
+  contentLang: string | null;
+  /** `html[lang]` declares a language the prose is clearly not written in. */
+  langMismatch: boolean;
+  /** Soft 404 evidence: 0 none, 1 "not found" wording on a thin 2xx page, 2 serves the site's 404 template. */
+  soft404: number;
+  /** Noindex added by JavaScript: 0 none, 1 inline-script evidence (text mode), 2 confirmed by rendering. */
+  jsNoindex: number;
+  /** `<title>` changed by JavaScript: 0 none, 1 inline-script evidence (text mode), 2 confirmed by rendering. */
+  jsTitleChanged: number;
+  /** The title JavaScript sets, when it can be read; null otherwise. */
+  jsTitle: string | null;
+  /** Highest estimated text similarity (%) to another page in its near-duplicate cluster. */
+  nearDupSimilarity: number | null;
   /** True when the page declares `<html ⚡>` / `<html amp>`. AMP-for-Ads
    *  / AMP-for-Email variants (`⚡4ads`, `⚡4email`) count too. */
   ampPage: boolean;
@@ -983,11 +1011,19 @@ export interface CrawlConfig {
   processPriority: 'normal' | 'below-normal' | 'idle';
   /**
    * Maximum SimHash hamming distance (0–64) at which two pages are still
-   * considered near-duplicates. 3 (~95% similarity over the body text
-   * shingles) is the default and matches Screaming Frog's tightest near-
-   * duplicate filter. 0 disables near-duplicate clustering entirely.
+   * considered near-duplicates. SimHash only catches near-identical
+   * copies — two posts sharing 96% of their wording sit ~15 bits apart —
+   * so text similarity is judged by `nearDuplicateSimilarity`. 0 turns
+   * the SimHash pass off.
    */
   nearDuplicateHammingThreshold: number;
+  /**
+   * Minimum estimated text similarity (percent, word level) at which two
+   * pages are clustered as near-duplicates, measured with MinHash over
+   * word 3-shingles of the main content. 90 matches Screaming Frog's
+   * default. 0 turns the similarity pass off.
+   */
+  nearDuplicateSimilarity: number;
   /**
    * If true, only pages flagged `indexability = 'indexable'` participate
    * in near-duplicate clustering. Indexability-blocked pages (noindex,
@@ -1186,8 +1222,9 @@ export interface CrawlConfig {
   /**
    * Follow `<link rel="canonical">` like a redirect — when a 200 page
    * declares a canonical pointing elsewhere, also enqueue the canonical
-   * target. Default `false` — most crawls treat canonicals as a
-   * signal, not a navigation hint.
+   * target. Default `true`: a canonical aimed at a page nothing links to
+   * (a deleted product, an old slug) is otherwise never fetched, and
+   * "Canonical → Non-200 / Redirect / Noindex" can't see it.
    */
   followCanonicals: boolean;
   /**
@@ -1301,9 +1338,10 @@ export interface CrawlConfig {
   storeCanonicals: boolean;
   /** Persist `<link rel="next">` / `<link rel="prev">`. */
   storePagination: boolean;
-  /** Enqueue every `<link rel="alternate" hreflang>` target. Off by
-   *  default: hreflang alternates are usually in scope anyway and get
-   *  discovered through ordinary links. */
+  /** Enqueue every in-scope `<link rel="alternate" hreflang>` target.
+   *  On by default: an alternate reached only through the annotation is
+   *  otherwise never fetched, so reciprocity and target checks can't run
+   *  on it. */
   crawlHreflang: boolean;
   /** Persist hreflang declarations (Hreflang tab + reciprocity audit). */
   storeHreflang: boolean;
@@ -1852,6 +1890,22 @@ export interface OverviewCounts {
     imageMissingAlt: number;
     metaRefreshUsed: number;
     charsetMissing: number;
+    /** HTTP `Content-Type` charset and the document's `<meta charset>` disagree. */
+    charsetMismatch: number;
+    /** `<link rel="canonical">` placed inside `<body>`, where search engines ignore it. */
+    canonicalOutsideHead: number;
+    /** `html[lang]` declares a language the main content isn't written in. */
+    langContentMismatch: number;
+    /** 2xx pages that present themselves as "not found". */
+    soft404: number;
+    /** Redirect chains of two or more hops, counted at the chain's first URL. */
+    redirectChain: number;
+    /** Internally linked URLs that robots.txt blocks. */
+    blockedByRobots: number;
+    /** Pages whose noindex only appears once JavaScript runs. */
+    noindexJsInjected: number;
+    /** Pages whose `<title>` is rewritten by JavaScript. */
+    titleChangedByJs: number;
     ampValidationErrors: number;
     canonicalConflictNearDuplicate: number;
     highBoilerplate: number;
@@ -2209,6 +2263,13 @@ export interface DiscoveredImage {
   width: number | null;
   height: number | null;
   isInternal: boolean;
+  /**
+   * A `srcset` / `<picture><source>` candidate of an `<img>` rather than
+   * an image slot of its own. It carries the slot's alt, so it must not
+   * count again toward Missing / Empty / Duplicate Alt — the probes
+   * (broken, large) still check it.
+   */
+  variant?: boolean;
 }
 
 /**
@@ -2433,6 +2494,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   maxQueueSize: 0,
   processPriority: 'normal',
   nearDuplicateHammingThreshold: 3,
+  nearDuplicateSimilarity: 90,
   duplicatesOnlyIndexable: true,
   contentAreaSelector: '',
   webhookUrl: '',
@@ -2460,7 +2522,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   maxLinksPerPage: 100,
   maxResponseTimeMs: 0,
   maxFileSizeBytes: 0,
-  followCanonicals: false,
+  followCanonicals: true,
   followPaginationLinks: true,
   followNofollow: false,
   followJsRedirects: false,
@@ -2476,7 +2538,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   storeExternalLinks: true,
   storeCanonicals: true,
   storePagination: true,
-  crawlHreflang: false,
+  crawlHreflang: true,
   storeHreflang: true,
   crawlAmp: false,
   storeAmp: true,
@@ -2489,7 +2551,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   checkLinksOutsideStartFolder: true,
   followExternalNofollow: false,
   crawlInvalidLinks: false,
-  crawlLinkedSitemaps: false,
+  crawlLinkedSitemaps: true,
   crawlQueryStrings: true,
   crawlQueryStringExceptions: [],
   analyseInlinks: true,

packages/shared-types/src/hreflang.ts

@@ -0,0 +1,79 @@
+/**
+ * hreflang value validation.
+ *
+ * Google accepts `x-default` or `language[-Script][-REGION]`: an ISO 639-1
+ * language, an optional ISO 15924 script and an optional ISO 3166-1
+ * alpha-2 (or UN M.49 numeric) region. A purely structural check let the
+ * two most common real-world mistakes through — `en-UK` (the country code
+ * is GB) and `jp` (the language code is ja) — while rejecting valid
+ * script subtags such as `zh-Hant`.
+ *
+ * The code tables come from ICU via `Intl` (Node and Electron ship full
+ * ICU), so there is no hand-maintained list to drift out of date. ICU also
+ * knows the deprecated aliases (`UK` → `GB`, `eng` → `en`); a subtag that
+ * only exists as an alias is reported as invalid, because search engines
+ * match hreflang against the canonical codes.
+ */
+
+/** Region subtags that are reserved or private-use, never a real country. */
+const RESERVED_REGIONS = new Set(['EU', 'EZ', 'UN', 'ZZ', 'QO', 'AA']);
+
+const SHAPE = /^([a-z]{2,3})(?:-([a-z]{4}))?(?:-([a-z]{2}|\d{3}))?$/i;
+
+type Names = { of(code: string): string | undefined };
+let names: { language: Names; script: Names; region: Names } | null | undefined;
+
+function displayNames(): typeof names {
+  if (names !== undefined) return names;
+  try {
+    const make = (type: 'language' | 'script' | 'region'): Names =>
+      new Intl.DisplayNames(['en'], { type, fallback: 'none' });
+    names = { language: make('language'), script: make('script'), region: make('region') };
+  } catch {
+    // A runtime without ICU display names: fall back to the shape check.
+    names = null;
+  }
+  return names;
+}
+
+function canonical(tag: string): string | null {
+  try {
+    return Intl.getCanonicalLocales(tag)[0] ?? null;
+  } catch {
+    return null;
+  }
+}
+
+export function isValidHreflangCode(raw: string): boolean {
+  const code = raw.trim();
+  if (!code) return false;
+  if (code.toLowerCase() === 'x-default') return true;
+  const m = SHAPE.exec(code);
+  if (!m) return false;
+  const lang = (m[1] ?? '').toLowerCase();
+  const script = m[2];
+  const region = m[3]?.toUpperCase();
+
+  const dn = displayNames();
+  if (!dn) return true;
+
+  if (!dn.language.of(lang)) return false;
+  // A three-letter code for a language that has a two-letter one (`eng`,
+  // `tur`) isn't ISO 639-1; three-letter-only languages (`fil`) are fine.
+  if (lang.length === 3 && (canonical(lang)?.length ?? 0) === 2) return false;
+
+  if (script) {
+    const s = script.charAt(0).toUpperCase() + script.slice(1).toLowerCase();
+    if (!dn.script.of(s)) return false;
+  }
+
+  if (region) {
+    if (/^\d{3}$/.test(region)) return dn.region.of(region) !== undefined;
+    if (RESERVED_REGIONS.has(region)) return false;
+    if (/^(Q[M-Z]|X[A-Z])$/.test(region) && region !== 'XK') return false;
+    if (!dn.region.of(region)) return false;
+    // `UK` resolves to a name only as an alias of `GB`.
+    if (canonical(`${lang}-${region}`)?.split('-').pop()?.toUpperCase() !== region) return false;
+  }
+  return true;
+}

packages/shared-types/src/index.ts

@@ -1,6 +1,8 @@
 export * from './crawl.js';
 export * from './issue-severity.js';
 export * from './issue-catalog.js';
+export * from './hreflang.js';
+export * from './minhash.js';
 export * from './languages.js';
 export * from './ipc.js';
 export * from './project.js';

packages/shared-types/src/ipc.ts

@@ -1515,6 +1515,8 @@ export interface UrlClusterMember {
   inlinks: number;
   /** SimHash hamming distance from the queried URL (0 = identical fingerprint). */
   hammingDistance: number;
+  /** Estimated text similarity to the queried URL in percent (MinHash); null when unknown. */
+  similarity: number | null;
 }
 
 export interface UrlRewritePreviewResult {

packages/shared-types/src/issue-catalog.ts

@@ -94,6 +94,7 @@ export const ISSUE_GROUPS: readonly IssueGroupDef[] = [
       { key: 'issues-canonical-not-absolute', label: 'Canonical Not Absolute', category: 'issues:canonical-not-absolute', count: 'canonicalNotAbsolute' },
       { key: 'issues-canonical-chain-multi-hop', label: 'Canonical Chain (Multi-hop)', category: 'issues:canonical-chain-multi-hop', count: 'canonicalChainMultiHop' },
       { key: 'issues-canonical-conflict-near-duplicate', label: 'Canonical Conflict (Near-Duplicate)', category: 'issues:canonical-conflict-near-duplicate', count: 'canonicalConflictNearDuplicate' },
+      { key: 'issues-canonical-outside-head', label: 'Canonical Outside <head>', category: 'issues:canonical-outside-head', count: 'canonicalOutsideHead' },
     ],
   },
   {
@@ -119,6 +120,8 @@ export const ISSUE_GROUPS: readonly IssueGroupDef[] = [
       { key: 'issues-ttfb-slow', label: 'TTFB Slow (>600ms)', category: 'issues:ttfb-slow', count: 'ttfbSlow' },
       { key: 'issues-ttfb-very-slow', label: 'TTFB Very Slow (>1.8s)', category: 'issues:ttfb-very-slow', count: 'ttfbVerySlow' },
       { key: 'issues-over-budget', label: 'Over Performance Budget', category: 'issues:over-budget', count: 'overBudget' },
+      { key: 'issues-soft-404', label: 'Soft 404', category: 'issues:soft-404', count: 'soft404' },
+      { key: 'issues-blocked-by-robots', label: 'Linked URL Blocked by robots.txt', category: 'issues:blocked-by-robots', count: 'blockedByRobots' },
     ],
   },
   {
@@ -144,6 +147,7 @@ export const ISSUE_GROUPS: readonly IssueGroupDef[] = [
     items: [
       { key: 'issues-meta-refresh', label: 'Meta Refresh Used', category: 'issues:meta-refresh-used', count: 'metaRefreshUsed' },
       { key: 'issues-charset-missing', label: 'Charset Missing', category: 'issues:charset-missing', count: 'charsetMissing' },
+      { key: 'issues-charset-mismatch', label: 'Charset Mismatch (HTTP vs HTML)', category: 'issues:charset-mismatch', count: 'charsetMismatch' },
       { key: 'issues-amp-validation', label: 'AMP Validation Errors', category: 'issues:amp-validation-errors', count: 'ampValidationErrors' },
     ],
   },
@@ -171,6 +175,7 @@ export const ISSUE_GROUPS: readonly IssueGroupDef[] = [
     label: 'Accessibility',
     items: [
       { key: 'issues-lang-missing', label: 'Lang Attribute Missing', category: 'issues:lang-missing', count: 'langMissing' },
+      { key: 'issues-lang-content-mismatch', label: 'Lang ≠ Content Language', category: 'issues:lang-content-mismatch', count: 'langContentMismatch' },
       { key: 'issues-form-input-unlabeled', label: 'Form Inputs Missing Label', category: 'issues:form-input-unlabeled', count: 'formInputUnlabeled' },
       { key: 'issues-landmark-main-missing', label: 'Main Landmark Missing', category: 'issues:landmark-main-missing', count: 'landmarkMainMissing' },
       { key: 'issues-skip-link-missing', label: 'Skip Link Missing', category: 'issues:skip-link-missing', count: 'skipLinkMissing' },
@@ -251,6 +256,7 @@ export const ISSUE_GROUPS: readonly IssueGroupDef[] = [
       { key: 'issues-redirect-chain-long', label: 'Long Chain (>3 hops)', category: 'issues:redirect-chain-long', count: 'redirectChainLong' },
       { key: 'issues-redirect-self', label: 'Self-Redirect', category: 'issues:redirect-self', count: 'redirectSelf' },
       { key: 'issues-redirect-to-noindex', label: 'Redirect → Noindex', category: 'issues:redirect-to-noindex', count: 'redirectToNoindex' },
+      { key: 'issues-redirect-chain', label: 'Redirect Chain (2+ hops)', category: 'issues:redirect-chain', count: 'redirectChain' },
     ],
   },
   {
@@ -355,6 +361,14 @@ export const ISSUE_GROUPS: readonly IssueGroupDef[] = [
       { key: 'issues-js-only-navigation', label: 'JS-Only Navigation', category: 'issues:js-only-navigation', count: 'jsOnlyNavigation' },
     ],
   },
+  {
+    key: 'issues-javascript',
+    label: 'JavaScript',
+    items: [
+      { key: 'issues-noindex-js-injected', label: 'Noindex Added by JavaScript', category: 'issues:noindex-js-injected', count: 'noindexJsInjected' },
+      { key: 'issues-title-changed-by-js', label: 'Title Changed by JavaScript', category: 'issues:title-changed-by-js', count: 'titleChangedByJs' },
+    ],
+  },
   {
     key: 'issues-pwa',
     label: 'PWA / Discovery',

packages/shared-types/src/issue-severity.ts

@@ -85,8 +85,15 @@ const WARNING: UrlCategory[] = [
   'issues:cors-wildcard-with-credentials',
   'issues:structured-data-invalid',
   'issues:pagination-broken',
-  'issues:hreflang-x-default-missing',
   'issues:redirect-chain-long',
+  'issues:redirect-chain',
+  'issues:charset-mismatch',
+  'issues:canonical-outside-head',
+  'issues:lang-content-mismatch',
+  'issues:soft-404',
+  'issues:blocked-by-robots',
+  'issues:noindex-js-injected',
+  'issues:title-changed-by-js',
   'issues:compression-missing',
   'issues:non-indexable-in-sitemap',
   'issues:non-200-in-sitemap',
@@ -158,6 +165,9 @@ const WARNING: UrlCategory[] = [
 /** Informational / stylistic / nice-to-have signals. */
 const INFO: UrlCategory[] = [
   'issues:hreflang-unlinked',
+  // Google recommends x-default but never requires it — a missing one is
+  // a suggestion, not a defect.
+  'issues:hreflang-x-default-missing',
   'issues:heading-skipped-level',
   'issues:canonical-self-referencing',
   'issues:canonical-non-self',

packages/shared-types/src/minhash.ts

@@ -0,0 +1,99 @@
+/**
+ * MinHash signatures for near-duplicate *similarity* — "these two posts
+ * are 96% the same text" — which SimHash can't express: SimHash with a
+ * Hamming cut-off of a few bits only catches near-identical copies, and
+ * two posts sharing 96% of their wording landed 15 bits apart.
+ *
+ * Signature: MINHASH_SLOTS independent min-hashes over the page's set of
+ * word 3-shingles, each stored as its low 16 bits (b-bit MinHash). Two
+ * signatures agree in a slot with probability ≈ the Jaccard similarity of
+ * the shingle sets, so the share of agreeing slots estimates it. 128 bytes
+ * per page.
+ *
+ * Jaccard over 3-shingles is a harsh scale for humans — changing one word
+ * in 25 removes three shingles per change and drops it to ~0.76 — so the
+ * estimate is mapped back to the word-level similarity a reader means:
+ * with a fraction p of words changed independently, the shared share of
+ * shingles is (1-p)^3 and J = (1-p)^3 / (2 - (1-p)^3), hence
+ * similarity = (2J / (1 + J))^(1/3).
+ */
+
+export const MINHASH_SLOTS = 64;
+/** Rows per LSH band; 21 bands × 3 rows use 63 of the 64 slots. */
+export const MINHASH_BAND_ROWS = 3;
+export const MINHASH_BANDS = Math.floor(MINHASH_SLOTS / MINHASH_BAND_ROWS);
+/** Shingle length the similarity mapping assumes. */
+const SHINGLE_WORDS = 3;
+
+/** Fixed per-slot seeds (splitmix32 sequence) — signatures must be stable across runs. */
+const SEEDS: Uint32Array = (() => {
+  const out = new Uint32Array(MINHASH_SLOTS);
+  let x = 0x9e3779b9;
+  for (let i = 0; i < MINHASH_SLOTS; i++) {
+    x = (x + 0x9e3779b9) >>> 0;
+    let z = x;
+    z = Math.imul(z ^ (z >>> 16), 0x85ebca6b) >>> 0;
+    z = Math.imul(z ^ (z >>> 13), 0xc2b2ae35) >>> 0;
+    out[i] = (z ^ (z >>> 16)) >>> 0;
+  }
+  return out;
+})();
+
+function fnv1a32(s: string): number {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+/** murmur3 finaliser — a cheap, well-mixed 32-bit permutation. */
+function mix32(h: number): number {
+  h = Math.imul(h ^ (h >>> 16), 0x85ebca6b);
+  h = Math.imul(h ^ (h >>> 13), 0xc2b2ae35);
+  return (h ^ (h >>> 16)) >>> 0;
+}
+
+/** Signature of a set of shingles, or null when the set is empty. */
+export function computeMinHash(shingles: Iterable<string>): Uint8Array | null {
+  const mins = new Uint32Array(MINHASH_SLOTS).fill(0xffffffff);
+  let any = false;
+  for (const sh of shingles) {
+    any = true;
+    const base = fnv1a32(sh);
+    for (let i = 0; i < MINHASH_SLOTS; i++) {
+      const v = mix32(base ^ SEEDS[i]!);
+      if (v < mins[i]!) mins[i] = v;
+    }
+  }
+  if (!any) return null;
+  const out = new Uint8Array(MINHASH_SLOTS * 2);
+  for (let i = 0; i < MINHASH_SLOTS; i++) {
+    out[i * 2] = mins[i]! & 0xff;
+    out[i * 2 + 1] = (mins[i]! >>> 8) & 0xff;
+  }
+  return out;
+}
+
+/** 16-bit value of one slot. */
+export function minhashSlot(sig: Uint8Array, slot: number): number {
+  return sig[slot * 2]! | (sig[slot * 2 + 1]! << 8);
+}
+
+/** Estimated Jaccard similarity of the two shingle sets (0–1). */
+export function minhashJaccard(a: Uint8Array, b: Uint8Array): number {
+  if (a.length !== MINHASH_SLOTS * 2 || b.length !== MINHASH_SLOTS * 2) return 0;
+  let equal = 0;
+  for (let i = 0; i < a.length; i += 2) {
+    if (a[i] === b[i] && a[i + 1] === b[i + 1]) equal++;
+  }
+  return equal / MINHASH_SLOTS;
+}
+
+/** Estimated word-level text similarity (0–1) — see the module comment. */
+export function minhashSimilarity(a: Uint8Array, b: Uint8Array): number {
+  const j = minhashJaccard(a, b);
+  if (j <= 0) return 0;
+  return Math.pow((2 * j) / (1 + j), 1 / SHINGLE_WORDS);
+}