pretecno/FreeCrawl-SEO-Tool
3
0 Fork 0
mirror of https://github.com/kemalai/FreeCrawl-SEO-Tool.git synced 2026-10-18 00:39:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: hreflang outside head, soft-404 site probe, target-crawl defaults

Browse Source
This commit is contained in:
kemalai committed 2026-10-17 11:41:46 +03:00
1 parent 8e3e065214
commit 3d99a340fc
45 files changed
+435 -55
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/cli/src/index.ts
+10 -1
View File
@@ -330,7 +330,9 @@ async function main(): Promise<void> {
exportedTo = outPath;
}
const issues = db.getOverviewCounts().issues;
const overview = db.getOverviewCounts();
const issues = overview.issues;
const soft404Probe = overview.site?.soft404Probe ?? null;
// A crawl that fetched nothing (unreachable seed, everything filtered) is a
// failure for CI purposes — a successful spider always fetches ≥1 page.
const hasErrors =
@@ -351,6 +353,7 @@ async function main(): Promise<void> {
dbPath,
summary,
issues,
site: { soft404Probe },
exportedTo,
rowsExported: exportedTo ? rowsWritten : null,
},
@@ -365,6 +368,12 @@ async function main(): Promise<void> {
if (exportedTo) {
console.log(`Wrote ${rowsWritten} rows -> ${exportedTo}`);
}
if (soft404Probe) {
console.log(
`Warning: missing pages return ${soft404Probe.status} (soft 404) — ${soft404Probe.url}` +
(soft404Probe.finalUrl ? ` -> ${soft404Probe.finalUrl}` : ''),
);
}
}
db.close();
apps/desktop/src/renderer/src/App.tsx
+29
View File
@@ -47,6 +47,8 @@ export function App() {
const reset = useAppStore((s) => s.reset);
const settingsOpen = useAppStore((s) => s.settingsOpen);
const setSettingsOpen = useAppStore((s) => s.setSettingsOpen);
const configNotice = useAppStore((s) => s.configNotice);
const dismissConfigNotice = useAppStore((s) => s.dismissConfigNotice);
const setConfig = useAppStore((s) => s.setConfig);
const applyProjectConfig = useAppStore((s) => s.applyProjectConfig);
const setActiveTab = useAppStore((s) => s.setActiveTab);
@@ -532,6 +534,33 @@ export function App() {
{dropFlash}
</div>
)}
{configNotice === 'target-crawl-defaults' && (
<div className="flex items-center gap-3 border-b border-blue-700/50 bg-blue-900/30 px-4 py-1 text-[11px] text-blue-100">
<span className="min-w-0 flex-1">
{t('app.targetCrawlDefaults', {
defaultValue:
'Crawl defaults updated: canonical targets, hreflang alternates and linked XML sitemaps are now crawled, so errors on pages nothing links to are caught.',
})}
</span>
<button
type="button"
className="shrink-0 font-medium underline-offset-2 hover:underline"
onClick={() => {
dismissConfigNotice();
setSettingsOpen(true, { section: 'spider-crawl' });
}}
>
{t('app.targetCrawlDefaultsReview', { defaultValue: 'Review settings' })}
</button>
<button
type="button"
className="shrink-0 text-blue-300 underline-offset-2 hover:underline"
onClick={dismissConfigNotice}
>
{t('app.noticeDismiss', { defaultValue: 'Dismiss' })}
</button>
</div>
)}
<TabsBar />
<main className="relative flex-1 overflow-hidden">
<PanelGroup
apps/desktop/src/renderer/src/components/OverviewSidebar.tsx
+29 -1
View File
@@ -1,5 +1,5 @@
import { memo, useEffect, useMemo, useRef, useState } from 'react';
import { ChevronDown, ChevronRight } from 'lucide-react';
import { AlertTriangle, ChevronDown, ChevronRight } from 'lucide-react';
import clsx from 'clsx';
import { useTranslation } from 'react-i18next';
import {
@@ -169,6 +169,7 @@ export function OverviewSidebar() {
const disabledSet = useMemo(() => new Set<string>(disabledIssues ?? []), [disabledIssues]);
const tree = useMemo(() => buildTree(overview, disabledSet), [overview, disabledSet]);
const totalForPercent = overview?.summary.totalInternalUrls ?? 0;
const soft404Probe = overview?.site?.soft404Probe ?? null;
return (
<div className="flex h-full flex-col bg-surface-900">
@@ -181,6 +182,33 @@ export function OverviewSidebar() {
<span>{t('sidebar.percentOfTotal', { defaultValue: '% of Total' })}</span>
</div>
</div>
{soft404Probe && (
// A finding about the whole site, not a URL list — so a notice
// rather than a tree node with a count and a filter behind it.
<div
className="flex items-start gap-1.5 border-b border-amber-700/40 bg-amber-900/20 px-2 py-1.5 text-[11px] text-amber-200"
title={t('sidebar.siteSoft404Hint', {
defaultValue:
'A request for a page that does not exist was answered {{status}} instead of 404, so broken links on this site look like working pages to search engines.',
status: soft404Probe.status,
})}
>
<AlertTriangle className="mt-0.5 h-3.5 w-3.5 shrink-0" />
<div className="min-w-0">
<div className="font-medium">
{t('sidebar.siteSoft404', {
defaultValue: 'Missing pages return {{status}} (soft 404)',
status: soft404Probe.status,
})}
</div>
<div className="break-all font-mono text-[10px] text-amber-300/80">
{soft404Probe.finalUrl
? `${soft404Probe.url} → ${soft404Probe.finalUrl}`
: soft404Probe.url}
</div>
</div>
</div>
)}
<div className="flex-1 overflow-auto">
<div className="py-1 text-[11px]">
{tree.map((node) => (
apps/desktop/src/renderer/src/components/SettingsDialog.tsx
+1 -1
View File
@@ -2172,7 +2172,7 @@ function SpiderCrawlPanel({ form, update }: PanelProps) {
<CrawlRow
label="Canonicals"
info="<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter."
example="Crawl off (default) — canonicals are a signal, not a route. Store on."
example="Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on."
crawl={{ checked: form.followCanonicals, onChange: (v) => update('followCanonicals', v) }}
store={{ checked: form.storeCanonicals, onChange: (v) => update('storeCanonicals', v) }}
/>
apps/desktop/src/renderer/src/i18n/info-tips/az.ts
+2 -2
View File
@@ -572,8 +572,8 @@ export const AZ_INFO_TIPS: Record<string, string> = {
'Tara söndürülü (standart) — bütün vebi taramadan xaricilərin statusunu yoxla.',
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
'<link rel=canonical> və onun HTTP Link: başlıq forması. Tara həm də kanonik hədəfi naviqasiya göstərişi kimi növbəyə qoyur. Saxla Kanoniklər sekmesini və hər kanonik problem filtrini doldurur.',
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'Tara söndürülü (standart) — kanoniklər siqnaldır, marşrut deyil. Saxla aktiv.',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'Tara aktiv (standart) — mövcud olmayan və ya yönləndirilmiş səhifəyə işarə edən kanonik aşkarlanır. Saxla aktiv.',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>. Standart kəşf qrafının hissəsi; keçid izləməni hər yerdə söndürmədən səhifələmə dövrünü təcrid etmək üçün Tara-nı söndürün.',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/es.ts
+2 -2
View File
@@ -575,8 +575,8 @@ export const ES_INFO_TIPS: Record<string, string> = {
'Rastrear desactivado (por defecto) — comprobar el estado de los externos sin rastrear toda la web.',
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
'<link rel=canonical> y su forma de cabecera HTTP Link:. Rastrear también encola el destino canonical, tratándolo como una indicación de navegación. Almacenar alimenta la pestaña Canonicals y todos los filtros de problemas de canonical.',
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'Rastrear desactivado (por defecto) — los canonicals son una señal, no una ruta. Almacenar activado.',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'Rastrear activado (por defecto) — se detecta un canonical que apunta a una página inexistente o redirigida. Almacenar activado.',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>. Parte del grafo de descubrimiento estándar; desactiva Rastrear para aislar un bucle de paginación sin desactivar el seguimiento de enlaces en todas partes.',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/fr.ts
+2 -2
View File
@@ -575,8 +575,8 @@ export const FR_INFO_TIPS: Record<string, string> = {
'Explorer désactivé (par défaut) — vérifier le statut des externes sans explorer tout le web.',
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
"<link rel=canonical> et sa forme d'en-tête HTTP Link:. Explorer met aussi la cible canonique en file, en la traitant comme une indication de navigation. Stocker alimente l'onglet Canoniques et tous les filtres de problèmes de canonique.",
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'Explorer désactivé (par défaut) — les canoniques sont un signal, pas un itinéraire. Stocker activé.',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'Explorer activé (par défaut) — une canonique pointant vers une page manquante ou redirigée est détectée. Stocker activé.',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>. Fait partie du graphe de découverte standard ; désactivez Explorer pour isoler une boucle de pagination sans désactiver le suivi de liens partout.',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/hi.ts
+2 -2
View File
@@ -570,8 +570,8 @@ export const HI_INFO_TIPS: Record<string, string> = {
'क्रॉल बंद (डिफ़ॉल्ट) — पूरा वेब स्पाइडर किए बिना बाहरी लिंक की स्टेटस-जाँच करें।',
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
'<link rel=canonical> और उसका HTTP Link: हेडर रूप। क्रॉल कैनोनिकल लक्ष्य को भी नेविगेशन संकेत मानकर कतार में डालता है। संग्रह कैनोनिकल टैब और हर कैनोनिकल समस्या फ़िल्टर को डेटा देता है।',
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'क्रॉल बंद (डिफ़ॉल्ट) — कैनोनिकल एक सिग्नल है, रास्ता नहीं। संग्रह चालू।',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'क्रॉल चालू (डिफ़ॉल्ट) — गायब या रीडायरेक्ट किए गए पेज की ओर इशारा करने वाला कैनोनिकल पकड़ा जाता है। संग्रह चालू।',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>। मानक खोज ग्राफ़ का हिस्सा; हर जगह लिंक-फ़ॉलो बंद किए बिना पेजिनेशन लूप अलग करने के लिए क्रॉल बंद करें।',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/it.ts
+2 -2
View File
@@ -575,8 +575,8 @@ export const IT_INFO_TIPS: Record<string, string> = {
"Scansiona disattivato (predefinito) — controlla lo stato degli esterni senza scansionare l'intero web.",
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
'<link rel=canonical> e la sua forma di header HTTP Link:. Scansiona accoda anche la destinazione canonical, trattandola come un suggerimento di navigazione. Memorizza alimenta la scheda Canonical e ogni filtro problema sui canonical.',
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'Scansiona disattivato (predefinito) — i canonical sono un segnale, non un percorso. Memorizza attivato.',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'Scansiona attivato (predefinito) — un canonical che punta a una pagina mancante o reindirizzata viene rilevato. Memorizza attivato.',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>. Parte del grafo di scoperta standard; disattiva Scansiona per isolare un ciclo di paginazione senza disabilitare il follow dei link ovunque.',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/ko.ts
+2 -2
View File
@@ -565,8 +565,8 @@ export const KO_INFO_TIPS: Record<string, string> = {
'크롤링 끔(기본값) — 웹 전체를 스파이더링하지 않고 외부 링크 상태만 검사합니다.',
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
'<link rel=canonical>과 그 HTTP Link: 헤더 형식. 크롤링은 canonical 대상도 탐색 힌트로 취급해 큐에 넣습니다. 저장은 표준 링크 탭과 모든 canonical 문제 필터에 데이터를 제공합니다.',
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'크롤링 끔(기본값) — canonical은 경로가 아니라 신호입니다. 저장 켬.',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'크롤링 켬(기본값) — 없는 페이지나 리다이렉트된 페이지를 가리키는 canonical을 찾아냅니다. 저장 켬.',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>. 표준 발견 그래프의 일부입니다. 모든 곳의 링크 추적을 끄지 않고 페이지네이션 루프만 격리하려면 크롤링을 끄세요.',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/pt-BR.ts
+2 -2
View File
@@ -573,8 +573,8 @@ export const PT_BR_INFO_TIPS: Record<string, string> = {
'Rastrear desligado (padrão) — verificar o status dos externos sem rastrear a web inteira.',
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
'<link rel=canonical> e sua forma de cabeçalho HTTP Link:. Rastrear também enfileira o destino canonical, tratando-o como dica de navegação. Armazenar alimenta a aba Canonicals e todos os filtros de problemas de canonical.',
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'Rastrear desligado (padrão) — canonicals são um sinal, não uma rota. Armazenar ligado.',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'Rastrear ligado (padrão) — um canonical que aponta para uma página inexistente ou redirecionada é detectado. Armazenar ligado.',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>. Parte do grafo de descoberta padrão; desligue Rastrear para isolar um loop de paginação sem desativar o seguimento de links em todo lugar.',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/ru.ts
+2 -2
View File
@@ -574,8 +574,8 @@ export const RU_INFO_TIPS: Record<string, string> = {
'«Сканировать» выкл. (по умолчанию) — проверять статус внешних, не обходя весь интернет.',
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
'<link rel=canonical> и его форма в HTTP-заголовке Link:. «Сканировать» также ставит цель canonical в очередь как подсказку навигации. «Сохранять» питает вкладку «Canonical» и все фильтры проблем canonical.',
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'«Сканировать» выкл. (по умолчанию) — canonical это сигнал, а не маршрут. «Сохранять» вкл.',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'«Сканировать» вкл. (по умолчанию) — canonical, указывающий на отсутствующую или перенаправленную страницу, будет обнаружен. «Сохранять» вкл.',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>. Часть стандартного графа обнаружения; выключите «Сканировать», чтобы изолировать петлю пагинации, не отключая переходы по ссылкам повсюду.',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/tr.ts
+2 -2
View File
@@ -574,8 +574,8 @@ export const TR_INFO_TIPS: Record<string, string> = {
"Tara kapalı (varsayılan) — tüm web'i taramadan dış bağlantıların durumunu kontrol edin.",
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
"<link rel=canonical> ve onun HTTP Link: başlığı biçimi. Tara ayrıca canonical hedefini de kuyruğa alır, yani onu bir navigasyon ipucu gibi değerlendirir. Sakla, Canonical'lar sekmesini ve tüm canonical sorun filtrelerini besler.",
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'Tara kapalı (varsayılan) — canonical bir sinyaldir, bir rota değil. Sakla açık.',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'Tara açık (varsayılan) — eksik ya da yönlendirilmiş bir sayfayı gösteren canonical yakalanır. Sakla açık.',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
"<link rel=next> / <link rel=prev>. Standart keşif grafiğinin parçasıdır; her yerde bağlantı takibini kapatmadan bir sayfalama döngüsünü izole etmek için Tara'yı kapatın.",
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/info-tips/zh-CN.ts
+2 -2
View File
@@ -561,8 +561,8 @@ export const ZH_CN_INFO_TIPS: Record<string, string> = {
'“抓取”关（默认）——检查外链状态但不去抓取整个互联网。',
['<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter.']:
'<link rel=canonical> 及其 HTTP Link: 响应头形式。“抓取”还会将 canonical 目标入队，视其为导航提示。“存储”为“规范链接”标签页和所有 canonical 问题筛选器提供数据。',
['Crawl off (default) — canonicals are a signal, not a route. Store on.']:
'“抓取”关（默认）——canonical 是信号而非路径。“存储”开。',
['Crawl on (default) — a canonical pointing at a missing or redirected page is caught. Store on.']:
'“抓取”开（默认）——指向缺失或已重定向页面的 canonical 会被发现。“存储”开。',
['<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere.']:
'<link rel=next> / <link rel=prev>。标准发现图的一部分；关闭“抓取”可隔离分页循环，而无需在各处禁用链接跟踪。',
['Both on unless you are debugging an infinite paginated series.']:
apps/desktop/src/renderer/src/i18n/labels/az.ts
+1
View File
@@ -199,6 +199,7 @@ export const AZ_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'lang atributu yoxdur',
'Charset Missing': 'Charset yoxdur',
'Canonical Outside <head>': 'Canonical <head> xaricində',
'Hreflang Outside <head>': 'Hreflang <head> xaricində',
'Soft 404': 'Soft 404',
'Linked URL Blocked by robots.txt': 'Keçid verilən URL robots.txt ilə bloklanıb',
'Charset Mismatch (HTTP vs HTML)': 'Charset uyğunsuzluğu (HTTP və HTML)',
apps/desktop/src/renderer/src/i18n/labels/es.ts
+1
View File
@@ -191,6 +191,7 @@ export const ES_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'Falta el atributo lang',
'Charset Missing': 'Falta el charset',
'Canonical Outside <head>': 'Canonical fuera de <head>',
'Hreflang Outside <head>': 'Hreflang fuera de <head>',
'Soft 404': 'Soft 404',
'Linked URL Blocked by robots.txt': 'URL enlazada bloqueada por robots.txt',
'Charset Mismatch (HTTP vs HTML)': 'Charset no coincide (HTTP vs HTML)',
apps/desktop/src/renderer/src/i18n/labels/fr.ts
+1
View File
@@ -191,6 +191,7 @@ export const FR_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'Attribut lang manquant',
'Charset Missing': 'Charset manquant',
'Canonical Outside <head>': 'Canonical hors de <head>',
'Hreflang Outside <head>': 'Hreflang hors de <head>',
'Soft 404': 'Soft 404',
'Linked URL Blocked by robots.txt': 'URL liée bloquée par robots.txt',
'Charset Mismatch (HTTP vs HTML)': 'Charset incohérent (HTTP vs HTML)',
apps/desktop/src/renderer/src/i18n/labels/hi.ts
+1
View File
@@ -191,6 +191,7 @@ export const HI_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'lang विशेषता नहीं',
'Charset Missing': 'charset नहीं',
'Canonical Outside <head>': 'Canonical <head> के बाहर',
'Hreflang Outside <head>': 'Hreflang <head> के बाहर',
'Soft 404': 'सॉफ़्ट 404',
'Linked URL Blocked by robots.txt': 'लिंक किया गया URL robots.txt से अवरुद्ध',
'Charset Mismatch (HTTP vs HTML)': 'charset बेमेल (HTTP बनाम HTML)',
apps/desktop/src/renderer/src/i18n/labels/it.ts
+1
View File
@@ -191,6 +191,7 @@ export const IT_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'Attributo lang mancante',
'Charset Missing': 'Charset mancante',
'Canonical Outside <head>': 'Canonical fuori da <head>',
'Hreflang Outside <head>': 'Hreflang fuori da <head>',
'Soft 404': 'Soft 404',
'Linked URL Blocked by robots.txt': 'URL collegato bloccato da robots.txt',
'Charset Mismatch (HTTP vs HTML)': 'Charset non corrispondente (HTTP vs HTML)',
apps/desktop/src/renderer/src/i18n/labels/ko.ts
+1
View File
@@ -191,6 +191,7 @@ export const KO_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'lang 속성 없음',
'Charset Missing': 'charset 없음',
'Canonical Outside <head>': '<head> 밖의 canonical',
'Hreflang Outside <head>': '<head> 밖의 hreflang',
'Soft 404': '소프트 404',
'Linked URL Blocked by robots.txt': 'robots.txt로 차단된 링크 URL',
'Charset Mismatch (HTTP vs HTML)': 'charset 불일치(HTTP vs HTML)',
apps/desktop/src/renderer/src/i18n/labels/pt-BR.ts
+1
View File
@@ -191,6 +191,7 @@ export const PT_BR_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'Falta o atributo lang',
'Charset Missing': 'Falta o charset',
'Canonical Outside <head>': 'Canonical fora do <head>',
'Hreflang Outside <head>': 'Hreflang fora do <head>',
'Soft 404': 'Soft 404',
'Linked URL Blocked by robots.txt': 'URL com link bloqueada pelo robots.txt',
'Charset Mismatch (HTTP vs HTML)': 'Charset divergente (HTTP vs HTML)',
apps/desktop/src/renderer/src/i18n/labels/ru.ts
+1
View File
@@ -191,6 +191,7 @@ export const RU_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'Нет атрибута lang',
'Charset Missing': 'Нет charset',
'Canonical Outside <head>': 'Canonical вне <head>',
'Hreflang Outside <head>': 'Hreflang вне <head>',
'Soft 404': 'Soft 404',
'Linked URL Blocked by robots.txt': 'URL со ссылками заблокирован robots.txt',
'Charset Mismatch (HTTP vs HTML)': 'Несовпадение charset (HTTP и HTML)',
apps/desktop/src/renderer/src/i18n/labels/tr.ts
+1
View File
@@ -190,6 +190,7 @@ export const TR_LABELS: Record<string, string> = {
'Lang Attribute Missing': 'Lang Niteliği Eksik',
'Charset Missing': 'Charset Eksik',
'Canonical Outside <head>': 'Canonical <head> Dışında',
'Hreflang Outside <head>': 'Hreflang <head> Dışında',
'Soft 404': 'Soft 404',
'Linked URL Blocked by robots.txt': 'Bağlantı Verilen URL robots.txt ile Engelli',
'Charset Mismatch (HTTP vs HTML)': 'Charset Uyuşmazlığı (HTTP ve HTML)',
apps/desktop/src/renderer/src/i18n/labels/zh-CN.ts
+1
View File
@@ -191,6 +191,7 @@ export const ZH_CN_LABELS: Record<string, string> = {
'Lang Attribute Missing': '缺少 lang 属性',
'Charset Missing': '缺少 charset',
'Canonical Outside <head>': 'Canonical 位于 <head> 之外',
'Hreflang Outside <head>': 'Hreflang 位于 <head> 之外',
'Soft 404': '软 404',
'Linked URL Blocked by robots.txt': '被 robots.txt 拦截的链接 URL',
'Charset Mismatch (HTTP vs HTML)': 'charset 不一致（HTTP 与 HTML）',
apps/desktop/src/renderer/src/i18n/locales/az.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "Ümumi baxış",
"urls": "URL",
"percentOfTotal": "Ümumidən %"
"percentOfTotal": "Ümumidən %",
"siteSoft404": "Mövcud olmayan səhifələr {{status}} qaytarır (soft 404)",
"siteSoft404Hint": "Mövcud olmayan səhifəyə edilən sorğuya 404 əvəzinə {{status}} cavabı verildi, buna görə də bu saytdakı qırıq keçidlər axtarış sistemlərinə işləyən səhifələr kimi görünür."
},
"integrations": {
"intro": "Xarici xidmətləri öz giriş məlumatlarınızla qoşun. API açarları və sirlər əməliyyat sisteminizin təhlükəsiz anbarı ilə şifrələnərək saxlanılır və heç vaxt bu maşından kənara çıxmır.",
@@ -297,7 +299,10 @@
"deleteDomainResult_one": "{{host}} adresindən {{formatted}} URL sətri silindi.",
"deleteDomainResult_other": "{{host}} adresindən {{formatted}} URL sətri silindi.",
"deleteDomainLinks_one": "Onlarla birlikdə {{formatted}} əlaqəli keçid də silindi.",
"deleteDomainLinks_other": "Onlarla birlikdə {{formatted}} əlaqəli keçid də silindi."
"deleteDomainLinks_other": "Onlarla birlikdə {{formatted}} əlaqəli keçid də silindi.",
"targetCrawlDefaults": "Standart tarama parametrləri yeniləndi: canonical hədəfləri, hreflang alternativləri və keçidli XML sitemap-lər artıq taranır, beləliklə heç bir keçidin aparmadığı səhifələrdəki xətalar da aşkarlanır.",
"targetCrawlDefaultsReview": "Parametrlərə bax",
"noticeDismiss": "Bağla"
},
"infoTip": {
"moreInfo": "Ətraflı məlumat",
apps/desktop/src/renderer/src/i18n/locales/en.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "Overview",
"urls": "URLs",
"percentOfTotal": "% of Total"
"percentOfTotal": "% of Total",
"siteSoft404": "Missing pages return {{status}} (soft 404)",
"siteSoft404Hint": "A request for a page that does not exist was answered {{status}} instead of 404, so broken links on this site look like working pages to search engines."
},
"integrations": {
"intro": "Connect external services with your own credentials. API keys and secrets are encrypted at rest using your operating system's secure storage and never leave this machine.",
@@ -297,7 +299,10 @@
"clearAllConfirm": "Clear ALL data in the active project?\n\nEvery URL, link, image, header, source snapshot, sitemap, and issue will be wiped. This cannot be undone — Save Project As… first if you want a backup.",
"clearFailed": "Could not clear the crawl: {{detail}}",
"deleteDomainLinks_one": "{{formatted}} associated link was removed with them.",
"deleteDomainLinks_other": "{{formatted}} associated links were removed with them."
"deleteDomainLinks_other": "{{formatted}} associated links were removed with them.",
"targetCrawlDefaults": "Crawl defaults updated: canonical targets, hreflang alternates and linked XML sitemaps are now crawled, so errors on pages nothing links to are caught.",
"targetCrawlDefaultsReview": "Review settings",
"noticeDismiss": "Dismiss"
},
"infoTip": {
"moreInfo": "More info",
apps/desktop/src/renderer/src/i18n/locales/es.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "Resumen general",
"urls": "URL",
"percentOfTotal": "% del total"
"percentOfTotal": "% del total",
"siteSoft404": "Las páginas inexistentes devuelven {{status}} (soft 404)",
"siteSoft404Hint": "Una solicitud a una página que no existe recibió {{status}} en lugar de 404, así que los enlaces rotos de este sitio parecen páginas válidas para los buscadores."
},
"integrations": {
"intro": "Conecta servicios externos con tus propias credenciales. Las claves de API y los secretos se cifran en reposo mediante el almacenamiento seguro de tu sistema operativo y nunca salen de esta máquina.",
@@ -304,7 +306,10 @@
"deleteDomainResult_other": "Se eliminaron {{formatted}} filas de URL de {{host}}.",
"deleteDomainLinks_one": "Junto con ellas se eliminó {{formatted}} enlace asociado.",
"deleteDomainLinks_many": "Junto con ellas se eliminaron {{formatted}} de enlaces asociados.",
"deleteDomainLinks_other": "Junto con ellas se eliminaron {{formatted}} enlaces asociados."
"deleteDomainLinks_other": "Junto con ellas se eliminaron {{formatted}} enlaces asociados.",
"targetCrawlDefaults": "Valores predeterminados de rastreo actualizados: ahora se rastrean los destinos canonical, las alternativas hreflang y los sitemaps XML enlazados, para detectar errores en páginas a las que nada enlaza.",
"targetCrawlDefaultsReview": "Revisar ajustes",
"noticeDismiss": "Descartar"
},
"infoTip": {
"moreInfo": "Más información",
apps/desktop/src/renderer/src/i18n/locales/fr.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "Vue d'ensemble",
"urls": "URL",
"percentOfTotal": "% du total"
"percentOfTotal": "% du total",
"siteSoft404": "Les pages inexistantes renvoient {{status}} (soft 404)",
"siteSoft404Hint": "Une requête vers une page qui n'existe pas a reçu {{status}} au lieu de 404 : les liens cassés de ce site ressemblent donc à des pages valides pour les moteurs de recherche."
},
"integrations": {
"intro": "Connectez des services externes avec vos propres identifiants. Les clés d'API et les secrets sont chiffrés au repos via le stockage sécurisé de votre système d'exploitation et ne quittent jamais cette machine.",
@@ -304,7 +306,10 @@
"deleteDomainResult_other": "{{formatted}} lignes d'URL supprimées de {{host}}.",
"deleteDomainLinks_one": "{{formatted}} lien associé a été supprimé avec elles.",
"deleteDomainLinks_many": "{{formatted}} de liens associés ont été supprimés avec elles.",
"deleteDomainLinks_other": "{{formatted}} liens associés ont été supprimés avec elles."
"deleteDomainLinks_other": "{{formatted}} liens associés ont été supprimés avec elles.",
"targetCrawlDefaults": "Paramètres d'exploration par défaut mis à jour : les cibles canonical, les alternatives hreflang et les sitemaps XML liés sont désormais explorés, pour détecter les erreurs sur les pages vers lesquelles aucun lien ne pointe.",
"targetCrawlDefaultsReview": "Vérifier les paramètres",
"noticeDismiss": "Ignorer"
},
"infoTip": {
"moreInfo": "Plus d'informations",
apps/desktop/src/renderer/src/i18n/locales/hi.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "अवलोकन",
"urls": "URL",
"percentOfTotal": "कुल का %"
"percentOfTotal": "कुल का %",
"siteSoft404": "मौजूद न होने वाले पेज {{status}} लौटाते हैं (soft 404)",
"siteSoft404Hint": "मौजूद न होने वाले पेज के अनुरोध का उत्तर 404 के बजाय {{status}} मिला, इसलिए इस साइट के टूटे लिंक सर्च इंजनों को काम करते पेज जैसे दिखते हैं।"
},
"integrations": {
"intro": "अपनी ही क्रेडेंशियल से बाहरी सेवाएँ जोड़ें। API कुंजियाँ और सीक्रेट आपके ऑपरेटिंग सिस्टम के सुरक्षित स्टोरेज से एन्क्रिप्ट होकर रखे जाते हैं और कभी इस मशीन से बाहर नहीं जाते।",
@@ -297,7 +299,10 @@
"deleteDomainResult_one": "{{host}} से {{formatted}} URL पंक्ति हटाई गई।",
"deleteDomainResult_other": "{{host}} से {{formatted}} URL पंक्तियाँ हटाई गईं।",
"deleteDomainLinks_one": "इनके साथ {{formatted}} संबंधित लिंक भी हटाया गया।",
"deleteDomainLinks_other": "इनके साथ {{formatted}} संबंधित लिंक भी हटाए गए।"
"deleteDomainLinks_other": "इनके साथ {{formatted}} संबंधित लिंक भी हटाए गए।",
"targetCrawlDefaults": "क्रॉल डिफ़ॉल्ट अपडेट किए गए: अब canonical लक्ष्य, hreflang विकल्प और लिंक किए गए XML साइटमैप भी क्रॉल होते हैं, ताकि उन पेजों की त्रुटियाँ भी पकड़ी जाएँ जिन पर कोई लिंक नहीं है।",
"targetCrawlDefaultsReview": "सेटिंग्स देखें",
"noticeDismiss": "खारिज करें"
},
"infoTip": {
"moreInfo": "अधिक जानकारी",
apps/desktop/src/renderer/src/i18n/locales/it.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "Panoramica",
"urls": "URL",
"percentOfTotal": "% del totale"
"percentOfTotal": "% del totale",
"siteSoft404": "Le pagine inesistenti restituiscono {{status}} (soft 404)",
"siteSoft404Hint": "Una richiesta a una pagina che non esiste ha ricevuto {{status}} invece di 404, quindi i link interrotti di questo sito sembrano pagine funzionanti ai motori di ricerca."
},
"integrations": {
"intro": "Collega servizi esterni con le tue credenziali. Le chiavi API e i segreti sono cifrati a riposo tramite l'archivio sicuro del tuo sistema operativo e non lasciano mai questa macchina.",
@@ -304,7 +306,10 @@
"deleteDomainResult_other": "{{formatted}} righe di URL eliminate da {{host}}.",
"deleteDomainLinks_one": "Insieme a esse è stato rimosso {{formatted}} link associato.",
"deleteDomainLinks_many": "Insieme a esse sono stati rimossi {{formatted}} di link associati.",
"deleteDomainLinks_other": "Insieme a esse sono stati rimossi {{formatted}} link associati."
"deleteDomainLinks_other": "Insieme a esse sono stati rimossi {{formatted}} link associati.",
"targetCrawlDefaults": "Impostazioni di scansione predefinite aggiornate: ora vengono scansionati i target canonical, le alternative hreflang e le sitemap XML collegate, così vengono rilevati gli errori nelle pagine che nessun link raggiunge.",
"targetCrawlDefaultsReview": "Rivedi impostazioni",
"noticeDismiss": "Ignora"
},
"infoTip": {
"moreInfo": "Maggiori informazioni",
apps/desktop/src/renderer/src/i18n/locales/ko.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "개요",
"urls": "URL",
"percentOfTotal": "전체 대비 %"
"percentOfTotal": "전체 대비 %",
"siteSoft404": "없는 페이지가 {{status}} 반환 (soft 404)",
"siteSoft404Hint": "존재하지 않는 페이지를 요청했을 때 404 대신 {{status}} 응답을 받았습니다. 따라서 이 사이트의 깨진 링크가 검색 엔진에는 정상 페이지처럼 보입니다."
},
"integrations": {
"intro": "자신의 자격 증명으로 외부 서비스를 연결하십시오. API 키와 비밀 값은 운영체제의 보안 저장소를 통해 암호화되어 저장되며 이 컴퓨터를 벗어나지 않습니다.",
@@ -290,7 +292,10 @@
"clearAllConfirm": "현재 프로젝트의 모든 데이터를 지우시겠습니까?\n\n모든 URL, 링크, 이미지, 헤더, 소스 스냅샷, 사이트맵, 문제 항목이 삭제됩니다. 되돌릴 수 없으므로 백업이 필요하면 먼저 다른 이름으로 프로젝트 저장…을 사용하십시오.",
"clearFailed": "크롤 데이터를 지우지 못했습니다: {{detail}}",
"deleteDomainResult_other": "{{host}}에서 URL 행 {{formatted}}개를 삭제했습니다.",
"deleteDomainLinks_other": "이와 함께 연관된 링크 {{formatted}}개도 제거되었습니다."
"deleteDomainLinks_other": "이와 함께 연관된 링크 {{formatted}}개도 제거되었습니다.",
"targetCrawlDefaults": "크롤 기본값이 업데이트되었습니다. 이제 canonical 대상, hreflang 대체 페이지, 연결된 XML 사이트맵도 크롤하므로 어떤 링크도 가리키지 않는 페이지의 오류도 발견됩니다.",
"targetCrawlDefaultsReview": "설정 검토",
"noticeDismiss": "닫기"
},
"infoTip": {
"moreInfo": "자세히",
apps/desktop/src/renderer/src/i18n/locales/pt-BR.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "Visão geral",
"urls": "URLs",
"percentOfTotal": "% do total"
"percentOfTotal": "% do total",
"siteSoft404": "Páginas inexistentes retornam {{status}} (soft 404)",
"siteSoft404Hint": "Uma solicitação para uma página que não existe recebeu {{status}} em vez de 404, então os links quebrados deste site parecem páginas válidas para os buscadores."
},
"integrations": {
"intro": "Conecte serviços externos com suas próprias credenciais. As chaves de API e os segredos são criptografados em repouso pelo armazenamento seguro do seu sistema operacional e nunca saem desta máquina.",
@@ -304,7 +306,10 @@
"deleteDomainResult_other": "{{formatted}} linhas de URL excluídas de {{host}}.",
"deleteDomainLinks_one": "Junto com elas, {{formatted}} link associado foi removido.",
"deleteDomainLinks_many": "Junto com elas, {{formatted}} de links associados foram removidos.",
"deleteDomainLinks_other": "Junto com elas, {{formatted}} links associados foram removidos."
"deleteDomainLinks_other": "Junto com elas, {{formatted}} links associados foram removidos.",
"targetCrawlDefaults": "Padrões de rastreamento atualizados: destinos canonical, alternativas hreflang e sitemaps XML vinculados agora são rastreados, para detectar erros em páginas que nenhum link alcança.",
"targetCrawlDefaultsReview": "Revisar configurações",
"noticeDismiss": "Dispensar"
},
"infoTip": {
"moreInfo": "Mais informações",
apps/desktop/src/renderer/src/i18n/locales/ru.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "Обзор",
"urls": "URL",
"percentOfTotal": "% от общего числа"
"percentOfTotal": "% от общего числа",
"siteSoft404": "Несуществующие страницы возвращают {{status}} (soft 404)",
"siteSoft404Hint": "Запрос к несуществующей странице получил ответ {{status}} вместо 404, поэтому битые ссылки на этом сайте выглядят для поисковых систем как рабочие страницы."
},
"integrations": {
"intro": "Подключайте внешние сервисы со своими собственными учётными данными. API-ключи и секреты шифруются на диске средствами защищённого хранилища вашей операционной системы и никогда не покидают эту машину.",
@@ -311,7 +313,10 @@
"deleteDomainLinks_one": "Вместе с ними удалена {{formatted}} связанная ссылка.",
"deleteDomainLinks_few": "Вместе с ними удалено {{formatted}} связанные ссылки.",
"deleteDomainLinks_many": "Вместе с ними удалено {{formatted}} связанных ссылок.",
"deleteDomainLinks_other": "Вместе с ними удалено {{formatted}} связанные ссылки."
"deleteDomainLinks_other": "Вместе с ними удалено {{formatted}} связанные ссылки.",
"targetCrawlDefaults": "Настройки обхода по умолчанию обновлены: теперь обходятся цели canonical, альтернативы hreflang и связанные XML-sitemap, чтобы находить ошибки на страницах, на которые нет ссылок.",
"targetCrawlDefaultsReview": "Проверить настройки",
"noticeDismiss": "Закрыть"
},
"infoTip": {
"moreInfo": "Подробнее",
apps/desktop/src/renderer/src/i18n/locales/tr.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "Genel Bakış",
"urls": "URL'ler",
"percentOfTotal": "Toplamın %"
"percentOfTotal": "Toplamın %",
"siteSoft404": "Olmayan sayfalar {{status}} döndürüyor (soft 404)",
"siteSoft404Hint": "Var olmayan bir sayfa için yapılan istek 404 yerine {{status}} ile yanıtlandı; bu yüzden sitedeki kırık bağlantılar arama motorlarına çalışan sayfalar gibi görünüyor."
},
"integrations": {
"intro": "Harici servisleri kendi kimlik bilgilerinle bağla. API anahtarları ve secret'lar işletim sisteminin güvenli depolaması ile şifreli saklanır ve bu makineden asla çıkmaz.",
@@ -297,7 +299,10 @@
"clearAllConfirm": "Aktif projedeki TÜM veriler temizlensin mi?\n\nHer URL, link, görsel, başlık, kaynak snapshot, sitemap ve issue silinecek. Geri alınamaz — yedek istiyorsan önce Projeyi Farklı Kaydet.",
"clearFailed": "Tarama temizlenemedi: {{detail}}",
"deleteDomainLinks_one": "Bunlarla birlikte {{formatted}} ilişkili bağlantı da kaldırıldı.",
"deleteDomainLinks_other": "Bunlarla birlikte {{formatted}} ilişkili bağlantı da kaldırıldı."
"deleteDomainLinks_other": "Bunlarla birlikte {{formatted}} ilişkili bağlantı da kaldırıldı.",
"targetCrawlDefaults": "Tarama varsayılanları güncellendi: canonical hedefleri, hreflang alternatifleri ve bağlantılı XML sitemap'ler artık taranıyor; böylece hiçbir bağlantının ulaşmadığı sayfalardaki hatalar da yakalanıyor.",
"targetCrawlDefaultsReview": "Ayarları İncele",
"noticeDismiss": "Kapat"
},
"infoTip": {
"moreInfo": "Daha fazla bilgi",
apps/desktop/src/renderer/src/i18n/locales/zh-CN.json
+7 -2
View File
@@ -2,7 +2,9 @@
"sidebar": {
"overview": "概览",
"urls": "URL",
"percentOfTotal": "占总数百分比"
"percentOfTotal": "占总数百分比",
"siteSoft404": "不存在的页面返回 {{status}}（soft 404）",
"siteSoft404Hint": "请求一个不存在的页面时得到的是 {{status}} 而不是 404，因此该站点上的失效链接在搜索引擎看来就像正常页面。"
},
"integrations": {
"intro": "使用你自己的凭据连接外部服务。API 密钥和机密会通过操作系统的安全存储在本地加密保存，绝不会离开这台机器。",
@@ -290,7 +292,10 @@
"clearAllConfirm": "清除当前项目中的全部数据吗？\n\n所有 URL、链接、图片、响应头、源码快照、站点地图和问题都会被清空。此操作无法撤销——如需备份，请先使用“项目另存为…”。",
"clearFailed": "无法清除抓取数据：{{detail}}",
"deleteDomainResult_other": "已从 {{host}} 删除 {{formatted}} 行 URL。",
"deleteDomainLinks_other": "随之一并移除了 {{formatted}} 条关联链接。"
"deleteDomainLinks_other": "随之一并移除了 {{formatted}} 条关联链接。",
"targetCrawlDefaults": "抓取默认设置已更新：现在会抓取 canonical 目标、hreflang 备用版本和关联的 XML 站点地图，从而发现没有任何链接指向的页面上的错误。",
"targetCrawlDefaultsReview": "查看设置",
"noticeDismiss": "关闭"
},
"infoTip": {
"moreInfo": "更多信息",
apps/desktop/src/renderer/src/store.ts
+52 -3
View File
@@ -306,6 +306,7 @@ export const TAB_QUICK_FILTERS: Partial<Record<TabKey, TabQuickFilter[]>> = {
{ label: 'Reciprocity Missing', category: 'issues:hreflang-reciprocity-missing' },
{ label: 'Target Issues', category: 'issues:hreflang-target-issues' },
{ label: 'Inconsistent Lang', category: 'issues:hreflang-inconsistent-lang' },
{ label: 'Hreflang Outside <head>', category: 'issues:hreflang-outside-head' },
],
// ─── Structured Data tab ────────────────────────────────────────────────
@@ -377,6 +378,9 @@ export interface TabViewState {
interface AppState {
config: CrawlConfig;
/** One-time notice shown under the top bar until dismissed. */
configNotice: 'target-crawl-defaults' | null;
dismissConfigNotice: () => void;
progress: CrawlProgress | null;
summary: CrawlSummary | null;
overview: OverviewCounts | null;
@@ -483,16 +487,57 @@ function loadRecentUrls(): string[] {
function loadInitialConfig(): CrawlConfig {
if (typeof window === 'undefined' || !window.freecrawl) return DEFAULT_CRAWL_CONFIG;
const saved = window.freecrawl.prefsGet('crawl-config');
if (!saved || typeof saved !== 'object') return { ...DEFAULT_CRAWL_CONFIG, startUrl: '' };
if (!saved || typeof saved !== 'object') {
// A fresh install already starts on the new defaults; mark it so a
// later `false` the user saves is never "upgraded" back.
window.freecrawl.prefsSet(TARGET_CRAWL_DEFAULTS_PREF, 1);
return { ...DEFAULT_CRAWL_CONFIG, startUrl: '' };
}
return {
...DEFAULT_CRAWL_CONFIG,
...(saved as Partial<CrawlConfig>),
...upgradeSavedConfig(saved as Partial<CrawlConfig>),
startUrl: '',
mode: 'spider',
deviceMode: 'desktop',
};
}
/**
* One-time move of a saved config onto the target-crawl defaults.
* Following canonical targets, hreflang alternates and linked XML sitemaps
* became the default because without them every error on a page nothing
* links to goes unseen. But the whole config is saved on every edit, so
* anyone who ever changed a setting still carries the old `false`s and
* would never get the new defaults — and a deliberate `false` can't be told
* apart from an inherited one. So the three are switched on once, a notice
* says so (with a way back), and any `false` saved after that is the user's
* own choice and stays. Saved projects keep the config they were crawled
* with — that is part of the document.
*/
const TARGET_CRAWL_DEFAULTS_PREF = 'crawl-config-target-defaults';
let targetCrawlDefaultsApplied = false;
function upgradeSavedConfig(saved: Partial<CrawlConfig>): Partial<CrawlConfig> {
if (window.freecrawl.prefsGet(TARGET_CRAWL_DEFAULTS_PREF)) return saved;
window.freecrawl.prefsSet(TARGET_CRAWL_DEFAULTS_PREF, 1);
if (
saved.followCanonicals !== false &&
saved.crawlHreflang !== false &&
saved.crawlLinkedSitemaps !== false
) {
return saved;
}
const upgraded: Partial<CrawlConfig> = {
...saved,
followCanonicals: true,
crawlHreflang: true,
crawlLinkedSitemaps: true,
};
window.freecrawl.prefsSet('crawl-config', upgraded);
targetCrawlDefaultsApplied = true;
return upgraded;
}
/**
* Debounced persistence for config edits. `setConfig` updates the in-memory
* Zustand state synchronously so the UI stays instant, but the two persistence
@@ -537,8 +582,12 @@ if (typeof window !== 'undefined') {
window.addEventListener('beforeunload', flushConfigPersist);
}
const initialConfig = loadInitialConfig();
export const useAppStore = create<AppState>((set, get) => ({
config: loadInitialConfig(),
config: initialConfig,
configNotice: targetCrawlDefaultsApplied ? 'target-crawl-defaults' : null,
dismissConfigNotice: () => set({ configNotice: null }),
progress: null,
summary: null,
overview: null,
packages/core/src/crawler.ts
+106 -1
View File
@@ -11,7 +11,12 @@ import type {
DiscoveredLink,
Indexability,
} from '@freecrawl/shared-types';
import { type ProjectDb, type ProbeWrite, EXPENSIVE_ISSUE_DEFINITIONS } from '@freecrawl/db';
import {
type ProjectDb,
type ProbeWrite,
EXPENSIVE_ISSUE_DEFINITIONS,
SOFT404_PROBE_META_KEY,
} from '@freecrawl/db';
import {
normalizeUrl,
isSameHost,
@@ -1172,6 +1177,9 @@ export class Crawler extends EventEmitter {
this.db.reset();
this.db.setMeta('startUrl', start);
}
// The missing-page probe describes the crawl that is starting; it is
// re-run at the end (a stopped crawl simply has no verdict).
if (!isRespider) this.db.setMeta(SOFT404_PROBE_META_KEY, '');
const isRefresh = isSameSite && !isRespider && !this.resumeOnly;
// Change markers describe the run that is starting now, so anything
// a previous refresh flagged is cleared first. A wiped project has
@@ -1590,6 +1598,9 @@ export class Crawler extends EventEmitter {
this.setOp('post-crawl:pdf-probes');
await this.runPdfMetadataProbes();
await yieldToEventLoop();
this.setOp('post-crawl:soft-404-probe');
await this.runSoft404Probe();
await yieldToEventLoop();
this.setOp('post-crawl:performance-budget');
this.runBudgetPass();
if (this.config.analyseIssues) {
@@ -1935,6 +1946,99 @@ export class Crawler extends EventEmitter {
);
}
/**
* Site-level soft-404 check. Asks for one page that cannot exist and
* looks at the answer: a 2xx (directly or after redirects) means every
* broken link on the site also "works" — search engines file those as
* soft 404s, and the 4xx reports here stay empty however many links are
* dead. One request, never stored as a URL row: the result is a notice
* about the site, not a page issue, so no crawled page is flagged.
*
* Only an unambiguous answer counts. A page that marks itself noindex
* (meta, `X-Robots-Tag`, or a script writing the meta — Google's advice
* for single-page apps) or moves on via meta refresh / JS redirect is
* handling the case; auth walls, bot challenges, 5xx and timeouts say
* nothing about it. Spider mode only — a list or sitemap crawl is not a
* crawl of one site.
*/
private async runSoft404Probe(): Promise<void> {
if (this.config.mode !== 'spider' || this.stopped) return;
let probeUrl: string;
try {
const start = new URL(this.config.startUrl);
if (start.protocol !== 'http:' && start.protocol !== 'https:') return;
// Resolved against the start URL, so a crawl scoped to /blog/ probes
// /blog/<token> rather than a path outside its scope.
const token = Math.random().toString(36).slice(2, 12);
probeUrl = new URL(`freecrawl-404-check-${token}`, start).href;
} catch {
return;
}
if (this.config.respectRobotsTxt && !this.robotsAllows(probeUrl)) return;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
let verdict: { status: number; finalUrl: string | null } | null = null;
try {
let url = probeUrl;
for (let hop = 0; hop <= 5; hop++) {
const res = await this.fetchWithRetry(url, controller.signal);
const status = res.status;
if (status >= 300 && status < 400) {
const location = res.headers.get('location');
await res.body?.cancel().catch(() => undefined);
if (!location) break;
url = new URL(location, url).href;
continue;
}
if (status < 200 || status >= 300) {
await res.body?.cancel().catch(() => undefined);
break;
}
const xRobots = res.headers.get('x-robots-tag');
if (
xRobots &&
xRobotsTagDirectives(xRobots, robotsUserAgentToken(this.config.userAgent)).noindex
) {
await res.body?.cancel().catch(() => undefined);
break;
}
const type = (res.headers.get('content-type') ?? '').toLowerCase();
if (type && !type.includes('html')) {
// A file served for any path is still a 2xx for a missing page.
await res.body?.cancel().catch(() => undefined);
verdict = { status, finalUrl: url === probeUrl ? null : url };
break;
}
const read = await readBodyCapped(res, this.config.maxFileSizeBytes);
if (read.overCap) break;
const page = parseHtml(new TextDecoder().decode(read.bytes), url);
if (page.hasNoindex || page.jsNoindexWrite || page.metaRefresh || page.jsRedirectUrl) break;
verdict = { status, finalUrl: url === probeUrl ? null : url };
break;
}
} catch {
/* inconclusive — timeouts and network errors say nothing about 404 handling */
} finally {
clearTimeout(timer);
}
if (!verdict) return;
this.emit(
'warn',
`Soft 404: ${probeUrl} (a page that does not exist) answered ${verdict.status}` +
(verdict.finalUrl ? ` after redirecting to ${verdict.finalUrl}` : '') +
' instead of 404 — broken links on this site look like working pages.',
);
try {
await this.dbCall<void>('setMeta', [
SOFT404_PROBE_META_KEY,
JSON.stringify({ url: probeUrl, ...verdict }),
]);
} catch {
/* the notice is best-effort; the crawl results are unaffected */
}
}
/**
* V2 Faz 16 #1 — Post-crawl social-image dimension probe. Ranged-GETs
* each distinct `og:image` / `twitter:image` (heavily deduplicated —
@@ -4588,6 +4692,7 @@ export class Crawler extends EventEmitter {
httpCharset,
metaCharset,
canonicalOutsideHead: this.config.storeCanonicals ? parsed.canonicalOutsideHead : 0,
hreflangOutsideHead: this.config.storeHreflang ? parsed.hreflangOutsideHead : 0,
contentLang: parsed.contentLang,
langMismatch: parsed.langMismatch ? 1 : 0,
soft404: parsed.soft404Signal && statusCode >= 200 && statusCode < 300 ? 1 : 0,
packages/core/src/html-parser.ts
+13 -1
View File
@@ -387,6 +387,8 @@ export interface ParsedPage {
minhash: Uint8Array | null;
/** `<link rel="canonical">` elements inside `<body>` (not honoured — see `canonical`). */
canonicalOutsideHead: number;
/** `<link rel="alternate" hreflang>` elements inside `<body>` (not honoured — see `hreflangs`). */
hreflangOutsideHead: number;
/** "Not found" wording in the title/H1 of a page with almost no content. */
soft404Signal: boolean;
/** An inline script assigns `document.title`. */
@@ -1172,10 +1174,19 @@ export function parseHtml(
const paginationPrev = paginationPrevRaw ? normalizeUrl(paginationPrevRaw, baseUrl, opts.urlRewrites) : null;
// Hreflang — `<link rel="alternate" hreflang="…" href="…">`. We dedupe
// by lang+href because some sites repeat tags accidentally.
// by lang+href because some sites repeat tags accidentally. Same rule as
// the canonical: search engines only read these links in `<head>`, so
// the ones in `<body>` are counted for their own issue and otherwise
// ignored — honouring them made the hreflang audits judge annotations
// Google never sees.
const hreflangSet = new Set<string>();
const hreflangs: HreflangEntry[] = [];
let hreflangOutsideHead = 0;
$('link[rel="alternate"][hreflang]').each((_, el) => {
if ($(el).closest('body').length > 0) {
hreflangOutsideHead++;
return;
}
const lang = ($(el).attr('hreflang') ?? '').trim();
const rawHref = ($(el).attr('href') ?? '').trim();
if (!lang || !rawHref) return;
@@ -2041,6 +2052,7 @@ export function parseHtml(
langMismatch: contentLanguage.mismatch,
minhash,
canonicalOutsideHead,
hreflangOutsideHead,
soft404Signal,
jsTitleWrite: headWrites.titleWrite,
jsTitle: headWrites.title,
packages/core/src/html-report.ts
+20 -2
View File
@@ -134,6 +134,7 @@ const ISSUES: IssueDef[] = [
{ key: 'charsetMissing', label: 'Charset Missing', severity: 'warn' },
{ key: 'charsetMismatch', label: 'Charset Mismatch (HTTP vs HTML)', severity: 'warn' },
{ key: 'canonicalOutsideHead', label: 'Canonical Outside <head>', severity: 'warn' },
{ key: 'hreflangOutsideHead', label: 'Hreflang Outside <head>', severity: 'warn' },
{ key: 'langContentMismatch', label: 'Lang ≠ Content Language', severity: 'warn' },
{ key: 'soft404', label: 'Soft 404', severity: 'warn' },
{ key: 'redirectChain', label: 'Redirect Chain (2+ hops)', severity: 'warn' },
@@ -240,6 +241,7 @@ const RECOMMENDATIONS: Partial<Record<IssueDef['key'], string>> = {
charsetMissing: 'Declare the character encoding (meta charset or Content-Type header).',
charsetMismatch: 'Make the Content-Type header charset match the meta charset — browsers obey the header.',
canonicalOutsideHead: 'Move rel=canonical into <head>; search engines ignore it in <body>.',
hreflangOutsideHead: 'Move hreflang <link> tags into <head>; search engines ignore them in <body>.',
langContentMismatch: 'Set html[lang] to the language the page is actually written in.',
soft404: 'Return 404/410 for pages that no longer exist instead of a 200 "not found" page.',
redirectChain: 'Point links and redirects straight at the final URL.',
@@ -310,7 +312,23 @@ function accentStyle(options: HtmlReportOptions): string {
return ok ? `:root{--accent:${raw}}` : '';
}
/** Site-wide findings that belong to no single URL (shown above the issue table). */
function renderSiteNotices(counts: OverviewCounts): string {
const probe = counts.site?.soft404Probe;
if (!probe) return '';
const via = probe.finalUrl
? ` after redirecting to <span class="mono">${escape(probe.finalUrl)}</span>`
: '';
return (
`<p><span class="sev-warn">WARN</span> Missing pages return ${probe.status}: ` +
`<span class="mono">${escape(probe.url)}</span>, a page that does not exist, was answered ` +
`${probe.status}${via} instead of 404. Broken links on this site look like working pages ` +
`to search engines (soft 404) — return a real 404 or 410 for missing URLs.</p>`
);
}
function renderIssues(counts: OverviewCounts): string {
const notices = renderSiteNotices(counts);
const rows = ISSUES.map((d) => {
const c = counts.issues[d.key] as number;
return { ...d, count: c };
@@ -320,7 +338,7 @@ function renderIssues(counts: OverviewCounts): string {
SEV_RANK[a.severity] - SEV_RANK[b.severity] || b.count - a.count,
);
if (rows.length === 0) {
return '<p class="muted">No issues detected. Nice.</p>';
return notices || '<p class="muted">No issues detected. Nice.</p>';
}
const body = rows
.map(
@@ -331,7 +349,7 @@ function renderIssues(counts: OverviewCounts): string {
</tr>`,
)
.join('');
return `<table><thead><tr><th>Severity</th><th>Issue</th><th class="right">URLs</th></tr></thead><tbody>${body}</tbody></table>`;
return `${notices}<table><thead><tr><th>Severity</th><th>Issue</th><th class="right">URLs</th></tr></thead><tbody>${body}</tbody></table>`;
}
export async function exportHtmlReport(
packages/db/src/index.ts
+1 -1
View File
@@ -1,4 +1,4 @@
export { ProjectDb, EXPENSIVE_ISSUE_DEFINITIONS } from './project-db.js';
export { ProjectDb, EXPENSIVE_ISSUE_DEFINITIONS, SOFT404_PROBE_META_KEY } from './project-db.js';
export type { UpsertUrlInput, ProbeWrite } from './project-db.js';
export { runMigrations } from './migrations.js';
export {
packages/db/src/migrations.ts
+14
View File
@@ -2423,6 +2423,20 @@ const MIGRATIONS: Migration[] = [
}
},
},
{
version: 98,
name: 'add_hreflang_outside_head',
// hreflang <link> elements inside <body> — ignored by search engines,
// counted for their own issue (mirrors canonical_outside_head).
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(urls)')
.all() as unknown as { name: string }[];
if (!cols.some((c) => c.name === 'hreflang_outside_head')) {
db.exec('ALTER TABLE urls ADD COLUMN hreflang_outside_head INTEGER NOT NULL DEFAULT 0');
}
},
},
];
export function runMigrations(db: DatabaseSync): void {
packages/db/src/project-db.ts
+29 -2
View File
@@ -17,6 +17,7 @@ import type {
InlinkRow,
OutlinkRow,
OverviewCounts,
Soft404ProbeResult,
PerformanceBudgetConfig,
PagespeedMetrics,
PagespeedRow,
@@ -81,6 +82,8 @@ import { runMigrations } from './migrations.js';
export const DISABLED_ISSUES_META_KEY = 'disabledIssues';
/** `meta` key mirroring `CrawlConfig.dedupePreNormalize` ('1' / '0'). */
export const DEDUPE_PRE_NORMALIZE_META_KEY = 'dedupePreNormalize';
/** JSON `Soft404ProbeResult` of the last crawl's missing-page probe; '' when clean. */
export const SOFT404_PROBE_META_KEY = 'soft404Probe';
/**
* Orphan page: an internal 2xx HTML page nothing links to. The start URL
* is exempt (nothing links to a homepage from inside the site by
@@ -218,6 +221,8 @@ interface UrlRowDb {
meta_charset: string | null;
/** `<link rel="canonical">` elements inside `<body>` — ignored by search engines. */
canonical_outside_head: number;
/** hreflang `<link>` elements inside `<body>` — ignored likewise. */
hreflang_outside_head: number;
/** Primary language the main-content prose is written in. */
content_lang: string | null;
/** 1 when `html[lang]` names a language the prose clearly isn't in. */
@@ -508,6 +513,7 @@ export interface UpsertUrlInput {
httpCharset?: string | null;
metaCharset?: string | null;
canonicalOutsideHead?: number;
hreflangOutsideHead?: number;
contentLang?: string | null;
langMismatch?: number;
soft404?: number;
@@ -699,7 +705,7 @@ const UPSERT_URL_SQL = `
heading_order_violations, subresource_request_count,
amp_page, amp_validation_errors, mobile_alternate,
http_charset, meta_charset, canonical_outside_head, content_lang, lang_mismatch,
soft_404, js_noindex, js_title_changed, js_title, minhash
soft_404, js_noindex, js_title_changed, js_title, minhash, hreflang_outside_head
) VALUES (
:url, :content_kind, :status_code, :status_text, :indexability, :indexability_reason,
:title, :title_length, :meta_description, :meta_description_length,
@@ -744,7 +750,7 @@ const UPSERT_URL_SQL = `
:heading_order_violations, :subresource_request_count,
:amp_page, :amp_validation_errors, :mobile_alternate,
:http_charset, :meta_charset, :canonical_outside_head, :content_lang, :lang_mismatch,
:soft_404, :js_noindex, :js_title_changed, :js_title, :minhash
:soft_404, :js_noindex, :js_title_changed, :js_title, :minhash, :hreflang_outside_head
)
ON CONFLICT(url) DO UPDATE SET
content_kind = excluded.content_kind,
@@ -890,6 +896,7 @@ const UPSERT_URL_SQL = `
http_charset = excluded.http_charset,
meta_charset = excluded.meta_charset,
canonical_outside_head = excluded.canonical_outside_head,
hreflang_outside_head = excluded.hreflang_outside_head,
content_lang = excluded.content_lang,
lang_mismatch = excluded.lang_mismatch,
soft_404 = excluded.soft_404,
@@ -2222,6 +2229,7 @@ export class ProjectDb {
http_charset: input.httpCharset ?? null,
meta_charset: input.metaCharset ?? null,
canonical_outside_head: input.canonicalOutsideHead ?? 0,
hreflang_outside_head: input.hreflangOutsideHead ?? 0,
content_lang: input.contentLang ?? null,
lang_mismatch: input.langMismatch ?? 0,
soft_404: input.soft404 ?? 0,
@@ -6461,9 +6469,22 @@ export class ProjectDb {
blockedRobots: countCategory('indexability:blocked-robots'),
},
issues: this.silenceDisabledIssues(this.getIssuesCounts()),
site: { soft404Probe: this.readSoft404Probe() },
};
}
private readSoft404Probe(): Soft404ProbeResult | null {
const raw = this.getMeta(SOFT404_PROBE_META_KEY);
if (!raw) return null;
try {
const v = JSON.parse(raw) as Partial<Soft404ProbeResult>;
if (typeof v.url !== 'string' || typeof v.status !== 'number') return null;
return { url: v.url, status: v.status, finalUrl: typeof v.finalUrl === 'string' ? v.finalUrl : null };
} catch {
return null;
}
}
private getIssuesCounts(): OverviewCounts['issues'] {
const countWhere = (clause: string): number => this.countUrlsWhere(clause);
// Materialised counters fan-out (I-3). Heavy issue checks read
@@ -6691,6 +6712,7 @@ export class ProjectDb {
AND http_charset != meta_charset`,
),
canonicalOutsideHead: countWhere(`${html} AND canonical_outside_head > 0`),
hreflangOutsideHead: countWhere(`${html} AND hreflang_outside_head > 0`),
langContentMismatch: countWhere(`${html} AND lang_mismatch = 1`),
soft404: countWhere(`${html} AND status_code BETWEEN 200 AND 299 AND soft_404 > 0`),
redirectChain: countWhere(
@@ -9978,6 +10000,7 @@ export class ProjectDb {
httpCharset: r.http_charset ?? null,
metaCharset: r.meta_charset ?? null,
canonicalOutsideHead: r.canonical_outside_head ?? 0,
hreflangOutsideHead: r.hreflang_outside_head ?? 0,
contentLang: r.content_lang ?? null,
langMismatch: (r.lang_mismatch ?? 0) === 1,
soft404: r.soft_404 ?? 0,
@@ -11070,6 +11093,10 @@ export function categoryWhereClause(
// `rel=canonical` inside <body> is ignored by search engines — the
// page has no canonical as far as they're concerned.
return "is_external = 0 AND content_kind = 'html' AND canonical_outside_head > 0";
case 'issues:hreflang-outside-head':
// Same as above for hreflang: links in <body> are never read, so the
// page has no alternates as far as search engines are concerned.
return "is_external = 0 AND content_kind = 'html' AND hreflang_outside_head > 0";
case 'issues:lang-content-mismatch':
// html[lang] names a language the main content clearly isn't in
// (different writing system, or none of its function words while
packages/shared-types/src/crawl.ts
+22
View File
@@ -118,6 +118,7 @@ export type UrlCategory =
| 'issues:charset-missing'
| 'issues:charset-mismatch'
| 'issues:canonical-outside-head'
| 'issues:hreflang-outside-head'
| 'issues:lang-content-mismatch'
| 'issues:soft-404'
| 'issues:redirect-chain'
@@ -597,6 +598,8 @@ export interface CrawlUrlRow {
metaCharset: string | null;
/** `<link rel="canonical">` elements placed inside `<body>` — search engines ignore them. */
canonicalOutsideHead: number;
/** `<link rel="alternate" hreflang>` elements placed inside `<body>` — ignored likewise. */
hreflangOutsideHead: number;
/** Primary language the main-content prose is written in (`tr`, `en`, …); null when unclear. */
contentLang: string | null;
/** `html[lang]` declares a language the prose is clearly not written in. */
@@ -1775,7 +1778,24 @@ export interface ExtractionRulesImportResult {
error?: string;
}
/**
* Site-level soft-404 check: what the server answered for a URL that cannot
* exist. Recorded only when that answer was a 2xx page that didn't mark
* itself noindex — i.e. a missing page looks like a real one, so a broken
* link on the site never shows up as a 404.
*/
export interface Soft404ProbeResult {
/** The made-up URL that was requested. */
url: string;
/** The 2xx status it was answered with. */
status: number;
/** Where redirects ended when the request was redirected, else null. */
finalUrl: string | null;
}
export interface OverviewCounts {
/** Findings about the site as a whole rather than any one URL. */
site?: { soft404Probe: Soft404ProbeResult | null };
summary: {
totalInternalUrls: number;
totalIndexable: number;
@@ -1894,6 +1914,8 @@ export interface OverviewCounts {
charsetMismatch: number;
/** `<link rel="canonical">` placed inside `<body>`, where search engines ignore it. */
canonicalOutsideHead: number;
/** hreflang `<link>` elements placed inside `<body>`, where search engines ignore them. */
hreflangOutsideHead: number;
/** `html[lang]` declares a language the main content isn't written in. */
langContentMismatch: number;
/** 2xx pages that present themselves as "not found". */
packages/shared-types/src/issue-catalog.ts
+1
View File
@@ -326,6 +326,7 @@ export const ISSUE_GROUPS: readonly IssueGroupDef[] = [
{ key: 'issues-hreflang-target-issues', label: 'Target Issues', category: 'issues:hreflang-target-issues', count: 'hreflangTargetIssues' },
{ key: 'issues-hreflang-unlinked', label: 'Unlinked Targets', category: 'issues:hreflang-unlinked', count: 'hreflangUnlinked' },
{ key: 'issues-hreflang-inconsistent-lang', label: 'Inconsistent Lang (same lang, two hrefs)', category: 'issues:hreflang-inconsistent-lang', count: 'hreflangInconsistentLang' },
{ key: 'issues-hreflang-outside-head', label: 'Hreflang Outside <head>', category: 'issues:hreflang-outside-head', count: 'hreflangOutsideHead' },
],
},
{
packages/shared-types/src/issue-severity.ts
+1
View File
@@ -89,6 +89,7 @@ const WARNING: UrlCategory[] = [
'issues:redirect-chain',
'issues:charset-mismatch',
'issues:canonical-outside-head',
'issues:hreflang-outside-head',
'issues:lang-content-mismatch',
'issues:soft-404',
'issues:blocked-by-robots',