feat: offline spell check + local language detection + single-file projects

This commit is contained in:
kemalai committed 2026-07-23 20:53:28 +03:00
1 parent 055d730af0
commit 08b423dd60
51 files changed
+5200 -557

No files matched your search

+42 -3
View File
@@ -23,21 +23,48 @@ jobs:
include:
- os: windows-latest
build_script: build:win
build_args: ''
artifact_name: windows-installer
artifact_paths: |
apps/desktop/release/*.exe
apps/desktop/release/*.exe.blockmap
apps/desktop/release/latest.yml
# macOS is built once per architecture rather than both arches
# in one job. The bundled Playwright browsers are arch-specific
# (`chrome-mac-arm64` vs `chrome-mac-x64`), so a single job on
# an Apple-silicon runner used to ship arm64 binaries inside the
# Intel .dmg — where Playwright looked for `chrome-mac-x64`,
# found nothing, and every Intel user landed on the "browser
# missing" dialog. Each job now downloads the browsers for the
# architecture it is packaging.
- os: macos-latest
build_script: build:mac
artifact_name: macos-installer
build_args: '-- --arm64'
artifact_name: macos-installer-arm64
artifact_paths: |
apps/desktop/release/*arm64*.dmg
apps/desktop/release/*arm64*.dmg.blockmap
apps/desktop/release/*arm64*.zip
apps/desktop/release/latest-mac.yml
- os: macos-latest
build_script: build:mac
build_args: '-- --x64'
# Cross-download the Intel browsers on the arm64 runner.
# Playwright picks its download by host platform, so this
# override is what makes the x64 bundle actually x64.
playwright_host_platform: mac15
artifact_name: macos-installer-x64
# latest-mac.yml is published by the arm64 job only — both
# jobs would otherwise upload a same-named file describing
# just their own arch, and the winner would be whichever
# finished last.
artifact_paths: |
apps/desktop/release/*.dmg
apps/desktop/release/*.dmg.blockmap
apps/desktop/release/*.zip
apps/desktop/release/latest-mac.yml
- os: ubuntu-latest
build_script: build:linux
build_args: ''
artifact_name: linux-installer
artifact_paths: |
apps/desktop/release/*.AppImage
@@ -55,6 +82,11 @@ jobs:
- name: Install dependencies
run: npm ci
env:
# Skip the postinstall browser download on cross-arch jobs so
# the cache holds only the target architecture — otherwise the
# installer would carry both (~+180 MB of dead weight).
PLAYWRIGHT_SKIP_BROWSER_INSTALL: ${{ matrix.playwright_host_platform && '1' || '0' }}
- name: Build shared packages
run: npx tsc -b
@@ -66,9 +98,11 @@ jobs:
# re-check if Playwright was upgraded since the lockfile was
# written.
run: npm run playwright:install
env:
PLAYWRIGHT_HOST_PLATFORM_OVERRIDE: ${{ matrix.playwright_host_platform }}
- name: Build installer (with bundled Playwright browsers)
run: npm --workspace apps/desktop run ${{ matrix.build_script }}
run: npm --workspace apps/desktop run ${{ matrix.build_script }} ${{ matrix.build_args }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Disable mac code-signing — we don't have an Apple Developer
@@ -82,6 +116,11 @@ jobs:
# apps/desktop/resources/playwright-browsers/ before
# electron-builder packs the installer.
BUNDLE_PLAYWRIGHT: '1'
# Also read by prepare-playwright-bundle.mjs, which refuses to
# package a browser cache that doesn't match the architecture
# being built — the exact mismatch that shipped an arm64
# Chromium inside the Intel .dmg.
PLAYWRIGHT_HOST_PLATFORM_OVERRIDE: ${{ matrix.playwright_host_platform }}
- name: Upload build artifacts
uses: actions/upload-artifact@v4
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.5 MiB

After

Width:  |  Height:  |  Size: 17 MiB

+2 -2
View File
@@ -33,9 +33,9 @@ FreeCrawl crawls your website and checks it for SEO problems — a free, desktop
It runs **150+ SEO checks across 32 tabs** — page titles, meta descriptions, headings, duplicate content, broken links, redirects, canonicals, hreflang, structured data, security headers, readability, and more. It can also render JavaScript pages with a real headless browser to capture the final DOM, screenshots, and Core Web Vitals.
Pull in real data from **Google Search Console, Analytics 4, PageSpeed Insights, and Chrome UX Report**, check spelling and grammar with **LanguageTool**, and run your own **AI prompts** on each page (OpenAI, Claude, or Ollama). Open several projects side by side, each in its own window running its own crawl, and manage them from a Projects dialog with search, tags, and starter templates.
Pull in real data from **Google Search Console, Analytics 4, PageSpeed Insights, and Chrome UX Report**, check spelling and grammar with **LanguageTool** — with a **bundled offline dictionary** for languages LanguageTool has no rules for, such as Turkish — and run your own **AI prompts** on each page (OpenAI, Claude, or Ollama). Open several projects side by side, each in its own window running its own crawl, and manage them from a Projects dialog with search, tags, and starter templates.
Export to **Excel, CSV, JSON, XML**, an HTML report, sitemaps, **Google Sheets**, or **BigQuery**, and save projects as **password-protected encrypted files**. A built-in **MCP server** (96 tools) lets AI agents like Claude drive crawls live and read every result — every action a human can take in the desktop is callable from an agent. Everything runs **fully local** — no telemetry, no cloud, MIT-licensed.
Export to **Excel, CSV, JSON, XML**, an HTML report, sitemaps, **Google Sheets**, or **BigQuery**, save each project as **one compressed file** (or a **password-protected encrypted file**), and export any URL Details sub-table on its own. A built-in **MCP server** (96 tools) lets AI agents like Claude drive crawls live and read every result — every action a human can take in the desktop is callable from an agent. Everything runs **fully local** — no telemetry, no cloud, MIT-licensed.
<br />
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/cli",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"type": "module",
"bin": {
+4
View File
@@ -14,6 +14,10 @@ files:
- package.json
- "!**/*.map"
- "!**/*.ts"
# The language detector loads only `eld/large`; the other three n-gram
# databases are ~4.5 MB of dead weight in the installer. ELD is a module
# singleton — one database per process — so the rest can never be reached.
- "!node_modules/eld/src/ngrams/{extrasmall,small,medium}.js"
asar: true
asarUnpack:
- "**/*.node"
+6 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/desktop",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"description": "FreeCrawl SEO — desktop application",
"homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
@@ -40,8 +40,12 @@
"cytoscape": "^3.33.2",
"cytoscape-dagre": "^2.5.0",
"dagre": "^0.8.5",
"dictionary-tr": "^2.0.0",
"eld": "^2.0.3",
"franc": "^6.2.0",
"i18next": "^26.2.0",
"lucide-react": "^0.544.0",
"nspell": "^2.1.5",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"react-i18next": "^17.0.8",
@@ -52,6 +56,7 @@
"devDependencies": {
"@types/cytoscape-dagre": "^2.3.4",
"@types/dagre": "^0.7.54",
"@types/nspell": "^2.1.6",
"@types/react": "^19.2.0",
"@types/react-dom": "^19.2.0",
"@vitejs/plugin-react": "^5.0.0",
Binary file not shown.
Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

@@ -16,9 +16,9 @@
* Idempotent: safe to run multiple times.
*/
import { cpSync, existsSync, mkdirSync, rmSync } from 'node:fs';
import { cpSync, existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { homedir, platform } from 'node:os';
import { cpus, homedir, platform } from 'node:os';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
@@ -38,6 +38,61 @@ function sourceCacheDir() {
return join(homedir(), '.cache', 'ms-playwright');
}
/**
* Per-platform folder names Playwright looks for inside a
* `chromium-<rev>` / `chromium_headless_shell-<rev>` directory. These are
* arch-specific, which is the entire reason this check exists: bundling
* an `chrome-mac-arm64` build into the Intel .dmg produced an installer
* that shipped 250 MB of browser the app could never use.
*
* `PLAYWRIGHT_HOST_PLATFORM_OVERRIDE` is how CI cross-downloads the
* Intel browsers on an Apple-silicon runner, so honour it here too.
*/
function expectedFolders() {
const override = process.env.PLAYWRIGHT_HOST_PLATFORM_OVERRIDE ?? '';
const isArm = override
? override.endsWith('arm64')
: platform() === 'darwin'
? cpus().some((c) => c.model.includes('Apple'))
: process.arch === 'arm64';
const os = override
? override.startsWith('mac')
? 'darwin'
: override.startsWith('win')
? 'win32'
: 'linux'
: platform();
if (os === 'win32') {
return { chromium: 'chrome-win64', headlessShell: 'chrome-headless-shell-win64' };
}
if (os === 'darwin') {
const arch = isArm ? 'arm64' : 'x64';
return {
chromium: `chrome-mac-${arch}`,
headlessShell: `chrome-headless-shell-mac-${arch}`,
};
}
if (isArm) return { chromium: 'chrome-linux', headlessShell: 'chrome-linux' };
return { chromium: 'chrome-linux64', headlessShell: 'chrome-headless-shell-linux64' };
}
/** True when `dir` holds a Chromium build for the platform being packaged. */
function hasUsableChromium(dir) {
const folders = expectedFolders();
if (!existsSync(dir)) return false;
const entries = readdirSync(dir);
const hasFull = entries.some(
(e) => e.startsWith('chromium-') && existsSync(join(dir, e, folders.chromium)),
);
const hasShell = entries.some(
(e) =>
e.startsWith('chromium_headless_shell-') &&
existsSync(join(dir, e, folders.headlessShell)),
);
return hasFull && hasShell;
}
if (process.env.BUNDLE_PLAYWRIGHT !== '1') {
// Bundling disabled — wipe any leftover from a previous build so the
// small-installer default produces a small file.
@@ -62,6 +117,18 @@ if (!existsSync(src)) {
process.exit(1);
}
const folders = expectedFolders();
if (!hasUsableChromium(src)) {
console.error(
`[prepare-playwright-bundle] The cache at ${src} has no Chromium build for the ` +
`platform being packaged (expected ${folders.chromium} + ${folders.headlessShell}).\n` +
'Bundling it would ship an installer whose browser this machine cannot launch.\n' +
'Run "npm run playwright:install" for the target platform first — cross-arch ' +
'builds need PLAYWRIGHT_HOST_PLATFORM_OVERRIDE set to the target (e.g. mac15).',
);
process.exit(1);
}
console.log(`[prepare-playwright-bundle] Copying ${src} → ${targetDir} …`);
rmSync(targetDir, { recursive: true, force: true });
mkdirSync(targetDir, { recursive: true });
@@ -70,4 +137,14 @@ mkdirSync(targetDir, { recursive: true });
// dereferenced, doubling the on-disk size and breaking the bundle's
// Mach-O loader paths.
cpSync(src, targetDir, { recursive: true, verbatimSymlinks: true });
console.log('[prepare-playwright-bundle] Done.');
if (!hasUsableChromium(targetDir)) {
console.error(
'[prepare-playwright-bundle] Copy finished but the bundled cache is not usable — aborting ' +
'rather than publishing an installer whose JS rendering is dead on arrival.',
);
process.exit(1);
}
console.log(
`[prepare-playwright-bundle] Done — bundled ${folders.chromium} + ${folders.headlessShell}.`,
);
Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

File diff suppressed because it is too large. Load diff
+671
View File
@@ -0,0 +1,671 @@
/**
* Page-language resolution for the spelling checker.
*
* LanguageTool's `auto` mode cannot be trusted for this job: it only knows
* the ~33 languages it ships rules for, so it answers with the *closest*
* supported language no matter what it is handed, at a confidence that is
* always ~0.99. Turkish prose comes back as "English (US), 0.99" with a
* misspelling reported on every single word; a longer sample comes back as
* "Crimean Tatar". Neither is recoverable downstream — the findings look
* like real data.
*
* So the language is decided here, before anything leaves the machine:
*
* 1. Fingerprint the prose with two independent detectors (see
* `detectLanguageRanking`).
* 2. Reconcile that against the page's declared `html[lang]`. The
* declaration wins when the two agree or are near-neighbours (pt vs
* gl, nl vs af, en-GB vs en-US) so the regional variant is preserved;
* the detector wins when they flatly disagree, because a stale
* `lang="en-US"` left in a CMS theme on an otherwise Turkish site is
* far more common than a page written in a language it doesn't claim.
* 3. Validate the result against what the endpoint actually offers, and
* refuse to check rather than substituting a wrong language.
*/
import { francAll } from 'franc';
import type { SpellingLanguageOption } from '@freecrawl/shared-types';
import * as logger from './logger.js';
/**
* Below this many characters of prose detection is guessing. Benchmarked
* over 161 native Wikipedia samples across 41 languages, scored on the
* decision this module actually makes (right LanguageTool language, or a
* correct refusal): ELD reads 83.9% at 50 characters, where franc alone
* managed 70.8% even at 60. Pages under the bar fall back to `html[lang]`.
*
* Prose shorter than 40 characters never reaches here — `MIN_PROSE_CHARS`
* in the checker skips those pages outright — so the band this actually
* governs is 40–60 characters of body copy.
*/
const MIN_DETECT_CHARS = 50;
/**
* How close a runner-up must score to the winner before a *declared*
* language is allowed to override it. franc normalises the top hit to
* 1.00, so this is a relative distance. Portuguese prose scores
* `glg:1.00 por:0.95`; a page declaring `pt` should stay Portuguese.
*/
const CLOSE_CALL = 0.9;
/** How deep into franc's ranking a declared language may be rescued. */
const CLOSE_CALL_DEPTH = 6;
/**
* ISO 639-3 (what franc returns) → ISO 639-1 (what LanguageTool speaks).
* Covers every language LanguageTool supports plus the widely-used ones it
* does not, so `unsupported` results can name the language they found
* instead of reporting a bare code.
*/
const ISO3_TO_ISO1: Record<string, string> = {
// — LanguageTool-supported —
arb: 'ar',
ara: 'ar',
ast: 'ast',
bel: 'be',
bre: 'br',
cat: 'ca',
crh: 'crh',
dan: 'da',
deu: 'de',
ell: 'el',
eng: 'en',
epo: 'eo',
spa: 'es',
pes: 'fa',
fas: 'fa',
fra: 'fr',
gle: 'ga',
glg: 'gl',
ita: 'it',
jpn: 'ja',
khm: 'km',
nob: 'nb',
nno: 'nn',
nor: 'no',
nld: 'nl',
pol: 'pl',
por: 'pt',
ron: 'ro',
rus: 'ru',
slk: 'sk',
slv: 'sl',
swe: 'sv',
tam: 'ta',
tgl: 'tl',
ukr: 'uk',
cmn: 'zh',
zho: 'zh',
yue: 'zh',
// — common, and *not* supported by LanguageTool —
tur: 'tr',
azj: 'az',
aze: 'az',
tuk: 'tk',
uzn: 'uz',
kaz: 'kk',
kir: 'ky',
hun: 'hu',
ces: 'cs',
fin: 'fi',
est: 'et',
lvs: 'lv',
lav: 'lv',
lit: 'lt',
heb: 'he',
hin: 'hi',
urd: 'ur',
ben: 'bn',
pan: 'pa',
mar: 'mr',
tel: 'te',
guj: 'gu',
kan: 'kn',
mal: 'ml',
tam_: 'ta',
ind: 'id',
zsm: 'ms',
zlm: 'ms',
vie: 'vi',
tha: 'th',
kor: 'ko',
bul: 'bg',
srp: 'sr',
hrv: 'hr',
bos: 'bs',
mkd: 'mk',
sqi: 'sq',
als: 'sq',
isl: 'is',
kat: 'ka',
hye: 'hy',
afr: 'af',
swh: 'sw',
amh: 'am',
npi: 'ne',
sin: 'si',
mya: 'my',
lao: 'lo',
ckb: 'ku',
kmr: 'ku',
pbu: 'ps',
hat: 'ht',
ltz: 'lb',
cym: 'cy',
eus: 'eu',
mlt: 'mt',
yid: 'yi',
};
/** Display names for the codes above — used in user-facing messages. */
const LANG_NAMES: Record<string, string> = {
af: 'Afrikaans',
am: 'Amharic',
ar: 'Arabic',
ast: 'Asturian',
az: 'Azerbaijani',
be: 'Belarusian',
bg: 'Bulgarian',
bn: 'Bengali',
br: 'Breton',
bs: 'Bosnian',
ca: 'Catalan',
crh: 'Crimean Tatar',
cs: 'Czech',
cy: 'Welsh',
da: 'Danish',
de: 'German',
el: 'Greek',
en: 'English',
eo: 'Esperanto',
es: 'Spanish',
et: 'Estonian',
eu: 'Basque',
fa: 'Persian',
fi: 'Finnish',
fr: 'French',
ga: 'Irish',
gl: 'Galician',
gu: 'Gujarati',
he: 'Hebrew',
hi: 'Hindi',
hr: 'Croatian',
ht: 'Haitian Creole',
hu: 'Hungarian',
hy: 'Armenian',
id: 'Indonesian',
is: 'Icelandic',
it: 'Italian',
ja: 'Japanese',
ka: 'Georgian',
kk: 'Kazakh',
km: 'Khmer',
kn: 'Kannada',
ko: 'Korean',
ku: 'Kurdish',
ky: 'Kyrgyz',
lb: 'Luxembourgish',
lo: 'Lao',
lt: 'Lithuanian',
lv: 'Latvian',
mk: 'Macedonian',
ml: 'Malayalam',
mr: 'Marathi',
ms: 'Malay',
mt: 'Maltese',
my: 'Burmese',
nb: 'Norwegian Bokmål',
ne: 'Nepali',
nl: 'Dutch',
nn: 'Norwegian Nynorsk',
no: 'Norwegian',
pa: 'Punjabi',
pl: 'Polish',
ps: 'Pashto',
pt: 'Portuguese',
ro: 'Romanian',
ru: 'Russian',
si: 'Sinhala',
sk: 'Slovak',
sl: 'Slovenian',
sq: 'Albanian',
sr: 'Serbian',
sv: 'Swedish',
sw: 'Swahili',
ta: 'Tamil',
te: 'Telugu',
th: 'Thai',
tk: 'Turkmen',
tl: 'Tagalog',
tr: 'Turkish',
uk: 'Ukrainian',
ur: 'Urdu',
uz: 'Uzbek',
vi: 'Vietnamese',
yi: 'Yiddish',
zh: 'Chinese',
};
/** Human name for a language code, falling back to the code itself. */
export function languageName(code: string | null): string {
if (!code) return 'unknown';
const primary = code.split('-')[0]?.toLowerCase() ?? '';
return LANG_NAMES[primary] ?? code;
}
/**
* Which concrete variant to prefer when a page only says `en` / `de` /
* `pt`. LanguageTool's bare codes exist but carry the weaker generic
* dictionary, so a real variant produces better findings.
*/
const PREFERRED_VARIANT: Record<string, string> = {
en: 'en-US',
de: 'de-DE',
pt: 'pt-PT',
ca: 'ca-ES',
zh: 'zh-CN',
nl: 'nl-NL',
fr: 'fr-FR',
es: 'es-ES',
no: 'nb-NO',
};
/** Normalise a raw `html[lang]` to its primary subtag, or null if unusable. */
export function declaredPrimary(pageLang: string | null): string | null {
const raw = (pageLang ?? '').trim().toLowerCase();
if (!raw) return null;
const primary = raw.split(/[-_]/)[0] ?? '';
// `x-default` (an hreflang value that leaks into lang attributes),
// `und`, and single letters carry no information.
if (primary.length < 2 || primary === 'x' || primary === 'und') return null;
if (!/^[a-z]{2,3}$/.test(primary)) return null;
return primary;
}
/** Full declared tag normalised to LanguageTool's casing, e.g. `en-GB`. */
function declaredLongCode(pageLang: string | null): string | null {
const raw = (pageLang ?? '').trim();
if (!raw) return null;
const parts = raw.split(/[-_]/);
const primary = parts[0]?.toLowerCase() ?? '';
const region = parts[1];
if (!primary || !region) return null;
return `${primary}-${region.toUpperCase()}`;
}
export interface DetectionRank {
/** ISO 639-1 (or 639-3 where no 639-1 exists) primary code. */
code: string;
/** Relative confidence — the winner is always 1. */
score: number;
}
export interface DetectionResult {
/** Candidate languages, best first. Empty when the sample is too short. */
ranks: DetectionRank[];
/**
* Whether the detectors actually recognised the text, as opposed to
* returning the nearest thing they know. False means the reading is too
* weak to overrule an explicit `html[lang]`.
*/
confident: boolean;
}
/**
* ELD's `eld/large` database, loaded on first use.
*
* Deliberately lazy: it is ~4.5 MB of n-gram tables and a spelling run is
* on-demand, so paying for it at app start would slow every launch for a
* feature most sessions never touch.
*
* ELD exports a module singleton and each size entry loads into the same
* object, so importing more than one silently leaves whichever landed last
* — `eld/large` is imported here and nowhere else.
*/
type EldDetector = {
detect: (text: string) => {
language: string;
getScores: () => Record<string, number>;
isReliable: () => boolean;
};
};
let eldPromise: Promise<EldDetector | null> | null = null;
function loadEld(): Promise<EldDetector | null> {
eldPromise ??= import('eld/large')
.then((m) => m.eld as EldDetector)
.catch((err: unknown) => {
logger.log(
'warn',
'languagetool',
`ELD language model failed to load (${
err instanceof Error ? err.message : String(err)
}) — falling back to franc alone`,
);
return null;
});
return eldPromise;
}
/**
* Start loading the detector without waiting for it. Called when a run is
* being set up so the model arrives while the endpoint's language list is
* still in flight, rather than stalling the first page of the batch.
*/
export function warmLanguageDetector(): void {
void loadEld();
}
/** Rescale a score map so the best entry is 1, matching franc's convention. */
function normaliseScores(scores: Record<string, number>): Map<string, number> {
const out = new Map<string, number>();
let top = 0;
for (const v of Object.values(scores)) if (v > top) top = v;
if (top <= 0) return out;
for (const [code, v] of Object.entries(scores)) {
const primary = code.split(/[-_]/)[0]?.toLowerCase();
if (primary) out.set(primary, v / top);
}
return out;
}
/**
* Fingerprint prose with both detectors and merge them into one ranking.
* Returns an empty ranking when the sample is too short to be meaningful
* rather than guessing.
*
* Two engines because each covers the other's blind spot, measured over
* 161 native Wikipedia samples in 41 languages:
*
* - **ELD leads.** On the decision that matters here it scored 97.5% /
* 96.3% / 91.3% at ~1000 / 150 / 70 characters, against franc's 96.9% /
* 91.3% / 72.7%, and runs ~7× faster. Short page copy is where franc
* fell apart, and short page copy is common.
* - **franc stays for coverage.** ELD knows 60 languages and cannot name
* seven that LanguageTool supports — Galician, Irish, Esperanto, Khmer,
* Asturian, Breton, Crimean Tatar. Handed Galician it answers
* Portuguese or Spanish, which is precisely the wrong-language check
* this module exists to prevent. franc knows all seven.
*
* So ELD's answer takes the top slot, while franc's ranking is preserved
* behind it. That placement is what makes the rescue work: a genuinely
* Galician page declaring `lang="gl"` has `gl` sitting in the ranking at
* franc's full score, and `resolveCheckLanguage`'s close-call reconciliation
* promotes it. A Portuguese page declaring `lang="pt"` keeps Portuguese,
* because ELD won the top slot and the declaration agrees with it.
*
* Note that a bigger model is *not* the same as a better one: `franc-all`,
* with 414 languages instead of 187, scored worse at every length (90.7% /
* 77.0% / 51.6% raw accuracy) because the extra candidates let it prefer
* Gagauz over Turkish and Ladino over Spanish.
*/
export async function detectLanguageRanking(
text: string,
): Promise<DetectionResult> {
const sample = text.trim();
if (sample.length < MIN_DETECT_CHARS) {
return { ranks: [], confident: false };
}
const francScores = new Map<string, number>();
for (const [iso3, score] of francAll(sample, {
minLength: MIN_DETECT_CHARS,
})) {
if (iso3 === 'und') continue;
const code = ISO3_TO_ISO1[iso3];
if (!code || francScores.has(code)) continue;
francScores.set(code, score);
}
const eld = await loadEld();
let eldTop: string | null = null;
let eldScores = new Map<string, number>();
// With no ELD at all (load failure) franc is the only opinion available,
// so its answer is taken at face value as it was before ELD existed.
let confident = true;
if (eld) {
try {
const res = eld.detect(sample);
const primary = res.language.split(/[-_]/)[0]?.toLowerCase();
eldScores = normaliseScores(res.getScores());
confident = res.isReliable();
// An unreliable verdict is a guess, and a guess must not outrank the
// broader model. Measured: where ELD reports unreliable its accuracy
// collapses from ~91% to 25%, and it reports unreliable exactly on
// the languages it has no model for — Irish prose comes back
// "Romanian, top score 0.45, second 0.42".
if (primary && confident) eldTop = primary;
} catch {
// A detector failure must not cost the page its check — franc alone
// still produces a usable ranking.
}
}
// A language either engine rates highly stays a candidate; the merged
// score is the better of the two so a close-call rescue can fire on
// either engine's evidence.
const merged = new Map<string, number>(francScores);
for (const [code, score] of eldScores) {
merged.set(code, Math.max(merged.get(code) ?? 0, score));
}
if (eldTop) merged.set(eldTop, 1);
const ranks = [...merged]
.map(([code, score]) => ({ code, score }))
.sort((a, b) => {
if (eldTop) {
if (a.code === eldTop) return -1;
if (b.code === eldTop) return 1;
}
return b.score - a.score;
})
.slice(0, 12);
return { ranks, confident };
}
/** Index the endpoint's `/v2/languages` payload for lookup. */
export interface SupportedLanguages {
/** Every concrete code the endpoint accepts, e.g. `en-GB`, `pl-PL`. */
longCodes: Set<string>;
/** Primary subtags with at least one usable variant, e.g. `en`, `pl`. */
primaries: Set<string>;
/** Best concrete code per primary subtag. */
bestByPrimary: Map<string, string>;
options: SpellingLanguageOption[];
}
export function indexSupportedLanguages(
options: SpellingLanguageOption[],
): SupportedLanguages {
const longCodes = new Set<string>();
const primaries = new Set<string>();
const byPrimary = new Map<string, string[]>();
for (const opt of options) {
const longCode = (opt.longCode || opt.code || '').trim();
if (!longCode) continue;
longCodes.add(longCode);
const primary = (longCode.split('-')[0] ?? '').toLowerCase();
if (!primary) continue;
primaries.add(primary);
const list = byPrimary.get(primary);
if (list) list.push(longCode);
else byPrimary.set(primary, [longCode]);
}
const bestByPrimary = new Map<string, string>();
for (const [primary, codes] of byPrimary) {
const preferred = PREFERRED_VARIANT[primary];
if (preferred && codes.includes(preferred)) {
bestByPrimary.set(primary, preferred);
continue;
}
// A concrete regional variant beats the bare code; `x-simple-language`
// and similar pseudo-variants are never a sensible default.
const regional = codes.find(
(c) => c.includes('-') && !c.includes('-x-') && c.length <= 6,
);
bestByPrimary.set(primary, regional ?? codes[0] ?? primary);
}
return { longCodes, primaries, bestByPrimary, options };
}
export interface LanguageDecision {
/** LanguageTool code to send. Null means: do not make the request. */
language: string | null;
/** Primary code read out of the prose (null when undetectable). */
detected: string | null;
/** Primary code declared by `html[lang]` (null when absent/unusable). */
declared: string | null;
/** Populated only when `language` is null. */
reason: 'unsupported' | 'undetermined' | null;
/** User-facing explanation for `reason`. */
message: string | null;
/** True when the declaration and the prose disagree outright. */
mismatch: boolean;
/**
* True when two independent signals — the page's own `html[lang]` and the
* trigram detector — named the same language. Callers use this to decide
* whether a noisy result means "wrong language" or "badly written page":
* with both signals agreeing, a high error rate is the finding, not a
* reason to doubt the language.
*/
agreed: boolean;
}
/**
* Decide which language to check a page in.
*
* `override` short-circuits everything — when the user pins a language in
* Settings they have told us the answer, including for sites the detector
* would read differently page by page.
*/
export async function resolveCheckLanguage(params: {
text: string;
pageLang: string | null;
supported: SupportedLanguages;
override?: string | undefined;
}): Promise<LanguageDecision> {
const { text, pageLang, supported, override } = params;
const declared = declaredPrimary(pageLang);
const { ranks: ranking, confident } = await detectLanguageRanking(text);
const detected = ranking[0]?.code ?? null;
if (override && override.trim().length > 0) {
const forced = override.trim();
if (supported.longCodes.has(forced)) {
return {
language: forced,
detected,
declared,
reason: null,
message: null,
mismatch: false,
agreed: !!detected && detected === declared,
};
}
const best = supported.bestByPrimary.get(forced.split('-')[0] ?? '');
if (best) {
return {
language: best,
detected,
declared,
reason: null,
message: null,
mismatch: false,
agreed: !!detected && detected === declared,
};
}
// A pinned language the endpoint dropped — fall through to detection
// rather than silently checking in the wrong one.
}
// Reconcile declaration against prose. `agreed` records the strongest
// case — both signals present and naming the same language — which lets
// the caller trust a noisy result instead of second-guessing it.
const agreed = !!detected && detected === declared;
let chosen: string | null;
let mismatch = false;
if (!detected) {
chosen = declared;
} else if (!declared) {
chosen = detected;
} else if (declared === detected) {
chosen = declared;
} else {
// Near-neighbour languages (pt/gl, nl/af, da/nb) are the detector's
// known weak spot — let a declaration that is still ranked highly win.
const near = ranking
.slice(0, CLOSE_CALL_DEPTH)
.find((r) => r.code === declared);
if (near && near.score >= CLOSE_CALL) {
chosen = declared;
} else if (!confident) {
// The detectors did not recognise this text; they returned the
// nearest thing they know. That is not evidence against an explicit
// declaration, so the declaration stands. Irish prose is the worked
// example: ELD has no model for it and answers "Romanian" at a flat
// 0.45, while franc does not rank Irish at all — overruling
// `lang="ga"` there would grade Irish against Romanian rules.
chosen = declared;
} else {
chosen = detected;
mismatch = true;
}
}
if (!chosen) {
return {
language: null,
detected,
declared,
reason: 'undetermined',
message:
'Page language could not be determined — it declares no html[lang] and carries too little prose to detect.',
mismatch: false,
agreed: false,
};
}
if (!supported.primaries.has(chosen)) {
return {
language: null,
detected,
declared,
reason: 'unsupported',
message: `${languageName(chosen)} is not supported by this LanguageTool endpoint.`,
mismatch,
agreed,
};
}
// Preserve the page's own regional variant when it is one the endpoint
// offers — `en-GB` prose should not be graded against `en-US` rules.
const longCode = declaredLongCode(pageLang);
if (
chosen === declared &&
longCode &&
supported.longCodes.has(longCode)
) {
return {
language: longCode,
detected,
declared,
reason: null,
message: null,
mismatch: false,
agreed,
};
}
return {
language: supported.bestByPrimary.get(chosen) ?? chosen,
detected,
declared,
reason: null,
message: null,
mismatch,
agreed,
};
}
+365 -99
View File
@@ -6,19 +6,35 @@
* endpoint / credentials come from the `languagetool` integration in the
* credential store and are resolved in the main process only.
*
* Two layers, mirroring `pagespeed.ts` / `crux.ts`:
* Three layers, mirroring `pagespeed.ts` / `crux.ts`:
* - `fetchSupportedLanguages` — what this endpoint actually offers,
* cached per endpoint. Every check is validated against it.
* - `checkText` — one page's prose → matches (never throws; failures
* resolve to an `error` result so a single bad page can't abort a run).
* - `runSpellingBatch` — a concurrency pool over many URLs, with progress
* and cooperative cancellation. Text is supplied lazily per item by the
* caller so a 1000-page run never holds every page body in memory.
*
* The check language is resolved locally (see `language-detect.ts`) and
* `auto` is never sent. LanguageTool's detector can only answer with a
* language it supports, so on a Turkish page it reports "English, 0.99
* confident" and flags every word — output that is indistinguishable from
* a genuinely error-ridden page once it reaches the UI.
*/
import type {
SpellingLanguageOption,
SpellingLevel,
SpellingMatch,
SpellingResult,
} from '@freecrawl/shared-types';
import { Agent, ProxyAgent, fetch as undiciFetch, type Dispatcher } from 'undici';
import {
indexSupportedLanguages,
languageName,
resolveCheckLanguage,
type SupportedLanguages,
} from './language-detect.js';
import { checkTextLocally } from './local-spell.js';
import * as logger from './logger.js';
export const PUBLIC_LT_ENDPOINT = 'https://api.languagetool.org';
@@ -39,30 +55,145 @@ const CONCURRENCY_PUBLIC = 1;
const CONCURRENCY_SELF_HOSTED = 4;
/**
* Languages LanguageTool refuses to accept as a bare code — it demands an
* explicit regional variant. Everything else passes through as-is.
* Share of words that may be flagged as *misspellings* before the result
* is treated as a language mismatch rather than a finding list.
*
* Calibrated against real responses: clean Spanish prose graded as
* Galician — two languages close enough that detection can confuse them —
* comes back at 24%, well under LanguageTool's own 60% bail-out, so
* relying on that alone lets a whole page of false positives through.
* Genuinely sloppy copy in the right language sits in the low single
* digits. 20% separates the two with room to spare.
*
* Misspellings only: grammar and style rules fire on correct prose too, so
* counting them would drag legitimate pages over the line. Applied only
* above `MISMATCH_MIN_WORDS`, and never when the user pinned the language.
*/
const VARIANT_REQUIRED: Record<string, string> = {
en: 'en-US',
de: 'de-DE',
pt: 'pt-PT',
ca: 'ca-ES',
};
const MISMATCH_RATIO = 0.2;
const MISMATCH_MIN_WORDS = 40;
/** How long a fetched `/v2/languages` list stays fresh. */
const LANGUAGES_TTL_MS = 6 * 60 * 60 * 1000;
/**
* Map a page's `html[lang]` to a LanguageTool language code. Returns
* `auto` when the page declares nothing, letting LanguageTool detect it.
* Languages the public API offered as of writing. Used only when
* `/v2/languages` cannot be reached, so a transient network failure
* degrades to a slightly stale list instead of blocking every check.
*/
export function resolveLanguage(pageLang: string | null): string {
const raw = (pageLang ?? '').trim().toLowerCase();
if (!raw) return 'auto';
const parts = raw.split('-');
const primary = parts[0] ?? '';
if (!primary) return 'auto';
if (parts.length >= 2 && parts[1]) {
return `${primary}-${parts[1].toUpperCase()}`;
const FALLBACK_LANGUAGES: SpellingLanguageOption[] = [
{ name: 'Arabic', code: 'ar', longCode: 'ar' },
{ name: 'Asturian', code: 'ast', longCode: 'ast-ES' },
{ name: 'Belarusian', code: 'be', longCode: 'be-BY' },
{ name: 'Breton', code: 'br', longCode: 'br-FR' },
{ name: 'Catalan', code: 'ca', longCode: 'ca-ES' },
{ name: 'Crimean Tatar', code: 'crh', longCode: 'crh-UA' },
{ name: 'Danish', code: 'da', longCode: 'da-DK' },
{ name: 'German (Germany)', code: 'de', longCode: 'de-DE' },
{ name: 'German (Austria)', code: 'de', longCode: 'de-AT' },
{ name: 'German (Swiss)', code: 'de', longCode: 'de-CH' },
{ name: 'Greek', code: 'el', longCode: 'el-GR' },
{ name: 'English (US)', code: 'en', longCode: 'en-US' },
{ name: 'English (GB)', code: 'en', longCode: 'en-GB' },
{ name: 'English (Australian)', code: 'en', longCode: 'en-AU' },
{ name: 'English (Canadian)', code: 'en', longCode: 'en-CA' },
{ name: 'English (New Zealand)', code: 'en', longCode: 'en-NZ' },
{ name: 'English (South African)', code: 'en', longCode: 'en-ZA' },
{ name: 'Esperanto', code: 'eo', longCode: 'eo' },
{ name: 'Spanish', code: 'es', longCode: 'es-ES' },
{ name: 'Spanish (Argentina)', code: 'es', longCode: 'es-AR' },
{ name: 'Persian', code: 'fa', longCode: 'fa-IR' },
{ name: 'French', code: 'fr', longCode: 'fr-FR' },
{ name: 'French (Belgium)', code: 'fr', longCode: 'fr-BE' },
{ name: 'French (Canada)', code: 'fr', longCode: 'fr-CA' },
{ name: 'French (Switzerland)', code: 'fr', longCode: 'fr-CH' },
{ name: 'Irish', code: 'ga', longCode: 'ga-IE' },
{ name: 'Galician', code: 'gl', longCode: 'gl-ES' },
{ name: 'Italian', code: 'it', longCode: 'it-IT' },
{ name: 'Japanese', code: 'ja', longCode: 'ja-JP' },
{ name: 'Khmer', code: 'km', longCode: 'km-KH' },
{ name: 'Norwegian (Bokmål)', code: 'nb', longCode: 'nb-NO' },
{ name: 'Dutch', code: 'nl', longCode: 'nl-NL' },
{ name: 'Dutch (Belgium)', code: 'nl', longCode: 'nl-BE' },
{ name: 'Polish', code: 'pl', longCode: 'pl-PL' },
{ name: 'Portuguese (Portugal)', code: 'pt', longCode: 'pt-PT' },
{ name: 'Portuguese (Brazil)', code: 'pt', longCode: 'pt-BR' },
{ name: 'Romanian', code: 'ro', longCode: 'ro-RO' },
{ name: 'Russian', code: 'ru', longCode: 'ru-RU' },
{ name: 'Slovak', code: 'sk', longCode: 'sk-SK' },
{ name: 'Slovenian', code: 'sl', longCode: 'sl-SI' },
{ name: 'Swedish', code: 'sv', longCode: 'sv-SE' },
{ name: 'Tamil', code: 'ta', longCode: 'ta-IN' },
{ name: 'Tagalog', code: 'tl', longCode: 'tl-PH' },
{ name: 'Ukrainian', code: 'uk', longCode: 'uk-UA' },
{ name: 'Chinese', code: 'zh', longCode: 'zh-CN' },
];
interface LanguagesCacheEntry {
at: number;
supported: SupportedLanguages;
}
const languagesCache = new Map<string, LanguagesCacheEntry>();
function normaliseEndpoint(endpoint: string): string {
return endpoint.replace(/\/+$/, '');
}
/**
* The set of languages one endpoint offers. Self-hosted servers ship
* different language modules than the public API — and an n-gram-less
* build offers fewer still — so this must be asked per endpoint rather
* than assumed. Cached for `LANGUAGES_TTL_MS`; a failure falls back to the
* bundled snapshot and is not cached, so the next run retries.
*/
export async function fetchSupportedLanguages(
endpoint: string,
): Promise<SupportedLanguages> {
const base = normaliseEndpoint(endpoint);
const cached = languagesCache.get(base);
if (cached && Date.now() - cached.at < LANGUAGES_TTL_MS) {
return cached.supported;
}
return VARIANT_REQUIRED[primary] ?? primary;
try {
const res = await undiciFetch(`${base}/v2/languages`, {
method: 'GET',
signal: AbortSignal.timeout(20_000),
headers: { Accept: 'application/json' },
dispatcher: ltDispatcher,
});
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const json = (await res.json()) as unknown;
if (!Array.isArray(json) || json.length === 0) {
throw new Error('empty language list');
}
const options: SpellingLanguageOption[] = [];
for (const entry of json as Record<string, unknown>[]) {
const name = typeof entry['name'] === 'string' ? entry['name'] : '';
const code = typeof entry['code'] === 'string' ? entry['code'] : '';
const longCode =
typeof entry['longCode'] === 'string' ? entry['longCode'] : code;
if (!code || !longCode) continue;
options.push({ name: name || longCode, code, longCode });
}
if (options.length === 0) throw new Error('no usable language entries');
const supported = indexSupportedLanguages(options);
languagesCache.set(base, { at: Date.now(), supported });
return supported;
} catch (err) {
logger.log(
'warn',
'languagetool',
`could not read supported languages from ${base} (${
err instanceof Error ? err.message : String(err)
}) — using the bundled list`,
);
return indexSupportedLanguages(FALLBACK_LANGUAGES);
}
}
/** Drop the cached language list — call when the endpoint changes. */
export function clearSupportedLanguagesCache(): void {
languagesCache.clear();
}
function createLtDispatcher(): Dispatcher {
@@ -82,12 +213,25 @@ function createLtDispatcher(): Dispatcher {
const ltDispatcher = createLtDispatcher();
function errorResult(fetchedAt: string, message: string): SpellingResult {
/** A result carrying no findings — the shape shared by every non-`ok` outcome. */
function emptyResult(
fetchedAt: string,
status: SpellingResult['status'],
message: string | null,
lang: {
language?: string | null;
detected?: string | null;
declared?: string | null;
} = {},
): SpellingResult {
return {
language: null,
language: lang.language ?? null,
detectedLanguage: lang.detected ?? null,
declaredLanguage: lang.declared ?? null,
matchCount: 0,
matches: [],
status: 'error',
status,
engine: null,
error: message,
fetchedAt,
};
@@ -152,16 +296,34 @@ export interface SpellingCheckOptions {
level: SpellingLevel;
/** Lower-cased surface forms to suppress (the custom dictionary). */
ignoreWords: ReadonlySet<string>;
/** Languages this endpoint offers — every check is validated against it. */
supported: SupportedLanguages;
/**
* User-pinned LanguageTool code from Settings. When set it wins over
* both the page's `html[lang]` and local detection, and disables the
* mismatch guard — the user has asserted the answer.
*/
languageOverride?: string | undefined;
}
/** undici's Response — inferred rather than cast to the DOM `Response`. */
type LtResponse = Awaited<ReturnType<typeof undiciFetch>>;
/**
* POST one check. The body is read as text first and only then parsed:
* LanguageTool reports failures as a plain-text `Error: …` line, not JSON,
* so parsing straight to JSON throws away the one thing that explains what
* went wrong — including its "this text isn't in that language" verdict.
*/
async function postCheck(
text: string,
language: string,
opts: SpellingCheckOptions,
): Promise<{ res: LtResponse; json: Record<string, unknown> | null }> {
): Promise<{
res: LtResponse;
json: Record<string, unknown> | null;
body: string;
}> {
const body = new URLSearchParams({
text,
language,
@@ -183,20 +345,48 @@ async function postCheck(
body: body.toString(),
dispatcher: ltDispatcher,
});
const json = (await res.json().catch(() => null)) as Record<
string,
unknown
> | null;
return { res, json };
const raw = await res.text().catch(() => '');
let json: Record<string, unknown> | null = null;
try {
const parsed: unknown = JSON.parse(raw);
if (parsed && typeof parsed === 'object') {
json = parsed as Record<string, unknown>;
}
} catch {
// Plain-text error body — `raw` carries the message.
}
return { res, json, body: raw };
}
/** Rough word count of the checked sample, for the mismatch ratio. */
function countWords(text: string): number {
const words = text.match(/[\p{L}\p{N}][\p{L}\p{N}'’-]*/gu);
return words ? words.length : 0;
}
/**
* Check one page's prose. Never throws. A page with too little prose
* resolves to `skipped`; a transport/API failure to `error`.
* LanguageTool's own bail-out when the text is clearly in another
* language: `Text checking was stopped due to too many errors (more than
* 60% of words seem to have an error)`. That is a language verdict, not a
* transport failure, and reads far better as one.
*/
function isLanguageBailout(message: string): boolean {
return /too many errors|correct text language/i.test(message);
}
/**
* Check one page's prose. Never throws.
*
* When the declared `html[lang]` is a code LanguageTool rejects, the call
* is retried once with `auto` so a bad lang attribute doesn't silently
* cost the user the whole page.
* The language is resolved locally first — from the prose itself,
* reconciled with `html[lang]` — and validated against the endpoint's own
* language list. A page whose language this endpoint does not offer
* resolves to `unsupported` *without* a request: substituting the nearest
* supported language (which is what LanguageTool's `auto` does) returns a
* finding on almost every word, and those findings are worse than none.
*
* A result that comes back with an implausible share of the page flagged
* is treated the same way — `mismatch` — because that is what a wrong
* language looks like from the outside.
*/
export async function checkText(
text: string,
@@ -206,49 +396,92 @@ export async function checkText(
const fetchedAt = new Date().toISOString();
const prose = text.trim();
if (prose.length < MIN_PROSE_CHARS) {
return {
language: null,
matchCount: 0,
matches: [],
status: 'skipped',
error: null,
fetchedAt,
};
return emptyResult(fetchedAt, 'skipped', null);
}
const clipped =
prose.length > MAX_TEXT_CHARS ? prose.slice(0, MAX_TEXT_CHARS) : prose;
const attempt = async (
language: string,
): Promise<{ ok: boolean; result: SpellingResult }> => {
const { res, json } = await postCheck(clipped, language, opts);
const decision = await resolveCheckLanguage({
text: clipped,
pageLang,
supported: opts.supported,
override: opts.languageOverride,
});
const langInfo = {
detected: decision.detected,
declared: decision.declared,
};
if (!decision.language) {
// LanguageTool cannot check this language — but a bundled Hunspell
// dictionary may still be able to. Spelling only; the result records
// which engine ran so nothing implies the grammar was examined.
const fallbackLang = decision.detected ?? decision.declared;
if (decision.reason === 'unsupported' && fallbackLang) {
const local = await checkTextLocally(
clipped,
fallbackLang,
opts.ignoreWords,
);
if (local) {
return {
language: fallbackLang,
detectedLanguage: decision.detected,
declaredLanguage: decision.declared,
matchCount: local.matches.length,
matches: local.matches,
status: 'ok',
engine: 'local',
error: null,
fetchedAt,
};
}
}
return emptyResult(
fetchedAt,
decision.reason === 'unsupported' ? 'unsupported' : 'skipped',
decision.message,
langInfo,
);
}
const language = decision.language;
try {
const { res, json, body } = await postCheck(clipped, language, opts);
if (!res.ok) {
const apiErr =
(json?.['message'] as string | undefined) ??
(json?.['error'] as string | undefined);
return {
ok: false,
result: errorResult(
(json?.['error'] as string | undefined) ??
// Plain-text `Error: …` body, trimmed to something a table cell
// and a tooltip can carry.
(body.trim().length > 0
? body.trim().replace(/^Error:\s*/i, '').slice(0, 400)
: `LanguageTool returned HTTP ${res.status}`);
if (isLanguageBailout(apiErr)) {
return emptyResult(
fetchedAt,
apiErr ?? `LanguageTool returned HTTP ${res.status}`,
),
};
'mismatch',
`LanguageTool stopped checking — the page does not read as ${languageName(language)}. Pin the language under Settings → Spelling if this is wrong.`,
{ ...langInfo, language },
);
}
return emptyResult(fetchedAt, 'error', apiErr, {
...langInfo,
language,
});
}
if (!json) {
return {
ok: false,
result: errorResult(fetchedAt, 'LanguageTool returned an empty response'),
};
return emptyResult(
fetchedAt,
'error',
'LanguageTool returned an empty response',
{ ...langInfo, language },
);
}
const langBlock = json['language'] as
| { code?: unknown; detectedLanguage?: { code?: unknown } }
| undefined;
const langBlock = json['language'] as { code?: unknown } | undefined;
const usedLang =
typeof langBlock?.code === 'string'
? langBlock.code
: typeof langBlock?.detectedLanguage?.code === 'string'
? langBlock.detectedLanguage.code
: null;
typeof langBlock?.code === 'string' ? langBlock.code : language;
const rawMatches = Array.isArray(json['matches'])
? (json['matches'] as LtMatch[])
@@ -260,30 +493,44 @@ export async function checkText(
.filter((m) => !opts.ignoreWords.has(m.text.toLowerCase()))
.slice(0, MAX_STORED_MATCHES);
return {
ok: true,
result: {
language: usedLang,
matchCount: matches.length,
matches,
status: 'ok',
error: null,
fetchedAt,
},
};
};
try {
const language = resolveLanguage(pageLang);
const first = await attempt(language);
if (first.ok) return first.result;
// A rejected language code is the one failure worth retrying — fall
// back to LanguageTool's own detection rather than losing the page.
if (language !== 'auto' && /lang/i.test(first.result.error ?? '')) {
const retry = await attempt('auto');
if (retry.ok) return retry.result;
// Density guard — the last line of defence against a wrong language
// reaching the UI as a finding list.
//
// Only consulted when the language was actually in doubt. If the page's
// own `html[lang]` and the trigram detector independently named the
// same language, two signals already agree and a high error rate is the
// finding, not a reason to doubt them — a genuinely typo-ridden English
// page must still report its typos. Likewise skipped when the user
// pinned the language: they overruled the detector on purpose.
if (!opts.languageOverride && !decision.agreed) {
const words = countWords(clipped);
const misspellings = matches.filter(
(m) => m.issueType === 'misspelling',
).length;
if (
words >= MISMATCH_MIN_WORDS &&
misspellings / words > MISMATCH_RATIO
) {
return emptyResult(
fetchedAt,
'mismatch',
`${Math.round((misspellings / words) * 100)}% of words were flagged when checked as ${languageName(usedLang)} — the page is almost certainly written in another language, so the findings were discarded. Pin the language under Settings → Spelling if this is wrong.`,
{ ...langInfo, language: usedLang },
);
}
}
return first.result;
return {
language: usedLang,
detectedLanguage: decision.detected,
declaredLanguage: decision.declared,
matchCount: matches.length,
matches,
status: 'ok',
engine: 'languagetool',
error: null,
fetchedAt,
};
} catch (err) {
const name = (err as { name?: string } | null)?.name;
const cause = (err as { cause?: unknown } | null)?.cause as
@@ -302,7 +549,10 @@ export async function checkText(
? `${err.message} (${cause.message})`
: err.message
: String(err);
return errorResult(fetchedAt, message);
return emptyResult(fetchedAt, 'error', message, {
...langInfo,
language,
});
}
}
@@ -324,6 +574,8 @@ export interface SpellingBatchOptions {
export interface SpellingBatchResult {
completed: number;
failed: number;
/** Pages whose language this endpoint does not support. */
unsupported: number;
cancelled: boolean;
}
@@ -340,6 +592,7 @@ export async function runSpellingBatch(
let done = 0;
let completed = 0;
let failed = 0;
let unsupported = 0;
let cursor = 0;
const ERROR_LOG_LIMIT = 3;
@@ -351,6 +604,8 @@ export async function runSpellingBatch(
logger.log('warn', 'languagetool', `${url} — ${message}`);
}
};
/** Languages seen that this endpoint cannot check, for the summary line. */
const unsupportedLangs = new Map<string, number>();
onProgress(0, total, null);
@@ -368,20 +623,14 @@ export async function runSpellingBatch(
try {
const page = await loadPage(url);
if (!page.text || page.text.trim().length === 0) {
result = {
language: null,
matchCount: 0,
matches: [],
status: 'skipped',
error: null,
fetchedAt,
};
result = emptyResult(fetchedAt, 'skipped', null);
} else {
result = await checkText(page.text, page.lang, check);
}
} catch (err) {
result = errorResult(
result = emptyResult(
fetchedAt,
'error',
err instanceof Error ? err.message : String(err),
);
}
@@ -404,6 +653,12 @@ export async function runSpellingBatch(
failed++;
if (result.status === 'error') {
recordFailure(url, result.error ?? 'Unknown error');
} else if (result.status === 'unsupported') {
unsupported++;
const lang = languageName(
result.detectedLanguage ?? result.declaredLanguage,
);
unsupportedLangs.set(lang, (unsupportedLangs.get(lang) ?? 0) + 1);
}
}
onProgress(done, total, null);
@@ -445,5 +700,16 @@ export async function runSpellingBatch(
);
}
}
return { completed, failed, cancelled };
if (unsupportedLangs.size > 0) {
const breakdown = [...unsupportedLangs.entries()]
.sort((a, b) => b[1] - a[1])
.map(([lang, count]) => `${count}× ${lang}`)
.join(', ');
logger.log(
'warn',
'languagetool',
`${unsupported} page(s) were not checked — LanguageTool has no rules for their language (${breakdown}). This is a limitation of LanguageTool itself, not of the crawl.`,
);
}
return { completed, failed, unsupported, cancelled };
}
+302
View File
@@ -0,0 +1,302 @@
/**
* Offline spelling checker for languages LanguageTool does not cover.
*
* LanguageTool ships rules for ~33 languages. Turkish is not among them,
* and neither are Hungarian, Czech, Finnish and most of the rest — so on a
* Turkish site the Spelling tab has nothing to report no matter how the
* endpoint is configured. This module fills that gap with a Hunspell
* dictionary run locally, in-process, with no network call.
*
* Scope is deliberately narrower than LanguageTool's: **spelling only, no
* grammar or style**. A dictionary can say "this word is not a word"; it
* cannot say "this clause disagrees with its subject". Results are tagged
* with the engine that produced them so the UI can say so rather than
* implying grammar was checked and came back clean.
*
* Why Hunspell via `nspell`, measured rather than assumed:
* - `nspell` (91 KB, pure JS) returns results *identical* to real
* Hunspell compiled to WebAssembly on the same dictionary, at twice
* the speed and without a 4.9 MB WASM payload.
* - `nodehun` binds native Hunspell and would reintroduce node-gyp,
* which this project avoids on purpose.
* - Turkish has no morphological analyser (Zemberek and friends) with a
* JavaScript or WebAssembly port.
*
* Only dictionaries whose licence is compatible with this project's MIT
* licence are bundled. That is a real constraint, not a preference: most
* Hunspell dictionaries are GPL-2.0, LGPL or AGPL-3.0, and shipping those
* inside an MIT application would encumber the whole distribution.
* Turkish qualifies — `dictionary-tr` is MIT.
*/
import type { SpellingMatch } from '@freecrawl/shared-types';
import * as logger from './logger.js';
/**
* Languages checkable without a network call. Keyed by primary subtag, so
* `tr`, `tr-TR` and a detected `tr` all resolve here.
*
* Adding a language means adding a dictionary dependency *and* clearing
* its licence against MIT — see the module note above.
*/
const LOCAL_DICTIONARIES: Record<string, () => Promise<HunspellDictionary>> = {
tr: async () => (await import('dictionary-tr')).default,
};
/** The `{aff, dic}` buffer pair every `dictionary-*` package exports. */
interface HunspellDictionary {
aff: Uint8Array;
dic: Uint8Array;
}
interface Speller {
correct: (word: string) => boolean;
suggest: (word: string) => string[];
}
/** Words shorter than this are prepositions, particles and noise. */
const MIN_WORD_CHARS = 3;
/** Match LanguageTool's cap so the stored blob stays bounded. */
const MAX_MATCHES = 500;
/** Suggestions kept per finding, matching the LanguageTool path. */
const MAX_REPLACEMENTS = 5;
/** Characters of surrounding text stored with each finding, per side. */
const CONTEXT_PAD = 40;
/**
* Turkish letters plus the circumflex vowels. Anything outside this — a
* digit, a Latin binomial, a foreign brand — is not something a Turkish
* dictionary can rule on, so it is left alone rather than reported.
*/
const CHECKABLE_WORD = /^[a-zçğıöşüâîû'’]+$/i;
/**
* ASCII spellings of Turkish letters. Typing `ogrenci` for `öğrenci` is by
* far the most common Turkish misspelling — a keyboard without the Turkish
* layout produces it constantly — and Hunspell's edit-distance suggester
* has no idea the two letters are related. Measured on 616 real words
* misspelled this way, its first suggestion was right 33.6% of the time;
* expanding these back and testing each candidate against the dictionary
* lifts that to 96.1%.
*/
const ASCII_EXPANSIONS: Record<string, string[]> = {
c: ['c', 'ç'],
g: ['g', 'ğ'],
i: ['i', 'ı'],
o: ['o', 'ö'],
s: ['s', 'ş'],
u: ['u', 'ü'],
};
/** Cap the expansion so a long all-vowel word cannot blow up. */
const MAX_EXPANSION_CANDIDATES = 512;
const spellerCache = new Map<string, Promise<Speller | null>>();
/** Is there an offline dictionary for this language? */
export function hasLocalDictionary(lang: string | null): boolean {
if (!lang) return false;
const primary = lang.split(/[-_]/)[0]?.toLowerCase() ?? '';
return primary in LOCAL_DICTIONARIES;
}
/**
* Load (and cache) the speller for a language. Lazy on purpose: the
* Turkish dictionary is ~9 MB of word list, and most sessions never open
* the Spelling tab at all.
*/
async function loadSpeller(primary: string): Promise<Speller | null> {
const cached = spellerCache.get(primary);
if (cached) return cached;
const load = LOCAL_DICTIONARIES[primary];
if (!load) return null;
const promise = (async (): Promise<Speller | null> => {
try {
const [{ default: nspell }, dictionary] = await Promise.all([
import('nspell'),
load(),
]);
// The `dictionary-*` packages type their payload as `Uint8Array` but
// produce it with `fs.readFile`, so it already is a Buffer at
// runtime — which is what nspell's types ask for. Guard anyway
// rather than assert, so a future plain-array payload still works.
const asBuffer = (data: Uint8Array): Buffer =>
Buffer.isBuffer(data) ? data : Buffer.from(data);
return nspell({
aff: asBuffer(dictionary.aff),
dic: asBuffer(dictionary.dic),
}) as unknown as Speller;
} catch (err) {
logger.log(
'warn',
'spelling',
`offline ${primary} dictionary failed to load (${
err instanceof Error ? err.message : String(err)
})`,
);
return null;
}
})();
spellerCache.set(primary, promise);
return promise;
}
/** Start loading a dictionary without waiting for it. */
export function warmLocalDictionary(lang: string | null): void {
if (!lang) return;
const primary = lang.split(/[-_]/)[0]?.toLowerCase() ?? '';
if (primary in LOCAL_DICTIONARIES) void loadSpeller(primary);
}
interface Token {
word: string;
offset: number;
/**
* Whether the word opens a sentence. Such words are capitalised by
* grammar rather than because they are names, so they still get checked
* — in lower case, since that is how the dictionary stores them.
*/
sentenceStart: boolean;
}
/** Split prose into words, tracking offsets and sentence boundaries. */
function tokenise(text: string): Token[] {
const tokens: Token[] = [];
const re = /[\p{L}][\p{L}'’]*/gu;
let match: RegExpExecArray | null;
while ((match = re.exec(text)) !== null) {
// Walk back over spaces and opening punctuation to find what actually
// precedes the word.
let i = match.index - 1;
while (i >= 0 && /[\s"'“‘([]/.test(text[i] ?? '')) i--;
const prev = i < 0 ? '' : (text[i] ?? '');
tokens.push({
word: match[0],
offset: match.index,
sentenceStart: prev === '' || /[.!?:;…]/.test(prev),
});
}
return tokens;
}
/**
* Whether a token is worth asking the dictionary about.
*
* Capitalised words that do not open a sentence are skipped. They are
* overwhelmingly proper nouns — brands, place names, people — which no
* general dictionary can vouch for. Measured on real Turkish prose,
* checking them anyway roughly doubles the noise (7.3% of words flagged
* rises to 15.6%) while adding almost nothing a reader would act on.
* All-caps tokens are acronyms and get the same treatment.
*/
function isCheckable(token: Token): boolean {
const { word, sentenceStart } = token;
if (word.length < MIN_WORD_CHARS) return false;
if (!CHECKABLE_WORD.test(word)) return false;
if (word === word.toUpperCase() && word !== word.toLowerCase()) return false;
const isCapitalised = word[0] !== word[0]?.toLowerCase();
if (isCapitalised && !sentenceStart) return false;
return true;
}
/** Candidate spellings obtained by restoring Turkish letters from ASCII. */
function asciiExpansions(word: string): string[] {
let candidates = [''];
for (const ch of word) {
const options = ASCII_EXPANSIONS[ch];
if (!options || candidates.length * options.length > MAX_EXPANSION_CANDIDATES) {
candidates = candidates.map((c) => c + ch);
continue;
}
const next: string[] = [];
for (const c of candidates) for (const o of options) next.push(c + o);
candidates = next;
}
return candidates;
}
/**
* Turkish-aware capitalisation. `i` uppercases to `İ`, not `I` — getting
* that wrong in a *correction* would be its own spelling mistake.
*/
function matchCapitalisation(suggestion: string, original: string): string {
const first = original[0];
if (!first || first === first.toLowerCase()) return suggestion;
const head = suggestion[0] ?? '';
return (head === 'i' ? 'İ' : head.toLocaleUpperCase('tr')) + suggestion.slice(1);
}
function suggestionsFor(speller: Speller, word: string): string[] {
const lower = word.toLowerCase();
const out: string[] = [];
const push = (s: string): void => {
const cased = matchCapitalisation(s, word);
if (!out.includes(cased)) out.push(cased);
};
// Restoring Turkish letters answers the dominant typo class far better
// than edit distance does, so those candidates come first.
for (const candidate of asciiExpansions(lower)) {
if (candidate !== lower && speller.correct(candidate)) push(candidate);
if (out.length >= MAX_REPLACEMENTS) return out;
}
for (const s of speller.suggest(word)) {
push(s);
if (out.length >= MAX_REPLACEMENTS) break;
}
return out;
}
export interface LocalCheckResult {
matches: SpellingMatch[];
/** Words actually submitted to the dictionary — the noise-rate divisor. */
checkedWords: number;
}
/**
* Spell-check prose offline. Returns null when no dictionary is bundled
* for the language, so the caller can fall back to reporting it as
* unsupported. Never throws.
*/
export async function checkTextLocally(
text: string,
lang: string,
ignoreWords: ReadonlySet<string>,
): Promise<LocalCheckResult | null> {
const primary = lang.split(/[-_]/)[0]?.toLowerCase() ?? '';
const speller = await loadSpeller(primary);
if (!speller) return null;
const matches: SpellingMatch[] = [];
let checkedWords = 0;
for (const token of tokenise(text)) {
if (!isCheckable(token)) continue;
checkedWords++;
const { word, offset, sentenceStart } = token;
// A sentence-opening word is capitalised by grammar; the dictionary
// holds it lower case, so ask about that form.
const probe = sentenceStart ? word.toLowerCase() : word;
if (speller.correct(word) || speller.correct(probe)) continue;
if (ignoreWords.has(word.toLowerCase())) continue;
if (matches.length >= MAX_MATCHES) break;
const from = Math.max(0, offset - CONTEXT_PAD);
const to = Math.min(text.length, offset + word.length + CONTEXT_PAD);
matches.push({
message: 'Possible spelling mistake found.',
shortMessage: 'Spelling mistake',
offset,
length: word.length,
text: word,
context: text.slice(from, to),
contextOffset: offset - from,
replacements: suggestionsFor(speller, word),
ruleId: `HUNSPELL_${primary.toUpperCase()}`,
category: 'Possible Typo',
issueType: 'misspelling',
});
}
return { matches, checkedWords };
}
+37
View File
@@ -39,7 +39,20 @@ export interface MenuLabels {
compareWith: string;
scheduledCrawl: string;
scheduledCrawlTooltip: string;
saveProject: string;
saveProjectAs: string;
/** Window title shown before a project has been saved to disk. */
titleUntitledProject: string;
dlgSaveProjectAsTitle: string;
dlgExportTableTitle: string;
dlgSaveFailedTitle: string;
/** `{size}` = archive MB, `{from}` = uncompressed MB. */
msgProjectSaved: string;
dlgUnsavedTitle: string;
msgUnsavedChanges: string;
detailUnsavedChanges: string;
btnSaveChanges: string;
btnDiscardChanges: string;
saveProjectEncrypted: string;
saveProjectEncryptedTooltip: string;
openProjectEncrypted: string;
@@ -172,7 +185,19 @@ const MENU_EN: MenuLabels = {
scheduledCrawl: 'Scheduled Crawl…',
scheduledCrawlTooltip:
'Set up an in-app recurring crawl for the currently-open project. Fires only while FreeCrawl is open; use the CLI + OS scheduler for triggers that survive restarts.',
saveProject: 'Save Project',
saveProjectAs: 'Save Project As…',
titleUntitledProject: 'Untitled project',
dlgSaveProjectAsTitle: 'Save Project As…',
dlgExportTableTitle: 'Export Table',
dlgSaveFailedTitle: 'Could Not Save Project',
msgProjectSaved: 'Saved as a single compressed file: {size} MB (from {from} MB).',
dlgUnsavedTitle: 'Unsaved Changes',
msgUnsavedChanges: 'This project has changes that are not saved yet.',
detailUnsavedChanges:
'Crawl results live in a working copy until you save them into the project file.',
btnSaveChanges: 'Save',
btnDiscardChanges: "Don't Save",
saveProjectEncrypted: 'Save Encrypted Snapshot…',
saveProjectEncryptedTooltip:
'Export the active project to an AES-256-GCM-encrypted .seoproject.enc file protected by a password.',
@@ -311,7 +336,19 @@ const MENU_TR: MenuLabels = {
scheduledCrawl: 'Zamanlanmış Crawl…',
scheduledCrawlTooltip:
'Şu an açık olan proje için uygulama içi tekrarlayan crawl kur. Yalnızca FreeCrawl açıkken çalışır; yeniden başlatmaya dayanan tetikler için CLI + OS zamanlayıcısını kullan.',
saveProject: 'Projeyi Kaydet',
saveProjectAs: 'Projeyi Farklı Kaydet…',
titleUntitledProject: 'Adsız proje',
dlgSaveProjectAsTitle: 'Projeyi Farklı Kaydet…',
dlgExportTableTitle: 'Tabloyu Dışa Aktar',
dlgSaveFailedTitle: 'Proje Kaydedilemedi',
msgProjectSaved: 'Tek sıkıştırılmış dosya olarak kaydedildi: {size} MB ({from} MB yerine).',
dlgUnsavedTitle: 'Kaydedilmemiş Değişiklikler',
msgUnsavedChanges: 'Bu projede henüz kaydedilmemiş değişiklikler var.',
detailUnsavedChanges:
'Crawl sonuçları, proje dosyasına kaydedene kadar bir çalışma kopyasında tutulur.',
btnSaveChanges: 'Kaydet',
btnDiscardChanges: 'Kaydetme',
saveProjectEncrypted: 'Şifreli Snapshot Kaydet…',
saveProjectEncryptedTooltip:
'Aktif projeyi parolayla korunan AES-256-GCM şifreli .seoproject.enc dosyasına dışa aktar.',
+5
View File
@@ -140,6 +140,11 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
toolTip: L.scheduledCrawlTooltip,
click: () => send('open-scheduled-crawl'),
},
{
label: L.saveProject,
accelerator: 'CmdOrCtrl+S',
click: () => send('save-project'),
},
{
label: L.saveProjectAs,
accelerator: 'CmdOrCtrl+Shift+S',
+324
View File
@@ -0,0 +1,324 @@
/**
* Playwright browser provisioning for the packaged desktop app.
*
* Three jobs, in the order they matter:
*
* 1. Decide which browser cache Playwright should read, and publish it
* through `PLAYWRIGHT_BROWSERS_PATH` BEFORE anything imports
* `playwright` (its registry latches the env var at module-eval
* time and never re-reads it). Preference order: the browsers
* bundled inside the installer → a writable per-user directory we
* download into → whatever Playwright would pick by default.
*
* 2. Report whether Chromium is actually usable on THIS machine. The
* bundle check is arch-aware on purpose: a release built on an
* Apple-silicon runner ships `chrome-mac-arm64`, which an Intel Mac
* cannot use, and silently falling back to a download beats
* pretending the bundle is fine.
*
* 3. Download the browsers when they're missing, WITHOUT `npx`. A
* packaged app has neither Node nor npm on its PATH — GUI processes
* on macOS inherit a minimal environment, so `spawn('npx', …)` dies
* with ENOENT and leaves the user stuck at a dialog telling them to
* open a terminal. Instead we re-exec our own Electron binary in
* Node mode (`ELECTRON_RUN_AS_NODE`) against Playwright's bundled
* CLI, which is always present inside the app bundle.
*/
import { app } from 'electron';
import { spawn, type ChildProcess } from 'node:child_process';
import { createRequire } from 'node:module';
import { existsSync, mkdirSync, readdirSync } from 'node:fs';
import { homedir, cpus } from 'node:os';
import { dirname, join } from 'node:path';
/** Browsers Playwright must have for JS rendering to work. */
const REQUIRED_BROWSERS = ['chromium', 'chromium-headless-shell'] as const;
/**
* Playwright's per-platform folder names inside a `chromium-<rev>` /
* `chromium_headless_shell-<rev>` directory. Mirrors its EXECUTABLE_PATHS
* table — kept here (rather than asked of Playwright) because this check
* has to run before `playwright` is loaded.
*/
function platformFolders(): { chromium: string; headlessShell: string } | null {
if (process.platform === 'win32') {
return { chromium: 'chrome-win64', headlessShell: 'chrome-headless-shell-win64' };
}
if (process.platform === 'darwin') {
// Playwright picks the mac arch from the CPU model, not process.arch,
// so an x64 build running under Rosetta still resolves the arm64
// folder. Matching that exactly is the whole point of this check.
const arch = cpus().some((c) => c.model.includes('Apple')) ? 'arm64' : 'x64';
return {
chromium: `chrome-mac-${arch}`,
headlessShell: `chrome-headless-shell-mac-${arch}`,
};
}
if (process.platform === 'linux') {
if (process.arch === 'arm64') {
// arm64 Linux still uses Playwright's own build, not Chrome-for-Testing.
return { chromium: 'chrome-linux', headlessShell: 'chrome-linux' };
}
return {
chromium: 'chrome-linux64',
headlessShell: 'chrome-headless-shell-linux64',
};
}
return null;
}
/**
* True when `dir` holds a Chromium build this machine can actually run.
* Both the full browser and the headless shell must be present — a
* partial cache would launch-fail later with Playwright's ASCII banner.
*/
function dirHasUsableChromium(dir: string): boolean {
const folders = platformFolders();
if (!folders || !existsSync(dir)) return false;
let entries: string[];
try {
entries = readdirSync(dir);
} catch {
return false;
}
const hasFull = entries.some(
(e) => e.startsWith('chromium-') && existsSync(join(dir, e, folders.chromium)),
);
const hasShell = entries.some(
(e) =>
e.startsWith('chromium_headless_shell-') &&
existsSync(join(dir, e, folders.headlessShell)),
);
return hasFull && hasShell;
}
/** Playwright's own default cache location, replicated. */
function defaultCacheDir(): string {
if (process.platform === 'win32') {
return join(
process.env.LOCALAPPDATA ?? join(homedir(), 'AppData', 'Local'),
'ms-playwright',
);
}
if (process.platform === 'darwin') {
return join(homedir(), 'Library', 'Caches', 'ms-playwright');
}
return join(homedir(), '.cache', 'ms-playwright');
}
/** Browsers shipped inside the installer, if this build has any. */
function bundledDir(): string | null {
if (!app.isPackaged || typeof process.resourcesPath !== 'string') return null;
if (!process.resourcesPath) return null;
return join(process.resourcesPath, 'playwright-browsers');
}
/**
* Writable download target. Lives under userData so it survives app
* upgrades and needs no elevation — the app bundle itself is read-only
* on macOS and under Program Files on Windows.
*/
export function installTargetDir(): string {
return join(app.getPath('userData'), 'playwright-browsers');
}
/** Set once by {@link setupPlaywrightBrowsersPath}; '' = Playwright default. */
let activeBrowsersPath = '';
/**
* Pick the browser cache and export it to the environment. Call this as
* early as possible in main — before the first `import('playwright')`.
* Returns a short description of the decision for the startup log.
*/
export function setupPlaywrightBrowsersPath(): string {
// Respect an explicit operator override (enterprise deployments that
// pre-seed a shared cache) — never second-guess it.
if (process.env.PLAYWRIGHT_BROWSERS_PATH) {
activeBrowsersPath = process.env.PLAYWRIGHT_BROWSERS_PATH;
return `using PLAYWRIGHT_BROWSERS_PATH from environment: ${activeBrowsersPath}`;
}
const bundled = bundledDir();
if (bundled && dirHasUsableChromium(bundled)) {
process.env.PLAYWRIGHT_BROWSERS_PATH = bundled;
activeBrowsersPath = bundled;
return `using browsers bundled in the installer: ${bundled}`;
}
const target = installTargetDir();
if (dirHasUsableChromium(target)) {
process.env.PLAYWRIGHT_BROWSERS_PATH = target;
activeBrowsersPath = target;
return `using previously downloaded browsers: ${target}`;
}
// Nothing installed yet. A dev machine (or a user who ran the
// Playwright CLI themselves) has the default cache populated — use it
// untouched. Otherwise point at the writable directory so the
// background install and the runtime lookup agree on one location.
if (dirHasUsableChromium(defaultCacheDir())) {
activeBrowsersPath = '';
return `using the default Playwright cache: ${defaultCacheDir()}`;
}
process.env.PLAYWRIGHT_BROWSERS_PATH = target;
activeBrowsersPath = target;
const why = bundled
? 'installer bundle missing or built for another architecture'
: 'no bundled browsers in this build';
return `no browsers found (${why}) — will download into ${target}`;
}
/**
* Cheap, playwright-free installation check against the directory we
* actually resolved. Safe to call at startup on the main thread.
*/
export function chromiumInstalled(): boolean {
return dirHasUsableChromium(activeBrowsersPath || defaultCacheDir());
}
/**
* Absolute path to Playwright's CLI entry point inside the app bundle.
*
* `cli.js` isn't listed in the package's `exports` map, so it can't be
* required by subpath — resolve `package.json` (which is exported) and
* walk to its sibling. Works inside app.asar; Electron's fs shim reads
* archived files transparently.
*/
function playwrightCliPath(): string | null {
const req = createRequire(import.meta.url);
for (const pkg of ['playwright/package.json', 'playwright-core/package.json']) {
try {
const cli = join(dirname(req.resolve(pkg)), 'cli.js');
if (existsSync(cli)) return cli;
} catch {
/* not resolvable from here — try the next candidate */
}
}
return null;
}
export interface BrowserInstallProgress {
/** 0-100, or null while Playwright is between downloads. */
percent: number | null;
/** Human-readable step, e.g. 'Downloading Chromium'. */
message: string;
}
export type InstallResult =
| { ok: true }
| { ok: false; reason: 'no-cli' | 'spawn-failed' | 'exit-code'; detail: string };
let running: Promise<InstallResult> | null = null;
let runningChild: ChildProcess | null = null;
/** True while a download is in flight (so callers don't stack them). */
export function installInProgress(): boolean {
return running !== null;
}
/**
* Download Chromium + chrome-headless-shell into the writable cache.
*
* Concurrent callers share one download. The child is our own Electron
* binary running in Node mode, so this works on a machine that has
* never seen Node, npm or a terminal.
*/
export function installChromiumBrowsers(
onProgress?: (p: BrowserInstallProgress) => void,
onLog?: (line: string) => void,
): Promise<InstallResult> {
if (running) return running;
running = new Promise<InstallResult>((resolveInstall) => {
const cli = playwrightCliPath();
if (!cli) {
resolveInstall({
ok: false,
reason: 'no-cli',
detail: 'Playwright CLI not found inside the application bundle.',
});
return;
}
// Downloads must land somewhere writable. When the resolved path is
// still the read-only installer bundle (partial cache) redirect the
// install — and the subsequent lookup — to userData.
let target = activeBrowsersPath;
const bundled = bundledDir();
if (!target || (bundled && target === bundled)) {
target = installTargetDir();
process.env.PLAYWRIGHT_BROWSERS_PATH = target;
activeBrowsersPath = target;
}
try {
mkdirSync(target, { recursive: true });
} catch {
/* creation also happens inside Playwright — ignore and let it report */
}
const child = spawn(process.execPath, [cli, 'install', ...REQUIRED_BROWSERS], {
env: {
...process.env,
// Turns the Electron binary into a plain Node runtime instead of
// launching a second copy of the app.
ELECTRON_RUN_AS_NODE: '1',
PLAYWRIGHT_BROWSERS_PATH: target,
},
windowsHide: true,
stdio: ['ignore', 'pipe', 'pipe'],
});
runningChild = child;
let stderr = '';
const handleChunk = (chunk: Buffer): void => {
for (const raw of chunk.toString().split('\n')) {
const line = raw.trim();
if (!line) continue;
onLog?.(line);
// Playwright renders an ASCII progress bar: "|███ | 42% of 158 MiB"
const pct = /(\d{1,3})%/.exec(line);
if (pct?.[1]) {
onProgress?.({ percent: Math.min(100, Number(pct[1])), message: 'download' });
} else if (/^Downloading/i.test(line)) {
onProgress?.({ percent: null, message: line });
}
}
};
child.stdout?.on('data', handleChunk);
child.stderr?.on('data', (chunk: Buffer) => {
stderr += chunk.toString();
// Playwright writes its progress bar to stderr on some platforms.
handleChunk(chunk);
});
child.on('error', (err) => {
resolveInstall({ ok: false, reason: 'spawn-failed', detail: err.message });
});
child.on('close', (code) => {
if (code === 0 && chromiumInstalled()) {
resolveInstall({ ok: true });
return;
}
resolveInstall({
ok: false,
reason: 'exit-code',
detail:
code === 0
? 'Playwright reported success but no usable Chromium is on disk.'
: `Playwright install exited with code ${code}. ${stderr.slice(-400)}`.trim(),
});
});
}).finally(() => {
running = null;
runningChild = null;
});
return running;
}
/** Kill an in-flight download (app shutdown). */
export function abortChromiumInstall(): void {
runningChild?.kill();
}
+173
View File
@@ -0,0 +1,173 @@
/**
* Compressed single-file project container.
*
* Container format (binary):
* ┌────────────┬─────────────┬────────────────────────────────┐
* │ 8 B magic │ 1 B version │ brotli-compressed SQLite file │
* │ "FCPROJ01" │ 0x01 │ │
* └────────────┴─────────────┴────────────────────────────────┘
*
* Why a container instead of the bare database?
* A `.seoproject` used to BE the live SQLite database, which is why
* saving produced three files: SQLite keeps `-wal` and `-shm` sidecars
* next to any database it has open. Those are an implementation detail
* of the engine, not something a user should have to keep together
* when they move or email a project. Packing the snapshot into one
* opaque file makes a project exactly one file again.
*
* Why brotli and not gzip?
* Measured on real 276–406 MB crawls: gzip -6 gives ~77% reduction,
* brotli quality 5 gives ~95% (406 MB → 17.6 MB) and is *faster* to
* write. SQLite pages are highly repetitive and brotli's much larger
* window catches redundancy gzip's 32 KB window cannot. Both ship
* inside Node's zlib, so neither costs a native dependency.
*
* Everything is streamed in both directions, so peak memory is O(chunk)
* regardless of project size — a multi-GB crawl must not have to fit in
* Electron's heap just to be saved.
*/
import { createReadStream, createWriteStream, openSync, closeSync, readSync } from 'node:fs';
import { rename, rm, stat } from 'node:fs/promises';
import { createBrotliCompress, createBrotliDecompress, constants } from 'node:zlib';
import { pipeline } from 'node:stream/promises';
const MAGIC = Buffer.from('FCPROJ01', 'utf8'); // 8 bytes
const VERSION = 0x01;
export const HEADER_LEN = MAGIC.length + 1;
/** SQLite's own file signature, used to recognise pre-container projects. */
const SQLITE_MAGIC = Buffer.from('SQLite format 3\0', 'utf8');
/**
* Quality 5 is the knee of the curve: q4 → 19.9 MB / 1.9 s, q5 → 17.6 MB
* / 2.8 s, q9 → 16.8 MB / 7.2 s on the same 406 MB project. Past q5 the
* extra seconds buy almost nothing.
*/
const BROTLI_QUALITY = 5;
/** How to read a file the user picked. */
export type ProjectFormat =
/** Compressed container written by this app. */
| 'archive'
/** A bare SQLite database — every project saved before compression landed. */
| 'sqlite'
/** Neither signature matched, or the file is too short / unreadable. */
| 'unknown';
/**
* Sniff a project file by signature rather than extension, so a renamed
* or extensionless file still opens and so legacy `.seoproject` files
* (which are bare databases) are detected as such.
*/
export function detectProjectFormat(path: string): ProjectFormat {
let fd: number | null = null;
try {
fd = openSync(path, 'r');
const head = Buffer.alloc(16);
const read = readSync(fd, head, 0, 16, 0);
if (read >= MAGIC.length && head.subarray(0, MAGIC.length).equals(MAGIC)) {
return 'archive';
}
if (read >= SQLITE_MAGIC.length && head.subarray(0, SQLITE_MAGIC.length).equals(SQLITE_MAGIC)) {
return 'sqlite';
}
return 'unknown';
} catch {
return 'unknown';
} finally {
if (fd !== null) {
try {
closeSync(fd);
} catch {
/* already closed */
}
}
}
}
/**
* Compress `srcDbPath` (a consistent SQLite snapshot) into the container
* at `dstPath`.
*
* Writes to a sibling `.part` file and renames on success: a crash or a
* full disk midway through leaves the previously saved project intact
* rather than truncating it to garbage.
*/
export async function packProject(
srcDbPath: string,
dstPath: string,
): Promise<{ bytesWritten: number; sourceBytes: number }> {
const sourceBytes = (await stat(srcDbPath)).size;
const partPath = `${dstPath}.part`;
try {
const out = createWriteStream(partPath);
// The header goes in uncompressed so `detectProjectFormat` only ever
// has to read 16 bytes to classify a file.
out.write(Buffer.concat([MAGIC, Buffer.from([VERSION])]));
await pipeline(
createReadStream(srcDbPath),
createBrotliCompress({
params: {
[constants.BROTLI_PARAM_QUALITY]: BROTLI_QUALITY,
// Letting brotli size its window to the input is most of the
// win over gzip on large database files.
[constants.BROTLI_PARAM_SIZE_HINT]: sourceBytes,
},
}),
out,
);
await rename(partPath, dstPath);
} catch (err) {
await rm(partPath, { force: true }).catch(() => {});
throw err;
}
return { bytesWritten: (await stat(dstPath)).size, sourceBytes };
}
/**
* Expand a container into a plain SQLite file at `dstDbPath`, which the
* app then opens as its working database. Throws when the header is
* missing or the version is newer than this build understands — better
* a clear error than a corrupt database.
*/
export async function unpackProject(
srcPath: string,
dstDbPath: string,
): Promise<{ bytesWritten: number }> {
const fd = openSync(srcPath, 'r');
let version: number;
try {
const head = Buffer.alloc(HEADER_LEN);
const read = readSync(fd, head, 0, HEADER_LEN, 0);
if (read < HEADER_LEN || !head.subarray(0, MAGIC.length).equals(MAGIC)) {
throw new Error('Not a FreeCrawl project archive (bad signature).');
}
version = head[MAGIC.length]!;
} finally {
closeSync(fd);
}
if (version > VERSION) {
throw new Error(
`This project was saved by a newer version of FreeCrawl (format v${version}). Please update to open it.`,
);
}
const partPath = `${dstDbPath}.part`;
try {
await pipeline(
createReadStream(srcPath, { start: HEADER_LEN }),
createBrotliDecompress(),
createWriteStream(partPath),
);
await rm(dstDbPath, { force: true });
await rename(partPath, dstDbPath);
} catch (err) {
await rm(partPath, { force: true }).catch(() => {});
throw err;
}
return { bytesWritten: (await stat(dstDbPath)).size };
}
+16
View File
@@ -1,6 +1,7 @@
import { contextBridge, ipcRenderer, type IpcRendererEvent } from 'electron';
import {
IPC,
type BrowserInstallState,
type ConfirmClearResult,
type CrawlConfig,
type RecentProject,
@@ -18,6 +19,8 @@ import {
type ExportImagesResult,
type ExportTabularInput,
type ExportTabularResult,
type ExportGridInput,
type ExportGridResult,
type DataDeleteByDomainInput,
type DataDeleteByDomainResult,
type CrashRecoveryStatus,
@@ -112,6 +115,7 @@ import {
type SpellingRunResult,
type SpellingProgress,
type SpellingMatchesResult,
type SpellingLanguageOption,
type GoogleAuthState,
type GoogleAuthResult,
type GscListSitesResult,
@@ -181,6 +185,8 @@ const api: FreeCrawlApi = {
ipcRenderer.invoke(IPC.crawlAddUrl, url),
projectSaveAs: (): Promise<{ filePath: string; bytesWritten: number } | null> =>
ipcRenderer.invoke(IPC.projectSaveAs),
projectSave: (): Promise<{ filePath: string; bytesWritten: number } | null> =>
ipcRenderer.invoke(IPC.projectSave),
projectOpen: (filePath?: string): Promise<{ filePath: string } | null> =>
ipcRenderer.invoke(IPC.projectOpen, filePath),
projectCurrentPath: (): Promise<string | null> =>
@@ -191,6 +197,12 @@ const api: FreeCrawlApi = {
ipcRenderer.invoke(IPC.projectConfigSet, config),
onProjectConfigChanged: (cb: (config: CrawlConfig | null) => void) =>
subscribe<CrawlConfig | null>(IPC.projectConfigChanged, cb),
browserInstallGet: (): Promise<BrowserInstallState> =>
ipcRenderer.invoke(IPC.browserInstallGet),
browserInstallStart: (): Promise<BrowserInstallState> =>
ipcRenderer.invoke(IPC.browserInstallStart),
onBrowserInstallState: (cb: (state: BrowserInstallState) => void) =>
subscribe<BrowserInstallState>(IPC.browserInstallState, cb),
recentProjectsList: (): Promise<RecentProject[]> =>
ipcRenderer.invoke(IPC.recentProjectsList),
recentProjectSetArchived: (path: string, archived: boolean): Promise<void> =>
@@ -241,6 +253,8 @@ const api: FreeCrawlApi = {
ipcRenderer.invoke(IPC.exportImages, input),
exportTabular: (input: ExportTabularInput): Promise<ExportTabularResult> =>
ipcRenderer.invoke(IPC.exportTabular, input),
exportGrid: (input: ExportGridInput): Promise<ExportGridResult> =>
ipcRenderer.invoke(IPC.exportGrid, input),
dataDeleteByDomain: (
input: DataDeleteByDomainInput,
): Promise<DataDeleteByDomainResult> =>
@@ -402,6 +416,8 @@ const api: FreeCrawlApi = {
subscribe<SpellingProgress>(IPC.spellingProgress, cb),
spellingMatches: (url: string): Promise<SpellingMatchesResult | null> =>
ipcRenderer.invoke(IPC.spellingMatches, url),
spellingLanguages: (): Promise<SpellingLanguageOption[]> =>
ipcRenderer.invoke(IPC.spellingLanguages),
googleAuthStart: (id: string): Promise<GoogleAuthResult> =>
ipcRenderer.invoke(IPC.googleAuthStart, id),
googleAuthStatus: (id: string): Promise<GoogleAuthState> =>
+5 -1
View File
@@ -127,7 +127,8 @@ export function App() {
return;
}
setConfig({ mode: 'list', urlList: urls, startUrl: urls[0] ?? '' });
setSettingsOpen(true);
// Land on Mode so the just-loaded list is what the user sees.
setSettingsOpen(true, { section: 'mode' });
setDropFlash(t('app.dropLoadedUrls', {
defaultValue: 'Loaded {{count}} URLs from {{name}} into List mode.',
count: urls.length,
@@ -412,6 +413,9 @@ export function App() {
case 'manage-projects':
setProjectsOpen(true);
break;
case 'save-project':
void window.freecrawl.projectSave();
break;
case 'save-project-as':
void window.freecrawl.projectSaveAs();
break;
@@ -18,6 +18,7 @@ import type {
UrlAnalyticsDetail,
SpellingMatchesResult,
} from '@freecrawl/shared-types';
import { GridExportButton } from './GridExportButton.js';
import { useAppStore } from '../store.js';
import { diagnoseStatus, type StatusDiagnosis } from '../utils/statusDiagnosis.js';
@@ -40,6 +41,59 @@ type SubTab =
| 'analytics'
| 'spelling';
/**
* Right-aligned Export control shared by the detail-panel sub-tabs, with
* optional status text (e.g. "Showing 500 of 1,703") on the left.
*/
function SubTabExportBar({
fileName,
sheetName,
disabled,
getData,
children,
}: {
fileName: string;
sheetName?: string;
disabled?: boolean;
getData: () => { headers: string[]; rows: (string | number | boolean | null)[][] };
children?: ReactNode;
}) {
return (
<div className="flex shrink-0 items-center gap-3 px-3 pt-2 text-[11px] text-surface-500">
{children}
<GridExportButton
className="ml-auto"
fileName={fileName}
sheetName={sheetName}
disabled={disabled}
getData={getData}
/>
</div>
);
}
/**
* Turn a page URL into a file-name stem for the detail-panel exports:
* host + path, punctuation collapsed to single dashes. Falls back to
* "url-details" when nothing is selected so the dialog still opens with
* a sane name.
*/
function urlExportStem(url: string | undefined): string {
if (!url) return 'url-details';
let raw = url;
try {
const u = new URL(url);
raw = u.hostname.replace(/^www\./i, '') + u.pathname;
} catch {
/* not parseable — fall back to the raw string */
}
const stem = raw
.replace(/[^a-zA-Z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 60);
return stem || 'url-details';
}
const SUB_TABS: { key: SubTab; label: string; disabled?: boolean }[] = [
{ key: 'url-details', label: 'URL Details' },
{ key: 'outline', label: 'Outline' },
@@ -246,9 +300,20 @@ export function BottomDetailPanel() {
const showSingleScopeBanner = isMulti && SINGLE_URL_ONLY_TABS.has(subTab);
// Shared file-name stem for every sub-tab's Export button, so a saved
// file says which page it came from ("example.com-blog-post-1-inlinks")
// instead of a generic name the user has to rename.
const exportStem = useMemo(
() => (isMulti ? `${multiPages.length}-urls` : urlExportStem(detail?.row.url)),
[isMulti, multiPages.length, detail?.row.url],
);
return (
<div className="flex h-full flex-col bg-surface-950">
<div className="flex items-center border-b border-surface-800 bg-surface-900">
{/* Single row that scrolls sideways, matching the main TabsBar. The
scrollbar sits directly under the tabs; `shrink-0` stops the flex
column from squeezing the bar when the panel is short. */}
<div className="flex shrink-0 items-center overflow-x-auto border-b border-surface-800 bg-surface-900">
{SUB_TABS.map((tab) => (
<button
key={tab.key}
@@ -306,12 +371,15 @@ export function BottomDetailPanel() {
{effectiveIds.length > 0 && !detail && loading && (
<div className="p-4 text-xs text-surface-500">{tx('common.loading', { defaultValue: 'Loading…' })}</div>
)}
{detail && subTab === 'url-details' && <NameValueView row={detail.row} />}
{detail && subTab === 'url-details' && (
<NameValueView row={detail.row} exportName={exportStem} />
)}
{subTab === 'inlinks' &&
(isMulti ? (
details.length > 0 && (
<LinksView
tableId="inlinks-multi"
exportName={`${exportStem}-inlinks`}
selectedUrlId={detail?.row.id ?? null}
total={aggregatedInlinksTotal}
shown={aggregatedInlinks.length}
@@ -323,6 +391,7 @@ export function BottomDetailPanel() {
detail && (
<LinksView
tableId="inlinks"
exportName={`${exportStem}-inlinks`}
selectedUrlId={detail.row.id}
total={detail.inlinksTotal}
shown={detail.inlinks.length}
@@ -352,6 +421,7 @@ export function BottomDetailPanel() {
details.length > 0 && (
<LinksView
tableId="outlinks-multi"
exportName={`${exportStem}-outlinks`}
selectedUrlId={detail?.row.id ?? null}
total={aggregatedOutlinksTotal}
shown={aggregatedOutlinks.length}
@@ -363,6 +433,7 @@ export function BottomDetailPanel() {
detail && (
<LinksView
tableId="outlinks"
exportName={`${exportStem}-outlinks`}
selectedUrlId={detail.row.id}
total={detail.outlinksTotal}
shown={detail.outlinks.length}
@@ -387,31 +458,57 @@ export function BottomDetailPanel() {
/>
)
))}
{detail && subTab === 'outline' && <OutlineView row={detail.row} />}
{detail && subTab === 'outline' && (
<OutlineView row={detail.row} exportName={exportStem} />
)}
{subTab === 'images' &&
(isMulti ? (
<MultiImagesView pages={multiPages} />
<MultiImagesView pages={multiPages} exportName={exportStem} />
) : (
detail && <ImagesView urlId={detail.row.id} row={detail.row} />
detail && (
<ImagesView
urlId={detail.row.id}
row={detail.row}
exportName={exportStem}
/>
)
))}
{subTab === 'resources' &&
(isMulti ? (
<MultiResourcesView pages={multiPages} />
<MultiResourcesView pages={multiPages} exportName={exportStem} />
) : (
detail && <ResourcesView urlId={detail.row.id} row={detail.row} />
detail && (
<ResourcesView
urlId={detail.row.id}
row={detail.row}
exportName={exportStem}
/>
)
))}
{detail && subTab === 'extracted-data' && (
<ExtractedDataView row={detail.row} />
<ExtractedDataView row={detail.row} exportName={exportStem} />
)}
{detail && subTab === 'serp-snippet' && <SerpSnippet row={detail.row} />}
{detail && subTab === 'http-headers' && (
<HttpHeadersView headers={detail.headers} row={detail.row} />
<HttpHeadersView
headers={detail.headers}
row={detail.row}
exportName={exportStem}
/>
)}
{detail && subTab === 'cookies' && (
<CookiesView row={detail.row} headers={detail.headers} />
<CookiesView
row={detail.row}
headers={detail.headers}
exportName={exportStem}
/>
)}
{detail && subTab === 'structured-data' && (
<StructuredDataView urlId={detail.row.id} row={detail.row} />
<StructuredDataView
urlId={detail.row.id}
row={detail.row}
exportName={exportStem}
/>
)}
{detail && subTab === 'view-source' && (
<ViewSourceView urlId={detail.row.id} pageUrl={detail.row.url} kind="raw" />
@@ -423,13 +520,17 @@ export function BottomDetailPanel() {
<ScreenshotView urlId={detail.row.id} />
)}
{detail && subTab === 'duplicates' && (
<DuplicatesView urlId={detail.row.id} row={detail.row} />
<DuplicatesView
urlId={detail.row.id}
row={detail.row}
exportName={exportStem}
/>
)}
{detail && subTab === 'analytics' && (
<AnalyticsView pageUrl={detail.row.url} />
<AnalyticsView pageUrl={detail.row.url} exportName={exportStem} />
)}
{detail && subTab === 'spelling' && (
<SpellingView pageUrl={detail.row.url} />
<SpellingView pageUrl={detail.row.url} exportName={exportStem} />
)}
</div>
</div>
@@ -453,7 +554,13 @@ function issueTypeClass(issueType: string): string {
* context so the user can see the error in situ, with suggested
* replacements listed beside it.
*/
function SpellingView({ pageUrl }: { pageUrl: string }) {
function SpellingView({
pageUrl,
exportName,
}: {
pageUrl: string;
exportName: string;
}) {
const { t } = useTranslation();
const [data, setData] = useState<SpellingMatchesResult | null>(null);
const [loaded, setLoaded] = useState(false);
@@ -496,21 +603,51 @@ function SpellingView({ pageUrl }: { pageUrl: string }) {
</div>
);
}
// Not failures — LanguageTool has no rules for this language, or the
// results were so noisy the language must have been wrong. Either way the
// honest answer is "no findings", not a list of false positives.
if (data.status === 'unsupported' || data.status === 'mismatch') {
return (
<div className="p-3 text-[11px] text-amber-300">
{data.error ??
t('spelling.unsupported', {
defaultValue:
'Not checked — LanguageTool has no rules for this page’s language.',
})}
{data.detectedLanguage && (
<span className="ml-1 text-surface-500">
{t('spelling.detectedAs', {
defaultValue: '(detected: {{lang}})',
lang: data.detectedLanguage,
})}
</span>
)}
</div>
);
}
if (data.status === 'skipped') {
return (
<div className="p-3 text-[11px] text-surface-500">
{t('spelling.skipped', {
defaultValue: 'Skipped — the page has too little prose to check.',
})}
{data.error ??
t('spelling.skipped', {
defaultValue: 'Skipped — the page has too little prose to check.',
})}
</div>
);
}
if (data.matches.length === 0) {
return (
<div className="p-3 text-[11px] text-emerald-400">
{t('spelling.clean', {
defaultValue: 'No spelling or grammar issues found.',
})}
{/* Saying "no grammar issues" would be a claim the offline
dictionary never made — it only ever looked at spelling. */}
{data.engine === 'local'
? t('spelling.cleanLocal', {
defaultValue:
'No spelling mistakes found. Grammar was not checked — LanguageTool has no rules for this language, so an offline dictionary was used instead.',
})
: t('spelling.clean', {
defaultValue: 'No spelling or grammar issues found.',
})}
</div>
);
}
@@ -530,6 +667,53 @@ function SpellingView({ pageUrl }: { pageUrl: string }) {
<span className="text-surface-200">{data.language}</span>
</span>
)}
{/* The engine determines what these findings can and cannot cover,
so it belongs next to them rather than buried in a tooltip. */}
{data.engine === 'local' && (
<span className="text-amber-400">
{t('spelling.engineLocal', {
defaultValue: 'offline dictionary — spelling only, no grammar',
})}
</span>
)}
{/* A `lang` attribute that contradicts the page's own text is worth
surfacing on its own — it misleads search engines and screen
readers, not just this checker. */}
{data.detectedLanguage &&
data.declaredLanguage &&
data.detectedLanguage !== data.declaredLanguage && (
<span className="text-amber-400">
{t('spelling.langConflict', {
defaultValue:
'html[lang] says {{declared}}, the text reads as {{detected}}',
declared: data.declaredLanguage,
detected: data.detectedLanguage,
})}
</span>
)}
<GridExportButton
className="ml-auto"
fileName={`${exportName}-spelling`}
sheetName="Spelling"
getData={() => ({
headers: [
t('spelling.colType', { defaultValue: 'Type' }),
t('spelling.colCategory', { defaultValue: 'Category' }),
t('spelling.colText', { defaultValue: 'Text' }),
t('spelling.colSuggestions', { defaultValue: 'Suggestions' }),
t('spelling.colMessage', { defaultValue: 'Message' }),
t('spelling.colContext', { defaultValue: 'Context' }),
],
rows: data.matches.map((m) => [
m.issueType,
m.category,
m.text,
m.replacements.join(', '),
m.message,
m.context,
]),
})}
/>
</div>
<ol className="flex-1 overflow-y-auto text-[11px]">
{data.matches.map((m, i) => {
@@ -581,7 +765,13 @@ function SpellingView({ pageUrl }: { pageUrl: string }) {
);
}
function AnalyticsView({ pageUrl }: { pageUrl: string }) {
function AnalyticsView({
pageUrl,
exportName,
}: {
pageUrl: string;
exportName: string;
}) {
const { t } = useTranslation();
const [data, setData] = useState<UrlAnalyticsDetail | null>(null);
const [loaded, setLoaded] = useState(false);
@@ -620,7 +810,55 @@ function AnalyticsView({ pageUrl }: { pageUrl: string }) {
}
const { gsc, ga4, inspection } = data;
// The cards are three unrelated metric sets; exporting them as one
// metric/value sheet keeps a single file per URL rather than three.
const exportRows = (): (string | number | boolean | null)[][] => {
const out: (string | number | boolean | null)[][] = [];
if (gsc) {
out.push(['Search Console', 'Clicks', gsc.clicks]);
out.push(['Search Console', 'Impressions', gsc.impressions]);
out.push(['Search Console', 'CTR', `${(gsc.ctr * 100).toFixed(1)}%`]);
out.push(['Search Console', 'Avg position', gsc.position.toFixed(1)]);
}
if (ga4) {
out.push(['Analytics 4', 'Sessions', ga4.sessions]);
out.push(['Analytics 4', 'Users', ga4.users]);
out.push(['Analytics 4', 'Pageviews', ga4.pageviews]);
out.push([
'Analytics 4',
'Engagement rate',
`${(ga4.engagementRate * 100).toFixed(1)}%`,
]);
out.push([
'Analytics 4',
'Avg session duration',
Math.round(ga4.avgSessionDuration),
]);
}
if (inspection) {
out.push(['URL Inspection', 'Verdict', inspection.verdict ?? '']);
out.push(['URL Inspection', 'Coverage state', inspection.coverageState ?? '']);
out.push(['URL Inspection', 'Robots state', inspection.robotsTxtState ?? '']);
out.push(['URL Inspection', 'Indexing state', inspection.indexingState ?? '']);
out.push(['URL Inspection', 'Canonical (Google)', inspection.googleCanonical ?? '']);
out.push(['URL Inspection', 'Last crawl', inspection.lastCrawlTime ?? '']);
}
return out;
};
return (
<div className="flex h-full flex-col">
<SubTabExportBar
fileName={`${exportName}-analytics`}
sheetName="Analytics"
getData={() => ({
headers: [
t('analyticsView.colSource', { defaultValue: 'Source' }),
t('analyticsView.colMetric', { defaultValue: 'Metric' }),
t('analyticsView.colValue', { defaultValue: 'Value' }),
],
rows: exportRows(),
})}
/>
<div className="grid grid-cols-1 gap-3 overflow-y-auto p-3 lg:grid-cols-3">
<AnalyticsCard
title={t('analyticsView.gscTitle', { defaultValue: 'Search Console' })}
@@ -686,6 +924,7 @@ function AnalyticsView({ pageUrl }: { pageUrl: string }) {
)}
</AnalyticsCard>
</div>
</div>
);
}
@@ -1177,7 +1416,15 @@ function ViewSourceScrollEffect({
return null;
}
function DuplicatesView({ urlId, row }: { urlId: number; row: CrawlUrlRow }) {
function DuplicatesView({
urlId,
row,
exportName,
}: {
urlId: number;
row: CrawlUrlRow;
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
const [members, setMembers] = useState<UrlClusterMember[] | null>(null);
@@ -1230,6 +1477,32 @@ function DuplicatesView({ urlId, row }: { urlId: number; row: CrawlUrlRow }) {
<span className="text-surface-500">
{t('duplicates.hammingHint', { defaultValue: '(Hamming distance from this URL — lower means more similar)' })}
</span>
<GridExportButton
className="ml-auto"
fileName={`${exportName}-duplicates`}
sheetName="Duplicates"
disabled={members.length === 0}
getData={() => ({
headers: [
translateLabel('Hamming', lang),
'URL',
translateLabel('Status', lang),
translateLabel('Indexability', lang),
translateLabel('Words', lang),
translateLabel('Inlinks', lang),
translateLabel('Title', lang),
],
rows: members.map((m) => [
m.hammingDistance,
m.url,
m.statusCode ?? '',
m.indexability ?? '',
m.wordCount ?? '',
m.inlinks ?? '',
m.title ?? '',
]),
})}
/>
</div>
<div className="flex-1 overflow-auto">
<table className="w-full text-[11px]">
@@ -1344,9 +1617,11 @@ function renderHighlighted(
function HttpHeadersView({
headers,
row,
exportName,
}: {
headers: { name: string; value: string }[];
row: CrawlUrlRow;
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
@@ -1413,6 +1688,15 @@ function HttpHeadersView({
})
: t('detail.headersCapturedNote', { defaultValue: 'Captured at crawl time.' })}
</span>
<GridExportButton
fileName={`${exportName}-http-headers-${side}`}
sheetName={side === 'request' ? 'Request Headers' : 'Response Headers'}
disabled={rows.length === 0}
getData={() => ({
headers: [translateLabel('Header', lang), translateLabel('Value', lang)],
rows: rows.map((h) => [h.name, h.value]),
})}
/>
</div>
<div className="flex-1 overflow-auto">
{showDiag && <StatusDiagnosisBanner diag={diag} />}
@@ -1593,7 +1877,7 @@ function fleschBand(score: number): string {
return 'very difficult';
}
function NameValueView({ row }: { row: CrawlUrlRow }) {
function NameValueView({ row, exportName }: { row: CrawlUrlRow; exportName: string }) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
// Server-side pixel-width is the source of truth (drives the issue
@@ -1911,6 +2195,17 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
return (
<div className="p-3">
<SubTabExportBar
fileName={`${exportName}-url-details`}
sheetName="URL Details"
getData={() => ({
headers: [translateLabel('Name', lang), translateLabel('Value', lang)],
rows: fields.map(([label, value]) => [
translateLabel(label, lang),
value === null || value === undefined ? '' : String(value),
]),
})}
/>
<table className="w-full text-[11px]">
<thead className="sticky top-0 bg-surface-900">
<tr className="text-surface-400">
@@ -2046,6 +2341,7 @@ function LinksView({
shown,
columns,
rows,
exportName,
}: {
tableId: string;
selectedUrlId: number | null;
@@ -2053,6 +2349,8 @@ function LinksView({
shown: number;
columns: LinksColumn[];
rows: string[][];
/** File-name stem for the Export button (already page-specific). */
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
@@ -2345,11 +2643,22 @@ function LinksView({
return (
<div ref={rootRef} className="relative flex h-full select-none flex-col">
<div className="shrink-0 px-3 pt-2 text-[11px] text-surface-500">
{t('links.showing', { defaultValue: 'Showing' })}{' '}
<span className="font-mono text-surface-200">{shown.toLocaleString()}</span>{' '}
{t('links.of', { defaultValue: 'of' })}{' '}
<span className="font-mono text-surface-200">{total.toLocaleString()}</span>
<div className="flex shrink-0 items-center gap-3 px-3 pt-2 text-[11px] text-surface-500">
<span>
{t('links.showing', { defaultValue: 'Showing' })}{' '}
<span className="font-mono text-surface-200">{shown.toLocaleString()}</span>{' '}
{t('links.of', { defaultValue: 'of' })}{' '}
<span className="font-mono text-surface-200">{total.toLocaleString()}</span>
</span>
<GridExportButton
className="ml-auto"
fileName={exportName}
disabled={rows.length === 0}
getData={() => ({
headers: columns.map((c) => translateLabel(c.header, lang)),
rows,
})}
/>
</div>
{rows.length === 0 ? (
<div className="py-8 text-center text-xs text-surface-500">{t('links.noLinks', { defaultValue: 'No links.' })}</div>
@@ -2955,9 +3264,11 @@ function splitJoinedSetCookie(joined: string): string[] {
function CookiesView({
row,
headers,
exportName,
}: {
row: CrawlUrlRow;
headers: { name: string; value: string }[];
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
@@ -3004,6 +3315,31 @@ function CookiesView({
{t('cookies.missing', { defaultValue: '{{n}} missing', n: row.cookiesNoSameSite })} <code>SameSite</code>
</span>
)}
<GridExportButton
className="ml-auto"
fileName={`${exportName}-cookies`}
sheetName="Cookies"
getData={() => ({
headers: [
translateLabel('Name', lang),
translateLabel('Domain', lang),
translateLabel('Path', lang),
translateLabel('Expires', lang),
'Secure',
'HttpOnly',
'SameSite',
],
rows: cookies.map((c) => [
c.name,
c.domain ?? '',
c.path ?? '/',
c.expires ?? (c.maxAge ? `Max-Age ${c.maxAge}` : 'session'),
c.secure,
c.httpOnly,
c.sameSite ?? '',
]),
})}
/>
</div>
<table className="w-full text-[11px]">
<thead className="sticky top-0 bg-surface-900">
@@ -3094,9 +3430,11 @@ function extractJsonLdBlocks(html: string): JsonLdBlock[] {
function StructuredDataView({
urlId,
row,
exportName,
}: {
urlId: number | null;
row: CrawlUrlRow;
exportName: string;
}) {
const { t } = useTranslation();
const [src, setSrc] = useState<UrlSourceResult | null>(null);
@@ -3156,6 +3494,26 @@ function StructuredDataView({
<span>
<span className="font-medium text-surface-200">{row.rdfaCount}</span> {t('structured.rdfaAttrs', { defaultValue: 'RDFa attrs' })}
</span>
<GridExportButton
className="ml-auto"
fileName={`${exportName}-structured-data`}
sheetName="Structured Data"
disabled={blocks.length === 0}
getData={() => ({
headers: [
t('structured.colBlock', { defaultValue: 'Block' }),
t('structured.colValid', { defaultValue: 'Valid JSON' }),
t('structured.colTypes', { defaultValue: 'Types' }),
t('structured.colJson', { defaultValue: 'JSON' }),
],
rows: blocks.map((b) => [
b.index + 1,
b.ok,
types.join(', '),
b.raw,
]),
})}
/>
</div>
<div className="flex-1 overflow-auto p-3">
@@ -3230,9 +3588,11 @@ function StructuredDataView({
function ImagesView({
urlId,
row,
exportName,
}: {
urlId: number | null;
row: CrawlUrlRow;
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
@@ -3311,6 +3671,29 @@ function ImagesView({
})})
</span>
)}
<GridExportButton
className="ml-auto"
fileName={`${exportName}-images`}
sheetName="Images"
getData={() => ({
headers: [
translateLabel('Source', lang),
translateLabel('Alt', lang),
'W',
'H',
translateLabel('Size', lang),
translateLabel('Scope', lang),
],
rows: rows.map((r) => [
r.src,
r.alt ?? '',
r.width ?? '',
r.height ?? '',
r.byteSize ?? '',
r.isInternal ? 'internal' : 'external',
]),
})}
/>
</div>
<div className="flex-1 overflow-auto p-3">
<table className="w-full text-[11px]">
@@ -3483,12 +3866,32 @@ function extractResources(html: string, pageUrl: string): ResourceEntry[] {
return out;
}
/**
* Loading hints shown as chips next to a subresource (async / defer /
* module / SRI / …). Shared by the Resources table and its Export button
* so the exported "Hints" column always matches what is on screen.
*/
function resourceHints(r: { attrs: Record<string, string> }): string[] {
const hints: string[] = [];
if (r.attrs['async'] !== undefined) hints.push('async');
if (r.attrs['defer'] !== undefined) hints.push('defer');
if ((r.attrs['type'] ?? '').toLowerCase() === 'module') hints.push('module');
if (r.attrs['crossorigin']) {
hints.push(`crossorigin=${r.attrs['crossorigin'] || 'anonymous'}`);
}
if (r.attrs['integrity']) hints.push('SRI');
if ((r.attrs['media'] ?? '').toLowerCase() === 'print') hints.push('print-only');
return hints;
}
function ResourcesView({
urlId,
row,
exportName,
}: {
urlId: number | null;
row: CrawlUrlRow;
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
@@ -3576,6 +3979,28 @@ function ResourcesView({
{translateLabel(f.label, lang)} <span className="text-surface-500">({f.count})</span>
</button>
))}
<GridExportButton
className="ml-auto"
fileName={`${exportName}-resources`}
sheetName="Resources"
disabled={filtered.length === 0}
getData={() => ({
headers: [
translateLabel('Type', lang),
'URL',
'3rd-party',
translateLabel('Hints', lang),
],
// Mirrors the visible table: the active filter applies, and
// the hint chips are flattened to one space-separated cell.
rows: filtered.map((r) => [
r.type,
r.url,
r.isExternal,
resourceHints(r).join(' '),
]),
})}
/>
</div>
<div className="flex-1 overflow-auto p-3">
{filtered.length === 0 ? (
@@ -3592,21 +4017,7 @@ function ResourcesView({
</thead>
<tbody>
{filtered.map((r, i) => {
const hints: string[] = [];
if (r.attrs['async'] !== undefined) hints.push('async');
if (r.attrs['defer'] !== undefined) hints.push('defer');
if (
(r.attrs['type'] ?? '').toLowerCase() === 'module'
) {
hints.push('module');
}
if (r.attrs['crossorigin']) {
hints.push(`crossorigin=${r.attrs['crossorigin'] || 'anonymous'}`);
}
if (r.attrs['integrity']) hints.push('SRI');
if ((r.attrs['media'] ?? '').toLowerCase() === 'print') {
hints.push('print-only');
}
const hints = resourceHints(r);
return (
<tr
key={`${r.url}-${i}`}
@@ -3659,7 +4070,13 @@ interface MultiPage {
* shows them in a single flat table with a leading Page column so the
* source URL of each image is unambiguous.
*/
function MultiImagesView({ pages }: { pages: MultiPage[] }) {
function MultiImagesView({
pages,
exportName,
}: {
pages: MultiPage[];
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
const [byPage, setByPage] = useState<Map<number, UrlPageImageRow[]>>(new Map());
@@ -3734,6 +4151,33 @@ function MultiImagesView({ pages }: { pages: MultiPage[] }) {
{largeCount > 0 && (
<span className="text-amber-400">{t('imagesView.largeCount', { defaultValue: '{{n}} > 100 KB', n: largeCount })}</span>
)}
<GridExportButton
className="ml-auto"
fileName={`${exportName}-images`}
sheetName="Images"
getData={() => ({
headers: [
translateLabel('Page', lang),
translateLabel('Source', lang),
translateLabel('Alt', lang),
'W',
'H',
translateLabel('Size', lang),
translateLabel('Scope', lang),
],
rows: pages.flatMap((p) =>
(byPage.get(p.id) ?? []).map((r) => [
p.row.url,
r.src,
r.alt ?? '',
r.width ?? '',
r.height ?? '',
r.byteSize ?? '',
r.isInternal ? 'internal' : 'external',
]),
),
})}
/>
</div>
<div className="flex-1 overflow-auto p-3">
<table className="w-full text-[11px]">
@@ -3821,7 +4265,13 @@ function MultiImagesView({ pages }: { pages: MultiPage[] }) {
* parallel, runs the same `extractResources` parser per page, and merges
* everything into one flat table with a leading Page column.
*/
function MultiResourcesView({ pages }: { pages: MultiPage[] }) {
function MultiResourcesView({
pages,
exportName,
}: {
pages: MultiPage[];
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
const [byPage, setByPage] = useState<Map<number, ResourceEntry[]>>(new Map());
@@ -3938,6 +4388,26 @@ function MultiResourcesView({ pages }: { pages: MultiPage[] }) {
{t('resources.missingSnapshots', { defaultValue: '{{missing}} of {{total}} pages have no stored body', missing: missingSnapshots, total: pages.length })}
</span>
)}
<GridExportButton
className="ml-auto"
fileName={`${exportName}-resources`}
sheetName="Resources"
disabled={filtered.length === 0}
getData={() => ({
headers: [
translateLabel('Page', lang),
translateLabel('Type', lang),
'URL',
'3rd-party',
],
rows: filtered.map((x) => [
x.page.row.url,
x.entry.type,
x.entry.url,
x.entry.isExternal,
]),
})}
/>
</div>
<div className="flex-1 overflow-auto p-3">
{filtered.length === 0 ? (
@@ -3996,7 +4466,13 @@ function MultiResourcesView({ pages }: { pages: MultiPage[] }) {
);
}
function ExtractedDataView({ row }: { row: CrawlUrlRow }) {
function ExtractedDataView({
row,
exportName,
}: {
row: CrawlUrlRow;
exportName: string;
}) {
const { t, i18n } = useTranslation();
const lang = i18n.language;
const extraction = row.extractionResults
@@ -4029,6 +4505,41 @@ function ExtractedDataView({ row }: { row: CrawlUrlRow }) {
return (
<div className="flex h-full flex-col gap-3 overflow-auto p-3">
<SubTabExportBar
fileName={`${exportName}-extracted-data`}
sheetName="Extracted Data"
getData={() => ({
headers: [
t('extraction.colSource', { defaultValue: 'Source' }),
translateLabel('Name', lang),
translateLabel('Value', lang),
],
// Both blocks land in one sheet, tagged by source, so a rule
// named the same as a search term stays distinguishable.
rows: [
...(hasExtraction
? Object.entries(extraction as Record<string, unknown>).map(
([k, v]) => [
t('extraction.customExtraction', { defaultValue: 'Custom Extraction' }),
k,
v === null || v === undefined
? ''
: typeof v === 'object'
? JSON.stringify(v)
: String(v),
],
)
: []),
...(hasSearch
? Object.entries(search as Record<string, unknown>).map(([k, v]) => [
t('extraction.customSearch', { defaultValue: 'Custom Search' }),
k,
v === null || v === undefined ? '' : String(v),
])
: []),
] as (string | number | boolean | null)[][],
})}
/>
{hasExtraction && (
<section>
<div className="mb-2 text-[10px] uppercase tracking-wide text-surface-500">
@@ -4136,7 +4647,7 @@ function parseHeadings(json: string | null): HeadingEntry[] {
}
}
function OutlineView({ row }: { row: CrawlUrlRow }) {
function OutlineView({ row, exportName }: { row: CrawlUrlRow; exportName: string }) {
const { t } = useTranslation();
const outline = parseHeadings(row.headings);
@@ -4188,6 +4699,19 @@ function OutlineView({ row }: { row: CrawlUrlRow }) {
{outline.length === 200 && (
<span className="text-surface-500">{t('outline.capped', { defaultValue: '(capped at 200)' })}</span>
)}
<GridExportButton
className="ml-auto"
fileName={`${exportName}-outline`}
sheetName="Outline"
getData={() => ({
headers: [
t('outline.colLevel', { defaultValue: 'Level' }),
t('outline.colText', { defaultValue: 'Heading' }),
t('outline.colIssue', { defaultValue: 'Issue' }),
],
rows: annotated.map((h) => [`h${h.level}`, h.text, h.skipped ?? '']),
})}
/>
</div>
<div className="flex-1 overflow-auto p-3">
<ol className="space-y-1 text-[11px]">
@@ -0,0 +1,81 @@
import { useState } from 'react';
import { Download } from 'lucide-react';
import clsx from 'clsx';
import { useTranslation } from 'react-i18next';
export interface GridExportData {
headers: string[];
rows: (string | number | boolean | null)[][];
}
/**
* Per-sub-tab export button for the URL Details panel.
*
* Every sub-tab shows a different table, so rather than one panel-wide
* export that would have to know all of them, each view drops this button
* in its own header and hands over its own rows. The data is produced
* lazily by `getData()` — building a few thousand cells is wasted work on
* every render when the user may never click.
*
* Format (CSV vs Excel) is chosen in the native save dialog by the file
* type the user picks, which keeps the button itself to a single control.
*/
export function GridExportButton({
fileName,
sheetName,
getData,
disabled,
className,
}: {
/** File name stem, without extension. Sanitised in the main process. */
fileName: string;
/** Worksheet name for xlsx. Defaults to the file name. */
sheetName?: string;
getData: () => GridExportData;
disabled?: boolean;
className?: string;
}) {
const { t } = useTranslation();
const [busy, setBusy] = useState(false);
const run = async (): Promise<void> => {
if (busy || disabled) return;
setBusy(true);
try {
const { headers, rows } = getData();
if (headers.length === 0) return;
await window.freecrawl.exportGrid({ fileName, sheetName, headers, rows });
} catch {
// The main process logs the cause; a failed export must not take
// the panel down with it.
} finally {
setBusy(false);
}
};
const inert = disabled || busy;
return (
<button
type="button"
onClick={() => void run()}
disabled={inert}
className={clsx(
'inline-flex shrink-0 items-center gap-1.5 rounded border px-2 py-0.5 text-[11px] transition',
inert
? 'cursor-not-allowed border-surface-800 text-surface-600'
: 'border-surface-700 text-surface-300 hover:bg-surface-800',
className,
)}
title={
disabled
? t('detailExport.nothing', { defaultValue: 'Nothing to export' })
: t('detailExport.title', {
defaultValue: 'Export this table to Excel or CSV',
})
}
>
<Download className="h-3.5 w-3.5" />
<span>{t('detailExport.label', { defaultValue: 'Export' })}</span>
</button>
);
}
@@ -1,5 +1,6 @@
import { Info } from 'lucide-react';
import { useTranslation } from 'react-i18next';
import { translateInfoTip } from '../i18n/info-tips.js';
export interface FieldInfo {
/** One-line description of what the field/column means. */
@@ -16,24 +17,31 @@ export interface FieldInfo {
* column-header sort/resize controls underneath.
*/
export function InfoTip({ info, example }: FieldInfo) {
const { t } = useTranslation();
const { t, i18n } = useTranslation();
if (!info && !example) return null;
// Tooltip bodies are passed in as English literals by ~170 column
// specs and settings fields; the translation happens here, once, so
// none of those call sites has to thread a `t()` call through. Unknown
// strings — and deliberately untranslated technical examples like
// regexes and header values — fall through unchanged.
const localizedInfo = translateInfoTip(info, i18n.language);
const localizedExample = translateInfoTip(example, i18n.language);
return (
<span className="group relative inline-flex">
<Info
className="h-3 w-3 cursor-help text-surface-500 transition-colors group-hover:text-surface-200"
tabIndex={0}
aria-label={info ?? t('infoTip.moreInfo', { defaultValue: 'More info' })}
aria-label={localizedInfo ?? t('infoTip.moreInfo', { defaultValue: 'More info' })}
/>
<span className="pointer-events-none invisible absolute left-4 top-0 z-50 w-64 rounded border border-surface-700 bg-surface-900 p-2 text-[10px] leading-relaxed text-surface-200 opacity-0 shadow-xl transition-opacity duration-150 group-hover:visible group-hover:opacity-100 group-focus-within:visible group-focus-within:opacity-100">
{info && <span className="block">{info}</span>}
{example && (
{localizedInfo && <span className="block">{localizedInfo}</span>}
{localizedExample && (
<>
<span className="mt-1.5 block text-[9px] font-semibold uppercase tracking-wider text-surface-500">
{t('infoTip.example', { defaultValue: 'Example' })}
</span>
<span className="mt-0.5 block break-words font-mono text-[10px] text-surface-300">
{example}
{localizedExample}
</span>
</>
)}
@@ -1,4 +1,4 @@
import { useEffect, useMemo, useState } from 'react';
import { useEffect, useMemo, useRef, useState, type RefObject } from 'react';
import { useTranslation, Trans } from 'react-i18next';
import {
SUPPORTED_LANGUAGES,
@@ -40,6 +40,7 @@ import {
BookOpen,
Chrome,
Target,
Check,
type LucideIcon,
} from 'lucide-react';
import clsx from 'clsx';
@@ -52,6 +53,7 @@ import type {
HttpAuth,
IntegrationDef,
IntegrationsState,
SpellingLanguageOption,
} from '@freecrawl/shared-types';
import { DEFAULT_CRAWL_CONFIG, INTEGRATIONS } from '@freecrawl/shared-types';
import { useAppStore } from '../store.js';
@@ -189,7 +191,11 @@ interface FormState {
budgetMaxCls: string;
}
type SectionKey =
/**
* Exported so callers elsewhere in the app can deep-link into a panel —
* see `SettingsTarget` in the store.
*/
export type SettingsSectionKey =
| 'presets'
| 'mode'
| 'crawler'
@@ -218,6 +224,9 @@ type SectionKey =
| 'privacy'
| 'language';
/** Local alias — the exported name is the one other modules import. */
type SectionKey = SettingsSectionKey;
interface SectionDef {
key: SectionKey;
label: string;
@@ -550,6 +559,7 @@ export function SettingsDialog({ open, onClose }: Props) {
const lang = i18n.language;
const config = useAppStore((s) => s.config);
const setConfig = useAppStore((s) => s.setConfig);
const target = useAppStore((s) => s.settingsTarget);
const [form, setForm] = useState<FormState>(() => configToForm(config));
const [active, setActive] = useState<SectionKey>('mode');
const [search, setSearch] = useState('');
@@ -564,6 +574,12 @@ export function SettingsDialog({ open, onClose }: Props) {
}
}, [open, config]);
// Follow a deep link ("Add API Key…" and friends). Without a target the
// dialog stays on whichever section the user last used.
useEffect(() => {
if (open && target) setActive(target.section);
}, [open, target]);
// ESC closes — common modal expectation.
useEffect(() => {
if (!open) return;
@@ -948,7 +964,9 @@ export function SettingsDialog({ open, onClose }: Props) {
{active === 'per-host-ua' && (
<PerHostUaPanel form={form} update={update} />
)}
{active === 'integrations' && <IntegrationsPanel />}
{active === 'integrations' && (
<IntegrationsPanel focusId={target?.integration ?? null} />
)}
{active === 'rendering' && (
<RenderingPanel form={form} update={update} />
)}
@@ -1127,36 +1145,89 @@ function PresetsPanel({
const { t, i18n } = useTranslation();
const [busy, setBusy] = useState(false);
const [message, setMessage] = useState<string | null>(null);
// Which preset was just applied. Drives the confirmation banner + the
// button's "Applied ✓" state so the click has a visible result — the
// fields it changes live on other panels, so without this the button
// looks inert.
const [appliedKey, setAppliedKey] = useState<string | null>(null);
const appliedTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
useEffect(
() => () => {
if (appliedTimer.current) clearTimeout(appliedTimer.current);
},
[],
);
const onApply = (p: PresetDef) => {
apply(p);
setAppliedKey(p.key);
if (appliedTimer.current) clearTimeout(appliedTimer.current);
appliedTimer.current = setTimeout(() => setAppliedKey(null), 4000);
};
const appliedPreset = PRESETS.find((p) => p.key === appliedKey) ?? null;
return (
<>
<p className="mb-3 text-[11px] text-surface-400">
{t('settingsPanels.presets.intro', { defaultValue: 'One-click profiles for common crawl scenarios. Clicking a preset overwrites the affected fields only — your URL list, custom rules, filters, and extraction rules are preserved.' })}
</p>
{appliedPreset && (
<div className="mb-3 flex items-center gap-2 rounded border border-emerald-700/50 bg-emerald-900/25 px-3 py-1.5 text-[11px] text-emerald-200">
<Check className="h-3.5 w-3.5 shrink-0" />
<span>
{t('settingsPanels.presets.applied', {
defaultValue:
'"{{name}}" preset applied — review the panels, then press Save to keep the changes.',
name: translateLabel(appliedPreset.label, i18n.language),
})}
</span>
</div>
)}
<div className="space-y-2">
{PRESETS.map((p) => (
<div
key={p.key}
className="flex items-start gap-3 rounded border border-surface-800 bg-surface-950/40 p-3"
>
<div className="flex-1 min-w-0">
<div className="text-[12px] font-medium text-surface-100">{translateLabel(p.label, i18n.language)}</div>
<div className="mt-0.5 text-[11px] text-surface-400">{translateLabel(p.description, i18n.language)}</div>
<div className="mt-1.5 flex flex-wrap gap-1 font-mono text-[10px] text-surface-500">
{Object.entries(p.overrides).map(([k, v]) => (
<span key={k} className="rounded border border-surface-800 px-1.5 py-0.5">
{k}={String(v)}
</span>
))}
</div>
</div>
<button
className="rounded border border-blue-700/60 bg-blue-900/30 px-3 py-1 text-[11px] text-blue-200 hover:bg-blue-900/50"
onClick={() => apply(p)}
{PRESETS.map((p) => {
const justApplied = appliedKey === p.key;
return (
<div
key={p.key}
className={clsx(
'flex items-start gap-3 rounded border bg-surface-950/40 p-3 transition-colors',
justApplied ? 'border-emerald-700/50' : 'border-surface-800',
)}
>
{t('common.apply', { defaultValue: 'Apply' })}
</button>
</div>
))}
<div className="flex-1 min-w-0">
<div className="text-[12px] font-medium text-surface-100">{translateLabel(p.label, i18n.language)}</div>
<div className="mt-0.5 text-[11px] text-surface-400">{translateLabel(p.description, i18n.language)}</div>
<div className="mt-1.5 flex flex-wrap gap-1 font-mono text-[10px] text-surface-500">
{Object.entries(p.overrides).map(([k, v]) => (
<span key={k} className="rounded border border-surface-800 px-1.5 py-0.5">
{k}={String(v)}
</span>
))}
</div>
</div>
<button
className={clsx(
'inline-flex items-center gap-1 rounded border px-3 py-1 text-[11px] transition-colors',
justApplied
? 'border-emerald-600/70 bg-emerald-900/40 text-emerald-200'
: 'border-blue-700/60 bg-blue-900/30 text-blue-200 hover:bg-blue-900/50',
)}
onClick={() => onApply(p)}
>
{justApplied ? (
<>
<Check className="h-3.5 w-3.5" />
{t('common.applied', { defaultValue: 'Applied' })}
</>
) : (
t('common.apply', { defaultValue: 'Apply' })
)}
</button>
</div>
);
})}
</div>
<div className="mt-5 border-t border-surface-800 pt-4">
@@ -3744,9 +3815,16 @@ function PerformanceBudgetPanel({ form, update }: PanelProps) {
);
}
function IntegrationsPanel() {
/**
* @param focusId Integration the user was sent here to configure. The
* panel lists a dozen-plus cards across four categories, so landing on
* the section alone would still leave them hunting; the matching card
* is scrolled into view and outlined for as long as the panel is open.
*/
function IntegrationsPanel({ focusId }: { focusId: string | null }) {
const { t, i18n } = useTranslation();
const [state, setState] = useState<IntegrationsState | null>(null);
const focusRef = useRef<HTMLDivElement | null>(null);
// Per-integration → per-field draft text. Only fields the user has
// actually typed into are tracked; a save sends just these so an
// untouched "saved" secret is never overwritten.
@@ -3758,6 +3836,10 @@ function IntegrationsPanel() {
void window.freecrawl.integrationsGetAll().then(setState);
}, []);
useEffect(() => {
focusRef.current?.scrollIntoView({ block: 'center' });
}, [focusId]);
const setDraft = (id: string, key: string, value: string) => {
setDrafts((d) => ({ ...d, [id]: { ...(d[id] ?? {}), [key]: value } }));
};
@@ -3813,6 +3895,8 @@ function IntegrationsPanel() {
<IntegrationCard
key={def.id}
def={def}
cardRef={def.id === focusId ? focusRef : undefined}
focused={def.id === focusId}
state={state?.[def.id]}
draft={drafts[def.id] ?? {}}
busy={busy === def.id}
@@ -3832,6 +3916,8 @@ function IntegrationsPanel() {
function IntegrationCard({
def,
cardRef,
focused,
state,
draft,
busy,
@@ -3841,6 +3927,8 @@ function IntegrationCard({
onClear,
}: {
def: IntegrationDef;
cardRef?: RefObject<HTMLDivElement | null>;
focused?: boolean;
state: IntegrationsState[string] | undefined;
draft: Record<string, string>;
busy: boolean;
@@ -3855,7 +3943,15 @@ function IntegrationCard({
const [guideOpen, setGuideOpen] = useState(false);
return (
<div className="rounded border border-surface-800 bg-surface-950/40 p-3">
<div
ref={cardRef}
className={clsx(
'rounded border bg-surface-950/40 p-3',
focused
? 'border-accent-500 ring-1 ring-accent-500/40'
: 'border-surface-800',
)}
>
<div className="mb-1 flex items-center gap-2">
<span className="text-[12px] font-medium text-surface-100">{def.name}</span>
<span className="rounded bg-surface-800 px-1.5 py-0.5 text-[9px] uppercase tracking-wide text-surface-400">
@@ -4005,6 +4101,38 @@ function SpellingPanel() {
const raw = window.freecrawl.prefsGet('spellingIgnoreWords');
return typeof raw === 'string' ? raw : '';
});
const [language, setLanguage] = useState<string>(() => {
const raw = window.freecrawl.prefsGet('spellingLanguage');
return typeof raw === 'string' ? raw : '';
});
const [languages, setLanguages] = useState<SpellingLanguageOption[]>([]);
const [languagesLoaded, setLanguagesLoaded] = useState(false);
// Asked of the configured endpoint rather than hardcoded — a self-hosted
// LanguageTool ships whichever language modules its operator installed.
useEffect(() => {
let cancelled = false;
void window.freecrawl
.spellingLanguages()
.then((list) => {
if (cancelled) return;
setLanguages(list);
setLanguagesLoaded(true);
})
.catch(() => {
if (!cancelled) setLanguagesLoaded(true);
});
return () => {
cancelled = true;
};
}, []);
function pickLanguage(code: string) {
setLanguage(code);
// Empty means auto — clear the pref rather than storing a sentinel.
if (code) window.freecrawl.prefsSet('spellingLanguage', code);
else window.freecrawl.prefsDelete('spellingLanguage');
}
function pickLevel(l: 'default' | 'picky') {
setLevel(l);
@@ -4030,9 +4158,41 @@ function SpellingPanel() {
return (
<>
<p className="mb-3 text-[11px] text-surface-400">
{t('settings.spelling.intro', { defaultValue: "Spelling, grammar and style checks run through LanguageTool against pages you select in the Spelling tab. The page's own html[lang] picks the language; pages that don't declare one are auto-detected. Configure the endpoint (public API or self-hosted) under Integrations → LanguageTool." })}
{t('settings.spelling.intro', { defaultValue: "Spelling, grammar and style checks run through LanguageTool against pages you select in the Spelling tab. Each page's language is detected from its own text and cross-checked against html[lang]; pages in a language LanguageTool has no rules for are reported as unsupported rather than graded against the wrong dictionary. Configure the endpoint (public API or self-hosted) under Integrations → LanguageTool." })}
</p>
<div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('settings.spelling.languageLabel', { defaultValue: "Language" })}
</div>
<p className="mb-2 text-[11px] text-surface-400">
{t('settings.spelling.languageHint', { defaultValue: "Leave on auto unless detection gets a site wrong. Pinning a language forces every page to be checked in it — useful for a single-language site whose html[lang] is wrong, and it also turns off the safety check that discards implausibly noisy results." })}
</p>
<select
className="h-7 w-full rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={language}
onChange={(e) => pickLanguage(e.target.value)}
disabled={!languagesLoaded}
>
<option value="">
{t('settings.spelling.languageAuto', { defaultValue: "Auto — detect from each page's text (recommended)" })}
</option>
{languages.map((l) => (
<option key={l.longCode} value={l.longCode}>
{l.name} ({l.longCode})
</option>
))}
</select>
{languagesLoaded && languages.length > 0 && (
<p className="mt-1 text-[10px] text-surface-500">
{t('settings.spelling.languageCount', {
defaultValue: "This endpoint offers {{n}} language variant(s). Languages outside that list — Turkish, Hungarian, Czech and others — cannot be checked by LanguageTool at all.",
n: languages.length,
})}
</p>
)}
</div>
<div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('settings.spelling.levelLabel', { defaultValue: "Rule Level" })}
@@ -3,6 +3,7 @@ import { useTranslation } from 'react-i18next';
import { useAppStore } from '../store.js';
import { usePerfMeter } from '../hooks/usePerfMeter.js';
import { useMemoryMonitor } from '../hooks/useMemoryMonitor.js';
import { useBrowserInstall } from '../hooks/useBrowserInstall.js';
function Stat({
label,
@@ -108,6 +109,7 @@ export function StatsBar() {
const setError = useAppStore((s) => s.setError);
const perf = usePerfMeter();
const mem = useMemoryMonitor();
const browser = useBrowserInstall();
const elapsed = progress?.elapsedMs ?? 0;
const elapsedStr = formatElapsed(elapsed);
@@ -225,6 +227,35 @@ export function StatsBar() {
/>
<div className="ml-auto flex items-center gap-2">
{/* JS-rendering browser provisioning. Silent when everything is
already in place — only surfaces while downloading or after a
failure, where it doubles as the retry button. */}
{browser.state.state === 'downloading' && (
<span
className="inline-flex items-center gap-1.5 text-sky-300"
title={t('browserInstall.downloadingTitle', {
defaultValue:
'Downloading the Chromium browser used for JavaScript rendering. Crawling in text mode is unaffected.',
})}
>
<span className="h-2 w-2 animate-pulse rounded-full bg-sky-400" />
<span>
{t('browserInstall.downloading', { defaultValue: 'Browser' })}{' '}
{browser.state.percent !== null ? `${browser.state.percent}%` : '…'}
</span>
</span>
)}
{browser.state.state === 'failed' && (
<button
className="rounded bg-amber-900/50 px-2 py-0.5 text-amber-200 hover:bg-amber-900/70"
onClick={browser.retry}
title={browser.state.error}
>
{t('browserInstall.failed', {
defaultValue: 'Browser download failed — retry',
})}
</button>
)}
{progress?.running ? (
progress.paused ? (
<span className="inline-flex items-center gap-1.5">
@@ -0,0 +1,48 @@
import { useEffect, useState } from 'react';
import type { BrowserInstallState } from '@freecrawl/shared-types';
/**
* Tracks provisioning of the Chromium build JS rendering needs.
*
* The main process downloads it in the background when the installer
* didn't ship a browser this machine can run (e.g. an Intel Mac given an
* Apple-silicon bundle). Nothing about a crawl blocks on it, so this is
* a pure status feed: the current state is fetched once on mount — a
* window opened mid-download still shows the progress — and every later
* change arrives by push.
*/
export function useBrowserInstall(): {
state: BrowserInstallState;
retry: () => void;
} {
const [state, setState] = useState<BrowserInstallState>({
state: 'unknown',
percent: null,
});
useEffect(() => {
let cancelled = false;
void window.freecrawl
.browserInstallGet()
.then((s) => {
if (!cancelled) setState(s);
})
.catch(() => {
/* main not ready yet — the push below will catch us up */
});
const off = window.freecrawl.onBrowserInstallState((s) => setState(s));
return () => {
cancelled = true;
off();
};
}, []);
return {
state,
retry: () => {
void window.freecrawl.browserInstallStart().catch(() => {
/* the pushed state carries the failure */
});
},
};
}
@@ -0,0 +1,540 @@
/**
* Turkish translations for InfoTip ([i] icon) tooltip bodies.
*
* Keyed by the verbatim English source string, exactly like
* {@link ./labels.ts} — the ~170 `info` / `example` props scattered
* across columns.ts and SettingsDialog.tsx stay as plain English
* literals, and `InfoTip` looks the text up at render time. That keeps
* one lookup in one component instead of threading `t()` calls through
* every column spec and settings field.
*
* Anything absent from this table renders in English (graceful
* degradation), which is also what happens to technical example values
* — regexes, header values, URLs — that are deliberately left untranslated.
*
* GENERATED FILE — do not hand-edit. Regenerate when tooltip copy changes.
*/
const TR_INFO_TIPS: Record<string, string> = {
["?page=1 / ?page=2 / ?page=4 → flags 'Sequence Break' on every member of the broken cluster."]:
"?page=1 / ?page=2 / ?page=4 → bozuk kümenin her üyesine 'Sıra Kopukluğu' işareti koyar.",
["`<a>` elements that look clickable but aren't crawlable (no href + onclick, `href=\"javascript:…\"`, or `href=\"#\"` with onclick)."]:
"Tıklanabilir görünen ama taranamayan `<a>` öğeleri (href yok + onclick var, `href=\"javascript:…\"` ya da onclick ile birlikte `href=\"#\"`).",
["`<link rel=\"amphtml\" href=\"…\">` value — the AMP version of this page. Empty when the page does not declare an AMP alternate."]:
"`<link rel=\"amphtml\" href=\"…\">` değeri — bu sayfanın AMP sürümü. Sayfa AMP alternatifi tanımlamıyorsa boştur.",
["`<link rel=\"next\" href=\"…\">` value resolved to absolute. Empty when the page is not paginated forward."]:
"`<link rel=\"next\" href=\"…\">` değerinin mutlak URL'e çözümlenmiş hali. Sayfanın ileri yönde sayfalaması yoksa boştur.",
["`<link rel=\"prev\" href=\"…\">` value resolved to absolute. Empty when the page is the first in its pagination cluster."]:
"`<link rel=\"prev\" href=\"…\">` değerinin mutlak URL'e çözümlenmiş hali. Sayfa, sayfalama kümesindeki ilk sayfaysa boştur.",
["`css` runs against the parsed DOM; `regex` runs against raw HTML."]:
"`css` ayrıştırılmış DOM üzerinde çalışır; `regex` ham HTML üzerinde çalışır.",
["`none` disables auth; `basic` adds `Authorization: Basic <base64>`; `bearer` adds `Authorization: Bearer <token>`; `digest` performs the RFC 2617 challenge-response on the first 401."]:
"`none` kimlik doğrulamayı kapatır; `basic` `Authorization: Basic <base64>` ekler; `bearer` `Authorization: Bearer <token>` ekler; `digest` ilk 401 yanıtında RFC 2617 challenge-response akışını uygular.",
["`POST <url>` is fired when the `done` event emits. 10 s timeout. Failures are logged as info events but never break the crawl."]:
"`done` olayı tetiklendiğinde `POST <url>` isteği gönderilir. 10 sn zaman aşımı vardır. Hatalar bilgi olayı olarak loglanır ama crawl'ı asla bozmaz.",
["0 (no duplicates), 7 (member of cluster #7)"]:
"0 (yinelenen yok), 7 (#7 numaralı kümenin üyesi)",
["0 = auto. 4 for 8GB RAM machines, 8+ for 16GB+."]:
"0 = otomatik. 8 GB RAM'li makinelerde 4, 16 GB ve üzeri için 8+.",
["0 default; 250 ms when a host returns 429 with a 'too fast' message."]:
"Varsayılan 0; bir sunucu 'çok hızlı' mesajıyla 429 dönüyorsa 250 ms.",
["0 for SSR sites, 2000 for typical SPAs, 5000+ for heavy client-rendered apps."]:
"SSR siteler için 0, tipik SPA'lar için 2000, ağır istemci taraflı uygulamalar için 5000+.",
["0.1 default (Google 'good'); 0 to disable."]:
"Varsayılan 0,1 (Google 'iyi' eşiği); kapatmak için 0.",
["1 = unique, 5 = part of a 5-page near-duplicate group"]:
"1 = benzersiz, 5 = 5 sayfalık yakın-yinelenen grubun parçası",
["10 (default), 3 for very tight chains, 0 to remove the cap"]:
"10 (varsayılan), çok kısa zincirler için 3, sınırı kaldırmak için 0",
["10 covers most sites; 3 limits crawls to top-of-funnel pages only."]:
"10 çoğu siteyi kapsar; 3 crawl'ı yalnızca üst seviye sayfalarla sınırlar.",
["100 default for most audits; 0 to disable the check."]:
"Çoğu denetim için varsayılan 100; kontrolü kapatmak için 0.",
["100 default; 50 for tight on-page link discipline; 0 to disable the issue."]:
"Varsayılan 100; sıkı sayfa içi bağlantı disiplini için 50; sorunu kapatmak için 0.",
["1000000 (1M) for a full site audit; 5000 for spot checks."]:
"Tam site denetimi için 1000000 (1M); nokta kontroller için 5000.",
["1024 (1 MB) default; 150 for a lean HTML budget; 0 to disable."]:
"Varsayılan 1024 (1 MB); yalın HTML bütçesi için 150; kapatmak için 0.",
["1048576 (1 MB) default; 524288 (512 KB) on tight disks; 0 to disable truncation entirely."]:
"Varsayılan 1048576 (1 MB); disk darsa 524288 (512 KB); kırpmayı tamamen kapatmak için 0.",
["10485760 (10 MB) on bandwidth-tight crawls; 0 to download anything."]:
"Bant genişliği kısıtlı crawl'larda 10485760 (10 MB); her şeyi indirmek için 0.",
["1366 = standard laptop, 1920 = full HD desktop, 375 = iPhone width."]:
"1366 = standart dizüstü, 1920 = full HD masaüstü, 375 = iPhone genişliği.",
["2 default; 0 to record errors immediately without retrying; 5 for unreliable upstreams."]:
"Varsayılan 2; hataları yeniden denemeden hemen kaydetmek için 0; güvenilmez sunucular için 5.",
["20 default; 50 on fast first-party servers; 5 if the site rate-limits or returns 429s."]:
"Varsayılan 20; hızlı kendi sunucularınızda 50; site hız sınırlaması yapıyor veya 429 dönüyorsa 5.",
["20 for typical sites; 5 to be polite on shared hosting; 60+ when crawling your own infra."]:
"Tipik siteler için 20; paylaşımlı hostingde nazik olmak için 5; kendi altyapınızı tararken 60+.",
["20000 (20 s) for typical use; 5000 for fast spot checks; 60000 for slow APIs."]:
"Tipik kullanım için 20000 (20 sn); hızlı nokta kontroller için 5000; yavaş API'ler için 60000.",
["2048 (≈2 GB) on a 4 GB laptop; 8192 on a 16 GB workstation; 0 to disable."]:
"4 GB dizüstüde 2048 (≈2 GB); 16 GB iş istasyonunda 8192; kapatmak için 0.",
["2048 default (RFC-suggested practical ceiling)."]:
"Varsayılan 2048 (RFC'nin önerdiği pratik tavan).",
["2500 default (Google 'good'); 0 to disable."]:
"Varsayılan 2500 (Google 'iyi' eşiği); kapatmak için 0.",
["3 = recommended; 5 catches looser duplicates (templated content with light variation); 0 turns the post-crawl pass off."]:
"3 = önerilen; 5 daha gevşek yinelenenleri yakalar (hafif farklarla şablonlanmış içerik); 0 crawl sonrası geçişi kapatır.",
["4 default; 6 on documentation sites with deep TOC trees; 0 to disable."]:
"Varsayılan 4; derin içindekiler ağacı olan dokümantasyon sitelerinde 6; kapatmak için 0.",
["500 default. Bump to 2000 when retrying against a flaky API."]:
"Varsayılan 500. Kararsız bir API'ye karşı yeniden denerken 2000'e çıkarın.",
["50000 keeps RAM bounded during big sitemap fan-outs; 0 for typical crawls."]:
"50000, büyük sitemap açılımlarında RAM'i sınırlı tutar; tipik crawl'lar için 0.",
["60000 (1 minute) for huge resources; 0 to rely solely on the fetch timeout."]:
"Çok büyük kaynaklar için 60000 (1 dakika); yalnızca fetch zaman aşımına güvenmek için 0.",
["64-bit SimHash + LSH bucketing + Union-Find clustering on body shingles. Most expensive pass — typical 5–10 s on a 100k crawl."]:
"Gövde metni parçaları üzerinde 64-bit SimHash + LSH gruplama + Union-Find kümeleme. En pahalı geçiştir — 100 bin URL'lik bir crawl'da tipik olarak 5–10 sn.",
["768 = standard laptop, 1080 = full HD desktop, 667 = iPhone 8 height."]:
"768 = standart dizüstü, 1080 = full HD masaüstü, 667 = iPhone 8 yüksekliği.",
["800 default; 200 for CDN-backed static; 0 to disable."]:
"Varsayılan 800; CDN destekli statik siteler için 200; kapatmak için 0.",
["Aborts @font-face / Google Fonts / WOFF2 requests. FOUT visible but text still renders."]:
"@font-face / Google Fonts / WOFF2 isteklerini iptal eder. Yazı tipi geç yüklenir (FOUT) ama metin yine de görüntülenir.",
["Aborts <img>, <picture>, background-image requests. Recommended for SEO crawls — image metadata still comes from <img> tag attributes."]:
"<img>, <picture> ve background-image isteklerini iptal eder. SEO crawl'ları için önerilir — görsel meta verisi yine <img> etiketi özniteliklerinden gelir.",
["Aborts <video> / <audio> sources. Page DOM still includes the <video> tag."]:
"<video> / <audio> kaynaklarını iptal eder. Sayfa DOM'unda <video> etiketi yine yer alır.",
["Aborts all <script> requests. This defeats the purpose of JS rendering — use Text Only mode instead."]:
"Tüm <script> isteklerini iptal eder. Bu, JS render'ın amacını ortadan kaldırır — bunun yerine Yalnızca Metin modunu kullanın.",
["Aborts external CSS. Inline styles still load. WARNING: many SPAs use CSS-driven visibility / lazy classes — blocking CSS may hide content that JS depends on."]:
"Harici CSS'i iptal eder. Satır içi stiller yüklenmeye devam eder. UYARI: birçok SPA, CSS tabanlı görünürlük / lazy sınıfları kullanır — CSS'i engellemek JS'in bağımlı olduğu içeriği gizleyebilir.",
["Aborts requests whose total lifetime (connect + headers + body) exceeds this. Distinct from `requestTimeoutMs` which is the headers timeout. Useful for capping individual slow pages without lowering the overall fetch timeout."]:
"Toplam ömrü (bağlantı + başlıklar + gövde) bu değeri aşan istekleri iptal eder. Başlık zaman aşımı olan `requestTimeoutMs`'ten farklıdır. Genel fetch zaman aşımını düşürmeden tek tek yavaş sayfaları sınırlamak için kullanışlıdır.",
["Absolute redirect target parsed from the meta-refresh content. Empty when meta-refresh sets only a delay."]:
"meta-refresh içeriğinden ayrıştırılan mutlak yönlendirme hedefi. meta-refresh yalnızca bir gecikme tanımlıyorsa boştur.",
["Additional time to wait after the chosen wait condition fires, for SPA hydration / late XHRs. 0 = no extra wait. Bounded by the request timeout."]:
"Seçilen bekleme koşulu gerçekleştikten sonra SPA hydration / geç XHR istekleri için ek bekleme süresi. 0 = ek bekleme yok. İstek zaman aşımıyla sınırlıdır.",
["Anchor text of the broken link as rendered in the source page."]:
"Kaynak sayfada göründüğü haliyle kırık bağlantının bağlantı metni.",
["Audits the rendered DOM for WCAG AA colour-contrast failures (4.5:1 normal text, 3:1 large text) and stylesheet rules that suppress the keyboard focus outline without a :focus-visible fallback. Surfaces the Low-Contrast Text and Focus Outline Suppressed issue filters."]:
"Render edilmiş DOM'u WCAG AA renk kontrastı hataları (normal metin 4.5:1, büyük metin 3:1) ve :focus-visible yedeği olmadan klavye odak çerçevesini bastıran stil kuralları açısından denetler. Düşük Kontrastlı Metin ve Odak Çerçevesi Bastırılmış sorun filtrelerini besler.",
["basic/digest for /staging behind nginx; bearer for protected APIs"]:
"nginx arkasındaki /staging için basic/digest; korumalı API'ler için bearer",
["Below Normal while you keep working in other apps; Idle for overnight unattended runs."]:
"Diğer uygulamalarda çalışmaya devam ederken Normalin Altı; gece boyu gözetimsiz çalıştırmalar için Boşta.",
["BFS click depth from the start URL. Start URL = 0; its outlinks = 1; etc. High depth often correlates with low importance."]:
"Başlangıç URL'inden itibaren BFS tıklama derinliği. Başlangıç URL = 0; onun giden bağlantıları = 1; vb. Yüksek derinlik çoğu zaman düşük önemle ilişkilidir.",
["Bodies over this are truncated and flagged. 1 MB covers the 99.9th percentile of HTML pages without letting one adversarial 50 MB page bloat the project file."]:
"Bu değeri aşan gövdeler kırpılır ve işaretlenir. 1 MB, HTML sayfalarının %99,9'unu kapsar ve 50 MB'lık uç bir sayfanın proje dosyasını şişirmesini engeller.",
["Buy Affordable Game Keys | Example Store"]:
"Uygun Fiyatlı Oyun Anahtarları | Örnek Mağaza",
["Character count of the first H1."]:
"İlk H1'in karakter sayısı.",
["Character count of the meta description. Recommended: 70–155 characters; over 155 risks truncation."]:
"Meta açıklamanın karakter sayısı. Önerilen: 70–155 karakter; 155 üzeri kırpılma riski taşır.",
["Character count of the title. Recommended: 30–60 characters; over 60 risks truncation in SERPs."]:
"Başlığın karakter sayısı. Önerilen: 30–60 karakter; 60 üzeri SERP'lerde kırpılma riski taşır.",
["Charikar 64-bit SimHash of body shingles. Used by the post-crawl near-duplicate clustering pass. Two SimHashes within the configured Hamming threshold are considered similar."]:
"Gövde metni parçalarının Charikar 64-bit SimHash değeri. Crawl sonrası yakın-yinelenen kümeleme geçişi tarafından kullanılır. Yapılandırılan Hamming eşiği içindeki iki SimHash benzer kabul edilir.",
["Coarse content classification derived from URL extension and Content-Type header."]:
"URL uzantısı ve Content-Type başlığından türetilen kaba içerik sınıflandırması.",
["Comma-joined sorted unique JSON-LD `@type` values declared on the page (Article, BreadcrumbList, Product, …)."]:
"Sayfada tanımlanan benzersiz JSON-LD `@type` değerlerinin sıralı, virgülle birleştirilmiş listesi (Article, BreadcrumbList, Product, …).",
["Contents of the first <meta name=\"description\"> tag. May be used as the SERP snippet."]:
"İlk <meta name=\"description\"> etiketinin içeriği. SERP snippet'i olarak kullanılabilir.",
["Contents of the first <meta name=\"robots\"> tag. Controls per-page indexing/following behaviour."]:
"İlk <meta name=\"robots\"> etiketinin içeriği. Sayfa bazında indeksleme/takip davranışını denetler.",
["Contents of the first <title> element. Google primarily uses this in SERP titles."]:
"İlk <title> öğesinin içeriği. Google, SERP başlıklarında öncelikle bunu kullanır.",
["Counts how many internal pages link to each URL. Drives the Most-Linked URLs report and the per-row Inlinks column."]:
"Her URL'e kaç iç sayfanın bağlantı verdiğini sayar. En Çok Bağlantı Alan URL'ler raporunu ve satır bazındaki Gelen Bağlantı kolonunu besler.",
["Crawl 3xx redirect targets. Each hop is its own row; the chain is reconstructed in the Response Codes view. Screaming Frog: 'Always Follow Redirects' (Configuration → Spider → Advanced)."]:
"3xx yönlendirme hedeflerini de tarar. Her adım kendi satırıdır; zincir Yanıt Kodları görünümünde yeniden kurulur. Screaming Frog karşılığı: 'Always Follow Redirects' (Configuration → Spider → Advanced).",
["Crawler RSS auto-pauses the queue when this is exceeded; resumes once memory drops to 80% of the cap. Soft cap — does not enforce a hard heap limit."]:
"Crawler'ın bellek kullanımı bu değeri aştığında kuyruk otomatik duraklatılır; bellek sınırın %80'ine indiğinde devam eder. Esnek sınırdır — katı bir heap limiti uygulamaz.",
["css for selectors, regex for free-form patterns"]:
"seçiciler için css, serbest biçimli desenler için regex",
["CSS selector that pins the duplicate-fingerprint text extraction to a specific page region. When set, the heuristic (main / role=main / article / body-minus-chrome) is bypassed and the selector wins. Useful on sites where the heuristic misclassifies — e.g. CMSes that wrap navigation inside `<main>` or sites with no semantic landmarks at all. Empty = use the heuristic. Invalid selectors silently fall back to the heuristic so a typo doesn't break the crawl."]:
"Yinelenen içerik parmak izi için metin çıkarımını belirli bir sayfa bölgesine sabitleyen CSS seçicisi. Ayarlandığında sezgisel yöntem (main / role=main / article / gövde eksi çerçeve) devre dışı kalır ve seçici kazanır. Sezgiselin yanlış sınıflandırdığı sitelerde kullanışlıdır — ör. navigasyonu `<main>` içine saran CMS'ler ya da hiç semantik landmark'ı olmayan siteler. Boş = sezgiseli kullan. Geçersiz seçiciler sessizce sezgisele döner, böylece bir yazım hatası crawl'ı bozmaz.",
["Cumulative Layout Shift from PageSpeed Insights, when present. Google's 'good' CLS threshold is 0.1. Unitless; accepts decimals. Pages without PSI data are never flagged."]:
"Varsa PageSpeed Insights'tan gelen Cumulative Layout Shift değeri. Google'ın 'iyi' CLS eşiği 0,1'dir. Birimsizdir; ondalık kabul eder. PSI verisi olmayan sayfalar asla işaretlenmez.",
["Drives the View Source detail tab. ~30–200 KB on disk per HTML page; turn off if you only need metadata and not full source viewing."]:
"Kaynağı Görüntüle detay sekmesini besler. HTML sayfa başına diskte ~30–200 KB; yalnızca meta veriye ihtiyacınız varsa ve kaynak görüntülemeyecekseniz kapatın.",
["Each rule runs JavaScript RegExp.replace on the fully-normalised URL. Flags default to 'g'. After all rules run, the result is re-parsed as a URL — if the rewrite produces an invalid URL, the link is dropped at normalisation time."]:
"Her kural, tam normalize edilmiş URL üzerinde JavaScript RegExp.replace çalıştırır. Bayraklar varsayılan olarak 'g'dir. Tüm kurallar çalıştıktan sonra sonuç yeniden URL olarak ayrıştırılır — yeniden yazım geçersiz bir URL üretirse bağlantı normalizasyon aşamasında düşürülür.",
["Empty = safest. 'chrome' if you want the same Chrome version your users see."]:
"Boş = en güvenlisi. Kullanıcılarınızın gördüğü Chrome sürümünü istiyorsanız 'chrome'.",
["Empty = use the bundled Playwright Chromium build (recommended — pinned version, works everywhere). 'chrome' / 'msedge' uses the system-installed browser. Beta channels for testing newer features."]:
"Boş = uygulamayla gelen Playwright Chromium sürümünü kullan (önerilir — sabit sürüm, her yerde çalışır). 'chrome' / 'msedge' sistemde kurulu tarayıcıyı kullanır. Beta kanalları yeni özellikleri test etmek içindir.",
["Fetch internal <img> resources (incl. srcset / <picture> sources) so they appear in the Internal tab with their own status code, content type, and size. Each counts toward Max URLs. Screaming Frog: 'Check Images' (Configuration → Spider → Crawl)."]:
"İç <img> kaynaklarını (srcset / <picture> kaynakları dahil) çeker; böylece kendi durum kodu, içerik tipi ve boyutuyla İç Kaynaklar sekmesinde görünürler. Her biri Maksimum URL sayısına dahil olur. Screaming Frog karşılığı: 'Check Images' (Configuration → Spider → Crawl).",
["Fetch internal <link rel=stylesheet> resources so they appear in the Internal tab with status code, content type, and size. Each counts toward Max URLs. Screaming Frog: 'Check CSS' (Configuration → Spider → Crawl)."]:
"İç <link rel=stylesheet> kaynaklarını çeker; böylece durum kodu, içerik tipi ve boyutuyla İç Kaynaklar sekmesinde görünürler. Her biri Maksimum URL sayısına dahil olur. Screaming Frog karşılığı: 'Check CSS' (Configuration → Spider → Crawl).",
["Fetch internal <script src> resources so they appear in the Internal tab with status code, content type, and size. Each counts toward Max URLs. Screaming Frog: 'Check JavaScript' (Configuration → Spider → Crawl)."]:
"İç <script src> kaynaklarını çeker; böylece durum kodu, içerik tipi ve boyutuyla İç Kaynaklar sekmesinde görünürler. Her biri Maksimum URL sayısına dahil olur. Screaming Frog karşılığı: 'Check JavaScript' (Configuration → Spider → Crawl).",
["Fetches /robots.txt sitemap directives + /sitemap.xml fallbacks. Powers the 'Non-Indexable in Sitemap' issue filter. Screaming Frog: 'Auto Discover XML Sitemaps via robots.txt' (Configuration → Spider → Crawl)."]:
"/robots.txt içindeki sitemap direktiflerini ve /sitemap.xml yedeklerini getirir. 'Sitemap'te İndekslenemez' sorun filtresini besler. Screaming Frog karşılığı: 'Auto Discover XML Sitemaps via robots.txt' (Configuration → Spider → Crawl).",
["first/last for single value, all for JSON array, concat for ' | ' joined string"]:
"tek değer için first/last, JSON dizisi için all, ' | ' ile birleştirilmiş dize için concat",
["FNV-1a 64-bit hash of the normalised body token stream. Two pages sharing this hash are byte-identical post-tokenisation — the basis of the Exact Duplicate filter."]:
"Normalize edilmiş gövde token akışının FNV-1a 64-bit hash değeri. Bu hash'i paylaşan iki sayfa, tokenizasyon sonrası bayt bazında aynıdır — Tam Yinelenen filtresinin temeli budur.",
["For Basic, sent base64-encoded; for Digest, hashed into the challenge response."]:
"Basic için base64 kodlanmış olarak gönderilir; Digest için challenge yanıtına hash'lenir.",
["For regex: `regex_group` extracts capture group 1; otherwise the whole match is used."]:
"Regex için: `regex_group` 1 numaralı yakalama grubunu çıkarır; aksi halde eşleşmenin tamamı kullanılır.",
["Full-page renders the entire scrollable canvas; Above-the-fold captures just the initial viewport (cheaper). Both writes two PNGs per URL."]:
"Tam Sayfa, kaydırılabilir alanın tamamını render eder; Ekranın Üstü yalnızca ilk görünümü yakalar (daha ucuz). Her İkisi seçeneği URL başına iki PNG yazar.",
["Google's index status, pulled from the URL Inspection API — not the Fetch button. Click \"Inspect (top 100)\" to fill this column; Fetch only pulls clicks / impressions / position."]:
"Google'ın indeksleme durumu; URL İnceleme API'sinden alınır — Getir düğmesinden değil. Bu kolonu doldurmak için \"İncele (ilk 100)\" seçeneğine tıklayın; Getir yalnızca tıklama / gösterim / pozisyon verisi çeker.",
["Googlebot — Smartphone matches Google's mobile-first indexing crawler."]:
"Googlebot — Akıllı Telefon, Google'ın mobil öncelikli indeksleme crawler'ıyla eşleşir.",
["Hard cap on pending URLs held in memory. Excess discoveries are dropped silently — bounds peak heap during fan-out bursts (big sitemaps, dense link graphs)."]:
"Bellekte tutulan bekleyen URL sayısı için katı üst sınır. Fazlası sessizce düşürülür — yoğun keşif patlamalarında (büyük sitemap'ler, sık bağlantılı grafikler) tepe bellek kullanımını sınırlar.",
["Hard cap on the number of 3xx hops we follow for a single chain. Each hop is recorded as its own URL row regardless. 0 disables the cap (chain still ends at `redirect_loop`)."]:
"Tek bir zincir için takip edilen 3xx adım sayısına katı üst sınır. Her adım yine de kendi URL satırı olarak kaydedilir. 0 sınırı kaldırır (zincir yine `redirect_loop` ile biter).",
["Hard cap on total URLs crawled. The crawl stops as soon as this is reached. Screaming Frog: 'Limit Crawl Total'."]:
"Taranan toplam URL sayısına katı üst sınır. Bu değere ulaşıldığı anda crawl durur. Screaming Frog karşılığı: 'Limit Crawl Total'.",
["Hard ceiling on requests per second across all workers combined. Equivalent to Screaming Frog's 'Max URL/s'. Acts as a token bucket — even with high concurrency the crawler waits between bursts to stay below this rate."]:
"Tüm işçiler toplamında saniyedeki istek sayısına katı tavan. Screaming Frog'daki 'Max URL/s' karşılığıdır. Token bucket gibi çalışır — eşzamanlılık yüksek olsa bile crawler bu hızın altında kalmak için istek grupları arasında bekler.",
["Height attribute value (in pixels) declared on the <img> tag, when present."]:
"Varsa <img> etiketinde tanımlanan height özniteliğinin değeri (piksel).",
["Honor Disallow rules + crawl-delay declared in /robots.txt for the configured User-Agent. Screaming Frog: 'Respect robots.txt' (Configuration → robots.txt)."]:
"Yapılandırılan User-Agent için /robots.txt'te tanımlı Disallow kurallarına ve crawl-delay değerine uyar. Screaming Frog karşılığı: 'Respect robots.txt' (Configuration → robots.txt).",
["Hop count from the start URL. Start URL is depth 0; its outlinks are depth 1, theirs depth 2, and so on. Screaming Frog: 'Limit Crawl Depth' (Configuration → Spider → Limits)."]:
"Başlangıç URL'inden itibaren adım sayısı. Başlangıç URL derinlik 0'dır; onun giden bağlantıları derinlik 1, onlarınki 2 ve böyle devam eder. Screaming Frog karşılığı: 'Limit Crawl Depth' (Configuration → Spider → Limits).",
["How many distinct pages reference this image. High values typically indicate site-wide assets (logos, icons)."]:
"Bu görsele kaç farklı sayfanın referans verdiği. Yüksek değerler genellikle site geneli varlıkları (logolar, ikonlar) gösterir.",
["How to canonicalise paths with/without a trailing slash. 'Add' is file-extension aware — won't add a slash to /file.pdf or /image.png."]:
"Sonunda eğik çizgi olan/olmayan yolların nasıl kanonikleştirileceği. 'Ekle' seçeneği dosya uzantısına duyarlıdır — /file.pdf veya /image.png sonuna eğik çizgi eklemez.",
["HTML attribute name to read."]:
"Okunacak HTML özniteliğinin adı.",
["HTML transfer size of the page document. Heavy HTML payloads delay first paint. Stored as bytes internally; entered here in kilobytes."]:
"Sayfa belgesinin HTML transfer boyutu. Ağır HTML yükleri ilk boyamayı geciktirir. Dahili olarak bayt saklanır; burada kilobayt olarak girilir.",
["HTTP `<img>` / `<video>` / `<audio>` / `<source>` references on an HTTPS page — rendered but the URL bar reads \"Not Secure\"."]:
"HTTPS bir sayfadaki HTTP `<img>` / `<video>` / `<audio>` / `<source>` referansları — render edilir ama adres çubuğunda \"Güvenli değil\" yazar.",
["HTTP `<script>` / `<link rel=stylesheet>` / `<iframe>` / `<object>` / `<embed>` references on an HTTPS page — browsers BLOCK these silently."]:
"HTTPS bir sayfadaki HTTP `<script>` / `<link rel=stylesheet>` / `<iframe>` / `<object>` / `<embed>` referansları — tarayıcılar bunları sessizce ENGELLER.",
["HTTP response status code. Empty/Failed indicates a network error before any response was received."]:
"HTTP yanıt durum kodu. Boş/Başarısız, herhangi bir yanıt alınmadan önce ağ hatası oluştuğunu gösterir.",
["HTTP status of the source page itself. Usually 200; if non-2xx the broken link may be inherited."]:
"Kaynak sayfanın kendi HTTP durumu. Genellikle 200'dür; 2xx dışıysa kırık bağlantı devralınmış olabilir.",
["HTTP status returned by the target. 0 = network failure (DNS, TLS, timeout)."]:
"Hedefin döndürdüğü HTTP durumu. 0 = ağ hatası (DNS, TLS, zaman aşımı).",
["HTTP/HTTPS proxies route via undici's ProxyAgent; SOCKS proxies (socks5://, socks5h://, socks4://, socks4a://) tunnel via the socks client. The `h`/`4a` variants resolve DNS at the proxy. Leave empty to inherit HTTPS_PROXY/HTTP_PROXY env vars."]:
"HTTP/HTTPS proxy'ler undici'nin ProxyAgent'ı üzerinden yönlendirilir; SOCKS proxy'ler (socks5://, socks5h://, socks4://, socks4a://) socks istemcisi üzerinden tünellenir. `h`/`4a` varyantları DNS'i proxy tarafında çözer. HTTPS_PROXY/HTTP_PROXY ortam değişkenlerini devralmak için boş bırakın.",
["Identifies the largest element visible in the initial viewport (likely LCP candidate per Google's heuristic) and stores its CSS selector, dimensions, and resource URL. Useful for spotting unoptimised LCP images without a PSI API call."]:
"İlk görünümde yer alan en büyük öğeyi (Google'ın sezgiseline göre olası LCP adayı) belirler ve CSS seçicisini, boyutlarını ve kaynak URL'ini saklar. PSI API çağrısı yapmadan optimize edilmemiş LCP görsellerini tespit etmek için kullanışlıdır.",
["If set, Playwright waits for this CSS selector to appear in the DOM before extracting HTML. Overrides the extra-wait timeout when present. Useful when you know the SPA reveals a specific element after hydration."]:
"Ayarlanırsa Playwright, HTML'i çıkarmadan önce bu CSS seçicisinin DOM'da görünmesini bekler. Tanımlıysa ek bekleme süresini geçersiz kılar. SPA'nın hydration sonrası belirli bir öğeyi gösterdiğini biliyorsanız kullanışlıdır.",
["Images on this page that have no alt attribute. WCAG accessibility issue + missed alt-as-anchor SEO opportunity."]:
"Bu sayfada alt özniteliği olmayan görseller. WCAG erişilebilirlik sorunu ve kaçırılmış alt-metin SEO fırsatı.",
["Indexable / Non-Indexable"]:
"İndekslenebilir / İndekslenemez",
["Internal PageRank, 0–100. Computed over the internal link graph (damping 0.85) and normalised so the most-linked page scores 100. Higher = more internal link equity."]:
"Dahili PageRank, 0–100. İç bağlantı grafiği üzerinden hesaplanır (damping 0,85) ve en çok bağlantı alan sayfa 100 olacak şekilde normalize edilir. Yüksek = daha fazla iç bağlantı değeri.",
["internal / external"]:
"iç / dış",
["JavaScript executed in every page BEFORE navigation begins (init script). Use to set localStorage / cookies / mock APIs / disable animations. Runs in page context — no Node access."]:
"Gezinme başlamadan ÖNCE her sayfada çalıştırılan JavaScript (init script). localStorage / çerez ayarlamak, API'leri taklit etmek veya animasyonları kapatmak için kullanın. Sayfa bağlamında çalışır — Node erişimi yoktur.",
["JavaScript regex (no flags — /g is implicit). Use a capture group with `output=regex_group` to extract just part of the match."]:
"JavaScript regex'i (bayrak yok — /g örtük olarak uygulanır). Eşleşmenin yalnızca bir kısmını almak için `output=regex_group` ile bir yakalama grubu kullanın.",
["JavaScript regex tested against the full URL. Empty = all URLs allowed. URL must match at least one to be enqueued. The start URL is always permitted regardless."]:
"Tam URL'e karşı test edilen JavaScript regex'i. Boş = tüm URL'lere izin verilir. Bir URL'in kuyruğa alınması için en az biriyle eşleşmesi gerekir. Başlangıç URL'ine her durumda izin verilir.",
["JavaScript regex. Any match → URL is skipped, even if it would otherwise pass the include list. Common uses: skip admin areas, large file types, session-id query params."]:
"JavaScript regex'i. Herhangi bir eşleşme → URL atlanır, dahil etme listesinden geçecek olsa bile. Yaygın kullanımlar: yönetim alanlarını, büyük dosya tiplerini ve oturum kimliği parametrelerini atlamak.",
["JSON map of `{ term: count }` literal-substring hits from the configured Custom Search terms."]:
"Yapılandırılan Özel Arama terimlerinden gelen birebir alt dize eşleşmelerinin `{ terim: sayı }` biçimindeki JSON eşlemesi.",
["JSON-stringified array of `{ lang, href }` pairs. Heavy column — better consumed via the URL Details panel."]:
"`{ lang, href }` çiftlerinin JSON dizisi. Ağır kolondur — URL Detayları panelinden okumak daha rahattır.",
["JSON-stringified custom-extraction results map. Heavy column — render verbatim, easier to read in the URL Details panel."]:
"Özel çıkarım sonuçlarının JSON eşlemesi. Ağır kolondur — birebir gösterilir, URL Detayları panelinde okumak daha kolaydır.",
["JSONPath against a JSON response body (e.g. `application/json` APIs). Only runs on responses that parse as JSON — ignored on HTML pages."]:
"JSON yanıt gövdesine karşı çalışan JSONPath (ör. `application/json` API'leri). Yalnızca JSON olarak ayrıştırılabilen yanıtlarda çalışır — HTML sayfalarda yok sayılır.",
["JSONPath returns the matched JSON value as-is; choose `Count` to return the number of matches instead."]:
"JSONPath, eşleşen JSON değerini olduğu gibi döner; bunun yerine eşleşme sayısını almak için `Count` seçin.",
["Largest Contentful Paint from PageSpeed Insights lab data, when the URL has been audited. Google's 'good' LCP threshold is 2500 ms. Pages without PSI data are never flagged on this metric."]:
"URL denetlenmişse PageSpeed Insights laboratuvar verisinden gelen Largest Contentful Paint değeri. Google'ın 'iyi' LCP eşiği 2500 ms'dir. PSI verisi olmayan sayfalar bu metrikte asla işaretlenmez.",
["load = good default. networkidle for heavy SPAs. domcontentloaded if you only need raw HTML."]:
"load = iyi bir varsayılan. Ağır SPA'lar için networkidle. Yalnızca ham HTML gerekiyorsa domcontentloaded.",
["Location header value when status is 3xx. The URL the server points to next; chain length is in the URL Details panel."]:
"Durum 3xx olduğunda Location başlığının değeri. Sunucunun işaret ettiği bir sonraki URL'dir; zincir uzunluğu URL Detayları panelindedir.",
["Lowercases the URL path component. Host is already case-insensitive per the URL spec, so this only affects the path."]:
"URL'in yol bileşenini küçük harfe çevirir. Host, URL standardına göre zaten büyük/küçük harf duyarsızdır, bu yüzden bu yalnızca yolu etkiler.",
["Near-duplicate cluster ID assigned by the post-crawl SimHash pass. 0 = singleton (no near-duplicates within the configured Hamming threshold). Pages sharing a non-zero cluster ID are mutually similar."]:
"Crawl sonrası SimHash geçişinin atadığı yakın-yinelenen küme kimliği. 0 = tekil (yapılandırılan Hamming eşiği içinde yakın yinelenen yok). Sıfır olmayan aynı küme kimliğini paylaşan sayfalar karşılıklı olarak benzerdir.",
["noindex, canonicalised, redirected, blocked-by-robots"]:
"noindex, canonical edilmiş, yönlendirildi, robots.txt tarafından engellendi",
["None for fastest crawl. Above-the-fold for SERP-thumbnail-style preview. Full page when you need long-page snapshots."]:
"En hızlı crawl için Hiçbiri. SERP küçük resmi tarzı önizleme için Ekranın Üstü. Uzun sayfa anlık görüntüleri gerekiyorsa Tam Sayfa.",
["Number of `<form action=\"http://…\">` declarations on an HTTPS page. Submitting one downgrades the connection."]:
"HTTPS bir sayfadaki `<form action=\"http://…\">` tanımlarının sayısı. Böyle bir formu göndermek bağlantıyı güvensize düşürür.",
["Number of `<link rel=\"alternate\" hreflang>` entries declared on this page. 0 = no alternates declared."]:
"Bu sayfada tanımlanan `<link rel=\"alternate\" hreflang>` girdilerinin sayısı. 0 = alternatif tanımlanmamış.",
["Number of `<link rel=\"canonical\">` tags on the page. >1 is a \"Multiple Canonicals\" issue."]:
"Sayfadaki `<link rel=\"canonical\">` etiketlerinin sayısı. >1 olması \"Birden Fazla Canonical\" sorunudur.",
["Number of `<script type=\"application/ld+json\">` blocks parsed successfully on the page."]:
"Sayfada başarıyla ayrıştırılan `<script type=\"application/ld+json\">` bloklarının sayısı.",
["Number of `<script type=\"application/ld+json\">` blocks that failed to parse as JSON."]:
"JSON olarak ayrıştırılamayan `<script type=\"application/ld+json\">` bloklarının sayısı.",
["Number of <img> elements on the page."]:
"Sayfadaki <img> öğelerinin sayısı.",
["Number of browser tabs the pool keeps warm in parallel. 0 = auto (matches crawler concurrency, capped at 8). More tabs = faster crawl but more RAM (each tab ~80–150 MB)."]:
"Havuzun paralel olarak hazır tuttuğu tarayıcı sekmesi sayısı. 0 = otomatik (crawler eşzamanlılığına eşitlenir, en fazla 8). Daha fazla sekme = daha hızlı crawl ama daha fazla RAM (sekme başına ~80–150 MB).",
["Number of hreflang targets that are non-200, noindex, or canonicalised away. Aggregated by the post-crawl pass."]:
"200 dönmeyen, noindex olan veya canonical ile başka sayfaya yönlendirilen hreflang hedeflerinin sayısı. Crawl sonrası geçişte toplanır.",
["Number of HTTP requests in flight at any one time. Equivalent to Screaming Frog's 'Max Threads'. Higher = faster crawl + more load on the target server."]:
"Aynı anda açık olan HTTP isteklerinin sayısı. Screaming Frog'daki 'Max Threads' karşılığıdır. Yüksek = daha hızlı crawl ve hedef sunucuda daha fazla yük.",
["Number of internal `<a>` elements with no usable anchor text or alt — accessibility / SEO regression."]:
"Kullanılabilir bağlantı metni veya alt metni olmayan iç `<a>` öğelerinin sayısı — erişilebilirlik / SEO açısından gerileme.",
["Number of internal pages that link to this URL. A rough internal-PageRank signal."]:
"Bu URL'e bağlantı veren iç sayfaların sayısı. Kabaca bir dahili PageRank sinyalidir.",
["Number of pages in this URL's near-duplicate cluster (1 = no duplicates, ≥2 = part of a duplicate group). Tunable via Settings → Duplicates."]:
"Bu URL'in yakın-yinelenen kümesindeki sayfa sayısı (1 = yinelenen yok, ≥2 = bir yinelenen grubunun parçası). Ayarlar → Yinelenenler bölümünden ayarlanabilir.",
["Number of redirect hops from this URL to its terminal target. Filled by the post-crawl `recomputeRedirectChains` walker. >3 trips the \"Long Chain\" issue."]:
"Bu URL'den son hedefe kadar olan yönlendirme adımı sayısı. Crawl sonrası `recomputeRedirectChains` yürüteci tarafından doldurulur. >3 olması \"Uzun Zincir\" sorununu tetikler.",
["Number of unique <a> links emitted from this page (internal + external)."]:
"Bu sayfadan çıkan benzersiz <a> bağlantılarının sayısı (iç + dış).",
["Off — only enable for testing edge cases."]:
"Kapalı — yalnızca uç durumları test etmek için açın.",
["Off — small speed gain not worth the fidelity loss."]:
"Kapalı — küçük hız kazancı, doğruluk kaybına değmez.",
["On — fonts add overhead without changing SEO output."]:
"Açık — yazı tipleri SEO çıktısını değiştirmeden yük getirir.",
["On (default) — cheap I/O, high SEO value."]:
"Açık (varsayılan) — düşük maliyetli, yüksek SEO değeri.",
["On (default) — media is heavy and rarely SEO-relevant."]:
"Açık (varsayılan) — medya ağırdır ve nadiren SEO açısından anlamlıdır.",
["On (default) so the Internal tab shows images, not just HTML; off for HTML-only crawls."]:
"Açık (varsayılan), böylece İç Kaynaklar sekmesi yalnızca HTML'i değil görselleri de gösterir; yalnızca HTML taramaları için kapalı.",
["On (default); off for HTML-only crawls."]:
"Açık (varsayılan); yalnızca HTML taramaları için kapalı.",
["On (default). Off only when crawling sites you own and need to bypass."]:
"Açık (varsayılan). Yalnızca size ait olan ve atlamanız gereken siteleri tararken kapalı.",
["On for accessibility / WCAG audits."]:
"Erişilebilirlik / WCAG denetimleri için açık.",
["On for max speed. Off if you need LCP candidate detection or visual screenshots later."]:
"Maksimum hız için açık. Sonrasında LCP aday tespiti veya görsel ekran görüntüleri gerekiyorsa kapalı.",
["On for modern sites that 301 http→https anyway; off for legacy intranet."]:
"Zaten http→https 301 yapan modern siteler için açık; eski intranet siteleri için kapalı.",
["On for normal audits; off when you only want to inspect raw 3xx behaviour."]:
"Normal denetimler için açık; yalnızca ham 3xx davranışını incelemek istediğinizde kapalı.",
["On for outbound link audits; off for fast internal-only crawls."]:
"Dış bağlantı denetimleri için açık; yalnızca iç sayfaları hızlıca taramak için kapalı.",
["On for performance-focused audits that should fail pages over a target."]:
"Hedefi aşan sayfaları başarısız saymak isteyen performans odaklı denetimler için açık.",
["On for performance-focused audits."]:
"Performans odaklı denetimler için açık.",
["On for production crawls. Off when debugging selector-not-found / hydration issues."]:
"Canlı crawl'lar için açık. Seçici bulunamadı / hydration sorunlarını incelerken kapalı.",
["ON for SEO audits (the typical case). Turn OFF to also cluster paginated / canonical-blocked variants for completeness."]:
"SEO denetimleri için AÇIK (tipik durum). Sayfalanmış / canonical ile engellenmiş varyantları da eksiksizlik için kümelemek istiyorsanız KAPATIN.",
["On for SEO audits that include Google's Mobile-Friendly checks."]:
"Google'ın Mobil Uyumluluk kontrollerini içeren SEO denetimleri için açık.",
["On for SEO audits where View Source matters; off for 1M-URL crawls where disk is tight."]:
"Kaynağı Görüntüle'nin önemli olduğu SEO denetimleri için açık; diskin dar olduğu 1M URL'lik crawl'larda kapalı.",
["ON for SEO audits. OFF only when you specifically need to inspect raw-URL collisions (e.g. case-sensitive filesystem CMSes)."]:
"SEO denetimleri için AÇIK. Yalnızca ham URL çakışmalarını incelemeniz gerekiyorsa KAPALI (ör. büyük/küçük harfe duyarlı dosya sistemli CMS'ler).",
["On if you need nofollow attribute audits; off keeps the link graph cleaner."]:
"nofollow özniteliği denetimi gerekiyorsa açık; kapalı olması bağlantı grafiğini daha temiz tutar.",
["On if your CMS serves the same page at mixed casing (/Foo and /foo)."]:
"CMS'iniz aynı sayfayı farklı harf düzenleriyle sunuyorsa açık (/Foo ve /foo).",
["On if your site canonicalises to non-www but emits www links somewhere."]:
"Siteniz www'siz sürüme kanonikleşiyor ama bir yerlerde www'li bağlantılar üretiyorsa açık.",
["On network errors, 408/425/429/5xx responses, retry up to N more times before giving up. Each retry counts toward the URL's response time budget."]:
"Ağ hatalarında ve 408/425/429/5xx yanıtlarında, vazgeçmeden önce en fazla N kez daha dener. Her deneme URL'in yanıt süresi bütçesine dahil olur.",
["On when auditing mobile UX or capturing PageSpeed-style mobile previews."]:
"Mobil kullanıcı deneyimini denetlerken veya PageSpeed tarzı mobil önizlemeler alırken açık.",
["One header per line in 'Key: Value' format. Added to every request — useful for auth tokens or custom routing hints. User values override defaults when keys collide."]:
"Her satıra bir başlık, 'Anahtar: Değer' biçiminde. Her isteğe eklenir — kimlik doğrulama token'ları veya özel yönlendirme ipuçları için kullanışlıdır. Anahtarlar çakıştığında kullanıcı değerleri varsayılanları geçersiz kılar.",
["One sitemap URL per line. On top of following links from the start URL, the crawler fetches these sitemaps and queues every page they list as an extra seed — faster/more complete discovery, and reliable orphan detection even when the sitemap lives at a non-standard path. Leave empty to disable."]:
"Her satıra bir sitemap URL'i. Başlangıç URL'inden bağlantı takibine ek olarak crawler bu sitemap'leri çeker ve listeledikleri her sayfayı ekstra başlangıç noktası olarak kuyruğa alır — daha hızlı ve eksiksiz keşif, ayrıca sitemap standart dışı bir yolda olsa bile güvenilir orphan tespiti. Devre dışı bırakmak için boş bırakın.",
["One URL per line. Each is fetched exactly once; outlinks are NOT followed. Comments starting with # are ignored."]:
"Her satıra bir URL. Her biri tam olarak bir kez çekilir; giden bağlantılar takip EDİLMEZ. # ile başlayan satırlar yok sayılır.",
["OS scheduler hint applied at crawl start. Lowering priority lets the rest of the machine stay responsive during heavy crawls. May require elevated privileges on some platforms."]:
"Crawl başlangıcında uygulanan işletim sistemi zamanlayıcı ipucu. Önceliği düşürmek, ağır crawl'lar sırasında makinenin geri kalanının akıcı kalmasını sağlar. Bazı platformlarda yükseltilmiş yetki gerektirebilir.",
["Page A→B declared but B→A absent flags 'Reciprocity Missing'; same lang on two hrefs flags 'Inconsistent Lang'."]:
"A→B tanımlanmış ama B→A yoksa 'Karşılıklılık Eksik' işaretlenir; iki farklı href'te aynı dil kullanılıyorsa 'Tutarsız Dil' işaretlenir.",
["Page that contains the broken link."]:
"Kırık bağlantıyı içeren sayfa.",
["Pages with > this many outgoing links (internal + external) trip the 'Total Links per Page' issue. Google's historic recommendation is 100; mega-menus/hub-pages routinely blow past this."]:
"Bu sayıdan fazla giden bağlantısı (iç + dış) olan sayfalar 'Sayfa Başına Toplam Bağlantı' sorununu tetikler. Google'ın geçmişteki önerisi 100'dür; mega menüler ve hub sayfaları bunu rutin olarak aşar.",
["PASS = indexed · FAIL = not indexed · PART/NEU = discovered but not yet indexed"]:
"PASS = indekslendi · FAIL = indekslenmedi · PART/NEU = keşfedildi ama henüz indekslenmedi",
["Pattern: ^https://m\\.(.+) · Replacement: https://www.$1 · Flags: i (collapse mobile subdomain to www)"]:
"Desen: ^https://m\\.(.+) · Değişim: https://www.$1 · Bayraklar: i (mobil alt alan adını www'ye indirger)",
["Per-request abort threshold. Pages that take longer than this are recorded as network errors. Screaming Frog: 'Response Timeout (secs)' (Configuration → Spider → Advanced) — that one's in seconds, this is in milliseconds."]:
"İstek başına iptal eşiği. Bundan uzun süren sayfalar ağ hatası olarak kaydedilir. Screaming Frog karşılığı: 'Response Timeout (secs)' (Configuration → Spider → Advanced) — o saniye cinsindendir, bu milisaniye.",
["Persist rel=\"nofollow\" links in the link graph. When off, nofollow links are dropped entirely (not counted in outlinks, not probed as externals). Screaming Frog inverse: turning this ON ≈ unchecking \"Follow Internal/External Nofollow\"."]:
"rel=\"nofollow\" bağlantılarını bağlantı grafiğinde saklar. Kapalıyken nofollow bağlantıları tamamen düşürülür (giden bağlantılara sayılmaz, dış bağlantı olarak sınanmaz). Screaming Frog'da tersi geçerlidir: bunu AÇMAK ≈ \"Follow Internal/External Nofollow\" seçeneğinin işaretini kaldırmak.",
["Picking a preset fills the User-Agent field below — you can still hand-edit it afterwards. Switch between Googlebot Smartphone / Desktop to compare how a site responds to mobile vs desktop crawlers."]:
"Hazır ayar seçmek aşağıdaki User-Agent alanını doldurur — sonrasında elle düzenleyebilirsiniz. Bir sitenin mobil ve masaüstü crawler'lara nasıl yanıt verdiğini karşılaştırmak için Googlebot Akıllı Telefon / Masaüstü arasında geçiş yapın.",
["Picks one of the saved profiles by name. Empty = use the Proxy URL field above (or env vars when that's also empty)."]:
"Kayıtlı profillerden birini adıyla seçer. Boş = yukarıdaki Proxy URL alanını kullan (o da boşsa ortam değişkenlerini).",
["Pre-computes Dead External Domain, Duplicate URL post-norm, Canonical Chain Multi-hop. Without this the sidebar shows 0 for those three."]:
"Ölü Dış Alan Adı, Normalizasyon Sonrası Yinelenen URL ve Çok Adımlı Canonical Zinciri değerlerini önceden hesaplar. Bu olmadan kenar çubuğu bu üçü için 0 gösterir.",
["Probe outbound links to other hosts (HEAD only) so the Broken Links view catches dead externals. Screaming Frog: 'External Links' (Configuration → Spider → Crawl)."]:
"Diğer sunuculara giden bağlantıları sınar (yalnızca HEAD); böylece Kırık Bağlantılar görünümü ölü dış bağlantıları yakalar. Screaming Frog karşılığı: 'External Links' (Configuration → Spider → Crawl).",
["Raw `Content-Security-Policy` response header. Empty when missing."]:
"Ham `Content-Security-Policy` yanıt başlığı. Yoksa boştur.",
["Raw `content` attribute of `<meta http-equiv=\"refresh\">`, e.g. \"5; url=/foo\"."]:
"`<meta http-equiv=\"refresh\">` etiketinin ham `content` özniteliği, ör. \"5; url=/foo\".",
["Raw `Strict-Transport-Security` header. Empty when missing — for HTTPS pages this is a security regression."]:
"Ham `Strict-Transport-Security` başlığı. Yoksa boştur — HTTPS sayfalar için bu bir güvenlik eksikliğidir.",
["Raw `X-Content-Type-Options` header. `nosniff` blocks MIME sniffing — prevents some XSS via content-type confusion."]:
"Ham `X-Content-Type-Options` başlığı. `nosniff`, MIME türü tahminini engeller — içerik tipi karışıklığından kaynaklanan bazı XSS'leri önler.",
["Raw `X-Frame-Options` header. SAMEORIGIN / DENY / ALLOW-FROM. Clickjacking defence."]:
"Ham `X-Frame-Options` başlığı. SAMEORIGIN / DENY / ALLOW-FROM. Clickjacking savunmasıdır.",
["Raw value of the Content-Type response header (incl. charset)."]:
"Content-Type yanıt başlığının ham değeri (charset dahil).",
["Re-renders each page on a mobile viewport and checks viewport meta tag, horizontal overflow, font size legibility, and tap-target spacing. Stores a pass/fail verdict on the urls table."]:
"Her sayfayı mobil görünümde yeniden render eder; viewport meta etiketini, yatay taşmayı, yazı tipi okunabilirliğini ve dokunma hedefi aralıklarını denetler. Geçti/kaldı sonucunu urls tablosuna yazar.",
["Read the full article →"]:
"Makalenin tamamını oku →",
["Reject-all = ignore Set-Cookie entirely (zero counts on cookie-flag issues). Block-third-party = analyse only first-party cookies (Domain attribute matches the page's registrable domain). Accept-all = analyse every Set-Cookie regardless of scope."]:
"Tümünü reddet = Set-Cookie tamamen yok sayılır (çerez bayrağı sorunları sıfır çıkar). Üçüncü tarafı engelle = yalnızca birinci taraf çerezler analiz edilir (Domain özniteliği sayfanın kayıtlı alan adıyla eşleşenler). Tümünü kabul et = kapsamdan bağımsız olarak her Set-Cookie analiz edilir.",
["Reject-all for stateless audits; Block-third-party to focus on the site's own cookie hygiene; Accept-all to also see ad/analytics tracker cookies."]:
"Durumsuz denetimler için Tümünü reddet; sitenin kendi çerez hijyenine odaklanmak için Üçüncü tarafı engelle; reklam/analitik takip çerezlerini de görmek için Tümünü kabul et.",
["Removes the leading 'www.' from the host at normalisation time. The seen-set, redirect graph, and link extraction all use the rewritten form, so duplicates collapse correctly."]:
"Normalizasyon sırasında host'un başındaki 'www.' ifadesini kaldırır. Görülenler kümesi, yönlendirme grafiği ve bağlantı çıkarımı yeniden yazılmış biçimi kullanır, böylece yinelenenler doğru şekilde birleşir.",
["Renders the page a second time on a mobile viewport and stores an above-the-fold PNG. Adds another full render + screenshot per URL."]:
"Sayfayı mobil görünümde ikinci kez render eder ve ekranın üst kısmının PNG'sini saklar. URL başına bir tam render ve ekran görüntüsü daha ekler.",
["Resolved absolute URL of the <img src> attribute."]:
"<img src> özniteliğinin mutlak URL'e çözümlenmiş hali.",
["Response body size in bytes (compressed transfer size, post-Content-Encoding)."]:
"Yanıt gövdesinin bayt cinsinden boyutu (Content-Encoding sonrası sıkıştırılmış transfer boyutu).",
["Rewrites http:// to https:// before fetching. Breaks HTTP-only sites."]:
"Çekmeden önce http:// adresini https:// olarak yeniden yazar. Yalnızca HTTP üzerinden yayın yapan siteleri bozar.",
["Run Chromium without a visible window. Turn off to debug rendering visually — useful when a page renders correctly in a normal browser but not under Playwright."]:
"Chromium'u görünür pencere olmadan çalıştırır. Render'ı görsel olarak incelemek için kapatın — bir sayfa normal tarayıcıda doğru render edilip Playwright altında edilmiyorsa kullanışlıdır.",
["Run the login steps once before the crawl, then replay the session cookies on every request."]:
"Giriş adımlarını crawl'dan önce bir kez çalıştırır, ardından oturum çerezlerini her istekte tekrar kullanır.",
["Runs iterative PageRank (damping 0.85) over the internal link graph and normalises it to a 0–100 Link Score per page. Drives the Link Score column and the 'By Link Score' visualization colour mode."]:
"İç bağlantı grafiği üzerinde yinelemeli PageRank (damping 0,85) çalıştırır ve sayfa başına 0–100 arası Bağlantı Puanına normalize eder. Bağlantı Puanı kolonunu ve görselleştirmedeki 'Bağlantı Puanına Göre' renk modunu besler.",
["Sends the URL through the same normalisation pipeline used by the crawler, with your unsaved settings applied. Useful for verifying regex rules before kicking off a crawl."]:
"URL'i, kaydedilmemiş ayarlarınız uygulanmış halde crawler'ın kullandığı normalizasyon hattından geçirir. Crawl başlatmadan önce regex kurallarını doğrulamak için kullanışlıdır.",
["Sent on every request as the User-Agent header. Identifies the crawler to servers; some sites serve different content based on UA."]:
"Her istekte User-Agent başlığı olarak gönderilir. Crawler'ı sunuculara tanıtır; bazı siteler UA'ya göre farklı içerik sunar.",
["Sent on every request. Affects which locale a multi-lingual site serves you."]:
"Her istekte gönderilir. Çok dilli bir sitenin size hangi dili sunacağını etkiler.",
["Sent verbatim as `Bearer <token>`. Don't include the `Bearer ` prefix yourself."]:
"`Bearer <token>` olarak birebir gönderilir. `Bearer ` ön ekini kendiniz eklemeyin.",
["Server response time (a TTFB proxy) measured during the crawl. Pages slower than this are flagged. Google considers a good server response time under 800 ms."]:
"Crawl sırasında ölçülen sunucu yanıt süresi (TTFB yaklaşığı). Bundan yavaş sayfalar işaretlenir. Google, 800 ms altındaki sunucu yanıt süresini iyi kabul eder.",
["Shop the latest game keys at unbeatable prices…"]:
"En yeni oyun anahtarlarını rakipsiz fiyatlarla keşfedin…",
["Skips body parsing for pages whose Content-Length header exceeds this. The page row is still created so links to it aren't lost; only body parsing and source snapshot capture are skipped."]:
"Content-Length başlığı bu değeri aşan sayfalarda gövde ayrıştırmasını atlar. Sayfa satırı yine oluşturulur, böylece ona giden bağlantılar kaybolmaz; yalnızca gövde ayrıştırma ve kaynak anlık görüntüsü atlanır.",
["Sleep this long on each worker AFTER a response completes, before it picks up the next URL. Stacks with the global RPS cap — useful for sites that rate-limit on inter-request gap rather than total throughput."]:
"Her işçinin, bir yanıt tamamlandıktan SONRA bir sonraki URL'i almadan önce bekleyeceği süre. Global RPS sınırıyla birlikte uygulanır — toplam hız yerine istekler arası boşluğa göre sınırlama yapan siteler için kullanışlıdır.",
["Specific reason a URL is non-indexable. For Indexable URLs this column is empty."]:
"Bir URL'in neden indekslenemez olduğunun spesifik nedeni. İndekslenebilir URL'lerde bu kolon boştur.",
["Spider follows links from the start URL across the chosen scope. List fetches a fixed set of URLs once with no link-following. Sitemap fetches a sitemap URL and crawls every page it lists (no link-following)."]:
"Spider, seçilen kapsam içinde başlangıç URL'inden bağlantıları takip eder. Liste, sabit bir URL kümesini bağlantı takibi yapmadan bir kez çeker. Sitemap, bir sitemap URL'ini çeker ve listelediği her sayfayı tarar (bağlantı takibi yok).",
["Spider for full site audits; List for re-checking a known set of pages; Sitemap to audit exactly what's published in sitemap.xml."]:
"Tam site denetimleri için Spider; bilinen bir sayfa kümesini yeniden kontrol etmek için Liste; sitemap.xml'de yayımlananı birebir denetlemek için Sitemap.",
["Standard CSS selector — same syntax as `document.querySelectorAll`."]:
"Standart CSS seçicisi — `document.querySelectorAll` ile aynı sözdizimi.",
["Stored in your local prefs file as plain text. Treat the file accordingly."]:
"Yerel tercihler dosyanızda düz metin olarak saklanır. Dosyayı buna göre koruyun.",
["Strip if your site canonicalises /foo (no slash); Add for sites that canonicalise /foo/."]:
"Siteniz /foo (eğik çizgisiz) biçimine kanonikleşiyorsa Kaldır; /foo/ biçimine kanonikleşen siteler için Ekle.",
["Sunset over the mountain ridge"]:
"Dağ sırtında gün batımı",
["Surplus `@id` occurrences across all JSON-LD blocks (page declares the same `@id` more than once)."]:
"Tüm JSON-LD blokları genelinde fazladan `@id` kullanımları (sayfa aynı `@id` değerini birden fazla kez tanımlıyor).",
["Terminal URL the redirect chain resolves to. Empty when this row is itself the terminal (i.e. status is 2xx/4xx/5xx) or when the chain hits a loop."]:
"Yönlendirme zincirinin çözümlendiği son URL. Bu satırın kendisi son nokta ise (yani durum 2xx/4xx/5xx) veya zincir bir döngüye girerse boştur.",
["text for visible content, attribute for href/src, count for occurrence count"]:
"görünür içerik için text, href/src için attribute, tekrar sayısı için count",
["Text of the first <h1> on the page. Should match user intent and ideally complement (not duplicate) the title."]:
"Sayfadaki ilk <h1> metni. Kullanıcı niyetiyle örtüşmeli ve ideal olarak başlığı tekrar etmek yerine tamamlamalıdır.",
["Text Only fetches the raw HTML response as-is — fast and deterministic. Old AJAX Crawling Scheme rewrites hashbang (#!) URLs to Google's deprecated ?_escaped_fragment_= form so a pre-rendering server returns the snapshot. Full JavaScript rendering is a V2 item."]:
"Yalnızca Metin, ham HTML yanıtını olduğu gibi çeker — hızlı ve öngörülebilir. Eski AJAX Tarama Şeması, hashbang (#!) URL'lerini Google'ın kullanımdan kaldırdığı ?_escaped_fragment_= biçimine çevirir, böylece ön render yapan sunucu anlık görüntüyü döner. Tam JavaScript render'ı bir V2 maddesidir.",
["Text Only for server-rendered / static sites; Old AJAX only for legacy hashbang SPAs."]:
"Sunucu tarafında render edilen / statik siteler için Yalnızca Metin; yalnızca eski hashbang SPA'lar için Eski AJAX.",
["The column / JSON-key name for this rule's output. Free-form."]:
"Bu kuralın çıktısı için kolon / JSON anahtar adı. Serbest biçimlidir.",
["The fully normalised URL of the crawled resource (post URL-rewriting)."]:
"Taranan kaynağın tam normalize edilmiş URL'i (URL yeniden yazımı sonrası).",
["The URL that fails to resolve (4xx/5xx/network error)."]:
"Çözümlenemeyen URL (4xx/5xx/ağ hatası).",
["Third-party `<script>` / `<link rel=stylesheet>` references without an `integrity=` attribute. SRI is recommended for any cross-origin subresource."]:
"`integrity=` özniteliği olmayan üçüncü taraf `<script>` / `<link rel=stylesheet>` referansları. Çapraz kaynaklı her alt kaynak için SRI önerilir.",
["Time-to-first-byte in milliseconds (network + server, excluding parse). Lower is better; >2000 ms is slow."]:
"Milisaniye cinsinden ilk bayta kadar geçen süre (ağ + sunucu, ayrıştırma hariç). Düşük olması iyidir; >2000 ms yavaştır.",
["Total number of <h1> elements on the page. SEO best practice is exactly 1."]:
"Sayfadaki toplam <h1> öğesi sayısı. SEO açısından en iyi uygulama tam olarak 1'dir.",
["Total number of <h2> elements on the page."]:
"Sayfadaki toplam <h2> öğesi sayısı.",
["tr,en;q=0.8 — Turkish first, English fallback."]:
"tr,en;q=0.8 — önce Türkçe, yedek olarak İngilizce.",
["Trips 'Folder Depth Too Deep' when the URL path's `/`-segment count exceeds this. Useful for spotting over-nested URL structures that bury content from crawlers."]:
"URL yolundaki `/` bölüm sayısı bu değeri aştığında 'Klasör Derinliği Çok Fazla' sorununu tetikler. İçeriği crawler'lardan gizleyen aşırı iç içe URL yapılarını tespit etmek için kullanışlıdır.",
["Trips 'Long Query String' when LENGTH(query) > this. Typical session-id sprawl + UTM tracking hits 100+ chars; over 200 starts to look like a bug."]:
"LENGTH(query) bu değeri aştığında 'Uzun Sorgu Dizesi' sorununu tetikler. Tipik oturum kimliği ve UTM takip parametreleri 100+ karaktere ulaşır; 200 üzeri artık bir hataya benzer.",
["Trips the 'URL Too Long' issue when LENGTH(url) > this. RFC 7230 doesn't mandate a max but most servers + middleboxes fail above ~2 KB; Chrome itself caps at ~32 KB."]:
"LENGTH(url) bu değeri aştığında 'URL Çok Uzun' sorununu tetikler. RFC 7230 bir üst sınır zorunlu kılmaz ama çoğu sunucu ve ara katman ~2 KB üzerinde hata verir; Chrome'un kendi sınırı ~32 KB'dir.",
["Two modes per line. (1) Wrap in slashes for a regex: /pattern/flags — supported flags imsuy (g is forced). Invalid patterns appear with count -1 in the detail panel so you can spot the typo. (2) Anything else is a literal case-insensitive substring — the legacy behaviour. Each term's per-page hit count is surfaced in the URL Details panel."]:
"Satır başına iki mod. (1) Regex için eğik çizgiler arasına alın: /desen/bayraklar — desteklenen bayraklar imsuy (g zorunlu uygulanır). Geçersiz desenler detay panelinde -1 sayısıyla görünür, böylece yazım hatasını fark edebilirsiniz. (2) Diğer her şey büyük/küçük harf duyarsız birebir alt dizedir — eski davranış. Her terimin sayfa başına eşleşme sayısı URL Detayları panelinde gösterilir.",
["Two pages are flagged as near-duplicates if their 64-bit SimHash differs by at most this many bits. 3 ≈ 95% similarity over body-text shingles (Screaming Frog's tightest filter). Set to 0 to skip clustering entirely."]:
"İki sayfanın 64-bit SimHash değeri en fazla bu kadar bit farklıysa yakın-yinelenen olarak işaretlenir. 3 ≈ gövde metni parçaları üzerinde %95 benzerlik (Screaming Frog'un en sıkı filtresi). Kümelemeyi tamamen atlamak için 0 yapın.",
["URL declared by the first <link rel=\"canonical\"> tag. Tells search engines which version to index when duplicates exist."]:
"İlk <link rel=\"canonical\"> etiketinde tanımlanan URL. Yinelenenler varken arama motorlarına hangi sürümün indeksleneceğini söyler.",
["URL paths ending in any of these extensions are not enqueued. Case-insensitive. Start URL is always crawled regardless."]:
"Bu uzantılardan biriyle biten URL yolları kuyruğa alınmaz. Büyük/küçük harf duyarsızdır. Başlangıç URL'i her durumda taranır.",
["Value of the alt attribute. Empty cell = no alt declared (accessibility/SEO issue)."]:
"alt özniteliğinin değeri. Boş hücre = alt tanımlanmamış (erişilebilirlik/SEO sorunu).",
["Value of the X-Robots-Tag HTTP response header. Same semantics as meta robots but applied at the server."]:
"X-Robots-Tag HTTP yanıt başlığının değeri. meta robots ile aynı anlama gelir ama sunucu tarafında uygulanır.",
["Viewport height — affects above-the-fold detection and lazy-load triggers."]:
"Görünüm yüksekliği — ekranın üstü tespitini ve lazy-load tetikleyicilerini etkiler.",
["Viewport width applied to every rendered page. Mobile audits typically use 360–414, desktop 1280–1920."]:
"Render edilen her sayfaya uygulanan görünüm genişliği. Mobil denetimlerde tipik olarak 360–414, masaüstünde 1280–1920 kullanılır.",
["Visible body text word count (excludes <script>/<style>). Useful for identifying thin content."]:
"Görünür gövde metninin kelime sayısı (<script>/<style> hariç). Zayıf içeriği tespit etmek için kullanışlıdır.",
["Wait this long before the FIRST retry, doubling on each subsequent attempt (500 → 1000 → 2000 …)."]:
"İLK yeniden denemeden önce bu kadar bekler, sonraki her denemede süre ikiye katlanır (500 → 1000 → 2000 …).",
["Walks 3xx redirect chains, fills `redirect_chain_length` / `redirect_loop`. Drives the 'Long Chain' and 'Redirect Loop' issues + the Redirects tab."]:
"3xx yönlendirme zincirlerini yürür, `redirect_chain_length` / `redirect_loop` alanlarını doldurur. 'Uzun Zincir' ve 'Yönlendirme Döngüsü' sorunlarını ve Yönlendirmeler sekmesini besler.",
["Welcome to Example Store"]:
"Örnek Mağazaya Hoş Geldiniz",
["What to do when multiple matches exist."]:
"Birden fazla eşleşme olduğunda ne yapılacağı.",
["What to read off each matched element. Ignored for an XPath `/@attr` or `/text()` terminal — that value is used directly."]:
"Eşleşen her öğeden neyin okunacağı. XPath `/@attr` veya `/text()` ile biten ifadelerde yok sayılır — o değer doğrudan kullanılır.",
["When non-empty, ALL query parameters not on this list are dropped during normalisation (case-insensitive name match). Leave empty to keep the default behaviour, which strips just utm_*, fbclid, gclid, mc_cid, and mc_eid."]:
"Boş değilse, bu listede olmayan TÜM sorgu parametreleri normalizasyon sırasında düşürülür (ad eşleşmesi büyük/küçük harf duyarsızdır). Yalnızca utm_*, fbclid, gclid, mc_cid ve mc_eid parametrelerini temizleyen varsayılan davranış için boş bırakın.",
["When off, no budget evaluation runs and the verdict column is cleared. When on, the post-crawl pass scores every internal 200 HTML page against the ceilings below."]:
"Kapalıyken bütçe değerlendirmesi çalışmaz ve sonuç kolonu temizlenir. Açıkken crawl sonrası geçiş, 200 dönen her iç HTML sayfasını aşağıdaki tavanlara göre puanlar.",
["When on (default), pagination_next + pagination_prev URLs are post-fetch enqueued. Off only to debug pagination-only loops without disabling all link follow."]:
"Açıkken (varsayılan), pagination_next ve pagination_prev URL'leri çekim sonrası kuyruğa alınır. Yalnızca tüm bağlantı takibini kapatmadan sayfalama döngülerini incelemek için kapatın.",
["When ON (default), the Duplicate URL filter compares URLs after lowercasing the host, dropping the query string, and trimming the trailing slash — the canonical SEO behaviour. When OFF, comparison is byte-exact, so the filter only fires on rows that share an identical raw URL string (rare since URLs are deduped at insert time)."]:
"AÇIK olduğunda (varsayılan), Yinelenen URL filtresi URL'leri host küçük harfe çevrilmiş, sorgu dizesi atılmış ve sondaki eğik çizgi kırpılmış halde karşılaştırır — standart SEO davranışı. KAPALI olduğunda karşılaştırma bayt bazındadır, bu yüzden filtre yalnızca birebir aynı ham URL dizesini paylaşan satırlarda tetiklenir (URL'ler ekleme sırasında zaten tekilleştirildiği için nadirdir).",
["When on, `<meta http-equiv='refresh'>` content URLs are enqueued like a redirect target. window.location body redirects are heuristic-only and currently out of scope."]:
"Açıkken `<meta http-equiv='refresh'>` içeriğindeki URL'ler bir yönlendirme hedefi gibi kuyruğa alınır. window.location ile yapılan gövde yönlendirmeleri yalnızca sezgiseldir ve şu an kapsam dışıdır.",
["When on, a 200 page declaring a canonical pointing elsewhere also enqueues that target. Default off — most crawls treat canonicals as a signal, not a navigation hint."]:
"Açıkken, başka bir yeri gösteren canonical tanımlayan 200 sayfaları o hedefi de kuyruğa alır. Varsayılan kapalıdır — çoğu crawl canonical'ı gezinme ipucu değil, sinyal olarak değerlendirir.",
["When on, pages with noindex / canonicalised / robots-blocked indexability are excluded from clustering — the Near-Duplicate report then surfaces only issues that affect search visibility."]:
"Açıkken noindex / canonical ile yönlendirilmiş / robots tarafından engellenmiş sayfalar kümelemenin dışında tutulur — Yakın Yinelenen raporu böylece yalnızca arama görünürlüğünü etkileyen sorunları gösterir.",
["When on, rel=nofollow links are recursed into like any other link. Default off — Screaming Frog 'Respect Nofollow' default."]:
"Açıkken rel=nofollow bağlantıları diğer bağlantılar gibi takip edilir. Varsayılan kapalıdır — Screaming Frog'un 'Respect Nofollow' varsayılanı.",
["When Playwright considers navigation complete. domcontentloaded = HTML parsed but resources still loading. load = window.load fired. networkidle = no network activity for 500ms (best for SPA but slower). commit = just response committed (fastest, riskiest)."]:
"Playwright'ın gezinmeyi ne zaman tamamlanmış sayacağı. domcontentloaded = HTML ayrıştırıldı ama kaynaklar hâlâ yükleniyor. load = window.load tetiklendi. networkidle = 500 ms boyunca ağ etkinliği yok (SPA için en iyisi ama daha yavaş). commit = yalnızca yanıt teslim edildi (en hızlı, en riskli).",
["Whether the broken target is on the same site (internal) or a different host (external)."]:
"Kırık hedefin aynı sitede (iç) mi yoksa farklı bir sunucuda (dış) mı olduğu.",
["Whether the URL is eligible to appear in search results. Combines status code, robots directives, canonical, and meta-refresh signals."]:
"URL'in arama sonuçlarında görünmeye uygun olup olmadığı. Durum kodu, robots direktifleri, canonical ve meta-refresh sinyallerini birleştirir.",
["Width attribute value (in pixels) declared on the <img> tag, when present."]:
"Varsa <img> etiketinde tanımlanan width özniteliğinin değeri (piksel).",
["XPath 1.0 subset over the parsed DOM. End in `/@attr` or `/text()` to read an attribute / text node. Predicates: `[n]`, `[@class=\"x\"]`, `[contains(@class,\"x\")]`, `[last()]`."]:
"Ayrıştırılmış DOM üzerinde XPath 1.0 alt kümesi. Bir özniteliği / metin düğümünü okumak için `/@attr` veya `/text()` ile bitirin. Yüklemler: `[n]`, `[@class=\"x\"]`, `[contains(@class,\"x\")]`, `[last()]`.",
["Y when the page declares hreflang alternates but no entry whose `href` matches the page URL. Google requires a self-reference."]:
"Sayfa hreflang alternatifleri tanımlıyor ama `href` değeri sayfa URL'iyle eşleşen bir girdi yoksa Y olur. Google kendine referans verilmesini zorunlu tutar.",
["Y when the redirect chain originating at this URL contains a cycle (A → B → A) detected by the cycle-safe walker; the chain is otherwise unwalked."]:
"Bu URL'den başlayan yönlendirme zinciri, döngüye dayanıklı yürütecin tespit ettiği bir çevrim (A → B → A) içeriyorsa Y olur; aksi halde zincir yürünmemiştir.",
["Y when this URL belongs to a paginated cluster whose ordinal sequence has a gap (e.g. ?page=1, 2, 4 — page 3 missing). Set by the post-crawl `recomputePaginationSequence` pass."]:
"Bu URL, sıra numaralarında boşluk olan bir sayfalama kümesine aitse Y olur (ör. ?page=1, 2, 4 — 3. sayfa eksik). Crawl sonrası `recomputePaginationSequence` geçişi tarafından ayarlanır.",
};
/**
* Translate an InfoTip body. Returns the input unchanged for English, for
* unknown strings, and for empty input.
*/
export function translateInfoTip(text: string | undefined, lang: string): string | undefined {
if (!text || !lang.startsWith('tr')) return text;
return TR_INFO_TIPS[text] ?? text;
}
@@ -33,6 +33,7 @@
"cancel": "Cancel",
"close": "Close",
"apply": "Apply",
"applied": "Applied",
"ok": "OK",
"remove": "Remove",
"reset": "Reset",
@@ -93,6 +94,11 @@
"mobile": "Mobile crawl — crawl the site’s mobile version (mobile User-Agent)"
}
},
"browserInstall": {
"downloading": "Browser",
"downloadingTitle": "Downloading the Chromium browser used for JavaScript rendering. Crawling in text mode is unaffected.",
"failed": "Browser download failed — retry"
},
"stats": {
"discovered": "Discovered",
"crawled": "Crawled",
@@ -392,6 +398,7 @@
"settingsPanels": {
"presets": {
"intro": "One-click profiles for common crawl scenarios. Clicking a preset overwrites the affected fields only — your URL list, custom rules, filters, and extraction rules are preserved.",
"applied": "\"{{name}}\" preset applied — review the panels, then press Save to keep the changes.",
"importExport": "Import / Export",
"importExportIntro": "Save the current settings to a JSON file (e.g. for sharing with a teammate or version control), or load a previously-exported settings file. Importing replaces the current form with the file's contents — press Save at the bottom to persist.",
"exportBtn": "Export…",
@@ -818,7 +825,8 @@
"empty3": "or search terms in",
"empty4": "then re-crawl.",
"customExtraction": "Custom Extraction",
"customSearchHits": "Custom Search hits"
"customSearchHits": "Custom Search hits",
"colSource": "Source"
},
"extractionPreview": {
"title": "Custom Extraction Preview",
@@ -850,7 +858,11 @@
"parsedOk": "parsed OK",
"parseFailed": "parse failed",
"snapshotUnavailable1": "JSON-LD blocks were detected during crawl but the page body snapshot is unavailable. Re-crawl with",
"snapshotUnavailable2": "enabled to view the raw payload."
"snapshotUnavailable2": "enabled to view the raw payload.",
"colBlock": "Block",
"colValid": "Valid JSON",
"colTypes": "Types",
"colJson": "JSON"
},
"imagesView": {
"loading": "Loading images…",
@@ -896,7 +908,10 @@
"skipped": "Skipped: previous was h{{n}}",
"skippedCount": "{{n}} skipped level(s)",
"capped": "(capped at 200)",
"emptyHeading": "(empty heading)"
"emptyHeading": "(empty heading)",
"colLevel": "Level",
"colText": "Heading",
"colIssue": "Issue"
},
"viewSource": {
"searchPlaceholder": "Search source…",
@@ -1123,11 +1138,20 @@
"statusError": "ERR",
"statusSkipped": "skipped",
"statusSkippedTip": "Too little prose on the page to check.",
"summary": "{{shown}} pages · {{checked}} checked · {{errors}} with findings · {{selected}} selected",
"summary": "{{shown}} pages · {{checked}} checked · {{errors}} with findings · {{unsupported}} unsupported · {{selected}} selected",
"firstN": "(first {{n}})",
"checking": "Checking: {{url}}",
"emptyNoData": "No internal HTML pages crawled yet — run a crawl, then select pages to check.",
"emptyNoMatch": "No pages match the current filter."
"emptyNoMatch": "No pages match the current filter.",
"filterUnsupported": "Unsupported language",
"statusUnsupported": "n/a",
"statusUnsupportedTip": "LanguageTool has no rules for this page’s language, so it was not checked.",
"statusMismatch": "lang?",
"statusMismatchTip": "The findings were discarded — the page does not read as the language it was checked in.",
"langConflict": "The page declares lang=\"{{declared}}\" but its text reads as {{detected}}. The check used {{used}}.",
"unsupportedBanner": "{{count}} page(s) left unchecked: LanguageTool has no rules for {{langs}}, and grading that text against a substitute language would report an error on nearly every word. Nothing here is misconfigured — no setting makes this engine check {{langs}}.",
"statusOkLocal": "ok*",
"statusOkLocalTip": "Checked with the offline dictionary: spelling only, no grammar. LanguageTool has no rules for this language."
},
"projects": {
"title": "Projects",
@@ -1169,7 +1193,18 @@
"clean": "No spelling or grammar issues found.",
"summary": "{{n}} finding(s)",
"language": "Language",
"suggestions": "Suggestions"
"suggestions": "Suggestions",
"colType": "Type",
"colCategory": "Category",
"colText": "Text",
"colSuggestions": "Suggestions",
"colMessage": "Message",
"colContext": "Context",
"unsupported": "Not checked — LanguageTool has no rules for this page’s language.",
"detectedAs": "(detected: {{lang}})",
"langConflict": "html[lang] says {{declared}}, the text reads as {{detected}}",
"cleanLocal": "No spelling mistakes found. Grammar was not checked — LanguageTool has no rules for this language, so an offline dictionary was used instead.",
"engineLocal": "offline dictionary — spelling only, no grammar"
},
"gscTab": {
"title": "Search Console",
@@ -1301,7 +1336,10 @@
"ga4Title": "Analytics 4",
"ga4Empty": "No GA4 traffic data yet.",
"inspectionTitle": "URL Inspection",
"inspectionEmpty": "No URL Inspection result yet — run it from the Search Console tab."
"inspectionEmpty": "No URL Inspection result yet — run it from the Search Console tab.",
"colSource": "Source",
"colMetric": "Metric",
"colValue": "Value"
},
"brokenTab": {
"searchPlaceholder": "Search broken target URLs…",
@@ -1389,13 +1427,17 @@
},
"spelling": {
"title": "Spelling",
"intro": "Spelling, grammar and style checks run through LanguageTool against pages you select in the Spelling tab. The page's own html[lang] picks the language; pages that don't declare one are auto-detected. Configure the endpoint (public API or self-hosted) under Integrations → LanguageTool.",
"intro": "Spelling, grammar and style checks run through LanguageTool against pages you select in the Spelling tab. Each page's language is detected from its own text and cross-checked against html[lang]; pages in a language LanguageTool has no rules for are reported as unsupported rather than graded against the wrong dictionary. Configure the endpoint (public API or self-hosted) under Integrations → LanguageTool.",
"levelLabel": "Rule Level",
"levelDefault": "Default — spelling and grammar errors only",
"levelPicky": "Picky — also flags style, typography and wordiness",
"dictionaryLabel": "Custom Dictionary",
"dictionaryHint": "Words to never flag — brand names, product names, industry jargon. One per line, or separated by commas. Case-insensitive.",
"dictionaryCount": "{{n}} word(s) ignored"
"dictionaryCount": "{{n}} word(s) ignored",
"languageLabel": "Language",
"languageHint": "Leave on auto unless detection gets a site wrong. Pinning a language forces every page to be checked in it — useful for a single-language site whose html[lang] is wrong, and it also turns off the safety check that discards implausibly noisy results.",
"languageAuto": "Auto — detect from each page's text (recommended)",
"languageCount": "This endpoint offers {{n}} language variant(s). Languages outside that list — Turkish, Hungarian, Czech and others — cannot be checked by LanguageTool at all."
},
"storage": {
"title": "Storage",
@@ -1501,5 +1543,10 @@
"seedNoActive": "No active crawl — start a crawl first, then seed.",
"seedNoOrigin": "No crawled URLs yet to derive a base origin from.",
"seedNone": "Nothing new to seed."
},
"detailExport": {
"label": "Export",
"title": "Export this table to Excel or CSV",
"nothing": "Nothing to export"
}
}
@@ -33,6 +33,7 @@
"cancel": "İptal",
"close": "Kapat",
"apply": "Uygula",
"applied": "Uygulandı",
"ok": "Tamam",
"remove": "Kaldır",
"reset": "Sıfırla",
@@ -93,6 +94,11 @@
"mobile": "Mobil tarama — sitenin mobil sürümünü tara (mobil User-Agent)"
}
},
"browserInstall": {
"downloading": "Tarayıcı",
"downloadingTitle": "JavaScript render için kullanılan Chromium tarayıcısı indiriliyor. Metin modunda tarama bundan etkilenmez.",
"failed": "Tarayıcı indirme başarısız — tekrar dene"
},
"stats": {
"discovered": "Keşfedilen",
"crawled": "Taranan",
@@ -392,6 +398,7 @@
"settingsPanels": {
"presets": {
"intro": "Yaygın crawl senaryoları için tek tık profilleri. Bir preset'e tıklamak yalnızca etkilenen alanları üzerine yazar — URL listen, özel kuralların, filtrelerin ve çıkarım kuralların korunur.",
"applied": "\"{{name}}\" preset'i uygulandı — panelleri gözden geçir, sonra değişiklikleri korumak için Kaydet'e bas.",
"importExport": "İçe / Dışa Aktar",
"importExportIntro": "Mevcut ayarları bir JSON dosyasına kaydet (örn. ekip arkadaşıyla paylaşmak veya versiyon kontrolü için) ya da daha önce dışa aktarılmış bir ayar dosyasını yükle. İçe aktarmak mevcut formu dosyanın içeriğiyle değiştirir — kalıcı kaydetmek için alttaki Kaydet'e bas.",
"exportBtn": "Dışa Aktar…",
@@ -818,7 +825,8 @@
"empty3": "üzerinden veya arama terimlerini",
"empty4": "üzerinden ayarla, sonra yeniden crawl çalıştır.",
"customExtraction": "Özel Çıkarım",
"customSearchHits": "Özel Arama Sonuçları"
"customSearchHits": "Özel Arama Sonuçları",
"colSource": "Kaynak"
},
"extractionPreview": {
"title": "Özel Çıkarım Önizlemesi",
@@ -850,7 +858,11 @@
"parsedOk": "parse edildi",
"parseFailed": "parse başarısız",
"snapshotUnavailable1": "Crawl sırasında JSON-LD blokları algılandı ama sayfa gövde snapshot'ı mevcut değil. Ham payload'u görmek için",
"snapshotUnavailable2": "etkin halde yeniden crawl çalıştır."
"snapshotUnavailable2": "etkin halde yeniden crawl çalıştır.",
"colBlock": "Blok",
"colValid": "Geçerli JSON",
"colTypes": "Türler",
"colJson": "JSON"
},
"imagesView": {
"loading": "Görseller yükleniyor…",
@@ -896,7 +908,10 @@
"skipped": "Atlandı: önceki h{{n}}",
"skippedCount": "{{n}} seviye atlandı",
"capped": "(200 ile sınırlı)",
"emptyHeading": "(boş başlık)"
"emptyHeading": "(boş başlık)",
"colLevel": "Seviye",
"colText": "Başlık",
"colIssue": "Sorun"
},
"viewSource": {
"searchPlaceholder": "Kaynakta ara…",
@@ -1123,11 +1138,20 @@
"statusError": "HATA",
"statusSkipped": "atlandı",
"statusSkippedTip": "Sayfada denetlenecek kadar düzyazı yok.",
"summary": "{{shown}} sayfa · {{checked}} denetlendi · {{errors}} bulgulu · {{selected}} seçildi",
"summary": "{{shown}} sayfa · {{checked}} denetlendi · {{errors}} bulgulu · {{unsupported}} desteklenmeyen · {{selected}} seçildi",
"firstN": "(ilk {{n}})",
"checking": "Denetleniyor: {{url}}",
"emptyNoData": "Henüz taranmış iç HTML sayfası yok — önce bir crawl çalıştır, sonra denetlenecek sayfaları seç.",
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok."
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok.",
"filterUnsupported": "Desteklenmeyen dil",
"statusUnsupported": "yok",
"statusUnsupportedTip": "LanguageTool bu sayfanın dili için kural içermiyor, bu yüzden denetlenmedi.",
"statusMismatch": "dil?",
"statusMismatchTip": "Bulgular atıldı — sayfa, denetlendiği dilde yazılmış görünmüyor.",
"langConflict": "Sayfa lang=\"{{declared}}\" bildiriyor ama metni {{detected}} olarak okunuyor. Denetim {{used}} ile yapıldı.",
"unsupportedBanner": "{{count}} sayfa denetlenmeden bırakıldı: LanguageTool’un {{langs}} için kuralı yok ve bu metni yerine geçecek bir dille değerlendirmek neredeyse her kelimeye hata bastırırdı. Yanlış bir ayar yok — hiçbir ayar bu motora {{langs}} denetletemez.",
"statusOkLocal": "tamam*",
"statusOkLocalTip": "Çevrimdışı sözlükle denetlendi: yalnızca yazım, dilbilgisi yok. LanguageTool bu dil için kural içermiyor."
},
"projects": {
"title": "Projeler",
@@ -1169,7 +1193,18 @@
"clean": "Yazım veya dilbilgisi sorunu bulunamadı.",
"summary": "{{n}} bulgu",
"language": "Dil",
"suggestions": "Öneriler"
"suggestions": "Öneriler",
"colType": "Tür",
"colCategory": "Kategori",
"colText": "Metin",
"colSuggestions": "Öneriler",
"colMessage": "Mesaj",
"colContext": "Bağlam",
"unsupported": "Denetlenmedi — LanguageTool bu sayfanın dili için kural içermiyor.",
"detectedAs": "(algılanan: {{lang}})",
"langConflict": "html[lang] {{declared}} diyor, metin {{detected}} olarak okunuyor",
"cleanLocal": "Yazım hatası bulunamadı. Dilbilgisi denetlenmedi — LanguageTool bu dil için kural içermediğinden çevrimdışı sözlük kullanıldı.",
"engineLocal": "çevrimdışı sözlük — yalnızca yazım, dilbilgisi yok"
},
"gscTab": {
"title": "Search Console",
@@ -1301,7 +1336,10 @@
"ga4Title": "Analytics 4",
"ga4Empty": "Henüz GA4 trafik verisi yok.",
"inspectionTitle": "URL Inspection",
"inspectionEmpty": "URL Inspection sonucu yok — Search Console sekmesinden çalıştır."
"inspectionEmpty": "URL Inspection sonucu yok — Search Console sekmesinden çalıştır.",
"colSource": "Kaynak",
"colMetric": "Metrik",
"colValue": "Değer"
},
"brokenTab": {
"searchPlaceholder": "Kırık hedef URL ara…",
@@ -1389,13 +1427,17 @@
},
"spelling": {
"title": "Yazım",
"intro": "Yazım, dilbilgisi ve stil denetimleri, Yazım sekmesinde seçtiğiniz sayfalar üzerinde LanguageTool ile çalışır. Dil, sayfanın kendi html[lang] değerinden seçilir; bildirmeyen sayfalar otomatik algılanır. Uç noktayı (genel API veya kendi sunucunuz) Entegrasyonlar → LanguageTool altından ayarlayın.",
"intro": "Yazım, dilbilgisi ve stil denetimleri, Yazım sekmesinde seçtiğiniz sayfalar üzerinde LanguageTool ile çalışır. Her sayfanın dili kendi metninden algılanır ve html[lang] ile karşılaştırılır; LanguageTool’un kuralı olmayan bir dildeki sayfalar yanlış sözlükle değerlendirilmek yerine desteklenmiyor olarak raporlanır. Uç noktayı (genel API veya kendi sunucunuz) Entegrasyonlar → LanguageTool altından ayarlayın.",
"levelLabel": "Kural Seviyesi",
"levelDefault": "Varsayılan — yalnızca yazım ve dilbilgisi hataları",
"levelPicky": "Titiz — stil, tipografi ve gereksiz uzunluğu da işaretler",
"dictionaryLabel": "Özel Sözlük",
"dictionaryHint": "Asla işaretlenmeyecek kelimeler — marka adları, ürün adları, sektör jargonu. Her satıra bir tane veya virgülle ayrılmış. Büyük/küçük harf duyarsız.",
"dictionaryCount": "{{n}} kelime yok sayılıyor"
"dictionaryCount": "{{n}} kelime yok sayılıyor",
"languageLabel": "Dil",
"languageHint": "Algılama bir siteyi yanlış okumadıkça otomatik bırakın. Bir dil sabitlemek her sayfayı o dilde denetler — html[lang] değeri yanlış olan tek dilli siteler için yararlıdır ve inandırıcı olmayan gürültülü sonuçları atan güvenlik kontrolünü de kapatır.",
"languageAuto": "Otomatik — her sayfanın metninden algıla (önerilen)",
"languageCount": "Bu uç nokta {{n}} dil varyantı sunuyor. Bu listenin dışındaki diller — Türkçe, Macarca, Çekçe ve diğerleri — LanguageTool ile hiç denetlenemez."
},
"storage": {
"title": "Depolama",
@@ -1501,5 +1543,10 @@
"seedNoActive": "Aktif crawl yok — önce bir crawl başlatın, sonra ekleyin.",
"seedNoOrigin": "Temel origin türetmek için henüz taranmış URL yok.",
"seedNone": "Eklenecek yeni bir şey yok."
},
"detailExport": {
"label": "Dışa Aktar",
"title": "Bu tabloyu Excel veya CSV olarak dışa aktar",
"nothing": "Dışa aktarılacak veri yok"
}
}
+24 -2
View File
@@ -7,6 +7,21 @@ import {
type OverviewCounts,
type UrlCategory,
} from '@freecrawl/shared-types';
import type { SettingsSectionKey } from './components/SettingsDialog.js';
/**
* Deep link into the Settings dialog. A banner that says "you need an API
* key" should drop the user on the field that fixes it, not on whichever
* panel happened to be open last.
*
* The import above is type-only, so it is erased at build time and no
* module cycle with SettingsDialog reaches the bundle.
*/
export interface SettingsTarget {
section: SettingsSectionKey;
/** Integration to scroll to and highlight — Integrations panel only. */
integration?: string;
}
export type TabKey =
| 'internal'
@@ -341,6 +356,9 @@ interface AppState {
sidebarOpen: boolean;
detailPanelOpen: boolean;
settingsOpen: boolean;
/** Where the Settings dialog should land when it opens. Null keeps
* whichever section the user was last on. */
settingsTarget: SettingsTarget | null;
recentUrls: string[];
dataVersion: number;
setConfig: (patch: Partial<CrawlConfig>) => void;
@@ -358,7 +376,7 @@ interface AppState {
setSelectedUrlIds: (ids: number[]) => void;
toggleSidebar: () => void;
toggleDetailPanel: () => void;
setSettingsOpen: (open: boolean) => void;
setSettingsOpen: (open: boolean, target?: SettingsTarget) => void;
addRecentUrl: (url: string) => void;
bumpDataVersion: () => void;
reset: () => void;
@@ -454,6 +472,7 @@ export const useAppStore = create<AppState>((set) => ({
sidebarOpen: true,
detailPanelOpen: true,
settingsOpen: false,
settingsTarget: null,
recentUrls: loadRecentUrls(),
dataVersion: 0,
setConfig: (patch) =>
@@ -487,7 +506,10 @@ export const useAppStore = create<AppState>((set) => ({
setSelectedUrlIds: (ids) => set({ selectedUrlIds: ids }),
toggleSidebar: () => set((s) => ({ sidebarOpen: !s.sidebarOpen })),
toggleDetailPanel: () => set((s) => ({ detailPanelOpen: !s.detailPanelOpen })),
setSettingsOpen: (open) => set({ settingsOpen: open }),
// Clearing the target on close matters: reopening from the menu should
// not re-run a deep link the user has already followed.
setSettingsOpen: (open, target) =>
set({ settingsOpen: open, settingsTarget: open ? (target ?? null) : null }),
addRecentUrl: (url) =>
set((state) => {
const trimmed = url.trim();
+5 -1
View File
@@ -52,8 +52,12 @@
.input {
@apply w-full rounded border border-surface-700 bg-surface-900 px-2 py-1.5 text-xs text-surface-100 placeholder:text-surface-500 focus:border-accent-500 focus:outline-none focus:ring-1 focus:ring-accent-500;
}
/* `shrink-0` + `whitespace-nowrap` keep every tab on one line: without
them the flex row compresses the buttons until long labels wrap to a
second line, which made the detail panel's tab bar twice as tall. The
row scrolls horizontally instead (see BottomDetailPanel). */
.tab {
@apply cursor-pointer border-b-2 border-transparent px-3 py-2 text-xs font-medium text-surface-400 transition hover:text-surface-100;
@apply shrink-0 cursor-pointer whitespace-nowrap border-b-2 border-transparent px-3 py-2 text-xs font-medium text-surface-400 transition hover:text-surface-100;
}
.tab-active {
@apply border-accent-500 text-surface-50;
@@ -310,7 +310,9 @@ export function CruxTab() {
</span>
<button
type="button"
onClick={() => setSettingsOpen(true)}
onClick={() =>
setSettingsOpen(true, { section: 'integrations', integration: 'crux' })
}
className="rounded border border-amber-500/60 bg-amber-700/40 px-2 py-0.5 text-[10px] font-medium text-amber-50 hover:bg-amber-700/70"
>
{t('cruxTab.addKeyCta', { defaultValue: 'Add API Key…' })}
@@ -336,7 +336,9 @@ export function PageSpeedTab() {
</span>
<button
type="button"
onClick={() => setSettingsOpen(true)}
onClick={() =>
setSettingsOpen(true, { section: 'integrations', integration: 'pagespeed' })
}
className="rounded border border-amber-500/60 bg-amber-700/40 px-2 py-0.5 text-[10px] font-medium text-amber-50 hover:bg-amber-700/70"
>
{t('pagespeedTab.addKeyCta', { defaultValue: 'Add API Key…' })}
@@ -14,7 +14,7 @@ const POLL_MS_RUNNING = 5000;
/** The public LanguageTool API allows ~20 requests/min — warn past this. */
const CONFIRM_THRESHOLD = 20;
type FilterMode = 'all' | 'checked' | 'unchecked' | 'errors';
type FilterMode = 'all' | 'checked' | 'unchecked' | 'errors' | 'unsupported';
/** Colour the match count by severity. */
function countClass(n: number | null): string {
@@ -149,13 +149,32 @@ export function SpellingTab() {
}, []);
const checkedCount = useMemo(
() => rows.filter((r) => r.status !== null).length,
() => rows.filter((r) => r.status === 'ok').length,
[rows],
);
const withErrors = useMemo(
() => rows.filter((r) => (r.matchCount ?? 0) > 0).length,
() => rows.filter((r) => r.status === 'ok' && (r.matchCount ?? 0) > 0).length,
[rows],
);
const unsupportedCount = useMemo(
() => rows.filter((r) => r.status === 'unsupported').length,
[rows],
);
/**
* The languages that came back unsupported, for the banner. Every
* unsupported row carries the language it was detected as, so a Turkish
* site names Turkish rather than leaving the user to guess why an entire
* crawl produced no findings.
*/
const unsupportedLangs = useMemo(() => {
const set = new Set<string>();
for (const r of rows) {
if (r.status === 'unsupported') {
set.add(r.detectedLanguage ?? r.lang ?? '?');
}
}
return [...set].sort();
}, [rows]);
return (
<div className="flex h-full w-full flex-col bg-surface-950">
@@ -193,6 +212,11 @@ export function SpellingTab() {
<option value="errors">
{t('spellingTab.filterErrors', { defaultValue: 'With findings' })}
</option>
<option value="unsupported">
{t('spellingTab.filterUnsupported', {
defaultValue: 'Unsupported language',
})}
</option>
</select>
<button
type="button"
@@ -204,7 +228,7 @@ export function SpellingTab() {
</button>
<button
type="button"
onClick={() => setSettingsOpen(true)}
onClick={() => setSettingsOpen(true, { section: 'spelling' })}
className="h-6 rounded border border-surface-700 bg-surface-800 px-2 text-[11px] text-surface-300 hover:bg-surface-700"
title={t('spellingTab.optionsTitle', {
defaultValue: 'Rule level and custom dictionary live in Settings → Spelling.',
@@ -248,6 +272,21 @@ export function SpellingTab() {
</div>
</div>
{/* LanguageTool ships rules for ~33 languages; Turkish, Hungarian,
Czech, Finnish and many others are simply not among them. Saying
so once, plainly, beats leaving the user to wonder why a whole
crawl came back empty. */}
{unsupportedCount > 0 && (
<div className="shrink-0 border-b border-amber-900/60 bg-amber-950/40 px-3 py-1.5 text-[11px] text-amber-200">
{t('spellingTab.unsupportedBanner', {
defaultValue:
'{{count}} page(s) left unchecked: LanguageTool has no rules for {{langs}}, and grading that text against a substitute language would report an error on nearly every word. Nothing here is misconfigured — no setting makes this engine check {{langs}}.',
count: unsupportedCount,
langs: unsupportedLangs.join(', '),
})}
</div>
)}
{/* Column header */}
<div
className="flex shrink-0 select-none items-center border-b border-surface-800 bg-surface-900/60 text-[10px] font-medium text-surface-400"
@@ -268,7 +307,7 @@ export function SpellingTab() {
<div className="flex-1 px-2">
{t('spellingTab.colUrl', { defaultValue: 'URL' })}
</div>
<div className="w-[60px] shrink-0 text-center">
<div className="w-[76px] shrink-0 text-center">
{t('spellingTab.colLang', { defaultValue: 'Lang' })}
</div>
<div className="w-[76px] shrink-0 text-right">
@@ -341,10 +380,11 @@ export function SpellingTab() {
<span>
{t('spellingTab.summary', {
defaultValue:
'{{shown}} pages · {{checked}} checked · {{errors}} with findings · {{selected}} selected',
'{{shown}} pages · {{checked}} checked · {{errors}} with findings · {{unsupported}} unsupported · {{selected}} selected',
shown: rows.length.toLocaleString(),
checked: checkedCount.toLocaleString(),
errors: withErrors.toLocaleString(),
unsupported: unsupportedCount.toLocaleString(),
selected: selected.size.toLocaleString(),
})}
{total > rows.length &&
@@ -382,18 +422,75 @@ function StatusBadge({ row }: { row: SpellingRow }) {
</span>
);
}
// Neither of these is a failure — LanguageTool simply has no rules for
// the page's language, or the check came back so noisy that the language
// must have been wrong. Both are amber, not red, and both explain
// themselves on hover.
if (row.status === 'unsupported') {
return (
<span
className="cursor-help text-amber-400"
title={
row.error ??
t('spellingTab.statusUnsupportedTip', {
defaultValue:
'LanguageTool has no rules for this page’s language, so it was not checked.',
})
}
>
{t('spellingTab.statusUnsupported', { defaultValue: 'n/a' })}
</span>
);
}
if (row.status === 'mismatch') {
return (
<span
className="cursor-help text-amber-400"
title={
row.error ??
t('spellingTab.statusMismatchTip', {
defaultValue:
'The findings were discarded — the page does not read as the language it was checked in.',
})
}
>
{t('spellingTab.statusMismatch', { defaultValue: 'lang?' })}
</span>
);
}
if (row.status === 'skipped') {
return (
<span
className="text-surface-500"
title={t('spellingTab.statusSkippedTip', {
defaultValue: 'Too little prose on the page to check.',
})}
className="cursor-help text-surface-500"
// A page can also be skipped for declaring no language and carrying
// too little text to detect one — that reason is stored, so prefer
// it over the generic tooltip.
title={
row.error ??
t('spellingTab.statusSkippedTip', {
defaultValue: 'Too little prose on the page to check.',
})
}
>
{t('spellingTab.statusSkipped', { defaultValue: 'skipped' })}
</span>
);
}
// Checked, but by the offline dictionary — spelling only. Marked so a
// clean row is not read as "grammar checked and fine".
if (row.engine === 'local') {
return (
<span
className="cursor-help text-emerald-400"
title={t('spellingTab.statusOkLocalTip', {
defaultValue:
'Checked with the offline dictionary: spelling only, no grammar. LanguageTool has no rules for this language.',
})}
>
{t('spellingTab.statusOkLocal', { defaultValue: 'ok*' })}
</span>
);
}
return (
<span className="text-emerald-400">
{t('spellingTab.statusOk', { defaultValue: 'ok' })}
@@ -401,6 +498,44 @@ function StatusBadge({ row }: { row: SpellingRow }) {
);
}
/**
* Language cell — the code the check ran under, or the code detected from
* the page's prose when it hasn't run. When the page's own `html[lang]`
* disagrees with what the prose reads as, the cell is marked: that is a
* genuine SEO defect (a stale `lang="en"` on a theme is common) and it is
* also the reason the old auto-detected results were nonsense.
*/
function LangCell({ row }: { row: SpellingRow }) {
const { t } = useTranslation();
const used = row.language ?? row.detectedLanguage ?? row.lang ?? null;
const declaredPrimary = row.lang?.split(/[-_]/)[0]?.toLowerCase() ?? null;
const detected = row.detectedLanguage;
const conflict =
!!detected && !!declaredPrimary && detected !== declaredPrimary;
return (
<div
className={`w-[76px] shrink-0 truncate text-center ${
conflict ? 'cursor-help text-amber-400' : 'text-surface-400'
}`}
title={
conflict
? t('spellingTab.langConflict', {
defaultValue:
'The page declares lang="{{declared}}" but its text reads as {{detected}}. The check used {{used}}.',
declared: row.lang ?? '',
detected,
used: used ?? detected,
})
: undefined
}
>
{used ?? '—'}
{conflict && ' ⚠'}
</div>
);
}
function SpellingTableRow({
row,
selected,
@@ -431,16 +566,19 @@ function SpellingTableRow({
<div className="flex-1 truncate px-2 text-surface-200" title={row.url}>
{row.url}
</div>
<div className="w-[60px] shrink-0 text-center text-surface-400">
{row.language ?? row.lang ?? '—'}
</div>
<LangCell row={row} />
<div className="w-[76px] shrink-0 text-right tabular-nums text-surface-400">
{row.wordCount?.toLocaleString() ?? '—'}
</div>
<div
className={`w-[80px] shrink-0 text-right font-mono tabular-nums ${countClass(row.matchCount)}`}
className={`w-[80px] shrink-0 text-right font-mono tabular-nums ${
row.status === 'ok' ? countClass(row.matchCount) : 'text-surface-700'
}`}
>
{row.matchCount?.toLocaleString() ?? '—'}
{/* A count is only meaningful when the page was actually checked —
an unsupported or mismatched page has zero findings because none
were collected, not because the copy is clean. */}
{row.status === 'ok' ? (row.matchCount?.toLocaleString() ?? '—') : '—'}
</div>
<div className="w-[78px] shrink-0 text-center">
<StatusBadge row={row} />
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/mcp-server",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"type": "module",
"bin": {
+3 -2
View File
@@ -663,14 +663,14 @@ export function buildTools(): Tool[] {
{
name: 'query_spelling',
description:
'Spelling & grammar check summary per crawled page (LanguageTool): match count, language used, status. Use filter "errors" for pages with ≥1 finding. Populate via Settings → Integrations → LanguageTool, then run a batch from the desktop Spelling tab (or spelling_run). Call get_url_spelling for a page\'s full match list.',
'Spelling & grammar check summary per crawled page (LanguageTool): match count, language used and detected, status. Use filter "errors" for pages with ≥1 finding, "unsupported" for pages LanguageTool has no dictionary for (it covers ~33 languages — Turkish, Hungarian, Czech and others are absent, and those pages are reported rather than graded against a substitute language). Status "mismatch" means the findings were discarded because the page did not read as the language it was checked in. Populate via Settings → Integrations → LanguageTool, then run a batch from the desktop Spelling tab (or spelling_run). Call get_url_spelling for a page\'s full match list.',
inputSchema: {
type: 'object',
properties: {
search: { type: 'string' },
filter: {
type: 'string',
enum: ['all', 'checked', 'unchecked', 'errors'],
enum: ['all', 'checked', 'unchecked', 'errors', 'unsupported'],
},
limit: { type: 'integer', minimum: 1, maximum: 1000 },
offset: { type: 'integer', minimum: 0 },
@@ -686,6 +686,7 @@ export function buildTools(): Tool[] {
| 'checked'
| 'unchecked'
| 'errors'
| 'unsupported'
| undefined,
}),
},
+121 -8
View File
@@ -1,12 +1,12 @@
{
"name": "freecrawl-seo",
"version": "0.9.0",
"version": "0.9.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "freecrawl-seo",
"version": "0.9.0",
"version": "0.9.1",
"hasInstallScript": true,
"license": "MIT",
"workspaces": [
@@ -27,7 +27,7 @@
},
"apps/cli": {
"name": "@freecrawl/cli",
"version": "0.9.0",
"version": "0.9.1",
"dependencies": {
"@freecrawl/core": "*",
"@freecrawl/db": "*",
@@ -44,7 +44,7 @@
},
"apps/desktop": {
"name": "@freecrawl/desktop",
"version": "0.9.0",
"version": "0.9.1",
"dependencies": {
"@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0",
@@ -60,8 +60,12 @@
"cytoscape": "^3.33.2",
"cytoscape-dagre": "^2.5.0",
"dagre": "^0.8.5",
"dictionary-tr": "^2.0.0",
"eld": "^2.0.3",
"franc": "^6.2.0",
"i18next": "^26.2.0",
"lucide-react": "^0.544.0",
"nspell": "^2.1.5",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"react-i18next": "^17.0.8",
@@ -72,6 +76,7 @@
"devDependencies": {
"@types/cytoscape-dagre": "^2.3.4",
"@types/dagre": "^0.7.54",
"@types/nspell": "^2.1.6",
"@types/react": "^19.2.0",
"@types/react-dom": "^19.2.0",
"@vitejs/plugin-react": "^5.0.0",
@@ -97,7 +102,7 @@
},
"apps/mcp-server": {
"name": "@freecrawl/mcp-server",
"version": "0.9.0",
"version": "0.9.1",
"dependencies": {
"@freecrawl/db": "*",
"@freecrawl/shared-types": "*"
@@ -2374,6 +2379,16 @@
"undici-types": "~6.21.0"
}
},
"node_modules/@types/nspell": {
"version": "2.1.6",
"resolved": "https://registry.npmjs.org/@types/nspell/-/nspell-2.1.6.tgz",
"integrity": "sha512-PnWYqrBZhkKAreX2kIZDJcBeigZJzXC8VmSuNiXeILUNreczBZHLxGEv/pRT+zL8rsXLxT5bRbiIF9wuh6cGgg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/plist": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/@types/plist/-/plist-3.0.5.tgz",
@@ -3782,6 +3797,16 @@
"node": ">=6"
}
},
"node_modules/collapse-white-space": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-2.1.0.tgz",
"integrity": "sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -4262,6 +4287,16 @@
"license": "MIT",
"optional": true
},
"node_modules/dictionary-tr": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/dictionary-tr/-/dictionary-tr-2.0.0.tgz",
"integrity": "sha512-Du0PHWIqAlNheOUEDgGea0lmEV4L/T57EvG4231IUeSWySSDZ8wj5TKo2Vlkx7j9RVD07XqZQLDCtOZvhll6xA==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/didyoumean": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz",
@@ -4485,6 +4520,15 @@
"node": ">=0.10.0"
}
},
"node_modules/eld": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/eld/-/eld-2.0.3.tgz",
"integrity": "sha512-KiwFrycPPJ2XgKb7MGsJ3m3grUFXq6+55R8ZuS9fj+Qv0WRFgeEBhmorHO/YwHV9VqhdJURKFGJ+CY24sEikOw==",
"license": "Apache-2.0",
"funding": {
"url": "https://linktr.ee/nitotm"
}
},
"node_modules/electron": {
"version": "41.2.2",
"resolved": "https://registry.npmjs.org/electron/-/electron-41.2.2.tgz",
@@ -5358,6 +5402,19 @@
"url": "https://github.com/sponsors/rawify"
}
},
"node_modules/franc": {
"version": "6.2.0",
"resolved": "https://registry.npmjs.org/franc/-/franc-6.2.0.tgz",
"integrity": "sha512-rcAewP7PSHvjq7Kgd7dhj82zE071kX5B4W1M4ewYMf/P+i6YsDQmj62Xz3VQm9zyUzUXwhIde/wHLGCMrM+yGg==",
"license": "MIT",
"dependencies": {
"trigram-utils": "^2.0.0"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/fs-extra": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
@@ -5993,6 +6050,29 @@
"node": ">=8"
}
},
"node_modules/is-buffer": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
"integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/is-core-module": {
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz",
@@ -6507,6 +6587,16 @@
"thenify-all": "^1.0.0"
}
},
"node_modules/n-gram": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/n-gram/-/n-gram-2.0.2.tgz",
"integrity": "sha512-S24aGsn+HLBxUGVAUFOwGpKs7LBcG4RudKU//eWzt/mQ97/NMKQxDWHyHx63UNWk/OOdihgmzoETn1tf5nQDzQ==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/nanoid": {
"version": "3.3.11",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz",
@@ -6709,6 +6799,15 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/nspell": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/nspell/-/nspell-2.1.5.tgz",
"integrity": "sha512-PSStyugKMiD9mHmqI/CR5xXrSIGejUXPlo88FBRq5Og1kO5QwQ5Ilu8D8O5I/SHpoS+mibpw6uKA8rd3vXd2Sg==",
"license": "MIT",
"dependencies": {
"is-buffer": "^2.0.0"
}
},
"node_modules/nth-check": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
@@ -8453,6 +8552,20 @@
"node": ">=8.0"
}
},
"node_modules/trigram-utils": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/trigram-utils/-/trigram-utils-2.0.1.tgz",
"integrity": "sha512-nfWIXHEaB+HdyslAfMxSqWKDdmqY9I32jS7GnqpdWQnLH89r6A5sdk3fDVYqGAZ0CrT8ovAFSAo6HRiWcWNIGQ==",
"license": "MIT",
"dependencies": {
"collapse-white-space": "^2.0.0",
"n-gram": "^2.0.0"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/truncate-utf8-bytes": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz",
@@ -9438,7 +9551,7 @@
},
"packages/core": {
"name": "@freecrawl/core",
"version": "0.9.0",
"version": "0.9.1",
"dependencies": {
"@freecrawl/db": "*",
"@freecrawl/shared-types": "*",
@@ -9467,7 +9580,7 @@
},
"packages/db": {
"name": "@freecrawl/db",
"version": "0.9.0",
"version": "0.9.1",
"dependencies": {
"@freecrawl/shared-types": "*"
},
@@ -9479,7 +9592,7 @@
},
"packages/shared-types": {
"name": "@freecrawl/shared-types",
"version": "0.9.0",
"version": "0.9.1",
"devDependencies": {
"rimraf": "^6.0.1",
"typescript": "^5.7.0"
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "freecrawl-seo",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
"license": "MIT",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/core",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"type": "module",
"main": "./dist/index.js",
+13 -13
View File
@@ -18,9 +18,8 @@
* wipe the session we just established).
*/
import { existsSync } from 'node:fs';
import { chromium, type LaunchOptions } from 'playwright';
import { PlaywrightBrowserMissingError } from './browser-pool.js';
import type { LaunchOptions } from 'playwright';
import { PlaywrightBrowserMissingError, resolveChromiumBinaries } from './browser-pool.js';
export interface BrowserLoginInput {
loginUrl: string;
@@ -82,23 +81,24 @@ export async function runBrowserLogin(
// Pre-flight the bundled-browser case so a missing binary surfaces a
// single clean install prompt instead of Playwright's per-call banner
// (mirrors BrowserPool.doStart).
const launchOpts: LaunchOptions = { headless: opts.headless };
if (!opts.channel && !opts.executablePath) {
let resolved = '';
try {
resolved = chromium.executablePath();
} catch {
/* unresolved → treat as missing below */
}
if (!resolved || !existsSync(resolved)) {
throw new PlaywrightBrowserMissingError(resolved, opts.channel);
const bins = await resolveChromiumBinaries();
if (!bins.installed) {
if (opts.headless && bins.chromiumInstalled) {
launchOpts.channel = 'chromium';
} else if (!bins.chromiumInstalled) {
throw new PlaywrightBrowserMissingError(bins.chromiumPath, opts.channel);
}
}
}
const launchOpts: LaunchOptions = { headless: opts.headless };
if (opts.channel) launchOpts.channel = opts.channel;
if (opts.executablePath) launchOpts.executablePath = opts.executablePath;
const timeout = Math.max(5000, opts.timeoutMs ?? 30_000);
// Lazy import — see the note in browser-pool.ts: a module-scope
// import would freeze PLAYWRIGHT_BROWSERS_PATH before the app sets it.
const { chromium } = await import('playwright');
const browser = await chromium.launch(launchOpts);
try {
const context = await browser.newContext({
+108 -22
View File
@@ -1,11 +1,22 @@
import { existsSync } from 'node:fs';
import {
chromium,
type Browser,
type BrowserContext,
type LaunchOptions,
type Page,
} from 'playwright';
import type { Browser, BrowserContext, LaunchOptions, Page } from 'playwright';
/**
* Playwright is imported lazily, never at module scope.
*
* Its browser registry reads `PLAYWRIGHT_BROWSERS_PATH` exactly once —
* while the module is being evaluated — and caches the resolved cache
* directory forever. A static `import { chromium } from 'playwright'`
* here therefore froze the lookup to the per-user cache before the
* Electron main process had a chance to point it at the browsers
* shipped inside the installer, which made the bundled browsers dead
* weight and sent every packaged user down the "browser missing"
* path. Loading on first use keeps that env var assignment effective.
*/
async function loadChromium() {
const { chromium } = await import('playwright');
return chromium;
}
/**
* Distinct error code so callers (main-process diagnostic dialog,
@@ -28,6 +39,82 @@ export class PlaywrightBrowserMissingError extends Error {
}
}
/** Where both Chromium builds live, and whether each is on disk. */
export interface ChromiumBinaries {
/** Full Chrome-for-Testing build — used for headed launches. */
chromiumPath: string;
/** chrome-headless-shell — what Playwright launches when headless. */
headlessShellPath: string;
chromiumInstalled: boolean;
headlessShellInstalled: boolean;
/** True only when both binaries are present. */
installed: boolean;
}
/**
* Derive the chrome-headless-shell path from the full Chromium path.
*
* Playwright installs the two as siblings under the same revision
* (`chromium-1223` / `chromium_headless_shell-1223`) but exposes only
* the former through `chromium.executablePath()`. The per-platform
* folder names mirror Playwright's own EXECUTABLE_PATHS table; note
* that the macOS and Linux-x64 layouts are arch-suffixed since the
* switch to Chrome-for-Testing builds. Returns '' when the layout is
* unrecognised — callers then treat the shell as "unknown" rather
* than "missing".
*/
function deriveHeadlessShellPath(chromiumPath: string): string {
if (!chromiumPath) return '';
const base = chromiumPath.replace(/chromium-(\d+)/, 'chromium_headless_shell-$1');
// Windows — chrome-win64\chrome.exe
const win = base.replace(
/chrome-win64[\\/]chrome\.exe$/i,
'chrome-headless-shell-win64\\chrome-headless-shell.exe',
);
if (win !== base) return win;
// macOS — chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/…
const mac = base.replace(
/chrome-mac-(arm64|x64)[\\/].*$/i,
'chrome-headless-shell-mac-$1/chrome-headless-shell',
);
if (mac !== base) return mac;
// Linux x64 — chrome-linux64/chrome
const linux = base.replace(
/chrome-linux64[\\/]chrome$/,
'chrome-headless-shell-linux64/chrome-headless-shell',
);
if (linux !== base) return linux;
// Linux arm64 — Playwright's own build, chrome-linux/chrome
const linuxArm = base.replace(/chrome-linux[\\/]chrome$/, 'chrome-linux/headless_shell');
if (linuxArm !== base) return linuxArm;
return '';
}
/**
* Resolve both Chromium binaries and report which are on disk. Used by
* the pool pre-flight and by the desktop app's startup check that
* decides whether to download the browser in the background.
*/
export async function resolveChromiumBinaries(): Promise<ChromiumBinaries> {
const chromium = await loadChromium();
let chromiumPath = '';
try {
chromiumPath = chromium.executablePath();
} catch {
/* no browser registered at all → both count as missing */
}
const headlessShellPath = deriveHeadlessShellPath(chromiumPath);
const chromiumInstalled = Boolean(chromiumPath) && existsSync(chromiumPath);
const headlessShellInstalled = Boolean(headlessShellPath) && existsSync(headlessShellPath);
return {
chromiumPath,
headlessShellPath,
chromiumInstalled,
headlessShellInstalled,
installed: chromiumInstalled && headlessShellInstalled,
};
}
export interface BrowserPoolOptions {
/** Number of pages kept warm in parallel. Matches crawl maxConcurrency. */
maxPages: number;
@@ -122,25 +209,24 @@ export class BrowserPool {
// surface a single install prompt and abort the JS render mode.
//
// When no explicit executable path or system channel is set,
// Playwright resolves the bundled browser. `chromium.executablePath()`
// throws when none is registered yet (very old install) — we treat
// that as "missing" too. For `headless: true` (default) Playwright
// 1.49+ also requires the chrome-headless-shell binary, which lives
// in a sibling directory; checking the regular chromium path is a
// good-enough proxy because both binaries are installed together
// by `npx playwright install`.
// Playwright resolves the bundled browser. A headless launch uses
// chrome-headless-shell, a headed one the full Chrome-for-Testing
// build — so both are checked independently. If only the shell is
// absent we degrade to the full build via `channel: 'chromium'`
// (Playwright's new-headless mode) instead of failing the crawl.
if (!this.opts.channel && !this.opts.executablePath) {
let resolved = '';
try {
resolved = chromium.executablePath();
} catch {
/* unresolved → fall through to the missing check below */
}
if (!resolved || !existsSync(resolved)) {
throw new PlaywrightBrowserMissingError(resolved, this.opts.channel);
const bins = await resolveChromiumBinaries();
if (!bins.installed) {
const needsShell = this.opts.headless;
if (needsShell && bins.chromiumInstalled) {
launchOpts.channel = 'chromium';
} else if (!bins.chromiumInstalled) {
throw new PlaywrightBrowserMissingError(bins.chromiumPath, this.opts.channel);
}
}
}
const chromium = await loadChromium();
this.browser = await chromium.launch(launchOpts);
this.context = await this.browser.newContext({
viewport: this.opts.viewport,
+5
View File
@@ -3,7 +3,9 @@ export type { CrawlerEvents } from './crawler.js';
export {
BrowserPool,
PlaywrightBrowserMissingError,
resolveChromiumBinaries,
type BrowserPoolOptions,
type ChromiumBinaries,
} from './browser-pool.js';
export {
renderUrl,
@@ -49,9 +51,12 @@ export { exportUrlsToJson, type JsonExportOptions } from './json-export.js';
export { exportUrlsToXml } from './xml-export.js';
export {
exportTabular,
exportGrid,
type TabularExportOptions,
type TabularExportResult,
type TabularSection,
type GridExportOptions,
type GridExportResult,
} from './tabular-export.js';
export { ensureHeapHeadroom, heapHeadroomBytes } from './heap-guard.js';
export {
+146
View File
@@ -263,6 +263,51 @@ function sanitizeSheetName(label: string, used: Set<string>): string {
return candidate;
}
/**
* Sheet XML from a plain cell matrix. Split out of the CrawlUrlRow
* variant below so the detail-panel grids — whose rows are already
* formatted strings, not database records — can reuse the same writer.
*/
function buildSheetXmlFromCells(
headers: readonly string[],
rows: readonly (readonly unknown[])[],
): string {
const parts: string[] = [];
parts.push(
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
'<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">' +
'<sheetData>',
);
parts.push('<row r="1">');
headers.forEach((h, i) => {
parts.push(
`<c r="${colLetter(i)}1" t="inlineStr"><is><t>${xmlEscape(h)}</t></is></c>`,
);
});
parts.push('</row>');
rows.forEach((row, rIdx) => {
const r = rIdx + 2;
parts.push(`<row r="${r}">`);
row.forEach((v, i) => {
if (v === null || v === undefined || v === '') return;
const cellRef = `${colLetter(i)}${r}`;
if (typeof v === 'number' && Number.isFinite(v)) {
parts.push(`<c r="${cellRef}"><v>${v}</v></c>`);
} else if (typeof v === 'boolean') {
parts.push(`<c r="${cellRef}" t="b"><v>${v ? 1 : 0}</v></c>`);
} else {
const safe = sanitizeFormulaPrefix(String(v));
parts.push(
`<c r="${cellRef}" t="inlineStr"><is><t>${xmlEscape(safe)}</t></is></c>`,
);
}
});
parts.push('</row>');
});
parts.push('</sheetData></worksheet>');
return parts.join('');
}
function buildSheetXml(
rows: CrawlUrlRow[],
columns: string[],
@@ -579,3 +624,104 @@ export async function exportTabular(
}
return { filePath: outputPath, files: written, rowsWritten: total };
}
// ── Detail-panel grid export ────────────────────────────────────────
//
// The URL Details panel's sub-tabs (Inlinks, Images, HTTP Headers,
// Cookies, …) each show a small table that is already formatted for
// display — the rows are strings the renderer computed, not database
// records. `exportTabular` above can't serve them: it streams straight
// from `ProjectDb` over `CrawlUrlRow` columns. This writer takes the
// literal cells instead, and reuses the same CSV escaping, formula-
// injection guard and XLSX packaging so a detail-panel export is
// indistinguishable from a main-table one.
export interface GridExportOptions {
format: 'csv' | 'xlsx';
/** Column headers, in order. */
headers: readonly string[];
/** Row cells, aligned to `headers`. */
rows: readonly (readonly unknown[])[];
/** Worksheet name (xlsx only). Sanitised to Excel's 31-char rules. */
sheetName?: string;
/** Prefix CSV with a UTF-8 BOM so Excel-for-Windows picks UTF-8. */
csvBom?: boolean;
}
export interface GridExportResult {
filePath: string;
rowsWritten: number;
}
/** Write one in-memory grid to a CSV or XLSX file. */
export async function exportGrid(
filePath: string,
options: GridExportOptions,
): Promise<GridExportResult> {
const { format, headers, rows } = options;
mkdirSync(path.dirname(filePath), { recursive: true });
if (format === 'xlsx') {
const sheet = sanitizeSheetName(options.sheetName ?? 'Sheet1', new Set());
writeFileSync(
filePath,
buildZipForCells(sheet, headers, rows),
);
return { filePath, rowsWritten: rows.length };
}
const withBom = options.csvBom !== false;
const generator = async function* (): AsyncGenerator<string> {
const header = headers.map((h) => escapeCsv(h)).join(',') + '\n';
yield withBom ? '' + header : header;
for (const row of rows) {
yield row.map((cell) => escapeCsv(cell)).join(',') + '\n';
}
};
await pipeline(
Readable.from(generator()),
createWriteStream(filePath, { encoding: 'utf8' }),
);
return { filePath, rowsWritten: rows.length };
}
/** Single-sheet workbook around {@link buildSheetXmlFromCells}. */
function buildZipForCells(
sheetName: string,
headers: readonly string[],
rows: readonly (readonly unknown[])[],
): Buffer {
const contentTypes =
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
'<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">' +
'<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>' +
'<Default Extension="xml" ContentType="application/xml"/>' +
'<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>' +
'<Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>' +
'</Types>';
const rels =
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
'<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>' +
'</Relationships>';
const workbook =
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
'<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">' +
`<sheets><sheet name="${xmlEscape(sheetName)}" sheetId="1" r:id="rId1"/></sheets></workbook>`;
const workbookRels =
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
'<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>' +
'</Relationships>';
return buildZip([
{ name: '[Content_Types].xml', data: Buffer.from(contentTypes, 'utf8') },
{ name: '_rels/.rels', data: Buffer.from(rels, 'utf8') },
{ name: 'xl/workbook.xml', data: Buffer.from(workbook, 'utf8') },
{ name: 'xl/_rels/workbook.xml.rels', data: Buffer.from(workbookRels, 'utf8') },
{
name: 'xl/worksheets/sheet1.xml',
data: Buffer.from(buildSheetXmlFromCells(headers, rows), 'utf8'),
},
]);
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/db",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"type": "module",
"main": "./dist/index.js",
+50
View File
@@ -1892,6 +1892,56 @@ const MIGRATIONS: Migration[] = [
}
},
},
{
version: 79,
name: 'add_spelling_detected_language',
// The check language is now resolved locally from the page's prose
// instead of being handed to LanguageTool's `auto` mode, which can only
// answer with a language it supports and so mislabels everything else.
//
// detected_language — primary code read out of the prose (`tr`)
// declared_language — primary code from html[lang] (`en`)
//
// Keeping both makes a wrong `lang` attribute visible as its own
// finding. Existing rows predate the resolver and were produced by the
// old `auto` path, so their results are cleared: a stored match list
// graded against the wrong language is worse than no result at all,
// and the page simply shows as unchecked until it is re-run.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(spelling_results)')
.all() as unknown as { name: string }[];
for (const col of ['detected_language', 'declared_language']) {
if (!cols.some((c) => c.name === col)) {
db.exec(`ALTER TABLE spelling_results ADD COLUMN ${col} TEXT`);
}
}
db.exec('DELETE FROM spelling_results');
},
},
{
version: 80,
name: 'add_spelling_engine',
// Pages in a language LanguageTool has no rules for are now checked
// against a bundled Hunspell dictionary instead of being left blank.
// That checker finds spelling mistakes only, so which engine produced
// a row has to be stored: without it a clean result would read as
// "grammar checked, nothing wrong" when grammar was never examined.
//
// Rows written before this column exists came from LanguageTool by
// definition, so they are backfilled rather than cleared.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(spelling_results)')
.all() as unknown as { name: string }[];
if (!cols.some((c) => c.name === 'engine')) {
db.exec('ALTER TABLE spelling_results ADD COLUMN engine TEXT');
}
db.exec(
"UPDATE spelling_results SET engine = 'languagetool' WHERE engine IS NULL",
);
},
},
];
export function runMigrations(db: DatabaseSync): void {
+74 -30
View File
@@ -25,6 +25,8 @@ import type {
SpellingResult,
SpellingRow,
SpellingMatch,
SpellingStatus,
SpellingEngine,
GscRow,
Ga4Row,
AiProvider,
@@ -53,6 +55,31 @@ import type {
import { issueCategoriesBySeverity } from '@freecrawl/shared-types';
import { runMigrations } from './migrations.js';
const SPELLING_STATUSES: readonly SpellingStatus[] = [
'ok',
'skipped',
'unsupported',
'mismatch',
'error',
];
/**
* Narrow a stored `spelling_results.status` string. Anything unrecognised
* — a row written by an older build — reads as "never checked" so the page
* simply offers itself for a re-run.
*/
function parseSpellingStatus(value: unknown): SpellingStatus | null {
return typeof value === 'string' &&
(SPELLING_STATUSES as readonly string[]).includes(value)
? (value as SpellingStatus)
: null;
}
/** Narrow a stored `spelling_results.engine` string. */
function parseSpellingEngine(value: unknown): SpellingEngine | null {
return value === 'languagetool' || value === 'local' ? value : null;
}
/**
* Best-effort `Content-Type` → {@link ContentKind} classifier. Used to
* re-classify external probe stubs (which are inserted as 'other' before the
@@ -3676,22 +3703,29 @@ export class ProjectDb {
this.db
.prepare(
`INSERT INTO spelling_results
(url, language, match_count, matches, status, error, fetched_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
(url, language, detected_language, declared_language,
match_count, matches, status, engine, error, fetched_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(url) DO UPDATE SET
language = excluded.language,
detected_language = excluded.detected_language,
declared_language = excluded.declared_language,
match_count = excluded.match_count,
matches = excluded.matches,
status = excluded.status,
engine = excluded.engine,
error = excluded.error,
fetched_at = excluded.fetched_at`,
)
.run(
url,
r.language,
r.detectedLanguage,
r.declaredLanguage,
r.matchCount,
r.matches.length > 0 ? JSON.stringify(r.matches) : null,
r.status,
r.engine,
r.error,
r.fetchedAt,
);
@@ -3707,7 +3741,7 @@ export class ProjectDb {
limit: number;
offset: number;
search?: string;
filter?: 'all' | 'checked' | 'unchecked' | 'errors';
filter?: 'all' | 'checked' | 'unchecked' | 'errors' | 'unsupported';
}): { rows: SpellingRow[]; total: number } {
const where: string[] = [
"u.is_external = 0 AND u.content_kind = 'html'",
@@ -3722,6 +3756,7 @@ export class ProjectDb {
if (filter === 'checked') where.push('s.url IS NOT NULL');
else if (filter === 'unchecked') where.push('s.url IS NULL');
else if (filter === 'errors') where.push('s.match_count > 0');
else if (filter === 'unsupported') where.push("s.status = 'unsupported'");
const whereSql = `WHERE ${where.join(' AND ')}`;
const joinSql = `
@@ -3735,8 +3770,11 @@ export class ProjectDb {
const rowsDb = this.db
.prepare(
`SELECT u.url AS url, u.lang AS lang, u.word_count AS word_count,
s.language AS language, s.match_count AS match_count,
s.status AS status, s.error AS error, s.fetched_at AS fetched_at
s.language AS language,
s.detected_language AS detected_language,
s.match_count AS match_count,
s.status AS status, s.engine AS engine,
s.error AS error, s.fetched_at AS fetched_at
${joinSql}
${whereSql}
ORDER BY s.match_count DESC, u.inlinks DESC, u.url ASC
@@ -3749,22 +3787,18 @@ export class ProjectDb {
return {
total: totalRow.c,
rows: rowsDb.map((r) => {
const status = r['status'];
return {
url: r['url'] as string,
lang: (r['lang'] as string | null) ?? null,
wordCount: (r['word_count'] as number | null) ?? null,
language: (r['language'] as string | null) ?? null,
matchCount: (r['match_count'] as number | null) ?? null,
status:
status === 'ok' || status === 'skipped' || status === 'error'
? status
: null,
error: (r['error'] as string | null) ?? null,
fetchedAt: (r['fetched_at'] as string | null) ?? null,
};
}),
rows: rowsDb.map((r) => ({
url: r['url'] as string,
lang: (r['lang'] as string | null) ?? null,
wordCount: (r['word_count'] as number | null) ?? null,
language: (r['language'] as string | null) ?? null,
detectedLanguage: (r['detected_language'] as string | null) ?? null,
matchCount: (r['match_count'] as number | null) ?? null,
status: parseSpellingStatus(r['status']),
engine: parseSpellingEngine(r['engine']),
error: (r['error'] as string | null) ?? null,
fetchedAt: (r['fetched_at'] as string | null) ?? null,
})),
};
}
@@ -3777,22 +3811,29 @@ export class ProjectDb {
getSpellingMatches(url: string): {
url: string;
language: string | null;
status: 'ok' | 'skipped' | 'error' | null;
detectedLanguage: string | null;
declaredLanguage: string | null;
status: SpellingStatus | null;
engine: SpellingEngine | null;
error: string | null;
fetchedAt: string | null;
matches: SpellingMatch[];
} | null {
const row = this.db
.prepare(
`SELECT url, language, matches, status, error, fetched_at
`SELECT url, language, detected_language, declared_language,
matches, status, engine, error, fetched_at
FROM spelling_results WHERE url = ?`,
)
.get(url) as
| {
url: string;
language: string | null;
detected_language: string | null;
declared_language: string | null;
matches: string | null;
status: string;
engine: string | null;
error: string | null;
fetched_at: string | null;
}
@@ -3808,14 +3849,13 @@ export class ProjectDb {
// Corrupt blob — surface the summary without the list.
}
}
const status = row.status;
return {
url: row.url,
language: row.language,
status:
status === 'ok' || status === 'skipped' || status === 'error'
? status
: null,
detectedLanguage: row.detected_language,
declaredLanguage: row.declared_language,
status: parseSpellingStatus(row.status),
engine: parseSpellingEngine(row.engine),
error: row.error,
fetchedAt: row.fetched_at,
matches,
@@ -4780,10 +4820,14 @@ export class ProjectDb {
)`,
),
contentThin: countWhere(`${html} AND word_count IS NOT NULL AND word_count < 300`),
// Only a completed check yields a finding count — an unsupported or
// language-mismatched page stores zero because nothing was collected.
// Stating `status = 'ok'` keeps that explicit rather than leaning on
// the count happening to be zero.
spellingGrammar: countWhere(
`${html} AND EXISTS (
SELECT 1 FROM spelling_results s
WHERE s.url = urls.url AND s.match_count > 0
WHERE s.url = urls.url AND s.status = 'ok' AND s.match_count > 0
)`,
),
responseSlow: countWhere(`${html} AND response_time_ms > 1000`),
@@ -8369,7 +8413,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
case 'issues:spelling-grammar':
return `is_external = 0 AND content_kind = 'html' AND EXISTS (
SELECT 1 FROM spelling_results s
WHERE s.url = urls.url AND s.match_count > 0
WHERE s.url = urls.url AND s.status = 'ok' AND s.match_count > 0
)`;
case 'issues:response-slow':
return "is_external = 0 AND content_kind = 'html' AND response_time_ms > 1000";
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/shared-types",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"type": "module",
"main": "./dist/index.js",
+92 -5
View File
@@ -24,7 +24,13 @@ import type {
PagespeedRunStrategy,
} from './pagespeed.js';
import type { CruxRow, CruxRunFormFactor } from './crux.js';
import type { SpellingMatch, SpellingRow } from './spelling.js';
import type {
SpellingEngine,
SpellingLanguageOption,
SpellingMatch,
SpellingRow,
SpellingStatus,
} from './spelling.js';
import type { RecentProject } from './project.js';
import type {
GoogleAuthState,
@@ -64,6 +70,9 @@ export const IPC = {
crawlClear: 'crawl:clear',
crawlAddUrl: 'crawl:add-url',
projectSaveAs: 'project:save-as',
/** Save to the currently bound `.seoproject`; falls back to Save As when
* the project has never been saved. */
projectSave: 'project:save',
projectOpen: 'project:open',
projectCurrentPath: 'project:current-path',
/** Per-project crawl config. `projectConfigGet` reads the active
@@ -92,6 +101,14 @@ export const IPC = {
crawlProgress: 'crawl:progress',
crawlDone: 'crawl:done',
crawlError: 'crawl:error',
/** JS-rendering browser provisioning. `browserInstallState` is pushed
* main→renderer whenever the download starts, advances or ends;
* `browserInstallGet` returns the current state for a window that
* mounted after the last push; `browserInstallStart` lets the user
* retry a failed download from the UI. */
browserInstallState: 'browser:install-state',
browserInstallGet: 'browser:install-get',
browserInstallStart: 'browser:install-start',
urlsQuery: 'urls:query',
urlDetailGet: 'urls:detail',
urlSourceGet: 'urls:source',
@@ -115,6 +132,10 @@ export const IPC = {
/** New unified tabular export with column selection + multi-tab support
* (CSV / XLSX). Backs the in-table "Export" button. */
exportTabular: 'export:tabular',
/** Export one already-rendered grid (a URL Details sub-tab table) to
* CSV or XLSX. Rows travel as literal cells because these tables are
* built in the renderer, not streamed from the database. */
exportGrid: 'export:grid',
/** GDPR-aligned per-domain delete. Wipes every row whose URL host
* matches the given domain (and that domain's links/images/headers/
* url_sources). Used by Settings → "Delete Domain Data". */
@@ -276,11 +297,15 @@ export const IPC = {
* `spellingQuery` lists crawled internal HTML pages joined with any
* stored check summary; `spellingRun` checks a user-selected set of
* URLs (emits `spellingProgress`); `spellingCancel` stops a run early;
* `spellingMatches` loads the full match list for one URL. */
* `spellingMatches` loads the full match list for one URL;
* `spellingLanguages` reports which languages the configured endpoint
* actually offers, so Settings can list them and the checker can refuse
* pages written in one it does not cover. */
spellingQuery: 'spelling:query',
spellingRun: 'spelling:run',
spellingCancel: 'spelling:cancel',
spellingMatches: 'spelling:matches',
spellingLanguages: 'spelling:languages',
/** main → renderer: live progress of an in-flight spelling run. */
spellingProgress: 'spelling:progress',
/** Faz 7 — Google OAuth keystone (shared by Search Console, GA4,
@@ -468,6 +493,7 @@ export type MenuEvent =
| 'delete-domain-data'
| 'clear-all-data'
| 'compare-with-project'
| 'save-project'
| 'save-project-as'
| 'save-project-encrypted'
| 'open-project-encrypted'
@@ -585,6 +611,24 @@ export interface ExportTabularInput {
csvBom?: boolean;
}
/** One detail-panel sub-tab table, ready to write. */
export interface ExportGridInput {
/** Suggested file name without extension (e.g. 'example.com-inlinks'). */
fileName: string;
/** Column headers, in display order. */
headers: string[];
/** Row cells aligned to `headers`. */
rows: (string | number | boolean | null)[][];
/** Worksheet name for xlsx; defaults to the file name. */
sheetName?: string;
}
export interface ExportGridResult {
/** Written file, or null when the user cancelled the save dialog. */
filePath: string | null;
rowsWritten: number;
}
export interface ExportTabularResult {
/** Output file (xlsx + single-section csv) or folder (multi-section csv). */
filePath: string;
@@ -1321,8 +1365,11 @@ export interface SpellingQueryInput {
offset: number;
/** Substring match against the URL. */
search?: string;
/** `all` (default), `checked`, `unchecked`, `errors` (matchCount > 0). */
filter?: 'all' | 'checked' | 'unchecked' | 'errors';
/**
* `all` (default), `checked`, `unchecked`, `errors` (matchCount > 0),
* `unsupported` (language LanguageTool has no rules for).
*/
filter?: 'all' | 'checked' | 'unchecked' | 'errors' | 'unsupported';
}
export interface SpellingQueryResult {
@@ -1341,6 +1388,12 @@ export interface SpellingRunResult {
completed: number;
/** Pages that errored or were skipped for having no prose. */
failed: number;
/**
* Pages left unchecked because LanguageTool ships no rules for their
* language. Counted separately from `failed`: nothing went wrong, the
* language is simply outside the tool's coverage.
*/
unsupported: number;
cancelled: boolean;
}
@@ -1356,7 +1409,13 @@ export interface SpellingProgress {
export interface SpellingMatchesResult {
url: string;
language: string | null;
status: 'ok' | 'skipped' | 'error' | null;
/** Language detected from the page's own prose. */
detectedLanguage: string | null;
/** Language the page declares via `html[lang]`. */
declaredLanguage: string | null;
status: SpellingStatus | null;
/** Which checker produced this result — `local` means spelling only. */
engine: SpellingEngine | null;
error: string | null;
fetchedAt: string | null;
matches: SpellingMatch[];
@@ -1572,6 +1631,20 @@ export interface LogEntry {
windowId?: number;
}
/**
* Provisioning state of the Chromium build JS rendering needs.
*
* `unknown` until the app has looked; `downloading` while the in-app
* installer is fetching it (no terminal involved); `failed` carries the
* reason so the status bar can show it on hover.
*/
export interface BrowserInstallState {
state: 'unknown' | 'ready' | 'downloading' | 'failed';
/** 0-100 while downloading, null when the step reports no percentage. */
percent: number | null;
error?: string;
}
export interface FreeCrawlApi {
crawlStart(config: CrawlConfig): Promise<void>;
crawlStop(): Promise<void>;
@@ -1580,6 +1653,8 @@ export interface FreeCrawlApi {
crawlClear(): Promise<void>;
crawlAddUrl(url: string): Promise<{ accepted: boolean }>;
projectSaveAs(): Promise<{ filePath: string; bytesWritten: number } | null>;
/** Save to the bound project file; prompts for a location the first time. */
projectSave(): Promise<{ filePath: string; bytesWritten: number } | null>;
projectOpen(filePath?: string): Promise<{ filePath: string } | null>;
projectCurrentPath(): Promise<string | null>;
/** Active project's saved crawl config, or null on the default scratch
@@ -1590,6 +1665,12 @@ export interface FreeCrawlApi {
/** Fires when the active project changes; payload is the project's saved
* config, or null when it has none yet / is the scratch DB. */
onProjectConfigChanged(cb: (config: CrawlConfig | null) => void): () => void;
/** Current JS-rendering browser provisioning state. */
browserInstallGet(): Promise<BrowserInstallState>;
/** Start (or retry) the in-app Chromium download. */
browserInstallStart(): Promise<BrowserInstallState>;
/** Fires on every provisioning state change (download progress included). */
onBrowserInstallState(cb: (state: BrowserInstallState) => void): () => void;
/** Full recent-projects list including archived entries. */
recentProjectsList(): Promise<RecentProject[]>;
/** Archive / unarchive a recent project (hides it from Open Recent). */
@@ -1621,6 +1702,9 @@ export interface FreeCrawlApi {
exportBrokenLinks(input: ExportBrokenLinksInput): Promise<ExportBrokenLinksResult>;
exportImages(input: ExportImagesInput): Promise<ExportImagesResult>;
exportTabular(input: ExportTabularInput): Promise<ExportTabularResult>;
/** Save a URL Details sub-tab table as CSV or XLSX (format follows the
* extension picked in the save dialog). */
exportGrid(input: ExportGridInput): Promise<ExportGridResult>;
dataDeleteByDomain(
input: DataDeleteByDomainInput,
): Promise<DataDeleteByDomainResult>;
@@ -1753,6 +1837,9 @@ export interface FreeCrawlApi {
onSpellingProgress(cb: (p: SpellingProgress) => void): () => void;
/** Full stored match list for one URL. */
spellingMatches(url: string): Promise<SpellingMatchesResult | null>;
/** Languages the configured LanguageTool endpoint offers, for the
* Settings → Spelling language picker. Sorted by display name. */
spellingLanguages(): Promise<SpellingLanguageOption[]>;
/** Faz 7 — start the interactive Google OAuth consent flow for one
* integration. Resolves once the user finishes (or cancels) in the
* browser. */
+70 -12
View File
@@ -5,16 +5,52 @@
* endpoint — either the free public API, a self-hosted server, or a
* Premium account — and stores the returned matches per URL.
*
* Language is taken from the page's `html[lang]` when present, falling
* back to LanguageTool's own auto-detection. Results are fetched
* on-demand against a user-selected subset (checking a whole crawl would
* blow through the public API's rate limit), so this mirrors the
* PageSpeed / CrUX batch model.
* The check language is resolved locally before anything is sent: the
* page's prose is fingerprinted with a trigram detector and reconciled
* against its declared `html[lang]`, then validated against the languages
* the configured endpoint actually supports. LanguageTool's own `auto`
* mode is deliberately never used — it can only answer with a language it
* supports, so it labels Turkish prose "English (99% confident)" and
* returns a finding on every word. Results are fetched on-demand against a
* user-selected subset (checking a whole crawl would blow through the
* public API's rate limit), so this mirrors the PageSpeed / CrUX batch
* model.
*/
/** LanguageTool rule strictness. `picky` enables style/typography rules. */
export type SpellingLevel = 'default' | 'picky';
/**
* Outcome of one page's check.
*
* `ok` — checked, `matches` is authoritative
* `skipped` — too little prose to check
* `unsupported` — the page's language isn't offered by this endpoint
* (Turkish, Hungarian, Czech, … are simply absent from
* LanguageTool); no request was made
* `mismatch` — checked, but the finding density proves the resolved
* language was wrong, so the findings are discarded
* rather than shown as thousands of false positives
* `error` — transport or API failure
*/
export type SpellingStatus =
| 'ok'
| 'skipped'
| 'unsupported'
| 'mismatch'
| 'error';
/**
* Which checker produced a result.
*
* `languagetool` — the configured endpoint: spelling, grammar and style.
* `local` — a bundled Hunspell dictionary, used for languages
* LanguageTool has no rules for. **Spelling only** — a
* word list cannot judge grammar, and a clean result from
* it must not be read as "the grammar is fine".
*/
export type SpellingEngine = 'languagetool' | 'local';
/** One spelling / grammar / style finding inside a page's prose. */
export interface SpellingMatch {
/** Full human-readable explanation from LanguageTool. */
@@ -43,16 +79,24 @@ export interface SpellingMatch {
/** Stored check result for one URL. */
export interface SpellingResult {
/** Language code LanguageTool actually used (may be auto-detected). */
/** LanguageTool code the check ran under, e.g. `en-GB`. Null when no
* request was made (`skipped` / `unsupported`). */
language: string | null;
/** Primary code the trigram detector read out of the prose, e.g. `tr`.
* Null when the page carried too little text to fingerprint. */
detectedLanguage: string | null;
/** Primary code the page declared via `html[lang]`, e.g. `en`. Null when
* the page declares nothing usable. Differs from `detectedLanguage`
* when the declaration is boilerplate-wrong — a real SEO finding. */
declaredLanguage: string | null;
/** Number of matches after the ignore-dictionary filter. */
matchCount: number;
matches: SpellingMatch[];
/**
* `ok` — checked; `skipped` — page had too little prose to check;
* `error` — the request failed.
*/
status: 'ok' | 'skipped' | 'error';
status: SpellingStatus;
/** Which checker produced this result. Null when none ran. */
engine: SpellingEngine | null;
/** Failure reason for `error`, or the explanation for `unsupported` /
* `mismatch` — always user-facing prose naming the languages involved. */
error: string | null;
fetchedAt: string;
}
@@ -68,8 +112,22 @@ export interface SpellingRow {
wordCount: number | null;
/** Language LanguageTool used for the check. */
language: string | null;
/** Language detected from the page's own prose. */
detectedLanguage: string | null;
matchCount: number | null;
status: 'ok' | 'skipped' | 'error' | null;
status: SpellingStatus | null;
engine: SpellingEngine | null;
error: string | null;
fetchedAt: string | null;
}
/**
* One language a LanguageTool endpoint offers, as returned by
* `/v2/languages`. `code` is the primary subtag (`en`), `longCode` the
* concrete variant to send (`en-GB`).
*/
export interface SpellingLanguageOption {
name: string;
code: string;
longCode: string;
}
+23 -24
View File
@@ -39,33 +39,32 @@ async function main() {
}
const fullBrowserOk = chromiumPath && existsSync(chromiumPath);
// Headless-shell binary path — derive by replacing the chromium-NNNN
// directory with chromium_headless_shell-NNNN. Playwright stores them
// as siblings in the cache (same revision number, different folder
// and exe name). Best-effort; if the regex fails to normalise we
// assume the binary is missing and let `playwright install` redo
// both — the command is idempotent.
// Headless-shell binary path — derive by swapping the chromium-NNNN
// directory for chromium_headless_shell-NNNN and the per-platform
// folder for its headless-shell counterpart. Playwright stores the two
// as siblings in the cache (same revision, different folder and exe
// name) but only exposes the full browser through executablePath().
// Since the move to Chrome-for-Testing builds these folder names are
// arch-suffixed on macOS and Linux-x64 — matching that exactly is what
// keeps this check from reporting "missing" on every single run.
const headlessShellPath = (() => {
if (!chromiumPath) return '';
// Windows: ...\chromium-1223\chrome-win64\chrome.exe
// → ...\chromium_headless_shell-1223\chrome-headless-shell-win64\chrome-headless-shell.exe
if (process.platform === 'win32') {
return chromiumPath
.replace(/chromium-(\d+)/, 'chromium_headless_shell-$1')
.replace(/chrome-win64\\chrome\.exe$/, 'chrome-headless-shell-win64\\chrome-headless-shell.exe');
const base = chromiumPath.replace(/chromium-(\d+)/, 'chromium_headless_shell-$1');
const rules = [
// Windows: ...\chrome-win64\chrome.exe
[/chrome-win64[\\/]chrome\.exe$/i, 'chrome-headless-shell-win64\\chrome-headless-shell.exe'],
// macOS: .../chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/…
[/chrome-mac-(arm64|x64)[\\/].*$/i, 'chrome-headless-shell-mac-$1/chrome-headless-shell'],
// Linux x64: .../chrome-linux64/chrome
[/chrome-linux64[\\/]chrome$/, 'chrome-headless-shell-linux64/chrome-headless-shell'],
// Linux arm64: Playwright's own build, .../chrome-linux/chrome
[/chrome-linux[\\/]chrome$/, 'chrome-linux/headless_shell'],
];
for (const [pattern, replacement] of rules) {
const out = base.replace(pattern, replacement);
if (out !== base) return out;
}
// macOS: .../chromium-1223/chrome-mac/Chromium.app/Contents/MacOS/Chromium
// → .../chromium_headless_shell-1223/chrome-mac/headless_shell
if (process.platform === 'darwin') {
return chromiumPath
.replace(/chromium-(\d+)/, 'chromium_headless_shell-$1')
.replace(/chrome-mac\/Chromium\.app\/Contents\/MacOS\/Chromium$/, 'chrome-mac/headless_shell');
}
// Linux: .../chromium-1223/chrome-linux/chrome
// → .../chromium_headless_shell-1223/chrome-linux/headless_shell
return chromiumPath
.replace(/chromium-(\d+)/, 'chromium_headless_shell-$1')
.replace(/chrome-linux\/chrome$/, 'chrome-linux/headless_shell');
return '';
})();
const headlessShellOk = headlessShellPath && existsSync(headlessShellPath);