fix: crawl correctness, performance, and full EN/TR localization

This commit is contained in:
kemalai committed 2026-07-02 20:55:43 +03:00
1 parent a8c1b58547
commit 36ebd9042a
36 files changed
+1348 -428

No files matched your search

+40 -12
View File
@@ -193,22 +193,37 @@ async function main(): Promise<void> {
}
}
// Per-flag overrides win ONLY when the flag is actually present. Applying
// them unconditionally would let a `parseNumeric(undefined, DEFAULT)` /
// `?? DEFAULT` fallback silently overwrite a value the user set in the
// --config file (the help text promises "the command line still win[s]",
// not "defaults clobber the file").
const config = {
...DEFAULT_CRAWL_CONFIG,
...fileConfig,
mode: listFile ? ('list' as const) : ('spider' as const),
urlList: listUrls,
startUrl,
maxDepth: parseNumeric(values.depth, DEFAULT_CRAWL_CONFIG.maxDepth),
maxUrls: parseNumeric(values.max, DEFAULT_CRAWL_CONFIG.maxUrls),
maxConcurrency: parseNumeric(values.concurrency, DEFAULT_CRAWL_CONFIG.maxConcurrency),
maxRps: parseNumeric(values.rps, DEFAULT_CRAWL_CONFIG.maxRps),
userAgent: values['user-agent'] ?? DEFAULT_CRAWL_CONFIG.userAgent,
respectRobotsTxt: !values['no-robots'],
crawlExternal: Boolean(values.external),
customHeaders: parseHeaders(values.header),
includePatterns: values.include ?? [],
excludePatterns: values.exclude ?? [],
...(values.depth !== undefined
? { maxDepth: parseNumeric(values.depth, DEFAULT_CRAWL_CONFIG.maxDepth) }
: {}),
...(values.max !== undefined
? { maxUrls: parseNumeric(values.max, DEFAULT_CRAWL_CONFIG.maxUrls) }
: {}),
...(values.concurrency !== undefined
? { maxConcurrency: parseNumeric(values.concurrency, DEFAULT_CRAWL_CONFIG.maxConcurrency) }
: {}),
...(values.rps !== undefined
? { maxRps: parseNumeric(values.rps, DEFAULT_CRAWL_CONFIG.maxRps) }
: {}),
...(values['user-agent'] !== undefined ? { userAgent: values['user-agent'] } : {}),
// `--no-robots` / `--external` only flip the flag ON in their direction;
// absent, the file value (or default) stands.
...(values['no-robots'] ? { respectRobotsTxt: false } : {}),
...(values.external ? { crawlExternal: true } : {}),
...(values.header !== undefined ? { customHeaders: parseHeaders(values.header) } : {}),
...(values.include !== undefined ? { includePatterns: values.include } : {}),
...(values.exclude !== undefined ? { excludePatterns: values.exclude } : {}),
};
const crawler = new Crawler(config, db);
@@ -221,7 +236,13 @@ async function main(): Promise<void> {
);
});
}
let crawlError: string | null = null;
crawler.on('error', (msg) => {
// Record the first fatal error (invalid/unreachable start URL, seed
// failure, bad pattern) so the exit code and --json `ok` reflect it —
// otherwise an unreachable seed produces an empty DB with no 4xx/5xx
// rows and the CLI misreports success to a CI pipeline.
if (crawlError === null) crawlError = String(msg);
process.stderr.write(`\n[error] ${msg}\n`);
});
@@ -244,7 +265,12 @@ async function main(): Promise<void> {
}
const issues = db.getOverviewCounts().issues;
const hasErrors = Object.keys(summary.byStatus).some((k) => {
// A crawl that fetched nothing (unreachable seed, everything filtered) is a
// failure for CI purposes — a successful spider always fetches ≥1 page.
const hasErrors =
crawlError !== null ||
summary.total === 0 ||
Object.keys(summary.byStatus).some((k) => {
const n = Number.parseInt(k, 10);
return Number.isFinite(n) && n >= 400;
});
@@ -254,6 +280,7 @@ async function main(): Promise<void> {
JSON.stringify(
{
ok: !hasErrors,
error: crawlError,
startUrl,
dbPath,
summary,
@@ -469,7 +496,8 @@ function flattenLogResult(filePath: string, result: LogAnalysisResult): LogInges
lastTs: u.lastTs,
})),
daily: Array.from(result.daily.entries()).map(([key, hits]) => {
const sp = key.indexOf(' ');
// Key is `${day}\t${bucket}` (tab-separated in log-analyzer.ts).
const sp = key.indexOf('\t');
return { day: key.slice(0, sp), bucket: key.slice(sp + 1), hits };
}),
status: Array.from(result.status.entries()).map(([status, v]) => ({
+4 -3
View File
@@ -13,6 +13,7 @@
*/
import type { AiProvider } from '@freecrawl/shared-types';
import { resolveCredentials } from './credentials.js';
import { apiFetch } from './api-fetch.js';
const REQUEST_TIMEOUT_MS = 120_000;
const DEFAULT_MAX_TOKENS = 800;
@@ -95,7 +96,7 @@ async function runOpenAi(model: string, prompt: string): Promise<AiRunOutput> {
model,
);
}
const res = await fetch('https://api.openai.com/v1/chat/completions', {
const res = await apiFetch('https://api.openai.com/v1/chat/completions', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -137,7 +138,7 @@ async function runAnthropic(model: string, prompt: string): Promise<AiRunOutput>
model,
);
}
const res = await fetch('https://api.anthropic.com/v1/messages', {
const res = await apiFetch('https://api.anthropic.com/v1/messages', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -184,7 +185,7 @@ async function runOllama(model: string, prompt: string): Promise<AiRunOutput> {
const endpoint = (creds['endpoint'] ?? 'http://localhost:11434').trim() || 'http://localhost:11434';
let res: Response;
try {
res = await fetch(`${endpoint.replace(/\/+$/, '')}/api/generate`, {
res = await apiFetch(`${endpoint.replace(/\/+$/, '')}/api/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model, prompt, stream: false }),
+52
View File
@@ -0,0 +1,52 @@
/**
* Shared undici dispatcher for main-process API integrations — AI providers,
* Search Console, GA4, Google Sheets / BigQuery, OAuth, SEO providers and the
* update check.
*
* The crawler installs a *process-global* dispatcher via `setGlobalDispatcher`
* with `headersTimeout: 10_000` (see packages/core `http-client.ts`). That is
* correct for crawling — a stuck origin can't freeze the connection pool — but
* fatal for these clients: a plain global `fetch` inherits the global
* dispatcher, so any API whose response takes more than 10 s to the first byte
* (a slow LLM completion, a large GSC/GA4 report, a Sheets/BigQuery write) is
* killed at the transport layer as a generic "fetch failed"
* (UND_ERR_HEADERS_TIMEOUT), regardless of the client's own longer
* `AbortSignal`. The failure only appears *after* the first crawl of a session
* installs the global dispatcher, which makes it maddening to diagnose.
*
* This dedicated dispatcher uses generous header/body timeouts (each client
* still enforces its real cap via `AbortSignal`) and honours HTTPS_PROXY /
* HTTP_PROXY so corporate-proxy users keep working, mirroring the PSI client
* in `pagespeed.ts` that already solved this for its own requests.
*/
import { Agent, ProxyAgent, fetch as undiciFetch, type Dispatcher } from 'undici';
/** Above the longest client-side AbortSignal (120 s in sheets-export). */
const TIMEOUT_MS = 125_000;
function createApiDispatcher(): Dispatcher {
const timeouts = { headersTimeout: TIMEOUT_MS, bodyTimeout: TIMEOUT_MS };
const proxy =
process.env['HTTPS_PROXY'] ??
process.env['https_proxy'] ??
process.env['HTTP_PROXY'] ??
process.env['http_proxy'] ??
null;
if (proxy) {
return new ProxyAgent({ uri: proxy, ...timeouts });
}
return new Agent({ ...timeouts, connect: { autoSelectFamily: true } });
}
const apiDispatcher = createApiDispatcher();
type FetchParams = Parameters<typeof undiciFetch>;
/**
* Drop-in replacement for the global `fetch` in main-process integration
* clients. Pins every request to the dedicated integration dispatcher above so
* it never inherits the crawler's aggressive global `headersTimeout`.
*/
export function apiFetch(input: FetchParams[0], init?: FetchParams[1]) {
return undiciFetch(input, { ...(init ?? {}), dispatcher: apiDispatcher });
}
+5 -4
View File
@@ -20,6 +20,7 @@ import { createSign } from 'node:crypto';
import type { ProjectDb } from '@freecrawl/db';
import type { CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
import { resolveCredentials } from './credentials.js';
import { apiFetch } from './api-fetch.js';
import * as logger from './logger.js';
const TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token';
@@ -117,7 +118,7 @@ async function mintAccessToken(sa: ServiceAccount): Promise<string> {
.sign(sa.private_key, 'base64url');
const jwt = `${signingInput}.${signature}`;
const res = await fetch(TOKEN_ENDPOINT, {
const res = await apiFetch(TOKEN_ENDPOINT, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({
@@ -168,7 +169,7 @@ async function ensureDataset(
token: string,
datasetId: string,
): Promise<void> {
const res = await fetch(
const res = await apiFetch(
`${BIGQUERY_API}/projects/${encodeURIComponent(projectId)}/datasets`,
{
method: 'POST',
@@ -194,7 +195,7 @@ async function createTable(
datasetId: string,
tableId: string,
): Promise<void> {
const res = await fetch(
const res = await apiFetch(
`${BIGQUERY_API}/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(
datasetId,
)}/tables`,
@@ -227,7 +228,7 @@ async function insertAllChunk(
tableId: string,
rows: { json: Record<string, string | number | null> }[],
): Promise<number> {
const res = await fetch(
const res = await apiFetch(
`${BIGQUERY_API}/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(
datasetId,
)}/tables/${encodeURIComponent(tableId)}/insertAll`,
+3 -2
View File
@@ -14,6 +14,7 @@
*/
import type { Ga4Property } from '@freecrawl/shared-types';
import { getAccessToken } from './google-oauth.js';
import { apiFetch } from './api-fetch.js';
import * as logger from './logger.js';
const ADMIN_BASE = 'https://analyticsadmin.googleapis.com/v1beta';
@@ -53,7 +54,7 @@ export async function listProperties(): Promise<Ga4Property[]> {
const url = `${ADMIN_BASE}/accountSummaries?pageSize=200${
pageToken ? `&pageToken=${encodeURIComponent(pageToken)}` : ''
}`;
const res = await fetch(url, {
const res = await apiFetch(url, {
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(30_000),
});
@@ -119,7 +120,7 @@ export async function runReport(
let offset = 0;
for (;;) {
const res = await fetch(endpoint, {
const res = await apiFetch(endpoint, {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
+5 -4
View File
@@ -29,6 +29,7 @@ import { createHash, randomBytes } from 'node:crypto';
import { join } from 'node:path';
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import type { GoogleAuthState } from '@freecrawl/shared-types';
import { apiFetch } from './api-fetch.js';
import { resolveCredentials } from './credentials.js';
import * as logger from './logger.js';
@@ -286,7 +287,7 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
});
// ── Exchange the code for tokens ──────────────────────────────────
const tokenRes = await fetch(TOKEN_ENDPOINT, {
const tokenRes = await apiFetch(TOKEN_ENDPOINT, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({
@@ -322,7 +323,7 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
let email: string | null = null;
if (typeof accessToken === 'string' && accessToken) {
try {
const uiRes = await fetch(USERINFO_ENDPOINT, {
const uiRes = await apiFetch(USERINFO_ENDPOINT, {
headers: { Authorization: `Bearer ${accessToken}` },
signal: AbortSignal.timeout(15_000),
});
@@ -383,7 +384,7 @@ export async function getAccessToken(integrationId: string): Promise<string> {
throw new Error(`"${integrationId}" is not connected — sign in with Google first.`);
}
const { clientId, clientSecret } = resolveClient(integrationId);
const res = await fetch(TOKEN_ENDPOINT, {
const res = await apiFetch(TOKEN_ENDPOINT, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({
@@ -416,7 +417,7 @@ export async function revokeAuth(integrationId: string): Promise<GoogleAuthState
const record = readRecord(integrationId);
if (record) {
try {
await fetch(`${REVOKE_ENDPOINT}?token=${encodeURIComponent(record.refreshToken)}`, {
await apiFetch(`${REVOKE_ENDPOINT}?token=${encodeURIComponent(record.refreshToken)}`, {
method: 'POST',
signal: AbortSignal.timeout(15_000),
});
+4 -3
View File
@@ -13,6 +13,7 @@
*/
import type { GscSite } from '@freecrawl/shared-types';
import { getAccessToken } from './google-oauth.js';
import { apiFetch } from './api-fetch.js';
import * as logger from './logger.js';
const API_BASE = 'https://www.googleapis.com/webmasters/v3';
@@ -37,7 +38,7 @@ function apiError(json: unknown, status: number): string {
/** List the connected account's Search Console properties. */
export async function listSites(): Promise<GscSite[]> {
const token = await getAccessToken('gsc');
const res = await fetch(`${API_BASE}/sites`, {
const res = await apiFetch(`${API_BASE}/sites`, {
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(30_000),
});
@@ -82,7 +83,7 @@ export async function querySearchAnalytics(
let startRow = 0;
for (;;) {
const res = await fetch(endpoint, {
const res = await apiFetch(endpoint, {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
@@ -151,7 +152,7 @@ export async function inspectUrl(
url: string,
): Promise<GscInspectionRaw> {
const token = await getAccessToken('gsc');
const res = await fetch(
const res = await apiFetch(
'https://searchconsole.googleapis.com/v1/urlInspection/index:inspect',
{
method: 'POST',
+110 -88
View File
@@ -148,6 +148,7 @@ import {
type LogImportSummary,
type LogExportInput,
type LogExportResult,
type LogEntry,
} from '@freecrawl/shared-types';
import {
BrowserPool,
@@ -180,9 +181,11 @@ import {
type LogExportTables,
} from '@freecrawl/core';
import { fetch as undiciFetch } from 'undici';
import { apiFetch } from './api-fetch.js';
import { spawn } from 'node:child_process';
import { ProjectDb } from '@freecrawl/db';
import { buildAppMenu } from './menu.js';
import { isMenuLang, type MenuLang } from './menu-i18n.js';
import { getMenuLabels, isMenuLang, type MenuLang } from './menu-i18n.js';
import {
getState as getIntegrationsState,
setCredentials as setIntegrationCredentials,
@@ -932,6 +935,12 @@ function getMenuLang(): MenuLang {
return isMenuLang(raw) ? raw : 'en';
}
/** Localized menu/dialog labels for the active UI language. Lets native
* dialogs pull `L().dlgX` inline without threading a param through handlers. */
function L(): ReturnType<typeof getMenuLabels> {
return getMenuLabels(getMenuLang());
}
function rebuildMenu(): void {
Menu.setApplicationMenu(
buildAppMenu({
@@ -943,7 +952,7 @@ function rebuildMenu(): void {
openProjectAtPath(path);
} catch (err) {
dialog.showErrorBox(
'Open Project Failed',
L().dlgOpenProjectFailedTitle,
`Could not open ${path}.\n\n${(err as Error).message}`,
);
// Drop the bad entry so it doesn't keep failing.
@@ -974,9 +983,9 @@ function openLogsFolder(): void {
if (mainWindow) {
void dialog.showMessageBox(mainWindow, {
type: 'warning',
title: 'Logs Folder Unavailable',
message: 'Disk logging has not been initialised. Logs are kept in memory only for this session.',
buttons: ['OK'],
title: L().dlgLogsFolderUnavailableTitle,
message: L().dlgLogsFolderUnavailableMsg,
buttons: [L().btnOk],
noLink: true,
});
}
@@ -1009,12 +1018,12 @@ function resetDiagnosticDialogs(): void {
if (mainWindow) {
void dialog.showMessageBox(mainWindow, {
type: 'info',
title: 'Diagnostic Warnings Reset',
title: L().dlgDiagResetTitle,
message:
removed === 0
? 'No suppressed diagnostic warnings to reset.'
: `Re-enabled ${removed} previously dismissed warning${removed === 1 ? '' : 's'}. They will pop up again the next time the underlying issue occurs.`,
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
}
@@ -1223,7 +1232,7 @@ async function downloadAndRevealInstaller(asset: GitHubReleaseAsset): Promise<vo
}
void dialog.showMessageBox(win, {
type: 'info',
title: 'Download Complete',
title: L().dlgDownloadCompleteTitle,
message: `${asset.name} downloaded.`,
detail:
`Saved to:\n${savePath}\n\n` +
@@ -1233,16 +1242,16 @@ async function downloadAndRevealInstaller(asset: GitHubReleaseAsset): Promise<vo
: process.platform === 'darwin'
? 'macOS Gatekeeper may block the app on first open because it is not notarised. Right-click the .dmg → Open to bypass.'
: ''),
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
} else {
void dialog.showMessageBox(win, {
type: 'error',
title: 'Download Failed',
title: L().dlgDownloadFailedTitle,
message: `Could not download ${asset.name}`,
detail: `Download state: ${state}\n\nYou can retry from the GitHub Releases page.`,
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
}
@@ -1256,10 +1265,10 @@ async function downloadAndRevealInstaller(asset: GitHubReleaseAsset): Promise<vo
session.defaultSession.removeListener('will-download', handler);
void dialog.showMessageBox(win, {
type: 'error',
title: 'Download Failed',
message: 'Could not start the download.',
title: L().dlgDownloadFailedTitle,
message: L().dlgDownloadStartFailedMsg,
detail: err instanceof Error ? err.message : String(err),
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
resolve();
@@ -1326,7 +1335,7 @@ async function runUpdateCheck(silent: boolean): Promise<void> {
const ac = new AbortController();
const timeout = setTimeout(() => ac.abort(), 10_000);
try {
const res = await fetch(apiUrl, {
const res = await apiFetch(apiUrl, {
headers: {
Accept: 'application/vnd.github+json',
'User-Agent': `FreeCrawl-SEO-Tool/${installed}`,
@@ -1357,12 +1366,12 @@ async function runUpdateCheck(silent: boolean): Promise<void> {
}
void dialog.showMessageBox(win, {
type: 'warning',
title: 'Update Check Failed',
title: L().dlgUpdateCheckFailedTitle,
message: "Couldn't reach the GitHub Releases API.",
detail:
(fetchError ?? 'No release tag in response.') +
'\n\nYou can browse releases manually at:\nhttps://github.com/kemalai/FreeCrawl-SEO-Tool/releases',
buttons: ['Open Releases Page', 'Close'],
buttons: [L().btnOpenReleasesPage, L().btnClose],
defaultId: 0,
cancelId: 1,
noLink: true,
@@ -1382,12 +1391,12 @@ async function runUpdateCheck(silent: boolean): Promise<void> {
if (silent) return;
void dialog.showMessageBox(win, {
type: 'info',
title: 'Up to Date',
title: L().dlgUpToDateTitle,
message: `You're on the latest version (v${installed}).`,
detail: release.published_at
? `Latest GitHub release: ${latest}\nPublished: ${new Date(release.published_at).toLocaleString()}`
: `Latest GitHub release: ${latest}`,
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
return;
@@ -1428,7 +1437,7 @@ async function runUpdateCheck(silent: boolean): Promise<void> {
const result = await dialog.showMessageBox(win, {
type: 'info',
title: 'Update Available',
title: L().dlgUpdateAvailableTitle,
message: `${latest} is available.`,
detail,
buttons,
@@ -1458,7 +1467,7 @@ async function runUpdateCheck(silent: boolean): Promise<void> {
async function promptOpenProject(): Promise<void> {
if (!mainWindow) return;
const res = await dialog.showOpenDialog(mainWindow, {
title: 'Open Project',
title: L().dlgOpenProjectTitle,
properties: ['openFile'],
filters: [
{ name: 'FreeCrawl Project', extensions: ['seoproject', 'sqlite', 'db'] },
@@ -1470,7 +1479,7 @@ async function promptOpenProject(): Promise<void> {
openProjectAtPath(res.filePaths[0]!);
} catch (err) {
dialog.showErrorBox(
'Open Project Failed',
L().dlgOpenProjectFailedTitle,
`Could not open the selected file.\n\n${(err as Error).message}`,
);
}
@@ -1558,10 +1567,21 @@ function createWindow(): void {
});
mainWindow.webContents.setWindowOpenHandler(({ url }) => {
void shell.openExternal(url);
// Only hand real web / mail links to the OS. Without a scheme allowlist a
// crawl-derived `file://` / `smb://` / custom-protocol URL would be opened
// with the user's privileges.
if (isSafeExternalUrl(url)) void shell.openExternal(url);
return { action: 'deny' };
});
// Null the reference when the window closes so the ~15 `mainWindow?.…send()`
// sites short-circuit instead of throwing "Object has been destroyed" — a
// closed-but-non-null BrowserWindow still throws on `.webContents`. Matches
// the Logs / Visualization / Log Analyzer windows, which already do this.
mainWindow.on('closed', () => {
mainWindow = null;
});
if (process.env['ELECTRON_RENDERER_URL']) {
void mainWindow.loadURL(process.env['ELECTRON_RENDERER_URL']);
} else {
@@ -1569,6 +1589,20 @@ function createWindow(): void {
}
}
/**
* Whether a renderer-supplied URL is safe to hand to `shell.openExternal`.
* Restricts to the schemes a user actually expects a browser / mail client to
* handle, blocking `file:`, `smb:`, and OS custom-protocol handlers.
*/
function isSafeExternalUrl(raw: string): boolean {
try {
const scheme = new URL(raw).protocol.toLowerCase();
return scheme === 'http:' || scheme === 'https:' || scheme === 'mailto:';
} catch {
return false;
}
}
/**
* Open (or focus) the Logs popup window. Loads the same renderer bundle
* with `?logs=1` so the renderer entry branches to the LogsView component.
@@ -1826,7 +1860,8 @@ function ingestLogResult(filePath: string, result: LogAnalysisResult): LogImport
lastTs: u.lastTs,
})),
daily: Array.from(result.daily.entries()).map(([key, hits]) => {
const sp = key.indexOf(' ');
// Key is `${day}\t${bucket}` (tab-separated in log-analyzer.ts).
const sp = key.indexOf('\t');
return { day: key.slice(0, sp), bucket: key.slice(sp + 1), hits };
}),
status: Array.from(result.status.entries()).map(([status, v]) => ({
@@ -1955,7 +1990,7 @@ function registerIpc(): void {
return { ok: false, error: 'No window available for the file picker.', imported: null, overview: null };
}
const res = await dialog.showOpenDialog(parent, {
title: 'Open Access Log',
title: L().dlgOpenAccessLogTitle,
properties: ['openFile'],
filters: [
{ name: 'Log Files', extensions: ['log', 'txt', 'gz', 'out', 'access'] },
@@ -2027,7 +2062,7 @@ function registerIpc(): void {
BrowserWindow.fromWebContents(e.sender) ?? logAnalyzerWindow ?? mainWindow;
if (!parent) return { filePath: '', bytesWritten: 0 };
const res = await dialog.showSaveDialog(parent, {
title: 'Export Log Analysis',
title: L().dlgExportLogAnalysisTitle,
defaultPath: `log-analysis.${format}`,
filters: [
format === 'csv'
@@ -2236,7 +2271,7 @@ function registerIpc(): void {
): Promise<ExtractionRulesExportResult> => {
if (!mainWindow) return { filePath: '', bytesWritten: 0 };
const res = await dialog.showSaveDialog(mainWindow, {
title: 'Export Extraction Rules',
title: L().dlgExportExtractionRulesTitle,
defaultPath: 'freecrawl-extraction-rules.json',
filters: [{ name: 'JSON', extensions: ['json'] }],
});
@@ -2268,7 +2303,7 @@ function registerIpc(): void {
return { filePath: '', rules: [], skippedCount: 0 };
}
const res = await dialog.showOpenDialog(mainWindow, {
title: 'Import Extraction Rules',
title: L().dlgImportExtractionRulesTitle,
properties: ['openFile'],
filters: [{ name: 'JSON', extensions: ['json'] }],
});
@@ -2575,7 +2610,7 @@ function registerIpc(): void {
if (!filePath) {
if (!mainWindow) return { filePath: '', bytesWritten: 0 };
const res = await dialog.showSaveDialog(mainWindow, {
title: 'Export Settings',
title: L().dlgExportSettingsTitle,
defaultPath: 'freecrawl-settings.json',
filters: [{ name: 'JSON', extensions: ['json'] }],
});
@@ -2601,7 +2636,7 @@ function registerIpc(): void {
async (): Promise<SettingsImportResult> => {
if (!mainWindow) return { filePath: '', config: null, unknownFields: [] };
const res = await dialog.showOpenDialog(mainWindow, {
title: 'Import Settings',
title: L().dlgImportSettingsTitle,
properties: ['openFile'],
filters: [{ name: 'JSON', extensions: ['json'] }],
});
@@ -2659,7 +2694,7 @@ function registerIpc(): void {
const win = mainWindow;
if (!win) return null;
const res = await dialog.showOpenDialog(win, {
title: input?.title ?? 'Choose Folder',
title: input?.title ?? L().dlgChooseFolderTitle,
defaultPath: input?.defaultPath,
properties: ['openDirectory', 'createDirectory'],
});
@@ -3575,7 +3610,7 @@ function registerIpc(): void {
// costs serialise + V8-deserialise + main-thread work in the
// renderer process. Batching at 100 ms drops IPC volume by ~10–30×
// while still feeling "live" in the log panel.
let logsBatch: import('@freecrawl/shared-types').LogEntry[] = [];
let logsBatch: LogEntry[] = [];
let logsFlushTimer: NodeJS.Timeout | null = null;
const flushLogsBatch = (): void => {
logsFlushTimer = null;
@@ -3989,18 +4024,17 @@ function registerIpc(): void {
if (!mainWindow) return false;
const res = await dialog.showMessageBox(mainWindow, {
type: 'question',
title: 'JavaScript Rendering — Browser Missing',
title: L().dlgPlaywrightTitle,
message:
'Playwright needs to download a Chromium browser before JavaScript rendering can run.',
detail:
'This is a one-time ~250 MB download. The browser is stored in your user cache; FreeCrawl never sends any data to Playwright servers — only the binary is downloaded from cdn.playwright.dev.\n\nDownload now?',
buttons: ['Download now', 'Skip — disable JS render for this run'],
buttons: [L().btnDownloadNow, L().btnSkipJsRender],
defaultId: 0,
cancelId: 1,
});
if (res.response !== 0) return false;
playwrightInstallPromise = new Promise<boolean>((resolve) => {
const { spawn } = require('node:child_process') as typeof import('node:child_process');
const proc = spawn(
'npx',
['playwright', 'install', 'chromium', 'chromium-headless-shell'],
@@ -4030,7 +4064,7 @@ function registerIpc(): void {
`Browser install failed (exit ${code}). stderr: ${stderr.slice(0, 500)}`,
);
dialog.showErrorBox(
'Browser Install Failed',
L().dlgBrowserInstallFailedTitle,
`Playwright install exited with code ${code}.\n\n` +
`Try running "npx playwright install chromium chromium-headless-shell" manually from a terminal in the FreeCrawl source folder.\n\n${stderr.slice(0, 400)}`,
);
@@ -4041,7 +4075,7 @@ function registerIpc(): void {
playwrightInstallPromise = null;
logger.log('error', 'playwright-install', `spawn failed: ${err.message}`);
dialog.showErrorBox(
'Browser Install Failed',
L().dlgBrowserInstallFailedTitle,
`Could not run "npx playwright install": ${err.message}\n\nPlease install Playwright browsers manually from a terminal.`,
);
resolve(false);
@@ -5025,10 +5059,10 @@ function registerIpc(): void {
const bytes = statSync(target).size;
await dialog.showMessageBox(win, {
type: 'info',
title: 'Project Saved',
title: L().dlgProjectSavedTitle,
message: `Snapshot written: ${(bytes / (1024 * 1024)).toFixed(1)} MB.`,
detail: target,
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
// Saved snapshot is a valid project on disk — pin it as the active
@@ -5054,7 +5088,7 @@ function registerIpc(): void {
if (!target) {
if (!mainWindow) return null;
const res = await dialog.showOpenDialog(mainWindow, {
title: 'Open Project',
title: L().dlgOpenProjectTitle,
properties: ['openFile'],
filters: [
{ name: 'FreeCrawl Project', extensions: ['seoproject', 'sqlite', 'db'] },
@@ -5070,7 +5104,7 @@ function registerIpc(): void {
} catch (err) {
if (mainWindow) {
dialog.showErrorBox(
'Open Project Failed',
L().dlgOpenProjectFailedTitle,
`Could not open ${target}.\n\n${(err as Error).message}`,
);
}
@@ -5135,12 +5169,12 @@ function registerIpc(): void {
await dialog.showMessageBox(win, {
type: 'info',
title: 'Encrypted Snapshot Saved',
title: L().dlgEncSnapshotSavedTitle,
message: `Encrypted snapshot written: ${(result.bytesWritten / (1024 * 1024)).toFixed(1)} MB.`,
detail:
`${target}\n\n` +
'Keep the password safe — it cannot be recovered. Without it, the file is unreadable.',
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
return { filePath: target, bytesWritten: result.bytesWritten };
@@ -5188,7 +5222,7 @@ function registerIpc(): void {
return app.getPath('documents');
})();
const save = await dialog.showSaveDialog(win, {
title: 'Save Decrypted Project As…',
title: L().dlgSaveDecryptedProjectTitle,
defaultPath: join(baseDir, 'recovered.seoproject'),
filters: [
{ name: 'FreeCrawl Project', extensions: ['seoproject'] },
@@ -5214,18 +5248,16 @@ function registerIpc(): void {
ipcMain.handle(IPC.confirmClear, async (): Promise<ConfirmClearResult> => {
const win = mainWindow;
if (!win) return { confirmed: false, skipNext: false };
const isTr = getMenuLang() === 'tr';
const L = getMenuLabels(getMenuLang());
const res = await dialog.showMessageBox(win, {
type: 'warning',
buttons: isTr ? ['Temizle', 'İptal'] : ['Clear', 'Cancel'],
buttons: [L.btnClear, L.btnCancel],
defaultId: 1,
cancelId: 1,
title: isTr ? 'Crawl Verilerini Temizle' : 'Clear Crawl Data',
message: isTr ? 'Tüm crawl verileri temizlensin mi?' : 'Clear all crawl data?',
detail: isTr
? 'Bu, bu projedeki tüm keşfedilmiş URL\'leri, linkleri ve crawl meta verilerini siler. Bu işlem geri alınamaz.'
: 'This will remove all discovered URLs, links, and crawl metadata for this project. This action cannot be undone.',
checkboxLabel: isTr ? 'Bana bir daha sorma' : "Don't ask me again",
title: L.clearCrawlData,
message: L.dlgConfirmClearMsg,
detail: L.dlgConfirmClearDetail,
checkboxLabel: L.dlgDontAskAgain,
checkboxChecked: false,
noLink: true,
});
@@ -5320,21 +5352,24 @@ function registerIpc(): void {
ipcMain.handle(IPC.urlContextMenu, (e, input: UrlContextMenuInput) => {
const win = BrowserWindow.fromWebContents(e.sender);
const canRecrawl = activeCrawler !== null && activeCrawler.isRunning;
const L = getMenuLabels(getMenuLang());
const template: MenuItemConstructorOptions[] = [
{
label: 'Copy',
label: L.ctxCopy,
click: () => clipboard.writeText(input.url),
},
{
label: 'Open in Browser',
click: () => void shell.openExternal(input.url),
label: L.ctxOpenInBrowser,
click: () => {
if (isSafeExternalUrl(input.url)) void shell.openExternal(input.url);
},
},
{ type: 'separator' },
{
label: 'Re-Spider',
label: L.ctxRespider,
enabled: canRecrawl,
toolTip: canRecrawl ? undefined : 'Start a crawl first',
toolTip: canRecrawl ? undefined : L.ctxStartCrawlFirst,
click: () => {
const db = getDb();
db.markUrlForRecrawl(input.urlId);
@@ -5345,7 +5380,7 @@ function registerIpc(): void {
},
},
{
label: 'Remove',
label: L.ctxRemove,
click: () => {
getDb().deleteUrl(input.urlId);
fireDataChanged();
@@ -5353,20 +5388,7 @@ function registerIpc(): void {
},
{ type: 'separator' },
{
label: 'Export',
enabled: false,
submenu: [{ label: 'Selected URLs as CSV', enabled: false }],
},
{ label: 'Visualisations', enabled: false },
{ label: 'Check Index', enabled: false },
{ label: 'Backlinks', enabled: false },
{ label: 'Validation', enabled: false },
{ label: 'History', enabled: false },
{ label: 'Speed', enabled: false },
{ type: 'separator' },
{ label: 'Show Other Domains on IP', enabled: false },
{
label: 'Open robots.txt',
label: L.ctxOpenRobotsTxt,
click: () => {
try {
const origin = new URL(input.url).origin;
@@ -5393,27 +5415,27 @@ function registerIpc(): void {
const urls = db.getUrlsByIds(ids);
const canRecrawl = activeCrawler !== null && activeCrawler.isRunning;
const n = ids.length.toLocaleString();
const L = getMenuLabels(getMenuLang());
const template: MenuItemConstructorOptions[] = [
{
label: `Copy ${n} URLs`,
label: L.ctxCopyNUrls.replace('{n}', n),
click: () => clipboard.writeText(urls.join('\n')),
},
{
label: `Open ${n} URLs in Browser`,
label: L.ctxOpenNUrlsInBrowser.replace('{n}', n),
// Guard: opening hundreds of tabs at once is a bad default.
enabled: urls.length <= 20,
toolTip:
urls.length > 20 ? 'Limited to 20 URLs to avoid spawning too many tabs' : undefined,
toolTip: urls.length > 20 ? L.ctxOpenLimitTooltip : undefined,
click: () => {
for (const u of urls) void shell.openExternal(u);
for (const u of urls) if (isSafeExternalUrl(u)) void shell.openExternal(u);
},
},
{ type: 'separator' },
{
label: `Re-Spider ${n} URLs`,
label: L.ctxRespiderNUrls.replace('{n}', n),
enabled: canRecrawl,
toolTip: canRecrawl ? undefined : 'Start a crawl first',
toolTip: canRecrawl ? undefined : L.ctxStartCrawlFirst,
click: () => {
db.markUrlsForRecrawl(ids);
if (activeCrawler) {
@@ -5423,7 +5445,7 @@ function registerIpc(): void {
},
},
{
label: `Remove ${n} URLs`,
label: L.ctxRemoveNUrls.replace('{n}', n),
click: () => {
db.deleteUrls(ids);
fireDataChanged();
@@ -5431,7 +5453,7 @@ function registerIpc(): void {
},
{ type: 'separator' },
{
label: `Export ${n} URLs as CSV…`,
label: L.ctxExportNUrlsAsCsv.replace('{n}', n),
click: async () => {
const w = win ?? mainWindow;
if (!w) return;
@@ -5798,7 +5820,7 @@ function registerIpc(): void {
return { outputDir: '', files: [], errors: [] };
}
const dirRes = await dialog.showOpenDialog(mainWindow, {
title: 'Bulk Export — choose output folder',
title: L().dlgBulkExportFolderTitle,
properties: ['openDirectory', 'createDirectory'],
});
if (dirRes.canceled || dirRes.filePaths.length === 0) {
@@ -5812,14 +5834,14 @@ function registerIpc(): void {
const totalRows = files.reduce((s, f) => s + f.rowsWritten, 0);
await dialog.showMessageBox(mainWindow, {
type: 'info',
title: 'Bulk Export Complete',
title: L().dlgBulkExportCompleteTitle,
message: `${files.length} file(s) written, ${totalRows.toLocaleString()} row(s) total.`,
detail:
outputDir +
(errors.length > 0
? `\n\nErrors:\n${errors.map((e) => `• ${e.label}: ${e.error}`).join('\n')}`
: ''),
buttons: ['OK', 'Open Folder'],
buttons: [L().btnOk, L().btnOpenFolder],
defaultId: 0,
noLink: true,
}).then((res) => {
@@ -5852,10 +5874,10 @@ function registerIpc(): void {
if (mainWindow) {
await dialog.showMessageBox(mainWindow, {
type: 'info',
title: 'HTML Report Saved',
title: L().dlgHtmlReportSavedTitle,
message: `Report written: ${(result.bytesWritten / 1024).toFixed(1)} KB.`,
detail: result.filePath,
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
}
@@ -5901,7 +5923,7 @@ function registerIpc(): void {
: result.files[0] ?? filePath;
await dialog.showMessageBox(mainWindow, {
type: result.truncated ? 'warning' : 'info',
title: 'Sitemap Generated',
title: L().dlgSitemapGeneratedTitle,
message: result.sharded
? `Sharded sitemap written: ${result.urlsWritten.toLocaleString()} URLs across ${
result.files.length - 1
@@ -5910,7 +5932,7 @@ function registerIpc(): void {
result.truncated ? ' (truncated at the 50,000 limit).' : '.'
}`,
detail,
buttons: ['OK'],
buttons: [L().btnOk],
noLink: true,
});
}
+197
View File
@@ -65,6 +65,73 @@ export interface MenuLabels {
checkForUpdates: string;
checkForUpdatesTooltip: string;
about: string;
// ── Right-click context menus (URL table) ──
ctxCopy: string;
ctxOpenInBrowser: string;
ctxRespider: string;
ctxStartCrawlFirst: string;
ctxRemove: string;
ctxOpenRobotsTxt: string;
/** `{n}` placeholder → selection count. */
ctxCopyNUrls: string;
ctxOpenNUrlsInBrowser: string;
ctxOpenLimitTooltip: string;
ctxRespiderNUrls: string;
ctxRemoveNUrls: string;
ctxExportNUrlsAsCsv: string;
// ── Recurring dialog buttons ──
btnOk: string;
btnCancel: string;
btnClose: string;
btnClear: string;
btnOpenFolder: string;
btnLater: string;
btnOpenReleasePage: string;
btnOpenReleasesPage: string;
btnDownloadInstaller: string;
btnDownloadNow: string;
btnSkipJsRender: string;
// ── Native dialog titles ──
dlgOpenProjectTitle: string;
dlgOpenProjectFailedTitle: string;
dlgLogsFolderUnavailableTitle: string;
dlgLogsFolderUnavailableMsg: string;
dlgDiagResetTitle: string;
dlgDiagResetNoneMsg: string;
dlgDownloadCompleteTitle: string;
dlgDownloadFailedTitle: string;
dlgDownloadStartFailedMsg: string;
dlgUpdateCheckFailedTitle: string;
dlgUpdateCheckFailedMsg: string;
dlgUpToDateTitle: string;
dlgUpdateAvailableTitle: string;
dlgOpenAccessLogTitle: string;
dlgExportLogAnalysisTitle: string;
dlgExportExtractionRulesTitle: string;
dlgImportExtractionRulesTitle: string;
dlgExportSettingsTitle: string;
dlgImportSettingsTitle: string;
dlgImportFailedTitle: string;
dlgImportFailedNoSettingsMsg: string;
dlgChooseFolderTitle: string;
dlgPlaywrightTitle: string;
dlgPlaywrightMsg: string;
dlgBrowserInstallFailedTitle: string;
dlgProjectSavedTitle: string;
dlgEncSnapshotSavedTitle: string;
dlgSaveDecryptedProjectTitle: string;
dlgBulkExportFolderTitle: string;
dlgBulkExportCompleteTitle: string;
dlgHtmlReportSavedTitle: string;
dlgSitemapGeneratedTitle: string;
// ── Clear-crawl confirmation (migrated off inline isTr) ──
dlgConfirmClearMsg: string;
dlgConfirmClearDetail: string;
dlgDontAskAgain: string;
}
const MENU_EN: MenuLabels = {
@@ -126,6 +193,71 @@ const MENU_EN: MenuLabels = {
checkForUpdatesTooltip:
'Fetch the latest GitHub release and compare it with your installed version. No background polling — runs only when you click.',
about: 'About FreeCrawl SEO',
ctxCopy: 'Copy',
ctxOpenInBrowser: 'Open in Browser',
ctxRespider: 'Re-Spider',
ctxStartCrawlFirst: 'Start a crawl first',
ctxRemove: 'Remove',
ctxOpenRobotsTxt: 'Open robots.txt',
ctxCopyNUrls: 'Copy {n} URLs',
ctxOpenNUrlsInBrowser: 'Open {n} URLs in Browser',
ctxOpenLimitTooltip: 'Limited to 20 URLs to avoid spawning too many tabs',
ctxRespiderNUrls: 'Re-Spider {n} URLs',
ctxRemoveNUrls: 'Remove {n} URLs',
ctxExportNUrlsAsCsv: 'Export {n} URLs as CSV…',
btnOk: 'OK',
btnCancel: 'Cancel',
btnClose: 'Close',
btnClear: 'Clear',
btnOpenFolder: 'Open Folder',
btnLater: 'Later',
btnOpenReleasePage: 'Open Release Page',
btnOpenReleasesPage: 'Open Releases Page',
btnDownloadInstaller: 'Download Installer',
btnDownloadNow: 'Download now',
btnSkipJsRender: 'Skip — disable JS render for this run',
dlgOpenProjectTitle: 'Open Project',
dlgOpenProjectFailedTitle: 'Open Project Failed',
dlgLogsFolderUnavailableTitle: 'Logs Folder Unavailable',
dlgLogsFolderUnavailableMsg:
'Disk logging has not been initialised. Logs are kept in memory only for this session.',
dlgDiagResetTitle: 'Diagnostic Warnings Reset',
dlgDiagResetNoneMsg: 'No suppressed diagnostic warnings to reset.',
dlgDownloadCompleteTitle: 'Download Complete',
dlgDownloadFailedTitle: 'Download Failed',
dlgDownloadStartFailedMsg: 'Could not start the download.',
dlgUpdateCheckFailedTitle: 'Update Check Failed',
dlgUpdateCheckFailedMsg: "Couldn't reach the GitHub Releases API.",
dlgUpToDateTitle: 'Up to Date',
dlgUpdateAvailableTitle: 'Update Available',
dlgOpenAccessLogTitle: 'Open Access Log',
dlgExportLogAnalysisTitle: 'Export Log Analysis',
dlgExportExtractionRulesTitle: 'Export Extraction Rules',
dlgImportExtractionRulesTitle: 'Import Extraction Rules',
dlgExportSettingsTitle: 'Export Settings',
dlgImportSettingsTitle: 'Import Settings',
dlgImportFailedTitle: 'Import Failed',
dlgImportFailedNoSettingsMsg: 'Imported file does not contain a settings object.',
dlgChooseFolderTitle: 'Choose Folder',
dlgPlaywrightTitle: 'JavaScript Rendering — Browser Missing',
dlgPlaywrightMsg:
'Playwright needs to download a Chromium browser before JavaScript rendering can run.',
dlgBrowserInstallFailedTitle: 'Browser Install Failed',
dlgProjectSavedTitle: 'Project Saved',
dlgEncSnapshotSavedTitle: 'Encrypted Snapshot Saved',
dlgSaveDecryptedProjectTitle: 'Save Decrypted Project As…',
dlgBulkExportFolderTitle: 'Bulk Export — choose output folder',
dlgBulkExportCompleteTitle: 'Bulk Export Complete',
dlgHtmlReportSavedTitle: 'HTML Report Saved',
dlgSitemapGeneratedTitle: 'Sitemap Generated',
dlgConfirmClearMsg: 'Clear all crawl data?',
dlgConfirmClearDetail:
'This permanently deletes every crawled URL, link, image, header and source snapshot in the active project. This cannot be undone.',
dlgDontAskAgain: "Don't ask me again",
};
const MENU_TR: MenuLabels = {
@@ -187,6 +319,71 @@ const MENU_TR: MenuLabels = {
checkForUpdatesTooltip:
'En son GitHub release\'ini çek ve kurulu sürümünüzle karşılaştır. Arka planda yoklama yok — yalnızca tıkladığınızda çalışır.',
about: 'FreeCrawl SEO Hakkında',
ctxCopy: 'Kopyala',
ctxOpenInBrowser: 'Tarayıcıda Aç',
ctxRespider: 'Yeniden Tara',
ctxStartCrawlFirst: 'Önce bir crawl başlatın',
ctxRemove: 'Kaldır',
ctxOpenRobotsTxt: 'robots.txt\'yi Aç',
ctxCopyNUrls: '{n} URL\'yi Kopyala',
ctxOpenNUrlsInBrowser: '{n} URL\'yi Tarayıcıda Aç',
ctxOpenLimitTooltip: 'Çok fazla sekme açılmasını önlemek için 20 URL ile sınırlı',
ctxRespiderNUrls: '{n} URL\'yi Yeniden Tara',
ctxRemoveNUrls: '{n} URL\'yi Kaldır',
ctxExportNUrlsAsCsv: '{n} URL\'yi CSV Olarak Dışa Aktar…',
btnOk: 'Tamam',
btnCancel: 'İptal',
btnClose: 'Kapat',
btnClear: 'Temizle',
btnOpenFolder: 'Klasörü Aç',
btnLater: 'Sonra',
btnOpenReleasePage: 'Release Sayfasını Aç',
btnOpenReleasesPage: 'Release Sayfasını Aç',
btnDownloadInstaller: 'Kurulumu İndir',
btnDownloadNow: 'Şimdi indir',
btnSkipJsRender: 'Atla — bu çalıştırmada JS render\'ı devre dışı bırak',
dlgOpenProjectTitle: 'Proje Aç',
dlgOpenProjectFailedTitle: 'Proje Açılamadı',
dlgLogsFolderUnavailableTitle: 'Log Klasörü Kullanılamıyor',
dlgLogsFolderUnavailableMsg:
'Diske log yazma başlatılmadı. Loglar bu oturum için yalnızca bellekte tutuluyor.',
dlgDiagResetTitle: 'Tanı Uyarıları Sıfırlandı',
dlgDiagResetNoneMsg: 'Sıfırlanacak bastırılmış tanı uyarısı yok.',
dlgDownloadCompleteTitle: 'İndirme Tamamlandı',
dlgDownloadFailedTitle: 'İndirme Başarısız',
dlgDownloadStartFailedMsg: 'İndirme başlatılamadı.',
dlgUpdateCheckFailedTitle: 'Güncelleme Kontrolü Başarısız',
dlgUpdateCheckFailedMsg: 'GitHub Releases API\'sine ulaşılamadı.',
dlgUpToDateTitle: 'Güncel',
dlgUpdateAvailableTitle: 'Güncelleme Mevcut',
dlgOpenAccessLogTitle: 'Erişim Logu Aç',
dlgExportLogAnalysisTitle: 'Log Analizini Dışa Aktar',
dlgExportExtractionRulesTitle: 'Çıkarım Kurallarını Dışa Aktar',
dlgImportExtractionRulesTitle: 'Çıkarım Kurallarını İçe Aktar',
dlgExportSettingsTitle: 'Ayarları Dışa Aktar',
dlgImportSettingsTitle: 'Ayarları İçe Aktar',
dlgImportFailedTitle: 'İçe Aktarma Başarısız',
dlgImportFailedNoSettingsMsg: 'İçe aktarılan dosya bir ayarlar nesnesi içermiyor.',
dlgChooseFolderTitle: 'Klasör Seç',
dlgPlaywrightTitle: 'JavaScript Render — Tarayıcı Eksik',
dlgPlaywrightMsg:
'JavaScript render çalışabilmeden önce Playwright\'ın bir Chromium tarayıcısı indirmesi gerekiyor.',
dlgBrowserInstallFailedTitle: 'Tarayıcı Kurulumu Başarısız',
dlgProjectSavedTitle: 'Proje Kaydedildi',
dlgEncSnapshotSavedTitle: 'Şifreli Snapshot Kaydedildi',
dlgSaveDecryptedProjectTitle: 'Çözülmüş Projeyi Farklı Kaydet…',
dlgBulkExportFolderTitle: 'Toplu Dışa Aktarım — çıktı klasörünü seçin',
dlgBulkExportCompleteTitle: 'Toplu Dışa Aktarım Tamamlandı',
dlgHtmlReportSavedTitle: 'HTML Rapor Kaydedildi',
dlgSitemapGeneratedTitle: 'Sitemap Oluşturuldu',
dlgConfirmClearMsg: 'Tüm crawl verileri temizlensin mi?',
dlgConfirmClearDetail:
'Bu işlem, aktif projedeki taranmış her URL, link, görsel, başlık ve kaynak snapshot\'ını kalıcı olarak siler. Geri alınamaz.',
dlgDontAskAgain: 'Bir daha sorma',
};
export function getMenuLabels(lang: MenuLang): MenuLabels {
+5 -4
View File
@@ -25,6 +25,7 @@ import type {
SeoProvider,
} from '@freecrawl/shared-types';
import { resolveCredentials } from './credentials.js';
import { apiFetch } from './api-fetch.js';
const REQUEST_TIMEOUT_MS = 60_000;
@@ -97,7 +98,7 @@ async function ahrefsFetch(
});
let res: Response;
try {
res = await fetch(
res = await apiFetch(
`https://api.ahrefs.com/v3/site-explorer/${call.path}?${params.toString()}`,
{
headers: { Authorization: `Bearer ${apiKey}`, Accept: 'application/json' },
@@ -222,7 +223,7 @@ async function fetchMajestic(url: string): Promise<MajesticMetrics> {
item0: url,
app_api_key: apiKey,
});
const res = await fetch(`https://api.majestic.com/api/json?${params.toString()}`, {
const res = await apiFetch(`https://api.majestic.com/api/json?${params.toString()}`, {
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
const json = (await res.json().catch(() => null)) as Record<string, unknown> | null;
@@ -259,7 +260,7 @@ async function fetchMoz(url: string): Promise<MozMetrics> {
);
}
const basic = Buffer.from(`${accessId}:${secretKey}`).toString('base64');
const res = await fetch('https://lsapi.seomoz.com/v2/url_metrics', {
const res = await apiFetch('https://lsapi.seomoz.com/v2/url_metrics', {
method: 'POST',
headers: {
Authorization: `Basic ${basic}`,
@@ -304,7 +305,7 @@ async function fetchSemrush(url: string): Promise<SemrushMetrics> {
display_format: 'json',
export_columns: 'Or,Ot,Oc,Ad',
});
const res = await fetch(`https://api.semrush.com/?${params.toString()}`, {
const res = await apiFetch(`https://api.semrush.com/?${params.toString()}`, {
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
const text = await res.text();
+3 -2
View File
@@ -15,6 +15,7 @@
import type { ProjectDb } from '@freecrawl/db';
import type { CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
import { getAccessToken } from './google-oauth.js';
import { apiFetch } from './api-fetch.js';
import * as logger from './logger.js';
const MAX_ROWS = 50_000;
@@ -63,7 +64,7 @@ export async function exportCategoryToSheets(
const token = await getAccessToken('sheets');
// 1. Create the spreadsheet.
const createRes = await fetch(
const createRes = await apiFetch(
'https://sheets.googleapis.com/v4/spreadsheets',
{
method: 'POST',
@@ -103,7 +104,7 @@ export async function exportCategoryToSheets(
// 3. Write the values.
if (count > 0) {
const writeRes = await fetch(
const writeRes = await apiFetch(
`https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/A1?valueInputOption=RAW`,
{
method: 'PUT',
+2 -2
View File
@@ -24,7 +24,7 @@ import { AnalyticsTab } from './tabs/AnalyticsTab.js';
import { AiTab } from './tabs/AiTab.js';
import { SeoTab } from './tabs/SeoTab.js';
import { useAppStore } from './store.js';
import type { MenuEvent } from '@freecrawl/shared-types';
import type { MenuEvent, CrawlProgress } from '@freecrawl/shared-types';
import { clearCrawlWithConfirm } from './utils/clearCrawl.js';
export function App() {
@@ -199,7 +199,7 @@ export function App() {
// frame caps the React work at 60 Hz worst-case (and at 5 Hz in
// the common case where the crawler's throttle dominates), so a
// user click never queues behind a stack of pending re-renders.
let pendingProgress: import('@freecrawl/shared-types').CrawlProgress | null = null;
let pendingProgress: CrawlProgress | null = null;
let progressRafScheduled = false;
const off1 = window.freecrawl.onProgress((p) => {
pendingProgress = p;
@@ -383,12 +383,21 @@ function UrlStatsTab({ version, mode }: { version: number; mode: 'all' | 'orphan
}, [search, filter, bot, sortBy, version]);
useEffect(() => {
// Cancellation guard: fast typing / filter changes fire overlapping
// queries, and without this the slower (older) response can resolve last
// and overwrite the newer result set, showing rows for a filter the user
// no longer has selected.
let cancelled = false;
const input: LogUrlStatsInput = { limit: PAGE, offset, search, sortBy, filter, bot: bot || undefined };
const call = mode === 'orphans' ? window.freecrawl.logOrphans(input) : window.freecrawl.logUrlStats(input);
void call.then((r) => {
if (cancelled) return;
setRows(r.rows);
setTotal(r.total);
});
return () => {
cancelled = true;
};
}, [offset, search, filter, bot, sortBy, version, mode]);
// The single filter <select> carries either a membership token
@@ -1,4 +1,5 @@
import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react';
import { useVirtualizer } from '@tanstack/react-virtual';
import clsx from 'clsx';
import { useTranslation } from 'react-i18next';
import { translateLabel } from '../i18n/labels.js';
@@ -562,9 +563,10 @@ function AnalyticsCard({
}
function AnalyticsRow({ label, value }: { label: string; value: string }) {
const { i18n } = useTranslation();
return (
<div className="flex items-baseline gap-3 text-[11px]">
<div className="w-32 shrink-0 text-surface-500">{label}</div>
<div className="w-32 shrink-0 text-surface-500">{translateLabel(label, i18n.language)}</div>
<div className="min-w-0 flex-1 truncate text-surface-100" title={value}>
{value}
</div>
@@ -803,10 +805,26 @@ function ViewSourceView({
setActiveMatch(0);
}, [search, urlId]);
const activeBody = kind === 'rendered' ? src?.renderedBody ?? null : src?.body ?? null;
// Memoize the full-body scans so they only re-run when the body, query, or
// active-match index actually change — NOT on every keystroke or parent
// re-render. On a multi-MB snapshot, an unmemoized countMatches +
// renderHighlighted per render stalls typing in the search box by 100 ms+.
const matches = useMemo(
() => (activeBody && search ? countMatches(activeBody, search) : 0),
[activeBody, search],
);
// Clamp active match into the current range so a shrinking match count
// (after the user typed an extra character) doesn't point past the end.
const safeActive = matches > 0 ? activeMatch % matches : 0;
const highlightedBody = useMemo(
() => (activeBody && search ? renderHighlighted(activeBody, search, safeActive) : null),
[activeBody, search, safeActive],
);
if (loading && !src) {
return <div className="p-4 text-[11px] text-surface-500">{t('viewSource.loadingSource', { defaultValue: 'Loading source…' })}</div>;
}
const activeBody = kind === 'rendered' ? src?.renderedBody ?? null : src?.body ?? null;
if (!src || activeBody === null) {
if (kind === 'rendered') {
return (
@@ -838,11 +856,6 @@ function ViewSourceView({
const body = activeBody;
const activeLength =
kind === 'rendered' ? src.renderedBodyLength ?? body.length : src.bodyLength;
const matches = search ? countMatches(body, search) : 0;
// Clamp active match into the current range so a shrinking match
// count (after the user typed an extra character) doesn't leave us
// pointing past the end.
const safeActive = matches > 0 ? activeMatch % matches : 0;
function step(delta: number): void {
if (matches === 0) return;
@@ -970,9 +983,7 @@ function ViewSourceView({
wrap ? 'whitespace-pre-wrap break-all' : 'whitespace-pre',
)}
>
{search
? renderHighlighted(body, search, safeActive)
: body}
{search ? highlightedBody : body}
</pre>
</div>
<ViewSourceScrollEffect
@@ -1223,7 +1234,10 @@ function HttpHeadersView({
)}
onClick={() => setSide('response')}
>
Response ({headers.length})
{t('detail.httpResponseTab', {
defaultValue: 'Response ({{count}})',
count: headers.length,
})}
</button>
<button
className={clsx(
@@ -1234,12 +1248,18 @@ function HttpHeadersView({
)}
onClick={() => setSide('request')}
>
Request ({requestHeaders.length})
{t('detail.httpRequestTab', {
defaultValue: 'Request ({{count}})',
count: requestHeaders.length,
})}
</button>
<span className="ml-auto text-surface-500">
{side === 'request'
? 'Reconstructed from active crawl config — actual values may have differed if config changed since crawl.'
: 'Captured at crawl time.'}
? t('detail.headersReconstructedNote', {
defaultValue:
'Reconstructed from active crawl config — actual values may have differed if config changed since crawl.',
})
: t('detail.headersCapturedNote', { defaultValue: 'Captured at crawl time.' })}
</span>
</div>
<div className="flex-1 overflow-auto">
@@ -1247,8 +1267,12 @@ function HttpHeadersView({
{rows.length === 0 ? (
<div className="p-4 text-[11px] text-surface-500">
{side === 'response'
? 'No response headers captured for this URL.'
: 'No request headers — config may be empty.'}
? t('detail.noResponseHeaders', {
defaultValue: 'No response headers captured for this URL.',
})
: t('detail.noRequestHeaders', {
defaultValue: 'No request headers — config may be empty.',
})}
</div>
) : (
<div className="p-3">
@@ -1906,6 +1930,18 @@ function LinksView({
| null
>(null);
const rootRef = useRef<HTMLDivElement | null>(null);
// Virtualize the rows — multi-select aggregates up to 50 URLs' links
// (MULTI_DETAIL_LIMIT), so this table can hit 25k+ rows and a plain
// rows.map would render ~400k DOM nodes and freeze the renderer. Fixed
// row height keeps estimateSize exact; overscan masks the sticky-header
// offset the same way the main URLs table does (CLAUDE.md 4.5).
const scrollRef = useRef<HTMLDivElement | null>(null);
const rowVirtualizer = useVirtualizer({
count: rows.length,
getScrollElement: () => scrollRef.current,
estimateSize: () => LINKS_ROW_HEIGHT,
overscan: 20,
});
// `selected` and `rows` change every render; keep the latest in refs
// so the document-level keydown listener can read them without being
// re-attached on every state change.
@@ -2165,7 +2201,7 @@ function LinksView({
{rows.length === 0 ? (
<div className="py-8 text-center text-xs text-surface-500">{t('links.noLinks', { defaultValue: 'No links.' })}</div>
) : (
<div className="mt-2 flex-1 overflow-auto">
<div ref={scrollRef} className="mt-2 flex-1 overflow-auto">
<div style={{ minWidth: totalWidth, width: '100%' }}>
<div
className="sticky top-0 z-10 flex bg-surface-900 text-[11px]"
@@ -2209,11 +2245,24 @@ function LinksView({
})}
<div className="flex-1 border-b border-surface-800" />
</div>
{rows.map((r, ri) => (
<div
className="relative"
style={{
height: rowVirtualizer.getTotalSize(),
minWidth: totalWidth,
width: '100%',
}}
>
{rowVirtualizer.getVirtualItems().map((vi) => {
const ri = vi.index;
const r = rows[ri];
if (!r) return null;
return (
<div
key={ri}
className="flex border-b border-surface-900 text-[11px]"
className="absolute left-0 top-0 flex border-b border-surface-900 text-[11px]"
style={{
transform: `translateY(${vi.start}px)`,
minWidth: totalWidth,
width: '100%',
height: LINKS_ROW_HEIGHT,
@@ -2262,7 +2311,9 @@ function LinksView({
})}
<div className="flex-1" />
</div>
))}
);
})}
</div>
</div>
</div>
)}
@@ -3100,7 +3151,11 @@ function ImagesView({
)}
{row.imagesCount > rows.length && (
<span className="text-surface-500">
(showing first {rows.length} of {row.imagesCount})
({t('imagesView.showingFirst', {
defaultValue: 'showing first {{shown}} of {{total}}',
shown: rows.length,
total: row.imagesCount,
})})
</span>
)}
</div>
@@ -1,4 +1,5 @@
import { Component, type ReactNode } from 'react';
import i18n from '../i18n';
interface Props {
/** Rendered when the boundary's subtree throws. Defaults to a tiny notice. */
@@ -45,14 +46,16 @@ export class ErrorBoundary extends Component<Props, State> {
if (this.props.fallback !== undefined) return this.props.fallback;
return (
<div className="fixed inset-x-0 top-2 z-[60] mx-auto w-fit max-w-[90vw] rounded border border-red-700/60 bg-red-950/90 px-3 py-2 text-[12px] text-red-200 shadow-lg">
<div className="font-semibold">Something broke in this panel.</div>
<div className="font-semibold">
{i18n.t('errorBoundary.panelBroke', { defaultValue: 'Something broke in this panel.' })}
</div>
<div className="mt-0.5 truncate font-mono text-[11px] text-red-300">{err.message}</div>
<button
type="button"
onClick={this.reset}
className="mt-1 rounded border border-red-700/60 px-2 py-0.5 text-[11px] text-red-200 hover:bg-red-900/50"
>
Dismiss
{i18n.t('errorBoundary.dismiss', { defaultValue: 'Dismiss' })}
</button>
</div>
);
@@ -1,4 +1,5 @@
import { useEffect, useMemo, useState } from 'react';
import { useEffect, useMemo, useRef, useState } from 'react';
import { useVirtualizer } from '@tanstack/react-virtual';
import { X } from 'lucide-react';
import { useTranslation } from 'react-i18next';
import type {
@@ -471,6 +472,20 @@ export function ReportsDialog({ open, onClose }: Props) {
return () => window.removeEventListener('keydown', onKey);
}, [open, onClose]);
// Reports fetch up to 1000 rows (pages-per-dir, orphans, …); rendering them
// all as <tr> inside a modal is a visible jank hit and violates the >500-row
// rule (CLAUDE.md 4.5). Virtualize with dynamic measurement because the key
// column wraps (break-all on long URLs / paths), so rows aren't fixed-height.
// Declared BEFORE the `!open` early return so the hook order stays stable.
const scrollRef = useRef<HTMLDivElement | null>(null);
const rowVirtualizer = useVirtualizer({
count: rows.length,
getScrollElement: () => scrollRef.current,
estimateSize: () => 28,
overscan: 15,
measureElement: (el) => el.getBoundingClientRect().height,
});
if (!open) return null;
const total = rows.reduce((sum, r) => sum + r.count, 0);
@@ -537,42 +552,45 @@ export function ReportsDialog({ open, onClose }: Props) {
</span>
</div>
<div className="flex-1 overflow-auto px-4 py-3 text-[11px]">
<div ref={scrollRef} className="flex-1 overflow-auto px-4 py-3 text-[11px]">
{rows.length === 0 && !loading && (
<div className="p-6 text-center text-surface-500">
{t('reports.noData', { defaultValue: 'No data — run a crawl first.' })}
</div>
)}
{rows.length > 0 && (
<table className="w-full">
<thead className="sticky top-0 bg-surface-900">
<tr className="text-surface-400">
<th className="w-2/3 py-1 pr-3 text-left font-medium">{keyLabels[kind]}</th>
<th className="w-24 py-1 pr-3 text-right font-medium">{t('reports.count', { defaultValue: 'Count' })}</th>
<th className="py-1 text-left font-medium">{t('reports.share', { defaultValue: 'Share' })}</th>
</tr>
</thead>
<tbody>
{rows.map((r) => {
<div className="w-full">
<div className="sticky top-0 z-10 flex bg-surface-900 text-surface-400">
<div className="w-2/3 py-1 pr-3 text-left font-medium">{keyLabels[kind]}</div>
<div className="w-24 py-1 pr-3 text-right font-medium">{t('reports.count', { defaultValue: 'Count' })}</div>
<div className="flex-1 py-1 text-left font-medium">{t('reports.share', { defaultValue: 'Share' })}</div>
</div>
<div className="relative" style={{ height: rowVirtualizer.getTotalSize() }}>
{rowVirtualizer.getVirtualItems().map((vi) => {
const r = rows[vi.index];
if (!r) return null;
const widthPct = max > 0 ? Math.round((r.count / max) * 100) : 0;
const sharePct = total > 0 ? ((r.count / total) * 100).toFixed(1) : '0.0';
return (
<tr
<div
key={r.key}
className="border-b border-surface-900 last:border-0 hover:bg-surface-900/50"
data-index={vi.index}
ref={rowVirtualizer.measureElement}
className="absolute left-0 top-0 flex w-full border-b border-surface-900 hover:bg-surface-900/50"
style={{ transform: `translateY(${vi.start}px)` }}
>
<td className="break-all py-1 pr-3 align-top font-mono text-surface-100">
<div className="w-2/3 break-all py-1 pr-3 align-top font-mono text-surface-100">
{r.badge && (
<span className="mr-2 rounded bg-surface-800 px-1.5 py-0.5 text-[9px] uppercase text-surface-400">
{r.badge}
</span>
)}
{r.key}
</td>
<td className="py-1 pr-3 text-right align-top font-mono text-surface-100">
</div>
<div className="w-24 py-1 pr-3 text-right align-top font-mono text-surface-100">
{r.valueLabel ?? r.count.toLocaleString()}
</td>
<td className="py-1 align-top">
</div>
<div className="flex-1 py-1 align-top">
<div className="flex items-center gap-2">
<div className="h-1.5 w-32 rounded bg-surface-800">
<div
@@ -584,12 +602,12 @@ export function ReportsDialog({ open, onClose }: Props) {
{sharePct}%
</span>
</div>
</td>
</tr>
</div>
</div>
);
})}
</tbody>
</table>
</div>
</div>
)}
</div>
@@ -1,5 +1,5 @@
import { useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useTranslation, Trans } from 'react-i18next';
import {
SUPPORTED_LANGUAGES,
changeLanguage,
@@ -1109,7 +1109,7 @@ function PresetsPanel({
exportSettings: () => Promise<void>;
importSettings: () => Promise<void>;
}) {
const { t } = useTranslation();
const { t, i18n } = useTranslation();
const [busy, setBusy] = useState(false);
const [message, setMessage] = useState<string | null>(null);
return (
@@ -1124,8 +1124,8 @@ function PresetsPanel({
className="flex items-start gap-3 rounded border border-surface-800 bg-surface-950/40 p-3"
>
<div className="flex-1 min-w-0">
<div className="text-[12px] font-medium text-surface-100">{p.label}</div>
<div className="mt-0.5 text-[11px] text-surface-400">{p.description}</div>
<div className="text-[12px] font-medium text-surface-100">{translateLabel(p.label, i18n.language)}</div>
<div className="mt-0.5 text-[11px] text-surface-400">{translateLabel(p.description, i18n.language)}</div>
<div className="mt-1.5 flex flex-wrap gap-1 font-mono text-[10px] text-surface-500">
{Object.entries(p.overrides).map(([k, v]) => (
<span key={k} className="rounded border border-surface-800 px-1.5 py-0.5">
@@ -1393,18 +1393,24 @@ function SpeedPanel({ form, update }: PanelProps) {
<div className="mb-4 rounded border border-blue-700/40 bg-blue-900/15 px-3 py-2 text-[11px] text-blue-200">
<div className="mb-0.5 font-medium">{t('settingsPanels.speed.effectiveCeiling', { defaultValue: 'Effective ceiling' })}</div>
<div className="text-blue-300/90">
{effectiveRps !== null ? (
<>
~{effectiveRps.toLocaleString()} URL/s ({Number.isFinite(conc) ? conc : '—'} parallel
workers, ≤ {Number.isFinite(rps) ? rps : '—'} RPS rate-limit
{Number.isFinite(delay) && delay > 0
? `, +${delay} ms post-request delay per worker`
: ''}
)
</>
) : (
'Set Max Concurrency and Max RPS to see the projected throughput.'
)}
{effectiveRps !== null
? t('settingsPanels.speed.ceilingValue', {
defaultValue:
'~{{rps}} URL/s ({{workers}} parallel workers, ≤ {{cap}} RPS rate-limit{{delaySuffix}})',
rps: effectiveRps.toLocaleString(),
workers: Number.isFinite(conc) ? conc : '—',
cap: Number.isFinite(rps) ? rps : '—',
delaySuffix:
Number.isFinite(delay) && delay > 0
? t('settingsPanels.speed.ceilingDelaySuffix', {
defaultValue: ', +{{delay}} ms post-request delay per worker',
delay,
})
: '',
})
: t('settingsPanels.speed.ceilingEmpty', {
defaultValue: 'Set Max Concurrency and Max RPS to see the projected throughput.',
})}
</div>
</div>
@@ -1448,27 +1454,43 @@ function SpeedPanel({ form, update }: PanelProps) {
example="500 default. Bump to 2000 when retrying against a flaky API."
/>
<p className="mt-1 text-[10px] text-surface-500">
Worst-case delay per failed URL ≈ initialDelay × (2 ^ attempts − 1) ={' '}
{Number.isFinite(retries) && Number.isFinite(delay)
? `${(Number.parseInt(form.retryInitialDelayMs, 10) || 500) *
(2 ** Math.max(0, retries) - 1)} ms`
: '—'}
{t('settingsPanels.speed.worstCaseDelay', {
defaultValue:
'Worst-case delay per failed URL ≈ initialDelay × (2 ^ attempts − 1) = {{value}}',
value:
Number.isFinite(retries) && Number.isFinite(delay)
? `${
(Number.parseInt(form.retryInitialDelayMs, 10) || 500) *
(2 ** Math.max(0, retries) - 1)
} ms`
: '—',
})}
</p>
<div className="mt-5 rounded border border-surface-800 bg-surface-950/40 px-3 py-2 text-[10px] text-surface-400">
<div className="mb-1 font-medium text-surface-300">Throughput tips</div>
<div className="mb-1 font-medium text-surface-300">
{t('settingsPanels.speed.tipsHeading', { defaultValue: 'Throughput tips' })}
</div>
<ul className="list-disc space-y-0.5 pl-4">
<li>
For a quick first sweep on a healthy site try the <strong>Fast</strong> preset (Settings
→ Presets) — concurrency 40, RPS 40, no retries.
<Trans
i18nKey="settingsPanels.speed.tip1"
defaults="For a quick first sweep on a healthy site try the <bold>Fast</bold> preset (Settings → Presets) — concurrency 40, RPS 40, no retries."
components={{ bold: <strong /> }}
/>
</li>
<li>
For unreliable origins or slow APIs try <strong>Thorough</strong> — concurrency 10, 3
retries, 30 s timeout.
<Trans
i18nKey="settingsPanels.speed.tip2"
defaults="For unreliable origins or slow APIs try <bold>Thorough</bold> — concurrency 10, 3 retries, 30 s timeout."
components={{ bold: <strong /> }}
/>
</li>
<li>
Concurrency × HTTP keep-alive = the steady-state connection count. Most servers comfortably
handle 20–40; corporate proxies / WAFs often cap at 8–10.
{t('settingsPanels.speed.tip3', {
defaultValue:
'Concurrency × HTTP keep-alive = the steady-state connection count. Most servers comfortably handle 20–40; corporate proxies / WAFs often cap at 8–10.',
})}
</li>
</ul>
</div>
@@ -1477,7 +1499,7 @@ function SpeedPanel({ form, update }: PanelProps) {
}
function RequestsPanel({ form, update }: PanelProps) {
const { t } = useTranslation();
const { t, i18n } = useTranslation();
return (
<>
<p className="mb-3 text-[11px] text-surface-400">
@@ -1501,7 +1523,7 @@ function RequestsPanel({ form, update }: PanelProps) {
</option>
{UA_PRESETS.map((p) => (
<option key={p.label} value={p.ua}>
{p.label}
{translateLabel(p.label, i18n.language)}
</option>
))}
</select>
@@ -1870,7 +1892,12 @@ function CustomExtractionPanel({ form, update }: PanelProps) {
// silently dropping — invalid JSON is the most common
// import failure mode and the user needs to know.
// eslint-disable-next-line no-alert
alert(`Import failed: ${result.error}`);
alert(
t('settingsPanels.customExtraction.importFailedAlert', {
defaultValue: 'Import failed: {{error}}',
error: result.error,
}),
);
}
return;
}
@@ -1878,7 +1905,9 @@ function CustomExtractionPanel({ form, update }: PanelProps) {
// eslint-disable-next-line no-alert
alert(
result.error ??
'No valid rules found in the file.',
t('settingsPanels.customExtraction.noValidRules', {
defaultValue: 'No valid rules found in the file.',
}),
);
return;
}
@@ -3559,10 +3588,10 @@ function RenderingPanel({ form, update }: PanelProps) {
)
}
>
<option value="none">None (default)</option>
<option value="fold">Above the fold</option>
<option value="fullpage">Full page</option>
<option value="both">Both (full page + fold)</option>
<option value="none">{t('settingsPanels.rendering.screenshotNone', { defaultValue: 'None (default)' })}</option>
<option value="fold">{t('settingsPanels.rendering.screenshotFold', { defaultValue: 'Above the fold' })}</option>
<option value="fullpage">{t('settingsPanels.rendering.screenshotFull', { defaultValue: 'Full page' })}</option>
<option value="both">{t('settingsPanels.rendering.screenshotBoth', { defaultValue: 'Both (full page + fold)' })}</option>
</select>
</label>
<Bool
@@ -3673,7 +3702,7 @@ function PerformanceBudgetPanel({ form, update }: PanelProps) {
}
function IntegrationsPanel() {
const { t } = useTranslation();
const { t, i18n } = useTranslation();
const [state, setState] = useState<IntegrationsState | null>(null);
// Per-integration → per-field draft text. Only fields the user has
// actually typed into are tracked; a save sends just these so an
@@ -3734,7 +3763,7 @@ function IntegrationsPanel() {
return (
<div key={cat.key} className="mb-4">
<div className="mb-1.5 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{cat.label}
{translateLabel(cat.label, i18n.language)}
</div>
<div className="space-y-2">
{items.map((def) => (
@@ -3777,7 +3806,7 @@ function IntegrationCard({
onSave: () => void;
onClear: () => void;
}) {
const { t } = useTranslation();
const { t, i18n } = useTranslation();
const configured = state?.configured ?? false;
const hasDraft = Object.values(draft).some((v) => v.length > 0);
const [guideOpen, setGuideOpen] = useState(false);
@@ -3787,7 +3816,7 @@ function IntegrationCard({
<div className="mb-1 flex items-center gap-2">
<span className="text-[12px] font-medium text-surface-100">{def.name}</span>
<span className="rounded bg-surface-800 px-1.5 py-0.5 text-[9px] uppercase tracking-wide text-surface-400">
{AUTH_TYPE_LABEL[def.authType]}
{translateLabel(AUTH_TYPE_LABEL[def.authType], i18n.language)}
</span>
<span
className={clsx(
@@ -3956,17 +3985,17 @@ function StoragePanel() {
return (
<>
<p className="mb-3 text-[11px] text-surface-400">
{t('settings.storage.intro')}
{t('settings.storage.intro', { defaultValue: "Default folder for new .seoproject files. Save As dialogs open here by default; the Open dialog is not constrained." })}
</p>
<div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('settings.storage.defaultSaveFolder')}
{t('settings.storage.defaultSaveFolder', { defaultValue: "Default Save Folder" })}
</div>
<div className="flex items-center gap-2">
<input
className="flex-1 rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
placeholder={t('settings.storage.placeholder')}
placeholder={t('settings.storage.placeholder', { defaultValue: "(System Documents folder)" })}
value={override}
readOnly
spellCheck={false}
@@ -3981,20 +4010,20 @@ function StoragePanel() {
<button
className="rounded border border-surface-700 px-3 py-1 text-[11px] text-surface-400 hover:bg-surface-800 hover:text-surface-100"
onClick={reset}
title={t('settings.storage.resetTitle')}
title={t('settings.storage.resetTitle', { defaultValue: "Reset to the OS Documents folder" })}
>
{t('common.reset')}
</button>
)}
</div>
<p className="mt-2 text-[10px] text-surface-500">
{t('settings.storage.resolvedPath')}:{' '}
{t('settings.storage.resolvedPath', { defaultValue: "Resolved path" })}:{' '}
<span className="text-surface-300">{resolved || '…'}</span>
</p>
</div>
<div className="rounded border border-surface-800 bg-surface-950/40 p-3 text-[11px] text-surface-400">
{t('settings.storage.note')}
{t('settings.storage.note', { defaultValue: "Active crawl data lives in the app's user-data directory until you Save As — the path above only controls where Save As starts. Changing this won't move any existing project files." })}
</div>
</>
);
@@ -4020,12 +4049,12 @@ function PrivacyPanel() {
return (
<>
<p className="mb-3 text-[11px] text-surface-400">
{t('settings.privacy.intro')}
{t('settings.privacy.intro', { defaultValue: "FreeCrawl is local-first — crawl results, project files, and URLs you visit never leave your machine." })}
</p>
<div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('settings.privacy.telemetryHeading')}
{t('settings.privacy.telemetryHeading', { defaultValue: "Anonymous Telemetry" })}
</div>
<label className="flex items-start gap-2">
<input
@@ -4036,17 +4065,17 @@ function PrivacyPanel() {
/>
<div className="flex flex-col gap-0.5">
<span className="text-[12px] text-surface-100">
{t('settings.privacy.telemetryLabel')}
{t('settings.privacy.telemetryLabel', { defaultValue: "Send anonymous usage telemetry (opt-in)" })}
</span>
<span className="text-[10px] text-surface-500">
{t('settings.privacy.telemetryHelp')}
{t('settings.privacy.telemetryHelp', { defaultValue: "Off by default. When on, aggregated counts of feature use (e.g. \"export ran\", \"report opened\") may be sent — never URLs, project contents, or crawl results." })}
</span>
</div>
</label>
</div>
<div className="rounded border border-amber-700/40 bg-amber-900/10 p-3 text-[11px] text-amber-200">
{t('settings.privacy.statusNote')}
{t('settings.privacy.statusNote', { defaultValue: "No telemetry endpoint is currently wired up. Toggling this preference is recorded but nothing is transmitted in the current build. The toggle is exposed now so consent state persists across upgrades when a future build adds an endpoint." })}
</div>
</>
);
@@ -4069,12 +4098,12 @@ function LanguagePanel() {
return (
<>
<p className="mb-3 text-[11px] text-surface-400">
{t('settings.language.intro')}
{t('settings.language.intro', { defaultValue: "Select the UI display language. Applied immediately." })}
</p>
<div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('settings.language.label')}
{t('settings.language.label', { defaultValue: "UI language" })}
</div>
<div className="flex flex-col gap-2">
{SUPPORTED_LANGUAGES.map((lng) => (
@@ -4094,7 +4123,7 @@ function LanguagePanel() {
</div>
<div className="rounded border border-amber-700/40 bg-amber-900/10 p-3 text-[11px] text-amber-200">
{t('settings.language.note')}
{t('settings.language.note', { defaultValue: "Some deep panels and dialogs may still display English while broader coverage is being added." })}
</div>
</>
);
@@ -138,10 +138,15 @@ export function StatsBar() {
valueClassName={fpsClass(perf.fps)}
title={
perf.fps >= 50
? 'Renderer is smooth (≥ 50 fps)'
? t('stats.fpsSmoothTitle', { defaultValue: 'Renderer is smooth (≥ 50 fps)' })
: perf.fps >= 30
? 'Renderer is degraded (30–49 fps) — likely competing with crawl IPC'
: 'Renderer is stalled (< 30 fps) — main thread starved; pause crawl or close Logs window'
? t('stats.fpsDegradedTitle', {
defaultValue: 'Renderer is degraded (30–49 fps) — likely competing with crawl IPC',
})
: t('stats.fpsStalledTitle', {
defaultValue:
'Renderer is stalled (< 30 fps) — main thread starved; pause crawl or close Logs window',
})
}
/>
{perf.heapMb !== null && (
@@ -149,7 +154,9 @@ export function StatsBar() {
label={t('stats.heap')}
value={`${perf.heapMb} MB`}
valueClassName={heapClass(perf.heapMb)}
title="Renderer JS heap. >500 MB = warm, >1 GB = likely a listener / cache leak"
title={t('stats.heapTitle', {
defaultValue: 'Renderer JS heap. >500 MB = warm, >1 GB = likely a listener / cache leak',
})}
/>
)}
{mem && (
@@ -158,26 +165,41 @@ export function StatsBar() {
label={t('stats.rss')}
value={formatBytes(mem.rss)}
valueClassName={rssClass(mem.rss, mem.systemTotal)}
title={`Main-process resident set size (Electron + workers). System total: ${formatBytes(mem.systemTotal)}.`}
title={t('stats.rssTitle', {
defaultValue:
'Main-process resident set size (Electron + workers). System total: {{total}}.',
total: formatBytes(mem.systemTotal),
})}
/>
<Stat
label={t('stats.sysFree')}
value={formatBytes(mem.systemFree)}
valueClassName={systemFreeClass(mem.systemFree, mem.systemTotal)}
title={`OS-reported free memory. < 10% of system total triggers swap; pause the crawl before that. System total: ${formatBytes(mem.systemTotal)}.`}
title={t('stats.sysFreeTitle', {
defaultValue:
'OS-reported free memory. < 10% of system total triggers swap; pause the crawl before that. System total: {{total}}.',
total: formatBytes(mem.systemTotal),
})}
/>
{perUrlCost !== null && (
<Stat
label={t('stats.perUrl')}
value={formatBytes(perUrlCost)}
title={`Average bytes of RSS per crawled URL (${mem.urlsCrawled.toLocaleString()} URLs). Includes Electron overhead, so the marginal cost on real-world large crawls is typically lower.`}
title={t('stats.perUrlTitle', {
defaultValue:
'Average bytes of RSS per crawled URL ({{urls}} URLs). Includes Electron overhead, so the marginal cost on real-world large crawls is typically lower.',
urls: mem.urlsCrawled.toLocaleString(),
})}
/>
)}
{capacity !== null && (
<Stat
label={t('stats.capacity')}
value={formatCount(capacity)}
title={`Estimated additional URLs that fit in remaining system memory at the current per-URL cost. Calc: systemFree / perUrlCost. Treat as an upper bound — sustained throughput is usually CPU-bound first.`}
title={t('stats.capacityTitle', {
defaultValue:
'Estimated additional URLs that fit in remaining system memory at the current per-URL cost. Calc: systemFree / perUrlCost. Treat as an upper bound — sustained throughput is usually CPU-bound first.',
})}
/>
)}
</>
@@ -188,10 +210,17 @@ export function StatsBar() {
valueClassName={lagClass(perf.inputLagMs)}
title={
perf.inputLagMs < 16
? 'Main thread is responsive — input feels instant'
? t('stats.lagResponsiveTitle', {
defaultValue: 'Main thread is responsive — input feels instant',
})
: perf.inputLagMs < 50
? 'Main thread is contended — light click stutter'
: 'Main thread is busy — IPC backed up; most likely sidebar SQL or table chunk fetch competing with the crawler'
? t('stats.lagContendedTitle', {
defaultValue: 'Main thread is contended — light click stutter',
})
: t('stats.lagBusyTitle', {
defaultValue:
'Main thread is busy — IPC backed up; most likely sidebar SQL or table chunk fetch competing with the crawler',
})
}
/>
@@ -53,7 +53,7 @@ export function usePerfMeter(): PerfSample {
useEffect(() => {
let cancelled = false;
let rafId = 0;
let frameTimes: number[] = [];
const frameTimes: number[] = [];
let lastEmittedFps = -1;
let lastEmittedHeap = -1;
let lastEmittedLag = -1;
+54 -1
View File
@@ -18,6 +18,20 @@
*/
const TR_LABELS: Record<string, string> = {
// Settings — User-Agent presets
'Googlebot — Smartphone': 'Googlebot — Akıllı Telefon',
'Googlebot — Desktop': 'Googlebot — Masaüstü',
'Chrome — Desktop': 'Chrome — Masaüstü',
'Chrome — Mobile (Pixel)': 'Chrome — Mobil (Pixel)',
'FreeCrawl SEO (default)': 'FreeCrawl SEO (varsayılan)',
// Settings — Integration categories (Performance is defined below with the sub-tab labels)
AI: 'Yapay Zeka',
'SEO Data': 'SEO Verisi',
// Settings — Integration auth types
'API Key': 'API Anahtarı',
'OAuth (your own client)': 'OAuth (kendi istemciniz)',
'Service Account': 'Servis Hesabı',
Local: 'Yerel',
// Top-level groups
Summary: 'Özet',
'Crawl Data': 'Crawl Verisi',
@@ -83,9 +97,48 @@ const TR_LABELS: Record<string, string> = {
// Indexability
Indexable: 'İndekslenebilir',
Noindex: 'Noindex',
Canonicalised: 'Canonicalised',
Canonicalised: 'Canonical edilmiş',
'Canonicalised (→ other)': 'Canonicalised (→ başka)',
'Non-Indexable': 'İndekslenemez',
// Indexability status (indexabilityStatusLabel values)
'Blocked by robots.txt': 'robots.txt tarafından engellendi',
Redirected: 'Yönlendirildi',
'Client Error': 'İstemci Hatası',
'Server Error': 'Sunucu Hatası',
// Analytics detail panel (AnalyticsRow labels)
Clicks: 'Tıklamalar',
Impressions: 'Gösterimler',
CTR: 'TO',
'Avg position': 'Ort. konum',
Sessions: 'Oturumlar',
Users: 'Kullanıcılar',
Pageviews: 'Sayfa görüntülemeleri',
'Engagement rate': 'Etkileşim oranı',
'Avg session duration': 'Ort. oturum süresi',
Verdict: 'Karar',
Coverage: 'Kapsam',
Indexing: 'İndeksleme',
'Last crawl': 'Son tarama',
'Google canonical': 'Google canonical',
'User canonical': 'Kullanıcı canonical',
// Settings crawl presets (PRESETS labels + descriptions)
Fast: 'Hızlı',
Thorough: 'Kapsamlı',
'Mobile-only': 'Yalnızca mobil',
'Desktop-only': 'Yalnızca masaüstü',
Aggressive: 'Agresif',
'High concurrency, short timeouts — for a quick first sweep on a healthy site. Skips media + retries.':
'Yüksek eşzamanlılık, kısa zaman aşımları — sağlıklı bir sitede hızlı ilk tarama için. Medya + yeniden denemeleri atlar.',
'Lower concurrency + extra retries; captures more on flaky origins. The default for large audits.':
'Daha düşük eşzamanlılık + ek yeniden denemeler; kararsız kaynaklarda daha fazla yakalar. Büyük denetimler için varsayılan.',
'Mimic Googlebot Smartphone — primary signal for mobile-first indexing. Combine with viewport audits.':
'Googlebot Smartphone taklidi — mobil öncelikli indeksleme için birincil sinyal. Viewport denetimleriyle birleştirin.',
'Mimic legacy Googlebot Desktop. Useful for comparing mobile vs. desktop renders.':
'Eski Googlebot Desktop taklidi. Mobil ve masaüstü render karşılaştırması için kullanışlı.',
'High parallelism + ignore robots — only for sites you own. Can trip rate-limit / WAF rules; use with caution.':
'Yüksek paralellik + robots yok sayılır — yalnızca sahip olduğunuz siteler için. Hız sınırı / WAF kurallarını tetikleyebilir; dikkatli kullanın.',
// Issues parent
'Page Titles': 'Sayfa Başlıkları',
@@ -90,7 +90,18 @@
"running": "Running",
"paused": "Paused",
"idle": "Idle",
"dismiss": "dismiss"
"dismiss": "dismiss",
"fpsSmoothTitle": "Renderer is smooth (≥ 50 fps)",
"fpsDegradedTitle": "Renderer is degraded (30–49 fps) — likely competing with crawl IPC",
"fpsStalledTitle": "Renderer is stalled (< 30 fps) — main thread starved; pause crawl or close Logs window",
"heapTitle": "Renderer JS heap. >500 MB = warm, >1 GB = likely a listener / cache leak",
"rssTitle": "Main-process resident set size (Electron + workers). System total: {{total}}.",
"sysFreeTitle": "OS-reported free memory. < 10% of system total triggers swap; pause the crawl before that. System total: {{total}}.",
"perUrlTitle": "Average bytes of RSS per crawled URL ({{urls}} URLs). Includes Electron overhead, so the marginal cost on real-world large crawls is typically lower.",
"capacityTitle": "Estimated additional URLs that fit in remaining system memory at the current per-URL cost. Calc: systemFree / perUrlCost. Treat as an upper bound — sustained throughput is usually CPU-bound first.",
"lagResponsiveTitle": "Main thread is responsive — input feels instant",
"lagContendedTitle": "Main thread is contended — light click stutter",
"lagBusyTitle": "Main thread is busy — IPC backed up; most likely sidebar SQL or table chunk fetch competing with the crawler"
},
"menu": {
"file": "File",
@@ -233,7 +244,13 @@
"aggregatingFirst": "aggregating first {{n}}",
"perPagePrefix": "This tab is per-page — showing data for",
"perPageSuffix": "(primary URL of the selection).",
"selectUrl": "Select a URL from the table to see details."
"selectUrl": "Select a URL from the table to see details.",
"httpResponseTab": "Response ({{count}})",
"httpRequestTab": "Request ({{count}})",
"headersReconstructedNote": "Reconstructed from active crawl config — actual values may have differed if config changed since crawl.",
"headersCapturedNote": "Captured at crawl time.",
"noResponseHeaders": "No response headers captured for this URL.",
"noRequestHeaders": "No request headers — config may be empty."
},
"filter": {
"title": "Advanced Table Search",
@@ -404,7 +421,15 @@
"crawlDelayMs": "Per-Worker Delay (ms after each request)",
"retriesHeading": "Retries",
"retryAttempts": "Retry Attempts (per URL on transient errors)",
"retryInitialDelayMs": "Initial Retry Delay (ms — exponential backoff)"
"retryInitialDelayMs": "Initial Retry Delay (ms — exponential backoff)",
"tipsHeading": "Throughput tips",
"ceilingValue": "~{{rps}} URL/s ({{workers}} parallel workers, ≤ {{cap}} RPS rate-limit{{delaySuffix}})",
"ceilingDelaySuffix": ", +{{delay}} ms post-request delay per worker",
"ceilingEmpty": "Set Max Concurrency and Max RPS to see the projected throughput.",
"worstCaseDelay": "Worst-case delay per failed URL ≈ initialDelay × (2 ^ attempts − 1) = {{value}}",
"tip1": "For a quick first sweep on a healthy site try the <bold>Fast</bold> preset (Settings → Presets) — concurrency 40, RPS 40, no retries.",
"tip2": "For unreliable origins or slow APIs try <bold>Thorough</bold> — concurrency 10, 3 retries, 30 s timeout.",
"tip3": "Concurrency × HTTP keep-alive = the steady-state connection count. Most servers comfortably handle 20–40; corporate proxies / WAFs often cap at 8–10."
},
"requests": {
"intro": "HTTP headers sent with every request.",
@@ -466,7 +491,9 @@
"exportTitle": "Save the current rule set as a JSON file. Versioned envelope, safe to commit to source control.",
"exportDisabled": "Add at least one rule before exporting.",
"importConfirm": "Replace {{existing}} existing rule(s) with {{incoming}} imported rule(s)?",
"importSkipped": "Imported {{ok}} rule(s). {{skipped}} entries were skipped (invalid shape or over the 10-rule cap)."
"importSkipped": "Imported {{ok}} rule(s). {{skipped}} entries were skipped (invalid shape or over the 10-rule cap).",
"importFailedAlert": "Import failed: {{error}}",
"noValidRules": "No valid rules found in the file."
},
"urlRewriting": {
"intro": "Normalisation applied before URLs are deduplicated and queued.",
@@ -638,7 +665,11 @@
"advancedAudits": "Advanced Audits",
"mobileUsability": "Mobile Usability audit",
"lcpCandidate": "LCP candidate detection",
"a11yAudit": "Accessibility audit (contrast + focus)"
"a11yAudit": "Accessibility audit (contrast + focus)",
"screenshotNone": "None (default)",
"screenshotFold": "Above the fold",
"screenshotFull": "Full page",
"screenshotBoth": "Both (full page + fold)"
},
"performanceBudget": {
"intro": "Set per-page performance ceilings. After each crawl, internal HTML pages that exceed any enabled threshold are flagged under the \"Over Performance Budget\" issue filter. Response time and page size come from the crawl itself; LCP and CLS use PageSpeed Insights data when present. Set any field to 0 to disable that individual check.",
@@ -806,7 +837,8 @@
"external": "external",
"largeCount": "{{n}} > 100 KB",
"multiEmpty": "No <img> tags discovered across the {{n}} selected pages.",
"multiSummary": "images across {{n}} pages"
"multiSummary": "images across {{n}} pages",
"showingFirst": "showing first {{shown}} of {{total}}"
},
"links": {
"showing": "Showing",
@@ -928,7 +960,9 @@
"edgeOpacity": "Edge opacity",
"edgeOpacityHint": "Lower = less visual noise on dense graphs.",
"changesNeedRerun": "Some changes need a layout re-run.",
"rerunLayout": "Re-run layout"
"rerunLayout": "Re-run layout",
"title": "Visualization",
"occurrences": "{{count}} occurrences"
},
"imagesTab": {
"searchPlaceholder": "Search image URLs / alt…",
@@ -1161,7 +1195,27 @@
"treeLoading": "Loading URL tree…",
"treeEmpty": "No URLs to display — crawl a site or change the filter.",
"expandAll": "Expand all",
"collapseAll": "Collapse all"
"collapseAll": "Collapse all",
"noUrlsHint": "Start a crawl or choose a different category.",
"statusUrls": "URLs:",
"statusLoaded": "{{count}} loaded",
"statusSelectedRows": "Selected Rows:",
"statusSelectedCells": "Selected Cells:",
"statusSelectedColumns": "Selected Columns:",
"rowHeader": "Row",
"rowNumberTitle": "Row number",
"rowSelectHint": "Click to select row · drag to select multiple rows",
"resizeHint": "Drag to resize · double-click to reset",
"sortByColumn": "Sort by this column",
"columnSortedAscHint": "Sorted ascending — click to flip",
"columnSortedDescHint": "Sorted descending — click to flip",
"columnSortedHint": "Sorted {{dir}} — click to flip",
"sortAsc": "ascending",
"sortDesc": "descending"
},
"errorBoundary": {
"panelBroke": "Something broke in this panel.",
"dismiss": "Dismiss"
},
"settings": {
"title": "Settings",
@@ -90,7 +90,18 @@
"running": "Çalışıyor",
"paused": "Duraklatıldı",
"idle": "Boşta",
"dismiss": "kapat"
"dismiss": "kapat",
"fpsSmoothTitle": "Arayüz akıcı (≥ 50 fps)",
"fpsDegradedTitle": "Arayüz yavaşladı (30–49 fps) — muhtemelen crawl IPC ile yarışıyor",
"fpsStalledTitle": "Arayüz takıldı (< 30 fps) — ana iş parçacığı aç kaldı; crawl'ı duraklatın veya Loglar penceresini kapatın",
"heapTitle": "Renderer JS belleği (heap). >500 MB = ısınmış, >1 GB = muhtemelen listener / önbellek sızıntısı",
"rssTitle": "Ana süreç yerleşik bellek boyutu (Electron + worker'lar). Sistem toplamı: {{total}}.",
"sysFreeTitle": "İşletim sisteminin bildirdiği boş bellek. Sistem toplamının < %10'u takas (swap) tetikler; ondan önce crawl'ı duraklatın. Sistem toplamı: {{total}}.",
"perUrlTitle": "Taranan URL başına ortalama RSS baytı ({{urls}} URL). Electron ek yükünü içerir; gerçek dünyadaki büyük crawl'larda marjinal maliyet genelde daha düşüktür.",
"capacityTitle": "Mevcut URL-başına maliyetle kalan sistem belleğine sığacak tahmini ek URL sayısı. Hesap: boşBellek / urlBaşınaMaliyet. Üst sınır olarak değerlendirin — sürekli verim genelde önce CPU'ya bağlıdır.",
"lagResponsiveTitle": "Ana iş parçacığı yanıt veriyor — girdi anında hissediliyor",
"lagContendedTitle": "Ana iş parçacığı yoğun — hafif tıklama takılması",
"lagBusyTitle": "Ana iş parçacığı meşgul — IPC birikti; büyük olasılıkla kenar çubuğu SQL'i veya tablo parça getirme crawler ile yarışıyor"
},
"menu": {
"file": "Dosya",
@@ -233,7 +244,13 @@
"aggregatingFirst": "ilk {{n}} toplanıyor",
"perPagePrefix": "Bu sekme sayfa başınadır — şu URL için veri gösteriliyor:",
"perPageSuffix": "(seçimin birincil URL'i).",
"selectUrl": "Detayları görmek için tablodan bir URL seç."
"selectUrl": "Detayları görmek için tablodan bir URL seç.",
"httpResponseTab": "Yanıt ({{count}})",
"httpRequestTab": "İstek ({{count}})",
"headersReconstructedNote": "Aktif tarama yapılandırmasından yeniden oluşturuldu — taramadan sonra yapılandırma değiştiyse gerçek değerler farklı olabilir.",
"headersCapturedNote": "Tarama sırasında yakalandı.",
"noResponseHeaders": "Bu URL için yanıt başlığı yakalanmadı.",
"noRequestHeaders": "İstek başlığı yok — yapılandırma boş olabilir."
},
"filter": {
"title": "Gelişmiş Tablo Arama",
@@ -250,6 +267,7 @@
"noValue": "(değer gerekmiyor)"
},
"reports": {
"reportLabel": "Rapor",
"report": {
"pagesPerDir": "Dizin Başına Sayfa",
"statusCodes": "Durum Kodu Histogramı",
@@ -403,7 +421,15 @@
"crawlDelayMs": "Worker Başına Gecikme (her istekten sonra, ms)",
"retriesHeading": "Yeniden Denemeler",
"retryAttempts": "Yeniden Deneme Sayısı (geçici hatada URL başına)",
"retryInitialDelayMs": "İlk Yeniden Deneme Gecikmesi (ms — üstel backoff)"
"retryInitialDelayMs": "İlk Yeniden Deneme Gecikmesi (ms — üstel backoff)",
"tipsHeading": "Verimlilik ipuçları",
"ceilingValue": "~{{rps}} URL/sn ({{workers}} paralel worker, ≤ {{cap}} RPS rate-limit{{delaySuffix}})",
"ceilingDelaySuffix": ", worker başına istek sonrası +{{delay}} ms gecikme",
"ceilingEmpty": "Öngörülen verimi görmek için Maks Paralellik ve Maks RPS değerlerini ayarlayın.",
"worstCaseDelay": "Başarısız URL başına en kötü durum gecikmesi ≈ ilkGecikme × (2 ^ deneme − 1) = {{value}}",
"tip1": "Sağlıklı bir sitede hızlı bir ilk tarama için <bold>Fast</bold> ön ayarını deneyin (Ayarlar → Ön Ayarlar) — paralellik 40, RPS 40, yeniden deneme yok.",
"tip2": "Güvenilmez kaynaklar veya yavaş API'ler için <bold>Thorough</bold> deneyin — paralellik 10, 3 yeniden deneme, 30 sn zaman aşımı.",
"tip3": "Paralellik × HTTP keep-alive = kararlı durumdaki bağlantı sayısı. Çoğu sunucu rahatça 20–40 taşır; kurumsal proxy'ler / WAF'lar genelde 8–10'da sınırlar."
},
"requests": {
"intro": "Her istekle gönderilen HTTP başlıkları.",
@@ -465,7 +491,9 @@
"exportTitle": "Mevcut kural setini JSON dosyasına kaydet. Versiyonlu sarmal, kaynak kontrolüne commit'lenebilir.",
"exportDisabled": "Dışa aktarmadan önce en az bir kural ekle.",
"importConfirm": "Mevcut {{existing}} kural, içe aktarılan {{incoming}} kuralla değiştirilsin mi?",
"importSkipped": "{{ok}} kural içe aktarıldı. {{skipped}} kayıt atlandı (geçersiz şema veya 10 kuralın üzerinde)."
"importSkipped": "{{ok}} kural içe aktarıldı. {{skipped}} kayıt atlandı (geçersiz şema veya 10 kuralın üzerinde).",
"importFailedAlert": "İçe aktarma başarısız: {{error}}",
"noValidRules": "Dosyada geçerli kural bulunamadı."
},
"urlRewriting": {
"intro": "URL'ler yinelemeden ve kuyruğa alınmadan önce uygulanan normalleştirme.",
@@ -637,7 +665,11 @@
"advancedAudits": "Gelişmiş Denetimler",
"mobileUsability": "Mobil Kullanılabilirlik denetimi",
"lcpCandidate": "LCP aday tespiti",
"a11yAudit": "Erişilebilirlik denetimi (kontrast + odak)"
"a11yAudit": "Erişilebilirlik denetimi (kontrast + odak)",
"screenshotNone": "Yok (varsayılan)",
"screenshotFold": "Katlama üstü",
"screenshotFull": "Tam sayfa",
"screenshotBoth": "Her ikisi (tam sayfa + katlama)"
},
"performanceBudget": {
"intro": "Sayfa başına performans tavanları belirleyin. Her crawl'dan sonra, etkin eşiklerden herhangi birini aşan dahili HTML sayfaları \"Performans Bütçesini Aşan\" issue filtresinde işaretlenir. Yanıt süresi ve sayfa boyutu crawl'dan; LCP ve CLS mevcutsa PageSpeed Insights verisinden gelir. Bir kontrolü devre dışı bırakmak için ilgili alanı 0 yapın.",
@@ -805,7 +837,8 @@
"external": "harici",
"largeCount": "{{n}} > 100 KB",
"multiEmpty": "Seçili {{n}} sayfada <img> etiketi bulunamadı.",
"multiSummary": "{{n}} sayfada görseller"
"multiSummary": "{{n}} sayfada görseller",
"showingFirst": "ilk {{shown}} / {{total}} gösteriliyor"
},
"links": {
"showing": "Gösterilen",
@@ -927,7 +960,9 @@
"edgeOpacity": "Kenar opaklığı",
"edgeOpacityHint": "Düşük = yoğun grafiklerde daha az görsel gürültü.",
"changesNeedRerun": "Bazı değişiklikler layout yeniden çalıştırması gerektirir.",
"rerunLayout": "Layout'u yeniden çalıştır"
"rerunLayout": "Layout'u yeniden çalıştır",
"title": "Görselleştirme",
"occurrences": "{{count}} kez geçiyor"
},
"imagesTab": {
"searchPlaceholder": "Görsel URL / alt ara…",
@@ -1160,7 +1195,27 @@
"treeLoading": "URL ağacı yükleniyor…",
"treeEmpty": "Gösterilecek URL yok — bir site tara veya filtreyi değiştir.",
"expandAll": "Tümünü genişlet",
"collapseAll": "Tümünü daralt"
"collapseAll": "Tümünü daralt",
"noUrlsHint": "Bir crawl başlatın veya farklı bir kategori seçin.",
"statusUrls": "URL:",
"statusLoaded": "{{count}} yüklendi",
"statusSelectedRows": "Seçili Satırlar:",
"statusSelectedCells": "Seçili Hücreler:",
"statusSelectedColumns": "Seçili Sütunlar:",
"rowHeader": "Satır",
"rowNumberTitle": "Satır numarası",
"rowSelectHint": "Satır seçmek için tıkla · birden çok satır için sürükle",
"resizeHint": "Yeniden boyutlandırmak için sürükle · sıfırlamak için çift tıkla",
"sortByColumn": "Bu sütuna göre sırala",
"columnSortedAscHint": "Artan sıralandı — çevirmek için tıkla",
"columnSortedDescHint": "Azalan sıralandı — çevirmek için tıkla",
"columnSortedHint": "{{dir}} sıralandı — ters çevirmek için tıklayın",
"sortAsc": "artan",
"sortDesc": "azalan"
},
"errorBoundary": {
"panelBroke": "Bu panelde bir şeyler bozuldu.",
"dismiss": "Kapat"
},
"settings": {
"title": "Ayarlar",
+44 -18
View File
@@ -1030,9 +1030,9 @@ export function UrlsTab() {
zIndex: 13,
background: 'rgb(23 23 23)',
}}
title="Row number"
title={t('urlsTab.rowNumberTitle', { defaultValue: 'Row number' })}
>
Row
{t('urlsTab.rowHeader', { defaultValue: 'Row' })}
<div
className="absolute -right-1 top-0 bottom-0 z-20 w-2 cursor-col-resize group hover:bg-accent-500/40"
onMouseDown={(e) => {
@@ -1046,7 +1046,9 @@ export function UrlsTab() {
resetColumn(ROW_NUM_KEY);
}}
onClick={(e) => e.stopPropagation()}
title="Drag to resize · double-click to reset"
title={t('urlsTab.resizeHint', {
defaultValue: 'Drag to resize · double-click to reset',
})}
/>
</div>
<DndContext
@@ -1185,7 +1187,9 @@ export function UrlsTab() {
onMouseEnter={() => {
if (dragRef.current?.kind === 'row') applyRowDrag(vi.index);
}}
title="Click to select row · drag to select multiple rows"
title={t('urlsTab.rowSelectHint', {
defaultValue: 'Click to select row · drag to select multiple rows',
})}
>
{vi.index + 1}
</div>
@@ -1236,7 +1240,7 @@ export function UrlsTab() {
}
}}
>
<Cell row={row} spec={c} />
<Cell row={row} spec={c} lang={lang} />
</div>
);
})}
@@ -1263,7 +1267,9 @@ export function UrlsTab() {
<div className="max-w-md text-center">
<div className="mb-1 text-sm font-semibold text-surface-300">{t('urlsTab.noUrlsToShow', { defaultValue: 'No URLs to show' })}</div>
<div className="text-xs text-surface-500">
Start a crawl or choose a different category.
{t('urlsTab.noUrlsHint', {
defaultValue: 'Start a crawl or choose a different category.',
})}
</div>
</div>
</div>
@@ -1275,28 +1281,31 @@ export function UrlsTab() {
style={{ height: STATUS_BAR_HEIGHT }}
>
<span>
URLs:{' '}
{t('urlsTab.statusUrls', { defaultValue: 'URLs:' })}{' '}
<span className="font-mono tabular-nums text-surface-200">
{lazy.total.toLocaleString()}
</span>
<span className="ml-1 text-surface-600">
({lazy.loadedRows.toLocaleString()} loaded)
({t('urlsTab.statusLoaded', {
defaultValue: '{{count}} loaded',
count: lazy.loadedRows.toLocaleString(),
})})
</span>
</span>
<span>
Selected Rows:{' '}
{t('urlsTab.statusSelectedRows', { defaultValue: 'Selected Rows:' })}{' '}
<span className="font-mono tabular-nums text-surface-200">
{selectedIds.size.toLocaleString()}
</span>
</span>
<span>
Selected Cells:{' '}
{t('urlsTab.statusSelectedCells', { defaultValue: 'Selected Cells:' })}{' '}
<span className="font-mono tabular-nums text-surface-200">
{selectedCells.size.toLocaleString()}
</span>
</span>
<span>
Selected Columns:{' '}
{t('urlsTab.statusSelectedColumns', { defaultValue: 'Selected Columns:' })}{' '}
<span className="font-mono tabular-nums text-surface-200">
{selectedColumns.size.toLocaleString()}
</span>
@@ -1466,8 +1475,14 @@ function SortableHeaderCell(props: {
onMouseDown={(e) => e.stopPropagation()}
title={
isSortCol
? `Sorted ${sortDir === 'asc' ? 'ascending' : 'descending'} — click to flip`
: 'Sort by this column'
? t('urlsTab.columnSortedHint', {
defaultValue: 'Sorted {{dir}} — click to flip',
dir:
sortDir === 'asc'
? t('urlsTab.sortAsc', { defaultValue: 'ascending' })
: t('urlsTab.sortDesc', { defaultValue: 'descending' }),
})
: t('urlsTab.sortByColumn', { defaultValue: 'Sort by this column' })
}
>
{isSortCol ? (
@@ -1489,7 +1504,9 @@ function SortableHeaderCell(props: {
onResetColumn(id);
}}
onClick={(e) => e.stopPropagation()}
title="Drag to resize · double-click to reset"
title={t('urlsTab.resizeHint', {
defaultValue: 'Drag to resize · double-click to reset',
})}
/>
</div>
);
@@ -1687,7 +1704,15 @@ function ColumnPickerPopover({
);
}
function Cell({ row, spec }: { row: CrawlUrlRow | null; spec: ColumnSpec }) {
function Cell({
row,
spec,
lang,
}: {
row: CrawlUrlRow | null;
spec: ColumnSpec;
lang: string;
}) {
if (row === null) {
return <span className="text-surface-700">…</span>;
}
@@ -1723,7 +1748,7 @@ function Cell({ row, spec }: { row: CrawlUrlRow | null; spec: ColumnSpec }) {
className={clsx('truncate', v === 'indexable' ? 'text-emerald-400' : 'text-amber-400')}
title={v}
>
{v === 'indexable' ? 'Indexable' : 'Non-Indexable'}
{translateLabel(v === 'indexable' ? 'Indexable' : 'Non-Indexable', lang)}
</span>
);
}
@@ -1733,9 +1758,10 @@ function Cell({ row, spec }: { row: CrawlUrlRow | null; spec: ColumnSpec }) {
if (label === '') {
return <span className="text-surface-700">—</span>;
}
const localized = translateLabel(label, lang);
return (
<span className="block truncate text-surface-200" title={label}>
{label}
<span className="block truncate text-surface-200" title={localized}>
{localized}
</span>
);
}
@@ -750,7 +750,7 @@ export function VisualizationTab() {
<div className="flex h-full w-full flex-col bg-surface-950">
<div className="flex items-center gap-2 border-b border-surface-800 bg-surface-900/40 px-3 py-1.5">
<div className="text-[12px] font-semibold tracking-wide text-surface-100">
Visualization
{t('viz.title', { defaultValue: 'Visualization' })}
</div>
<div className="ml-2 flex flex-wrap items-center gap-2 text-[11px]">
<label className="flex items-center gap-1 text-surface-400">
@@ -1047,7 +1047,10 @@ export function VisualizationTab() {
key={a.anchor}
className="text-surface-200"
style={{ fontSize: `${size}px` }}
title={`${a.count.toLocaleString()} occurrences`}
title={t('viz.occurrences', {
defaultValue: '{{count}} occurrences',
count: a.count.toLocaleString(),
})}
>
{a.anchor}
</span>
+22 -6
View File
@@ -26,7 +26,7 @@ import { buildTools, type Tool } from './tools.js';
import { defaultProjectPath, listProjectFiles } from './project-resolver.js';
const SERVER_NAME = 'freecrawl-seo';
const SERVER_VERSION = '0.2.7';
const SERVER_VERSION = '0.8.0';
const PROTOCOL_VERSION = '2024-11-05';
interface Session {
@@ -58,6 +58,10 @@ function setProject(projectPath: string): Session {
} catch {
// best-effort close
}
// Null it first so that if openProject throws (missing / corrupt file)
// we don't leave the session pointing at a now-closed DB handle, which
// would make every later DB tool fail with "database is not open".
session = null;
}
session = openProject(projectPath);
return session;
@@ -70,6 +74,11 @@ const tools: Tool[] = [
description:
'List `.seoproject` files in the desktop app\'s default projects folder. Returns absolute paths the user can pass to `set_project`.',
inputSchema: { type: 'object', properties: {} },
// Must NOT force-open the default project — on a fresh install (or when
// only non-default projects exist) that file doesn't exist yet, and
// requiring it would deadlock: the very tools you'd use to point the
// session at a valid file would themselves fail to open the missing one.
requiresDb: false,
handler: () => {
const files = listProjectFiles();
return { files, defaultPath: defaultProjectPath() };
@@ -89,6 +98,8 @@ const tools: Tool[] = [
},
},
},
// Opens the *target* file itself; must not first open the default.
requiresDb: false,
handler: (args) => {
const target = args.path;
if (typeof target !== 'string' || target.trim() === '') {
@@ -103,7 +114,10 @@ const tools: Tool[] = [
description:
'Report which `.seoproject` the MCP server is currently reading from.',
inputSchema: { type: 'object', properties: {} },
handler: () => ({ projectPath: getSession().projectPath }),
requiresDb: false,
// Report the open project without forcing one open — fall back to the
// default path (which may not exist yet) purely as a hint.
handler: () => ({ projectPath: session ? session.projectPath : defaultProjectPath() }),
},
];
@@ -158,9 +172,13 @@ server.on('tools/call', async (raw) => {
// work on a fresh install where the local `.seoproject` file doesn't
// exist yet (only the desktop app has it).
const needsDb = tool.requiresDb !== false;
let result: unknown;
try {
// `db` is unused when `needsDb` is false; we pass the session's DB
// when available, else a never-used null cast — type system lets us
// do this because the handler ignores `db` for those tools.
// when available, else a never-used null cast — the handler ignores it
// for those tools. Resolving the session INSIDE the try means a failed
// project open (e.g. a missing/corrupt file) surfaces as a proper tool
// error (`isError`) instead of a raw JSON-RPC -32000 protocol error.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let db: any = null;
if (needsDb) {
@@ -170,8 +188,6 @@ server.on('tools/call', async (raw) => {
// the DB just to satisfy the param), but don't force-open.
db = session.db;
}
let result: unknown;
try {
result = await tool.handler(args, db);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
+16 -34
View File
@@ -30,32 +30,19 @@ export interface Tool {
requiresDb?: boolean;
}
const URL_CATEGORY_VALUES: UrlCategory[] = [
'all',
'internal:all',
'internal:html',
'internal:js',
'internal:css',
'internal:image',
'internal:pdf',
'internal:font',
'internal:other',
'external:all',
'external:html',
'external:other',
'status:2xx',
'status:3xx',
'status:4xx',
'status:5xx',
'status:no-response',
'status:blocked-robots',
'security:https',
'security:http',
'indexability:indexable',
'indexability:non-indexable',
'indexability:noindex',
'indexability:canonicalised',
];
// The `category` filter accepts the full UrlCategory union (200+ values:
// internal/external content kinds, status:* buckets, security:*,
// indexability:*, tab:* groupings, and per-issue drill-downs like
// "issues:title-missing" / "issues:severity-critical"). The union is a
// type, not a runtime list, so we validate at the DB layer (categoryWhereClause)
// and describe rather than `enum`-constrain — an enum of a hand-picked subset
// would reject the very `issues:*` drill-downs that `top_issues` returns.
const URL_CATEGORY_DESC =
'UrlCategory filter matching the desktop UI categories. Accepts "all"; ' +
'internal:*/external:* content kinds; status:2xx/3xx/4xx/5xx/no-response/blocked-robots; ' +
'security:https/http; indexability:*; tab:* groupings; and per-issue drill-downs ' +
'such as "issues:title-missing" or severity rollups like "issues:severity-critical". ' +
'Defaults to "all".';
const COMMON_URL_FIELDS = [
'id',
@@ -157,7 +144,6 @@ export function buildTools(): Tool[] {
category: {
type: 'string',
description: 'UrlCategory filter (e.g. "all", "internal:html", "issues:title-missing", "status:4xx"). Defaults to "all".',
enum: URL_CATEGORY_VALUES,
},
search: {
type: 'string',
@@ -1054,7 +1040,6 @@ export function buildTools(): Tool[] {
filePath: { type: 'string', description: 'Absolute output path.' },
category: {
type: 'string',
enum: URL_CATEGORY_VALUES,
description: 'Optional UrlCategory subset filter. Default exports the whole project.',
},
selectedIds: {
@@ -1081,7 +1066,7 @@ export function buildTools(): Tool[] {
type: 'object',
properties: {
filePath: { type: 'string' },
category: { type: 'string', enum: URL_CATEGORY_VALUES },
category: { type: 'string', description: URL_CATEGORY_DESC },
selectedIds: { type: 'array', items: { type: 'integer' } },
pretty: { type: 'boolean' },
},
@@ -1103,7 +1088,7 @@ export function buildTools(): Tool[] {
type: 'object',
properties: {
filePath: { type: 'string' },
category: { type: 'string', enum: URL_CATEGORY_VALUES },
category: { type: 'string', description: URL_CATEGORY_DESC },
selectedIds: { type: 'array', items: { type: 'integer' } },
},
required: ['filePath'],
@@ -1408,7 +1393,7 @@ export function buildTools(): Tool[] {
type: 'object',
properties: {
label: { type: 'string' },
category: { type: 'string', enum: URL_CATEGORY_VALUES },
category: { type: 'string', description: URL_CATEGORY_DESC },
subdir: { type: 'string' },
filename: { type: 'string' },
},
@@ -1675,7 +1660,6 @@ export function buildTools(): Tool[] {
},
category: {
type: 'string',
enum: URL_CATEGORY_VALUES,
description: 'Resolve target URLs from the active project by category (e.g. "internal:html") when `urls` is omitted.',
},
search: {
@@ -1726,7 +1710,6 @@ export function buildTools(): Tool[] {
},
category: {
type: 'string',
enum: URL_CATEGORY_VALUES,
description: 'Resolve targets by category when `urls` is omitted.',
},
search: { type: 'string' },
@@ -1762,7 +1745,6 @@ export function buildTools(): Tool[] {
},
category: {
type: 'string',
enum: URL_CATEGORY_VALUES,
description: 'Resolve targets by category when `urls` is omitted.',
},
search: { type: 'string' },
+34 -1
View File
@@ -1,4 +1,6 @@
import js from '@eslint/js';
import globals from 'globals';
import reactHooks from 'eslint-plugin-react-hooks';
import tseslint from 'typescript-eslint';
export default tseslint.config(
@@ -15,6 +17,37 @@ export default tseslint.config(
},
},
{
ignores: ['**/dist/**', '**/out/**', '**/node_modules/**', '**/build/**', '.claude/**'],
// React Hooks linting for the renderer. Previously the codebase carried
// `eslint-disable react-hooks/exhaustive-deps` comments but the plugin was
// never installed, so ESLint errored "rule not found" AND hooks-deps
// linting never actually ran. rules-of-hooks catches conditional/looped
// hook calls; exhaustive-deps runs as a warning so existing disables are
// honoured without failing the gate.
files: ['**/*.{ts,tsx}'],
plugins: { 'react-hooks': reactHooks },
rules: {
'react-hooks/rules-of-hooks': 'error',
'react-hooks/exhaustive-deps': 'warn',
},
},
{
// Node-run build/setup scripts (plain .js/.mjs, not the bundled app) use
// process/console/__dirname etc. Give them the Node global environment so
// they don't trip `no-undef`.
files: ['scripts/**/*.{js,mjs,cjs}', '**/*.config.{js,mjs,cjs}', 'apps/*/scripts/**/*.{js,mjs,cjs}'],
languageOptions: {
globals: { ...globals.node },
},
},
{
ignores: [
'**/dist/**',
'**/dist_bak_claude/**',
'**/out/**',
'**/out-types/**',
'**/node_modules/**',
'**/build/**',
'.claude/**',
],
},
);
+6 -1
View File
@@ -119,11 +119,16 @@ export function compareCrawls(
const counts: Record<CompareCategory, number> = { ...EMPTY_COUNTS };
const samples: CompareDiffRow[] = [];
// Per-category sample tally so the cap check is O(1). The previous
// `samples.filter(...).length` scanned the whole samples array on every
// push — O(n²) that hung for minutes comparing two large divergent crawls.
const sampleCounts: Record<CompareCategory, number> = { ...EMPTY_COUNTS };
const push = (row: CompareDiffRow): void => {
counts[row.category]++;
if (samples.filter((s) => s.category === row.category).length < limit) {
if (sampleCounts[row.category] < limit) {
samples.push(row);
sampleCounts[row.category]++;
}
};
+220 -85
View File
@@ -22,6 +22,7 @@ import {
resolveStartUrl,
compileUrlRegexRewrites,
toEscapedFragmentUrl,
type UrlRewriteOptions,
} from './url-utils.js';
import { parseHtml, estimatePixelWidth, type ParsedPage } from './html-parser.js';
import { analyseCookies, extractSetCookies } from './cookies.js';
@@ -111,6 +112,10 @@ export class Crawler extends EventEmitter {
* sequence, replayed on every request. Null when form login is off. */
private sessionJar: SessionCookieJar | null = null;
private robots: RobotsChecker | null = null;
/** robots.txt `Crawl-delay` for the start origin, in ms (0 = none). */
private robotsCrawlDelayMs = 0;
/** Monotonic dispatch clock for the maxRps gate (see acquireRateSlot). */
private nextDispatchAt = 0;
private progressTimer: NodeJS.Timeout | null = null;
private memoryWatchdogTimer: NodeJS.Timeout | null = null;
/** Periodic post-crawl-style recompute of expensive issue counters
@@ -205,7 +210,10 @@ export class Crawler extends EventEmitter {
* nothing per call (no `?:` chains, no per-link `if`s) and so changing
* config mid-crawl can't desync the seen-set's keying.
*/
private readonly urlRewrites: import('./url-utils.js').UrlRewriteOptions;
private readonly urlRewrites: UrlRewriteOptions;
/** Effective proxy resolved in the constructor, reused by the start-URL
* probe so it doesn't reset the global dispatcher back to direct. */
private readonly resolvedProxy: string;
/**
* Optional freeze-watchdog hook. The desktop main process injects a
@@ -421,6 +429,7 @@ export class Crawler extends EventEmitter {
}
return config.proxyUrl ?? '';
})();
this.resolvedProxy = resolvedProxy;
initHttpClient({ proxyOverride: resolvedProxy });
this.config = config;
this.db = db;
@@ -539,14 +548,20 @@ export class Crawler extends EventEmitter {
}
const startProbeT0 = Date.now();
const start = await resolveStartUrl(this.config.startUrl, this.config.userAgent, 5000, (info) => {
const start = await resolveStartUrl(
this.config.startUrl,
this.config.userAgent,
5000,
(info) => {
this.emit(
'debug',
`resolveStartUrl: ${info.method} ${info.url} -> ${info.outcome}${
info.detail ? ` (${info.detail})` : ''
}`,
);
});
},
{ proxyOverride: this.resolvedProxy, rewrites: this.urlRewrites },
);
if (!start) {
this.emit(
'error',
@@ -598,7 +613,18 @@ export class Crawler extends EventEmitter {
// recompute and sitemap-derived issue counts use the full data set.
const robotsPromise = this.config.respectRobotsTxt
? loadRobots(origin, this.config.userAgent).then((r) => {
if (!this.stopped) this.robots = r;
if (!this.stopped) {
this.robots = r;
// Honour a robots.txt `Crawl-delay` (seconds) by folding it into
// the per-worker politeness delay. Cached once — the start
// origin's directive applies for the crawl, matching the single
// robots checker used for allow/deny.
const rd = r.getCrawlDelay();
this.robotsCrawlDelayMs =
typeof rd === 'number' && Number.isFinite(rd) && rd > 0
? Math.round(rd * 1000)
: 0;
}
})
: Promise.resolve();
const sitemapPromise = this.config.discoverSitemaps
@@ -722,17 +748,38 @@ export class Crawler extends EventEmitter {
this.stopQueueCheckpointTimer();
}
// Post-crawl heavy lifting. Each step is a synchronous SQL pass
// that can take 1–3 s on a 1M-URL crawl; if we run them all in a
// tight sequence the main process JS thread stays blocked for the
// entire duration, IPC dispatch starves, and any other window
// (especially Logs) freezes visibly. Yielding to the event loop
// between steps lets queued IPC mesajları (logs:batch, progress,
// dataChanged) get serviced.
// Wave 6 — Per-pass crawl-analysis toggles. Each pass is gated
// by its config flag so users running tight time-budget audits
// can skip steps they don't need (e.g. duplicate clustering can
// be 5–10 s on a 100k-URL crawl).
// Post-crawl heavy lifting — shared with list mode via runPostCrawlPasses.
// Each pass is gated by its config flag so users running tight time-budget
// audits can skip steps they don't need, and each yields to the event loop
// so IPC dispatch (logs:batch, progress, dataChanged) never starves.
await this.runPostCrawlPasses();
this.running = false;
this.stopMemoryMonitor();
// Release per-URL dedup sets — at 1M URLs this is ~80–120 MB of string
// heap that's no longer needed once the queue is drained.
this.seen.clear();
this.externalSeen.clear();
this.emitProgress();
this.setOp('idle');
// Suppress 'done' if a stop() ran during teardown — otherwise the
// zombie crawler's done-event clobbers the new crawl's UI state.
if (!this.stopped) {
this.emit('done', this.db.getSummary());
this.fireWebhook();
}
}
/**
* Post-crawl analysis pipeline shared by spider and list mode. Each step is
* a synchronous SQL pass that can take 1–3 s on a 1M-URL crawl; routing DB
* passes through `runDbPass` / `recomputeIssues` dispatches them to the
* writer-worker in the desktop host (CLI / tests fall back to inline), and
* `yieldToEventLoop` between steps lets queued IPC (logs, progress,
* dataChanged) get serviced so no window freezes. List mode used to run
* these inline on the main thread and skipped boilerplate + manifest probes
* — see rule 1.7 (root-cause perf, never mask by degrading UX).
*/
private async runPostCrawlPasses(): Promise<void> {
if (this.config.analyseInlinks) {
await yieldToEventLoop();
this.emit('info', 'Recomputing inlinks…');
@@ -762,12 +809,10 @@ export class Crawler extends EventEmitter {
// the crawler instance, not the DB; can't blindly off-thread it.
this.runDuplicateClustering();
// V2 Faz 14 #5 — Template / boilerplate detection. Runs alongside
// duplicate clustering because both are content-similarity passes;
// this one specifically surfaces the repeated chrome (nav, footer,
// sidebar) that bleeds into every page and dilutes thin-content
// detection. Memory-bounded (samples ~2K bodies), skipped when
// no body snapshots are stored.
// Template / boilerplate detection — runs alongside duplicate
// clustering because both are content-similarity passes; surfaces the
// repeated chrome (nav, footer, sidebar) that dilutes thin-content
// detection. Memory-bounded, skipped when no body snapshots are stored.
await yieldToEventLoop();
this.emit('info', 'Detecting boilerplate coverage…');
this.setOp('post-crawl:boilerplate-coverage');
@@ -801,10 +846,7 @@ export class Crawler extends EventEmitter {
// Route through the injected hook — in the desktop host this
// dispatches to the writer-worker, keeping the 70+ correlated
// subqueries off the main thread. CLI / tests fall back to
// running inline via `db.recomputeUrlsIssuesYielding`. Without
// offloading, a 5K-URL crawl freezes the main process for
// ~45 s in post-crawl, tripping Electron's "Not Responding"
// dialog.
// running inline via `db.recomputeUrlsIssuesYielding`.
await this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS);
}
await yieldToEventLoop();
@@ -825,20 +867,6 @@ export class Crawler extends EventEmitter {
await yieldToEventLoop();
this.setOp('post-crawl:performance-budget');
this.runBudgetPass();
this.running = false;
this.stopMemoryMonitor();
// Release per-URL dedup sets — at 1M URLs this is ~80–120 MB of string
// heap that's no longer needed once the queue is drained.
this.seen.clear();
this.externalSeen.clear();
this.emitProgress();
this.setOp('idle');
// Suppress 'done' if a stop() ran during teardown — otherwise the
// zombie crawler's done-event clobbers the new crawl's UI state.
if (!this.stopped) {
this.emit('done', this.db.getSummary());
this.fireWebhook();
}
}
/**
@@ -1447,47 +1475,11 @@ export class Crawler extends EventEmitter {
this.stopQueueCheckpointTimer();
}
// Same yielding strategy as spider mode — see the comment block
// above the spider-mode recompute. SQL aggregates blocking the JS
// thread starve IPC dispatch and freeze every other window.
// Toggles match the spider-mode gates so a single config controls
// both modes' post-crawl pipeline.
if (this.config.analyseInlinks) {
await yieldToEventLoop();
this.db.recomputeInlinks();
}
if (this.config.analyseRedirectChains) {
await yieldToEventLoop();
this.db.recomputeRedirectChains();
}
if (this.config.analyseHreflang) {
await yieldToEventLoop();
this.db.recomputeHreflangAnalysis();
await yieldToEventLoop();
this.db.recomputeHreflangInconsistent();
}
if (this.config.analyseDuplicates) {
await yieldToEventLoop();
this.runDuplicateClustering();
}
if (this.config.analysePagination) {
await yieldToEventLoop();
this.db.recomputePaginationSequence();
}
if (this.config.analyseIssues) {
await yieldToEventLoop();
this.db.recomputeUrlsIssues(EXPENSIVE_ISSUE_DEFINITIONS);
}
await yieldToEventLoop();
await this.runImageSizeProbes();
await yieldToEventLoop();
await this.runTlsCertProbes();
await yieldToEventLoop();
await this.runSocialImageProbes();
await yieldToEventLoop();
await this.runPdfMetadataProbes();
await yieldToEventLoop();
this.runBudgetPass();
// Identical post-crawl pipeline to spider mode — routed through the
// writer-worker hooks (runDbPass / recomputeIssues) instead of blocking
// SQL on the main thread, and now includes boilerplate coverage +
// manifest probes that list mode previously skipped. See rule 1.7.
await this.runPostCrawlPasses();
this.running = false;
this.stopMemoryMonitor();
this.seen.clear();
@@ -2129,6 +2121,23 @@ export class Crawler extends EventEmitter {
}
}
/**
* Token gate for the global requests-per-second cap (`config.maxRps`).
* Each caller reserves the next dispatch slot on a monotonic clock and
* sleeps until it arrives; the read-compute-write of `nextDispatchAt` is
* synchronous (no await between), so concurrent workers get distinct,
* evenly-spaced slots. `maxRps <= 0` disables the gate (unlimited).
*/
private async acquireRateSlot(): Promise<void> {
if (this.config.maxRps <= 0) return;
const minInterval = 1000 / this.config.maxRps;
const now = Date.now();
const scheduled = Math.max(now, this.nextDispatchAt);
this.nextDispatchAt = scheduled + minInterval;
const waitMs = scheduled - now;
if (waitMs > 0 && !this.stopped) await sleep(waitMs);
}
private async fetchAndProcess(item: QueueItem): Promise<void> {
if (this.stopped) return;
this.setOp(`crawl:fetch:${item.url}`);
@@ -2143,6 +2152,14 @@ export class Crawler extends EventEmitter {
// and progress event listeners run between URLs.
await new Promise<void>((r) => setImmediate(r));
// Global requests-per-second cap. Gates the DISPATCH time (before the
// timeout starts and before responseTime is measured) rather than using
// p-queue's intervalCap, which bucket-refills in bursts and can freeze
// the queue between windows (see the constructor's queue comment). Waits
// are bounded by ~concurrency / maxRps, so Stop stays responsive.
await this.acquireRateSlot();
if (this.stopped) return;
const t0 = Date.now();
const controller = new AbortController();
this.inFlightFetchControllers.add(controller);
@@ -2525,8 +2542,33 @@ export class Crawler extends EventEmitter {
return;
}
const body = await res.text();
const bodyLength = parseIntSafe(contentLengthHeader) ?? Buffer.byteLength(body, 'utf8');
const capRead = await readBodyCapped(res, this.config.maxFileSizeBytes);
if (capRead.overCap) {
// Chunked/streaming HTML with no Content-Length blew past the cap.
// Mirror the declared-length skip branch: keep the row (so inlinks
// survive) but drop the body and downstream parsing.
this.failed++;
this.emit(
'warn',
`Skipped ${item.url}: streamed body exceeded maxFileSizeBytes ${this.config.maxFileSizeBytes}`,
);
await this.dbCall<number>('upsertUrl', [
{
url: item.url,
urlMalformed: isUrlMalformed(item.url) ? 1 : 0,
contentKind: 'other',
statusCode,
statusText: 'size-cap-exceeded',
indexability: 'non-indexable:client-error',
indexabilityReason: `Body skipped — streamed size exceeds maxFileSizeBytes ${this.config.maxFileSizeBytes}`,
responseTimeMs,
depth: item.depth,
},
]);
return;
}
const body = decodeBody(capRead.bytes, contentType);
const bodyLength = parseIntSafe(contentLengthHeader) ?? capRead.bytes.length;
// V2 Faz 1 — JavaScript rendering pass. When `renderingMode === 'js'`
// and the host injected a Playwright renderer, fetch the post-JS DOM
@@ -2984,9 +3026,14 @@ export class Crawler extends EventEmitter {
if (respTimeTimer) clearTimeout(respTimeTimer);
// Politeness delay — applied per worker *after* each request so a
// higher concurrency still honours a "one request every N ms per slot"
// contract on top of the global RPS cap.
if (this.config.crawlDelayMs > 0 && !this.stopped) {
await sleep(this.config.crawlDelayMs);
// contract on top of the global RPS cap. Takes the larger of the
// configured delay and any robots.txt Crawl-delay for the origin.
const politenessMs = Math.max(
this.config.crawlDelayMs,
this.config.respectRobotsTxt ? this.robotsCrawlDelayMs : 0,
);
if (politenessMs > 0 && !this.stopped) {
await sleep(politenessMs);
}
}
}
@@ -3559,6 +3606,94 @@ function parseIntSafe(v: string | null | undefined): number | null {
return Number.isFinite(n) ? n : null;
}
/**
* Read a fetch Response body as raw bytes with a hard byte cap. undici streams
* the DECODED body (content-encoding already stripped), so this bounds the
* actual memory a single page can consume. Content-Length is only advisory and
* is absent on chunked / streaming responses — without this cap the body
* buffers unbounded and a mislabeled multi-hundred-MB `text/html` endpoint OOMs
* the process. Returns `{ overCap: true }` once the accumulated bytes exceed
* `capBytes` (after cancelling the stream); otherwise the raw bytes, which the
* caller decodes with the page's actual charset (NOT forced UTF-8).
*/
async function readBodyCapped(
res: Response,
capBytes: number,
): Promise<{ overCap: false; bytes: Buffer } | { overCap: true; bytes: null }> {
if (capBytes <= 0 || !res.body) {
return { overCap: false, bytes: Buffer.from(await res.arrayBuffer()) };
}
const reader = (res.body as ReadableStream<Uint8Array>).getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
if (!value) continue;
total += value.byteLength;
if (total > capBytes) {
try {
await reader.cancel();
} catch {
/* ignore */
}
return { overCap: true, bytes: null };
}
chunks.push(value);
}
} finally {
try {
reader.releaseLock();
} catch {
/* ignore */
}
}
return { overCap: false, bytes: Buffer.concat(chunks) };
}
/**
* Detect the character encoding of an HTML/text body. undici (per the fetch
* spec) always decodes bodies as UTF-8, which mojibakes legacy pages served as
* ISO-8859-9 / windows-1254 (common on older Turkish sites), corrupting
* titles, meta, word counts and duplicate fingerprints. Precedence follows the
* HTML spec: Content-Type header charset → BOM → `<meta charset>` in the head.
* Returns null when no signal is found (caller defaults to UTF-8).
*/
function detectCharset(bytes: Buffer, contentType: string | null): string | null {
if (contentType) {
const m = /charset\s*=\s*["']?\s*([^"';,\s]+)/i.exec(contentType);
if (m && m[1]) return m[1].toLowerCase();
}
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return 'utf-8';
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return 'utf-16le';
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return 'utf-16be';
// Scan the first 2 KB as latin1 (ASCII-safe) for a <meta charset> declaration.
const head = bytes.subarray(0, Math.min(bytes.length, 2048)).toString('latin1');
const metaCharset = /<meta[^>]+charset\s*=\s*["']?\s*([a-z0-9_-]+)/i.exec(head);
if (metaCharset && metaCharset[1]) return metaCharset[1].toLowerCase();
const metaHttp = /<meta[^>]+content\s*=\s*["'][^"']*charset\s*=\s*([a-z0-9_-]+)/i.exec(head);
if (metaHttp && metaHttp[1]) return metaHttp[1].toLowerCase();
return null;
}
/**
* Decode a body buffer using its detected charset. Falls back to UTF-8 when no
* charset is declared or the label is unknown to the platform's TextDecoder
* (Electron bundles full ICU, so legacy labels like `windows-1254` resolve).
*/
function decodeBody(bytes: Buffer, contentType: string | null): string {
const charset = detectCharset(bytes, contentType);
if (!charset || charset === 'utf-8' || charset === 'utf8') {
return bytes.toString('utf8');
}
try {
return new TextDecoder(charset).decode(bytes);
} catch {
return bytes.toString('utf8');
}
}
/**
* Extract the URL of the first `rel="canonical"` entry from an RFC 8288
* `Link:` HTTP header. Returns null when the header is absent or contains
+1 -1
View File
@@ -800,7 +800,7 @@ export function parseHtml(
.first()
.attr('content') ?? ''
).trim();
let metaRefresh: string | null = metaRefreshRaw || null;
const metaRefresh: string | null = metaRefreshRaw || null;
let metaRefreshUrl: string | null = null;
if (metaRefresh) {
// Format is `<seconds>[; url=<URL>]`. Parameters are case-insensitive
Binary file not shown.
+1 -1
View File
@@ -258,7 +258,7 @@ export function validateRobotsTxt(text: string): RobotsValidationIssue[] {
const lineNo = i + 1;
const raw = lines[i] ?? '';
// Strip BOM on first line — some Windows editors prepend U+FEFF.
const noBom = i === 0 ? raw.replace(/^/, '') : raw;
const noBom = i === 0 ? raw.replace(/^\uFEFF/, '') : raw;
// Strip trailing CR (handled by split, but defensive) + comments.
// Comments start at `#` and consume the rest of the line (including
// `#` mid-line); per RFC 9309 §2.2 leading whitespace is allowed.
+4 -3
View File
@@ -197,7 +197,7 @@ async function writeXmlFile(
}
function sanitizeFilename(label: string): string {
return label.replace(/[\\/?*\[\]:<>|"]/g, '_').replace(/\s+/g, '-').toLowerCase();
return label.replace(/[\\/?*[\]:<>|"]/g, '_').replace(/\s+/g, '-').toLowerCase();
}
function sanitizeSubdir(subdir: string): string {
@@ -206,7 +206,7 @@ function sanitizeSubdir(subdir: string): string {
// filename component.
return subdir
.split(/[\\/]+/)
.map((seg) => seg.replace(/[\\/?*\[\]:<>|"]/g, '_').replace(/\s+/g, '-').toLowerCase())
.map((seg) => seg.replace(/[\\/?*[\]:<>|"]/g, '_').replace(/\s+/g, '-').toLowerCase())
.filter((seg) => seg.length > 0)
.join(path.sep);
}
@@ -232,6 +232,7 @@ function xmlEscape(str: string): string {
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
// strip control chars XML 1.0 forbids
// eslint-disable-next-line no-control-regex -- intentionally matching control chars to strip them
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '');
}
@@ -249,7 +250,7 @@ function colLetter(idx: number): string {
}
function sanitizeSheetName(label: string, used: Set<string>): string {
let base = label.replace(/[\\/?*\[\]:]/g, ' ').trim().slice(0, 31);
let base = label.replace(/[\\/?*[\]:]/g, ' ').trim().slice(0, 31);
if (!base) base = 'Sheet';
let candidate = base;
let i = 2;
+89 -10
View File
@@ -71,7 +71,16 @@ export function normalizeUrl(
rewrites: UrlRewriteOptions = {},
): string | null {
try {
const u = new URL(raw, base);
// HTML/XML attribute values escape `&` as an entity, so an href like
// `/p?a=1&amp;b=2` arrives with a literal `amp;`. Decode the ampersand
// entity family up-front so the query splits on the real `&` — otherwise
// `new URL` keeps `amp;b=2` as a bogus parameter, the crawler fetches the
// wrong page (or 404s) and stores a duplicate row for the escaped variant.
// Safe for already-decoded callers: a real URL never carries `&amp;` /
// `&#38;` / `&#x26;` as literal data, and the trailing `;` requirement
// means genuine separators like `?a=1&b=2` are left untouched.
const decoded = raw.replace(/&(?:amp|#0*38|#x0*26);/gi, '&');
const u = new URL(decoded, base);
u.hash = '';
const keep = rewrites.keepQueryParams;
@@ -130,6 +139,67 @@ export function normalizeUrl(
}
}
/**
* Two-label public suffixes where the registrable domain is the last THREE
* labels (e.g. `example.co.uk`, `site.com.tr`) rather than the last two.
* A curated subset of the Mozilla Public Suffix List covering the common
* ccTLD second levels — the full PSL is megabytes and unnecessary for scope
* matching. Without this, `foo.com.tr` and `bar.com.tr` collapse to the same
* registrable domain (`com.tr`), so `all-subdomains` scope escapes onto every
* unrelated site on the ccTLD.
*/
const MULTI_LEVEL_SUFFIXES: ReadonlySet<string> = new Set([
// United Kingdom
'co.uk', 'org.uk', 'me.uk', 'ltd.uk', 'plc.uk', 'net.uk', 'sch.uk', 'ac.uk', 'gov.uk', 'nhs.uk',
// Turkey
'com.tr', 'net.tr', 'org.tr', 'gov.tr', 'edu.tr', 'k12.tr', 'av.tr', 'bel.tr', 'biz.tr', 'gen.tr',
'info.tr', 'name.tr', 'web.tr', 'tv.tr',
// Australia / New Zealand
'com.au', 'net.au', 'org.au', 'edu.au', 'gov.au', 'asn.au', 'id.au',
'co.nz', 'net.nz', 'org.nz', 'govt.nz', 'ac.nz', 'school.nz',
// Japan / Korea / China / Hong Kong
'co.jp', 'or.jp', 'ne.jp', 'ac.jp', 'go.jp', 'gr.jp',
'co.kr', 'or.kr', 'ne.kr', 're.kr', 'go.kr', 'ac.kr',
'com.cn', 'net.cn', 'org.cn', 'gov.cn', 'edu.cn',
'com.hk', 'org.hk', 'net.hk', 'gov.hk', 'edu.hk',
// Brazil / Mexico / Argentina / Colombia
'com.br', 'net.br', 'org.br', 'gov.br', 'edu.br',
'com.mx', 'net.mx', 'org.mx', 'gob.mx', 'edu.mx',
'com.ar', 'net.ar', 'org.ar', 'gob.ar', 'edu.ar',
'com.co', 'net.co', 'org.co', 'gov.co', 'edu.co',
// India / Singapore / Malaysia / Thailand / Vietnam / Indonesia / Philippines
'co.in', 'net.in', 'org.in', 'gen.in', 'firm.in', 'ind.in', 'ac.in', 'edu.in', 'gov.in', 'res.in',
'com.sg', 'net.sg', 'org.sg', 'gov.sg', 'edu.sg',
'com.my', 'net.my', 'org.my', 'gov.my', 'edu.my',
'co.th', 'in.th', 'ac.th', 'go.th', 'or.th',
'com.vn', 'net.vn', 'org.vn', 'gov.vn', 'edu.vn',
'co.id', 'or.id', 'go.id', 'ac.id', 'web.id',
'com.ph', 'net.ph', 'org.ph', 'gov.ph', 'edu.ph',
// South Africa / Israel / Russia / Ukraine / Spain / Poland
'co.za', 'org.za', 'net.za', 'gov.za', 'ac.za',
'co.il', 'org.il', 'net.il', 'gov.il', 'ac.il',
'com.ru', 'net.ru', 'org.ru',
'com.ua', 'net.ua', 'org.ua',
'com.es', 'org.es', 'gob.es', 'edu.es', 'nom.es',
'com.pl', 'net.pl', 'org.pl', 'gov.pl', 'edu.pl',
]);
/**
* Registrable ("root") domain of a hostname, honouring the multi-level
* public-suffix table above. `www.example.com` → `example.com`;
* `shop.example.co.uk` → `example.co.uk`.
*/
export function registrableDomain(hostname: string): string {
const host = hostname.toLowerCase().replace(/\.$/, '');
const labels = host.split('.');
if (labels.length <= 2) return host;
const lastTwo = labels.slice(-2).join('.');
if (MULTI_LEVEL_SUFFIXES.has(lastTwo) && labels.length >= 3) {
return labels.slice(-3).join('.');
}
return lastTwo;
}
export function isSameHost(
urlA: string,
urlB: string,
@@ -139,8 +209,7 @@ export function isSameHost(
const a = new URL(urlA);
const b = new URL(urlB);
if (opts.includeSubdomains) {
const root = (h: string) => h.split('.').slice(-2).join('.');
return root(a.hostname) === root(b.hostname);
return registrableDomain(a.hostname) === registrableDomain(b.hostname);
}
if (a.hostname === b.hostname) return true;
// CDN list — either side matching a configured CDN host counts as
@@ -197,8 +266,7 @@ export function isInScope(
return c.pathname === start.pathname || c.pathname.startsWith(prefix);
}
case 'all-subdomains': {
const root = (h: string) => h.split('.').slice(-2).join('.');
return root(start.hostname) === root(c.hostname);
return registrableDomain(start.hostname) === registrableDomain(c.hostname);
}
default:
return false;
@@ -281,6 +349,7 @@ export async function resolveStartUrl(
userAgent = 'FreeCrawlSEO/0.1',
probeTimeoutMs = 3000,
onAttempt?: (a: ResolveStartUrlAttempt) => void,
opts: { proxyOverride?: string; rewrites?: UrlRewriteOptions } = {},
): Promise<string | null> {
const trimmed = raw.trim();
if (!trimmed) return null;
@@ -304,7 +373,11 @@ export async function resolveStartUrl(
async function resolveVia(url: string): Promise<string | null> {
const { fetch: undiciFetch } = await import('undici');
const { initHttpClient, formatFetchError } = await import('./http-client.js');
initHttpClient();
// Preserve the crawler's configured proxy. Calling initHttpClient() with
// no argument recomputes the effective proxy from env vars only, which
// silently rebuilds the global dispatcher as direct and drops a
// Settings/profile proxy for the rest of the crawl.
initHttpClient({ proxyOverride: opts.proxyOverride });
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), probeTimeoutMs);
try {
@@ -372,23 +445,29 @@ export async function resolveStartUrl(
}
}
// The start URL must be normalized with the SAME rewrite rules the
// crawler applies to discovered links; otherwise (e.g. stripWww on a
// www. site) the scope anchor and every in-scope link disagree and the
// crawl stops after one page.
const rewrites = opts.rewrites ?? {};
if (/^https?:\/\//i.test(trimmed)) {
const resolved = (await resolveVia(trimmed)) ?? trimmed;
return normalizeUrl(resolved);
return normalizeUrl(resolved, undefined, rewrites);
}
const bare = trimmed.replace(/^\/\//, '').replace(/^\/+/, '');
const httpsUrl = `https://${bare}`;
const viaHttps = await resolveVia(httpsUrl);
if (viaHttps) return normalizeUrl(viaHttps);
if (viaHttps) return normalizeUrl(viaHttps, undefined, rewrites);
const httpUrl = `http://${bare}`;
const viaHttp = await resolveVia(httpUrl);
if (viaHttp) return normalizeUrl(viaHttp);
if (viaHttp) return normalizeUrl(viaHttp, undefined, rewrites);
// Neither protocol responded — return the secure candidate anyway;
// the crawler will surface a network error with that URL.
return normalizeUrl(httpsUrl);
return normalizeUrl(httpsUrl, undefined, rewrites);
}
/**
+1
View File
@@ -175,6 +175,7 @@ function xmlEscape(raw: string): string {
.replace(/'/g, '&apos;')
// XML 1.0 forbids most C0 control characters; strip everything below
// 0x20 except tab (0x09), LF (0x0A), and CR (0x0D).
// eslint-disable-next-line no-control-regex -- intentionally matching control chars to strip them
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '');
}
+41 -13
View File
@@ -1063,6 +1063,19 @@ export class ProjectDb {
// DELETEs are index-backed) so the page's link + image set
// always reflects the latest fetch.
this.db.prepare('DELETE FROM links WHERE from_url_id = ?').run(urlId);
// `images.occurrences` is a running tally of how many pages use each
// image. Deleting this page's usage rows must first decrement that
// tally for the affected images, otherwise the re-insert via
// `insertImages` (which does `occurrences + 1`) drifts the count
// upward without bound on every re-crawl / Re-Spider / resume.
// Decrement-then-reinsert nets zero for images still on the page and
// correctly drops images that vanished from it.
this.db
.prepare(
`UPDATE images SET occurrences = occurrences - 1
WHERE id IN (SELECT image_id FROM image_usages WHERE from_url_id = ?)`,
)
.run(urlId);
this.db
.prepare('DELETE FROM image_usages WHERE from_url_id = ?')
.run(urlId);
@@ -1098,19 +1111,31 @@ export class ProjectDb {
* not yet present in the urls table (never crawled). Returns the minimum
* depth at which each pending URL was discovered.
*/
/** url-keyed audit tables with no FK to `urls` — must be wiped explicitly
* on delete (ON DELETE CASCADE doesn't reach them). Kept in sync with
* `deleteByDomain`, which clears the same six. */
private static readonly URL_KEYED_AUDIT_TABLES = [
'pagespeed_results',
'gsc_results',
'ga4_results',
'ai_results',
'seo_results',
'gsc_inspection_results',
] as const;
deleteUrl(id: number): void {
// `urls_issues` and `pagespeed_results` have no FK to `urls`, so the
// `ON DELETE CASCADE` that cleans links/headers/images doesn't reach
// them — wipe them explicitly or a deleted URL leaves orphan rows
// that inflate `getIssueCounts()` and re-attach stale audit data on
// re-crawl. The `pagespeed_results` delete must run before the parent
// `urls` row vanishes (it resolves the URL string from `urls`).
// `urls_issues` and the url-keyed audit tables have no FK to `urls`, so
// the `ON DELETE CASCADE` that cleans links/headers/images doesn't reach
// them — wipe them explicitly or a deleted URL leaves orphan rows that
// inflate `getIssueCounts()`, re-attach stale audit data on re-crawl, and
// resurface the URL as a GSC/GA4 cross-source "orphan". These must run
// before the parent `urls` row vanishes (they resolve the URL from it).
this.runInTransaction(() => {
for (const table of ProjectDb.URL_KEYED_AUDIT_TABLES) {
this.db
.prepare(
'DELETE FROM pagespeed_results WHERE url = (SELECT url FROM urls WHERE id = ?)',
)
.prepare(`DELETE FROM ${table} WHERE url = (SELECT url FROM urls WHERE id = ?)`)
.run(id);
}
this.db.prepare('DELETE FROM urls_issues WHERE url_id = ?').run(id);
this.db.prepare('DELETE FROM urls WHERE id = ?').run(id);
});
@@ -1154,20 +1179,23 @@ export class ProjectDb {
deleteUrls(ids: number[]): void {
if (ids.length === 0) return;
// Chunk to stay under SQLite's bound-parameter limit; wrap in one
// transaction so the urls + urls_issues + pagespeed_results deletes
// are atomic. `urls_issues` / `pagespeed_results` have no FK to
// `urls` (see `deleteUrl`), so they're wiped explicitly.
// transaction so the urls + urls_issues + audit-table deletes are atomic.
// `urls_issues` and the url-keyed audit tables have no FK to `urls`
// (see `deleteUrl`), so they're wiped explicitly — same six tables as
// `deleteByDomain`, else deleted URLs leave orphan analytics/AI rows.
const CHUNK = 500;
this.runInTransaction(() => {
for (let i = 0; i < ids.length; i += CHUNK) {
const slice = ids.slice(i, i + CHUNK);
const placeholders = slice.map(() => '?').join(',');
for (const table of ProjectDb.URL_KEYED_AUDIT_TABLES) {
this.db
.prepare(
`DELETE FROM pagespeed_results
`DELETE FROM ${table}
WHERE url IN (SELECT url FROM urls WHERE id IN (${placeholders}))`,
)
.run(...slice);
}
this.db
.prepare(`DELETE FROM urls_issues WHERE url_id IN (${placeholders})`)
.run(...slice);