mirror of
https://github.com/kemalai/FreeCrawl-SEO-Tool.git
synced 2026-10-18 00:39:38 +02:00
feat: PageSpeed Insights + crawl stability fixes
This commit is contained in:
kemalai committed
1 parent
902cf98fa5
commit
8214718e9e
33 files changed
+1732
-103
No files matched your search
@@ -43,7 +43,7 @@ A high-performance website crawler built for serious SEO audits. Targets <kbd>1M
|
||||
<td width="50%" valign="top">
|
||||
|
||||
### 🔍 Analysis
|
||||
- **150+ SEO issue checks** across 24 top-level tabs
|
||||
- **150+ SEO issue checks** across 25 top-level tabs
|
||||
- Near-duplicate clustering (SimHash + LSH) + exact-duplicate content hash
|
||||
- Full hreflang validation (reciprocity, self-ref, inconsistent lang)
|
||||
- OpenGraph / Twitter Card / JSON-LD / AMP / **Web App Manifest** parsing
|
||||
@@ -53,6 +53,7 @@ A high-performance website crawler built for serious SEO audits. Targets <kbd>1M
|
||||
- **Active vs passive mixed content** split
|
||||
- Readability (Flesch, Flesch–Kincaid, Gunning Fog)
|
||||
- Custom CSS + Regex extraction (10 rules) + `/regex/` literal Custom Search
|
||||
- **PageSpeed Insights** — on-demand Lighthouse audits (mobile + desktop) per URL
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
@@ -439,9 +440,9 @@ graph LR
|
||||
## 📈 Status
|
||||
|
||||
> [!NOTE]
|
||||
> **Active development.** All 24 analysis tabs, advanced search, 150+ issue categories, sitemap export variants, JSON / CSV / XLSX / HTML reports, list mode, custom extraction, near-duplicate + exact-duplicate detection, hreflang validation, project compare, Cytoscape visualization, auth, proxy, webhook, **MCP server with crawl control + live progress**, OS notifications, **robots.txt syntax validator**, URL rewriting + preview, **status-code diagnosis banner**, **live memory monitor**, **in-app scheduled crawl**, **multi-language UI (EN + TR)**, **`.seoproject` file association**, in-app logs, and diagnostic popups are working. Cross-platform installers (Windows `.exe` + portable, macOS `.dmg`, Linux `.AppImage` / `.deb` / `.rpm`). Live-streaming UX with **first row in ~1 s**.
|
||||
> **Active development.** All 25 analysis tabs, advanced search, 150+ issue categories, sitemap export variants, JSON / CSV / XLSX / HTML reports, list mode, custom extraction, near-duplicate + exact-duplicate detection, hreflang validation, project compare, Cytoscape visualization, auth, proxy, webhook, **MCP server with crawl control + live progress**, **Google PageSpeed Insights integration**, OS notifications, **robots.txt syntax validator**, URL rewriting + preview, **status-code diagnosis banner**, **live memory monitor**, **in-app scheduled crawl**, **multi-language UI (EN + TR)**, **`.seoproject` file association**, in-app logs, and diagnostic popups are working. Cross-platform installers (Windows `.exe` + portable, macOS `.dmg`, Linux `.AppImage` / `.deb` / `.rpm`). Live-streaming UX with **first row in ~1 s**.
|
||||
>
|
||||
> **Upcoming:** plugin system, JavaScript rendering, log analyzer, PageSpeed API integration.
|
||||
> **Upcoming:** plugin system, JavaScript rendering, log analyzer.
|
||||
|
||||
<br />
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/cli",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/desktop",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"private": true,
|
||||
"description": "FreeCrawl SEO — desktop application",
|
||||
"homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
|
||||
|
||||
@@ -69,6 +69,7 @@ const ALLOWED_METHODS = new Set<string>([
|
||||
'queryUrls',
|
||||
'queryImages',
|
||||
'queryBrokenLinks',
|
||||
'queryPagespeed',
|
||||
'getOverviewCounts',
|
||||
'getOverviewCountsAsync',
|
||||
'getSummary',
|
||||
|
||||
@@ -88,6 +88,11 @@ import {
|
||||
type UrlCertInfoResult,
|
||||
type UrlsQueryInput,
|
||||
type UrlsQueryResult,
|
||||
type PagespeedQueryInput,
|
||||
type PagespeedQueryResult,
|
||||
type PagespeedRunInput,
|
||||
type PagespeedRunResult,
|
||||
type PagespeedStrategy,
|
||||
} from '@freecrawl/shared-types';
|
||||
import {
|
||||
Crawler,
|
||||
@@ -114,7 +119,9 @@ import {
|
||||
getState as getIntegrationsState,
|
||||
setCredentials as setIntegrationCredentials,
|
||||
clearCredentials as clearIntegrationCredentials,
|
||||
resolveCredentials,
|
||||
} from './credentials.js';
|
||||
import { runPagespeedBatch, type PagespeedBatchItem } from './pagespeed.js';
|
||||
import * as logger from './logger.js';
|
||||
import { dbReaderPool, callReaderOrFallback } from './db-reader-pool.js';
|
||||
import { dbWriterPool } from './db-writer-pool.js';
|
||||
@@ -141,6 +148,10 @@ let mainWindow: BrowserWindow | null = null;
|
||||
let logsWindow: BrowserWindow | null = null;
|
||||
let db: ProjectDb | null = null;
|
||||
let activeCrawler: Crawler | null = null;
|
||||
/** Faz 7 — PageSpeed run guard. Only one PSI batch may run at a time;
|
||||
* `pagespeedCancelRequested` is the cooperative cancel flag. */
|
||||
let pagespeedRunActive = false;
|
||||
let pagespeedCancelRequested = false;
|
||||
/** Most-recent CrawlConfig — needed by the crash-recovery resume
|
||||
* flow so we can re-create the Crawler with the same knobs the user
|
||||
* had set, without re-prompting. Hydrated lazily from `project_meta`
|
||||
@@ -213,6 +224,11 @@ function flushPrefs(): void {
|
||||
clearTimeout(prefsWriteTimer);
|
||||
prefsWriteTimer = null;
|
||||
}
|
||||
// Never write a cache that was never loaded — `prefsCache` is still
|
||||
// the empty default. This guards the rejected second instance: its
|
||||
// `before-quit` → `performShutdown` runs `flushPrefs`, and writing
|
||||
// `{}` here would clobber the running first instance's saved prefs.
|
||||
if (!prefsLoaded) return;
|
||||
try {
|
||||
writeFileSync(prefsFilePath(), JSON.stringify(prefsCache, null, 2), 'utf8');
|
||||
} catch {
|
||||
@@ -872,7 +888,18 @@ function pickInstallerAsset(assets: GitHubReleaseAsset[]): GitHubReleaseAsset |
|
||||
(a) => a && typeof a.name === 'string' && !a.name.endsWith('.blockmap'),
|
||||
);
|
||||
if (process.platform === 'win32') {
|
||||
return list.find((a) => /\.exe$/i.test(a.name)) ?? null;
|
||||
// The Windows release ships two `.exe` assets — the NSIS installer
|
||||
// (`…Setup….exe`) and the standalone portable build
|
||||
// (`…-portable.exe`). "Check for Updates" must offer the installer:
|
||||
// prefer the `Setup` asset, fall back to any non-portable `.exe`,
|
||||
// and only as a last resort take whatever `.exe` exists.
|
||||
const exes = list.filter((a) => /\.exe$/i.test(a.name));
|
||||
return (
|
||||
exes.find((a) => /setup/i.test(a.name)) ??
|
||||
exes.find((a) => !/portable/i.test(a.name)) ??
|
||||
exes[0] ??
|
||||
null
|
||||
);
|
||||
}
|
||||
if (process.platform === 'darwin') {
|
||||
const isArm = process.arch === 'arm64';
|
||||
@@ -1276,6 +1303,13 @@ function createWindow(): void {
|
||||
sandbox: false,
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
// Keep the renderer's frame loop at full rate even when the window
|
||||
// loses focus — otherwise focusing the Logs window throttles this
|
||||
// window's requestAnimationFrame, dropping its FPS and starving
|
||||
// the freeze-watchdog's renderer heartbeat (it then logs a false
|
||||
// STALL:RENDERER). The Logs window already sets this; the main
|
||||
// window needs it for the same reason.
|
||||
backgroundThrottling: false,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1859,6 +1893,117 @@ function registerIpc(): void {
|
||||
return clearIntegrationCredentials(id);
|
||||
});
|
||||
|
||||
// V1 Faz 7 — Google PageSpeed Insights. `pagespeedQuery` lists the
|
||||
// crawled internal HTML pages joined with any stored audit results;
|
||||
// `pagespeedRun` audits a user-selected subset (the slow part — PSI
|
||||
// takes ~10–30 s per URL — so it streams `pagespeedProgress` events
|
||||
// and can be cancelled mid-run).
|
||||
ipcMain.handle(
|
||||
IPC.pagespeedQuery,
|
||||
async (_e, input: PagespeedQueryInput): Promise<PagespeedQueryResult> => {
|
||||
const params = {
|
||||
limit: input.limit,
|
||||
offset: input.offset,
|
||||
search: input.search,
|
||||
filter: input.filter,
|
||||
};
|
||||
return callReaderOrFallback<PagespeedQueryResult>(
|
||||
'queryPagespeed',
|
||||
[params],
|
||||
() => getDb().queryPagespeed(params),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
ipcMain.handle(
|
||||
IPC.pagespeedRun,
|
||||
async (_e, input: PagespeedRunInput): Promise<PagespeedRunResult> => {
|
||||
if (pagespeedRunActive) {
|
||||
return { completed: 0, failed: 0, cancelled: true };
|
||||
}
|
||||
const urls = Array.from(
|
||||
new Set(
|
||||
(input.urls ?? []).filter(
|
||||
(u): u is string => typeof u === 'string' && u.length > 0,
|
||||
),
|
||||
),
|
||||
);
|
||||
if (urls.length === 0) {
|
||||
return { completed: 0, failed: 0, cancelled: false };
|
||||
}
|
||||
const strategies: PagespeedStrategy[] =
|
||||
input.strategy === 'both' ? ['mobile', 'desktop'] : [input.strategy];
|
||||
const items: PagespeedBatchItem[] = [];
|
||||
for (const url of urls) {
|
||||
for (const strategy of strategies) items.push({ url, strategy });
|
||||
}
|
||||
const apiKeyRaw = resolveCredentials('pagespeed')['apiKey'];
|
||||
const apiKey =
|
||||
apiKeyRaw && apiKeyRaw.trim().length > 0 ? apiKeyRaw.trim() : undefined;
|
||||
|
||||
pagespeedRunActive = true;
|
||||
pagespeedCancelRequested = false;
|
||||
// Pin writes to the project that was active when the run started —
|
||||
// a PSI run can last minutes, and resolving `getDb()` per result
|
||||
// would leak audits into a different project if the user opens
|
||||
// one mid-run.
|
||||
const runDb = getDb();
|
||||
logger.log(
|
||||
'info',
|
||||
'pagespeed',
|
||||
`run started — ${items.length} audit(s): ${urls.length} URL(s) × ${strategies.length} strategy${
|
||||
apiKey ? ' (keyed)' : ' (keyless — low rate limit)'
|
||||
}`,
|
||||
);
|
||||
try {
|
||||
const result = await runPagespeedBatch({
|
||||
items,
|
||||
apiKey,
|
||||
isCancelled: () => pagespeedCancelRequested,
|
||||
onResult: (url, strategy, metrics) => {
|
||||
try {
|
||||
runDb.upsertPagespeedResult(url, strategy, metrics);
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'error',
|
||||
'pagespeed',
|
||||
`DB write failed for ${url}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
onProgress: (done, total, currentUrl) => {
|
||||
mainWindow?.webContents.send(IPC.pagespeedProgress, {
|
||||
done,
|
||||
total,
|
||||
currentUrl,
|
||||
running: true,
|
||||
});
|
||||
},
|
||||
});
|
||||
mainWindow?.webContents.send(IPC.pagespeedProgress, {
|
||||
done: result.completed + result.failed,
|
||||
total: items.length,
|
||||
currentUrl: null,
|
||||
running: false,
|
||||
});
|
||||
fireDataChanged();
|
||||
return result;
|
||||
} finally {
|
||||
pagespeedRunActive = false;
|
||||
pagespeedCancelRequested = false;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
ipcMain.handle(IPC.pagespeedCancel, () => {
|
||||
if (pagespeedRunActive) {
|
||||
pagespeedCancelRequested = true;
|
||||
logger.log('info', 'pagespeed', 'run cancellation requested');
|
||||
}
|
||||
});
|
||||
|
||||
// Stream new entries to the logs window in coalesced batches. A heavy
|
||||
// crawl emits 100–300 logs/s; sending each as its own IPC message
|
||||
// saturated the renderer's event loop and caused visible UI stutters
|
||||
@@ -3403,7 +3548,15 @@ void app.whenReady().then(async () => {
|
||||
});
|
||||
});
|
||||
|
||||
app.on('window-all-closed', () => {
|
||||
// Full teardown — DB, worker pools, watchdog, MCP bridge, prefs flush.
|
||||
// This lives in `before-quit`, NOT `window-all-closed`: on macOS the app
|
||||
// stays resident after its window closes (standard platform behaviour),
|
||||
// and terminating the worker pools there would make a later reopen from
|
||||
// the dock throw `cannot init after terminate()`. Runs exactly once.
|
||||
let shutdownComplete = false;
|
||||
function performShutdown(): void {
|
||||
if (shutdownComplete) return;
|
||||
shutdownComplete = true;
|
||||
activeCrawler?.stop();
|
||||
stopScheduler();
|
||||
// Tear down the MCP bridge first so any in-flight HTTP calls fail
|
||||
@@ -3428,5 +3581,16 @@ app.on('window-all-closed', () => {
|
||||
// Flush the disk log stream before exit so the last lines aren't lost
|
||||
// if the OS kills the process during a Quit-All cycle.
|
||||
logger.flushFileLogging();
|
||||
}
|
||||
|
||||
app.on('before-quit', () => {
|
||||
performShutdown();
|
||||
});
|
||||
|
||||
app.on('window-all-closed', () => {
|
||||
// Stop any active crawl when the UI goes away, but keep the DB and
|
||||
// worker pools alive so macOS reopen-from-dock works. The full
|
||||
// teardown happens in `before-quit`.
|
||||
activeCrawler?.stop();
|
||||
if (process.platform !== 'darwin') app.quit();
|
||||
});
|
||||
@@ -0,0 +1,229 @@
|
||||
/**
|
||||
* Faz 7 — Google PageSpeed Insights API client.
|
||||
*
|
||||
* PSI runs a Lighthouse audit on a publicly reachable URL and returns
|
||||
* lab performance metrics. It works without a key at a low shared rate
|
||||
* limit; a free Google Cloud API key (the `pagespeed` integration in
|
||||
* the credential store) raises throughput. The key is resolved in the
|
||||
* main process only and never crosses into the renderer.
|
||||
*
|
||||
* Two layers:
|
||||
* - `fetchPagespeedAudit` — one URL + strategy → metrics (or an
|
||||
* `error` result on any failure, so a single bad URL never aborts
|
||||
* a batch).
|
||||
* - `runPagespeedBatch` — a small concurrency pool over many
|
||||
* (url, strategy) items, with progress callbacks and cooperative
|
||||
* cancellation. PSI is slow (~10–30 s per audit), so the renderer
|
||||
* drives this on-demand against a user-selected subset, never the
|
||||
* whole crawl.
|
||||
*/
|
||||
import type { PagespeedMetrics, PagespeedStrategy } from '@freecrawl/shared-types';
|
||||
import * as logger from './logger.js';
|
||||
|
||||
const PSI_ENDPOINT =
|
||||
'https://www.googleapis.com/pagespeedonline/v5/runPagespeed';
|
||||
|
||||
/** A single Lighthouse audit can legitimately take ~30 s; cap at 90 s. */
|
||||
const REQUEST_TIMEOUT_MS = 90_000;
|
||||
|
||||
/** Concurrency: a key lifts the quota enough for modest parallelism;
|
||||
* keyless calls share a tiny global pool and 429 instantly, so they
|
||||
* run strictly one at a time. */
|
||||
const CONCURRENCY_WITH_KEY = 4;
|
||||
const CONCURRENCY_NO_KEY = 1;
|
||||
|
||||
function errorMetrics(fetchedAt: string, message: string): PagespeedMetrics {
|
||||
return {
|
||||
performance: null,
|
||||
lcp: null,
|
||||
cls: null,
|
||||
fcp: null,
|
||||
tbt: null,
|
||||
speedIndex: null,
|
||||
status: 'error',
|
||||
error: message,
|
||||
fetchedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Pull `numericValue` out of a Lighthouse audit object, rounded. */
|
||||
function auditValue(
|
||||
audits: Record<string, unknown>,
|
||||
key: string,
|
||||
decimals = 0,
|
||||
): number | null {
|
||||
const audit = audits[key];
|
||||
if (!audit || typeof audit !== 'object') return null;
|
||||
const v = (audit as { numericValue?: unknown }).numericValue;
|
||||
if (typeof v !== 'number' || !Number.isFinite(v)) return null;
|
||||
const factor = 10 ** decimals;
|
||||
return Math.round(v * factor) / factor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Audit one URL for one form factor. Never throws — any network /
|
||||
* quota / Lighthouse failure resolves to an `error` PagespeedMetrics
|
||||
* so the caller can record it and move on.
|
||||
*/
|
||||
export async function fetchPagespeedAudit(
|
||||
url: string,
|
||||
strategy: PagespeedStrategy,
|
||||
apiKey: string | undefined,
|
||||
): Promise<PagespeedMetrics> {
|
||||
const fetchedAt = new Date().toISOString();
|
||||
const params = new URLSearchParams({
|
||||
url,
|
||||
strategy,
|
||||
category: 'performance',
|
||||
});
|
||||
if (apiKey) params.set('key', apiKey);
|
||||
|
||||
try {
|
||||
const res = await fetch(`${PSI_ENDPOINT}?${params.toString()}`, {
|
||||
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
const json = (await res.json().catch(() => null)) as Record<
|
||||
string,
|
||||
unknown
|
||||
> | null;
|
||||
|
||||
if (!res.ok) {
|
||||
const apiErr = json?.['error'] as { message?: string } | undefined;
|
||||
return errorMetrics(
|
||||
fetchedAt,
|
||||
apiErr?.message ?? `PageSpeed API returned HTTP ${res.status}`,
|
||||
);
|
||||
}
|
||||
if (!json) {
|
||||
return errorMetrics(fetchedAt, 'PageSpeed API returned an empty response');
|
||||
}
|
||||
|
||||
const lhr = json['lighthouseResult'] as Record<string, unknown> | undefined;
|
||||
if (!lhr) {
|
||||
return errorMetrics(fetchedAt, 'No Lighthouse result in PageSpeed response');
|
||||
}
|
||||
const audits = (lhr['audits'] as Record<string, unknown>) ?? {};
|
||||
const categories = lhr['categories'] as
|
||||
| { performance?: { score?: unknown } }
|
||||
| undefined;
|
||||
const score = categories?.performance?.score;
|
||||
|
||||
return {
|
||||
performance:
|
||||
typeof score === 'number' ? Math.round(score * 100) : null,
|
||||
lcp: auditValue(audits, 'largest-contentful-paint'),
|
||||
cls: auditValue(audits, 'cumulative-layout-shift', 3),
|
||||
fcp: auditValue(audits, 'first-contentful-paint'),
|
||||
tbt: auditValue(audits, 'total-blocking-time'),
|
||||
speedIndex: auditValue(audits, 'speed-index'),
|
||||
status: 'ok',
|
||||
error: null,
|
||||
fetchedAt,
|
||||
};
|
||||
} catch (err) {
|
||||
// `AbortSignal.timeout` rejects with a DOMException, which is not
|
||||
// always an `instanceof Error` in Node — sniff `.name` directly.
|
||||
const name = (err as { name?: string } | null)?.name;
|
||||
const message =
|
||||
name === 'TimeoutError' || name === 'AbortError'
|
||||
? `PageSpeed audit timed out after ${REQUEST_TIMEOUT_MS / 1000}s`
|
||||
: err instanceof Error
|
||||
? err.message
|
||||
: String(err);
|
||||
return errorMetrics(fetchedAt, message);
|
||||
}
|
||||
}
|
||||
|
||||
/** One (url, strategy) audit unit of work. */
|
||||
export interface PagespeedBatchItem {
|
||||
url: string;
|
||||
strategy: PagespeedStrategy;
|
||||
}
|
||||
|
||||
export interface PagespeedBatchOptions {
|
||||
items: PagespeedBatchItem[];
|
||||
/** Resolved PSI API key, or undefined for keyless (low-rate) calls. */
|
||||
apiKey: string | undefined;
|
||||
/** Cooperative cancel — checked before each item is dequeued. */
|
||||
isCancelled: () => boolean;
|
||||
/** Persist one completed audit. */
|
||||
onResult: (
|
||||
url: string,
|
||||
strategy: PagespeedStrategy,
|
||||
metrics: PagespeedMetrics,
|
||||
) => void;
|
||||
/** Live progress — `currentUrl` is the URL just about to be audited. */
|
||||
onProgress: (done: number, total: number, currentUrl: string | null) => void;
|
||||
}
|
||||
|
||||
export interface PagespeedBatchResult {
|
||||
completed: number;
|
||||
failed: number;
|
||||
cancelled: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a PageSpeed audit over many items with a small concurrency pool.
|
||||
* Cancellation gates new items only — audits already in flight finish
|
||||
* and are persisted (a completed audit is always worth keeping).
|
||||
*/
|
||||
export async function runPagespeedBatch(
|
||||
opts: PagespeedBatchOptions,
|
||||
): Promise<PagespeedBatchResult> {
|
||||
const { items, apiKey, isCancelled, onResult, onProgress } = opts;
|
||||
const total = items.length;
|
||||
let done = 0;
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
let cursor = 0;
|
||||
|
||||
onProgress(0, total, null);
|
||||
|
||||
const worker = async (): Promise<void> => {
|
||||
for (;;) {
|
||||
if (isCancelled()) return;
|
||||
const index = cursor++;
|
||||
if (index >= total) return;
|
||||
const item = items[index];
|
||||
if (!item) return;
|
||||
onProgress(done, total, item.url);
|
||||
const metrics = await fetchPagespeedAudit(
|
||||
item.url,
|
||||
item.strategy,
|
||||
apiKey,
|
||||
);
|
||||
try {
|
||||
onResult(item.url, item.strategy, metrics);
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'error',
|
||||
'pagespeed',
|
||||
`failed to persist audit for ${item.url}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
done++;
|
||||
if (metrics.status === 'ok') completed++;
|
||||
else failed++;
|
||||
onProgress(done, total, null);
|
||||
}
|
||||
};
|
||||
|
||||
const poolSize = Math.min(
|
||||
apiKey ? CONCURRENCY_WITH_KEY : CONCURRENCY_NO_KEY,
|
||||
Math.max(1, total),
|
||||
);
|
||||
await Promise.all(Array.from({ length: poolSize }, () => worker()));
|
||||
|
||||
const cancelled = isCancelled() && done < total;
|
||||
logger.log(
|
||||
'info',
|
||||
'pagespeed',
|
||||
`batch finished — ${completed} ok, ${failed} failed${
|
||||
cancelled ? ', cancelled early' : ''
|
||||
} (${done}/${total})`,
|
||||
);
|
||||
return { completed, failed, cancelled };
|
||||
}
|
||||
@@ -72,7 +72,7 @@ class ParserPool {
|
||||
size ?? Math.max(2, Math.min(8, cpuCount - 2));
|
||||
for (let i = 0; i < target; i++) {
|
||||
try {
|
||||
this.workers.push(this.spawnWorker(i));
|
||||
this.workers.push(this.spawnWorker());
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'warn',
|
||||
@@ -140,35 +140,50 @@ class ParserPool {
|
||||
|
||||
// ── private ──────────────────────────────────────────────────────
|
||||
|
||||
private spawnWorker(index: number): Worker {
|
||||
private spawnWorker(): Worker {
|
||||
const w = new Worker(WORKER_PATH);
|
||||
w.on('message', (msg: ResponseMessage) => this.handleResponse(msg));
|
||||
w.on('error', (err) => {
|
||||
logger.log('error', 'main', `parser-worker[${index}] error: ${err.message}`);
|
||||
logger.log('error', 'main', `parser-worker error: ${err.message}`);
|
||||
});
|
||||
w.on('exit', (code) => {
|
||||
if (this.terminated) return;
|
||||
if (code === 0) return;
|
||||
// Resolve this worker's CURRENT slot by identity — never trust a
|
||||
// captured index. An init-time spawn failure leaves the array
|
||||
// shorter than the logical worker count, so a captured index
|
||||
// would point at the wrong slot (or past the end).
|
||||
const slot = this.workers.indexOf(w);
|
||||
if (slot < 0) return; // already replaced / removed
|
||||
if (code === 0) {
|
||||
// Clean exit — drop the slot so `parse()` never dispatches to it.
|
||||
this.workers.splice(slot, 1);
|
||||
return;
|
||||
}
|
||||
logger.log(
|
||||
'warn',
|
||||
'main',
|
||||
`parser-worker[${index}] exited (code=${code}) — respawning`,
|
||||
`parser-worker exited (code=${code}) — respawning`,
|
||||
);
|
||||
const respawn = this.tryRespawn(index);
|
||||
if (respawn) this.workers[index] = respawn;
|
||||
const respawn = this.tryRespawn();
|
||||
if (respawn) {
|
||||
this.workers[slot] = respawn;
|
||||
} else {
|
||||
// Respawn failed — remove the dead worker so it is never picked.
|
||||
this.workers.splice(slot, 1);
|
||||
}
|
||||
});
|
||||
return w;
|
||||
}
|
||||
|
||||
private tryRespawn(index: number): Worker | null {
|
||||
private tryRespawn(): Worker | null {
|
||||
if (this.terminated) return null;
|
||||
try {
|
||||
return this.spawnWorker(index);
|
||||
return this.spawnWorker();
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'error',
|
||||
'main',
|
||||
`parser-worker[${index}] respawn failed: ${
|
||||
`parser-worker respawn failed: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
|
||||
@@ -144,17 +144,24 @@ export function computeNextFire(
|
||||
now: number,
|
||||
lastFiredAt: number | null,
|
||||
): number {
|
||||
if (spec.cadence === 'hourly') {
|
||||
const base = lastFiredAt ?? now;
|
||||
return base + 60 * 60 * 1000;
|
||||
}
|
||||
if (spec.cadence === 'custom') {
|
||||
if (spec.cadence === 'hourly' || spec.cadence === 'custom') {
|
||||
const interval =
|
||||
Math.max(MIN_CUSTOM_INTERVAL_MIN, spec.intervalMinutes ?? MIN_CUSTOM_INTERVAL_MIN) *
|
||||
60 *
|
||||
1000;
|
||||
const base = lastFiredAt ?? now;
|
||||
return base + interval;
|
||||
spec.cadence === 'hourly'
|
||||
? 60 * 60 * 1000
|
||||
: Math.max(
|
||||
MIN_CUSTOM_INTERVAL_MIN,
|
||||
spec.intervalMinutes ?? MIN_CUSTOM_INTERVAL_MIN,
|
||||
) *
|
||||
60 *
|
||||
1000;
|
||||
// Advance from the last fire, but never return a time in the past:
|
||||
// if a crawl ran longer than the interval, `base + interval` would
|
||||
// already be behind `now` and the next tick would fire immediately,
|
||||
// back-to-back with no gap. Step forward by whole intervals until
|
||||
// the result is in the future.
|
||||
let next = (lastFiredAt ?? now) + interval;
|
||||
while (next <= now) next += interval;
|
||||
return next;
|
||||
}
|
||||
// Daily / weekly — pin to time-of-day in local timezone. We always
|
||||
// advance by at least 1 minute past `now` to avoid an immediate
|
||||
|
||||
@@ -83,6 +83,11 @@ import {
|
||||
type UrlsQueryInput,
|
||||
type UrlsQueryResult,
|
||||
type MemoryStats,
|
||||
type PagespeedQueryInput,
|
||||
type PagespeedQueryResult,
|
||||
type PagespeedRunInput,
|
||||
type PagespeedRunResult,
|
||||
type PagespeedProgress,
|
||||
} from '@freecrawl/shared-types';
|
||||
|
||||
function subscribe<T>(
|
||||
@@ -257,6 +262,13 @@ const api: FreeCrawlApi = {
|
||||
): Promise<IntegrationsState> => ipcRenderer.invoke(IPC.integrationsSet, id, fields),
|
||||
integrationsClear: (id: string): Promise<IntegrationsState> =>
|
||||
ipcRenderer.invoke(IPC.integrationsClear, id),
|
||||
pagespeedQuery: (input: PagespeedQueryInput): Promise<PagespeedQueryResult> =>
|
||||
ipcRenderer.invoke(IPC.pagespeedQuery, input),
|
||||
pagespeedRun: (input: PagespeedRunInput): Promise<PagespeedRunResult> =>
|
||||
ipcRenderer.invoke(IPC.pagespeedRun, input),
|
||||
pagespeedCancel: (): Promise<void> => ipcRenderer.invoke(IPC.pagespeedCancel),
|
||||
onPagespeedProgress: (cb) =>
|
||||
subscribe<PagespeedProgress>(IPC.pagespeedProgress, cb),
|
||||
onLogEntry: (cb) => subscribe<LogEntry>(IPC.logsEntry, cb),
|
||||
onLogsBatch: (cb) => subscribe<LogEntry[]>(IPC.logsBatch, cb),
|
||||
onLogsBusy: (cb) => subscribe<boolean>(IPC.logsBusy, cb),
|
||||
|
||||
@@ -16,6 +16,7 @@ import { UrlsTab } from './tabs/UrlsTab.js';
|
||||
import { ImagesTab } from './tabs/ImagesTab.js';
|
||||
import { BrokenLinksTab } from './tabs/BrokenLinksTab.js';
|
||||
import { SerpTab } from './tabs/SerpTab.js';
|
||||
import { PageSpeedTab } from './tabs/PageSpeedTab.js';
|
||||
import { VisualizationTab } from './tabs/VisualizationTab.js';
|
||||
import { useAppStore } from './store.js';
|
||||
import type { MenuEvent } from '@freecrawl/shared-types';
|
||||
@@ -157,9 +158,11 @@ export function App() {
|
||||
// behind the main window on some Windows configurations — leaves
|
||||
// the user staring at an "unresponsive" UI: the URL input can't
|
||||
// be clicked because a hidden modal is blocking the renderer.
|
||||
let recoveryTimer: number | undefined;
|
||||
let recoveryCancelled = false;
|
||||
void window.freecrawl.crashRecoveryStatus().then((status) => {
|
||||
if (status.pendingCount === 0) return;
|
||||
window.setTimeout(() => {
|
||||
if (recoveryCancelled || status.pendingCount === 0) return;
|
||||
recoveryTimer = window.setTimeout(() => {
|
||||
const proceed = window.confirm(
|
||||
t('app.crashRecovery', {
|
||||
defaultValue:
|
||||
@@ -308,6 +311,10 @@ export function App() {
|
||||
}
|
||||
});
|
||||
return () => {
|
||||
// Cancel the deferred crash-recovery prompt so it can't fire a
|
||||
// `window.confirm` after this component has unmounted.
|
||||
recoveryCancelled = true;
|
||||
if (recoveryTimer !== undefined) window.clearTimeout(recoveryTimer);
|
||||
off1();
|
||||
off2();
|
||||
off3();
|
||||
@@ -357,6 +364,8 @@ export function App() {
|
||||
<BrokenLinksTab />
|
||||
) : activeTab === 'serp' ? (
|
||||
<SerpTab />
|
||||
) : activeTab === 'pagespeed' ? (
|
||||
<PageSpeedTab />
|
||||
) : activeTab === 'visualization' ? (
|
||||
<VisualizationTab />
|
||||
) : (
|
||||
|
||||
@@ -42,6 +42,7 @@ const TAB_CATEGORY: Record<TabKey, UrlCategory> = {
|
||||
links: 'all',
|
||||
'broken-links': 'all',
|
||||
serp: 'tab:serp',
|
||||
pagespeed: 'all',
|
||||
visualization: 'all',
|
||||
};
|
||||
|
||||
|
||||
@@ -29,6 +29,13 @@ export function OverviewSidebar() {
|
||||
const progressCrawled = useAppStore((s) => s.progress?.crawled ?? 0);
|
||||
const activeCategory = useAppStore((s) => s.activeCategory);
|
||||
const navigateToCategory = useAppStore((s) => s.navigateToCategory);
|
||||
// Monotonic guard shared by the polling effect and the progress-driven
|
||||
// refetch — both call overviewGet(), and on the multi-worker reader
|
||||
// pool a slower response can resolve after a fresher one. Applying
|
||||
// results strictly in issue order stops a stale count overwriting a
|
||||
// newer one.
|
||||
const overviewSeqRef = useRef(0);
|
||||
const overviewAppliedRef = useRef(0);
|
||||
const [expanded, setExpanded] = useState<Set<string>>(
|
||||
new Set([
|
||||
'summary',
|
||||
@@ -62,9 +69,13 @@ export function OverviewSidebar() {
|
||||
// overlapping calls just queue up and starve other IPCs.
|
||||
if (inFlight) return;
|
||||
inFlight = true;
|
||||
const seq = ++overviewSeqRef.current;
|
||||
try {
|
||||
const o = await window.freecrawl.overviewGet();
|
||||
if (!cancelled) setOverview(o);
|
||||
if (!cancelled && seq > overviewAppliedRef.current) {
|
||||
overviewAppliedRef.current = seq;
|
||||
setOverview(o);
|
||||
}
|
||||
} finally {
|
||||
inFlight = false;
|
||||
}
|
||||
@@ -113,7 +124,13 @@ export function OverviewSidebar() {
|
||||
const bucket = Math.floor(progressCrawled / 50);
|
||||
if (bucket === lastRefetchAtRef.current) return;
|
||||
lastRefetchAtRef.current = bucket;
|
||||
void window.freecrawl.overviewGet().then((o) => setOverview(o));
|
||||
const seq = ++overviewSeqRef.current;
|
||||
void window.freecrawl.overviewGet().then((o) => {
|
||||
if (seq > overviewAppliedRef.current) {
|
||||
overviewAppliedRef.current = seq;
|
||||
setOverview(o);
|
||||
}
|
||||
});
|
||||
}, [progressCrawled, progressRunning, setOverview]);
|
||||
|
||||
const toggle = (k: string) => {
|
||||
|
||||
@@ -104,6 +104,31 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
[opts.category, opts.search, opts.sortBy, opts.sortDir, opts.filter],
|
||||
);
|
||||
|
||||
// Single entry point for writing a chunk into the cache. Guarantees
|
||||
// `chunkOrder` tracks every chunk present in `chunks` — the live tick
|
||||
// previously called `chunks.set` directly, leaving untracked chunks
|
||||
// the LRU could never evict (a slow heap leak on long live crawls,
|
||||
// and `chunkOrder` undercounting so eviction kicked in too late).
|
||||
const storeChunk = useCallback(
|
||||
(idx: number, rows: CrawlUrlRow[]): void => {
|
||||
const isNew = !chunks.current.has(idx);
|
||||
chunks.current.set(idx, rows);
|
||||
if (isNew) chunkOrder.current.push(idx);
|
||||
// Evict chunks not currently visible once the cache exceeds cap.
|
||||
while (chunkOrder.current.length > MAX_CACHED_CHUNKS) {
|
||||
const evict = chunkOrder.current.shift();
|
||||
if (evict === undefined) break;
|
||||
const { first, last } = activeRange.current;
|
||||
if (evict >= first && evict <= last) {
|
||||
chunkOrder.current.push(evict);
|
||||
continue;
|
||||
}
|
||||
chunks.current.delete(evict);
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
// Shape change (tab/sort/filter switch): clear immediately and load
|
||||
// fresh data in parallel. Showing the previous tab's rows under the
|
||||
// new tab's header is the wrong UX — "Internal" rows visible while
|
||||
@@ -133,8 +158,7 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
const [{ total: t }, ...newChunks] = await Promise.all([metaPromise, ...chunkPromises]);
|
||||
if (cancelled || token !== resetToken.current) return;
|
||||
for (const ch of newChunks) {
|
||||
chunks.current.set(ch.idx, ch.rows);
|
||||
chunkOrder.current.push(ch.idx);
|
||||
storeChunk(ch.idx, ch.rows);
|
||||
}
|
||||
setTotal(t);
|
||||
setVersion((v) => v + 1);
|
||||
@@ -193,7 +217,7 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
try {
|
||||
const { rows } = await queryChunk(i);
|
||||
if (cancelled || token !== resetToken.current) return;
|
||||
chunks.current.set(i, rows);
|
||||
storeChunk(i, rows);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
@@ -258,26 +282,14 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
try {
|
||||
const { rows, total: t } = await queryChunk(chunkIdx);
|
||||
if (token !== resetToken.current) return;
|
||||
chunks.current.set(chunkIdx, rows);
|
||||
chunkOrder.current.push(chunkIdx);
|
||||
// Evict chunks not currently visible once cache exceeds cap.
|
||||
while (chunkOrder.current.length > MAX_CACHED_CHUNKS) {
|
||||
const evict = chunkOrder.current.shift();
|
||||
if (evict === undefined) break;
|
||||
const { first, last } = activeRange.current;
|
||||
if (evict >= first && evict <= last) {
|
||||
chunkOrder.current.push(evict);
|
||||
continue;
|
||||
}
|
||||
chunks.current.delete(evict);
|
||||
}
|
||||
storeChunk(chunkIdx, rows);
|
||||
setTotal(t);
|
||||
setVersion((v) => v + 1);
|
||||
} finally {
|
||||
fetching.current.delete(chunkIdx);
|
||||
}
|
||||
},
|
||||
[queryChunk],
|
||||
[queryChunk, storeChunk],
|
||||
);
|
||||
|
||||
const ensureRange = useCallback(
|
||||
|
||||
@@ -143,6 +143,7 @@
|
||||
"links": "Links",
|
||||
"brokenLinks": "Broken Links",
|
||||
"serp": "SERP",
|
||||
"pagespeed": "PageSpeed",
|
||||
"visualization": "Visualization"
|
||||
},
|
||||
"export": {
|
||||
@@ -617,6 +618,37 @@
|
||||
"emptyNoData": "No indexable HTML pages with a title yet — run a crawl to populate SERP previews.",
|
||||
"emptyNoMatch": "No pages match the current filter."
|
||||
},
|
||||
"pagespeedTab": {
|
||||
"title": "PageSpeed Insights",
|
||||
"filterPlaceholder": "Filter by URL…",
|
||||
"filterAll": "All pages",
|
||||
"filterTested": "Tested only",
|
||||
"filterUntested": "Untested only",
|
||||
"strategy": "Device:",
|
||||
"deviceMobile": "Mobile",
|
||||
"deviceDesktop": "Desktop",
|
||||
"deviceBoth": "Mobile + Desktop",
|
||||
"selectUntested": "Select untested",
|
||||
"run": "Run PageSpeed ({{count}})",
|
||||
"runProgress": "Auditing {{done}} / {{total}}",
|
||||
"cancel": "Cancel",
|
||||
"confirmRun": "This will request {{count}} PageSpeed audits. Each audit takes ~10–30s and{{keyless}} may take a while. Continue?",
|
||||
"confirmKeyless": " — without an API key (low rate limit) —",
|
||||
"noKeyBanner": "No PageSpeed API key — audits run keyless at a low rate limit. Add a free key in Settings → Integrations for faster runs.",
|
||||
"colUrl": "URL",
|
||||
"colStatus": "Status",
|
||||
"colPerf": "Perf",
|
||||
"colLcp": "LCP",
|
||||
"colCls": "CLS",
|
||||
"colLastRun": "Last run",
|
||||
"groupMobile": "Mobile",
|
||||
"groupDesktop": "Desktop",
|
||||
"summary": "{{shown}} pages · {{tested}} tested · {{selected}} selected",
|
||||
"firstN": "(first {{n}})",
|
||||
"auditing": "Auditing: {{url}}",
|
||||
"emptyNoData": "No internal HTML pages crawled yet — run a crawl, then select pages to audit.",
|
||||
"emptyNoMatch": "No pages match the current filter."
|
||||
},
|
||||
"brokenTab": {
|
||||
"searchPlaceholder": "Search broken target URLs…",
|
||||
"scopeAll": "All",
|
||||
@@ -625,6 +657,9 @@
|
||||
"internalOnly": "Internal only",
|
||||
"externalOnly": "External only",
|
||||
"linksUnit": "broken links",
|
||||
"uniqueUnit": "unique URLs",
|
||||
"uniqueLabel": "Unique broken URLs",
|
||||
"totalLabel": "Total rows",
|
||||
"noBrokenLinks": "No broken links",
|
||||
"noBrokenLinksHint": "Run a crawl that includes external links to see broken links.",
|
||||
"cellsSelected": "{{n}} cell(s) selected"
|
||||
|
||||
@@ -143,6 +143,7 @@
|
||||
"links": "Linkler",
|
||||
"brokenLinks": "Kırık Linkler",
|
||||
"serp": "SERP",
|
||||
"pagespeed": "PageSpeed",
|
||||
"visualization": "Görselleştirme"
|
||||
},
|
||||
"export": {
|
||||
@@ -616,6 +617,37 @@
|
||||
"emptyNoData": "Henüz başlığı olan indekslenebilir HTML sayfası yok — SERP önizlemelerini doldurmak için bir crawl çalıştır.",
|
||||
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok."
|
||||
},
|
||||
"pagespeedTab": {
|
||||
"title": "PageSpeed Insights",
|
||||
"filterPlaceholder": "URL'ye göre filtrele…",
|
||||
"filterAll": "Tüm sayfalar",
|
||||
"filterTested": "Yalnızca test edilenler",
|
||||
"filterUntested": "Yalnızca test edilmeyenler",
|
||||
"strategy": "Cihaz:",
|
||||
"deviceMobile": "Mobil",
|
||||
"deviceDesktop": "Masaüstü",
|
||||
"deviceBoth": "Mobil + Masaüstü",
|
||||
"selectUntested": "Test edilmeyenleri seç",
|
||||
"run": "PageSpeed Çalıştır ({{count}})",
|
||||
"runProgress": "Denetleniyor {{done}} / {{total}}",
|
||||
"cancel": "İptal",
|
||||
"confirmRun": "Bu işlem {{count}} PageSpeed denetimi isteyecek. Her denetim ~10–30sn sürer ve{{keyless}} biraz zaman alabilir. Devam edilsin mi?",
|
||||
"confirmKeyless": " — API anahtarı olmadan (düşük hız sınırı) —",
|
||||
"noKeyBanner": "PageSpeed API anahtarı yok — denetimler düşük hız sınırıyla anahtarsız çalışır. Daha hızlı çalıştırmak için Ayarlar → Entegrasyonlar'dan ücretsiz bir anahtar ekleyin.",
|
||||
"colUrl": "URL",
|
||||
"colStatus": "Durum",
|
||||
"colPerf": "Perf",
|
||||
"colLcp": "LCP",
|
||||
"colCls": "CLS",
|
||||
"colLastRun": "Son çalıştırma",
|
||||
"groupMobile": "Mobil",
|
||||
"groupDesktop": "Masaüstü",
|
||||
"summary": "{{shown}} sayfa · {{tested}} test edildi · {{selected}} seçildi",
|
||||
"firstN": "(ilk {{n}})",
|
||||
"auditing": "Denetleniyor: {{url}}",
|
||||
"emptyNoData": "Henüz taranmış iç HTML sayfası yok — önce bir crawl çalıştır, sonra denetlenecek sayfaları seç.",
|
||||
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok."
|
||||
},
|
||||
"brokenTab": {
|
||||
"searchPlaceholder": "Kırık hedef URL ara…",
|
||||
"scopeAll": "Tümü",
|
||||
@@ -624,6 +656,9 @@
|
||||
"internalOnly": "Yalnızca Dahili",
|
||||
"externalOnly": "Yalnızca Harici",
|
||||
"linksUnit": "kırık link",
|
||||
"uniqueUnit": "benzersiz URL",
|
||||
"uniqueLabel": "Benzersiz kırık URL",
|
||||
"totalLabel": "Toplam satır",
|
||||
"noBrokenLinks": "Kırık link yok",
|
||||
"noBrokenLinksHint": "Kırık linkleri görmek için harici linkleri de içeren bir crawl çalıştır.",
|
||||
"cellsSelected": "{{n}} hücre seçili"
|
||||
|
||||
@@ -34,6 +34,7 @@ export type TabKey =
|
||||
| 'links'
|
||||
| 'broken-links'
|
||||
| 'serp'
|
||||
| 'pagespeed'
|
||||
| 'visualization';
|
||||
|
||||
export const TAB_ORDER: { key: TabKey; label: string }[] = [
|
||||
@@ -62,6 +63,7 @@ export const TAB_ORDER: { key: TabKey; label: string }[] = [
|
||||
{ key: 'links', label: 'Links' },
|
||||
{ key: 'broken-links', label: 'Broken Links' },
|
||||
{ key: 'serp', label: 'SERP' },
|
||||
{ key: 'pagespeed', label: 'PageSpeed' },
|
||||
{ key: 'visualization', label: 'Visualization' },
|
||||
];
|
||||
|
||||
|
||||
@@ -96,6 +96,21 @@ function cellValue(row: BrokenLinkRow, colIdx: number): string {
|
||||
}
|
||||
}
|
||||
|
||||
/** True when the row set shifted enough to invalidate index-keyed
|
||||
* cell selection (a live-crawl poll can insert / reorder rows). */
|
||||
function brokenRowsChanged(
|
||||
prev: BrokenLinkRow[],
|
||||
next: BrokenLinkRow[],
|
||||
): boolean {
|
||||
if (prev.length !== next.length) return true;
|
||||
for (let i = 0; i < next.length; i++) {
|
||||
const p = prev[i];
|
||||
const n = next[i];
|
||||
if (!p || !n || p.fromUrl !== n.fromUrl || p.toUrl !== n.toUrl) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function BrokenLinksTab() {
|
||||
const { t, i18n } = useTranslation();
|
||||
const lang = i18n.language;
|
||||
@@ -104,6 +119,10 @@ export function BrokenLinksTab() {
|
||||
const progress = useAppStore((s) => s.progress);
|
||||
const [rows, setRows] = useState<BrokenLinkRow[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
// Distinct broken target URLs — stable across crawls of different
|
||||
// sizes, unlike the row `total` which scales with how many pages the
|
||||
// crawl reached.
|
||||
const [uniqueTargets, setUniqueTargets] = useState(0);
|
||||
const [search, setSearch] = useState('');
|
||||
const [exporting, setExporting] = useState(false);
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
@@ -148,8 +167,22 @@ export function BrokenLinksTab() {
|
||||
internal,
|
||||
});
|
||||
if (cancelled) return;
|
||||
// A live-crawl poll can insert / reorder broken-link rows. The
|
||||
// cell selection is index-keyed ("rowIdx:colIdx"), so once the row
|
||||
// set shifts the old keys point at different rows — drop the
|
||||
// selection rather than highlight / copy the wrong cells.
|
||||
if (
|
||||
selectedRef.current.size > 0 &&
|
||||
brokenRowsChanged(rowsRef.current, res.rows)
|
||||
) {
|
||||
setSelected(new Set());
|
||||
setMenu(null);
|
||||
anchorRef.current = null;
|
||||
dragRef.current = null;
|
||||
}
|
||||
setRows(res.rows);
|
||||
setTotal(res.total);
|
||||
setUniqueTargets(res.uniqueTargets);
|
||||
};
|
||||
void load();
|
||||
const cadence = progress?.running ? POLL_MS_RUNNING : POLL_MS_IDLE;
|
||||
@@ -310,6 +343,8 @@ export function BrokenLinksTab() {
|
||||
)}
|
||||
<div className="ml-auto text-[11px] text-surface-500">
|
||||
<span className="font-mono text-surface-200">{total.toLocaleString()}</span> {t('brokenTab.linksUnit', { defaultValue: 'broken links' })}
|
||||
<span className="mx-1 text-surface-700">·</span>
|
||||
<span className="font-mono text-surface-200">{uniqueTargets.toLocaleString()}</span> {t('brokenTab.uniqueUnit', { defaultValue: 'unique URLs' })}
|
||||
<span className="ml-2 text-surface-600">({t('imagesTab.loaded', { defaultValue: '{{n}} loaded', n: rows.length.toLocaleString() })})</span>
|
||||
</div>
|
||||
<button
|
||||
@@ -454,7 +489,13 @@ export function BrokenLinksTab() {
|
||||
</span>
|
||||
)}
|
||||
<span>
|
||||
Total:{' '}
|
||||
{t('brokenTab.uniqueLabel', { defaultValue: 'Unique broken URLs' })}:{' '}
|
||||
<span className="font-mono tabular-nums text-surface-200">
|
||||
{uniqueTargets.toLocaleString()}
|
||||
</span>
|
||||
</span>
|
||||
<span>
|
||||
{t('brokenTab.totalLabel', { defaultValue: 'Total rows' })}:{' '}
|
||||
<span className="font-mono tabular-nums text-surface-200">
|
||||
{total.toLocaleString()}
|
||||
</span>
|
||||
|
||||
@@ -0,0 +1,584 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useVirtualizer } from '@tanstack/react-virtual';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Smartphone, Monitor, Loader2 } from 'lucide-react';
|
||||
import type {
|
||||
PagespeedMetrics,
|
||||
PagespeedProgress,
|
||||
PagespeedRow,
|
||||
PagespeedRunStrategy,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { useAppStore } from '../store.js';
|
||||
|
||||
const TOOLBAR_HEIGHT = 36;
|
||||
const HEADER_HEIGHT = 46;
|
||||
const STATUS_BAR_HEIGHT = 22;
|
||||
const ROW_HEIGHT = 30;
|
||||
const PAGE_SIZE = 2000;
|
||||
const POLL_MS_RUNNING = 5000;
|
||||
/** Above this many audits, warn before kicking off the run. */
|
||||
const CONFIRM_THRESHOLD = 30;
|
||||
|
||||
type FilterMode = 'all' | 'tested' | 'untested';
|
||||
|
||||
/** Lighthouse score bands → cell colour. */
|
||||
function scoreClass(v: number | null): string {
|
||||
if (v === null) return 'text-surface-600';
|
||||
if (v >= 90) return 'text-emerald-400';
|
||||
if (v >= 50) return 'text-amber-400';
|
||||
return 'text-red-400';
|
||||
}
|
||||
|
||||
/** Core Web Vitals thresholds → cell colour. */
|
||||
function cwvClass(
|
||||
v: number | null,
|
||||
good: number,
|
||||
poor: number,
|
||||
): string {
|
||||
if (v === null) return 'text-surface-600';
|
||||
if (v <= good) return 'text-emerald-400';
|
||||
if (v <= poor) return 'text-amber-400';
|
||||
return 'text-red-400';
|
||||
}
|
||||
|
||||
/** Format a millisecond timing compactly (e.g. `1.8s`, `420ms`). */
|
||||
function fmtMs(v: number | null): string {
|
||||
if (v === null) return '—';
|
||||
if (v >= 1000) return `${(v / 1000).toFixed(1)}s`;
|
||||
return `${Math.round(v)}ms`;
|
||||
}
|
||||
|
||||
function fmtCls(v: number | null): string {
|
||||
return v === null ? '—' : v.toFixed(3);
|
||||
}
|
||||
|
||||
function fmtScore(v: number | null): string {
|
||||
return v === null ? '—' : String(v);
|
||||
}
|
||||
|
||||
/** Short relative time for the "Last run" column. */
|
||||
function relativeTime(iso: string): string {
|
||||
if (!iso) return '—';
|
||||
const then = new Date(iso).getTime();
|
||||
if (Number.isNaN(then)) return '—';
|
||||
const diff = Date.now() - then;
|
||||
const min = Math.floor(diff / 60_000);
|
||||
if (min < 1) return 'just now';
|
||||
if (min < 60) return `${min}m ago`;
|
||||
const hr = Math.floor(min / 60);
|
||||
if (hr < 24) return `${hr}h ago`;
|
||||
const day = Math.floor(hr / 24);
|
||||
if (day < 30) return `${day}d ago`;
|
||||
return new Date(iso).toLocaleDateString();
|
||||
}
|
||||
|
||||
export function PageSpeedTab() {
|
||||
const { t } = useTranslation();
|
||||
const dataVersion = useAppStore((s) => s.dataVersion);
|
||||
const crawlProgress = useAppStore((s) => s.progress);
|
||||
|
||||
const [rows, setRows] = useState<PagespeedRow[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [search, setSearch] = useState('');
|
||||
const [filter, setFilter] = useState<FilterMode>('all');
|
||||
const [strategy, setStrategy] = useState<PagespeedRunStrategy>('mobile');
|
||||
const [selected, setSelected] = useState<Set<string>>(new Set());
|
||||
const [running, setRunning] = useState(false);
|
||||
const [psProgress, setPsProgress] = useState<PagespeedProgress | null>(null);
|
||||
const [hasApiKey, setHasApiKey] = useState(true);
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const reload = useCallback(async () => {
|
||||
const res = await window.freecrawl.pagespeedQuery({
|
||||
limit: PAGE_SIZE,
|
||||
offset: 0,
|
||||
search: search || undefined,
|
||||
filter,
|
||||
});
|
||||
setRows(res.rows);
|
||||
setTotal(res.total);
|
||||
}, [search, filter]);
|
||||
|
||||
// Candidate list — reloads on filter/search/crawl-data changes, and
|
||||
// polls while a crawl is discovering new pages.
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void reload();
|
||||
if (!crawlProgress?.running) return;
|
||||
const id = setInterval(() => {
|
||||
if (!cancelled) void reload();
|
||||
}, POLL_MS_RUNNING);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(id);
|
||||
};
|
||||
}, [reload, dataVersion, crawlProgress?.running]);
|
||||
|
||||
// Whether the PageSpeed integration has an API key — surfaces the
|
||||
// keyless-mode warning banner when it doesn't.
|
||||
useEffect(() => {
|
||||
void window.freecrawl.integrationsGetAll().then((state) => {
|
||||
setHasApiKey(state['pagespeed']?.fields?.['apiKey']?.set ?? false);
|
||||
});
|
||||
}, []);
|
||||
|
||||
// Live progress of an in-flight run. Refresh the table whenever an
|
||||
// audit completes (events without a `currentUrl`) so partial results
|
||||
// appear as the run proceeds.
|
||||
useEffect(() => {
|
||||
const off = window.freecrawl.onPagespeedProgress((p) => {
|
||||
setPsProgress(p.running ? p : null);
|
||||
if (!p.currentUrl) void reload();
|
||||
});
|
||||
return off;
|
||||
}, [reload]);
|
||||
|
||||
const virtualizer = useVirtualizer({
|
||||
count: rows.length,
|
||||
getScrollElement: () => scrollRef.current,
|
||||
estimateSize: () => ROW_HEIGHT,
|
||||
overscan: 16,
|
||||
getItemKey: (index) => rows[index]?.url ?? `idx-${index}`,
|
||||
});
|
||||
|
||||
const allLoadedSelected =
|
||||
rows.length > 0 && rows.every((r) => selected.has(r.url));
|
||||
|
||||
const toggleRow = useCallback((url: string) => {
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(url)) next.delete(url);
|
||||
else next.add(url);
|
||||
return next;
|
||||
});
|
||||
}, []);
|
||||
|
||||
const toggleAll = useCallback(() => {
|
||||
setSelected((prev) => {
|
||||
if (rows.length > 0 && rows.every((r) => prev.has(r.url))) {
|
||||
return new Set();
|
||||
}
|
||||
return new Set(rows.map((r) => r.url));
|
||||
});
|
||||
}, [rows]);
|
||||
|
||||
const selectUntested = useCallback(() => {
|
||||
setSelected(
|
||||
new Set(
|
||||
rows.filter((r) => !r.mobile && !r.desktop).map((r) => r.url),
|
||||
),
|
||||
);
|
||||
}, [rows]);
|
||||
|
||||
const strategyCount = strategy === 'both' ? 2 : 1;
|
||||
const auditCount = selected.size * strategyCount;
|
||||
|
||||
const runAudit = useCallback(async () => {
|
||||
const urls = [...selected];
|
||||
if (urls.length === 0 || running) return;
|
||||
if (
|
||||
auditCount > CONFIRM_THRESHOLD &&
|
||||
!window.confirm(
|
||||
t('pagespeedTab.confirmRun', {
|
||||
defaultValue:
|
||||
'This will request {{count}} PageSpeed audits. Each audit takes ~10–30s and{{keyless}} may take a while. Continue?',
|
||||
count: auditCount,
|
||||
keyless: hasApiKey
|
||||
? ''
|
||||
: t('pagespeedTab.confirmKeyless', {
|
||||
defaultValue: ' — without an API key (low rate limit) —',
|
||||
}),
|
||||
}),
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
setRunning(true);
|
||||
setPsProgress({ done: 0, total: auditCount, currentUrl: null, running: true });
|
||||
try {
|
||||
await window.freecrawl.pagespeedRun({ urls, strategy });
|
||||
} finally {
|
||||
setRunning(false);
|
||||
setPsProgress(null);
|
||||
void reload();
|
||||
}
|
||||
}, [selected, running, auditCount, strategy, hasApiKey, t, reload]);
|
||||
|
||||
const cancelRun = useCallback(() => {
|
||||
void window.freecrawl.pagespeedCancel();
|
||||
}, []);
|
||||
|
||||
const testedCount = useMemo(
|
||||
() => rows.filter((r) => r.mobile || r.desktop).length,
|
||||
[rows],
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="flex h-full w-full flex-col bg-surface-950">
|
||||
{/* Toolbar */}
|
||||
<div
|
||||
className="flex items-center gap-2 border-b border-surface-800 bg-surface-900/40 px-3"
|
||||
style={{ height: TOOLBAR_HEIGHT }}
|
||||
>
|
||||
<div className="text-[12px] font-semibold tracking-wide text-surface-100">
|
||||
{t('pagespeedTab.title', { defaultValue: 'PageSpeed Insights' })}
|
||||
</div>
|
||||
<input
|
||||
type="text"
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
placeholder={t('pagespeedTab.filterPlaceholder', {
|
||||
defaultValue: 'Filter by URL…',
|
||||
})}
|
||||
className="h-6 w-56 rounded border border-surface-700 bg-surface-950 px-2 text-[11px] text-surface-100 placeholder-surface-500 focus:border-blue-500 focus:outline-none"
|
||||
/>
|
||||
<select
|
||||
className="h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={filter}
|
||||
onChange={(e) => setFilter(e.target.value as FilterMode)}
|
||||
>
|
||||
<option value="all">
|
||||
{t('pagespeedTab.filterAll', { defaultValue: 'All pages' })}
|
||||
</option>
|
||||
<option value="tested">
|
||||
{t('pagespeedTab.filterTested', { defaultValue: 'Tested only' })}
|
||||
</option>
|
||||
<option value="untested">
|
||||
{t('pagespeedTab.filterUntested', { defaultValue: 'Untested only' })}
|
||||
</option>
|
||||
</select>
|
||||
<label className="flex items-center gap-1 text-[11px] text-surface-400">
|
||||
{t('pagespeedTab.strategy', { defaultValue: 'Device:' })}
|
||||
<select
|
||||
className="h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={strategy}
|
||||
onChange={(e) =>
|
||||
setStrategy(e.target.value as PagespeedRunStrategy)
|
||||
}
|
||||
disabled={running}
|
||||
>
|
||||
<option value="mobile">
|
||||
{t('pagespeedTab.deviceMobile', { defaultValue: 'Mobile' })}
|
||||
</option>
|
||||
<option value="desktop">
|
||||
{t('pagespeedTab.deviceDesktop', { defaultValue: 'Desktop' })}
|
||||
</option>
|
||||
<option value="both">
|
||||
{t('pagespeedTab.deviceBoth', { defaultValue: 'Mobile + Desktop' })}
|
||||
</option>
|
||||
</select>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
onClick={selectUntested}
|
||||
disabled={running || rows.length === 0}
|
||||
className="h-6 rounded border border-surface-700 bg-surface-800 px-2 text-[11px] text-surface-200 hover:bg-surface-700 disabled:cursor-not-allowed disabled:opacity-40"
|
||||
>
|
||||
{t('pagespeedTab.selectUntested', { defaultValue: 'Select untested' })}
|
||||
</button>
|
||||
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
{running && psProgress ? (
|
||||
<>
|
||||
<span className="flex items-center gap-1.5 text-[11px] text-surface-300">
|
||||
<Loader2 size={12} className="animate-spin text-blue-400" />
|
||||
{t('pagespeedTab.runProgress', {
|
||||
defaultValue: 'Auditing {{done}} / {{total}}',
|
||||
done: psProgress.done,
|
||||
total: psProgress.total,
|
||||
})}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={cancelRun}
|
||||
className="h-6 rounded border border-red-700 bg-red-900/40 px-2.5 text-[11px] font-medium text-red-200 hover:bg-red-900/70"
|
||||
>
|
||||
{t('pagespeedTab.cancel', { defaultValue: 'Cancel' })}
|
||||
</button>
|
||||
</>
|
||||
) : (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void runAudit()}
|
||||
disabled={selected.size === 0 || running}
|
||||
className="h-6 rounded bg-blue-600 px-3 text-[11px] font-medium text-white hover:bg-blue-500 disabled:cursor-not-allowed disabled:bg-surface-700 disabled:text-surface-500"
|
||||
>
|
||||
{t('pagespeedTab.run', {
|
||||
defaultValue: 'Run PageSpeed ({{count}})',
|
||||
count: auditCount,
|
||||
})}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Keyless-mode banner */}
|
||||
{!hasApiKey && (
|
||||
<div className="border-b border-amber-700/40 bg-amber-900/20 px-3 py-1 text-[10px] text-amber-200">
|
||||
{t('pagespeedTab.noKeyBanner', {
|
||||
defaultValue:
|
||||
'No PageSpeed API key — audits run keyless at a low rate limit. Add a free key in Settings → Integrations for faster runs.',
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Column header (two-tier: device groups + metric labels) */}
|
||||
<div
|
||||
className="flex shrink-0 select-none items-stretch border-b border-surface-800 bg-surface-900/60 text-[10px] font-medium text-surface-400"
|
||||
style={{ height: HEADER_HEIGHT }}
|
||||
>
|
||||
<div className="flex w-[30px] items-center justify-center border-r border-surface-800">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={allLoadedSelected}
|
||||
ref={(el) => {
|
||||
if (el) {
|
||||
el.indeterminate =
|
||||
selected.size > 0 && !allLoadedSelected;
|
||||
}
|
||||
}}
|
||||
onChange={toggleAll}
|
||||
disabled={rows.length === 0}
|
||||
className="h-3 w-3 accent-blue-500"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-1 flex-col justify-end border-r border-surface-800 px-2 pb-1">
|
||||
{t('pagespeedTab.colUrl', { defaultValue: 'URL' })}
|
||||
</div>
|
||||
<div className="flex w-[54px] flex-col justify-end border-r border-surface-800 px-1 pb-1 text-center">
|
||||
{t('pagespeedTab.colStatus', { defaultValue: 'Status' })}
|
||||
</div>
|
||||
{/* Mobile group */}
|
||||
<div className="flex w-[162px] flex-col border-r border-surface-800">
|
||||
<div className="flex h-[22px] items-center justify-center gap-1 border-b border-surface-800 bg-surface-900/80 text-surface-300">
|
||||
<Smartphone size={11} />
|
||||
{t('pagespeedTab.groupMobile', { defaultValue: 'Mobile' })}
|
||||
</div>
|
||||
<div className="flex flex-1 items-end pb-1 text-center">
|
||||
<div className="w-[50px]">{t('pagespeedTab.colPerf', { defaultValue: 'Perf' })}</div>
|
||||
<div className="w-[60px]">{t('pagespeedTab.colLcp', { defaultValue: 'LCP' })}</div>
|
||||
<div className="w-[52px]">{t('pagespeedTab.colCls', { defaultValue: 'CLS' })}</div>
|
||||
</div>
|
||||
</div>
|
||||
{/* Desktop group */}
|
||||
<div className="flex w-[162px] flex-col border-r border-surface-800">
|
||||
<div className="flex h-[22px] items-center justify-center gap-1 border-b border-surface-800 bg-surface-900/80 text-surface-300">
|
||||
<Monitor size={11} />
|
||||
{t('pagespeedTab.groupDesktop', { defaultValue: 'Desktop' })}
|
||||
</div>
|
||||
<div className="flex flex-1 items-end pb-1 text-center">
|
||||
<div className="w-[50px]">{t('pagespeedTab.colPerf', { defaultValue: 'Perf' })}</div>
|
||||
<div className="w-[60px]">{t('pagespeedTab.colLcp', { defaultValue: 'LCP' })}</div>
|
||||
<div className="w-[52px]">{t('pagespeedTab.colCls', { defaultValue: 'CLS' })}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex w-[96px] flex-col justify-end px-2 pb-1 text-right">
|
||||
{t('pagespeedTab.colLastRun', { defaultValue: 'Last run' })}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Body */}
|
||||
<div ref={scrollRef} className="flex-1 overflow-y-auto">
|
||||
{rows.length === 0 ? (
|
||||
<div className="flex h-full items-center justify-center px-6 text-center text-[12px] text-surface-500">
|
||||
{total === 0
|
||||
? t('pagespeedTab.emptyNoData', {
|
||||
defaultValue:
|
||||
'No internal HTML pages crawled yet — run a crawl, then select pages to audit.',
|
||||
})
|
||||
: t('pagespeedTab.emptyNoMatch', {
|
||||
defaultValue: 'No pages match the current filter.',
|
||||
})}
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
style={{
|
||||
height: virtualizer.getTotalSize(),
|
||||
width: '100%',
|
||||
position: 'relative',
|
||||
}}
|
||||
>
|
||||
{virtualizer.getVirtualItems().map((vRow) => {
|
||||
const row = rows[vRow.index];
|
||||
if (!row) return null;
|
||||
return (
|
||||
<div
|
||||
key={vRow.key}
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 0,
|
||||
left: 0,
|
||||
width: '100%',
|
||||
height: ROW_HEIGHT,
|
||||
transform: `translateY(${vRow.start}px)`,
|
||||
}}
|
||||
>
|
||||
<PageSpeedRow
|
||||
row={row}
|
||||
selected={selected.has(row.url)}
|
||||
onToggle={() => toggleRow(row.url)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Status bar */}
|
||||
<div
|
||||
className="flex items-center gap-3 border-t border-surface-800 bg-surface-900/40 px-3 text-[11px] text-surface-400"
|
||||
style={{ height: STATUS_BAR_HEIGHT }}
|
||||
>
|
||||
<span>
|
||||
{t('pagespeedTab.summary', {
|
||||
defaultValue: '{{shown}} pages · {{tested}} tested · {{selected}} selected',
|
||||
shown: rows.length.toLocaleString(),
|
||||
tested: testedCount.toLocaleString(),
|
||||
selected: selected.size.toLocaleString(),
|
||||
})}
|
||||
{total > rows.length &&
|
||||
' ' +
|
||||
t('pagespeedTab.firstN', {
|
||||
defaultValue: '(first {{n}})',
|
||||
n: PAGE_SIZE.toLocaleString(),
|
||||
})}
|
||||
</span>
|
||||
{running && psProgress?.currentUrl && (
|
||||
<span className="truncate text-surface-500">
|
||||
{t('pagespeedTab.auditing', {
|
||||
defaultValue: 'Auditing: {{url}}',
|
||||
url: psProgress.currentUrl,
|
||||
})}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MetricCell({
|
||||
text,
|
||||
cls,
|
||||
width,
|
||||
title,
|
||||
}: {
|
||||
text: string;
|
||||
cls: string;
|
||||
width: number;
|
||||
title?: string;
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
className={`shrink-0 text-center tabular-nums ${cls}`}
|
||||
style={{ width }}
|
||||
title={title}
|
||||
>
|
||||
{text}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function StrategyCells({ m }: { m: PagespeedMetrics | null }) {
|
||||
if (!m) {
|
||||
return (
|
||||
<>
|
||||
<MetricCell text="—" cls="text-surface-700" width={50} />
|
||||
<MetricCell text="—" cls="text-surface-700" width={60} />
|
||||
<MetricCell text="—" cls="text-surface-700" width={52} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
if (m.status === 'error') {
|
||||
return (
|
||||
<div
|
||||
className="flex w-[162px] shrink-0 items-center justify-center text-red-400"
|
||||
title={m.error ?? 'Audit failed'}
|
||||
>
|
||||
ERR
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<MetricCell
|
||||
text={fmtScore(m.performance)}
|
||||
cls={scoreClass(m.performance)}
|
||||
width={50}
|
||||
/>
|
||||
<MetricCell
|
||||
text={fmtMs(m.lcp)}
|
||||
cls={cwvClass(m.lcp, 2500, 4000)}
|
||||
width={60}
|
||||
/>
|
||||
<MetricCell
|
||||
text={fmtCls(m.cls)}
|
||||
cls={cwvClass(m.cls, 0.1, 0.25)}
|
||||
width={52}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function PageSpeedRow({
|
||||
row,
|
||||
selected,
|
||||
onToggle,
|
||||
}: {
|
||||
row: PagespeedRow;
|
||||
selected: boolean;
|
||||
onToggle: () => void;
|
||||
}) {
|
||||
const lastRun = useMemo(() => {
|
||||
const candidates = [row.mobile?.fetchedAt, row.desktop?.fetchedAt].filter(
|
||||
(v): v is string => !!v,
|
||||
);
|
||||
if (candidates.length === 0) return '';
|
||||
return candidates.sort().reverse()[0] ?? '';
|
||||
}, [row.mobile, row.desktop]);
|
||||
|
||||
const status = row.statusCode;
|
||||
const statusCls =
|
||||
status === null
|
||||
? 'text-surface-600'
|
||||
: status >= 200 && status < 300
|
||||
? 'text-emerald-400'
|
||||
: status >= 300 && status < 400
|
||||
? 'text-blue-400'
|
||||
: 'text-red-400';
|
||||
|
||||
return (
|
||||
<div
|
||||
onClick={onToggle}
|
||||
className={`flex h-full cursor-pointer items-center text-[11px] ${
|
||||
selected
|
||||
? 'bg-blue-900/30'
|
||||
: 'odd:bg-surface-900/20 hover:bg-surface-800/40'
|
||||
}`}
|
||||
>
|
||||
<div className="flex w-[30px] items-center justify-center">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={selected}
|
||||
onChange={onToggle}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
className="h-3 w-3 accent-blue-500"
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
className="flex-1 truncate px-2 text-surface-200"
|
||||
title={row.url}
|
||||
>
|
||||
{row.url}
|
||||
</div>
|
||||
<div className={`w-[54px] shrink-0 text-center tabular-nums ${statusCls}`}>
|
||||
{status ?? '—'}
|
||||
</div>
|
||||
<StrategyCells m={row.mobile} />
|
||||
<StrategyCells m={row.desktop} />
|
||||
<div className="w-[96px] shrink-0 truncate px-2 text-right text-surface-500">
|
||||
{lastRun ? relativeTime(lastRun) : '—'}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -219,6 +219,16 @@ export function VisualizationTab() {
|
||||
const [hover, setHover] = useState<string | null>(null);
|
||||
const [selectedUrl, setSelectedUrl] = useState<string | null>(null);
|
||||
const [tuning, setTuning] = useState<VisTuning>(() => loadTuning());
|
||||
// Debounced copy that drives the expensive cytoscape rebuild. A tuning
|
||||
// slider drag fires onChange on every step; rebuilding the graph and
|
||||
// re-running the cose layout on each step thrashes the view. The
|
||||
// slider itself stays instant (bound to `tuning`) — the graph settles
|
||||
// ~250 ms after the last change.
|
||||
const [debouncedTuning, setDebouncedTuning] = useState<VisTuning>(tuning);
|
||||
useEffect(() => {
|
||||
const id = setTimeout(() => setDebouncedTuning(tuning), 250);
|
||||
return () => clearTimeout(id);
|
||||
}, [tuning]);
|
||||
const [tunerOpen, setTunerOpen] = useState(false);
|
||||
const [exportMenuOpen, setExportMenuOpen] = useState(false);
|
||||
|
||||
@@ -249,17 +259,24 @@ export function VisualizationTab() {
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [nodeLimit, dataVersion]);
|
||||
|
||||
// Monotonic token so a slow graphSnapshot (e.g. a large `nodeLimit`)
|
||||
// can't resolve after a newer request and render a graph that no
|
||||
// longer matches the controls. Covers both the effect and the
|
||||
// Reload / Tuning buttons.
|
||||
const loadTokenRef = useRef(0);
|
||||
async function loadGraph() {
|
||||
const token = ++loadTokenRef.current;
|
||||
setLoading(true);
|
||||
try {
|
||||
const [g, a] = await Promise.all([
|
||||
window.freecrawl.graphSnapshot({ nodeLimit }),
|
||||
window.freecrawl.topAnchorTexts(120),
|
||||
]);
|
||||
if (token !== loadTokenRef.current) return;
|
||||
setGraph(g);
|
||||
setAnchors(a);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
if (token === loadTokenRef.current) setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,7 +304,7 @@ export function VisualizationTab() {
|
||||
statusCode: n.statusCode ?? '',
|
||||
inlinks: n.inlinks,
|
||||
color: colorFn(n),
|
||||
size: nodeSize(n.inlinks, tuning.nodeSizeScale),
|
||||
size: nodeSize(n.inlinks, debouncedTuning.nodeSizeScale),
|
||||
isTop: topIds.has(String(n.id)) ? 1 : 0,
|
||||
},
|
||||
})),
|
||||
@@ -318,15 +335,15 @@ export function VisualizationTab() {
|
||||
padding: 30,
|
||||
};
|
||||
if (layout === 'cose') {
|
||||
layoutCfg.nodeRepulsion = () => 1_000_000 * tuning.repulsionScale;
|
||||
layoutCfg.idealEdgeLength = () => 400 * tuning.edgeLengthScale;
|
||||
layoutCfg.nodeRepulsion = () => 1_000_000 * debouncedTuning.repulsionScale;
|
||||
layoutCfg.idealEdgeLength = () => 400 * debouncedTuning.edgeLengthScale;
|
||||
layoutCfg.edgeElasticity = () => 20;
|
||||
layoutCfg.gravity = 0;
|
||||
layoutCfg.gravityRange = 5.0;
|
||||
layoutCfg.gravityCompound = 0;
|
||||
layoutCfg.numIter = 6000;
|
||||
layoutCfg.nodeOverlap = 200;
|
||||
layoutCfg.componentSpacing = 400 * tuning.componentSpacingScale;
|
||||
layoutCfg.componentSpacing = 400 * debouncedTuning.componentSpacingScale;
|
||||
layoutCfg.nestingFactor = 1.2;
|
||||
layoutCfg.initialTemp = 2000;
|
||||
layoutCfg.coolingFactor = 0.995;
|
||||
@@ -412,7 +429,7 @@ export function VisualizationTab() {
|
||||
'target-arrow-color': '#525252',
|
||||
'target-arrow-shape': 'triangle',
|
||||
'arrow-scale': 0.6,
|
||||
opacity: tuning.edgeOpacity,
|
||||
opacity: debouncedTuning.edgeOpacity,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -534,7 +551,7 @@ export function VisualizationTab() {
|
||||
cy.destroy();
|
||||
cyRef.current = null;
|
||||
};
|
||||
}, [graph, layout, colorMode, labelMode, tuning]);
|
||||
}, [graph, layout, colorMode, labelMode, debouncedTuning]);
|
||||
|
||||
return (
|
||||
<div className="flex h-full w-full flex-col bg-surface-950">
|
||||
|
||||
@@ -605,6 +605,7 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
|
||||
images: [],
|
||||
'broken-links': [],
|
||||
serp: [],
|
||||
pagespeed: [],
|
||||
visualization: [],
|
||||
canonicals: [
|
||||
COL.url,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/mcp-server",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
Generated
+8
-8
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "freecrawl-seo",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "freecrawl-seo",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
@@ -25,7 +25,7 @@
|
||||
},
|
||||
"apps/cli": {
|
||||
"name": "@freecrawl/cli",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"dependencies": {
|
||||
"@freecrawl/core": "*",
|
||||
"@freecrawl/db": "*",
|
||||
@@ -42,7 +42,7 @@
|
||||
},
|
||||
"apps/desktop": {
|
||||
"name": "@freecrawl/desktop",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"dependencies": {
|
||||
"@freecrawl/core": "*",
|
||||
"@freecrawl/db": "*",
|
||||
@@ -86,7 +86,7 @@
|
||||
},
|
||||
"apps/mcp-server": {
|
||||
"name": "@freecrawl/mcp-server",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"dependencies": {
|
||||
"@freecrawl/db": "*",
|
||||
"@freecrawl/shared-types": "*"
|
||||
@@ -8826,7 +8826,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@freecrawl/core",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"dependencies": {
|
||||
"@freecrawl/db": "*",
|
||||
"@freecrawl/shared-types": "*",
|
||||
@@ -8852,7 +8852,7 @@
|
||||
},
|
||||
"packages/db": {
|
||||
"name": "@freecrawl/db",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"dependencies": {
|
||||
"@freecrawl/shared-types": "*"
|
||||
},
|
||||
@@ -8864,7 +8864,7 @@
|
||||
},
|
||||
"packages/shared-types": {
|
||||
"name": "@freecrawl/shared-types",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"devDependencies": {
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "^5.7.0"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "freecrawl-seo",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"private": true,
|
||||
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
|
||||
"license": "MIT",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/core",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -1246,6 +1246,18 @@ export class Crawler extends EventEmitter {
|
||||
private enqueueExternal(url: string): void {
|
||||
if (this.stopped) return;
|
||||
if (this.externalSeen.has(url)) return;
|
||||
// Bound external discovery the same way internal crawling is bounded.
|
||||
// Without these caps `externalSeen` and the probe queue grow without
|
||||
// limit on sites with a huge external-link graph (directories,
|
||||
// aggregators) — the internal `enqueue` has the same guards.
|
||||
if (this.externalSeen.size >= this.config.maxUrls) return;
|
||||
if (
|
||||
this.config.maxQueueSize > 0 &&
|
||||
this.externalQueue.size + this.externalQueue.pending >=
|
||||
this.config.maxQueueSize
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (!this.passesUrlFilter(url)) return;
|
||||
this.externalSeen.add(url);
|
||||
this.externalQueue
|
||||
|
||||
@@ -1314,7 +1314,23 @@ function detectAnalyticsTrackers(
|
||||
rawHtml: string,
|
||||
): AnalyticsTracker[] {
|
||||
const found = new Map<string, AnalyticsTracker>();
|
||||
const add = (name: string, id: string | null = null): void => {
|
||||
const add = (
|
||||
name: string,
|
||||
id: string | null = null,
|
||||
allowMultiple = false,
|
||||
): void => {
|
||||
// `allowMultiple` trackers (e.g. GA4) keep a separate entry per
|
||||
// distinct id — two GA4 property tags on one page is a real
|
||||
// misconfiguration the "Multiple GA4" issue filter needs to see.
|
||||
// Without this the name-keyed map collapses them into one entry and
|
||||
// the filter can never fire.
|
||||
if (allowMultiple && id !== null) {
|
||||
const key = `${name}|${id}`;
|
||||
if (!found.has(key)) found.set(key, { name, id });
|
||||
// Drop a prior id-less placeholder for this tracker, if any.
|
||||
found.delete(name);
|
||||
return;
|
||||
}
|
||||
const existing = found.get(name);
|
||||
// Keep the more specific (id-bearing) record if we see the same tracker twice.
|
||||
if (!existing || (existing.id === null && id !== null)) {
|
||||
@@ -1348,10 +1364,13 @@ function detectAnalyticsTrackers(
|
||||
// them as a separate tracker since Google deprecated UA in July 2023.
|
||||
for (const src of scriptSrcs) {
|
||||
const m = src.match(/gtag\/js\?id=(G-[A-Z0-9]+)/i);
|
||||
if (m && m[1]) add('Google Analytics 4', m[1].toUpperCase());
|
||||
if (m && m[1]) add('Google Analytics 4', m[1].toUpperCase(), true);
|
||||
}
|
||||
// Match every distinct inline `G-…` id, not just the first — a page
|
||||
// wiring up two GA4 properties typically lists both inline.
|
||||
for (const m of inlineBlob.matchAll(/['"](G-[A-Z0-9]{6,})['"]/g)) {
|
||||
if (m[1]) add('Google Analytics 4', m[1].toUpperCase(), true);
|
||||
}
|
||||
const ga4Inline = inlineBlob.match(/['"](G-[A-Z0-9]{6,})['"]/);
|
||||
if (ga4Inline && ga4Inline[1]) add('Google Analytics 4', ga4Inline[1].toUpperCase());
|
||||
const uaInline = inlineBlob.match(/UA-\d{4,10}-\d{1,4}/);
|
||||
if (uaInline) add('Google Analytics (UA)', uaInline[0]);
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/db",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -1242,6 +1242,56 @@ const MIGRATIONS: Migration[] = [
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 55,
|
||||
name: 'add_pagespeed_results_table',
|
||||
// V1 Faz 7 — Google PageSpeed Insights integration. Audit results
|
||||
// live in their own table rather than as `urls` columns: PSI is run
|
||||
// on-demand against a user-selected subset (never the whole crawl),
|
||||
// each URL can have up to two rows (mobile + desktop), and keeping
|
||||
// it separate avoids widening the already-very-wide `urls` table.
|
||||
//
|
||||
// url — the audited page URL (joins back to `urls.url`).
|
||||
// strategy — 'mobile' | 'desktop' form factor.
|
||||
// performance — Lighthouse Performance score 0–100 (null if N/A).
|
||||
// lcp/fcp/tbt/speed_index — lab timing metrics in milliseconds.
|
||||
// cls — Cumulative Layout Shift (unitless).
|
||||
// status — 'ok' when the audit completed, 'error' otherwise.
|
||||
// error — failure reason when status='error'.
|
||||
// fetched_at — ISO timestamp of the audit.
|
||||
up: (db) => {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS pagespeed_results (
|
||||
url TEXT NOT NULL,
|
||||
strategy TEXT NOT NULL,
|
||||
performance INTEGER,
|
||||
lcp REAL,
|
||||
cls REAL,
|
||||
fcp REAL,
|
||||
tbt REAL,
|
||||
speed_index REAL,
|
||||
status TEXT NOT NULL,
|
||||
error TEXT,
|
||||
fetched_at TEXT NOT NULL,
|
||||
PRIMARY KEY (url, strategy)
|
||||
);
|
||||
`);
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 56,
|
||||
name: 'add_image_usages_from_index',
|
||||
// `image_usages` had an index on `image_id` but none on
|
||||
// `from_url_id`. The per-URL write path now deletes a page's prior
|
||||
// `image_usages` rows before re-inserting (so a re-crawl doesn't
|
||||
// duplicate them) — without this index that DELETE would full-scan
|
||||
// the table on every URL write, turning a re-crawl into O(n²).
|
||||
up: (db) => {
|
||||
db.exec(
|
||||
'CREATE INDEX IF NOT EXISTS idx_image_usages_from ON image_usages(from_url_id);',
|
||||
);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export function runMigrations(db: DatabaseSync): void {
|
||||
|
||||
+257
-30
@@ -15,6 +15,9 @@ import type {
|
||||
InlinkRow,
|
||||
OutlinkRow,
|
||||
OverviewCounts,
|
||||
PagespeedMetrics,
|
||||
PagespeedRow,
|
||||
PagespeedStrategy,
|
||||
UrlCategory,
|
||||
UrlDetail,
|
||||
} from '@freecrawl/shared-types';
|
||||
@@ -671,7 +674,8 @@ export class ProjectDb {
|
||||
DELETE FROM project_meta;
|
||||
DELETE FROM urls_issues;
|
||||
DELETE FROM crawl_queue;
|
||||
DELETE FROM host_certs;`,
|
||||
DELETE FROM host_certs;
|
||||
DELETE FROM pagespeed_results;`,
|
||||
);
|
||||
});
|
||||
// Reclaim freed pages back to the OS so the project file size
|
||||
@@ -792,6 +796,27 @@ export class ProjectDb {
|
||||
WHERE id NOT IN (SELECT DISTINCT image_id FROM image_usages)`,
|
||||
);
|
||||
|
||||
// PageSpeed audit rows are keyed by URL string (no FK to `urls`),
|
||||
// so wipe them by the same host match — otherwise a GDPR delete
|
||||
// leaves stale PSI data that would re-attach if the URL is
|
||||
// re-crawled later.
|
||||
this.db
|
||||
.prepare(
|
||||
`DELETE FROM pagespeed_results
|
||||
WHERE LOWER(
|
||||
SUBSTR(
|
||||
url,
|
||||
INSTR(url, '://') + 3,
|
||||
CASE
|
||||
WHEN INSTR(SUBSTR(url, INSTR(url, '://') + 3), '/') > 0
|
||||
THEN INSTR(SUBSTR(url, INSTR(url, '://') + 3), '/') - 1
|
||||
ELSE LENGTH(url)
|
||||
END
|
||||
)
|
||||
) = ?`,
|
||||
)
|
||||
.run(target);
|
||||
|
||||
return { urlsDeleted, linksDeleted };
|
||||
});
|
||||
}
|
||||
@@ -901,6 +926,19 @@ export class ProjectDb {
|
||||
}
|
||||
}
|
||||
if (urlId) {
|
||||
// Re-crawl safety. A URL can be fetched more than once — manual
|
||||
// "Re-Spider", crash-recovery resume, or re-running a crawl
|
||||
// without Clear. `insertLinks` / `insertImages` are plain
|
||||
// INSERTs, so without clearing the page's previous rows first
|
||||
// every re-fetch would DUPLICATE its link + image-usage rows,
|
||||
// inflating broken-link counts and every other link/image
|
||||
// metric. Delete unconditionally (a no-op on a first crawl, both
|
||||
// DELETEs are index-backed) so the page's link + image set
|
||||
// always reflects the latest fetch.
|
||||
this.db.prepare('DELETE FROM links WHERE from_url_id = ?').run(urlId);
|
||||
this.db
|
||||
.prepare('DELETE FROM image_usages WHERE from_url_id = ?')
|
||||
.run(urlId);
|
||||
if (payload.links.length > 0) {
|
||||
this.insertLinks(urlId, payload.links, payload.fromDepth);
|
||||
}
|
||||
@@ -934,7 +972,21 @@ export class ProjectDb {
|
||||
* depth at which each pending URL was discovered.
|
||||
*/
|
||||
deleteUrl(id: number): void {
|
||||
this.db.prepare('DELETE FROM urls WHERE id = ?').run(id);
|
||||
// `urls_issues` and `pagespeed_results` have no FK to `urls`, so the
|
||||
// `ON DELETE CASCADE` that cleans links/headers/images doesn't reach
|
||||
// them — wipe them explicitly or a deleted URL leaves orphan rows
|
||||
// that inflate `getIssueCounts()` and re-attach stale audit data on
|
||||
// re-crawl. The `pagespeed_results` delete must run before the parent
|
||||
// `urls` row vanishes (it resolves the URL string from `urls`).
|
||||
this.runInTransaction(() => {
|
||||
this.db
|
||||
.prepare(
|
||||
'DELETE FROM pagespeed_results WHERE url = (SELECT url FROM urls WHERE id = ?)',
|
||||
)
|
||||
.run(id);
|
||||
this.db.prepare('DELETE FROM urls_issues WHERE url_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM urls WHERE id = ?').run(id);
|
||||
});
|
||||
}
|
||||
|
||||
markUrlForRecrawl(id: number): void {
|
||||
@@ -952,33 +1004,73 @@ export class ProjectDb {
|
||||
|
||||
markUrlsForRecrawl(ids: number[]): void {
|
||||
if (ids.length === 0) return;
|
||||
const placeholders = ids.map(() => '?').join(',');
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE urls SET
|
||||
status_code = NULL,
|
||||
status_text = NULL,
|
||||
indexability = 'indexable',
|
||||
indexability_reason = NULL
|
||||
WHERE id IN (${placeholders}) AND is_external = 0`,
|
||||
)
|
||||
.run(...ids);
|
||||
// Chunk the id list — a multi-row table selection can exceed SQLite's
|
||||
// ~32 K bound-parameter limit, which would throw `too many SQL
|
||||
// variables` and fail the whole operation.
|
||||
const CHUNK = 500;
|
||||
for (let i = 0; i < ids.length; i += CHUNK) {
|
||||
const slice = ids.slice(i, i + CHUNK);
|
||||
const placeholders = slice.map(() => '?').join(',');
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE urls SET
|
||||
status_code = NULL,
|
||||
status_text = NULL,
|
||||
indexability = 'indexable',
|
||||
indexability_reason = NULL
|
||||
WHERE id IN (${placeholders}) AND is_external = 0`,
|
||||
)
|
||||
.run(...slice);
|
||||
}
|
||||
}
|
||||
|
||||
deleteUrls(ids: number[]): void {
|
||||
if (ids.length === 0) return;
|
||||
const placeholders = ids.map(() => '?').join(',');
|
||||
this.db.prepare(`DELETE FROM urls WHERE id IN (${placeholders})`).run(...ids);
|
||||
// Chunk to stay under SQLite's bound-parameter limit; wrap in one
|
||||
// transaction so the urls + urls_issues + pagespeed_results deletes
|
||||
// are atomic. `urls_issues` / `pagespeed_results` have no FK to
|
||||
// `urls` (see `deleteUrl`), so they're wiped explicitly.
|
||||
const CHUNK = 500;
|
||||
this.runInTransaction(() => {
|
||||
for (let i = 0; i < ids.length; i += CHUNK) {
|
||||
const slice = ids.slice(i, i + CHUNK);
|
||||
const placeholders = slice.map(() => '?').join(',');
|
||||
this.db
|
||||
.prepare(
|
||||
`DELETE FROM pagespeed_results
|
||||
WHERE url IN (SELECT url FROM urls WHERE id IN (${placeholders}))`,
|
||||
)
|
||||
.run(...slice);
|
||||
this.db
|
||||
.prepare(`DELETE FROM urls_issues WHERE url_id IN (${placeholders})`)
|
||||
.run(...slice);
|
||||
this.db
|
||||
.prepare(`DELETE FROM urls WHERE id IN (${placeholders})`)
|
||||
.run(...slice);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Look up the URL strings for a batch of ids (preserves DB order). */
|
||||
/** Look up the URL strings for a batch of ids, in ascending id order. */
|
||||
getUrlsByIds(ids: number[]): string[] {
|
||||
if (ids.length === 0) return [];
|
||||
const placeholders = ids.map(() => '?').join(',');
|
||||
const rows = this.db
|
||||
.prepare(`SELECT url FROM urls WHERE id IN (${placeholders})`)
|
||||
.all(...ids) as unknown as { url: string }[];
|
||||
return rows.map((r) => r.url);
|
||||
// Sort then chunk: each chunk query is `ORDER BY id`, so the
|
||||
// concatenated result stays globally id-ordered while staying under
|
||||
// SQLite's bound-parameter limit.
|
||||
const CHUNK = 500;
|
||||
const sorted = [...ids].sort((a, b) => a - b);
|
||||
const out: string[] = [];
|
||||
for (let i = 0; i < sorted.length; i += CHUNK) {
|
||||
const slice = sorted.slice(i, i + CHUNK);
|
||||
const placeholders = slice.map(() => '?').join(',');
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT url FROM urls WHERE id IN (${placeholders}) ORDER BY id`,
|
||||
)
|
||||
.all(...slice) as unknown as { url: string }[];
|
||||
for (const r of rows) out.push(r.url);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
getUrlRowById(id: number): { url: string; depth: number; isExternal: number } | null {
|
||||
@@ -2444,7 +2536,7 @@ export class ProjectDb {
|
||||
offset: number;
|
||||
internal?: 'all' | 'internal' | 'external';
|
||||
search?: string;
|
||||
}): { rows: BrokenLinkRow[]; total: number } {
|
||||
}): { rows: BrokenLinkRow[]; total: number; uniqueTargets: number } {
|
||||
const where: string[] = ['t.status_code >= 400 AND t.status_code < 600'];
|
||||
const args: (string | number)[] = [];
|
||||
const internal = params.internal ?? 'all';
|
||||
@@ -2458,13 +2550,13 @@ export class ProjectDb {
|
||||
const whereSql = `WHERE ${where.join(' AND ')}`;
|
||||
const totalRow = this.db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS c
|
||||
`SELECT COUNT(*) AS c, COUNT(DISTINCT l.to_url) AS u
|
||||
FROM links l
|
||||
JOIN urls f ON l.from_url_id = f.id
|
||||
JOIN urls t ON l.to_url = t.url
|
||||
${whereSql}`,
|
||||
)
|
||||
.get(...args) as { c: number };
|
||||
.get(...args) as { c: number; u: number };
|
||||
const rowsDb = this.db
|
||||
.prepare(
|
||||
`SELECT f.url AS from_url, f.status_code AS from_status,
|
||||
@@ -2488,6 +2580,7 @@ export class ProjectDb {
|
||||
}[];
|
||||
return {
|
||||
total: totalRow.c,
|
||||
uniqueTargets: totalRow.u,
|
||||
rows: rowsDb.map((r) => ({
|
||||
fromUrl: r.from_url,
|
||||
fromStatusCode: r.from_status,
|
||||
@@ -2500,6 +2593,132 @@ export class ProjectDb {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Faz 7 — store one PageSpeed Insights audit result. Keyed by
|
||||
* (url, strategy); re-running an audit overwrites the previous row.
|
||||
*/
|
||||
upsertPagespeedResult(
|
||||
url: string,
|
||||
strategy: PagespeedStrategy,
|
||||
m: PagespeedMetrics,
|
||||
): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO pagespeed_results
|
||||
(url, strategy, performance, lcp, cls, fcp, tbt, speed_index,
|
||||
status, error, fetched_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(url, strategy) DO UPDATE SET
|
||||
performance = excluded.performance,
|
||||
lcp = excluded.lcp,
|
||||
cls = excluded.cls,
|
||||
fcp = excluded.fcp,
|
||||
tbt = excluded.tbt,
|
||||
speed_index = excluded.speed_index,
|
||||
status = excluded.status,
|
||||
error = excluded.error,
|
||||
fetched_at = excluded.fetched_at`,
|
||||
)
|
||||
.run(
|
||||
url,
|
||||
strategy,
|
||||
m.performance,
|
||||
m.lcp,
|
||||
m.cls,
|
||||
m.fcp,
|
||||
m.tbt,
|
||||
m.speedIndex,
|
||||
m.status,
|
||||
m.error,
|
||||
m.fetchedAt,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Faz 7 — candidate query for the PageSpeed tab. Returns crawled
|
||||
* internal HTML 2xx pages (the universe PSI can be run against)
|
||||
* left-joined with any stored mobile + desktop audit results.
|
||||
*/
|
||||
queryPagespeed(params: {
|
||||
limit: number;
|
||||
offset: number;
|
||||
search?: string;
|
||||
filter?: 'all' | 'tested' | 'untested';
|
||||
}): { rows: PagespeedRow[]; total: number } {
|
||||
const where: string[] = [
|
||||
"u.is_external = 0 AND u.content_kind = 'html'",
|
||||
'u.status_code >= 200 AND u.status_code < 300',
|
||||
];
|
||||
const args: (string | number)[] = [];
|
||||
if (params.search) {
|
||||
where.push('u.url LIKE ?');
|
||||
args.push(`%${params.search}%`);
|
||||
}
|
||||
const filter = params.filter ?? 'all';
|
||||
if (filter === 'tested') {
|
||||
where.push('(pm.url IS NOT NULL OR pd.url IS NOT NULL)');
|
||||
} else if (filter === 'untested') {
|
||||
where.push('pm.url IS NULL AND pd.url IS NULL');
|
||||
}
|
||||
const whereSql = `WHERE ${where.join(' AND ')}`;
|
||||
const joinSql = `
|
||||
FROM urls u
|
||||
LEFT JOIN pagespeed_results pm ON pm.url = u.url AND pm.strategy = 'mobile'
|
||||
LEFT JOIN pagespeed_results pd ON pd.url = u.url AND pd.strategy = 'desktop'`;
|
||||
|
||||
const totalRow = this.db
|
||||
.prepare(`SELECT COUNT(*) AS c ${joinSql} ${whereSql}`)
|
||||
.get(...args) as { c: number };
|
||||
|
||||
const rowsDb = this.db
|
||||
.prepare(
|
||||
`SELECT u.url AS url, u.status_code AS status_code,
|
||||
pm.performance AS m_perf, pm.lcp AS m_lcp, pm.cls AS m_cls,
|
||||
pm.fcp AS m_fcp, pm.tbt AS m_tbt, pm.speed_index AS m_si,
|
||||
pm.status AS m_status, pm.error AS m_error, pm.fetched_at AS m_fetched,
|
||||
pd.performance AS d_perf, pd.lcp AS d_lcp, pd.cls AS d_cls,
|
||||
pd.fcp AS d_fcp, pd.tbt AS d_tbt, pd.speed_index AS d_si,
|
||||
pd.status AS d_status, pd.error AS d_error, pd.fetched_at AS d_fetched
|
||||
${joinSql}
|
||||
${whereSql}
|
||||
ORDER BY u.inlinks DESC, u.url ASC
|
||||
LIMIT ? OFFSET ?`,
|
||||
)
|
||||
.all(...args, params.limit, params.offset) as unknown as Record<
|
||||
string,
|
||||
unknown
|
||||
>[];
|
||||
|
||||
const metrics = (
|
||||
r: Record<string, unknown>,
|
||||
p: string,
|
||||
): PagespeedMetrics | null => {
|
||||
const status = r[`${p}_status`];
|
||||
if (typeof status !== 'string') return null;
|
||||
return {
|
||||
performance: (r[`${p}_perf`] as number | null) ?? null,
|
||||
lcp: (r[`${p}_lcp`] as number | null) ?? null,
|
||||
cls: (r[`${p}_cls`] as number | null) ?? null,
|
||||
fcp: (r[`${p}_fcp`] as number | null) ?? null,
|
||||
tbt: (r[`${p}_tbt`] as number | null) ?? null,
|
||||
speedIndex: (r[`${p}_si`] as number | null) ?? null,
|
||||
status: status === 'ok' ? 'ok' : 'error',
|
||||
error: (r[`${p}_error`] as string | null) ?? null,
|
||||
fetchedAt: (r[`${p}_fetched`] as string | null) ?? '',
|
||||
};
|
||||
};
|
||||
|
||||
return {
|
||||
total: totalRow.c,
|
||||
rows: rowsDb.map((r) => ({
|
||||
url: r['url'] as string,
|
||||
statusCode: (r['status_code'] as number | null) ?? null,
|
||||
mobile: metrics(r, 'm'),
|
||||
desktop: metrics(r, 'd'),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
queryImages(params: {
|
||||
limit: number;
|
||||
offset: number;
|
||||
@@ -3656,12 +3875,20 @@ export class ProjectDb {
|
||||
|
||||
*iterateUrlsByIds(ids: number[]): IterableIterator<CrawlUrlRow> {
|
||||
if (ids.length === 0) return;
|
||||
const placeholders = ids.map(() => '?').join(',');
|
||||
const rows = this.db
|
||||
.prepare(`SELECT * FROM urls WHERE id IN (${placeholders}) ORDER BY id`)
|
||||
.all(...ids) as unknown as UrlRowDb[];
|
||||
for (const row of rows) {
|
||||
yield this.rowFromDb(row);
|
||||
// Sort then chunk: keeps the yielded rows globally id-ordered while
|
||||
// staying under SQLite's ~32 K bound-parameter limit (a large table
|
||||
// selection can otherwise throw `too many SQL variables`).
|
||||
const CHUNK = 500;
|
||||
const sorted = [...ids].sort((a, b) => a - b);
|
||||
for (let i = 0; i < sorted.length; i += CHUNK) {
|
||||
const slice = sorted.slice(i, i + CHUNK);
|
||||
const placeholders = slice.map(() => '?').join(',');
|
||||
const rows = this.db
|
||||
.prepare(`SELECT * FROM urls WHERE id IN (${placeholders}) ORDER BY id`)
|
||||
.all(...slice) as unknown as UrlRowDb[];
|
||||
for (const row of rows) {
|
||||
yield this.rowFromDb(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/shared-types",
|
||||
"version": "0.5.5",
|
||||
"version": "0.5.6",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -2,3 +2,4 @@ export * from './crawl.js';
|
||||
export * from './ipc.js';
|
||||
export * from './project.js';
|
||||
export * from './integrations.js';
|
||||
export * from './pagespeed.js';
|
||||
@@ -12,6 +12,10 @@ import type {
|
||||
UrlDetail,
|
||||
} from './crawl.js';
|
||||
import type { IntegrationsState } from './integrations.js';
|
||||
import type {
|
||||
PagespeedRow,
|
||||
PagespeedRunStrategy,
|
||||
} from './pagespeed.js';
|
||||
|
||||
export const IPC = {
|
||||
crawlStart: 'crawl:start',
|
||||
@@ -143,6 +147,16 @@ export const IPC = {
|
||||
* `projectSaveDir` pref when set, otherwise the OS Documents folder.
|
||||
* Used by Settings → Storage to display the active path. */
|
||||
defaultProjectDir: 'app:default-project-dir',
|
||||
/** Faz 7 — Google PageSpeed Insights integration.
|
||||
* `pagespeedQuery` lists crawled internal HTML pages joined with any
|
||||
* stored audit results; `pagespeedRun` audits a user-selected set of
|
||||
* URLs (the slow part — emits `pagespeedProgress` as it goes);
|
||||
* `pagespeedCancel` stops an in-flight run early. */
|
||||
pagespeedQuery: 'pagespeed:query',
|
||||
pagespeedRun: 'pagespeed:run',
|
||||
pagespeedCancel: 'pagespeed:cancel',
|
||||
/** main → renderer: live progress of an in-flight PageSpeed run. */
|
||||
pagespeedProgress: 'pagespeed:progress',
|
||||
} as const;
|
||||
|
||||
export type IpcChannel = (typeof IPC)[keyof typeof IPC];
|
||||
@@ -220,7 +234,13 @@ export interface BrokenLinksQueryInput {
|
||||
|
||||
export interface BrokenLinksQueryResult {
|
||||
rows: BrokenLinkRow[];
|
||||
/** Total broken-link rows (each = one from→to link instance). Scales
|
||||
* with how many pages the crawl reached. */
|
||||
total: number;
|
||||
/** Distinct broken target URLs. Far more stable across crawls than
|
||||
* `total` — one broken URL counts once no matter how many pages link
|
||||
* to it. */
|
||||
uniqueTargets: number;
|
||||
}
|
||||
|
||||
export interface ExportCsvInput {
|
||||
@@ -887,6 +907,54 @@ export interface UrlRewritePreviewResult {
|
||||
parseError?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inputs for the PageSpeed tab's candidate query. Returns crawled
|
||||
* internal HTML pages (the universe PSI can be run against) left-joined
|
||||
* with any audit results already stored for them.
|
||||
*/
|
||||
export interface PagespeedQueryInput {
|
||||
limit: number;
|
||||
offset: number;
|
||||
/** Substring match against the URL. */
|
||||
search?: string;
|
||||
/** `all` (default), `tested` (has ≥1 stored result), `untested`. */
|
||||
filter?: 'all' | 'tested' | 'untested';
|
||||
}
|
||||
|
||||
export interface PagespeedQueryResult {
|
||||
rows: PagespeedRow[];
|
||||
/** Total candidate rows matching the filter (before pagination). */
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface PagespeedRunInput {
|
||||
/** URLs to audit — taken from the candidate rows in the tab. */
|
||||
urls: string[];
|
||||
/** Which form factor(s) to audit. `both` doubles the API calls. */
|
||||
strategy: PagespeedRunStrategy;
|
||||
}
|
||||
|
||||
export interface PagespeedRunResult {
|
||||
/** Number of (url × strategy) audits that completed successfully. */
|
||||
completed: number;
|
||||
/** Number that returned an error (URL unreachable, quota, etc.). */
|
||||
failed: number;
|
||||
/** True when the user cancelled the run before it finished. */
|
||||
cancelled: boolean;
|
||||
}
|
||||
|
||||
/** main → renderer live progress while a PageSpeed run is in flight. */
|
||||
export interface PagespeedProgress {
|
||||
/** Audits finished so far (success + failure). */
|
||||
done: number;
|
||||
/** Total audits in this run (urls × strategy count). */
|
||||
total: number;
|
||||
/** URL currently being audited, or null between items. */
|
||||
currentUrl: string | null;
|
||||
/** False once the run has fully stopped. */
|
||||
running: boolean;
|
||||
}
|
||||
|
||||
export interface LogEntry {
|
||||
/** Monotonic sequence id, increments on every log call this session. */
|
||||
id: number;
|
||||
@@ -993,6 +1061,17 @@ export interface FreeCrawlApi {
|
||||
pickDirectory(input?: { title?: string; defaultPath?: string }): Promise<string | null>;
|
||||
/** Resolved default save directory for new projects (Documents fallback when unset). */
|
||||
defaultProjectDir(): Promise<string>;
|
||||
/** Faz 7 — list crawled internal HTML pages joined with their stored
|
||||
* PageSpeed Insights audit results. */
|
||||
pagespeedQuery(input: PagespeedQueryInput): Promise<PagespeedQueryResult>;
|
||||
/** Run a PageSpeed Insights audit over the given URLs. Resolves when
|
||||
* the whole run finishes; subscribe to `onPagespeedProgress` for
|
||||
* live progress. */
|
||||
pagespeedRun(input: PagespeedRunInput): Promise<PagespeedRunResult>;
|
||||
/** Request cancellation of an in-flight PageSpeed run. */
|
||||
pagespeedCancel(): Promise<void>;
|
||||
/** Live progress of an in-flight PageSpeed run. */
|
||||
onPagespeedProgress(cb: (p: PagespeedProgress) => void): () => void;
|
||||
onLogEntry(cb: (entry: LogEntry) => void): () => void;
|
||||
onLogsBatch(cb: (entries: LogEntry[]) => void): () => void;
|
||||
onLogsBusy(cb: (busy: boolean) => void): () => void;
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Faz 7 — Google PageSpeed Insights integration types.
|
||||
*
|
||||
* PSI runs a Lighthouse audit on a public URL and returns lab
|
||||
* performance metrics. FreeCrawl drives it on-demand against a
|
||||
* user-selected subset of crawled pages (PSI is slow — ~10–30 s per
|
||||
* URL — and rate-limited, so auto-probing a full crawl is never an
|
||||
* option). Results are stored per URL + strategy in the project DB.
|
||||
*
|
||||
* The PSI API works without a key at a low shared rate limit; a free
|
||||
* Google Cloud API key (the `pagespeed` integration in the credential
|
||||
* store) raises the throughput. The key is resolved in the main
|
||||
* process only — it never crosses into the renderer.
|
||||
*/
|
||||
|
||||
/** Which form factor Lighthouse emulates for the audit. */
|
||||
export type PagespeedStrategy = 'mobile' | 'desktop';
|
||||
|
||||
/** A run can target one form factor or both. */
|
||||
export type PagespeedRunStrategy = PagespeedStrategy | 'both';
|
||||
|
||||
/**
|
||||
* Lighthouse lab metrics for one URL + strategy. All timing values are
|
||||
* milliseconds; `cls` is the unitless Cumulative Layout Shift score.
|
||||
* Any metric may be `null` when Lighthouse could not compute it.
|
||||
*/
|
||||
export interface PagespeedMetrics {
|
||||
/** Lighthouse Performance category score, 0–100. */
|
||||
performance: number | null;
|
||||
/** Largest Contentful Paint (ms). */
|
||||
lcp: number | null;
|
||||
/** Cumulative Layout Shift (unitless). */
|
||||
cls: number | null;
|
||||
/** First Contentful Paint (ms). */
|
||||
fcp: number | null;
|
||||
/** Total Blocking Time (ms). */
|
||||
tbt: number | null;
|
||||
/** Speed Index (ms). */
|
||||
speedIndex: number | null;
|
||||
/** `ok` when the audit completed, `error` when PSI/Lighthouse failed. */
|
||||
status: 'ok' | 'error';
|
||||
/** Human-readable failure reason when `status` is `error`. */
|
||||
error: string | null;
|
||||
/** ISO 8601 timestamp of when the audit was fetched. */
|
||||
fetchedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* One row in the PageSpeed tab — a crawled internal HTML page with its
|
||||
* stored audit results. `mobile` / `desktop` are `null` until that
|
||||
* strategy has been run for the URL.
|
||||
*/
|
||||
export interface PagespeedRow {
|
||||
url: string;
|
||||
statusCode: number | null;
|
||||
mobile: PagespeedMetrics | null;
|
||||
desktop: PagespeedMetrics | null;
|
||||
}
|
||||
Reference in new issue
Block a user