feat: multi-account GSC/GA4 + crawl resume & screenshot fixes

This commit is contained in:
kemalai committed 2026-08-08 19:51:02 +03:00
1 parent b91a2b7eae
commit 98cc62154d
35 files changed
+3716 -676

No files matched your search

+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/cli",
"version": "0.9.2",
"version": "0.9.3",
"private": true,
"type": "module",
"bin": {
+54 -22
View File
@@ -26,8 +26,8 @@ Usage:
freecrawl audit-robots <url> [--user-agent UA] Test if a URL is allowed by robots.txt
freecrawl compare <before.seoproject> <after.seoproject>
Cross-project diff (added / removed / status / title / meta / h1 / canonical / indexability / response_time)
freecrawl analyze-logs <access.log> [--project file]
Parse a server access log (Apache / Nginx / IIS / custom), detect bots,
freecrawl analyze-logs <access.log> [more.log …] [--project file]
Parse one or more server access logs (Apache / Nginx / IIS / custom), detect bots,
and merge the aggregates into a project for crawl × log analysis
Options:
@@ -133,16 +133,19 @@ async function main(): Promise<void> {
}
if (positionals[0] === 'analyze-logs') {
if (!positionals[1]) {
console.error('Usage: freecrawl analyze-logs <access.log> [--project file] [--format f] [--regex p] [--verify-bots]');
console.error('Usage: freecrawl analyze-logs <access.log> [more.log …] [--project file] [--format f] [--regex p] [--verify-bots]');
process.exit(2);
}
const exitCode = await runAnalyzeLogs(resolve(positionals[1]), {
projectPath: resolve(values.project ?? values.db ?? 'crawl.seoproject'),
format: (values.format as LogIngestInput['file']['format'] | 'auto' | undefined) ?? 'auto',
regex: values.regex,
verifyBots: Boolean(values['verify-bots']),
json: Boolean(values.json),
});
const exitCode = await runAnalyzeLogs(
positionals.slice(1).map((p) => resolve(p)),
{
projectPath: resolve(values.project ?? values.db ?? 'crawl.seoproject'),
format: (values.format as LogIngestInput['file']['format'] | 'auto' | undefined) ?? 'auto',
regex: values.regex,
verifyBots: Boolean(values['verify-bots']),
json: Boolean(values.json),
},
);
process.exit(exitCode);
}
@@ -526,11 +529,12 @@ function flattenLogResult(filePath: string, result: LogAnalysisResult): LogInges
}
/**
* Parse an access log, detect bots, and merge the aggregates into a
* project DB. Prints a summary (or JSON). Exit 0 on success, 2 on error.
* Parse one or more access logs, detect bots, and merge every file's
* aggregates into the same project DB. Prints a per-file line plus a combined
* summary (or JSON). Exit 0 when at least one file ingested, 2 on error.
*/
async function runAnalyzeLogs(
logPath: string,
logPaths: string[],
opts: {
projectPath: string;
format: 'auto' | 'apache-combined' | 'apache-common' | 'nginx' | 'iis-w3c' | 'custom';
@@ -547,12 +551,37 @@ async function runAnalyzeLogs(
return 2;
}
try {
const result = await analyzeLogFile(logPath, {
format: opts.format,
customRegex: opts.regex,
verifyBots: opts.verifyBots,
});
const summary = db.ingestLogAnalysis(flattenLogResult(logPath, result));
const imported: ReturnType<ProjectDb['ingestLogAnalysis']>[] = [];
const failures: Array<{ file: string; error: string }> = [];
for (const logPath of logPaths) {
try {
const result = await analyzeLogFile(logPath, {
format: opts.format,
customRegex: opts.regex,
verifyBots: opts.verifyBots,
});
const summary = db.ingestLogAnalysis(flattenLogResult(logPath, result));
imported.push(summary);
if (!opts.json) {
console.log(`Log: ${summary.fileName} (${summary.format})`);
console.log(` Lines: ${summary.parsedLines}/${summary.totalLines} parsed, ${summary.skippedLines} skipped`);
}
} catch (err) {
const error = (err as Error).message;
failures.push({ file: logPath, error });
if (!opts.json) console.error(` Skipped ${logPath}: ${error}`);
}
}
if (imported.length === 0) {
if (opts.json) {
process.stdout.write(JSON.stringify({ ok: false, imported: [], errors: failures }, null, 2) + '\n');
} else {
console.error('analyze-logs failed: no files could be ingested.');
}
return 2;
}
const overview = db.getLogOverview();
const bots = db.getLogBots().slice(0, 10);
const topUrls = db
@@ -561,11 +590,14 @@ async function runAnalyzeLogs(
if (opts.json) {
process.stdout.write(
JSON.stringify({ ok: true, imported: summary, overview, topBots: bots, topBotUrls: topUrls }, null, 2) + '\n',
JSON.stringify(
{ ok: true, imported, errors: failures, overview, topBots: bots, topBotUrls: topUrls },
null,
2,
) + '\n',
);
} else {
console.log(`Log: ${summary.fileName} (${summary.format})`);
console.log(` Lines: ${summary.parsedLines}/${summary.totalLines} parsed, ${summary.skippedLines} skipped`);
if (imported.length > 1) console.log(`Ingested ${imported.length} files.`);
console.log(` Hits: ${overview.totalHits} total · ${overview.botHits} bot · ${overview.humanHits} human`);
if (overview.minTs) {
const d = (ms: number): string => new Date(ms).toISOString().slice(0, 16).replace('T', ' ');
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/desktop",
"version": "0.9.2",
"version": "0.9.3",
"private": true,
"description": "FreeCrawl SEO — desktop application",
"homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
+15 -7
View File
@@ -73,6 +73,9 @@ interface PendingRequest {
resolve: (v: unknown) => void;
reject: (e: Error) => void;
timer: ReturnType<typeof setTimeout>;
/** Caller already gave up (timeout) but the worker is still executing
* the query. The slot stays counted as busy until the reply lands. */
abandoned?: boolean;
}
interface ResponseMessage {
@@ -166,14 +169,16 @@ export class DbReaderPool {
slot.pending++;
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
// A timeout frees the *caller*, not the worker: the query is
// already running inside a synchronous SQLite call that cannot
// be cancelled. Dropping the slot's pending count here made
// `pickSlot` treat a worker that was minutes deep in a heavy
// aggregate as idle and pile more work onto it, until every
// worker in the pool was stacked with abandoned queries and no
// UI read could get through. Keep the slot counted as busy and
// release it when the reply actually arrives.
const p = this.pending.get(requestId);
if (p) {
this.pending.delete(requestId);
this.slots[p.workerIdx]!.pending = Math.max(
0,
this.slots[p.workerIdx]!.pending - 1,
);
}
if (p) p.abandoned = true;
reject(new Error(`reader-timeout: ${method} > ${timeoutMs}ms`));
}, timeoutMs);
this.pending.set(requestId, {
@@ -253,6 +258,9 @@ export class DbReaderPool {
clearTimeout(pending.timer);
const slot = this.slots[pending.workerIdx];
if (slot) slot.pending = Math.max(0, slot.pending - 1);
// The caller was already rejected on timeout; this reply only
// releases the slot.
if (pending.abandoned) return;
if (msg.ok) {
pending.resolve(msg.result);
} else {
+17 -2
View File
@@ -55,6 +55,20 @@ const watchdog = init.freezeWatchdogSab
? new FreezeWatchdogSharedState(init.freezeWatchdogSab)
: null;
// Publish into the writer's own slot and keep its own heartbeat. This
// thread used to write the *reader* op string while never ticking any
// heartbeat, which meant a stalled reader pool got logged with whatever
// write happened to run last pinned to it, and a genuinely stuck writer
// produced no evidence at all.
const WATCHDOG_HEARTBEAT_INTERVAL_MS = 100;
if (watchdog) {
watchdog.tickWriterHeartbeat();
watchdog.setWriterOp('idle');
setInterval(() => {
watchdog.tickWriterHeartbeat();
}, WATCHDOG_HEARTBEAT_INTERVAL_MS).unref();
}
interface RequestMessage {
requestId: number;
method: string;
@@ -93,6 +107,7 @@ const ALLOWED_METHODS = new Set<string>([
'checkpointQueue',
'clearQueueCheckpoint',
'walCheckpoint',
'optimize',
'markUrlForRecrawl',
'markUrlsForRecrawl',
'deleteUrl',
@@ -104,7 +119,7 @@ const ALLOWED_METHODS = new Set<string>([
parentPort.on('message', async (msg: RequestMessage) => {
if (!msg || typeof msg.requestId !== 'number') return;
const { requestId, method, args } = msg;
if (watchdog) watchdog.setReaderOp('writer:' + method);
if (watchdog) watchdog.setWriterOp(method);
try {
if (!ALLOWED_METHODS.has(method)) {
throw new Error(`db-writer-worker: method '${method}' is not whitelisted`);
@@ -123,7 +138,7 @@ parentPort.on('message', async (msg: RequestMessage) => {
error: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
});
} finally {
if (watchdog) watchdog.setReaderOp('idle');
if (watchdog) watchdog.setWriterOp('idle');
}
});
@@ -1,12 +1,17 @@
/**
* Shared SharedArrayBuffer layout for the freeze watchdog.
*
* Three execution contexts read/write to the same SAB:
* Four execution contexts read/write to the same SAB:
* 1. The Electron main process (the writer for `mainHB`, counters,
* `mainOp`, and rendererLag forwarded from IPC).
* 2. The db-reader worker thread (writes its own `readerHB` and
* 2. The db-reader worker threads (write their own `readerHB` and
* `readerOp`).
* 3. The freeze-watchdog worker thread (reads everything; writes
* 3. The db-writer worker thread (writes `writerHB` and `writerOp`).
* It used to publish into the *reader* op slot without ever
* ticking the reader heartbeat, so a reader-pool stall could be
* logged with a writer's method name attached to it and the writer
* itself was invisible to the watchdog.
* 4. The freeze-watchdog worker thread (reads everything; writes
* nothing — its job is purely observational).
*
* All scalar reads/writes use `Atomics.{load,store}` so the watchdog
@@ -18,11 +23,11 @@
* thread is alive, and that IS atomic.
*
* Layout (1024 bytes total, generous for future fields):
* [0..32) BigInt64Array(4) — heartbeat timestamps + spare
* [0..32) BigInt64Array(4) — heartbeat timestamps
* [0] main heartbeat (ms since epoch)
* [1] db-reader heartbeat
* [2] last renderer lag-report timestamp
* [3] spare
* [3] db-writer heartbeat
* [32..96) Int32Array(16) — counters + op-string lengths
* [0] renderer lag (ms)
* [1] crawled
@@ -31,10 +36,12 @@
* [4] failed
* [5] main op string byte length
* [6] reader op string byte length
* [7..15] spare
* [7] writer op string byte length
* [8..15] spare
* [96..) Uint8Array — op string bytes (UTF-8)
* [0..256) main op
* [256..512) reader op
* [512..768) writer op
*/
const I64_BYTE_OFFSET = 0;
@@ -45,11 +52,13 @@ const U8_BYTE_OFFSET = 96;
const MAIN_OP_OFFSET_IN_U8 = 0;
const READER_OP_OFFSET_IN_U8 = 256;
const WRITER_OP_OFFSET_IN_U8 = 512;
const OP_MAX_BYTES = 255;
const I64_IDX_MAIN_HB = 0;
const I64_IDX_READER_HB = 1;
const I64_IDX_RENDERER_REPORT_TS = 2;
const I64_IDX_WRITER_HB = 3;
const I32_IDX_RENDERER_LAG = 0;
const I32_IDX_CRAWLED = 1;
@@ -58,6 +67,7 @@ const I32_IDX_PENDING = 3;
const I32_IDX_FAILED = 4;
const I32_IDX_MAIN_OP_LEN = 5;
const I32_IDX_READER_OP_LEN = 6;
const I32_IDX_WRITER_OP_LEN = 7;
export const SAB_BYTE_LENGTH = 1024;
@@ -106,6 +116,10 @@ export class FreezeWatchdogSharedState {
Atomics.store(this.i64, I64_IDX_READER_HB, BigInt(Date.now()));
}
tickWriterHeartbeat(): void {
Atomics.store(this.i64, I64_IDX_WRITER_HB, BigInt(Date.now()));
}
reportRendererLag(lagMs: number): void {
Atomics.store(this.i32, I32_IDX_RENDERER_LAG, lagMs | 0);
Atomics.store(this.i64, I64_IDX_RENDERER_REPORT_TS, BigInt(Date.now()));
@@ -119,6 +133,10 @@ export class FreezeWatchdogSharedState {
this.writeOpString(op, READER_OP_OFFSET_IN_U8, I32_IDX_READER_OP_LEN);
}
setWriterOp(op: string): void {
this.writeOpString(op, WRITER_OP_OFFSET_IN_U8, I32_IDX_WRITER_OP_LEN);
}
updateCounters(c: CounterPatch): void {
if (c.crawled !== undefined) {
Atomics.store(this.i32, I32_IDX_CRAWLED, c.crawled | 0);
@@ -144,6 +162,10 @@ export class FreezeWatchdogSharedState {
return Number(Atomics.load(this.i64, I64_IDX_READER_HB));
}
readWriterHeartbeatMs(): number {
return Number(Atomics.load(this.i64, I64_IDX_WRITER_HB));
}
readRendererReportTsMs(): number {
return Number(Atomics.load(this.i64, I64_IDX_RENDERER_REPORT_TS));
}
@@ -169,6 +191,10 @@ export class FreezeWatchdogSharedState {
return this.readOpString(READER_OP_OFFSET_IN_U8, I32_IDX_READER_OP_LEN);
}
readWriterOp(): string {
return this.readOpString(WRITER_OP_OFFSET_IN_U8, I32_IDX_WRITER_OP_LEN);
}
// ── Internals ─────────────────────────────────────────────────────
private writeOpString(op: string, offset: number, lenIdx: number): void {
+128 -56
View File
@@ -25,8 +25,23 @@ import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js';
const CHECK_INTERVAL_MS = 250;
const STALL_THRESHOLD_MAIN_MS = 500;
const STALL_THRESHOLD_READER_MS = 1000;
const STALL_THRESHOLD_WRITER_MS = 1000;
const STALL_THRESHOLD_RENDERER_LAG_MS = 500;
const STALL_THRESHOLD_RENDERER_SILENCE_MS = 1500;
/**
* How long a stall must persist past detection before it is written to
* the file.
*
* Detection thresholds are deliberately twitchy so the recorded gap and
* op string come from the real onset of a freeze. Writing at that same
* sensitivity is what made the log unreadable: with a 250 ms sample
* interval and a 500 ms main threshold, ordinary GC pauses tripped the
* detector constantly and over a third of the `STALL:MAIN` lines in a
* real capture were `end after 250ms` — noise that buried the
* multi-second freezes worth investigating. Detection is unchanged;
* only the reporting waits for the stall to prove itself.
*/
const STALL_REPORT_FLOOR_MS = 1000;
const HEARTBEAT_LOG_INTERVAL_MS = 5000;
interface InitData {
@@ -68,14 +83,62 @@ function sanitizeOp(op: string): string {
}
interface StallTracker {
/** Gap exceeded the threshold and we're timing it. */
active: boolean;
/** The `start` line has been written — so an `end` line is owed. */
reported: boolean;
startTs: number;
startOp: string;
/** `gap=612ms` / `silence=1709ms`, captured at onset. */
detail: string;
}
const mainStall: StallTracker = { active: false, startTs: 0, startOp: '' };
const readerStall: StallTracker = { active: false, startTs: 0, startOp: '' };
const rendererStall: StallTracker = { active: false, startTs: 0, startOp: '' };
const newTracker = (): StallTracker => ({
active: false,
reported: false,
startTs: 0,
startOp: '',
detail: '',
});
const mainStall = newTracker();
const readerStall = newTracker();
const writerStall = newTracker();
const rendererStall = newTracker();
/**
* Shared start/report/end bookkeeping. `onStart` and `onEnd` build the
* log lines; neither runs unless the stall outlives
* `STALL_REPORT_FLOOR_MS`, so a blip costs nothing but a few field
* writes.
*/
function trackStall(
t: StallTracker,
stalled: boolean,
now: number,
op: string,
detail: string,
onStart: (t: StallTracker) => string,
onEnd: (t: StallTracker, durationMs: number) => string,
): void {
if (stalled) {
if (!t.active) {
t.active = true;
t.reported = false;
t.startTs = now;
t.startOp = op;
t.detail = detail;
} else if (!t.reported && now - t.startTs >= STALL_REPORT_FLOOR_MS) {
t.reported = true;
appendLine(onStart(t));
}
return;
}
if (!t.active) return;
if (t.reported) appendLine(onEnd(t, now - t.startTs));
t.active = false;
t.reported = false;
}
let lastHeartbeatLogTs = 0;
let bootLogged = false;
@@ -85,52 +148,64 @@ function check(): void {
const counters = state.readCounters();
const mainHb = state.readMainHeartbeatMs();
const readerHb = state.readReaderHeartbeatMs();
const writerHb = state.readWriterHeartbeatMs();
const rendererTs = state.readRendererReportTsMs();
const rendererLag = state.readRendererLagMs();
const mainOp = sanitizeOp(state.readMainOp());
const readerOp = sanitizeOp(state.readReaderOp());
const writerOp = sanitizeOp(state.readWriterOp());
const mainGap = mainHb > 0 ? now - mainHb : 0;
const readerGap = readerHb > 0 ? now - readerHb : 0;
const writerGap = writerHb > 0 ? now - writerHb : 0;
const rendererSilence = rendererTs > 0 ? now - rendererTs : 0;
// ── Main thread stall ──
if (mainHb > 0 && mainGap > STALL_THRESHOLD_MAIN_MS) {
if (!mainStall.active) {
mainStall.active = true;
mainStall.startTs = now;
mainStall.startOp = mainOp;
appendLine(
`${isoNow()} [STALL:MAIN start gap=${mainGap}ms] op="${mainOp || '<unknown>'}" ` +
`crawled=${counters.crawled} discovered=${counters.discovered} ` +
`pending=${counters.pending} failed=${counters.failed}`,
);
}
} else if (mainStall.active) {
const dur = now - mainStall.startTs;
appendLine(
`${isoNow()} [STALL:MAIN end after ${dur}ms] startOp="${mainStall.startOp}" endOp="${mainOp}"`,
);
mainStall.active = false;
}
trackStall(
mainStall,
mainHb > 0 && mainGap > STALL_THRESHOLD_MAIN_MS,
now,
mainOp,
`gap=${mainGap}ms`,
(t) =>
`${isoNow()} [STALL:MAIN start ${t.detail}] op="${t.startOp || '<unknown>'}" ` +
`crawled=${counters.crawled} discovered=${counters.discovered} ` +
`pending=${counters.pending} failed=${counters.failed}`,
(t, dur) =>
`${isoNow()} [STALL:MAIN end after ${dur}ms] startOp="${t.startOp}" endOp="${mainOp}"`,
);
// ── DB reader thread stall ──
if (readerHb > 0 && readerGap > STALL_THRESHOLD_READER_MS) {
if (!readerStall.active) {
readerStall.active = true;
readerStall.startTs = now;
readerStall.startOp = readerOp;
appendLine(
`${isoNow()} [STALL:READER start gap=${readerGap}ms] op="${readerOp || '<unknown>'}"`,
);
}
} else if (readerStall.active) {
const dur = now - readerStall.startTs;
appendLine(
`${isoNow()} [STALL:READER end after ${dur}ms] startOp="${readerStall.startOp}" endOp="${readerOp}"`,
);
readerStall.active = false;
}
// ── DB reader pool stall ──
// Every reader worker ticks the same heartbeat, so this only fires
// when *none* of them got a turn — i.e. the whole pool is blocked.
trackStall(
readerStall,
readerHb > 0 && readerGap > STALL_THRESHOLD_READER_MS,
now,
readerOp,
`gap=${readerGap}ms`,
(t) => `${isoNow()} [STALL:READER start ${t.detail}] op="${t.startOp || '<unknown>'}"`,
(t, dur) =>
`${isoNow()} [STALL:READER end after ${dur}ms] startOp="${t.startOp}" endOp="${readerOp}"`,
);
// ── DB writer thread stall ──
// Its own heartbeat and its own op slot: a blocked writer holds up
// every per-URL page write behind it, which surfaces as a crawl that
// has stopped advancing, and that is worth telling apart from a
// blocked reader pool (which only affects what the UI can display).
trackStall(
writerStall,
writerHb > 0 && writerGap > STALL_THRESHOLD_WRITER_MS,
now,
writerOp,
`gap=${writerGap}ms`,
(t) =>
`${isoNow()} [STALL:WRITER start ${t.detail}] op="${t.startOp || '<unknown>'}" ` +
`crawled=${counters.crawled} pending=${counters.pending}`,
(t, dur) =>
`${isoNow()} [STALL:WRITER end after ${dur}ms] startOp="${t.startOp}" endOp="${writerOp}"`,
);
// ── Renderer stall ──
// We treat both an explicit high-lag report AND a long silence
@@ -146,23 +221,17 @@ function check(): void {
rendererStalled = true;
rendererReason = `silence=${rendererSilence}ms`;
}
if (rendererStalled) {
if (!rendererStall.active) {
rendererStall.active = true;
rendererStall.startTs = now;
rendererStall.startOp = mainOp;
appendLine(
`${isoNow()} [STALL:RENDERER start ${rendererReason}] last_main_op="${mainOp || '<unknown>'}" ` +
`crawled=${counters.crawled} pending=${counters.pending}`,
);
}
} else if (rendererStall.active) {
const dur = now - rendererStall.startTs;
appendLine(
`${isoNow()} [STALL:RENDERER end after ${dur}ms] last_lag=${rendererLag}ms`,
);
rendererStall.active = false;
}
trackStall(
rendererStall,
rendererStalled,
now,
mainOp,
rendererReason,
(t) =>
`${isoNow()} [STALL:RENDERER start ${t.detail}] last_main_op="${t.startOp || '<unknown>'}" ` +
`crawled=${counters.crawled} pending=${counters.pending}`,
(_t, dur) => `${isoNow()} [STALL:RENDERER end after ${dur}ms] last_lag=${rendererLag}ms`,
);
// ── Periodic heartbeat (so the file shows the app is alive even
// when nothing has stalled) ──
@@ -171,13 +240,16 @@ function check(): void {
appendLine(
`${isoNow()} [BOOT] freeze-watchdog started ` +
`thresholds: main>${STALL_THRESHOLD_MAIN_MS}ms reader>${STALL_THRESHOLD_READER_MS}ms ` +
`renderer_lag>${STALL_THRESHOLD_RENDERER_LAG_MS}ms renderer_silence>${STALL_THRESHOLD_RENDERER_SILENCE_MS}ms`,
`writer>${STALL_THRESHOLD_WRITER_MS}ms ` +
`renderer_lag>${STALL_THRESHOLD_RENDERER_LAG_MS}ms renderer_silence>${STALL_THRESHOLD_RENDERER_SILENCE_MS}ms ` +
`report_floor=${STALL_REPORT_FLOOR_MS}ms`,
);
}
if (now - lastHeartbeatLogTs >= HEARTBEAT_LOG_INTERVAL_MS) {
lastHeartbeatLogTs = now;
appendLine(
`${isoNow()} [HEARTBEAT] main_op="${mainOp}" reader_op="${readerOp}" ` +
`writer_op="${writerOp}" ` +
`crawled=${counters.crawled} discovered=${counters.discovered} ` +
`pending=${counters.pending} renderer_lag=${rendererLag}ms`,
);
+6 -3
View File
@@ -46,8 +46,10 @@ function apiError(json: unknown, status: number): string {
}
/** Flatten the GA4 Admin `accountSummaries` tree into a flat property list. */
export async function listProperties(): Promise<Ga4Property[]> {
const token = await getAccessToken('ga4');
export async function listProperties(
accountId?: string,
): Promise<Ga4Property[]> {
const token = await getAccessToken('ga4', accountId);
const out: Ga4Property[] = [];
let pageToken: string | undefined;
for (;;) {
@@ -113,8 +115,9 @@ export async function runReport(
property: string,
startDate: string,
endDate: string,
accountId?: string,
): Promise<Ga4PageRow[]> {
const token = await getAccessToken('ga4');
const token = await getAccessToken('ga4', accountId);
const endpoint = `${DATA_BASE}/${property}:runReport`;
const out: Ga4PageRow[] = [];
let offset = 0;
+220 -40
View File
@@ -28,7 +28,7 @@ import { createServer, type Server, type IncomingMessage, type ServerResponse }
import { createHash, randomBytes } from 'node:crypto';
import { join } from 'node:path';
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import type { GoogleAuthState } from '@freecrawl/shared-types';
import type { GoogleAccount, GoogleAuthState } from '@freecrawl/shared-types';
import { apiFetch } from './api-fetch.js';
import { resolveCredentials } from './credentials.js';
import * as logger from './logger.js';
@@ -60,17 +60,43 @@ interface TokenRecord {
refreshToken: string;
email: string | null;
connectedAt: string;
/** Opaque per-account id. Absent on records written before
* multi-account support — `loadStore` backfills those. */
accountId?: string;
}
/** On-disk shape: integration id → encrypted JSON of a `TokenRecord`. */
/**
* On-disk shape: **account key** → encrypted JSON of a `TokenRecord`.
*
* The account key is `<integrationId>#<accountId>`, which is what lets a
* user link several Google accounts to the same integration. Records
* written before multi-account support used the bare integration id as
* the key; `loadStore` migrates those in place on first read.
*/
type TokenFile = Record<string, string>;
/** Separator between the integration id and the account id in a key. */
const KEY_SEP = '#';
let cache: TokenFile | null = null;
/** In-memory short-lived access tokens — never persisted. */
/** In-memory short-lived access tokens — never persisted. Keyed by the
* same account key as the store. */
const accessTokens = new Map<string, { token: string; expiresAt: number }>();
/** Guards against two consent flows running at once. */
let authInProgress = false;
/** Compose the store key for one account of one integration. */
function accountKey(integrationId: string, accountId: string): string {
return `${integrationId}${KEY_SEP}${accountId}`;
}
/** Split a store key back into its integration + account ids. */
function parseKey(key: string): { integrationId: string; accountId: string } | null {
const i = key.indexOf(KEY_SEP);
if (i <= 0) return null;
return { integrationId: key.slice(0, i), accountId: key.slice(i + 1) };
}
function filePath(): string {
return join(app.getPath('userData'), 'google-tokens.enc.json');
}
@@ -83,6 +109,7 @@ function loadStore(): TokenFile {
const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown;
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
cache = parsed as TokenFile;
migrateLegacyKeys();
return cache;
}
}
@@ -97,6 +124,47 @@ function loadStore(): TokenFile {
return cache;
}
/**
* Rewrite pre-multi-account entries — keyed by the bare integration id —
* into `<integrationId>#<accountId>` form so an existing connection
* survives the upgrade as the integration's first linked account. Runs
* once per process, right after the store is first read.
*/
function migrateLegacyKeys(): void {
const store = cache;
if (!store) return;
let changed = false;
for (const key of Object.keys(store)) {
// Already namespaced, or not one of ours — leave alone.
if (key.includes(KEY_SEP) || !(key in INTEGRATION_SCOPES)) continue;
const raw = store[key]!;
let record: TokenRecord | null = null;
try {
record = JSON.parse(decrypt(raw)) as TokenRecord;
} catch {
/* corrupt — drop it below */
}
delete store[key];
changed = true;
if (!record?.refreshToken) continue;
const accountId = record.accountId || newAccountId();
store[accountKey(key, accountId)] = encrypt(
JSON.stringify({ ...record, accountId }),
);
logger.log(
'info',
'main',
`google-oauth: migrated "${key}" to multi-account (${record.email ?? 'unknown account'})`,
);
}
if (changed) persist();
}
/** Short opaque id for a newly linked account. */
function newAccountId(): string {
return randomBytes(6).toString('hex');
}
function persist(): void {
try {
writeFileSync(filePath(), JSON.stringify(cache ?? {}, null, 2), 'utf8');
@@ -132,8 +200,9 @@ function decrypt(stored: string): string {
return '';
}
function readRecord(integrationId: string): TokenRecord | null {
const raw = loadStore()[integrationId];
/** Read one account's token record by its full store key. */
function readRecordByKey(key: string): TokenRecord | null {
const raw = loadStore()[key];
if (!raw) return null;
try {
const json = decrypt(raw);
@@ -150,18 +219,60 @@ function readRecord(integrationId: string): TokenRecord | null {
function writeRecord(integrationId: string, record: TokenRecord): void {
const store = loadStore();
store[integrationId] = encrypt(JSON.stringify(record));
store[accountKey(integrationId, record.accountId ?? newAccountId())] = encrypt(
JSON.stringify(record),
);
persist();
}
/**
* Every account linked to one integration, in connection order. This is
* the list the account dropdowns render.
*/
export function listAccounts(integrationId: string): GoogleAccount[] {
const store = loadStore();
const out: GoogleAccount[] = [];
for (const key of Object.keys(store)) {
const parsed = parseKey(key);
if (!parsed || parsed.integrationId !== integrationId) continue;
const record = readRecordByKey(key);
if (!record) continue;
out.push({
accountId: parsed.accountId,
email: record.email,
connectedAt: record.connectedAt,
});
}
out.sort((a, b) => a.connectedAt.localeCompare(b.connectedAt));
return out;
}
/**
* Resolve which account an API call should use. An explicit id wins; a
* blank one falls back to the first linked account, which is what every
* pre-multi-account caller (and project) implicitly means.
*/
export function resolveAccountId(
integrationId: string,
requested?: string,
): string | null {
const accounts = listAccounts(integrationId);
if (accounts.length === 0) return null;
if (requested && accounts.some((a) => a.accountId === requested)) {
return requested;
}
return accounts[0]!.accountId;
}
/** Redacted connection state for the renderer — no tokens cross IPC. */
export function getAuthState(integrationId: string): GoogleAuthState {
const record = readRecord(integrationId);
if (!record) return { connected: false, email: null, connectedAt: null };
const accounts = listAccounts(integrationId);
const first = accounts[0];
return {
connected: true,
email: record.email,
connectedAt: record.connectedAt,
connected: accounts.length > 0,
email: first?.email ?? null,
connectedAt: first?.connectedAt ?? null,
accounts,
};
}
@@ -174,6 +285,23 @@ function base64url(buf: Buffer): string {
return buf.toString('base64url');
}
/**
* Turn an OAuth `error=` callback code into something the user can act
* on. `access_denied` is the one that matters: with the BYOC model the
* consent screen starts in "Testing", where only accounts on the test
* user list may authorise — so linking a second account fails here
* until that account is added.
*/
function consentError(code: string): string {
if (code === 'access_denied') {
return 'Access blocked by Google (access_denied). If your OAuth consent screen is in "Testing" status, only accounts on its test-user list can connect — add this Google account under Google Cloud Console → APIs & Services → OAuth consent screen → Audience → Test users, then try again. Otherwise you declined the permission prompt.';
}
if (code === 'admin_policy_enforced') {
return 'Your Google Workspace admin blocks this app (admin_policy_enforced). Ask them to allow the Search Console / Analytics scopes for your OAuth client, or connect a personal Google account instead.';
}
return `Consent denied (${code})`;
}
/** Resolve the user's BYOC client id + secret, or throw a clear error. */
function resolveClient(integrationId: string): {
clientId: string;
@@ -240,7 +368,7 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
res.setHeader('Content-Type', 'text/html; charset=utf-8');
if (error || !code) {
res.end(resultPage('FreeCrawl — sign-in cancelled', false));
reject(new Error(error ? `Consent denied (${error})` : 'No authorization code returned'));
reject(new Error(error ? consentError(error) : 'No authorization code returned'));
return;
}
if (returnedState !== stateToken) {
@@ -282,7 +410,15 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
});
setTimeout(() => {
reject(new Error('Timed out waiting for the browser sign-in.'));
// Google blocks a non-test-user at the consent screen without
// ever redirecting to the loopback, so that failure reaches us
// as a timeout rather than an `error=` callback. Say so, since
// it is by far the most common reason this expires.
reject(
new Error(
'Timed out waiting for the browser sign-in. If Google showed "Access blocked / 403 access_denied", add this Google account under Google Cloud Console → APIs & Services → OAuth consent screen → Audience → Test users, then try again.',
),
);
}, CONSENT_TIMEOUT_MS);
});
@@ -319,8 +455,11 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
);
}
// Best-effort: resolve the connected account email for the UI.
// Best-effort: resolve the connected account email for the UI. The
// access token is held aside until the account id is known, since
// the in-memory cache is keyed per account.
let email: string | null = null;
let pendingAccessToken: { token: string; expiresAt: number } | null = null;
if (typeof accessToken === 'string' && accessToken) {
try {
const uiRes = await apiFetch(USERINFO_ENDPOINT, {
@@ -338,21 +477,35 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
typeof tokenJson['expires_in'] === 'number'
? (tokenJson['expires_in'] as number)
: 3600;
accessTokens.set(integrationId, {
pendingAccessToken = {
token: accessToken,
expiresAt: Date.now() + expiresIn * 1000,
});
};
}
// Re-authorising an address that is already linked refreshes that
// account in place rather than creating a duplicate row in every
// dropdown. A brand-new address is appended as another account.
const existing = email
? listAccounts(integrationId).find((a) => a.email === email)
: undefined;
const accountId = existing?.accountId ?? newAccountId();
writeRecord(integrationId, {
refreshToken,
email,
connectedAt: new Date().toISOString(),
connectedAt: existing?.connectedAt ?? new Date().toISOString(),
accountId,
});
if (pendingAccessToken) {
accessTokens.set(accountKey(integrationId, accountId), pendingAccessToken);
}
logger.log(
'info',
'main',
`google-oauth: "${integrationId}" connected${email ? ` as ${email}` : ''}`,
`google-oauth: "${integrationId}" ${existing ? 'reconnected' : 'connected'}${
email ? ` as ${email}` : ''
}`,
);
return getAuthState(integrationId);
} finally {
@@ -374,12 +527,20 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
* token when still fresh, otherwise refreshes it from the stored
* refresh token. Main-process only — never expose the result over IPC.
*/
export async function getAccessToken(integrationId: string): Promise<string> {
const cached = accessTokens.get(integrationId);
export async function getAccessToken(
integrationId: string,
requestedAccountId?: string,
): Promise<string> {
const accountId = resolveAccountId(integrationId, requestedAccountId);
if (!accountId) {
throw new Error(`"${integrationId}" is not connected — sign in with Google first.`);
}
const key = accountKey(integrationId, accountId);
const cached = accessTokens.get(key);
if (cached && cached.expiresAt - TOKEN_EXPIRY_SKEW_MS > Date.now()) {
return cached.token;
}
const record = readRecord(integrationId);
const record = readRecordByKey(key);
if (!record) {
throw new Error(`"${integrationId}" is not connected — sign in with Google first.`);
}
@@ -405,33 +566,52 @@ export async function getAccessToken(integrationId: string): Promise<string> {
}
const token = json['access_token'] as string;
const expiresIn = typeof json['expires_in'] === 'number' ? (json['expires_in'] as number) : 3600;
accessTokens.set(integrationId, {
accessTokens.set(key, {
token,
expiresAt: Date.now() + expiresIn * 1000,
});
return token;
}
/** Disconnect — revoke at Google (best-effort) and wipe the stored token. */
export async function revokeAuth(integrationId: string): Promise<GoogleAuthState> {
const record = readRecord(integrationId);
if (record) {
try {
await apiFetch(`${REVOKE_ENDPOINT}?token=${encodeURIComponent(record.refreshToken)}`, {
method: 'POST',
signal: AbortSignal.timeout(15_000),
});
} catch {
/* best-effort — local wipe below is what matters */
}
}
/**
* Disconnect one linked account — or, when `accountId` is omitted, every
* account for the integration. Revokes at Google (best-effort) and wipes
* the stored refresh token either way.
*/
export async function revokeAuth(
integrationId: string,
accountId?: string,
): Promise<GoogleAuthState> {
const store = loadStore();
if (store[integrationId]) {
delete store[integrationId];
persist();
const targets = accountId
? [accountKey(integrationId, accountId)]
: Object.keys(store).filter(
(k) => parseKey(k)?.integrationId === integrationId,
);
for (const key of targets) {
const record = readRecordByKey(key);
if (record) {
try {
await apiFetch(`${REVOKE_ENDPOINT}?token=${encodeURIComponent(record.refreshToken)}`, {
method: 'POST',
signal: AbortSignal.timeout(15_000),
});
} catch {
/* best-effort — local wipe below is what matters */
}
}
if (store[key]) {
delete store[key];
persist();
}
accessTokens.delete(key);
}
accessTokens.delete(integrationId);
logger.log('info', 'main', `google-oauth: "${integrationId}" disconnected`);
logger.log(
'info',
'main',
`google-oauth: "${integrationId}" disconnected${accountId ? ` (account ${accountId})` : ' (all accounts)'}`,
);
return getAuthState(integrationId);
}
+125 -9
View File
@@ -11,7 +11,11 @@
* (`google-oauth.ts`) — `getAccessToken('gsc')` mints / refreshes the
* bearer token; nothing here touches credentials directly.
*/
import type { GscSite } from '@freecrawl/shared-types';
import type {
GscSite,
GscIntegrationSettings,
GscSearchType,
} from '@freecrawl/shared-types';
import { getAccessToken } from './google-oauth.js';
import { apiFetch } from './api-fetch.js';
import * as logger from './logger.js';
@@ -35,9 +39,9 @@ function apiError(json: unknown, status: number): string {
return `Search Console API error (HTTP ${status})`;
}
/** List the connected account's Search Console properties. */
export async function listSites(): Promise<GscSite[]> {
const token = await getAccessToken('gsc');
/** List one connected account's Search Console properties. */
export async function listSites(accountId?: string): Promise<GscSite[]> {
const token = await getAccessToken('gsc', accountId);
const res = await apiFetch(`${API_BASE}/sites`, {
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(30_000),
@@ -65,20 +69,107 @@ export interface GscPageRow {
position: number;
}
/** One API dimension filter (`dimensionFilterGroups[].filters[]`). */
interface GscDimensionFilter {
dimension: string;
operator: string;
expression: string;
}
/**
* Shaping options for a Search Analytics pull, derived from the
* per-project GSC settings (search type, device/country/query dimension
* filters, total-row cap). Empty options reproduce the previous
* behaviour exactly (web search, no filters, 200 K row ceiling).
*/
export interface GscQueryOptions {
type?: GscSearchType;
dimensionFilterGroups?: { groupType: 'and'; filters: GscDimensionFilter[] }[];
/** Total-row cap across all pages (SF "Limit Max Results"). */
maxRows?: number;
}
/**
* Resolve the `[startDate, endDate]` window from the settings' date
* range. Non-custom ranges end ~2 days ago (Search Console's data lag)
* and span back the preset's width; `16m` uses the API's 16-month max.
*/
export function resolveGscDateRange(settings: GscIntegrationSettings): {
startDate: string;
endDate: string;
} {
const isoDate = (d: Date): string => d.toISOString().slice(0, 10);
if (settings.dateRange === 'custom' && settings.startDate && settings.endDate) {
return { startDate: settings.startDate, endDate: settings.endDate };
}
const daysBack =
settings.dateRange === '7d'
? 7
: settings.dateRange === '90d'
? 90
: settings.dateRange === '16m'
? 480
: 28;
const end = new Date(Date.now() - 2 * 86_400_000);
const start = new Date(end.getTime() - (daysBack - 1) * 86_400_000);
return { startDate: isoDate(start), endDate: isoDate(end) };
}
/** Build the API query options (type + dimension filters + row cap) from
* the per-project GSC settings. */
export function buildGscQueryOptions(
settings: GscIntegrationSettings,
): GscQueryOptions {
const filters: GscDimensionFilter[] = [];
if (settings.deviceFilter !== 'all') {
filters.push({
dimension: 'device',
operator: 'equals',
expression: settings.deviceFilter,
});
}
if (settings.countryFilter) {
filters.push({
dimension: 'country',
operator: 'equals',
expression: settings.countryFilter,
});
}
if (settings.queryFilterMode !== 'none' && settings.queryFilterValue) {
filters.push({
dimension: 'query',
operator: settings.queryFilterMode,
expression: settings.queryFilterValue,
});
}
const opts: GscQueryOptions = { type: settings.searchType };
if (filters.length > 0) {
opts.dimensionFilterGroups = [{ groupType: 'and', filters }];
}
if (settings.limitMaxResults) {
opts.maxRows = Math.max(1, Math.min(1_000_000, settings.maxResults || 100000));
}
return opts;
}
/**
* Pull per-page Search Console metrics for `[startDate, endDate]`
* (`YYYY-MM-DD`, inclusive). Pages transparently past the 25 000-row
* API limit and returns the flattened list.
* API limit and returns the flattened list. `opts` shapes the query
* (search type, dimension filters, total-row cap).
*/
export async function querySearchAnalytics(
property: string,
startDate: string,
endDate: string,
opts: GscQueryOptions = {},
accountId?: string,
): Promise<GscPageRow[]> {
const token = await getAccessToken('gsc');
const token = await getAccessToken('gsc', accountId);
const endpoint = `${API_BASE}/sites/${encodeURIComponent(
property,
)}/searchAnalytics/query`;
const maxRows = Math.max(1, Math.min(1_000_000, opts.maxRows ?? MAX_ROWS));
const out: GscPageRow[] = [];
let startRow = 0;
@@ -95,6 +186,10 @@ export async function querySearchAnalytics(
dimensions: ['page'],
rowLimit: ROW_LIMIT,
startRow,
...(opts.type ? { type: opts.type } : {}),
...(opts.dimensionFilterGroups
? { dimensionFilterGroups: opts.dimensionFilterGroups }
: {}),
}),
signal: AbortSignal.timeout(60_000),
});
@@ -121,9 +216,10 @@ export async function querySearchAnalytics(
position: Number(r.position) || 0,
});
}
if (rows.length < ROW_LIMIT || out.length >= MAX_ROWS) break;
if (rows.length < ROW_LIMIT || out.length >= maxRows) break;
startRow += ROW_LIMIT;
}
if (out.length > maxRows) out.length = maxRows;
logger.log(
'info',
'gsc',
@@ -141,6 +237,9 @@ export interface GscInspectionRaw {
lastCrawlTime: string | null;
googleCanonical: string | null;
userCanonical: string | null;
mobileVerdict: string | null;
ampVerdict: string | null;
richResultsVerdict: string | null;
}
/**
@@ -150,8 +249,9 @@ export interface GscInspectionRaw {
export async function inspectUrl(
property: string,
url: string,
accountId?: string,
): Promise<GscInspectionRaw> {
const token = await getAccessToken('gsc');
const token = await getAccessToken('gsc', accountId);
const res = await apiFetch(
'https://searchconsole.googleapis.com/v1/urlInspection/index:inspect',
{
@@ -165,12 +265,25 @@ export async function inspectUrl(
},
);
const json = (await res.json().catch(() => null)) as {
inspectionResult?: { indexStatusResult?: Record<string, unknown> };
inspectionResult?: {
indexStatusResult?: Record<string, unknown>;
mobileUsabilityResult?: Record<string, unknown>;
ampResult?: Record<string, unknown>;
richResultsResult?: Record<string, unknown>;
};
} | null;
if (!res.ok || !json) throw new Error(apiError(json, res.status));
const idx = json.inspectionResult?.indexStatusResult ?? {};
const s = (k: string): string | null =>
typeof idx[k] === 'string' ? (idx[k] as string) : null;
// The mobile / AMP / rich-result facets each carry their own verdict;
// absent facets (page has no AMP, no rich results) stay null.
const facetVerdict = (
facet: Record<string, unknown> | undefined,
): string | null =>
facet && typeof facet['verdict'] === 'string'
? (facet['verdict'] as string)
: null;
return {
verdict: s('verdict'),
coverageState: s('coverageState'),
@@ -179,5 +292,8 @@ export async function inspectUrl(
lastCrawlTime: s('lastCrawlTime'),
googleCanonical: s('googleCanonical'),
userCanonical: s('userCanonical'),
mobileVerdict: facetVerdict(json.inspectionResult?.mobileUsabilityResult),
ampVerdict: facetVerdict(json.inspectionResult?.ampResult),
richResultsVerdict: facetVerdict(json.inspectionResult?.richResultsResult),
};
}
+499 -95
View File
@@ -21,6 +21,7 @@ import { AsyncLocalStorage } from 'node:async_hooks';
import { fileURLToPath } from 'node:url';
import { basename, dirname, join, resolve as resolvePath, sep as pathSep } from 'node:path';
import {
cpSync,
mkdirSync,
readFileSync,
readdirSync,
@@ -59,6 +60,7 @@ import {
type ExportGridResult,
type DataDeleteByDomainInput,
type DataDeleteByDomainResult,
type CrashRecoveryResumeResult,
type CrashRecoveryStatus,
type ExportHtmlReportInput,
type ExportHtmlReportResult,
@@ -148,6 +150,12 @@ import {
type GscFetchMeta,
type GscQueryInput,
type GscQueryResult,
type GscCrawlNewUrlsResult,
type GscIntegrationSettings,
type Ga4IntegrationSettings,
type GoogleAccount,
defaultGscSettings,
defaultGa4Settings,
type Ga4ListPropertiesResult,
type Ga4FetchInput,
type Ga4FetchResult,
@@ -258,8 +266,20 @@ import {
} from './languagetool.js';
import { warmLanguageDetector } from './language-detect.js';
import { warmLocalDictionary } from './local-spell.js';
import { startAuth, getAuthState, revokeAuth } from './google-oauth.js';
import { listSites, querySearchAnalytics, inspectUrl } from './gsc.js';
import {
startAuth,
getAuthState,
revokeAuth,
listAccounts,
resolveAccountId,
} from './google-oauth.js';
import {
listSites,
querySearchAnalytics,
inspectUrl,
resolveGscDateRange,
buildGscQueryOptions,
} from './gsc.js';
import { exportCategoryToSheets } from './sheets-export.js';
import { exportCategoryToBigQuery } from './bigquery-export.js';
import { listProperties as ga4ListProperties, runReport as ga4RunReport } from './ga4.js';
@@ -473,6 +493,16 @@ class ProjectSession {
this.pendingRecoveryCheckpoint = [];
}
database.reset();
// The scratch database is wiped on every launch, so the PNGs the last
// session captured into its sidecar are now unreferenced. Drop them with
// it — otherwise the directory in userData grows forever across crawls
// of unrelated sites. A saved project keeps its own copy (see
// `saveProjectDocument`), so nothing the user chose to keep is at risk.
try {
rmSync(screenshotDirFor(path), { recursive: true, force: true });
} catch {
/* best-effort — a locked file just means it is cleaned next launch */
}
if (ramMode) {
logger.log(
'info',
@@ -1466,6 +1496,27 @@ function markDirty(s: ProjectSession): void {
updateWindowTitle(s);
}
/**
* V2 Faz 1 — Screenshot PNGs are too big to sit in the database, so they
* live on disk in a sidecar directory named after the file the session is
* bound to: `<name>.screenshots/`.
*/
function screenshotDirFor(basePath: string): string {
return join(dirname(basePath), `${basename(basePath)}.screenshots`);
}
/**
* Sidecar for whatever this session is actually writing to. An unsaved
* session has no document, so it falls back to the working/scratch database
* in userData — keying this on the document alone meant screenshot capture
* was silently disabled for every crawl until the user saved a project,
* which is the common case and left the Screenshot tab permanently empty.
*/
function activeScreenshotDir(s: ProjectSession): string | null {
const base = s.currentProjectPath || s.scratchPath;
return base ? screenshotDirFor(base) : null;
}
/**
* Write the working database to `documentPath` as a compressed container.
*
@@ -1477,8 +1528,46 @@ function markDirty(s: ProjectSession): void {
async function saveProjectDocument(
s: ProjectSession,
documentPath: string,
opts: {
/**
* False for the MCP snapshot action, which writes a copy without moving
* the session onto it. The sidecar handling below rewrites the LIVE
* database, so it must only run when this save is the session's new
* home — a snapshot doing it would leave the open project's rows
* pointing into the snapshot's directory.
*/
rebind?: boolean;
} = {},
): Promise<{ bytesWritten: number; sourceBytes: number }> {
const database = s.getDb();
// Screenshots sit beside the document rather than inside the archive, so
// saving has to bring them along and re-point the rows at the new sidecar
// — otherwise the saved copy names a directory that belongs to the
// previous location and its Screenshot tab comes up blank. Covers the
// first save too: captures taken while unsaved live in the userData
// scratch sidecar. Runs BEFORE the snapshot below so the archive carries
// the rewritten paths.
const previousShotDir = activeScreenshotDir(s);
const nextShotDir = screenshotDirFor(documentPath);
if (
opts.rebind !== false &&
previousShotDir &&
previousShotDir !== nextShotDir &&
existsSync(previousShotDir)
) {
try {
cpSync(previousShotDir, nextShotDir, { recursive: true });
database.remapScreenshotDir(previousShotDir, nextShotDir);
} catch (err) {
logger.log(
'warn',
'main',
`Could not move the screenshots sidecar to ${nextShotDir}: ${
err instanceof Error ? err.message : String(err)
} — the saved project will have no screenshots.`,
);
}
}
try {
database.walCheckpoint();
} catch {
@@ -3139,8 +3228,15 @@ function registerIpc(): void {
registerHandle(
IPC.logAnalyze,
async (e, input: LogAnalyzeInput): Promise<LogAnalyzeResult> => {
let filePath = input?.filePath;
if (!filePath) {
// Resolve the file list: explicit `filePaths` (bulk) wins, then a single
// `filePath`, then a multi-select picker.
let filePaths =
input?.filePaths && input.filePaths.length > 0
? input.filePaths
: input?.filePath
? [input.filePath]
: undefined;
if (!filePaths) {
const parent =
BrowserWindow.fromWebContents(e.sender) ?? logAnalyzerWindow ?? mainWindow;
if (!parent) {
@@ -3148,7 +3244,7 @@ function registerIpc(): void {
}
const res = await dialog.showOpenDialog(parent, {
title: L().dlgOpenAccessLogTitle,
properties: ['openFile'],
properties: ['openFile', 'multiSelections'],
filters: [
{ name: 'Log Files', extensions: ['log', 'txt', 'gz', 'out', 'access'] },
{ name: 'All Files', extensions: ['*'] },
@@ -3157,30 +3253,52 @@ function registerIpc(): void {
if (res.canceled || res.filePaths.length === 0) {
return { ok: false, error: undefined, imported: null, overview: getDb().getLogOverview() };
}
filePath = res.filePaths[0]!;
filePaths = res.filePaths;
}
try {
const result = await analyzeLogFile(filePath, {
format: input?.format ?? 'auto',
customRegex: input?.customRegex,
verifyBots: input?.verifyBots === true,
});
const imported = ingestLogResult(filePath, result);
if (result.urlCapHit) {
logger.log('warn', 'main', `Log Analyzer: distinct-URL cap hit while ingesting ${basename(filePath)}; some long-tail URLs were dropped.`);
// Ingest each file in turn; one bad file must not abort the batch.
const batch: LogImportSummary[] = [];
const errors: Array<{ fileName: string; error: string }> = [];
for (const filePath of filePaths) {
try {
const result = await analyzeLogFile(filePath, {
format: input?.format ?? 'auto',
customRegex: input?.customRegex,
verifyBots: input?.verifyBots === true,
});
batch.push(ingestLogResult(filePath, result));
if (result.urlCapHit) {
logger.log('warn', 'main', `Log Analyzer: distinct-URL cap hit while ingesting ${basename(filePath)}; some long-tail URLs were dropped.`);
}
logger.log(
'info',
'main',
`Log Analyzer ingested ${basename(filePath)} (${result.format}): ${result.parsedLines}/${result.totalLines} lines, ${result.bots.size} bot type(s).`,
);
} catch (err) {
const error = err instanceof Error ? err.message : String(err);
logger.log('warn', 'main', `Log Analyzer failed on ${filePath}: ${error}`);
errors.push({ fileName: basename(filePath), error });
}
logger.log(
'info',
'main',
`Log Analyzer ingested ${basename(filePath)} (${result.format}): ${result.parsedLines}/${result.totalLines} lines, ${result.bots.size} bot type(s).`,
);
fireDataChanged();
return { ok: true, imported, overview: getDb().getLogOverview() };
} catch (err) {
const error = err instanceof Error ? err.message : String(err);
logger.log('warn', 'main', `Log Analyzer failed on ${filePath}: ${error}`);
return { ok: false, error, imported: null, overview: getDb().getLogOverview() };
}
fireDataChanged();
const overview = getDb().getLogOverview();
if (batch.length === 0) {
return {
ok: false,
error: errors[0]?.error ?? 'No log files could be ingested.',
imported: null,
batch: [],
errors,
overview,
};
}
return {
ok: true,
imported: batch[0] ?? null,
batch,
errors: errors.length > 0 ? errors : undefined,
overview,
};
},
);
@@ -3271,9 +3389,11 @@ function registerIpc(): void {
// naive startsWith() lets `<root>/../../../etc/passwd` pass.
const resolved = resolvePath(absolutePath);
const allowedRoots: string[] = [];
if (currentSession().currentProjectPath) {
const dir = resolveScreenshotDir(currentSession().currentProjectPath);
if (dir) allowedRoots.push(resolvePath(dir));
const sess = currentSession();
// Both sidecars: the document's own, and the working/scratch one —
// rows captured before the project was ever saved point at the latter.
for (const base of [sess.currentProjectPath, sess.scratchPath]) {
if (base) allowedRoots.push(resolvePath(screenshotDirFor(base)));
}
// Append the separator so that `<root>` doesn't match `<root>2`.
const safe = allowedRoots.some(
@@ -3733,12 +3853,20 @@ function registerIpc(): void {
registerHandle(
IPC.reportsOrphanCrossSource,
(_e, limit?: number): Promise<OrphanCrossSourceRow[]> =>
callReaderOrFallback<OrphanCrossSourceRow[]>(
(_e, limit?: number): Promise<OrphanCrossSourceRow[]> => {
// Scoped to the project's selected accounts — the cross-source
// joins are only 1:1 when pinned to one GSC + one GA4 account.
const args: [number, string, string] = [
limit ?? 1000,
resolveGoogleAccount('gsc'),
resolveGoogleAccount('ga4'),
];
return callReaderOrFallback<OrphanCrossSourceRow[]>(
'orphanPagesCrossSource',
[limit ?? 1000],
() => getDb().orphanPagesCrossSource(limit ?? 1000),
),
args,
() => getDb().orphanPagesCrossSource(...args),
);
},
);
registerHandle(IPC.reportsServerHeaders, () =>
@@ -3899,6 +4027,19 @@ function registerIpc(): void {
return clearIntegrationCredentials(id);
});
// Per-project, per-integration behaviour settings (date ranges, filter
// options, "crawl new URLs"). Stored as a JSON blob in project_meta
// under `integrationSettings:<id>` — separate from the encrypted,
// app-global credential store above.
registerHandle(IPC.integrationSettingsGet, (_e, id: string) =>
readIntegrationSettings(id),
);
registerHandle(
IPC.integrationSettingsSet,
(_e, id: string, settings: Record<string, unknown>) =>
writeIntegrationSettings(id, settings),
);
// V1 Faz 7 — Google PageSpeed Insights. `pagespeedQuery` lists the
// crawled internal HTML pages joined with any stored audit results;
// `pagespeedRun` audits a user-selected subset (the slow part — PSI
@@ -4494,7 +4635,8 @@ function registerIpc(): void {
);
registerHandle(
IPC.googleAuthRevoke,
(_e, id: string): Promise<GoogleAuthState> => revokeAuth(id),
(_e, id: string, accountId?: string): Promise<GoogleAuthState> =>
revokeAuth(id, accountId),
);
// V1 Faz 7 — Google Search Console. `gscFetch` pulls per-page metrics
@@ -4502,9 +4644,10 @@ function registerIpc(): void {
// `gscQuery` lists crawled pages joined with that data.
registerHandle(
IPC.gscListSites,
async (): Promise<GscListSitesResult> => {
async (_e, accountId?: string): Promise<GscListSitesResult> => {
try {
return { ok: true, error: null, sites: await listSites() };
const account = resolveGoogleAccount('gsc', accountId);
return { ok: true, error: null, sites: await listSites(account) };
} catch (err) {
return {
ok: false,
@@ -4514,6 +4657,10 @@ function registerIpc(): void {
}
},
);
registerHandle(
IPC.googleAccountsList,
(_e, integrationId: string): GoogleAccount[] => listAccounts(integrationId),
);
registerHandle(
IPC.gscFetch,
async (_e, input: GscFetchInput): Promise<GscFetchResult> => {
@@ -4525,23 +4672,27 @@ function registerIpc(): void {
meta: null,
};
}
// Search Console data lags ~2 days — end the window there and
// span `days` back so the range only covers data that exists.
const isoDate = (d: Date): string => d.toISOString().slice(0, 10);
const end = new Date(Date.now() - 2 * 86_400_000);
const start = new Date(end.getTime() - (input.days - 1) * 86_400_000);
const startDate = isoDate(start);
const endDate = isoDate(end);
// Pin the DB to the project active when the pull started.
// The date range + dimension filters + row cap come from the stored
// per-project GSC settings (Settings → Integrations). Persist the
// property the user pulled from so it's remembered next session.
const settings = resolveGscSettings();
const accountId = resolveGoogleAccount('gsc', input.accountId);
writeIntegrationSettings('gsc', { property: input.property, accountId });
const { startDate, endDate } = resolveGscDateRange(settings);
const queryOpts = buildGscQueryOptions(settings);
// Pin the DB + session to the project active when the pull started.
const runDb = getDb();
const runSession = currentSession();
try {
const rows = await querySearchAnalytics(
input.property,
startDate,
endDate,
queryOpts,
accountId,
);
const fetchedAt = new Date().toISOString();
runDb.replaceGscResults(rows, fetchedAt);
runDb.replaceGscResults(rows, fetchedAt, accountId);
const meta: GscFetchMeta = {
property: input.property,
startDate,
@@ -4549,14 +4700,38 @@ function registerIpc(): void {
fetchedAt,
rowCount: rows.length,
};
runDb.setMeta('gscFetchMeta', JSON.stringify(meta));
runDb.setMeta(googleMetaKey('gscFetchMeta', accountId), JSON.stringify(meta));
fireDataChanged();
logger.log(
'info',
'gsc',
`Search Console pull stored — ${rows.length} page row(s)`,
);
return { ok: true, error: null, rowCount: rows.length, meta };
// How many GSC URLs are missing from the crawl — the candidates
// for "Crawl New URLs Discovered In Google Search Console".
const orphans = runDb.gscOrphanUrls({
matchSlash: settings.matchSlash,
matchCase: settings.matchCase,
accountId,
});
let queuedNewUrls = 0;
if (settings.crawlNewUrls && orphans.length > 0) {
await launchRespider(runSession, orphans);
queuedNewUrls = orphans.length;
logger.log(
'info',
'gsc',
`Crawling ${orphans.length} new URL(s) discovered in Search Console`,
);
}
return {
ok: true,
error: null,
rowCount: rows.length,
meta,
newUrlCount: orphans.length,
queuedNewUrls,
};
} catch (err) {
return {
ok: false,
@@ -4567,14 +4742,47 @@ function registerIpc(): void {
}
},
);
registerHandle(
IPC.gscCrawlNewUrls,
async (): Promise<GscCrawlNewUrlsResult> => {
const settings = resolveGscSettings();
const runSession = currentSession();
try {
const orphans = getDb().gscOrphanUrls({
matchSlash: settings.matchSlash,
matchCase: settings.matchCase,
accountId: resolveGoogleAccount('gsc'),
});
const hasActiveCrawl = runSession.activeCrawler?.isRunning === true;
if (orphans.length > 0) await launchRespider(runSession, orphans);
return {
ok: true,
error: null,
candidateCount: orphans.length,
queued: orphans.length,
hasActiveCrawl,
};
} catch (err) {
return {
ok: false,
error: err instanceof Error ? err.message : String(err),
candidateCount: 0,
queued: 0,
hasActiveCrawl: false,
};
}
},
);
registerHandle(
IPC.gscQuery,
async (_e, input: GscQueryInput): Promise<GscQueryResult> => {
const accountId = resolveGoogleAccount('gsc', input.accountId);
const params = {
limit: input.limit,
offset: input.offset,
search: input.search,
filter: input.filter,
accountId,
};
const result = await callReaderOrFallback<{
rows: GscQueryResult['rows'];
@@ -4582,7 +4790,7 @@ function registerIpc(): void {
}>('queryGsc', [params], () => getDb().queryGsc(params));
let meta: GscFetchMeta | null = null;
try {
const raw = getDb().getMeta('gscFetchMeta');
const raw = getDb().getMeta(googleMetaKey('gscFetchMeta', accountId));
if (raw) meta = JSON.parse(raw) as GscFetchMeta;
} catch {
/* no / corrupt meta — render without the context line */
@@ -4596,9 +4804,14 @@ function registerIpc(): void {
// stored data. Auth runs through the shared OAuth keystone.
registerHandle(
IPC.ga4ListProperties,
async (): Promise<Ga4ListPropertiesResult> => {
async (_e, accountId?: string): Promise<Ga4ListPropertiesResult> => {
try {
return { ok: true, error: null, properties: await ga4ListProperties() };
const account = resolveGoogleAccount('ga4', accountId);
return {
ok: true,
error: null,
properties: await ga4ListProperties(account),
};
} catch (err) {
return {
ok: false,
@@ -4626,10 +4839,22 @@ function registerIpc(): void {
const startDate = isoDate(start);
const endDate = isoDate(end);
const runDb = getDb();
const accountId = resolveGoogleAccount('ga4', input.accountId);
writeIntegrationSettings('ga4', {
accountId,
property: input.property,
propertyName: input.propertyName || input.property,
days: input.days,
});
try {
const rows = await ga4RunReport(input.property, startDate, endDate);
const rows = await ga4RunReport(
input.property,
startDate,
endDate,
accountId,
);
const fetchedAt = new Date().toISOString();
runDb.replaceGa4Results(rows, fetchedAt);
runDb.replaceGa4Results(rows, fetchedAt, accountId);
const meta: Ga4FetchMeta = {
property: input.property,
propertyName: input.propertyName || input.property,
@@ -4638,7 +4863,7 @@ function registerIpc(): void {
fetchedAt,
rowCount: rows.length,
};
runDb.setMeta('ga4FetchMeta', JSON.stringify(meta));
runDb.setMeta(googleMetaKey('ga4FetchMeta', accountId), JSON.stringify(meta));
fireDataChanged();
logger.log(
'info',
@@ -4659,11 +4884,13 @@ function registerIpc(): void {
registerHandle(
IPC.ga4Query,
async (_e, input: Ga4QueryInput): Promise<Ga4QueryResult> => {
const accountId = resolveGoogleAccount('ga4', input.accountId);
const params = {
limit: input.limit,
offset: input.offset,
search: input.search,
filter: input.filter,
accountId,
};
const result = await callReaderOrFallback<{
rows: Ga4QueryResult['rows'];
@@ -4671,7 +4898,7 @@ function registerIpc(): void {
}>('queryGa4', [params], () => getDb().queryGa4(params));
let meta: Ga4FetchMeta | null = null;
try {
const raw = getDb().getMeta('ga4FetchMeta');
const raw = getDb().getMeta(googleMetaKey('ga4FetchMeta', accountId));
if (raw) meta = JSON.parse(raw) as Ga4FetchMeta;
} catch {
/* no / corrupt meta — render without the context line */
@@ -5050,6 +5277,7 @@ function registerIpc(): void {
return { completed: 0, failed: 0, cancelled: false };
}
const runDb = runSession.getDb();
const inspectAccountId = resolveGoogleAccount('gsc', input.accountId);
run.active = true;
run.cancelRequested = false;
let completed = 0;
@@ -5077,14 +5305,14 @@ function registerIpc(): void {
});
const fetchedAt = new Date().toISOString();
try {
const raw = await inspectUrl(input.property, url);
const raw = await inspectUrl(input.property, url, inspectAccountId);
const result: GscInspectionResult = {
...raw,
status: 'ok',
error: null,
fetchedAt,
};
runDb.upsertGscInspection(url, result);
runDb.upsertGscInspection(url, result, inspectAccountId);
completed++;
} catch (err) {
runDb.upsertGscInspection(url, {
@@ -5095,10 +5323,13 @@ function registerIpc(): void {
lastCrawlTime: null,
googleCanonical: null,
userCanonical: null,
mobileVerdict: null,
ampVerdict: null,
richResultsVerdict: null,
status: 'error',
error: err instanceof Error ? err.message : String(err),
fetchedAt,
});
}, inspectAccountId);
failed++;
}
done++;
@@ -5165,10 +5396,17 @@ function registerIpc(): void {
IPC.urlAnalyticsGet,
async (_e, url: string): Promise<UrlAnalyticsDetail | null> => {
if (!url) return null;
// Scoped to the project's selected accounts so the panel agrees
// with what the Search Console / GA4 tabs are showing.
const args: [string, string, string] = [
url,
resolveGoogleAccount('gsc'),
resolveGoogleAccount('ga4'),
];
return callReaderOrFallback<UrlAnalyticsDetail | null>(
'getUrlAnalytics',
[url],
() => getDb().getUrlAnalytics(url),
args,
() => getDb().getUrlAnalytics(...args),
);
},
);
@@ -5527,7 +5765,7 @@ function registerIpc(): void {
renderHook = undefined;
}
}
const screenshotDir = resolveScreenshotDir(currentSession().currentProjectPath);
const screenshotDir = activeScreenshotDir(sess);
const wantsScreenshot = config.jsRender.screenshotMode !== 'none';
const wantsMobile =
config.jsRender.mobileScreenshot || config.jsRender.mobileUsability;
@@ -5684,6 +5922,129 @@ function registerIpc(): void {
* different site is being crawled, and the re-spider must never read as
* one.
*/
/** project_meta key for one integration's per-project settings blob. */
function integrationSettingsKey(id: string): string {
return `integrationSettings:${id}`;
}
/** Read one integration's stored settings blob (or null). */
function readIntegrationSettings(
id: string,
): Record<string, unknown> | null {
const raw = getDb().getMeta(integrationSettingsKey(id));
if (!raw) return null;
try {
const parsed = JSON.parse(raw) as unknown;
return parsed && typeof parsed === 'object'
? (parsed as Record<string, unknown>)
: null;
} catch {
return null;
}
}
/** Merge-save one integration's settings blob; returns the merged result. */
function writeIntegrationSettings(
id: string,
settings: Record<string, unknown>,
): Record<string, unknown> {
const merged = { ...(readIntegrationSettings(id) ?? {}), ...settings };
getDb().setMeta(integrationSettingsKey(id), JSON.stringify(merged));
return merged;
}
/** Resolve the effective GSC settings (defaults + stored overrides). */
function resolveGscSettings(): GscIntegrationSettings {
return {
...defaultGscSettings(),
...(readIntegrationSettings('gsc') ?? {}),
} as GscIntegrationSettings;
}
/** Resolve the effective GA4 settings (defaults + stored overrides). */
function resolveGa4Settings(): Ga4IntegrationSettings {
return {
...defaultGa4Settings(),
...(readIntegrationSettings('ga4') ?? {}),
} as Ga4IntegrationSettings;
}
/**
* Which linked Google account a call should act on. An explicit id from
* the caller wins, then the project's stored choice, then the first
* linked account (what every pre-multi-account project means).
*
* Also folds in the legacy-row adoption: the first time an account is
* resolved for an integration, any rows still tagged `account_id = ''`
* — written before multi-account existed — are relabelled to it, so an
* upgraded project keeps the data it already pulled.
*/
function resolveGoogleAccount(
integrationId: 'gsc' | 'ga4',
requested?: string,
): string {
const stored =
integrationId === 'gsc'
? resolveGscSettings().accountId
: resolveGa4Settings().accountId;
const accountId = resolveAccountId(integrationId, requested || stored) ?? '';
if (accountId) adoptLegacyRowsOnce(integrationId, accountId);
return accountId;
}
/**
* `project_meta` key holding the last pull's metadata for one account.
* Pre-multi-account projects stored it under the bare base key; that
* row is moved across during legacy adoption.
*/
function googleMetaKey(base: string, accountId: string): string {
return accountId ? `${base}:${accountId}` : base;
}
/** Projects whose legacy rows have already been adopted this session. */
const legacyAdopted = new Set<string>();
function adoptLegacyRowsOnce(
integrationId: 'gsc' | 'ga4',
accountId: string,
): void {
const sess = currentSession();
const marker = `${sess.currentProjectPath || 'default'}:${integrationId}:${accountId}`;
if (legacyAdopted.has(marker)) return;
legacyAdopted.add(marker);
try {
const db = sess.getDb();
const tables =
integrationId === 'gsc'
? ['gsc_results', 'gsc_inspection_results']
: ['ga4_results'];
let moved = 0;
for (const table of tables) {
moved += db.adoptLegacyGoogleRows(table, accountId);
}
// Move the matching fetch-meta row so the tab's context line
// ("property · range · fetched N ago") survives the upgrade too.
const metaBase = integrationId === 'gsc' ? 'gscFetchMeta' : 'ga4FetchMeta';
const legacyMeta = db.getMeta(metaBase);
if (legacyMeta && !db.getMeta(googleMetaKey(metaBase, accountId))) {
db.setMeta(googleMetaKey(metaBase, accountId), legacyMeta);
}
if (moved > 0) {
logger.log(
'info',
integrationId,
`Adopted ${moved} pre-multi-account row(s) into the connected account`,
);
}
} catch (err) {
logger.log(
'warn',
integrationId,
`legacy row adoption skipped: ${(err as Error).message}`,
);
}
}
async function launchRespider(sess: ProjectSession, urls: string[]): Promise<void> {
if (urls.length === 0) return;
const db = sess.getDb();
@@ -5721,18 +6082,6 @@ function registerIpc(): void {
await launchCrawl(respiderConfig, { respiderSeeds: urls, session: sess });
}
/**
* V2 Faz 1 — Resolve the per-project screenshots sidecar directory.
* Lives alongside the `.seoproject` file as `<project>.screenshots/`
* so it tracks with Save As / move operations.
*/
function resolveScreenshotDir(projectPath: string): string | null {
if (!projectPath) return null;
const dir = dirname(projectPath);
const base = basename(projectPath);
return join(dir, `${base}.screenshots`);
}
/**
* Stable 16-char hex stem derived from the URL. Used as the screenshot
* filename prefix so re-crawls overwrite the previous capture instead
@@ -6203,9 +6552,13 @@ function registerIpc(): void {
// batches with progress events + cancellation flags; those stay
// desktop-only until a polling tool pattern is designed.
'gsc-list-sites': async () => {
'gsc-list-sites': async (input) => {
try {
return { ok: true, error: null, sites: await listSites() };
const account = resolveGoogleAccount(
'gsc',
(input as { accountId?: string } | undefined)?.accountId,
);
return { ok: true, error: null, sites: await listSites(account) };
} catch (err) {
return {
ok: false,
@@ -6235,10 +6588,17 @@ function registerIpc(): void {
const startDate = isoDate(start);
const endDate = isoDate(end);
const runDb = getDb();
const accountId = resolveGoogleAccount('gsc', i.accountId);
try {
const rows = await querySearchAnalytics(i.property, startDate, endDate);
const rows = await querySearchAnalytics(
i.property,
startDate,
endDate,
{},
accountId,
);
const fetchedAt = new Date().toISOString();
runDb.replaceGscResults(rows, fetchedAt);
runDb.replaceGscResults(rows, fetchedAt, accountId);
const meta = {
property: i.property,
startDate,
@@ -6246,7 +6606,7 @@ function registerIpc(): void {
fetchedAt,
rowCount: rows.length,
};
runDb.setMeta('gscFetchMeta', JSON.stringify(meta));
runDb.setMeta(googleMetaKey('gscFetchMeta', accountId), JSON.stringify(meta));
fireDataChanged();
return { ok: true, error: null, rowCount: rows.length, meta };
} catch (err) {
@@ -6259,9 +6619,17 @@ function registerIpc(): void {
}
},
'ga4-list-properties': async () => {
'ga4-list-properties': async (input) => {
try {
return { ok: true, error: null, properties: await ga4ListProperties() };
const account = resolveGoogleAccount(
'ga4',
(input as { accountId?: string } | undefined)?.accountId,
);
return {
ok: true,
error: null,
properties: await ga4ListProperties(account),
};
} catch (err) {
return {
ok: false,
@@ -6290,10 +6658,16 @@ function registerIpc(): void {
const startDate = isoDate(start);
const endDate = isoDate(end);
const runDb = getDb();
const accountId = resolveGoogleAccount('ga4', i.accountId);
try {
const rows = await ga4RunReport(i.property, startDate, endDate);
const rows = await ga4RunReport(
i.property,
startDate,
endDate,
accountId,
);
const fetchedAt = new Date().toISOString();
runDb.replaceGa4Results(rows, fetchedAt);
runDb.replaceGa4Results(rows, fetchedAt, accountId);
const meta = {
property: i.property,
propertyName: i.propertyName || i.property,
@@ -6302,7 +6676,7 @@ function registerIpc(): void {
fetchedAt,
rowCount: rows.length,
};
runDb.setMeta('ga4FetchMeta', JSON.stringify(meta));
runDb.setMeta(googleMetaKey('ga4FetchMeta', accountId), JSON.stringify(meta));
fireDataChanged();
return { ok: true, error: null, rowCount: rows.length, meta };
} catch (err) {
@@ -6673,7 +7047,7 @@ function registerIpc(): void {
const sess = currentSession();
const wasDirty = sess.dirty;
const boundPath = sess.currentProjectPath;
const result = await saveProjectDocument(sess, target);
const result = await saveProjectDocument(sess, target, { rebind: false });
// Restore the binding: this was a snapshot, not a Save As.
sess.currentProjectPath = boundPath;
sess.dirty = wasDirty;
@@ -7712,27 +8086,53 @@ function registerIpc(): void {
async (): Promise<CrashRecoveryStatus> => {
// Read from the in-memory snapshot we captured before
// `db.reset()` wiped the previous session's data.
if (currentSession().pendingRecoveryCheckpoint.length === 0) {
return { pendingCount: 0, seedUrl: '' };
const sess = currentSession();
if (sess.pendingRecoveryCheckpoint.length === 0) {
return { pendingCount: 0, seedUrl: '', mode: 'spider' };
}
const mode = sess.lastCrawlConfig?.mode ?? 'spider';
// Show what the crawl was actually pointed at. A List/Sitemap run
// checkpoints the first URL of its list rather than its own seed
// (see the resume handler), so quoting it back would name a random
// content page as the "start URL" of a sitemap crawl.
const seedUrl =
mode === 'spider'
? (sess.pendingRecoveryCheckpoint[0]?.seedUrl ?? '')
: (sess.lastCrawlConfig?.startUrl ?? '');
return {
pendingCount: currentSession().pendingRecoveryCheckpoint.length,
seedUrl: currentSession().pendingRecoveryCheckpoint[0]?.seedUrl ?? '',
pendingCount: sess.pendingRecoveryCheckpoint.length,
seedUrl,
mode,
};
},
);
registerHandle(
IPC.crashRecoveryResume,
async (): Promise<{ accepted: boolean }> => {
async (): Promise<CrashRecoveryResumeResult> => {
const sess = currentSession();
if (sess.activeCrawler) return { accepted: false };
if (sess.pendingRecoveryCheckpoint.length === 0) return { accepted: false };
if (sess.activeCrawler) return { accepted: false, config: null };
if (sess.pendingRecoveryCheckpoint.length === 0)
return { accepted: false, config: null };
const seedUrl = sess.pendingRecoveryCheckpoint[0]?.seedUrl ?? '';
if (!seedUrl) return { accepted: false };
const savedCfg = sess.lastCrawlConfig;
const cfg = savedCfg ? { ...savedCfg, startUrl: seedUrl } : null;
if (!cfg) return { accepted: false };
if (!savedCfg) return { accepted: false, config: null };
// Spider mode checkpoints the *resolved* start URL (post-redirect),
// which is also what `project_meta.startUrl` holds — taking it from
// the checkpoint is what stops the resume from reading as a different
// site and wiping the project. List/Sitemap runs rewrite
// `config.startUrl` to the first URL of their list before the first
// checkpoint tick, so their seed is a content page, not a start URL:
// resuming with it would hand a plain HTML page to the sitemap parser
// and fail. Their saved config already carries the right value.
const cfg =
savedCfg.mode === 'spider' && seedUrl
? { ...savedCfg, startUrl: seedUrl }
: savedCfg;
// List mode seeds from `urlList`, so only the other two need a URL.
if (cfg.mode !== 'list' && !cfg.startUrl.trim()) {
return { accepted: false, config: null };
}
const crawler = new Crawler(cfg, sess.getDb(), {
setOp: (op: string) => freezeWatchdog.setMainOp(op),
parseHtml: parseHtmlViaPool,
@@ -7755,7 +8155,11 @@ function registerIpc(): void {
sess.pendingRecoveryCheckpoint = [];
crawler.enqueueCheckpointed(items);
void crawler.start();
return { accepted: true };
// Hand the config back so the toolbar can adopt the crawl that is
// now actually running. Without this the renderer keeps the blank
// Spider defaults it booted with, and a Sitemap/List resume reads
// as "it didn't continue where it left off".
return { accepted: true, config: cfg };
},
);
+23 -7
View File
@@ -23,6 +23,7 @@ import {
type ExportGridResult,
type DataDeleteByDomainInput,
type DataDeleteByDomainResult,
type CrashRecoveryResumeResult,
type CrashRecoveryStatus,
type ExportHtmlReportInput,
type ExportHtmlReportResult,
@@ -119,12 +120,14 @@ import {
type SpellingMatchesResult,
type SpellingLanguageOption,
type GoogleAuthState,
type GoogleAccount,
type GoogleAuthResult,
type GscListSitesResult,
type GscFetchInput,
type GscFetchResult,
type GscQueryInput,
type GscQueryResult,
type GscCrawlNewUrlsResult,
type Ga4ListPropertiesResult,
type Ga4FetchInput,
type Ga4FetchResult,
@@ -265,7 +268,7 @@ const api: FreeCrawlApi = {
ipcRenderer.invoke(IPC.dataDeleteByDomain, input),
crashRecoveryStatus: (): Promise<CrashRecoveryStatus> =>
ipcRenderer.invoke(IPC.crashRecoveryStatus),
crashRecoveryResume: (): Promise<{ accepted: boolean }> =>
crashRecoveryResume: (): Promise<CrashRecoveryResumeResult> =>
ipcRenderer.invoke(IPC.crashRecoveryResume),
crashRecoveryDiscard: (): Promise<void> =>
ipcRenderer.invoke(IPC.crashRecoveryDiscard),
@@ -398,6 +401,15 @@ const api: FreeCrawlApi = {
): Promise<IntegrationsState> => ipcRenderer.invoke(IPC.integrationsSet, id, fields),
integrationsClear: (id: string): Promise<IntegrationsState> =>
ipcRenderer.invoke(IPC.integrationsClear, id),
integrationSettingsGet: (
id: string,
): Promise<Record<string, unknown> | null> =>
ipcRenderer.invoke(IPC.integrationSettingsGet, id),
integrationSettingsSet: (
id: string,
settings: Record<string, unknown>,
): Promise<Record<string, unknown>> =>
ipcRenderer.invoke(IPC.integrationSettingsSet, id, settings),
pagespeedQuery: (input: PagespeedQueryInput): Promise<PagespeedQueryResult> =>
ipcRenderer.invoke(IPC.pagespeedQuery, input),
pagespeedRun: (input: PagespeedRunInput): Promise<PagespeedRunResult> =>
@@ -426,16 +438,20 @@ const api: FreeCrawlApi = {
ipcRenderer.invoke(IPC.googleAuthStart, id),
googleAuthStatus: (id: string): Promise<GoogleAuthState> =>
ipcRenderer.invoke(IPC.googleAuthStatus, id),
googleAuthRevoke: (id: string): Promise<GoogleAuthState> =>
ipcRenderer.invoke(IPC.googleAuthRevoke, id),
gscListSites: (): Promise<GscListSitesResult> =>
ipcRenderer.invoke(IPC.gscListSites),
googleAuthRevoke: (id: string, accountId?: string): Promise<GoogleAuthState> =>
ipcRenderer.invoke(IPC.googleAuthRevoke, id, accountId),
gscListSites: (accountId?: string): Promise<GscListSitesResult> =>
ipcRenderer.invoke(IPC.gscListSites, accountId),
googleAccountsList: (integrationId: string): Promise<GoogleAccount[]> =>
ipcRenderer.invoke(IPC.googleAccountsList, integrationId),
gscFetch: (input: GscFetchInput): Promise<GscFetchResult> =>
ipcRenderer.invoke(IPC.gscFetch, input),
gscQuery: (input: GscQueryInput): Promise<GscQueryResult> =>
ipcRenderer.invoke(IPC.gscQuery, input),
ga4ListProperties: (): Promise<Ga4ListPropertiesResult> =>
ipcRenderer.invoke(IPC.ga4ListProperties),
gscCrawlNewUrls: (): Promise<GscCrawlNewUrlsResult> =>
ipcRenderer.invoke(IPC.gscCrawlNewUrls),
ga4ListProperties: (accountId?: string): Promise<Ga4ListPropertiesResult> =>
ipcRenderer.invoke(IPC.ga4ListProperties, accountId),
ga4Fetch: (input: Ga4FetchInput): Promise<Ga4FetchResult> =>
ipcRenderer.invoke(IPC.ga4Fetch, input),
ga4Query: (input: Ga4QueryInput): Promise<Ga4QueryResult> =>
+9 -2
View File
@@ -185,13 +185,20 @@ export function App() {
const proceed = window.confirm(
t('app.crashRecovery', {
defaultValue:
"FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n Start URL: {{seed}}\n Pending URLs: {{pending}}\n\nResume the crawl from where it stopped?\n\nClick OK to resume, Cancel to discard the recovery state and start fresh.",
"FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n Mode: {{mode}}\n Start URL: {{seed}}\n Pending URLs: {{pending}}\n\nResume the crawl from where it stopped?\n\nClick OK to resume, Cancel to discard the recovery state and start fresh.",
mode: t(`topbar.mode.${status.mode}`, { defaultValue: status.mode }),
seed: status.seedUrl,
pending: status.pendingCount.toLocaleString(),
}),
);
if (proceed) {
void window.freecrawl.crashRecoveryResume();
// Adopt the resumed run's config so the toolbar shows the mode
// and start URL actually being crawled — the renderer boots
// with blank Spider defaults, which made a Sitemap/List resume
// look like it had started over.
void window.freecrawl.crashRecoveryResume().then((res) => {
if (res.accepted && res.config) applyProjectConfig(res.config);
});
} else {
void window.freecrawl.crashRecoveryDiscard();
}
@@ -85,17 +85,35 @@ export function LogAnalyzerView() {
verifyBots,
});
if (res.overview) setOverview(res.overview);
if (res.ok && res.imported) {
const i = res.imported;
showToast(
t('logAnalyzer.imported', {
defaultValue: '{{file}} — parsed {{parsed}}/{{total}} lines ({{format}})',
file: i.fileName,
parsed: i.parsedLines.toLocaleString(),
total: i.totalLines.toLocaleString(),
format: i.format,
}),
);
const batch = res.batch ?? (res.imported ? [res.imported] : []);
const failed = res.errors?.length ?? 0;
if (res.ok && batch.length > 0) {
if (batch.length === 1) {
const i = batch[0]!;
showToast(
t('logAnalyzer.imported', {
defaultValue: '{{file}} — parsed {{parsed}}/{{total}} lines ({{format}})',
file: i.fileName,
parsed: i.parsedLines.toLocaleString(),
total: i.totalLines.toLocaleString(),
format: i.format,
}),
);
} else {
const parsed = batch.reduce((s, b) => s + b.parsedLines, 0);
const total = batch.reduce((s, b) => s + b.totalLines, 0);
showToast(
t('logAnalyzer.importedBatch', {
defaultValue: '{{files}} files — parsed {{parsed}}/{{total}} lines',
files: batch.length,
parsed: parsed.toLocaleString(),
total: total.toLocaleString(),
}) +
(failed > 0
? ` · ${t('logAnalyzer.importedFailed', { defaultValue: '{{n}} failed', n: failed })}`
: ''),
);
}
setVersion((v) => v + 1);
} else if (res.error) {
showToast(t('logAnalyzer.importError', { defaultValue: 'Import failed: {{err}}', err: res.error }));
@@ -184,7 +202,7 @@ export function LogAnalyzerView() {
>
{busy
? t('logAnalyzer.analyzing', { defaultValue: 'Analyzing…' })
: t('logAnalyzer.import', { defaultValue: 'Import Log File…' })}
: t('logAnalyzer.import', { defaultValue: 'Import Log Files…' })}
</button>
{hasData && <ExportMenu onExport={onExport} />}
{hasData && (
@@ -0,0 +1,115 @@
import { useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import {
defaultGa4Settings,
type GoogleAccount,
type Ga4IntegrationSettings,
} from '@freecrawl/shared-types';
/**
* Per-integration behaviour settings for Google Analytics 4, rendered
* inside the GA4 card in Settings → Integrations. Smaller than the GSC
* equivalent — the Data API call takes no dimension filters — but it
* carries the same per-project account + date-range choice, so a project
* remembers which of several linked GA4 accounts it reports on.
*/
export function Ga4SettingsSection() {
const { t } = useTranslation();
const [settings, setSettings] = useState<Ga4IntegrationSettings | null>(null);
const [accounts, setAccounts] = useState<GoogleAccount[]>([]);
useEffect(() => {
let cancelled = false;
void (async () => {
const [stored, list] = await Promise.all([
window.freecrawl.integrationSettingsGet('ga4'),
window.freecrawl.googleAccountsList('ga4'),
]);
if (cancelled) return;
setAccounts(list);
setSettings({
...defaultGa4Settings(),
...(stored ?? {}),
} as Ga4IntegrationSettings);
})();
return () => {
cancelled = true;
};
}, []);
if (!settings) return null;
const update = <K extends keyof Ga4IntegrationSettings>(
key: K,
value: Ga4IntegrationSettings[K],
) => {
setSettings((prev) => (prev ? { ...prev, [key]: value } : prev));
void window.freecrawl.integrationSettingsSet('ga4', {
[key]: value,
} as Record<string, unknown>);
};
const selectCls =
'h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none';
return (
<div className="mt-3 rounded border border-surface-800 bg-surface-950/60 p-2.5">
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('ga4Settings.title', { defaultValue: 'Analytics 4 Settings' })}
</div>
{accounts.length > 1 && (
<div className="mb-3">
<div className="mb-1 text-[10px] font-medium text-surface-300">
{t('ga4Settings.account', { defaultValue: 'Account' })}
</div>
<select
className={`${selectCls} max-w-full`}
value={settings.accountId || accounts[0]?.accountId || ''}
onChange={(e) => update('accountId', e.target.value)}
title={t('ga4Settings.accountHint', {
defaultValue:
'Which linked Google account this project pulls Analytics data with.',
})}
>
{accounts.map((a) => (
<option key={a.accountId} value={a.accountId}>
{a.email ?? a.accountId}
</option>
))}
</select>
</div>
)}
<div>
<div className="mb-1 text-[10px] font-medium text-surface-300">
{t('ga4Settings.dateRange', { defaultValue: 'Date Range' })}
</div>
<select
className={selectCls}
value={settings.days}
onChange={(e) =>
update('days', Number(e.target.value) as Ga4IntegrationSettings['days'])
}
>
<option value={7}>
{t('ga4Settings.range7', { defaultValue: 'Last 7 days' })}
</option>
<option value={28}>
{t('ga4Settings.range28', { defaultValue: 'Last 28 days' })}
</option>
<option value={90}>
{t('ga4Settings.range90', { defaultValue: 'Last 90 days' })}
</option>
</select>
</div>
<p className="mt-2 text-[10px] leading-relaxed text-surface-500">
{t('ga4Settings.hint', {
defaultValue:
'The property is picked in the GA4 tab toolbar. Analytics data is near-realtime, so the range ends today.',
})}
</p>
</div>
);
}
@@ -0,0 +1,155 @@
import { useCallback, useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { Loader2, Plus, Trash2, Check } from 'lucide-react';
import type { GoogleAccount } from '@freecrawl/shared-types';
/**
* Linked-account manager for an OAuth-BYOC Google integration (Search
* Console / Analytics 4 / Sheets).
*
* A user may connect several Google accounts to the same integration —
* their own property plus a client's, say — so this lists every linked
* account, lets them add another, and disconnects them one at a time.
* Which account a project actually reports on is chosen per project in
* the tab toolbar; this panel only manages the links themselves.
*
* @param integrationId `gsc` / `ga4` / `sheets`.
* @param configured False until the BYOC client ID + secret are saved —
* consent can't start without them, so the connect button is disabled.
* @param onChange Notifies the parent so a settings section showing an
* account dropdown can refresh.
*/
export function GoogleAccountsSection({
integrationId,
configured,
onChange,
}: {
integrationId: string;
configured: boolean;
onChange?: (accounts: GoogleAccount[]) => void;
}) {
const { t } = useTranslation();
const [accounts, setAccounts] = useState<GoogleAccount[] | null>(null);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const load = useCallback(async () => {
const list = await window.freecrawl.googleAccountsList(integrationId);
setAccounts(list);
onChange?.(list);
// `onChange` is a render-scoped callback in practice; depending on it
// would re-run this on every parent render.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [integrationId]);
useEffect(() => {
void load();
}, [load]);
const connect = async () => {
setBusy(true);
setError(null);
try {
const res = await window.freecrawl.googleAuthStart(integrationId);
if (!res.ok && res.error) setError(res.error);
await load();
} finally {
setBusy(false);
}
};
const disconnect = async (account: GoogleAccount) => {
const label = account.email ?? account.accountId;
if (
!window.confirm(
t('googleAccounts.confirmDisconnect', {
defaultValue: 'Disconnect {{account}}? Data already pulled with it stays until the next fetch.',
account: label,
}),
)
) {
return;
}
setBusy(true);
setError(null);
try {
await window.freecrawl.googleAuthRevoke(integrationId, account.accountId);
await load();
} finally {
setBusy(false);
}
};
if (!accounts) return null;
return (
<div className="mt-3 rounded border border-surface-800 bg-surface-950/60 p-2.5">
<div className="mb-2 flex items-center gap-2">
<span className="text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('googleAccounts.title', { defaultValue: 'Connected Accounts' })}
</span>
<span className="rounded bg-surface-800 px-1.5 py-0.5 text-[9px] tabular-nums text-surface-400">
{accounts.length}
</span>
<button
type="button"
onClick={() => void connect()}
disabled={busy || !configured}
className="ml-auto inline-flex items-center gap-1 rounded bg-blue-600 px-2 py-0.5 text-[10px] font-medium text-white hover:bg-blue-500 disabled:cursor-not-allowed disabled:bg-surface-700 disabled:text-surface-500"
title={
configured
? undefined
: t('googleAccounts.needClient', {
defaultValue: 'Save your OAuth client ID + secret first.',
})
}
>
{busy ? <Loader2 size={11} className="animate-spin" /> : <Plus size={11} />}
{accounts.length === 0
? t('googleAccounts.connect', { defaultValue: 'Connect account' })
: t('googleAccounts.connectAnother', { defaultValue: 'Connect another' })}
</button>
</div>
{accounts.length === 0 ? (
<p className="text-[10px] leading-relaxed text-surface-500">
{t('googleAccounts.empty', {
defaultValue:
'No account linked yet. Connect one — you can link several and pick which one each project uses.',
})}
</p>
) : (
<ul className="space-y-1">
{accounts.map((a) => (
<li
key={a.accountId}
className="flex items-center gap-2 rounded border border-surface-800 bg-surface-900/40 px-2 py-1"
>
<Check size={11} className="shrink-0 text-emerald-400" />
<span className="truncate text-[11px] text-surface-200" title={a.email ?? a.accountId}>
{a.email ?? t('googleAccounts.unknownEmail', { defaultValue: 'Google account' })}
</span>
<button
type="button"
onClick={() => void disconnect(a)}
disabled={busy}
className="ml-auto inline-flex items-center gap-1 rounded border border-red-700/60 px-1.5 py-0.5 text-[10px] text-red-300 hover:bg-red-900/20 disabled:opacity-50"
title={t('googleAccounts.disconnect', { defaultValue: 'Disconnect' })}
>
<Trash2 size={10} />
</button>
</li>
))}
</ul>
)}
{error && <div className="mt-1.5 text-[10px] text-red-400">{error}</div>}
<p className="mt-2 text-[10px] leading-relaxed text-surface-500">
{t('googleAccounts.hint', {
defaultValue:
'Each project picks which linked account it reports on, in the tab toolbar. Data pulled with different accounts is stored separately.',
})}
</p>
</div>
);
}
@@ -0,0 +1,361 @@
import { useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import {
defaultGscSettings,
type GoogleAccount,
type GscIntegrationSettings,
} from '@freecrawl/shared-types';
/**
* Per-integration behaviour settings for Google Search Console — the
* FreeCrawl equivalent of Screaming Frog's "API Access → Google Search
* Console" dialog. Rendered inside the GSC card in Settings →
* Integrations. Stored per project via `integrationSettingsSet('gsc')`;
* the Search Console tab's Fetch button reads these to shape the pull.
*/
export function GscSettingsSection() {
const { t } = useTranslation();
const [settings, setSettings] = useState<GscIntegrationSettings | null>(null);
const [accounts, setAccounts] = useState<GoogleAccount[]>([]);
useEffect(() => {
let cancelled = false;
void (async () => {
const [stored, list] = await Promise.all([
window.freecrawl.integrationSettingsGet('gsc'),
window.freecrawl.googleAccountsList('gsc'),
]);
if (cancelled) return;
setAccounts(list);
setSettings({
...defaultGscSettings(),
...(stored ?? {}),
} as GscIntegrationSettings);
})();
return () => {
cancelled = true;
};
}, []);
if (!settings) return null;
const update = <K extends keyof GscIntegrationSettings>(
key: K,
value: GscIntegrationSettings[K],
) => {
setSettings((prev) => (prev ? { ...prev, [key]: value } : prev));
void window.freecrawl.integrationSettingsSet('gsc', {
[key]: value,
} as Record<string, unknown>);
};
const selectCls =
'h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none';
const inputCls =
'h-6 rounded border border-surface-700 bg-surface-950 px-2 text-[11px] text-surface-100 placeholder-surface-600 focus:border-blue-500 focus:outline-none';
return (
<div className="mt-3 rounded border border-surface-800 bg-surface-950/60 p-2.5">
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('gscSettings.title', { defaultValue: 'Search Console Settings' })}
</div>
{/* ── Account ────────────────────────────────────────────── */}
{accounts.length > 1 && (
<div className="mb-3">
<div className="mb-1 text-[10px] font-medium text-surface-300">
{t('gscSettings.account', { defaultValue: 'Account' })}
</div>
<select
className={`${selectCls} max-w-full`}
value={settings.accountId || accounts[0]?.accountId || ''}
onChange={(e) => update('accountId', e.target.value)}
title={t('gscSettings.accountHint', {
defaultValue:
'Which linked Google account this project pulls Search Console data with.',
})}
>
{accounts.map((a) => (
<option key={a.accountId} value={a.accountId}>
{a.email ?? a.accountId}
</option>
))}
</select>
</div>
)}
{/* ── Date Range ─────────────────────────────────────────── */}
<div className="mb-3">
<div className="mb-1 text-[10px] font-medium text-surface-300">
{t('gscSettings.dateRange', { defaultValue: 'Date Range' })}
</div>
<div className="flex flex-wrap items-center gap-2">
<select
className={selectCls}
value={settings.dateRange}
onChange={(e) =>
update(
'dateRange',
e.target.value as GscIntegrationSettings['dateRange'],
)
}
>
<option value="7d">
{t('gscSettings.range7', { defaultValue: 'Last 7 days' })}
</option>
<option value="28d">
{t('gscSettings.range28', { defaultValue: 'Last 28 days' })}
</option>
<option value="90d">
{t('gscSettings.range90', { defaultValue: 'Last 90 days' })}
</option>
<option value="16m">
{t('gscSettings.range16m', { defaultValue: 'Last 16 months' })}
</option>
<option value="custom">
{t('gscSettings.rangeCustom', { defaultValue: 'Custom' })}
</option>
</select>
{settings.dateRange === 'custom' && (
<>
<input
type="date"
className={inputCls}
value={settings.startDate}
onChange={(e) => update('startDate', e.target.value)}
title={t('gscSettings.startDate', { defaultValue: 'Start date' })}
/>
<span className="text-[11px] text-surface-500">→</span>
<input
type="date"
className={inputCls}
value={settings.endDate}
onChange={(e) => update('endDate', e.target.value)}
title={t('gscSettings.endDate', { defaultValue: 'End date' })}
/>
</>
)}
</div>
</div>
{/* ── Dimension Filter ───────────────────────────────────── */}
<div className="mb-3">
<div className="mb-1 text-[10px] font-medium text-surface-300">
{t('gscSettings.dimensionFilter', { defaultValue: 'Dimension Filter' })}
</div>
<div className="grid grid-cols-2 gap-2">
<label className="flex flex-col gap-0.5">
<span className="text-[10px] text-surface-500">
{t('gscSettings.device', { defaultValue: 'Device' })}
</span>
<select
className={selectCls}
value={settings.deviceFilter}
onChange={(e) =>
update(
'deviceFilter',
e.target.value as GscIntegrationSettings['deviceFilter'],
)
}
>
<option value="all">
{t('gscSettings.deviceAll', { defaultValue: 'All' })}
</option>
<option value="DESKTOP">
{t('gscSettings.deviceDesktop', { defaultValue: 'Desktop' })}
</option>
<option value="MOBILE">
{t('gscSettings.deviceMobile', { defaultValue: 'Mobile' })}
</option>
<option value="TABLET">
{t('gscSettings.deviceTablet', { defaultValue: 'Tablet' })}
</option>
</select>
</label>
<label className="flex flex-col gap-0.5">
<span className="text-[10px] text-surface-500">
{t('gscSettings.type', { defaultValue: 'Search Type' })}
</span>
<select
className={selectCls}
value={settings.searchType}
onChange={(e) =>
update(
'searchType',
e.target.value as GscIntegrationSettings['searchType'],
)
}
>
<option value="web">
{t('gscSettings.typeWeb', { defaultValue: 'Web' })}
</option>
<option value="image">
{t('gscSettings.typeImage', { defaultValue: 'Image' })}
</option>
<option value="video">
{t('gscSettings.typeVideo', { defaultValue: 'Video' })}
</option>
<option value="news">
{t('gscSettings.typeNews', { defaultValue: 'News' })}
</option>
<option value="discover">
{t('gscSettings.typeDiscover', { defaultValue: 'Discover' })}
</option>
<option value="googleNews">
{t('gscSettings.typeGoogleNews', { defaultValue: 'Google News' })}
</option>
</select>
</label>
<label className="flex flex-col gap-0.5">
<span className="text-[10px] text-surface-500">
{t('gscSettings.country', { defaultValue: 'Country' })}
</span>
<input
type="text"
className={inputCls}
value={settings.countryFilter}
placeholder={t('gscSettings.countryPlaceholder', {
defaultValue: 'e.g. usa (ISO code) — blank for all',
})}
onChange={(e) =>
update('countryFilter', e.target.value.trim().toLowerCase())
}
spellCheck={false}
/>
</label>
<label className="flex flex-col gap-0.5">
<span className="text-[10px] text-surface-500">
{t('gscSettings.queryFilter', { defaultValue: 'Search Query' })}
</span>
<div className="flex gap-1">
<select
className={selectCls}
value={settings.queryFilterMode}
onChange={(e) =>
update(
'queryFilterMode',
e.target.value as GscIntegrationSettings['queryFilterMode'],
)
}
>
<option value="none">
{t('gscSettings.queryNone', { defaultValue: 'None' })}
</option>
<option value="contains">
{t('gscSettings.queryContains', { defaultValue: 'Contains' })}
</option>
<option value="notContains">
{t('gscSettings.queryNotContains', {
defaultValue: "Doesn't contain",
})}
</option>
<option value="equals">
{t('gscSettings.queryEquals', { defaultValue: 'Equals' })}
</option>
<option value="notEquals">
{t('gscSettings.queryNotEquals', {
defaultValue: "Doesn't equal",
})}
</option>
<option value="includingRegex">
{t('gscSettings.queryRegex', { defaultValue: 'Matches regex' })}
</option>
<option value="excludingRegex">
{t('gscSettings.queryNotRegex', {
defaultValue: "Doesn't match regex",
})}
</option>
</select>
<input
type="text"
className={`${inputCls} min-w-0 flex-1`}
value={settings.queryFilterValue}
disabled={settings.queryFilterMode === 'none'}
placeholder={t('gscSettings.queryValuePlaceholder', {
defaultValue: 'Query…',
})}
onChange={(e) => update('queryFilterValue', e.target.value)}
spellCheck={false}
/>
</div>
</label>
</div>
</div>
{/* ── General ────────────────────────────────────────────── */}
<div>
<div className="mb-1 text-[10px] font-medium text-surface-300">
{t('gscSettings.general', { defaultValue: 'General' })}
</div>
<div className="space-y-1.5">
<label className="flex items-center gap-2 text-[11px] text-surface-200">
<input
type="checkbox"
className="accent-blue-500"
checked={settings.matchSlash}
onChange={(e) => update('matchSlash', e.target.checked)}
/>
{t('gscSettings.matchSlash', {
defaultValue: 'Match Trailing and Non-Trailing Slash URLs',
})}
</label>
<label className="flex items-center gap-2 text-[11px] text-surface-200">
<input
type="checkbox"
className="accent-blue-500"
checked={settings.matchCase}
onChange={(e) => update('matchCase', e.target.checked)}
/>
{t('gscSettings.matchCase', {
defaultValue: 'Match Uppercase & Lowercase URLs',
})}
</label>
<div className="flex items-center gap-2">
<label className="flex items-center gap-2 text-[11px] text-surface-200">
<input
type="checkbox"
className="accent-blue-500"
checked={settings.limitMaxResults}
onChange={(e) => update('limitMaxResults', e.target.checked)}
/>
{t('gscSettings.limitMaxResults', {
defaultValue: 'Limit Max Results',
})}
</label>
<input
type="number"
min={1}
step={1000}
className={`${inputCls} w-24`}
value={settings.maxResults}
disabled={!settings.limitMaxResults}
onChange={(e) =>
update(
'maxResults',
Math.max(1, Math.floor(Number(e.target.value) || 0)),
)
}
/>
</div>
<label className="flex items-center gap-2 text-[11px] text-surface-200">
<input
type="checkbox"
className="accent-blue-500"
checked={settings.crawlNewUrls}
onChange={(e) => update('crawlNewUrls', e.target.checked)}
/>
{t('gscSettings.crawlNewUrls', {
defaultValue: 'Crawl New URLs Discovered In Google Search Console',
})}
</label>
<p className="pl-6 text-[10px] leading-relaxed text-surface-500">
{t('gscSettings.crawlNewUrlsHint', {
defaultValue:
'After each Search Console pull, URLs that appear in GSC but were not found by the crawl are added to the crawl queue.',
})}
</p>
</div>
</div>
</div>
);
}
@@ -1,4 +1,4 @@
import { useEffect, useMemo, useRef, useState, type ReactNode, type RefObject } from 'react';
import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react';
import { useTranslation, Trans } from 'react-i18next';
import {
SUPPORTED_LANGUAGES,
@@ -42,6 +42,8 @@ import {
Target,
Check,
Workflow,
ChevronRight,
ChevronDown,
type LucideIcon,
} from 'lucide-react';
import clsx from 'clsx';
@@ -62,6 +64,9 @@ import { useAppStore } from '../store.js';
import { InfoTip, type FieldInfo } from './InfoTip.js';
import { ExtractionPreviewDialog } from './ExtractionPreviewDialog.js';
import { IntegrationSetupGuideModal } from './IntegrationSetupGuideModal.js';
import { GscSettingsSection } from './GscSettingsSection.js';
import { Ga4SettingsSection } from './Ga4SettingsSection.js';
import { GoogleAccountsSection } from './GoogleAccountsSection.js';
interface Props {
open: boolean;
@@ -252,19 +257,53 @@ export type SettingsSectionKey =
| 'storage'
| 'spelling'
| 'privacy'
| 'language';
| 'language'
/** Per-integration sub-page, e.g. `integration:gsc`. Each integration
* gets its own page under the "Integrations" group header, so a
* provider's credentials and its behaviour settings are never mixed
* in with a dozen other providers on one scrolling page. */
| `integration:${string}`;
/** Local alias — the exported name is the one other modules import. */
type SectionKey = SettingsSectionKey;
/** Sidebar key prefix for per-integration sub-pages. */
const INTEGRATION_KEY_PREFIX = 'integration:';
/** Build the sidebar key for one integration's sub-page. */
function integrationSectionKey(id: string): SectionKey {
return `${INTEGRATION_KEY_PREFIX}${id}` as SectionKey;
}
/** Extract the integration id from a sub-page key (null when not one). */
function integrationIdFromSection(key: SectionKey): string | null {
return key.startsWith(INTEGRATION_KEY_PREFIX)
? key.slice(INTEGRATION_KEY_PREFIX.length)
: null;
}
interface SectionDef {
key: SectionKey;
label: string;
icon: LucideIcon;
/** Searchable keywords beyond the label. */
keywords: string;
/** A collapsible header rather than a page — clicking it expands its
* children instead of selecting content. */
group?: true;
/** Parent group key, for entries nested under a header. */
parent?: SectionKey;
}
/** Sidebar icon per integration category — keeps the sub-page list
* scannable without inventing a bespoke icon for every provider. */
const INTEGRATION_CATEGORY_ICON: Record<IntegrationDef['category'], LucideIcon> = {
ai: Sparkles,
performance: Gauge,
seo: Target,
google: Chrome,
};
const SECTIONS: SectionDef[] = [
{
key: 'presets',
@@ -405,7 +444,19 @@ const SECTIONS: SectionDef[] = [
icon: Plug,
keywords:
'integrations api key oauth google search console analytics gsc ga4 pagespeed ahrefs moz semrush majestic openai anthropic claude ollama sheets bigquery credentials',
group: true,
},
// One sub-page per integration, generated from the shared catalog so a
// new provider shows up in the sidebar without touching this file.
...INTEGRATIONS.map(
(def): SectionDef => ({
key: integrationSectionKey(def.id),
label: def.name,
icon: INTEGRATION_CATEGORY_ICON[def.category],
keywords: `integration ${def.id} ${def.name} ${def.category} ${def.authType} ${def.description}`,
parent: 'integrations',
}),
),
{
key: 'rendering',
label: 'Rendering',
@@ -626,6 +677,10 @@ export function SettingsDialog({ open, onClose }: Props) {
const [form, setForm] = useState<FormState>(() => configToForm(config));
const [active, setActive] = useState<SectionKey>('mode');
const [search, setSearch] = useState('');
/** Collapsible sidebar groups the user has opened. */
const [expandedGroups, setExpandedGroups] = useState<Set<SectionKey>>(
() => new Set(),
);
// Re-seed the form whenever the dialog reopens — picks up any external
// config change (e.g. URL/scope edits in the top bar) so the dialog
@@ -638,11 +693,33 @@ export function SettingsDialog({ open, onClose }: Props) {
}, [open, config]);
// Follow a deep link ("Add API Key…" and friends). Without a target the
// dialog stays on whichever section the user last used.
// dialog stays on whichever section the user last used. A target naming
// an integration lands on that provider's own sub-page (and opens the
// group so the selection is visible in the sidebar); a bare
// `integrations` target falls back to the first provider, since the
// group header itself is no longer a page.
useEffect(() => {
if (open && target) setActive(target.section);
if (!open || !target) return;
if (target.section === 'integrations') {
const id = target.integration ?? INTEGRATIONS[0]?.id;
if (id) {
setActive(integrationSectionKey(id));
setExpandedGroups((prev) => new Set(prev).add('integrations'));
}
return;
}
setActive(target.section);
}, [open, target]);
// Selecting a nested page opens its group so the highlighted row is
// actually visible in the sidebar. Applied as a state change (not a
// derived override) so the header stays collapsible afterwards.
useEffect(() => {
const parent = SECTIONS.find((s) => s.key === active)?.parent;
if (!parent) return;
setExpandedGroups((prev) => (prev.has(parent) ? prev : new Set(prev).add(parent)));
}, [active]);
// ESC closes — common modal expectation.
useEffect(() => {
if (!open) return;
@@ -656,17 +733,38 @@ export function SettingsDialog({ open, onClose }: Props) {
const visibleSections = useMemo(() => {
const q = search.trim().toLowerCase();
if (!q) return SECTIONS;
return SECTIONS.filter(
(s) =>
s.label.toLowerCase().includes(q) || s.keywords.toLowerCase().includes(q),
);
const matches = (s: SectionDef): boolean =>
s.label.toLowerCase().includes(q) || s.keywords.toLowerCase().includes(q);
// Keep parent/child pairs intact. A matching child always brings its
// header along, so no row is orphaned. A matching group expands to all
// its children only when no child matched on its own — otherwise
// searching "gsc" would list every provider, since the group's own
// keywords name them all.
const keep = new Set<SectionKey>();
for (const s of SECTIONS) {
if (matches(s) && !s.group) {
keep.add(s.key);
if (s.parent) keep.add(s.parent);
}
}
for (const s of SECTIONS) {
if (!s.group || !matches(s)) continue;
keep.add(s.key);
const children = SECTIONS.filter((c) => c.parent === s.key);
if (!children.some((c) => keep.has(c.key))) {
for (const c of children) keep.add(c.key);
}
}
return SECTIONS.filter((s) => keep.has(s.key));
}, [search]);
// If the search filter hides the active section, jump to the first visible.
// If the search filter hides the active section, jump to the first visible
// page — group headers are skipped since they render no content.
useEffect(() => {
if (visibleSections.length === 0) return;
if (!visibleSections.some((s) => s.key === active)) {
setActive(visibleSections[0]!.key);
const pages = visibleSections.filter((s) => !s.group);
if (pages.length === 0) return;
if (!pages.some((s) => s.key === active)) {
setActive(pages[0]!.key);
}
}, [visibleSections, active]);
@@ -897,6 +995,28 @@ export function SettingsDialog({ open, onClose }: Props) {
}
const activeDef = SECTIONS.find((s) => s.key === active) ?? SECTIONS[0]!;
const parentDef = activeDef.parent
? SECTIONS.find((s) => s.key === activeDef.parent)
: undefined;
const searching = search.trim().length > 0;
/** A group renders open when the user opened it, or while a search is
* narrowing the list (so matches are never hidden behind a collapsed
* header). Selecting a child auto-opens its group once, via the effect
* above — deriving openness from the selection instead would make the
* header un-collapsible while one of its pages is active. */
function isGroupOpen(key: SectionKey): boolean {
return searching || expandedGroups.has(key);
}
function toggleGroup(key: SectionKey) {
setExpandedGroups((prev) => {
const next = new Set(prev);
if (next.has(key)) next.delete(key);
else next.add(key);
return next;
});
}
return (
<div
@@ -943,20 +1063,46 @@ export function SettingsDialog({ open, onClose }: Props) {
)}
{visibleSections.map((s) => {
const Icon = s.icon;
// Group header — expands/collapses its children instead of
// selecting a page of its own.
if (s.group) {
const open = isGroupOpen(s.key);
const Chevron = open ? ChevronDown : ChevronRight;
return (
<button
key={s.key}
className="flex w-full items-center gap-2 border-l-2 border-transparent px-3 py-1.5 text-left text-[12px] text-surface-300 transition-colors hover:bg-surface-800 hover:text-surface-100"
onClick={() => toggleGroup(s.key)}
aria-expanded={open}
>
<Icon className="h-3.5 w-3.5" />
<span>{translateLabel(s.label, lang)}</span>
<Chevron className="ml-auto h-3.5 w-3.5 text-surface-500" />
</button>
);
}
// Child page — only rendered while its group is open.
if (s.parent && !isGroupOpen(s.parent)) return null;
const isActive = s.key === active;
return (
<button
key={s.key}
className={clsx(
'flex w-full items-center gap-2 px-3 py-1.5 text-left text-[12px] transition-colors',
'flex w-full items-center gap-2 py-1.5 text-left text-[12px] transition-colors',
s.parent ? 'pl-8 pr-3' : 'px-3',
isActive
? 'bg-accent-600/20 text-accent-200 border-l-2 border-accent-500'
: 'border-l-2 border-transparent text-surface-300 hover:bg-surface-800 hover:text-surface-100',
)}
onClick={() => setActive(s.key)}
// Long provider names truncate in the narrow sidebar.
title={translateLabel(s.label, lang)}
>
<Icon className="h-3.5 w-3.5" />
<span>{translateLabel(s.label, lang)}</span>
<Icon className="h-3.5 w-3.5 shrink-0" />
<span className="truncate">{translateLabel(s.label, lang)}</span>
</button>
);
})}
@@ -968,6 +1114,12 @@ export function SettingsDialog({ open, onClose }: Props) {
<div className="border-b border-surface-800 px-5 py-2 text-[11px] text-surface-400">
{t('settings.title', { defaultValue: 'Settings' })}{' '}
<span className="mx-1 text-surface-600">›</span>
{parentDef && (
<>
<span>{translateLabel(parentDef.label, lang)}</span>
<span className="mx-1 text-surface-600">›</span>
</>
)}
<span className="text-surface-200">{translateLabel(activeDef.label, lang)}</span>
</div>
<div className="flex-1 overflow-auto px-5 py-4 text-[12px]">
@@ -1056,8 +1208,8 @@ export function SettingsDialog({ open, onClose }: Props) {
{active === 'per-host-ua' && (
<PerHostUaPanel form={form} update={update} />
)}
{active === 'integrations' && (
<IntegrationsPanel focusId={target?.integration ?? null} />
{integrationIdFromSection(active) && (
<IntegrationPage id={integrationIdFromSection(active)!} />
)}
{active === 'rendering' && (
<RenderingPanel form={form} update={update} />
@@ -4326,108 +4478,119 @@ function PerformanceBudgetPanel({ form, update }: PanelProps) {
}
/**
* @param focusId Integration the user was sent here to configure. The
* panel lists a dozen-plus cards across four categories, so landing on
* the section alone would still leave them hunting; the matching card
* is scrolled into view and outlined for as long as the panel is open.
* One integration's own settings page. Every provider gets a dedicated
* sub-page under the "Integrations" sidebar group — credentials plus any
* behaviour settings the provider defines (e.g. the GSC date range and
* dimension filters). The previous single scrolling page listed a
* dozen-plus providers at once, which buried each one's settings.
*
* @param id Integration id from the shared `INTEGRATIONS` catalog.
*/
function IntegrationsPanel({ focusId }: { focusId: string | null }) {
function IntegrationPage({ id }: { id: string }) {
const { t, i18n } = useTranslation();
const def = INTEGRATIONS.find((i) => i.id === id);
const [state, setState] = useState<IntegrationsState | null>(null);
const focusRef = useRef<HTMLDivElement | null>(null);
// Per-integration → per-field draft text. Only fields the user has
// actually typed into are tracked; a save sends just these so an
// untouched "saved" secret is never overwritten.
const [drafts, setDrafts] = useState<Record<string, Record<string, string>>>({});
const [busy, setBusy] = useState<string | null>(null);
const [notice, setNotice] = useState<Record<string, string>>({});
// Per-field draft text. Only fields the user has actually typed into
// are tracked; a save sends just these so an untouched "saved" secret
// is never overwritten.
const [draft, setDraft] = useState<Record<string, string>>({});
const [busy, setBusy] = useState(false);
const [notice, setNotice] = useState<string | undefined>();
/** Bumped whenever an account is linked/unlinked, to re-seed the
* provider settings section's account dropdown. */
const [accountsVersion, setAccountsVersion] = useState(0);
useEffect(() => {
void window.freecrawl.integrationsGetAll().then(setState);
}, []);
// Switching provider must not carry the previous page's draft or
// "Saved." notice across.
useEffect(() => {
focusRef.current?.scrollIntoView({ block: 'center' });
}, [focusId]);
setDraft({});
setNotice(undefined);
}, [id]);
const setDraft = (id: string, key: string, value: string) => {
setDrafts((d) => ({ ...d, [id]: { ...(d[id] ?? {}), [key]: value } }));
};
if (!def) return null;
const save = async (id: string) => {
const fields = drafts[id];
if (!fields || Object.keys(fields).length === 0) return;
setBusy(id);
const category = INTEGRATION_CATEGORIES.find((c) => c.key === def.category);
const save = async () => {
if (Object.keys(draft).length === 0) return;
setBusy(true);
try {
const next = await window.freecrawl.integrationsSet(id, fields);
setState(next);
setDrafts((d) => ({ ...d, [id]: {} }));
setNotice((n) => ({ ...n, [id]: t('integrations.saved', { defaultValue: 'Saved.' }) }));
setState(await window.freecrawl.integrationsSet(def.id, draft));
setDraft({});
setNotice(t('integrations.saved', { defaultValue: 'Saved.' }));
} finally {
setBusy(null);
setBusy(false);
}
};
const clear = async (id: string) => {
const clear = async () => {
if (!window.confirm(t('integrations.confirmClear', { defaultValue: 'Remove all stored credentials for this integration?' }))) {
return;
}
setBusy(id);
setBusy(true);
try {
const next = await window.freecrawl.integrationsClear(id);
setState(next);
setDrafts((d) => ({ ...d, [id]: {} }));
setNotice((n) => ({ ...n, [id]: t('integrations.cleared', { defaultValue: 'Removed.' }) }));
setState(await window.freecrawl.integrationsClear(def.id));
setDraft({});
setNotice(t('integrations.cleared', { defaultValue: 'Removed.' }));
} finally {
setBusy(null);
setBusy(false);
}
};
return (
<>
<div className="mb-2 flex items-center gap-2">
<span className="text-[13px] font-semibold text-surface-100">{def.name}</span>
{category && (
<span className="rounded bg-surface-800 px-1.5 py-0.5 text-[9px] uppercase tracking-wide text-surface-400">
{translateLabel(category.label, i18n.language)}
</span>
)}
</div>
<p className="mb-3 text-[11px] text-surface-400">
{t('integrations.intro', { defaultValue: 'Connect external services with your own credentials. API keys and secrets are encrypted at rest using your operating system\'s secure storage and never leave this machine.' })}
</p>
<div className="mb-3 rounded border border-blue-700/40 bg-blue-900/15 px-3 py-2 text-[11px] text-blue-200">
{t('integrations.byocNote', { defaultValue: 'Google integrations use a "bring your own client" model — you create your own Google Cloud OAuth client and paste its ID/secret, so no shared FreeCrawl app or verification is involved.' })}
</div>
{def.authType === 'oauth-byoc' && (
<div className="mb-3 rounded border border-blue-700/40 bg-blue-900/15 px-3 py-2 text-[11px] text-blue-200">
{t('integrations.byocNote', { defaultValue: 'Google integrations use a "bring your own client" model — you create your own Google Cloud OAuth client and paste its ID/secret, so no shared FreeCrawl app or verification is involved.' })}
</div>
)}
{INTEGRATION_CATEGORIES.map((cat) => {
const items = INTEGRATIONS.filter((i) => i.category === cat.key);
if (items.length === 0) return null;
return (
<div key={cat.key} className="mb-4">
<div className="mb-1.5 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{translateLabel(cat.label, i18n.language)}
</div>
<div className="space-y-2">
{items.map((def) => (
<IntegrationCard
key={def.id}
def={def}
cardRef={def.id === focusId ? focusRef : undefined}
focused={def.id === focusId}
state={state?.[def.id]}
draft={drafts[def.id] ?? {}}
busy={busy === def.id}
notice={notice[def.id]}
onDraft={(k, v) => setDraft(def.id, k, v)}
onSave={() => void save(def.id)}
onClear={() => void clear(def.id)}
/>
))}
</div>
</div>
);
})}
<IntegrationCard
def={def}
state={state?.[def.id]}
draft={draft}
busy={busy}
notice={notice}
onDraft={(k, v) => setDraft((d) => ({ ...d, [k]: v }))}
onSave={() => void save()}
onClear={() => void clear()}
/>
{/* Linked Google accounts — several can share one OAuth client. */}
{def.authType === 'oauth-byoc' && (
<GoogleAccountsSection
integrationId={def.id}
configured={state?.[def.id]?.configured ?? false}
onChange={() => setAccountsVersion((v) => v + 1)}
/>
)}
{/* Provider-specific behaviour settings, kept separate from the
credential card above. Re-mounted when the account list changes
so their account dropdowns pick up a new or removed link. */}
{def.id === 'gsc' && <GscSettingsSection key={accountsVersion} />}
{def.id === 'ga4' && <Ga4SettingsSection key={accountsVersion} />}
</>
);
}
function IntegrationCard({
def,
cardRef,
focused,
state,
draft,
busy,
@@ -4437,8 +4600,6 @@ function IntegrationCard({
onClear,
}: {
def: IntegrationDef;
cardRef?: RefObject<HTMLDivElement | null>;
focused?: boolean;
state: IntegrationsState[string] | undefined;
draft: Record<string, string>;
busy: boolean;
@@ -4453,17 +4614,10 @@ function IntegrationCard({
const [guideOpen, setGuideOpen] = useState(false);
return (
<div
ref={cardRef}
className={clsx(
'rounded border bg-surface-950/40 p-3',
focused
? 'border-accent-500 ring-1 ring-accent-500/40'
: 'border-surface-800',
)}
>
<div className="rounded border border-surface-800 bg-surface-950/40 p-3">
<div className="mb-1 flex items-center gap-2">
<span className="text-[12px] font-medium text-surface-100">{def.name}</span>
{/* The provider name lives in the page header above — the card
only carries its credential model + connection status. */}
<span className="rounded bg-surface-800 px-1.5 py-0.5 text-[9px] uppercase tracking-wide text-surface-400">
{translateLabel(AUTH_TYPE_LABEL[def.authType], i18n.language)}
</span>
@@ -255,7 +255,7 @@
"dropFailedProject": "Failed to open project: {{name}}",
"dropNoUrls": "No URLs found in {{name}}.",
"dropLoadedUrls": "Loaded {{count}} URLs from {{name}} into List mode.",
"crashRecovery": "FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n Start URL: {{seed}}\n Pending URLs: {{pending}}\n\nResume the crawl from where it stopped?\n\nClick OK to resume, Cancel to discard the recovery state and start fresh.",
"crashRecovery": "FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n Mode: {{mode}}\n Start URL: {{seed}}\n Pending URLs: {{pending}}\n\nResume the crawl from where it stopped?\n\nClick OK to resume, Cancel to discard the recovery state and start fresh.",
"deleteDomainPrompt": "GDPR Domain Wipe — enter the host to delete (case-insensitive, exact match).\n\nExamples: example.com · blog.example.com\n\nThis cannot be undone. Save Project As… first to keep a backup.",
"deleteDomainConfirm": "Delete every URL hosted on \"{{host}}\", along with its links, headers, images, and source snapshots?\n\nThis action cannot be undone.",
"deleteDomainResult": "Deleted {{urls}} URL row(s) from {{host}} (and {{links}} associated link(s)).",
@@ -1266,7 +1266,91 @@
"summary": "{{shown}} pages",
"firstN": "(first {{n}})",
"emptyNoData": "No internal HTML pages crawled yet — run a crawl, then fetch Search Console data.",
"emptyNoMatch": "No pages match the current filter."
"emptyNoMatch": "No pages match the current filter.",
"days16m": "Last 16 months",
"daysCustom": "Custom (Settings)",
"dateRangeTitle": "Date range for the pull. Full dimension filters live in Settings → Integrations → Google Search Console.",
"presetTitle": "Pre-defined filters for URLs verified in GSC",
"presetAll": "All",
"presetClicks": "Clicks Above 0",
"presetNoData": "No Search Analytics Data",
"presetNonIndexableData": "Non-Indexable with Search Analytics Data",
"presetOrphan": "Orphan URLs",
"presetNotOnGoogle": "URL is Not on Google",
"presetIndexableNotIndexed": "Indexable URL Not Indexed",
"presetOnGoogleIssues": "URL is on Google But Has Issues",
"presetCanonical": "User-Declared Canonical Not Selected",
"presetMobile": "Page is Not Mobile Friendly",
"presetAmp": "AMP URL Invalid",
"presetRich": "Rich Result Invalid",
"newUrlsFound": "{{n}} URL(s) appear in Search Console but were not found by the crawl.",
"newUrlsQueued": "{{n}} new URL(s) discovered in Search Console were added to the crawl.",
"crawlNewUrls": "Crawl {{n}} new URLs",
"accountTitle": "Which linked Google account this project reports on. Each account keeps its own stored data."
},
"googleAccounts": {
"title": "Connected Accounts",
"connect": "Connect account",
"connectAnother": "Connect another",
"disconnect": "Disconnect",
"confirmDisconnect": "Disconnect {{account}}? Data already pulled with it stays until the next fetch.",
"empty": "No account linked yet. Connect one — you can link several and pick which one each project uses.",
"unknownEmail": "Google account",
"needClient": "Save your OAuth client ID + secret first.",
"hint": "Each project picks which linked account it reports on, in the tab toolbar. Data pulled with different accounts is stored separately."
},
"ga4Settings": {
"title": "Analytics 4 Settings",
"account": "Account",
"accountHint": "Which linked Google account this project pulls Analytics data with.",
"dateRange": "Date Range",
"range7": "Last 7 days",
"range28": "Last 28 days",
"range90": "Last 90 days",
"hint": "The property is picked in the GA4 tab toolbar. Analytics data is near-realtime, so the range ends today."
},
"gscSettings": {
"title": "Search Console Settings",
"account": "Account",
"accountHint": "Which linked Google account this project pulls Search Console data with.",
"dateRange": "Date Range",
"range7": "Last 7 days",
"range28": "Last 28 days",
"range90": "Last 90 days",
"range16m": "Last 16 months",
"rangeCustom": "Custom",
"startDate": "Start date",
"endDate": "End date",
"dimensionFilter": "Dimension Filter",
"device": "Device",
"deviceAll": "All",
"deviceDesktop": "Desktop",
"deviceMobile": "Mobile",
"deviceTablet": "Tablet",
"type": "Search Type",
"typeWeb": "Web",
"typeImage": "Image",
"typeVideo": "Video",
"typeNews": "News",
"typeDiscover": "Discover",
"typeGoogleNews": "Google News",
"country": "Country",
"countryPlaceholder": "e.g. usa (ISO code) — blank for all",
"queryFilter": "Search Query",
"queryNone": "None",
"queryContains": "Contains",
"queryNotContains": "Doesn't contain",
"queryEquals": "Equals",
"queryNotEquals": "Doesn't equal",
"queryRegex": "Matches regex",
"queryNotRegex": "Doesn't match regex",
"queryValuePlaceholder": "Query…",
"general": "General",
"matchSlash": "Match Trailing and Non-Trailing Slash URLs",
"matchCase": "Match Uppercase & Lowercase URLs",
"limitMaxResults": "Limit Max Results",
"crawlNewUrls": "Crawl New URLs Discovered In Google Search Console",
"crawlNewUrlsHint": "After each Search Console pull, URLs that appear in GSC but were not found by the crawl are added to the crawl queue."
},
"ga4Tab": {
"title": "Google Analytics 4",
@@ -1300,7 +1384,8 @@
"summary": "{{shown}} pages",
"firstN": "(first {{n}})",
"emptyNoData": "No internal HTML pages crawled yet — run a crawl, then fetch GA4 data.",
"emptyNoMatch": "No pages match the current filter."
"emptyNoMatch": "No pages match the current filter.",
"accountTitle": "Which linked Google account this project reports on. Each account keeps its own stored data."
},
"aiTab": {
"title": "AI",
@@ -1503,7 +1588,7 @@
"customRegexPlaceholder": "Named-group regex: (?<ip>…) (?<ts>…) (?<method>…) (?<path>…) (?<status>…) (?<ua>…)",
"verifyBots": "Verify bots (rDNS)",
"verifyBotsHint": "Reverse-DNS verify a sample of bot IPs (slower; catches spoofed user-agents)",
"import": "Import Log File…",
"import": "Import Log Files…",
"analyzing": "Analyzing…",
"clear": "Clear",
"clearConfirm": "Clear all ingested log data from this project?",
@@ -1512,6 +1597,8 @@
"exportXlsx": "Excel (.xlsx, one sheet each)",
"exported": "Exported {{size}} KB → {{path}}",
"imported": "{{file}} — parsed {{parsed}}/{{total}} lines ({{format}})",
"importedBatch": "{{files}} files — parsed {{parsed}}/{{total}} lines",
"importedFailed": "{{n}} failed",
"importError": "Import failed: {{err}}",
"noData": "No log data yet",
"noDataHint": "Import a server access log (Apache, Nginx, or IIS W3C) to see which URLs search-engine bots crawl, how response codes are distributed in the log, and which logged URLs your crawl never reached.",
@@ -255,7 +255,7 @@
"dropFailedProject": "Proje açılamadı: {{name}}",
"dropNoUrls": "{{name}} içinde URL bulunamadı.",
"dropLoadedUrls": "{{name}} dosyasından {{count}} URL List moduna yüklendi.",
"crashRecovery": "FreeCrawl, temiz bitmemiş bir önceki crawl tespit etti:\n\n Başlangıç URL: {{seed}}\n Bekleyen URL: {{pending}}\n\nCrawl kaldığı yerden devam etsin mi?\n\nDevam etmek için Tamam, recovery durumunu iptal edip baştan başlamak için İptal'e tıkla.",
"crashRecovery": "FreeCrawl, temiz bitmemiş bir önceki crawl tespit etti:\n\n Mod: {{mode}}\n Başlangıç URL: {{seed}}\n Bekleyen URL: {{pending}}\n\nCrawl kaldığı yerden devam etsin mi?\n\nDevam etmek için Tamam, recovery durumunu iptal edip baştan başlamak için İptal'e tıkla.",
"deleteDomainPrompt": "GDPR Alan Temizleme — silinecek host'u gir (büyük/küçük harf duyarsız, tam eşleşme).\n\nÖrnek: example.com · blog.example.com\n\nGeri alınamaz. Yedek istiyorsan önce Projeyi Farklı Kaydet.",
"deleteDomainConfirm": "\"{{host}}\" üzerindeki tüm URL'leri, linklerini, başlıklarını, görsellerini ve kaynak snapshot'larını sil?\n\nBu işlem geri alınamaz.",
"deleteDomainResult": "{{host}} alanından {{urls}} URL satırı silindi (ve {{links}} bağlı link).",
@@ -1266,7 +1266,91 @@
"summary": "{{shown}} sayfa",
"firstN": "(ilk {{n}})",
"emptyNoData": "Henüz taranmış iç HTML sayfası yok — önce bir crawl çalıştır, sonra Search Console verisini getir.",
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok."
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok.",
"days16m": "Son 16 ay",
"daysCustom": "Özel (Ayarlar)",
"dateRangeTitle": "Getirme için tarih aralığı. Tüm dimension filtreleri Ayarlar → Entegrasyonlar → Google Search Console'da.",
"presetTitle": "GSC'de doğrulanmış URL'ler için hazır filtreler",
"presetAll": "Tümü",
"presetClicks": "Tıklaması 0'dan Fazla",
"presetNoData": "Search Analytics Verisi Yok",
"presetNonIndexableData": "Search Analytics Verisi Olan İndekslenemez",
"presetOrphan": "Yetim URL'ler",
"presetNotOnGoogle": "URL Google'da Değil",
"presetIndexableNotIndexed": "İndekslenebilir ama İndekslenmemiş",
"presetOnGoogleIssues": "Google'da ama Sorunlu",
"presetCanonical": "Kullanıcı Canonical'ı Seçilmemiş",
"presetMobile": "Sayfa Mobil Uyumlu Değil",
"presetAmp": "AMP URL Geçersiz",
"presetRich": "Zengin Sonuç Geçersiz",
"newUrlsFound": "{{n}} URL Search Console'da görünüyor ama crawl'da bulunamadı.",
"newUrlsQueued": "Search Console'da keşfedilen {{n}} yeni URL crawl'a eklendi.",
"crawlNewUrls": "{{n}} yeni URL'yi crawl et",
"accountTitle": "Bu projenin hangi bağlı Google hesabını raporlayacağı. Her hesap kendi verisini ayrı saklar."
},
"googleAccounts": {
"title": "Bağlı Hesaplar",
"connect": "Hesap bağla",
"connectAnother": "Başka hesap bağla",
"disconnect": "Bağlantıyı kes",
"confirmDisconnect": "{{account}} bağlantısı kesilsin mi? Bu hesapla çekilmiş veriler bir sonraki getirmeye kadar kalır.",
"empty": "Henüz bağlı hesap yok. Bir tane bağla — birden fazla hesap bağlayıp her projede hangisinin kullanılacağını seçebilirsin.",
"unknownEmail": "Google hesabı",
"needClient": "Önce OAuth client ID + secret'ını kaydet.",
"hint": "Her proje hangi bağlı hesabı raporlayacağını sekme araç çubuğundan seçer. Farklı hesaplarla çekilen veriler ayrı saklanır."
},
"ga4Settings": {
"title": "Analytics 4 Ayarları",
"account": "Hesap",
"accountHint": "Bu projenin Analytics verisini hangi bağlı Google hesabıyla çekeceği.",
"dateRange": "Tarih Aralığı",
"range7": "Son 7 gün",
"range28": "Son 28 gün",
"range90": "Son 90 gün",
"hint": "Property, GA4 sekmesi araç çubuğundan seçilir. Analytics verisi neredeyse gerçek zamanlı olduğu için aralık bugün biter."
},
"gscSettings": {
"title": "Search Console Ayarları",
"account": "Hesap",
"accountHint": "Bu projenin Search Console verisini hangi bağlı Google hesabıyla çekeceği.",
"dateRange": "Tarih Aralığı",
"range7": "Son 7 gün",
"range28": "Son 28 gün",
"range90": "Son 90 gün",
"range16m": "Son 16 ay",
"rangeCustom": "Özel",
"startDate": "Başlangıç tarihi",
"endDate": "Bitiş tarihi",
"dimensionFilter": "Dimension Filtresi",
"device": "Cihaz",
"deviceAll": "Tümü",
"deviceDesktop": "Masaüstü",
"deviceMobile": "Mobil",
"deviceTablet": "Tablet",
"type": "Arama Türü",
"typeWeb": "Web",
"typeImage": "Görsel",
"typeVideo": "Video",
"typeNews": "Haberler",
"typeDiscover": "Discover",
"typeGoogleNews": "Google Haberler",
"country": "Ülke",
"countryPlaceholder": "örn. usa (ISO kodu) — tümü için boş bırak",
"queryFilter": "Arama Sorgusu",
"queryNone": "Yok",
"queryContains": "İçerir",
"queryNotContains": "İçermez",
"queryEquals": "Eşittir",
"queryNotEquals": "Eşit değil",
"queryRegex": "Regex eşleşir",
"queryNotRegex": "Regex eşleşmez",
"queryValuePlaceholder": "Sorgu…",
"general": "Genel",
"matchSlash": "Sondaki Slash'lı ve Slash'sız URL'leri Eşleştir",
"matchCase": "Büyük & Küçük Harfli URL'leri Eşleştir",
"limitMaxResults": "Maks. Sonuç Sınırı",
"crawlNewUrls": "Google Search Console'da Keşfedilen Yeni URL'leri Crawl Et",
"crawlNewUrlsHint": "Her Search Console getirmesinden sonra, GSC'de görünen ama crawl'da bulunamayan URL'ler crawl kuyruğuna eklenir."
},
"ga4Tab": {
"title": "Google Analytics 4",
@@ -1300,7 +1384,8 @@
"summary": "{{shown}} sayfa",
"firstN": "(ilk {{n}})",
"emptyNoData": "Henüz taranmış iç HTML sayfası yok — önce bir crawl çalıştır, sonra GA4 verisini getir.",
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok."
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok.",
"accountTitle": "Bu projenin hangi bağlı Google hesabını raporlayacağı. Her hesap kendi verisini ayrı saklar."
},
"aiTab": {
"title": "AI",
@@ -1503,7 +1588,7 @@
"customRegexPlaceholder": "Adlandırılmış grup regex: (?<ip>…) (?<ts>…) (?<method>…) (?<path>…) (?<status>…) (?<ua>…)",
"verifyBots": "Botları doğrula (rDNS)",
"verifyBotsHint": "Bot IP'lerinin bir örneğini ters DNS ile doğrula (daha yavaş; sahte user-agent'ları yakalar)",
"import": "Log Dosyası İçe Aktar…",
"import": "Log Dosyalarını İçe Aktar…",
"analyzing": "Analiz ediliyor…",
"clear": "Temizle",
"clearConfirm": "Bu projedeki tüm log verileri temizlensin mi?",
@@ -1512,6 +1597,8 @@
"exportXlsx": "Excel (.xlsx, her tablo ayrı sayfa)",
"exported": "{{size}} KB dışa aktarıldı → {{path}}",
"imported": "{{file}} — {{parsed}}/{{total}} satır ayrıştırıldı ({{format}})",
"importedBatch": "{{files}} dosya — {{parsed}}/{{total}} satır ayrıştırıldı",
"importedFailed": "{{n}} başarısız",
"importError": "İçe aktarım başarısız: {{err}}",
"noData": "Henüz log verisi yok",
"noDataHint": "Hangi URL'leri arama motoru botlarının taradığını, logdaki yanıt kodu dağılımını ve crawl'ın hiç ulaşmadığı URL'leri görmek için bir sunucu erişim logu (Apache, Nginx veya IIS W3C) içe aktarın.",
+6 -2
View File
@@ -491,10 +491,14 @@ export const useAppStore = create<AppState>((set) => ({
return { config: next };
}),
applyProjectConfig: (config) =>
// Swap the UI to a project's saved config on open. Deliberately does
// Swap the UI to a project's saved config on open — including its
// mode and start URL, which are part of the document: a project
// crawled from a sitemap must reopen in Sitemap mode showing that
// sitemap, not blank in Spider mode while its tables hold the data.
// Also the resume path after an interrupted crawl. Deliberately does
// NOT touch global prefs — opening a project must not overwrite the
// user's default settings.
set({ config: { ...config, startUrl: '' } }),
set({ config }),
setProgress: (p) => set({ progress: p }),
setSummary: (s) => set({ summary: s }),
markCrawlFinished: () =>
@@ -2,11 +2,13 @@ import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react'
import { useVirtualizer } from '@tanstack/react-virtual';
import { useTranslation } from 'react-i18next';
import { Loader2, Search, RefreshCw } from 'lucide-react';
import type {
Ga4FetchMeta,
Ga4Property,
Ga4Row,
GoogleAuthState,
import {
defaultGa4Settings,
type Ga4FetchMeta,
type Ga4Property,
type Ga4Row,
type GoogleAccount,
type GoogleAuthState,
} from '@freecrawl/shared-types';
import { useAppStore } from '../store.js';
@@ -57,6 +59,8 @@ export function AnalyticsTab() {
const [stage, setStage] = useState<Stage>('loading');
const [authState, setAuthState] = useState<GoogleAuthState | null>(null);
const [properties, setProperties] = useState<Ga4Property[]>([]);
const [accounts, setAccounts] = useState<GoogleAccount[]>([]);
const [accountId, setAccountId] = useState('');
const [propertyId, setPropertyId] = useState('');
const [days, setDays] = useState<RangeDays>(28);
const [rows, setRows] = useState<Ga4Row[]>([]);
@@ -69,26 +73,55 @@ export function AnalyticsTab() {
const [error, setError] = useState<string | null>(null);
const scrollRef = useRef<HTMLDivElement>(null);
const loadProperties = useCallback(async () => {
const res = await window.freecrawl.ga4ListProperties();
const loadProperties = useCallback(async (account?: string) => {
const res = await window.freecrawl.ga4ListProperties(account);
if (!res.ok) {
setError(res.error);
setProperties([]);
return;
}
setError(null);
setProperties(res.properties);
setPropertyId((prev) => prev || res.properties[0]?.propertyId || '');
setPropertyId((prev) =>
// Keep the current property only if the account can still see it.
prev && res.properties.some((p) => p.propertyId === prev)
? prev
: (res.properties[0]?.propertyId ?? ''),
);
}, []);
/** Switch the project to another linked Google account: persist it,
* re-list that account's properties, reload the table. */
const changeAccount = useCallback(
(next: string) => {
setAccountId(next);
void window.freecrawl.integrationSettingsSet('ga4', { accountId: next });
void loadProperties(next);
},
[loadProperties],
);
useEffect(() => {
let cancelled = false;
void (async () => {
const [integrations, auth] = await Promise.all([
const [integrations, auth, ga4Settings] = await Promise.all([
window.freecrawl.integrationsGetAll(),
window.freecrawl.googleAuthStatus('ga4'),
window.freecrawl.integrationSettingsGet('ga4'),
]);
if (cancelled) return;
setAuthState(auth);
setAccounts(auth.accounts);
// Seed from the stored per-project settings; a stored account that
// has since been unlinked falls back to the first one.
const s = { ...defaultGa4Settings(), ...(ga4Settings ?? {}) };
const account =
auth.accounts.find((a) => a.accountId === s.accountId)?.accountId ??
auth.accounts[0]?.accountId ??
'';
setAccountId(account);
setDays(s.days);
if (s.property) setPropertyId(s.property);
const configured = integrations['ga4']?.configured ?? false;
if (!configured) {
setStage('unconfigured');
@@ -96,7 +129,7 @@ export function AnalyticsTab() {
setStage('disconnected');
} else {
setStage('ready');
void loadProperties();
void loadProperties(account);
}
})();
return () => {
@@ -110,11 +143,12 @@ export function AnalyticsTab() {
offset: 0,
search: search || undefined,
filter,
accountId: accountId || undefined,
});
setRows(res.rows);
setTotal(res.total);
setMeta(res.meta);
}, [search, filter]);
}, [search, filter, accountId]);
useEffect(() => {
if (stage !== 'ready') return;
@@ -144,9 +178,12 @@ export function AnalyticsTab() {
try {
const res = await window.freecrawl.googleAuthStart('ga4');
setAuthState(res.state);
setAccounts(res.state.accounts);
if (res.ok && res.state.connected) {
const account = res.state.accounts[0]?.accountId ?? '';
setAccountId(account);
setStage('ready');
void loadProperties();
void loadProperties(account);
} else if (res.error) {
setError(res.error);
}
@@ -155,13 +192,27 @@ export function AnalyticsTab() {
}
}, [loadProperties]);
/** Unlink the account currently selected in the toolbar; with several
* linked, the tab stays usable and falls back to the next one. */
const disconnect = useCallback(async () => {
const next = await window.freecrawl.googleAuthRevoke('ga4');
const next = await window.freecrawl.googleAuthRevoke(
'ga4',
accountId || undefined,
);
setAuthState(next);
setAccounts(next.accounts);
setProperties([]);
setPropertyId('');
setStage('disconnected');
}, []);
if (!next.connected) {
setAccountId('');
setStage('disconnected');
return;
}
const fallback = next.accounts[0]!.accountId;
setAccountId(fallback);
void window.freecrawl.integrationSettingsSet('ga4', { accountId: fallback });
void loadProperties(fallback);
}, [accountId, loadProperties]);
const runFetch = useCallback(async () => {
if (!propertyId || fetching) return;
@@ -173,6 +224,7 @@ export function AnalyticsTab() {
property: propertyId,
propertyName: selected?.displayName || propertyId,
days,
accountId: accountId || undefined,
});
if (!res.ok) {
setError(res.error);
@@ -183,7 +235,7 @@ export function AnalyticsTab() {
} finally {
setFetching(false);
}
}, [propertyId, properties, days, fetching, reload]);
}, [propertyId, properties, days, accountId, fetching, reload]);
// ── Gate states ──────────────────────────────────────────────────────
if (stage === 'loading') {
@@ -277,6 +329,24 @@ export function AnalyticsTab() {
</select>
<div className="ml-auto flex items-center gap-2">
{accounts.length > 1 && (
<select
className="h-6 max-w-[170px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={accountId}
onChange={(e) => changeAccount(e.target.value)}
disabled={fetching}
title={t('ga4Tab.accountTitle', {
defaultValue:
'Which linked Google account this project reports on. Each account keeps its own stored data.',
})}
>
{accounts.map((a) => (
<option key={a.accountId} value={a.accountId}>
{a.email ?? a.accountId}
</option>
))}
</select>
)}
<select
className="h-6 max-w-[260px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={propertyId}
@@ -327,7 +397,10 @@ export function AnalyticsTab() {
<span>
{t('ga4Tab.connectedAs', {
defaultValue: 'Connected as {{email}}',
email: authState?.email ?? 'Google account',
email:
accounts.find((a) => a.accountId === accountId)?.email ??
authState?.email ??
'Google account',
})}
</span>
<button
@@ -1,14 +1,18 @@
import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react';
import { useVirtualizer } from '@tanstack/react-virtual';
import { useTranslation } from 'react-i18next';
import { Loader2, Search, RefreshCw } from 'lucide-react';
import { Loader2, Search, RefreshCw, Plus } from 'lucide-react';
import { InfoTip } from '../components/InfoTip.js';
import type {
GoogleAuthState,
GscFetchMeta,
GscInspectProgress,
GscRow,
GscSite,
import {
defaultGscSettings,
type GoogleAccount,
type GoogleAuthState,
type GscDateRange,
type GscFetchMeta,
type GscInspectProgress,
type GscPreset,
type GscRow,
type GscSite,
} from '@freecrawl/shared-types';
import { useAppStore } from '../store.js';
@@ -22,11 +26,26 @@ const POLL_MS_RUNNING = 5000;
* small enough chunks that one click is reversible. */
const INSPECT_BATCH = 100;
type FilterMode = 'all' | 'with-data' | 'without-data';
type RangeDays = 7 | 28 | 90;
/** What the tab is currently showing — drives the three-way render. */
type Stage = 'loading' | 'unconfigured' | 'disconnected' | 'ready';
/** Quick-filter presets, mirroring Screaming Frog's GSC filter dropdown.
* `needsInspection` entries only match once URL Inspection has been run. */
const FILTER_PRESETS: { value: GscPreset; labelKey: string; label: string }[] = [
{ value: 'all', labelKey: 'gscTab.presetAll', label: 'All' },
{ value: 'clicks-above-0', labelKey: 'gscTab.presetClicks', label: 'Clicks Above 0' },
{ value: 'without-data', labelKey: 'gscTab.presetNoData', label: 'No Search Analytics Data' },
{ value: 'non-indexable-with-data', labelKey: 'gscTab.presetNonIndexableData', label: 'Non-Indexable with Search Analytics Data' },
{ value: 'orphan', labelKey: 'gscTab.presetOrphan', label: 'Orphan URLs' },
{ value: 'not-on-google', labelKey: 'gscTab.presetNotOnGoogle', label: 'URL is Not on Google' },
{ value: 'indexable-not-indexed', labelKey: 'gscTab.presetIndexableNotIndexed', label: 'Indexable URL Not Indexed' },
{ value: 'on-google-with-issues', labelKey: 'gscTab.presetOnGoogleIssues', label: 'URL is on Google But Has Issues' },
{ value: 'canonical-mismatch', labelKey: 'gscTab.presetCanonical', label: 'User-Declared Canonical Not Selected' },
{ value: 'not-mobile-friendly', labelKey: 'gscTab.presetMobile', label: 'Page is Not Mobile Friendly' },
{ value: 'amp-invalid', labelKey: 'gscTab.presetAmp', label: 'AMP URL Invalid' },
{ value: 'rich-result-invalid', labelKey: 'gscTab.presetRich', label: 'Rich Result Invalid' },
];
/** Average-position bands → cell colour (lower is better). */
function positionClass(v: number): string {
if (v <= 0) return 'text-surface-600';
@@ -55,42 +74,80 @@ export function SearchConsoleTab() {
const [stage, setStage] = useState<Stage>('loading');
const [authState, setAuthState] = useState<GoogleAuthState | null>(null);
const [sites, setSites] = useState<GscSite[]>([]);
const [accounts, setAccounts] = useState<GoogleAccount[]>([]);
const [accountId, setAccountId] = useState('');
const [property, setProperty] = useState('');
const [days, setDays] = useState<RangeDays>(28);
const [dateRange, setDateRange] = useState<GscDateRange>('28d');
const [rows, setRows] = useState<GscRow[]>([]);
const [total, setTotal] = useState(0);
const [meta, setMeta] = useState<GscFetchMeta | null>(null);
const [search, setSearch] = useState('');
const [filter, setFilter] = useState<FilterMode>('all');
const [filter, setFilter] = useState<GscPreset>('all');
const [connecting, setConnecting] = useState(false);
const [fetching, setFetching] = useState(false);
const [inspecting, setInspecting] = useState(false);
const [inspectProgress, setInspectProgress] = useState<GscInspectProgress | null>(null);
const [error, setError] = useState<string | null>(null);
/** After a pull: how many GSC URLs were missing from the crawl, and
* whether they were auto-queued (crawl-new-urls setting on). */
const [newUrls, setNewUrls] = useState<{ count: number; queued: number } | null>(null);
const [crawlingNew, setCrawlingNew] = useState(false);
const scrollRef = useRef<HTMLDivElement>(null);
const loadSites = useCallback(async () => {
const res = await window.freecrawl.gscListSites();
const loadSites = useCallback(async (account?: string) => {
const res = await window.freecrawl.gscListSites(account);
if (!res.ok) {
setError(res.error);
setSites([]);
return;
}
setError(null);
setSites(res.sites);
setProperty((prev) => prev || res.sites[0]?.siteUrl || '');
setProperty((prev) =>
// Keep the current property only if the account can still see it.
prev && res.sites.some((s) => s.siteUrl === prev)
? prev
: (res.sites[0]?.siteUrl ?? ''),
);
}, []);
/** Switch the project to another linked Google account: persist the
* choice, then re-list that account's properties and reload the table
* (each account's rows are stored separately). */
const changeAccount = useCallback(
(next: string) => {
setAccountId(next);
setNewUrls(null);
void window.freecrawl.integrationSettingsSet('gsc', { accountId: next });
void loadSites(next);
},
[loadSites],
);
// Resolve the tab's stage on mount: needs OAuth credentials, then a
// connected account, before the data view is meaningful.
useEffect(() => {
let cancelled = false;
void (async () => {
const [integrations, auth] = await Promise.all([
const [integrations, auth, gscSettings] = await Promise.all([
window.freecrawl.integrationsGetAll(),
window.freecrawl.googleAuthStatus('gsc'),
window.freecrawl.integrationSettingsGet('gsc'),
]);
if (cancelled) return;
setAuthState(auth);
setAccounts(auth.accounts);
// Seed the toolbar from the stored per-project GSC settings so the
// account + property + date range persist across sessions. A stored
// account that has since been unlinked falls back to the first one.
const s = { ...defaultGscSettings(), ...(gscSettings ?? {}) };
const account =
auth.accounts.find((a) => a.accountId === s.accountId)?.accountId ??
auth.accounts[0]?.accountId ??
'';
setAccountId(account);
setDateRange(s.dateRange);
if (s.property) setProperty(s.property);
const configured = integrations['gsc']?.configured ?? false;
if (!configured) {
setStage('unconfigured');
@@ -98,7 +155,7 @@ export function SearchConsoleTab() {
setStage('disconnected');
} else {
setStage('ready');
void loadSites();
void loadSites(account);
}
})();
return () => {
@@ -112,11 +169,12 @@ export function SearchConsoleTab() {
offset: 0,
search: search || undefined,
filter,
accountId: accountId || undefined,
});
setRows(res.rows);
setTotal(res.total);
setMeta(res.meta);
}, [search, filter]);
}, [search, filter, accountId]);
// Candidate list — only while the data view is showing.
useEffect(() => {
@@ -147,9 +205,12 @@ export function SearchConsoleTab() {
try {
const res = await window.freecrawl.googleAuthStart('gsc');
setAuthState(res.state);
setAccounts(res.state.accounts);
if (res.ok && res.state.connected) {
const account = res.state.accounts[0]?.accountId ?? '';
setAccountId(account);
setStage('ready');
void loadSites();
void loadSites(account);
} else if (res.error) {
setError(res.error);
}
@@ -158,30 +219,72 @@ export function SearchConsoleTab() {
}
}, [loadSites]);
/** Unlink the account currently selected in the toolbar. With several
* linked, the tab stays usable and falls back to the next one. */
const disconnect = useCallback(async () => {
const next = await window.freecrawl.googleAuthRevoke('gsc');
const next = await window.freecrawl.googleAuthRevoke(
'gsc',
accountId || undefined,
);
setAuthState(next);
setAccounts(next.accounts);
setSites([]);
setProperty('');
setStage('disconnected');
if (!next.connected) {
setAccountId('');
setStage('disconnected');
return;
}
const fallback = next.accounts[0]!.accountId;
setAccountId(fallback);
void window.freecrawl.integrationSettingsSet('gsc', { accountId: fallback });
void loadSites(fallback);
}, [accountId, loadSites]);
const changeDateRange = useCallback((next: GscDateRange) => {
setDateRange(next);
void window.freecrawl.integrationSettingsSet('gsc', { dateRange: next });
}, []);
const runFetch = useCallback(async () => {
if (!property || fetching) return;
setFetching(true);
setError(null);
setNewUrls(null);
try {
const res = await window.freecrawl.gscFetch({ property, days });
// Date range + dimension filters + row cap come from the stored GSC
// settings; the main process reads them. We only pass the property.
const res = await window.freecrawl.gscFetch({
property,
accountId: accountId || undefined,
});
if (!res.ok) {
setError(res.error);
} else {
setMeta(res.meta);
setNewUrls({ count: res.newUrlCount ?? 0, queued: res.queuedNewUrls ?? 0 });
await reload();
}
} finally {
setFetching(false);
}
}, [property, days, fetching, reload]);
}, [property, accountId, fetching, reload]);
const crawlNew = useCallback(async () => {
if (crawlingNew) return;
setCrawlingNew(true);
setError(null);
try {
const res = await window.freecrawl.gscCrawlNewUrls();
if (!res.ok) {
setError(res.error);
} else {
setNewUrls({ count: res.candidateCount, queued: res.queued });
}
} finally {
setCrawlingNew(false);
}
}, [crawlingNew]);
const runInspect = useCallback(async () => {
if (!property || inspecting) return;
@@ -201,13 +304,17 @@ export function SearchConsoleTab() {
setInspecting(true);
setError(null);
try {
await window.freecrawl.gscInspectRun({ property, urls });
await window.freecrawl.gscInspectRun({
property,
urls,
accountId: accountId || undefined,
});
} finally {
setInspecting(false);
setInspectProgress(null);
void reload();
}
}, [property, inspecting, rows, t, reload]);
}, [property, accountId, inspecting, rows, t, reload]);
// Live URL Inspection progress.
useEffect(() => {
@@ -295,22 +402,39 @@ export function SearchConsoleTab() {
/>
</div>
<select
className="h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
className="h-6 max-w-[220px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={filter}
onChange={(e) => setFilter(e.target.value as FilterMode)}
onChange={(e) => setFilter(e.target.value as GscPreset)}
title={t('gscTab.presetTitle', {
defaultValue: 'Pre-defined filters for URLs verified in GSC',
})}
>
<option value="all">
{t('gscTab.filterAll', { defaultValue: 'All pages' })}
</option>
<option value="with-data">
{t('gscTab.filterWith', { defaultValue: 'With GSC data' })}
</option>
<option value="without-data">
{t('gscTab.filterWithout', { defaultValue: 'No GSC data' })}
</option>
{FILTER_PRESETS.map((p) => (
<option key={p.value} value={p.value}>
{t(p.labelKey, { defaultValue: p.label })}
</option>
))}
</select>
<div className="ml-auto flex items-center gap-2">
{accounts.length > 1 && (
<select
className="h-6 max-w-[170px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={accountId}
onChange={(e) => changeAccount(e.target.value)}
disabled={fetching || inspecting}
title={t('gscTab.accountTitle', {
defaultValue:
'Which linked Google account this project reports on. Each account keeps its own stored data.',
})}
>
{accounts.map((a) => (
<option key={a.accountId} value={a.accountId}>
{a.email ?? a.accountId}
</option>
))}
</select>
)}
<select
className="h-6 max-w-[200px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={property}
@@ -331,13 +455,19 @@ export function SearchConsoleTab() {
</select>
<select
className="h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={days}
onChange={(e) => setDays(Number(e.target.value) as RangeDays)}
value={dateRange}
onChange={(e) => changeDateRange(e.target.value as GscDateRange)}
disabled={fetching}
title={t('gscTab.dateRangeTitle', {
defaultValue:
'Date range for the pull. Full dimension filters live in Settings → Integrations → Google Search Console.',
})}
>
<option value={7}>{t('gscTab.days7', { defaultValue: 'Last 7 days' })}</option>
<option value={28}>{t('gscTab.days28', { defaultValue: 'Last 28 days' })}</option>
<option value={90}>{t('gscTab.days90', { defaultValue: 'Last 90 days' })}</option>
<option value="7d">{t('gscTab.days7', { defaultValue: 'Last 7 days' })}</option>
<option value="28d">{t('gscTab.days28', { defaultValue: 'Last 28 days' })}</option>
<option value="90d">{t('gscTab.days90', { defaultValue: 'Last 90 days' })}</option>
<option value="16m">{t('gscTab.days16m', { defaultValue: 'Last 16 months' })}</option>
<option value="custom">{t('gscTab.daysCustom', { defaultValue: 'Custom (Settings)' })}</option>
</select>
<button
type="button"
@@ -384,12 +514,52 @@ export function SearchConsoleTab() {
</div>
)}
{/* New-URLs-in-GSC banner — offer to crawl orphan GSC URLs */}
{newUrls && newUrls.count > 0 && (
<div className="flex items-center gap-2 border-b border-amber-700/40 bg-amber-900/15 px-3 py-1 text-[10px] text-amber-200">
<span>
{newUrls.queued > 0
? t('gscTab.newUrlsQueued', {
defaultValue:
'{{n}} new URL(s) discovered in Search Console were added to the crawl.',
n: newUrls.queued,
})
: t('gscTab.newUrlsFound', {
defaultValue:
'{{n}} URL(s) appear in Search Console but were not found by the crawl.',
n: newUrls.count,
})}
</span>
{newUrls.queued === 0 && (
<button
type="button"
onClick={() => void crawlNew()}
disabled={crawlingNew}
className="ml-auto inline-flex items-center gap-1 rounded bg-amber-600/80 px-2 py-0.5 text-[10px] font-medium text-white hover:bg-amber-600 disabled:opacity-50"
>
{crawlingNew ? (
<Loader2 size={11} className="animate-spin" />
) : (
<Plus size={11} />
)}
{t('gscTab.crawlNewUrls', {
defaultValue: 'Crawl {{n}} new URLs',
n: newUrls.count,
})}
</button>
)}
</div>
)}
{/* Context line — connection + last pull */}
<div className="flex items-center gap-2 border-b border-surface-800 bg-surface-900/20 px-3 py-1 text-[10px] text-surface-500">
<span>
{t('gscTab.connectedAs', {
defaultValue: 'Connected as {{email}}',
email: authState?.email ?? 'Google account',
email:
accounts.find((a) => a.accountId === accountId)?.email ??
authState?.email ??
'Google account',
})}
</span>
<button
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/mcp-server",
"version": "0.9.2",
"version": "0.9.3",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "freecrawl-seo",
"version": "0.9.2",
"version": "0.9.3",
"private": true,
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
"license": "MIT",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/core",
"version": "0.9.2",
"version": "0.9.3",
"private": true,
"type": "module",
"main": "./dist/index.js",
+111 -13
View File
@@ -236,22 +236,75 @@ export class Crawler extends EventEmitter {
/** Pending queue snapshot used by the checkpoint timer. Mirrors
* what's in `this.queue` minus already-completed items. */
private pendingItems = new Map<string, number>();
/**
* Enqueues and completions accumulated since the last checkpoint.
* Shipping this delta instead of the whole of `pendingItems` is what
* keeps the checkpoint proportional to crawl throughput: on a crawl
* with a 200k-URL frontier the old full snapshot rebuilt a 200k-entry
* array on the main thread, structured-cloned it to the writer worker
* and re-inserted every row twice a minute, which blocked the writer
* FIFO the per-URL page writes share and stalled the crawl loop for
* tens of seconds each pass. The delta is a few hundred rows.
*/
private queueAdded = new Map<string, number>();
private queueRemoved = new Set<string>();
/** First tick of a run does a full replace so the table starts in a
* known state; set again after a failed write to resynchronise. */
private queueCheckpointNeedsFull = true;
private startQueueCheckpointTimer(): void {
if (this.queueCheckpointTimer) return;
// A resumed run inherits rows written by the previous process, and
// a fresh run may inherit a discarded checkpoint, so the first tick
// always rebuilds from `pendingItems` rather than trusting the
// table to match.
this.queueCheckpointNeedsFull = true;
this.queueAdded.clear();
this.queueRemoved.clear();
this.queueCheckpointTimer = setInterval(() => {
if (this.stopped || !this.running) return;
const items = Array.from(this.pendingItems.entries()).map(([url, depth]) => ({
url,
depth,
}));
void this.dbCall<void>('checkpointQueue', [items, this.config.startUrl]).catch((err) => {
this.emit(
'debug',
`queue checkpoint failed: ${err instanceof Error ? err.message : String(err)}`,
);
void this.flushQueueCheckpoint();
// Piggyback the planner-statistics refresh on the same tick. A
// crawl starts with empty tables, so any plan chosen at row zero
// assumes tables that never grow; `PRAGMA optimize` re-analyses
// only what has actually drifted, so this is a no-op most ticks
// and tens of milliseconds when it isn't.
void this.dbCall<void>('optimize', []).catch(() => {
/* statistics are an optimisation, never a correctness input */
});
}, Crawler.QUEUE_CHECKPOINT_INTERVAL_MS);
}
private async flushQueueCheckpoint(): Promise<void> {
const full = this.queueCheckpointNeedsFull;
if (!full && this.queueAdded.size === 0 && this.queueRemoved.size === 0) return;
const patch = full
? {
replaceAll: true,
added: Array.from(this.pendingItems, ([url, depth]) => ({ url, depth })),
removed: [],
}
: {
replaceAll: false,
added: Array.from(this.queueAdded, ([url, depth]) => ({ url, depth })),
removed: Array.from(this.queueRemoved),
};
// Clear before awaiting so events arriving during the write land in
// the next delta instead of being written twice.
this.queueAdded.clear();
this.queueRemoved.clear();
this.queueCheckpointNeedsFull = false;
try {
await this.dbCall<void>('checkpointQueue', [patch, this.config.startUrl]);
} catch (err) {
// The delta is already gone, so fall back to a full rebuild next
// tick — otherwise the checkpoint would silently drift from the
// live queue and a resume would replay the wrong set.
this.queueCheckpointNeedsFull = true;
this.emit(
'debug',
`queue checkpoint failed: ${err instanceof Error ? err.message : String(err)}`,
);
}
}
private stopQueueCheckpointTimer(): void {
if (this.queueCheckpointTimer) {
clearInterval(this.queueCheckpointTimer);
@@ -952,6 +1005,18 @@ export class Crawler extends EventEmitter {
* — see rule 1.7 (root-cause perf, never mask by degrading UX).
*/
private async runPostCrawlPasses(): Promise<void> {
// Refresh the planner's statistics before anything else runs. Every
// pass below is a correlated-subquery / self-join workload, and the
// tables they read only reached their final size moments ago — the
// plans SQLite would pick from stale (or absent) stats are the ones
// that turn a sub-second pass into a multi-minute one.
await yieldToEventLoop();
this.setOp('post-crawl:optimize');
try {
await this.dbCall<void>('optimize', []);
} catch {
/* statistics are an optimisation, never a correctness input */
}
if (this.config.analyseInlinks) {
await yieldToEventLoop();
this.emit('info', 'Recomputing inlinks…');
@@ -1740,10 +1805,18 @@ export class Crawler extends EventEmitter {
afterReset?: () => Promise<void>,
): Promise<void> {
const previousStart = this.db.getMeta('startUrl');
if (previousStart !== fingerprint) {
this.db.reset();
// A crash-recovery resume continues the interrupted crawl, so its rows
// must survive even though the fingerprint may no longer match — a
// sitemap that gained or lost a single URL changes the count baked into
// it, and wiping there would destroy exactly the data the user asked to
// keep. The fingerprint is left pointing at the list that was actually
// crawled, so a later Start still reads a drifted list as a new crawl.
if (!this.resumeOnly) {
if (previousStart !== fingerprint) {
this.db.reset();
}
this.db.setMeta('startUrl', fingerprint);
}
this.db.setMeta('startUrl', fingerprint);
if (afterReset) await afterReset();
// Force exact-url scope so anything fetched in fetchAndProcess never
@@ -1762,6 +1835,22 @@ export class Crawler extends EventEmitter {
this.startIssueRecomputeTimer();
this.startQueueCheckpointTimer();
// Resume: mark everything already in the project as seen so the loop
// below only picks up what is genuinely still outstanding. Without it a
// resumed List/Sitemap run re-fetches the whole source list — the exact
// "it didn't carry on where it left off" the checkpoint exists to
// prevent. Spider mode gets the same treatment from `hydrateFromDb`.
// The checkpointed pending URLs are already queued by the caller
// (`enqueueCheckpointed`) before `start()`, so they survive this sweep;
// list entries added since the interruption are still picked up below.
if (this.resumeOnly) {
for (const known of this.db.getAllUrls()) {
this.seen.add(known);
}
this.crawled = this.db.countCrawledUrls();
this.resourceAdmitted = this.db.countSubresourceUrls();
}
for (const u of urls) {
this.enqueue({ url: u, depth: 0 });
}
@@ -2568,6 +2657,8 @@ export class Crawler extends EventEmitter {
if (item.resourceRow) this.resourceAdmitted++;
this.pending++;
this.pendingItems.set(item.url, item.depth);
this.queueRemoved.delete(item.url);
this.queueAdded.set(item.url, item.depth);
this.queue
.add(() => this.fetchAndProcess(item), { priority: Crawler.priorityFor(item) })
.catch((err: unknown) => {
@@ -2582,6 +2673,8 @@ export class Crawler extends EventEmitter {
// failures are recorded in the urls table and shouldn't be
// retried by a resumed crawl.
this.pendingItems.delete(item.url);
this.queueAdded.delete(item.url);
this.queueRemoved.add(item.url);
});
}
@@ -3268,7 +3361,12 @@ export class Crawler extends EventEmitter {
}
const xRobotsLower = xRobotsTag?.toLowerCase() ?? '';
const headerNoindex = xRobotsLower.includes('noindex');
// Same `none` shorthand as the meta tag, plus the header's optional
// `<bot>: <directives>` form (`X-Robots-Tag: googlebot: none`) — the
// colon is a token separator here, not part of a directive.
const headerNoindex =
xRobotsLower.includes('noindex') ||
xRobotsLower.split(/[\s,;:]+/).includes('none');
let indexability: Indexability = 'indexable';
let reason: string | null = null;
+39 -3
View File
@@ -57,6 +57,12 @@ export interface ParsedPage {
canonical: string | null;
/** Number of `<link rel="canonical">` elements declared. >1 is a confusion signal. */
canonicalCount: number;
/**
* Every `<meta name="robots">` content value on the page, lowercased
* and joined with `, ` in document order. Multiple declarations are
* kept because search engines union them and honour the most
* restrictive directive.
*/
metaRobots: string | null;
lang: string | null;
viewport: string | null;
@@ -524,7 +530,28 @@ export function parseHtml(
const canonicalEls = $('link[rel="canonical"]');
const canonical = (canonicalEls.first().attr('href') ?? '').trim() || null;
const canonicalCount = canonicalEls.length;
const metaRobots = ($('meta[name="robots"]').attr('content') ?? '').trim().toLowerCase() || null;
// Robots directives — collected from EVERY `<meta name="robots">` in
// the document, not just the first. Two details that a plain
// `$('meta[name="robots"]').attr('content')` gets wrong on real sites:
//
// 1. `name` is matched case-insensitively. CSS attribute *values* are
// case-sensitive outside a small HTML whitelist that `name` is not
// part of, so `<meta name="ROBOTS">` matched nothing at all and the
// page came back "Indexable" with an empty Meta Robots column.
// 2. A page may declare the tag more than once — a theme default plus
// an SEO plugin's own output is the classic WordPress case. Search
// engines union the directives and apply the most restrictive one,
// so reading only the first tag reported `index,follow` + `noindex`
// pages as indexable. The contents are joined (comma-separated, in
// document order) into one value: the UI's Meta Robots column then
// shows the conflict instead of hiding half of it.
const robotsMetaContents: string[] = [];
$('meta').each((_, el) => {
if (($(el).attr('name') ?? '').trim().toLowerCase() !== 'robots') return;
const content = ($(el).attr('content') ?? '').trim().toLowerCase();
if (content) robotsMetaContents.push(content);
});
const metaRobots = robotsMetaContents.length > 0 ? robotsMetaContents.join(', ') : null;
const lang = ($('html').attr('lang') ?? '').trim() || null;
const viewport = ($('meta[name="viewport"]').attr('content') ?? '').trim() || null;
const ogTitle =
@@ -1076,8 +1103,17 @@ export function parseHtml(
}
}
const hasNoindex = metaRobots !== null && metaRobots.includes('noindex');
const hasNofollow = metaRobots !== null && metaRobots.includes('nofollow');
// `none` is the spec's shorthand for `noindex, nofollow` — a page
// carrying only that directive used to slip through as indexable. It is
// matched as a whole token (split on comma / semicolon / whitespace so
// malformed separators still tokenize) rather than as a substring,
// which would fire on unrelated values. The substring check on the
// combined string is kept alongside it so any separator we haven't
// thought of still yields the old, lenient detection.
const robotsTokens = new Set((metaRobots ?? '').split(/[\s,;]+/).filter(Boolean));
const hasNone = robotsTokens.has('none');
const hasNoindex = metaRobots !== null && (metaRobots.includes('noindex') || hasNone);
const hasNofollow = metaRobots !== null && (metaRobots.includes('nofollow') || hasNone);
// Text/code ratio — visible body text bytes divided by raw HTML bytes,
// rounded to integer percent. Reuses the script/style/noscript-stripped
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/db",
"version": "0.9.2",
"version": "0.9.3",
"private": true,
"type": "module",
"main": "./dist/index.js",
+126
View File
@@ -1984,6 +1984,132 @@ const MIGRATIONS: Migration[] = [
}
},
},
{
version: 83,
name: 'add_gsc_inspection_appearance_verdicts',
// GSC URL Inspection returns more than the index-status verdict:
// mobileUsabilityResult / ampResult / richResultsResult each carry
// their own PASS/FAIL verdict. Storing them powers the "Page is Not
// Mobile Friendly", "AMP URL Invalid" and "Rich Result Invalid"
// filter presets on the Search Console tab.
//
// Left NULL for rows inspected before these columns existed — absent
// and "not inspected for this facet" are the same thing to consumers.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(gsc_inspection_results)')
.all() as unknown as { name: string }[];
if (!cols.some((c) => c.name === 'mobile_verdict')) {
db.exec('ALTER TABLE gsc_inspection_results ADD COLUMN mobile_verdict TEXT');
}
if (!cols.some((c) => c.name === 'amp_verdict')) {
db.exec('ALTER TABLE gsc_inspection_results ADD COLUMN amp_verdict TEXT');
}
if (!cols.some((c) => c.name === 'rich_results_verdict')) {
db.exec(
'ALTER TABLE gsc_inspection_results ADD COLUMN rich_results_verdict TEXT',
);
}
},
},
{
version: 84,
name: 'add_google_account_id',
// Multi-account GSC / GA4: a user can link several Google accounts to
// the same integration (their own property plus a client's), so these
// three snapshot tables can no longer be keyed by `url` alone — two
// accounts may legitimately report on the same URL.
//
// SQLite cannot ALTER a PRIMARY KEY, so each table is rebuilt with the
// composite `(url, account_id)` key and its rows copied across.
// Pre-existing rows came from the single connected account and get
// `account_id = ''`; the main process adopts them into that account's
// id the first time the account list is resolved, so an upgrade keeps
// its data instead of silently showing an empty table.
up: (db) => {
const hasAccountCol = (table: string): boolean =>
(db.prepare(`PRAGMA table_info(${table})`).all() as unknown as {
name: string;
}[]).some((c) => c.name === 'account_id');
if (!hasAccountCol('gsc_results')) {
db.exec(`
CREATE TABLE gsc_results_new (
url TEXT NOT NULL,
account_id TEXT NOT NULL DEFAULT '',
clicks INTEGER,
impressions INTEGER,
ctr REAL,
position REAL,
fetched_at TEXT NOT NULL,
PRIMARY KEY (url, account_id)
);
INSERT INTO gsc_results_new (url, account_id, clicks, impressions, ctr, position, fetched_at)
SELECT url, '', clicks, impressions, ctr, position, fetched_at FROM gsc_results;
DROP TABLE gsc_results;
ALTER TABLE gsc_results_new RENAME TO gsc_results;
CREATE INDEX IF NOT EXISTS idx_gsc_results_account ON gsc_results(account_id);
`);
}
if (!hasAccountCol('ga4_results')) {
db.exec(`
CREATE TABLE ga4_results_new (
url TEXT NOT NULL,
account_id TEXT NOT NULL DEFAULT '',
sessions INTEGER,
users INTEGER,
pageviews INTEGER,
engagement_rate REAL,
avg_session_duration REAL,
fetched_at TEXT NOT NULL,
PRIMARY KEY (url, account_id)
);
INSERT INTO ga4_results_new (url, account_id, sessions, users, pageviews,
engagement_rate, avg_session_duration, fetched_at)
SELECT url, '', sessions, users, pageviews,
engagement_rate, avg_session_duration, fetched_at FROM ga4_results;
DROP TABLE ga4_results;
ALTER TABLE ga4_results_new RENAME TO ga4_results;
CREATE INDEX IF NOT EXISTS idx_ga4_results_account ON ga4_results(account_id);
`);
}
if (!hasAccountCol('gsc_inspection_results')) {
db.exec(`
CREATE TABLE gsc_inspection_results_new (
url TEXT NOT NULL,
account_id TEXT NOT NULL DEFAULT '',
verdict TEXT,
coverage_state TEXT,
robots_txt_state TEXT,
indexing_state TEXT,
last_crawl_time TEXT,
google_canonical TEXT,
user_canonical TEXT,
mobile_verdict TEXT,
amp_verdict TEXT,
rich_results_verdict TEXT,
status TEXT NOT NULL,
error TEXT,
fetched_at TEXT NOT NULL,
PRIMARY KEY (url, account_id)
);
INSERT INTO gsc_inspection_results_new
(url, account_id, verdict, coverage_state, robots_txt_state, indexing_state,
last_crawl_time, google_canonical, user_canonical,
mobile_verdict, amp_verdict, rich_results_verdict, status, error, fetched_at)
SELECT url, '', verdict, coverage_state, robots_txt_state, indexing_state,
last_crawl_time, google_canonical, user_canonical,
mobile_verdict, amp_verdict, rich_results_verdict, status, error, fetched_at
FROM gsc_inspection_results;
DROP TABLE gsc_inspection_results;
ALTER TABLE gsc_inspection_results_new RENAME TO gsc_inspection_results;
CREATE INDEX IF NOT EXISTS idx_gsc_inspection_account ON gsc_inspection_results(account_id);
`);
}
},
},
];
export function runMigrations(db: DatabaseSync): void {
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/shared-types",
"version": "0.9.2",
"version": "0.9.3",
"private": true,
"type": "module",
"main": "./dist/index.js",
+159 -3
View File
@@ -10,14 +10,33 @@
* encrypted at rest with `safeStorage`, exactly like API keys.
*/
/**
* One connected Google account for an integration. A user can link
* several accounts to the same integration (e.g. their own Search
* Console plus a client's), each with its own refresh token; the
* per-project settings pick which one a crawl's data comes from.
*/
export interface GoogleAccount {
/** Opaque stable id — the key the token store and `account_id`
* columns use. Never the email, so a renamed/aliased address can't
* orphan the stored rows. */
accountId: string;
/** The Google account address, when the userinfo call succeeded. */
email: string | null;
/** ISO timestamp this account was connected. */
connectedAt: string;
}
/** Connection state of one OAuth-BYOC Google integration. */
export interface GoogleAuthState {
/** True once a refresh token is stored — the integration is connected. */
/** True once at least one account is connected. */
connected: boolean;
/** The Google account email the tokens belong to, when known. */
/** First connected account's email — kept for single-account callers. */
email: string | null;
/** ISO timestamp the connection was established. */
/** ISO timestamp the first connection was established. */
connectedAt: string | null;
/** Every connected account, in connection order. */
accounts: GoogleAccount[];
}
/** Result of an interactive `googleAuthStart` consent flow. */
@@ -73,11 +92,125 @@ export interface GscInspectionResult {
googleCanonical: string | null;
/** The canonical the page declared. */
userCanonical: string | null;
/** Mobile Usability verdict — `PASS` / `FAIL` / `NEUTRAL` /
* `VERDICT_UNSPECIFIED`. Null when inspection predates this column. */
mobileVerdict: string | null;
/** AMP result verdict, when the page declares an AMP version. */
ampVerdict: string | null;
/** Rich-results verdict across all detected structured-data items. */
richResultsVerdict: string | null;
status: 'ok' | 'error';
error: string | null;
fetchedAt: string;
}
/**
* Quick-filter presets for the Search Console tab, mirroring Screaming
* Frog's "verified in GSC" filter dropdown. `all` / `with-data` /
* `without-data` are the legacy values the MCP `query_gsc` tool still
* sends; the rest are the SF-parity presets. Presets past
* `non-indexable-with-data` need URL Inspection data to have been run;
* `orphan` widens the candidate set to GSC URLs missing from the crawl.
*/
export type GscPreset =
| 'all'
| 'with-data'
| 'without-data'
| 'clicks-above-0'
| 'non-indexable-with-data'
| 'orphan'
| 'not-on-google'
| 'indexable-not-indexed'
| 'on-google-with-issues'
| 'canonical-mismatch'
| 'not-mobile-friendly'
| 'amp-invalid'
| 'rich-result-invalid';
/** Date-range presets for a GSC pull. `16m` is the API's 16-month max;
* `custom` uses the explicit `startDate` / `endDate`. */
export type GscDateRange = '7d' | '28d' | '90d' | '16m' | 'custom';
/** Device dimension filter — API values (or `all` for no filter). */
export type GscDeviceFilter = 'all' | 'DESKTOP' | 'MOBILE' | 'TABLET';
/** Search-appearance type the metrics are pulled for. */
export type GscSearchType =
| 'web'
| 'image'
| 'video'
| 'news'
| 'discover'
| 'googleNews';
/** Query dimension filter operator. */
export type GscQueryFilterMode =
| 'none'
| 'contains'
| 'notContains'
| 'equals'
| 'notEquals'
| 'includingRegex'
| 'excludingRegex';
/**
* Per-project Google Search Console integration settings — the FreeCrawl
* equivalent of Screaming Frog's "API Access → Google Search Console"
* dialog. Stored per project (encrypted credentials stay separate); the
* fetch reads these to shape the Search Analytics query and to decide
* whether newly-discovered GSC URLs are crawled.
*/
export interface GscIntegrationSettings {
/** Which connected Google account this project pulls from. Empty means
* "the first connected account" — the state every pre-multi-account
* project is in. */
accountId: string;
/** Last-selected Search Console property (`siteUrl`). */
property: string;
dateRange: GscDateRange;
/** `YYYY-MM-DD`, only used when `dateRange === 'custom'`. */
startDate: string;
endDate: string;
deviceFilter: GscDeviceFilter;
/** ISO-3166-1 alpha-3 country code, lowercase (e.g. `usa`); '' = none. */
countryFilter: string;
searchType: GscSearchType;
queryFilterMode: GscQueryFilterMode;
queryFilterValue: string;
/** Match trailing vs non-trailing-slash URL variants when comparing
* GSC URLs against crawled URLs (SF: "Match Trailing and Non-Trailing
* Slash URLs"). */
matchSlash: boolean;
/** Match uppercase vs lowercase URL variants (SF: "Match Uppercase &
* Lowercase URLs"). */
matchCase: boolean;
/** Cap the number of rows pulled (SF: "Limit Max Results"). */
limitMaxResults: boolean;
maxResults: number;
/** SF: "Crawl New URLs Discovered In Google Search Console" — after a
* pull, feed GSC URLs that aren't in the crawl into the crawler. */
crawlNewUrls: boolean;
}
/** Factory for the default GSC settings (a fresh object each call so the
* shared default can't be mutated). */
export function defaultGscSettings(): GscIntegrationSettings {
return {
accountId: '',
property: '',
dateRange: '28d',
startDate: '',
endDate: '',
deviceFilter: 'all',
countryFilter: '',
searchType: 'web',
queryFilterMode: 'none',
queryFilterValue: '',
matchSlash: true,
matchCase: false,
limitMaxResults: true,
maxResults: 100000,
crawlNewUrls: false,
};
}
/** Metadata describing the most recent GSC pull for the active project. */
export interface GscFetchMeta {
/** The Search Console property the data was pulled from. */
@@ -130,6 +263,29 @@ export interface UrlAnalyticsDetail {
inspection: GscInspectionResult | null;
}
/**
* Per-project Google Analytics 4 integration settings. Smaller than the
* GSC equivalent — the Data API call takes no dimension filters here —
* but it carries the same account/property selection so a project
* remembers which of several linked GA4 accounts it reports on.
*/
export interface Ga4IntegrationSettings {
/** Which connected Google account this project pulls from. Empty means
* "the first connected account". */
accountId: string;
/** Last-selected property resource name (`properties/<id>`). */
property: string;
/** Friendly label for that property, for the fetch meta line. */
propertyName: string;
/** Trailing window in days. */
days: 7 | 28 | 90;
}
/** Factory for the default GA4 settings (fresh object each call). */
export function defaultGa4Settings(): Ga4IntegrationSettings {
return { accountId: '', property: '', propertyName: '', days: 28 };
}
/** Metadata describing the most recent GA4 pull for the active project. */
export interface Ga4FetchMeta {
/** API resource name (`properties/<id>`) the data was pulled from. */
+95 -13
View File
@@ -2,6 +2,7 @@ import type {
AdvancedFilter,
BrokenLinkRow,
CrawlConfig,
CrawlMode,
CrawlProgress,
CrawlSummary,
CrawlUrlRow,
@@ -35,9 +36,11 @@ import type { RecentProject } from './project.js';
import type {
GoogleAuthState,
GoogleAuthResult,
GoogleAccount,
GscSite,
GscRow,
GscFetchMeta,
GscPreset,
Ga4Property,
Ga4Row,
Ga4FetchMeta,
@@ -265,6 +268,16 @@ export const IPC = {
integrationsGetAll: 'integrations:get-all',
integrationsSet: 'integrations:set',
integrationsClear: 'integrations:clear',
/** Per-project, per-integration behaviour settings (distinct from the
* encrypted credential store). `get` reads the stored JSON blob for
* one integration id (or null); `set` merges the supplied object.
* Currently used by Google Search Console (date range, dimension
* filters, "crawl new URLs"). */
integrationSettingsGet: 'integrations:settings-get',
integrationSettingsSet: 'integrations:settings-set',
/** Feed GSC-discovered URLs missing from the crawl into the crawler
* (SF: "Crawl New URLs Discovered In Google Search Console"). */
gscCrawlNewUrls: 'gsc:crawl-new-urls',
/** Open a native directory picker. Used by Settings → Storage to let the
* user choose the default folder for new `.seoproject` files. */
pickDirectory: 'app:pick-directory',
@@ -317,6 +330,9 @@ export const IPC = {
googleAuthStart: 'google:auth-start',
googleAuthStatus: 'google:auth-status',
googleAuthRevoke: 'google:auth-revoke',
/** List every Google account linked to one integration — a user can
* connect several (their own property plus a client's). */
googleAccountsList: 'google:accounts-list',
/** Faz 7 — Google Search Console. `gscListSites` lists the connected
* account's verified properties; `gscFetch` pulls per-page clicks /
* impressions / CTR / position for a date range; `gscQuery` lists
@@ -650,6 +666,18 @@ export interface CrashRecoveryStatus {
pendingCount: number;
/** The start URL the previous crawl was running against. */
seedUrl: string;
/** Mode of the interrupted crawl, so the prompt can say whether it
* was a Spider / List / Sitemap run instead of only naming a URL. */
mode: CrawlMode;
}
/** Outcome of a resume request. `config` is the crawl config the run
* was restarted with — the renderer applies it so the toolbar shows
* the mode + start URL that are actually being crawled rather than
* the blank Spider defaults it booted with. */
export interface CrashRecoveryResumeResult {
accepted: boolean;
config: CrawlConfig | null;
}
/**
@@ -1462,8 +1490,13 @@ export interface GscListSitesResult {
export interface GscFetchInput {
/** Search Console property to pull from (`siteUrl` from `GscSite`). */
property: string;
/** Trailing window in days (Search Console data lags ~2-3 days). */
days: 7 | 28 | 90;
/** Which connected Google account to pull with. Omitted / empty falls
* back to the project's configured account, then the first one. */
accountId?: string;
/** Legacy trailing window in days — used only by the MCP `gsc_fetch`
* tool. The desktop pull reads the date range from the stored GSC
* integration settings instead. */
days?: 7 | 28 | 90;
}
export interface GscFetchResult {
@@ -1472,14 +1505,36 @@ export interface GscFetchResult {
/** Page rows stored from this pull. */
rowCount: number;
meta: GscFetchMeta | null;
/** GSC URLs from this pull that aren't in the crawl (orphan candidates
* for "Crawl New URLs"). Absent on older callers. */
newUrlCount?: number;
/** How many of those were queued into the crawler because the
* "crawl new URLs" setting is on (0 when the setting is off). */
queuedNewUrls?: number;
}
export interface GscQueryInput {
limit: number;
offset: number;
search?: string;
/** `all` (default), `with-data` (has GSC impressions), `without-data`. */
filter?: 'all' | 'with-data' | 'without-data';
/** Filter preset — see `GscPreset`. `all` (default). Legacy
* `with-data` / `without-data` still accepted. */
filter?: GscPreset;
/** Which account's stored rows to join. Omitted / empty falls back to
* the project's configured account, then the first one. */
accountId?: string;
}
export interface GscCrawlNewUrlsResult {
ok: boolean;
error: string | null;
/** GSC URLs found missing from the crawl. */
candidateCount: number;
/** How many were fed into the crawler (== candidateCount unless empty). */
queued: number;
/** True when the URLs were injected into a live crawl (vs. spinning up
* a fresh re-spider pass). */
hasActiveCrawl: boolean;
}
export interface GscQueryResult {
@@ -1502,6 +1557,9 @@ export interface Ga4FetchInput {
/** Friendly label persisted into the fetch meta for the UI. */
propertyName: string;
days: 7 | 28 | 90;
/** Which connected Google account to pull with. Omitted / empty falls
* back to the project's configured account, then the first one. */
accountId?: string;
}
export interface Ga4FetchResult {
@@ -1516,6 +1574,9 @@ export interface Ga4QueryInput {
offset: number;
search?: string;
filter?: 'all' | 'with-data' | 'without-data';
/** Which account's stored rows to join. Omitted / empty falls back to
* the project's configured account, then the first one. */
accountId?: string;
}
export interface Ga4QueryResult {
@@ -1601,6 +1662,9 @@ export interface GscInspectRunInput {
/** Search Console property the URLs belong to. */
property: string;
urls: string[];
/** Which connected Google account owns that property. Omitted / empty
* falls back to the project's configured account. */
accountId?: string;
}
export interface GscInspectRunResult {
@@ -1742,7 +1806,7 @@ export interface FreeCrawlApi {
input: DataDeleteByDomainInput,
): Promise<DataDeleteByDomainResult>;
crashRecoveryStatus(): Promise<CrashRecoveryStatus>;
crashRecoveryResume(): Promise<{ accepted: boolean }>;
crashRecoveryResume(): Promise<CrashRecoveryResumeResult>;
crashRecoveryDiscard(): Promise<void>;
exportHtmlReport(input: ExportHtmlReportInput): Promise<ExportHtmlReportResult>;
exportBulk(): Promise<BulkExportResult>;
@@ -1877,18 +1941,33 @@ export interface FreeCrawlApi {
* integration. Resolves once the user finishes (or cancels) in the
* browser. */
googleAuthStart(integrationId: string): Promise<GoogleAuthResult>;
/** Read the stored OAuth connection state of one Google integration. */
/** Read the stored OAuth connection state of one Google integration,
* including every connected account. */
googleAuthStatus(integrationId: string): Promise<GoogleAuthState>;
/** Disconnect — wipe the stored OAuth tokens for one integration. */
googleAuthRevoke(integrationId: string): Promise<GoogleAuthState>;
/** List the connected account's Search Console properties. */
gscListSites(): Promise<GscListSitesResult>;
/** Disconnect one linked account, or every account for the integration
* when `accountId` is omitted. */
googleAuthRevoke(integrationId: string, accountId?: string): Promise<GoogleAuthState>;
/** List a connected account's Search Console properties. Omitting
* `accountId` uses the project's configured account. */
gscListSites(accountId?: string): Promise<GscListSitesResult>;
/** Pull per-page Search Console metrics for the given property + range. */
gscFetch(input: GscFetchInput): Promise<GscFetchResult>;
/** List crawled pages joined with their stored Search Console metrics. */
gscQuery(input: GscQueryInput): Promise<GscQueryResult>;
/** List the connected account's Google Analytics 4 properties. */
ga4ListProperties(): Promise<Ga4ListPropertiesResult>;
/** Feed GSC-discovered URLs missing from the crawl into the crawler
* (SF: "Crawl New URLs Discovered In Google Search Console"). */
gscCrawlNewUrls(): Promise<GscCrawlNewUrlsResult>;
/** Read one integration's per-project behaviour settings (or null when
* none stored). Currently used by Google Search Console. */
integrationSettingsGet(id: string): Promise<Record<string, unknown> | null>;
/** Merge-save one integration's per-project behaviour settings. */
integrationSettingsSet(
id: string,
settings: Record<string, unknown>,
): Promise<Record<string, unknown>>;
/** List a connected account's Google Analytics 4 properties. Omitting
* `accountId` uses the project's configured account. */
ga4ListProperties(accountId?: string): Promise<Ga4ListPropertiesResult>;
/** Pull per-page GA4 metrics for the given property + range. */
ga4Fetch(input: Ga4FetchInput): Promise<Ga4FetchResult>;
/** List crawled pages joined with their stored GA4 metrics. */
@@ -1921,8 +2000,11 @@ export interface FreeCrawlApi {
* GCP project. Uses the `bigquery` service-account credentials. */
exportBigquery(input: ExportBigqueryInput): Promise<ExportBigqueryResult>;
/** Read the combined Analytics detail (GSC + GA4 + URL Inspection)
* for one URL. Returns null when the URL hasn't been crawled. */
* for one URL, scoped to the project's configured accounts. Returns
* null when the URL hasn't been crawled. */
urlAnalyticsGet(url: string): Promise<UrlAnalyticsDetail | null>;
/** List the Google accounts linked to one integration. */
googleAccountsList(integrationId: string): Promise<GoogleAccount[]>;
onLogEntry(cb: (entry: LogEntry) => void): () => void;
onLogsBatch(cb: (entries: LogEntry[]) => void): () => void;
onLogsBusy(cb: (busy: boolean) => void): () => void;
+14 -1
View File
@@ -24,6 +24,12 @@ export type LogFormatChoice = LogFormat | 'auto';
export interface LogAnalyzeInput {
/** Absolute log file path. Omitted → the main process opens a picker. */
filePath?: string;
/**
* Absolute log file paths for a bulk import (each analyzed + ingested in
* turn). Takes precedence over `filePath`. When both are omitted the main
* process opens a multi-select picker.
*/
filePaths?: string[];
/** Dialect. Default `auto`. */
format?: LogFormatChoice;
/**
@@ -99,8 +105,15 @@ export interface LogImportSummary {
export interface LogAnalyzeResult {
ok: boolean;
error?: string;
/** Null when the user cancelled the file picker. */
/**
* Single-file convenience: the first file successfully ingested in this
* call. Null when the user cancelled the picker or nothing was ingested.
*/
imported: LogImportSummary | null;
/** Every file ingested in this (possibly bulk) call. */
batch?: LogImportSummary[];
/** Per-file failures that did not abort the rest of the batch. */
errors?: Array<{ fileName: string; error: string }>;
overview: LogOverview | null;
}