mirror of
https://github.com/kemalai/FreeCrawl-SEO-Tool.git
synced 2026-10-18 00:39:38 +02:00
feat: multi-account GSC/GA4 + crawl resume & screenshot fixes
This commit is contained in:
1 parent
b91a2b7eae
commit
98cc62154d
35 files changed
+3716
-676
No files matched your search
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/cli",
|
||||
"version": "0.9.2",
|
||||
"version": "0.9.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
+54
-22
@@ -26,8 +26,8 @@ Usage:
|
||||
freecrawl audit-robots <url> [--user-agent UA] Test if a URL is allowed by robots.txt
|
||||
freecrawl compare <before.seoproject> <after.seoproject>
|
||||
Cross-project diff (added / removed / status / title / meta / h1 / canonical / indexability / response_time)
|
||||
freecrawl analyze-logs <access.log> [--project file]
|
||||
Parse a server access log (Apache / Nginx / IIS / custom), detect bots,
|
||||
freecrawl analyze-logs <access.log> [more.log …] [--project file]
|
||||
Parse one or more server access logs (Apache / Nginx / IIS / custom), detect bots,
|
||||
and merge the aggregates into a project for crawl × log analysis
|
||||
|
||||
Options:
|
||||
@@ -133,16 +133,19 @@ async function main(): Promise<void> {
|
||||
}
|
||||
if (positionals[0] === 'analyze-logs') {
|
||||
if (!positionals[1]) {
|
||||
console.error('Usage: freecrawl analyze-logs <access.log> [--project file] [--format f] [--regex p] [--verify-bots]');
|
||||
console.error('Usage: freecrawl analyze-logs <access.log> [more.log …] [--project file] [--format f] [--regex p] [--verify-bots]');
|
||||
process.exit(2);
|
||||
}
|
||||
const exitCode = await runAnalyzeLogs(resolve(positionals[1]), {
|
||||
projectPath: resolve(values.project ?? values.db ?? 'crawl.seoproject'),
|
||||
format: (values.format as LogIngestInput['file']['format'] | 'auto' | undefined) ?? 'auto',
|
||||
regex: values.regex,
|
||||
verifyBots: Boolean(values['verify-bots']),
|
||||
json: Boolean(values.json),
|
||||
});
|
||||
const exitCode = await runAnalyzeLogs(
|
||||
positionals.slice(1).map((p) => resolve(p)),
|
||||
{
|
||||
projectPath: resolve(values.project ?? values.db ?? 'crawl.seoproject'),
|
||||
format: (values.format as LogIngestInput['file']['format'] | 'auto' | undefined) ?? 'auto',
|
||||
regex: values.regex,
|
||||
verifyBots: Boolean(values['verify-bots']),
|
||||
json: Boolean(values.json),
|
||||
},
|
||||
);
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
@@ -526,11 +529,12 @@ function flattenLogResult(filePath: string, result: LogAnalysisResult): LogInges
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an access log, detect bots, and merge the aggregates into a
|
||||
* project DB. Prints a summary (or JSON). Exit 0 on success, 2 on error.
|
||||
* Parse one or more access logs, detect bots, and merge every file's
|
||||
* aggregates into the same project DB. Prints a per-file line plus a combined
|
||||
* summary (or JSON). Exit 0 when at least one file ingested, 2 on error.
|
||||
*/
|
||||
async function runAnalyzeLogs(
|
||||
logPath: string,
|
||||
logPaths: string[],
|
||||
opts: {
|
||||
projectPath: string;
|
||||
format: 'auto' | 'apache-combined' | 'apache-common' | 'nginx' | 'iis-w3c' | 'custom';
|
||||
@@ -547,12 +551,37 @@ async function runAnalyzeLogs(
|
||||
return 2;
|
||||
}
|
||||
try {
|
||||
const result = await analyzeLogFile(logPath, {
|
||||
format: opts.format,
|
||||
customRegex: opts.regex,
|
||||
verifyBots: opts.verifyBots,
|
||||
});
|
||||
const summary = db.ingestLogAnalysis(flattenLogResult(logPath, result));
|
||||
const imported: ReturnType<ProjectDb['ingestLogAnalysis']>[] = [];
|
||||
const failures: Array<{ file: string; error: string }> = [];
|
||||
for (const logPath of logPaths) {
|
||||
try {
|
||||
const result = await analyzeLogFile(logPath, {
|
||||
format: opts.format,
|
||||
customRegex: opts.regex,
|
||||
verifyBots: opts.verifyBots,
|
||||
});
|
||||
const summary = db.ingestLogAnalysis(flattenLogResult(logPath, result));
|
||||
imported.push(summary);
|
||||
if (!opts.json) {
|
||||
console.log(`Log: ${summary.fileName} (${summary.format})`);
|
||||
console.log(` Lines: ${summary.parsedLines}/${summary.totalLines} parsed, ${summary.skippedLines} skipped`);
|
||||
}
|
||||
} catch (err) {
|
||||
const error = (err as Error).message;
|
||||
failures.push({ file: logPath, error });
|
||||
if (!opts.json) console.error(` Skipped ${logPath}: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (imported.length === 0) {
|
||||
if (opts.json) {
|
||||
process.stdout.write(JSON.stringify({ ok: false, imported: [], errors: failures }, null, 2) + '\n');
|
||||
} else {
|
||||
console.error('analyze-logs failed: no files could be ingested.');
|
||||
}
|
||||
return 2;
|
||||
}
|
||||
|
||||
const overview = db.getLogOverview();
|
||||
const bots = db.getLogBots().slice(0, 10);
|
||||
const topUrls = db
|
||||
@@ -561,11 +590,14 @@ async function runAnalyzeLogs(
|
||||
|
||||
if (opts.json) {
|
||||
process.stdout.write(
|
||||
JSON.stringify({ ok: true, imported: summary, overview, topBots: bots, topBotUrls: topUrls }, null, 2) + '\n',
|
||||
JSON.stringify(
|
||||
{ ok: true, imported, errors: failures, overview, topBots: bots, topBotUrls: topUrls },
|
||||
null,
|
||||
2,
|
||||
) + '\n',
|
||||
);
|
||||
} else {
|
||||
console.log(`Log: ${summary.fileName} (${summary.format})`);
|
||||
console.log(` Lines: ${summary.parsedLines}/${summary.totalLines} parsed, ${summary.skippedLines} skipped`);
|
||||
if (imported.length > 1) console.log(`Ingested ${imported.length} files.`);
|
||||
console.log(` Hits: ${overview.totalHits} total · ${overview.botHits} bot · ${overview.humanHits} human`);
|
||||
if (overview.minTs) {
|
||||
const d = (ms: number): string => new Date(ms).toISOString().slice(0, 16).replace('T', ' ');
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/desktop",
|
||||
"version": "0.9.2",
|
||||
"version": "0.9.3",
|
||||
"private": true,
|
||||
"description": "FreeCrawl SEO — desktop application",
|
||||
"homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
|
||||
|
||||
@@ -73,6 +73,9 @@ interface PendingRequest {
|
||||
resolve: (v: unknown) => void;
|
||||
reject: (e: Error) => void;
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
/** Caller already gave up (timeout) but the worker is still executing
|
||||
* the query. The slot stays counted as busy until the reply lands. */
|
||||
abandoned?: boolean;
|
||||
}
|
||||
|
||||
interface ResponseMessage {
|
||||
@@ -166,14 +169,16 @@ export class DbReaderPool {
|
||||
slot.pending++;
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
// A timeout frees the *caller*, not the worker: the query is
|
||||
// already running inside a synchronous SQLite call that cannot
|
||||
// be cancelled. Dropping the slot's pending count here made
|
||||
// `pickSlot` treat a worker that was minutes deep in a heavy
|
||||
// aggregate as idle and pile more work onto it, until every
|
||||
// worker in the pool was stacked with abandoned queries and no
|
||||
// UI read could get through. Keep the slot counted as busy and
|
||||
// release it when the reply actually arrives.
|
||||
const p = this.pending.get(requestId);
|
||||
if (p) {
|
||||
this.pending.delete(requestId);
|
||||
this.slots[p.workerIdx]!.pending = Math.max(
|
||||
0,
|
||||
this.slots[p.workerIdx]!.pending - 1,
|
||||
);
|
||||
}
|
||||
if (p) p.abandoned = true;
|
||||
reject(new Error(`reader-timeout: ${method} > ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
this.pending.set(requestId, {
|
||||
@@ -253,6 +258,9 @@ export class DbReaderPool {
|
||||
clearTimeout(pending.timer);
|
||||
const slot = this.slots[pending.workerIdx];
|
||||
if (slot) slot.pending = Math.max(0, slot.pending - 1);
|
||||
// The caller was already rejected on timeout; this reply only
|
||||
// releases the slot.
|
||||
if (pending.abandoned) return;
|
||||
if (msg.ok) {
|
||||
pending.resolve(msg.result);
|
||||
} else {
|
||||
|
||||
@@ -55,6 +55,20 @@ const watchdog = init.freezeWatchdogSab
|
||||
? new FreezeWatchdogSharedState(init.freezeWatchdogSab)
|
||||
: null;
|
||||
|
||||
// Publish into the writer's own slot and keep its own heartbeat. This
|
||||
// thread used to write the *reader* op string while never ticking any
|
||||
// heartbeat, which meant a stalled reader pool got logged with whatever
|
||||
// write happened to run last pinned to it, and a genuinely stuck writer
|
||||
// produced no evidence at all.
|
||||
const WATCHDOG_HEARTBEAT_INTERVAL_MS = 100;
|
||||
if (watchdog) {
|
||||
watchdog.tickWriterHeartbeat();
|
||||
watchdog.setWriterOp('idle');
|
||||
setInterval(() => {
|
||||
watchdog.tickWriterHeartbeat();
|
||||
}, WATCHDOG_HEARTBEAT_INTERVAL_MS).unref();
|
||||
}
|
||||
|
||||
interface RequestMessage {
|
||||
requestId: number;
|
||||
method: string;
|
||||
@@ -93,6 +107,7 @@ const ALLOWED_METHODS = new Set<string>([
|
||||
'checkpointQueue',
|
||||
'clearQueueCheckpoint',
|
||||
'walCheckpoint',
|
||||
'optimize',
|
||||
'markUrlForRecrawl',
|
||||
'markUrlsForRecrawl',
|
||||
'deleteUrl',
|
||||
@@ -104,7 +119,7 @@ const ALLOWED_METHODS = new Set<string>([
|
||||
parentPort.on('message', async (msg: RequestMessage) => {
|
||||
if (!msg || typeof msg.requestId !== 'number') return;
|
||||
const { requestId, method, args } = msg;
|
||||
if (watchdog) watchdog.setReaderOp('writer:' + method);
|
||||
if (watchdog) watchdog.setWriterOp(method);
|
||||
try {
|
||||
if (!ALLOWED_METHODS.has(method)) {
|
||||
throw new Error(`db-writer-worker: method '${method}' is not whitelisted`);
|
||||
@@ -123,7 +138,7 @@ parentPort.on('message', async (msg: RequestMessage) => {
|
||||
error: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
|
||||
});
|
||||
} finally {
|
||||
if (watchdog) watchdog.setReaderOp('idle');
|
||||
if (watchdog) watchdog.setWriterOp('idle');
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
/**
|
||||
* Shared SharedArrayBuffer layout for the freeze watchdog.
|
||||
*
|
||||
* Three execution contexts read/write to the same SAB:
|
||||
* Four execution contexts read/write to the same SAB:
|
||||
* 1. The Electron main process (the writer for `mainHB`, counters,
|
||||
* `mainOp`, and rendererLag forwarded from IPC).
|
||||
* 2. The db-reader worker thread (writes its own `readerHB` and
|
||||
* 2. The db-reader worker threads (write their own `readerHB` and
|
||||
* `readerOp`).
|
||||
* 3. The freeze-watchdog worker thread (reads everything; writes
|
||||
* 3. The db-writer worker thread (writes `writerHB` and `writerOp`).
|
||||
* It used to publish into the *reader* op slot without ever
|
||||
* ticking the reader heartbeat, so a reader-pool stall could be
|
||||
* logged with a writer's method name attached to it and the writer
|
||||
* itself was invisible to the watchdog.
|
||||
* 4. The freeze-watchdog worker thread (reads everything; writes
|
||||
* nothing — its job is purely observational).
|
||||
*
|
||||
* All scalar reads/writes use `Atomics.{load,store}` so the watchdog
|
||||
@@ -18,11 +23,11 @@
|
||||
* thread is alive, and that IS atomic.
|
||||
*
|
||||
* Layout (1024 bytes total, generous for future fields):
|
||||
* [0..32) BigInt64Array(4) — heartbeat timestamps + spare
|
||||
* [0..32) BigInt64Array(4) — heartbeat timestamps
|
||||
* [0] main heartbeat (ms since epoch)
|
||||
* [1] db-reader heartbeat
|
||||
* [2] last renderer lag-report timestamp
|
||||
* [3] spare
|
||||
* [3] db-writer heartbeat
|
||||
* [32..96) Int32Array(16) — counters + op-string lengths
|
||||
* [0] renderer lag (ms)
|
||||
* [1] crawled
|
||||
@@ -31,10 +36,12 @@
|
||||
* [4] failed
|
||||
* [5] main op string byte length
|
||||
* [6] reader op string byte length
|
||||
* [7..15] spare
|
||||
* [7] writer op string byte length
|
||||
* [8..15] spare
|
||||
* [96..) Uint8Array — op string bytes (UTF-8)
|
||||
* [0..256) main op
|
||||
* [256..512) reader op
|
||||
* [512..768) writer op
|
||||
*/
|
||||
|
||||
const I64_BYTE_OFFSET = 0;
|
||||
@@ -45,11 +52,13 @@ const U8_BYTE_OFFSET = 96;
|
||||
|
||||
const MAIN_OP_OFFSET_IN_U8 = 0;
|
||||
const READER_OP_OFFSET_IN_U8 = 256;
|
||||
const WRITER_OP_OFFSET_IN_U8 = 512;
|
||||
const OP_MAX_BYTES = 255;
|
||||
|
||||
const I64_IDX_MAIN_HB = 0;
|
||||
const I64_IDX_READER_HB = 1;
|
||||
const I64_IDX_RENDERER_REPORT_TS = 2;
|
||||
const I64_IDX_WRITER_HB = 3;
|
||||
|
||||
const I32_IDX_RENDERER_LAG = 0;
|
||||
const I32_IDX_CRAWLED = 1;
|
||||
@@ -58,6 +67,7 @@ const I32_IDX_PENDING = 3;
|
||||
const I32_IDX_FAILED = 4;
|
||||
const I32_IDX_MAIN_OP_LEN = 5;
|
||||
const I32_IDX_READER_OP_LEN = 6;
|
||||
const I32_IDX_WRITER_OP_LEN = 7;
|
||||
|
||||
export const SAB_BYTE_LENGTH = 1024;
|
||||
|
||||
@@ -106,6 +116,10 @@ export class FreezeWatchdogSharedState {
|
||||
Atomics.store(this.i64, I64_IDX_READER_HB, BigInt(Date.now()));
|
||||
}
|
||||
|
||||
tickWriterHeartbeat(): void {
|
||||
Atomics.store(this.i64, I64_IDX_WRITER_HB, BigInt(Date.now()));
|
||||
}
|
||||
|
||||
reportRendererLag(lagMs: number): void {
|
||||
Atomics.store(this.i32, I32_IDX_RENDERER_LAG, lagMs | 0);
|
||||
Atomics.store(this.i64, I64_IDX_RENDERER_REPORT_TS, BigInt(Date.now()));
|
||||
@@ -119,6 +133,10 @@ export class FreezeWatchdogSharedState {
|
||||
this.writeOpString(op, READER_OP_OFFSET_IN_U8, I32_IDX_READER_OP_LEN);
|
||||
}
|
||||
|
||||
setWriterOp(op: string): void {
|
||||
this.writeOpString(op, WRITER_OP_OFFSET_IN_U8, I32_IDX_WRITER_OP_LEN);
|
||||
}
|
||||
|
||||
updateCounters(c: CounterPatch): void {
|
||||
if (c.crawled !== undefined) {
|
||||
Atomics.store(this.i32, I32_IDX_CRAWLED, c.crawled | 0);
|
||||
@@ -144,6 +162,10 @@ export class FreezeWatchdogSharedState {
|
||||
return Number(Atomics.load(this.i64, I64_IDX_READER_HB));
|
||||
}
|
||||
|
||||
readWriterHeartbeatMs(): number {
|
||||
return Number(Atomics.load(this.i64, I64_IDX_WRITER_HB));
|
||||
}
|
||||
|
||||
readRendererReportTsMs(): number {
|
||||
return Number(Atomics.load(this.i64, I64_IDX_RENDERER_REPORT_TS));
|
||||
}
|
||||
@@ -169,6 +191,10 @@ export class FreezeWatchdogSharedState {
|
||||
return this.readOpString(READER_OP_OFFSET_IN_U8, I32_IDX_READER_OP_LEN);
|
||||
}
|
||||
|
||||
readWriterOp(): string {
|
||||
return this.readOpString(WRITER_OP_OFFSET_IN_U8, I32_IDX_WRITER_OP_LEN);
|
||||
}
|
||||
|
||||
// ── Internals ─────────────────────────────────────────────────────
|
||||
|
||||
private writeOpString(op: string, offset: number, lenIdx: number): void {
|
||||
|
||||
@@ -25,8 +25,23 @@ import { FreezeWatchdogSharedState } from './freeze-watchdog-shared.js';
|
||||
const CHECK_INTERVAL_MS = 250;
|
||||
const STALL_THRESHOLD_MAIN_MS = 500;
|
||||
const STALL_THRESHOLD_READER_MS = 1000;
|
||||
const STALL_THRESHOLD_WRITER_MS = 1000;
|
||||
const STALL_THRESHOLD_RENDERER_LAG_MS = 500;
|
||||
const STALL_THRESHOLD_RENDERER_SILENCE_MS = 1500;
|
||||
/**
|
||||
* How long a stall must persist past detection before it is written to
|
||||
* the file.
|
||||
*
|
||||
* Detection thresholds are deliberately twitchy so the recorded gap and
|
||||
* op string come from the real onset of a freeze. Writing at that same
|
||||
* sensitivity is what made the log unreadable: with a 250 ms sample
|
||||
* interval and a 500 ms main threshold, ordinary GC pauses tripped the
|
||||
* detector constantly and over a third of the `STALL:MAIN` lines in a
|
||||
* real capture were `end after 250ms` — noise that buried the
|
||||
* multi-second freezes worth investigating. Detection is unchanged;
|
||||
* only the reporting waits for the stall to prove itself.
|
||||
*/
|
||||
const STALL_REPORT_FLOOR_MS = 1000;
|
||||
const HEARTBEAT_LOG_INTERVAL_MS = 5000;
|
||||
|
||||
interface InitData {
|
||||
@@ -68,14 +83,62 @@ function sanitizeOp(op: string): string {
|
||||
}
|
||||
|
||||
interface StallTracker {
|
||||
/** Gap exceeded the threshold and we're timing it. */
|
||||
active: boolean;
|
||||
/** The `start` line has been written — so an `end` line is owed. */
|
||||
reported: boolean;
|
||||
startTs: number;
|
||||
startOp: string;
|
||||
/** `gap=612ms` / `silence=1709ms`, captured at onset. */
|
||||
detail: string;
|
||||
}
|
||||
|
||||
const mainStall: StallTracker = { active: false, startTs: 0, startOp: '' };
|
||||
const readerStall: StallTracker = { active: false, startTs: 0, startOp: '' };
|
||||
const rendererStall: StallTracker = { active: false, startTs: 0, startOp: '' };
|
||||
const newTracker = (): StallTracker => ({
|
||||
active: false,
|
||||
reported: false,
|
||||
startTs: 0,
|
||||
startOp: '',
|
||||
detail: '',
|
||||
});
|
||||
|
||||
const mainStall = newTracker();
|
||||
const readerStall = newTracker();
|
||||
const writerStall = newTracker();
|
||||
const rendererStall = newTracker();
|
||||
|
||||
/**
|
||||
* Shared start/report/end bookkeeping. `onStart` and `onEnd` build the
|
||||
* log lines; neither runs unless the stall outlives
|
||||
* `STALL_REPORT_FLOOR_MS`, so a blip costs nothing but a few field
|
||||
* writes.
|
||||
*/
|
||||
function trackStall(
|
||||
t: StallTracker,
|
||||
stalled: boolean,
|
||||
now: number,
|
||||
op: string,
|
||||
detail: string,
|
||||
onStart: (t: StallTracker) => string,
|
||||
onEnd: (t: StallTracker, durationMs: number) => string,
|
||||
): void {
|
||||
if (stalled) {
|
||||
if (!t.active) {
|
||||
t.active = true;
|
||||
t.reported = false;
|
||||
t.startTs = now;
|
||||
t.startOp = op;
|
||||
t.detail = detail;
|
||||
} else if (!t.reported && now - t.startTs >= STALL_REPORT_FLOOR_MS) {
|
||||
t.reported = true;
|
||||
appendLine(onStart(t));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!t.active) return;
|
||||
if (t.reported) appendLine(onEnd(t, now - t.startTs));
|
||||
t.active = false;
|
||||
t.reported = false;
|
||||
}
|
||||
|
||||
let lastHeartbeatLogTs = 0;
|
||||
let bootLogged = false;
|
||||
@@ -85,52 +148,64 @@ function check(): void {
|
||||
const counters = state.readCounters();
|
||||
const mainHb = state.readMainHeartbeatMs();
|
||||
const readerHb = state.readReaderHeartbeatMs();
|
||||
const writerHb = state.readWriterHeartbeatMs();
|
||||
const rendererTs = state.readRendererReportTsMs();
|
||||
const rendererLag = state.readRendererLagMs();
|
||||
const mainOp = sanitizeOp(state.readMainOp());
|
||||
const readerOp = sanitizeOp(state.readReaderOp());
|
||||
const writerOp = sanitizeOp(state.readWriterOp());
|
||||
|
||||
const mainGap = mainHb > 0 ? now - mainHb : 0;
|
||||
const readerGap = readerHb > 0 ? now - readerHb : 0;
|
||||
const writerGap = writerHb > 0 ? now - writerHb : 0;
|
||||
const rendererSilence = rendererTs > 0 ? now - rendererTs : 0;
|
||||
|
||||
// ── Main thread stall ──
|
||||
if (mainHb > 0 && mainGap > STALL_THRESHOLD_MAIN_MS) {
|
||||
if (!mainStall.active) {
|
||||
mainStall.active = true;
|
||||
mainStall.startTs = now;
|
||||
mainStall.startOp = mainOp;
|
||||
appendLine(
|
||||
`${isoNow()} [STALL:MAIN start gap=${mainGap}ms] op="${mainOp || '<unknown>'}" ` +
|
||||
`crawled=${counters.crawled} discovered=${counters.discovered} ` +
|
||||
`pending=${counters.pending} failed=${counters.failed}`,
|
||||
);
|
||||
}
|
||||
} else if (mainStall.active) {
|
||||
const dur = now - mainStall.startTs;
|
||||
appendLine(
|
||||
`${isoNow()} [STALL:MAIN end after ${dur}ms] startOp="${mainStall.startOp}" endOp="${mainOp}"`,
|
||||
);
|
||||
mainStall.active = false;
|
||||
}
|
||||
trackStall(
|
||||
mainStall,
|
||||
mainHb > 0 && mainGap > STALL_THRESHOLD_MAIN_MS,
|
||||
now,
|
||||
mainOp,
|
||||
`gap=${mainGap}ms`,
|
||||
(t) =>
|
||||
`${isoNow()} [STALL:MAIN start ${t.detail}] op="${t.startOp || '<unknown>'}" ` +
|
||||
`crawled=${counters.crawled} discovered=${counters.discovered} ` +
|
||||
`pending=${counters.pending} failed=${counters.failed}`,
|
||||
(t, dur) =>
|
||||
`${isoNow()} [STALL:MAIN end after ${dur}ms] startOp="${t.startOp}" endOp="${mainOp}"`,
|
||||
);
|
||||
|
||||
// ── DB reader thread stall ──
|
||||
if (readerHb > 0 && readerGap > STALL_THRESHOLD_READER_MS) {
|
||||
if (!readerStall.active) {
|
||||
readerStall.active = true;
|
||||
readerStall.startTs = now;
|
||||
readerStall.startOp = readerOp;
|
||||
appendLine(
|
||||
`${isoNow()} [STALL:READER start gap=${readerGap}ms] op="${readerOp || '<unknown>'}"`,
|
||||
);
|
||||
}
|
||||
} else if (readerStall.active) {
|
||||
const dur = now - readerStall.startTs;
|
||||
appendLine(
|
||||
`${isoNow()} [STALL:READER end after ${dur}ms] startOp="${readerStall.startOp}" endOp="${readerOp}"`,
|
||||
);
|
||||
readerStall.active = false;
|
||||
}
|
||||
// ── DB reader pool stall ──
|
||||
// Every reader worker ticks the same heartbeat, so this only fires
|
||||
// when *none* of them got a turn — i.e. the whole pool is blocked.
|
||||
trackStall(
|
||||
readerStall,
|
||||
readerHb > 0 && readerGap > STALL_THRESHOLD_READER_MS,
|
||||
now,
|
||||
readerOp,
|
||||
`gap=${readerGap}ms`,
|
||||
(t) => `${isoNow()} [STALL:READER start ${t.detail}] op="${t.startOp || '<unknown>'}"`,
|
||||
(t, dur) =>
|
||||
`${isoNow()} [STALL:READER end after ${dur}ms] startOp="${t.startOp}" endOp="${readerOp}"`,
|
||||
);
|
||||
|
||||
// ── DB writer thread stall ──
|
||||
// Its own heartbeat and its own op slot: a blocked writer holds up
|
||||
// every per-URL page write behind it, which surfaces as a crawl that
|
||||
// has stopped advancing, and that is worth telling apart from a
|
||||
// blocked reader pool (which only affects what the UI can display).
|
||||
trackStall(
|
||||
writerStall,
|
||||
writerHb > 0 && writerGap > STALL_THRESHOLD_WRITER_MS,
|
||||
now,
|
||||
writerOp,
|
||||
`gap=${writerGap}ms`,
|
||||
(t) =>
|
||||
`${isoNow()} [STALL:WRITER start ${t.detail}] op="${t.startOp || '<unknown>'}" ` +
|
||||
`crawled=${counters.crawled} pending=${counters.pending}`,
|
||||
(t, dur) =>
|
||||
`${isoNow()} [STALL:WRITER end after ${dur}ms] startOp="${t.startOp}" endOp="${writerOp}"`,
|
||||
);
|
||||
|
||||
// ── Renderer stall ──
|
||||
// We treat both an explicit high-lag report AND a long silence
|
||||
@@ -146,23 +221,17 @@ function check(): void {
|
||||
rendererStalled = true;
|
||||
rendererReason = `silence=${rendererSilence}ms`;
|
||||
}
|
||||
if (rendererStalled) {
|
||||
if (!rendererStall.active) {
|
||||
rendererStall.active = true;
|
||||
rendererStall.startTs = now;
|
||||
rendererStall.startOp = mainOp;
|
||||
appendLine(
|
||||
`${isoNow()} [STALL:RENDERER start ${rendererReason}] last_main_op="${mainOp || '<unknown>'}" ` +
|
||||
`crawled=${counters.crawled} pending=${counters.pending}`,
|
||||
);
|
||||
}
|
||||
} else if (rendererStall.active) {
|
||||
const dur = now - rendererStall.startTs;
|
||||
appendLine(
|
||||
`${isoNow()} [STALL:RENDERER end after ${dur}ms] last_lag=${rendererLag}ms`,
|
||||
);
|
||||
rendererStall.active = false;
|
||||
}
|
||||
trackStall(
|
||||
rendererStall,
|
||||
rendererStalled,
|
||||
now,
|
||||
mainOp,
|
||||
rendererReason,
|
||||
(t) =>
|
||||
`${isoNow()} [STALL:RENDERER start ${t.detail}] last_main_op="${t.startOp || '<unknown>'}" ` +
|
||||
`crawled=${counters.crawled} pending=${counters.pending}`,
|
||||
(_t, dur) => `${isoNow()} [STALL:RENDERER end after ${dur}ms] last_lag=${rendererLag}ms`,
|
||||
);
|
||||
|
||||
// ── Periodic heartbeat (so the file shows the app is alive even
|
||||
// when nothing has stalled) ──
|
||||
@@ -171,13 +240,16 @@ function check(): void {
|
||||
appendLine(
|
||||
`${isoNow()} [BOOT] freeze-watchdog started ` +
|
||||
`thresholds: main>${STALL_THRESHOLD_MAIN_MS}ms reader>${STALL_THRESHOLD_READER_MS}ms ` +
|
||||
`renderer_lag>${STALL_THRESHOLD_RENDERER_LAG_MS}ms renderer_silence>${STALL_THRESHOLD_RENDERER_SILENCE_MS}ms`,
|
||||
`writer>${STALL_THRESHOLD_WRITER_MS}ms ` +
|
||||
`renderer_lag>${STALL_THRESHOLD_RENDERER_LAG_MS}ms renderer_silence>${STALL_THRESHOLD_RENDERER_SILENCE_MS}ms ` +
|
||||
`report_floor=${STALL_REPORT_FLOOR_MS}ms`,
|
||||
);
|
||||
}
|
||||
if (now - lastHeartbeatLogTs >= HEARTBEAT_LOG_INTERVAL_MS) {
|
||||
lastHeartbeatLogTs = now;
|
||||
appendLine(
|
||||
`${isoNow()} [HEARTBEAT] main_op="${mainOp}" reader_op="${readerOp}" ` +
|
||||
`writer_op="${writerOp}" ` +
|
||||
`crawled=${counters.crawled} discovered=${counters.discovered} ` +
|
||||
`pending=${counters.pending} renderer_lag=${rendererLag}ms`,
|
||||
);
|
||||
|
||||
@@ -46,8 +46,10 @@ function apiError(json: unknown, status: number): string {
|
||||
}
|
||||
|
||||
/** Flatten the GA4 Admin `accountSummaries` tree into a flat property list. */
|
||||
export async function listProperties(): Promise<Ga4Property[]> {
|
||||
const token = await getAccessToken('ga4');
|
||||
export async function listProperties(
|
||||
accountId?: string,
|
||||
): Promise<Ga4Property[]> {
|
||||
const token = await getAccessToken('ga4', accountId);
|
||||
const out: Ga4Property[] = [];
|
||||
let pageToken: string | undefined;
|
||||
for (;;) {
|
||||
@@ -113,8 +115,9 @@ export async function runReport(
|
||||
property: string,
|
||||
startDate: string,
|
||||
endDate: string,
|
||||
accountId?: string,
|
||||
): Promise<Ga4PageRow[]> {
|
||||
const token = await getAccessToken('ga4');
|
||||
const token = await getAccessToken('ga4', accountId);
|
||||
const endpoint = `${DATA_BASE}/${property}:runReport`;
|
||||
const out: Ga4PageRow[] = [];
|
||||
let offset = 0;
|
||||
|
||||
@@ -28,7 +28,7 @@ import { createServer, type Server, type IncomingMessage, type ServerResponse }
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import type { GoogleAuthState } from '@freecrawl/shared-types';
|
||||
import type { GoogleAccount, GoogleAuthState } from '@freecrawl/shared-types';
|
||||
import { apiFetch } from './api-fetch.js';
|
||||
import { resolveCredentials } from './credentials.js';
|
||||
import * as logger from './logger.js';
|
||||
@@ -60,17 +60,43 @@ interface TokenRecord {
|
||||
refreshToken: string;
|
||||
email: string | null;
|
||||
connectedAt: string;
|
||||
/** Opaque per-account id. Absent on records written before
|
||||
* multi-account support — `loadStore` backfills those. */
|
||||
accountId?: string;
|
||||
}
|
||||
|
||||
/** On-disk shape: integration id → encrypted JSON of a `TokenRecord`. */
|
||||
/**
|
||||
* On-disk shape: **account key** → encrypted JSON of a `TokenRecord`.
|
||||
*
|
||||
* The account key is `<integrationId>#<accountId>`, which is what lets a
|
||||
* user link several Google accounts to the same integration. Records
|
||||
* written before multi-account support used the bare integration id as
|
||||
* the key; `loadStore` migrates those in place on first read.
|
||||
*/
|
||||
type TokenFile = Record<string, string>;
|
||||
|
||||
/** Separator between the integration id and the account id in a key. */
|
||||
const KEY_SEP = '#';
|
||||
|
||||
let cache: TokenFile | null = null;
|
||||
/** In-memory short-lived access tokens — never persisted. */
|
||||
/** In-memory short-lived access tokens — never persisted. Keyed by the
|
||||
* same account key as the store. */
|
||||
const accessTokens = new Map<string, { token: string; expiresAt: number }>();
|
||||
/** Guards against two consent flows running at once. */
|
||||
let authInProgress = false;
|
||||
|
||||
/** Compose the store key for one account of one integration. */
|
||||
function accountKey(integrationId: string, accountId: string): string {
|
||||
return `${integrationId}${KEY_SEP}${accountId}`;
|
||||
}
|
||||
|
||||
/** Split a store key back into its integration + account ids. */
|
||||
function parseKey(key: string): { integrationId: string; accountId: string } | null {
|
||||
const i = key.indexOf(KEY_SEP);
|
||||
if (i <= 0) return null;
|
||||
return { integrationId: key.slice(0, i), accountId: key.slice(i + 1) };
|
||||
}
|
||||
|
||||
function filePath(): string {
|
||||
return join(app.getPath('userData'), 'google-tokens.enc.json');
|
||||
}
|
||||
@@ -83,6 +109,7 @@ function loadStore(): TokenFile {
|
||||
const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown;
|
||||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||||
cache = parsed as TokenFile;
|
||||
migrateLegacyKeys();
|
||||
return cache;
|
||||
}
|
||||
}
|
||||
@@ -97,6 +124,47 @@ function loadStore(): TokenFile {
|
||||
return cache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite pre-multi-account entries — keyed by the bare integration id —
|
||||
* into `<integrationId>#<accountId>` form so an existing connection
|
||||
* survives the upgrade as the integration's first linked account. Runs
|
||||
* once per process, right after the store is first read.
|
||||
*/
|
||||
function migrateLegacyKeys(): void {
|
||||
const store = cache;
|
||||
if (!store) return;
|
||||
let changed = false;
|
||||
for (const key of Object.keys(store)) {
|
||||
// Already namespaced, or not one of ours — leave alone.
|
||||
if (key.includes(KEY_SEP) || !(key in INTEGRATION_SCOPES)) continue;
|
||||
const raw = store[key]!;
|
||||
let record: TokenRecord | null = null;
|
||||
try {
|
||||
record = JSON.parse(decrypt(raw)) as TokenRecord;
|
||||
} catch {
|
||||
/* corrupt — drop it below */
|
||||
}
|
||||
delete store[key];
|
||||
changed = true;
|
||||
if (!record?.refreshToken) continue;
|
||||
const accountId = record.accountId || newAccountId();
|
||||
store[accountKey(key, accountId)] = encrypt(
|
||||
JSON.stringify({ ...record, accountId }),
|
||||
);
|
||||
logger.log(
|
||||
'info',
|
||||
'main',
|
||||
`google-oauth: migrated "${key}" to multi-account (${record.email ?? 'unknown account'})`,
|
||||
);
|
||||
}
|
||||
if (changed) persist();
|
||||
}
|
||||
|
||||
/** Short opaque id for a newly linked account. */
|
||||
function newAccountId(): string {
|
||||
return randomBytes(6).toString('hex');
|
||||
}
|
||||
|
||||
function persist(): void {
|
||||
try {
|
||||
writeFileSync(filePath(), JSON.stringify(cache ?? {}, null, 2), 'utf8');
|
||||
@@ -132,8 +200,9 @@ function decrypt(stored: string): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
function readRecord(integrationId: string): TokenRecord | null {
|
||||
const raw = loadStore()[integrationId];
|
||||
/** Read one account's token record by its full store key. */
|
||||
function readRecordByKey(key: string): TokenRecord | null {
|
||||
const raw = loadStore()[key];
|
||||
if (!raw) return null;
|
||||
try {
|
||||
const json = decrypt(raw);
|
||||
@@ -150,18 +219,60 @@ function readRecord(integrationId: string): TokenRecord | null {
|
||||
|
||||
function writeRecord(integrationId: string, record: TokenRecord): void {
|
||||
const store = loadStore();
|
||||
store[integrationId] = encrypt(JSON.stringify(record));
|
||||
store[accountKey(integrationId, record.accountId ?? newAccountId())] = encrypt(
|
||||
JSON.stringify(record),
|
||||
);
|
||||
persist();
|
||||
}
|
||||
|
||||
/**
|
||||
* Every account linked to one integration, in connection order. This is
|
||||
* the list the account dropdowns render.
|
||||
*/
|
||||
export function listAccounts(integrationId: string): GoogleAccount[] {
|
||||
const store = loadStore();
|
||||
const out: GoogleAccount[] = [];
|
||||
for (const key of Object.keys(store)) {
|
||||
const parsed = parseKey(key);
|
||||
if (!parsed || parsed.integrationId !== integrationId) continue;
|
||||
const record = readRecordByKey(key);
|
||||
if (!record) continue;
|
||||
out.push({
|
||||
accountId: parsed.accountId,
|
||||
email: record.email,
|
||||
connectedAt: record.connectedAt,
|
||||
});
|
||||
}
|
||||
out.sort((a, b) => a.connectedAt.localeCompare(b.connectedAt));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve which account an API call should use. An explicit id wins; a
|
||||
* blank one falls back to the first linked account, which is what every
|
||||
* pre-multi-account caller (and project) implicitly means.
|
||||
*/
|
||||
export function resolveAccountId(
|
||||
integrationId: string,
|
||||
requested?: string,
|
||||
): string | null {
|
||||
const accounts = listAccounts(integrationId);
|
||||
if (accounts.length === 0) return null;
|
||||
if (requested && accounts.some((a) => a.accountId === requested)) {
|
||||
return requested;
|
||||
}
|
||||
return accounts[0]!.accountId;
|
||||
}
|
||||
|
||||
/** Redacted connection state for the renderer — no tokens cross IPC. */
|
||||
export function getAuthState(integrationId: string): GoogleAuthState {
|
||||
const record = readRecord(integrationId);
|
||||
if (!record) return { connected: false, email: null, connectedAt: null };
|
||||
const accounts = listAccounts(integrationId);
|
||||
const first = accounts[0];
|
||||
return {
|
||||
connected: true,
|
||||
email: record.email,
|
||||
connectedAt: record.connectedAt,
|
||||
connected: accounts.length > 0,
|
||||
email: first?.email ?? null,
|
||||
connectedAt: first?.connectedAt ?? null,
|
||||
accounts,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -174,6 +285,23 @@ function base64url(buf: Buffer): string {
|
||||
return buf.toString('base64url');
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn an OAuth `error=` callback code into something the user can act
|
||||
* on. `access_denied` is the one that matters: with the BYOC model the
|
||||
* consent screen starts in "Testing", where only accounts on the test
|
||||
* user list may authorise — so linking a second account fails here
|
||||
* until that account is added.
|
||||
*/
|
||||
function consentError(code: string): string {
|
||||
if (code === 'access_denied') {
|
||||
return 'Access blocked by Google (access_denied). If your OAuth consent screen is in "Testing" status, only accounts on its test-user list can connect — add this Google account under Google Cloud Console → APIs & Services → OAuth consent screen → Audience → Test users, then try again. Otherwise you declined the permission prompt.';
|
||||
}
|
||||
if (code === 'admin_policy_enforced') {
|
||||
return 'Your Google Workspace admin blocks this app (admin_policy_enforced). Ask them to allow the Search Console / Analytics scopes for your OAuth client, or connect a personal Google account instead.';
|
||||
}
|
||||
return `Consent denied (${code})`;
|
||||
}
|
||||
|
||||
/** Resolve the user's BYOC client id + secret, or throw a clear error. */
|
||||
function resolveClient(integrationId: string): {
|
||||
clientId: string;
|
||||
@@ -240,7 +368,7 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
|
||||
res.setHeader('Content-Type', 'text/html; charset=utf-8');
|
||||
if (error || !code) {
|
||||
res.end(resultPage('FreeCrawl — sign-in cancelled', false));
|
||||
reject(new Error(error ? `Consent denied (${error})` : 'No authorization code returned'));
|
||||
reject(new Error(error ? consentError(error) : 'No authorization code returned'));
|
||||
return;
|
||||
}
|
||||
if (returnedState !== stateToken) {
|
||||
@@ -282,7 +410,15 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
reject(new Error('Timed out waiting for the browser sign-in.'));
|
||||
// Google blocks a non-test-user at the consent screen without
|
||||
// ever redirecting to the loopback, so that failure reaches us
|
||||
// as a timeout rather than an `error=` callback. Say so, since
|
||||
// it is by far the most common reason this expires.
|
||||
reject(
|
||||
new Error(
|
||||
'Timed out waiting for the browser sign-in. If Google showed "Access blocked / 403 access_denied", add this Google account under Google Cloud Console → APIs & Services → OAuth consent screen → Audience → Test users, then try again.',
|
||||
),
|
||||
);
|
||||
}, CONSENT_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
@@ -319,8 +455,11 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
|
||||
);
|
||||
}
|
||||
|
||||
// Best-effort: resolve the connected account email for the UI.
|
||||
// Best-effort: resolve the connected account email for the UI. The
|
||||
// access token is held aside until the account id is known, since
|
||||
// the in-memory cache is keyed per account.
|
||||
let email: string | null = null;
|
||||
let pendingAccessToken: { token: string; expiresAt: number } | null = null;
|
||||
if (typeof accessToken === 'string' && accessToken) {
|
||||
try {
|
||||
const uiRes = await apiFetch(USERINFO_ENDPOINT, {
|
||||
@@ -338,21 +477,35 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
|
||||
typeof tokenJson['expires_in'] === 'number'
|
||||
? (tokenJson['expires_in'] as number)
|
||||
: 3600;
|
||||
accessTokens.set(integrationId, {
|
||||
pendingAccessToken = {
|
||||
token: accessToken,
|
||||
expiresAt: Date.now() + expiresIn * 1000,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
// Re-authorising an address that is already linked refreshes that
|
||||
// account in place rather than creating a duplicate row in every
|
||||
// dropdown. A brand-new address is appended as another account.
|
||||
const existing = email
|
||||
? listAccounts(integrationId).find((a) => a.email === email)
|
||||
: undefined;
|
||||
const accountId = existing?.accountId ?? newAccountId();
|
||||
|
||||
writeRecord(integrationId, {
|
||||
refreshToken,
|
||||
email,
|
||||
connectedAt: new Date().toISOString(),
|
||||
connectedAt: existing?.connectedAt ?? new Date().toISOString(),
|
||||
accountId,
|
||||
});
|
||||
if (pendingAccessToken) {
|
||||
accessTokens.set(accountKey(integrationId, accountId), pendingAccessToken);
|
||||
}
|
||||
logger.log(
|
||||
'info',
|
||||
'main',
|
||||
`google-oauth: "${integrationId}" connected${email ? ` as ${email}` : ''}`,
|
||||
`google-oauth: "${integrationId}" ${existing ? 'reconnected' : 'connected'}${
|
||||
email ? ` as ${email}` : ''
|
||||
}`,
|
||||
);
|
||||
return getAuthState(integrationId);
|
||||
} finally {
|
||||
@@ -374,12 +527,20 @@ export async function startAuth(integrationId: string): Promise<GoogleAuthState>
|
||||
* token when still fresh, otherwise refreshes it from the stored
|
||||
* refresh token. Main-process only — never expose the result over IPC.
|
||||
*/
|
||||
export async function getAccessToken(integrationId: string): Promise<string> {
|
||||
const cached = accessTokens.get(integrationId);
|
||||
export async function getAccessToken(
|
||||
integrationId: string,
|
||||
requestedAccountId?: string,
|
||||
): Promise<string> {
|
||||
const accountId = resolveAccountId(integrationId, requestedAccountId);
|
||||
if (!accountId) {
|
||||
throw new Error(`"${integrationId}" is not connected — sign in with Google first.`);
|
||||
}
|
||||
const key = accountKey(integrationId, accountId);
|
||||
const cached = accessTokens.get(key);
|
||||
if (cached && cached.expiresAt - TOKEN_EXPIRY_SKEW_MS > Date.now()) {
|
||||
return cached.token;
|
||||
}
|
||||
const record = readRecord(integrationId);
|
||||
const record = readRecordByKey(key);
|
||||
if (!record) {
|
||||
throw new Error(`"${integrationId}" is not connected — sign in with Google first.`);
|
||||
}
|
||||
@@ -405,33 +566,52 @@ export async function getAccessToken(integrationId: string): Promise<string> {
|
||||
}
|
||||
const token = json['access_token'] as string;
|
||||
const expiresIn = typeof json['expires_in'] === 'number' ? (json['expires_in'] as number) : 3600;
|
||||
accessTokens.set(integrationId, {
|
||||
accessTokens.set(key, {
|
||||
token,
|
||||
expiresAt: Date.now() + expiresIn * 1000,
|
||||
});
|
||||
return token;
|
||||
}
|
||||
|
||||
/** Disconnect — revoke at Google (best-effort) and wipe the stored token. */
|
||||
export async function revokeAuth(integrationId: string): Promise<GoogleAuthState> {
|
||||
const record = readRecord(integrationId);
|
||||
if (record) {
|
||||
try {
|
||||
await apiFetch(`${REVOKE_ENDPOINT}?token=${encodeURIComponent(record.refreshToken)}`, {
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
} catch {
|
||||
/* best-effort — local wipe below is what matters */
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Disconnect one linked account — or, when `accountId` is omitted, every
|
||||
* account for the integration. Revokes at Google (best-effort) and wipes
|
||||
* the stored refresh token either way.
|
||||
*/
|
||||
export async function revokeAuth(
|
||||
integrationId: string,
|
||||
accountId?: string,
|
||||
): Promise<GoogleAuthState> {
|
||||
const store = loadStore();
|
||||
if (store[integrationId]) {
|
||||
delete store[integrationId];
|
||||
persist();
|
||||
const targets = accountId
|
||||
? [accountKey(integrationId, accountId)]
|
||||
: Object.keys(store).filter(
|
||||
(k) => parseKey(k)?.integrationId === integrationId,
|
||||
);
|
||||
|
||||
for (const key of targets) {
|
||||
const record = readRecordByKey(key);
|
||||
if (record) {
|
||||
try {
|
||||
await apiFetch(`${REVOKE_ENDPOINT}?token=${encodeURIComponent(record.refreshToken)}`, {
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
} catch {
|
||||
/* best-effort — local wipe below is what matters */
|
||||
}
|
||||
}
|
||||
if (store[key]) {
|
||||
delete store[key];
|
||||
persist();
|
||||
}
|
||||
accessTokens.delete(key);
|
||||
}
|
||||
accessTokens.delete(integrationId);
|
||||
logger.log('info', 'main', `google-oauth: "${integrationId}" disconnected`);
|
||||
logger.log(
|
||||
'info',
|
||||
'main',
|
||||
`google-oauth: "${integrationId}" disconnected${accountId ? ` (account ${accountId})` : ' (all accounts)'}`,
|
||||
);
|
||||
return getAuthState(integrationId);
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,11 @@
|
||||
* (`google-oauth.ts`) — `getAccessToken('gsc')` mints / refreshes the
|
||||
* bearer token; nothing here touches credentials directly.
|
||||
*/
|
||||
import type { GscSite } from '@freecrawl/shared-types';
|
||||
import type {
|
||||
GscSite,
|
||||
GscIntegrationSettings,
|
||||
GscSearchType,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { getAccessToken } from './google-oauth.js';
|
||||
import { apiFetch } from './api-fetch.js';
|
||||
import * as logger from './logger.js';
|
||||
@@ -35,9 +39,9 @@ function apiError(json: unknown, status: number): string {
|
||||
return `Search Console API error (HTTP ${status})`;
|
||||
}
|
||||
|
||||
/** List the connected account's Search Console properties. */
|
||||
export async function listSites(): Promise<GscSite[]> {
|
||||
const token = await getAccessToken('gsc');
|
||||
/** List one connected account's Search Console properties. */
|
||||
export async function listSites(accountId?: string): Promise<GscSite[]> {
|
||||
const token = await getAccessToken('gsc', accountId);
|
||||
const res = await apiFetch(`${API_BASE}/sites`, {
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
@@ -65,20 +69,107 @@ export interface GscPageRow {
|
||||
position: number;
|
||||
}
|
||||
|
||||
/** One API dimension filter (`dimensionFilterGroups[].filters[]`). */
|
||||
interface GscDimensionFilter {
|
||||
dimension: string;
|
||||
operator: string;
|
||||
expression: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shaping options for a Search Analytics pull, derived from the
|
||||
* per-project GSC settings (search type, device/country/query dimension
|
||||
* filters, total-row cap). Empty options reproduce the previous
|
||||
* behaviour exactly (web search, no filters, 200 K row ceiling).
|
||||
*/
|
||||
export interface GscQueryOptions {
|
||||
type?: GscSearchType;
|
||||
dimensionFilterGroups?: { groupType: 'and'; filters: GscDimensionFilter[] }[];
|
||||
/** Total-row cap across all pages (SF "Limit Max Results"). */
|
||||
maxRows?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the `[startDate, endDate]` window from the settings' date
|
||||
* range. Non-custom ranges end ~2 days ago (Search Console's data lag)
|
||||
* and span back the preset's width; `16m` uses the API's 16-month max.
|
||||
*/
|
||||
export function resolveGscDateRange(settings: GscIntegrationSettings): {
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
} {
|
||||
const isoDate = (d: Date): string => d.toISOString().slice(0, 10);
|
||||
if (settings.dateRange === 'custom' && settings.startDate && settings.endDate) {
|
||||
return { startDate: settings.startDate, endDate: settings.endDate };
|
||||
}
|
||||
const daysBack =
|
||||
settings.dateRange === '7d'
|
||||
? 7
|
||||
: settings.dateRange === '90d'
|
||||
? 90
|
||||
: settings.dateRange === '16m'
|
||||
? 480
|
||||
: 28;
|
||||
const end = new Date(Date.now() - 2 * 86_400_000);
|
||||
const start = new Date(end.getTime() - (daysBack - 1) * 86_400_000);
|
||||
return { startDate: isoDate(start), endDate: isoDate(end) };
|
||||
}
|
||||
|
||||
/** Build the API query options (type + dimension filters + row cap) from
|
||||
* the per-project GSC settings. */
|
||||
export function buildGscQueryOptions(
|
||||
settings: GscIntegrationSettings,
|
||||
): GscQueryOptions {
|
||||
const filters: GscDimensionFilter[] = [];
|
||||
if (settings.deviceFilter !== 'all') {
|
||||
filters.push({
|
||||
dimension: 'device',
|
||||
operator: 'equals',
|
||||
expression: settings.deviceFilter,
|
||||
});
|
||||
}
|
||||
if (settings.countryFilter) {
|
||||
filters.push({
|
||||
dimension: 'country',
|
||||
operator: 'equals',
|
||||
expression: settings.countryFilter,
|
||||
});
|
||||
}
|
||||
if (settings.queryFilterMode !== 'none' && settings.queryFilterValue) {
|
||||
filters.push({
|
||||
dimension: 'query',
|
||||
operator: settings.queryFilterMode,
|
||||
expression: settings.queryFilterValue,
|
||||
});
|
||||
}
|
||||
const opts: GscQueryOptions = { type: settings.searchType };
|
||||
if (filters.length > 0) {
|
||||
opts.dimensionFilterGroups = [{ groupType: 'and', filters }];
|
||||
}
|
||||
if (settings.limitMaxResults) {
|
||||
opts.maxRows = Math.max(1, Math.min(1_000_000, settings.maxResults || 100000));
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull per-page Search Console metrics for `[startDate, endDate]`
|
||||
* (`YYYY-MM-DD`, inclusive). Pages transparently past the 25 000-row
|
||||
* API limit and returns the flattened list.
|
||||
* API limit and returns the flattened list. `opts` shapes the query
|
||||
* (search type, dimension filters, total-row cap).
|
||||
*/
|
||||
export async function querySearchAnalytics(
|
||||
property: string,
|
||||
startDate: string,
|
||||
endDate: string,
|
||||
opts: GscQueryOptions = {},
|
||||
accountId?: string,
|
||||
): Promise<GscPageRow[]> {
|
||||
const token = await getAccessToken('gsc');
|
||||
const token = await getAccessToken('gsc', accountId);
|
||||
const endpoint = `${API_BASE}/sites/${encodeURIComponent(
|
||||
property,
|
||||
)}/searchAnalytics/query`;
|
||||
const maxRows = Math.max(1, Math.min(1_000_000, opts.maxRows ?? MAX_ROWS));
|
||||
const out: GscPageRow[] = [];
|
||||
let startRow = 0;
|
||||
|
||||
@@ -95,6 +186,10 @@ export async function querySearchAnalytics(
|
||||
dimensions: ['page'],
|
||||
rowLimit: ROW_LIMIT,
|
||||
startRow,
|
||||
...(opts.type ? { type: opts.type } : {}),
|
||||
...(opts.dimensionFilterGroups
|
||||
? { dimensionFilterGroups: opts.dimensionFilterGroups }
|
||||
: {}),
|
||||
}),
|
||||
signal: AbortSignal.timeout(60_000),
|
||||
});
|
||||
@@ -121,9 +216,10 @@ export async function querySearchAnalytics(
|
||||
position: Number(r.position) || 0,
|
||||
});
|
||||
}
|
||||
if (rows.length < ROW_LIMIT || out.length >= MAX_ROWS) break;
|
||||
if (rows.length < ROW_LIMIT || out.length >= maxRows) break;
|
||||
startRow += ROW_LIMIT;
|
||||
}
|
||||
if (out.length > maxRows) out.length = maxRows;
|
||||
logger.log(
|
||||
'info',
|
||||
'gsc',
|
||||
@@ -141,6 +237,9 @@ export interface GscInspectionRaw {
|
||||
lastCrawlTime: string | null;
|
||||
googleCanonical: string | null;
|
||||
userCanonical: string | null;
|
||||
mobileVerdict: string | null;
|
||||
ampVerdict: string | null;
|
||||
richResultsVerdict: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -150,8 +249,9 @@ export interface GscInspectionRaw {
|
||||
export async function inspectUrl(
|
||||
property: string,
|
||||
url: string,
|
||||
accountId?: string,
|
||||
): Promise<GscInspectionRaw> {
|
||||
const token = await getAccessToken('gsc');
|
||||
const token = await getAccessToken('gsc', accountId);
|
||||
const res = await apiFetch(
|
||||
'https://searchconsole.googleapis.com/v1/urlInspection/index:inspect',
|
||||
{
|
||||
@@ -165,12 +265,25 @@ export async function inspectUrl(
|
||||
},
|
||||
);
|
||||
const json = (await res.json().catch(() => null)) as {
|
||||
inspectionResult?: { indexStatusResult?: Record<string, unknown> };
|
||||
inspectionResult?: {
|
||||
indexStatusResult?: Record<string, unknown>;
|
||||
mobileUsabilityResult?: Record<string, unknown>;
|
||||
ampResult?: Record<string, unknown>;
|
||||
richResultsResult?: Record<string, unknown>;
|
||||
};
|
||||
} | null;
|
||||
if (!res.ok || !json) throw new Error(apiError(json, res.status));
|
||||
const idx = json.inspectionResult?.indexStatusResult ?? {};
|
||||
const s = (k: string): string | null =>
|
||||
typeof idx[k] === 'string' ? (idx[k] as string) : null;
|
||||
// The mobile / AMP / rich-result facets each carry their own verdict;
|
||||
// absent facets (page has no AMP, no rich results) stay null.
|
||||
const facetVerdict = (
|
||||
facet: Record<string, unknown> | undefined,
|
||||
): string | null =>
|
||||
facet && typeof facet['verdict'] === 'string'
|
||||
? (facet['verdict'] as string)
|
||||
: null;
|
||||
return {
|
||||
verdict: s('verdict'),
|
||||
coverageState: s('coverageState'),
|
||||
@@ -179,5 +292,8 @@ export async function inspectUrl(
|
||||
lastCrawlTime: s('lastCrawlTime'),
|
||||
googleCanonical: s('googleCanonical'),
|
||||
userCanonical: s('userCanonical'),
|
||||
mobileVerdict: facetVerdict(json.inspectionResult?.mobileUsabilityResult),
|
||||
ampVerdict: facetVerdict(json.inspectionResult?.ampResult),
|
||||
richResultsVerdict: facetVerdict(json.inspectionResult?.richResultsResult),
|
||||
};
|
||||
}
|
||||
+499
-95
@@ -21,6 +21,7 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { basename, dirname, join, resolve as resolvePath, sep as pathSep } from 'node:path';
|
||||
import {
|
||||
cpSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
@@ -59,6 +60,7 @@ import {
|
||||
type ExportGridResult,
|
||||
type DataDeleteByDomainInput,
|
||||
type DataDeleteByDomainResult,
|
||||
type CrashRecoveryResumeResult,
|
||||
type CrashRecoveryStatus,
|
||||
type ExportHtmlReportInput,
|
||||
type ExportHtmlReportResult,
|
||||
@@ -148,6 +150,12 @@ import {
|
||||
type GscFetchMeta,
|
||||
type GscQueryInput,
|
||||
type GscQueryResult,
|
||||
type GscCrawlNewUrlsResult,
|
||||
type GscIntegrationSettings,
|
||||
type Ga4IntegrationSettings,
|
||||
type GoogleAccount,
|
||||
defaultGscSettings,
|
||||
defaultGa4Settings,
|
||||
type Ga4ListPropertiesResult,
|
||||
type Ga4FetchInput,
|
||||
type Ga4FetchResult,
|
||||
@@ -258,8 +266,20 @@ import {
|
||||
} from './languagetool.js';
|
||||
import { warmLanguageDetector } from './language-detect.js';
|
||||
import { warmLocalDictionary } from './local-spell.js';
|
||||
import { startAuth, getAuthState, revokeAuth } from './google-oauth.js';
|
||||
import { listSites, querySearchAnalytics, inspectUrl } from './gsc.js';
|
||||
import {
|
||||
startAuth,
|
||||
getAuthState,
|
||||
revokeAuth,
|
||||
listAccounts,
|
||||
resolveAccountId,
|
||||
} from './google-oauth.js';
|
||||
import {
|
||||
listSites,
|
||||
querySearchAnalytics,
|
||||
inspectUrl,
|
||||
resolveGscDateRange,
|
||||
buildGscQueryOptions,
|
||||
} from './gsc.js';
|
||||
import { exportCategoryToSheets } from './sheets-export.js';
|
||||
import { exportCategoryToBigQuery } from './bigquery-export.js';
|
||||
import { listProperties as ga4ListProperties, runReport as ga4RunReport } from './ga4.js';
|
||||
@@ -473,6 +493,16 @@ class ProjectSession {
|
||||
this.pendingRecoveryCheckpoint = [];
|
||||
}
|
||||
database.reset();
|
||||
// The scratch database is wiped on every launch, so the PNGs the last
|
||||
// session captured into its sidecar are now unreferenced. Drop them with
|
||||
// it — otherwise the directory in userData grows forever across crawls
|
||||
// of unrelated sites. A saved project keeps its own copy (see
|
||||
// `saveProjectDocument`), so nothing the user chose to keep is at risk.
|
||||
try {
|
||||
rmSync(screenshotDirFor(path), { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort — a locked file just means it is cleaned next launch */
|
||||
}
|
||||
if (ramMode) {
|
||||
logger.log(
|
||||
'info',
|
||||
@@ -1466,6 +1496,27 @@ function markDirty(s: ProjectSession): void {
|
||||
updateWindowTitle(s);
|
||||
}
|
||||
|
||||
/**
|
||||
* V2 Faz 1 — Screenshot PNGs are too big to sit in the database, so they
|
||||
* live on disk in a sidecar directory named after the file the session is
|
||||
* bound to: `<name>.screenshots/`.
|
||||
*/
|
||||
function screenshotDirFor(basePath: string): string {
|
||||
return join(dirname(basePath), `${basename(basePath)}.screenshots`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sidecar for whatever this session is actually writing to. An unsaved
|
||||
* session has no document, so it falls back to the working/scratch database
|
||||
* in userData — keying this on the document alone meant screenshot capture
|
||||
* was silently disabled for every crawl until the user saved a project,
|
||||
* which is the common case and left the Screenshot tab permanently empty.
|
||||
*/
|
||||
function activeScreenshotDir(s: ProjectSession): string | null {
|
||||
const base = s.currentProjectPath || s.scratchPath;
|
||||
return base ? screenshotDirFor(base) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the working database to `documentPath` as a compressed container.
|
||||
*
|
||||
@@ -1477,8 +1528,46 @@ function markDirty(s: ProjectSession): void {
|
||||
async function saveProjectDocument(
|
||||
s: ProjectSession,
|
||||
documentPath: string,
|
||||
opts: {
|
||||
/**
|
||||
* False for the MCP snapshot action, which writes a copy without moving
|
||||
* the session onto it. The sidecar handling below rewrites the LIVE
|
||||
* database, so it must only run when this save is the session's new
|
||||
* home — a snapshot doing it would leave the open project's rows
|
||||
* pointing into the snapshot's directory.
|
||||
*/
|
||||
rebind?: boolean;
|
||||
} = {},
|
||||
): Promise<{ bytesWritten: number; sourceBytes: number }> {
|
||||
const database = s.getDb();
|
||||
// Screenshots sit beside the document rather than inside the archive, so
|
||||
// saving has to bring them along and re-point the rows at the new sidecar
|
||||
// — otherwise the saved copy names a directory that belongs to the
|
||||
// previous location and its Screenshot tab comes up blank. Covers the
|
||||
// first save too: captures taken while unsaved live in the userData
|
||||
// scratch sidecar. Runs BEFORE the snapshot below so the archive carries
|
||||
// the rewritten paths.
|
||||
const previousShotDir = activeScreenshotDir(s);
|
||||
const nextShotDir = screenshotDirFor(documentPath);
|
||||
if (
|
||||
opts.rebind !== false &&
|
||||
previousShotDir &&
|
||||
previousShotDir !== nextShotDir &&
|
||||
existsSync(previousShotDir)
|
||||
) {
|
||||
try {
|
||||
cpSync(previousShotDir, nextShotDir, { recursive: true });
|
||||
database.remapScreenshotDir(previousShotDir, nextShotDir);
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'warn',
|
||||
'main',
|
||||
`Could not move the screenshots sidecar to ${nextShotDir}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
} — the saved project will have no screenshots.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
try {
|
||||
database.walCheckpoint();
|
||||
} catch {
|
||||
@@ -3139,8 +3228,15 @@ function registerIpc(): void {
|
||||
registerHandle(
|
||||
IPC.logAnalyze,
|
||||
async (e, input: LogAnalyzeInput): Promise<LogAnalyzeResult> => {
|
||||
let filePath = input?.filePath;
|
||||
if (!filePath) {
|
||||
// Resolve the file list: explicit `filePaths` (bulk) wins, then a single
|
||||
// `filePath`, then a multi-select picker.
|
||||
let filePaths =
|
||||
input?.filePaths && input.filePaths.length > 0
|
||||
? input.filePaths
|
||||
: input?.filePath
|
||||
? [input.filePath]
|
||||
: undefined;
|
||||
if (!filePaths) {
|
||||
const parent =
|
||||
BrowserWindow.fromWebContents(e.sender) ?? logAnalyzerWindow ?? mainWindow;
|
||||
if (!parent) {
|
||||
@@ -3148,7 +3244,7 @@ function registerIpc(): void {
|
||||
}
|
||||
const res = await dialog.showOpenDialog(parent, {
|
||||
title: L().dlgOpenAccessLogTitle,
|
||||
properties: ['openFile'],
|
||||
properties: ['openFile', 'multiSelections'],
|
||||
filters: [
|
||||
{ name: 'Log Files', extensions: ['log', 'txt', 'gz', 'out', 'access'] },
|
||||
{ name: 'All Files', extensions: ['*'] },
|
||||
@@ -3157,30 +3253,52 @@ function registerIpc(): void {
|
||||
if (res.canceled || res.filePaths.length === 0) {
|
||||
return { ok: false, error: undefined, imported: null, overview: getDb().getLogOverview() };
|
||||
}
|
||||
filePath = res.filePaths[0]!;
|
||||
filePaths = res.filePaths;
|
||||
}
|
||||
try {
|
||||
const result = await analyzeLogFile(filePath, {
|
||||
format: input?.format ?? 'auto',
|
||||
customRegex: input?.customRegex,
|
||||
verifyBots: input?.verifyBots === true,
|
||||
});
|
||||
const imported = ingestLogResult(filePath, result);
|
||||
if (result.urlCapHit) {
|
||||
logger.log('warn', 'main', `Log Analyzer: distinct-URL cap hit while ingesting ${basename(filePath)}; some long-tail URLs were dropped.`);
|
||||
// Ingest each file in turn; one bad file must not abort the batch.
|
||||
const batch: LogImportSummary[] = [];
|
||||
const errors: Array<{ fileName: string; error: string }> = [];
|
||||
for (const filePath of filePaths) {
|
||||
try {
|
||||
const result = await analyzeLogFile(filePath, {
|
||||
format: input?.format ?? 'auto',
|
||||
customRegex: input?.customRegex,
|
||||
verifyBots: input?.verifyBots === true,
|
||||
});
|
||||
batch.push(ingestLogResult(filePath, result));
|
||||
if (result.urlCapHit) {
|
||||
logger.log('warn', 'main', `Log Analyzer: distinct-URL cap hit while ingesting ${basename(filePath)}; some long-tail URLs were dropped.`);
|
||||
}
|
||||
logger.log(
|
||||
'info',
|
||||
'main',
|
||||
`Log Analyzer ingested ${basename(filePath)} (${result.format}): ${result.parsedLines}/${result.totalLines} lines, ${result.bots.size} bot type(s).`,
|
||||
);
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
logger.log('warn', 'main', `Log Analyzer failed on ${filePath}: ${error}`);
|
||||
errors.push({ fileName: basename(filePath), error });
|
||||
}
|
||||
logger.log(
|
||||
'info',
|
||||
'main',
|
||||
`Log Analyzer ingested ${basename(filePath)} (${result.format}): ${result.parsedLines}/${result.totalLines} lines, ${result.bots.size} bot type(s).`,
|
||||
);
|
||||
fireDataChanged();
|
||||
return { ok: true, imported, overview: getDb().getLogOverview() };
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
logger.log('warn', 'main', `Log Analyzer failed on ${filePath}: ${error}`);
|
||||
return { ok: false, error, imported: null, overview: getDb().getLogOverview() };
|
||||
}
|
||||
fireDataChanged();
|
||||
const overview = getDb().getLogOverview();
|
||||
if (batch.length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
error: errors[0]?.error ?? 'No log files could be ingested.',
|
||||
imported: null,
|
||||
batch: [],
|
||||
errors,
|
||||
overview,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
imported: batch[0] ?? null,
|
||||
batch,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
overview,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
@@ -3271,9 +3389,11 @@ function registerIpc(): void {
|
||||
// naive startsWith() lets `<root>/../../../etc/passwd` pass.
|
||||
const resolved = resolvePath(absolutePath);
|
||||
const allowedRoots: string[] = [];
|
||||
if (currentSession().currentProjectPath) {
|
||||
const dir = resolveScreenshotDir(currentSession().currentProjectPath);
|
||||
if (dir) allowedRoots.push(resolvePath(dir));
|
||||
const sess = currentSession();
|
||||
// Both sidecars: the document's own, and the working/scratch one —
|
||||
// rows captured before the project was ever saved point at the latter.
|
||||
for (const base of [sess.currentProjectPath, sess.scratchPath]) {
|
||||
if (base) allowedRoots.push(resolvePath(screenshotDirFor(base)));
|
||||
}
|
||||
// Append the separator so that `<root>` doesn't match `<root>2`.
|
||||
const safe = allowedRoots.some(
|
||||
@@ -3733,12 +3853,20 @@ function registerIpc(): void {
|
||||
|
||||
registerHandle(
|
||||
IPC.reportsOrphanCrossSource,
|
||||
(_e, limit?: number): Promise<OrphanCrossSourceRow[]> =>
|
||||
callReaderOrFallback<OrphanCrossSourceRow[]>(
|
||||
(_e, limit?: number): Promise<OrphanCrossSourceRow[]> => {
|
||||
// Scoped to the project's selected accounts — the cross-source
|
||||
// joins are only 1:1 when pinned to one GSC + one GA4 account.
|
||||
const args: [number, string, string] = [
|
||||
limit ?? 1000,
|
||||
resolveGoogleAccount('gsc'),
|
||||
resolveGoogleAccount('ga4'),
|
||||
];
|
||||
return callReaderOrFallback<OrphanCrossSourceRow[]>(
|
||||
'orphanPagesCrossSource',
|
||||
[limit ?? 1000],
|
||||
() => getDb().orphanPagesCrossSource(limit ?? 1000),
|
||||
),
|
||||
args,
|
||||
() => getDb().orphanPagesCrossSource(...args),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
registerHandle(IPC.reportsServerHeaders, () =>
|
||||
@@ -3899,6 +4027,19 @@ function registerIpc(): void {
|
||||
return clearIntegrationCredentials(id);
|
||||
});
|
||||
|
||||
// Per-project, per-integration behaviour settings (date ranges, filter
|
||||
// options, "crawl new URLs"). Stored as a JSON blob in project_meta
|
||||
// under `integrationSettings:<id>` — separate from the encrypted,
|
||||
// app-global credential store above.
|
||||
registerHandle(IPC.integrationSettingsGet, (_e, id: string) =>
|
||||
readIntegrationSettings(id),
|
||||
);
|
||||
registerHandle(
|
||||
IPC.integrationSettingsSet,
|
||||
(_e, id: string, settings: Record<string, unknown>) =>
|
||||
writeIntegrationSettings(id, settings),
|
||||
);
|
||||
|
||||
// V1 Faz 7 — Google PageSpeed Insights. `pagespeedQuery` lists the
|
||||
// crawled internal HTML pages joined with any stored audit results;
|
||||
// `pagespeedRun` audits a user-selected subset (the slow part — PSI
|
||||
@@ -4494,7 +4635,8 @@ function registerIpc(): void {
|
||||
);
|
||||
registerHandle(
|
||||
IPC.googleAuthRevoke,
|
||||
(_e, id: string): Promise<GoogleAuthState> => revokeAuth(id),
|
||||
(_e, id: string, accountId?: string): Promise<GoogleAuthState> =>
|
||||
revokeAuth(id, accountId),
|
||||
);
|
||||
|
||||
// V1 Faz 7 — Google Search Console. `gscFetch` pulls per-page metrics
|
||||
@@ -4502,9 +4644,10 @@ function registerIpc(): void {
|
||||
// `gscQuery` lists crawled pages joined with that data.
|
||||
registerHandle(
|
||||
IPC.gscListSites,
|
||||
async (): Promise<GscListSitesResult> => {
|
||||
async (_e, accountId?: string): Promise<GscListSitesResult> => {
|
||||
try {
|
||||
return { ok: true, error: null, sites: await listSites() };
|
||||
const account = resolveGoogleAccount('gsc', accountId);
|
||||
return { ok: true, error: null, sites: await listSites(account) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -4514,6 +4657,10 @@ function registerIpc(): void {
|
||||
}
|
||||
},
|
||||
);
|
||||
registerHandle(
|
||||
IPC.googleAccountsList,
|
||||
(_e, integrationId: string): GoogleAccount[] => listAccounts(integrationId),
|
||||
);
|
||||
registerHandle(
|
||||
IPC.gscFetch,
|
||||
async (_e, input: GscFetchInput): Promise<GscFetchResult> => {
|
||||
@@ -4525,23 +4672,27 @@ function registerIpc(): void {
|
||||
meta: null,
|
||||
};
|
||||
}
|
||||
// Search Console data lags ~2 days — end the window there and
|
||||
// span `days` back so the range only covers data that exists.
|
||||
const isoDate = (d: Date): string => d.toISOString().slice(0, 10);
|
||||
const end = new Date(Date.now() - 2 * 86_400_000);
|
||||
const start = new Date(end.getTime() - (input.days - 1) * 86_400_000);
|
||||
const startDate = isoDate(start);
|
||||
const endDate = isoDate(end);
|
||||
// Pin the DB to the project active when the pull started.
|
||||
// The date range + dimension filters + row cap come from the stored
|
||||
// per-project GSC settings (Settings → Integrations). Persist the
|
||||
// property the user pulled from so it's remembered next session.
|
||||
const settings = resolveGscSettings();
|
||||
const accountId = resolveGoogleAccount('gsc', input.accountId);
|
||||
writeIntegrationSettings('gsc', { property: input.property, accountId });
|
||||
const { startDate, endDate } = resolveGscDateRange(settings);
|
||||
const queryOpts = buildGscQueryOptions(settings);
|
||||
// Pin the DB + session to the project active when the pull started.
|
||||
const runDb = getDb();
|
||||
const runSession = currentSession();
|
||||
try {
|
||||
const rows = await querySearchAnalytics(
|
||||
input.property,
|
||||
startDate,
|
||||
endDate,
|
||||
queryOpts,
|
||||
accountId,
|
||||
);
|
||||
const fetchedAt = new Date().toISOString();
|
||||
runDb.replaceGscResults(rows, fetchedAt);
|
||||
runDb.replaceGscResults(rows, fetchedAt, accountId);
|
||||
const meta: GscFetchMeta = {
|
||||
property: input.property,
|
||||
startDate,
|
||||
@@ -4549,14 +4700,38 @@ function registerIpc(): void {
|
||||
fetchedAt,
|
||||
rowCount: rows.length,
|
||||
};
|
||||
runDb.setMeta('gscFetchMeta', JSON.stringify(meta));
|
||||
runDb.setMeta(googleMetaKey('gscFetchMeta', accountId), JSON.stringify(meta));
|
||||
fireDataChanged();
|
||||
logger.log(
|
||||
'info',
|
||||
'gsc',
|
||||
`Search Console pull stored — ${rows.length} page row(s)`,
|
||||
);
|
||||
return { ok: true, error: null, rowCount: rows.length, meta };
|
||||
// How many GSC URLs are missing from the crawl — the candidates
|
||||
// for "Crawl New URLs Discovered In Google Search Console".
|
||||
const orphans = runDb.gscOrphanUrls({
|
||||
matchSlash: settings.matchSlash,
|
||||
matchCase: settings.matchCase,
|
||||
accountId,
|
||||
});
|
||||
let queuedNewUrls = 0;
|
||||
if (settings.crawlNewUrls && orphans.length > 0) {
|
||||
await launchRespider(runSession, orphans);
|
||||
queuedNewUrls = orphans.length;
|
||||
logger.log(
|
||||
'info',
|
||||
'gsc',
|
||||
`Crawling ${orphans.length} new URL(s) discovered in Search Console`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
error: null,
|
||||
rowCount: rows.length,
|
||||
meta,
|
||||
newUrlCount: orphans.length,
|
||||
queuedNewUrls,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -4567,14 +4742,47 @@ function registerIpc(): void {
|
||||
}
|
||||
},
|
||||
);
|
||||
registerHandle(
|
||||
IPC.gscCrawlNewUrls,
|
||||
async (): Promise<GscCrawlNewUrlsResult> => {
|
||||
const settings = resolveGscSettings();
|
||||
const runSession = currentSession();
|
||||
try {
|
||||
const orphans = getDb().gscOrphanUrls({
|
||||
matchSlash: settings.matchSlash,
|
||||
matchCase: settings.matchCase,
|
||||
accountId: resolveGoogleAccount('gsc'),
|
||||
});
|
||||
const hasActiveCrawl = runSession.activeCrawler?.isRunning === true;
|
||||
if (orphans.length > 0) await launchRespider(runSession, orphans);
|
||||
return {
|
||||
ok: true,
|
||||
error: null,
|
||||
candidateCount: orphans.length,
|
||||
queued: orphans.length,
|
||||
hasActiveCrawl,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
candidateCount: 0,
|
||||
queued: 0,
|
||||
hasActiveCrawl: false,
|
||||
};
|
||||
}
|
||||
},
|
||||
);
|
||||
registerHandle(
|
||||
IPC.gscQuery,
|
||||
async (_e, input: GscQueryInput): Promise<GscQueryResult> => {
|
||||
const accountId = resolveGoogleAccount('gsc', input.accountId);
|
||||
const params = {
|
||||
limit: input.limit,
|
||||
offset: input.offset,
|
||||
search: input.search,
|
||||
filter: input.filter,
|
||||
accountId,
|
||||
};
|
||||
const result = await callReaderOrFallback<{
|
||||
rows: GscQueryResult['rows'];
|
||||
@@ -4582,7 +4790,7 @@ function registerIpc(): void {
|
||||
}>('queryGsc', [params], () => getDb().queryGsc(params));
|
||||
let meta: GscFetchMeta | null = null;
|
||||
try {
|
||||
const raw = getDb().getMeta('gscFetchMeta');
|
||||
const raw = getDb().getMeta(googleMetaKey('gscFetchMeta', accountId));
|
||||
if (raw) meta = JSON.parse(raw) as GscFetchMeta;
|
||||
} catch {
|
||||
/* no / corrupt meta — render without the context line */
|
||||
@@ -4596,9 +4804,14 @@ function registerIpc(): void {
|
||||
// stored data. Auth runs through the shared OAuth keystone.
|
||||
registerHandle(
|
||||
IPC.ga4ListProperties,
|
||||
async (): Promise<Ga4ListPropertiesResult> => {
|
||||
async (_e, accountId?: string): Promise<Ga4ListPropertiesResult> => {
|
||||
try {
|
||||
return { ok: true, error: null, properties: await ga4ListProperties() };
|
||||
const account = resolveGoogleAccount('ga4', accountId);
|
||||
return {
|
||||
ok: true,
|
||||
error: null,
|
||||
properties: await ga4ListProperties(account),
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -4626,10 +4839,22 @@ function registerIpc(): void {
|
||||
const startDate = isoDate(start);
|
||||
const endDate = isoDate(end);
|
||||
const runDb = getDb();
|
||||
const accountId = resolveGoogleAccount('ga4', input.accountId);
|
||||
writeIntegrationSettings('ga4', {
|
||||
accountId,
|
||||
property: input.property,
|
||||
propertyName: input.propertyName || input.property,
|
||||
days: input.days,
|
||||
});
|
||||
try {
|
||||
const rows = await ga4RunReport(input.property, startDate, endDate);
|
||||
const rows = await ga4RunReport(
|
||||
input.property,
|
||||
startDate,
|
||||
endDate,
|
||||
accountId,
|
||||
);
|
||||
const fetchedAt = new Date().toISOString();
|
||||
runDb.replaceGa4Results(rows, fetchedAt);
|
||||
runDb.replaceGa4Results(rows, fetchedAt, accountId);
|
||||
const meta: Ga4FetchMeta = {
|
||||
property: input.property,
|
||||
propertyName: input.propertyName || input.property,
|
||||
@@ -4638,7 +4863,7 @@ function registerIpc(): void {
|
||||
fetchedAt,
|
||||
rowCount: rows.length,
|
||||
};
|
||||
runDb.setMeta('ga4FetchMeta', JSON.stringify(meta));
|
||||
runDb.setMeta(googleMetaKey('ga4FetchMeta', accountId), JSON.stringify(meta));
|
||||
fireDataChanged();
|
||||
logger.log(
|
||||
'info',
|
||||
@@ -4659,11 +4884,13 @@ function registerIpc(): void {
|
||||
registerHandle(
|
||||
IPC.ga4Query,
|
||||
async (_e, input: Ga4QueryInput): Promise<Ga4QueryResult> => {
|
||||
const accountId = resolveGoogleAccount('ga4', input.accountId);
|
||||
const params = {
|
||||
limit: input.limit,
|
||||
offset: input.offset,
|
||||
search: input.search,
|
||||
filter: input.filter,
|
||||
accountId,
|
||||
};
|
||||
const result = await callReaderOrFallback<{
|
||||
rows: Ga4QueryResult['rows'];
|
||||
@@ -4671,7 +4898,7 @@ function registerIpc(): void {
|
||||
}>('queryGa4', [params], () => getDb().queryGa4(params));
|
||||
let meta: Ga4FetchMeta | null = null;
|
||||
try {
|
||||
const raw = getDb().getMeta('ga4FetchMeta');
|
||||
const raw = getDb().getMeta(googleMetaKey('ga4FetchMeta', accountId));
|
||||
if (raw) meta = JSON.parse(raw) as Ga4FetchMeta;
|
||||
} catch {
|
||||
/* no / corrupt meta — render without the context line */
|
||||
@@ -5050,6 +5277,7 @@ function registerIpc(): void {
|
||||
return { completed: 0, failed: 0, cancelled: false };
|
||||
}
|
||||
const runDb = runSession.getDb();
|
||||
const inspectAccountId = resolveGoogleAccount('gsc', input.accountId);
|
||||
run.active = true;
|
||||
run.cancelRequested = false;
|
||||
let completed = 0;
|
||||
@@ -5077,14 +5305,14 @@ function registerIpc(): void {
|
||||
});
|
||||
const fetchedAt = new Date().toISOString();
|
||||
try {
|
||||
const raw = await inspectUrl(input.property, url);
|
||||
const raw = await inspectUrl(input.property, url, inspectAccountId);
|
||||
const result: GscInspectionResult = {
|
||||
...raw,
|
||||
status: 'ok',
|
||||
error: null,
|
||||
fetchedAt,
|
||||
};
|
||||
runDb.upsertGscInspection(url, result);
|
||||
runDb.upsertGscInspection(url, result, inspectAccountId);
|
||||
completed++;
|
||||
} catch (err) {
|
||||
runDb.upsertGscInspection(url, {
|
||||
@@ -5095,10 +5323,13 @@ function registerIpc(): void {
|
||||
lastCrawlTime: null,
|
||||
googleCanonical: null,
|
||||
userCanonical: null,
|
||||
mobileVerdict: null,
|
||||
ampVerdict: null,
|
||||
richResultsVerdict: null,
|
||||
status: 'error',
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
fetchedAt,
|
||||
});
|
||||
}, inspectAccountId);
|
||||
failed++;
|
||||
}
|
||||
done++;
|
||||
@@ -5165,10 +5396,17 @@ function registerIpc(): void {
|
||||
IPC.urlAnalyticsGet,
|
||||
async (_e, url: string): Promise<UrlAnalyticsDetail | null> => {
|
||||
if (!url) return null;
|
||||
// Scoped to the project's selected accounts so the panel agrees
|
||||
// with what the Search Console / GA4 tabs are showing.
|
||||
const args: [string, string, string] = [
|
||||
url,
|
||||
resolveGoogleAccount('gsc'),
|
||||
resolveGoogleAccount('ga4'),
|
||||
];
|
||||
return callReaderOrFallback<UrlAnalyticsDetail | null>(
|
||||
'getUrlAnalytics',
|
||||
[url],
|
||||
() => getDb().getUrlAnalytics(url),
|
||||
args,
|
||||
() => getDb().getUrlAnalytics(...args),
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -5527,7 +5765,7 @@ function registerIpc(): void {
|
||||
renderHook = undefined;
|
||||
}
|
||||
}
|
||||
const screenshotDir = resolveScreenshotDir(currentSession().currentProjectPath);
|
||||
const screenshotDir = activeScreenshotDir(sess);
|
||||
const wantsScreenshot = config.jsRender.screenshotMode !== 'none';
|
||||
const wantsMobile =
|
||||
config.jsRender.mobileScreenshot || config.jsRender.mobileUsability;
|
||||
@@ -5684,6 +5922,129 @@ function registerIpc(): void {
|
||||
* different site is being crawled, and the re-spider must never read as
|
||||
* one.
|
||||
*/
|
||||
/** project_meta key for one integration's per-project settings blob. */
|
||||
function integrationSettingsKey(id: string): string {
|
||||
return `integrationSettings:${id}`;
|
||||
}
|
||||
|
||||
/** Read one integration's stored settings blob (or null). */
|
||||
function readIntegrationSettings(
|
||||
id: string,
|
||||
): Record<string, unknown> | null {
|
||||
const raw = getDb().getMeta(integrationSettingsKey(id));
|
||||
if (!raw) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
return parsed && typeof parsed === 'object'
|
||||
? (parsed as Record<string, unknown>)
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Merge-save one integration's settings blob; returns the merged result. */
|
||||
function writeIntegrationSettings(
|
||||
id: string,
|
||||
settings: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
const merged = { ...(readIntegrationSettings(id) ?? {}), ...settings };
|
||||
getDb().setMeta(integrationSettingsKey(id), JSON.stringify(merged));
|
||||
return merged;
|
||||
}
|
||||
|
||||
/** Resolve the effective GSC settings (defaults + stored overrides). */
|
||||
function resolveGscSettings(): GscIntegrationSettings {
|
||||
return {
|
||||
...defaultGscSettings(),
|
||||
...(readIntegrationSettings('gsc') ?? {}),
|
||||
} as GscIntegrationSettings;
|
||||
}
|
||||
|
||||
/** Resolve the effective GA4 settings (defaults + stored overrides). */
|
||||
function resolveGa4Settings(): Ga4IntegrationSettings {
|
||||
return {
|
||||
...defaultGa4Settings(),
|
||||
...(readIntegrationSettings('ga4') ?? {}),
|
||||
} as Ga4IntegrationSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which linked Google account a call should act on. An explicit id from
|
||||
* the caller wins, then the project's stored choice, then the first
|
||||
* linked account (what every pre-multi-account project means).
|
||||
*
|
||||
* Also folds in the legacy-row adoption: the first time an account is
|
||||
* resolved for an integration, any rows still tagged `account_id = ''`
|
||||
* — written before multi-account existed — are relabelled to it, so an
|
||||
* upgraded project keeps the data it already pulled.
|
||||
*/
|
||||
function resolveGoogleAccount(
|
||||
integrationId: 'gsc' | 'ga4',
|
||||
requested?: string,
|
||||
): string {
|
||||
const stored =
|
||||
integrationId === 'gsc'
|
||||
? resolveGscSettings().accountId
|
||||
: resolveGa4Settings().accountId;
|
||||
const accountId = resolveAccountId(integrationId, requested || stored) ?? '';
|
||||
if (accountId) adoptLegacyRowsOnce(integrationId, accountId);
|
||||
return accountId;
|
||||
}
|
||||
|
||||
/**
|
||||
* `project_meta` key holding the last pull's metadata for one account.
|
||||
* Pre-multi-account projects stored it under the bare base key; that
|
||||
* row is moved across during legacy adoption.
|
||||
*/
|
||||
function googleMetaKey(base: string, accountId: string): string {
|
||||
return accountId ? `${base}:${accountId}` : base;
|
||||
}
|
||||
|
||||
/** Projects whose legacy rows have already been adopted this session. */
|
||||
const legacyAdopted = new Set<string>();
|
||||
|
||||
function adoptLegacyRowsOnce(
|
||||
integrationId: 'gsc' | 'ga4',
|
||||
accountId: string,
|
||||
): void {
|
||||
const sess = currentSession();
|
||||
const marker = `${sess.currentProjectPath || 'default'}:${integrationId}:${accountId}`;
|
||||
if (legacyAdopted.has(marker)) return;
|
||||
legacyAdopted.add(marker);
|
||||
try {
|
||||
const db = sess.getDb();
|
||||
const tables =
|
||||
integrationId === 'gsc'
|
||||
? ['gsc_results', 'gsc_inspection_results']
|
||||
: ['ga4_results'];
|
||||
let moved = 0;
|
||||
for (const table of tables) {
|
||||
moved += db.adoptLegacyGoogleRows(table, accountId);
|
||||
}
|
||||
// Move the matching fetch-meta row so the tab's context line
|
||||
// ("property · range · fetched N ago") survives the upgrade too.
|
||||
const metaBase = integrationId === 'gsc' ? 'gscFetchMeta' : 'ga4FetchMeta';
|
||||
const legacyMeta = db.getMeta(metaBase);
|
||||
if (legacyMeta && !db.getMeta(googleMetaKey(metaBase, accountId))) {
|
||||
db.setMeta(googleMetaKey(metaBase, accountId), legacyMeta);
|
||||
}
|
||||
if (moved > 0) {
|
||||
logger.log(
|
||||
'info',
|
||||
integrationId,
|
||||
`Adopted ${moved} pre-multi-account row(s) into the connected account`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'warn',
|
||||
integrationId,
|
||||
`legacy row adoption skipped: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function launchRespider(sess: ProjectSession, urls: string[]): Promise<void> {
|
||||
if (urls.length === 0) return;
|
||||
const db = sess.getDb();
|
||||
@@ -5721,18 +6082,6 @@ function registerIpc(): void {
|
||||
await launchCrawl(respiderConfig, { respiderSeeds: urls, session: sess });
|
||||
}
|
||||
|
||||
/**
|
||||
* V2 Faz 1 — Resolve the per-project screenshots sidecar directory.
|
||||
* Lives alongside the `.seoproject` file as `<project>.screenshots/`
|
||||
* so it tracks with Save As / move operations.
|
||||
*/
|
||||
function resolveScreenshotDir(projectPath: string): string | null {
|
||||
if (!projectPath) return null;
|
||||
const dir = dirname(projectPath);
|
||||
const base = basename(projectPath);
|
||||
return join(dir, `${base}.screenshots`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable 16-char hex stem derived from the URL. Used as the screenshot
|
||||
* filename prefix so re-crawls overwrite the previous capture instead
|
||||
@@ -6203,9 +6552,13 @@ function registerIpc(): void {
|
||||
// batches with progress events + cancellation flags; those stay
|
||||
// desktop-only until a polling tool pattern is designed.
|
||||
|
||||
'gsc-list-sites': async () => {
|
||||
'gsc-list-sites': async (input) => {
|
||||
try {
|
||||
return { ok: true, error: null, sites: await listSites() };
|
||||
const account = resolveGoogleAccount(
|
||||
'gsc',
|
||||
(input as { accountId?: string } | undefined)?.accountId,
|
||||
);
|
||||
return { ok: true, error: null, sites: await listSites(account) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -6235,10 +6588,17 @@ function registerIpc(): void {
|
||||
const startDate = isoDate(start);
|
||||
const endDate = isoDate(end);
|
||||
const runDb = getDb();
|
||||
const accountId = resolveGoogleAccount('gsc', i.accountId);
|
||||
try {
|
||||
const rows = await querySearchAnalytics(i.property, startDate, endDate);
|
||||
const rows = await querySearchAnalytics(
|
||||
i.property,
|
||||
startDate,
|
||||
endDate,
|
||||
{},
|
||||
accountId,
|
||||
);
|
||||
const fetchedAt = new Date().toISOString();
|
||||
runDb.replaceGscResults(rows, fetchedAt);
|
||||
runDb.replaceGscResults(rows, fetchedAt, accountId);
|
||||
const meta = {
|
||||
property: i.property,
|
||||
startDate,
|
||||
@@ -6246,7 +6606,7 @@ function registerIpc(): void {
|
||||
fetchedAt,
|
||||
rowCount: rows.length,
|
||||
};
|
||||
runDb.setMeta('gscFetchMeta', JSON.stringify(meta));
|
||||
runDb.setMeta(googleMetaKey('gscFetchMeta', accountId), JSON.stringify(meta));
|
||||
fireDataChanged();
|
||||
return { ok: true, error: null, rowCount: rows.length, meta };
|
||||
} catch (err) {
|
||||
@@ -6259,9 +6619,17 @@ function registerIpc(): void {
|
||||
}
|
||||
},
|
||||
|
||||
'ga4-list-properties': async () => {
|
||||
'ga4-list-properties': async (input) => {
|
||||
try {
|
||||
return { ok: true, error: null, properties: await ga4ListProperties() };
|
||||
const account = resolveGoogleAccount(
|
||||
'ga4',
|
||||
(input as { accountId?: string } | undefined)?.accountId,
|
||||
);
|
||||
return {
|
||||
ok: true,
|
||||
error: null,
|
||||
properties: await ga4ListProperties(account),
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -6290,10 +6658,16 @@ function registerIpc(): void {
|
||||
const startDate = isoDate(start);
|
||||
const endDate = isoDate(end);
|
||||
const runDb = getDb();
|
||||
const accountId = resolveGoogleAccount('ga4', i.accountId);
|
||||
try {
|
||||
const rows = await ga4RunReport(i.property, startDate, endDate);
|
||||
const rows = await ga4RunReport(
|
||||
i.property,
|
||||
startDate,
|
||||
endDate,
|
||||
accountId,
|
||||
);
|
||||
const fetchedAt = new Date().toISOString();
|
||||
runDb.replaceGa4Results(rows, fetchedAt);
|
||||
runDb.replaceGa4Results(rows, fetchedAt, accountId);
|
||||
const meta = {
|
||||
property: i.property,
|
||||
propertyName: i.propertyName || i.property,
|
||||
@@ -6302,7 +6676,7 @@ function registerIpc(): void {
|
||||
fetchedAt,
|
||||
rowCount: rows.length,
|
||||
};
|
||||
runDb.setMeta('ga4FetchMeta', JSON.stringify(meta));
|
||||
runDb.setMeta(googleMetaKey('ga4FetchMeta', accountId), JSON.stringify(meta));
|
||||
fireDataChanged();
|
||||
return { ok: true, error: null, rowCount: rows.length, meta };
|
||||
} catch (err) {
|
||||
@@ -6673,7 +7047,7 @@ function registerIpc(): void {
|
||||
const sess = currentSession();
|
||||
const wasDirty = sess.dirty;
|
||||
const boundPath = sess.currentProjectPath;
|
||||
const result = await saveProjectDocument(sess, target);
|
||||
const result = await saveProjectDocument(sess, target, { rebind: false });
|
||||
// Restore the binding: this was a snapshot, not a Save As.
|
||||
sess.currentProjectPath = boundPath;
|
||||
sess.dirty = wasDirty;
|
||||
@@ -7712,27 +8086,53 @@ function registerIpc(): void {
|
||||
async (): Promise<CrashRecoveryStatus> => {
|
||||
// Read from the in-memory snapshot we captured before
|
||||
// `db.reset()` wiped the previous session's data.
|
||||
if (currentSession().pendingRecoveryCheckpoint.length === 0) {
|
||||
return { pendingCount: 0, seedUrl: '' };
|
||||
const sess = currentSession();
|
||||
if (sess.pendingRecoveryCheckpoint.length === 0) {
|
||||
return { pendingCount: 0, seedUrl: '', mode: 'spider' };
|
||||
}
|
||||
const mode = sess.lastCrawlConfig?.mode ?? 'spider';
|
||||
// Show what the crawl was actually pointed at. A List/Sitemap run
|
||||
// checkpoints the first URL of its list rather than its own seed
|
||||
// (see the resume handler), so quoting it back would name a random
|
||||
// content page as the "start URL" of a sitemap crawl.
|
||||
const seedUrl =
|
||||
mode === 'spider'
|
||||
? (sess.pendingRecoveryCheckpoint[0]?.seedUrl ?? '')
|
||||
: (sess.lastCrawlConfig?.startUrl ?? '');
|
||||
return {
|
||||
pendingCount: currentSession().pendingRecoveryCheckpoint.length,
|
||||
seedUrl: currentSession().pendingRecoveryCheckpoint[0]?.seedUrl ?? '',
|
||||
pendingCount: sess.pendingRecoveryCheckpoint.length,
|
||||
seedUrl,
|
||||
mode,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
registerHandle(
|
||||
IPC.crashRecoveryResume,
|
||||
async (): Promise<{ accepted: boolean }> => {
|
||||
async (): Promise<CrashRecoveryResumeResult> => {
|
||||
const sess = currentSession();
|
||||
if (sess.activeCrawler) return { accepted: false };
|
||||
if (sess.pendingRecoveryCheckpoint.length === 0) return { accepted: false };
|
||||
if (sess.activeCrawler) return { accepted: false, config: null };
|
||||
if (sess.pendingRecoveryCheckpoint.length === 0)
|
||||
return { accepted: false, config: null };
|
||||
const seedUrl = sess.pendingRecoveryCheckpoint[0]?.seedUrl ?? '';
|
||||
if (!seedUrl) return { accepted: false };
|
||||
const savedCfg = sess.lastCrawlConfig;
|
||||
const cfg = savedCfg ? { ...savedCfg, startUrl: seedUrl } : null;
|
||||
if (!cfg) return { accepted: false };
|
||||
if (!savedCfg) return { accepted: false, config: null };
|
||||
// Spider mode checkpoints the *resolved* start URL (post-redirect),
|
||||
// which is also what `project_meta.startUrl` holds — taking it from
|
||||
// the checkpoint is what stops the resume from reading as a different
|
||||
// site and wiping the project. List/Sitemap runs rewrite
|
||||
// `config.startUrl` to the first URL of their list before the first
|
||||
// checkpoint tick, so their seed is a content page, not a start URL:
|
||||
// resuming with it would hand a plain HTML page to the sitemap parser
|
||||
// and fail. Their saved config already carries the right value.
|
||||
const cfg =
|
||||
savedCfg.mode === 'spider' && seedUrl
|
||||
? { ...savedCfg, startUrl: seedUrl }
|
||||
: savedCfg;
|
||||
// List mode seeds from `urlList`, so only the other two need a URL.
|
||||
if (cfg.mode !== 'list' && !cfg.startUrl.trim()) {
|
||||
return { accepted: false, config: null };
|
||||
}
|
||||
const crawler = new Crawler(cfg, sess.getDb(), {
|
||||
setOp: (op: string) => freezeWatchdog.setMainOp(op),
|
||||
parseHtml: parseHtmlViaPool,
|
||||
@@ -7755,7 +8155,11 @@ function registerIpc(): void {
|
||||
sess.pendingRecoveryCheckpoint = [];
|
||||
crawler.enqueueCheckpointed(items);
|
||||
void crawler.start();
|
||||
return { accepted: true };
|
||||
// Hand the config back so the toolbar can adopt the crawl that is
|
||||
// now actually running. Without this the renderer keeps the blank
|
||||
// Spider defaults it booted with, and a Sitemap/List resume reads
|
||||
// as "it didn't continue where it left off".
|
||||
return { accepted: true, config: cfg };
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
type ExportGridResult,
|
||||
type DataDeleteByDomainInput,
|
||||
type DataDeleteByDomainResult,
|
||||
type CrashRecoveryResumeResult,
|
||||
type CrashRecoveryStatus,
|
||||
type ExportHtmlReportInput,
|
||||
type ExportHtmlReportResult,
|
||||
@@ -119,12 +120,14 @@ import {
|
||||
type SpellingMatchesResult,
|
||||
type SpellingLanguageOption,
|
||||
type GoogleAuthState,
|
||||
type GoogleAccount,
|
||||
type GoogleAuthResult,
|
||||
type GscListSitesResult,
|
||||
type GscFetchInput,
|
||||
type GscFetchResult,
|
||||
type GscQueryInput,
|
||||
type GscQueryResult,
|
||||
type GscCrawlNewUrlsResult,
|
||||
type Ga4ListPropertiesResult,
|
||||
type Ga4FetchInput,
|
||||
type Ga4FetchResult,
|
||||
@@ -265,7 +268,7 @@ const api: FreeCrawlApi = {
|
||||
ipcRenderer.invoke(IPC.dataDeleteByDomain, input),
|
||||
crashRecoveryStatus: (): Promise<CrashRecoveryStatus> =>
|
||||
ipcRenderer.invoke(IPC.crashRecoveryStatus),
|
||||
crashRecoveryResume: (): Promise<{ accepted: boolean }> =>
|
||||
crashRecoveryResume: (): Promise<CrashRecoveryResumeResult> =>
|
||||
ipcRenderer.invoke(IPC.crashRecoveryResume),
|
||||
crashRecoveryDiscard: (): Promise<void> =>
|
||||
ipcRenderer.invoke(IPC.crashRecoveryDiscard),
|
||||
@@ -398,6 +401,15 @@ const api: FreeCrawlApi = {
|
||||
): Promise<IntegrationsState> => ipcRenderer.invoke(IPC.integrationsSet, id, fields),
|
||||
integrationsClear: (id: string): Promise<IntegrationsState> =>
|
||||
ipcRenderer.invoke(IPC.integrationsClear, id),
|
||||
integrationSettingsGet: (
|
||||
id: string,
|
||||
): Promise<Record<string, unknown> | null> =>
|
||||
ipcRenderer.invoke(IPC.integrationSettingsGet, id),
|
||||
integrationSettingsSet: (
|
||||
id: string,
|
||||
settings: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> =>
|
||||
ipcRenderer.invoke(IPC.integrationSettingsSet, id, settings),
|
||||
pagespeedQuery: (input: PagespeedQueryInput): Promise<PagespeedQueryResult> =>
|
||||
ipcRenderer.invoke(IPC.pagespeedQuery, input),
|
||||
pagespeedRun: (input: PagespeedRunInput): Promise<PagespeedRunResult> =>
|
||||
@@ -426,16 +438,20 @@ const api: FreeCrawlApi = {
|
||||
ipcRenderer.invoke(IPC.googleAuthStart, id),
|
||||
googleAuthStatus: (id: string): Promise<GoogleAuthState> =>
|
||||
ipcRenderer.invoke(IPC.googleAuthStatus, id),
|
||||
googleAuthRevoke: (id: string): Promise<GoogleAuthState> =>
|
||||
ipcRenderer.invoke(IPC.googleAuthRevoke, id),
|
||||
gscListSites: (): Promise<GscListSitesResult> =>
|
||||
ipcRenderer.invoke(IPC.gscListSites),
|
||||
googleAuthRevoke: (id: string, accountId?: string): Promise<GoogleAuthState> =>
|
||||
ipcRenderer.invoke(IPC.googleAuthRevoke, id, accountId),
|
||||
gscListSites: (accountId?: string): Promise<GscListSitesResult> =>
|
||||
ipcRenderer.invoke(IPC.gscListSites, accountId),
|
||||
googleAccountsList: (integrationId: string): Promise<GoogleAccount[]> =>
|
||||
ipcRenderer.invoke(IPC.googleAccountsList, integrationId),
|
||||
gscFetch: (input: GscFetchInput): Promise<GscFetchResult> =>
|
||||
ipcRenderer.invoke(IPC.gscFetch, input),
|
||||
gscQuery: (input: GscQueryInput): Promise<GscQueryResult> =>
|
||||
ipcRenderer.invoke(IPC.gscQuery, input),
|
||||
ga4ListProperties: (): Promise<Ga4ListPropertiesResult> =>
|
||||
ipcRenderer.invoke(IPC.ga4ListProperties),
|
||||
gscCrawlNewUrls: (): Promise<GscCrawlNewUrlsResult> =>
|
||||
ipcRenderer.invoke(IPC.gscCrawlNewUrls),
|
||||
ga4ListProperties: (accountId?: string): Promise<Ga4ListPropertiesResult> =>
|
||||
ipcRenderer.invoke(IPC.ga4ListProperties, accountId),
|
||||
ga4Fetch: (input: Ga4FetchInput): Promise<Ga4FetchResult> =>
|
||||
ipcRenderer.invoke(IPC.ga4Fetch, input),
|
||||
ga4Query: (input: Ga4QueryInput): Promise<Ga4QueryResult> =>
|
||||
|
||||
@@ -185,13 +185,20 @@ export function App() {
|
||||
const proceed = window.confirm(
|
||||
t('app.crashRecovery', {
|
||||
defaultValue:
|
||||
"FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n Start URL: {{seed}}\n Pending URLs: {{pending}}\n\nResume the crawl from where it stopped?\n\nClick OK to resume, Cancel to discard the recovery state and start fresh.",
|
||||
"FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n Mode: {{mode}}\n Start URL: {{seed}}\n Pending URLs: {{pending}}\n\nResume the crawl from where it stopped?\n\nClick OK to resume, Cancel to discard the recovery state and start fresh.",
|
||||
mode: t(`topbar.mode.${status.mode}`, { defaultValue: status.mode }),
|
||||
seed: status.seedUrl,
|
||||
pending: status.pendingCount.toLocaleString(),
|
||||
}),
|
||||
);
|
||||
if (proceed) {
|
||||
void window.freecrawl.crashRecoveryResume();
|
||||
// Adopt the resumed run's config so the toolbar shows the mode
|
||||
// and start URL actually being crawled — the renderer boots
|
||||
// with blank Spider defaults, which made a Sitemap/List resume
|
||||
// look like it had started over.
|
||||
void window.freecrawl.crashRecoveryResume().then((res) => {
|
||||
if (res.accepted && res.config) applyProjectConfig(res.config);
|
||||
});
|
||||
} else {
|
||||
void window.freecrawl.crashRecoveryDiscard();
|
||||
}
|
||||
|
||||
@@ -85,17 +85,35 @@ export function LogAnalyzerView() {
|
||||
verifyBots,
|
||||
});
|
||||
if (res.overview) setOverview(res.overview);
|
||||
if (res.ok && res.imported) {
|
||||
const i = res.imported;
|
||||
showToast(
|
||||
t('logAnalyzer.imported', {
|
||||
defaultValue: '{{file}} — parsed {{parsed}}/{{total}} lines ({{format}})',
|
||||
file: i.fileName,
|
||||
parsed: i.parsedLines.toLocaleString(),
|
||||
total: i.totalLines.toLocaleString(),
|
||||
format: i.format,
|
||||
}),
|
||||
);
|
||||
const batch = res.batch ?? (res.imported ? [res.imported] : []);
|
||||
const failed = res.errors?.length ?? 0;
|
||||
if (res.ok && batch.length > 0) {
|
||||
if (batch.length === 1) {
|
||||
const i = batch[0]!;
|
||||
showToast(
|
||||
t('logAnalyzer.imported', {
|
||||
defaultValue: '{{file}} — parsed {{parsed}}/{{total}} lines ({{format}})',
|
||||
file: i.fileName,
|
||||
parsed: i.parsedLines.toLocaleString(),
|
||||
total: i.totalLines.toLocaleString(),
|
||||
format: i.format,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
const parsed = batch.reduce((s, b) => s + b.parsedLines, 0);
|
||||
const total = batch.reduce((s, b) => s + b.totalLines, 0);
|
||||
showToast(
|
||||
t('logAnalyzer.importedBatch', {
|
||||
defaultValue: '{{files}} files — parsed {{parsed}}/{{total}} lines',
|
||||
files: batch.length,
|
||||
parsed: parsed.toLocaleString(),
|
||||
total: total.toLocaleString(),
|
||||
}) +
|
||||
(failed > 0
|
||||
? ` · ${t('logAnalyzer.importedFailed', { defaultValue: '{{n}} failed', n: failed })}`
|
||||
: ''),
|
||||
);
|
||||
}
|
||||
setVersion((v) => v + 1);
|
||||
} else if (res.error) {
|
||||
showToast(t('logAnalyzer.importError', { defaultValue: 'Import failed: {{err}}', err: res.error }));
|
||||
@@ -184,7 +202,7 @@ export function LogAnalyzerView() {
|
||||
>
|
||||
{busy
|
||||
? t('logAnalyzer.analyzing', { defaultValue: 'Analyzing…' })
|
||||
: t('logAnalyzer.import', { defaultValue: 'Import Log File…' })}
|
||||
: t('logAnalyzer.import', { defaultValue: 'Import Log Files…' })}
|
||||
</button>
|
||||
{hasData && <ExportMenu onExport={onExport} />}
|
||||
{hasData && (
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import {
|
||||
defaultGa4Settings,
|
||||
type GoogleAccount,
|
||||
type Ga4IntegrationSettings,
|
||||
} from '@freecrawl/shared-types';
|
||||
|
||||
/**
|
||||
* Per-integration behaviour settings for Google Analytics 4, rendered
|
||||
* inside the GA4 card in Settings → Integrations. Smaller than the GSC
|
||||
* equivalent — the Data API call takes no dimension filters — but it
|
||||
* carries the same per-project account + date-range choice, so a project
|
||||
* remembers which of several linked GA4 accounts it reports on.
|
||||
*/
|
||||
export function Ga4SettingsSection() {
|
||||
const { t } = useTranslation();
|
||||
const [settings, setSettings] = useState<Ga4IntegrationSettings | null>(null);
|
||||
const [accounts, setAccounts] = useState<GoogleAccount[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void (async () => {
|
||||
const [stored, list] = await Promise.all([
|
||||
window.freecrawl.integrationSettingsGet('ga4'),
|
||||
window.freecrawl.googleAccountsList('ga4'),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
setAccounts(list);
|
||||
setSettings({
|
||||
...defaultGa4Settings(),
|
||||
...(stored ?? {}),
|
||||
} as Ga4IntegrationSettings);
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (!settings) return null;
|
||||
|
||||
const update = <K extends keyof Ga4IntegrationSettings>(
|
||||
key: K,
|
||||
value: Ga4IntegrationSettings[K],
|
||||
) => {
|
||||
setSettings((prev) => (prev ? { ...prev, [key]: value } : prev));
|
||||
void window.freecrawl.integrationSettingsSet('ga4', {
|
||||
[key]: value,
|
||||
} as Record<string, unknown>);
|
||||
};
|
||||
|
||||
const selectCls =
|
||||
'h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none';
|
||||
|
||||
return (
|
||||
<div className="mt-3 rounded border border-surface-800 bg-surface-950/60 p-2.5">
|
||||
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
|
||||
{t('ga4Settings.title', { defaultValue: 'Analytics 4 Settings' })}
|
||||
</div>
|
||||
|
||||
{accounts.length > 1 && (
|
||||
<div className="mb-3">
|
||||
<div className="mb-1 text-[10px] font-medium text-surface-300">
|
||||
{t('ga4Settings.account', { defaultValue: 'Account' })}
|
||||
</div>
|
||||
<select
|
||||
className={`${selectCls} max-w-full`}
|
||||
value={settings.accountId || accounts[0]?.accountId || ''}
|
||||
onChange={(e) => update('accountId', e.target.value)}
|
||||
title={t('ga4Settings.accountHint', {
|
||||
defaultValue:
|
||||
'Which linked Google account this project pulls Analytics data with.',
|
||||
})}
|
||||
>
|
||||
{accounts.map((a) => (
|
||||
<option key={a.accountId} value={a.accountId}>
|
||||
{a.email ?? a.accountId}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<div className="mb-1 text-[10px] font-medium text-surface-300">
|
||||
{t('ga4Settings.dateRange', { defaultValue: 'Date Range' })}
|
||||
</div>
|
||||
<select
|
||||
className={selectCls}
|
||||
value={settings.days}
|
||||
onChange={(e) =>
|
||||
update('days', Number(e.target.value) as Ga4IntegrationSettings['days'])
|
||||
}
|
||||
>
|
||||
<option value={7}>
|
||||
{t('ga4Settings.range7', { defaultValue: 'Last 7 days' })}
|
||||
</option>
|
||||
<option value={28}>
|
||||
{t('ga4Settings.range28', { defaultValue: 'Last 28 days' })}
|
||||
</option>
|
||||
<option value={90}>
|
||||
{t('ga4Settings.range90', { defaultValue: 'Last 90 days' })}
|
||||
</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<p className="mt-2 text-[10px] leading-relaxed text-surface-500">
|
||||
{t('ga4Settings.hint', {
|
||||
defaultValue:
|
||||
'The property is picked in the GA4 tab toolbar. Analytics data is near-realtime, so the range ends today.',
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Loader2, Plus, Trash2, Check } from 'lucide-react';
|
||||
import type { GoogleAccount } from '@freecrawl/shared-types';
|
||||
|
||||
/**
|
||||
* Linked-account manager for an OAuth-BYOC Google integration (Search
|
||||
* Console / Analytics 4 / Sheets).
|
||||
*
|
||||
* A user may connect several Google accounts to the same integration —
|
||||
* their own property plus a client's, say — so this lists every linked
|
||||
* account, lets them add another, and disconnects them one at a time.
|
||||
* Which account a project actually reports on is chosen per project in
|
||||
* the tab toolbar; this panel only manages the links themselves.
|
||||
*
|
||||
* @param integrationId `gsc` / `ga4` / `sheets`.
|
||||
* @param configured False until the BYOC client ID + secret are saved —
|
||||
* consent can't start without them, so the connect button is disabled.
|
||||
* @param onChange Notifies the parent so a settings section showing an
|
||||
* account dropdown can refresh.
|
||||
*/
|
||||
export function GoogleAccountsSection({
|
||||
integrationId,
|
||||
configured,
|
||||
onChange,
|
||||
}: {
|
||||
integrationId: string;
|
||||
configured: boolean;
|
||||
onChange?: (accounts: GoogleAccount[]) => void;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const [accounts, setAccounts] = useState<GoogleAccount[] | null>(null);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
const list = await window.freecrawl.googleAccountsList(integrationId);
|
||||
setAccounts(list);
|
||||
onChange?.(list);
|
||||
// `onChange` is a render-scoped callback in practice; depending on it
|
||||
// would re-run this on every parent render.
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [integrationId]);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
const connect = async () => {
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
const res = await window.freecrawl.googleAuthStart(integrationId);
|
||||
if (!res.ok && res.error) setError(res.error);
|
||||
await load();
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const disconnect = async (account: GoogleAccount) => {
|
||||
const label = account.email ?? account.accountId;
|
||||
if (
|
||||
!window.confirm(
|
||||
t('googleAccounts.confirmDisconnect', {
|
||||
defaultValue: 'Disconnect {{account}}? Data already pulled with it stays until the next fetch.',
|
||||
account: label,
|
||||
}),
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
await window.freecrawl.googleAuthRevoke(integrationId, account.accountId);
|
||||
await load();
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (!accounts) return null;
|
||||
|
||||
return (
|
||||
<div className="mt-3 rounded border border-surface-800 bg-surface-950/60 p-2.5">
|
||||
<div className="mb-2 flex items-center gap-2">
|
||||
<span className="text-[10px] font-semibold uppercase tracking-wider text-surface-400">
|
||||
{t('googleAccounts.title', { defaultValue: 'Connected Accounts' })}
|
||||
</span>
|
||||
<span className="rounded bg-surface-800 px-1.5 py-0.5 text-[9px] tabular-nums text-surface-400">
|
||||
{accounts.length}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void connect()}
|
||||
disabled={busy || !configured}
|
||||
className="ml-auto inline-flex items-center gap-1 rounded bg-blue-600 px-2 py-0.5 text-[10px] font-medium text-white hover:bg-blue-500 disabled:cursor-not-allowed disabled:bg-surface-700 disabled:text-surface-500"
|
||||
title={
|
||||
configured
|
||||
? undefined
|
||||
: t('googleAccounts.needClient', {
|
||||
defaultValue: 'Save your OAuth client ID + secret first.',
|
||||
})
|
||||
}
|
||||
>
|
||||
{busy ? <Loader2 size={11} className="animate-spin" /> : <Plus size={11} />}
|
||||
{accounts.length === 0
|
||||
? t('googleAccounts.connect', { defaultValue: 'Connect account' })
|
||||
: t('googleAccounts.connectAnother', { defaultValue: 'Connect another' })}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{accounts.length === 0 ? (
|
||||
<p className="text-[10px] leading-relaxed text-surface-500">
|
||||
{t('googleAccounts.empty', {
|
||||
defaultValue:
|
||||
'No account linked yet. Connect one — you can link several and pick which one each project uses.',
|
||||
})}
|
||||
</p>
|
||||
) : (
|
||||
<ul className="space-y-1">
|
||||
{accounts.map((a) => (
|
||||
<li
|
||||
key={a.accountId}
|
||||
className="flex items-center gap-2 rounded border border-surface-800 bg-surface-900/40 px-2 py-1"
|
||||
>
|
||||
<Check size={11} className="shrink-0 text-emerald-400" />
|
||||
<span className="truncate text-[11px] text-surface-200" title={a.email ?? a.accountId}>
|
||||
{a.email ?? t('googleAccounts.unknownEmail', { defaultValue: 'Google account' })}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void disconnect(a)}
|
||||
disabled={busy}
|
||||
className="ml-auto inline-flex items-center gap-1 rounded border border-red-700/60 px-1.5 py-0.5 text-[10px] text-red-300 hover:bg-red-900/20 disabled:opacity-50"
|
||||
title={t('googleAccounts.disconnect', { defaultValue: 'Disconnect' })}
|
||||
>
|
||||
<Trash2 size={10} />
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
|
||||
{error && <div className="mt-1.5 text-[10px] text-red-400">{error}</div>}
|
||||
<p className="mt-2 text-[10px] leading-relaxed text-surface-500">
|
||||
{t('googleAccounts.hint', {
|
||||
defaultValue:
|
||||
'Each project picks which linked account it reports on, in the tab toolbar. Data pulled with different accounts is stored separately.',
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import {
|
||||
defaultGscSettings,
|
||||
type GoogleAccount,
|
||||
type GscIntegrationSettings,
|
||||
} from '@freecrawl/shared-types';
|
||||
|
||||
/**
|
||||
* Per-integration behaviour settings for Google Search Console — the
|
||||
* FreeCrawl equivalent of Screaming Frog's "API Access → Google Search
|
||||
* Console" dialog. Rendered inside the GSC card in Settings →
|
||||
* Integrations. Stored per project via `integrationSettingsSet('gsc')`;
|
||||
* the Search Console tab's Fetch button reads these to shape the pull.
|
||||
*/
|
||||
export function GscSettingsSection() {
|
||||
const { t } = useTranslation();
|
||||
const [settings, setSettings] = useState<GscIntegrationSettings | null>(null);
|
||||
const [accounts, setAccounts] = useState<GoogleAccount[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void (async () => {
|
||||
const [stored, list] = await Promise.all([
|
||||
window.freecrawl.integrationSettingsGet('gsc'),
|
||||
window.freecrawl.googleAccountsList('gsc'),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
setAccounts(list);
|
||||
setSettings({
|
||||
...defaultGscSettings(),
|
||||
...(stored ?? {}),
|
||||
} as GscIntegrationSettings);
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (!settings) return null;
|
||||
|
||||
const update = <K extends keyof GscIntegrationSettings>(
|
||||
key: K,
|
||||
value: GscIntegrationSettings[K],
|
||||
) => {
|
||||
setSettings((prev) => (prev ? { ...prev, [key]: value } : prev));
|
||||
void window.freecrawl.integrationSettingsSet('gsc', {
|
||||
[key]: value,
|
||||
} as Record<string, unknown>);
|
||||
};
|
||||
|
||||
const selectCls =
|
||||
'h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none';
|
||||
const inputCls =
|
||||
'h-6 rounded border border-surface-700 bg-surface-950 px-2 text-[11px] text-surface-100 placeholder-surface-600 focus:border-blue-500 focus:outline-none';
|
||||
|
||||
return (
|
||||
<div className="mt-3 rounded border border-surface-800 bg-surface-950/60 p-2.5">
|
||||
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
|
||||
{t('gscSettings.title', { defaultValue: 'Search Console Settings' })}
|
||||
</div>
|
||||
|
||||
{/* ── Account ────────────────────────────────────────────── */}
|
||||
{accounts.length > 1 && (
|
||||
<div className="mb-3">
|
||||
<div className="mb-1 text-[10px] font-medium text-surface-300">
|
||||
{t('gscSettings.account', { defaultValue: 'Account' })}
|
||||
</div>
|
||||
<select
|
||||
className={`${selectCls} max-w-full`}
|
||||
value={settings.accountId || accounts[0]?.accountId || ''}
|
||||
onChange={(e) => update('accountId', e.target.value)}
|
||||
title={t('gscSettings.accountHint', {
|
||||
defaultValue:
|
||||
'Which linked Google account this project pulls Search Console data with.',
|
||||
})}
|
||||
>
|
||||
{accounts.map((a) => (
|
||||
<option key={a.accountId} value={a.accountId}>
|
||||
{a.email ?? a.accountId}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ── Date Range ─────────────────────────────────────────── */}
|
||||
<div className="mb-3">
|
||||
<div className="mb-1 text-[10px] font-medium text-surface-300">
|
||||
{t('gscSettings.dateRange', { defaultValue: 'Date Range' })}
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<select
|
||||
className={selectCls}
|
||||
value={settings.dateRange}
|
||||
onChange={(e) =>
|
||||
update(
|
||||
'dateRange',
|
||||
e.target.value as GscIntegrationSettings['dateRange'],
|
||||
)
|
||||
}
|
||||
>
|
||||
<option value="7d">
|
||||
{t('gscSettings.range7', { defaultValue: 'Last 7 days' })}
|
||||
</option>
|
||||
<option value="28d">
|
||||
{t('gscSettings.range28', { defaultValue: 'Last 28 days' })}
|
||||
</option>
|
||||
<option value="90d">
|
||||
{t('gscSettings.range90', { defaultValue: 'Last 90 days' })}
|
||||
</option>
|
||||
<option value="16m">
|
||||
{t('gscSettings.range16m', { defaultValue: 'Last 16 months' })}
|
||||
</option>
|
||||
<option value="custom">
|
||||
{t('gscSettings.rangeCustom', { defaultValue: 'Custom' })}
|
||||
</option>
|
||||
</select>
|
||||
{settings.dateRange === 'custom' && (
|
||||
<>
|
||||
<input
|
||||
type="date"
|
||||
className={inputCls}
|
||||
value={settings.startDate}
|
||||
onChange={(e) => update('startDate', e.target.value)}
|
||||
title={t('gscSettings.startDate', { defaultValue: 'Start date' })}
|
||||
/>
|
||||
<span className="text-[11px] text-surface-500">→</span>
|
||||
<input
|
||||
type="date"
|
||||
className={inputCls}
|
||||
value={settings.endDate}
|
||||
onChange={(e) => update('endDate', e.target.value)}
|
||||
title={t('gscSettings.endDate', { defaultValue: 'End date' })}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* ── Dimension Filter ───────────────────────────────────── */}
|
||||
<div className="mb-3">
|
||||
<div className="mb-1 text-[10px] font-medium text-surface-300">
|
||||
{t('gscSettings.dimensionFilter', { defaultValue: 'Dimension Filter' })}
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<label className="flex flex-col gap-0.5">
|
||||
<span className="text-[10px] text-surface-500">
|
||||
{t('gscSettings.device', { defaultValue: 'Device' })}
|
||||
</span>
|
||||
<select
|
||||
className={selectCls}
|
||||
value={settings.deviceFilter}
|
||||
onChange={(e) =>
|
||||
update(
|
||||
'deviceFilter',
|
||||
e.target.value as GscIntegrationSettings['deviceFilter'],
|
||||
)
|
||||
}
|
||||
>
|
||||
<option value="all">
|
||||
{t('gscSettings.deviceAll', { defaultValue: 'All' })}
|
||||
</option>
|
||||
<option value="DESKTOP">
|
||||
{t('gscSettings.deviceDesktop', { defaultValue: 'Desktop' })}
|
||||
</option>
|
||||
<option value="MOBILE">
|
||||
{t('gscSettings.deviceMobile', { defaultValue: 'Mobile' })}
|
||||
</option>
|
||||
<option value="TABLET">
|
||||
{t('gscSettings.deviceTablet', { defaultValue: 'Tablet' })}
|
||||
</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="flex flex-col gap-0.5">
|
||||
<span className="text-[10px] text-surface-500">
|
||||
{t('gscSettings.type', { defaultValue: 'Search Type' })}
|
||||
</span>
|
||||
<select
|
||||
className={selectCls}
|
||||
value={settings.searchType}
|
||||
onChange={(e) =>
|
||||
update(
|
||||
'searchType',
|
||||
e.target.value as GscIntegrationSettings['searchType'],
|
||||
)
|
||||
}
|
||||
>
|
||||
<option value="web">
|
||||
{t('gscSettings.typeWeb', { defaultValue: 'Web' })}
|
||||
</option>
|
||||
<option value="image">
|
||||
{t('gscSettings.typeImage', { defaultValue: 'Image' })}
|
||||
</option>
|
||||
<option value="video">
|
||||
{t('gscSettings.typeVideo', { defaultValue: 'Video' })}
|
||||
</option>
|
||||
<option value="news">
|
||||
{t('gscSettings.typeNews', { defaultValue: 'News' })}
|
||||
</option>
|
||||
<option value="discover">
|
||||
{t('gscSettings.typeDiscover', { defaultValue: 'Discover' })}
|
||||
</option>
|
||||
<option value="googleNews">
|
||||
{t('gscSettings.typeGoogleNews', { defaultValue: 'Google News' })}
|
||||
</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="flex flex-col gap-0.5">
|
||||
<span className="text-[10px] text-surface-500">
|
||||
{t('gscSettings.country', { defaultValue: 'Country' })}
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
className={inputCls}
|
||||
value={settings.countryFilter}
|
||||
placeholder={t('gscSettings.countryPlaceholder', {
|
||||
defaultValue: 'e.g. usa (ISO code) — blank for all',
|
||||
})}
|
||||
onChange={(e) =>
|
||||
update('countryFilter', e.target.value.trim().toLowerCase())
|
||||
}
|
||||
spellCheck={false}
|
||||
/>
|
||||
</label>
|
||||
<label className="flex flex-col gap-0.5">
|
||||
<span className="text-[10px] text-surface-500">
|
||||
{t('gscSettings.queryFilter', { defaultValue: 'Search Query' })}
|
||||
</span>
|
||||
<div className="flex gap-1">
|
||||
<select
|
||||
className={selectCls}
|
||||
value={settings.queryFilterMode}
|
||||
onChange={(e) =>
|
||||
update(
|
||||
'queryFilterMode',
|
||||
e.target.value as GscIntegrationSettings['queryFilterMode'],
|
||||
)
|
||||
}
|
||||
>
|
||||
<option value="none">
|
||||
{t('gscSettings.queryNone', { defaultValue: 'None' })}
|
||||
</option>
|
||||
<option value="contains">
|
||||
{t('gscSettings.queryContains', { defaultValue: 'Contains' })}
|
||||
</option>
|
||||
<option value="notContains">
|
||||
{t('gscSettings.queryNotContains', {
|
||||
defaultValue: "Doesn't contain",
|
||||
})}
|
||||
</option>
|
||||
<option value="equals">
|
||||
{t('gscSettings.queryEquals', { defaultValue: 'Equals' })}
|
||||
</option>
|
||||
<option value="notEquals">
|
||||
{t('gscSettings.queryNotEquals', {
|
||||
defaultValue: "Doesn't equal",
|
||||
})}
|
||||
</option>
|
||||
<option value="includingRegex">
|
||||
{t('gscSettings.queryRegex', { defaultValue: 'Matches regex' })}
|
||||
</option>
|
||||
<option value="excludingRegex">
|
||||
{t('gscSettings.queryNotRegex', {
|
||||
defaultValue: "Doesn't match regex",
|
||||
})}
|
||||
</option>
|
||||
</select>
|
||||
<input
|
||||
type="text"
|
||||
className={`${inputCls} min-w-0 flex-1`}
|
||||
value={settings.queryFilterValue}
|
||||
disabled={settings.queryFilterMode === 'none'}
|
||||
placeholder={t('gscSettings.queryValuePlaceholder', {
|
||||
defaultValue: 'Query…',
|
||||
})}
|
||||
onChange={(e) => update('queryFilterValue', e.target.value)}
|
||||
spellCheck={false}
|
||||
/>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* ── General ────────────────────────────────────────────── */}
|
||||
<div>
|
||||
<div className="mb-1 text-[10px] font-medium text-surface-300">
|
||||
{t('gscSettings.general', { defaultValue: 'General' })}
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
<label className="flex items-center gap-2 text-[11px] text-surface-200">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="accent-blue-500"
|
||||
checked={settings.matchSlash}
|
||||
onChange={(e) => update('matchSlash', e.target.checked)}
|
||||
/>
|
||||
{t('gscSettings.matchSlash', {
|
||||
defaultValue: 'Match Trailing and Non-Trailing Slash URLs',
|
||||
})}
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-[11px] text-surface-200">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="accent-blue-500"
|
||||
checked={settings.matchCase}
|
||||
onChange={(e) => update('matchCase', e.target.checked)}
|
||||
/>
|
||||
{t('gscSettings.matchCase', {
|
||||
defaultValue: 'Match Uppercase & Lowercase URLs',
|
||||
})}
|
||||
</label>
|
||||
<div className="flex items-center gap-2">
|
||||
<label className="flex items-center gap-2 text-[11px] text-surface-200">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="accent-blue-500"
|
||||
checked={settings.limitMaxResults}
|
||||
onChange={(e) => update('limitMaxResults', e.target.checked)}
|
||||
/>
|
||||
{t('gscSettings.limitMaxResults', {
|
||||
defaultValue: 'Limit Max Results',
|
||||
})}
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
step={1000}
|
||||
className={`${inputCls} w-24`}
|
||||
value={settings.maxResults}
|
||||
disabled={!settings.limitMaxResults}
|
||||
onChange={(e) =>
|
||||
update(
|
||||
'maxResults',
|
||||
Math.max(1, Math.floor(Number(e.target.value) || 0)),
|
||||
)
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
<label className="flex items-center gap-2 text-[11px] text-surface-200">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="accent-blue-500"
|
||||
checked={settings.crawlNewUrls}
|
||||
onChange={(e) => update('crawlNewUrls', e.target.checked)}
|
||||
/>
|
||||
{t('gscSettings.crawlNewUrls', {
|
||||
defaultValue: 'Crawl New URLs Discovered In Google Search Console',
|
||||
})}
|
||||
</label>
|
||||
<p className="pl-6 text-[10px] leading-relaxed text-surface-500">
|
||||
{t('gscSettings.crawlNewUrlsHint', {
|
||||
defaultValue:
|
||||
'After each Search Console pull, URLs that appear in GSC but were not found by the crawl are added to the crawl queue.',
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useEffect, useMemo, useRef, useState, type ReactNode, type RefObject } from 'react';
|
||||
import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react';
|
||||
import { useTranslation, Trans } from 'react-i18next';
|
||||
import {
|
||||
SUPPORTED_LANGUAGES,
|
||||
@@ -42,6 +42,8 @@ import {
|
||||
Target,
|
||||
Check,
|
||||
Workflow,
|
||||
ChevronRight,
|
||||
ChevronDown,
|
||||
type LucideIcon,
|
||||
} from 'lucide-react';
|
||||
import clsx from 'clsx';
|
||||
@@ -62,6 +64,9 @@ import { useAppStore } from '../store.js';
|
||||
import { InfoTip, type FieldInfo } from './InfoTip.js';
|
||||
import { ExtractionPreviewDialog } from './ExtractionPreviewDialog.js';
|
||||
import { IntegrationSetupGuideModal } from './IntegrationSetupGuideModal.js';
|
||||
import { GscSettingsSection } from './GscSettingsSection.js';
|
||||
import { Ga4SettingsSection } from './Ga4SettingsSection.js';
|
||||
import { GoogleAccountsSection } from './GoogleAccountsSection.js';
|
||||
|
||||
interface Props {
|
||||
open: boolean;
|
||||
@@ -252,19 +257,53 @@ export type SettingsSectionKey =
|
||||
| 'storage'
|
||||
| 'spelling'
|
||||
| 'privacy'
|
||||
| 'language';
|
||||
| 'language'
|
||||
/** Per-integration sub-page, e.g. `integration:gsc`. Each integration
|
||||
* gets its own page under the "Integrations" group header, so a
|
||||
* provider's credentials and its behaviour settings are never mixed
|
||||
* in with a dozen other providers on one scrolling page. */
|
||||
| `integration:${string}`;
|
||||
|
||||
/** Local alias — the exported name is the one other modules import. */
|
||||
type SectionKey = SettingsSectionKey;
|
||||
|
||||
/** Sidebar key prefix for per-integration sub-pages. */
|
||||
const INTEGRATION_KEY_PREFIX = 'integration:';
|
||||
|
||||
/** Build the sidebar key for one integration's sub-page. */
|
||||
function integrationSectionKey(id: string): SectionKey {
|
||||
return `${INTEGRATION_KEY_PREFIX}${id}` as SectionKey;
|
||||
}
|
||||
|
||||
/** Extract the integration id from a sub-page key (null when not one). */
|
||||
function integrationIdFromSection(key: SectionKey): string | null {
|
||||
return key.startsWith(INTEGRATION_KEY_PREFIX)
|
||||
? key.slice(INTEGRATION_KEY_PREFIX.length)
|
||||
: null;
|
||||
}
|
||||
|
||||
interface SectionDef {
|
||||
key: SectionKey;
|
||||
label: string;
|
||||
icon: LucideIcon;
|
||||
/** Searchable keywords beyond the label. */
|
||||
keywords: string;
|
||||
/** A collapsible header rather than a page — clicking it expands its
|
||||
* children instead of selecting content. */
|
||||
group?: true;
|
||||
/** Parent group key, for entries nested under a header. */
|
||||
parent?: SectionKey;
|
||||
}
|
||||
|
||||
/** Sidebar icon per integration category — keeps the sub-page list
|
||||
* scannable without inventing a bespoke icon for every provider. */
|
||||
const INTEGRATION_CATEGORY_ICON: Record<IntegrationDef['category'], LucideIcon> = {
|
||||
ai: Sparkles,
|
||||
performance: Gauge,
|
||||
seo: Target,
|
||||
google: Chrome,
|
||||
};
|
||||
|
||||
const SECTIONS: SectionDef[] = [
|
||||
{
|
||||
key: 'presets',
|
||||
@@ -405,7 +444,19 @@ const SECTIONS: SectionDef[] = [
|
||||
icon: Plug,
|
||||
keywords:
|
||||
'integrations api key oauth google search console analytics gsc ga4 pagespeed ahrefs moz semrush majestic openai anthropic claude ollama sheets bigquery credentials',
|
||||
group: true,
|
||||
},
|
||||
// One sub-page per integration, generated from the shared catalog so a
|
||||
// new provider shows up in the sidebar without touching this file.
|
||||
...INTEGRATIONS.map(
|
||||
(def): SectionDef => ({
|
||||
key: integrationSectionKey(def.id),
|
||||
label: def.name,
|
||||
icon: INTEGRATION_CATEGORY_ICON[def.category],
|
||||
keywords: `integration ${def.id} ${def.name} ${def.category} ${def.authType} ${def.description}`,
|
||||
parent: 'integrations',
|
||||
}),
|
||||
),
|
||||
{
|
||||
key: 'rendering',
|
||||
label: 'Rendering',
|
||||
@@ -626,6 +677,10 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
const [form, setForm] = useState<FormState>(() => configToForm(config));
|
||||
const [active, setActive] = useState<SectionKey>('mode');
|
||||
const [search, setSearch] = useState('');
|
||||
/** Collapsible sidebar groups the user has opened. */
|
||||
const [expandedGroups, setExpandedGroups] = useState<Set<SectionKey>>(
|
||||
() => new Set(),
|
||||
);
|
||||
|
||||
// Re-seed the form whenever the dialog reopens — picks up any external
|
||||
// config change (e.g. URL/scope edits in the top bar) so the dialog
|
||||
@@ -638,11 +693,33 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
}, [open, config]);
|
||||
|
||||
// Follow a deep link ("Add API Key…" and friends). Without a target the
|
||||
// dialog stays on whichever section the user last used.
|
||||
// dialog stays on whichever section the user last used. A target naming
|
||||
// an integration lands on that provider's own sub-page (and opens the
|
||||
// group so the selection is visible in the sidebar); a bare
|
||||
// `integrations` target falls back to the first provider, since the
|
||||
// group header itself is no longer a page.
|
||||
useEffect(() => {
|
||||
if (open && target) setActive(target.section);
|
||||
if (!open || !target) return;
|
||||
if (target.section === 'integrations') {
|
||||
const id = target.integration ?? INTEGRATIONS[0]?.id;
|
||||
if (id) {
|
||||
setActive(integrationSectionKey(id));
|
||||
setExpandedGroups((prev) => new Set(prev).add('integrations'));
|
||||
}
|
||||
return;
|
||||
}
|
||||
setActive(target.section);
|
||||
}, [open, target]);
|
||||
|
||||
// Selecting a nested page opens its group so the highlighted row is
|
||||
// actually visible in the sidebar. Applied as a state change (not a
|
||||
// derived override) so the header stays collapsible afterwards.
|
||||
useEffect(() => {
|
||||
const parent = SECTIONS.find((s) => s.key === active)?.parent;
|
||||
if (!parent) return;
|
||||
setExpandedGroups((prev) => (prev.has(parent) ? prev : new Set(prev).add(parent)));
|
||||
}, [active]);
|
||||
|
||||
// ESC closes — common modal expectation.
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
@@ -656,17 +733,38 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
const visibleSections = useMemo(() => {
|
||||
const q = search.trim().toLowerCase();
|
||||
if (!q) return SECTIONS;
|
||||
return SECTIONS.filter(
|
||||
(s) =>
|
||||
s.label.toLowerCase().includes(q) || s.keywords.toLowerCase().includes(q),
|
||||
);
|
||||
const matches = (s: SectionDef): boolean =>
|
||||
s.label.toLowerCase().includes(q) || s.keywords.toLowerCase().includes(q);
|
||||
// Keep parent/child pairs intact. A matching child always brings its
|
||||
// header along, so no row is orphaned. A matching group expands to all
|
||||
// its children only when no child matched on its own — otherwise
|
||||
// searching "gsc" would list every provider, since the group's own
|
||||
// keywords name them all.
|
||||
const keep = new Set<SectionKey>();
|
||||
for (const s of SECTIONS) {
|
||||
if (matches(s) && !s.group) {
|
||||
keep.add(s.key);
|
||||
if (s.parent) keep.add(s.parent);
|
||||
}
|
||||
}
|
||||
for (const s of SECTIONS) {
|
||||
if (!s.group || !matches(s)) continue;
|
||||
keep.add(s.key);
|
||||
const children = SECTIONS.filter((c) => c.parent === s.key);
|
||||
if (!children.some((c) => keep.has(c.key))) {
|
||||
for (const c of children) keep.add(c.key);
|
||||
}
|
||||
}
|
||||
return SECTIONS.filter((s) => keep.has(s.key));
|
||||
}, [search]);
|
||||
|
||||
// If the search filter hides the active section, jump to the first visible.
|
||||
// If the search filter hides the active section, jump to the first visible
|
||||
// page — group headers are skipped since they render no content.
|
||||
useEffect(() => {
|
||||
if (visibleSections.length === 0) return;
|
||||
if (!visibleSections.some((s) => s.key === active)) {
|
||||
setActive(visibleSections[0]!.key);
|
||||
const pages = visibleSections.filter((s) => !s.group);
|
||||
if (pages.length === 0) return;
|
||||
if (!pages.some((s) => s.key === active)) {
|
||||
setActive(pages[0]!.key);
|
||||
}
|
||||
}, [visibleSections, active]);
|
||||
|
||||
@@ -897,6 +995,28 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
}
|
||||
|
||||
const activeDef = SECTIONS.find((s) => s.key === active) ?? SECTIONS[0]!;
|
||||
const parentDef = activeDef.parent
|
||||
? SECTIONS.find((s) => s.key === activeDef.parent)
|
||||
: undefined;
|
||||
const searching = search.trim().length > 0;
|
||||
|
||||
/** A group renders open when the user opened it, or while a search is
|
||||
* narrowing the list (so matches are never hidden behind a collapsed
|
||||
* header). Selecting a child auto-opens its group once, via the effect
|
||||
* above — deriving openness from the selection instead would make the
|
||||
* header un-collapsible while one of its pages is active. */
|
||||
function isGroupOpen(key: SectionKey): boolean {
|
||||
return searching || expandedGroups.has(key);
|
||||
}
|
||||
|
||||
function toggleGroup(key: SectionKey) {
|
||||
setExpandedGroups((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(key)) next.delete(key);
|
||||
else next.add(key);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -943,20 +1063,46 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
)}
|
||||
{visibleSections.map((s) => {
|
||||
const Icon = s.icon;
|
||||
|
||||
// Group header — expands/collapses its children instead of
|
||||
// selecting a page of its own.
|
||||
if (s.group) {
|
||||
const open = isGroupOpen(s.key);
|
||||
const Chevron = open ? ChevronDown : ChevronRight;
|
||||
return (
|
||||
<button
|
||||
key={s.key}
|
||||
className="flex w-full items-center gap-2 border-l-2 border-transparent px-3 py-1.5 text-left text-[12px] text-surface-300 transition-colors hover:bg-surface-800 hover:text-surface-100"
|
||||
onClick={() => toggleGroup(s.key)}
|
||||
aria-expanded={open}
|
||||
>
|
||||
<Icon className="h-3.5 w-3.5" />
|
||||
<span>{translateLabel(s.label, lang)}</span>
|
||||
<Chevron className="ml-auto h-3.5 w-3.5 text-surface-500" />
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
// Child page — only rendered while its group is open.
|
||||
if (s.parent && !isGroupOpen(s.parent)) return null;
|
||||
|
||||
const isActive = s.key === active;
|
||||
return (
|
||||
<button
|
||||
key={s.key}
|
||||
className={clsx(
|
||||
'flex w-full items-center gap-2 px-3 py-1.5 text-left text-[12px] transition-colors',
|
||||
'flex w-full items-center gap-2 py-1.5 text-left text-[12px] transition-colors',
|
||||
s.parent ? 'pl-8 pr-3' : 'px-3',
|
||||
isActive
|
||||
? 'bg-accent-600/20 text-accent-200 border-l-2 border-accent-500'
|
||||
: 'border-l-2 border-transparent text-surface-300 hover:bg-surface-800 hover:text-surface-100',
|
||||
)}
|
||||
onClick={() => setActive(s.key)}
|
||||
// Long provider names truncate in the narrow sidebar.
|
||||
title={translateLabel(s.label, lang)}
|
||||
>
|
||||
<Icon className="h-3.5 w-3.5" />
|
||||
<span>{translateLabel(s.label, lang)}</span>
|
||||
<Icon className="h-3.5 w-3.5 shrink-0" />
|
||||
<span className="truncate">{translateLabel(s.label, lang)}</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
@@ -968,6 +1114,12 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
<div className="border-b border-surface-800 px-5 py-2 text-[11px] text-surface-400">
|
||||
{t('settings.title', { defaultValue: 'Settings' })}{' '}
|
||||
<span className="mx-1 text-surface-600">›</span>
|
||||
{parentDef && (
|
||||
<>
|
||||
<span>{translateLabel(parentDef.label, lang)}</span>
|
||||
<span className="mx-1 text-surface-600">›</span>
|
||||
</>
|
||||
)}
|
||||
<span className="text-surface-200">{translateLabel(activeDef.label, lang)}</span>
|
||||
</div>
|
||||
<div className="flex-1 overflow-auto px-5 py-4 text-[12px]">
|
||||
@@ -1056,8 +1208,8 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
{active === 'per-host-ua' && (
|
||||
<PerHostUaPanel form={form} update={update} />
|
||||
)}
|
||||
{active === 'integrations' && (
|
||||
<IntegrationsPanel focusId={target?.integration ?? null} />
|
||||
{integrationIdFromSection(active) && (
|
||||
<IntegrationPage id={integrationIdFromSection(active)!} />
|
||||
)}
|
||||
{active === 'rendering' && (
|
||||
<RenderingPanel form={form} update={update} />
|
||||
@@ -4326,108 +4478,119 @@ function PerformanceBudgetPanel({ form, update }: PanelProps) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param focusId Integration the user was sent here to configure. The
|
||||
* panel lists a dozen-plus cards across four categories, so landing on
|
||||
* the section alone would still leave them hunting; the matching card
|
||||
* is scrolled into view and outlined for as long as the panel is open.
|
||||
* One integration's own settings page. Every provider gets a dedicated
|
||||
* sub-page under the "Integrations" sidebar group — credentials plus any
|
||||
* behaviour settings the provider defines (e.g. the GSC date range and
|
||||
* dimension filters). The previous single scrolling page listed a
|
||||
* dozen-plus providers at once, which buried each one's settings.
|
||||
*
|
||||
* @param id Integration id from the shared `INTEGRATIONS` catalog.
|
||||
*/
|
||||
function IntegrationsPanel({ focusId }: { focusId: string | null }) {
|
||||
function IntegrationPage({ id }: { id: string }) {
|
||||
const { t, i18n } = useTranslation();
|
||||
const def = INTEGRATIONS.find((i) => i.id === id);
|
||||
const [state, setState] = useState<IntegrationsState | null>(null);
|
||||
const focusRef = useRef<HTMLDivElement | null>(null);
|
||||
// Per-integration → per-field draft text. Only fields the user has
|
||||
// actually typed into are tracked; a save sends just these so an
|
||||
// untouched "saved" secret is never overwritten.
|
||||
const [drafts, setDrafts] = useState<Record<string, Record<string, string>>>({});
|
||||
const [busy, setBusy] = useState<string | null>(null);
|
||||
const [notice, setNotice] = useState<Record<string, string>>({});
|
||||
// Per-field draft text. Only fields the user has actually typed into
|
||||
// are tracked; a save sends just these so an untouched "saved" secret
|
||||
// is never overwritten.
|
||||
const [draft, setDraft] = useState<Record<string, string>>({});
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [notice, setNotice] = useState<string | undefined>();
|
||||
/** Bumped whenever an account is linked/unlinked, to re-seed the
|
||||
* provider settings section's account dropdown. */
|
||||
const [accountsVersion, setAccountsVersion] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
void window.freecrawl.integrationsGetAll().then(setState);
|
||||
}, []);
|
||||
|
||||
// Switching provider must not carry the previous page's draft or
|
||||
// "Saved." notice across.
|
||||
useEffect(() => {
|
||||
focusRef.current?.scrollIntoView({ block: 'center' });
|
||||
}, [focusId]);
|
||||
setDraft({});
|
||||
setNotice(undefined);
|
||||
}, [id]);
|
||||
|
||||
const setDraft = (id: string, key: string, value: string) => {
|
||||
setDrafts((d) => ({ ...d, [id]: { ...(d[id] ?? {}), [key]: value } }));
|
||||
};
|
||||
if (!def) return null;
|
||||
|
||||
const save = async (id: string) => {
|
||||
const fields = drafts[id];
|
||||
if (!fields || Object.keys(fields).length === 0) return;
|
||||
setBusy(id);
|
||||
const category = INTEGRATION_CATEGORIES.find((c) => c.key === def.category);
|
||||
|
||||
const save = async () => {
|
||||
if (Object.keys(draft).length === 0) return;
|
||||
setBusy(true);
|
||||
try {
|
||||
const next = await window.freecrawl.integrationsSet(id, fields);
|
||||
setState(next);
|
||||
setDrafts((d) => ({ ...d, [id]: {} }));
|
||||
setNotice((n) => ({ ...n, [id]: t('integrations.saved', { defaultValue: 'Saved.' }) }));
|
||||
setState(await window.freecrawl.integrationsSet(def.id, draft));
|
||||
setDraft({});
|
||||
setNotice(t('integrations.saved', { defaultValue: 'Saved.' }));
|
||||
} finally {
|
||||
setBusy(null);
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const clear = async (id: string) => {
|
||||
const clear = async () => {
|
||||
if (!window.confirm(t('integrations.confirmClear', { defaultValue: 'Remove all stored credentials for this integration?' }))) {
|
||||
return;
|
||||
}
|
||||
setBusy(id);
|
||||
setBusy(true);
|
||||
try {
|
||||
const next = await window.freecrawl.integrationsClear(id);
|
||||
setState(next);
|
||||
setDrafts((d) => ({ ...d, [id]: {} }));
|
||||
setNotice((n) => ({ ...n, [id]: t('integrations.cleared', { defaultValue: 'Removed.' }) }));
|
||||
setState(await window.freecrawl.integrationsClear(def.id));
|
||||
setDraft({});
|
||||
setNotice(t('integrations.cleared', { defaultValue: 'Removed.' }));
|
||||
} finally {
|
||||
setBusy(null);
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="mb-2 flex items-center gap-2">
|
||||
<span className="text-[13px] font-semibold text-surface-100">{def.name}</span>
|
||||
{category && (
|
||||
<span className="rounded bg-surface-800 px-1.5 py-0.5 text-[9px] uppercase tracking-wide text-surface-400">
|
||||
{translateLabel(category.label, i18n.language)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="mb-3 text-[11px] text-surface-400">
|
||||
{t('integrations.intro', { defaultValue: 'Connect external services with your own credentials. API keys and secrets are encrypted at rest using your operating system\'s secure storage and never leave this machine.' })}
|
||||
</p>
|
||||
<div className="mb-3 rounded border border-blue-700/40 bg-blue-900/15 px-3 py-2 text-[11px] text-blue-200">
|
||||
{t('integrations.byocNote', { defaultValue: 'Google integrations use a "bring your own client" model — you create your own Google Cloud OAuth client and paste its ID/secret, so no shared FreeCrawl app or verification is involved.' })}
|
||||
</div>
|
||||
{def.authType === 'oauth-byoc' && (
|
||||
<div className="mb-3 rounded border border-blue-700/40 bg-blue-900/15 px-3 py-2 text-[11px] text-blue-200">
|
||||
{t('integrations.byocNote', { defaultValue: 'Google integrations use a "bring your own client" model — you create your own Google Cloud OAuth client and paste its ID/secret, so no shared FreeCrawl app or verification is involved.' })}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{INTEGRATION_CATEGORIES.map((cat) => {
|
||||
const items = INTEGRATIONS.filter((i) => i.category === cat.key);
|
||||
if (items.length === 0) return null;
|
||||
return (
|
||||
<div key={cat.key} className="mb-4">
|
||||
<div className="mb-1.5 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
|
||||
{translateLabel(cat.label, i18n.language)}
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
{items.map((def) => (
|
||||
<IntegrationCard
|
||||
key={def.id}
|
||||
def={def}
|
||||
cardRef={def.id === focusId ? focusRef : undefined}
|
||||
focused={def.id === focusId}
|
||||
state={state?.[def.id]}
|
||||
draft={drafts[def.id] ?? {}}
|
||||
busy={busy === def.id}
|
||||
notice={notice[def.id]}
|
||||
onDraft={(k, v) => setDraft(def.id, k, v)}
|
||||
onSave={() => void save(def.id)}
|
||||
onClear={() => void clear(def.id)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<IntegrationCard
|
||||
def={def}
|
||||
state={state?.[def.id]}
|
||||
draft={draft}
|
||||
busy={busy}
|
||||
notice={notice}
|
||||
onDraft={(k, v) => setDraft((d) => ({ ...d, [k]: v }))}
|
||||
onSave={() => void save()}
|
||||
onClear={() => void clear()}
|
||||
/>
|
||||
|
||||
{/* Linked Google accounts — several can share one OAuth client. */}
|
||||
{def.authType === 'oauth-byoc' && (
|
||||
<GoogleAccountsSection
|
||||
integrationId={def.id}
|
||||
configured={state?.[def.id]?.configured ?? false}
|
||||
onChange={() => setAccountsVersion((v) => v + 1)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Provider-specific behaviour settings, kept separate from the
|
||||
credential card above. Re-mounted when the account list changes
|
||||
so their account dropdowns pick up a new or removed link. */}
|
||||
{def.id === 'gsc' && <GscSettingsSection key={accountsVersion} />}
|
||||
{def.id === 'ga4' && <Ga4SettingsSection key={accountsVersion} />}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function IntegrationCard({
|
||||
def,
|
||||
cardRef,
|
||||
focused,
|
||||
state,
|
||||
draft,
|
||||
busy,
|
||||
@@ -4437,8 +4600,6 @@ function IntegrationCard({
|
||||
onClear,
|
||||
}: {
|
||||
def: IntegrationDef;
|
||||
cardRef?: RefObject<HTMLDivElement | null>;
|
||||
focused?: boolean;
|
||||
state: IntegrationsState[string] | undefined;
|
||||
draft: Record<string, string>;
|
||||
busy: boolean;
|
||||
@@ -4453,17 +4614,10 @@ function IntegrationCard({
|
||||
const [guideOpen, setGuideOpen] = useState(false);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={cardRef}
|
||||
className={clsx(
|
||||
'rounded border bg-surface-950/40 p-3',
|
||||
focused
|
||||
? 'border-accent-500 ring-1 ring-accent-500/40'
|
||||
: 'border-surface-800',
|
||||
)}
|
||||
>
|
||||
<div className="rounded border border-surface-800 bg-surface-950/40 p-3">
|
||||
<div className="mb-1 flex items-center gap-2">
|
||||
<span className="text-[12px] font-medium text-surface-100">{def.name}</span>
|
||||
{/* The provider name lives in the page header above — the card
|
||||
only carries its credential model + connection status. */}
|
||||
<span className="rounded bg-surface-800 px-1.5 py-0.5 text-[9px] uppercase tracking-wide text-surface-400">
|
||||
{translateLabel(AUTH_TYPE_LABEL[def.authType], i18n.language)}
|
||||
</span>
|
||||
|
||||
@@ -255,7 +255,7 @@
|
||||
"dropFailedProject": "Failed to open project: {{name}}",
|
||||
"dropNoUrls": "No URLs found in {{name}}.",
|
||||
"dropLoadedUrls": "Loaded {{count}} URLs from {{name}} into List mode.",
|
||||
"crashRecovery": "FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n Start URL: {{seed}}\n Pending URLs: {{pending}}\n\nResume the crawl from where it stopped?\n\nClick OK to resume, Cancel to discard the recovery state and start fresh.",
|
||||
"crashRecovery": "FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n Mode: {{mode}}\n Start URL: {{seed}}\n Pending URLs: {{pending}}\n\nResume the crawl from where it stopped?\n\nClick OK to resume, Cancel to discard the recovery state and start fresh.",
|
||||
"deleteDomainPrompt": "GDPR Domain Wipe — enter the host to delete (case-insensitive, exact match).\n\nExamples: example.com · blog.example.com\n\nThis cannot be undone. Save Project As… first to keep a backup.",
|
||||
"deleteDomainConfirm": "Delete every URL hosted on \"{{host}}\", along with its links, headers, images, and source snapshots?\n\nThis action cannot be undone.",
|
||||
"deleteDomainResult": "Deleted {{urls}} URL row(s) from {{host}} (and {{links}} associated link(s)).",
|
||||
@@ -1266,7 +1266,91 @@
|
||||
"summary": "{{shown}} pages",
|
||||
"firstN": "(first {{n}})",
|
||||
"emptyNoData": "No internal HTML pages crawled yet — run a crawl, then fetch Search Console data.",
|
||||
"emptyNoMatch": "No pages match the current filter."
|
||||
"emptyNoMatch": "No pages match the current filter.",
|
||||
"days16m": "Last 16 months",
|
||||
"daysCustom": "Custom (Settings)",
|
||||
"dateRangeTitle": "Date range for the pull. Full dimension filters live in Settings → Integrations → Google Search Console.",
|
||||
"presetTitle": "Pre-defined filters for URLs verified in GSC",
|
||||
"presetAll": "All",
|
||||
"presetClicks": "Clicks Above 0",
|
||||
"presetNoData": "No Search Analytics Data",
|
||||
"presetNonIndexableData": "Non-Indexable with Search Analytics Data",
|
||||
"presetOrphan": "Orphan URLs",
|
||||
"presetNotOnGoogle": "URL is Not on Google",
|
||||
"presetIndexableNotIndexed": "Indexable URL Not Indexed",
|
||||
"presetOnGoogleIssues": "URL is on Google But Has Issues",
|
||||
"presetCanonical": "User-Declared Canonical Not Selected",
|
||||
"presetMobile": "Page is Not Mobile Friendly",
|
||||
"presetAmp": "AMP URL Invalid",
|
||||
"presetRich": "Rich Result Invalid",
|
||||
"newUrlsFound": "{{n}} URL(s) appear in Search Console but were not found by the crawl.",
|
||||
"newUrlsQueued": "{{n}} new URL(s) discovered in Search Console were added to the crawl.",
|
||||
"crawlNewUrls": "Crawl {{n}} new URLs",
|
||||
"accountTitle": "Which linked Google account this project reports on. Each account keeps its own stored data."
|
||||
},
|
||||
"googleAccounts": {
|
||||
"title": "Connected Accounts",
|
||||
"connect": "Connect account",
|
||||
"connectAnother": "Connect another",
|
||||
"disconnect": "Disconnect",
|
||||
"confirmDisconnect": "Disconnect {{account}}? Data already pulled with it stays until the next fetch.",
|
||||
"empty": "No account linked yet. Connect one — you can link several and pick which one each project uses.",
|
||||
"unknownEmail": "Google account",
|
||||
"needClient": "Save your OAuth client ID + secret first.",
|
||||
"hint": "Each project picks which linked account it reports on, in the tab toolbar. Data pulled with different accounts is stored separately."
|
||||
},
|
||||
"ga4Settings": {
|
||||
"title": "Analytics 4 Settings",
|
||||
"account": "Account",
|
||||
"accountHint": "Which linked Google account this project pulls Analytics data with.",
|
||||
"dateRange": "Date Range",
|
||||
"range7": "Last 7 days",
|
||||
"range28": "Last 28 days",
|
||||
"range90": "Last 90 days",
|
||||
"hint": "The property is picked in the GA4 tab toolbar. Analytics data is near-realtime, so the range ends today."
|
||||
},
|
||||
"gscSettings": {
|
||||
"title": "Search Console Settings",
|
||||
"account": "Account",
|
||||
"accountHint": "Which linked Google account this project pulls Search Console data with.",
|
||||
"dateRange": "Date Range",
|
||||
"range7": "Last 7 days",
|
||||
"range28": "Last 28 days",
|
||||
"range90": "Last 90 days",
|
||||
"range16m": "Last 16 months",
|
||||
"rangeCustom": "Custom",
|
||||
"startDate": "Start date",
|
||||
"endDate": "End date",
|
||||
"dimensionFilter": "Dimension Filter",
|
||||
"device": "Device",
|
||||
"deviceAll": "All",
|
||||
"deviceDesktop": "Desktop",
|
||||
"deviceMobile": "Mobile",
|
||||
"deviceTablet": "Tablet",
|
||||
"type": "Search Type",
|
||||
"typeWeb": "Web",
|
||||
"typeImage": "Image",
|
||||
"typeVideo": "Video",
|
||||
"typeNews": "News",
|
||||
"typeDiscover": "Discover",
|
||||
"typeGoogleNews": "Google News",
|
||||
"country": "Country",
|
||||
"countryPlaceholder": "e.g. usa (ISO code) — blank for all",
|
||||
"queryFilter": "Search Query",
|
||||
"queryNone": "None",
|
||||
"queryContains": "Contains",
|
||||
"queryNotContains": "Doesn't contain",
|
||||
"queryEquals": "Equals",
|
||||
"queryNotEquals": "Doesn't equal",
|
||||
"queryRegex": "Matches regex",
|
||||
"queryNotRegex": "Doesn't match regex",
|
||||
"queryValuePlaceholder": "Query…",
|
||||
"general": "General",
|
||||
"matchSlash": "Match Trailing and Non-Trailing Slash URLs",
|
||||
"matchCase": "Match Uppercase & Lowercase URLs",
|
||||
"limitMaxResults": "Limit Max Results",
|
||||
"crawlNewUrls": "Crawl New URLs Discovered In Google Search Console",
|
||||
"crawlNewUrlsHint": "After each Search Console pull, URLs that appear in GSC but were not found by the crawl are added to the crawl queue."
|
||||
},
|
||||
"ga4Tab": {
|
||||
"title": "Google Analytics 4",
|
||||
@@ -1300,7 +1384,8 @@
|
||||
"summary": "{{shown}} pages",
|
||||
"firstN": "(first {{n}})",
|
||||
"emptyNoData": "No internal HTML pages crawled yet — run a crawl, then fetch GA4 data.",
|
||||
"emptyNoMatch": "No pages match the current filter."
|
||||
"emptyNoMatch": "No pages match the current filter.",
|
||||
"accountTitle": "Which linked Google account this project reports on. Each account keeps its own stored data."
|
||||
},
|
||||
"aiTab": {
|
||||
"title": "AI",
|
||||
@@ -1503,7 +1588,7 @@
|
||||
"customRegexPlaceholder": "Named-group regex: (?<ip>…) (?<ts>…) (?<method>…) (?<path>…) (?<status>…) (?<ua>…)",
|
||||
"verifyBots": "Verify bots (rDNS)",
|
||||
"verifyBotsHint": "Reverse-DNS verify a sample of bot IPs (slower; catches spoofed user-agents)",
|
||||
"import": "Import Log File…",
|
||||
"import": "Import Log Files…",
|
||||
"analyzing": "Analyzing…",
|
||||
"clear": "Clear",
|
||||
"clearConfirm": "Clear all ingested log data from this project?",
|
||||
@@ -1512,6 +1597,8 @@
|
||||
"exportXlsx": "Excel (.xlsx, one sheet each)",
|
||||
"exported": "Exported {{size}} KB → {{path}}",
|
||||
"imported": "{{file}} — parsed {{parsed}}/{{total}} lines ({{format}})",
|
||||
"importedBatch": "{{files}} files — parsed {{parsed}}/{{total}} lines",
|
||||
"importedFailed": "{{n}} failed",
|
||||
"importError": "Import failed: {{err}}",
|
||||
"noData": "No log data yet",
|
||||
"noDataHint": "Import a server access log (Apache, Nginx, or IIS W3C) to see which URLs search-engine bots crawl, how response codes are distributed in the log, and which logged URLs your crawl never reached.",
|
||||
|
||||
@@ -255,7 +255,7 @@
|
||||
"dropFailedProject": "Proje açılamadı: {{name}}",
|
||||
"dropNoUrls": "{{name}} içinde URL bulunamadı.",
|
||||
"dropLoadedUrls": "{{name}} dosyasından {{count}} URL List moduna yüklendi.",
|
||||
"crashRecovery": "FreeCrawl, temiz bitmemiş bir önceki crawl tespit etti:\n\n Başlangıç URL: {{seed}}\n Bekleyen URL: {{pending}}\n\nCrawl kaldığı yerden devam etsin mi?\n\nDevam etmek için Tamam, recovery durumunu iptal edip baştan başlamak için İptal'e tıkla.",
|
||||
"crashRecovery": "FreeCrawl, temiz bitmemiş bir önceki crawl tespit etti:\n\n Mod: {{mode}}\n Başlangıç URL: {{seed}}\n Bekleyen URL: {{pending}}\n\nCrawl kaldığı yerden devam etsin mi?\n\nDevam etmek için Tamam, recovery durumunu iptal edip baştan başlamak için İptal'e tıkla.",
|
||||
"deleteDomainPrompt": "GDPR Alan Temizleme — silinecek host'u gir (büyük/küçük harf duyarsız, tam eşleşme).\n\nÖrnek: example.com · blog.example.com\n\nGeri alınamaz. Yedek istiyorsan önce Projeyi Farklı Kaydet.",
|
||||
"deleteDomainConfirm": "\"{{host}}\" üzerindeki tüm URL'leri, linklerini, başlıklarını, görsellerini ve kaynak snapshot'larını sil?\n\nBu işlem geri alınamaz.",
|
||||
"deleteDomainResult": "{{host}} alanından {{urls}} URL satırı silindi (ve {{links}} bağlı link).",
|
||||
@@ -1266,7 +1266,91 @@
|
||||
"summary": "{{shown}} sayfa",
|
||||
"firstN": "(ilk {{n}})",
|
||||
"emptyNoData": "Henüz taranmış iç HTML sayfası yok — önce bir crawl çalıştır, sonra Search Console verisini getir.",
|
||||
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok."
|
||||
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok.",
|
||||
"days16m": "Son 16 ay",
|
||||
"daysCustom": "Özel (Ayarlar)",
|
||||
"dateRangeTitle": "Getirme için tarih aralığı. Tüm dimension filtreleri Ayarlar → Entegrasyonlar → Google Search Console'da.",
|
||||
"presetTitle": "GSC'de doğrulanmış URL'ler için hazır filtreler",
|
||||
"presetAll": "Tümü",
|
||||
"presetClicks": "Tıklaması 0'dan Fazla",
|
||||
"presetNoData": "Search Analytics Verisi Yok",
|
||||
"presetNonIndexableData": "Search Analytics Verisi Olan İndekslenemez",
|
||||
"presetOrphan": "Yetim URL'ler",
|
||||
"presetNotOnGoogle": "URL Google'da Değil",
|
||||
"presetIndexableNotIndexed": "İndekslenebilir ama İndekslenmemiş",
|
||||
"presetOnGoogleIssues": "Google'da ama Sorunlu",
|
||||
"presetCanonical": "Kullanıcı Canonical'ı Seçilmemiş",
|
||||
"presetMobile": "Sayfa Mobil Uyumlu Değil",
|
||||
"presetAmp": "AMP URL Geçersiz",
|
||||
"presetRich": "Zengin Sonuç Geçersiz",
|
||||
"newUrlsFound": "{{n}} URL Search Console'da görünüyor ama crawl'da bulunamadı.",
|
||||
"newUrlsQueued": "Search Console'da keşfedilen {{n}} yeni URL crawl'a eklendi.",
|
||||
"crawlNewUrls": "{{n}} yeni URL'yi crawl et",
|
||||
"accountTitle": "Bu projenin hangi bağlı Google hesabını raporlayacağı. Her hesap kendi verisini ayrı saklar."
|
||||
},
|
||||
"googleAccounts": {
|
||||
"title": "Bağlı Hesaplar",
|
||||
"connect": "Hesap bağla",
|
||||
"connectAnother": "Başka hesap bağla",
|
||||
"disconnect": "Bağlantıyı kes",
|
||||
"confirmDisconnect": "{{account}} bağlantısı kesilsin mi? Bu hesapla çekilmiş veriler bir sonraki getirmeye kadar kalır.",
|
||||
"empty": "Henüz bağlı hesap yok. Bir tane bağla — birden fazla hesap bağlayıp her projede hangisinin kullanılacağını seçebilirsin.",
|
||||
"unknownEmail": "Google hesabı",
|
||||
"needClient": "Önce OAuth client ID + secret'ını kaydet.",
|
||||
"hint": "Her proje hangi bağlı hesabı raporlayacağını sekme araç çubuğundan seçer. Farklı hesaplarla çekilen veriler ayrı saklanır."
|
||||
},
|
||||
"ga4Settings": {
|
||||
"title": "Analytics 4 Ayarları",
|
||||
"account": "Hesap",
|
||||
"accountHint": "Bu projenin Analytics verisini hangi bağlı Google hesabıyla çekeceği.",
|
||||
"dateRange": "Tarih Aralığı",
|
||||
"range7": "Son 7 gün",
|
||||
"range28": "Son 28 gün",
|
||||
"range90": "Son 90 gün",
|
||||
"hint": "Property, GA4 sekmesi araç çubuğundan seçilir. Analytics verisi neredeyse gerçek zamanlı olduğu için aralık bugün biter."
|
||||
},
|
||||
"gscSettings": {
|
||||
"title": "Search Console Ayarları",
|
||||
"account": "Hesap",
|
||||
"accountHint": "Bu projenin Search Console verisini hangi bağlı Google hesabıyla çekeceği.",
|
||||
"dateRange": "Tarih Aralığı",
|
||||
"range7": "Son 7 gün",
|
||||
"range28": "Son 28 gün",
|
||||
"range90": "Son 90 gün",
|
||||
"range16m": "Son 16 ay",
|
||||
"rangeCustom": "Özel",
|
||||
"startDate": "Başlangıç tarihi",
|
||||
"endDate": "Bitiş tarihi",
|
||||
"dimensionFilter": "Dimension Filtresi",
|
||||
"device": "Cihaz",
|
||||
"deviceAll": "Tümü",
|
||||
"deviceDesktop": "Masaüstü",
|
||||
"deviceMobile": "Mobil",
|
||||
"deviceTablet": "Tablet",
|
||||
"type": "Arama Türü",
|
||||
"typeWeb": "Web",
|
||||
"typeImage": "Görsel",
|
||||
"typeVideo": "Video",
|
||||
"typeNews": "Haberler",
|
||||
"typeDiscover": "Discover",
|
||||
"typeGoogleNews": "Google Haberler",
|
||||
"country": "Ülke",
|
||||
"countryPlaceholder": "örn. usa (ISO kodu) — tümü için boş bırak",
|
||||
"queryFilter": "Arama Sorgusu",
|
||||
"queryNone": "Yok",
|
||||
"queryContains": "İçerir",
|
||||
"queryNotContains": "İçermez",
|
||||
"queryEquals": "Eşittir",
|
||||
"queryNotEquals": "Eşit değil",
|
||||
"queryRegex": "Regex eşleşir",
|
||||
"queryNotRegex": "Regex eşleşmez",
|
||||
"queryValuePlaceholder": "Sorgu…",
|
||||
"general": "Genel",
|
||||
"matchSlash": "Sondaki Slash'lı ve Slash'sız URL'leri Eşleştir",
|
||||
"matchCase": "Büyük & Küçük Harfli URL'leri Eşleştir",
|
||||
"limitMaxResults": "Maks. Sonuç Sınırı",
|
||||
"crawlNewUrls": "Google Search Console'da Keşfedilen Yeni URL'leri Crawl Et",
|
||||
"crawlNewUrlsHint": "Her Search Console getirmesinden sonra, GSC'de görünen ama crawl'da bulunamayan URL'ler crawl kuyruğuna eklenir."
|
||||
},
|
||||
"ga4Tab": {
|
||||
"title": "Google Analytics 4",
|
||||
@@ -1300,7 +1384,8 @@
|
||||
"summary": "{{shown}} sayfa",
|
||||
"firstN": "(ilk {{n}})",
|
||||
"emptyNoData": "Henüz taranmış iç HTML sayfası yok — önce bir crawl çalıştır, sonra GA4 verisini getir.",
|
||||
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok."
|
||||
"emptyNoMatch": "Mevcut filtreyle eşleşen sayfa yok.",
|
||||
"accountTitle": "Bu projenin hangi bağlı Google hesabını raporlayacağı. Her hesap kendi verisini ayrı saklar."
|
||||
},
|
||||
"aiTab": {
|
||||
"title": "AI",
|
||||
@@ -1503,7 +1588,7 @@
|
||||
"customRegexPlaceholder": "Adlandırılmış grup regex: (?<ip>…) (?<ts>…) (?<method>…) (?<path>…) (?<status>…) (?<ua>…)",
|
||||
"verifyBots": "Botları doğrula (rDNS)",
|
||||
"verifyBotsHint": "Bot IP'lerinin bir örneğini ters DNS ile doğrula (daha yavaş; sahte user-agent'ları yakalar)",
|
||||
"import": "Log Dosyası İçe Aktar…",
|
||||
"import": "Log Dosyalarını İçe Aktar…",
|
||||
"analyzing": "Analiz ediliyor…",
|
||||
"clear": "Temizle",
|
||||
"clearConfirm": "Bu projedeki tüm log verileri temizlensin mi?",
|
||||
@@ -1512,6 +1597,8 @@
|
||||
"exportXlsx": "Excel (.xlsx, her tablo ayrı sayfa)",
|
||||
"exported": "{{size}} KB dışa aktarıldı → {{path}}",
|
||||
"imported": "{{file}} — {{parsed}}/{{total}} satır ayrıştırıldı ({{format}})",
|
||||
"importedBatch": "{{files}} dosya — {{parsed}}/{{total}} satır ayrıştırıldı",
|
||||
"importedFailed": "{{n}} başarısız",
|
||||
"importError": "İçe aktarım başarısız: {{err}}",
|
||||
"noData": "Henüz log verisi yok",
|
||||
"noDataHint": "Hangi URL'leri arama motoru botlarının taradığını, logdaki yanıt kodu dağılımını ve crawl'ın hiç ulaşmadığı URL'leri görmek için bir sunucu erişim logu (Apache, Nginx veya IIS W3C) içe aktarın.",
|
||||
|
||||
@@ -491,10 +491,14 @@ export const useAppStore = create<AppState>((set) => ({
|
||||
return { config: next };
|
||||
}),
|
||||
applyProjectConfig: (config) =>
|
||||
// Swap the UI to a project's saved config on open. Deliberately does
|
||||
// Swap the UI to a project's saved config on open — including its
|
||||
// mode and start URL, which are part of the document: a project
|
||||
// crawled from a sitemap must reopen in Sitemap mode showing that
|
||||
// sitemap, not blank in Spider mode while its tables hold the data.
|
||||
// Also the resume path after an interrupted crawl. Deliberately does
|
||||
// NOT touch global prefs — opening a project must not overwrite the
|
||||
// user's default settings.
|
||||
set({ config: { ...config, startUrl: '' } }),
|
||||
set({ config }),
|
||||
setProgress: (p) => set({ progress: p }),
|
||||
setSummary: (s) => set({ summary: s }),
|
||||
markCrawlFinished: () =>
|
||||
|
||||
@@ -2,11 +2,13 @@ import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react'
|
||||
import { useVirtualizer } from '@tanstack/react-virtual';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Loader2, Search, RefreshCw } from 'lucide-react';
|
||||
import type {
|
||||
Ga4FetchMeta,
|
||||
Ga4Property,
|
||||
Ga4Row,
|
||||
GoogleAuthState,
|
||||
import {
|
||||
defaultGa4Settings,
|
||||
type Ga4FetchMeta,
|
||||
type Ga4Property,
|
||||
type Ga4Row,
|
||||
type GoogleAccount,
|
||||
type GoogleAuthState,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { useAppStore } from '../store.js';
|
||||
|
||||
@@ -57,6 +59,8 @@ export function AnalyticsTab() {
|
||||
const [stage, setStage] = useState<Stage>('loading');
|
||||
const [authState, setAuthState] = useState<GoogleAuthState | null>(null);
|
||||
const [properties, setProperties] = useState<Ga4Property[]>([]);
|
||||
const [accounts, setAccounts] = useState<GoogleAccount[]>([]);
|
||||
const [accountId, setAccountId] = useState('');
|
||||
const [propertyId, setPropertyId] = useState('');
|
||||
const [days, setDays] = useState<RangeDays>(28);
|
||||
const [rows, setRows] = useState<Ga4Row[]>([]);
|
||||
@@ -69,26 +73,55 @@ export function AnalyticsTab() {
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const loadProperties = useCallback(async () => {
|
||||
const res = await window.freecrawl.ga4ListProperties();
|
||||
const loadProperties = useCallback(async (account?: string) => {
|
||||
const res = await window.freecrawl.ga4ListProperties(account);
|
||||
if (!res.ok) {
|
||||
setError(res.error);
|
||||
setProperties([]);
|
||||
return;
|
||||
}
|
||||
setError(null);
|
||||
setProperties(res.properties);
|
||||
setPropertyId((prev) => prev || res.properties[0]?.propertyId || '');
|
||||
setPropertyId((prev) =>
|
||||
// Keep the current property only if the account can still see it.
|
||||
prev && res.properties.some((p) => p.propertyId === prev)
|
||||
? prev
|
||||
: (res.properties[0]?.propertyId ?? ''),
|
||||
);
|
||||
}, []);
|
||||
|
||||
/** Switch the project to another linked Google account: persist it,
|
||||
* re-list that account's properties, reload the table. */
|
||||
const changeAccount = useCallback(
|
||||
(next: string) => {
|
||||
setAccountId(next);
|
||||
void window.freecrawl.integrationSettingsSet('ga4', { accountId: next });
|
||||
void loadProperties(next);
|
||||
},
|
||||
[loadProperties],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void (async () => {
|
||||
const [integrations, auth] = await Promise.all([
|
||||
const [integrations, auth, ga4Settings] = await Promise.all([
|
||||
window.freecrawl.integrationsGetAll(),
|
||||
window.freecrawl.googleAuthStatus('ga4'),
|
||||
window.freecrawl.integrationSettingsGet('ga4'),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
setAuthState(auth);
|
||||
setAccounts(auth.accounts);
|
||||
// Seed from the stored per-project settings; a stored account that
|
||||
// has since been unlinked falls back to the first one.
|
||||
const s = { ...defaultGa4Settings(), ...(ga4Settings ?? {}) };
|
||||
const account =
|
||||
auth.accounts.find((a) => a.accountId === s.accountId)?.accountId ??
|
||||
auth.accounts[0]?.accountId ??
|
||||
'';
|
||||
setAccountId(account);
|
||||
setDays(s.days);
|
||||
if (s.property) setPropertyId(s.property);
|
||||
const configured = integrations['ga4']?.configured ?? false;
|
||||
if (!configured) {
|
||||
setStage('unconfigured');
|
||||
@@ -96,7 +129,7 @@ export function AnalyticsTab() {
|
||||
setStage('disconnected');
|
||||
} else {
|
||||
setStage('ready');
|
||||
void loadProperties();
|
||||
void loadProperties(account);
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
@@ -110,11 +143,12 @@ export function AnalyticsTab() {
|
||||
offset: 0,
|
||||
search: search || undefined,
|
||||
filter,
|
||||
accountId: accountId || undefined,
|
||||
});
|
||||
setRows(res.rows);
|
||||
setTotal(res.total);
|
||||
setMeta(res.meta);
|
||||
}, [search, filter]);
|
||||
}, [search, filter, accountId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (stage !== 'ready') return;
|
||||
@@ -144,9 +178,12 @@ export function AnalyticsTab() {
|
||||
try {
|
||||
const res = await window.freecrawl.googleAuthStart('ga4');
|
||||
setAuthState(res.state);
|
||||
setAccounts(res.state.accounts);
|
||||
if (res.ok && res.state.connected) {
|
||||
const account = res.state.accounts[0]?.accountId ?? '';
|
||||
setAccountId(account);
|
||||
setStage('ready');
|
||||
void loadProperties();
|
||||
void loadProperties(account);
|
||||
} else if (res.error) {
|
||||
setError(res.error);
|
||||
}
|
||||
@@ -155,13 +192,27 @@ export function AnalyticsTab() {
|
||||
}
|
||||
}, [loadProperties]);
|
||||
|
||||
/** Unlink the account currently selected in the toolbar; with several
|
||||
* linked, the tab stays usable and falls back to the next one. */
|
||||
const disconnect = useCallback(async () => {
|
||||
const next = await window.freecrawl.googleAuthRevoke('ga4');
|
||||
const next = await window.freecrawl.googleAuthRevoke(
|
||||
'ga4',
|
||||
accountId || undefined,
|
||||
);
|
||||
setAuthState(next);
|
||||
setAccounts(next.accounts);
|
||||
setProperties([]);
|
||||
setPropertyId('');
|
||||
setStage('disconnected');
|
||||
}, []);
|
||||
if (!next.connected) {
|
||||
setAccountId('');
|
||||
setStage('disconnected');
|
||||
return;
|
||||
}
|
||||
const fallback = next.accounts[0]!.accountId;
|
||||
setAccountId(fallback);
|
||||
void window.freecrawl.integrationSettingsSet('ga4', { accountId: fallback });
|
||||
void loadProperties(fallback);
|
||||
}, [accountId, loadProperties]);
|
||||
|
||||
const runFetch = useCallback(async () => {
|
||||
if (!propertyId || fetching) return;
|
||||
@@ -173,6 +224,7 @@ export function AnalyticsTab() {
|
||||
property: propertyId,
|
||||
propertyName: selected?.displayName || propertyId,
|
||||
days,
|
||||
accountId: accountId || undefined,
|
||||
});
|
||||
if (!res.ok) {
|
||||
setError(res.error);
|
||||
@@ -183,7 +235,7 @@ export function AnalyticsTab() {
|
||||
} finally {
|
||||
setFetching(false);
|
||||
}
|
||||
}, [propertyId, properties, days, fetching, reload]);
|
||||
}, [propertyId, properties, days, accountId, fetching, reload]);
|
||||
|
||||
// ── Gate states ──────────────────────────────────────────────────────
|
||||
if (stage === 'loading') {
|
||||
@@ -277,6 +329,24 @@ export function AnalyticsTab() {
|
||||
</select>
|
||||
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
{accounts.length > 1 && (
|
||||
<select
|
||||
className="h-6 max-w-[170px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={accountId}
|
||||
onChange={(e) => changeAccount(e.target.value)}
|
||||
disabled={fetching}
|
||||
title={t('ga4Tab.accountTitle', {
|
||||
defaultValue:
|
||||
'Which linked Google account this project reports on. Each account keeps its own stored data.',
|
||||
})}
|
||||
>
|
||||
{accounts.map((a) => (
|
||||
<option key={a.accountId} value={a.accountId}>
|
||||
{a.email ?? a.accountId}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
<select
|
||||
className="h-6 max-w-[260px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={propertyId}
|
||||
@@ -327,7 +397,10 @@ export function AnalyticsTab() {
|
||||
<span>
|
||||
{t('ga4Tab.connectedAs', {
|
||||
defaultValue: 'Connected as {{email}}',
|
||||
email: authState?.email ?? 'Google account',
|
||||
email:
|
||||
accounts.find((a) => a.accountId === accountId)?.email ??
|
||||
authState?.email ??
|
||||
'Google account',
|
||||
})}
|
||||
</span>
|
||||
<button
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
import { useCallback, useEffect, useRef, useState, type ReactNode } from 'react';
|
||||
import { useVirtualizer } from '@tanstack/react-virtual';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Loader2, Search, RefreshCw } from 'lucide-react';
|
||||
import { Loader2, Search, RefreshCw, Plus } from 'lucide-react';
|
||||
import { InfoTip } from '../components/InfoTip.js';
|
||||
import type {
|
||||
GoogleAuthState,
|
||||
GscFetchMeta,
|
||||
GscInspectProgress,
|
||||
GscRow,
|
||||
GscSite,
|
||||
import {
|
||||
defaultGscSettings,
|
||||
type GoogleAccount,
|
||||
type GoogleAuthState,
|
||||
type GscDateRange,
|
||||
type GscFetchMeta,
|
||||
type GscInspectProgress,
|
||||
type GscPreset,
|
||||
type GscRow,
|
||||
type GscSite,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { useAppStore } from '../store.js';
|
||||
|
||||
@@ -22,11 +26,26 @@ const POLL_MS_RUNNING = 5000;
|
||||
* small enough chunks that one click is reversible. */
|
||||
const INSPECT_BATCH = 100;
|
||||
|
||||
type FilterMode = 'all' | 'with-data' | 'without-data';
|
||||
type RangeDays = 7 | 28 | 90;
|
||||
/** What the tab is currently showing — drives the three-way render. */
|
||||
type Stage = 'loading' | 'unconfigured' | 'disconnected' | 'ready';
|
||||
|
||||
/** Quick-filter presets, mirroring Screaming Frog's GSC filter dropdown.
|
||||
* `needsInspection` entries only match once URL Inspection has been run. */
|
||||
const FILTER_PRESETS: { value: GscPreset; labelKey: string; label: string }[] = [
|
||||
{ value: 'all', labelKey: 'gscTab.presetAll', label: 'All' },
|
||||
{ value: 'clicks-above-0', labelKey: 'gscTab.presetClicks', label: 'Clicks Above 0' },
|
||||
{ value: 'without-data', labelKey: 'gscTab.presetNoData', label: 'No Search Analytics Data' },
|
||||
{ value: 'non-indexable-with-data', labelKey: 'gscTab.presetNonIndexableData', label: 'Non-Indexable with Search Analytics Data' },
|
||||
{ value: 'orphan', labelKey: 'gscTab.presetOrphan', label: 'Orphan URLs' },
|
||||
{ value: 'not-on-google', labelKey: 'gscTab.presetNotOnGoogle', label: 'URL is Not on Google' },
|
||||
{ value: 'indexable-not-indexed', labelKey: 'gscTab.presetIndexableNotIndexed', label: 'Indexable URL Not Indexed' },
|
||||
{ value: 'on-google-with-issues', labelKey: 'gscTab.presetOnGoogleIssues', label: 'URL is on Google But Has Issues' },
|
||||
{ value: 'canonical-mismatch', labelKey: 'gscTab.presetCanonical', label: 'User-Declared Canonical Not Selected' },
|
||||
{ value: 'not-mobile-friendly', labelKey: 'gscTab.presetMobile', label: 'Page is Not Mobile Friendly' },
|
||||
{ value: 'amp-invalid', labelKey: 'gscTab.presetAmp', label: 'AMP URL Invalid' },
|
||||
{ value: 'rich-result-invalid', labelKey: 'gscTab.presetRich', label: 'Rich Result Invalid' },
|
||||
];
|
||||
|
||||
/** Average-position bands → cell colour (lower is better). */
|
||||
function positionClass(v: number): string {
|
||||
if (v <= 0) return 'text-surface-600';
|
||||
@@ -55,42 +74,80 @@ export function SearchConsoleTab() {
|
||||
const [stage, setStage] = useState<Stage>('loading');
|
||||
const [authState, setAuthState] = useState<GoogleAuthState | null>(null);
|
||||
const [sites, setSites] = useState<GscSite[]>([]);
|
||||
const [accounts, setAccounts] = useState<GoogleAccount[]>([]);
|
||||
const [accountId, setAccountId] = useState('');
|
||||
const [property, setProperty] = useState('');
|
||||
const [days, setDays] = useState<RangeDays>(28);
|
||||
const [dateRange, setDateRange] = useState<GscDateRange>('28d');
|
||||
const [rows, setRows] = useState<GscRow[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [meta, setMeta] = useState<GscFetchMeta | null>(null);
|
||||
const [search, setSearch] = useState('');
|
||||
const [filter, setFilter] = useState<FilterMode>('all');
|
||||
const [filter, setFilter] = useState<GscPreset>('all');
|
||||
const [connecting, setConnecting] = useState(false);
|
||||
const [fetching, setFetching] = useState(false);
|
||||
const [inspecting, setInspecting] = useState(false);
|
||||
const [inspectProgress, setInspectProgress] = useState<GscInspectProgress | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
/** After a pull: how many GSC URLs were missing from the crawl, and
|
||||
* whether they were auto-queued (crawl-new-urls setting on). */
|
||||
const [newUrls, setNewUrls] = useState<{ count: number; queued: number } | null>(null);
|
||||
const [crawlingNew, setCrawlingNew] = useState(false);
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const loadSites = useCallback(async () => {
|
||||
const res = await window.freecrawl.gscListSites();
|
||||
const loadSites = useCallback(async (account?: string) => {
|
||||
const res = await window.freecrawl.gscListSites(account);
|
||||
if (!res.ok) {
|
||||
setError(res.error);
|
||||
setSites([]);
|
||||
return;
|
||||
}
|
||||
setError(null);
|
||||
setSites(res.sites);
|
||||
setProperty((prev) => prev || res.sites[0]?.siteUrl || '');
|
||||
setProperty((prev) =>
|
||||
// Keep the current property only if the account can still see it.
|
||||
prev && res.sites.some((s) => s.siteUrl === prev)
|
||||
? prev
|
||||
: (res.sites[0]?.siteUrl ?? ''),
|
||||
);
|
||||
}, []);
|
||||
|
||||
/** Switch the project to another linked Google account: persist the
|
||||
* choice, then re-list that account's properties and reload the table
|
||||
* (each account's rows are stored separately). */
|
||||
const changeAccount = useCallback(
|
||||
(next: string) => {
|
||||
setAccountId(next);
|
||||
setNewUrls(null);
|
||||
void window.freecrawl.integrationSettingsSet('gsc', { accountId: next });
|
||||
void loadSites(next);
|
||||
},
|
||||
[loadSites],
|
||||
);
|
||||
|
||||
// Resolve the tab's stage on mount: needs OAuth credentials, then a
|
||||
// connected account, before the data view is meaningful.
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void (async () => {
|
||||
const [integrations, auth] = await Promise.all([
|
||||
const [integrations, auth, gscSettings] = await Promise.all([
|
||||
window.freecrawl.integrationsGetAll(),
|
||||
window.freecrawl.googleAuthStatus('gsc'),
|
||||
window.freecrawl.integrationSettingsGet('gsc'),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
setAuthState(auth);
|
||||
setAccounts(auth.accounts);
|
||||
// Seed the toolbar from the stored per-project GSC settings so the
|
||||
// account + property + date range persist across sessions. A stored
|
||||
// account that has since been unlinked falls back to the first one.
|
||||
const s = { ...defaultGscSettings(), ...(gscSettings ?? {}) };
|
||||
const account =
|
||||
auth.accounts.find((a) => a.accountId === s.accountId)?.accountId ??
|
||||
auth.accounts[0]?.accountId ??
|
||||
'';
|
||||
setAccountId(account);
|
||||
setDateRange(s.dateRange);
|
||||
if (s.property) setProperty(s.property);
|
||||
const configured = integrations['gsc']?.configured ?? false;
|
||||
if (!configured) {
|
||||
setStage('unconfigured');
|
||||
@@ -98,7 +155,7 @@ export function SearchConsoleTab() {
|
||||
setStage('disconnected');
|
||||
} else {
|
||||
setStage('ready');
|
||||
void loadSites();
|
||||
void loadSites(account);
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
@@ -112,11 +169,12 @@ export function SearchConsoleTab() {
|
||||
offset: 0,
|
||||
search: search || undefined,
|
||||
filter,
|
||||
accountId: accountId || undefined,
|
||||
});
|
||||
setRows(res.rows);
|
||||
setTotal(res.total);
|
||||
setMeta(res.meta);
|
||||
}, [search, filter]);
|
||||
}, [search, filter, accountId]);
|
||||
|
||||
// Candidate list — only while the data view is showing.
|
||||
useEffect(() => {
|
||||
@@ -147,9 +205,12 @@ export function SearchConsoleTab() {
|
||||
try {
|
||||
const res = await window.freecrawl.googleAuthStart('gsc');
|
||||
setAuthState(res.state);
|
||||
setAccounts(res.state.accounts);
|
||||
if (res.ok && res.state.connected) {
|
||||
const account = res.state.accounts[0]?.accountId ?? '';
|
||||
setAccountId(account);
|
||||
setStage('ready');
|
||||
void loadSites();
|
||||
void loadSites(account);
|
||||
} else if (res.error) {
|
||||
setError(res.error);
|
||||
}
|
||||
@@ -158,30 +219,72 @@ export function SearchConsoleTab() {
|
||||
}
|
||||
}, [loadSites]);
|
||||
|
||||
/** Unlink the account currently selected in the toolbar. With several
|
||||
* linked, the tab stays usable and falls back to the next one. */
|
||||
const disconnect = useCallback(async () => {
|
||||
const next = await window.freecrawl.googleAuthRevoke('gsc');
|
||||
const next = await window.freecrawl.googleAuthRevoke(
|
||||
'gsc',
|
||||
accountId || undefined,
|
||||
);
|
||||
setAuthState(next);
|
||||
setAccounts(next.accounts);
|
||||
setSites([]);
|
||||
setProperty('');
|
||||
setStage('disconnected');
|
||||
if (!next.connected) {
|
||||
setAccountId('');
|
||||
setStage('disconnected');
|
||||
return;
|
||||
}
|
||||
const fallback = next.accounts[0]!.accountId;
|
||||
setAccountId(fallback);
|
||||
void window.freecrawl.integrationSettingsSet('gsc', { accountId: fallback });
|
||||
void loadSites(fallback);
|
||||
}, [accountId, loadSites]);
|
||||
|
||||
const changeDateRange = useCallback((next: GscDateRange) => {
|
||||
setDateRange(next);
|
||||
void window.freecrawl.integrationSettingsSet('gsc', { dateRange: next });
|
||||
}, []);
|
||||
|
||||
const runFetch = useCallback(async () => {
|
||||
if (!property || fetching) return;
|
||||
setFetching(true);
|
||||
setError(null);
|
||||
setNewUrls(null);
|
||||
try {
|
||||
const res = await window.freecrawl.gscFetch({ property, days });
|
||||
// Date range + dimension filters + row cap come from the stored GSC
|
||||
// settings; the main process reads them. We only pass the property.
|
||||
const res = await window.freecrawl.gscFetch({
|
||||
property,
|
||||
accountId: accountId || undefined,
|
||||
});
|
||||
if (!res.ok) {
|
||||
setError(res.error);
|
||||
} else {
|
||||
setMeta(res.meta);
|
||||
setNewUrls({ count: res.newUrlCount ?? 0, queued: res.queuedNewUrls ?? 0 });
|
||||
await reload();
|
||||
}
|
||||
} finally {
|
||||
setFetching(false);
|
||||
}
|
||||
}, [property, days, fetching, reload]);
|
||||
}, [property, accountId, fetching, reload]);
|
||||
|
||||
const crawlNew = useCallback(async () => {
|
||||
if (crawlingNew) return;
|
||||
setCrawlingNew(true);
|
||||
setError(null);
|
||||
try {
|
||||
const res = await window.freecrawl.gscCrawlNewUrls();
|
||||
if (!res.ok) {
|
||||
setError(res.error);
|
||||
} else {
|
||||
setNewUrls({ count: res.candidateCount, queued: res.queued });
|
||||
}
|
||||
} finally {
|
||||
setCrawlingNew(false);
|
||||
}
|
||||
}, [crawlingNew]);
|
||||
|
||||
const runInspect = useCallback(async () => {
|
||||
if (!property || inspecting) return;
|
||||
@@ -201,13 +304,17 @@ export function SearchConsoleTab() {
|
||||
setInspecting(true);
|
||||
setError(null);
|
||||
try {
|
||||
await window.freecrawl.gscInspectRun({ property, urls });
|
||||
await window.freecrawl.gscInspectRun({
|
||||
property,
|
||||
urls,
|
||||
accountId: accountId || undefined,
|
||||
});
|
||||
} finally {
|
||||
setInspecting(false);
|
||||
setInspectProgress(null);
|
||||
void reload();
|
||||
}
|
||||
}, [property, inspecting, rows, t, reload]);
|
||||
}, [property, accountId, inspecting, rows, t, reload]);
|
||||
|
||||
// Live URL Inspection progress.
|
||||
useEffect(() => {
|
||||
@@ -295,22 +402,39 @@ export function SearchConsoleTab() {
|
||||
/>
|
||||
</div>
|
||||
<select
|
||||
className="h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
className="h-6 max-w-[220px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={filter}
|
||||
onChange={(e) => setFilter(e.target.value as FilterMode)}
|
||||
onChange={(e) => setFilter(e.target.value as GscPreset)}
|
||||
title={t('gscTab.presetTitle', {
|
||||
defaultValue: 'Pre-defined filters for URLs verified in GSC',
|
||||
})}
|
||||
>
|
||||
<option value="all">
|
||||
{t('gscTab.filterAll', { defaultValue: 'All pages' })}
|
||||
</option>
|
||||
<option value="with-data">
|
||||
{t('gscTab.filterWith', { defaultValue: 'With GSC data' })}
|
||||
</option>
|
||||
<option value="without-data">
|
||||
{t('gscTab.filterWithout', { defaultValue: 'No GSC data' })}
|
||||
</option>
|
||||
{FILTER_PRESETS.map((p) => (
|
||||
<option key={p.value} value={p.value}>
|
||||
{t(p.labelKey, { defaultValue: p.label })}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
{accounts.length > 1 && (
|
||||
<select
|
||||
className="h-6 max-w-[170px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={accountId}
|
||||
onChange={(e) => changeAccount(e.target.value)}
|
||||
disabled={fetching || inspecting}
|
||||
title={t('gscTab.accountTitle', {
|
||||
defaultValue:
|
||||
'Which linked Google account this project reports on. Each account keeps its own stored data.',
|
||||
})}
|
||||
>
|
||||
{accounts.map((a) => (
|
||||
<option key={a.accountId} value={a.accountId}>
|
||||
{a.email ?? a.accountId}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
<select
|
||||
className="h-6 max-w-[200px] rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={property}
|
||||
@@ -331,13 +455,19 @@ export function SearchConsoleTab() {
|
||||
</select>
|
||||
<select
|
||||
className="h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={days}
|
||||
onChange={(e) => setDays(Number(e.target.value) as RangeDays)}
|
||||
value={dateRange}
|
||||
onChange={(e) => changeDateRange(e.target.value as GscDateRange)}
|
||||
disabled={fetching}
|
||||
title={t('gscTab.dateRangeTitle', {
|
||||
defaultValue:
|
||||
'Date range for the pull. Full dimension filters live in Settings → Integrations → Google Search Console.',
|
||||
})}
|
||||
>
|
||||
<option value={7}>{t('gscTab.days7', { defaultValue: 'Last 7 days' })}</option>
|
||||
<option value={28}>{t('gscTab.days28', { defaultValue: 'Last 28 days' })}</option>
|
||||
<option value={90}>{t('gscTab.days90', { defaultValue: 'Last 90 days' })}</option>
|
||||
<option value="7d">{t('gscTab.days7', { defaultValue: 'Last 7 days' })}</option>
|
||||
<option value="28d">{t('gscTab.days28', { defaultValue: 'Last 28 days' })}</option>
|
||||
<option value="90d">{t('gscTab.days90', { defaultValue: 'Last 90 days' })}</option>
|
||||
<option value="16m">{t('gscTab.days16m', { defaultValue: 'Last 16 months' })}</option>
|
||||
<option value="custom">{t('gscTab.daysCustom', { defaultValue: 'Custom (Settings)' })}</option>
|
||||
</select>
|
||||
<button
|
||||
type="button"
|
||||
@@ -384,12 +514,52 @@ export function SearchConsoleTab() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* New-URLs-in-GSC banner — offer to crawl orphan GSC URLs */}
|
||||
{newUrls && newUrls.count > 0 && (
|
||||
<div className="flex items-center gap-2 border-b border-amber-700/40 bg-amber-900/15 px-3 py-1 text-[10px] text-amber-200">
|
||||
<span>
|
||||
{newUrls.queued > 0
|
||||
? t('gscTab.newUrlsQueued', {
|
||||
defaultValue:
|
||||
'{{n}} new URL(s) discovered in Search Console were added to the crawl.',
|
||||
n: newUrls.queued,
|
||||
})
|
||||
: t('gscTab.newUrlsFound', {
|
||||
defaultValue:
|
||||
'{{n}} URL(s) appear in Search Console but were not found by the crawl.',
|
||||
n: newUrls.count,
|
||||
})}
|
||||
</span>
|
||||
{newUrls.queued === 0 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void crawlNew()}
|
||||
disabled={crawlingNew}
|
||||
className="ml-auto inline-flex items-center gap-1 rounded bg-amber-600/80 px-2 py-0.5 text-[10px] font-medium text-white hover:bg-amber-600 disabled:opacity-50"
|
||||
>
|
||||
{crawlingNew ? (
|
||||
<Loader2 size={11} className="animate-spin" />
|
||||
) : (
|
||||
<Plus size={11} />
|
||||
)}
|
||||
{t('gscTab.crawlNewUrls', {
|
||||
defaultValue: 'Crawl {{n}} new URLs',
|
||||
n: newUrls.count,
|
||||
})}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Context line — connection + last pull */}
|
||||
<div className="flex items-center gap-2 border-b border-surface-800 bg-surface-900/20 px-3 py-1 text-[10px] text-surface-500">
|
||||
<span>
|
||||
{t('gscTab.connectedAs', {
|
||||
defaultValue: 'Connected as {{email}}',
|
||||
email: authState?.email ?? 'Google account',
|
||||
email:
|
||||
accounts.find((a) => a.accountId === accountId)?.email ??
|
||||
authState?.email ??
|
||||
'Google account',
|
||||
})}
|
||||
</span>
|
||||
<button
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/mcp-server",
|
||||
"version": "0.9.2",
|
||||
"version": "0.9.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "freecrawl-seo",
|
||||
"version": "0.9.2",
|
||||
"version": "0.9.3",
|
||||
"private": true,
|
||||
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
|
||||
"license": "MIT",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/core",
|
||||
"version": "0.9.2",
|
||||
"version": "0.9.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
+111
-13
@@ -236,22 +236,75 @@ export class Crawler extends EventEmitter {
|
||||
/** Pending queue snapshot used by the checkpoint timer. Mirrors
|
||||
* what's in `this.queue` minus already-completed items. */
|
||||
private pendingItems = new Map<string, number>();
|
||||
/**
|
||||
* Enqueues and completions accumulated since the last checkpoint.
|
||||
* Shipping this delta instead of the whole of `pendingItems` is what
|
||||
* keeps the checkpoint proportional to crawl throughput: on a crawl
|
||||
* with a 200k-URL frontier the old full snapshot rebuilt a 200k-entry
|
||||
* array on the main thread, structured-cloned it to the writer worker
|
||||
* and re-inserted every row twice a minute, which blocked the writer
|
||||
* FIFO the per-URL page writes share and stalled the crawl loop for
|
||||
* tens of seconds each pass. The delta is a few hundred rows.
|
||||
*/
|
||||
private queueAdded = new Map<string, number>();
|
||||
private queueRemoved = new Set<string>();
|
||||
/** First tick of a run does a full replace so the table starts in a
|
||||
* known state; set again after a failed write to resynchronise. */
|
||||
private queueCheckpointNeedsFull = true;
|
||||
private startQueueCheckpointTimer(): void {
|
||||
if (this.queueCheckpointTimer) return;
|
||||
// A resumed run inherits rows written by the previous process, and
|
||||
// a fresh run may inherit a discarded checkpoint, so the first tick
|
||||
// always rebuilds from `pendingItems` rather than trusting the
|
||||
// table to match.
|
||||
this.queueCheckpointNeedsFull = true;
|
||||
this.queueAdded.clear();
|
||||
this.queueRemoved.clear();
|
||||
this.queueCheckpointTimer = setInterval(() => {
|
||||
if (this.stopped || !this.running) return;
|
||||
const items = Array.from(this.pendingItems.entries()).map(([url, depth]) => ({
|
||||
url,
|
||||
depth,
|
||||
}));
|
||||
void this.dbCall<void>('checkpointQueue', [items, this.config.startUrl]).catch((err) => {
|
||||
this.emit(
|
||||
'debug',
|
||||
`queue checkpoint failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
void this.flushQueueCheckpoint();
|
||||
// Piggyback the planner-statistics refresh on the same tick. A
|
||||
// crawl starts with empty tables, so any plan chosen at row zero
|
||||
// assumes tables that never grow; `PRAGMA optimize` re-analyses
|
||||
// only what has actually drifted, so this is a no-op most ticks
|
||||
// and tens of milliseconds when it isn't.
|
||||
void this.dbCall<void>('optimize', []).catch(() => {
|
||||
/* statistics are an optimisation, never a correctness input */
|
||||
});
|
||||
}, Crawler.QUEUE_CHECKPOINT_INTERVAL_MS);
|
||||
}
|
||||
private async flushQueueCheckpoint(): Promise<void> {
|
||||
const full = this.queueCheckpointNeedsFull;
|
||||
if (!full && this.queueAdded.size === 0 && this.queueRemoved.size === 0) return;
|
||||
const patch = full
|
||||
? {
|
||||
replaceAll: true,
|
||||
added: Array.from(this.pendingItems, ([url, depth]) => ({ url, depth })),
|
||||
removed: [],
|
||||
}
|
||||
: {
|
||||
replaceAll: false,
|
||||
added: Array.from(this.queueAdded, ([url, depth]) => ({ url, depth })),
|
||||
removed: Array.from(this.queueRemoved),
|
||||
};
|
||||
// Clear before awaiting so events arriving during the write land in
|
||||
// the next delta instead of being written twice.
|
||||
this.queueAdded.clear();
|
||||
this.queueRemoved.clear();
|
||||
this.queueCheckpointNeedsFull = false;
|
||||
try {
|
||||
await this.dbCall<void>('checkpointQueue', [patch, this.config.startUrl]);
|
||||
} catch (err) {
|
||||
// The delta is already gone, so fall back to a full rebuild next
|
||||
// tick — otherwise the checkpoint would silently drift from the
|
||||
// live queue and a resume would replay the wrong set.
|
||||
this.queueCheckpointNeedsFull = true;
|
||||
this.emit(
|
||||
'debug',
|
||||
`queue checkpoint failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
private stopQueueCheckpointTimer(): void {
|
||||
if (this.queueCheckpointTimer) {
|
||||
clearInterval(this.queueCheckpointTimer);
|
||||
@@ -952,6 +1005,18 @@ export class Crawler extends EventEmitter {
|
||||
* — see rule 1.7 (root-cause perf, never mask by degrading UX).
|
||||
*/
|
||||
private async runPostCrawlPasses(): Promise<void> {
|
||||
// Refresh the planner's statistics before anything else runs. Every
|
||||
// pass below is a correlated-subquery / self-join workload, and the
|
||||
// tables they read only reached their final size moments ago — the
|
||||
// plans SQLite would pick from stale (or absent) stats are the ones
|
||||
// that turn a sub-second pass into a multi-minute one.
|
||||
await yieldToEventLoop();
|
||||
this.setOp('post-crawl:optimize');
|
||||
try {
|
||||
await this.dbCall<void>('optimize', []);
|
||||
} catch {
|
||||
/* statistics are an optimisation, never a correctness input */
|
||||
}
|
||||
if (this.config.analyseInlinks) {
|
||||
await yieldToEventLoop();
|
||||
this.emit('info', 'Recomputing inlinks…');
|
||||
@@ -1740,10 +1805,18 @@ export class Crawler extends EventEmitter {
|
||||
afterReset?: () => Promise<void>,
|
||||
): Promise<void> {
|
||||
const previousStart = this.db.getMeta('startUrl');
|
||||
if (previousStart !== fingerprint) {
|
||||
this.db.reset();
|
||||
// A crash-recovery resume continues the interrupted crawl, so its rows
|
||||
// must survive even though the fingerprint may no longer match — a
|
||||
// sitemap that gained or lost a single URL changes the count baked into
|
||||
// it, and wiping there would destroy exactly the data the user asked to
|
||||
// keep. The fingerprint is left pointing at the list that was actually
|
||||
// crawled, so a later Start still reads a drifted list as a new crawl.
|
||||
if (!this.resumeOnly) {
|
||||
if (previousStart !== fingerprint) {
|
||||
this.db.reset();
|
||||
}
|
||||
this.db.setMeta('startUrl', fingerprint);
|
||||
}
|
||||
this.db.setMeta('startUrl', fingerprint);
|
||||
if (afterReset) await afterReset();
|
||||
|
||||
// Force exact-url scope so anything fetched in fetchAndProcess never
|
||||
@@ -1762,6 +1835,22 @@ export class Crawler extends EventEmitter {
|
||||
this.startIssueRecomputeTimer();
|
||||
this.startQueueCheckpointTimer();
|
||||
|
||||
// Resume: mark everything already in the project as seen so the loop
|
||||
// below only picks up what is genuinely still outstanding. Without it a
|
||||
// resumed List/Sitemap run re-fetches the whole source list — the exact
|
||||
// "it didn't carry on where it left off" the checkpoint exists to
|
||||
// prevent. Spider mode gets the same treatment from `hydrateFromDb`.
|
||||
// The checkpointed pending URLs are already queued by the caller
|
||||
// (`enqueueCheckpointed`) before `start()`, so they survive this sweep;
|
||||
// list entries added since the interruption are still picked up below.
|
||||
if (this.resumeOnly) {
|
||||
for (const known of this.db.getAllUrls()) {
|
||||
this.seen.add(known);
|
||||
}
|
||||
this.crawled = this.db.countCrawledUrls();
|
||||
this.resourceAdmitted = this.db.countSubresourceUrls();
|
||||
}
|
||||
|
||||
for (const u of urls) {
|
||||
this.enqueue({ url: u, depth: 0 });
|
||||
}
|
||||
@@ -2568,6 +2657,8 @@ export class Crawler extends EventEmitter {
|
||||
if (item.resourceRow) this.resourceAdmitted++;
|
||||
this.pending++;
|
||||
this.pendingItems.set(item.url, item.depth);
|
||||
this.queueRemoved.delete(item.url);
|
||||
this.queueAdded.set(item.url, item.depth);
|
||||
this.queue
|
||||
.add(() => this.fetchAndProcess(item), { priority: Crawler.priorityFor(item) })
|
||||
.catch((err: unknown) => {
|
||||
@@ -2582,6 +2673,8 @@ export class Crawler extends EventEmitter {
|
||||
// failures are recorded in the urls table and shouldn't be
|
||||
// retried by a resumed crawl.
|
||||
this.pendingItems.delete(item.url);
|
||||
this.queueAdded.delete(item.url);
|
||||
this.queueRemoved.add(item.url);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3268,7 +3361,12 @@ export class Crawler extends EventEmitter {
|
||||
}
|
||||
|
||||
const xRobotsLower = xRobotsTag?.toLowerCase() ?? '';
|
||||
const headerNoindex = xRobotsLower.includes('noindex');
|
||||
// Same `none` shorthand as the meta tag, plus the header's optional
|
||||
// `<bot>: <directives>` form (`X-Robots-Tag: googlebot: none`) — the
|
||||
// colon is a token separator here, not part of a directive.
|
||||
const headerNoindex =
|
||||
xRobotsLower.includes('noindex') ||
|
||||
xRobotsLower.split(/[\s,;:]+/).includes('none');
|
||||
|
||||
let indexability: Indexability = 'indexable';
|
||||
let reason: string | null = null;
|
||||
|
||||
@@ -57,6 +57,12 @@ export interface ParsedPage {
|
||||
canonical: string | null;
|
||||
/** Number of `<link rel="canonical">` elements declared. >1 is a confusion signal. */
|
||||
canonicalCount: number;
|
||||
/**
|
||||
* Every `<meta name="robots">` content value on the page, lowercased
|
||||
* and joined with `, ` in document order. Multiple declarations are
|
||||
* kept because search engines union them and honour the most
|
||||
* restrictive directive.
|
||||
*/
|
||||
metaRobots: string | null;
|
||||
lang: string | null;
|
||||
viewport: string | null;
|
||||
@@ -524,7 +530,28 @@ export function parseHtml(
|
||||
const canonicalEls = $('link[rel="canonical"]');
|
||||
const canonical = (canonicalEls.first().attr('href') ?? '').trim() || null;
|
||||
const canonicalCount = canonicalEls.length;
|
||||
const metaRobots = ($('meta[name="robots"]').attr('content') ?? '').trim().toLowerCase() || null;
|
||||
// Robots directives — collected from EVERY `<meta name="robots">` in
|
||||
// the document, not just the first. Two details that a plain
|
||||
// `$('meta[name="robots"]').attr('content')` gets wrong on real sites:
|
||||
//
|
||||
// 1. `name` is matched case-insensitively. CSS attribute *values* are
|
||||
// case-sensitive outside a small HTML whitelist that `name` is not
|
||||
// part of, so `<meta name="ROBOTS">` matched nothing at all and the
|
||||
// page came back "Indexable" with an empty Meta Robots column.
|
||||
// 2. A page may declare the tag more than once — a theme default plus
|
||||
// an SEO plugin's own output is the classic WordPress case. Search
|
||||
// engines union the directives and apply the most restrictive one,
|
||||
// so reading only the first tag reported `index,follow` + `noindex`
|
||||
// pages as indexable. The contents are joined (comma-separated, in
|
||||
// document order) into one value: the UI's Meta Robots column then
|
||||
// shows the conflict instead of hiding half of it.
|
||||
const robotsMetaContents: string[] = [];
|
||||
$('meta').each((_, el) => {
|
||||
if (($(el).attr('name') ?? '').trim().toLowerCase() !== 'robots') return;
|
||||
const content = ($(el).attr('content') ?? '').trim().toLowerCase();
|
||||
if (content) robotsMetaContents.push(content);
|
||||
});
|
||||
const metaRobots = robotsMetaContents.length > 0 ? robotsMetaContents.join(', ') : null;
|
||||
const lang = ($('html').attr('lang') ?? '').trim() || null;
|
||||
const viewport = ($('meta[name="viewport"]').attr('content') ?? '').trim() || null;
|
||||
const ogTitle =
|
||||
@@ -1076,8 +1103,17 @@ export function parseHtml(
|
||||
}
|
||||
}
|
||||
|
||||
const hasNoindex = metaRobots !== null && metaRobots.includes('noindex');
|
||||
const hasNofollow = metaRobots !== null && metaRobots.includes('nofollow');
|
||||
// `none` is the spec's shorthand for `noindex, nofollow` — a page
|
||||
// carrying only that directive used to slip through as indexable. It is
|
||||
// matched as a whole token (split on comma / semicolon / whitespace so
|
||||
// malformed separators still tokenize) rather than as a substring,
|
||||
// which would fire on unrelated values. The substring check on the
|
||||
// combined string is kept alongside it so any separator we haven't
|
||||
// thought of still yields the old, lenient detection.
|
||||
const robotsTokens = new Set((metaRobots ?? '').split(/[\s,;]+/).filter(Boolean));
|
||||
const hasNone = robotsTokens.has('none');
|
||||
const hasNoindex = metaRobots !== null && (metaRobots.includes('noindex') || hasNone);
|
||||
const hasNofollow = metaRobots !== null && (metaRobots.includes('nofollow') || hasNone);
|
||||
|
||||
// Text/code ratio — visible body text bytes divided by raw HTML bytes,
|
||||
// rounded to integer percent. Reuses the script/style/noscript-stripped
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/db",
|
||||
"version": "0.9.2",
|
||||
"version": "0.9.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -1984,6 +1984,132 @@ const MIGRATIONS: Migration[] = [
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 83,
|
||||
name: 'add_gsc_inspection_appearance_verdicts',
|
||||
// GSC URL Inspection returns more than the index-status verdict:
|
||||
// mobileUsabilityResult / ampResult / richResultsResult each carry
|
||||
// their own PASS/FAIL verdict. Storing them powers the "Page is Not
|
||||
// Mobile Friendly", "AMP URL Invalid" and "Rich Result Invalid"
|
||||
// filter presets on the Search Console tab.
|
||||
//
|
||||
// Left NULL for rows inspected before these columns existed — absent
|
||||
// and "not inspected for this facet" are the same thing to consumers.
|
||||
up: (db) => {
|
||||
const cols = db
|
||||
.prepare('PRAGMA table_info(gsc_inspection_results)')
|
||||
.all() as unknown as { name: string }[];
|
||||
if (!cols.some((c) => c.name === 'mobile_verdict')) {
|
||||
db.exec('ALTER TABLE gsc_inspection_results ADD COLUMN mobile_verdict TEXT');
|
||||
}
|
||||
if (!cols.some((c) => c.name === 'amp_verdict')) {
|
||||
db.exec('ALTER TABLE gsc_inspection_results ADD COLUMN amp_verdict TEXT');
|
||||
}
|
||||
if (!cols.some((c) => c.name === 'rich_results_verdict')) {
|
||||
db.exec(
|
||||
'ALTER TABLE gsc_inspection_results ADD COLUMN rich_results_verdict TEXT',
|
||||
);
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 84,
|
||||
name: 'add_google_account_id',
|
||||
// Multi-account GSC / GA4: a user can link several Google accounts to
|
||||
// the same integration (their own property plus a client's), so these
|
||||
// three snapshot tables can no longer be keyed by `url` alone — two
|
||||
// accounts may legitimately report on the same URL.
|
||||
//
|
||||
// SQLite cannot ALTER a PRIMARY KEY, so each table is rebuilt with the
|
||||
// composite `(url, account_id)` key and its rows copied across.
|
||||
// Pre-existing rows came from the single connected account and get
|
||||
// `account_id = ''`; the main process adopts them into that account's
|
||||
// id the first time the account list is resolved, so an upgrade keeps
|
||||
// its data instead of silently showing an empty table.
|
||||
up: (db) => {
|
||||
const hasAccountCol = (table: string): boolean =>
|
||||
(db.prepare(`PRAGMA table_info(${table})`).all() as unknown as {
|
||||
name: string;
|
||||
}[]).some((c) => c.name === 'account_id');
|
||||
|
||||
if (!hasAccountCol('gsc_results')) {
|
||||
db.exec(`
|
||||
CREATE TABLE gsc_results_new (
|
||||
url TEXT NOT NULL,
|
||||
account_id TEXT NOT NULL DEFAULT '',
|
||||
clicks INTEGER,
|
||||
impressions INTEGER,
|
||||
ctr REAL,
|
||||
position REAL,
|
||||
fetched_at TEXT NOT NULL,
|
||||
PRIMARY KEY (url, account_id)
|
||||
);
|
||||
INSERT INTO gsc_results_new (url, account_id, clicks, impressions, ctr, position, fetched_at)
|
||||
SELECT url, '', clicks, impressions, ctr, position, fetched_at FROM gsc_results;
|
||||
DROP TABLE gsc_results;
|
||||
ALTER TABLE gsc_results_new RENAME TO gsc_results;
|
||||
CREATE INDEX IF NOT EXISTS idx_gsc_results_account ON gsc_results(account_id);
|
||||
`);
|
||||
}
|
||||
|
||||
if (!hasAccountCol('ga4_results')) {
|
||||
db.exec(`
|
||||
CREATE TABLE ga4_results_new (
|
||||
url TEXT NOT NULL,
|
||||
account_id TEXT NOT NULL DEFAULT '',
|
||||
sessions INTEGER,
|
||||
users INTEGER,
|
||||
pageviews INTEGER,
|
||||
engagement_rate REAL,
|
||||
avg_session_duration REAL,
|
||||
fetched_at TEXT NOT NULL,
|
||||
PRIMARY KEY (url, account_id)
|
||||
);
|
||||
INSERT INTO ga4_results_new (url, account_id, sessions, users, pageviews,
|
||||
engagement_rate, avg_session_duration, fetched_at)
|
||||
SELECT url, '', sessions, users, pageviews,
|
||||
engagement_rate, avg_session_duration, fetched_at FROM ga4_results;
|
||||
DROP TABLE ga4_results;
|
||||
ALTER TABLE ga4_results_new RENAME TO ga4_results;
|
||||
CREATE INDEX IF NOT EXISTS idx_ga4_results_account ON ga4_results(account_id);
|
||||
`);
|
||||
}
|
||||
|
||||
if (!hasAccountCol('gsc_inspection_results')) {
|
||||
db.exec(`
|
||||
CREATE TABLE gsc_inspection_results_new (
|
||||
url TEXT NOT NULL,
|
||||
account_id TEXT NOT NULL DEFAULT '',
|
||||
verdict TEXT,
|
||||
coverage_state TEXT,
|
||||
robots_txt_state TEXT,
|
||||
indexing_state TEXT,
|
||||
last_crawl_time TEXT,
|
||||
google_canonical TEXT,
|
||||
user_canonical TEXT,
|
||||
mobile_verdict TEXT,
|
||||
amp_verdict TEXT,
|
||||
rich_results_verdict TEXT,
|
||||
status TEXT NOT NULL,
|
||||
error TEXT,
|
||||
fetched_at TEXT NOT NULL,
|
||||
PRIMARY KEY (url, account_id)
|
||||
);
|
||||
INSERT INTO gsc_inspection_results_new
|
||||
(url, account_id, verdict, coverage_state, robots_txt_state, indexing_state,
|
||||
last_crawl_time, google_canonical, user_canonical,
|
||||
mobile_verdict, amp_verdict, rich_results_verdict, status, error, fetched_at)
|
||||
SELECT url, '', verdict, coverage_state, robots_txt_state, indexing_state,
|
||||
last_crawl_time, google_canonical, user_canonical,
|
||||
mobile_verdict, amp_verdict, rich_results_verdict, status, error, fetched_at
|
||||
FROM gsc_inspection_results;
|
||||
DROP TABLE gsc_inspection_results;
|
||||
ALTER TABLE gsc_inspection_results_new RENAME TO gsc_inspection_results;
|
||||
CREATE INDEX IF NOT EXISTS idx_gsc_inspection_account ON gsc_inspection_results(account_id);
|
||||
`);
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export function runMigrations(db: DatabaseSync): void {
|
||||
|
||||
+638
-212
File diff suppressed because it is too large.
Load diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/shared-types",
|
||||
"version": "0.9.2",
|
||||
"version": "0.9.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -10,14 +10,33 @@
|
||||
* encrypted at rest with `safeStorage`, exactly like API keys.
|
||||
*/
|
||||
|
||||
/**
|
||||
* One connected Google account for an integration. A user can link
|
||||
* several accounts to the same integration (e.g. their own Search
|
||||
* Console plus a client's), each with its own refresh token; the
|
||||
* per-project settings pick which one a crawl's data comes from.
|
||||
*/
|
||||
export interface GoogleAccount {
|
||||
/** Opaque stable id — the key the token store and `account_id`
|
||||
* columns use. Never the email, so a renamed/aliased address can't
|
||||
* orphan the stored rows. */
|
||||
accountId: string;
|
||||
/** The Google account address, when the userinfo call succeeded. */
|
||||
email: string | null;
|
||||
/** ISO timestamp this account was connected. */
|
||||
connectedAt: string;
|
||||
}
|
||||
|
||||
/** Connection state of one OAuth-BYOC Google integration. */
|
||||
export interface GoogleAuthState {
|
||||
/** True once a refresh token is stored — the integration is connected. */
|
||||
/** True once at least one account is connected. */
|
||||
connected: boolean;
|
||||
/** The Google account email the tokens belong to, when known. */
|
||||
/** First connected account's email — kept for single-account callers. */
|
||||
email: string | null;
|
||||
/** ISO timestamp the connection was established. */
|
||||
/** ISO timestamp the first connection was established. */
|
||||
connectedAt: string | null;
|
||||
/** Every connected account, in connection order. */
|
||||
accounts: GoogleAccount[];
|
||||
}
|
||||
|
||||
/** Result of an interactive `googleAuthStart` consent flow. */
|
||||
@@ -73,11 +92,125 @@ export interface GscInspectionResult {
|
||||
googleCanonical: string | null;
|
||||
/** The canonical the page declared. */
|
||||
userCanonical: string | null;
|
||||
/** Mobile Usability verdict — `PASS` / `FAIL` / `NEUTRAL` /
|
||||
* `VERDICT_UNSPECIFIED`. Null when inspection predates this column. */
|
||||
mobileVerdict: string | null;
|
||||
/** AMP result verdict, when the page declares an AMP version. */
|
||||
ampVerdict: string | null;
|
||||
/** Rich-results verdict across all detected structured-data items. */
|
||||
richResultsVerdict: string | null;
|
||||
status: 'ok' | 'error';
|
||||
error: string | null;
|
||||
fetchedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Quick-filter presets for the Search Console tab, mirroring Screaming
|
||||
* Frog's "verified in GSC" filter dropdown. `all` / `with-data` /
|
||||
* `without-data` are the legacy values the MCP `query_gsc` tool still
|
||||
* sends; the rest are the SF-parity presets. Presets past
|
||||
* `non-indexable-with-data` need URL Inspection data to have been run;
|
||||
* `orphan` widens the candidate set to GSC URLs missing from the crawl.
|
||||
*/
|
||||
export type GscPreset =
|
||||
| 'all'
|
||||
| 'with-data'
|
||||
| 'without-data'
|
||||
| 'clicks-above-0'
|
||||
| 'non-indexable-with-data'
|
||||
| 'orphan'
|
||||
| 'not-on-google'
|
||||
| 'indexable-not-indexed'
|
||||
| 'on-google-with-issues'
|
||||
| 'canonical-mismatch'
|
||||
| 'not-mobile-friendly'
|
||||
| 'amp-invalid'
|
||||
| 'rich-result-invalid';
|
||||
|
||||
/** Date-range presets for a GSC pull. `16m` is the API's 16-month max;
|
||||
* `custom` uses the explicit `startDate` / `endDate`. */
|
||||
export type GscDateRange = '7d' | '28d' | '90d' | '16m' | 'custom';
|
||||
/** Device dimension filter — API values (or `all` for no filter). */
|
||||
export type GscDeviceFilter = 'all' | 'DESKTOP' | 'MOBILE' | 'TABLET';
|
||||
/** Search-appearance type the metrics are pulled for. */
|
||||
export type GscSearchType =
|
||||
| 'web'
|
||||
| 'image'
|
||||
| 'video'
|
||||
| 'news'
|
||||
| 'discover'
|
||||
| 'googleNews';
|
||||
/** Query dimension filter operator. */
|
||||
export type GscQueryFilterMode =
|
||||
| 'none'
|
||||
| 'contains'
|
||||
| 'notContains'
|
||||
| 'equals'
|
||||
| 'notEquals'
|
||||
| 'includingRegex'
|
||||
| 'excludingRegex';
|
||||
|
||||
/**
|
||||
* Per-project Google Search Console integration settings — the FreeCrawl
|
||||
* equivalent of Screaming Frog's "API Access → Google Search Console"
|
||||
* dialog. Stored per project (encrypted credentials stay separate); the
|
||||
* fetch reads these to shape the Search Analytics query and to decide
|
||||
* whether newly-discovered GSC URLs are crawled.
|
||||
*/
|
||||
export interface GscIntegrationSettings {
|
||||
/** Which connected Google account this project pulls from. Empty means
|
||||
* "the first connected account" — the state every pre-multi-account
|
||||
* project is in. */
|
||||
accountId: string;
|
||||
/** Last-selected Search Console property (`siteUrl`). */
|
||||
property: string;
|
||||
dateRange: GscDateRange;
|
||||
/** `YYYY-MM-DD`, only used when `dateRange === 'custom'`. */
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
deviceFilter: GscDeviceFilter;
|
||||
/** ISO-3166-1 alpha-3 country code, lowercase (e.g. `usa`); '' = none. */
|
||||
countryFilter: string;
|
||||
searchType: GscSearchType;
|
||||
queryFilterMode: GscQueryFilterMode;
|
||||
queryFilterValue: string;
|
||||
/** Match trailing vs non-trailing-slash URL variants when comparing
|
||||
* GSC URLs against crawled URLs (SF: "Match Trailing and Non-Trailing
|
||||
* Slash URLs"). */
|
||||
matchSlash: boolean;
|
||||
/** Match uppercase vs lowercase URL variants (SF: "Match Uppercase &
|
||||
* Lowercase URLs"). */
|
||||
matchCase: boolean;
|
||||
/** Cap the number of rows pulled (SF: "Limit Max Results"). */
|
||||
limitMaxResults: boolean;
|
||||
maxResults: number;
|
||||
/** SF: "Crawl New URLs Discovered In Google Search Console" — after a
|
||||
* pull, feed GSC URLs that aren't in the crawl into the crawler. */
|
||||
crawlNewUrls: boolean;
|
||||
}
|
||||
|
||||
/** Factory for the default GSC settings (a fresh object each call so the
|
||||
* shared default can't be mutated). */
|
||||
export function defaultGscSettings(): GscIntegrationSettings {
|
||||
return {
|
||||
accountId: '',
|
||||
property: '',
|
||||
dateRange: '28d',
|
||||
startDate: '',
|
||||
endDate: '',
|
||||
deviceFilter: 'all',
|
||||
countryFilter: '',
|
||||
searchType: 'web',
|
||||
queryFilterMode: 'none',
|
||||
queryFilterValue: '',
|
||||
matchSlash: true,
|
||||
matchCase: false,
|
||||
limitMaxResults: true,
|
||||
maxResults: 100000,
|
||||
crawlNewUrls: false,
|
||||
};
|
||||
}
|
||||
|
||||
/** Metadata describing the most recent GSC pull for the active project. */
|
||||
export interface GscFetchMeta {
|
||||
/** The Search Console property the data was pulled from. */
|
||||
@@ -130,6 +263,29 @@ export interface UrlAnalyticsDetail {
|
||||
inspection: GscInspectionResult | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-project Google Analytics 4 integration settings. Smaller than the
|
||||
* GSC equivalent — the Data API call takes no dimension filters here —
|
||||
* but it carries the same account/property selection so a project
|
||||
* remembers which of several linked GA4 accounts it reports on.
|
||||
*/
|
||||
export interface Ga4IntegrationSettings {
|
||||
/** Which connected Google account this project pulls from. Empty means
|
||||
* "the first connected account". */
|
||||
accountId: string;
|
||||
/** Last-selected property resource name (`properties/<id>`). */
|
||||
property: string;
|
||||
/** Friendly label for that property, for the fetch meta line. */
|
||||
propertyName: string;
|
||||
/** Trailing window in days. */
|
||||
days: 7 | 28 | 90;
|
||||
}
|
||||
|
||||
/** Factory for the default GA4 settings (fresh object each call). */
|
||||
export function defaultGa4Settings(): Ga4IntegrationSettings {
|
||||
return { accountId: '', property: '', propertyName: '', days: 28 };
|
||||
}
|
||||
|
||||
/** Metadata describing the most recent GA4 pull for the active project. */
|
||||
export interface Ga4FetchMeta {
|
||||
/** API resource name (`properties/<id>`) the data was pulled from. */
|
||||
|
||||
@@ -2,6 +2,7 @@ import type {
|
||||
AdvancedFilter,
|
||||
BrokenLinkRow,
|
||||
CrawlConfig,
|
||||
CrawlMode,
|
||||
CrawlProgress,
|
||||
CrawlSummary,
|
||||
CrawlUrlRow,
|
||||
@@ -35,9 +36,11 @@ import type { RecentProject } from './project.js';
|
||||
import type {
|
||||
GoogleAuthState,
|
||||
GoogleAuthResult,
|
||||
GoogleAccount,
|
||||
GscSite,
|
||||
GscRow,
|
||||
GscFetchMeta,
|
||||
GscPreset,
|
||||
Ga4Property,
|
||||
Ga4Row,
|
||||
Ga4FetchMeta,
|
||||
@@ -265,6 +268,16 @@ export const IPC = {
|
||||
integrationsGetAll: 'integrations:get-all',
|
||||
integrationsSet: 'integrations:set',
|
||||
integrationsClear: 'integrations:clear',
|
||||
/** Per-project, per-integration behaviour settings (distinct from the
|
||||
* encrypted credential store). `get` reads the stored JSON blob for
|
||||
* one integration id (or null); `set` merges the supplied object.
|
||||
* Currently used by Google Search Console (date range, dimension
|
||||
* filters, "crawl new URLs"). */
|
||||
integrationSettingsGet: 'integrations:settings-get',
|
||||
integrationSettingsSet: 'integrations:settings-set',
|
||||
/** Feed GSC-discovered URLs missing from the crawl into the crawler
|
||||
* (SF: "Crawl New URLs Discovered In Google Search Console"). */
|
||||
gscCrawlNewUrls: 'gsc:crawl-new-urls',
|
||||
/** Open a native directory picker. Used by Settings → Storage to let the
|
||||
* user choose the default folder for new `.seoproject` files. */
|
||||
pickDirectory: 'app:pick-directory',
|
||||
@@ -317,6 +330,9 @@ export const IPC = {
|
||||
googleAuthStart: 'google:auth-start',
|
||||
googleAuthStatus: 'google:auth-status',
|
||||
googleAuthRevoke: 'google:auth-revoke',
|
||||
/** List every Google account linked to one integration — a user can
|
||||
* connect several (their own property plus a client's). */
|
||||
googleAccountsList: 'google:accounts-list',
|
||||
/** Faz 7 — Google Search Console. `gscListSites` lists the connected
|
||||
* account's verified properties; `gscFetch` pulls per-page clicks /
|
||||
* impressions / CTR / position for a date range; `gscQuery` lists
|
||||
@@ -650,6 +666,18 @@ export interface CrashRecoveryStatus {
|
||||
pendingCount: number;
|
||||
/** The start URL the previous crawl was running against. */
|
||||
seedUrl: string;
|
||||
/** Mode of the interrupted crawl, so the prompt can say whether it
|
||||
* was a Spider / List / Sitemap run instead of only naming a URL. */
|
||||
mode: CrawlMode;
|
||||
}
|
||||
|
||||
/** Outcome of a resume request. `config` is the crawl config the run
|
||||
* was restarted with — the renderer applies it so the toolbar shows
|
||||
* the mode + start URL that are actually being crawled rather than
|
||||
* the blank Spider defaults it booted with. */
|
||||
export interface CrashRecoveryResumeResult {
|
||||
accepted: boolean;
|
||||
config: CrawlConfig | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1462,8 +1490,13 @@ export interface GscListSitesResult {
|
||||
export interface GscFetchInput {
|
||||
/** Search Console property to pull from (`siteUrl` from `GscSite`). */
|
||||
property: string;
|
||||
/** Trailing window in days (Search Console data lags ~2-3 days). */
|
||||
days: 7 | 28 | 90;
|
||||
/** Which connected Google account to pull with. Omitted / empty falls
|
||||
* back to the project's configured account, then the first one. */
|
||||
accountId?: string;
|
||||
/** Legacy trailing window in days — used only by the MCP `gsc_fetch`
|
||||
* tool. The desktop pull reads the date range from the stored GSC
|
||||
* integration settings instead. */
|
||||
days?: 7 | 28 | 90;
|
||||
}
|
||||
|
||||
export interface GscFetchResult {
|
||||
@@ -1472,14 +1505,36 @@ export interface GscFetchResult {
|
||||
/** Page rows stored from this pull. */
|
||||
rowCount: number;
|
||||
meta: GscFetchMeta | null;
|
||||
/** GSC URLs from this pull that aren't in the crawl (orphan candidates
|
||||
* for "Crawl New URLs"). Absent on older callers. */
|
||||
newUrlCount?: number;
|
||||
/** How many of those were queued into the crawler because the
|
||||
* "crawl new URLs" setting is on (0 when the setting is off). */
|
||||
queuedNewUrls?: number;
|
||||
}
|
||||
|
||||
export interface GscQueryInput {
|
||||
limit: number;
|
||||
offset: number;
|
||||
search?: string;
|
||||
/** `all` (default), `with-data` (has GSC impressions), `without-data`. */
|
||||
filter?: 'all' | 'with-data' | 'without-data';
|
||||
/** Filter preset — see `GscPreset`. `all` (default). Legacy
|
||||
* `with-data` / `without-data` still accepted. */
|
||||
filter?: GscPreset;
|
||||
/** Which account's stored rows to join. Omitted / empty falls back to
|
||||
* the project's configured account, then the first one. */
|
||||
accountId?: string;
|
||||
}
|
||||
|
||||
export interface GscCrawlNewUrlsResult {
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
/** GSC URLs found missing from the crawl. */
|
||||
candidateCount: number;
|
||||
/** How many were fed into the crawler (== candidateCount unless empty). */
|
||||
queued: number;
|
||||
/** True when the URLs were injected into a live crawl (vs. spinning up
|
||||
* a fresh re-spider pass). */
|
||||
hasActiveCrawl: boolean;
|
||||
}
|
||||
|
||||
export interface GscQueryResult {
|
||||
@@ -1502,6 +1557,9 @@ export interface Ga4FetchInput {
|
||||
/** Friendly label persisted into the fetch meta for the UI. */
|
||||
propertyName: string;
|
||||
days: 7 | 28 | 90;
|
||||
/** Which connected Google account to pull with. Omitted / empty falls
|
||||
* back to the project's configured account, then the first one. */
|
||||
accountId?: string;
|
||||
}
|
||||
|
||||
export interface Ga4FetchResult {
|
||||
@@ -1516,6 +1574,9 @@ export interface Ga4QueryInput {
|
||||
offset: number;
|
||||
search?: string;
|
||||
filter?: 'all' | 'with-data' | 'without-data';
|
||||
/** Which account's stored rows to join. Omitted / empty falls back to
|
||||
* the project's configured account, then the first one. */
|
||||
accountId?: string;
|
||||
}
|
||||
|
||||
export interface Ga4QueryResult {
|
||||
@@ -1601,6 +1662,9 @@ export interface GscInspectRunInput {
|
||||
/** Search Console property the URLs belong to. */
|
||||
property: string;
|
||||
urls: string[];
|
||||
/** Which connected Google account owns that property. Omitted / empty
|
||||
* falls back to the project's configured account. */
|
||||
accountId?: string;
|
||||
}
|
||||
|
||||
export interface GscInspectRunResult {
|
||||
@@ -1742,7 +1806,7 @@ export interface FreeCrawlApi {
|
||||
input: DataDeleteByDomainInput,
|
||||
): Promise<DataDeleteByDomainResult>;
|
||||
crashRecoveryStatus(): Promise<CrashRecoveryStatus>;
|
||||
crashRecoveryResume(): Promise<{ accepted: boolean }>;
|
||||
crashRecoveryResume(): Promise<CrashRecoveryResumeResult>;
|
||||
crashRecoveryDiscard(): Promise<void>;
|
||||
exportHtmlReport(input: ExportHtmlReportInput): Promise<ExportHtmlReportResult>;
|
||||
exportBulk(): Promise<BulkExportResult>;
|
||||
@@ -1877,18 +1941,33 @@ export interface FreeCrawlApi {
|
||||
* integration. Resolves once the user finishes (or cancels) in the
|
||||
* browser. */
|
||||
googleAuthStart(integrationId: string): Promise<GoogleAuthResult>;
|
||||
/** Read the stored OAuth connection state of one Google integration. */
|
||||
/** Read the stored OAuth connection state of one Google integration,
|
||||
* including every connected account. */
|
||||
googleAuthStatus(integrationId: string): Promise<GoogleAuthState>;
|
||||
/** Disconnect — wipe the stored OAuth tokens for one integration. */
|
||||
googleAuthRevoke(integrationId: string): Promise<GoogleAuthState>;
|
||||
/** List the connected account's Search Console properties. */
|
||||
gscListSites(): Promise<GscListSitesResult>;
|
||||
/** Disconnect one linked account, or every account for the integration
|
||||
* when `accountId` is omitted. */
|
||||
googleAuthRevoke(integrationId: string, accountId?: string): Promise<GoogleAuthState>;
|
||||
/** List a connected account's Search Console properties. Omitting
|
||||
* `accountId` uses the project's configured account. */
|
||||
gscListSites(accountId?: string): Promise<GscListSitesResult>;
|
||||
/** Pull per-page Search Console metrics for the given property + range. */
|
||||
gscFetch(input: GscFetchInput): Promise<GscFetchResult>;
|
||||
/** List crawled pages joined with their stored Search Console metrics. */
|
||||
gscQuery(input: GscQueryInput): Promise<GscQueryResult>;
|
||||
/** List the connected account's Google Analytics 4 properties. */
|
||||
ga4ListProperties(): Promise<Ga4ListPropertiesResult>;
|
||||
/** Feed GSC-discovered URLs missing from the crawl into the crawler
|
||||
* (SF: "Crawl New URLs Discovered In Google Search Console"). */
|
||||
gscCrawlNewUrls(): Promise<GscCrawlNewUrlsResult>;
|
||||
/** Read one integration's per-project behaviour settings (or null when
|
||||
* none stored). Currently used by Google Search Console. */
|
||||
integrationSettingsGet(id: string): Promise<Record<string, unknown> | null>;
|
||||
/** Merge-save one integration's per-project behaviour settings. */
|
||||
integrationSettingsSet(
|
||||
id: string,
|
||||
settings: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>>;
|
||||
/** List a connected account's Google Analytics 4 properties. Omitting
|
||||
* `accountId` uses the project's configured account. */
|
||||
ga4ListProperties(accountId?: string): Promise<Ga4ListPropertiesResult>;
|
||||
/** Pull per-page GA4 metrics for the given property + range. */
|
||||
ga4Fetch(input: Ga4FetchInput): Promise<Ga4FetchResult>;
|
||||
/** List crawled pages joined with their stored GA4 metrics. */
|
||||
@@ -1921,8 +2000,11 @@ export interface FreeCrawlApi {
|
||||
* GCP project. Uses the `bigquery` service-account credentials. */
|
||||
exportBigquery(input: ExportBigqueryInput): Promise<ExportBigqueryResult>;
|
||||
/** Read the combined Analytics detail (GSC + GA4 + URL Inspection)
|
||||
* for one URL. Returns null when the URL hasn't been crawled. */
|
||||
* for one URL, scoped to the project's configured accounts. Returns
|
||||
* null when the URL hasn't been crawled. */
|
||||
urlAnalyticsGet(url: string): Promise<UrlAnalyticsDetail | null>;
|
||||
/** List the Google accounts linked to one integration. */
|
||||
googleAccountsList(integrationId: string): Promise<GoogleAccount[]>;
|
||||
onLogEntry(cb: (entry: LogEntry) => void): () => void;
|
||||
onLogsBatch(cb: (entries: LogEntry[]) => void): () => void;
|
||||
onLogsBusy(cb: (busy: boolean) => void): () => void;
|
||||
|
||||
@@ -24,6 +24,12 @@ export type LogFormatChoice = LogFormat | 'auto';
|
||||
export interface LogAnalyzeInput {
|
||||
/** Absolute log file path. Omitted → the main process opens a picker. */
|
||||
filePath?: string;
|
||||
/**
|
||||
* Absolute log file paths for a bulk import (each analyzed + ingested in
|
||||
* turn). Takes precedence over `filePath`. When both are omitted the main
|
||||
* process opens a multi-select picker.
|
||||
*/
|
||||
filePaths?: string[];
|
||||
/** Dialect. Default `auto`. */
|
||||
format?: LogFormatChoice;
|
||||
/**
|
||||
@@ -99,8 +105,15 @@ export interface LogImportSummary {
|
||||
export interface LogAnalyzeResult {
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
/** Null when the user cancelled the file picker. */
|
||||
/**
|
||||
* Single-file convenience: the first file successfully ingested in this
|
||||
* call. Null when the user cancelled the picker or nothing was ingested.
|
||||
*/
|
||||
imported: LogImportSummary | null;
|
||||
/** Every file ingested in this (possibly bulk) call. */
|
||||
batch?: LogImportSummary[];
|
||||
/** Per-file failures that did not abort the rest of the batch. */
|
||||
errors?: Array<{ fileName: string; error: string }>;
|
||||
overview: LogOverview | null;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user