mirror of
https://github.com/kemalai/FreeCrawl-SEO-Tool.git
synced 2026-10-18 00:39:38 +02:00
feat: export dialog + MCP server for AI agents
This commit is contained in:
kemalai committed
1 parent
85230246d8
commit
cec358be9f
28 files changed
+2327
-174
No files matched your search
@@ -215,6 +215,97 @@ npm --workspace apps/desktop run build:linux # AppImage / .deb
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>🤖 MCP server</b> — query your crawl from Claude / any MCP client</summary>
|
||||
|
||||
<br />
|
||||
|
||||
FreeCrawl ships an **MCP (Model Context Protocol) server** that exposes the active `.seoproject` to AI agents over stdio. Read-only — runs alongside the desktop app without contention (SQLite WAL).
|
||||
|
||||
**Tools**: `get_summary`, `get_overview_counts`, `top_issues`, `query_urls`, `get_url_detail`, `list_projects`, `set_project`, `current_project`.
|
||||
|
||||
**1. Build it once:**
|
||||
|
||||
```bash
|
||||
npm run build:mcp
|
||||
```
|
||||
|
||||
This produces `apps/mcp-server/dist/index.js`.
|
||||
|
||||
**2. Register it with your MCP client.**
|
||||
|
||||
<details>
|
||||
<summary><b>Claude Desktop</b></summary>
|
||||
|
||||
Edit your Claude Desktop config:
|
||||
|
||||
| Platform | Path |
|
||||
| :--- | :--- |
|
||||
| Windows | `%APPDATA%\Claude\claude_desktop_config.json` |
|
||||
| macOS | `~/Library/Application Support/Claude/claude_desktop_config.json` |
|
||||
| Linux | `~/.config/Claude/claude_desktop_config.json` |
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"freecrawl": {
|
||||
"command": "node",
|
||||
"args": ["/absolute/path/to/FreeCrawl-SEO-Tool/apps/mcp-server/dist/index.js"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Restart Claude Desktop. The `freecrawl` server appears under the tool 🔌 icon.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Claude Code (CLI)</b></summary>
|
||||
|
||||
```bash
|
||||
claude mcp add freecrawl -- node /absolute/path/to/FreeCrawl-SEO-Tool/apps/mcp-server/dist/index.js
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Other MCP clients</b></summary>
|
||||
|
||||
Run the binary directly with stdio transport:
|
||||
|
||||
```bash
|
||||
node apps/mcp-server/dist/index.js
|
||||
```
|
||||
|
||||
The server speaks newline-delimited JSON-RPC 2.0 — point any MCP-compatible client at it.
|
||||
|
||||
</details>
|
||||
|
||||
**3. Try it.** Ask your agent things like:
|
||||
|
||||
> *"Show the 10 URLs with the longest response time in my last crawl."*
|
||||
> *"What are the top 5 issue categories with the most affected pages?"*
|
||||
> *"List every URL with a missing meta description."*
|
||||
|
||||
**Pointing at a non-default project:**
|
||||
|
||||
By default the server reads `<userData>/projects/default.seoproject` (the same file the desktop app uses). Override with the `FREECRAWL_PROJECT` env var, or call the `set_project` tool mid-session:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"freecrawl": {
|
||||
"command": "node",
|
||||
"args": ["/path/to/apps/mcp-server/dist/index.js"],
|
||||
"env": { "FREECRAWL_PROJECT": "/path/to/audit.seoproject" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<br />
|
||||
|
||||
---
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/cli",
|
||||
"version": "0.2.7",
|
||||
"version": "0.2.8",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/desktop",
|
||||
"version": "0.2.7",
|
||||
"version": "0.2.8",
|
||||
"private": true,
|
||||
"description": "FreeCrawl SEO — desktop application",
|
||||
"main": "./out/main/index.js",
|
||||
|
||||
@@ -4,43 +4,48 @@ import { fileURLToPath } from 'node:url';
|
||||
import * as logger from './logger.js';
|
||||
|
||||
/**
|
||||
* Pool of one worker_thread holding a read-only SQLite connection.
|
||||
* "Pool" is forward-compatible naming — for now we run a single worker
|
||||
* because SQLite's WAL allows arbitrary concurrent readers but adding a
|
||||
* second worker would just double our cache footprint without gaining
|
||||
* parallelism on a single physical disk.
|
||||
* Pool of N worker_threads, each holding its own read-only SQLite
|
||||
* connection. SQLite WAL mode allows arbitrary concurrent readers, so
|
||||
* spreading queries across multiple workers gives genuine parallelism
|
||||
* for UI reads — critical because:
|
||||
*
|
||||
* - The OverviewSidebar refresh issues a 130-clause `getOverviewCounts`
|
||||
* every few seconds. On a single-worker pool this serialises every
|
||||
* other read behind it, freezing the URL table refresh and the
|
||||
* bottom panel's per-URL detail / inlinks queries until the heavy
|
||||
* pass completes.
|
||||
* - Pool now lets the heavy aggregate run on one worker while the
|
||||
* short, latency-sensitive queries (urlsQuery for the visible
|
||||
* chunks, urlDetail for the bottom panel) run on the others.
|
||||
*
|
||||
* Routing: round-robin with a "least busy" tie-break — for each call
|
||||
* we pick the worker with the fewest pending requests, and on a tie
|
||||
* fall back to a rotating cursor so two same-loaded workers don't
|
||||
* always get the next call dropped on the same one.
|
||||
*
|
||||
* Lifecycle:
|
||||
* - `init(dbPath)` spawns the worker, points it at the file.
|
||||
* - `swap(newPath)` is called on Open Project — terminates the old
|
||||
* worker and spawns a fresh one; in-flight requests are rejected
|
||||
* - `init(dbPath)` spawns all workers, points them at the file.
|
||||
* - `swap(newPath)` is called on Open Project — terminates every old
|
||||
* worker and spawns a fresh set; in-flight requests are rejected
|
||||
* with `Error('reader-swapped')` so callers can retry safely.
|
||||
* - `terminate()` on app quit. After terminate, all `call()` resolve
|
||||
* with `Error('reader-terminated')`.
|
||||
*
|
||||
* Crash recovery:
|
||||
* - If the worker exits unexpectedly (`exit` event with non-zero code
|
||||
* while we still have a path) we auto-respawn up to MAX_RESTARTS
|
||||
* within RESTART_WINDOW_MS. After the cap we surface the failure
|
||||
* and stop trying — the user gets a clear log entry instead of a
|
||||
* restart loop.
|
||||
* - In-flight requests during a crash are rejected with
|
||||
* - If a worker exits unexpectedly, that one slot auto-respawns up to
|
||||
* MAX_RESTARTS within RESTART_WINDOW_MS. Other workers in the pool
|
||||
* keep serving traffic.
|
||||
* - In-flight requests on the crashed worker are rejected with
|
||||
* `Error('reader-crashed')`. Callers fall back to the main-process
|
||||
* ProjectDb (see `dbReaderCallOrFallback`) so the UI stays alive.
|
||||
* ProjectDb (see `callReaderOrFallback`) so the UI stays alive.
|
||||
*/
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
// The worker is built alongside the main bundle as `db-reader-worker.js`
|
||||
// (electron-vite produces flat output in `out/main/`). In dev mode
|
||||
// electron-vite uses the same convention.
|
||||
const WORKER_PATH = path.join(__dirname, 'db-reader-worker.js');
|
||||
|
||||
// 30 s default leaves headroom for SQLite read contention during the
|
||||
// post-crawl recompute phase (recomputeUrlsIssues holds a writer
|
||||
// transaction that can keep reader queries blocked behind earlier
|
||||
// queued requests for several seconds at a time on large crawls).
|
||||
// Genuine worker hangs are still surfaced — just with a slightly
|
||||
// longer detection window than before.
|
||||
// post-crawl recompute phase. Genuine worker hangs are still surfaced
|
||||
// — just with a slightly longer detection window than before.
|
||||
const REQUEST_TIMEOUT_MS = 30_000;
|
||||
// Heavy aggregate queries (the overview sidebar's 130-counter pass and
|
||||
// the post-crawl materialiser's counter fan-out) can run past the
|
||||
@@ -54,7 +59,17 @@ const HEAVY_METHODS = new Set<string>([
|
||||
const MAX_RESTARTS = 3;
|
||||
const RESTART_WINDOW_MS = 60_000;
|
||||
|
||||
// Reader pool size. Four workers gives the typical UI workload (one
|
||||
// long-running aggregate from the sidebar + visible-table chunk fetch
|
||||
// + bottom-panel detail + crawl-engine background reads) a free slot
|
||||
// for every concurrent request; bigger doesn't help because most UI
|
||||
// reads are short and a dedicated worker per query becomes pure
|
||||
// memory cost. Each worker keeps its own SQLite handle (~5 MB heap)
|
||||
// and prepared-statement cache (~10 MB on a populated DB).
|
||||
const POOL_SIZE = 4;
|
||||
|
||||
interface PendingRequest {
|
||||
workerIdx: number;
|
||||
resolve: (v: unknown) => void;
|
||||
reject: (e: Error) => void;
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
@@ -67,94 +82,143 @@ interface ResponseMessage {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface WorkerSlot {
|
||||
worker: Worker | null;
|
||||
pending: number;
|
||||
restartTimes: number[];
|
||||
}
|
||||
|
||||
class DbReaderPool {
|
||||
private worker: Worker | null = null;
|
||||
private slots: WorkerSlot[] = [];
|
||||
private dbPath: string | null = null;
|
||||
private freezeWatchdogSab: SharedArrayBuffer | null = null;
|
||||
private nextRequestId = 1;
|
||||
private pending = new Map<number, PendingRequest>();
|
||||
private restartTimes: number[] = [];
|
||||
private rrCursor = 0;
|
||||
private terminated = false;
|
||||
|
||||
/** Spawn (or respawn) the worker pointed at `dbPath`. Idempotent.
|
||||
*
|
||||
* `freezeWatchdogSab` — if provided, the worker will tick its
|
||||
* own heartbeat + publish its current method into this shared
|
||||
* buffer so the freeze-watchdog thread can detect a stuck
|
||||
* reader. Pass `null` to disable diagnostics. */
|
||||
/** Spawn (or respawn) the pool pointed at `dbPath`. Idempotent. */
|
||||
init(dbPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
|
||||
if (this.terminated) {
|
||||
throw new Error('DbReaderPool: cannot init after terminate()');
|
||||
}
|
||||
this.freezeWatchdogSab = freezeWatchdogSab;
|
||||
if (this.dbPath === dbPath && this.worker !== null) return;
|
||||
if (this.dbPath === dbPath && this.slots.some((s) => s.worker !== null)) return;
|
||||
this.dbPath = dbPath;
|
||||
this.spawn();
|
||||
this.slots = Array.from({ length: POOL_SIZE }, () => ({
|
||||
worker: null,
|
||||
pending: 0,
|
||||
restartTimes: [],
|
||||
}));
|
||||
for (let i = 0; i < POOL_SIZE; i++) this.spawn(i);
|
||||
}
|
||||
|
||||
/** Switch the worker to a different .seoproject file. */
|
||||
/** Switch the pool to a different .seoproject file. */
|
||||
swap(newPath: string, freezeWatchdogSab: SharedArrayBuffer | null = null): void {
|
||||
if (freezeWatchdogSab !== null) this.freezeWatchdogSab = freezeWatchdogSab;
|
||||
if (this.dbPath === newPath && this.worker !== null) return;
|
||||
if (this.dbPath === newPath && this.slots.some((s) => s.worker !== null)) return;
|
||||
this.failPendingWith('reader-swapped');
|
||||
this.dbPath = newPath;
|
||||
this.spawn();
|
||||
if (this.slots.length === 0) {
|
||||
this.slots = Array.from({ length: POOL_SIZE }, () => ({
|
||||
worker: null,
|
||||
pending: 0,
|
||||
restartTimes: [],
|
||||
}));
|
||||
}
|
||||
for (let i = 0; i < POOL_SIZE; i++) this.spawn(i);
|
||||
}
|
||||
|
||||
/** Permanent shutdown — used on app quit. */
|
||||
async terminate(): Promise<void> {
|
||||
this.terminated = true;
|
||||
this.failPendingWith('reader-terminated');
|
||||
if (this.worker) {
|
||||
const w = this.worker;
|
||||
this.worker = null;
|
||||
try {
|
||||
await w.terminate();
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
}
|
||||
const workers = this.slots.map((s) => s.worker).filter((w): w is Worker => w !== null);
|
||||
for (const slot of this.slots) slot.worker = null;
|
||||
await Promise.all(
|
||||
workers.map((w) =>
|
||||
w.terminate().catch(() => undefined),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** True while a worker is up and ready to receive requests. */
|
||||
/** True while at least one worker is up and ready. */
|
||||
isReady(): boolean {
|
||||
return this.worker !== null && !this.terminated;
|
||||
return !this.terminated && this.slots.some((s) => s.worker !== null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch a method call to the worker. Callers should use the typed
|
||||
* wrapper `callReader<T>(method, args)` declared in the main process
|
||||
* IPC layer. Resolves with the method's return value, rejects with
|
||||
* an Error on worker error / timeout / crash.
|
||||
* Dispatch a method call to the least-busy ready worker. Resolves
|
||||
* with the method's return value, rejects with an Error on worker
|
||||
* error / timeout / crash.
|
||||
*/
|
||||
call<T>(method: string, args: unknown[] = []): Promise<T> {
|
||||
if (this.terminated) return Promise.reject(new Error('reader-terminated'));
|
||||
if (!this.worker) return Promise.reject(new Error('reader-not-initialised'));
|
||||
const slotIdx = this.pickSlot();
|
||||
if (slotIdx < 0) return Promise.reject(new Error('reader-not-initialised'));
|
||||
const slot = this.slots[slotIdx]!;
|
||||
const worker = slot.worker;
|
||||
if (!worker) return Promise.reject(new Error('reader-not-initialised'));
|
||||
const requestId = this.nextRequestId++;
|
||||
const timeoutMs = HEAVY_METHODS.has(method)
|
||||
? HEAVY_REQUEST_TIMEOUT_MS
|
||||
: REQUEST_TIMEOUT_MS;
|
||||
slot.pending++;
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
const p = this.pending.get(requestId);
|
||||
if (p) {
|
||||
this.pending.delete(requestId);
|
||||
this.slots[p.workerIdx]!.pending = Math.max(
|
||||
0,
|
||||
this.slots[p.workerIdx]!.pending - 1,
|
||||
);
|
||||
}
|
||||
reject(new Error(`reader-timeout: ${method} > ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
this.pending.set(requestId, {
|
||||
workerIdx: slotIdx,
|
||||
resolve: resolve as (v: unknown) => void,
|
||||
reject,
|
||||
timer,
|
||||
});
|
||||
this.worker!.postMessage({ requestId, method, args });
|
||||
worker.postMessage({ requestId, method, args });
|
||||
});
|
||||
}
|
||||
|
||||
// ── private ──────────────────────────────────────────────────────────────
|
||||
|
||||
private spawn(): void {
|
||||
/**
|
||||
* Pick the slot with the fewest pending requests. Ties are broken
|
||||
* by a round-robin cursor so identically-loaded workers don't all
|
||||
* receive their next call on the same slot.
|
||||
*/
|
||||
private pickSlot(): number {
|
||||
let best = -1;
|
||||
let bestPending = Number.POSITIVE_INFINITY;
|
||||
// Start the scan from the rotating cursor so RR is honoured on ties.
|
||||
for (let i = 0; i < this.slots.length; i++) {
|
||||
const idx = (this.rrCursor + i) % this.slots.length;
|
||||
const s = this.slots[idx]!;
|
||||
if (!s.worker) continue;
|
||||
if (s.pending < bestPending) {
|
||||
best = idx;
|
||||
bestPending = s.pending;
|
||||
}
|
||||
}
|
||||
if (best >= 0) {
|
||||
this.rrCursor = (best + 1) % this.slots.length;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
private spawn(slotIdx: number): void {
|
||||
if (!this.dbPath) return;
|
||||
if (this.worker) {
|
||||
const old = this.worker;
|
||||
this.worker = null;
|
||||
const slot = this.slots[slotIdx];
|
||||
if (!slot) return;
|
||||
if (slot.worker) {
|
||||
const old = slot.worker;
|
||||
slot.worker = null;
|
||||
void old.terminate().catch(() => undefined);
|
||||
}
|
||||
try {
|
||||
@@ -166,18 +230,19 @@ class DbReaderPool {
|
||||
});
|
||||
w.on('message', (msg: ResponseMessage) => this.handleResponse(msg));
|
||||
w.on('error', (err) => {
|
||||
logger.log('error', 'db-reader', `worker error: ${err.message}`);
|
||||
logger.log('error', 'db-reader', `worker[${slotIdx}] error: ${err.message}`);
|
||||
});
|
||||
w.on('exit', (code) => this.handleExit(code));
|
||||
this.worker = w;
|
||||
logger.log('info', 'db-reader', `worker spawned for ${this.dbPath}`);
|
||||
w.on('exit', (code) => this.handleExit(slotIdx, code));
|
||||
slot.worker = w;
|
||||
slot.pending = 0;
|
||||
logger.log('info', 'db-reader', `worker[${slotIdx}] spawned for ${this.dbPath}`);
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'error',
|
||||
'db-reader',
|
||||
`worker spawn failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
`worker[${slotIdx}] spawn failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
this.worker = null;
|
||||
slot.worker = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,6 +251,8 @@ class DbReaderPool {
|
||||
if (!pending) return;
|
||||
this.pending.delete(msg.requestId);
|
||||
clearTimeout(pending.timer);
|
||||
const slot = this.slots[pending.workerIdx];
|
||||
if (slot) slot.pending = Math.max(0, slot.pending - 1);
|
||||
if (msg.ok) {
|
||||
pending.resolve(msg.result);
|
||||
} else {
|
||||
@@ -193,31 +260,38 @@ class DbReaderPool {
|
||||
}
|
||||
}
|
||||
|
||||
private handleExit(code: number): void {
|
||||
this.worker = null;
|
||||
private handleExit(slotIdx: number, code: number): void {
|
||||
const slot = this.slots[slotIdx];
|
||||
if (!slot) return;
|
||||
slot.worker = null;
|
||||
if (this.terminated) return;
|
||||
// Reject only this slot's in-flight requests.
|
||||
for (const [id, p] of this.pending) {
|
||||
if (p.workerIdx === slotIdx) {
|
||||
clearTimeout(p.timer);
|
||||
p.reject(new Error('reader-crashed'));
|
||||
this.pending.delete(id);
|
||||
}
|
||||
}
|
||||
slot.pending = 0;
|
||||
if (code === 0) return;
|
||||
this.failPendingWith('reader-crashed');
|
||||
// Restart bookkeeping: drop entries older than the window so a
|
||||
// long-lived process that crashes once a day doesn't permanently
|
||||
// exhaust its restart budget.
|
||||
const now = Date.now();
|
||||
this.restartTimes = this.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS);
|
||||
if (this.restartTimes.length >= MAX_RESTARTS) {
|
||||
slot.restartTimes = slot.restartTimes.filter((t) => now - t < RESTART_WINDOW_MS);
|
||||
if (slot.restartTimes.length >= MAX_RESTARTS) {
|
||||
logger.log(
|
||||
'error',
|
||||
'db-reader',
|
||||
`worker crashed ${MAX_RESTARTS}× within ${RESTART_WINDOW_MS}ms — giving up. UI queries fall back to main-process DB.`,
|
||||
`worker[${slotIdx}] crashed ${MAX_RESTARTS}× within ${RESTART_WINDOW_MS}ms — giving up on this slot.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.restartTimes.push(now);
|
||||
slot.restartTimes.push(now);
|
||||
logger.log(
|
||||
'warn',
|
||||
'db-reader',
|
||||
`worker exited with code ${code}; respawning (${this.restartTimes.length}/${MAX_RESTARTS})`,
|
||||
`worker[${slotIdx}] exited with code ${code}; respawning (${slot.restartTimes.length}/${MAX_RESTARTS})`,
|
||||
);
|
||||
this.spawn();
|
||||
this.spawn(slotIdx);
|
||||
}
|
||||
|
||||
private failPendingWith(reason: string): void {
|
||||
@@ -226,6 +300,7 @@ class DbReaderPool {
|
||||
p.reject(new Error(reason));
|
||||
}
|
||||
this.pending.clear();
|
||||
for (const slot of this.slots) slot.pending = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,10 +308,10 @@ class DbReaderPool {
|
||||
export const dbReaderPool = new DbReaderPool();
|
||||
|
||||
/**
|
||||
* Thin helper for IPC handlers: try the worker first, fall back to the
|
||||
* synchronous main-process DB on any worker error. The fallback path
|
||||
* keeps the UI working even if the worker is crashed/restarting, at
|
||||
* the cost of running on the main thread for that one query.
|
||||
* Thin helper for IPC handlers: try the worker pool first, fall back
|
||||
* to the synchronous main-process DB on any worker error. The fallback
|
||||
* path keeps the UI working even if every worker is crashed/restarting,
|
||||
* at the cost of running on the main thread for that one query.
|
||||
*/
|
||||
export async function callReaderOrFallback<T>(
|
||||
method: string,
|
||||
|
||||
@@ -25,6 +25,8 @@ import {
|
||||
type ExportJsonResult,
|
||||
type ExportXmlInput,
|
||||
type ExportXmlResult,
|
||||
type ExportTabularInput,
|
||||
type ExportTabularResult,
|
||||
type DataDeleteByDomainInput,
|
||||
type DataDeleteByDomainResult,
|
||||
type CrashRecoveryStatus,
|
||||
@@ -81,6 +83,7 @@ import {
|
||||
exportUrlsToCsv,
|
||||
exportUrlsToJson,
|
||||
exportUrlsToXml,
|
||||
exportTabular,
|
||||
exportSitemap,
|
||||
exportHtmlReport,
|
||||
compareCrawls,
|
||||
@@ -366,6 +369,64 @@ async function writeFetchedUrlViaPool(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the post-crawl issue recompute on the writer worker. This pass
|
||||
* has 70+ correlated SQL subqueries against the urls table; on a
|
||||
* 5K-URL crawl it takes ~45 s, and running it on the main thread
|
||||
* freezes the renderer for that entire window (the "Not Responding"
|
||||
* dialog). Routing through the writer worker keeps main free.
|
||||
*/
|
||||
async function recomputeIssuesViaPool(
|
||||
definitions: ReadonlyArray<readonly [string, string]>,
|
||||
): Promise<void> {
|
||||
if (!dbWriterPool.isReady()) {
|
||||
return getDb().recomputeUrlsIssuesYielding(definitions);
|
||||
}
|
||||
try {
|
||||
await dbWriterPool.call<void>('recomputeUrlsIssuesYielding', [definitions]);
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'warn',
|
||||
'main',
|
||||
`recomputeUrlsIssuesYielding dispatch failed (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}) — falling back to main-thread recompute.`,
|
||||
);
|
||||
await getDb().recomputeUrlsIssuesYielding(definitions);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic post-crawl pass dispatcher. Routes recomputeInlinks,
|
||||
* recomputeRedirectChains, recomputeHreflangAnalysis,
|
||||
* recomputeHreflangInconsistent, and recomputePaginationSequence
|
||||
* through the writer worker so the main thread stays responsive to
|
||||
* Stop/Pause clicks and IPC traffic during the post-crawl phase.
|
||||
* Without this, those five sync SQL passes combined freeze main for
|
||||
* 5–15 s on a 5K-URL crawl — exactly the window where users hit
|
||||
* Stop/Pause and "nothing happens".
|
||||
*/
|
||||
async function runDbPassViaPool(method: string): Promise<void> {
|
||||
if (!dbWriterPool.isReady()) {
|
||||
const fn = (getDb() as unknown as Record<string, unknown>)[method];
|
||||
if (typeof fn === 'function') (fn as () => void).call(getDb());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await dbWriterPool.call<void>(method, []);
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'warn',
|
||||
'main',
|
||||
`${method} dispatch failed (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}) — falling back to main-thread.`,
|
||||
);
|
||||
const fn = (getDb() as unknown as Record<string, unknown>)[method];
|
||||
if (typeof fn === 'function') (fn as () => void).call(getDb());
|
||||
}
|
||||
}
|
||||
|
||||
function getDb(): ProjectDb {
|
||||
if (!db) {
|
||||
const dataDir = join(app.getPath('userData'), 'projects');
|
||||
@@ -1160,6 +1221,8 @@ function registerIpc(): void {
|
||||
setOp: (op: string) => freezeWatchdog.setMainOp(op),
|
||||
parseHtml: parseHtmlViaPool,
|
||||
writeFetchedUrl: writeFetchedUrlViaPool,
|
||||
recomputeIssues: recomputeIssuesViaPool,
|
||||
runDbPass: runDbPassViaPool,
|
||||
});
|
||||
activeCrawler = crawler;
|
||||
attachCrawlerListeners(crawler);
|
||||
@@ -1178,11 +1241,29 @@ function registerIpc(): void {
|
||||
activeCrawler?.resume();
|
||||
});
|
||||
|
||||
ipcMain.handle(IPC.crawlClear, () => {
|
||||
ipcMain.handle(IPC.crawlClear, async () => {
|
||||
activeCrawler?.stop();
|
||||
activeCrawler = null;
|
||||
const database = getDb();
|
||||
// Route reset() through the writer pool so it sits behind any
|
||||
// in-flight writeFetchedUrl RPCs in the worker's FIFO. Without
|
||||
// this, the main process truncates the urls table while the
|
||||
// writer worker is still draining the last batch of per-URL
|
||||
// writes — those writes win the SQLite lock race and resurrect
|
||||
// ~19 rows in the table after Clear. Going through the pool
|
||||
// serialises naturally: every queued write completes, then reset
|
||||
// runs against an already-stopped crawler.
|
||||
if (dbWriterPool.isReady()) {
|
||||
try {
|
||||
await dbWriterPool.call('reset', []);
|
||||
} catch {
|
||||
// Worker terminated or RPC timed out — fall back to direct
|
||||
// reset rather than leaving the table half-cleared.
|
||||
database.reset();
|
||||
}
|
||||
} else {
|
||||
database.reset();
|
||||
}
|
||||
// Belt-and-braces: reset() above already includes a DELETE FROM
|
||||
// crawl_queue, but if anything (a late-firing checkpoint timer, a
|
||||
// half-finished post-crawl pass) writes to that table after the
|
||||
@@ -1661,6 +1742,47 @@ function registerIpc(): void {
|
||||
},
|
||||
);
|
||||
|
||||
ipcMain.handle(
|
||||
IPC.exportTabular,
|
||||
async (_e, input: ExportTabularInput): Promise<ExportTabularResult> => {
|
||||
const { format, sections, columns, selectedIds } = input;
|
||||
let outputPath = input.filePath ?? '';
|
||||
const isSelection = (selectedIds?.length ?? 0) > 0;
|
||||
const multiSection = sections.length > 1;
|
||||
if (!outputPath) {
|
||||
if (format === 'csv' && multiSection) {
|
||||
const res = await dialog.showOpenDialog(mainWindow!, {
|
||||
properties: ['openDirectory', 'createDirectory'],
|
||||
title: 'Choose folder for CSV export',
|
||||
});
|
||||
if (res.canceled || res.filePaths.length === 0) {
|
||||
return { filePath: '', files: [], rowsWritten: 0 };
|
||||
}
|
||||
outputPath = res.filePaths[0]!;
|
||||
} else {
|
||||
const ext = format === 'xlsx' ? 'xlsx' : 'csv';
|
||||
const filterName = format === 'xlsx' ? 'Excel Workbook' : 'CSV';
|
||||
const baseName = isSelection ? 'freecrawl-selected' : 'freecrawl-export';
|
||||
const res = await dialog.showSaveDialog(mainWindow!, {
|
||||
defaultPath: `${baseName}.${ext}`,
|
||||
filters: [{ name: filterName, extensions: [ext] }],
|
||||
});
|
||||
if (res.canceled || !res.filePath) {
|
||||
return { filePath: '', files: [], rowsWritten: 0 };
|
||||
}
|
||||
outputPath = res.filePath;
|
||||
}
|
||||
}
|
||||
const result = await exportTabular(getDb(), outputPath, {
|
||||
format,
|
||||
sections,
|
||||
columns,
|
||||
selectedIds,
|
||||
});
|
||||
return result;
|
||||
},
|
||||
);
|
||||
|
||||
ipcMain.handle(
|
||||
IPC.dataDeleteByDomain,
|
||||
async (
|
||||
@@ -1707,6 +1829,8 @@ function registerIpc(): void {
|
||||
setOp: (op: string) => freezeWatchdog.setMainOp(op),
|
||||
parseHtml: parseHtmlViaPool,
|
||||
writeFetchedUrl: writeFetchedUrlViaPool,
|
||||
recomputeIssues: recomputeIssuesViaPool,
|
||||
runDbPass: runDbPassViaPool,
|
||||
});
|
||||
activeCrawler = crawler;
|
||||
attachCrawlerListeners(crawler);
|
||||
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
type ExportJsonResult,
|
||||
type ExportXmlInput,
|
||||
type ExportXmlResult,
|
||||
type ExportTabularInput,
|
||||
type ExportTabularResult,
|
||||
type DataDeleteByDomainInput,
|
||||
type DataDeleteByDomainResult,
|
||||
type CrashRecoveryStatus,
|
||||
@@ -123,6 +125,8 @@ const api: FreeCrawlApi = {
|
||||
ipcRenderer.invoke(IPC.exportJson, input),
|
||||
exportXml: (input: ExportXmlInput): Promise<ExportXmlResult> =>
|
||||
ipcRenderer.invoke(IPC.exportXml, input),
|
||||
exportTabular: (input: ExportTabularInput): Promise<ExportTabularResult> =>
|
||||
ipcRenderer.invoke(IPC.exportTabular, input),
|
||||
dataDeleteByDomain: (
|
||||
input: DataDeleteByDomainInput,
|
||||
): Promise<DataDeleteByDomainResult> =>
|
||||
|
||||
@@ -137,8 +137,17 @@ export function App() {
|
||||
// previous session left a non-empty checkpoint we ask whether to
|
||||
// resume. Discarding wipes the snapshot so the prompt doesn't
|
||||
// recur on next launch.
|
||||
//
|
||||
// The confirm() is wrapped in a 250 ms setTimeout so the initial
|
||||
// render finishes (URL input mounts, status bar paints) before
|
||||
// the modal opens. Without the defer, the synchronous `confirm`
|
||||
// can fire on first paint and — when Electron renders the modal
|
||||
// behind the main window on some Windows configurations — leaves
|
||||
// the user staring at an "unresponsive" UI: the URL input can't
|
||||
// be clicked because a hidden modal is blocking the renderer.
|
||||
void window.freecrawl.crashRecoveryStatus().then((status) => {
|
||||
if (status.pendingCount === 0) return;
|
||||
window.setTimeout(() => {
|
||||
const proceed = window.confirm(
|
||||
`FreeCrawl detected a previous crawl that didn't finish cleanly:\n\n` +
|
||||
` Start URL: ${status.seedUrl}\n` +
|
||||
@@ -151,6 +160,7 @@ export function App() {
|
||||
} else {
|
||||
void window.freecrawl.crashRecoveryDiscard();
|
||||
}
|
||||
}, 250);
|
||||
});
|
||||
|
||||
// rAF-throttled progress dispatch. The crawler emits up to 5
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { Component, type ReactNode } from 'react';
|
||||
|
||||
interface Props {
|
||||
/** Rendered when the boundary's subtree throws. Defaults to a tiny notice. */
|
||||
fallback?: ReactNode;
|
||||
/** Surfaced to console + main-process log so the user can copy/paste. */
|
||||
context?: string;
|
||||
children: ReactNode;
|
||||
}
|
||||
|
||||
interface State {
|
||||
err: Error | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Catches render-time errors in a subtree so a single misbehaving dialog
|
||||
* (e.g. ExportDialog) can't unmount the whole renderer and leave the user
|
||||
* staring at the Electron window's bare background colour. The boundary
|
||||
* logs the error and shows a small inline notice; the rest of the app
|
||||
* keeps working.
|
||||
*/
|
||||
export class ErrorBoundary extends Component<Props, State> {
|
||||
override state: State = { err: null };
|
||||
|
||||
static getDerivedStateFromError(err: Error): State {
|
||||
return { err };
|
||||
}
|
||||
|
||||
override componentDidCatch(err: Error, info: { componentStack?: string }): void {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(
|
||||
`[ErrorBoundary${this.props.context ? `:${this.props.context}` : ''}]`,
|
||||
err,
|
||||
info.componentStack,
|
||||
);
|
||||
}
|
||||
|
||||
reset = (): void => {
|
||||
this.setState({ err: null });
|
||||
};
|
||||
|
||||
override render(): ReactNode {
|
||||
const { err } = this.state;
|
||||
if (!err) return this.props.children;
|
||||
if (this.props.fallback !== undefined) return this.props.fallback;
|
||||
return (
|
||||
<div className="fixed inset-x-0 top-2 z-[60] mx-auto w-fit max-w-[90vw] rounded border border-red-700/60 bg-red-950/90 px-3 py-2 text-[12px] text-red-200 shadow-lg">
|
||||
<div className="font-semibold">Something broke in this panel.</div>
|
||||
<div className="mt-0.5 truncate font-mono text-[11px] text-red-300">{err.message}</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={this.reset}
|
||||
className="mt-1 rounded border border-red-700/60 px-2 py-0.5 text-[11px] text-red-200 hover:bg-red-900/50"
|
||||
>
|
||||
Dismiss
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { X, Loader2 } from 'lucide-react';
|
||||
import clsx from 'clsx';
|
||||
import type { CrawlUrlRow, ExportTabularSection, UrlCategory } from '@freecrawl/shared-types';
|
||||
import { TAB_ORDER, type TabKey } from '../store.js';
|
||||
import { COLUMN_SPECS, columnId } from '../tabs/columns.js';
|
||||
|
||||
type Format = 'csv' | 'xlsx';
|
||||
|
||||
interface ExportTab {
|
||||
key: TabKey;
|
||||
label: string;
|
||||
category: UrlCategory;
|
||||
columnKeys: (keyof CrawlUrlRow)[];
|
||||
}
|
||||
|
||||
const TAB_CATEGORY: Record<TabKey, UrlCategory> = {
|
||||
internal: 'internal:html',
|
||||
external: 'external:all',
|
||||
'response-codes': 'all',
|
||||
url: 'internal:html',
|
||||
'page-titles': 'internal:html',
|
||||
'meta-description': 'internal:html',
|
||||
h1: 'internal:html',
|
||||
h2: 'internal:html',
|
||||
content: 'internal:html',
|
||||
images: 'internal:html',
|
||||
canonicals: 'internal:html',
|
||||
directives: 'internal:html',
|
||||
redirects: 'all',
|
||||
links: 'all',
|
||||
'broken-links': 'all',
|
||||
};
|
||||
|
||||
function buildExportableTabs(): ExportTab[] {
|
||||
const out: ExportTab[] = [];
|
||||
for (const t of TAB_ORDER) {
|
||||
const cols = COLUMN_SPECS[t.key];
|
||||
// Tabs with their own non-URL-row layouts (Images / Broken Links)
|
||||
// have empty COLUMN_SPECS — skip them so the export grid only shows
|
||||
// categories that share the URL-row schema.
|
||||
if (!cols || cols.length === 0) continue;
|
||||
out.push({
|
||||
key: t.key,
|
||||
label: t.label,
|
||||
category: TAB_CATEGORY[t.key],
|
||||
columnKeys: cols.map((c) => c.key),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface ExportDialogProps {
|
||||
open: boolean;
|
||||
onClose: () => void;
|
||||
/** The tab the user clicked Export from — drives the default selection. */
|
||||
defaultTab: TabKey;
|
||||
/** Row ids selected in the current table; falsy → export the whole filter set. */
|
||||
selectedIds?: number[];
|
||||
}
|
||||
|
||||
export function ExportDialog({
|
||||
open,
|
||||
onClose,
|
||||
defaultTab,
|
||||
selectedIds,
|
||||
}: ExportDialogProps) {
|
||||
const tabs = useMemo(() => buildExportableTabs(), []);
|
||||
const initialTabKey: TabKey = useMemo(() => {
|
||||
return tabs.find((t) => t.key === defaultTab)?.key ?? tabs[0]?.key ?? 'internal';
|
||||
}, [defaultTab, tabs]);
|
||||
|
||||
const selectionCount = selectedIds?.length ?? 0;
|
||||
|
||||
const [format, setFormat] = useState<Format>('csv');
|
||||
const [pickedTabs, setPickedTabs] = useState<Set<TabKey>>(() => new Set([initialTabKey]));
|
||||
const [pickedColumns, setPickedColumns] = useState<Set<string>>(() => {
|
||||
const t = tabs.find((x) => x.key === initialTabKey);
|
||||
return new Set((t?.columnKeys ?? []) as string[]);
|
||||
});
|
||||
const [useSelection, setUseSelection] = useState<boolean>(selectionCount > 0);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [status, setStatus] = useState<string>('');
|
||||
|
||||
// The dialog is conditionally mounted by the parent (only when open=true),
|
||||
// so useState initializers above already populate the right defaults on
|
||||
// each open. Escape key dismissal is the only ambient behaviour we need.
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
const onKey = (e: KeyboardEvent): void => {
|
||||
if (e.key === 'Escape') onClose();
|
||||
};
|
||||
document.addEventListener('keydown', onKey);
|
||||
return () => document.removeEventListener('keydown', onKey);
|
||||
}, [open, onClose]);
|
||||
|
||||
// Union of column keys exposed by every picked tab — these are the
|
||||
// columns the dialog offers; checked state lives in `pickedColumns`.
|
||||
const offeredColumns = useMemo(() => {
|
||||
const seen = new Set<string>();
|
||||
const out: { key: string; header: string; tab: TabKey }[] = [];
|
||||
for (const t of tabs) {
|
||||
if (!pickedTabs.has(t.key)) continue;
|
||||
const specs = COLUMN_SPECS[t.key];
|
||||
for (const spec of specs) {
|
||||
const id = columnId(spec);
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
out.push({ key: spec.key as string, header: spec.header, tab: t.key });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}, [pickedTabs, tabs]);
|
||||
|
||||
const orderedColumnList = useMemo(
|
||||
() => offeredColumns.filter((c) => pickedColumns.has(c.key)).map((c) => c.key),
|
||||
[offeredColumns, pickedColumns],
|
||||
);
|
||||
|
||||
if (!open) return null;
|
||||
|
||||
const toggleTab = (key: TabKey) => {
|
||||
setPickedTabs((prev) => {
|
||||
const next = new Set(prev);
|
||||
let willBeRemoved = false;
|
||||
if (next.has(key)) {
|
||||
if (next.size === 1) return prev; // keep at least one
|
||||
next.delete(key);
|
||||
willBeRemoved = true;
|
||||
} else {
|
||||
next.add(key);
|
||||
}
|
||||
// Auto-tick the new tab's columns when it joins the picked set, so
|
||||
// the user doesn't have to re-check them. When a tab leaves we keep
|
||||
// the user's column picks alone — orderedColumnList already drops
|
||||
// any column not offered by remaining tabs.
|
||||
if (!willBeRemoved) {
|
||||
const t = tabs.find((x) => x.key === key);
|
||||
if (t) {
|
||||
setPickedColumns((prevCols) => {
|
||||
const nextCols = new Set(prevCols);
|
||||
for (const k of t.columnKeys) nextCols.add(k as string);
|
||||
return nextCols;
|
||||
});
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const toggleColumn = (key: string) => {
|
||||
setPickedColumns((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(key)) next.delete(key);
|
||||
else next.add(key);
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const checkAllColumns = () => {
|
||||
setPickedColumns(new Set(offeredColumns.map((c) => c.key)));
|
||||
};
|
||||
|
||||
const uncheckAllColumns = () => {
|
||||
setPickedColumns(new Set());
|
||||
};
|
||||
|
||||
const submit = async () => {
|
||||
if (orderedColumnList.length === 0) {
|
||||
setStatus('Pick at least one column.');
|
||||
return;
|
||||
}
|
||||
if (pickedTabs.size === 0) {
|
||||
setStatus('Pick at least one table.');
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
setStatus('');
|
||||
try {
|
||||
const sections: ExportTabularSection[] = tabs
|
||||
.filter((t) => pickedTabs.has(t.key))
|
||||
.map((t) => ({ label: t.label, category: t.category }));
|
||||
const result = await window.freecrawl.exportTabular({
|
||||
format,
|
||||
sections,
|
||||
columns: orderedColumnList,
|
||||
selectedIds:
|
||||
useSelection && selectedIds && selectedIds.length > 0
|
||||
? selectedIds
|
||||
: undefined,
|
||||
});
|
||||
if (!result.filePath) {
|
||||
// User cancelled save / folder dialog.
|
||||
setBusy(false);
|
||||
return;
|
||||
}
|
||||
setStatus(
|
||||
`Exported ${result.rowsWritten.toLocaleString()} row${result.rowsWritten === 1 ? '' : 's'} to ${result.filePath}`,
|
||||
);
|
||||
setBusy(false);
|
||||
setTimeout(() => onClose(), 700);
|
||||
} catch (err) {
|
||||
setBusy(false);
|
||||
setStatus(`Export failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
className="fixed inset-0 z-50 flex items-center justify-center bg-black/60"
|
||||
onClick={onClose}
|
||||
>
|
||||
<div
|
||||
className="flex max-h-[85vh] w-[780px] max-w-[95vw] flex-col rounded-md border border-surface-700 bg-surface-900 shadow-2xl"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<div className="flex items-center justify-between border-b border-surface-800 px-4 py-2.5">
|
||||
<div className="text-sm font-semibold text-surface-100">Export</div>
|
||||
<button
|
||||
className="rounded p-1 text-surface-400 hover:bg-surface-800 hover:text-surface-100"
|
||||
onClick={onClose}
|
||||
title="Close"
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="flex-1 overflow-auto px-4 py-3">
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<section>
|
||||
<div className="mb-1.5 text-[11px] font-semibold uppercase tracking-wide text-surface-400">
|
||||
Tables
|
||||
</div>
|
||||
<div className="rounded border border-surface-800 bg-surface-950/50 p-2">
|
||||
<div className="grid grid-cols-1 gap-1">
|
||||
{tabs.map((t) => {
|
||||
const checked = pickedTabs.has(t.key);
|
||||
const isDefault = t.key === initialTabKey;
|
||||
return (
|
||||
<label
|
||||
key={t.key}
|
||||
className={clsx(
|
||||
'flex cursor-pointer items-center gap-2 rounded px-1.5 py-1 text-[12px]',
|
||||
checked
|
||||
? 'bg-accent-500/10 text-surface-100'
|
||||
: 'text-surface-300 hover:bg-surface-800/60',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
className="h-3.5 w-3.5"
|
||||
checked={checked}
|
||||
onChange={() => toggleTab(t.key)}
|
||||
/>
|
||||
<span className="truncate">{t.label}</span>
|
||||
{isDefault && (
|
||||
<span className="ml-auto rounded bg-surface-800 px-1 text-[9px] uppercase tracking-wide text-surface-400">
|
||||
Active
|
||||
</span>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="flex min-h-0 flex-col">
|
||||
<div className="mb-1.5 flex items-center justify-between">
|
||||
<div className="text-[11px] font-semibold uppercase tracking-wide text-surface-400">
|
||||
Columns
|
||||
</div>
|
||||
<div className="flex gap-1 text-[10px]">
|
||||
<button
|
||||
type="button"
|
||||
className="rounded border border-surface-700 px-1.5 py-0.5 text-surface-300 hover:bg-surface-800"
|
||||
onClick={checkAllColumns}
|
||||
>
|
||||
All
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="rounded border border-surface-700 px-1.5 py-0.5 text-surface-300 hover:bg-surface-800"
|
||||
onClick={uncheckAllColumns}
|
||||
>
|
||||
None
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="max-h-[320px] overflow-auto rounded border border-surface-800 bg-surface-950/50 p-2">
|
||||
<div className="grid grid-cols-1 gap-1">
|
||||
{offeredColumns.map((c) => {
|
||||
const checked = pickedColumns.has(c.key);
|
||||
return (
|
||||
<label
|
||||
key={c.key}
|
||||
className={clsx(
|
||||
'flex cursor-pointer items-center gap-2 rounded px-1.5 py-1 text-[12px]',
|
||||
checked
|
||||
? 'bg-accent-500/10 text-surface-100'
|
||||
: 'text-surface-300 hover:bg-surface-800/60',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
className="h-3.5 w-3.5"
|
||||
checked={checked}
|
||||
onChange={() => toggleColumn(c.key)}
|
||||
/>
|
||||
<span className="truncate">{c.header}</span>
|
||||
<span className="ml-auto truncate font-mono text-[10px] text-surface-500">
|
||||
{c.key}
|
||||
</span>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 flex flex-wrap items-center gap-4">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-[11px] font-semibold uppercase tracking-wide text-surface-400">
|
||||
Format
|
||||
</span>
|
||||
<label className="flex cursor-pointer items-center gap-1 text-[12px] text-surface-200">
|
||||
<input
|
||||
type="radio"
|
||||
name="export-format"
|
||||
checked={format === 'csv'}
|
||||
onChange={() => setFormat('csv')}
|
||||
/>
|
||||
CSV
|
||||
</label>
|
||||
<label className="flex cursor-pointer items-center gap-1 text-[12px] text-surface-200">
|
||||
<input
|
||||
type="radio"
|
||||
name="export-format"
|
||||
checked={format === 'xlsx'}
|
||||
onChange={() => setFormat('xlsx')}
|
||||
/>
|
||||
Excel (.xlsx)
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{selectionCount > 0 && (
|
||||
<label className="flex cursor-pointer items-center gap-2 text-[12px] text-surface-200">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={useSelection}
|
||||
onChange={(e) => setUseSelection(e.target.checked)}
|
||||
/>
|
||||
<span>
|
||||
Selected rows only{' '}
|
||||
<span className="text-surface-500">
|
||||
({selectionCount.toLocaleString()})
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{format === 'csv' && pickedTabs.size > 1 && (
|
||||
<div className="mt-2 rounded border border-amber-700/40 bg-amber-900/20 px-2 py-1 text-[11px] text-amber-200">
|
||||
Multiple tables + CSV → you'll be asked to pick a folder; one file is written per table.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status && (
|
||||
<div className="mt-3 truncate rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-200">
|
||||
{status}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-end gap-2 border-t border-surface-800 px-4 py-2.5">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className="rounded border border-surface-700 px-3 py-1 text-[12px] text-surface-300 hover:bg-surface-800"
|
||||
disabled={busy}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={submit}
|
||||
disabled={busy || pickedColumns.size === 0 || pickedTabs.size === 0}
|
||||
className={clsx(
|
||||
'inline-flex items-center gap-1 rounded border border-surface-700 px-3 py-1 text-[12px] text-surface-200 hover:bg-surface-800',
|
||||
(busy || pickedColumns.size === 0 || pickedTabs.size === 0) &&
|
||||
'cursor-not-allowed opacity-50 hover:bg-transparent',
|
||||
)}
|
||||
>
|
||||
{busy && <Loader2 className="h-3.5 w-3.5 animate-spin" />}
|
||||
Export
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -2,18 +2,27 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import type { AdvancedFilter, CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
|
||||
|
||||
const CHUNK_SIZE = 500;
|
||||
const MAX_CACHED_CHUNKS = 8;
|
||||
// Live-refresh cadence during an active crawl. 750 ms strikes a balance
|
||||
// between perceived "row streaming" smoothness and the cost of a COUNT
|
||||
// query + per-chunk fetch every tick. At 100 URL/s the user still sees
|
||||
// ~75 new rows per tick (continuous flow), while leaving 3× more main-
|
||||
// thread headroom than the previous 250 ms cadence.
|
||||
// Cadence for the live-tail tick. 1500 ms strikes the balance between
|
||||
// "feels fresh" and "doesn't pin SQLite". At 750 ms a 2k+ row crawl was
|
||||
// burning ~30% of one core in COUNT + chunk fetches, blocking input IPC
|
||||
// for the duration. The user sees the same effect: numbers tick at half
|
||||
// speed but the window stops kasma'ing.
|
||||
const LIVE_REFRESH_MS = 1500;
|
||||
// Doubled the cache (was 8) so a fast scroll across 4-5 chunks doesn't
|
||||
// thrash the reader pool: each chunk evicted = a re-fetch the next time
|
||||
// the user scrolls back to it. 16 × 500 = 8000 rows kept warm; at
|
||||
// ~2 KB/row that's ~16 MB heap, well within budget.
|
||||
const MAX_CACHED_CHUNKS = 16;
|
||||
// Number of chunks to pre-fetch ahead/behind the visible range. The
|
||||
// visible range is what the virtualizer asks for; pre-fetching the
|
||||
// neighbours hides "..." placeholders during fast scrolls because the
|
||||
// next chunk has usually already landed by the time the user reaches
|
||||
// it. 1 chunk = 500 rows of look-ahead each direction, cheap on the
|
||||
// reader pool.
|
||||
const PREFETCH_CHUNKS = 1;
|
||||
// Min gap between progress-driven ticks. Progress events arrive at
|
||||
// ~5 Hz from the crawler; without throttle the visible-chunk re-query
|
||||
// would fire that often too. 100 ms keeps the table feeling live
|
||||
// (10 refreshes/sec ceiling) without pinning the reader-pool.
|
||||
const TICK_THROTTLE_MS = 100;
|
||||
// Safety-net periodic tick for cases where progress events stop
|
||||
// arriving (crawl finished, manual DB edits, etc.). Long interval —
|
||||
// the progress-event path drives the in-crawl experience.
|
||||
const LIVE_REFRESH_MS = 5000;
|
||||
|
||||
export interface LazyRowsOpts {
|
||||
category: UrlCategory;
|
||||
@@ -31,6 +40,12 @@ export interface LazyRowsState {
|
||||
loadedRows: number;
|
||||
rowAt: (index: number) => CrawlUrlRow | null;
|
||||
ensureRange: (start: number, end: number) => void;
|
||||
/** True between the moment a shape change is requested (tab/sort/filter
|
||||
* flip) and the moment the new data has fully landed. Lets the
|
||||
* consumer distinguish "0 URLs because the category genuinely has
|
||||
* none" from "0 URLs because we haven't fetched yet" — which the
|
||||
* empty-state UI cares about a lot. */
|
||||
isLoading: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -49,6 +64,7 @@ export interface LazyRowsState {
|
||||
export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
const [total, setTotal] = useState(0);
|
||||
const [version, setVersion] = useState(0);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const chunks = useRef(new Map<number, CrawlUrlRow[]>());
|
||||
const chunkOrder = useRef<number[]>([]);
|
||||
const fetching = useRef(new Set<number>());
|
||||
@@ -88,20 +104,52 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
[opts.category, opts.search, opts.sortBy, opts.sortDir, opts.filter],
|
||||
);
|
||||
|
||||
// Shape change: rebuild from scratch. The virtualizer will call
|
||||
// ensureRange again on its next measurement pass.
|
||||
// Shape change (tab/sort/filter switch): clear immediately and load
|
||||
// fresh data in parallel. Showing the previous tab's rows under the
|
||||
// new tab's header is the wrong UX — "Internal" rows visible while
|
||||
// "External" is selected makes the user mistrust what they see.
|
||||
// Instead we set `isLoading=true` and let the consumer render a
|
||||
// proper loading state until the new total + visible chunks land.
|
||||
// The reader-pool has multiple workers now, so meta + visible-chunk
|
||||
// queries run in parallel and typically resolve in ~50-150 ms.
|
||||
useEffect(() => {
|
||||
resetToken.current++;
|
||||
chunks.current.clear();
|
||||
chunkOrder.current = [];
|
||||
fetching.current.clear();
|
||||
setIsLoading(true);
|
||||
setVersion((v) => v + 1);
|
||||
const token = resetToken.current;
|
||||
let cancelled = false;
|
||||
void queryMeta().then(({ total: t }) => {
|
||||
|
||||
const load = async (): Promise<void> => {
|
||||
const metaPromise = queryMeta();
|
||||
const { first, last } = activeRange.current;
|
||||
const chunkPromises: Promise<{ rows: CrawlUrlRow[]; total: number; idx: number }>[] = [];
|
||||
for (let i = first; i <= last; i++) {
|
||||
chunkPromises.push(queryChunk(i).then((r) => ({ ...r, idx: i })));
|
||||
}
|
||||
try {
|
||||
const [{ total: t }, ...newChunks] = await Promise.all([metaPromise, ...chunkPromises]);
|
||||
if (cancelled || token !== resetToken.current) return;
|
||||
for (const ch of newChunks) {
|
||||
chunks.current.set(ch.idx, ch.rows);
|
||||
chunkOrder.current.push(ch.idx);
|
||||
}
|
||||
setTotal(t);
|
||||
});
|
||||
setVersion((v) => v + 1);
|
||||
} catch {
|
||||
if (cancelled || token !== resetToken.current) return;
|
||||
// Leave chunks empty on error; the live tick / ensureRange
|
||||
// will retry. Still need to flip loading off so the empty
|
||||
// state can render an actual error/empty UI.
|
||||
} finally {
|
||||
if (!cancelled && token === resetToken.current) {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
void load();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
@@ -111,16 +159,26 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
// Live tick: re-query only the visible chunks and patch them in place.
|
||||
// Never calls .clear(), so rowAt never returns null for a
|
||||
// previously-loaded index. The table streams rather than flashes.
|
||||
// A leading tick fires immediately so the first rows surface without
|
||||
// the LIVE_REFRESH_MS dead window after Start.
|
||||
//
|
||||
// Drive model: push, not poll. Each crawler progress event (≈ every
|
||||
// time `crawled` advances, which lands at ~5 Hz on a healthy crawl)
|
||||
// triggers an immediate visible-chunk re-query. The previous design
|
||||
// polled every 1500 ms regardless of activity — newly-written rows
|
||||
// could sit in the DB for up to that long before surfacing in the
|
||||
// table. The push path closes that window to ~100 ms (the throttle
|
||||
// floor that keeps the reader-pool from being hammered).
|
||||
//
|
||||
// A 5 s safety-net `setInterval` still fires for code paths that
|
||||
// mutate the DB without going through a progress event (manual row
|
||||
// delete, project re-open, etc.).
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
let inFlight = false;
|
||||
let lastTickTs = 0;
|
||||
const tick = async () => {
|
||||
// Coalesce overlapping ticks — at 250 ms cadence a slow COUNT or
|
||||
// chunk fetch can otherwise queue up multiple in-flight ticks.
|
||||
if (inFlight) return;
|
||||
inFlight = true;
|
||||
lastTickTs = Date.now();
|
||||
const token = resetToken.current;
|
||||
try {
|
||||
try {
|
||||
@@ -146,33 +204,47 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
inFlight = false;
|
||||
}
|
||||
};
|
||||
// Wrap the tick in requestIdleCallback so the live-tail SQL only
|
||||
// fires when the renderer's event loop has idle slack. If the user
|
||||
// is dragging / clicking / typing, the tick is deferred until the
|
||||
// input handler frame finishes — eliminates the "click → kasma"
|
||||
// pattern even when the underlying SQL is fast. Falls back to a
|
||||
// direct call when the browser doesn't expose `requestIdleCallback`
|
||||
// (older Chromium builds; Electron 41 ships modern Chromium so this
|
||||
// branch is mostly defensive).
|
||||
interface RequestIdleCallback {
|
||||
(cb: () => void, opts?: { timeout: number }): number;
|
||||
}
|
||||
const w = window as Window & { requestIdleCallback?: RequestIdleCallback };
|
||||
const scheduleTick = (): void => {
|
||||
if (typeof w.requestIdleCallback === 'function') {
|
||||
// 2 s timeout guarantees the tick eventually runs even if the
|
||||
// main thread is permanently busy — better stale data than no
|
||||
// data at all.
|
||||
w.requestIdleCallback(() => void tick(), { timeout: 2000 });
|
||||
// 300 ms timeout — short enough that the user perceives an
|
||||
// event-driven refresh as instant; long enough that an
|
||||
// in-progress click handler isn't preempted.
|
||||
w.requestIdleCallback(() => void tick(), { timeout: 300 });
|
||||
} else {
|
||||
void tick();
|
||||
}
|
||||
};
|
||||
scheduleTick();
|
||||
// Leading tick fires directly (no idle wrap) so the first row
|
||||
// surfaces as soon as the crawler has written one — without this,
|
||||
// requestIdleCallback could defer the initial fetch up to 300 ms
|
||||
// on a busy main thread right after Start.
|
||||
void tick();
|
||||
|
||||
// Push: progress event drives the live-tail tick. We tick on every
|
||||
// event where `crawled` changed, throttled to TICK_THROTTLE_MS so
|
||||
// a 100 URL/s crawl doesn't force 100 SQL roundtrips per second.
|
||||
// Direct call (no idle wrap) — progress events are sparse enough
|
||||
// that input-handler interference is a non-concern, and pulling
|
||||
// through requestIdleCallback was adding 50-300 ms before each
|
||||
// newly-written URL surfaced in the table.
|
||||
let lastCrawled = -1;
|
||||
const offProgress = window.freecrawl?.onProgress?.((p) => {
|
||||
if (p.crawled === lastCrawled) return;
|
||||
lastCrawled = p.crawled;
|
||||
if (Date.now() - lastTickTs < TICK_THROTTLE_MS) return;
|
||||
void tick();
|
||||
});
|
||||
|
||||
// Safety-net poll for crawl-idle DB mutations.
|
||||
const id = setInterval(scheduleTick, LIVE_REFRESH_MS);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(id);
|
||||
offProgress?.();
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [keyStr]);
|
||||
@@ -213,7 +285,13 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
const first = Math.max(0, Math.floor(start / CHUNK_SIZE));
|
||||
const last = Math.max(0, Math.floor(end / CHUNK_SIZE));
|
||||
activeRange.current = { first, last };
|
||||
for (let i = first; i <= last; i++) {
|
||||
// Pre-fetch a window of neighbouring chunks so a fast scroll
|
||||
// doesn't show "..." placeholders while the next chunk loads —
|
||||
// by the time the virtualizer scrolls into the next chunk's
|
||||
// range, its rows are usually already in the cache.
|
||||
const fetchFrom = Math.max(0, first - PREFETCH_CHUNKS);
|
||||
const fetchTo = last + PREFETCH_CHUNKS;
|
||||
for (let i = fetchFrom; i <= fetchTo; i++) {
|
||||
if (!chunks.current.has(i)) void fetchChunk(i);
|
||||
}
|
||||
},
|
||||
@@ -237,5 +315,5 @@ export function useLazyUrlRows(opts: LazyRowsOpts): LazyRowsState {
|
||||
[version],
|
||||
);
|
||||
|
||||
return { total, loadedRows, rowAt, ensureRange };
|
||||
return { total, loadedRows, rowAt, ensureRange, isLoading };
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useVirtualizer } from '@tanstack/react-virtual';
|
||||
import { ChevronsUpDown, Filter, X } from 'lucide-react';
|
||||
import { ChevronsUpDown, Download, Filter, X } from 'lucide-react';
|
||||
import clsx from 'clsx';
|
||||
import type { AdvancedFilter, CrawlUrlRow } from '@freecrawl/shared-types';
|
||||
import { useAppStore, type TabKey } from '../store.js';
|
||||
import { COLUMN_SPECS, columnId, type ColumnSpec } from './columns.js';
|
||||
import { useLazyUrlRows } from '../hooks/useLazyUrlRows.js';
|
||||
import { AdvancedFilterDialog } from '../components/AdvancedFilterDialog.js';
|
||||
import { ErrorBoundary } from '../components/ErrorBoundary.js';
|
||||
import { ExportDialog } from '../components/ExportDialog.js';
|
||||
import { InfoTip } from '../components/InfoTip.js';
|
||||
|
||||
const ROW_HEIGHT = 24;
|
||||
@@ -36,6 +38,7 @@ export function UrlsTab() {
|
||||
const [sortDir, setSortDir] = useState<'asc' | 'desc'>('asc');
|
||||
const [filter, setFilter] = useState<AdvancedFilter | null>(null);
|
||||
const [filterDialogOpen, setFilterDialogOpen] = useState(false);
|
||||
const [exportDialogOpen, setExportDialogOpen] = useState(false);
|
||||
// Three orthogonal selection layers:
|
||||
// - selectedIds: row-level selection driven by the Row-number column.
|
||||
// Feeds the bulk context menu (Copy/Open/Re-Spider/Remove/Export).
|
||||
@@ -515,12 +518,21 @@ export function UrlsTab() {
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
<div className="ml-auto text-[11px] text-surface-500">
|
||||
<span className="font-mono text-surface-200">{lazy.total.toLocaleString()}</span> URLs
|
||||
<span className="ml-2 text-surface-600">
|
||||
({lazy.loadedRows.toLocaleString()} loaded)
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setExportDialogOpen(true)}
|
||||
disabled={lazy.total === 0}
|
||||
className={clsx(
|
||||
'ml-auto inline-flex items-center gap-1.5 rounded border px-2 py-1 text-[11px] transition',
|
||||
lazy.total === 0
|
||||
? 'cursor-not-allowed border-surface-800 text-surface-600'
|
||||
: 'border-surface-700 text-surface-300 hover:bg-surface-800',
|
||||
)}
|
||||
title={lazy.total === 0 ? 'No data to export' : 'Export this table'}
|
||||
>
|
||||
<Download className="h-3.5 w-3.5" />
|
||||
<span>Export</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div ref={scrollRef} className="relative flex-1 select-none overflow-auto">
|
||||
@@ -753,7 +765,15 @@ export function UrlsTab() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{lazy.total === 0 && (
|
||||
{lazy.total === 0 && lazy.isLoading && (
|
||||
<div
|
||||
className="pointer-events-none absolute inset-0 flex items-center justify-center"
|
||||
style={{ top: HEADER_HEIGHT }}
|
||||
>
|
||||
<div className="text-xs text-surface-500">Loading…</div>
|
||||
</div>
|
||||
)}
|
||||
{lazy.total === 0 && !lazy.isLoading && (
|
||||
<div
|
||||
className="pointer-events-none absolute inset-0 flex items-center justify-center"
|
||||
style={{ top: HEADER_HEIGHT }}
|
||||
@@ -771,6 +791,15 @@ export function UrlsTab() {
|
||||
className="flex shrink-0 items-center justify-end gap-4 border-t border-surface-800 bg-surface-900/60 px-3 text-[11px] text-surface-400"
|
||||
style={{ height: STATUS_BAR_HEIGHT }}
|
||||
>
|
||||
<span>
|
||||
URLs:{' '}
|
||||
<span className="font-mono tabular-nums text-surface-200">
|
||||
{lazy.total.toLocaleString()}
|
||||
</span>
|
||||
<span className="ml-1 text-surface-600">
|
||||
({lazy.loadedRows.toLocaleString()} loaded)
|
||||
</span>
|
||||
</span>
|
||||
<span>
|
||||
Selected Rows:{' '}
|
||||
<span className="font-mono tabular-nums text-surface-200">
|
||||
@@ -789,12 +818,6 @@ export function UrlsTab() {
|
||||
{selectedColumns.size.toLocaleString()}
|
||||
</span>
|
||||
</span>
|
||||
<span>
|
||||
Filter Total:{' '}
|
||||
<span className="font-mono tabular-nums text-surface-200">
|
||||
{lazy.total.toLocaleString()}
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<AdvancedFilterDialog
|
||||
@@ -803,6 +826,17 @@ export function UrlsTab() {
|
||||
onClose={() => setFilterDialogOpen(false)}
|
||||
onApply={(f) => setFilter(f)}
|
||||
/>
|
||||
|
||||
<ErrorBoundary context="ExportDialog">
|
||||
{exportDialogOpen && (
|
||||
<ExportDialog
|
||||
open={exportDialogOpen}
|
||||
onClose={() => setExportDialogOpen(false)}
|
||||
defaultTab={activeTab}
|
||||
selectedIds={selectedIds.size > 0 ? [...selectedIds] : undefined}
|
||||
/>
|
||||
)}
|
||||
</ErrorBoundary>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "@freecrawl/mcp-server",
|
||||
"version": "0.2.8",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"freecrawl-mcp": "./dist/index.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"start": "node ./dist/index.js",
|
||||
"dev": "tsc -p tsconfig.json --watch",
|
||||
"clean": "rimraf dist"
|
||||
},
|
||||
"dependencies": {
|
||||
"@freecrawl/db": "*",
|
||||
"@freecrawl/shared-types": "*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.0",
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "^5.7.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* FreeCrawl SEO — MCP server (stdio transport, JSON-RPC 2.0).
|
||||
*
|
||||
* Exposes the active `.seoproject` to MCP-compatible clients (Claude
|
||||
* Desktop, Claude Code, …) so an AI agent can run SEO queries against
|
||||
* the most recent crawl directly:
|
||||
*
|
||||
* - `get_summary` → headline numbers
|
||||
* - `get_overview_counts` → sidebar issue/category breakdown
|
||||
* - `top_issues` → ranked non-empty issue list
|
||||
* - `query_urls` → paginated category/search/sort
|
||||
* - `get_url_detail` → full row + inlinks/outlinks/images
|
||||
* - `list_projects` → enumerate `.seoproject` files
|
||||
* - `set_project` → switch sessions to a different file
|
||||
*
|
||||
* The DB connection is read-only (`new ProjectDb(path, { readOnly: true })`)
|
||||
* so the desktop app's writer thread can keep operating concurrently —
|
||||
* SQLite WAL allows arbitrary concurrent readers.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import { ProjectDb } from '@freecrawl/db';
|
||||
import { JsonRpcServer } from './rpc.js';
|
||||
import { buildTools, type Tool } from './tools.js';
|
||||
import { defaultProjectPath, listProjectFiles } from './project-resolver.js';
|
||||
|
||||
const SERVER_NAME = 'freecrawl-seo';
|
||||
const SERVER_VERSION = '0.2.7';
|
||||
const PROTOCOL_VERSION = '2024-11-05';
|
||||
|
||||
interface Session {
|
||||
projectPath: string;
|
||||
db: ProjectDb;
|
||||
}
|
||||
|
||||
let session: Session | null = null;
|
||||
|
||||
function openProject(projectPath: string): Session {
|
||||
if (!fs.existsSync(projectPath)) {
|
||||
throw new Error(`Project file does not exist: ${projectPath}`);
|
||||
}
|
||||
const db = new ProjectDb(projectPath, { readOnly: true });
|
||||
return { projectPath, db };
|
||||
}
|
||||
|
||||
function getSession(): Session {
|
||||
if (session) return session;
|
||||
const initialPath = defaultProjectPath();
|
||||
session = openProject(initialPath);
|
||||
return session;
|
||||
}
|
||||
|
||||
function setProject(projectPath: string): Session {
|
||||
if (session) {
|
||||
try {
|
||||
session.db.close();
|
||||
} catch {
|
||||
// best-effort close
|
||||
}
|
||||
}
|
||||
session = openProject(projectPath);
|
||||
return session;
|
||||
}
|
||||
|
||||
const tools: Tool[] = [
|
||||
...buildTools(),
|
||||
{
|
||||
name: 'list_projects',
|
||||
description:
|
||||
'List `.seoproject` files in the desktop app\'s default projects folder. Returns absolute paths the user can pass to `set_project`.',
|
||||
inputSchema: { type: 'object', properties: {} },
|
||||
handler: () => {
|
||||
const files = listProjectFiles();
|
||||
return { files, defaultPath: defaultProjectPath() };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'set_project',
|
||||
description:
|
||||
'Switch the active project for this MCP session. Subsequent tool calls hit the new file. Closes the previous DB handle.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
required: ['path'],
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string',
|
||||
description: 'Absolute path to a `.seoproject` file.',
|
||||
},
|
||||
},
|
||||
},
|
||||
handler: (args) => {
|
||||
const target = args.path;
|
||||
if (typeof target !== 'string' || target.trim() === '') {
|
||||
throw new Error('set_project requires `path` to be a non-empty string.');
|
||||
}
|
||||
const next = setProject(target);
|
||||
return { ok: true, projectPath: next.projectPath };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'current_project',
|
||||
description:
|
||||
'Report which `.seoproject` the MCP server is currently reading from.',
|
||||
inputSchema: { type: 'object', properties: {} },
|
||||
handler: () => ({ projectPath: getSession().projectPath }),
|
||||
},
|
||||
];
|
||||
|
||||
const server = new JsonRpcServer();
|
||||
|
||||
// ── Required MCP lifecycle methods ───────────────────────────────────
|
||||
|
||||
server.on('initialize', (params) => {
|
||||
const requestedVersion =
|
||||
typeof (params as Record<string, unknown> | undefined)?.protocolVersion === 'string'
|
||||
? ((params as Record<string, string>).protocolVersion)
|
||||
: PROTOCOL_VERSION;
|
||||
return {
|
||||
protocolVersion: requestedVersion,
|
||||
capabilities: {
|
||||
tools: { listChanged: false },
|
||||
},
|
||||
serverInfo: {
|
||||
name: SERVER_NAME,
|
||||
version: SERVER_VERSION,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
// MCP clients send `notifications/initialized` after `initialize`. It
|
||||
// has no `id`, so the JSON-RPC layer won't write a response.
|
||||
server.on('notifications/initialized', () => undefined);
|
||||
|
||||
// ── Tools surface ────────────────────────────────────────────────────
|
||||
|
||||
server.on('tools/list', () => ({
|
||||
tools: tools.map((t) => ({
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
inputSchema: t.inputSchema,
|
||||
})),
|
||||
}));
|
||||
|
||||
server.on('tools/call', async (raw) => {
|
||||
const params = (raw ?? {}) as { name?: string; arguments?: Record<string, unknown> };
|
||||
const name = params.name;
|
||||
const args = params.arguments ?? {};
|
||||
if (typeof name !== 'string') {
|
||||
throw new Error('tools/call requires `name` to be a string.');
|
||||
}
|
||||
const tool = tools.find((t) => t.name === name);
|
||||
if (!tool) throw new Error(`Unknown tool: ${name}`);
|
||||
|
||||
const sess = getSession();
|
||||
let result: unknown;
|
||||
try {
|
||||
result = await tool.handler(args, sess.db);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
isError: true,
|
||||
content: [{ type: 'text', text: msg }],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: typeof result === 'string' ? result : JSON.stringify(result, null, 2),
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
// ── Bootstrap ────────────────────────────────────────────────────────
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
if (session) {
|
||||
try {
|
||||
session.db.close();
|
||||
} catch {
|
||||
// best-effort close
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
server.log(`starting ${SERVER_NAME} v${SERVER_VERSION}`);
|
||||
server.log(`default project: ${defaultProjectPath()}`);
|
||||
server.start();
|
||||
@@ -0,0 +1,57 @@
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import fs from 'node:fs';
|
||||
|
||||
/**
|
||||
* Locate the desktop app's `default.seoproject` without depending on
|
||||
* Electron. The desktop app stores it under
|
||||
* `app.getPath('userData')/projects/default.seoproject`, where
|
||||
* `userData` resolves per-platform using the same rules Electron uses.
|
||||
*
|
||||
* The Electron-resolved userData path is:
|
||||
* - Windows: %APPDATA%/<productName>
|
||||
* - macOS: ~/Library/Application Support/<productName>
|
||||
* - Linux: $XDG_CONFIG_HOME/<productName> or ~/.config/<productName>
|
||||
*
|
||||
* `productName` for FreeCrawl is `@freecrawl/desktop` (taken from
|
||||
* `apps/desktop/package.json`'s `name` field).
|
||||
*/
|
||||
|
||||
const PRODUCT_NAME = '@freecrawl/desktop';
|
||||
const PROJECT_FILENAME = 'default.seoproject';
|
||||
|
||||
export function defaultProjectPath(): string {
|
||||
// Explicit override always wins — handy for headless testing or
|
||||
// pointing at a non-default project.
|
||||
const override = process.env.FREECRAWL_PROJECT;
|
||||
if (override && override.trim().length > 0) return override.trim();
|
||||
|
||||
const home = os.homedir();
|
||||
let userData: string;
|
||||
if (process.platform === 'win32') {
|
||||
userData = path.join(
|
||||
process.env.APPDATA ?? path.join(home, 'AppData', 'Roaming'),
|
||||
PRODUCT_NAME,
|
||||
);
|
||||
} else if (process.platform === 'darwin') {
|
||||
userData = path.join(home, 'Library', 'Application Support', PRODUCT_NAME);
|
||||
} else {
|
||||
userData = path.join(
|
||||
process.env.XDG_CONFIG_HOME ?? path.join(home, '.config'),
|
||||
PRODUCT_NAME,
|
||||
);
|
||||
}
|
||||
return path.join(userData, 'projects', PROJECT_FILENAME);
|
||||
}
|
||||
|
||||
export function listProjectFiles(): string[] {
|
||||
const dir = path.dirname(defaultProjectPath());
|
||||
try {
|
||||
return fs
|
||||
.readdirSync(dir)
|
||||
.filter((f) => f.endsWith('.seoproject'))
|
||||
.map((f) => path.join(dir, f));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* Minimal newline-delimited JSON-RPC 2.0 server for stdio MCP transport.
|
||||
*
|
||||
* MCP's stdio transport is line-delimited JSON — one request per line on
|
||||
* stdin, one response per line on stdout. stderr is reserved for human
|
||||
* logs; writing anything non-JSON to stdout corrupts the protocol stream.
|
||||
*
|
||||
* We deliberately do NOT depend on `@modelcontextprotocol/sdk`. The
|
||||
* protocol surface we need (`initialize`, `tools/list`, `tools/call`,
|
||||
* the `initialized` notification) is small, well-specified, and stable;
|
||||
* the dep-free path keeps the install footprint minimal and avoids a
|
||||
* version-pin we'd otherwise have to chase.
|
||||
*/
|
||||
|
||||
export interface JsonRpcRequest {
|
||||
jsonrpc: '2.0';
|
||||
id?: number | string | null;
|
||||
method: string;
|
||||
params?: unknown;
|
||||
}
|
||||
|
||||
export interface JsonRpcSuccess {
|
||||
jsonrpc: '2.0';
|
||||
id: number | string | null;
|
||||
result: unknown;
|
||||
}
|
||||
|
||||
export interface JsonRpcError {
|
||||
jsonrpc: '2.0';
|
||||
id: number | string | null;
|
||||
error: { code: number; message: string; data?: unknown };
|
||||
}
|
||||
|
||||
export type Handler = (params: unknown) => unknown | Promise<unknown>;
|
||||
|
||||
export class JsonRpcServer {
|
||||
private handlers = new Map<string, Handler>();
|
||||
private buffer = '';
|
||||
|
||||
on(method: string, handler: Handler): void {
|
||||
this.handlers.set(method, handler);
|
||||
}
|
||||
|
||||
log(...parts: unknown[]): void {
|
||||
// Always to stderr — stdout is reserved for protocol frames.
|
||||
const line = parts
|
||||
.map((p) => (typeof p === 'string' ? p : JSON.stringify(p)))
|
||||
.join(' ');
|
||||
process.stderr.write(`[freecrawl-mcp] ${line}\n`);
|
||||
}
|
||||
|
||||
start(): void {
|
||||
process.stdin.setEncoding('utf8');
|
||||
process.stdin.on('data', (chunk: string) => {
|
||||
this.buffer += chunk;
|
||||
let nl: number;
|
||||
while ((nl = this.buffer.indexOf('\n')) !== -1) {
|
||||
const line = this.buffer.slice(0, nl).trim();
|
||||
this.buffer = this.buffer.slice(nl + 1);
|
||||
if (line) void this.handleLine(line);
|
||||
}
|
||||
});
|
||||
process.stdin.on('end', () => process.exit(0));
|
||||
}
|
||||
|
||||
private write(msg: JsonRpcSuccess | JsonRpcError): void {
|
||||
process.stdout.write(JSON.stringify(msg) + '\n');
|
||||
}
|
||||
|
||||
private async handleLine(line: string): Promise<void> {
|
||||
let req: JsonRpcRequest;
|
||||
try {
|
||||
req = JSON.parse(line) as JsonRpcRequest;
|
||||
} catch {
|
||||
this.log('parse error', line);
|
||||
return;
|
||||
}
|
||||
const id = req.id ?? null;
|
||||
const handler = this.handlers.get(req.method);
|
||||
// Notifications (no `id`) get no response, even on miss.
|
||||
const isNotification = req.id === undefined || req.id === null;
|
||||
if (!handler) {
|
||||
if (!isNotification) {
|
||||
this.write({
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
error: { code: -32601, message: `Method not found: ${req.method}` },
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await handler(req.params);
|
||||
if (!isNotification) {
|
||||
this.write({ jsonrpc: '2.0', id, result });
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
this.log('handler error', req.method, msg);
|
||||
if (!isNotification) {
|
||||
this.write({
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
error: { code: -32000, message: msg },
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Tool definitions and handlers exposed by the FreeCrawl MCP server.
|
||||
*
|
||||
* The server opens the project DB in read-only mode (see `index.ts`),
|
||||
* so it can coexist with the desktop app's writer without contention.
|
||||
* SQLite WAL mode permits arbitrary concurrent readers.
|
||||
*/
|
||||
|
||||
import type { ProjectDb } from '@freecrawl/db';
|
||||
import type { UrlCategory } from '@freecrawl/shared-types';
|
||||
|
||||
export interface Tool {
|
||||
name: string;
|
||||
description: string;
|
||||
inputSchema: Record<string, unknown>;
|
||||
handler: (args: Record<string, unknown>, db: ProjectDb) => unknown | Promise<unknown>;
|
||||
}
|
||||
|
||||
const URL_CATEGORY_VALUES: UrlCategory[] = [
|
||||
'all',
|
||||
'internal:all',
|
||||
'internal:html',
|
||||
'internal:js',
|
||||
'internal:css',
|
||||
'internal:image',
|
||||
'internal:pdf',
|
||||
'internal:font',
|
||||
'internal:other',
|
||||
'external:all',
|
||||
'external:html',
|
||||
'external:other',
|
||||
'status:2xx',
|
||||
'status:3xx',
|
||||
'status:4xx',
|
||||
'status:5xx',
|
||||
'status:no-response',
|
||||
'status:blocked-robots',
|
||||
'security:https',
|
||||
'security:http',
|
||||
'indexability:indexable',
|
||||
'indexability:non-indexable',
|
||||
'indexability:noindex',
|
||||
'indexability:canonicalised',
|
||||
];
|
||||
|
||||
const COMMON_URL_FIELDS = [
|
||||
'id',
|
||||
'url',
|
||||
'statusCode',
|
||||
'contentKind',
|
||||
'indexability',
|
||||
'title',
|
||||
'titleLength',
|
||||
'metaDescription',
|
||||
'metaDescriptionLength',
|
||||
'h1',
|
||||
'h1Count',
|
||||
'wordCount',
|
||||
'canonical',
|
||||
'metaRobots',
|
||||
'contentLength',
|
||||
'responseTimeMs',
|
||||
'depth',
|
||||
'inlinks',
|
||||
'outlinks',
|
||||
'imagesCount',
|
||||
'imagesMissingAlt',
|
||||
'redirectTarget',
|
||||
'crawledAt',
|
||||
];
|
||||
|
||||
function projectFromRow(
|
||||
row: Record<string, unknown>,
|
||||
fields: string[],
|
||||
): Record<string, unknown> {
|
||||
if (fields.length === 0) {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const k of COMMON_URL_FIELDS) out[k] = row[k];
|
||||
return out;
|
||||
}
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const k of fields) out[k] = row[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
export function buildTools(): Tool[] {
|
||||
return [
|
||||
{
|
||||
name: 'get_summary',
|
||||
description:
|
||||
'Top-level crawl summary for the active FreeCrawl project: total URLs crawled, indexable/non-indexable counts, status-code distribution, average response time, plus crawl start/end timestamps. Use this first when reporting on a crawl.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
},
|
||||
handler: (_args, db) => db.getSummary(),
|
||||
},
|
||||
|
||||
{
|
||||
name: 'get_overview_counts',
|
||||
description:
|
||||
'Sidebar overview counts — every category and issue filter the desktop app surfaces, with row counts. Useful to find which issue categories are non-empty before drilling in with query_urls.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
},
|
||||
handler: (_args, db) => db.getOverviewCounts(),
|
||||
},
|
||||
|
||||
{
|
||||
name: 'query_urls',
|
||||
description:
|
||||
'Paginated query against the URLs table with the same category filter the UI uses. Returns rows + total count. Use `category` to narrow (e.g. "issues:title-missing"), `search` to substring-match URLs/titles, and `limit`/`offset` to page through.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
category: {
|
||||
type: 'string',
|
||||
description: 'UrlCategory filter (e.g. "all", "internal:html", "issues:title-missing", "status:4xx"). Defaults to "all".',
|
||||
enum: URL_CATEGORY_VALUES,
|
||||
},
|
||||
search: {
|
||||
type: 'string',
|
||||
description: 'Optional substring match against URL and title.',
|
||||
},
|
||||
limit: {
|
||||
type: 'integer',
|
||||
minimum: 1,
|
||||
maximum: 500,
|
||||
description: 'Max rows to return (1–500). Default 50.',
|
||||
},
|
||||
offset: {
|
||||
type: 'integer',
|
||||
minimum: 0,
|
||||
description: 'Row offset for pagination. Default 0.',
|
||||
},
|
||||
sortBy: {
|
||||
type: 'string',
|
||||
description: 'Column to sort by (e.g. "depth", "responseTimeMs", "inlinks").',
|
||||
},
|
||||
sortDir: {
|
||||
type: 'string',
|
||||
enum: ['asc', 'desc'],
|
||||
description: 'Sort direction. Default "asc".',
|
||||
},
|
||||
fields: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
description:
|
||||
'Columns to include per row. Empty/omitted → a sensible default subset (URL, status, indexability, title, depth, inlinks, etc.). Use ["*"] for the full row.',
|
||||
},
|
||||
},
|
||||
},
|
||||
handler: (args, db) => {
|
||||
const category = (args.category as UrlCategory | undefined) ?? 'all';
|
||||
const search = typeof args.search === 'string' ? args.search : undefined;
|
||||
const limit = Math.min(
|
||||
Math.max(typeof args.limit === 'number' ? args.limit : 50, 1),
|
||||
500,
|
||||
);
|
||||
const offset = Math.max(
|
||||
typeof args.offset === 'number' ? args.offset : 0,
|
||||
0,
|
||||
);
|
||||
const sortBy = typeof args.sortBy === 'string' ? args.sortBy : undefined;
|
||||
const sortDir = args.sortDir === 'desc' ? 'desc' : 'asc';
|
||||
const fieldsArg = Array.isArray(args.fields) ? (args.fields as string[]) : [];
|
||||
const wantsAll = fieldsArg.length === 1 && fieldsArg[0] === '*';
|
||||
|
||||
const { rows, total } = db.queryUrls({
|
||||
category,
|
||||
search,
|
||||
limit,
|
||||
offset,
|
||||
sortBy,
|
||||
sortDir,
|
||||
});
|
||||
const projected = wantsAll
|
||||
? rows
|
||||
: rows.map((r) =>
|
||||
projectFromRow(
|
||||
r as unknown as Record<string, unknown>,
|
||||
fieldsArg,
|
||||
),
|
||||
);
|
||||
return { rows: projected, total, limit, offset };
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: 'get_url_detail',
|
||||
description:
|
||||
'Full details for one URL: every column, plus inlinks, outlinks, images, and HTTP headers. Pass either `id` (preferred — from query_urls) or `url`.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: {
|
||||
type: 'integer',
|
||||
description: 'Numeric URL id from query_urls / get_url_id.',
|
||||
},
|
||||
url: {
|
||||
type: 'string',
|
||||
description: 'Exact URL string (slower path — uses an indexed lookup).',
|
||||
},
|
||||
linkLimit: {
|
||||
type: 'integer',
|
||||
description: 'Cap on inlinks + outlinks returned. Default 500.',
|
||||
},
|
||||
},
|
||||
},
|
||||
handler: (args, db) => {
|
||||
let id: number | null = null;
|
||||
if (typeof args.id === 'number') {
|
||||
id = args.id;
|
||||
} else if (typeof args.url === 'string') {
|
||||
const found = db.getUrlIdByUrl(args.url);
|
||||
if (found === null) {
|
||||
throw new Error(`URL not found: ${args.url}`);
|
||||
}
|
||||
id = found;
|
||||
} else {
|
||||
throw new Error('get_url_detail requires either `id` or `url`.');
|
||||
}
|
||||
const linkLimit =
|
||||
typeof args.linkLimit === 'number' ? args.linkLimit : 500;
|
||||
const detail = db.getUrlDetail(id, linkLimit);
|
||||
if (detail === null) {
|
||||
throw new Error(`URL id ${id} not found in this project.`);
|
||||
}
|
||||
return detail;
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: 'top_issues',
|
||||
description:
|
||||
'Ranked list of non-empty issue categories from get_overview_counts, sorted by count descending. Quick way to ask "what are the biggest problems with this crawl?".',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
limit: {
|
||||
type: 'integer',
|
||||
description: 'Max categories to return. Default 20.',
|
||||
minimum: 1,
|
||||
maximum: 200,
|
||||
},
|
||||
},
|
||||
},
|
||||
handler: (args, db) => {
|
||||
const limit = Math.min(
|
||||
Math.max(typeof args.limit === 'number' ? args.limit : 20, 1),
|
||||
200,
|
||||
);
|
||||
const counts = db.getOverviewCounts() as unknown as Record<string, unknown>;
|
||||
const issues = (counts.issues as Record<string, number>) ?? {};
|
||||
const ranked = Object.entries(issues)
|
||||
.filter(([, n]) => n > 0)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, limit)
|
||||
.map(([category, count]) => ({ category: `issues:${category}`, count }));
|
||||
return { issues: ranked };
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"composite": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"references": [
|
||||
{ "path": "../../packages/shared-types" },
|
||||
{ "path": "../../packages/db" }
|
||||
]
|
||||
}
|
||||
Generated
+27
-7
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "freecrawl-seo",
|
||||
"version": "0.1.10",
|
||||
"version": "0.2.8",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "freecrawl-seo",
|
||||
"version": "0.1.10",
|
||||
"version": "0.2.8",
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
@@ -25,7 +25,7 @@
|
||||
},
|
||||
"apps/cli": {
|
||||
"name": "@freecrawl/cli",
|
||||
"version": "0.1.10",
|
||||
"version": "0.2.8",
|
||||
"dependencies": {
|
||||
"@freecrawl/core": "*",
|
||||
"@freecrawl/db": "*",
|
||||
@@ -42,7 +42,7 @@
|
||||
},
|
||||
"apps/desktop": {
|
||||
"name": "@freecrawl/desktop",
|
||||
"version": "0.1.10",
|
||||
"version": "0.2.8",
|
||||
"dependencies": {
|
||||
"@freecrawl/core": "*",
|
||||
"@freecrawl/db": "*",
|
||||
@@ -82,6 +82,22 @@
|
||||
"react-dom": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
|
||||
}
|
||||
},
|
||||
"apps/mcp-server": {
|
||||
"name": "@freecrawl/mcp-server",
|
||||
"version": "0.2.8",
|
||||
"dependencies": {
|
||||
"@freecrawl/db": "*",
|
||||
"@freecrawl/shared-types": "*"
|
||||
},
|
||||
"bin": {
|
||||
"freecrawl-mcp": "dist/index.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.0",
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "^5.7.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@alloc/quick-lru": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",
|
||||
@@ -1431,6 +1447,10 @@
|
||||
"resolved": "apps/desktop",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@freecrawl/mcp-server": {
|
||||
"resolved": "apps/mcp-server",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@freecrawl/shared-types": {
|
||||
"resolved": "packages/shared-types",
|
||||
"link": true
|
||||
@@ -8712,7 +8732,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@freecrawl/core",
|
||||
"version": "0.1.10",
|
||||
"version": "0.2.8",
|
||||
"dependencies": {
|
||||
"@freecrawl/db": "*",
|
||||
"@freecrawl/shared-types": "*",
|
||||
@@ -8738,7 +8758,7 @@
|
||||
},
|
||||
"packages/db": {
|
||||
"name": "@freecrawl/db",
|
||||
"version": "0.1.10",
|
||||
"version": "0.2.8",
|
||||
"dependencies": {
|
||||
"@freecrawl/shared-types": "*"
|
||||
},
|
||||
@@ -8750,7 +8770,7 @@
|
||||
},
|
||||
"packages/shared-types": {
|
||||
"name": "@freecrawl/shared-types",
|
||||
"version": "0.1.10",
|
||||
"version": "0.2.8",
|
||||
"devDependencies": {
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "^5.7.0"
|
||||
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "freecrawl-seo",
|
||||
"version": "0.2.7",
|
||||
"version": "0.2.8",
|
||||
"private": true,
|
||||
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
|
||||
"license": "MIT",
|
||||
@@ -18,6 +18,7 @@
|
||||
"build": "tsc -b && npm run build --workspace apps/desktop && npm run build --workspace apps/cli",
|
||||
"build:desktop": "npm run build --workspace apps/desktop",
|
||||
"build:cli": "npm run build --workspace apps/cli",
|
||||
"build:mcp": "npm run build --workspace apps/mcp-server",
|
||||
"typecheck": "npm run typecheck --workspaces --if-present",
|
||||
"lint": "eslint .",
|
||||
"format": "prettier --write \"**/*.{ts,tsx,js,json,md,css,html}\"",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/core",
|
||||
"version": "0.2.7",
|
||||
"version": "0.2.8",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
+174
-31
@@ -89,13 +89,23 @@ export class Crawler extends EventEmitter {
|
||||
*/
|
||||
private memoryAutoPaused = false;
|
||||
private memoryMonitorTimer: NodeJS.Timeout | null = null;
|
||||
/**
|
||||
* Live AbortControllers for in-flight HTTP fetches in the main URL
|
||||
* pipeline. Pause / Stop abort every member so the crawl actually
|
||||
* stops instead of letting `concurrency` (often 20) more URLs land
|
||||
* in the table after the user clicked the button. Each call site
|
||||
* adds itself in `try` and removes itself in `finally`.
|
||||
*/
|
||||
private inFlightFetchControllers = new Set<AbortController>();
|
||||
private robots: RobotsChecker | null = null;
|
||||
private progressTimer: NodeJS.Timeout | null = null;
|
||||
/** Periodic post-crawl-style recompute of expensive issue counters
|
||||
* while the crawl is still running, so the sidebar number is < 30 s
|
||||
* stale instead of "0 until crawl ends". 30 s cadence picked
|
||||
* conservatively — the recompute wraps in `runInTransaction` and is
|
||||
* not free on a 100k-URL DB. */
|
||||
* stale instead of "0 until crawl ends". The recompute is dispatched
|
||||
* to the writer-worker via the injected `recomputeIssues` hook, so
|
||||
* it does NOT block the main thread — the freezes that an earlier
|
||||
* version produced were a result of running the recompute inline on
|
||||
* the main-thread writer connection, which has since been fixed. */
|
||||
private issueRecomputeTimer: NodeJS.Timeout | null = null;
|
||||
private static readonly ISSUE_RECOMPUTE_INTERVAL_MS = 30_000;
|
||||
private issueRecomputeInFlight = false;
|
||||
@@ -103,20 +113,16 @@ export class Crawler extends EventEmitter {
|
||||
if (this.issueRecomputeTimer) return;
|
||||
this.issueRecomputeTimer = setInterval(() => {
|
||||
if (this.stopped || !this.running || this.paused) return;
|
||||
// Drop overlapping ticks. The yielding recompute can take longer
|
||||
// than the 30 s tick on very large projects; piling up calls
|
||||
// would queue 70-statement transactions back-to-back and starve
|
||||
// the crawler's own writes.
|
||||
// Drop overlapping ticks — on a 1M-URL DB the recompute can run
|
||||
// longer than the 30 s tick, and piling RPCs on the writer worker
|
||||
// would starve per-URL writes behind a queue of issue passes.
|
||||
if (this.issueRecomputeInFlight) return;
|
||||
this.issueRecomputeInFlight = true;
|
||||
// Cooperative variant: yields between definitions so a 1–3 s
|
||||
// recompute doesn't freeze the main thread (and the user's
|
||||
// clicks) for the full duration. Trades atomic visibility of
|
||||
// the rebuild for UI responsiveness — fine for the live tick
|
||||
// since the post-crawl pass still uses the atomic version for
|
||||
// the final state.
|
||||
void this.db
|
||||
.recomputeUrlsIssuesYielding(EXPENSIVE_ISSUE_DEFINITIONS)
|
||||
// The injected hook routes through the writer worker in the
|
||||
// desktop host, so this is off-thread; main stays free to
|
||||
// service IPC and pump the crawl queue. CLI fallback runs the
|
||||
// yielding variant inline, which is fine for that context.
|
||||
void this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS)
|
||||
.catch((err) => {
|
||||
this.emit(
|
||||
'debug',
|
||||
@@ -230,6 +236,30 @@ export class Crawler extends EventEmitter {
|
||||
payload: Parameters<ProjectDb['writeFetchedUrl']>[0],
|
||||
) => Promise<{ urlId: number }>;
|
||||
|
||||
/**
|
||||
* Optional async hook for the post-crawl issue recompute. When the
|
||||
* desktop main process injects a writer-worker bridge here, the
|
||||
* 70+ correlated subqueries run off the main thread — without it,
|
||||
* a 5K-URL crawl freezes the renderer for ~45 s in the post-crawl
|
||||
* phase ("materialise-issues"). Defaults to running the recompute
|
||||
* inline (CLI / test contexts).
|
||||
*/
|
||||
private readonly recomputeIssues: (
|
||||
definitions: ReadonlyArray<readonly [string, string]>,
|
||||
) => Promise<void>;
|
||||
|
||||
/**
|
||||
* Generic post-crawl SQL-pass dispatch. Methods like
|
||||
* `recomputeInlinks`, `recomputeRedirectChains`,
|
||||
* `recomputeHreflangAnalysis`, `recomputeHreflangInconsistent`,
|
||||
* `recomputePaginationSequence` are sync void calls on `ProjectDb`
|
||||
* — running them inline on the main thread blocks the renderer for
|
||||
* ~5-15 s combined on a 5K+ URL crawl. The desktop host injects a
|
||||
* writer-worker dispatcher here so each pass runs off-thread; CLI /
|
||||
* tests fall back to calling the method directly on `this.db`.
|
||||
*/
|
||||
private readonly runDbPass: (methodName: string) => Promise<void>;
|
||||
|
||||
constructor(
|
||||
config: CrawlConfig,
|
||||
db: ProjectDb,
|
||||
@@ -243,6 +273,10 @@ export class Crawler extends EventEmitter {
|
||||
writeFetchedUrl?: (
|
||||
payload: Parameters<ProjectDb['writeFetchedUrl']>[0],
|
||||
) => Promise<{ urlId: number }>;
|
||||
recomputeIssues?: (
|
||||
definitions: ReadonlyArray<readonly [string, string]>,
|
||||
) => Promise<void>;
|
||||
runDbPass?: (methodName: string) => Promise<void>;
|
||||
} = {},
|
||||
) {
|
||||
super();
|
||||
@@ -252,6 +286,15 @@ export class Crawler extends EventEmitter {
|
||||
((html, pageUrl, parseOpts) => Promise.resolve(parseHtml(html, pageUrl, parseOpts)));
|
||||
this.writeFetchedUrl =
|
||||
opts.writeFetchedUrl ?? ((payload) => Promise.resolve(this.db.writeFetchedUrl(payload)));
|
||||
this.recomputeIssues =
|
||||
opts.recomputeIssues ?? ((defs) => this.db.recomputeUrlsIssuesYielding(defs));
|
||||
this.runDbPass =
|
||||
opts.runDbPass ??
|
||||
((method) => {
|
||||
const fn = (this.db as unknown as Record<string, unknown>)[method];
|
||||
if (typeof fn === 'function') (fn as () => void).call(this.db);
|
||||
return Promise.resolve();
|
||||
});
|
||||
// Wave 9 — Resolve the active proxy. If a named profile is selected
|
||||
// and present in `proxyProfiles`, its URL wins over the legacy
|
||||
// `proxyUrl` field; if the named profile doesn't resolve we fall
|
||||
@@ -268,14 +311,27 @@ export class Crawler extends EventEmitter {
|
||||
this.config = config;
|
||||
this.db = db;
|
||||
const concurrency = Math.max(1, Math.min(200, config.maxConcurrency));
|
||||
const intervalCap = Math.max(1, config.maxRps);
|
||||
this.queue = new PQueue({ concurrency, interval: 1000, intervalCap });
|
||||
// Rate-limit (interval + intervalCap) intentionally NOT set on
|
||||
// the queue. p-queue's bucket-refill semantics interact poorly
|
||||
// with long-tailed response times: after the first burst lands
|
||||
// and slots free up, dispatch can stall for an entire bucket
|
||||
// before the next batch goes out — and on a 1 s/intervalCap=20
|
||||
// bucket the queue would freeze entirely once the first ~20
|
||||
// tasks completed. Throughput is now bounded purely by the
|
||||
// worker pool (concurrency × 1/avg_response), which on a slow
|
||||
// remote server gives the user the 5-10 URL/s they expect
|
||||
// instead of ~1 URL/s. config.maxRps is honoured indirectly via
|
||||
// maxConcurrency — users who need a hard RPS cap should drop
|
||||
// concurrency rather than trust per-second buckets.
|
||||
this.queue = new PQueue({ concurrency });
|
||||
// Track queue concurrency from the start so adaptive shrinkage
|
||||
// doesn't have to wait for the first reportRendererLag call to
|
||||
// materialise the running value.
|
||||
this.currentConcurrency = concurrency;
|
||||
// External probes run on a separate queue so slow third-party hosts
|
||||
// don't block the main crawl.
|
||||
this.externalQueue = new PQueue({
|
||||
concurrency: Math.max(2, Math.min(10, concurrency)),
|
||||
interval: 1000,
|
||||
intervalCap: Math.max(2, intervalCap),
|
||||
});
|
||||
// Compile include/exclude patterns once — an invalid pattern should
|
||||
// surface to the user as a crawler error, not a silent miss.
|
||||
@@ -445,7 +501,17 @@ export class Crawler extends EventEmitter {
|
||||
try {
|
||||
// Wait for internal crawl first, then drain any external probes still
|
||||
// in flight or queued (externals may have been enqueued during internal).
|
||||
// p-queue's onIdle resolves the moment size+pending hits 0, but
|
||||
// fetchAndProcess can race with that: a worker finishes the
|
||||
// *last* in-flight task, link extraction enqueues new URLs a
|
||||
// microtask later, and onIdle has already resolved. Loop while
|
||||
// counters disagree — one extra 50 ms wait per pass guards the
|
||||
// race window without spinning.
|
||||
while (!this.stopped) {
|
||||
await this.queue.onIdle();
|
||||
await new Promise<void>((r) => setTimeout(r, 50));
|
||||
if (this.queue.size + this.queue.pending === 0) break;
|
||||
}
|
||||
await this.externalQueue.onIdle();
|
||||
// robots.txt + sitemap discovery may still be running — wait for
|
||||
// both before the post-crawl recompute so issue filters depending
|
||||
@@ -512,44 +578,49 @@ export class Crawler extends EventEmitter {
|
||||
await yieldToEventLoop();
|
||||
this.emit('info', 'Recomputing inlinks…');
|
||||
this.setOp('post-crawl:recompute-inlinks');
|
||||
this.db.recomputeInlinks();
|
||||
await this.runDbPass('recomputeInlinks');
|
||||
}
|
||||
if (this.config.analyseRedirectChains) {
|
||||
await yieldToEventLoop();
|
||||
this.emit('info', 'Recomputing redirect chains…');
|
||||
this.setOp('post-crawl:recompute-redirect-chains');
|
||||
this.db.recomputeRedirectChains();
|
||||
await this.runDbPass('recomputeRedirectChains');
|
||||
}
|
||||
if (this.config.analyseHreflang) {
|
||||
await yieldToEventLoop();
|
||||
this.emit('info', 'Recomputing hreflang analysis…');
|
||||
this.setOp('post-crawl:recompute-hreflang');
|
||||
this.db.recomputeHreflangAnalysis();
|
||||
await this.runDbPass('recomputeHreflangAnalysis');
|
||||
await yieldToEventLoop();
|
||||
this.setOp('post-crawl:recompute-hreflang-inconsistent');
|
||||
this.db.recomputeHreflangInconsistent();
|
||||
await this.runDbPass('recomputeHreflangInconsistent');
|
||||
}
|
||||
if (this.config.analyseDuplicates) {
|
||||
await yieldToEventLoop();
|
||||
this.emit('info', 'Clustering duplicates…');
|
||||
this.setOp('post-crawl:cluster-duplicates');
|
||||
// Duplicate clustering has JS-side simhash work that lives on
|
||||
// the crawler instance, not the DB; can't blindly off-thread it.
|
||||
this.runDuplicateClustering();
|
||||
}
|
||||
if (this.config.analysePagination) {
|
||||
await yieldToEventLoop();
|
||||
this.emit('info', 'Detecting pagination sequence gaps…');
|
||||
this.setOp('post-crawl:pagination-sequence');
|
||||
this.db.recomputePaginationSequence();
|
||||
await this.runDbPass('recomputePaginationSequence');
|
||||
}
|
||||
if (this.config.analyseIssues) {
|
||||
await yieldToEventLoop();
|
||||
this.emit('info', 'Materialising issue counters…');
|
||||
this.setOp('post-crawl:materialise-issues');
|
||||
// Use the cooperatively-scheduled variant (yields between each
|
||||
// of the 70+ definitions). Trades atomic visibility of the
|
||||
// rebuild for a responsive main thread — on a 5k-URL DB the
|
||||
// sync version was blocking the event loop for 22-26 s.
|
||||
await this.db.recomputeUrlsIssuesYielding(EXPENSIVE_ISSUE_DEFINITIONS);
|
||||
// Route through the injected hook — in the desktop host this
|
||||
// dispatches to the writer-worker, keeping the 70+ correlated
|
||||
// subqueries off the main thread. CLI / tests fall back to
|
||||
// running inline via `db.recomputeUrlsIssuesYielding`. Without
|
||||
// offloading, a 5K-URL crawl freezes the main process for
|
||||
// ~45 s in post-crawl, tripping Electron's "Not Responding"
|
||||
// dialog.
|
||||
await this.recomputeIssues(EXPENSIVE_ISSUE_DEFINITIONS);
|
||||
}
|
||||
await yieldToEventLoop();
|
||||
this.setOp('post-crawl:image-probes');
|
||||
@@ -1022,6 +1093,24 @@ export class Crawler extends EventEmitter {
|
||||
|
||||
// If the start URL isn't in the DB yet, kick off a brand-new crawl from it.
|
||||
if (!this.db.hasUrl(this.config.startUrl)) {
|
||||
// Stub upsert before enqueue so the seed surfaces in the URL
|
||||
// table within ~100 ms of clicking Start instead of waiting for
|
||||
// the full network round-trip + parse + write (~2 s on a typical
|
||||
// site). status_code=null reads as "fetching" in the table.
|
||||
// fetchAndProcess's writeFetchedUrl later overwrites every field
|
||||
// with the real response data via INSERT … ON CONFLICT UPDATE.
|
||||
try {
|
||||
this.db.upsertUrl({
|
||||
url: this.config.startUrl,
|
||||
contentKind: 'html',
|
||||
statusCode: null,
|
||||
statusText: 'fetching',
|
||||
indexability: 'indexable',
|
||||
depth: 0,
|
||||
});
|
||||
} catch {
|
||||
/* non-fatal: fetchAndProcess will still write the real row */
|
||||
}
|
||||
this.enqueue({ url: this.config.startUrl, depth: 0 });
|
||||
}
|
||||
|
||||
@@ -1140,6 +1229,19 @@ export class Crawler extends EventEmitter {
|
||||
}
|
||||
this.sitemapAbort = null;
|
||||
}
|
||||
// Abort every in-flight HTTP fetch. Without this, Stop only halts
|
||||
// *new* dispatch — the ~20 fetches already in flight would each
|
||||
// finish their request and write a row, which races with Clear's
|
||||
// db.reset() and resurrects ~19 rows in the table. The catch
|
||||
// branch in fetchAndProcess detects the stopped-abort and exits
|
||||
// silently without writing.
|
||||
for (const c of this.inFlightFetchControllers) {
|
||||
try {
|
||||
c.abort();
|
||||
} catch {
|
||||
/* ignore — already aborted */
|
||||
}
|
||||
}
|
||||
// Drop any queued work. If paused, unblock onIdle() so start() can resolve.
|
||||
this.queue.clear();
|
||||
this.externalQueue.clear();
|
||||
@@ -1150,9 +1252,22 @@ export class Crawler extends EventEmitter {
|
||||
pause(): void {
|
||||
if (this.stopped || this.paused) return;
|
||||
this.paused = true;
|
||||
// PQueue.pause() halts dispatch but lets in-flight tasks finish naturally.
|
||||
// PQueue.pause() halts new dispatch. By itself that lets every
|
||||
// in-flight fetch run to completion — at concurrency 20 the user
|
||||
// sees ~20 more URLs land in the table before things stop, which
|
||||
// reads as "Pause didn't work." Aborting the live controllers
|
||||
// turns Pause into "stop now": fetchAndProcess detects a paused-
|
||||
// abort, re-enqueues the URL, and returns without writing a row.
|
||||
this.queue.pause();
|
||||
this.externalQueue.pause();
|
||||
for (const c of this.inFlightFetchControllers) {
|
||||
try {
|
||||
c.abort();
|
||||
} catch {
|
||||
/* ignore — already aborted */
|
||||
}
|
||||
}
|
||||
this.setOp('paused');
|
||||
this.emitProgress();
|
||||
}
|
||||
|
||||
@@ -1161,6 +1276,7 @@ export class Crawler extends EventEmitter {
|
||||
this.paused = false;
|
||||
this.queue.start();
|
||||
this.externalQueue.start();
|
||||
this.setOp('idle');
|
||||
this.emitProgress();
|
||||
}
|
||||
|
||||
@@ -1446,6 +1562,7 @@ export class Crawler extends EventEmitter {
|
||||
|
||||
const t0 = Date.now();
|
||||
const controller = new AbortController();
|
||||
this.inFlightFetchControllers.add(controller);
|
||||
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
||||
// Wave 3 — Optional max-response-time hard cap. Distinct from
|
||||
// `requestTimeoutMs` (which is the connect+headers timeout): this
|
||||
@@ -1998,6 +2115,25 @@ export class Crawler extends EventEmitter {
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// Stop / Clear: abandon silently. No row is written, no failure
|
||||
// counter is bumped, the URL is not re-enqueued. Without this
|
||||
// gate the ~20 in-flight URLs after Stop would each land in the
|
||||
// table — and Clear's db.reset() races with their writes,
|
||||
// resurrecting "ghost" rows in a freshly-cleared table.
|
||||
if (this.stopped && controller.signal.aborted) {
|
||||
return;
|
||||
}
|
||||
// User-initiated Pause aborts every live controller. Don't treat
|
||||
// those as fetch failures — the URL hasn't been seen yet, so we
|
||||
// re-enqueue it directly (bypassing `enqueue` because the URL is
|
||||
// already in the `seen` set from the original dispatch). Without
|
||||
// this, Pause would mark `concurrency` (often 20) URLs as failed
|
||||
// and surface them as errors in the URL table. Re-enqueue lands
|
||||
// in a paused queue → resume() picks them up cleanly.
|
||||
if (this.paused && controller.signal.aborted) {
|
||||
this.queue.add(() => this.fetchAndProcess(item)).catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
this.failed++;
|
||||
const detail = formatFetchError(err);
|
||||
const elapsed = Date.now() - t0;
|
||||
@@ -2020,6 +2156,7 @@ export class Crawler extends EventEmitter {
|
||||
depth: item.depth,
|
||||
});
|
||||
} finally {
|
||||
this.inFlightFetchControllers.delete(controller);
|
||||
clearTimeout(timeout);
|
||||
if (respTimeTimer) clearTimeout(respTimeTimer);
|
||||
// Politeness delay — applied per worker *after* each request so a
|
||||
@@ -2171,8 +2308,14 @@ export class Crawler extends EventEmitter {
|
||||
const ceiling = Math.max(1, Math.min(200, this.config.maxConcurrency));
|
||||
if (this.currentConcurrency === 0) this.currentConcurrency = ceiling;
|
||||
let next = this.currentConcurrency;
|
||||
// Floor 5 (was 1). The renderer's lag probe occasionally spikes
|
||||
// for unrelated reasons (GC, an input burst, devtools open) and
|
||||
// a 1-floor would let those one-off spikes corner the queue at
|
||||
// single-task dispatch — which on a slow remote server (avg resp
|
||||
// 1.5-2 s) cuts throughput from 10 URL/s to 0.5 URL/s and the
|
||||
// queue never recovers because the user never produces zero lag.
|
||||
if (lagMs > 200) {
|
||||
next = Math.max(1, this.currentConcurrency - 1);
|
||||
next = Math.max(5, this.currentConcurrency - 1);
|
||||
} else if (lagMs < 30) {
|
||||
next = Math.min(ceiling, this.currentConcurrency + 1);
|
||||
}
|
||||
|
||||
@@ -10,6 +10,12 @@ export {
|
||||
export { exportUrlsToCsv } from './csv-export.js';
|
||||
export { exportUrlsToJson, type JsonExportOptions } from './json-export.js';
|
||||
export { exportUrlsToXml } from './xml-export.js';
|
||||
export {
|
||||
exportTabular,
|
||||
type TabularExportOptions,
|
||||
type TabularExportResult,
|
||||
type TabularSection,
|
||||
} from './tabular-export.js';
|
||||
export { testUrlAgainstRobots, type RobotsTestResult } from './robots.js';
|
||||
export {
|
||||
exportSitemap,
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
import { createWriteStream, writeFileSync } from 'node:fs';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { Readable } from 'node:stream';
|
||||
import path from 'node:path';
|
||||
import zlib from 'node:zlib';
|
||||
import type { ProjectDb } from '@freecrawl/db';
|
||||
import type { CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
|
||||
|
||||
export interface TabularSection {
|
||||
label: string;
|
||||
category: UrlCategory;
|
||||
}
|
||||
|
||||
export interface TabularExportOptions {
|
||||
format: 'csv' | 'xlsx';
|
||||
sections: TabularSection[];
|
||||
/** Keys of CrawlUrlRow to include, in order. */
|
||||
columns: string[];
|
||||
/** When provided AND non-empty, restrict every section to these row ids. */
|
||||
selectedIds?: number[];
|
||||
}
|
||||
|
||||
export interface TabularExportResult {
|
||||
/** File (xlsx / single-section csv) or folder (multi-section csv). */
|
||||
filePath: string;
|
||||
files: string[];
|
||||
rowsWritten: number;
|
||||
}
|
||||
|
||||
function escapeCsv(value: unknown): string {
|
||||
if (value === null || value === undefined) return '';
|
||||
const str = String(value);
|
||||
if (/[",\n\r]/.test(str)) {
|
||||
return `"${str.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
function rowSource(
|
||||
db: ProjectDb,
|
||||
category: UrlCategory,
|
||||
selectedIds: number[] | undefined,
|
||||
): Iterable<CrawlUrlRow> {
|
||||
if (selectedIds && selectedIds.length > 0) {
|
||||
// Apply selectedIds first, then narrow by category client-side. The
|
||||
// sidebar's category predicates can be expensive; for export the
|
||||
// selection is always the small side.
|
||||
const ids = new Set(selectedIds);
|
||||
return (function* () {
|
||||
for (const row of db.iterateUrlsByCategory(category)) {
|
||||
if (ids.has(row.id)) yield row;
|
||||
}
|
||||
})();
|
||||
}
|
||||
return db.iterateUrlsByCategory(category);
|
||||
}
|
||||
|
||||
async function writeCsvFile(
|
||||
db: ProjectDb,
|
||||
filePath: string,
|
||||
columns: string[],
|
||||
category: UrlCategory,
|
||||
selectedIds: number[] | undefined,
|
||||
): Promise<number> {
|
||||
let rowsWritten = 0;
|
||||
const header = columns.join(',') + '\n';
|
||||
const source = rowSource(db, category, selectedIds);
|
||||
const generator = async function* (): AsyncGenerator<string> {
|
||||
yield '' + header;
|
||||
for (const row of source) {
|
||||
const line =
|
||||
columns
|
||||
.map((c) => escapeCsv((row as unknown as Record<string, unknown>)[c]))
|
||||
.join(',') + '\n';
|
||||
rowsWritten++;
|
||||
yield line;
|
||||
}
|
||||
};
|
||||
await pipeline(
|
||||
Readable.from(generator()),
|
||||
createWriteStream(filePath, { encoding: 'utf8' }),
|
||||
);
|
||||
return rowsWritten;
|
||||
}
|
||||
|
||||
// ── Minimal XLSX writer ─────────────────────────────────────────────
|
||||
//
|
||||
// Office Open XML SpreadsheetML, packaged as a ZIP with DEFLATE entries.
|
||||
// Uses inline strings (no shared-string table) — slightly fatter on disk
|
||||
// but lets us stream-build sheet XML row-by-row without a second pass.
|
||||
//
|
||||
// Worksheet shape:
|
||||
// <worksheet><sheetData>
|
||||
// <row r="1"><c r="A1" t="inlineStr"><is><t>URL</t></is></c>...</row>
|
||||
// <row r="2"><c r="A2" t="inlineStr"><is><t>https://...</t></is></c>...</row>
|
||||
// </sheetData></worksheet>
|
||||
//
|
||||
// Numbers go in as <c><v>123</v></c> (no `t=` attribute, default = number).
|
||||
|
||||
function xmlEscape(str: string): string {
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
// strip control chars XML 1.0 forbids
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '');
|
||||
}
|
||||
|
||||
function colLetter(idx: number): string {
|
||||
// 0-based → A, B, ..., Z, AA, AB, ...
|
||||
let n = idx;
|
||||
let s = '';
|
||||
while (true) {
|
||||
const r = n % 26;
|
||||
s = String.fromCharCode(65 + r) + s;
|
||||
n = Math.floor(n / 26) - 1;
|
||||
if (n < 0) break;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
function sanitizeSheetName(label: string, used: Set<string>): string {
|
||||
let base = label.replace(/[\\/?*\[\]:]/g, ' ').trim().slice(0, 31);
|
||||
if (!base) base = 'Sheet';
|
||||
let candidate = base;
|
||||
let i = 2;
|
||||
while (used.has(candidate.toLowerCase())) {
|
||||
const suffix = ` (${i++})`;
|
||||
candidate = base.slice(0, 31 - suffix.length) + suffix;
|
||||
}
|
||||
used.add(candidate.toLowerCase());
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function buildSheetXml(
|
||||
rows: CrawlUrlRow[],
|
||||
columns: string[],
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
|
||||
'<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">' +
|
||||
'<sheetData>',
|
||||
);
|
||||
// Header row
|
||||
parts.push('<row r="1">');
|
||||
columns.forEach((c, i) => {
|
||||
parts.push(
|
||||
`<c r="${colLetter(i)}1" t="inlineStr"><is><t>${xmlEscape(c)}</t></is></c>`,
|
||||
);
|
||||
});
|
||||
parts.push('</row>');
|
||||
// Data rows
|
||||
rows.forEach((row, rIdx) => {
|
||||
const r = rIdx + 2;
|
||||
parts.push(`<row r="${r}">`);
|
||||
columns.forEach((col, i) => {
|
||||
const v = (row as unknown as Record<string, unknown>)[col];
|
||||
if (v === null || v === undefined || v === '') return;
|
||||
const cellRef = `${colLetter(i)}${r}`;
|
||||
if (typeof v === 'number' && Number.isFinite(v)) {
|
||||
parts.push(`<c r="${cellRef}"><v>${v}</v></c>`);
|
||||
} else if (typeof v === 'boolean') {
|
||||
parts.push(`<c r="${cellRef}" t="b"><v>${v ? 1 : 0}</v></c>`);
|
||||
} else {
|
||||
parts.push(
|
||||
`<c r="${cellRef}" t="inlineStr"><is><t>${xmlEscape(String(v))}</t></is></c>`,
|
||||
);
|
||||
}
|
||||
});
|
||||
parts.push('</row>');
|
||||
});
|
||||
parts.push('</sheetData></worksheet>');
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
// ── ZIP writer (DEFLATE, no central directory streaming — small enough
|
||||
// to buffer per-entry, write central dir at the end). ────────────────
|
||||
|
||||
interface ZipEntry {
|
||||
name: string;
|
||||
raw: Buffer;
|
||||
compressed: Buffer;
|
||||
crc32: number;
|
||||
offset: number;
|
||||
}
|
||||
|
||||
const CRC_TABLE = (() => {
|
||||
const table = new Uint32Array(256);
|
||||
for (let n = 0; n < 256; n++) {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k++) {
|
||||
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
}
|
||||
table[n] = c >>> 0;
|
||||
}
|
||||
return table;
|
||||
})();
|
||||
|
||||
function crc32(buf: Buffer): number {
|
||||
let c = 0xffffffff;
|
||||
for (let i = 0; i < buf.length; i++) {
|
||||
c = CRC_TABLE[(c ^ buf[i]!) & 0xff]! ^ (c >>> 8);
|
||||
}
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function buildZip(files: { name: string; data: Buffer }[]): Buffer {
|
||||
const entries: ZipEntry[] = [];
|
||||
const localChunks: Buffer[] = [];
|
||||
let offset = 0;
|
||||
|
||||
for (const f of files) {
|
||||
const raw = f.data;
|
||||
const compressed = zlib.deflateRawSync(raw);
|
||||
const crc = crc32(raw);
|
||||
const nameBuf = Buffer.from(f.name, 'utf8');
|
||||
|
||||
const localHeader = Buffer.alloc(30);
|
||||
localHeader.writeUInt32LE(0x04034b50, 0); // local file header sig
|
||||
localHeader.writeUInt16LE(20, 4); // version needed
|
||||
localHeader.writeUInt16LE(0x0800, 6); // gp flag (UTF-8 name)
|
||||
localHeader.writeUInt16LE(8, 8); // method = deflate
|
||||
localHeader.writeUInt16LE(0, 10); // mod time
|
||||
localHeader.writeUInt16LE(0, 12); // mod date
|
||||
localHeader.writeUInt32LE(crc, 14);
|
||||
localHeader.writeUInt32LE(compressed.length, 18);
|
||||
localHeader.writeUInt32LE(raw.length, 22);
|
||||
localHeader.writeUInt16LE(nameBuf.length, 26);
|
||||
localHeader.writeUInt16LE(0, 28); // extra field len
|
||||
|
||||
entries.push({
|
||||
name: f.name,
|
||||
raw,
|
||||
compressed,
|
||||
crc32: crc,
|
||||
offset,
|
||||
});
|
||||
localChunks.push(localHeader, nameBuf, compressed);
|
||||
offset += localHeader.length + nameBuf.length + compressed.length;
|
||||
}
|
||||
|
||||
const centralChunks: Buffer[] = [];
|
||||
let cdSize = 0;
|
||||
for (const e of entries) {
|
||||
const nameBuf = Buffer.from(e.name, 'utf8');
|
||||
const cd = Buffer.alloc(46);
|
||||
cd.writeUInt32LE(0x02014b50, 0); // central dir sig
|
||||
cd.writeUInt16LE(20, 4); // version made by
|
||||
cd.writeUInt16LE(20, 6); // version needed
|
||||
cd.writeUInt16LE(0x0800, 8); // gp flag
|
||||
cd.writeUInt16LE(8, 10); // method
|
||||
cd.writeUInt16LE(0, 12);
|
||||
cd.writeUInt16LE(0, 14);
|
||||
cd.writeUInt32LE(e.crc32, 16);
|
||||
cd.writeUInt32LE(e.compressed.length, 20);
|
||||
cd.writeUInt32LE(e.raw.length, 24);
|
||||
cd.writeUInt16LE(nameBuf.length, 28);
|
||||
cd.writeUInt16LE(0, 30);
|
||||
cd.writeUInt16LE(0, 32);
|
||||
cd.writeUInt16LE(0, 34);
|
||||
cd.writeUInt16LE(0, 36);
|
||||
cd.writeUInt32LE(0, 38);
|
||||
cd.writeUInt32LE(e.offset, 42);
|
||||
centralChunks.push(cd, nameBuf);
|
||||
cdSize += cd.length + nameBuf.length;
|
||||
}
|
||||
|
||||
const eocd = Buffer.alloc(22);
|
||||
eocd.writeUInt32LE(0x06054b50, 0);
|
||||
eocd.writeUInt16LE(0, 4);
|
||||
eocd.writeUInt16LE(0, 6);
|
||||
eocd.writeUInt16LE(entries.length, 8);
|
||||
eocd.writeUInt16LE(entries.length, 10);
|
||||
eocd.writeUInt32LE(cdSize, 12);
|
||||
eocd.writeUInt32LE(offset, 16);
|
||||
eocd.writeUInt16LE(0, 20);
|
||||
|
||||
return Buffer.concat([...localChunks, ...centralChunks, eocd]);
|
||||
}
|
||||
|
||||
function buildXlsxBuffer(
|
||||
sheets: { name: string; rows: CrawlUrlRow[] }[],
|
||||
columns: string[],
|
||||
): Buffer {
|
||||
const contentTypes =
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
|
||||
'<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">' +
|
||||
'<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>' +
|
||||
'<Default Extension="xml" ContentType="application/xml"/>' +
|
||||
'<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>' +
|
||||
sheets
|
||||
.map(
|
||||
(_, i) =>
|
||||
`<Override PartName="/xl/worksheets/sheet${i + 1}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>`,
|
||||
)
|
||||
.join('') +
|
||||
'</Types>';
|
||||
|
||||
const rels =
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
|
||||
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
|
||||
'<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>' +
|
||||
'</Relationships>';
|
||||
|
||||
const workbook =
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
|
||||
'<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">' +
|
||||
'<sheets>' +
|
||||
sheets
|
||||
.map(
|
||||
(s, i) =>
|
||||
`<sheet name="${xmlEscape(s.name)}" sheetId="${i + 1}" r:id="rId${i + 1}"/>`,
|
||||
)
|
||||
.join('') +
|
||||
'</sheets></workbook>';
|
||||
|
||||
const workbookRels =
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' +
|
||||
'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' +
|
||||
sheets
|
||||
.map(
|
||||
(_, i) =>
|
||||
`<Relationship Id="rId${i + 1}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet${i + 1}.xml"/>`,
|
||||
)
|
||||
.join('') +
|
||||
'</Relationships>';
|
||||
|
||||
const files: { name: string; data: Buffer }[] = [
|
||||
{ name: '[Content_Types].xml', data: Buffer.from(contentTypes, 'utf8') },
|
||||
{ name: '_rels/.rels', data: Buffer.from(rels, 'utf8') },
|
||||
{ name: 'xl/workbook.xml', data: Buffer.from(workbook, 'utf8') },
|
||||
{ name: 'xl/_rels/workbook.xml.rels', data: Buffer.from(workbookRels, 'utf8') },
|
||||
];
|
||||
sheets.forEach((s, i) => {
|
||||
files.push({
|
||||
name: `xl/worksheets/sheet${i + 1}.xml`,
|
||||
data: Buffer.from(buildSheetXml(s.rows, columns), 'utf8'),
|
||||
});
|
||||
});
|
||||
|
||||
return buildZip(files);
|
||||
}
|
||||
|
||||
export async function exportTabular(
|
||||
db: ProjectDb,
|
||||
outputPath: string,
|
||||
options: TabularExportOptions,
|
||||
): Promise<TabularExportResult> {
|
||||
const { format, sections, columns, selectedIds } = options;
|
||||
if (sections.length === 0) {
|
||||
throw new Error('exportTabular: at least one section is required');
|
||||
}
|
||||
if (columns.length === 0) {
|
||||
throw new Error('exportTabular: at least one column is required');
|
||||
}
|
||||
|
||||
if (format === 'csv') {
|
||||
if (sections.length === 1) {
|
||||
const rows = await writeCsvFile(
|
||||
db,
|
||||
outputPath,
|
||||
columns,
|
||||
sections[0]!.category,
|
||||
selectedIds,
|
||||
);
|
||||
return { filePath: outputPath, files: [outputPath], rowsWritten: rows };
|
||||
}
|
||||
// Multi-section CSV → write one file per section into the chosen
|
||||
// folder. `outputPath` is treated as a folder.
|
||||
let total = 0;
|
||||
const written: string[] = [];
|
||||
for (const section of sections) {
|
||||
const safe = section.label.replace(/[\\/?*\[\]:<>|"]/g, '_');
|
||||
const file = path.join(outputPath, `${safe}.csv`);
|
||||
total += await writeCsvFile(db, file, columns, section.category, selectedIds);
|
||||
written.push(file);
|
||||
}
|
||||
return { filePath: outputPath, files: written, rowsWritten: total };
|
||||
}
|
||||
|
||||
// xlsx — collect rows per section into memory (bounded by typical
|
||||
// section size), build workbook, write once.
|
||||
const usedNames = new Set<string>();
|
||||
let total = 0;
|
||||
const sheets = sections.map((section) => {
|
||||
const rows: CrawlUrlRow[] = [];
|
||||
for (const row of rowSource(db, section.category, selectedIds)) {
|
||||
rows.push(row);
|
||||
}
|
||||
total += rows.length;
|
||||
return { name: sanitizeSheetName(section.label, usedNames), rows };
|
||||
});
|
||||
const buf = buildXlsxBuffer(sheets, columns);
|
||||
writeFileSync(outputPath, buf);
|
||||
return { filePath: outputPath, files: [outputPath], rowsWritten: total };
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/db",
|
||||
"version": "0.2.7",
|
||||
"version": "0.2.8",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -3036,6 +3036,13 @@ export class ProjectDb {
|
||||
return row ? this.rowFromDb(row) : null;
|
||||
}
|
||||
|
||||
/** Resolve a URL string to its numeric id, or null if absent. Used by
|
||||
* the MCP server when a caller supplies a URL instead of an id. */
|
||||
getUrlIdByUrl(url: string): number | null {
|
||||
const row = this.stmtGetUrlId.get(url) as { id: number } | undefined;
|
||||
return row ? row.id : null;
|
||||
}
|
||||
|
||||
getInlinks(url: string, limit: number): { rows: InlinkRow[]; total: number } {
|
||||
const total = (
|
||||
this.db.prepare('SELECT COUNT(*) AS c FROM links WHERE to_url = ?').get(url) as { c: number }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@freecrawl/shared-types",
|
||||
"version": "0.2.7",
|
||||
"version": "0.2.8",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
|
||||
@@ -39,6 +39,9 @@ export const IPC = {
|
||||
exportCsv: 'export:csv',
|
||||
exportJson: 'export:json',
|
||||
exportXml: 'export:xml',
|
||||
/** New unified tabular export with column selection + multi-tab support
|
||||
* (CSV / XLSX). Backs the in-table "Export" button. */
|
||||
exportTabular: 'export:tabular',
|
||||
/** GDPR-aligned per-domain delete. Wipes every row whose URL host
|
||||
* matches the given domain (and that domain's links/images/headers/
|
||||
* url_sources). Used by Settings → "Delete Domain Data". */
|
||||
@@ -213,6 +216,38 @@ export interface ExportXmlResult {
|
||||
rowsWritten: number;
|
||||
}
|
||||
|
||||
/** One section / sheet of a tabular export — a single category fed into
|
||||
* the workbook (xlsx) or split into its own file (csv). */
|
||||
export interface ExportTabularSection {
|
||||
/** Display label — becomes the xlsx sheet name and the csv file suffix. */
|
||||
label: string;
|
||||
category: UrlCategory;
|
||||
}
|
||||
|
||||
export interface ExportTabularInput {
|
||||
format: 'csv' | 'xlsx';
|
||||
/**
|
||||
* One or more sections. With CSV + multiple sections, files are written
|
||||
* into a folder the user picks; xlsx always produces a single workbook
|
||||
* with one sheet per section.
|
||||
*/
|
||||
sections: ExportTabularSection[];
|
||||
/** Keys of `CrawlUrlRow` to emit, in order. */
|
||||
columns: string[];
|
||||
/** Optional pre-resolved output path (skip the dialog). */
|
||||
filePath?: string;
|
||||
/** When set, restrict every section to these row ids. */
|
||||
selectedIds?: number[];
|
||||
}
|
||||
|
||||
export interface ExportTabularResult {
|
||||
/** Output file (xlsx + single-section csv) or folder (multi-section csv). */
|
||||
filePath: string;
|
||||
/** Files actually written. Multi-section CSV writes one file per section. */
|
||||
files: string[];
|
||||
rowsWritten: number;
|
||||
}
|
||||
|
||||
export interface DataDeleteByDomainInput {
|
||||
/** Hostname to wipe — case-insensitive, no scheme/port. */
|
||||
domain: string;
|
||||
@@ -676,6 +711,7 @@ export interface FreeCrawlApi {
|
||||
exportCsv(input: ExportCsvInput): Promise<ExportCsvResult>;
|
||||
exportJson(input: ExportJsonInput): Promise<ExportJsonResult>;
|
||||
exportXml(input: ExportXmlInput): Promise<ExportXmlResult>;
|
||||
exportTabular(input: ExportTabularInput): Promise<ExportTabularResult>;
|
||||
dataDeleteByDomain(
|
||||
input: DataDeleteByDomainInput,
|
||||
): Promise<DataDeleteByDomainResult>;
|
||||
|
||||
+2
-1
@@ -5,6 +5,7 @@
|
||||
{ "path": "./packages/db" },
|
||||
{ "path": "./packages/core" },
|
||||
{ "path": "./apps/desktop" },
|
||||
{ "path": "./apps/cli" }
|
||||
{ "path": "./apps/cli" },
|
||||
{ "path": "./apps/mcp-server" }
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user