feat: 3D link visualiser + macOS performance and shortcuts

README.md

@@ -400,12 +400,14 @@ graph LR
 ## 📈 Status
 
 > [!NOTE]
-> **Active development.** All 31 analysis tabs are working — the core set (Internal, External, Response Codes, URL, Page Titles, Meta Description, H1, H2, Content, Images, Canonicals, Directives, Redirects, Pagination, Hreflang, AMP, Structured Data, Meta Refresh, Custom Extraction, Custom Search, Security, Duplicates, Links, Broken Links, SERP) plus the integration tabs (**PageSpeed, CrUX, Spelling, Search Console, GA4, AI, SEO Authority**).
+> **Active development.** All 32 analysis tabs are working — the core set (Internal, External, Response Codes, URL, Page Titles, Meta Description, H1, H2, Content, Images, Canonicals, Directives, Redirects, Pagination, Hreflang, AMP, Structured Data, Meta Refresh, Custom Extraction, Custom Search, Security, Duplicates, Links, Broken Links, SERP) plus the integration tabs (**PageSpeed, CrUX, Spelling, Search Console, GA4, AI, SEO Authority**).
 >
-> Also shipping: 150+ issue checks, **JavaScript rendering** (Playwright — post-JS DOM, screenshots, LCP candidate, Mobile Usability audit), **multi-window projects** (independent per-window crawls), near/exact-duplicate detection, hreflang validation, **internal PageRank link scoring**, project compare, the **Cytoscape link graph** (force-directed / BFS / radial / Sugiyama / directory-tree layouts + LCP and link-score overlays + click-to-trace Crawl Path Report), **Basic / Digest / Bearer + form-based & browser-driven SPA login**, **HTTP / SOCKS5 proxy**, the **MCP server with live crawl control**, all **Google integrations** (Search Console / URL Inspection / GA4 / PageSpeed / CrUX), **LanguageTool spelling & grammar**, **AI per-URL prompts** (OpenAI / Anthropic / Ollama), **SEO Authority providers** (Ahrefs / Majestic / Moz / Semrush), **Google Sheets + BigQuery export**, **encrypted project snapshots** (AES-256-GCM), **RAM-only storage mode**, scheduled crawls, robots.txt validator, URL rewriting, EN + TR UI, and the **standalone Log File Analyzer** (Apache / Nginx / IIS access logs → bot hits, crawl budget, crawl × log orphans, 40+ bot detection, CSV / Excel export, and a headless `analyze-logs` CLI).
+> Also shipping: 150+ issue checks, **JavaScript rendering** (Playwright — post-JS DOM, screenshots, LCP candidate, Mobile Usability audit), **multi-window projects** (independent per-window crawls), near/exact-duplicate detection, hreflang validation, **internal PageRank link scoring**, project compare, the **link visualiser** (WebGL 3D force-directed crawl tree and link graph, plus 2D Crawl Tree and Directory Tree with click-to-collapse branches, hover summary cards, LCP and link-score overlays, and a click-to-trace Crawl Path Report), **Basic / Digest / Bearer + form-based & browser-driven SPA login**, **HTTP / SOCKS5 proxy**, the **MCP server with live crawl control**, all **Google integrations** (Search Console / URL Inspection / GA4 / PageSpeed / CrUX), **LanguageTool spelling & grammar**, **AI per-URL prompts** (OpenAI / Anthropic / Ollama), **SEO Authority providers** (Ahrefs / Majestic / Moz / Semrush), **Google Sheets + BigQuery export**, **encrypted project snapshots** (AES-256-GCM), **RAM-only storage mode**, scheduled crawls, robots.txt validator, URL rewriting, EN + TR UI, and the **standalone Log File Analyzer** (Apache / Nginx / IIS access logs → bot hits, crawl budget, crawl × log orphans, 40+ bot detection, CSV / Excel export, and a headless `analyze-logs` CLI).
 >
 > Cross-platform installers ship for Windows (`.exe` + portable), macOS (`.dmg`), and Linux (`.AppImage` / `.deb` / `.rpm`), with Playwright Chromium bundled so JS rendering works on first launch. The virtualized 1M-row table streams the **first row in ~1 s**.
 >
+> **Apple Silicon tuning:** long crawls hold a power assertion so macOS App Nap can't throttle them in the background, concurrency scales down automatically as the thermal state rises (so a fanless MacBook Air degrades gracefully instead of stalling), heavy aggregates never run on the main thread, and every large read is batch-streamed to stay clear of Electron's hard per-process heap ceiling. macOS also gets the full Windows-style `Cmd`-key shortcut set.
+>
 > **Upcoming:** Plugin system, Light theme, Code-signing + auto-update.
 
 <br />

SECURITY.md

@@ -13,7 +13,7 @@ upgrading is the recommended remediation path.
 
 | Version | Supported          |
 | ------- | ------------------ |
-| Latest stable (currently `0.3.x`) | Yes |
+| Latest stable (currently `0.9.x`) | Yes |
 | Older `0.x` releases              | No  |
 
 Release artifacts are published at:
@@ -60,8 +60,8 @@ possible:
   layer (`@freecrawl/db`), desktop shell, MCP server, or CLI
 - **Affected version(s)** — output of `Help → About`, or the `version`
   field in `package.json`
-- **Operating system + architecture** — Windows 11 / macOS 14 (Intel or
-  ARM64) / Linux distro
+- **Operating system + architecture** — Windows 10/11, macOS 13 or newer
+  (Intel or Apple Silicon), or your Linux distribution
 - **Reproduction steps** — minimal sequence that triggers the issue
 - **Proof-of-concept** — screenshot, video, or minimal code
 - **Impact assessment** — confidentiality / integrity / availability
@@ -143,12 +143,16 @@ Releases. The artifacts are produced by an automated GitHub Actions
 workflow defined in `.github/workflows/release.yml`. SHA512 digests are
 published alongside each release in `latest.yml` / `latest-mac.yml`.
 
-Code-signing for Windows installers is being established through
-[SignPath.io](https://signpath.io)'s open-source signing program. Once
-active, all `.exe` artifacts will be signed and the application will
-verify the publisher signature before applying any in-app update. macOS
-artifacts are currently unsigned; users should verify the SHA512 digest
-manually until Apple Developer ID notarization is in place.
+**Release artifacts are currently unsigned on every platform.** Verify
+the SHA512 digest of anything you download against the value published
+with the release before running it. Expect the usual first-launch
+warnings: SmartScreen on Windows, and Gatekeeper on macOS (documented
+in the README, along with the `xattr -cr` workaround).
+
+Code-signing for Windows installers is being pursued through
+[SignPath.io](https://signpath.io)'s open-source signing program, and
+macOS notarization requires an Apple Developer ID. Neither is active
+yet; this section will be updated when that changes.
 
 If you encounter an installer that fails signature verification or
 appears to come from any source other than the official GitHub Releases
@@ -167,9 +171,19 @@ page above, **do not run it** — please report it via the channels above.
   the crawls the user explicitly initiates and (when enabled) the
   on-demand "Check for Updates" call to the GitHub Releases API.
 - **Project files (`.seoproject`)** are SQLite databases containing
-  crawled content. They are not encrypted at rest in the current
-  release — users handling sensitive data should rely on full-disk
-  encryption at the OS level.
+  crawled content, and are **not** encrypted by default. Two options
+  exist for sensitive crawls:
+  - **Save As Encrypted** writes a password-protected snapshot
+    (AES-256-GCM, per-file random salt and IV, PBKDF2-SHA256 key
+    derivation at 200 000 iterations, authenticated header). Opening it
+    decrypts to a working copy, so the *open* project is still plaintext
+    on disk for the duration of the session.
+  - **RAM-only storage mode** keeps the crawl in memory and never writes
+    a project file at all.
+
+  Users handling sensitive data should still enable full-disk encryption
+  at the OS level — it is the only thing that protects the working copy
+  and SQLite's temporary files.
 
 ---
 

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.8.9",
+  "version": "0.9.0",
   "private": true,
   "type": "module",
   "bin": {

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.8.9",
+  "version": "0.9.0",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
@@ -35,6 +35,7 @@
     "@tanstack/react-table": "^8.20.5",
     "@tanstack/react-virtual": "^3.10.0",
     "@types/cytoscape": "^3.21.9",
+    "3d-force-graph": "^1.80.0",
     "clsx": "^2.1.1",
     "cytoscape": "^3.33.2",
     "cytoscape-dagre": "^2.5.0",

apps/desktop/src/main/db-reader-pool.ts

@@ -323,6 +323,15 @@ export function setReaderPoolResolver(fn: () => DbReaderPool): void {
  * to the synchronous main-process DB on any worker error. The fallback
  * path keeps the UI working even if every worker is crashed/restarting,
  * at the cost of running on the main thread for that one query.
+ *
+ * HEAVY methods never fall back. If a heavy aggregate crashed or timed
+ * out its reader worker, re-running the exact same query on the main
+ * thread escalates a contained worker failure into a main-process
+ * stall (multi-second aggregate blocking every IPC) or — for
+ * OOM-class failures — a process crash, since Electron's V8 heap is
+ * hard-capped at 4 GB per process. Callers get a rejection instead;
+ * the sidebar keeps its previous counts and the next poll tick retries
+ * against the (by then respawned) worker.
  */
 export async function callReaderOrFallback<T>(
   method: string,
@@ -330,13 +339,25 @@ export async function callReaderOrFallback<T>(
   fallback: () => T | Promise<T>,
 ): Promise<T> {
   const pool = resolveReaderPool();
+  const heavy = HEAVY_METHODS.has(method);
   if (!pool.isReady()) {
+    if (heavy) {
+      throw new Error(`reader-unavailable: '${method}' skips main-thread fallback`);
+    }
     return fallback();
   }
   try {
     return await pool.call<T>(method, args);
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
+    if (heavy) {
+      logger.log(
+        'warn',
+        'db-reader',
+        `'${method}' failed on reader pool (${msg}) — heavy query, NOT retrying on main thread.`,
+      );
+      throw err;
+    }
     logger.log(
       'warn',
       'db-reader',

apps/desktop/src/main/db-writer-worker.ts

@@ -86,11 +86,13 @@ const ALLOWED_METHODS = new Set<string>([
   'recomputeHreflangAnalysis',
   'recomputeHreflangInconsistent',
   'recomputeDuplicateClusters',
+  'recomputeBoilerplateCoverage',
   'recomputePaginationSequence',
   'recomputeUrlsIssues',
   'recomputeUrlsIssuesYielding',
   'checkpointQueue',
   'clearQueueCheckpoint',
+  'walCheckpoint',
   'markUrlForRecrawl',
   'markUrlsForRecrawl',
   'deleteUrl',

apps/desktop/src/main/index.ts

@@ -6,6 +6,8 @@ import {
   dialog,
   Menu,
   Notification,
+  powerMonitor,
+  powerSaveBlocker,
   session,
   shell,
   Tray,
@@ -24,6 +26,7 @@ import { totalmem, freemem } from 'node:os';
 import {
   DEFAULT_CRAWL_CONFIG,
   IPC,
+  type MenuEvent,
   type ConfirmClearResult,
   type CrawlConfig,
   type CrawlProgress,
@@ -184,6 +187,7 @@ import {
   exportTabular,
   exportSitemap,
   exportHtmlReport,
+  ensureHeapHeadroom,
   compareCrawls,
   testUrlAgainstRobots,
   validateRobotsTxt,
@@ -1035,6 +1039,92 @@ async function recomputeIssuesViaPool(
  * 5–15 s on a 5K-URL crawl — exactly the window where users hit
  * Stop/Pause and "nothing happens".
  */
+/**
+ * Ref-counted power-save blocker held while any window's crawl runs.
+ * First active crawl starts the blocker, last one to end releases it —
+ * keyed by Crawler instance so double-release (e.g. stop() during
+ * clearCrawlDb after the Stop IPC already fired) is a no-op. Without
+ * this, macOS App Nap throttles a backgrounded crawl's timers, CPU
+ * share and disk I/O, and laptop sleep kills it entirely.
+ */
+const crawlsHoldingPowerBlocker = new Set<Crawler>();
+let crawlPowerBlockerId: number | null = null;
+
+function acquireCrawlPowerBlocker(crawler: Crawler): void {
+  if (crawlsHoldingPowerBlocker.has(crawler)) return;
+  crawlsHoldingPowerBlocker.add(crawler);
+  if (crawlPowerBlockerId !== null) return;
+  try {
+    crawlPowerBlockerId = powerSaveBlocker.start('prevent-app-suspension');
+    logger.log('info', 'main', 'Power-save blocker acquired (crawl running)');
+  } catch (err) {
+    // Non-fatal — the crawl still runs, just without suspension immunity.
+    crawlPowerBlockerId = null;
+    logger.log(
+      'warn',
+      'main',
+      `powerSaveBlocker.start failed: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+}
+
+function releaseCrawlPowerBlocker(crawler: Crawler): void {
+  if (!crawlsHoldingPowerBlocker.delete(crawler)) return;
+  if (crawlsHoldingPowerBlocker.size > 0 || crawlPowerBlockerId === null) return;
+  try {
+    powerSaveBlocker.stop(crawlPowerBlockerId);
+  } catch {
+    /* already stopped — ignore */
+  }
+  crawlPowerBlockerId = null;
+  logger.log('info', 'main', 'Power-save blocker released (no crawls running)');
+}
+
+/**
+ * Thermal-adaptive crawl throttling (macOS-only — Electron emits
+ * `thermal-state-change` only there). On fanless Apple Silicon
+ * machines (MacBook Air) a 40-minute crawl at full concurrency drives
+ * the SoC into sustained throttling; Apple's guidance is to shed CPU
+ * and I/O proactively from the `fair` state onward, before the OS
+ * starts enacting its own countermeasures at `serious`. The scale is
+ * applied as a concurrency *ceiling* on every live crawler — the
+ * crawler's lag-adaptive loop keeps operating underneath it.
+ * `nominal`/`unknown` map to 1 (unsupported systems always report
+ * `nominal`, so this path is a safe no-op for them).
+ */
+function thermalScaleFor(state: string): number {
+  switch (state) {
+    case 'critical':
+      return 0.25;
+    case 'serious':
+      return 0.5;
+    case 'fair':
+      return 0.75;
+    default:
+      return 1;
+  }
+}
+
+function applyThermalStateToCrawlers(state: string): void {
+  const scale = thermalScaleFor(state);
+  for (const s of [defaultSession, ...sessionsByWc.values()]) {
+    s.activeCrawler?.setThrottleScale(scale, `thermal:${state}`);
+  }
+}
+
+/** Seed a freshly launched crawler with the current thermal state so a
+ *  crawl started on an already-hot machine begins throttled instead of
+ *  waiting for the next state *change*. */
+function applyCurrentThermalState(crawler: Crawler): void {
+  if (process.platform !== 'darwin') return;
+  try {
+    const state = powerMonitor.getCurrentThermalState();
+    crawler.setThrottleScale(thermalScaleFor(state), `thermal:${state}`);
+  } catch {
+    /* powerMonitor not ready pre-app-ready — crawls can't start then */
+  }
+}
+
 async function runDbPassViaPool(
   session: ProjectSession,
   method: string,
@@ -1352,6 +1442,22 @@ function rebuildMenu(): void {
       onOpenLogsFolder: () => openLogsFolder(),
       onCheckForUpdates: () => void checkForUpdates(),
       onOpenLogAnalyzer: () => openLogAnalyzerWindow(),
+      onEditCopy: () => {
+        const win = BrowserWindow.getFocusedWindow();
+        if (!win || win.isDestroyed()) return;
+        // Project windows understand the 'edit-copy' menu event and run
+        // the custom-table-copy → native-copy round-trip. Auxiliary
+        // windows (Logs, Log Analyzer, viz popouts) don't subscribe to
+        // menu events — give them the stock native copy directly so
+        // Cmd+C on selected text keeps working there.
+        const isProjectWindow =
+          win === mainWindow || sessionsByWc.has(win.webContents.id);
+        if (isProjectWindow) {
+          win.webContents.send(IPC.menuEvent, 'edit-copy' satisfies MenuEvent);
+        } else {
+          win.webContents.copy();
+        }
+      },
     }),
   );
 }
@@ -2466,6 +2572,13 @@ function seedDiscoveryUrls(limit: number): LogSeedDiscoveryResult {
 
 function registerIpc(): void {
   registerHandle(IPC.appVersion, () => app.getVersion());
+  // Native-copy fallback leg of the macOS Edit▸Copy round-trip: the
+  // renderer calls this when no custom table selection consumed the
+  // copy, so the stock behaviour (copy the DOM text selection / the
+  // focused input's selection) still happens.
+  registerHandle(IPC.editCopyNative, (e): void => {
+    e.sender.copy();
+  });
 
   // Live memory snapshot for the in-app monitor (status bar). Returns
   // process RSS / heap + system total/free + the active project's URL
@@ -2622,6 +2735,10 @@ function registerIpc(): void {
         filePath = res.filePath;
       }
       const db = getDb();
+      // Bounded but chunky: up to 250K+ small rows plus the workbook
+      // buffer, all materialised at once. Verify the heap can absorb
+      // it before starting — a clear error beats a V8 OOM abort.
+      ensureHeapHeadroom('Log analysis export', 256 * 1024 * 1024);
       const tables: LogExportTables = {
         overview: db.getLogOverview(),
         urlStats: db.queryLogUrlStats({ limit: 100_000, offset: 0, sortBy: 'totalHits', filter: 'all' }).rows,
@@ -4593,6 +4710,23 @@ function registerIpc(): void {
    * in the Start dialog again. Persisted across restarts via DB
    * project_meta so it survives the very crash we're recovering from. */
   function attachCrawlerListeners(crawler: Crawler, sess: ProjectSession): void {
+    // Keep the OS from suspending/App-Napping us while this crawl runs.
+    // On macOS a backgrounded crawl otherwise gets App Nap's priority
+    // reduction + timer throttling + disk-I/O throttling (and on Apple
+    // Silicon, background-QoS work is confined to E-cores); on laptops
+    // generally, system sleep would kill the crawl outright.
+    // `prevent-app-suspension` is the cheapest blocker level — the
+    // display may still turn off. Released on 'done' AND 'stopped'
+    // (done is suppressed after stop(), so both hooks are required),
+    // deliberately BEFORE the zombie-crawler guard in the handlers
+    // below: a superseded crawler must still drop its blocker hold.
+    acquireCrawlPowerBlocker(crawler);
+    crawler.on('done', () => releaseCrawlPowerBlocker(crawler));
+    crawler.on('stopped', () => releaseCrawlPowerBlocker(crawler));
+    // Start throttled if the machine is already hot (macOS-only no-op
+    // elsewhere) — the thermal-state-change listener only covers
+    // *changes* after launch.
+    applyCurrentThermalState(crawler);
     // Bound to the launching session so progress/done/error reach THAT
     // window (multi-window) — these callbacks run outside any IPC handler's
     // AsyncLocalStorage context, so they can't use `currentSession()`.
@@ -6002,7 +6136,7 @@ function registerIpc(): void {
         const rawDb = (database as any).db as { exec: (sql: string) => void };
         rawDb.exec(`VACUUM INTO '${tmpPath.replace(/'/g, "''")}'`);
         const { encryptFile } = await import('./project-encryption.js');
-        const result = encryptFile(tmpPath, i.filePath, i.password);
+        const result = await encryptFile(tmpPath, i.filePath, i.password);
         return { filePath: i.filePath, bytesWritten: result.bytesWritten };
       } finally {
         try {
@@ -6022,7 +6156,7 @@ function registerIpc(): void {
       if (!i.password) throw new Error('password is required.');
       if (!i.destPath) throw new Error('destPath is required (output .seoproject path).');
       const { decryptFile } = await import('./project-encryption.js');
-      decryptFile(i.filePath, i.destPath, i.password);
+      await decryptFile(i.filePath, i.destPath, i.password);
       openProjectAtPath(i.destPath);
       return { filePath: i.destPath };
     },
@@ -6310,7 +6444,7 @@ function registerIpc(): void {
         rawDb.exec(`VACUUM INTO '${escaped}'`);
 
         const { encryptFile } = await import('./project-encryption.js');
-        const result = encryptFile(tmpPath, target, password);
+        const result = await encryptFile(tmpPath, target, password);
 
         await dialog.showMessageBox(win, {
           type: 'info',
@@ -6379,7 +6513,7 @@ function registerIpc(): void {
 
       try {
         const { decryptFile } = await import('./project-encryption.js');
-        decryptFile(srcPath, dstPath, password);
+        await decryptFile(srcPath, dstPath, password);
         openProjectAtPath(dstPath);
         return { filePath: dstPath };
       } catch (err) {
@@ -7347,6 +7481,16 @@ void app.whenReady().then(async () => {
   logger.log('info', 'main', `App ready — version ${app.getVersion()}`);
   freezeWatchdog.setMainOp('idle');
 
+  // Thermal-adaptive crawl throttling — macOS-only (Electron emits
+  // `thermal-state-change` only on macOS; other platforms never fire it).
+  // See applyThermalStateToCrawlers for the state → concurrency mapping.
+  if (process.platform === 'darwin') {
+    powerMonitor.on('thermal-state-change', (details) => {
+      logger.log('info', 'main', `Thermal state → ${details.state}`);
+      applyThermalStateToCrawlers(details.state);
+    });
+  }
+
   // Open the localhost MCP bridge so Claude Code / Claude Desktop can
   // drive crawls + poll progress against this running instance. Bind
   // failure is logged but non-fatal: the desktop app keeps working,

apps/desktop/src/main/menu-i18n.ts

@@ -45,6 +45,8 @@ export interface MenuLabels {
   openProjectEncrypted: string;
   openProjectEncryptedTooltip: string;
   settings: string;
+  edit: string;
+  copy: string;
   view: string;
   overviewSidebar: string;
   detailPanel: string;
@@ -178,6 +180,8 @@ const MENU_EN: MenuLabels = {
   openProjectEncryptedTooltip:
     'Decrypt a .seoproject.enc snapshot with its password and open the recovered project.',
   settings: 'Settings…',
+  edit: 'Edit',
+  copy: 'Copy',
   view: 'View',
   overviewSidebar: 'Overview Sidebar',
   detailPanel: 'Detail Panel',
@@ -315,6 +319,8 @@ const MENU_TR: MenuLabels = {
   openProjectEncryptedTooltip:
     'Bir .seoproject.enc snapshot\'ını parolasıyla çöz ve kurtarılan projeyi aç.',
   settings: 'Ayarlar…',
+  edit: 'Düzen',
+  copy: 'Kopyala',
   view: 'Görünüm',
   overviewSidebar: 'Genel Bakış Kenar Çubuğu',
   detailPanel: 'Detay Paneli',

apps/desktop/src/main/menu.ts

@@ -32,6 +32,17 @@ export interface AppMenuHandlers {
   onCheckForUpdates: () => void;
   /** V2 Faz 2 — open the standalone Log File Analyzer window. */
   onOpenLogAnalyzer: () => void;
+  /**
+   * macOS Edit▸Copy dispatcher. On macOS the native menu's key
+   * equivalent swallows Cmd+C before the renderer ever sees a keydown
+   * — the exact opposite of Windows/Linux, where the page gets the key
+   * first. So on macOS the Copy item is a custom click that lets the
+   * host decide: project windows get a 'edit-copy' menu event (their
+   * renderer tries the custom table-selection copy, then falls back to
+   * native), auxiliary windows (Logs, Log Analyzer) get a plain
+   * `webContents.copy()`.
+   */
+  onEditCopy: () => void;
   /** UI language for menu labels. Falls back to `en` when missing. */
   lang?: MenuLang;
 }
@@ -154,7 +165,33 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
         isMac ? { role: 'close' } : { role: 'quit' },
       ],
     },
-    { role: 'editMenu' as const },
+    // Windows/Linux keep the stock role — there the renderer receives
+    // Ctrl+C keydown BEFORE any menu accelerator, so the app's custom
+    // table-copy handlers work with zero interception. On macOS the
+    // native menu's key equivalent fires FIRST and the page never sees
+    // Cmd+C, so Copy must be a custom item that routes through
+    // `onEditCopy` (custom table copy → native copy fallback). Every
+    // other edit command keeps its native role.
+    isMac
+      ? {
+          label: L.edit,
+          submenu: [
+            { role: 'undo' as const },
+            { role: 'redo' as const },
+            { type: 'separator' as const },
+            { role: 'cut' as const },
+            {
+              label: L.copy,
+              accelerator: 'CmdOrCtrl+C',
+              click: () => handlers.onEditCopy(),
+            },
+            { role: 'paste' as const },
+            { role: 'pasteAndMatchStyle' as const },
+            { role: 'delete' as const },
+            { role: 'selectAll' as const },
+          ],
+        }
+      : { role: 'editMenu' as const },
     {
       label: L.view,
       submenu: [

apps/desktop/src/main/project-encryption.ts

@@ -31,7 +31,18 @@ import {
   pbkdf2Sync,
   randomBytes,
 } from 'node:crypto';
-import { readFileSync, writeFileSync } from 'node:fs';
+import {
+  closeSync,
+  createReadStream,
+  createWriteStream,
+  openSync,
+  readSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { finished, pipeline } from 'node:stream/promises';
 
 const MAGIC = Buffer.from('FCRYPT01', 'utf8'); // 8 bytes
 const VERSION = 0x01;
@@ -42,22 +53,29 @@ const TAG_LEN = 16;
 const PBKDF2_ITERATIONS = 200_000;
 const HEADER_LEN = MAGIC.length + 1 + SALT_LEN + IV_LEN; // magic + version + salt + iv
 
+/** Streaming chunk size — large enough that syscall overhead is noise,
+ *  small enough that peak memory stays flat for any project size. */
+const STREAM_HIGH_WATER_MARK = 8 * 1024 * 1024;
+
 /**
  * Encrypt the file at `srcPath` and write the encrypted container to
- * `dstPath`. The source file is read into memory in one shot — fine for
- * `.seoproject` snapshots which are tens of MB; for hundreds of MB we'd
- * want a streaming variant. Throws if the password is empty or the
- * source file can't be read.
+ * `dstPath`. Fully streaming: read → AES-GCM transform → write in 8 MB
+ * chunks, so peak memory is O(chunk) no matter how large the project
+ * file is. The previous whole-file-in-a-Buffer version needed
+ * plaintext + ciphertext resident at once — on a multi-GB project
+ * that alone could blow Electron's 4 GB heap cap (Buffers count
+ * against it, unlike plain Node). Throws if the password is empty or
+ * the source file can't be read.
  */
-export function encryptFile(
+export async function encryptFile(
   srcPath: string,
   dstPath: string,
   password: string,
-): { bytesWritten: number } {
+): Promise<{ bytesWritten: number }> {
   if (!password || password.length === 0) {
     throw new Error('Password is required.');
   }
-  const plaintext = readFileSync(srcPath);
+  const srcSize = statSync(srcPath).size;
   const salt = randomBytes(SALT_LEN);
   const iv = randomBytes(IV_LEN);
   const key = pbkdf2Sync(password, salt, PBKDF2_ITERATIONS, KEY_LEN, 'sha256');
@@ -65,12 +83,21 @@ export function encryptFile(
   const header = Buffer.concat([MAGIC, Buffer.from([VERSION]), salt, iv]);
   const cipher = createCipheriv('aes-256-gcm', key, iv);
   cipher.setAAD(header);
-  const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
-  const tag = cipher.getAuthTag();
 
-  const out = Buffer.concat([header, ciphertext, tag]);
-  writeFileSync(dstPath, out);
-  return { bytesWritten: out.length };
+  const out = createWriteStream(dstPath);
+  out.write(header);
+  // `end: false` keeps the sink open — the GCM auth tag only exists
+  // after the cipher flushes, and it goes at the end of the container.
+  await pipeline(
+    createReadStream(srcPath, { highWaterMark: STREAM_HIGH_WATER_MARK }),
+    cipher,
+    out,
+    { end: false },
+  );
+  out.end(cipher.getAuthTag());
+  await finished(out);
+  // GCM ciphertext length equals plaintext length.
+  return { bytesWritten: HEADER_LEN + srcSize + TAG_LEN };
 }
 
 /**
@@ -78,49 +105,91 @@ export function encryptFile(
  * `.seoproject` bytes to `dstPath`. Throws on bad magic, unsupported
  * version, or an authentication-tag mismatch (wrong password / file
  * tampered with).
+ *
+ * Streaming, with the same all-or-nothing contract as the old
+ * in-memory version: plaintext is streamed into a sibling temp file
+ * and only renamed onto `dstPath` after `decipher.final()` verifies
+ * the GCM tag — so `dstPath` never contains unauthenticated bytes.
  */
-export function decryptFile(
+export async function decryptFile(
   srcPath: string,
   dstPath: string,
   password: string,
-): { bytesWritten: number } {
+): Promise<{ bytesWritten: number }> {
   if (!password || password.length === 0) {
     throw new Error('Password is required.');
   }
-  const buf = readFileSync(srcPath);
-  if (buf.length < HEADER_LEN + TAG_LEN) {
+  const srcSize = statSync(srcPath).size;
+  if (srcSize < HEADER_LEN + TAG_LEN) {
     throw new Error('Encrypted file is truncated or not a FreeCrawl encrypted project.');
   }
-  if (!buf.subarray(0, MAGIC.length).equals(MAGIC)) {
+  // Positional reads for the fixed-size header and the trailing tag —
+  // the ciphertext between them is never held in memory.
+  const header = Buffer.alloc(HEADER_LEN);
+  const tag = Buffer.alloc(TAG_LEN);
+  const fd = openSync(srcPath, 'r');
+  try {
+    if (
+      readSync(fd, header, 0, HEADER_LEN, 0) !== HEADER_LEN ||
+      readSync(fd, tag, 0, TAG_LEN, srcSize - TAG_LEN) !== TAG_LEN
+    ) {
+      throw new Error('Encrypted file is truncated or not a FreeCrawl encrypted project.');
+    }
+  } finally {
+    closeSync(fd);
+  }
+  if (!header.subarray(0, MAGIC.length).equals(MAGIC)) {
     throw new Error('File is not a FreeCrawl encrypted project (bad magic).');
   }
-  const version = buf[MAGIC.length];
+  const version = header[MAGIC.length];
   if (version !== VERSION) {
     throw new Error(`Unsupported encrypted-project version: 0x${version?.toString(16) ?? '??'}.`);
   }
-  const salt = buf.subarray(MAGIC.length + 1, MAGIC.length + 1 + SALT_LEN);
-  const iv = buf.subarray(
+  const salt = header.subarray(MAGIC.length + 1, MAGIC.length + 1 + SALT_LEN);
+  const iv = header.subarray(
     MAGIC.length + 1 + SALT_LEN,
     MAGIC.length + 1 + SALT_LEN + IV_LEN,
   );
-  const header = buf.subarray(0, HEADER_LEN);
-  const tag = buf.subarray(buf.length - TAG_LEN);
-  const ciphertext = buf.subarray(HEADER_LEN, buf.length - TAG_LEN);
 
   const key = pbkdf2Sync(password, salt, PBKDF2_ITERATIONS, KEY_LEN, 'sha256');
   const decipher = createDecipheriv('aes-256-gcm', key, iv);
   decipher.setAAD(header);
   decipher.setAuthTag(tag);
-  let plaintext: Buffer;
+
+  const ciphertextLen = srcSize - HEADER_LEN - TAG_LEN;
+  if (ciphertextLen === 0) {
+    // Degenerate empty-payload container — still verify the tag.
     try {
-    plaintext = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+      decipher.final();
     } catch {
-    // GCM auth failure — wrong password or corrupted ciphertext. Don't
-    // leak which one; surface a single user-friendly error.
       throw new Error(
         'Could not decrypt — the password is wrong, or the file has been altered.',
       );
     }
-  writeFileSync(dstPath, plaintext);
-  return { bytesWritten: plaintext.length };
+    writeFileSync(dstPath, Buffer.alloc(0));
+    return { bytesWritten: 0 };
+  }
+
+  const tmpPath = `${dstPath}.decrypt-tmp`;
+  try {
+    await pipeline(
+      createReadStream(srcPath, {
+        start: HEADER_LEN,
+        end: srcSize - TAG_LEN - 1, // `end` is inclusive
+        highWaterMark: STREAM_HIGH_WATER_MARK,
+      }),
+      decipher,
+      createWriteStream(tmpPath),
+    );
+  } catch {
+    // GCM auth failure (decipher.final threw inside the pipeline) or
+    // an I/O error — either way, drop the partial output. Don't leak
+    // which failure it was; surface a single user-friendly error.
+    rmSync(tmpPath, { force: true });
+    throw new Error(
+      'Could not decrypt — the password is wrong, or the file has been altered.',
+    );
+  }
+  renameSync(tmpPath, dstPath);
+  return { bytesWritten: ciphertextLen };
 }

apps/desktop/src/preload/index.ts

@@ -471,6 +471,7 @@ const api: FreeCrawlApi = {
   onDone: (cb) => subscribe<CrawlSummary>(IPC.crawlDone, cb),
   onError: (cb) => subscribe<string>(IPC.crawlError, cb),
   onMenuEvent: (cb) => subscribe<MenuEvent>(IPC.menuEvent, cb),
+  editCopyNative: (): Promise<void> => ipcRenderer.invoke(IPC.editCopyNative),
   onDataChanged: (cb) => {
     const listener = (): void => cb();
     ipcRenderer.on(IPC.dataChanged, listener);

apps/desktop/src/renderer/src/App.tsx

@@ -228,6 +228,31 @@ export function App() {
     });
     const off4 = window.freecrawl.onMenuEvent((event: MenuEvent) => {
       switch (event) {
+        case 'edit-copy': {
+          // macOS-only round-trip (see menu.ts): the native menu eats
+          // Cmd+C before the page sees a keydown, so the Copy item
+          // forwards here instead. Re-dispatch a synthetic keydown so
+          // the existing document-level Ctrl/Cmd+C table-copy handlers
+          // (detail panel cells, broken-links cells) run unchanged; if
+          // none of them preventDefault'ed — no table selection — fall
+          // back to the stock native copy for text selections/inputs.
+          const ae = document.activeElement as HTMLElement | null;
+          const editable =
+            ae !== null &&
+            (ae.tagName === 'INPUT' || ae.tagName === 'TEXTAREA' || ae.isContentEditable);
+          if (!editable) {
+            const synthetic = new KeyboardEvent('keydown', {
+              key: 'c',
+              metaKey: true,
+              bubbles: true,
+              cancelable: true,
+            });
+            document.dispatchEvent(synthetic);
+            if (synthetic.defaultPrevented) break;
+          }
+          void window.freecrawl.editCopyNative();
+          break;
+        }
         case 'toggle-sidebar':
           toggleSidebar();
           break;

apps/desktop/src/renderer/src/components/OverviewSidebar.tsx

@@ -80,6 +80,11 @@ export function OverviewSidebar() {
           overviewAppliedRef.current = seq;
           setOverview(o);
         }
+      } catch {
+        // Heavy aggregates no longer fall back to the main thread when
+        // the reader worker is down — the call rejects instead. Keep
+        // the previous counts; the next poll tick retries against the
+        // respawned worker.
       } finally {
         inFlight = false;
       }
@@ -129,11 +134,16 @@ export function OverviewSidebar() {
     if (bucket === lastRefetchAtRef.current) return;
     lastRefetchAtRef.current = bucket;
     const seq = ++overviewSeqRef.current;
-    void window.freecrawl.overviewGet().then((o) => {
+    void window.freecrawl
+      .overviewGet()
+      .then((o) => {
         if (seq > overviewAppliedRef.current) {
           overviewAppliedRef.current = seq;
           setOverview(o);
         }
+      })
+      .catch(() => {
+        // Reader worker down — keep previous counts, retry next bucket.
       });
   }, [progressCrawled, progressRunning, setOverview]);
 

apps/desktop/src/renderer/src/i18n/labels.ts

@@ -452,20 +452,15 @@ const TR_LABELS: Record<string, string> = {
   Pages: 'Sayfalar',
 
   // VisualizationTab layout names + hints
-  'Force-Directed': 'Force-Directed',
-  'Compound spring embedder': 'Bileşik yay yerleştirici',
-  'Tree (BFS)': 'Ağaç (BFS)',
-  'Roots-to-leaves layered': 'Kök-yaprak katmanlı',
-  'Force-Directed Tree': 'Force-Directed Ağaç',
-  'Sugiyama layered DAG (dagre)': 'Sugiyama katmanlı DAG (dagre)',
-  'Radial Tree': 'Radyal Ağaç',
-  'Roots at centre, leaves outward': 'Kök merkezde, yapraklar dışa',
+  '3D Force-Directed Crawl Tree': '3D Force-Directed Tarama Ağacı',
+  'Discovery tree in 3D — drag to rotate, scroll to zoom':
+    '3B keşif ağacı — döndürmek için sürükle, yakınlaştırmak için kaydır',
+  '3D Force-Directed Link Graph': '3D Force-Directed Bağlantı Grafiği',
+  'Full internal link mesh in 3D': 'Tam iç bağlantı ağı, 3B',
+  'Crawl Tree': 'Tarama Ağacı',
+  'Left-to-right labelled discovery tree (2D)': 'Soldan sağa etiketli keşif ağacı (2D)',
   'Directory Tree': 'Dizin Ağacı',
-  'Grouped by URL path segments': 'URL yol segmentlerine göre gruplu',
-  Circle: 'Çember',
-  'Equal radial spacing': 'Eşit radyal boşluk',
-  Concentric: 'Konsantrik',
-  'By inlinks (centre = most-linked)': 'Inlinks\'e göre (merkez = en çok linklenen)',
+  'Grouped by URL path segments (2D)': 'URL yol segmentlerine göre gruplu (2D)',
 
   // BrokenLinksTab column headers
   Source: 'Kaynak',

apps/desktop/src/renderer/src/i18n/locales/en.json

@@ -944,6 +944,7 @@
     "color": "Color:",
     "nodes": "Nodes:",
     "labels": "Labels:",
+    "byCrawl": "Crawl Diagram (Depth × Indexability)",
     "byStatus": "By Status",
     "byDepth": "By Depth",
     "byIndexability": "By Indexability",
@@ -969,7 +970,8 @@
     "exportSvg": "SVG (vector — Illustrator/Figma)",
     "exportHtml": "Standalone HTML (shareable)",
     "nodesEdgesCount": "{{nodes}} nodes · {{edges}} edges",
-    "interactionHint": "Hover = neighbours · Click = select · Empty click = clear · Double-click node = open · Double-click canvas = fit",
+    "interactionHint": "Hover = summary · Click branch = collapse/expand · Double-click node = open · Double-click canvas = fit",
+    "interactionHint3d": "Drag = rotate · Scroll = zoom · Right-drag = pan · Click = fly to node · Right-click = open",
     "loadingGraph": "Loading graph…",
     "emptyGraph": "No URLs crawled yet — start a crawl to populate the graph.",
     "topAnchorTexts": "Top Anchor Texts",
@@ -988,7 +990,18 @@
     "changesNeedRerun": "Some changes need a layout re-run.",
     "rerunLayout": "Re-run layout",
     "title": "Visualization",
-    "occurrences": "{{count}} occurrences"
+    "occurrences": "{{count}} occurrences",
+    "cardTitle": "Page Title",
+    "cardResponse": "Response Code",
+    "cardIndexability": "Indexability",
+    "cardH1": "H1",
+    "cardH2Count": "H2 Count",
+    "cardDepth": "Crawl Depth",
+    "cardInlinks": "Unique Inlinks",
+    "cardOutlinks": "Unique Outlinks",
+    "cardFollowed": "Followed Outlinks",
+    "cardPctTotal": "% of Total",
+    "cardWordCount": "Word Count"
   },
   "imagesTab": {
     "searchPlaceholder": "Search image URLs / alt…",

apps/desktop/src/renderer/src/i18n/locales/tr.json

@@ -944,6 +944,7 @@
     "color": "Renk:",
     "nodes": "Düğüm:",
     "labels": "Etiket:",
+    "byCrawl": "Tarama Diyagramı (Derinlik × İndekslenebilirlik)",
     "byStatus": "Duruma Göre",
     "byDepth": "Derinliğe Göre",
     "byIndexability": "İndekslenebilirliğe Göre",
@@ -969,7 +970,8 @@
     "exportSvg": "SVG (vektör — Illustrator/Figma)",
     "exportHtml": "Standalone HTML (paylaşılabilir)",
     "nodesEdgesCount": "{{nodes}} düğüm · {{edges}} kenar",
-    "interactionHint": "Hover = komşular · Tıkla = seç · Boş tıklama = temizle · Çift tık düğüm = aç · Çift tık tuval = sığdır",
+    "interactionHint": "Üzerine gel = özet · Dala tıkla = kapat/aç · Çift tık düğüm = aç · Çift tık tuval = sığdır",
+    "interactionHint3d": "Sürükle = döndür · Kaydır = yakınlaştır · Sağ-sürükle = kaydır · Tıkla = düğüme uç · Sağ tık = aç",
     "loadingGraph": "Grafik yükleniyor…",
     "emptyGraph": "Henüz crawl edilmiş URL yok — grafiği doldurmak için bir crawl başlat.",
     "topAnchorTexts": "En Çok Anchor Metinleri",
@@ -988,7 +990,18 @@
     "changesNeedRerun": "Bazı değişiklikler layout yeniden çalıştırması gerektirir.",
     "rerunLayout": "Layout'u yeniden çalıştır",
     "title": "Görselleştirme",
-    "occurrences": "{{count}} kez geçiyor"
+    "occurrences": "{{count}} kez geçiyor",
+    "cardTitle": "Sayfa Başlığı",
+    "cardResponse": "Yanıt Kodu",
+    "cardIndexability": "İndekslenebilirlik",
+    "cardH1": "H1",
+    "cardH2Count": "H2 Sayısı",
+    "cardDepth": "Tarama Derinliği",
+    "cardInlinks": "Benzersiz İç Bağlantı",
+    "cardOutlinks": "Benzersiz Dış Bağlantı",
+    "cardFollowed": "Takip Edilen Bağlantı",
+    "cardPctTotal": "Toplamın %'si",
+    "cardWordCount": "Kelime Sayısı"
   },
   "imagesTab": {
     "searchPlaceholder": "Görsel URL / alt ara…",

apps/mcp-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/mcp-server",
-  "version": "0.8.9",
+  "version": "0.9.0",
   "private": true,
   "type": "module",
   "bin": {

docs/macos-apple-silicon-performance.md

@@ -0,0 +1,149 @@
+# macOS / Apple Silicon Performans Optimizasyonu — Araştırma Raporu
+
+> **Tarih:** 2026-07-21
+> **Tetikleyici:** MacBook Air M2 (Mac14,2), macOS 26.1 (25B78) üzerinde 40 dakikalık crawl sonrası
+> post-crawl fazında main process V8 heap OOM crash'i (`hata.txt` — EXC_BREAKPOINT / brk 0,
+> `LowMemoryNotification` → `FatalProcessOutOfMemory`, `node::sqlite::StatementSync::All` içinde).
+> **Kapsam:** FreeCrawl SEO'nun düşük donanımlı Apple Silicon Mac'lerde (8–16 GB unified memory,
+> fansız MacBook Air) darboğazsız çalışması.
+>
+> **Kanıt durumu:** 26 kaynak tarandı, 125 iddia çıkarıldı. Doğrulama fazının bir kısmı oturum
+> limitine takıldığından 25 iddianın 6'sı 3-0 çapraz doğrulamadan geçti, 3'ü çürütüldü (rapora
+> alınmadı), kalanlar kaynak alıntılı ancak tek kaynaklı — ⚠️ ile işaretli.
+
+---
+
+## 1. En kritik bulgu: Crash'in tavanı RAM değil, Electron'un 4 GB V8 kafesi ✅
+
+**hata.txt'deki çökme 8 GB'lık Air'de değil, 64 GB'lık Mac Studio'da da aynen yaşanırdı.**
+
+- Electron 14'ten beri **pointer compression** etkin: V8 heap'i process başına **maksimum 4 GB**
+  ile sınırlı — makinedeki fiziksel RAM'den bağımsız.
+  ([electronjs.org/blog/v8-memory-cage](https://www.electronjs.org/blog/v8-memory-cage)) ✅ 3-0
+- Electron 21'den beri **sandboxed pointers** de etkin; crash raporundaki
+  `Memory Tag 255 = 1.3 TB` bu sandbox'ın **sanal adres rezervasyonu** — gerçek RAM tüketimi
+  değil, zararsız. ([v8.dev/blog/sandbox](https://v8.dev/blog/sandbox)) ✅ 3-0 (rezervasyon boyutu ⚠️)
+- ⚠️ `--max-old-space-size` **işe yaramaz**: pointer-compression kafesi flag'i ezer;
+  `app.commandLine.appendSwitch('js-flags', ...)` main process için no-op (isolate zaten
+  yaratılmış). Maintainer'lara göre runtime workaround yok; tek çıkış özel Electron derlemesi.
+  ([electron#31330](https://github.com/electron/electron/issues/31330))
+- ⚠️ **En sinsi detay:** Electron'da ArrayBuffer/Buffer verisi de Node.js'in aksine kafes
+  **içinde** tutuluyor (maintainer nornagon'un tespiti). "Büyük veriyi Buffer'a al, heap
+  rahatlasın" taktiği Electron'da **çalışmaz**.
+- ⚠️ **Mimari sonuç:** `worker_threads` aynı process'te yaşadığından main isolate + reader
+  worker'lar + writer worker'lar **tek 4 GB kafesi paylaşıyor**. Worker havuzları CPU
+  paralelliği sağlıyor ama bellek tavanını genişletmiyor; worker→main `postMessage` structured
+  clone anlık olarak veriyi ikiye katlıyor.
+
+**Windows/Intel kafasından fark:** "Page file büyür, idare eder" varsayımı Electron'da her
+platformda geçersiz — kafes 4 GB. Düşük RAM'li Mac bunu sadece daha erken hissettirir
+(compression/swap baskısı kafes OOM'undan önce başlar).
+
+**Gerçek çözüm — veri hacmini heap'e hiç sokmamak:**
+- `StatementSync.prototype.iterate()` — tam bu amaçla eklendi, Node v22.13.0'a backport edildi,
+  Electron 41.2.2'nin Node 22.x'inde mevcut.
+  ([nodejs.org/api/sqlite.html](https://nodejs.org/api/sqlite.html),
+  [nodejs/node#54213](https://github.com/nodejs/node/pull/54213))
+- Gerçekten büyük heap gerektiren işler için maintainer önerisi: işi Electron dışına taşımak.
+  Pratik karşılığı **`utilityProcess`** — ayrı OS process = ayrı 4 GB kafes + ayrı crash domain'i.
+
+---
+
+## 2. `procRole: Background` — App Nap + E-core hapsi ✅
+
+Crash raporundaki `Role: Background` tesadüf değil; crawl arka planda çalışıyordu:
+
+- **App Nap** koşulları: ön planda değil + görünür içerik güncellenmiyor + ses yok + assertion
+  alınmamış → aday. ✅ 3-0
+- App Nap üç mekanizmayla vurur: ✅ 3-0
+  1. **Priority reduction** → daha az CPU payı
+  2. **Timer throttling** → `setInterval`/`setTimeout` tabanlı PQueue iş dağıtımı seyrekleşir
+  3. **I/O throttling** → SQLite disk yazışları kısılır
+- Resmi çıkış: NSProcessInfo activity assertion → sistem App Nap'e sokmaz. ✅ 3-0
+  Electron karşılığı: `powerSaveBlocker.start('prevent-app-suspension')`.
+  ([Apple App Nap docs](https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/AppNap.html))
+- ⚠️ Ödünleşim: `prevent-app-suspension` sistem uykusunu da engeller; "sadece App Nap'i engelle"
+  granülerliği Electron'da yok ('not planned' kapatıldı). Crawl aktifken aç, bitince `stop()`.
+- ⚠️ **E-core hapsi** (Apple Silicon'a özgü): QoS `background` (9) thread'ler **yalnızca
+  E-core'larda** çalışır; P-core'lar boşta olsa bile terfi yok, dışarıdan terfi de mümkün değil.
+  ([eclecticlight.co](https://eclecticlight.co/2024/12/17/tune-for-performance-core-types/))
+- ⚠️ App Nap'e girmiş uygulamalar bellek baskısında **compression/swap için öncelikli hedef**.
+
+---
+
+## 3. Fansız Air'de sustained load — termal adaptasyon ⚠️
+
+- macOS termal durum API'si: `nominal / fair / serious / critical`. Apple önerisi: `fair`'de
+  proaktif azalt, `serious`'ta CPU+I/O düşür, `critical`'de minimuma in.
+  ([Apple thermal docs](https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/RespondToThermalStateChanges.html))
+- **Native modül gerekmez:** Electron 41.2.2'de `powerMonitor.getCurrentThermalState()` ve
+  `'thermal-state-change'` event'i mevcut (electron.d.ts'ten doğrulandı).
+- Uyarı: desteklenmeyen sistemler her zaman `nominal` döndürür → tek sinyal olarak kullanma;
+  mevcut adaptive-throttle (yanıt süresi bazlı) ile birleştir.
+- Önerilen merdiven: `fair` → concurrency %75, `serious` → %50, `critical` → minimum.
+
+---
+
+## 4. SQLite × APFS — Windows'tan gerçekten farklı davranan katman ⚠️
+
+- **macOS'ta `fsync()` dayanıklılık garantisi vermez** — güç kesintisi garantisi için
+  `F_FULLFSYNC` gerekir; o da çok pahalı. Apple'ın sistem SQLite'ı bile `fullfsync=on`'u sessizce
+  `F_BARRIERFSYNC`'e çevirir.
+  ([mjtsai.com](https://mjtsai.com/blog/2025/09/05/sqlite-on-macos-not-acid/),
+  [avi.im/blag/2025/sqlite-fsync](https://avi.im/blag/2025/sqlite-fsync/))
+- Crawl projesi için doğru nokta: **`WAL` + `synchronous=NORMAL`** — corruption-safe, fsync
+  yalnızca checkpoint'te; kaybedilebilecek şey son commit'ler (yeniden fetch edilebilir veri).
+  `fullfsync` açma.
+- **WAL sınırsız büyüyebilir:** 40+ dk sürekli yazan crawl'da `wal_autocheckpoint` +
+  faz aralarında `PRAGMA wal_checkpoint(TRUNCATE)`.
+  ([phiresky](https://phiresky.github.io/blog/2020/sqlite-performance-tuning/))
+- `mmap_size` (256 MB–1 GB): read syscall'larını page-cache mapping'e çevirir; yalnızca sanal
+  bellek rezerve eder.
+- **Apple Silicon 16 KB VM sayfası kullanır** — 1 MB'lık TEXT blob'lar için `page_size=8192/16384`
+  düşünülebilir (yeni DB'lerde; mevcutlara VACUUM gerekir).
+- Sorgu hijyeni: body kolonunu list/aggregate sorgularından dışla; Google eşiği "4 KB'ın birkaç
+  katı üstü dosyaya" — 1 MB gövdeler için uzun vadede dosya + path kolonu meşru seçenek.
+
+---
+
+## 5. macOS bellek yönetimi — düşük RAM'li Mac'te ne olur? ⚠️
+
+- Kademeli davranış: önce **memory compression** (10.9+), sonra **swap**; process kill son çare
+  ve önce düşük öncelik bandı. Jetsam'in masaüstü agresifliği konusunda kaynaklar çelişkili
+  (iki karşıt iddia da çürütüldü) — net olan: **bizim crash sistem kill'i değil, V8'in 4 GB
+  tavanına çarpması** (`LowMemoryNotification` → abort zinciri kanıtı).
+- `DISPATCH_SOURCE_TYPE_MEMORYPRESSURE` (10.9+) native baskı bildirimi verir; native köprüsüz
+  pratik alternatif: büyük materialization öncesi `v8.getHeapStatistics()` headroom kontrolü.
+- Unified memory'de RAM aynı zamanda VRAM: 8 GB Air'de efektif bellek nominalden az.
+
+---
+
+## 6. macOS 26 (Tahoe) — bilinen Electron sorunları ⚠️
+
+| Sorun | Durum | FreeCrawl'a etkisi |
+|---|---|---|
+| `_cornerMask` private-API override → WindowServer GPU spike, sistem geneli kasma | Electron 36+'ya backport (PR #48376, 2025-09-26) ile düzeltildi | **Yok** — 41.2.2 düzeltmeyi içerir ([theregister](https://www.theregister.com/2025/10/02/macos_26_electron_slowdown/), [#48311](https://github.com/electron/electron/issues/48311)) |
+| MAS/App Sandbox + arm64 + macOS 26'da başlangıçta `EXC_BREAKPOINT` (MAP_JIT sıkılaşması hipotezi) | Açık; Apple DTS "JIT rehberi değişmedi" diyor, maintainer onayı yok | **App Store dağıtımı planlanırsa kritik**; sandbox'sız DMG'de üremiyor ([#51351](https://github.com/electron/electron/issues/51351), [Apple forums](https://developer.apple.com/forums/thread/821584)) |
+| 26.2'de Electron uygulamalarının crash log'suz SIGKILL'lenmesi | Tek kullanıcı raporu, 'not planned', doğrulanmamış | İzlenmeli ([#49261](https://github.com/electron/electron/issues/49261)) |
+| AutoFill (`NSAutoFillHeuristicController`) scroll yavaşlığı | 26.0.1'de kısmen düzeltildi | Düşük |
+
+**Teşhis ipucu:** Bu regresyon sınıfının istisna imzası bizimkiyle aynı (`EXC_BREAKPOINT` /
+brk 0, V8 içinde). Ayrım: OOM'da stack'te `LowMemoryNotification` + `node::sqlite` frame'leri
+var ve çökme dakikalar sonra; JIT regresyonunda çökme başlangıçta ve JIT/compiler thread'inde.
+
+---
+
+## 7. Eylem planı ve durum
+
+| # | Öncelik | İş | Durum |
+|---|---|---|---|
+| 1 | P0 | `recomputeBoilerplateCoverage`: `.all(sampleSize)` → 64'lük keyset batch + `SUBSTR(body,1,200000)`; `dbCall` ile writer worker'a taşındı (tepe bellek ~2–4 GB → ~25 MB) | ✅ 2026-07-21 |
+| 2 | P0 | Diğer toplu `url_sources.body` okuyucuları tarandı — `seoTextCorpus` body'yi zaten dışlıyor, duplicate clustering yalnızca `simhash` okuyor; başka toplu okuyucu yok | ✅ 2026-07-21 |
+| 3 | P0 | Crawl sırasında `powerSaveBlocker('prevent-app-suspension')` — ref-count'lu, `done` + yeni `stopped` event'iyle bırakılıyor | ✅ 2026-07-21 |
+| 4 | P1 | `callReaderOrFallback`: ağır metotlarda (HEAVY_METHODS) main-thread fallback kaldırıldı — rejection döner, sidebar eski sayıları korur ve sonraki tick'te yeniden dener | ✅ 2026-07-21 |
+| 5 | P1 | Termal-adaptif concurrency: `Crawler.setThrottleScale` (dış tavan, lag-adaptasyonuyla kompoze) + macOS'ta `powerMonitor` `thermal-state-change` → nominal 1 / fair 0.75 / serious 0.5 / critical 0.25; crawl başlangıcında mevcut durum da uygulanır | ✅ 2026-07-21 |
+| 6 | P1 | SQLite pragma seti — inceleme sonucu **zaten mevcuttu** (`WAL`, `synchronous=NORMAL`, `mmap_size=30GB`, `wal_autocheckpoint=2000`, `busy_timeout`, `temp_store=MEMORY`); eksik olan tek parça eklendi: post-crawl `wal_checkpoint(TRUNCATE)` (writer worker üzerinden) | ✅ 2026-07-21 |
+| 7 | P2 | DB worker'ları `worker_threads` → `utilityProcess` (ayrı 4 GB kafes + crash izolasyonu) — **bilinçli ertelendi** (2026-07-21): crawl yazma yolunu değiştiren en riskli iş, gerçek crawl smoke-testleriyle doğrulanabileceği ayrı bir oturuma bırakıldı; watchdog SAB'ı cross-process çalışmadığından heartbeat'lerin mesaj tabanlıya çevrilmesi de gerekecek | ⏸ |
+| 8 | P2 | Heap headroom guard: `ensureHeapHeadroom` (core, `v8.getHeapStatistics` tabanlı, 256 MB rezerv) — XLSX export satır biriktirme döngüsü (50K'da bir) + workbook build öncesi + Log Analyzer export'una uygulandı; OOM abort yerine anlaşılır "use CSV" hatası | ✅ 2026-07-21 |
+| 9 | P2 | Sorgu/export hijyeni: (a) `iterateUrlsByCategory` + `iterateIndexableUrls` **sahte streamer'dı** (içeride `.all()`) — keyset-paginated gerçek streaming'e çevrildi (1M-URL CSV export tepe belleği ~yüzlerce MB → sabit); (b) `queryUrls`'a server-side limit clamp (≤10K); (c) `SELECT *` bilinçli korundu — `urls`'ta sınırsız kolon yok, mapping tüm kolonları tüketiyor, 187 kolonluk el listesi drift riski; (d) bonus: proje şifreleme/çözme tam-dosya-Buffer'dan 8 MB streaming'e çevrildi (format aynı; 9 senaryoluk round-trip testiyle doğrulandı) | ✅ 2026-07-21 |
+| 10 | P2 | Uzun vade: `url_sources.body`'yi DB dışına (dosya + path) taşıma — şema migrasyonu + tüm okuma/yazma yollarını değiştiren büyük iş; P0–P2 sonrası acil bellek baskısı kalmadığından ihtiyaç doğana kadar ertelendi | ⏸ |

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "freecrawl-seo",
-      "version": "0.8.1",
+      "version": "0.9.0",
       "hasInstallScript": true,
       "license": "MIT",
       "workspaces": [
@@ -27,7 +27,7 @@
     },
     "apps/cli": {
       "name": "@freecrawl/cli",
-      "version": "0.8.1",
+      "version": "0.9.0",
       "dependencies": {
         "@freecrawl/core": "*",
         "@freecrawl/db": "*",
@@ -44,7 +44,7 @@
     },
     "apps/desktop": {
       "name": "@freecrawl/desktop",
-      "version": "0.8.1",
+      "version": "0.9.0",
       "dependencies": {
         "@dnd-kit/core": "^6.3.1",
         "@dnd-kit/sortable": "^10.0.0",
@@ -55,6 +55,7 @@
         "@tanstack/react-table": "^8.20.5",
         "@tanstack/react-virtual": "^3.10.0",
         "@types/cytoscape": "^3.21.9",
+        "3d-force-graph": "^1.80.0",
         "clsx": "^2.1.1",
         "cytoscape": "^3.33.2",
         "cytoscape-dagre": "^2.5.0",
@@ -96,7 +97,7 @@
     },
     "apps/mcp-server": {
       "name": "@freecrawl/mcp-server",
-      "version": "0.8.1",
+      "version": "0.9.0",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*"
@@ -2218,6 +2219,12 @@
         "url": "https://github.com/sponsors/tannerlinsley"
       }
     },
+    "node_modules/@tweenjs/tween.js": {
+      "version": "25.0.0",
+      "resolved": "https://registry.npmjs.org/@tweenjs/tween.js/-/tween.js-25.0.0.tgz",
+      "integrity": "sha512-XKLA6syeBUaPzx4j3qwMqzzq+V4uo72BnlbOjmuljLrRqdsd3qnzvZZoxvMHZ23ndsRS4aufU6JOZYpCbU6T1A==",
+      "license": "MIT"
+    },
     "node_modules/@types/babel__core": {
       "version": "7.20.5",
       "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",
@@ -2743,6 +2750,22 @@
         "node": ">=10.0.0"
       }
     },
+    "node_modules/3d-force-graph": {
+      "version": "1.80.0",
+      "resolved": "https://registry.npmjs.org/3d-force-graph/-/3d-force-graph-1.80.0.tgz",
+      "integrity": "sha512-tzI353gW1nXPpnC7VTa3JjMg+3cp77qOLUFO0vucPTfF+q5R6sQsNsIqVTbRIb7RSypn14nBa4yfkOe9ThxASw==",
+      "license": "MIT",
+      "dependencies": {
+        "accessor-fn": "1",
+        "kapsule": "^1.16",
+        "three": ">=0.179 <1",
+        "three-forcegraph": "1",
+        "three-render-objects": "^1.41"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/7zip-bin": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.2.0.tgz",
@@ -2760,6 +2783,15 @@
         "node": "^20.17.0 || >=22.9.0"
       }
     },
+    "node_modules/accessor-fn": {
+      "version": "1.5.3",
+      "resolved": "https://registry.npmjs.org/accessor-fn/-/accessor-fn-1.5.3.tgz",
+      "integrity": "sha512-rkAofCwe/FvYFUlMB0v0gWmhqtfAtV1IUkdPbfhTUyYniu5LrC0A0UJkTH0Jv3S8SvwkmfuAlY+mQIJATdocMA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/acorn": {
       "version": "8.16.0",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz",
@@ -3929,6 +3961,165 @@
         "cytoscape": "^3.2.22"
       }
     },
+    "node_modules/d3-array": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
+      "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
+      "license": "ISC",
+      "dependencies": {
+        "internmap": "1 - 2"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-binarytree": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/d3-binarytree/-/d3-binarytree-1.0.2.tgz",
+      "integrity": "sha512-cElUNH+sHu95L04m92pG73t2MEJXKu+GeKUN1TJkFsu93E5W8E9Sc3kHEGJKgenGvj19m6upSn2EunvMgMD2Yw==",
+      "license": "MIT"
+    },
+    "node_modules/d3-color": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
+      "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-dispatch": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz",
+      "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-force-3d": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/d3-force-3d/-/d3-force-3d-3.0.6.tgz",
+      "integrity": "sha512-4tsKHUPLOVkyfEffZo1v6sFHvGFwAIIjt/W8IThbp08DYAsXZck+2pSHEG5W1+gQgEvFLdZkYvmJAbRM2EzMnA==",
+      "license": "MIT",
+      "dependencies": {
+        "d3-binarytree": "1",
+        "d3-dispatch": "1 - 3",
+        "d3-octree": "1",
+        "d3-quadtree": "1 - 3",
+        "d3-timer": "1 - 3"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-format": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz",
+      "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-interpolate": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
+      "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
+      "license": "ISC",
+      "dependencies": {
+        "d3-color": "1 - 3"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-octree": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/d3-octree/-/d3-octree-1.1.0.tgz",
+      "integrity": "sha512-F8gPlqpP+HwRPMO/8uOu5wjH110+6q4cgJvgJT6vlpy3BEaDIKlTZrgHKZSp/i1InRpVfh4puY/kvL6MxK930A==",
+      "license": "MIT"
+    },
+    "node_modules/d3-quadtree": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz",
+      "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-scale": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
+      "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
+      "license": "ISC",
+      "dependencies": {
+        "d3-array": "2.10.0 - 3",
+        "d3-format": "1 - 3",
+        "d3-interpolate": "1.2.0 - 3",
+        "d3-time": "2.1.1 - 3",
+        "d3-time-format": "2 - 4"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-scale-chromatic": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz",
+      "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==",
+      "license": "ISC",
+      "dependencies": {
+        "d3-color": "1 - 3",
+        "d3-interpolate": "1 - 3"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-selection": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz",
+      "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-time": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz",
+      "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==",
+      "license": "ISC",
+      "dependencies": {
+        "d3-array": "2 - 3"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-time-format": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz",
+      "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==",
+      "license": "ISC",
+      "dependencies": {
+        "d3-time": "1 - 3"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-timer": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz",
+      "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/dagre": {
       "version": "0.8.5",
       "resolved": "https://registry.npmjs.org/dagre/-/dagre-0.8.5.tgz",
@@ -3939,6 +4130,18 @@
         "lodash": "^4.17.15"
       }
     },
+    "node_modules/data-bind-mapper": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/data-bind-mapper/-/data-bind-mapper-1.0.3.tgz",
+      "integrity": "sha512-QmU3lyEnbENQPo0M1F9BMu4s6cqNNp8iJA+b/HP2sSb7pf3dxwF3+EP1eO69rwBfH9kFJ1apmzrtogAmVt2/Xw==",
+      "license": "MIT",
+      "dependencies": {
+        "accessor-fn": "1"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/debug": {
       "version": "4.4.3",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
@@ -5110,6 +5313,20 @@
       "dev": true,
       "license": "ISC"
     },
+    "node_modules/float-tooltip": {
+      "version": "1.7.5",
+      "resolved": "https://registry.npmjs.org/float-tooltip/-/float-tooltip-1.7.5.tgz",
+      "integrity": "sha512-/kXzuDnnBqyyWyhDMH7+PfP8J/oXiAavGzcRxASOMRHFuReDtofizLLJsf7nnDLAfEaMW4pVWaXrAjtnglpEkg==",
+      "license": "MIT",
+      "dependencies": {
+        "d3-selection": "2 - 3",
+        "kapsule": "^1.16",
+        "preact": "10"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/form-data": {
       "version": "4.0.5",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
@@ -5745,6 +5962,15 @@
       "dev": true,
       "license": "ISC"
     },
+    "node_modules/internmap": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz",
+      "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/ip-address": {
       "version": "10.2.0",
       "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
@@ -5959,6 +6185,18 @@
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/kapsule": {
+      "version": "1.16.3",
+      "resolved": "https://registry.npmjs.org/kapsule/-/kapsule-1.16.3.tgz",
+      "integrity": "sha512-4+5mNNf4vZDSwPhKprKwz3330iisPrb08JyMgbsdFrimBCKNHecua/WBwvVg3n7vwx0C1ARjfhwIpbrbd9n5wg==",
+      "license": "MIT",
+      "dependencies": {
+        "lodash-es": "4"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/keyv": {
       "version": "4.5.4",
       "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
@@ -6032,6 +6270,12 @@
       "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
       "license": "MIT"
     },
+    "node_modules/lodash-es": {
+      "version": "4.18.1",
+      "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz",
+      "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==",
+      "license": "MIT"
+    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -6289,6 +6533,44 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/ngraph.events": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/ngraph.events/-/ngraph.events-1.4.0.tgz",
+      "integrity": "sha512-NeDGI4DSyjBNBRtA86222JoYietsmCXbs8CEB0dZ51Xeh4lhVl1y3wpWLumczvnha8sFQIW4E0vvVWwgmX2mGw==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/ngraph.forcelayout": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/ngraph.forcelayout/-/ngraph.forcelayout-3.3.1.tgz",
+      "integrity": "sha512-MKBuEh1wujyQHFTW57y5vd/uuEOK0XfXYxm3lC7kktjJLRdt/KEKEknyOlc6tjXflqBKEuYBBcu7Ax5VY+S6aw==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "ngraph.events": "^1.0.0",
+        "ngraph.merge": "^1.0.0",
+        "ngraph.random": "^1.0.0"
+      }
+    },
+    "node_modules/ngraph.graph": {
+      "version": "20.1.2",
+      "resolved": "https://registry.npmjs.org/ngraph.graph/-/ngraph.graph-20.1.2.tgz",
+      "integrity": "sha512-W/G3GBR3Y5UxMLHTUCPP9v+pbtpzwuAEIqP5oZV+9IwgxAIEZwh+Foc60iPc1idlnK7Zxu0p3puxAyNmDvBd0Q==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "ngraph.events": "^1.4.0"
+      }
+    },
+    "node_modules/ngraph.merge": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/ngraph.merge/-/ngraph.merge-1.0.0.tgz",
+      "integrity": "sha512-5J8YjGITUJeapsomtTALYsw7rFveYkM+lBj3QiYZ79EymQcuri65Nw3knQtFxQBU1r5iOaVRXrSwMENUPK62Vg==",
+      "license": "MIT"
+    },
+    "node_modules/ngraph.random": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/ngraph.random/-/ngraph.random-1.2.0.tgz",
+      "integrity": "sha512-4EUeAGbB2HWX9njd6bP6tciN6ByJfoaAvmVL9QTaZSeXrW46eNGA9GajiXiPBbvFqxUWFkEbyo6x5qsACUuVfA==",
+      "license": "BSD-3-Clause"
+    },
     "node_modules/node-abi": {
       "version": "4.28.0",
       "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-4.28.0.tgz",
@@ -6822,6 +7104,18 @@
         "node": ">=10.4.0"
       }
     },
+    "node_modules/polished": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/polished/-/polished-4.3.1.tgz",
+      "integrity": "sha512-OBatVyC/N7SCW/FaDHrSd+vn0o5cS855TOmYi4OkdWUMSJCET/xip//ch8xGUvtr3i44X9LVyWwQlRMTN3pwSA==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.17.8"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/postcss": {
       "version": "8.5.10",
       "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz",
@@ -7014,6 +7308,24 @@
         "node": "^12.20.0 || >=14"
       }
     },
+    "node_modules/preact": {
+      "version": "10.29.7",
+      "resolved": "https://registry.npmjs.org/preact/-/preact-10.29.7.tgz",
+      "integrity": "sha512-DCHYrK/B10yUD3ZjLfhZ3WIE/9Vf9VFUODcRE2dRomTYDpJk6z6L9wecSfhfE6M9ZTHUdyQkoC46arIDhEV84Q==",
+      "license": "MIT",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/preact"
+      },
+      "peerDependencies": {
+        "preact-render-to-string": ">=5"
+      },
+      "peerDependenciesMeta": {
+        "preact-render-to-string": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/prelude-ls": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -8015,6 +8327,56 @@
         "node": ">=0.8"
       }
     },
+    "node_modules/three": {
+      "version": "0.185.1",
+      "resolved": "https://registry.npmjs.org/three/-/three-0.185.1.tgz",
+      "integrity": "sha512-5aojFCXKwnjBRZvUnt3WFfEcvUJgkN5LlijRFN95hMy8WVkG4I0QNcJE+OuWvuJ0bOdStrbfXn0pkd6/QyiAlg==",
+      "license": "MIT",
+      "peer": true
+    },
+    "node_modules/three-forcegraph": {
+      "version": "1.43.4",
+      "resolved": "https://registry.npmjs.org/three-forcegraph/-/three-forcegraph-1.43.4.tgz",
+      "integrity": "sha512-FtmiZP/T16ZQaHza3JDaDn0YTXFtg9e7pGnTeU8nzu0NNkx7MpWbF/GvmpbQsWHx3rukHtkRv1fTorLPB3FDEA==",
+      "license": "MIT",
+      "dependencies": {
+        "accessor-fn": "1",
+        "d3-array": "1 - 3",
+        "d3-force-3d": "2 - 3",
+        "d3-scale": "1 - 4",
+        "d3-scale-chromatic": "1 - 3",
+        "data-bind-mapper": "1",
+        "kapsule": "^1.16",
+        "ngraph.forcelayout": "3",
+        "ngraph.graph": "20",
+        "tinycolor2": "1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "peerDependencies": {
+        "three": ">=0.118.3"
+      }
+    },
+    "node_modules/three-render-objects": {
+      "version": "1.42.0",
+      "resolved": "https://registry.npmjs.org/three-render-objects/-/three-render-objects-1.42.0.tgz",
+      "integrity": "sha512-KYfkPrYGEbIK8ChFocWqOF1aAN80FBUBWVYB8mB2oBpVuVN+52FvvngVYB5ieFANQu7Rt21rPYZ/xKaAgVWWRQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@tweenjs/tween.js": "18 - 25",
+        "accessor-fn": "1",
+        "float-tooltip": "^1.7",
+        "kapsule": "^1.16",
+        "polished": "4"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "peerDependencies": {
+        "three": ">=0.179"
+      }
+    },
     "node_modules/tiny-async-pool": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.3.0.tgz",
@@ -8035,6 +8397,12 @@
         "semver": "bin/semver"
       }
     },
+    "node_modules/tinycolor2": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz",
+      "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==",
+      "license": "MIT"
+    },
     "node_modules/tinyglobby": {
       "version": "0.2.16",
       "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz",
@@ -9070,7 +9438,7 @@
     },
     "packages/core": {
       "name": "@freecrawl/core",
-      "version": "0.8.1",
+      "version": "0.9.0",
       "dependencies": {
         "@freecrawl/db": "*",
         "@freecrawl/shared-types": "*",
@@ -9099,7 +9467,7 @@
     },
     "packages/db": {
       "name": "@freecrawl/db",
-      "version": "0.8.1",
+      "version": "0.9.0",
       "dependencies": {
         "@freecrawl/shared-types": "*"
       },
@@ -9111,7 +9479,7 @@
     },
     "packages/shared-types": {
       "name": "@freecrawl/shared-types",
-      "version": "0.8.1",
+      "version": "0.9.0",
       "devDependencies": {
         "rimraf": "^6.0.1",
         "typescript": "^5.7.0"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.8.9",
+  "version": "0.9.0",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.8.9",
+  "version": "0.9.0",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/crawler.ts

@@ -49,6 +49,15 @@ import { runBrowserLogin } from './browser-login.js';
 export interface CrawlerEvents {
   progress: (p: CrawlProgress) => void;
   done: (summary: CrawlSummary) => void;
+  /**
+   * Fired exactly once when `stop()` transitions the crawler into the
+   * stopped state. `done` is deliberately suppressed after a stop (a
+   * zombie crawler's done-event would clobber the next crawl's UI
+   * state), so hosts that need a "this crawl is over, whatever the
+   * reason" signal must listen to BOTH `done` and `stopped` — e.g. the
+   * desktop app's power-save blocker release.
+   */
+  stopped: () => void;
   error: (message: string) => void;
   warn: (message: string) => void;
   info: (message: string) => void;
@@ -875,11 +884,20 @@ export class Crawler extends EventEmitter {
       // clustering because both are content-similarity passes; surfaces the
       // repeated chrome (nav, footer, sidebar) that dilutes thin-content
       // detection. Memory-bounded, skipped when no body snapshots are stored.
+      // Routed through dbCall (writer worker on desktop) rather than the
+      // main-thread ProjectDb: the pass reads thousands of body snapshots
+      // and its JS-string churn belongs off the UI-serving thread — a
+      // main-thread run of this exact pass was the post-crawl V8 OOM
+      // crash on macOS (Electron's 4 GB pointer-compression heap cap).
       await yieldToEventLoop();
       this.emit('info', 'Detecting boilerplate coverage…');
       this.setOp('post-crawl:boilerplate-coverage');
       try {
-        const result = this.db.recomputeBoilerplateCoverage();
+        const result = await this.dbCall<{
+          sampled: number;
+          boilerplateShingles: number;
+          pagesAboveHighThreshold: number;
+        }>('recomputeBoilerplateCoverage', []);
         if (result.sampled > 0) {
           this.emit(
             'info',
@@ -929,6 +947,20 @@ export class Crawler extends EventEmitter {
     await yieldToEventLoop();
     this.setOp('post-crawl:performance-budget');
     this.runBudgetPass();
+    // Fold the WAL back into the main DB file now that the write burst
+    // is over. wal_autocheckpoint bounds WAL growth *during* the crawl;
+    // this TRUNCATE pass reclaims the file afterwards so the project
+    // on disk is compact and the first post-crawl reader queries don't
+    // pay WAL-frame lookup overhead. Matters double on macOS/APFS
+    // where checkpoint fsyncs are the expensive operation — better one
+    // deliberate pass here than implicit ones during the next crawl.
+    await yieldToEventLoop();
+    this.setOp('post-crawl:wal-checkpoint');
+    try {
+      await this.dbCall<void>('walCheckpoint', []);
+    } catch {
+      /* non-fatal — a reader mid-query can block the checkpoint */
+    }
   }
 
   /**
@@ -1944,6 +1976,9 @@ export class Crawler extends EventEmitter {
   }
 
   stop(): void {
+    // Emit 'stopped' only on the first transition — clearCrawlDb and
+    // the Stop IPC handler can both call stop() on the same instance.
+    const wasStopped = this.stopped;
     this.stopped = true;
     this.running = false;
     this.paused = false;
@@ -1985,6 +2020,7 @@ export class Crawler extends EventEmitter {
     this.externalQueue.clear();
     this.queue.start();
     this.externalQueue.start();
+    if (!wasStopped) this.emit('stopped');
   }
 
   pause(): void {
@@ -3736,12 +3772,53 @@ export class Crawler extends EventEmitter {
   private currentConcurrency = 0;
   private lastAdaptTs = 0;
   private static readonly ADAPT_COOLDOWN_MS = 2_000;
+
+  /**
+   * Externally imposed concurrency scale, 0.1–1. Fed by the desktop
+   * host from OS pressure signals — today macOS thermal state
+   * (nominal 1 / fair 0.75 / serious 0.5 / critical 0.25), by design
+   * open to other sources (memory pressure) later. Scales the ceiling
+   * the lag-adaptive loop is allowed to reach, so the two mechanisms
+   * compose instead of fighting: shrink is applied immediately
+   * (Apple's guidance — react to `serious` at once, the OS is already
+   * enacting countermeasures), recovery is gradual via
+   * `reportRendererLag`'s +1-per-cooldown growth once the ceiling
+   * lifts.
+   */
+  private throttleScale = 1;
+
+  /** Configured ceiling with the external throttle scale applied. */
+  private effectiveCeiling(): number {
+    const base = Math.max(1, Math.min(200, this.config.maxConcurrency));
+    return Math.max(1, Math.round(base * this.throttleScale));
+  }
+
+  /** Public — the desktop main process maps thermal state changes here. */
+  setThrottleScale(scale: number, reason: string): void {
+    const s = Math.min(1, Math.max(0.1, scale));
+    if (s === this.throttleScale) return;
+    this.throttleScale = s;
+    const ceiling = this.effectiveCeiling();
+    if (this.currentConcurrency === 0 || this.currentConcurrency > ceiling) {
+      this.currentConcurrency = ceiling;
+      this.queue.concurrency = ceiling;
+    }
+    // External probes get the same cut. Constructor band is 2..10;
+    // under throttle we additionally allow dropping to 1 so a
+    // `critical` state really does quiesce the network.
+    this.externalQueue.concurrency = Math.max(1, Math.min(10, ceiling));
+    this.emit(
+      'info',
+      `Throttle: concurrency ceiling → ${ceiling}/${Math.max(1, Math.min(200, this.config.maxConcurrency))} (${reason})`,
+    );
+  }
+
   /** Public so the desktop main process can pipe in renderer Lag reports. */
   reportRendererLag(lagMs: number): void {
     if (!this.running || this.paused) return;
     const now = Date.now();
     if (now - this.lastAdaptTs < Crawler.ADAPT_COOLDOWN_MS) return;
-    const ceiling = Math.max(1, Math.min(200, this.config.maxConcurrency));
+    const ceiling = this.effectiveCeiling();
     if (this.currentConcurrency === 0) this.currentConcurrency = ceiling;
     let next = this.currentConcurrency;
     // Floor 5 (was 1). The renderer's lag probe occasionally spikes
@@ -3750,8 +3827,10 @@ export class Crawler extends EventEmitter {
     // single-task dispatch — which on a slow remote server (avg resp
     // 1.5-2 s) cuts throughput from 10 URL/s to 0.5 URL/s and the
     // queue never recovers because the user never produces zero lag.
+    // The floor never exceeds the throttled ceiling — under thermal
+    // pressure the external cap wins.
     if (lagMs > 200) {
-      next = Math.max(5, this.currentConcurrency - 1);
+      next = Math.max(Math.min(5, ceiling), this.currentConcurrency - 1);
     } else if (lagMs < 30) {
       next = Math.min(ceiling, this.currentConcurrency + 1);
     }

packages/core/src/heap-guard.ts

@@ -0,0 +1,50 @@
+/**
+ * JS-heap headroom guard for large single-shot materialisations.
+ *
+ * Electron enables V8 pointer compression, which hard-caps every
+ * process's JS heap at 4 GB regardless of machine RAM — and unlike
+ * plain Node, Buffer/ArrayBuffer data counts against the same cage.
+ * When V8 can't allocate it doesn't throw a catchable error; it aborts
+ * the whole process (`FatalProcessOutOfMemory`). So any code path that
+ * is about to build a large in-memory artefact (an XLSX workbook, a
+ * whole-file encryption buffer) must check headroom BEFORE allocating
+ * and fail with a normal, user-visible error instead of a crash.
+ *
+ * The check is advisory — `estimatedBytes` is a rough upper bound, and
+ * a healthy RESERVE is kept free for GC churn and concurrent
+ * allocations. Callers on genuinely unbounded paths should prefer
+ * streaming (see tabular-export's CSV writer) and use this guard only
+ * where the format itself forces full materialisation.
+ */
+
+import v8 from 'node:v8';
+
+/** Slack kept free below the heap limit — GC needs breathing room, and
+ *  other subsystems (IPC buffers, SQLite rows in flight) allocate
+ *  concurrently with the guarded operation. */
+const RESERVE_BYTES = 256 * 1024 * 1024;
+
+const MB = 1024 * 1024;
+
+/** Bytes currently available before the hard heap ceiling. */
+export function heapHeadroomBytes(): number {
+  const s = v8.getHeapStatistics();
+  return Math.max(0, s.heap_size_limit - s.used_heap_size);
+}
+
+/**
+ * Throw a descriptive Error when `estimatedBytes` more allocation
+ * would push the heap within RESERVE of its hard limit. No-op when
+ * there is room. `label` names the operation in the error message the
+ * user sees (e.g. "XLSX export").
+ */
+export function ensureHeapHeadroom(label: string, estimatedBytes: number): void {
+  const available = heapHeadroomBytes();
+  if (estimatedBytes + RESERVE_BYTES <= available) return;
+  throw new Error(
+    `${label}: not enough memory headroom — needs ~${Math.ceil(estimatedBytes / MB)} MB ` +
+      `but only ${Math.floor(available / MB)} MB of JS heap remains ` +
+      `(hard limit ${Math.round(v8.getHeapStatistics().heap_size_limit / MB)} MB). ` +
+      'Try a streaming format (CSV/JSON) or export a smaller selection.',
+  );
+}

packages/core/src/html-parser.ts

@@ -922,7 +922,18 @@ export function parseHtml(
   }
   const mixedContentCount = mixedContentActive + mixedContentPassive;
 
-  const text = $('body').text().replace(/\s+/g, ' ').trim();
+  // Visible body prose. `$('body').text()` alone also returns the text
+  // nodes *inside* `<script>` / `<style>` / `<noscript>`, so on a
+  // JS-heavy page the minified bundle is counted as content: a real
+  // 400-word product page measured 12,900 "words", and every readability
+  // score was computed over JavaScript source. Strip those subtrees
+  // first — the same thing `textCodeRatio` does below, for the same
+  // reason. Note this deliberately keeps nav / header / footer copy
+  // (unlike `extractMainContentText`) so thin-content thresholds keep
+  // measuring the whole rendered page.
+  const $visibleBody = $('body').clone();
+  $visibleBody.find('script, style, noscript').remove();
+  const text = $visibleBody.text().replace(/\s+/g, ' ').trim();
   const wordCount = text.length > 0 ? text.split(' ').filter(Boolean).length : 0;
   const readability = computeReadability(text, wordCount);
 
@@ -1008,19 +1019,14 @@ export function parseHtml(
   const hasNofollow = metaRobots !== null && metaRobots.includes('nofollow');
 
   // Text/code ratio — visible body text bytes divided by raw HTML bytes,
-  // rounded to integer percent. Cheap to compute right next to the body
-  // text-length pass below. We strip <script>/<style>/<noscript> contents
-  // out of the visible-text count because their inner text isn't user-
-  // facing content (and would inflate the ratio on JS-heavy SPAs that
-  // ship a 200 KB script tag with no markup). Null when the body is
-  // empty or the HTML body byte count is zero.
+  // rounded to integer percent. Reuses the script/style/noscript-stripped
+  // `text` computed above: their inner text isn't user-facing content and
+  // would inflate the ratio on JS-heavy SPAs that ship a 200 KB script tag
+  // with no markup. Null when the HTML body byte count is zero.
   const totalHtmlBytes = Buffer.byteLength(html, 'utf8');
   let textCodeRatio: number | null = null;
   if (totalHtmlBytes > 0) {
-    const $body = $('body').clone();
-    $body.find('script, style, noscript').remove();
-    const visibleText = $body.text().replace(/\s+/g, ' ').trim();
-    const visibleBytes = Buffer.byteLength(visibleText, 'utf8');
+    const visibleBytes = Buffer.byteLength(text, 'utf8');
     textCodeRatio = Math.min(100, Math.round((visibleBytes / totalHtmlBytes) * 100));
   }
 

packages/core/src/index.ts

@@ -53,6 +53,7 @@ export {
   type TabularExportResult,
   type TabularSection,
 } from './tabular-export.js';
+export { ensureHeapHeadroom, heapHeadroomBytes } from './heap-guard.js';
 export {
   testUrlAgainstRobots,
   validateRobotsTxt,

packages/core/src/tabular-export.ts

@@ -5,6 +5,7 @@ import path from 'node:path';
 import zlib from 'node:zlib';
 import type { ProjectDb } from '@freecrawl/db';
 import type { CrawlUrlRow, UrlCategory } from '@freecrawl/shared-types';
+import { ensureHeapHeadroom } from './heap-guard.js';
 
 export interface TabularSection {
   label: string;
@@ -506,17 +507,26 @@ export async function exportTabular(
 
   if (format === 'xlsx') {
     // xlsx — single workbook, one sheet per section. subdir/filename
-    // are ignored; the workbook lives at `outputPath`.
+    // are ignored; the workbook lives at `outputPath`. Unlike the
+    // CSV/JSON/XML writers below (which stream row-by-row), the xlsx
+    // container format forces every sheet into memory at once — so
+    // guard the heap while accumulating: better a clear "use CSV for
+    // datasets this big" error than a V8 OOM abort of the whole app.
     const usedNames = new Set<string>();
     let total = 0;
     const sheets = sections.map((section) => {
       const rows: CrawlUrlRow[] = [];
       for (const row of rowSource(db, section.category, selectedIds)) {
         rows.push(row);
+        if (rows.length % 50_000 === 0) {
+          ensureHeapHeadroom('XLSX export', 128 * 1024 * 1024);
+        }
       }
       total += rows.length;
       return { name: sanitizeSheetName(section.label, usedNames), rows };
     });
+    // Workbook buffer is roughly rows × columns × ~24 B of XML.
+    ensureHeapHeadroom('XLSX export', total * columns.length * 24);
     const buf = buildXlsxBuffer(sheets, columns);
     writeFileSync(outputPath, buf);
     return { filePath: outputPath, files: [outputPath], rowsWritten: total };

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.8.9",
+  "version": "0.9.0",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/project-db.ts

@@ -2304,8 +2304,11 @@ export class ProjectDb {
    * Why sampling instead of full-crawl pass: at 100K URLs × 50KB body
    * × 100 unique 5-word shingles, a full pass needs ~5GB of working
    * memory for the shingle→pageSet inverted index. Sampling 2K bodies
-   * keeps it under 200MB while still surfacing the same boilerplate
-   * (templated chrome doesn't change between page 2000 and page 100000).
+   * keeps the shingle index under ~60MB while still surfacing the same
+   * boilerplate (templated chrome doesn't change between page 2000 and
+   * page 100000). Bodies themselves are streamed in small batches with
+   * a per-body SUBSTR cap (see Phase 1) so raw HTML never accumulates
+   * in the JS heap — the sampled-index budget above is the whole cost.
    *
    * Pages with < shingleSize words contribute nothing (no shingles
    * generated) and get coverage = 0.
@@ -2350,6 +2353,15 @@ export class ProjectDb {
         .replace(/<script[\s\S]*?<\/script>/gi, ' ')
         .replace(/<style[\s\S]*?<\/style>/gi, ' ')
         .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
+        // Unterminated trailing block. Bodies arrive truncated (the
+        // crawler's 1 MB snapshot cap, plus the SUBSTR cap below), so the
+        // cut routinely lands inside a <script>. The paired patterns above
+        // need a closing tag to match, so without this the whole minified
+        // bundle survives tag-stripping and becomes "text" — on a real
+        // crawl that turned 400 words of prose into 8,300 JS tokens, and
+        // since those tokens are near-identical across pages every shingle
+        // looked like boilerplate and coverage pinned at 100% site-wide.
+        .replace(/<(?:script|style|noscript)\b[\s\S]*$/i, ' ')
         .replace(/<!--[\s\S]*?-->/g, ' ')
         .replace(/<[^>]+>/g, ' ')
         .replace(/&[#a-z0-9]+;/gi, ' ')
@@ -2374,26 +2386,55 @@ export class ProjectDb {
     const pageShingles = new Map<number, Set<number>>();
     const shingleCounts = new Map<number, number>();
 
-    const rows = this.db
-      .prepare(
-        `SELECT u.id AS id, us.body AS body
+    // Bodies are read in small keyset-paginated batches, each body
+    // capped via SUBSTR, then shingled and dropped before the next
+    // batch loads. The previous single `.all(sampleSize)` materialised
+    // every sampled body at once — with 1 MB snapshot caps that's up
+    // to 2 GB of JS strings (4 GB the moment a body contains one
+    // non-Latin-1 char and V8 switches to two-byte strings), which
+    // blew Electron's hard 4 GB pointer-compression heap ceiling and
+    // crashed the app post-crawl. Peak is now ~BATCH × cap ≈ 25 MB.
+    //
+    // The cap matches the crawler's 1 MB body-snapshot ceiling, so a
+    // stored body is read whole and the shingle set covers the real
+    // page rather than its first third. An earlier 200K cap looked
+    // safer but skewed the result badly: it cut deep inside the inline
+    // JS most pages carry, and everything after that cut is unique
+    // *content*, so pages lost their distinguishing shingles while
+    // keeping the shared chrome — coverage collapsed to 100% for every
+    // sampled page. Peak heap stays bounded by BATCH × cap ≈ 64 MB,
+    // versus the ~2 GB the pre-batching `.all(sampleSize)` reached.
+    const BATCH = 64;
+    const BODY_CHAR_CAP = 1_048_576;
+    const pickBatch = this.db.prepare(
+      `SELECT u.id AS id, SUBSTR(us.body, 1, ${BODY_CHAR_CAP}) AS body
          FROM urls u
          JOIN url_sources us ON us.url_id = u.id
-          WHERE u.is_external = 0
+        WHERE u.id > ?
+          AND u.is_external = 0
           AND u.content_kind = 'html'
           AND us.body IS NOT NULL
           AND LENGTH(us.body) > 0
         ORDER BY u.id
         LIMIT ?`,
-      )
-      .all(sampleSize) as { id: number; body: string }[];
-
-    for (const row of rows) {
+    );
+    let lastId = 0;
+    let remaining = sampleSize;
+    while (remaining > 0) {
+      const batch = pickBatch.all(lastId, Math.min(BATCH, remaining)) as {
+        id: number;
+        body: string;
+      }[];
+      if (batch.length === 0) break;
+      for (const row of batch) {
         const shingles = shingleHashes(stripHtml(row.body));
         pageShingles.set(row.id, shingles);
         for (const h of shingles) {
           shingleCounts.set(h, (shingleCounts.get(h) ?? 0) + 1);
         }
+        lastId = row.id;
+      }
+      remaining -= batch.length;
     }
 
     // Phase 2 — identify boilerplate shingles (appear on more than
@@ -4510,6 +4551,15 @@ export class ProjectDb {
     });
     const sortCol = validSortColumn(params.sortBy);
     const sortDir = params.sortDir === 'desc' ? 'DESC' : 'ASC';
+    // Server-side clamp — this method is reachable from the renderer
+    // and the MCP bridge, so the limit can't be trusted to be sane. A
+    // runaway limit here materialises the whole table in one `.all()`;
+    // 10K rows is comfortably above every legitimate caller (virtual
+    // table chunks are ≤2K, MCP selection resolution caps at 5K).
+    // Callers that genuinely need the full table stream it through
+    // `iterateUrlsByCategory` instead.
+    const limit = Math.max(1, Math.min(10_000, Math.floor(params.limit) || 1));
+    const offset = Math.max(0, Math.floor(params.offset) || 0);
 
     const totalRow = this.db
       .prepare(`SELECT COUNT(*) AS c FROM urls ${whereSql}`)
@@ -4521,7 +4571,7 @@ export class ProjectDb {
       .prepare(
         `SELECT * FROM urls ${whereSql} ORDER BY ${sortCol} ${sortDir}${tieBreak} LIMIT ? OFFSET ?`,
       )
-      .all(...args, params.limit, params.offset) as unknown as UrlRowDb[];
+      .all(...args, limit, offset) as unknown as UrlRowDb[];
 
     return { rows: rowsDb.map(this.rowFromDb), total: totalRow.c };
   }
@@ -4564,6 +4614,17 @@ export class ProjectDb {
       }
       return out;
     };
+    // Count a sidebar node through the *same* predicate its click applies.
+    // These counters used to carry their own hand-written WHERE clauses,
+    // which had silently drifted from `categoryWhereClause`: the sidebar
+    // scoped Response Codes / Security / Indexability to internal URLs
+    // while the filter didn't, so "Success (2xx) 7,327" opened a table of
+    // 7,329 rows. Deriving the count from the clause makes the number and
+    // the row set the same query by construction.
+    const countCategory = (cat: UrlCategory): number => {
+      const clause = categoryWhereClause(cat);
+      return countWhere(clause ?? '1 = 1');
+    };
 
     const totalInternalUrls = countWhere('is_external = 0');
     const totalExternalUrls = countWhere('is_external = 1');
@@ -4571,12 +4632,10 @@ export class ProjectDb {
     // Headline indexability reflects PAGES only (content_kind = 'html').
     // Resources (image/css/js/font) are all `indexable` with a 2xx status,
     // so without this guard they would inflate the "Indexable URLs" stat.
-    const totalIndexable = countWhere(
-      "is_external = 0 AND content_kind = 'html' AND indexability = 'indexable' AND status_code IS NOT NULL",
-    );
-    const totalNonIndexable = countWhere(
-      "is_external = 0 AND content_kind = 'html' AND indexability LIKE 'non-indexable%'",
-    );
+    // The page-only scope lives in `categoryWhereClause` so the Directives
+    // tab shows exactly the rows this headline counts.
+    const totalIndexable = countCategory('indexability:indexable');
+    const totalNonIndexable = countCategory('indexability:non-indexable');
     const totalImages = (
       this.db.prepare('SELECT COUNT(*) AS c FROM images').get() as { c: number }
     ).c;
@@ -4604,33 +4663,28 @@ export class ProjectDb {
         html: countWhere("is_external = 1 AND content_kind = 'html'"),
         other: countWhere("is_external = 1 AND content_kind != 'html'"),
       },
+      // Response Codes covers every URL the crawl touched, external
+      // included — that group is how a user finds a broken outbound link,
+      // and the External tab keeps its own separate view.
       responseCodes: {
-        all: totalInternalUrls,
-        blockedRobots: countWhere(
-          "is_external = 0 AND indexability = 'non-indexable:robots-blocked'",
-        ),
-        noResponse: countWhere('is_external = 0 AND status_code IS NULL'),
-        success2xx: countWhere('is_external = 0 AND status_code >= 200 AND status_code < 300'),
-        redirect3xx: countWhere('is_external = 0 AND status_code >= 300 AND status_code < 400'),
-        clientError4xx: countWhere(
-          'is_external = 0 AND status_code >= 400 AND status_code < 500',
-        ),
-        serverError5xx: countWhere(
-          'is_external = 0 AND status_code >= 500 AND status_code < 600',
-        ),
+        all: countCategory('all'),
+        blockedRobots: countCategory('status:blocked-robots'),
+        noResponse: countCategory('status:no-response'),
+        success2xx: countCategory('status:2xx'),
+        redirect3xx: countCategory('status:3xx'),
+        clientError4xx: countCategory('status:4xx'),
+        serverError5xx: countCategory('status:5xx'),
       },
       security: {
-        https: countWhere("is_external = 0 AND url LIKE 'https://%'"),
-        http: countWhere("is_external = 0 AND url LIKE 'http://%'"),
+        https: countCategory('security:https'),
+        http: countCategory('security:http'),
       },
       indexability: {
         indexable: totalIndexable,
         nonIndexable: totalNonIndexable,
-        noindex: countWhere("is_external = 0 AND indexability = 'non-indexable:noindex'"),
-        canonicalised: countWhere("is_external = 0 AND indexability = 'non-indexable:canonical'"),
-        blockedRobots: countWhere(
-          "is_external = 0 AND indexability = 'non-indexable:robots-blocked'",
-        ),
+        noindex: countCategory('indexability:noindex'),
+        canonicalised: countCategory('indexability:canonicalised'),
+        blockedRobots: countCategory('indexability:blocked-robots'),
       },
       issues: this.getIssuesCounts(),
     };
@@ -5968,14 +6022,29 @@ export class ProjectDb {
    * `category === 'all'` walks the full urls table.
    */
   *iterateUrlsByCategory(category: UrlCategory): IterableIterator<CrawlUrlRow> {
+    // Keyset-paginated so this is a REAL streamer: the previous
+    // single `.all()` materialised every matching row up front, which
+    // on a 1M-URL export meant hundreds of MB of row objects pinned in
+    // the JS heap before the first CSV line was written — squarely
+    // inside Electron's 4 GB heap-cap budget. 2000-row batches keep
+    // peak memory flat regardless of table size while adding only
+    // ~500 statement executions per million rows.
     const where = categoryWhereClause(category);
+    const BATCH = 2000;
     const sql = where
-      ? `SELECT * FROM urls WHERE ${where} ORDER BY id`
-      : 'SELECT * FROM urls ORDER BY id';
-    const rows = this.db.prepare(sql).all() as unknown as UrlRowDb[];
+      ? `SELECT * FROM urls WHERE (${where}) AND id > ? ORDER BY id LIMIT ?`
+      : 'SELECT * FROM urls WHERE id > ? ORDER BY id LIMIT ?';
+    const stmt = this.db.prepare(sql);
+    let lastId = 0;
+    for (;;) {
+      const rows = stmt.all(lastId, BATCH) as unknown as UrlRowDb[];
+      if (rows.length === 0) return;
       for (const row of rows) {
+        lastId = row.id;
         yield this.rowFromDb(row);
       }
+      if (rows.length < BATCH) return;
+    }
   }
 
   /**
@@ -5984,19 +6053,33 @@ export class ProjectDb {
    * noindex, canonicalised, blocked, 4xx/5xx, and non-HTML resources.
    */
   *iterateIndexableUrls(): IterableIterator<CrawlUrlRow> {
-    const rows = this.db
-      .prepare(
+    // Composite (depth, id) keyset pagination — same real-streaming
+    // rationale as iterateUrlsByCategory, but this consumer (the XML
+    // sitemap generator) needs depth-then-id order, so the cursor
+    // carries both columns.
+    const BATCH = 2000;
+    const stmt = this.db.prepare(
       `SELECT * FROM urls
        WHERE is_external = 0
          AND content_kind = 'html'
          AND indexability = 'indexable'
          AND status_code >= 200 AND status_code < 300
-         ORDER BY depth, id`,
-      )
-      .all() as unknown as UrlRowDb[];
+         AND (depth > ? OR (depth = ? AND id > ?))
+       ORDER BY depth, id
+       LIMIT ?`,
+    );
+    let lastDepth = -1;
+    let lastId = 0;
+    for (;;) {
+      const rows = stmt.all(lastDepth, lastDepth, lastId, BATCH) as unknown as UrlRowDb[];
+      if (rows.length === 0) return;
       for (const row of rows) {
+        lastDepth = (row as unknown as { depth: number }).depth;
+        lastId = row.id;
         yield this.rowFromDb(row);
       }
+      if (rows.length < BATCH) return;
+    }
   }
 
   /**
@@ -6021,13 +6104,24 @@ export class ProjectDb {
       lcpResourceUrl: string | null;
       lcpCoverage: number | null;
       linkScore: number | null;
+      title: string | null;
+      h1: string | null;
+      h2Count: number;
+      wordCount: number | null;
+      outlinks: number;
+      followedOutlinks: number;
     }[];
     edges: { source: number; target: number }[];
+    /** Total internal HTML URLs in the project (NOT capped by
+     *  `nodeLimit`) — the denominator for the visualization tooltip's
+     *  "% of Total" inlink share. */
+    totalUrls: number;
   } {
     const nodes = this.db
       .prepare(
         `SELECT id, url, status_code, depth, inlinks, indexability,
-                lcp_tag, lcp_resource_url, lcp_coverage, link_score
+                lcp_tag, lcp_resource_url, lcp_coverage, link_score,
+                title, h1, h2_count, word_count, outlinks
            FROM urls
           WHERE is_external = 0 AND content_kind = 'html'
           ORDER BY inlinks DESC, id ASC
@@ -6044,9 +6138,23 @@ export class ProjectDb {
       lcp_resource_url: string | null;
       lcp_coverage: number | null;
       link_score: number | null;
+      title: string | null;
+      h1: string | null;
+      h2_count: number;
+      word_count: number | null;
+      outlinks: number;
     }[];
 
-    if (nodes.length === 0) return { nodes: [], edges: [] };
+    if (nodes.length === 0) return { nodes: [], edges: [], totalUrls: 0 };
+
+    const totalUrls = (
+      this.db
+        .prepare(
+          `SELECT COUNT(*) AS c FROM urls
+            WHERE is_external = 0 AND content_kind = 'html'`,
+        )
+        .get() as { c: number }
+    ).c;
 
     const idByUrl = new Map<string, number>();
     for (const n of nodes) idByUrl.set(n.url, n.id);
@@ -6058,12 +6166,27 @@ export class ProjectDb {
     const placeholders = fromIds.map(() => '?').join(',');
     const edgeRows = this.db
       .prepare(
-        `SELECT from_url_id AS source, to_url FROM links
+        `SELECT from_url_id AS source, to_url, rel FROM links
           WHERE is_internal = 1 AND from_url_id IN (${placeholders})`,
       )
-      .all(...fromIds) as { source: number; to_url: string }[];
+      .all(...fromIds) as { source: number; to_url: string; rel: string | null }[];
 
     const edges: { source: number; target: number }[] = [];
+    // Followed = distinct internal targets reached by a link whose rel
+    // doesn't carry nofollow — the outlinks that actually pass equity.
+    // Counted off the same rows we already fetched (no extra query) and
+    // BEFORE the node-cap filter, so the number reflects the real page
+    // rather than the visible slice of the graph.
+    const followedTargets = new Map<number, Set<string>>();
+    for (const e of edgeRows) {
+      if (e.rel && /\bnofollow\b/i.test(e.rel)) continue;
+      let set = followedTargets.get(e.source);
+      if (!set) {
+        set = new Set<string>();
+        followedTargets.set(e.source, set);
+      }
+      set.add(e.to_url);
+    }
     for (const e of edgeRows) {
       const target = idByUrl.get(e.to_url);
       if (target !== undefined && target !== e.source) {
@@ -6083,8 +6206,15 @@ export class ProjectDb {
         lcpResourceUrl: n.lcp_resource_url,
         lcpCoverage: n.lcp_coverage,
         linkScore: n.link_score,
+        title: n.title,
+        h1: n.h1,
+        h2Count: n.h2_count,
+        wordCount: n.word_count,
+        outlinks: n.outlinks,
+        followedOutlinks: followedTargets.get(n.id)?.size ?? 0,
       })),
       edges,
+      totalUrls,
     };
   }
 
@@ -8069,16 +8199,28 @@ function categoryWhereClause(cat: UrlCategory): string | null {
       return "url LIKE 'https://%'";
     case 'security:http':
       return "url LIKE 'http://%'";
+    // Indexability is a property of pages, not of the resources they
+    // pull in: a font or a stylesheet is stored as `indexable` with a
+    // 2xx status, and counting those as indexable URLs inflates the
+    // headline stat and fills the Directives tab with rows no directive
+    // can apply to. Every member of this group therefore carries the
+    // same internal-HTML scope. `indexable` additionally requires a
+    // response — a URL we never fetched isn't indexable.
     case 'indexability:indexable':
-      return "indexability = 'indexable'";
+      return `is_external = 0 AND content_kind = 'html'
+              AND indexability = 'indexable' AND status_code IS NOT NULL`;
     case 'indexability:non-indexable':
-      return "indexability LIKE 'non-indexable%'";
+      return `is_external = 0 AND content_kind = 'html'
+              AND indexability LIKE 'non-indexable%'`;
     case 'indexability:noindex':
-      return "indexability = 'non-indexable:noindex'";
+      return `is_external = 0 AND content_kind = 'html'
+              AND indexability = 'non-indexable:noindex'`;
     case 'indexability:canonicalised':
-      return "indexability = 'non-indexable:canonical'";
+      return `is_external = 0 AND content_kind = 'html'
+              AND indexability = 'non-indexable:canonical'`;
     case 'indexability:blocked-robots':
-      return "indexability = 'non-indexable:robots-blocked'";
+      return `is_external = 0 AND content_kind = 'html'
+              AND indexability = 'non-indexable:robots-blocked'`;
     case 'issues:title-missing':
       return "is_external = 0 AND content_kind = 'html' AND (title IS NULL OR title = '')";
     case 'issues:title-too-long':

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.8.9",
+  "version": "0.9.0",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/ipc.ts

@@ -138,6 +138,12 @@ export const IPC = {
   topAnchorTexts: 'graph:anchor-texts',
   sitemapGenerate: 'sitemap:generate',
   menuEvent: 'menu:event',
+  /** Renderer-requested native Copy (webContents.copy()). Used by the
+   * macOS Edit▸Copy round-trip: the native menu swallows Cmd+C before
+   * the page sees the keydown, so the menu item asks the renderer
+   * whether a custom table-selection copy applies — and when it
+   * doesn't, the renderer calls back here for the stock behaviour. */
+  editCopyNative: 'edit:copyNative',
   dataChanged: 'data:changed',
   appVersion: 'app:version',
   /** Returns live process + system memory stats for the in-app
@@ -477,6 +483,7 @@ export type MenuEvent =
   | 'open-log-analyzer'
   | 'open-settings'
   | 'open-scheduled-crawl'
+  | 'edit-copy'
   | 'about';
 
 export interface ExportCsvResult {
@@ -711,6 +718,18 @@ export interface GraphNode {
    *  Null until the post-crawl link-score pass runs. Powers the
    *  "By Link Score" visualization colour mode. */
   linkScore: number | null;
+  // ── Fields below feed the visualization's hover summary card ──
+  title: string | null;
+  h1: string | null;
+  /** Number of `<h2>` elements on the page (the crawler stores a count,
+   *  not the heading text). */
+  h2Count: number;
+  wordCount: number | null;
+  /** Unique outlinks recorded for the page (internal + external). */
+  outlinks: number;
+  /** Distinct internal outlink targets NOT marked rel="nofollow" — the
+   *  links that actually pass equity. */
+  followedOutlinks: number;
 }
 
 export interface GraphEdge {
@@ -725,6 +744,9 @@ export interface GraphSnapshotInput {
 export interface GraphSnapshotResult {
   nodes: GraphNode[];
   edges: GraphEdge[];
+  /** Total internal HTML URLs in the project, uncapped by `nodeLimit` —
+   *  denominator for the hover card's "% of Total" inlink share. */
+  totalUrls: number;
 }
 
 export interface CrawlPathInput {
@@ -1815,5 +1837,8 @@ export interface FreeCrawlApi {
   onDone(cb: (summary: CrawlSummary) => void): () => void;
   onError(cb: (message: string) => void): () => void;
   onMenuEvent(cb: (event: MenuEvent) => void): () => void;
+  /** Run the stock Copy command (native text-selection copy) on this
+   * window — the fallback leg of the macOS Edit▸Copy round-trip. */
+  editCopyNative(): Promise<void>;
   onDataChanged(cb: () => void): () => void;
 }