diff --git a/README.md b/README.md
index 653093c..8ac6810 100644
--- a/README.md
+++ b/README.md
@@ -400,12 +400,14 @@ graph LR
## 📈 Status
> [!NOTE]
-> **Active development.** All 31 analysis tabs are working — the core set (Internal, External, Response Codes, URL, Page Titles, Meta Description, H1, H2, Content, Images, Canonicals, Directives, Redirects, Pagination, Hreflang, AMP, Structured Data, Meta Refresh, Custom Extraction, Custom Search, Security, Duplicates, Links, Broken Links, SERP) plus the integration tabs (**PageSpeed, CrUX, Spelling, Search Console, GA4, AI, SEO Authority**).
+> **Active development.** All 32 analysis tabs are working — the core set (Internal, External, Response Codes, URL, Page Titles, Meta Description, H1, H2, Content, Images, Canonicals, Directives, Redirects, Pagination, Hreflang, AMP, Structured Data, Meta Refresh, Custom Extraction, Custom Search, Security, Duplicates, Links, Broken Links, SERP) plus the integration tabs (**PageSpeed, CrUX, Spelling, Search Console, GA4, AI, SEO Authority**).
>
-> Also shipping: 150+ issue checks, **JavaScript rendering** (Playwright — post-JS DOM, screenshots, LCP candidate, Mobile Usability audit), **multi-window projects** (independent per-window crawls), near/exact-duplicate detection, hreflang validation, **internal PageRank link scoring**, project compare, the **Cytoscape link graph** (force-directed / BFS / radial / Sugiyama / directory-tree layouts + LCP and link-score overlays + click-to-trace Crawl Path Report), **Basic / Digest / Bearer + form-based & browser-driven SPA login**, **HTTP / SOCKS5 proxy**, the **MCP server with live crawl control**, all **Google integrations** (Search Console / URL Inspection / GA4 / PageSpeed / CrUX), **LanguageTool spelling & grammar**, **AI per-URL prompts** (OpenAI / Anthropic / Ollama), **SEO Authority providers** (Ahrefs / Majestic / Moz / Semrush), **Google Sheets + BigQuery export**, **encrypted project snapshots** (AES-256-GCM), **RAM-only storage mode**, scheduled crawls, robots.txt validator, URL rewriting, EN + TR UI, and the **standalone Log File Analyzer** (Apache / Nginx / IIS access logs → bot hits, crawl budget, crawl × log orphans, 40+ bot detection, CSV / Excel export, and a headless `analyze-logs` CLI).
+> Also shipping: 150+ issue checks, **JavaScript rendering** (Playwright — post-JS DOM, screenshots, LCP candidate, Mobile Usability audit), **multi-window projects** (independent per-window crawls), near/exact-duplicate detection, hreflang validation, **internal PageRank link scoring**, project compare, the **link visualiser** (WebGL 3D force-directed crawl tree and link graph, plus 2D Crawl Tree and Directory Tree with click-to-collapse branches, hover summary cards, LCP and link-score overlays, and a click-to-trace Crawl Path Report), **Basic / Digest / Bearer + form-based & browser-driven SPA login**, **HTTP / SOCKS5 proxy**, the **MCP server with live crawl control**, all **Google integrations** (Search Console / URL Inspection / GA4 / PageSpeed / CrUX), **LanguageTool spelling & grammar**, **AI per-URL prompts** (OpenAI / Anthropic / Ollama), **SEO Authority providers** (Ahrefs / Majestic / Moz / Semrush), **Google Sheets + BigQuery export**, **encrypted project snapshots** (AES-256-GCM), **RAM-only storage mode**, scheduled crawls, robots.txt validator, URL rewriting, EN + TR UI, and the **standalone Log File Analyzer** (Apache / Nginx / IIS access logs → bot hits, crawl budget, crawl × log orphans, 40+ bot detection, CSV / Excel export, and a headless `analyze-logs` CLI).
>
> Cross-platform installers ship for Windows (`.exe` + portable), macOS (`.dmg`), and Linux (`.AppImage` / `.deb` / `.rpm`), with Playwright Chromium bundled so JS rendering works on first launch. The virtualized 1M-row table streams the **first row in ~1 s**.
>
+> **Apple Silicon tuning:** long crawls hold a power assertion so macOS App Nap can't throttle them in the background, concurrency scales down automatically as the thermal state rises (so a fanless MacBook Air degrades gracefully instead of stalling), heavy aggregates never run on the main thread, and every large read is batch-streamed to stay clear of Electron's hard per-process heap ceiling. macOS also gets the full Windows-style `Cmd`-key shortcut set.
+>
> **Upcoming:** Plugin system, Light theme, Code-signing + auto-update.
diff --git a/SECURITY.md b/SECURITY.md
index c531510..741ae10 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -13,7 +13,7 @@ upgrading is the recommended remediation path.
| Version | Supported |
| ------- | ------------------ |
-| Latest stable (currently `0.3.x`) | Yes |
+| Latest stable (currently `0.9.x`) | Yes |
| Older `0.x` releases | No |
Release artifacts are published at:
@@ -60,8 +60,8 @@ possible:
layer (`@freecrawl/db`), desktop shell, MCP server, or CLI
- **Affected version(s)** — output of `Help → About`, or the `version`
field in `package.json`
-- **Operating system + architecture** — Windows 11 / macOS 14 (Intel or
- ARM64) / Linux distro
+- **Operating system + architecture** — Windows 10/11, macOS 13 or newer
+ (Intel or Apple Silicon), or your Linux distribution
- **Reproduction steps** — minimal sequence that triggers the issue
- **Proof-of-concept** — screenshot, video, or minimal code
- **Impact assessment** — confidentiality / integrity / availability
@@ -143,12 +143,16 @@ Releases. The artifacts are produced by an automated GitHub Actions
workflow defined in `.github/workflows/release.yml`. SHA512 digests are
published alongside each release in `latest.yml` / `latest-mac.yml`.
-Code-signing for Windows installers is being established through
-[SignPath.io](https://signpath.io)'s open-source signing program. Once
-active, all `.exe` artifacts will be signed and the application will
-verify the publisher signature before applying any in-app update. macOS
-artifacts are currently unsigned; users should verify the SHA512 digest
-manually until Apple Developer ID notarization is in place.
+**Release artifacts are currently unsigned on every platform.** Verify
+the SHA512 digest of anything you download against the value published
+with the release before running it. Expect the usual first-launch
+warnings: SmartScreen on Windows, and Gatekeeper on macOS (documented
+in the README, along with the `xattr -cr` workaround).
+
+Code-signing for Windows installers is being pursued through
+[SignPath.io](https://signpath.io)'s open-source signing program, and
+macOS notarization requires an Apple Developer ID. Neither is active
+yet; this section will be updated when that changes.
If you encounter an installer that fails signature verification or
appears to come from any source other than the official GitHub Releases
@@ -167,9 +171,19 @@ page above, **do not run it** — please report it via the channels above.
the crawls the user explicitly initiates and (when enabled) the
on-demand "Check for Updates" call to the GitHub Releases API.
- **Project files (`.seoproject`)** are SQLite databases containing
- crawled content. They are not encrypted at rest in the current
- release — users handling sensitive data should rely on full-disk
- encryption at the OS level.
+ crawled content, and are **not** encrypted by default. Two options
+ exist for sensitive crawls:
+ - **Save As Encrypted** writes a password-protected snapshot
+ (AES-256-GCM, per-file random salt and IV, PBKDF2-SHA256 key
+ derivation at 200 000 iterations, authenticated header). Opening it
+ decrypts to a working copy, so the *open* project is still plaintext
+ on disk for the duration of the session.
+ - **RAM-only storage mode** keeps the crawl in memory and never writes
+ a project file at all.
+
+ Users handling sensitive data should still enable full-disk encryption
+ at the OS level — it is the only thing that protects the working copy
+ and SQLite's temporary files.
---
diff --git a/apps/cli/package.json b/apps/cli/package.json
index cd48ef6..b9e7f0a 100644
--- a/apps/cli/package.json
+++ b/apps/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/cli",
- "version": "0.8.9",
+ "version": "0.9.0",
"private": true,
"type": "module",
"bin": {
diff --git a/apps/desktop/package.json b/apps/desktop/package.json
index 91aa38b..623a185 100644
--- a/apps/desktop/package.json
+++ b/apps/desktop/package.json
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/desktop",
- "version": "0.8.9",
+ "version": "0.9.0",
"private": true,
"description": "FreeCrawl SEO — desktop application",
"homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
@@ -35,6 +35,7 @@
"@tanstack/react-table": "^8.20.5",
"@tanstack/react-virtual": "^3.10.0",
"@types/cytoscape": "^3.21.9",
+ "3d-force-graph": "^1.80.0",
"clsx": "^2.1.1",
"cytoscape": "^3.33.2",
"cytoscape-dagre": "^2.5.0",
diff --git a/apps/desktop/src/main/db-reader-pool.ts b/apps/desktop/src/main/db-reader-pool.ts
index 60c8750..0312466 100644
--- a/apps/desktop/src/main/db-reader-pool.ts
+++ b/apps/desktop/src/main/db-reader-pool.ts
@@ -323,6 +323,15 @@ export function setReaderPoolResolver(fn: () => DbReaderPool): void {
* to the synchronous main-process DB on any worker error. The fallback
* path keeps the UI working even if every worker is crashed/restarting,
* at the cost of running on the main thread for that one query.
+ *
+ * HEAVY methods never fall back. If a heavy aggregate crashed or timed
+ * out its reader worker, re-running the exact same query on the main
+ * thread escalates a contained worker failure into a main-process
+ * stall (multi-second aggregate blocking every IPC) or — for
+ * OOM-class failures — a process crash, since Electron's V8 heap is
+ * hard-capped at 4 GB per process. Callers get a rejection instead;
+ * the sidebar keeps its previous counts and the next poll tick retries
+ * against the (by then respawned) worker.
*/
export async function callReaderOrFallback(
method: string,
@@ -330,13 +339,25 @@ export async function callReaderOrFallback(
fallback: () => T | Promise,
): Promise {
const pool = resolveReaderPool();
+ const heavy = HEAVY_METHODS.has(method);
if (!pool.isReady()) {
+ if (heavy) {
+ throw new Error(`reader-unavailable: '${method}' skips main-thread fallback`);
+ }
return fallback();
}
try {
return await pool.call(method, args);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
+ if (heavy) {
+ logger.log(
+ 'warn',
+ 'db-reader',
+ `'${method}' failed on reader pool (${msg}) — heavy query, NOT retrying on main thread.`,
+ );
+ throw err;
+ }
logger.log(
'warn',
'db-reader',
diff --git a/apps/desktop/src/main/db-writer-worker.ts b/apps/desktop/src/main/db-writer-worker.ts
index 7036511..ea5e060 100644
--- a/apps/desktop/src/main/db-writer-worker.ts
+++ b/apps/desktop/src/main/db-writer-worker.ts
@@ -86,11 +86,13 @@ const ALLOWED_METHODS = new Set([
'recomputeHreflangAnalysis',
'recomputeHreflangInconsistent',
'recomputeDuplicateClusters',
+ 'recomputeBoilerplateCoverage',
'recomputePaginationSequence',
'recomputeUrlsIssues',
'recomputeUrlsIssuesYielding',
'checkpointQueue',
'clearQueueCheckpoint',
+ 'walCheckpoint',
'markUrlForRecrawl',
'markUrlsForRecrawl',
'deleteUrl',
diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts
index a0dee7c..66c9392 100644
--- a/apps/desktop/src/main/index.ts
+++ b/apps/desktop/src/main/index.ts
@@ -6,6 +6,8 @@ import {
dialog,
Menu,
Notification,
+ powerMonitor,
+ powerSaveBlocker,
session,
shell,
Tray,
@@ -24,6 +26,7 @@ import { totalmem, freemem } from 'node:os';
import {
DEFAULT_CRAWL_CONFIG,
IPC,
+ type MenuEvent,
type ConfirmClearResult,
type CrawlConfig,
type CrawlProgress,
@@ -184,6 +187,7 @@ import {
exportTabular,
exportSitemap,
exportHtmlReport,
+ ensureHeapHeadroom,
compareCrawls,
testUrlAgainstRobots,
validateRobotsTxt,
@@ -1035,6 +1039,92 @@ async function recomputeIssuesViaPool(
* 5–15 s on a 5K-URL crawl — exactly the window where users hit
* Stop/Pause and "nothing happens".
*/
+/**
+ * Ref-counted power-save blocker held while any window's crawl runs.
+ * First active crawl starts the blocker, last one to end releases it —
+ * keyed by Crawler instance so double-release (e.g. stop() during
+ * clearCrawlDb after the Stop IPC already fired) is a no-op. Without
+ * this, macOS App Nap throttles a backgrounded crawl's timers, CPU
+ * share and disk I/O, and laptop sleep kills it entirely.
+ */
+const crawlsHoldingPowerBlocker = new Set();
+let crawlPowerBlockerId: number | null = null;
+
+function acquireCrawlPowerBlocker(crawler: Crawler): void {
+ if (crawlsHoldingPowerBlocker.has(crawler)) return;
+ crawlsHoldingPowerBlocker.add(crawler);
+ if (crawlPowerBlockerId !== null) return;
+ try {
+ crawlPowerBlockerId = powerSaveBlocker.start('prevent-app-suspension');
+ logger.log('info', 'main', 'Power-save blocker acquired (crawl running)');
+ } catch (err) {
+ // Non-fatal — the crawl still runs, just without suspension immunity.
+ crawlPowerBlockerId = null;
+ logger.log(
+ 'warn',
+ 'main',
+ `powerSaveBlocker.start failed: ${err instanceof Error ? err.message : String(err)}`,
+ );
+ }
+}
+
+function releaseCrawlPowerBlocker(crawler: Crawler): void {
+ if (!crawlsHoldingPowerBlocker.delete(crawler)) return;
+ if (crawlsHoldingPowerBlocker.size > 0 || crawlPowerBlockerId === null) return;
+ try {
+ powerSaveBlocker.stop(crawlPowerBlockerId);
+ } catch {
+ /* already stopped — ignore */
+ }
+ crawlPowerBlockerId = null;
+ logger.log('info', 'main', 'Power-save blocker released (no crawls running)');
+}
+
+/**
+ * Thermal-adaptive crawl throttling (macOS-only — Electron emits
+ * `thermal-state-change` only there). On fanless Apple Silicon
+ * machines (MacBook Air) a 40-minute crawl at full concurrency drives
+ * the SoC into sustained throttling; Apple's guidance is to shed CPU
+ * and I/O proactively from the `fair` state onward, before the OS
+ * starts enacting its own countermeasures at `serious`. The scale is
+ * applied as a concurrency *ceiling* on every live crawler — the
+ * crawler's lag-adaptive loop keeps operating underneath it.
+ * `nominal`/`unknown` map to 1 (unsupported systems always report
+ * `nominal`, so this path is a safe no-op for them).
+ */
+function thermalScaleFor(state: string): number {
+ switch (state) {
+ case 'critical':
+ return 0.25;
+ case 'serious':
+ return 0.5;
+ case 'fair':
+ return 0.75;
+ default:
+ return 1;
+ }
+}
+
+function applyThermalStateToCrawlers(state: string): void {
+ const scale = thermalScaleFor(state);
+ for (const s of [defaultSession, ...sessionsByWc.values()]) {
+ s.activeCrawler?.setThrottleScale(scale, `thermal:${state}`);
+ }
+}
+
+/** Seed a freshly launched crawler with the current thermal state so a
+ * crawl started on an already-hot machine begins throttled instead of
+ * waiting for the next state *change*. */
+function applyCurrentThermalState(crawler: Crawler): void {
+ if (process.platform !== 'darwin') return;
+ try {
+ const state = powerMonitor.getCurrentThermalState();
+ crawler.setThrottleScale(thermalScaleFor(state), `thermal:${state}`);
+ } catch {
+ /* powerMonitor not ready pre-app-ready — crawls can't start then */
+ }
+}
+
async function runDbPassViaPool(
session: ProjectSession,
method: string,
@@ -1352,6 +1442,22 @@ function rebuildMenu(): void {
onOpenLogsFolder: () => openLogsFolder(),
onCheckForUpdates: () => void checkForUpdates(),
onOpenLogAnalyzer: () => openLogAnalyzerWindow(),
+ onEditCopy: () => {
+ const win = BrowserWindow.getFocusedWindow();
+ if (!win || win.isDestroyed()) return;
+ // Project windows understand the 'edit-copy' menu event and run
+ // the custom-table-copy → native-copy round-trip. Auxiliary
+ // windows (Logs, Log Analyzer, viz popouts) don't subscribe to
+ // menu events — give them the stock native copy directly so
+ // Cmd+C on selected text keeps working there.
+ const isProjectWindow =
+ win === mainWindow || sessionsByWc.has(win.webContents.id);
+ if (isProjectWindow) {
+ win.webContents.send(IPC.menuEvent, 'edit-copy' satisfies MenuEvent);
+ } else {
+ win.webContents.copy();
+ }
+ },
}),
);
}
@@ -2466,6 +2572,13 @@ function seedDiscoveryUrls(limit: number): LogSeedDiscoveryResult {
function registerIpc(): void {
registerHandle(IPC.appVersion, () => app.getVersion());
+ // Native-copy fallback leg of the macOS Edit▸Copy round-trip: the
+ // renderer calls this when no custom table selection consumed the
+ // copy, so the stock behaviour (copy the DOM text selection / the
+ // focused input's selection) still happens.
+ registerHandle(IPC.editCopyNative, (e): void => {
+ e.sender.copy();
+ });
// Live memory snapshot for the in-app monitor (status bar). Returns
// process RSS / heap + system total/free + the active project's URL
@@ -2622,6 +2735,10 @@ function registerIpc(): void {
filePath = res.filePath;
}
const db = getDb();
+ // Bounded but chunky: up to 250K+ small rows plus the workbook
+ // buffer, all materialised at once. Verify the heap can absorb
+ // it before starting — a clear error beats a V8 OOM abort.
+ ensureHeapHeadroom('Log analysis export', 256 * 1024 * 1024);
const tables: LogExportTables = {
overview: db.getLogOverview(),
urlStats: db.queryLogUrlStats({ limit: 100_000, offset: 0, sortBy: 'totalHits', filter: 'all' }).rows,
@@ -4593,6 +4710,23 @@ function registerIpc(): void {
* in the Start dialog again. Persisted across restarts via DB
* project_meta so it survives the very crash we're recovering from. */
function attachCrawlerListeners(crawler: Crawler, sess: ProjectSession): void {
+ // Keep the OS from suspending/App-Napping us while this crawl runs.
+ // On macOS a backgrounded crawl otherwise gets App Nap's priority
+ // reduction + timer throttling + disk-I/O throttling (and on Apple
+ // Silicon, background-QoS work is confined to E-cores); on laptops
+ // generally, system sleep would kill the crawl outright.
+ // `prevent-app-suspension` is the cheapest blocker level — the
+ // display may still turn off. Released on 'done' AND 'stopped'
+ // (done is suppressed after stop(), so both hooks are required),
+ // deliberately BEFORE the zombie-crawler guard in the handlers
+ // below: a superseded crawler must still drop its blocker hold.
+ acquireCrawlPowerBlocker(crawler);
+ crawler.on('done', () => releaseCrawlPowerBlocker(crawler));
+ crawler.on('stopped', () => releaseCrawlPowerBlocker(crawler));
+ // Start throttled if the machine is already hot (macOS-only no-op
+ // elsewhere) — the thermal-state-change listener only covers
+ // *changes* after launch.
+ applyCurrentThermalState(crawler);
// Bound to the launching session so progress/done/error reach THAT
// window (multi-window) — these callbacks run outside any IPC handler's
// AsyncLocalStorage context, so they can't use `currentSession()`.
@@ -6002,7 +6136,7 @@ function registerIpc(): void {
const rawDb = (database as any).db as { exec: (sql: string) => void };
rawDb.exec(`VACUUM INTO '${tmpPath.replace(/'/g, "''")}'`);
const { encryptFile } = await import('./project-encryption.js');
- const result = encryptFile(tmpPath, i.filePath, i.password);
+ const result = await encryptFile(tmpPath, i.filePath, i.password);
return { filePath: i.filePath, bytesWritten: result.bytesWritten };
} finally {
try {
@@ -6022,7 +6156,7 @@ function registerIpc(): void {
if (!i.password) throw new Error('password is required.');
if (!i.destPath) throw new Error('destPath is required (output .seoproject path).');
const { decryptFile } = await import('./project-encryption.js');
- decryptFile(i.filePath, i.destPath, i.password);
+ await decryptFile(i.filePath, i.destPath, i.password);
openProjectAtPath(i.destPath);
return { filePath: i.destPath };
},
@@ -6310,7 +6444,7 @@ function registerIpc(): void {
rawDb.exec(`VACUUM INTO '${escaped}'`);
const { encryptFile } = await import('./project-encryption.js');
- const result = encryptFile(tmpPath, target, password);
+ const result = await encryptFile(tmpPath, target, password);
await dialog.showMessageBox(win, {
type: 'info',
@@ -6379,7 +6513,7 @@ function registerIpc(): void {
try {
const { decryptFile } = await import('./project-encryption.js');
- decryptFile(srcPath, dstPath, password);
+ await decryptFile(srcPath, dstPath, password);
openProjectAtPath(dstPath);
return { filePath: dstPath };
} catch (err) {
@@ -7347,6 +7481,16 @@ void app.whenReady().then(async () => {
logger.log('info', 'main', `App ready — version ${app.getVersion()}`);
freezeWatchdog.setMainOp('idle');
+ // Thermal-adaptive crawl throttling — macOS-only (Electron emits
+ // `thermal-state-change` only on macOS; other platforms never fire it).
+ // See applyThermalStateToCrawlers for the state → concurrency mapping.
+ if (process.platform === 'darwin') {
+ powerMonitor.on('thermal-state-change', (details) => {
+ logger.log('info', 'main', `Thermal state → ${details.state}`);
+ applyThermalStateToCrawlers(details.state);
+ });
+ }
+
// Open the localhost MCP bridge so Claude Code / Claude Desktop can
// drive crawls + poll progress against this running instance. Bind
// failure is logged but non-fatal: the desktop app keeps working,
diff --git a/apps/desktop/src/main/menu-i18n.ts b/apps/desktop/src/main/menu-i18n.ts
index c08fe2c..aa0d790 100644
--- a/apps/desktop/src/main/menu-i18n.ts
+++ b/apps/desktop/src/main/menu-i18n.ts
@@ -45,6 +45,8 @@ export interface MenuLabels {
openProjectEncrypted: string;
openProjectEncryptedTooltip: string;
settings: string;
+ edit: string;
+ copy: string;
view: string;
overviewSidebar: string;
detailPanel: string;
@@ -178,6 +180,8 @@ const MENU_EN: MenuLabels = {
openProjectEncryptedTooltip:
'Decrypt a .seoproject.enc snapshot with its password and open the recovered project.',
settings: 'Settings…',
+ edit: 'Edit',
+ copy: 'Copy',
view: 'View',
overviewSidebar: 'Overview Sidebar',
detailPanel: 'Detail Panel',
@@ -315,6 +319,8 @@ const MENU_TR: MenuLabels = {
openProjectEncryptedTooltip:
'Bir .seoproject.enc snapshot\'ını parolasıyla çöz ve kurtarılan projeyi aç.',
settings: 'Ayarlar…',
+ edit: 'Düzen',
+ copy: 'Kopyala',
view: 'Görünüm',
overviewSidebar: 'Genel Bakış Kenar Çubuğu',
detailPanel: 'Detay Paneli',
diff --git a/apps/desktop/src/main/menu.ts b/apps/desktop/src/main/menu.ts
index 5e41690..010ce78 100644
--- a/apps/desktop/src/main/menu.ts
+++ b/apps/desktop/src/main/menu.ts
@@ -32,6 +32,17 @@ export interface AppMenuHandlers {
onCheckForUpdates: () => void;
/** V2 Faz 2 — open the standalone Log File Analyzer window. */
onOpenLogAnalyzer: () => void;
+ /**
+ * macOS Edit▸Copy dispatcher. On macOS the native menu's key
+ * equivalent swallows Cmd+C before the renderer ever sees a keydown
+ * — the exact opposite of Windows/Linux, where the page gets the key
+ * first. So on macOS the Copy item is a custom click that lets the
+ * host decide: project windows get a 'edit-copy' menu event (their
+ * renderer tries the custom table-selection copy, then falls back to
+ * native), auxiliary windows (Logs, Log Analyzer) get a plain
+ * `webContents.copy()`.
+ */
+ onEditCopy: () => void;
/** UI language for menu labels. Falls back to `en` when missing. */
lang?: MenuLang;
}
@@ -154,7 +165,33 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
isMac ? { role: 'close' } : { role: 'quit' },
],
},
- { role: 'editMenu' as const },
+ // Windows/Linux keep the stock role — there the renderer receives
+ // Ctrl+C keydown BEFORE any menu accelerator, so the app's custom
+ // table-copy handlers work with zero interception. On macOS the
+ // native menu's key equivalent fires FIRST and the page never sees
+ // Cmd+C, so Copy must be a custom item that routes through
+ // `onEditCopy` (custom table copy → native copy fallback). Every
+ // other edit command keeps its native role.
+ isMac
+ ? {
+ label: L.edit,
+ submenu: [
+ { role: 'undo' as const },
+ { role: 'redo' as const },
+ { type: 'separator' as const },
+ { role: 'cut' as const },
+ {
+ label: L.copy,
+ accelerator: 'CmdOrCtrl+C',
+ click: () => handlers.onEditCopy(),
+ },
+ { role: 'paste' as const },
+ { role: 'pasteAndMatchStyle' as const },
+ { role: 'delete' as const },
+ { role: 'selectAll' as const },
+ ],
+ }
+ : { role: 'editMenu' as const },
{
label: L.view,
submenu: [
diff --git a/apps/desktop/src/main/project-encryption.ts b/apps/desktop/src/main/project-encryption.ts
index 5fccc9e..9172874 100644
--- a/apps/desktop/src/main/project-encryption.ts
+++ b/apps/desktop/src/main/project-encryption.ts
@@ -31,7 +31,18 @@ import {
pbkdf2Sync,
randomBytes,
} from 'node:crypto';
-import { readFileSync, writeFileSync } from 'node:fs';
+import {
+ closeSync,
+ createReadStream,
+ createWriteStream,
+ openSync,
+ readSync,
+ renameSync,
+ rmSync,
+ statSync,
+ writeFileSync,
+} from 'node:fs';
+import { finished, pipeline } from 'node:stream/promises';
const MAGIC = Buffer.from('FCRYPT01', 'utf8'); // 8 bytes
const VERSION = 0x01;
@@ -42,22 +53,29 @@ const TAG_LEN = 16;
const PBKDF2_ITERATIONS = 200_000;
const HEADER_LEN = MAGIC.length + 1 + SALT_LEN + IV_LEN; // magic + version + salt + iv
+/** Streaming chunk size — large enough that syscall overhead is noise,
+ * small enough that peak memory stays flat for any project size. */
+const STREAM_HIGH_WATER_MARK = 8 * 1024 * 1024;
+
/**
* Encrypt the file at `srcPath` and write the encrypted container to
- * `dstPath`. The source file is read into memory in one shot — fine for
- * `.seoproject` snapshots which are tens of MB; for hundreds of MB we'd
- * want a streaming variant. Throws if the password is empty or the
- * source file can't be read.
+ * `dstPath`. Fully streaming: read → AES-GCM transform → write in 8 MB
+ * chunks, so peak memory is O(chunk) no matter how large the project
+ * file is. The previous whole-file-in-a-Buffer version needed
+ * plaintext + ciphertext resident at once — on a multi-GB project
+ * that alone could blow Electron's 4 GB heap cap (Buffers count
+ * against it, unlike plain Node). Throws if the password is empty or
+ * the source file can't be read.
*/
-export function encryptFile(
+export async function encryptFile(
srcPath: string,
dstPath: string,
password: string,
-): { bytesWritten: number } {
+): Promise<{ bytesWritten: number }> {
if (!password || password.length === 0) {
throw new Error('Password is required.');
}
- const plaintext = readFileSync(srcPath);
+ const srcSize = statSync(srcPath).size;
const salt = randomBytes(SALT_LEN);
const iv = randomBytes(IV_LEN);
const key = pbkdf2Sync(password, salt, PBKDF2_ITERATIONS, KEY_LEN, 'sha256');
@@ -65,12 +83,21 @@ export function encryptFile(
const header = Buffer.concat([MAGIC, Buffer.from([VERSION]), salt, iv]);
const cipher = createCipheriv('aes-256-gcm', key, iv);
cipher.setAAD(header);
- const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
- const tag = cipher.getAuthTag();
- const out = Buffer.concat([header, ciphertext, tag]);
- writeFileSync(dstPath, out);
- return { bytesWritten: out.length };
+ const out = createWriteStream(dstPath);
+ out.write(header);
+ // `end: false` keeps the sink open — the GCM auth tag only exists
+ // after the cipher flushes, and it goes at the end of the container.
+ await pipeline(
+ createReadStream(srcPath, { highWaterMark: STREAM_HIGH_WATER_MARK }),
+ cipher,
+ out,
+ { end: false },
+ );
+ out.end(cipher.getAuthTag());
+ await finished(out);
+ // GCM ciphertext length equals plaintext length.
+ return { bytesWritten: HEADER_LEN + srcSize + TAG_LEN };
}
/**
@@ -78,49 +105,91 @@ export function encryptFile(
* `.seoproject` bytes to `dstPath`. Throws on bad magic, unsupported
* version, or an authentication-tag mismatch (wrong password / file
* tampered with).
+ *
+ * Streaming, with the same all-or-nothing contract as the old
+ * in-memory version: plaintext is streamed into a sibling temp file
+ * and only renamed onto `dstPath` after `decipher.final()` verifies
+ * the GCM tag — so `dstPath` never contains unauthenticated bytes.
*/
-export function decryptFile(
+export async function decryptFile(
srcPath: string,
dstPath: string,
password: string,
-): { bytesWritten: number } {
+): Promise<{ bytesWritten: number }> {
if (!password || password.length === 0) {
throw new Error('Password is required.');
}
- const buf = readFileSync(srcPath);
- if (buf.length < HEADER_LEN + TAG_LEN) {
+ const srcSize = statSync(srcPath).size;
+ if (srcSize < HEADER_LEN + TAG_LEN) {
throw new Error('Encrypted file is truncated or not a FreeCrawl encrypted project.');
}
- if (!buf.subarray(0, MAGIC.length).equals(MAGIC)) {
+ // Positional reads for the fixed-size header and the trailing tag —
+ // the ciphertext between them is never held in memory.
+ const header = Buffer.alloc(HEADER_LEN);
+ const tag = Buffer.alloc(TAG_LEN);
+ const fd = openSync(srcPath, 'r');
+ try {
+ if (
+ readSync(fd, header, 0, HEADER_LEN, 0) !== HEADER_LEN ||
+ readSync(fd, tag, 0, TAG_LEN, srcSize - TAG_LEN) !== TAG_LEN
+ ) {
+ throw new Error('Encrypted file is truncated or not a FreeCrawl encrypted project.');
+ }
+ } finally {
+ closeSync(fd);
+ }
+ if (!header.subarray(0, MAGIC.length).equals(MAGIC)) {
throw new Error('File is not a FreeCrawl encrypted project (bad magic).');
}
- const version = buf[MAGIC.length];
+ const version = header[MAGIC.length];
if (version !== VERSION) {
throw new Error(`Unsupported encrypted-project version: 0x${version?.toString(16) ?? '??'}.`);
}
- const salt = buf.subarray(MAGIC.length + 1, MAGIC.length + 1 + SALT_LEN);
- const iv = buf.subarray(
+ const salt = header.subarray(MAGIC.length + 1, MAGIC.length + 1 + SALT_LEN);
+ const iv = header.subarray(
MAGIC.length + 1 + SALT_LEN,
MAGIC.length + 1 + SALT_LEN + IV_LEN,
);
- const header = buf.subarray(0, HEADER_LEN);
- const tag = buf.subarray(buf.length - TAG_LEN);
- const ciphertext = buf.subarray(HEADER_LEN, buf.length - TAG_LEN);
const key = pbkdf2Sync(password, salt, PBKDF2_ITERATIONS, KEY_LEN, 'sha256');
const decipher = createDecipheriv('aes-256-gcm', key, iv);
decipher.setAAD(header);
decipher.setAuthTag(tag);
- let plaintext: Buffer;
+
+ const ciphertextLen = srcSize - HEADER_LEN - TAG_LEN;
+ if (ciphertextLen === 0) {
+ // Degenerate empty-payload container — still verify the tag.
+ try {
+ decipher.final();
+ } catch {
+ throw new Error(
+ 'Could not decrypt — the password is wrong, or the file has been altered.',
+ );
+ }
+ writeFileSync(dstPath, Buffer.alloc(0));
+ return { bytesWritten: 0 };
+ }
+
+ const tmpPath = `${dstPath}.decrypt-tmp`;
try {
- plaintext = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+ await pipeline(
+ createReadStream(srcPath, {
+ start: HEADER_LEN,
+ end: srcSize - TAG_LEN - 1, // `end` is inclusive
+ highWaterMark: STREAM_HIGH_WATER_MARK,
+ }),
+ decipher,
+ createWriteStream(tmpPath),
+ );
} catch {
- // GCM auth failure — wrong password or corrupted ciphertext. Don't
- // leak which one; surface a single user-friendly error.
+ // GCM auth failure (decipher.final threw inside the pipeline) or
+ // an I/O error — either way, drop the partial output. Don't leak
+ // which failure it was; surface a single user-friendly error.
+ rmSync(tmpPath, { force: true });
throw new Error(
'Could not decrypt — the password is wrong, or the file has been altered.',
);
}
- writeFileSync(dstPath, plaintext);
- return { bytesWritten: plaintext.length };
+ renameSync(tmpPath, dstPath);
+ return { bytesWritten: ciphertextLen };
}
diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts
index 9e6146b..4f9af15 100644
--- a/apps/desktop/src/preload/index.ts
+++ b/apps/desktop/src/preload/index.ts
@@ -471,6 +471,7 @@ const api: FreeCrawlApi = {
onDone: (cb) => subscribe(IPC.crawlDone, cb),
onError: (cb) => subscribe(IPC.crawlError, cb),
onMenuEvent: (cb) => subscribe(IPC.menuEvent, cb),
+ editCopyNative: (): Promise => ipcRenderer.invoke(IPC.editCopyNative),
onDataChanged: (cb) => {
const listener = (): void => cb();
ipcRenderer.on(IPC.dataChanged, listener);
diff --git a/apps/desktop/src/renderer/src/App.tsx b/apps/desktop/src/renderer/src/App.tsx
index 6836b4a..888fa24 100644
--- a/apps/desktop/src/renderer/src/App.tsx
+++ b/apps/desktop/src/renderer/src/App.tsx
@@ -228,6 +228,31 @@ export function App() {
});
const off4 = window.freecrawl.onMenuEvent((event: MenuEvent) => {
switch (event) {
+ case 'edit-copy': {
+ // macOS-only round-trip (see menu.ts): the native menu eats
+ // Cmd+C before the page sees a keydown, so the Copy item
+ // forwards here instead. Re-dispatch a synthetic keydown so
+ // the existing document-level Ctrl/Cmd+C table-copy handlers
+ // (detail panel cells, broken-links cells) run unchanged; if
+ // none of them preventDefault'ed — no table selection — fall
+ // back to the stock native copy for text selections/inputs.
+ const ae = document.activeElement as HTMLElement | null;
+ const editable =
+ ae !== null &&
+ (ae.tagName === 'INPUT' || ae.tagName === 'TEXTAREA' || ae.isContentEditable);
+ if (!editable) {
+ const synthetic = new KeyboardEvent('keydown', {
+ key: 'c',
+ metaKey: true,
+ bubbles: true,
+ cancelable: true,
+ });
+ document.dispatchEvent(synthetic);
+ if (synthetic.defaultPrevented) break;
+ }
+ void window.freecrawl.editCopyNative();
+ break;
+ }
case 'toggle-sidebar':
toggleSidebar();
break;
diff --git a/apps/desktop/src/renderer/src/components/OverviewSidebar.tsx b/apps/desktop/src/renderer/src/components/OverviewSidebar.tsx
index c10e737..7d4c2d7 100644
--- a/apps/desktop/src/renderer/src/components/OverviewSidebar.tsx
+++ b/apps/desktop/src/renderer/src/components/OverviewSidebar.tsx
@@ -80,6 +80,11 @@ export function OverviewSidebar() {
overviewAppliedRef.current = seq;
setOverview(o);
}
+ } catch {
+ // Heavy aggregates no longer fall back to the main thread when
+ // the reader worker is down — the call rejects instead. Keep
+ // the previous counts; the next poll tick retries against the
+ // respawned worker.
} finally {
inFlight = false;
}
@@ -129,12 +134,17 @@ export function OverviewSidebar() {
if (bucket === lastRefetchAtRef.current) return;
lastRefetchAtRef.current = bucket;
const seq = ++overviewSeqRef.current;
- void window.freecrawl.overviewGet().then((o) => {
- if (seq > overviewAppliedRef.current) {
- overviewAppliedRef.current = seq;
- setOverview(o);
- }
- });
+ void window.freecrawl
+ .overviewGet()
+ .then((o) => {
+ if (seq > overviewAppliedRef.current) {
+ overviewAppliedRef.current = seq;
+ setOverview(o);
+ }
+ })
+ .catch(() => {
+ // Reader worker down — keep previous counts, retry next bucket.
+ });
}, [progressCrawled, progressRunning, setOverview]);
const toggle = (k: string) => {
diff --git a/apps/desktop/src/renderer/src/i18n/labels.ts b/apps/desktop/src/renderer/src/i18n/labels.ts
index 0f3a597..fd497b3 100644
--- a/apps/desktop/src/renderer/src/i18n/labels.ts
+++ b/apps/desktop/src/renderer/src/i18n/labels.ts
@@ -452,20 +452,15 @@ const TR_LABELS: Record = {
Pages: 'Sayfalar',
// VisualizationTab layout names + hints
- 'Force-Directed': 'Force-Directed',
- 'Compound spring embedder': 'Bileşik yay yerleştirici',
- 'Tree (BFS)': 'Ağaç (BFS)',
- 'Roots-to-leaves layered': 'Kök-yaprak katmanlı',
- 'Force-Directed Tree': 'Force-Directed Ağaç',
- 'Sugiyama layered DAG (dagre)': 'Sugiyama katmanlı DAG (dagre)',
- 'Radial Tree': 'Radyal Ağaç',
- 'Roots at centre, leaves outward': 'Kök merkezde, yapraklar dışa',
+ '3D Force-Directed Crawl Tree': '3D Force-Directed Tarama Ağacı',
+ 'Discovery tree in 3D — drag to rotate, scroll to zoom':
+ '3B keşif ağacı — döndürmek için sürükle, yakınlaştırmak için kaydır',
+ '3D Force-Directed Link Graph': '3D Force-Directed Bağlantı Grafiği',
+ 'Full internal link mesh in 3D': 'Tam iç bağlantı ağı, 3B',
+ 'Crawl Tree': 'Tarama Ağacı',
+ 'Left-to-right labelled discovery tree (2D)': 'Soldan sağa etiketli keşif ağacı (2D)',
'Directory Tree': 'Dizin Ağacı',
- 'Grouped by URL path segments': 'URL yol segmentlerine göre gruplu',
- Circle: 'Çember',
- 'Equal radial spacing': 'Eşit radyal boşluk',
- Concentric: 'Konsantrik',
- 'By inlinks (centre = most-linked)': 'Inlinks\'e göre (merkez = en çok linklenen)',
+ 'Grouped by URL path segments (2D)': 'URL yol segmentlerine göre gruplu (2D)',
// BrokenLinksTab column headers
Source: 'Kaynak',
diff --git a/apps/desktop/src/renderer/src/i18n/locales/en.json b/apps/desktop/src/renderer/src/i18n/locales/en.json
index 9ba768d..1db73a8 100644
--- a/apps/desktop/src/renderer/src/i18n/locales/en.json
+++ b/apps/desktop/src/renderer/src/i18n/locales/en.json
@@ -944,6 +944,7 @@
"color": "Color:",
"nodes": "Nodes:",
"labels": "Labels:",
+ "byCrawl": "Crawl Diagram (Depth × Indexability)",
"byStatus": "By Status",
"byDepth": "By Depth",
"byIndexability": "By Indexability",
@@ -969,7 +970,8 @@
"exportSvg": "SVG (vector — Illustrator/Figma)",
"exportHtml": "Standalone HTML (shareable)",
"nodesEdgesCount": "{{nodes}} nodes · {{edges}} edges",
- "interactionHint": "Hover = neighbours · Click = select · Empty click = clear · Double-click node = open · Double-click canvas = fit",
+ "interactionHint": "Hover = summary · Click branch = collapse/expand · Double-click node = open · Double-click canvas = fit",
+ "interactionHint3d": "Drag = rotate · Scroll = zoom · Right-drag = pan · Click = fly to node · Right-click = open",
"loadingGraph": "Loading graph…",
"emptyGraph": "No URLs crawled yet — start a crawl to populate the graph.",
"topAnchorTexts": "Top Anchor Texts",
@@ -988,7 +990,18 @@
"changesNeedRerun": "Some changes need a layout re-run.",
"rerunLayout": "Re-run layout",
"title": "Visualization",
- "occurrences": "{{count}} occurrences"
+ "occurrences": "{{count}} occurrences",
+ "cardTitle": "Page Title",
+ "cardResponse": "Response Code",
+ "cardIndexability": "Indexability",
+ "cardH1": "H1",
+ "cardH2Count": "H2 Count",
+ "cardDepth": "Crawl Depth",
+ "cardInlinks": "Unique Inlinks",
+ "cardOutlinks": "Unique Outlinks",
+ "cardFollowed": "Followed Outlinks",
+ "cardPctTotal": "% of Total",
+ "cardWordCount": "Word Count"
},
"imagesTab": {
"searchPlaceholder": "Search image URLs / alt…",
diff --git a/apps/desktop/src/renderer/src/i18n/locales/tr.json b/apps/desktop/src/renderer/src/i18n/locales/tr.json
index 47096a7..fa5aef3 100644
--- a/apps/desktop/src/renderer/src/i18n/locales/tr.json
+++ b/apps/desktop/src/renderer/src/i18n/locales/tr.json
@@ -944,6 +944,7 @@
"color": "Renk:",
"nodes": "Düğüm:",
"labels": "Etiket:",
+ "byCrawl": "Tarama Diyagramı (Derinlik × İndekslenebilirlik)",
"byStatus": "Duruma Göre",
"byDepth": "Derinliğe Göre",
"byIndexability": "İndekslenebilirliğe Göre",
@@ -969,7 +970,8 @@
"exportSvg": "SVG (vektör — Illustrator/Figma)",
"exportHtml": "Standalone HTML (paylaşılabilir)",
"nodesEdgesCount": "{{nodes}} düğüm · {{edges}} kenar",
- "interactionHint": "Hover = komşular · Tıkla = seç · Boş tıklama = temizle · Çift tık düğüm = aç · Çift tık tuval = sığdır",
+ "interactionHint": "Üzerine gel = özet · Dala tıkla = kapat/aç · Çift tık düğüm = aç · Çift tık tuval = sığdır",
+ "interactionHint3d": "Sürükle = döndür · Kaydır = yakınlaştır · Sağ-sürükle = kaydır · Tıkla = düğüme uç · Sağ tık = aç",
"loadingGraph": "Grafik yükleniyor…",
"emptyGraph": "Henüz crawl edilmiş URL yok — grafiği doldurmak için bir crawl başlat.",
"topAnchorTexts": "En Çok Anchor Metinleri",
@@ -988,7 +990,18 @@
"changesNeedRerun": "Bazı değişiklikler layout yeniden çalıştırması gerektirir.",
"rerunLayout": "Layout'u yeniden çalıştır",
"title": "Görselleştirme",
- "occurrences": "{{count}} kez geçiyor"
+ "occurrences": "{{count}} kez geçiyor",
+ "cardTitle": "Sayfa Başlığı",
+ "cardResponse": "Yanıt Kodu",
+ "cardIndexability": "İndekslenebilirlik",
+ "cardH1": "H1",
+ "cardH2Count": "H2 Sayısı",
+ "cardDepth": "Tarama Derinliği",
+ "cardInlinks": "Benzersiz İç Bağlantı",
+ "cardOutlinks": "Benzersiz Dış Bağlantı",
+ "cardFollowed": "Takip Edilen Bağlantı",
+ "cardPctTotal": "Toplamın %'si",
+ "cardWordCount": "Kelime Sayısı"
},
"imagesTab": {
"searchPlaceholder": "Görsel URL / alt ara…",
diff --git a/apps/desktop/src/renderer/src/tabs/VisualizationTab.tsx b/apps/desktop/src/renderer/src/tabs/VisualizationTab.tsx
index 2105fe1..7bd72e6 100644
--- a/apps/desktop/src/renderer/src/tabs/VisualizationTab.tsx
+++ b/apps/desktop/src/renderer/src/tabs/VisualizationTab.tsx
@@ -1,7 +1,8 @@
-import { useEffect, useRef, useState } from 'react';
+import { Fragment, useEffect, useRef, useState } from 'react';
import { RefreshCw, Sparkles, Settings2, RotateCcw, Download, Route } from 'lucide-react';
import cytoscape, { type Core } from 'cytoscape';
import dagre from 'cytoscape-dagre';
+import ForceGraph3D, { type ForceGraph3DInstance } from '3d-force-graph';
import { useTranslation } from 'react-i18next';
import type {
AnchorTextRow,
@@ -76,11 +77,14 @@ function downloadBlob(data: Blob, filename: string): void {
}, 5_000);
}
+// Matches the on-screen canvas colour so exports look identical.
+const CANVAS_BG = '#2F2F2F';
+
function exportPng(cy: Core, filename = 'freecrawl-graph.png'): void {
const dataUrl = cy.png({
output: 'base64uri',
full: true,
- bg: '#0a0a0f',
+ bg: CANVAS_BG,
scale: 2,
});
void fetch(dataUrl)
@@ -88,6 +92,15 @@ function exportPng(cy: Core, filename = 'freecrawl-graph.png'): void {
.then((b) => downloadBlob(b, filename));
}
+/** Snapshot the 3D view's WebGL canvas (renderer created with
+ * preserveDrawingBuffer so the backbuffer is readable). */
+function exportPng3d(fg: ForceGraph3DInstance, filename = 'freecrawl-graph-3d.png'): void {
+ const canvas = fg.renderer().domElement;
+ canvas.toBlob((b: Blob | null) => {
+ if (b) downloadBlob(b, filename);
+ }, 'image/png');
+}
+
function exportSvg(cy: Core, filename = 'freecrawl-graph.svg'): void {
const bbox = cy.elements().boundingBox({});
const pad = 40;
@@ -99,11 +112,11 @@ function exportSvg(cy: Core, filename = 'freecrawl-graph.svg'): void {
parts.push(
``);
@@ -140,7 +153,7 @@ function exportStandaloneHtml(
FreeCrawl Graph