feat: visualization + serp tabs, column show/hide, bulk export expansion, mcp bridge fix

This commit is contained in:
kemalai committed 2026-05-15 22:07:05 +03:00
1 parent 5d6ea5256f
commit 1497ec7231
19 files changed
+961 -445

No files matched your search

+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/cli",
"version": "0.3.7",
"version": "0.4.0",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/desktop",
"version": "0.3.7",
"version": "0.4.0",
"private": true,
"description": "FreeCrawl SEO — desktop application",
"main": "./out/main/index.js",
+174 -2
View File
@@ -2545,6 +2545,178 @@ function registerIpc(): void {
file: 'issues-near-duplicate.csv',
category: 'issues:near-duplicate',
},
// V1 Faz 4 — Redirects family.
{ label: 'All Redirects', file: 'all-redirects.csv', category: 'tab:redirects' },
{
label: 'Redirect Chains (Long)',
file: 'redirect-chains-long.csv',
category: 'issues:redirect-chain-long',
},
{ label: 'Redirect Loops', file: 'redirect-loops.csv', category: 'issues:redirect-loop' },
{
label: 'Self-Redirects',
file: 'redirect-self.csv',
category: 'issues:redirect-self',
},
// V1 Faz 4 — Canonical family.
{ label: 'All Canonicals', file: 'all-canonicals.csv', category: 'tab:canonicals' },
{
label: 'Canonical — Non Self-Referencing',
file: 'canonical-non-self.csv',
category: 'issues:canonical-non-self',
},
{
label: 'Canonical — To Non-200',
file: 'canonical-to-non-200.csv',
category: 'issues:canonical-to-non-200',
},
{
label: 'Canonical — To Redirect',
file: 'canonical-to-redirect.csv',
category: 'issues:canonical-to-redirect',
},
{
label: 'Canonical — To Noindex',
file: 'canonical-to-noindex.csv',
category: 'issues:canonical-to-noindex',
},
{
label: 'Canonical — Multi-Hop Chain',
file: 'canonical-chain-multi-hop.csv',
category: 'issues:canonical-chain-multi-hop',
},
{
label: 'Multiple Canonicals',
file: 'canonical-multiple.csv',
category: 'issues:multiple-canonicals',
},
// V1 Faz 4 — Pagination family.
{ label: 'All Pagination', file: 'all-pagination.csv', category: 'tab:pagination' },
{
label: 'Pagination — Sequence Break',
file: 'pagination-sequence-break.csv',
category: 'issues:pagination-sequence-break',
},
{
label: 'Pagination — Canonical Conflict',
file: 'pagination-canonical-conflict.csv',
category: 'issues:pagination-canonical-conflict',
},
// V1 Faz 4 — Hreflang family (7 variants).
{ label: 'Hreflang — All', file: 'hreflang-all.csv', category: 'tab:hreflang' },
{
label: 'Hreflang — x-default Missing',
file: 'hreflang-x-default-missing.csv',
category: 'issues:hreflang-x-default-missing',
},
{
label: 'Hreflang — Invalid Language Code',
file: 'hreflang-invalid-code.csv',
category: 'issues:hreflang-invalid-code',
},
{
label: 'Hreflang — Self-Ref Missing',
file: 'hreflang-self-ref-missing.csv',
category: 'issues:hreflang-self-ref-missing',
},
{
label: 'Hreflang — Target Issues',
file: 'hreflang-target-issues.csv',
category: 'issues:hreflang-target-issues',
},
{
label: 'Hreflang — Inconsistent Language',
file: 'hreflang-inconsistent-lang.csv',
category: 'issues:hreflang-inconsistent-lang',
},
// V1 Faz 4 — Duplicates (cluster + exact).
{ label: 'Duplicate Pages', file: 'duplicate-pages.csv', category: 'tab:duplicates' },
{
label: 'Duplicate Content — Exact',
file: 'duplicate-content-exact.csv',
category: 'issues:duplicate-content-exact',
},
// V1 Faz 4 — Custom data exports.
{
label: 'Custom Extraction Results',
file: 'custom-extraction.csv',
category: 'tab:custom-extraction',
},
{ label: 'Custom Search Hits', file: 'custom-search.csv', category: 'tab:custom-search' },
// V1 Faz 4 — Structured data validation errors.
{
label: 'Structured Data — Missing',
file: 'structured-data-missing.csv',
category: 'issues:structured-data-missing',
},
{
label: 'Structured Data — Invalid',
file: 'structured-data-invalid.csv',
category: 'issues:structured-data-invalid',
},
{
label: 'Structured Data — Duplicate @id',
file: 'structured-data-duplicate-id.csv',
category: 'issues:schema-duplicate-id',
},
{
label: 'Structured Data — Missing Required Property',
file: 'structured-data-missing-required.csv',
category: 'issues:schema-missing-required',
},
// V1 Faz 4 — AMP.
{ label: 'AMP Pages', file: 'amp-pages.csv', category: 'tab:amp' },
// V1 Faz 4 — Image variants.
{
label: 'Image — Missing Alt',
file: 'image-missing-alt.csv',
category: 'issues:image-missing-alt',
},
{
label: 'Image — Empty Alt',
file: 'image-empty-alt.csv',
category: 'issues:image-empty-alt',
},
{
label: 'Image — Too Large',
file: 'image-too-large.csv',
category: 'issues:image-too-large',
},
{
label: 'Image — Broken Src',
file: 'image-broken-src.csv',
category: 'issues:image-broken-src',
},
// V1 Faz 4 — Security.
{ label: 'Security Audit', file: 'security-audit.csv', category: 'tab:security' },
{
label: 'Mixed Content — Active',
file: 'mixed-content-active.csv',
category: 'issues:mixed-content-active',
},
{
label: 'Mixed Content — Passive',
file: 'mixed-content-passive.csv',
category: 'issues:mixed-content-passive',
},
{
label: 'HSTS Missing',
file: 'security-hsts-missing.csv',
category: 'issues:hsts-missing',
},
{
label: 'CSP Missing',
file: 'security-csp-missing.csv',
category: 'issues:csp-missing',
},
// V1 Faz 4 — SERP summary.
{ label: 'SERP Summary', file: 'serp-summary.csv', category: 'tab:serp' },
// V1 Faz 4 — Outlinks Zero (dead-end pages: no outbound links).
{
label: 'Outlinks Zero',
file: 'outlinks-zero.csv',
category: 'issues:outlinks-zero',
},
];
const files: BulkExportFile[] = [];
const errors: { label: string; error: string }[] = [];
@@ -2790,7 +2962,7 @@ if (!gotSingleInstanceLock) {
app.commandLine.appendSwitch('disable-gpu-disk-cache');
app.commandLine.appendSwitch('disable-features', 'HardwareMediaKeyHandling');
void app.whenReady().then(() => {
void app.whenReady().then(async () => {
// Disk logging is bootstrapped after `app.whenReady()` — `getPath('userData')`
// is only valid post-ready. Prior log lines are still in the ring buffer
// and will be flushed to disk lazily via subsequent writes.
@@ -2837,7 +3009,7 @@ void app.whenReady().then(() => {
// failure is logged but non-fatal: the desktop app keeps working,
// only MCP-driven crawl control is disabled.
try {
const bridge = startMcpBridge({
const bridge = await startMcpBridge({
userDataDir: app.getPath('userData'),
startCrawl: async (input: McpStartCrawlInput) => {
if (!crawlController) {
+49 -14
View File
@@ -118,12 +118,21 @@ let server: Server | null = null;
let discoveryPath: string | null = null;
/**
* Open the localhost HTTP bridge and write the discovery file. Returns
* `null` if it could not start (already running, permission denied,
* etc.) — non-fatal: the desktop app keeps running, MCP just won't be
* able to drive a crawl.
* Open the localhost HTTP bridge and write the discovery file. Resolves
* with `null` if it could not start (already running, permission denied,
* listen error, etc.) — non-fatal: the desktop app keeps running, MCP
* just won't be able to drive a crawl.
*
* Async because `server.listen()` is asynchronous: the OS only assigns
* the ephemeral port after the kernel binds the socket, and calling
* `server.address()` synchronously right after `.listen()` returns
* `null`. We resolve on the first `listening` event (or reject-style
* `null` on `error`) so the discovery file always reflects the real
* bound port.
*/
export function startMcpBridge(deps: McpBridgeDeps): { port: number; token: string } | null {
export async function startMcpBridge(
deps: McpBridgeDeps,
): Promise<{ port: number; token: string } | null> {
if (server) return null; // already started
const token = randomBytes(32).toString('hex');
@@ -141,21 +150,47 @@ export function startMcpBridge(deps: McpBridgeDeps): { port: number; token: stri
});
};
try {
server = createServer(handler);
// listen(0, '127.0.0.1') — random ephemeral port on loopback only.
server.listen(0, '127.0.0.1');
} catch (err) {
log('error', `mcp-bridge listen failed: ${err instanceof Error ? err.message : String(err)}`);
const localServer = createServer(handler);
server = localServer;
// listen(0, '127.0.0.1') — random ephemeral port on loopback only.
// The 'listening' event is what tells us the kernel has actually
// bound the socket; only then does `address()` return the assigned
// port. Pair with a one-shot 'error' listener so a bind failure
// (port-exhaustion, permission, etc.) resolves to `null` instead of
// hanging the promise.
const listened = await new Promise<boolean>((resolve) => {
const onError = (err: Error) => {
log('error', `mcp-bridge listen failed: ${err.message}`);
localServer.off('listening', onListening);
resolve(false);
};
const onListening = () => {
localServer.off('error', onError);
resolve(true);
};
localServer.once('error', onError);
localServer.once('listening', onListening);
try {
localServer.listen(0, '127.0.0.1');
} catch (err) {
log('error', `mcp-bridge listen threw: ${err instanceof Error ? err.message : String(err)}`);
localServer.off('listening', onListening);
localServer.off('error', onError);
resolve(false);
}
});
if (!listened) {
server = null;
return null;
}
const addr = server.address();
const addr = localServer.address();
if (!addr || typeof addr === 'string') {
log('error', 'mcp-bridge: invalid address after listen');
try {
server.close();
localServer.close();
} catch {
/* ignore */
}
@@ -181,7 +216,7 @@ export function startMcpBridge(deps: McpBridgeDeps): { port: number; token: stri
} catch (err) {
log('error', `mcp-bridge: failed to write discovery file: ${err instanceof Error ? err.message : String(err)}`);
try {
server.close();
localServer.close();
} catch {
/* ignore */
}
+1 -1
View File
@@ -135,7 +135,7 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
},
{ type: 'separator' },
{
label: 'Visualization…',
label: 'Show Visualization Tab',
accelerator: 'CmdOrCtrl+G',
click: () => send('open-visualization'),
},
+10 -8
View File
@@ -10,10 +10,11 @@ import { SitemapValidatorDialog } from './components/SitemapValidatorDialog.js';
import { ReportsDialog } from './components/ReportsDialog.js';
import { SettingsDialog } from './components/SettingsDialog.js';
import { CompareDialog } from './components/CompareDialog.js';
import { VisualizationDialog } from './components/VisualizationDialog.js';
import { UrlsTab } from './tabs/UrlsTab.js';
import { ImagesTab } from './tabs/ImagesTab.js';
import { BrokenLinksTab } from './tabs/BrokenLinksTab.js';
import { SerpTab } from './tabs/SerpTab.js';
import { VisualizationTab } from './tabs/VisualizationTab.js';
import { useAppStore } from './store.js';
import type { MenuEvent } from '@freecrawl/shared-types';
import { clearCrawlWithConfirm } from './utils/clearCrawl.js';
@@ -32,11 +33,11 @@ export function App() {
const settingsOpen = useAppStore((s) => s.settingsOpen);
const setSettingsOpen = useAppStore((s) => s.setSettingsOpen);
const setConfig = useAppStore((s) => s.setConfig);
const setActiveTab = useAppStore((s) => s.setActiveTab);
const [robotsTesterOpen, setRobotsTesterOpen] = useState(false);
const [sitemapValidatorOpen, setSitemapValidatorOpen] = useState(false);
const [reportsOpen, setReportsOpen] = useState(false);
const [compareOpen, setCompareOpen] = useState(false);
const [visualizationOpen, setVisualizationOpen] = useState(false);
const [dropFlash, setDropFlash] = useState<string | null>(null);
// Drag & drop URL list — drop a `.txt` / `.csv` of URLs anywhere on the
@@ -275,7 +276,7 @@ export function App() {
void window.freecrawl.projectSaveAs();
break;
case 'open-visualization':
setVisualizationOpen(true);
setActiveTab('visualization');
break;
}
});
@@ -295,6 +296,7 @@ export function App() {
reset,
bumpDataVersion,
setSettingsOpen,
setActiveTab,
]);
return (
@@ -325,11 +327,15 @@ export function App() {
<ImagesTab />
) : activeTab === 'broken-links' ? (
<BrokenLinksTab />
) : activeTab === 'serp' ? (
<SerpTab />
) : activeTab === 'visualization' ? (
<VisualizationTab />
) : (
<UrlsTab />
)}
</Panel>
{detailPanelOpen && (
{detailPanelOpen && activeTab !== 'visualization' && (
<>
<PanelResizeHandle className="group relative h-1.5 bg-surface-800 transition-colors hover:bg-accent-500/60 data-[resize-handle-state=drag]:bg-accent-500">
<div className="absolute inset-x-0 -top-1 -bottom-1" />
@@ -365,10 +371,6 @@ export function App() {
<ReportsDialog open={reportsOpen} onClose={() => setReportsOpen(false)} />
<SettingsDialog open={settingsOpen} onClose={() => setSettingsOpen(false)} />
<CompareDialog open={compareOpen} onClose={() => setCompareOpen(false)} />
<VisualizationDialog
open={visualizationOpen}
onClose={() => setVisualizationOpen(false)}
/>
</div>
);
}
@@ -39,6 +39,8 @@ const TAB_CATEGORY: Record<TabKey, UrlCategory> = {
duplicates: 'tab:duplicates',
links: 'all',
'broken-links': 'all',
serp: 'tab:serp',
visualization: 'all',
};
function buildExportableTabs(): ExportTab[] {
+7 -1
View File
@@ -32,7 +32,9 @@ export type TabKey =
| 'security'
| 'duplicates'
| 'links'
| 'broken-links';
| 'broken-links'
| 'serp'
| 'visualization';
export const TAB_ORDER: { key: TabKey; label: string }[] = [
{ key: 'internal', label: 'Internal' },
@@ -59,6 +61,8 @@ export const TAB_ORDER: { key: TabKey; label: string }[] = [
{ key: 'duplicates', label: 'Duplicates' },
{ key: 'links', label: 'Links' },
{ key: 'broken-links', label: 'Broken Links' },
{ key: 'serp', label: 'SERP' },
{ key: 'visualization', label: 'Visualization' },
];
interface AppState {
@@ -245,6 +249,8 @@ function categoryForTab(tab: TabKey): UrlCategory {
return 'tab:security';
case 'duplicates':
return 'tab:duplicates';
case 'serp':
return 'tab:serp';
default:
return 'internal:html';
}
@@ -0,0 +1,262 @@
import { useEffect, useRef, useState } from 'react';
import { useVirtualizer } from '@tanstack/react-virtual';
import type { CrawlUrlRow } from '@freecrawl/shared-types';
import { useAppStore } from '../store.js';
const CARD_HEIGHT = 132;
const HEADER_HEIGHT = 32;
const STATUS_BAR_HEIGHT = 22;
const PAGE_SIZE = 2000;
const POLL_MS_RUNNING = 3000;
const POLL_MS_IDLE = 30_000;
// Google's mobile SERP truncates titles around 600 px and descriptions
// around 990 px (rendered with their UI font). We approximate with Arial
// at the same point sizes — close enough to flag over-budget snippets
// without false negatives.
const TITLE_PX_LIMIT = 600;
const DESC_PX_LIMIT = 990;
const TITLE_CHAR_HARD_CAP = 100;
const DESC_CHAR_HARD_CAP = 200;
type SortMode = 'inlinks' | 'url' | 'title-length-desc' | 'desc-length-desc';
export function SerpTab() {
const dataVersion = useAppStore((s) => s.dataVersion);
const progress = useAppStore((s) => s.progress);
const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
const selectedUrlId = useAppStore((s) => s.selectedUrlId);
const [rows, setRows] = useState<CrawlUrlRow[]>([]);
const [total, setTotal] = useState(0);
const [search, setSearch] = useState('');
const [sortMode, setSortMode] = useState<SortMode>('inlinks');
const scrollRef = useRef<HTMLDivElement>(null);
useEffect(() => {
let cancelled = false;
const load = async () => {
const sortBy =
sortMode === 'inlinks'
? 'inlinks'
: sortMode === 'url'
? 'url'
: sortMode === 'title-length-desc'
? 'titleLength'
: 'metaDescriptionLength';
const sortDir: 'asc' | 'desc' = sortMode === 'url' ? 'asc' : 'desc';
const res = await window.freecrawl.urlsQuery({
limit: PAGE_SIZE,
offset: 0,
category: 'tab:serp',
search: search || undefined,
sortBy,
sortDir,
});
if (cancelled) return;
setRows(res.rows);
setTotal(res.total);
};
void load();
const cadence = progress?.running ? POLL_MS_RUNNING : POLL_MS_IDLE;
const id = setInterval(load, cadence);
return () => {
cancelled = true;
clearInterval(id);
};
}, [search, sortMode, dataVersion, progress?.running]);
const virtualizer = useVirtualizer({
count: rows.length,
getScrollElement: () => scrollRef.current,
estimateSize: () => CARD_HEIGHT,
overscan: 12,
getItemKey: (index) => {
const r = rows[index];
return r ? r.id : `idx-${index}`;
},
});
return (
<div className="flex h-full w-full flex-col bg-surface-950">
<div
className="flex items-center gap-3 border-b border-surface-800 bg-surface-900/40 px-3"
style={{ height: HEADER_HEIGHT }}
>
<div className="text-[12px] font-semibold tracking-wide text-surface-100">
SERP Preview
</div>
<input
type="text"
value={search}
onChange={(e) => setSearch(e.target.value)}
placeholder="Filter by URL or title…"
className="h-6 w-72 rounded border border-surface-700 bg-surface-950 px-2 text-[11px] text-surface-100 placeholder-surface-500 focus:border-blue-500 focus:outline-none"
/>
<label className="flex items-center gap-1 text-[11px] text-surface-400">
Sort:
<select
className="h-6 rounded border border-surface-700 bg-surface-950 px-2 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={sortMode}
onChange={(e) => setSortMode(e.target.value as SortMode)}
>
<option value="inlinks">Inlinks (most-linked first)</option>
<option value="url">URL (alphabetical)</option>
<option value="title-length-desc">Longest title first</option>
<option value="desc-length-desc">Longest description first</option>
</select>
</label>
<div className="ml-auto text-[11px] text-surface-500">
{rows.length.toLocaleString()} of {total.toLocaleString()} indexable pages
{total > rows.length && ' (first ' + PAGE_SIZE.toLocaleString() + ')'}
</div>
</div>
<div
ref={scrollRef}
className="flex-1 overflow-y-auto bg-surface-950 px-6 py-4"
>
{rows.length === 0 && (
<div className="flex h-full items-center justify-center text-[12px] text-surface-500">
{total === 0
? 'No indexable HTML pages with a title yet — run a crawl to populate SERP previews.'
: 'No pages match the current filter.'}
</div>
)}
{rows.length > 0 && (
<div
style={{
height: virtualizer.getTotalSize(),
width: '100%',
position: 'relative',
}}
>
{virtualizer.getVirtualItems().map((vRow) => {
const row = rows[vRow.index];
if (!row) return null;
return (
<div
key={vRow.key}
style={{
position: 'absolute',
top: 0,
left: 0,
width: '100%',
height: CARD_HEIGHT,
transform: `translateY(${vRow.start}px)`,
}}
className="pb-3"
>
<SerpCard
row={row}
active={selectedUrlId === row.id}
onClick={() => setSelectedUrlId(row.id)}
/>
</div>
);
})}
</div>
)}
</div>
<div
className="flex items-center gap-3 border-t border-surface-800 bg-surface-900/40 px-3 text-[11px] text-surface-400"
style={{ height: STATUS_BAR_HEIGHT }}
>
<span>
Click a card to select that URL · Title budget {TITLE_PX_LIMIT}px · Description budget {DESC_PX_LIMIT}px
</span>
</div>
</div>
);
}
function SerpCard({
row,
active,
onClick,
}: {
row: CrawlUrlRow;
active: boolean;
onClick: () => void;
}) {
const title = row.title ?? '(no title)';
const desc = row.metaDescription ?? '';
const titlePx = row.title ? measurePixelWidth(row.title, 15) : 0;
const descPx = desc ? measurePixelWidth(desc, 13) : 0;
const titleOver = titlePx > TITLE_PX_LIMIT;
const descOver = descPx > DESC_PX_LIMIT;
return (
<button
type="button"
onClick={onClick}
className={
'group block w-full max-w-[760px] cursor-pointer rounded border bg-surface-900 px-4 py-2.5 text-left transition ' +
(active
? 'border-blue-500 ring-1 ring-blue-500/40'
: 'border-surface-800 hover:border-blue-500/60 hover:bg-surface-900/80')
}
>
<div className="mb-0.5 truncate text-[12px] text-surface-400">
{displayUrl(row.url)}
</div>
<div
className="mb-0.5 line-clamp-1 text-[18px] leading-snug text-[#8ab4f8]"
title={title}
>
{title.length > TITLE_CHAR_HARD_CAP
? title.slice(0, TITLE_CHAR_HARD_CAP) + '…'
: title}
</div>
<div
className="mb-1 line-clamp-2 text-[13px] leading-snug text-surface-300"
title={desc || '(no meta description)'}
>
{desc ? (
desc.length > DESC_CHAR_HARD_CAP ? (
desc.slice(0, DESC_CHAR_HARD_CAP) + '…'
) : (
desc
)
) : (
<span className="italic text-surface-500">
(no meta description — Google falls back to page text)
</span>
)}
</div>
<div className="flex flex-wrap items-center gap-x-4 gap-y-0.5 text-[10px] text-surface-500">
<span className={titleOver ? 'text-amber-400' : ''}>
Title: {titlePx}px / {TITLE_PX_LIMIT}px · {row.titleLength ?? 0} chars
</span>
{desc && (
<span className={descOver ? 'text-amber-400' : ''}>
Desc: {descPx}px / {DESC_PX_LIMIT}px · {row.metaDescriptionLength ?? 0} chars
</span>
)}
<span>Inlinks: {(row.inlinks ?? 0).toLocaleString()}</span>
</div>
</button>
);
}
function displayUrl(url: string): string {
try {
const u = new URL(url);
const host = u.hostname.replace(/^www\./, '');
const path = u.pathname === '/' ? '' : u.pathname;
return `${host}${path}`;
} catch {
return url;
}
}
let canvas: HTMLCanvasElement | null = null;
function measurePixelWidth(text: string, fontPx: number): number {
if (!canvas) {
canvas = document.createElement('canvas');
}
const ctx = canvas.getContext('2d');
if (!ctx) return 0;
ctx.font = `${fontPx}px Arial, sans-serif`;
return Math.round(ctx.measureText(text).width);
}
+192 -5
View File
@@ -1,6 +1,6 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useVirtualizer } from '@tanstack/react-virtual';
import { ChevronsUpDown, Download, Filter, X } from 'lucide-react';
import { ChevronsUpDown, Columns3, Download, Filter, X } from 'lucide-react';
import clsx from 'clsx';
import type { AdvancedFilter, CrawlUrlRow } from '@freecrawl/shared-types';
import { useAppStore, type TabKey } from '../store.js';
@@ -18,6 +18,7 @@ const ROW_NUM_DEFAULT_WIDTH = 56;
const ROW_NUM_KEY = '__row_num__';
const STATUS_BAR_HEIGHT = 22;
const PREFS_PREFIX = 'col-widths:';
const PREFS_HIDDEN_PREFIX = 'col-hidden:';
function loadStoredWidths(tab: TabKey): Record<string, number> {
const value = window.freecrawl.prefsGet(PREFS_PREFIX + tab);
@@ -27,6 +28,22 @@ function loadStoredWidths(tab: TabKey): Record<string, number> {
return {};
}
function loadHiddenColumns(tab: TabKey): Set<string> {
const value = window.freecrawl.prefsGet(PREFS_HIDDEN_PREFIX + tab);
if (Array.isArray(value)) {
return new Set(value.filter((v): v is string => typeof v === 'string'));
}
return new Set();
}
function saveHiddenColumns(tab: TabKey, hidden: Set<string>): void {
try {
window.freecrawl.prefsSet(PREFS_HIDDEN_PREFIX + tab, [...hidden]);
} catch {
// best-effort persistence
}
}
export function UrlsTab() {
const activeTab = useAppStore((s) => s.activeTab);
const activeCategory = useAppStore((s) => s.activeCategory);
@@ -84,12 +101,34 @@ export function UrlsTab() {
const [columnWidths, setColumnWidths] = useState<Record<string, number>>(() =>
loadStoredWidths(activeTab),
);
const [hiddenColumns, setHiddenColumnsState] = useState<Set<string>>(() =>
loadHiddenColumns(activeTab),
);
const [columnPickerOpen, setColumnPickerOpen] = useState(false);
const scrollRef = useRef<HTMLDivElement>(null);
const columns = COLUMN_SPECS[activeTab];
const allColumns = COLUMN_SPECS[activeTab];
// Visible columns drive the rendered header + body. Hidden ids that no
// longer exist on the active tab (e.g. user switched between tabs with
// different specs) are silently filtered out — they remain in the saved
// set so toggling back to the originating tab restores them.
const columns = useMemo(
() => allColumns.filter((c) => !hiddenColumns.has(columnId(c))),
[allColumns, hiddenColumns],
);
const setHiddenColumns = useCallback(
(next: Set<string>) => {
setHiddenColumnsState(next);
saveHiddenColumns(activeTab, next);
},
[activeTab],
);
useEffect(() => {
setColumnWidths(loadStoredWidths(activeTab));
setHiddenColumnsState(loadHiddenColumns(activeTab));
setColumnPickerOpen(false);
}, [activeTab]);
useEffect(() => {
@@ -99,7 +138,7 @@ export function UrlsTab() {
selectionAnchor.current = null;
cellAnchor.current = null;
dragRef.current = null;
}, [activeCategory, activeTab, search, sortBy, sortDir]);
}, [activeCategory, activeTab, search, sortBy, sortDir, hiddenColumns]);
// Clear the drag flag on any mouseup anywhere — without this, releasing
// the button outside a cell would leave `dragRef` dangling and the next
@@ -518,12 +557,51 @@ export function UrlsTab() {
<X className="h-3 w-3" />
</button>
)}
<div className="relative ml-auto">
<button
type="button"
data-columns-anchor="1"
onClick={() => setColumnPickerOpen((v) => !v)}
disabled={allColumns.length === 0}
className={clsx(
'inline-flex items-center gap-1.5 rounded border px-2 py-1 text-[11px] transition',
allColumns.length === 0
? 'cursor-not-allowed border-surface-800 text-surface-600'
: columnPickerOpen
? 'border-accent-500 bg-accent-500/15 text-accent-300'
: hiddenColumns.size > 0
? 'border-accent-500/60 bg-accent-500/10 text-accent-300 hover:bg-accent-500/20'
: 'border-surface-700 text-surface-300 hover:bg-surface-800',
)}
title={
allColumns.length === 0
? 'This tab has no toggleable columns'
: 'Show/hide columns'
}
>
<Columns3 className="h-3.5 w-3.5" />
<span>Columns</span>
{hiddenColumns.size > 0 && (
<span className="rounded bg-accent-500/20 px-1 font-mono text-[10px]">
{allColumns.length - hiddenColumns.size}/{allColumns.length}
</span>
)}
</button>
{columnPickerOpen && (
<ColumnPickerPopover
allColumns={allColumns}
hiddenColumns={hiddenColumns}
onChange={setHiddenColumns}
onClose={() => setColumnPickerOpen(false)}
/>
)}
</div>
<button
type="button"
onClick={() => setExportDialogOpen(true)}
disabled={lazy.total === 0}
className={clsx(
'ml-auto inline-flex items-center gap-1.5 rounded border px-2 py-1 text-[11px] transition',
'inline-flex items-center gap-1.5 rounded border px-2 py-1 text-[11px] transition',
lazy.total === 0
? 'cursor-not-allowed border-surface-800 text-surface-600'
: 'border-surface-700 text-surface-300 hover:bg-surface-800',
@@ -587,7 +665,14 @@ export function UrlsTab() {
onMouseEnter={() => {
if (dragRef.current?.kind === 'column') applyColumnDrag(colIdx);
}}
title={c.header + ' (click to select column · drag to select multiple · click arrow to sort)'}
onContextMenu={(e) => {
e.preventDefault();
e.stopPropagation();
const next = new Set(hiddenColumns);
next.add(columnId(c));
setHiddenColumns(next);
}}
title={c.header + ' (click to select · drag to multi-select · right-click to hide)'}
>
<span className="cursor-pointer truncate">{c.header}</span>
{(c.info || c.example) && (
@@ -841,6 +926,108 @@ export function UrlsTab() {
);
}
function ColumnPickerPopover({
allColumns,
hiddenColumns,
onChange,
onClose,
}: {
allColumns: ColumnSpec[];
hiddenColumns: Set<string>;
onChange: (next: Set<string>) => void;
onClose: () => void;
}) {
const popRef = useRef<HTMLDivElement | null>(null);
// Outside-click closes the popover. We attach to mousedown so the
// listener fires before any click handler inside the table can fire
// and reopen us in the same gesture. Clicks on the anchoring "Columns"
// button are exempted via the `data-columns-anchor` guard.
useEffect(() => {
const onDocDown = (e: MouseEvent) => {
if (!popRef.current) return;
if (popRef.current.contains(e.target as Node)) return;
let n = e.target as HTMLElement | null;
while (n) {
if (n.dataset && n.dataset['columnsAnchor'] === '1') return;
n = n.parentElement;
}
onClose();
};
document.addEventListener('mousedown', onDocDown);
return () => document.removeEventListener('mousedown', onDocDown);
}, [onClose]);
// Escape closes too — keeps the popover keyboard-accessible.
useEffect(() => {
const onKey = (e: KeyboardEvent) => {
if (e.key === 'Escape') onClose();
};
window.addEventListener('keydown', onKey);
return () => window.removeEventListener('keydown', onKey);
}, [onClose]);
const toggle = (id: string) => {
const next = new Set(hiddenColumns);
if (next.has(id)) next.delete(id);
else next.add(id);
onChange(next);
};
const visibleCount = allColumns.length - hiddenColumns.size;
return (
<div
ref={popRef}
className="absolute right-0 top-full z-30 mt-1 w-80 rounded-md border border-surface-700 bg-surface-900 shadow-2xl"
onClick={(e) => e.stopPropagation()}
>
<div className="flex items-center justify-between border-b border-surface-800 px-3 py-2">
<div className="text-[12px] font-semibold text-surface-100">
Columns ({visibleCount}/{allColumns.length})
</div>
<div className="flex items-center gap-1 text-[10px]">
<button
type="button"
onClick={() => onChange(new Set())}
className="rounded border border-surface-700 px-1.5 py-0.5 text-surface-300 hover:bg-surface-800"
title="Make every column visible"
>
Show all
</button>
<button
type="button"
onClick={() => onChange(new Set(allColumns.map((c) => columnId(c))))}
className="rounded border border-surface-700 px-1.5 py-0.5 text-surface-300 hover:bg-surface-800"
title="Hide every column except Row #"
>
Hide all
</button>
</div>
</div>
<div className="max-h-[60vh] overflow-y-auto py-1">
{allColumns.map((c) => {
const id = columnId(c);
const visible = !hiddenColumns.has(id);
return (
<label
key={id}
className="flex cursor-pointer items-center gap-2 px-3 py-1 text-[11px] text-surface-200 hover:bg-surface-800"
>
<input
type="checkbox"
checked={visible}
onChange={() => toggle(id)}
className="h-3 w-3 accent-blue-500"
/>
<span className="truncate">{c.header}</span>
</label>
);
})}
</div>
</div>
);
}
function Cell({ row, spec }: { row: CrawlUrlRow | null; spec: ColumnSpec }) {
if (row === null) {
return <span className="text-surface-700">…</span>;
@@ -1,6 +1,12 @@
import { useEffect, useRef, useState } from 'react';
import { X, RefreshCw, Sparkles, Settings2, RotateCcw, Download } from 'lucide-react';
import { RefreshCw, Sparkles, Settings2, RotateCcw, Download } from 'lucide-react';
import cytoscape, { type Core } from 'cytoscape';
import type {
AnchorTextRow,
GraphSnapshotResult,
Indexability,
} from '@freecrawl/shared-types';
import { useAppStore } from '../store.js';
/**
* User-tunable layout knobs. Persisted in prefs under the
@@ -43,13 +49,6 @@ function loadTuning(): VisTuning {
return DEFAULT_TUNING;
}
/**
* Trigger a download of `data` as `filename`. Anchor-click pattern
* works in Electron's renderer because chromium honours the
* `download` attribute and the in-app save flow doesn't need shell
* permissions for a Blob URL. URL is revoked after a short delay so
* Chromium has time to attach the download.
*/
function downloadBlob(data: Blob, filename: string): void {
const url = URL.createObjectURL(data);
const a = document.createElement('a');
@@ -64,36 +63,18 @@ function downloadBlob(data: Blob, filename: string): void {
}, 5_000);
}
/**
* Export the current Cytoscape canvas as a high-DPI PNG. Cytoscape's
* `cy.png()` rasterises into the requested pixel dimensions (we use 2×
* the visible size for retina-quality print) and returns a base64 data
* URL which we convert to a Blob for the file download.
*/
function exportPng(cy: Core, filename = 'freecrawl-graph.png'): void {
const dataUrl = cy.png({
output: 'base64uri',
full: true, // include nodes outside the current viewport
full: true,
bg: '#0a0a0f',
scale: 2,
});
// base64uri → Blob via fetch — cleaner than manual atob() decoding
// and supported in Chromium since forever.
void fetch(dataUrl)
.then((r) => r.blob())
.then((b) => downloadBlob(b, filename));
}
/**
* Export the current canvas as SVG. We don't pull in cytoscape-svg as
* a dependency — Cytoscape's `jpg`/`png` API has no native SVG output,
* so we emit a minimal hand-rolled SVG: `<circle>` per node + `<line>`
* per edge, using the runtime renderedPosition coordinates so the
* output matches what the user sees on screen.
*
* The result is editable in Illustrator / Inkscape / Figma — the
* primary use case for "give me an SVG of my graph".
*/
function exportSvg(cy: Core, filename = 'freecrawl-graph.svg'): void {
const bbox = cy.elements().boundingBox({});
const pad = 40;
@@ -106,7 +87,6 @@ function exportSvg(cy: Core, filename = 'freecrawl-graph.svg'): void {
`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}">`,
);
parts.push(`<rect width="100%" height="100%" fill="#0a0a0f"/>`);
// Edges first so they sit beneath nodes.
cy.edges().forEach((e) => {
const src = e.source().position();
const tgt = e.target().position();
@@ -135,17 +115,6 @@ function escapeXml(s: string): string {
.replace(/"/g, '&quot;');
}
/**
* Export the graph as a self-contained HTML file. Embeds Cytoscape
* from a CDN + the snapshot data inline; the user can open it
* directly in any browser, share via email/Slack, or attach to a
* client report. No FreeCrawl runtime needed for the recipient.
*
* Trade-off: the resulting file requires internet to load Cytoscape
* the first time (CDN fetch). We could inline the library too at the
* cost of ~250 KB per export, which is reasonable for a one-shot
* deliverable but adds 5–10× to file size on most graphs.
*/
function exportStandaloneHtml(
cy: Core,
filename = 'freecrawl-graph.html',
@@ -188,16 +157,6 @@ function exportStandaloneHtml(
</html>`;
downloadBlob(new Blob([html], { type: 'text/html' }), filename);
}
import type {
AnchorTextRow,
GraphSnapshotResult,
Indexability,
} from '@freecrawl/shared-types';
interface Props {
open: boolean;
onClose: () => void;
}
type LayoutKind = 'cose' | 'breadthfirst' | 'circle' | 'concentric';
@@ -220,7 +179,6 @@ function statusColor(code: number | null): string {
}
function depthColor(d: number): string {
// Bluescale 0-10
const palette = [
'#1e3a8a', '#1e40af', '#1d4ed8', '#2563eb', '#3b82f6',
'#60a5fa', '#93c5fd', '#bfdbfe', '#dbeafe', '#eff6ff', '#a3a3a3',
@@ -235,21 +193,16 @@ function indexColor(i: Indexability): string {
}
function nodeSize(inlinks: number, scale = 1): number {
// Log-scale + hard cap so a hub with 10K inlinks doesn't dominate the
// canvas. Baseline range is 6–24 px; the user-tunable `scale` factor
// multiplies both ends linearly so 0.5× = 3–12 px, 2× = 12–48 px.
const raw = 6 + Math.log2(inlinks + 1) * 1.8;
return Math.min(raw, 24) * scale;
}
/**
* Label-display modes. The default is `hover` because graphs >100 nodes
* become a wall of overlapping text otherwise — the previous "always-on"
* default produced a dense unreadable mass on real sites.
*/
type LabelMode = 'hover' | 'top' | 'always';
export function VisualizationDialog({ open, onClose }: Props) {
export function VisualizationTab() {
// React to data refreshes so the graph re-snapshots when a new crawl
// populates URLs (or the active project is swapped).
const dataVersion = useAppStore((s) => s.dataVersion);
const [graph, setGraph] = useState<GraphSnapshotResult | null>(null);
const [anchors, setAnchors] = useState<AnchorTextRow[]>([]);
const [layout, setLayout] = useState<LayoutKind>('cose');
@@ -276,42 +229,21 @@ export function VisualizationDialog({ open, onClose }: Props) {
return next;
});
}
/**
* Floating label overlay state — rendered as plain HTML on top of the
* Cytoscape canvas so the font size stays fixed in CSS pixels regardless
* of the user's zoom level (built-in cytoscape labels scale with zoom,
* which makes them unreadable at low zoom and giant at high zoom).
*/
const [labelOverlay, setLabelOverlay] = useState<{
text: string;
/** Container-relative pixel position of the node centre (post-zoom). */
x: number;
y: number;
/** Node radius in CSS pixels — used to push the label below the dot. */
radius: number;
} | null>(null);
// Mirror selectedUrl into a ref so cytoscape event handlers (which
// close over the initial state) can read the latest value without
// tearing down the whole event-listener stack on every selection change.
const selectedUrlRef = useRef<string | null>(null);
useEffect(() => {
selectedUrlRef.current = selectedUrl;
}, [selectedUrl]);
useEffect(() => {
if (!open) return;
const onKey = (e: KeyboardEvent) => {
if (e.key === 'Escape') onClose();
};
window.addEventListener('keydown', onKey);
return () => window.removeEventListener('keydown', onKey);
}, [open, onClose]);
useEffect(() => {
if (!open) return;
void loadGraph();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [open, nodeLimit]);
}, [nodeLimit, dataVersion]);
async function loadGraph() {
setLoading(true);
@@ -327,9 +259,8 @@ export function VisualizationDialog({ open, onClose }: Props) {
}
}
// Render Cytoscape whenever graph / layout / colorMode / labelMode change.
useEffect(() => {
if (!open || !containerRef.current || !graph) return;
if (!containerRef.current || !graph) return;
const colorFn = (n: GraphSnapshotResult['nodes'][number]) => {
if (colorMode === 'depth') return depthColor(n.depth);
@@ -337,9 +268,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
return statusColor(n.statusCode);
};
// For "Top" mode: pick the 20 most-linked nodes — they're the ones
// people actually want to see labelled (homepage, hubs, category
// landing pages). Everything else stays unlabelled by default.
const TOP_LABEL_COUNT = 20;
const topByInlinks = [...graph.nodes]
.sort((a, b) => b.inlinks - a.inlinks)
@@ -373,10 +301,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
cyRef.current = null;
}
// Label visibility selector. `hover` keeps every label hidden until
// the user mouses over (the focus class added in the hover handler
// overrides this); `top` shows only the 20 most-linked nodes;
// `always` is the legacy unreadable mode for completeness.
const baseLabelSelector =
labelMode === 'always'
? 'node'
@@ -384,23 +308,12 @@ export function VisualizationDialog({ open, onClose }: Props) {
? 'node[isTop = 1]'
: 'node.focus';
// Layout-specific spacing — the previous default cose at 200 nodes
// produced overlapping clumps. We bump nodeRepulsion + idealEdgeLength
// so the layout runner pushes nodes apart aggressively, which is the
// biggest readability lever.
const layoutCfg: Record<string, unknown> = {
name: layout,
animate: false,
padding: 30,
};
if (layout === 'cose') {
// Hub-spoke link graphs (a navbar that points to every page) collapse
// into a tight ball with default cose forces because every short
// navbar edge contracts in the same direction. Counter with extreme
// node-pair repulsion + zero gravity so edges have to overcome
// ~half-a-million units of push to bring nodes adjacent. The
// boundingBox forces a wide canvas so the layout has room to breathe
// even when the dialog is small.
layoutCfg.nodeRepulsion = () => 1_000_000 * tuning.repulsionScale;
layoutCfg.idealEdgeLength = () => 400 * tuning.edgeLengthScale;
layoutCfg.edgeElasticity = () => 20;
@@ -432,18 +345,12 @@ export function VisualizationDialog({ open, onClose }: Props) {
const cy = cytoscape({
container: containerRef.current,
elements,
// Style is loosely typed — cytoscape's TS defs don't model
// `data(...)` mapper expressions cleanly. We cast to keep the
// declarative form readable.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
style: ([
{
selector: 'node',
style: {
'background-color': 'data(color)',
// Label visibility is controlled by separate selectors below;
// base node has empty label so nothing renders unless the
// selector promotes it.
label: '',
color: '#e5e5e5',
'font-size': 9,
@@ -464,16 +371,12 @@ export function VisualizationDialog({ open, onClose }: Props) {
},
},
{
// Selector for nodes that SHOULD show their label.
selector: baseLabelSelector,
style: {
label: 'data(label)',
},
},
{
// Hover/focus highlight — bright outline, on top. Label is
// rendered via the HTML overlay so it stays at fixed CSS size
// regardless of zoom; we don't promote `label` here.
selector: 'node.focus',
style: {
'border-width': 2,
@@ -482,8 +385,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
},
},
{
// Persistent selection — same yellow border as focus but stays
// until the user explicitly clicks empty canvas to deselect.
selector: 'node.selected',
style: {
'border-width': 3,
@@ -492,7 +393,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
},
},
{
// Faded non-neighbours when a node is selected.
selector: 'node.faded',
style: {
opacity: 0.25,
@@ -536,12 +436,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
maxZoom: 4,
});
/**
* Place the floating overlay label over a node. We translate cytoscape
* model coords → rendered (canvas) coords → screen-pixel coords inside
* the container so the label sits under the dot at a fixed CSS size
* regardless of zoom.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const placeLabel = (node: any) => {
const pos = node.renderedPosition();
@@ -559,10 +453,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
const node = e.target;
setHover(String(node.data('fullUrl')));
placeLabel(node);
// Highlight the hovered node + its directly-connected neighbours;
// fade everything else so the user can read the surrounding cluster
// even on a dense canvas. We avoid touching nodes that have the
// `selected` class so a persisted selection stays visible.
cy.batch(() => {
cy.elements().not('.selected').addClass('faded');
const neighbourhood = node.closedNeighborhood();
@@ -573,8 +463,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
});
cy.on('mouseout', 'node', () => {
setHover(null);
// Don't drop the label overlay if a node is selected — keep showing
// the selected node's URL. Otherwise clear it.
const sel = cy.$('node.selected');
if (sel.length > 0) {
placeLabel(sel[0]!);
@@ -584,8 +472,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
cy.batch(() => {
cy.elements().removeClass('faded');
cy.elements().removeClass('focus');
// Re-apply faded to non-selected when there IS a selection so the
// persistent highlight stays visible.
if (sel.length > 0) {
const keep = sel[0]!.closedNeighborhood();
cy.elements().not(keep).addClass('faded');
@@ -594,10 +480,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
});
});
// Click-to-select. Persists until the user clicks empty canvas (which
// first cancels the selection) or another node. We DO NOT animate the
// viewport on selection — the previous behaviour zoomed the camera,
// which felt jarring and lost the user's mental map of the graph.
cy.on('tap', 'node', (e) => {
const node = e.target;
setSelectedUrl(String(node.data('fullUrl')));
@@ -605,8 +487,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
cy.batch(() => {
cy.elements().removeClass('selected');
node.addClass('selected');
// Fade non-neighbours so the selection's locality is obvious even
// after the mouse leaves the dot.
cy.elements().addClass('faded');
const keep = node.closedNeighborhood();
keep.removeClass('faded');
@@ -614,14 +494,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
});
});
cy.on('tap', (e) => {
// Empty-canvas click. Two-stage behaviour:
// 1) If something is selected, the click clears the selection
// (and only that — no auto zoom-fit, the user might want to
// stay where they were panning).
// 2) If nothing is selected, the click is a no-op.
// Double-click on empty canvas is the explicit "fit to all" gesture
// (handled separately below) so a stray pan-click doesn't yank the
// viewport every time.
if (e.target !== cy) return;
if (selectedUrlRef.current) {
setSelectedUrl(null);
@@ -639,7 +511,6 @@ export function VisualizationDialog({ open, onClose }: Props) {
}
});
// Keep the overlay glued to the node as the user pans / zooms.
cy.on('pan zoom render', () => {
const sel = cy.$('node.selected');
if (sel.length > 0) {
@@ -647,13 +518,9 @@ export function VisualizationDialog({ open, onClose }: Props) {
}
});
// Double-click opens the URL in the system browser — the most-asked
// affordance in graph views ("which page is this?").
cy.on('dbltap', 'node', (e) => {
const url = String(e.target.data('fullUrl') ?? '');
if (url) {
// Renderer can use window.open with target=_blank — Electron's
// setWindowOpenHandler in main routes that to shell.openExternal.
window.open(url, '_blank');
}
});
@@ -663,275 +530,251 @@ export function VisualizationDialog({ open, onClose }: Props) {
cy.destroy();
cyRef.current = null;
};
}, [open, graph, layout, colorMode, labelMode, tuning]);
if (!open) return null;
}, [graph, layout, colorMode, labelMode, tuning]);
return (
<div
className="fixed inset-0 z-30 flex items-center justify-center bg-black/60"
onClick={onClose}
>
<div
className="flex h-[88vh] w-[1280px] max-w-[98vw] flex-col overflow-hidden rounded-md border border-surface-700 bg-surface-900 shadow-2xl"
onClick={(e) => e.stopPropagation()}
>
<div className="flex items-center gap-2 border-b border-surface-800 px-4 py-2">
<div className="text-sm font-semibold tracking-wide text-surface-100">
Visualization
</div>
<div className="ml-3 flex items-center gap-2 text-[11px]">
<label className="flex items-center gap-1 text-surface-400">
Layout:
<select
className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={layout}
onChange={(e) => setLayout(e.target.value as LayoutKind)}
>
{LAYOUTS.map((l) => (
<option key={l.key} value={l.key} title={l.hint}>
{l.label}
</option>
))}
</select>
</label>
<label className="flex items-center gap-1 text-surface-400">
Color:
<select
className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={colorMode}
onChange={(e) => setColorMode(e.target.value as ColorMode)}
>
<option value="status">By Status</option>
<option value="depth">By Depth</option>
<option value="indexability">By Indexability</option>
</select>
</label>
<label className="flex items-center gap-1 text-surface-400">
Nodes:
<select
className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={String(nodeLimit)}
onChange={(e) => setNodeLimit(Number(e.target.value))}
>
<option value="50">50</option>
<option value="100">100</option>
<option value="150">150</option>
<option value="300">300</option>
<option value="500">500</option>
<option value="1000">1,000</option>
<option value="2000">2,000</option>
</select>
</label>
<label className="flex items-center gap-1 text-surface-400">
Labels:
<select
className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={labelMode}
onChange={(e) => setLabelMode(e.target.value as LabelMode)}
title="Hover = on demand · Top 20 = only the most-linked hubs · All = every node"
>
<option value="hover">Hover Only</option>
<option value="top">Top 20</option>
<option value="always">All</option>
</select>
</label>
<button
className="rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-200 hover:border-blue-500 hover:bg-surface-800"
onClick={() => cyRef.current?.fit(undefined, 30)}
title="Fit graph to view"
<div className="flex h-full w-full flex-col bg-surface-950">
<div className="flex items-center gap-2 border-b border-surface-800 bg-surface-900/40 px-3 py-1.5">
<div className="text-[12px] font-semibold tracking-wide text-surface-100">
Visualization
</div>
<div className="ml-2 flex flex-wrap items-center gap-2 text-[11px]">
<label className="flex items-center gap-1 text-surface-400">
Layout:
<select
className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={layout}
onChange={(e) => setLayout(e.target.value as LayoutKind)}
>
Fit
</button>
<button
className="flex items-center gap-1 rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-200 hover:border-blue-500 hover:bg-surface-800"
onClick={() => loadGraph()}
disabled={loading}
{LAYOUTS.map((l) => (
<option key={l.key} value={l.key} title={l.hint}>
{l.label}
</option>
))}
</select>
</label>
<label className="flex items-center gap-1 text-surface-400">
Color:
<select
className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={colorMode}
onChange={(e) => setColorMode(e.target.value as ColorMode)}
>
{loading ? (
<RefreshCw className="h-3 w-3 animate-spin" />
) : (
<RefreshCw className="h-3 w-3" />
)}
Reload
</button>
<div className="relative">
<button
data-tuning-anchor="1"
className={`flex items-center gap-1 rounded border px-2 py-1 text-[11px] ${
tunerOpen
? 'border-blue-500 bg-surface-800 text-blue-200'
: 'border-surface-700 text-surface-200 hover:border-blue-500 hover:bg-surface-800'
}`}
onClick={() => setTunerOpen((v) => !v)}
title="Layout tuning"
aria-label="Layout tuning"
>
<Settings2 className="h-3 w-3" />
</button>
{tunerOpen && (
<TuningPopover
tuning={tuning}
patch={patchTuning}
reset={() => patchTuning(DEFAULT_TUNING)}
reload={() => loadGraph()}
close={() => setTunerOpen(false)}
/>
)}
</div>
<div className="relative">
<button
className={`flex items-center gap-1 rounded border px-2 py-1 text-[11px] ${
exportMenuOpen
? 'border-blue-500 bg-surface-800 text-blue-200'
: 'border-surface-700 text-surface-200 hover:border-blue-500 hover:bg-surface-800'
}`}
onClick={() => setExportMenuOpen((v) => !v)}
title="Export graph"
aria-label="Export graph"
>
<Download className="h-3 w-3" />
Export
</button>
{exportMenuOpen && (
<div
className="absolute right-0 top-full mt-1 z-10 min-w-[200px] rounded border border-surface-700 bg-surface-900 shadow-lg"
onMouseLeave={() => setExportMenuOpen(false)}
>
<button
className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
onClick={() => {
if (cyRef.current) exportPng(cyRef.current);
setExportMenuOpen(false);
}}
>
PNG (high-DPI raster)
</button>
<button
className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
onClick={() => {
if (cyRef.current) exportSvg(cyRef.current);
setExportMenuOpen(false);
}}
>
SVG (vector — Illustrator/Figma)
</button>
<button
className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
onClick={() => {
if (cyRef.current) exportStandaloneHtml(cyRef.current);
setExportMenuOpen(false);
}}
>
Standalone HTML (shareable)
</button>
</div>
)}
</div>
</div>
<option value="status">By Status</option>
<option value="depth">By Depth</option>
<option value="indexability">By Indexability</option>
</select>
</label>
<label className="flex items-center gap-1 text-surface-400">
Nodes:
<select
className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={String(nodeLimit)}
onChange={(e) => setNodeLimit(Number(e.target.value))}
>
<option value="50">50</option>
<option value="100">100</option>
<option value="150">150</option>
<option value="300">300</option>
<option value="500">500</option>
<option value="1000">1,000</option>
<option value="2000">2,000</option>
</select>
</label>
<label className="flex items-center gap-1 text-surface-400">
Labels:
<select
className="rounded border border-surface-700 bg-surface-950 px-2 py-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
value={labelMode}
onChange={(e) => setLabelMode(e.target.value as LabelMode)}
title="Hover = on demand · Top 20 = only the most-linked hubs · All = every node"
>
<option value="hover">Hover Only</option>
<option value="top">Top 20</option>
<option value="always">All</option>
</select>
</label>
<button
className="ml-auto rounded p-1 text-surface-400 hover:bg-surface-800 hover:text-surface-100"
onClick={onClose}
title="Close (Esc)"
className="rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-200 hover:border-blue-500 hover:bg-surface-800"
onClick={() => cyRef.current?.fit(undefined, 30)}
title="Fit graph to view"
>
<X className="h-4 w-4" />
Fit
</button>
</div>
<div className="flex flex-1 min-h-0">
<div className="relative flex-1 overflow-hidden bg-surface-950">
<div ref={containerRef} className="absolute inset-0" />
{labelOverlay && (
<button
className="flex items-center gap-1 rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-200 hover:border-blue-500 hover:bg-surface-800"
onClick={() => loadGraph()}
disabled={loading}
>
{loading ? (
<RefreshCw className="h-3 w-3 animate-spin" />
) : (
<RefreshCw className="h-3 w-3" />
)}
Reload
</button>
<div className="relative">
<button
data-tuning-anchor="1"
className={`flex items-center gap-1 rounded border px-2 py-1 text-[11px] ${
tunerOpen
? 'border-blue-500 bg-surface-800 text-blue-200'
: 'border-surface-700 text-surface-200 hover:border-blue-500 hover:bg-surface-800'
}`}
onClick={() => setTunerOpen((v) => !v)}
title="Layout tuning"
aria-label="Layout tuning"
>
<Settings2 className="h-3 w-3" />
</button>
{tunerOpen && (
<TuningPopover
tuning={tuning}
patch={patchTuning}
reset={() => patchTuning(DEFAULT_TUNING)}
reload={() => loadGraph()}
close={() => setTunerOpen(false)}
/>
)}
</div>
<div className="relative">
<button
className={`flex items-center gap-1 rounded border px-2 py-1 text-[11px] ${
exportMenuOpen
? 'border-blue-500 bg-surface-800 text-blue-200'
: 'border-surface-700 text-surface-200 hover:border-blue-500 hover:bg-surface-800'
}`}
onClick={() => setExportMenuOpen((v) => !v)}
title="Export graph"
aria-label="Export graph"
>
<Download className="h-3 w-3" />
Export
</button>
{exportMenuOpen && (
<div
className="pointer-events-none absolute z-10 -translate-x-1/2 whitespace-nowrap rounded border border-amber-500/70 bg-surface-950/95 px-2 py-0.5 font-mono text-[12px] text-amber-100 shadow-lg"
style={{
// Offset the label below the node by its rendered radius
// plus a small constant so the dot and the label never
// overlap. Position is in CSS pixels — fixed regardless
// of cytoscape zoom.
left: `${labelOverlay.x}px`,
top: `${labelOverlay.y + labelOverlay.radius + 6}px`,
maxWidth: '440px',
textOverflow: 'ellipsis',
overflow: 'hidden',
}}
className="absolute right-0 top-full mt-1 z-10 min-w-[200px] rounded border border-surface-700 bg-surface-900 shadow-lg"
onMouseLeave={() => setExportMenuOpen(false)}
>
{labelOverlay.text}
</div>
)}
{graph && (
<div className="pointer-events-none absolute left-3 top-3 rounded bg-surface-900/80 px-2 py-1 text-[10px] text-surface-300">
{graph.nodes.length.toLocaleString()} nodes ·{' '}
{graph.edges.length.toLocaleString()} edges
</div>
)}
{graph && (
<div className="pointer-events-none absolute right-3 top-3 rounded bg-surface-900/80 px-2 py-1 text-[10px] text-surface-400">
Hover = neighbours · Click = select · Empty click = clear · Double-click node = open · Double-click canvas = fit
</div>
)}
{hover && (
<div className="pointer-events-none absolute bottom-3 left-3 max-w-[60%] truncate rounded bg-surface-900/90 px-2 py-1 font-mono text-[11px] text-surface-100">
{hover}
</div>
)}
{loading && !graph && (
<div className="absolute inset-0 flex items-center justify-center text-[12px] text-surface-500">
Loading graph…
<button
className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
onClick={() => {
if (cyRef.current) exportPng(cyRef.current);
setExportMenuOpen(false);
}}
>
PNG (high-DPI raster)
</button>
<button
className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
onClick={() => {
if (cyRef.current) exportSvg(cyRef.current);
setExportMenuOpen(false);
}}
>
SVG (vector — Illustrator/Figma)
</button>
<button
className="block w-full px-3 py-1.5 text-left text-[11px] text-surface-200 hover:bg-surface-800"
onClick={() => {
if (cyRef.current) exportStandaloneHtml(cyRef.current);
setExportMenuOpen(false);
}}
>
Standalone HTML (shareable)
</button>
</div>
)}
</div>
<aside className="flex w-72 flex-col border-l border-surface-800 bg-surface-950/40">
<div className="border-b border-surface-800 px-3 py-2 text-[11px] font-semibold uppercase tracking-wider text-surface-400">
<div className="flex items-center gap-1">
<Sparkles className="h-3 w-3" /> Top Anchor Texts
</div>
</div>
<div className="flex-1 overflow-auto p-2 leading-snug">
{anchors.length === 0 && (
<div className="px-2 py-3 text-[11px] italic text-surface-500">
No internal-link anchors collected yet.
</div>
)}
{anchors.length > 0 && (
<div className="flex flex-wrap items-baseline gap-2">
{anchors.map((a) => {
// Linear interpolate font size from log(count) so the
// densest term is ~2.6x the rarest.
const max = anchors[0]?.count ?? 1;
const min = anchors[anchors.length - 1]?.count ?? 1;
const range = Math.max(1, Math.log2(max) - Math.log2(min));
const frac =
(Math.log2(a.count) - Math.log2(min)) / range;
const size = 9 + frac * 13;
return (
<span
key={a.anchor}
className="text-surface-200"
style={{ fontSize: `${size}px` }}
title={`${a.count.toLocaleString()} occurrences`}
>
{a.anchor}
</span>
);
})}
</div>
)}
</div>
</aside>
</div>
</div>
<div className="flex flex-1 min-h-0">
<div className="relative flex-1 overflow-hidden bg-surface-950">
<div ref={containerRef} className="absolute inset-0" />
{labelOverlay && (
<div
className="pointer-events-none absolute z-10 -translate-x-1/2 whitespace-nowrap rounded border border-amber-500/70 bg-surface-950/95 px-2 py-0.5 font-mono text-[12px] text-amber-100 shadow-lg"
style={{
left: `${labelOverlay.x}px`,
top: `${labelOverlay.y + labelOverlay.radius + 6}px`,
maxWidth: '440px',
textOverflow: 'ellipsis',
overflow: 'hidden',
}}
>
{labelOverlay.text}
</div>
)}
{graph && (
<div className="pointer-events-none absolute left-3 top-3 rounded bg-surface-900/80 px-2 py-1 text-[10px] text-surface-300">
{graph.nodes.length.toLocaleString()} nodes ·{' '}
{graph.edges.length.toLocaleString()} edges
</div>
)}
{graph && (
<div className="pointer-events-none absolute right-3 top-3 rounded bg-surface-900/80 px-2 py-1 text-[10px] text-surface-400">
Hover = neighbours · Click = select · Empty click = clear · Double-click node = open · Double-click canvas = fit
</div>
)}
{hover && (
<div className="pointer-events-none absolute bottom-3 left-3 max-w-[60%] truncate rounded bg-surface-900/90 px-2 py-1 font-mono text-[11px] text-surface-100">
{hover}
</div>
)}
{loading && !graph && (
<div className="absolute inset-0 flex items-center justify-center text-[12px] text-surface-500">
Loading graph…
</div>
)}
{!loading && graph && graph.nodes.length === 0 && (
<div className="absolute inset-0 flex items-center justify-center text-[12px] text-surface-500">
No URLs crawled yet — start a crawl to populate the graph.
</div>
)}
</div>
<aside className="flex w-72 flex-col border-l border-surface-800 bg-surface-950/40">
<div className="border-b border-surface-800 px-3 py-2 text-[11px] font-semibold uppercase tracking-wider text-surface-400">
<div className="flex items-center gap-1">
<Sparkles className="h-3 w-3" /> Top Anchor Texts
</div>
</div>
<div className="flex-1 overflow-auto p-2 leading-snug">
{anchors.length === 0 && (
<div className="px-2 py-3 text-[11px] italic text-surface-500">
No internal-link anchors collected yet.
</div>
)}
{anchors.length > 0 && (
<div className="flex flex-wrap items-baseline gap-2">
{anchors.map((a) => {
const max = anchors[0]?.count ?? 1;
const min = anchors[anchors.length - 1]?.count ?? 1;
const range = Math.max(1, Math.log2(max) - Math.log2(min));
const frac =
(Math.log2(a.count) - Math.log2(min)) / range;
const size = 9 + frac * 13;
return (
<span
key={a.anchor}
className="text-surface-200"
style={{ fontSize: `${size}px` }}
title={`${a.count.toLocaleString()} occurrences`}
>
{a.anchor}
</span>
);
})}
</div>
)}
</div>
</aside>
</div>
</div>
);
}
/**
* Floating tuning popover anchored under the gear button. Exposes the
* five layout knobs we expose to the user via sliders. Changes apply
* live (Cytoscape re-renders on every `tuning` state change) and persist
* to prefs.
*/
function TuningPopover({
tuning,
patch,
@@ -945,17 +788,11 @@ function TuningPopover({
reload: () => void;
close: () => void;
}) {
// Close on outside click — popover is anchor-relative so a global
// click listener on `mousedown` is the standard pattern. We attach to
// mousedown rather than click so it fires before any selection or
// drag-start inside the canvas.
const popRef = useRef<HTMLDivElement | null>(null);
useEffect(() => {
const onDocDown = (e: MouseEvent) => {
if (!popRef.current) return;
if (popRef.current.contains(e.target as Node)) return;
// Don't close when clicking the gear button itself — it owns the
// toggle. Detect by walking up to find a [data-tuning-anchor] guard.
let n = e.target as HTMLElement | null;
while (n) {
if (n.dataset && n.dataset['tuningAnchor'] === '1') return;
@@ -599,11 +599,13 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
COL.indexability,
COL.inlinks,
],
// Images / Broken Links tabs have their own table layouts (separate
// components) and don't share the URL-row column schema. Entries kept
// so Record<TabKey, ...> is total.
// Images / Broken Links / SERP / Visualization tabs have their own
// layouts (separate components) and don't share the URL-row column
// schema. Entries kept so Record<TabKey, ...> is total.
images: [],
'broken-links': [],
serp: [],
visualization: [],
canonicals: [
COL.url,
COL.canonical,
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/mcp-server",
"version": "0.3.7",
"version": "0.4.0",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "freecrawl-seo",
"version": "0.3.7",
"version": "0.4.0",
"private": true,
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
"license": "MIT",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/core",
"version": "0.3.7",
"version": "0.4.0",
"private": true,
"type": "module",
"main": "./dist/index.js",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/db",
"version": "0.3.7",
"version": "0.4.0",
"private": true,
"type": "module",
"main": "./dist/index.js",
+10
View File
@@ -6190,6 +6190,16 @@ function categoryWhereClause(cat: UrlCategory): string | null {
)
)
)`;
case 'tab:serp':
// V1 Faz 10 — SERP tab. Indexable internal HTML 2xx pages with a
// title so the SERP-snippet preview always has something to render.
// Meta description is optional (Google falls back to page text), so
// we don't require it here; the empty-state copy in the SERP card
// makes the missing description visible.
return `is_external = 0 AND content_kind = 'html'
AND indexability = 'indexable'
AND status_code >= 200 AND status_code < 300
AND title IS NOT NULL AND title != ''`;
default:
return null;
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/shared-types",
"version": "0.3.7",
"version": "0.4.0",
"private": true,
"type": "module",
"main": "./dist/index.js",
+1
View File
@@ -205,6 +205,7 @@ export type UrlCategory =
| 'tab:custom-search'
| 'tab:security'
| 'tab:duplicates'
| 'tab:serp'
| 'issues:hreflang-inconsistent-lang'
| 'issues:page-many-requests'