mirror of
https://github.com/kemalai/FreeCrawl-SEO-Tool.git
synced 2026-10-18 00:39:38 +02:00
feat: FreeCrawl SEO scaffold (Electron desktop + CLI + crawler core)
This commit is contained in:
kemalai committed
commit
984c960ab7
56 files changed
+11462
No files matched your search
@@ -0,0 +1,12 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
# Dependencies
|
||||
node_modules/
|
||||
.pnp
|
||||
.pnp.js
|
||||
|
||||
# Build output
|
||||
dist/
|
||||
build/
|
||||
out/
|
||||
out-types/
|
||||
release/
|
||||
*.tsbuildinfo
|
||||
*.exe
|
||||
*.dmg
|
||||
*.AppImage
|
||||
*.deb
|
||||
*.rpm
|
||||
|
||||
# Generated config artifacts (from tsc -b on .config.ts files)
|
||||
/apps/desktop/electron.vite.config.js
|
||||
/apps/desktop/electron.vite.config.js.map
|
||||
/apps/desktop/electron.vite.config.d.ts
|
||||
/apps/desktop/electron.vite.config.d.ts.map
|
||||
|
||||
# Environment
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
|
||||
# IDE / Editor
|
||||
.vscode/
|
||||
!.vscode/settings.json
|
||||
!.vscode/extensions.json
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
desktop.ini
|
||||
|
||||
# Testing
|
||||
coverage/
|
||||
.nyc_output/
|
||||
|
||||
# Cache
|
||||
.cache/
|
||||
.parcel-cache/
|
||||
.turbo/
|
||||
|
||||
# Local crawl data
|
||||
crawls/
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
|
||||
# Claude Code
|
||||
.claude
|
||||
.claude/settings.local.json
|
||||
.claude/*.local.*
|
||||
.claude/memory/
|
||||
.claude/projects/
|
||||
.claude/todos/
|
||||
.claude/shell-snapshots/
|
||||
.claude/statsig/
|
||||
.claude/ide/
|
||||
.claude/history.jsonl
|
||||
CLAUDE.md
|
||||
@@ -0,0 +1,7 @@
|
||||
node_modules
|
||||
dist
|
||||
out
|
||||
build
|
||||
*.min.js
|
||||
package-lock.json
|
||||
.claude
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"semi": true,
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all",
|
||||
"printWidth": 100,
|
||||
"tabWidth": 2,
|
||||
"useTabs": false,
|
||||
"arrowParens": "always",
|
||||
"endOfLine": "lf"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Kemal Acar
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,15 @@
|
||||
# FreeCrawl SEO
|
||||
|
||||
Open-source desktop SEO crawler — a free alternative to Screaming Frog.
|
||||
|
||||
## About
|
||||
|
||||
FreeCrawl SEO is a cross-platform desktop application for crawling websites and analyzing SEO metrics. It aims to provide the core functionality of commercial SEO crawlers in a free and open-source package.
|
||||
|
||||
## Status
|
||||
|
||||
Early development. Features and architecture are being actively designed.
|
||||
|
||||
## License
|
||||
|
||||
MIT — see [LICENSE](LICENSE).
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"name": "@freecrawl/cli",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"freecrawl": "./dist/index.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"start": "node ./dist/index.js",
|
||||
"dev": "tsc -p tsconfig.json --watch",
|
||||
"clean": "rimraf dist"
|
||||
},
|
||||
"dependencies": {
|
||||
"@freecrawl/core": "*",
|
||||
"@freecrawl/db": "*",
|
||||
"@freecrawl/shared-types": "*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.0",
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "^5.7.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env node
|
||||
import { parseArgs } from 'node:util';
|
||||
import { resolve } from 'node:path';
|
||||
import { DEFAULT_CRAWL_CONFIG } from '@freecrawl/shared-types';
|
||||
import { Crawler, exportUrlsToCsv } from '@freecrawl/core';
|
||||
import { ProjectDb } from '@freecrawl/db';
|
||||
|
||||
function help(): void {
|
||||
console.log(`freecrawl — headless SEO crawler
|
||||
|
||||
Usage:
|
||||
freecrawl <url> [options]
|
||||
|
||||
Options:
|
||||
--depth <n> Max crawl depth (default: ${DEFAULT_CRAWL_CONFIG.maxDepth})
|
||||
--max <n> Max URLs (default: ${DEFAULT_CRAWL_CONFIG.maxUrls})
|
||||
--concurrency <n> Max parallel requests (default: ${DEFAULT_CRAWL_CONFIG.maxConcurrency})
|
||||
--rps <n> Max requests per second (default: ${DEFAULT_CRAWL_CONFIG.maxRps})
|
||||
--user-agent <str> Custom User-Agent string
|
||||
--no-robots Ignore robots.txt
|
||||
--external Follow external links
|
||||
--db <file> SQLite project file (default: ./crawl.seoproject)
|
||||
--out <file.csv> Export CSV after crawl
|
||||
-h, --help Show this help
|
||||
`);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { values, positionals } = parseArgs({
|
||||
allowPositionals: true,
|
||||
options: {
|
||||
depth: { type: 'string' },
|
||||
max: { type: 'string' },
|
||||
concurrency: { type: 'string' },
|
||||
rps: { type: 'string' },
|
||||
'user-agent': { type: 'string' },
|
||||
'no-robots': { type: 'boolean' },
|
||||
external: { type: 'boolean' },
|
||||
db: { type: 'string' },
|
||||
out: { type: 'string' },
|
||||
help: { type: 'boolean', short: 'h' },
|
||||
},
|
||||
});
|
||||
|
||||
if (values.help || positionals.length === 0) {
|
||||
help();
|
||||
process.exit(values.help ? 0 : 1);
|
||||
}
|
||||
|
||||
const startUrl = positionals[0]!;
|
||||
const dbPath = resolve(values.db ?? 'crawl.seoproject');
|
||||
const db = new ProjectDb(dbPath);
|
||||
|
||||
const config = {
|
||||
...DEFAULT_CRAWL_CONFIG,
|
||||
startUrl,
|
||||
maxDepth: parseNumeric(values.depth, DEFAULT_CRAWL_CONFIG.maxDepth),
|
||||
maxUrls: parseNumeric(values.max, DEFAULT_CRAWL_CONFIG.maxUrls),
|
||||
maxConcurrency: parseNumeric(values.concurrency, DEFAULT_CRAWL_CONFIG.maxConcurrency),
|
||||
maxRps: parseNumeric(values.rps, DEFAULT_CRAWL_CONFIG.maxRps),
|
||||
userAgent: values['user-agent'] ?? DEFAULT_CRAWL_CONFIG.userAgent,
|
||||
respectRobotsTxt: !values['no-robots'],
|
||||
crawlExternal: Boolean(values.external),
|
||||
};
|
||||
|
||||
const crawler = new Crawler(config, db);
|
||||
|
||||
crawler.on('progress', (p) => {
|
||||
process.stdout.write(
|
||||
`\r[${p.crawled}/${p.discovered}] pending=${p.pending} failed=${p.failed} @ ${p.urlsPerSecond.toFixed(1)} URL/s avg=${p.avgResponseTimeMs}ms t=${Math.round(p.elapsedMs / 1000)}s `,
|
||||
);
|
||||
});
|
||||
crawler.on('error', (msg) => {
|
||||
process.stderr.write(`\n[error] ${msg}\n`);
|
||||
});
|
||||
|
||||
await crawler.start();
|
||||
process.stdout.write('\n');
|
||||
|
||||
const summary = db.getSummary();
|
||||
console.log(`Done. Total=${summary.total} Bytes=${summary.totalBytes} AvgResp=${summary.avgResponseTimeMs}ms`);
|
||||
|
||||
if (values.out) {
|
||||
const outPath = resolve(values.out);
|
||||
const { rowsWritten } = await exportUrlsToCsv(db, outPath);
|
||||
console.log(`Wrote ${rowsWritten} rows → ${outPath}`);
|
||||
}
|
||||
|
||||
const hasErrors = Object.keys(summary.byStatus).some((k) => {
|
||||
const n = Number.parseInt(k, 10);
|
||||
return Number.isFinite(n) && n >= 400;
|
||||
});
|
||||
|
||||
db.close();
|
||||
process.exit(hasErrors ? 1 : 0);
|
||||
}
|
||||
|
||||
function parseNumeric(value: string | undefined, fallback: number): number {
|
||||
if (value === undefined) return fallback;
|
||||
const n = Number.parseInt(value, 10);
|
||||
return Number.isFinite(n) ? n : fallback;
|
||||
}
|
||||
|
||||
main().catch((err: unknown) => {
|
||||
console.error(err instanceof Error ? err.stack : err);
|
||||
process.exit(2);
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"composite": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"references": [
|
||||
{ "path": "../../packages/shared-types" },
|
||||
{ "path": "../../packages/db" },
|
||||
{ "path": "../../packages/core" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
appId: com.kemalai.freecrawl
|
||||
productName: FreeCrawl SEO
|
||||
directories:
|
||||
buildResources: resources
|
||||
output: release
|
||||
files:
|
||||
- out/**/*
|
||||
- package.json
|
||||
- "!**/*.map"
|
||||
- "!**/*.ts"
|
||||
asar: true
|
||||
asarUnpack:
|
||||
- "**/*.node"
|
||||
win:
|
||||
target:
|
||||
- target: nsis
|
||||
arch:
|
||||
- x64
|
||||
nsis:
|
||||
oneClick: false
|
||||
allowToChangeInstallationDirectory: true
|
||||
createDesktopShortcut: true
|
||||
createStartMenuShortcut: true
|
||||
mac:
|
||||
target:
|
||||
- target: dmg
|
||||
arch:
|
||||
- arm64
|
||||
- x64
|
||||
category: public.app-category.developer-tools
|
||||
linux:
|
||||
target:
|
||||
- AppImage
|
||||
- deb
|
||||
category: Development
|
||||
@@ -0,0 +1,36 @@
|
||||
import { resolve } from 'node:path';
|
||||
import { defineConfig, externalizeDepsPlugin } from 'electron-vite';
|
||||
import react from '@vitejs/plugin-react';
|
||||
|
||||
export default defineConfig({
|
||||
main: {
|
||||
plugins: [externalizeDepsPlugin()],
|
||||
build: {
|
||||
rollupOptions: {
|
||||
input: { index: resolve(__dirname, 'src/main/index.ts') },
|
||||
},
|
||||
},
|
||||
},
|
||||
preload: {
|
||||
plugins: [externalizeDepsPlugin()],
|
||||
build: {
|
||||
rollupOptions: {
|
||||
input: { index: resolve(__dirname, 'src/preload/index.ts') },
|
||||
},
|
||||
},
|
||||
},
|
||||
renderer: {
|
||||
root: resolve(__dirname, 'src/renderer'),
|
||||
resolve: {
|
||||
alias: {
|
||||
'@renderer': resolve(__dirname, 'src/renderer/src'),
|
||||
},
|
||||
},
|
||||
plugins: [react()],
|
||||
build: {
|
||||
rollupOptions: {
|
||||
input: { index: resolve(__dirname, 'src/renderer/index.html') },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "@freecrawl/desktop",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "FreeCrawl SEO — desktop application",
|
||||
"main": "./out/main/index.js",
|
||||
"scripts": {
|
||||
"dev": "electron-vite dev",
|
||||
"start": "electron-vite preview",
|
||||
"build": "electron-vite build",
|
||||
"build:win": "electron-vite build && electron-builder --win",
|
||||
"build:mac": "electron-vite build && electron-builder --mac",
|
||||
"build:linux": "electron-vite build && electron-builder --linux",
|
||||
"typecheck:node": "tsc -p tsconfig.node.json --noEmit",
|
||||
"typecheck:web": "tsc -p tsconfig.web.json --noEmit",
|
||||
"typecheck": "npm run typecheck:node && npm run typecheck:web"
|
||||
},
|
||||
"dependencies": {
|
||||
"@freecrawl/core": "*",
|
||||
"@freecrawl/db": "*",
|
||||
"@freecrawl/shared-types": "*",
|
||||
"@tanstack/react-table": "^8.20.5",
|
||||
"@tanstack/react-virtual": "^3.10.0",
|
||||
"clsx": "^2.1.1",
|
||||
"lucide-react": "^0.544.0",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
"zustand": "^5.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.2.0",
|
||||
"@types/react-dom": "^19.2.0",
|
||||
"@vitejs/plugin-react": "^5.0.0",
|
||||
"autoprefixer": "^10.4.20",
|
||||
"electron": "^41.0.0",
|
||||
"electron-builder": "^26.0.0",
|
||||
"electron-vite": "^5.0.0",
|
||||
"postcss": "^8.5.0",
|
||||
"tailwindcss": "^3.4.17",
|
||||
"typescript": "^5.7.0",
|
||||
"vite": "^7.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,144 @@
|
||||
import { app, BrowserWindow, ipcMain, dialog, shell } from 'electron';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import {
|
||||
IPC,
|
||||
type CrawlConfig,
|
||||
type CrawlProgress,
|
||||
type CrawlSummary,
|
||||
type ExportCsvInput,
|
||||
type ExportCsvResult,
|
||||
type UrlsQueryInput,
|
||||
type UrlsQueryResult,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { Crawler, exportUrlsToCsv } from '@freecrawl/core';
|
||||
import { ProjectDb } from '@freecrawl/db';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
let mainWindow: BrowserWindow | null = null;
|
||||
let db: ProjectDb | null = null;
|
||||
let activeCrawler: Crawler | null = null;
|
||||
|
||||
function getDb(): ProjectDb {
|
||||
if (!db) {
|
||||
const dataDir = join(app.getPath('userData'), 'projects');
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
db = new ProjectDb(join(dataDir, 'default.seoproject'));
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
function createWindow(): void {
|
||||
mainWindow = new BrowserWindow({
|
||||
width: 1440,
|
||||
height: 900,
|
||||
minWidth: 1024,
|
||||
minHeight: 640,
|
||||
show: false,
|
||||
autoHideMenuBar: true,
|
||||
backgroundColor: '#0a0a0a',
|
||||
title: 'FreeCrawl SEO',
|
||||
webPreferences: {
|
||||
preload: join(__dirname, '../preload/index.js'),
|
||||
sandbox: false,
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
},
|
||||
});
|
||||
|
||||
mainWindow.on('ready-to-show', () => mainWindow?.show());
|
||||
|
||||
mainWindow.webContents.setWindowOpenHandler(({ url }) => {
|
||||
void shell.openExternal(url);
|
||||
return { action: 'deny' };
|
||||
});
|
||||
|
||||
if (process.env['ELECTRON_RENDERER_URL']) {
|
||||
void mainWindow.loadURL(process.env['ELECTRON_RENDERER_URL']);
|
||||
mainWindow.webContents.openDevTools({ mode: 'detach' });
|
||||
} else {
|
||||
void mainWindow.loadFile(join(__dirname, '../renderer/index.html'));
|
||||
}
|
||||
}
|
||||
|
||||
function registerIpc(): void {
|
||||
ipcMain.handle(IPC.appVersion, () => app.getVersion());
|
||||
|
||||
ipcMain.handle(IPC.crawlStart, async (_e, config: CrawlConfig) => {
|
||||
if (activeCrawler) {
|
||||
activeCrawler.stop();
|
||||
}
|
||||
const database = getDb();
|
||||
const crawler = new Crawler(config, database);
|
||||
activeCrawler = crawler;
|
||||
|
||||
crawler.on('progress', (p: CrawlProgress) => {
|
||||
mainWindow?.webContents.send(IPC.crawlProgress, p);
|
||||
});
|
||||
crawler.on('done', (summary: CrawlSummary) => {
|
||||
mainWindow?.webContents.send(IPC.crawlDone, summary);
|
||||
if (activeCrawler === crawler) activeCrawler = null;
|
||||
});
|
||||
crawler.on('error', (msg: string) => {
|
||||
mainWindow?.webContents.send(IPC.crawlError, msg);
|
||||
});
|
||||
|
||||
void crawler.start();
|
||||
});
|
||||
|
||||
ipcMain.handle(IPC.crawlStop, () => {
|
||||
activeCrawler?.stop();
|
||||
});
|
||||
|
||||
ipcMain.handle(IPC.urlsQuery, (_e, input: UrlsQueryInput): UrlsQueryResult => {
|
||||
return getDb().queryUrls({
|
||||
limit: input.limit,
|
||||
offset: input.offset,
|
||||
filter: input.filter ?? 'all',
|
||||
search: input.search,
|
||||
sortBy: input.sortBy as string | undefined,
|
||||
sortDir: input.sortDir,
|
||||
});
|
||||
});
|
||||
|
||||
ipcMain.handle(IPC.summaryGet, (): CrawlSummary => {
|
||||
return getDb().getSummary();
|
||||
});
|
||||
|
||||
ipcMain.handle(
|
||||
IPC.exportCsv,
|
||||
async (_e, input: ExportCsvInput): Promise<ExportCsvResult> => {
|
||||
let filePath = input.filePath;
|
||||
if (!filePath) {
|
||||
const res = await dialog.showSaveDialog(mainWindow!, {
|
||||
defaultPath: 'freecrawl-export.csv',
|
||||
filters: [{ name: 'CSV', extensions: ['csv'] }],
|
||||
});
|
||||
if (res.canceled || !res.filePath) {
|
||||
return { filePath: '', rowsWritten: 0 };
|
||||
}
|
||||
filePath = res.filePath;
|
||||
}
|
||||
const { rowsWritten } = await exportUrlsToCsv(getDb(), filePath);
|
||||
return { filePath, rowsWritten };
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
void app.whenReady().then(() => {
|
||||
registerIpc();
|
||||
createWindow();
|
||||
|
||||
app.on('activate', () => {
|
||||
if (BrowserWindow.getAllWindows().length === 0) createWindow();
|
||||
});
|
||||
});
|
||||
|
||||
app.on('window-all-closed', () => {
|
||||
activeCrawler?.stop();
|
||||
db?.close();
|
||||
db = null;
|
||||
if (process.platform !== 'darwin') app.quit();
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
import { contextBridge, ipcRenderer, type IpcRendererEvent } from 'electron';
|
||||
import {
|
||||
IPC,
|
||||
type CrawlConfig,
|
||||
type CrawlProgress,
|
||||
type CrawlSummary,
|
||||
type ExportCsvInput,
|
||||
type ExportCsvResult,
|
||||
type FreeCrawlApi,
|
||||
type UrlsQueryInput,
|
||||
type UrlsQueryResult,
|
||||
} from '@freecrawl/shared-types';
|
||||
|
||||
function subscribe<T>(
|
||||
channel: string,
|
||||
cb: (payload: T) => void,
|
||||
): () => void {
|
||||
const listener = (_e: IpcRendererEvent, payload: T): void => cb(payload);
|
||||
ipcRenderer.on(channel, listener);
|
||||
return () => ipcRenderer.removeListener(channel, listener);
|
||||
}
|
||||
|
||||
const api: FreeCrawlApi = {
|
||||
crawlStart: (config: CrawlConfig) => ipcRenderer.invoke(IPC.crawlStart, config),
|
||||
crawlStop: () => ipcRenderer.invoke(IPC.crawlStop),
|
||||
urlsQuery: (input: UrlsQueryInput): Promise<UrlsQueryResult> =>
|
||||
ipcRenderer.invoke(IPC.urlsQuery, input),
|
||||
summaryGet: (): Promise<CrawlSummary> => ipcRenderer.invoke(IPC.summaryGet),
|
||||
exportCsv: (input: ExportCsvInput): Promise<ExportCsvResult> =>
|
||||
ipcRenderer.invoke(IPC.exportCsv, input),
|
||||
appVersion: (): Promise<string> => ipcRenderer.invoke(IPC.appVersion),
|
||||
onProgress: (cb) => subscribe<CrawlProgress>(IPC.crawlProgress, cb),
|
||||
onDone: (cb) => subscribe<CrawlSummary>(IPC.crawlDone, cb),
|
||||
onError: (cb) => subscribe<string>(IPC.crawlError, cb),
|
||||
};
|
||||
|
||||
contextBridge.exposeInMainWorld('freecrawl', api);
|
||||
@@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>FreeCrawl SEO</title>
|
||||
</head>
|
||||
<body class="bg-surface-950 text-surface-100">
|
||||
<div id="root"></div>
|
||||
<script type="module" src="./src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,41 @@
|
||||
import { useEffect } from 'react';
|
||||
import { TopBar } from './components/TopBar.js';
|
||||
import { StatsBar } from './components/StatsBar.js';
|
||||
import { TabsBar } from './components/TabsBar.js';
|
||||
import { OverviewTab } from './tabs/OverviewTab.js';
|
||||
import { UrlsTab } from './tabs/UrlsTab.js';
|
||||
import { useAppStore } from './store.js';
|
||||
|
||||
export function App() {
|
||||
const activeTab = useAppStore((s) => s.activeTab);
|
||||
const setProgress = useAppStore((s) => s.setProgress);
|
||||
const setSummary = useAppStore((s) => s.setSummary);
|
||||
const setError = useAppStore((s) => s.setError);
|
||||
|
||||
useEffect(() => {
|
||||
const off1 = window.freecrawl.onProgress(setProgress);
|
||||
const off2 = window.freecrawl.onDone((s) => {
|
||||
setSummary(s);
|
||||
});
|
||||
const off3 = window.freecrawl.onError(setError);
|
||||
return () => {
|
||||
off1();
|
||||
off2();
|
||||
off3();
|
||||
};
|
||||
}, [setProgress, setSummary, setError]);
|
||||
|
||||
return (
|
||||
<div className="flex h-full flex-col bg-surface-950 text-surface-100">
|
||||
<TopBar />
|
||||
<StatsBar />
|
||||
<TabsBar />
|
||||
<main className="flex-1 overflow-hidden">
|
||||
{activeTab === 'overview' && <OverviewTab />}
|
||||
{activeTab === 'internal' && <UrlsTab filter="internal" />}
|
||||
{activeTab === 'external' && <UrlsTab filter="external" />}
|
||||
{activeTab === 'issues' && <UrlsTab filter="errors" />}
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { useAppStore } from '../store.js';
|
||||
|
||||
function Stat({ label, value }: { label: string; value: string | number }) {
|
||||
return (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="text-surface-500">{label}</span>
|
||||
<span className="font-mono font-medium text-surface-100">{value}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function StatsBar() {
|
||||
const progress = useAppStore((s) => s.progress);
|
||||
const error = useAppStore((s) => s.error);
|
||||
const setError = useAppStore((s) => s.setError);
|
||||
|
||||
const elapsed = progress?.elapsedMs ?? 0;
|
||||
const elapsedStr = formatElapsed(elapsed);
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-5 border-b border-surface-800 bg-surface-900/50 px-3 py-1.5 text-[11px]">
|
||||
<Stat label="Discovered" value={progress?.discovered ?? 0} />
|
||||
<Stat label="Crawled" value={progress?.crawled ?? 0} />
|
||||
<Stat label="Pending" value={progress?.pending ?? 0} />
|
||||
<Stat label="Failed" value={progress?.failed ?? 0} />
|
||||
<Stat label="URL/s" value={progress?.urlsPerSecond?.toFixed(1) ?? '0.0'} />
|
||||
<Stat label="Avg resp" value={`${progress?.avgResponseTimeMs ?? 0}ms`} />
|
||||
<Stat label="Elapsed" value={elapsedStr} />
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
{progress?.running ? (
|
||||
<span className="inline-flex items-center gap-1.5">
|
||||
<span className="h-2 w-2 animate-pulse rounded-full bg-emerald-500" />
|
||||
<span className="text-emerald-400">Running</span>
|
||||
</span>
|
||||
) : (
|
||||
<span className="inline-flex items-center gap-1.5">
|
||||
<span className="h-2 w-2 rounded-full bg-surface-600" />
|
||||
<span className="text-surface-500">Idle</span>
|
||||
</span>
|
||||
)}
|
||||
{error && (
|
||||
<button
|
||||
className="rounded bg-red-900/50 px-2 py-0.5 text-red-200 hover:bg-red-900/70"
|
||||
onClick={() => setError(null)}
|
||||
title={error}
|
||||
>
|
||||
⚠ {error.length > 60 ? error.slice(0, 60) + '…' : error} (dismiss)
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function formatElapsed(ms: number): string {
|
||||
const s = Math.floor(ms / 1000);
|
||||
const hh = Math.floor(s / 3600);
|
||||
const mm = Math.floor((s % 3600) / 60);
|
||||
const ss = s % 60;
|
||||
return `${hh.toString().padStart(2, '0')}:${mm.toString().padStart(2, '0')}:${ss.toString().padStart(2, '0')}`;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import clsx from 'clsx';
|
||||
import { useAppStore, type TabKey } from '../store.js';
|
||||
|
||||
const TABS: { key: TabKey; label: string }[] = [
|
||||
{ key: 'overview', label: 'Overview' },
|
||||
{ key: 'internal', label: 'Internal URLs' },
|
||||
{ key: 'external', label: 'External URLs' },
|
||||
{ key: 'issues', label: 'Issues' },
|
||||
];
|
||||
|
||||
export function TabsBar() {
|
||||
const activeTab = useAppStore((s) => s.activeTab);
|
||||
const setActiveTab = useAppStore((s) => s.setActiveTab);
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-0 border-b border-surface-800 bg-surface-900/30 px-2">
|
||||
{TABS.map((t) => (
|
||||
<button
|
||||
key={t.key}
|
||||
className={clsx('tab', activeTab === t.key && 'tab-active')}
|
||||
onClick={() => setActiveTab(t.key)}
|
||||
>
|
||||
{t.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { useState } from 'react';
|
||||
import { Play, Square, Download } from 'lucide-react';
|
||||
import { useAppStore } from '../store.js';
|
||||
|
||||
export function TopBar() {
|
||||
const config = useAppStore((s) => s.config);
|
||||
const setConfig = useAppStore((s) => s.setConfig);
|
||||
const progress = useAppStore((s) => s.progress);
|
||||
const reset = useAppStore((s) => s.reset);
|
||||
const setError = useAppStore((s) => s.setError);
|
||||
const [exporting, setExporting] = useState(false);
|
||||
|
||||
const running = progress?.running === true;
|
||||
|
||||
async function start() {
|
||||
if (!config.startUrl.trim()) {
|
||||
setError('Başlangıç URL girmelisin.');
|
||||
return;
|
||||
}
|
||||
reset();
|
||||
try {
|
||||
await window.freecrawl.crawlStart(config);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
async function stop() {
|
||||
await window.freecrawl.crawlStop();
|
||||
}
|
||||
|
||||
async function exportCsv() {
|
||||
setExporting(true);
|
||||
try {
|
||||
const res = await window.freecrawl.exportCsv({ filePath: '' });
|
||||
if (res.filePath) {
|
||||
console.log(`Exported ${res.rowsWritten} rows to ${res.filePath}`);
|
||||
}
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setExporting(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2 border-b border-surface-800 bg-surface-900 px-3 py-2">
|
||||
<div className="text-xs font-semibold uppercase tracking-wide text-surface-400">
|
||||
FreeCrawl
|
||||
</div>
|
||||
<div className="mx-2 h-5 w-px bg-surface-800" />
|
||||
<input
|
||||
className="input flex-1"
|
||||
placeholder="https://example.com"
|
||||
value={config.startUrl}
|
||||
onChange={(e) => setConfig({ startUrl: e.target.value })}
|
||||
disabled={running}
|
||||
spellCheck={false}
|
||||
/>
|
||||
<div className="flex items-center gap-1 text-xs text-surface-400">
|
||||
<label>Depth</label>
|
||||
<input
|
||||
type="number"
|
||||
className="input w-16"
|
||||
value={config.maxDepth}
|
||||
min={0}
|
||||
max={50}
|
||||
onChange={(e) =>
|
||||
setConfig({ maxDepth: Math.max(0, Number.parseInt(e.target.value, 10) || 0) })
|
||||
}
|
||||
disabled={running}
|
||||
/>
|
||||
<label>URLs</label>
|
||||
<input
|
||||
type="number"
|
||||
className="input w-24"
|
||||
value={config.maxUrls}
|
||||
min={1}
|
||||
onChange={(e) =>
|
||||
setConfig({ maxUrls: Math.max(1, Number.parseInt(e.target.value, 10) || 1) })
|
||||
}
|
||||
disabled={running}
|
||||
/>
|
||||
<label>Conc.</label>
|
||||
<input
|
||||
type="number"
|
||||
className="input w-14"
|
||||
value={config.maxConcurrency}
|
||||
min={1}
|
||||
max={50}
|
||||
onChange={(e) =>
|
||||
setConfig({
|
||||
maxConcurrency: Math.max(
|
||||
1,
|
||||
Math.min(50, Number.parseInt(e.target.value, 10) || 1),
|
||||
),
|
||||
})
|
||||
}
|
||||
disabled={running}
|
||||
/>
|
||||
</div>
|
||||
{running ? (
|
||||
<button className="btn btn-ghost" onClick={stop}>
|
||||
<Square className="h-3.5 w-3.5" /> Stop
|
||||
</button>
|
||||
) : (
|
||||
<button className="btn btn-primary" onClick={start}>
|
||||
<Play className="h-3.5 w-3.5" /> Start
|
||||
</button>
|
||||
)}
|
||||
<button className="btn btn-ghost" onClick={exportCsv} disabled={exporting}>
|
||||
<Download className="h-3.5 w-3.5" /> {exporting ? 'Exporting…' : 'Export CSV'}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
import type { FreeCrawlApi } from '@freecrawl/shared-types';
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
freecrawl: FreeCrawlApi;
|
||||
}
|
||||
}
|
||||
|
||||
export {};
|
||||
@@ -0,0 +1,13 @@
|
||||
import React from 'react';
|
||||
import { createRoot } from 'react-dom/client';
|
||||
import { App } from './App.js';
|
||||
import './styles.css';
|
||||
|
||||
const root = document.getElementById('root');
|
||||
if (root) {
|
||||
createRoot(root).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { create } from 'zustand';
|
||||
import {
|
||||
DEFAULT_CRAWL_CONFIG,
|
||||
type CrawlConfig,
|
||||
type CrawlProgress,
|
||||
type CrawlSummary,
|
||||
} from '@freecrawl/shared-types';
|
||||
|
||||
export type TabKey = 'overview' | 'internal' | 'external' | 'issues';
|
||||
|
||||
interface AppState {
|
||||
config: CrawlConfig;
|
||||
progress: CrawlProgress | null;
|
||||
summary: CrawlSummary | null;
|
||||
activeTab: TabKey;
|
||||
error: string | null;
|
||||
setConfig: (patch: Partial<CrawlConfig>) => void;
|
||||
setProgress: (p: CrawlProgress) => void;
|
||||
setSummary: (s: CrawlSummary) => void;
|
||||
setActiveTab: (t: TabKey) => void;
|
||||
setError: (e: string | null) => void;
|
||||
reset: () => void;
|
||||
}
|
||||
|
||||
export const useAppStore = create<AppState>((set) => ({
|
||||
config: DEFAULT_CRAWL_CONFIG,
|
||||
progress: null,
|
||||
summary: null,
|
||||
activeTab: 'overview',
|
||||
error: null,
|
||||
setConfig: (patch) =>
|
||||
set((state) => ({ config: { ...state.config, ...patch } })),
|
||||
setProgress: (p) => set({ progress: p }),
|
||||
setSummary: (s) => set({ summary: s }),
|
||||
setActiveTab: (t) => set({ activeTab: t }),
|
||||
setError: (e) => set({ error: e }),
|
||||
reset: () => set({ progress: null, summary: null, error: null }),
|
||||
}));
|
||||
@@ -0,0 +1,58 @@
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
@layer base {
|
||||
html,
|
||||
body,
|
||||
#root {
|
||||
height: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family:
|
||||
Inter,
|
||||
system-ui,
|
||||
-apple-system,
|
||||
sans-serif;
|
||||
font-size: 13px;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
|
||||
*::-webkit-scrollbar {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
}
|
||||
*::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
*::-webkit-scrollbar-thumb {
|
||||
background: theme('colors.surface.700');
|
||||
border-radius: 4px;
|
||||
}
|
||||
*::-webkit-scrollbar-thumb:hover {
|
||||
background: theme('colors.surface.600');
|
||||
}
|
||||
}
|
||||
|
||||
@layer components {
|
||||
.btn {
|
||||
@apply inline-flex items-center gap-1.5 rounded px-3 py-1.5 text-xs font-medium transition;
|
||||
}
|
||||
.btn-primary {
|
||||
@apply bg-accent-500 text-white hover:bg-accent-600 active:bg-accent-600 disabled:opacity-50;
|
||||
}
|
||||
.btn-ghost {
|
||||
@apply text-surface-300 hover:bg-surface-800 hover:text-surface-100 disabled:opacity-50;
|
||||
}
|
||||
.input {
|
||||
@apply w-full rounded border border-surface-700 bg-surface-900 px-2 py-1.5 text-xs text-surface-100 placeholder:text-surface-500 focus:border-accent-500 focus:outline-none focus:ring-1 focus:ring-accent-500;
|
||||
}
|
||||
.tab {
|
||||
@apply cursor-pointer border-b-2 border-transparent px-3 py-2 text-xs font-medium text-surface-400 transition hover:text-surface-100;
|
||||
}
|
||||
.tab-active {
|
||||
@apply border-accent-500 text-surface-50;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import type { CrawlSummary } from '@freecrawl/shared-types';
|
||||
import { useAppStore } from '../store.js';
|
||||
|
||||
export function OverviewTab() {
|
||||
const progress = useAppStore((s) => s.progress);
|
||||
const [summary, setSummary] = useState<CrawlSummary | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const load = async () => {
|
||||
const s = await window.freecrawl.summaryGet();
|
||||
setSummary(s);
|
||||
};
|
||||
void load();
|
||||
const id = setInterval(load, 1500);
|
||||
return () => clearInterval(id);
|
||||
}, [progress?.crawled]);
|
||||
|
||||
if (!summary || summary.total === 0) {
|
||||
return (
|
||||
<div className="flex h-full items-center justify-center">
|
||||
<div className="max-w-md text-center">
|
||||
<div className="mb-2 text-lg font-semibold text-surface-200">No crawl yet</div>
|
||||
<div className="text-sm text-surface-400">
|
||||
Enter a URL above and click <span className="font-medium text-surface-200">Start</span>{' '}
|
||||
to begin crawling.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto p-5">
|
||||
<div className="mx-auto grid max-w-5xl grid-cols-1 gap-4 md:grid-cols-2 lg:grid-cols-3">
|
||||
<SummaryCard title="Total URLs" value={summary.total.toLocaleString()} />
|
||||
<SummaryCard
|
||||
title="Avg Response"
|
||||
value={`${summary.avgResponseTimeMs.toLocaleString()} ms`}
|
||||
/>
|
||||
<SummaryCard title="Total Bytes" value={formatBytes(summary.totalBytes)} />
|
||||
|
||||
<BreakdownCard title="Status Codes" data={summary.byStatus} colorOf={statusColor} />
|
||||
<BreakdownCard title="Content Type" data={summary.byContentKind} />
|
||||
<BreakdownCard
|
||||
title="Indexability"
|
||||
data={summary.byIndexability}
|
||||
colorOf={indexabilityColor}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SummaryCard({ title, value }: { title: string; value: string }) {
|
||||
return (
|
||||
<div className="rounded-md border border-surface-800 bg-surface-900 p-4">
|
||||
<div className="text-xs uppercase tracking-wide text-surface-500">{title}</div>
|
||||
<div className="mt-2 font-mono text-2xl font-semibold text-surface-50">{value}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function BreakdownCard<T extends Record<string, number>>({
|
||||
title,
|
||||
data,
|
||||
colorOf,
|
||||
}: {
|
||||
title: string;
|
||||
data: T;
|
||||
colorOf?: (key: string) => string;
|
||||
}) {
|
||||
const entries = Object.entries(data).sort((a, b) => b[1] - a[1]);
|
||||
const total = entries.reduce((sum, [, v]) => sum + v, 0);
|
||||
return (
|
||||
<div className="rounded-md border border-surface-800 bg-surface-900 p-4">
|
||||
<div className="mb-3 text-xs uppercase tracking-wide text-surface-500">{title}</div>
|
||||
<div className="space-y-1.5">
|
||||
{entries.map(([key, value]) => {
|
||||
const pct = total > 0 ? (value / total) * 100 : 0;
|
||||
return (
|
||||
<div key={key} className="flex items-center gap-2 text-xs">
|
||||
<div className="w-28 truncate text-surface-300">{key}</div>
|
||||
<div className="relative h-4 flex-1 overflow-hidden rounded bg-surface-800">
|
||||
<div
|
||||
className={`h-full ${colorOf?.(key) ?? 'bg-accent-500'}`}
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="w-12 text-right font-mono text-surface-200">
|
||||
{value.toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function statusColor(status: string): string {
|
||||
const n = Number.parseInt(status, 10);
|
||||
if (Number.isNaN(n)) return 'bg-surface-600';
|
||||
if (n >= 200 && n < 300) return 'bg-emerald-600';
|
||||
if (n >= 300 && n < 400) return 'bg-amber-500';
|
||||
if (n >= 400 && n < 500) return 'bg-orange-600';
|
||||
if (n >= 500) return 'bg-red-600';
|
||||
return 'bg-surface-600';
|
||||
}
|
||||
|
||||
function indexabilityColor(key: string): string {
|
||||
if (key === 'indexable') return 'bg-emerald-600';
|
||||
if (key.startsWith('non-indexable:server-error')) return 'bg-red-600';
|
||||
if (key.startsWith('non-indexable:client-error')) return 'bg-orange-600';
|
||||
return 'bg-amber-500';
|
||||
}
|
||||
|
||||
function formatBytes(n: number): string {
|
||||
if (n < 1024) return `${n} B`;
|
||||
if (n < 1024 ** 2) return `${(n / 1024).toFixed(1)} KB`;
|
||||
if (n < 1024 ** 3) return `${(n / 1024 ** 2).toFixed(1)} MB`;
|
||||
return `${(n / 1024 ** 3).toFixed(2)} GB`;
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
import {
|
||||
createColumnHelper,
|
||||
flexRender,
|
||||
getCoreRowModel,
|
||||
useReactTable,
|
||||
type SortingState,
|
||||
} from '@tanstack/react-table';
|
||||
import { useVirtualizer } from '@tanstack/react-virtual';
|
||||
import clsx from 'clsx';
|
||||
import type { CrawlUrlRow } from '@freecrawl/shared-types';
|
||||
import { useAppStore } from '../store.js';
|
||||
|
||||
type Filter = 'all' | 'internal' | 'external' | 'errors' | 'redirects';
|
||||
|
||||
const PAGE_SIZE = 500;
|
||||
|
||||
const columnHelper = createColumnHelper<CrawlUrlRow>();
|
||||
const columns = [
|
||||
columnHelper.accessor('url', {
|
||||
header: 'URL',
|
||||
cell: (i) => (
|
||||
<span className="truncate font-mono text-[11px] text-surface-100" title={i.getValue()}>
|
||||
{i.getValue()}
|
||||
</span>
|
||||
),
|
||||
size: 520,
|
||||
}),
|
||||
columnHelper.accessor('statusCode', {
|
||||
header: 'Status',
|
||||
cell: (i) => {
|
||||
const v = i.getValue();
|
||||
return (
|
||||
<span
|
||||
className={clsx(
|
||||
'inline-block rounded px-1.5 font-mono text-[10px]',
|
||||
statusClasses(v),
|
||||
)}
|
||||
>
|
||||
{v ?? '—'}
|
||||
</span>
|
||||
);
|
||||
},
|
||||
size: 70,
|
||||
}),
|
||||
columnHelper.accessor('contentKind', { header: 'Type', size: 70 }),
|
||||
columnHelper.accessor('indexability', {
|
||||
header: 'Indexability',
|
||||
cell: (i) => {
|
||||
const v = i.getValue();
|
||||
return (
|
||||
<span
|
||||
className={clsx(
|
||||
'truncate',
|
||||
v === 'indexable' ? 'text-emerald-400' : 'text-amber-400',
|
||||
)}
|
||||
title={v}
|
||||
>
|
||||
{v === 'indexable' ? 'Indexable' : 'Non-indexable'}
|
||||
</span>
|
||||
);
|
||||
},
|
||||
size: 120,
|
||||
}),
|
||||
columnHelper.accessor('title', {
|
||||
header: 'Title',
|
||||
cell: (i) => <span className="truncate" title={i.getValue() ?? ''}>{i.getValue() ?? ''}</span>,
|
||||
size: 300,
|
||||
}),
|
||||
columnHelper.accessor('titleLength', { header: 'Title Len', size: 80 }),
|
||||
columnHelper.accessor('metaDescription', {
|
||||
header: 'Meta Description',
|
||||
cell: (i) => (
|
||||
<span className="truncate text-surface-300" title={i.getValue() ?? ''}>
|
||||
{i.getValue() ?? ''}
|
||||
</span>
|
||||
),
|
||||
size: 320,
|
||||
}),
|
||||
columnHelper.accessor('metaDescriptionLength', { header: 'Meta Len', size: 80 }),
|
||||
columnHelper.accessor('h1', {
|
||||
header: 'H1',
|
||||
cell: (i) => <span className="truncate" title={i.getValue() ?? ''}>{i.getValue() ?? ''}</span>,
|
||||
size: 240,
|
||||
}),
|
||||
columnHelper.accessor('wordCount', { header: 'Words', size: 70 }),
|
||||
columnHelper.accessor('responseTimeMs', { header: 'Resp (ms)', size: 90 }),
|
||||
columnHelper.accessor('depth', { header: 'Depth', size: 60 }),
|
||||
columnHelper.accessor('inlinks', { header: 'Inlinks', size: 70 }),
|
||||
columnHelper.accessor('outlinks', { header: 'Outlinks', size: 80 }),
|
||||
];
|
||||
|
||||
export function UrlsTab({ filter }: { filter: Filter }) {
|
||||
const progress = useAppStore((s) => s.progress);
|
||||
const [rows, setRows] = useState<CrawlUrlRow[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [search, setSearch] = useState('');
|
||||
const [sorting, setSorting] = useState<SortingState>([]);
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const load = async () => {
|
||||
const sortBy = (sorting[0]?.id as keyof CrawlUrlRow | undefined) ?? 'id';
|
||||
const sortDir = sorting[0]?.desc ? 'desc' : 'asc';
|
||||
const res = await window.freecrawl.urlsQuery({
|
||||
limit: PAGE_SIZE,
|
||||
offset: 0,
|
||||
filter,
|
||||
search: search.trim() || undefined,
|
||||
sortBy,
|
||||
sortDir,
|
||||
});
|
||||
setRows(res.rows);
|
||||
setTotal(res.total);
|
||||
};
|
||||
void load();
|
||||
const id = setInterval(load, 1500);
|
||||
return () => clearInterval(id);
|
||||
}, [filter, search, sorting, progress?.crawled]);
|
||||
|
||||
const table = useReactTable({
|
||||
data: rows,
|
||||
columns,
|
||||
getCoreRowModel: getCoreRowModel(),
|
||||
state: { sorting },
|
||||
onSortingChange: setSorting,
|
||||
manualSorting: true,
|
||||
});
|
||||
|
||||
const rowVirtualizer = useVirtualizer({
|
||||
count: table.getRowModel().rows.length,
|
||||
getScrollElement: () => scrollRef.current,
|
||||
estimateSize: () => 28,
|
||||
overscan: 12,
|
||||
});
|
||||
|
||||
const visibleColumnsWidth = useMemo(
|
||||
() => columns.reduce((sum, c) => sum + (c.size ?? 100), 0),
|
||||
[],
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="flex h-full flex-col">
|
||||
<div className="flex items-center gap-2 border-b border-surface-800 bg-surface-900/30 px-3 py-1.5">
|
||||
<input
|
||||
className="input w-80"
|
||||
placeholder="Search URLs / titles…"
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
spellCheck={false}
|
||||
/>
|
||||
<div className="ml-auto text-[11px] text-surface-500">
|
||||
Showing <span className="font-mono text-surface-200">{rows.length.toLocaleString()}</span>{' '}
|
||||
of <span className="font-mono text-surface-200">{total.toLocaleString()}</span>
|
||||
{total > PAGE_SIZE && (
|
||||
<span className="ml-1 text-surface-600">(first {PAGE_SIZE} shown)</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div ref={scrollRef} className="relative flex-1 overflow-auto">
|
||||
<table className="w-max border-collapse" style={{ minWidth: visibleColumnsWidth }}>
|
||||
<thead className="sticky top-0 z-10 bg-surface-900 text-[11px]">
|
||||
{table.getHeaderGroups().map((hg) => (
|
||||
<tr key={hg.id}>
|
||||
{hg.headers.map((h) => (
|
||||
<th
|
||||
key={h.id}
|
||||
className="select-none border-b border-surface-800 px-2 py-1.5 text-left font-medium text-surface-300"
|
||||
style={{ width: h.getSize() }}
|
||||
onClick={h.column.getToggleSortingHandler()}
|
||||
>
|
||||
<div className="flex items-center gap-1">
|
||||
{flexRender(h.column.columnDef.header, h.getContext())}
|
||||
{h.column.getIsSorted() === 'asc' && '▲'}
|
||||
{h.column.getIsSorted() === 'desc' && '▼'}
|
||||
</div>
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</thead>
|
||||
<tbody style={{ height: rowVirtualizer.getTotalSize(), display: 'block', position: 'relative' }}>
|
||||
{rowVirtualizer.getVirtualItems().map((vi) => {
|
||||
const row = table.getRowModel().rows[vi.index];
|
||||
if (!row) return null;
|
||||
return (
|
||||
<tr
|
||||
key={row.id}
|
||||
className="absolute left-0 top-0 flex items-center border-b border-surface-900 text-[11px] hover:bg-surface-900/60"
|
||||
style={{ transform: `translateY(${vi.start}px)`, height: vi.size }}
|
||||
>
|
||||
{row.getVisibleCells().map((cell) => (
|
||||
<td
|
||||
key={cell.id}
|
||||
className="overflow-hidden px-2"
|
||||
style={{ width: cell.column.getSize() }}
|
||||
>
|
||||
{flexRender(cell.column.columnDef.cell, cell.getContext())}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function statusClasses(code: number | null): string {
|
||||
if (code === null) return 'bg-surface-800 text-surface-400';
|
||||
if (code >= 200 && code < 300) return 'bg-emerald-900/60 text-emerald-300';
|
||||
if (code >= 300 && code < 400) return 'bg-amber-900/60 text-amber-300';
|
||||
if (code >= 400 && code < 500) return 'bg-orange-900/60 text-orange-300';
|
||||
if (code >= 500) return 'bg-red-900/60 text-red-300';
|
||||
return 'bg-surface-800 text-surface-400';
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
export default {
|
||||
content: ['./src/renderer/index.html', './src/renderer/src/**/*.{ts,tsx}'],
|
||||
darkMode: 'class',
|
||||
theme: {
|
||||
extend: {
|
||||
fontFamily: {
|
||||
sans: ['Inter', 'system-ui', 'sans-serif'],
|
||||
mono: ['JetBrains Mono', 'ui-monospace', 'SFMono-Regular', 'monospace'],
|
||||
},
|
||||
colors: {
|
||||
surface: {
|
||||
50: '#fafafa',
|
||||
100: '#f5f5f5',
|
||||
200: '#e5e5e5',
|
||||
300: '#d4d4d4',
|
||||
400: '#a3a3a3',
|
||||
500: '#737373',
|
||||
600: '#525252',
|
||||
700: '#404040',
|
||||
800: '#262626',
|
||||
850: '#1f1f1f',
|
||||
900: '#171717',
|
||||
950: '#0a0a0a',
|
||||
},
|
||||
accent: {
|
||||
500: '#2563eb',
|
||||
600: '#1d4ed8',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
plugins: [],
|
||||
};
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"files": [],
|
||||
"references": [
|
||||
{ "path": "./tsconfig.node.json" },
|
||||
{ "path": "./tsconfig.web.json" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"types": ["node", "electron-vite/node"],
|
||||
"outDir": "./out-types/node",
|
||||
"rootDir": "./src"
|
||||
},
|
||||
"include": ["src/main/**/*", "src/preload/**/*"],
|
||||
"references": [
|
||||
{ "path": "../../packages/shared-types" },
|
||||
{ "path": "../../packages/db" },
|
||||
{ "path": "../../packages/core" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"types": ["node"],
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["electron.vite.config.ts", "vite.config.ts"]
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2022",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"jsx": "react-jsx",
|
||||
"outDir": "./out-types/web",
|
||||
"rootDir": "./src/renderer",
|
||||
"baseUrl": "./src/renderer",
|
||||
"paths": {
|
||||
"@renderer/*": ["src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src/renderer/**/*"],
|
||||
"references": [{ "path": "../../packages/shared-types" }]
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import js from '@eslint/js';
|
||||
import tseslint from 'typescript-eslint';
|
||||
|
||||
export default tseslint.config(
|
||||
js.configs.recommended,
|
||||
...tseslint.configs.recommended,
|
||||
{
|
||||
languageOptions: {
|
||||
ecmaVersion: 2022,
|
||||
sourceType: 'module',
|
||||
},
|
||||
rules: {
|
||||
'@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }],
|
||||
'@typescript-eslint/consistent-type-imports': 'error',
|
||||
},
|
||||
},
|
||||
{
|
||||
ignores: ['**/dist/**', '**/out/**', '**/node_modules/**', '**/build/**', '.claude/**'],
|
||||
},
|
||||
);
|
||||
Generated
+8721
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"name": "freecrawl-seo",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
|
||||
"license": "MIT",
|
||||
"author": "kemalai",
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22.0.0"
|
||||
},
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
"packages/*"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "npm run dev --workspace apps/desktop",
|
||||
"build": "tsc -b && npm run build --workspace apps/desktop && npm run build --workspace apps/cli",
|
||||
"build:desktop": "npm run build --workspace apps/desktop",
|
||||
"build:cli": "npm run build --workspace apps/cli",
|
||||
"typecheck": "npm run typecheck --workspaces --if-present",
|
||||
"lint": "eslint .",
|
||||
"format": "prettier --write \"**/*.{ts,tsx,js,json,md,css,html}\"",
|
||||
"format:check": "prettier --check \"**/*.{ts,tsx,js,json,md,css,html}\"",
|
||||
"test": "npm run test --workspaces --if-present",
|
||||
"cli": "npm run start --workspace apps/cli --"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.0",
|
||||
"eslint": "^9.17.0",
|
||||
"prettier": "^3.4.0",
|
||||
"typescript": "^5.7.0",
|
||||
"typescript-eslint": "^8.20.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "@freecrawl/core",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"clean": "rimraf dist"
|
||||
},
|
||||
"dependencies": {
|
||||
"@freecrawl/db": "*",
|
||||
"@freecrawl/shared-types": "*",
|
||||
"cheerio": "^1.0.0",
|
||||
"p-queue": "^8.0.1",
|
||||
"robots-parser": "^3.0.1",
|
||||
"undici": "^8.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "^5.7.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { fetch as undiciFetch } from 'undici';
|
||||
import PQueue from 'p-queue';
|
||||
import type {
|
||||
ContentKind,
|
||||
CrawlConfig,
|
||||
CrawlProgress,
|
||||
CrawlSummary,
|
||||
DiscoveredLink,
|
||||
Indexability,
|
||||
} from '@freecrawl/shared-types';
|
||||
import type { ProjectDb } from '@freecrawl/db';
|
||||
import { normalizeUrl, isSameHost, extractExtension } from './url-utils.js';
|
||||
import { parseHtml } from './html-parser.js';
|
||||
import { loadRobots, type RobotsChecker } from './robots.js';
|
||||
|
||||
export interface CrawlerEvents {
|
||||
progress: (p: CrawlProgress) => void;
|
||||
done: (summary: CrawlSummary) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
interface QueueItem {
|
||||
url: string;
|
||||
depth: number;
|
||||
}
|
||||
|
||||
const EXT_TO_KIND: Record<string, ContentKind> = {
|
||||
css: 'css',
|
||||
js: 'js',
|
||||
mjs: 'js',
|
||||
png: 'image',
|
||||
jpg: 'image',
|
||||
jpeg: 'image',
|
||||
gif: 'image',
|
||||
webp: 'image',
|
||||
svg: 'image',
|
||||
ico: 'image',
|
||||
pdf: 'pdf',
|
||||
woff: 'font',
|
||||
woff2: 'font',
|
||||
ttf: 'font',
|
||||
otf: 'font',
|
||||
};
|
||||
|
||||
export class Crawler extends EventEmitter {
|
||||
private readonly config: CrawlConfig;
|
||||
private readonly db: ProjectDb;
|
||||
private queue: PQueue;
|
||||
private seen = new Set<string>();
|
||||
private pending = 0;
|
||||
private crawled = 0;
|
||||
private failed = 0;
|
||||
private totalResponseTimeMs = 0;
|
||||
private responseSamples = 0;
|
||||
private startedAt = 0;
|
||||
private stopped = false;
|
||||
private robots: RobotsChecker | null = null;
|
||||
private progressTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor(config: CrawlConfig, db: ProjectDb) {
|
||||
super();
|
||||
this.config = config;
|
||||
this.db = db;
|
||||
this.queue = new PQueue({
|
||||
concurrency: Math.max(1, Math.min(50, config.maxConcurrency)),
|
||||
interval: 1000,
|
||||
intervalCap: Math.max(1, config.maxRps),
|
||||
});
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
this.startedAt = Date.now();
|
||||
this.stopped = false;
|
||||
this.db.reset();
|
||||
|
||||
const start = normalizeUrl(this.config.startUrl);
|
||||
if (!start) {
|
||||
this.emit('error', `Invalid start URL: ${this.config.startUrl}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const origin = new URL(start).origin;
|
||||
if (this.config.respectRobotsTxt) {
|
||||
this.robots = await loadRobots(origin, this.config.userAgent);
|
||||
}
|
||||
|
||||
this.progressTimer = setInterval(() => this.emitProgress(), 500);
|
||||
|
||||
this.enqueue({ url: start, depth: 0 });
|
||||
|
||||
try {
|
||||
await this.queue.onIdle();
|
||||
} finally {
|
||||
if (this.progressTimer) clearInterval(this.progressTimer);
|
||||
this.progressTimer = null;
|
||||
}
|
||||
|
||||
this.db.recomputeInlinks();
|
||||
this.emitProgress();
|
||||
this.emit('done', this.db.getSummary());
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
this.queue.clear();
|
||||
}
|
||||
|
||||
private enqueue(item: QueueItem): void {
|
||||
if (this.stopped) return;
|
||||
if (this.seen.has(item.url)) return;
|
||||
if (this.seen.size >= this.config.maxUrls) return;
|
||||
if (item.depth > this.config.maxDepth) return;
|
||||
if (this.robots && !this.robots.isAllowed(item.url)) return;
|
||||
|
||||
this.seen.add(item.url);
|
||||
this.pending++;
|
||||
this.queue
|
||||
.add(() => this.fetchAndProcess(item))
|
||||
.catch((err: unknown) => {
|
||||
this.emit(
|
||||
'error',
|
||||
`Queue error: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
this.pending = Math.max(0, this.pending - 1);
|
||||
});
|
||||
}
|
||||
|
||||
private async fetchAndProcess(item: QueueItem): Promise<void> {
|
||||
if (this.stopped) return;
|
||||
|
||||
const t0 = Date.now();
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
||||
|
||||
try {
|
||||
const res = await undiciFetch(item.url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'user-agent': this.config.userAgent,
|
||||
'accept-language': this.config.acceptLanguage,
|
||||
accept:
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
},
|
||||
redirect: this.config.followRedirects ? 'follow' : 'manual',
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const responseTimeMs = Date.now() - t0;
|
||||
this.totalResponseTimeMs += responseTimeMs;
|
||||
this.responseSamples++;
|
||||
|
||||
const statusCode = res.status;
|
||||
const contentType = res.headers.get('content-type');
|
||||
const contentLengthHeader = res.headers.get('content-length');
|
||||
const xRobotsTag = res.headers.get('x-robots-tag');
|
||||
|
||||
const kind = detectContentKind(item.url, contentType);
|
||||
|
||||
if (kind !== 'html' || statusCode >= 400) {
|
||||
try {
|
||||
await res.text();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
const indexability: Indexability =
|
||||
statusCode >= 500
|
||||
? 'non-indexable:server-error'
|
||||
: statusCode >= 400
|
||||
? 'non-indexable:client-error'
|
||||
: 'indexable';
|
||||
this.db.upsertUrl({
|
||||
url: item.url,
|
||||
contentKind: kind,
|
||||
statusCode,
|
||||
statusText: null,
|
||||
indexability,
|
||||
indexabilityReason: indexability === 'indexable' ? null : `HTTP ${statusCode}`,
|
||||
contentType,
|
||||
contentLength: parseIntSafe(contentLengthHeader),
|
||||
xRobotsTag,
|
||||
responseTimeMs,
|
||||
depth: item.depth,
|
||||
});
|
||||
this.crawled++;
|
||||
return;
|
||||
}
|
||||
|
||||
const body = await res.text();
|
||||
const bodyLength = parseIntSafe(contentLengthHeader) ?? Buffer.byteLength(body, 'utf8');
|
||||
const parsed = parseHtml(body, item.url, {
|
||||
includeSubdomains: this.config.includeSubdomains,
|
||||
});
|
||||
|
||||
const xRobotsLower = xRobotsTag?.toLowerCase() ?? '';
|
||||
const headerNoindex = xRobotsLower.includes('noindex');
|
||||
|
||||
let indexability: Indexability = 'indexable';
|
||||
let reason: string | null = null;
|
||||
if (parsed.hasNoindex) {
|
||||
indexability = 'non-indexable:noindex';
|
||||
reason = 'meta robots: noindex';
|
||||
} else if (headerNoindex) {
|
||||
indexability = 'non-indexable:noindex';
|
||||
reason = 'X-Robots-Tag: noindex';
|
||||
} else if (parsed.canonical && normalizeUrl(parsed.canonical, item.url) !== item.url) {
|
||||
indexability = 'non-indexable:canonical';
|
||||
reason = `canonical points to ${parsed.canonical}`;
|
||||
}
|
||||
|
||||
const internalLinks = parsed.links.filter((l) => l.isInternal);
|
||||
const urlId = this.db.upsertUrl({
|
||||
url: item.url,
|
||||
contentKind: 'html',
|
||||
statusCode,
|
||||
statusText: null,
|
||||
indexability,
|
||||
indexabilityReason: reason,
|
||||
title: parsed.title,
|
||||
metaDescription: parsed.metaDescription,
|
||||
h1: parsed.h1,
|
||||
h2Count: parsed.h2Count,
|
||||
wordCount: parsed.wordCount,
|
||||
canonical: parsed.canonical,
|
||||
metaRobots: parsed.metaRobots,
|
||||
xRobotsTag,
|
||||
contentType,
|
||||
contentLength: bodyLength,
|
||||
responseTimeMs,
|
||||
depth: item.depth,
|
||||
outlinks: parsed.links.length,
|
||||
});
|
||||
this.db.insertLinks(urlId, parsed.links);
|
||||
this.crawled++;
|
||||
|
||||
if (parsed.hasNofollow || indexability === 'non-indexable:noindex') {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextDepth = item.depth + 1;
|
||||
for (const link of parsed.links) {
|
||||
if (!link.isInternal && !this.config.crawlExternal) continue;
|
||||
if (!isSameHost(this.config.startUrl, link.toUrl, {
|
||||
includeSubdomains: this.config.includeSubdomains,
|
||||
}) && !this.config.crawlExternal) {
|
||||
continue;
|
||||
}
|
||||
if (link.rel?.includes('nofollow')) continue;
|
||||
this.enqueue({ url: link.toUrl, depth: nextDepth });
|
||||
}
|
||||
} catch (err) {
|
||||
this.failed++;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
this.db.upsertUrl({
|
||||
url: item.url,
|
||||
contentKind: 'html',
|
||||
statusCode: null,
|
||||
statusText: message,
|
||||
indexability: 'non-indexable:client-error',
|
||||
indexabilityReason: `Network error: ${message}`,
|
||||
responseTimeMs: Date.now() - t0,
|
||||
depth: item.depth,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
private emitProgress(): void {
|
||||
const elapsedMs = Date.now() - this.startedAt;
|
||||
const urlsPerSecond = elapsedMs > 0 ? (this.crawled / elapsedMs) * 1000 : 0;
|
||||
const avgResponseTimeMs =
|
||||
this.responseSamples > 0 ? Math.round(this.totalResponseTimeMs / this.responseSamples) : 0;
|
||||
const progress: CrawlProgress = {
|
||||
discovered: this.seen.size,
|
||||
crawled: this.crawled,
|
||||
failed: this.failed,
|
||||
pending: this.pending,
|
||||
currentDepth: 0,
|
||||
urlsPerSecond: Math.round(urlsPerSecond * 10) / 10,
|
||||
elapsedMs,
|
||||
avgResponseTimeMs,
|
||||
running: !this.stopped && this.queue.size + this.queue.pending > 0,
|
||||
startUrl: this.config.startUrl,
|
||||
};
|
||||
this.emit('progress', progress);
|
||||
}
|
||||
}
|
||||
|
||||
function parseIntSafe(v: string | null | undefined): number | null {
|
||||
if (!v) return null;
|
||||
const n = Number.parseInt(v, 10);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
function detectContentKind(url: string, contentType: string | null): ContentKind {
|
||||
const ct = contentType?.toLowerCase() ?? '';
|
||||
if (ct.includes('text/html') || ct.includes('application/xhtml')) return 'html';
|
||||
if (ct.includes('text/css')) return 'css';
|
||||
if (ct.includes('javascript')) return 'js';
|
||||
if (ct.startsWith('image/')) return 'image';
|
||||
if (ct.includes('application/pdf')) return 'pdf';
|
||||
if (ct.includes('font/') || ct.includes('application/font')) return 'font';
|
||||
|
||||
const ext = extractExtension(url);
|
||||
return EXT_TO_KIND[ext] ?? 'other';
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { createWriteStream } from 'node:fs';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { Readable } from 'node:stream';
|
||||
import type { ProjectDb } from '@freecrawl/db';
|
||||
import type { CrawlUrlRow } from '@freecrawl/shared-types';
|
||||
|
||||
const CSV_COLUMNS: (keyof CrawlUrlRow)[] = [
|
||||
'url',
|
||||
'statusCode',
|
||||
'contentKind',
|
||||
'indexability',
|
||||
'indexabilityReason',
|
||||
'title',
|
||||
'titleLength',
|
||||
'metaDescription',
|
||||
'metaDescriptionLength',
|
||||
'h1',
|
||||
'h2Count',
|
||||
'wordCount',
|
||||
'canonical',
|
||||
'metaRobots',
|
||||
'xRobotsTag',
|
||||
'contentType',
|
||||
'contentLength',
|
||||
'responseTimeMs',
|
||||
'depth',
|
||||
'inlinks',
|
||||
'outlinks',
|
||||
'redirectTarget',
|
||||
'crawledAt',
|
||||
];
|
||||
|
||||
function escapeCsv(value: unknown): string {
|
||||
if (value === null || value === undefined) return '';
|
||||
const str = String(value);
|
||||
if (/[",\n\r]/.test(str)) {
|
||||
return `"${str.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
export async function exportUrlsToCsv(
|
||||
db: ProjectDb,
|
||||
filePath: string,
|
||||
): Promise<{ rowsWritten: number }> {
|
||||
let rowsWritten = 0;
|
||||
const header = CSV_COLUMNS.join(',') + '\n';
|
||||
|
||||
const generator = async function* (): AsyncGenerator<string> {
|
||||
yield '' + header;
|
||||
for (const row of db.iterateAllUrls()) {
|
||||
const line = CSV_COLUMNS.map((col) => escapeCsv(row[col])).join(',') + '\n';
|
||||
rowsWritten++;
|
||||
yield line;
|
||||
}
|
||||
};
|
||||
|
||||
await pipeline(Readable.from(generator()), createWriteStream(filePath, { encoding: 'utf8' }));
|
||||
|
||||
return { rowsWritten };
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import * as cheerio from 'cheerio';
|
||||
import type { DiscoveredLink } from '@freecrawl/shared-types';
|
||||
import { normalizeUrl, isSameHost } from './url-utils.js';
|
||||
|
||||
export interface ParsedPage {
|
||||
title: string | null;
|
||||
metaDescription: string | null;
|
||||
h1: string | null;
|
||||
h2Count: number;
|
||||
wordCount: number;
|
||||
canonical: string | null;
|
||||
metaRobots: string | null;
|
||||
lang: string | null;
|
||||
links: DiscoveredLink[];
|
||||
hasNoindex: boolean;
|
||||
hasNofollow: boolean;
|
||||
}
|
||||
|
||||
export function parseHtml(
|
||||
html: string,
|
||||
pageUrl: string,
|
||||
opts: { includeSubdomains?: boolean } = {},
|
||||
): ParsedPage {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const title = $('title').first().text().trim() || null;
|
||||
const metaDescription = ($('meta[name="description"]').attr('content') ?? '').trim() || null;
|
||||
const h1 = $('h1').first().text().trim() || null;
|
||||
const h2Count = $('h2').length;
|
||||
const canonical = ($('link[rel="canonical"]').attr('href') ?? '').trim() || null;
|
||||
const metaRobots = ($('meta[name="robots"]').attr('content') ?? '').trim().toLowerCase() || null;
|
||||
const lang = ($('html').attr('lang') ?? '').trim() || null;
|
||||
|
||||
const text = $('body').text().replace(/\s+/g, ' ').trim();
|
||||
const wordCount = text.length > 0 ? text.split(' ').filter(Boolean).length : 0;
|
||||
|
||||
const hasNoindex = metaRobots !== null && metaRobots.includes('noindex');
|
||||
const hasNofollow = metaRobots !== null && metaRobots.includes('nofollow');
|
||||
|
||||
const linkMap = new Map<string, DiscoveredLink>();
|
||||
$('a[href]').each((_, el) => {
|
||||
const href = $(el).attr('href');
|
||||
if (!href) return;
|
||||
const normalized = normalizeUrl(href, pageUrl);
|
||||
if (!normalized) return;
|
||||
if (!/^https?:/.test(normalized)) return;
|
||||
if (linkMap.has(normalized)) return;
|
||||
|
||||
const rel = ($(el).attr('rel') ?? '').trim().toLowerCase() || null;
|
||||
const anchor = $(el).text().replace(/\s+/g, ' ').trim().slice(0, 200) || null;
|
||||
const isInternal = isSameHost(pageUrl, normalized, opts);
|
||||
|
||||
linkMap.set(normalized, {
|
||||
fromUrl: pageUrl,
|
||||
toUrl: normalized,
|
||||
anchor,
|
||||
rel,
|
||||
isInternal,
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
title,
|
||||
metaDescription,
|
||||
h1,
|
||||
h2Count,
|
||||
wordCount,
|
||||
canonical,
|
||||
metaRobots,
|
||||
lang,
|
||||
links: [...linkMap.values()],
|
||||
hasNoindex,
|
||||
hasNofollow,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export { Crawler } from './crawler.js';
|
||||
export type { CrawlerEvents } from './crawler.js';
|
||||
export { normalizeUrl, isSameHost } from './url-utils.js';
|
||||
export { parseHtml } from './html-parser.js';
|
||||
export { exportUrlsToCsv } from './csv-export.js';
|
||||
@@ -0,0 +1,38 @@
|
||||
import { fetch as undiciFetch } from 'undici';
|
||||
import robotsParser from 'robots-parser';
|
||||
|
||||
export interface RobotsChecker {
|
||||
isAllowed(url: string): boolean;
|
||||
getCrawlDelay(): number | undefined;
|
||||
}
|
||||
|
||||
const NOOP: RobotsChecker = {
|
||||
isAllowed: () => true,
|
||||
getCrawlDelay: () => undefined,
|
||||
};
|
||||
|
||||
export async function loadRobots(origin: string, userAgent: string): Promise<RobotsChecker> {
|
||||
const robotsUrl = new URL('/robots.txt', origin).toString();
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 5000);
|
||||
try {
|
||||
const res = await undiciFetch(robotsUrl, {
|
||||
method: 'GET',
|
||||
headers: { 'user-agent': userAgent },
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (res.ok) {
|
||||
const body = await res.text();
|
||||
const parser = robotsParser(robotsUrl, body);
|
||||
return {
|
||||
isAllowed: (url: string) => parser.isAllowed(url, userAgent) ?? true,
|
||||
getCrawlDelay: () => parser.getCrawlDelay(userAgent),
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// ignore — default allow
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
return NOOP;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
export function normalizeUrl(raw: string, base?: string): string | null {
|
||||
try {
|
||||
const u = new URL(raw, base);
|
||||
u.hash = '';
|
||||
const tracking = [
|
||||
'utm_source',
|
||||
'utm_medium',
|
||||
'utm_campaign',
|
||||
'utm_term',
|
||||
'utm_content',
|
||||
'fbclid',
|
||||
'gclid',
|
||||
'mc_cid',
|
||||
'mc_eid',
|
||||
];
|
||||
for (const p of tracking) u.searchParams.delete(p);
|
||||
if (u.pathname === '') u.pathname = '/';
|
||||
if ((u.protocol === 'http:' && u.port === '80') || (u.protocol === 'https:' && u.port === '443')) {
|
||||
u.port = '';
|
||||
}
|
||||
return u.toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function isSameHost(
|
||||
urlA: string,
|
||||
urlB: string,
|
||||
opts: { includeSubdomains?: boolean } = {},
|
||||
): boolean {
|
||||
try {
|
||||
const a = new URL(urlA);
|
||||
const b = new URL(urlB);
|
||||
if (opts.includeSubdomains) {
|
||||
const root = (h: string) => h.split('.').slice(-2).join('.');
|
||||
return root(a.hostname) === root(b.hostname);
|
||||
}
|
||||
return a.hostname === b.hostname;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function extractExtension(url: string): string {
|
||||
try {
|
||||
const u = new URL(url);
|
||||
const match = u.pathname.match(/\.([a-z0-9]{1,6})$/i);
|
||||
return match ? match[1]!.toLowerCase() : '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"composite": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"references": [{ "path": "../shared-types" }, { "path": "../db" }]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "@freecrawl/db",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"clean": "rimraf dist"
|
||||
},
|
||||
"dependencies": {
|
||||
"@freecrawl/shared-types": "*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.0",
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "^5.7.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export { ProjectDb } from './project-db.js';
|
||||
export { runMigrations } from './migrations.js';
|
||||
@@ -0,0 +1,98 @@
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
interface Migration {
|
||||
version: number;
|
||||
name: string;
|
||||
up: string;
|
||||
}
|
||||
|
||||
const MIGRATIONS: Migration[] = [
|
||||
{
|
||||
version: 1,
|
||||
name: 'initial_schema',
|
||||
up: `
|
||||
CREATE TABLE IF NOT EXISTS project_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS urls (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
content_kind TEXT NOT NULL DEFAULT 'html',
|
||||
status_code INTEGER,
|
||||
status_text TEXT,
|
||||
indexability TEXT NOT NULL DEFAULT 'indexable',
|
||||
indexability_reason TEXT,
|
||||
title TEXT,
|
||||
title_length INTEGER,
|
||||
meta_description TEXT,
|
||||
meta_description_length INTEGER,
|
||||
h1 TEXT,
|
||||
h2_count INTEGER NOT NULL DEFAULT 0,
|
||||
word_count INTEGER,
|
||||
canonical TEXT,
|
||||
meta_robots TEXT,
|
||||
x_robots_tag TEXT,
|
||||
content_type TEXT,
|
||||
content_length INTEGER,
|
||||
response_time_ms INTEGER,
|
||||
depth INTEGER NOT NULL DEFAULT 0,
|
||||
inlinks INTEGER NOT NULL DEFAULT 0,
|
||||
outlinks INTEGER NOT NULL DEFAULT 0,
|
||||
redirect_target TEXT,
|
||||
crawled_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_urls_status ON urls(status_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_urls_indexability ON urls(indexability);
|
||||
CREATE INDEX IF NOT EXISTS idx_urls_content_kind ON urls(content_kind);
|
||||
CREATE INDEX IF NOT EXISTS idx_urls_depth ON urls(depth);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS links (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
from_url_id INTEGER NOT NULL REFERENCES urls(id) ON DELETE CASCADE,
|
||||
to_url TEXT NOT NULL,
|
||||
anchor TEXT,
|
||||
rel TEXT,
|
||||
is_internal INTEGER NOT NULL DEFAULT 1
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_links_from ON links(from_url_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_links_to ON links(to_url);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS headers (
|
||||
url_id INTEGER NOT NULL REFERENCES urls(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
PRIMARY KEY (url_id, name)
|
||||
);
|
||||
`,
|
||||
},
|
||||
];
|
||||
|
||||
export function runMigrations(db: DatabaseSync): void {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS schema_version (
|
||||
version INTEGER PRIMARY KEY,
|
||||
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
const currentVersion =
|
||||
(db.prepare('SELECT MAX(version) AS v FROM schema_version').get() as { v: number | null }).v ??
|
||||
0;
|
||||
|
||||
const pending = MIGRATIONS.filter((m) => m.version > currentVersion);
|
||||
for (const migration of pending) {
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
db.exec(migration.up);
|
||||
db.prepare('INSERT INTO schema_version (version) VALUES (?)').run(migration.version);
|
||||
db.exec('COMMIT');
|
||||
} catch (err) {
|
||||
db.exec('ROLLBACK');
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
import { DatabaseSync, type StatementSync } from 'node:sqlite';
|
||||
import type {
|
||||
ContentKind,
|
||||
CrawlSummary,
|
||||
CrawlUrlRow,
|
||||
DiscoveredLink,
|
||||
Indexability,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { runMigrations } from './migrations.js';
|
||||
|
||||
interface UrlRowDb {
|
||||
id: number;
|
||||
url: string;
|
||||
content_kind: ContentKind;
|
||||
status_code: number | null;
|
||||
status_text: string | null;
|
||||
indexability: Indexability;
|
||||
indexability_reason: string | null;
|
||||
title: string | null;
|
||||
title_length: number | null;
|
||||
meta_description: string | null;
|
||||
meta_description_length: number | null;
|
||||
h1: string | null;
|
||||
h2_count: number;
|
||||
word_count: number | null;
|
||||
canonical: string | null;
|
||||
meta_robots: string | null;
|
||||
x_robots_tag: string | null;
|
||||
content_type: string | null;
|
||||
content_length: number | null;
|
||||
response_time_ms: number | null;
|
||||
depth: number;
|
||||
inlinks: number;
|
||||
outlinks: number;
|
||||
redirect_target: string | null;
|
||||
crawled_at: string;
|
||||
}
|
||||
|
||||
export interface UpsertUrlInput {
|
||||
url: string;
|
||||
contentKind?: ContentKind;
|
||||
statusCode?: number | null;
|
||||
statusText?: string | null;
|
||||
indexability?: Indexability;
|
||||
indexabilityReason?: string | null;
|
||||
title?: string | null;
|
||||
metaDescription?: string | null;
|
||||
h1?: string | null;
|
||||
h2Count?: number;
|
||||
wordCount?: number | null;
|
||||
canonical?: string | null;
|
||||
metaRobots?: string | null;
|
||||
xRobotsTag?: string | null;
|
||||
contentType?: string | null;
|
||||
contentLength?: number | null;
|
||||
responseTimeMs?: number | null;
|
||||
depth: number;
|
||||
outlinks?: number;
|
||||
redirectTarget?: string | null;
|
||||
}
|
||||
|
||||
const UPSERT_URL_SQL = `
|
||||
INSERT INTO urls (
|
||||
url, content_kind, status_code, status_text, indexability, indexability_reason,
|
||||
title, title_length, meta_description, meta_description_length,
|
||||
h1, h2_count, word_count, canonical, meta_robots, x_robots_tag,
|
||||
content_type, content_length, response_time_ms, depth, outlinks, redirect_target
|
||||
) VALUES (
|
||||
:url, :content_kind, :status_code, :status_text, :indexability, :indexability_reason,
|
||||
:title, :title_length, :meta_description, :meta_description_length,
|
||||
:h1, :h2_count, :word_count, :canonical, :meta_robots, :x_robots_tag,
|
||||
:content_type, :content_length, :response_time_ms, :depth, :outlinks, :redirect_target
|
||||
)
|
||||
ON CONFLICT(url) DO UPDATE SET
|
||||
content_kind = excluded.content_kind,
|
||||
status_code = excluded.status_code,
|
||||
status_text = excluded.status_text,
|
||||
indexability = excluded.indexability,
|
||||
indexability_reason = excluded.indexability_reason,
|
||||
title = excluded.title,
|
||||
title_length = excluded.title_length,
|
||||
meta_description = excluded.meta_description,
|
||||
meta_description_length = excluded.meta_description_length,
|
||||
h1 = excluded.h1,
|
||||
h2_count = excluded.h2_count,
|
||||
word_count = excluded.word_count,
|
||||
canonical = excluded.canonical,
|
||||
meta_robots = excluded.meta_robots,
|
||||
x_robots_tag = excluded.x_robots_tag,
|
||||
content_type = excluded.content_type,
|
||||
content_length = excluded.content_length,
|
||||
response_time_ms = excluded.response_time_ms,
|
||||
depth = excluded.depth,
|
||||
outlinks = excluded.outlinks,
|
||||
redirect_target = excluded.redirect_target,
|
||||
crawled_at = CURRENT_TIMESTAMP
|
||||
`;
|
||||
|
||||
export class ProjectDb {
|
||||
private readonly db: DatabaseSync;
|
||||
private readonly stmtUpsertUrl: StatementSync;
|
||||
private readonly stmtGetUrlId: StatementSync;
|
||||
private readonly stmtInsertLink: StatementSync;
|
||||
|
||||
constructor(filePath: string) {
|
||||
this.db = new DatabaseSync(filePath);
|
||||
this.db.exec('PRAGMA journal_mode = WAL');
|
||||
this.db.exec('PRAGMA synchronous = NORMAL');
|
||||
this.db.exec('PRAGMA foreign_keys = ON');
|
||||
this.db.exec('PRAGMA temp_store = MEMORY');
|
||||
this.db.exec('PRAGMA cache_size = -64000');
|
||||
runMigrations(this.db);
|
||||
|
||||
this.stmtUpsertUrl = this.db.prepare(UPSERT_URL_SQL);
|
||||
this.stmtGetUrlId = this.db.prepare('SELECT id FROM urls WHERE url = ?');
|
||||
this.stmtInsertLink = this.db.prepare(
|
||||
'INSERT INTO links (from_url_id, to_url, anchor, rel, is_internal) VALUES (?, ?, ?, ?, ?)',
|
||||
);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.db.exec('DELETE FROM links; DELETE FROM headers; DELETE FROM urls;');
|
||||
}
|
||||
|
||||
countUrls(): number {
|
||||
return (this.db.prepare('SELECT COUNT(*) AS c FROM urls').get() as { c: number }).c;
|
||||
}
|
||||
|
||||
hasUrl(url: string): boolean {
|
||||
return this.db.prepare('SELECT 1 FROM urls WHERE url = ?').get(url) !== undefined;
|
||||
}
|
||||
|
||||
upsertUrl(input: UpsertUrlInput): number {
|
||||
const params = {
|
||||
url: input.url,
|
||||
content_kind: input.contentKind ?? 'html',
|
||||
status_code: input.statusCode ?? null,
|
||||
status_text: input.statusText ?? null,
|
||||
indexability: input.indexability ?? 'indexable',
|
||||
indexability_reason: input.indexabilityReason ?? null,
|
||||
title: input.title ?? null,
|
||||
title_length: input.title?.length ?? null,
|
||||
meta_description: input.metaDescription ?? null,
|
||||
meta_description_length: input.metaDescription?.length ?? null,
|
||||
h1: input.h1 ?? null,
|
||||
h2_count: input.h2Count ?? 0,
|
||||
word_count: input.wordCount ?? null,
|
||||
canonical: input.canonical ?? null,
|
||||
meta_robots: input.metaRobots ?? null,
|
||||
x_robots_tag: input.xRobotsTag ?? null,
|
||||
content_type: input.contentType ?? null,
|
||||
content_length: input.contentLength ?? null,
|
||||
response_time_ms: input.responseTimeMs ?? null,
|
||||
depth: input.depth,
|
||||
outlinks: input.outlinks ?? 0,
|
||||
redirect_target: input.redirectTarget ?? null,
|
||||
};
|
||||
|
||||
this.stmtUpsertUrl.run(params);
|
||||
const row = this.stmtGetUrlId.get(input.url) as { id: number } | undefined;
|
||||
return row?.id ?? 0;
|
||||
}
|
||||
|
||||
insertLinks(fromUrlId: number, links: DiscoveredLink[]): void {
|
||||
if (links.length === 0) return;
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const link of links) {
|
||||
this.stmtInsertLink.run(
|
||||
fromUrlId,
|
||||
link.toUrl,
|
||||
link.anchor,
|
||||
link.rel,
|
||||
link.isInternal ? 1 : 0,
|
||||
);
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (err) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
recomputeInlinks(): void {
|
||||
this.db.exec(`
|
||||
UPDATE urls SET inlinks = (
|
||||
SELECT COUNT(*) FROM links l
|
||||
WHERE l.to_url = urls.url AND l.is_internal = 1
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
queryUrls(params: {
|
||||
limit: number;
|
||||
offset: number;
|
||||
filter?: 'all' | 'internal' | 'external' | 'errors' | 'redirects';
|
||||
search?: string;
|
||||
sortBy?: string;
|
||||
sortDir?: 'asc' | 'desc';
|
||||
}): { rows: CrawlUrlRow[]; total: number } {
|
||||
const where: string[] = [];
|
||||
const args: (string | number)[] = [];
|
||||
|
||||
switch (params.filter) {
|
||||
case 'errors':
|
||||
where.push('(status_code >= 400 OR status_code IS NULL)');
|
||||
break;
|
||||
case 'redirects':
|
||||
where.push('status_code >= 300 AND status_code < 400');
|
||||
break;
|
||||
case 'internal':
|
||||
where.push("content_kind = 'html'");
|
||||
break;
|
||||
}
|
||||
|
||||
if (params.search) {
|
||||
where.push('(url LIKE ? OR title LIKE ?)');
|
||||
const like = `%${params.search}%`;
|
||||
args.push(like, like);
|
||||
}
|
||||
|
||||
const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '';
|
||||
|
||||
const validSort = new Set([
|
||||
'id',
|
||||
'url',
|
||||
'status_code',
|
||||
'title_length',
|
||||
'meta_description_length',
|
||||
'word_count',
|
||||
'response_time_ms',
|
||||
'depth',
|
||||
'inlinks',
|
||||
'outlinks',
|
||||
'crawled_at',
|
||||
]);
|
||||
const sortCol =
|
||||
params.sortBy && validSort.has(this.toSnake(params.sortBy))
|
||||
? this.toSnake(params.sortBy)
|
||||
: 'id';
|
||||
const sortDir = params.sortDir === 'desc' ? 'DESC' : 'ASC';
|
||||
|
||||
const totalRow = this.db
|
||||
.prepare(`SELECT COUNT(*) AS c FROM urls ${whereSql}`)
|
||||
.get(...args) as { c: number };
|
||||
|
||||
const rowsDb = this.db
|
||||
.prepare(
|
||||
`SELECT * FROM urls ${whereSql} ORDER BY ${sortCol} ${sortDir} LIMIT ? OFFSET ?`,
|
||||
)
|
||||
.all(...args, params.limit, params.offset) as unknown as UrlRowDb[];
|
||||
|
||||
return { rows: rowsDb.map(this.rowFromDb), total: totalRow.c };
|
||||
}
|
||||
|
||||
getSummary(): CrawlSummary {
|
||||
const total = this.countUrls();
|
||||
const byStatus: Record<string, number> = {};
|
||||
const byContentKind: Record<string, number> = {};
|
||||
const byIndexability: Record<string, number> = {};
|
||||
|
||||
for (const row of this.db
|
||||
.prepare(
|
||||
`SELECT COALESCE(CAST(status_code AS TEXT), 'unknown') AS status, COUNT(*) AS c FROM urls GROUP BY status`,
|
||||
)
|
||||
.all() as unknown as { status: string; c: number }[]) {
|
||||
byStatus[row.status] = row.c;
|
||||
}
|
||||
|
||||
for (const row of this.db
|
||||
.prepare('SELECT content_kind, COUNT(*) AS c FROM urls GROUP BY content_kind')
|
||||
.all() as unknown as { content_kind: ContentKind; c: number }[]) {
|
||||
byContentKind[row.content_kind] = row.c;
|
||||
}
|
||||
|
||||
for (const row of this.db
|
||||
.prepare('SELECT indexability, COUNT(*) AS c FROM urls GROUP BY indexability')
|
||||
.all() as unknown as { indexability: string; c: number }[]) {
|
||||
byIndexability[row.indexability] = row.c;
|
||||
}
|
||||
|
||||
const avg = this.db
|
||||
.prepare('SELECT AVG(response_time_ms) AS avg FROM urls WHERE response_time_ms IS NOT NULL')
|
||||
.get() as { avg: number | null };
|
||||
const bytes = this.db
|
||||
.prepare('SELECT COALESCE(SUM(content_length), 0) AS total FROM urls')
|
||||
.get() as { total: number };
|
||||
|
||||
return {
|
||||
total,
|
||||
byStatus,
|
||||
byContentKind: byContentKind as CrawlSummary['byContentKind'],
|
||||
byIndexability,
|
||||
avgResponseTimeMs: Math.round(avg.avg ?? 0),
|
||||
totalBytes: bytes.total,
|
||||
};
|
||||
}
|
||||
|
||||
*iterateAllUrls(): IterableIterator<CrawlUrlRow> {
|
||||
const rows = this.db.prepare('SELECT * FROM urls ORDER BY id').all() as unknown as UrlRowDb[];
|
||||
for (const row of rows) {
|
||||
yield this.rowFromDb(row);
|
||||
}
|
||||
}
|
||||
|
||||
private rowFromDb = (r: UrlRowDb): CrawlUrlRow => ({
|
||||
id: r.id,
|
||||
url: r.url,
|
||||
contentKind: r.content_kind,
|
||||
statusCode: r.status_code,
|
||||
statusText: r.status_text,
|
||||
indexability: r.indexability,
|
||||
indexabilityReason: r.indexability_reason,
|
||||
title: r.title,
|
||||
titleLength: r.title_length,
|
||||
metaDescription: r.meta_description,
|
||||
metaDescriptionLength: r.meta_description_length,
|
||||
h1: r.h1,
|
||||
h2Count: r.h2_count,
|
||||
wordCount: r.word_count,
|
||||
canonical: r.canonical,
|
||||
metaRobots: r.meta_robots,
|
||||
xRobotsTag: r.x_robots_tag,
|
||||
contentType: r.content_type,
|
||||
contentLength: r.content_length,
|
||||
responseTimeMs: r.response_time_ms,
|
||||
depth: r.depth,
|
||||
inlinks: r.inlinks,
|
||||
outlinks: r.outlinks,
|
||||
redirectTarget: r.redirect_target,
|
||||
crawledAt: r.crawled_at,
|
||||
});
|
||||
|
||||
private toSnake(s: string): string {
|
||||
return s.replace(/([A-Z])/g, '_$1').toLowerCase();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"composite": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"references": [{ "path": "../shared-types" }]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "@freecrawl/shared-types",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"clean": "rimraf dist"
|
||||
},
|
||||
"devDependencies": {
|
||||
"rimraf": "^6.0.1",
|
||||
"typescript": "^5.7.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
export type Indexability =
|
||||
| 'indexable'
|
||||
| 'non-indexable:noindex'
|
||||
| 'non-indexable:canonical'
|
||||
| 'non-indexable:robots-blocked'
|
||||
| 'non-indexable:redirect'
|
||||
| 'non-indexable:client-error'
|
||||
| 'non-indexable:server-error';
|
||||
|
||||
export type ContentKind = 'html' | 'css' | 'js' | 'image' | 'pdf' | 'font' | 'other';
|
||||
|
||||
export interface CrawlUrlRow {
|
||||
id: number;
|
||||
url: string;
|
||||
contentKind: ContentKind;
|
||||
statusCode: number | null;
|
||||
statusText: string | null;
|
||||
indexability: Indexability;
|
||||
indexabilityReason: string | null;
|
||||
title: string | null;
|
||||
titleLength: number | null;
|
||||
metaDescription: string | null;
|
||||
metaDescriptionLength: number | null;
|
||||
h1: string | null;
|
||||
h2Count: number;
|
||||
wordCount: number | null;
|
||||
canonical: string | null;
|
||||
metaRobots: string | null;
|
||||
xRobotsTag: string | null;
|
||||
contentType: string | null;
|
||||
contentLength: number | null;
|
||||
responseTimeMs: number | null;
|
||||
depth: number;
|
||||
inlinks: number;
|
||||
outlinks: number;
|
||||
redirectTarget: string | null;
|
||||
crawledAt: string;
|
||||
}
|
||||
|
||||
export interface CrawlConfig {
|
||||
startUrl: string;
|
||||
maxDepth: number;
|
||||
maxUrls: number;
|
||||
maxConcurrency: number;
|
||||
maxRps: number;
|
||||
requestTimeoutMs: number;
|
||||
userAgent: string;
|
||||
followRedirects: boolean;
|
||||
respectRobotsTxt: boolean;
|
||||
crawlExternal: boolean;
|
||||
includeSubdomains: boolean;
|
||||
acceptLanguage: string;
|
||||
}
|
||||
|
||||
export interface CrawlProgress {
|
||||
discovered: number;
|
||||
crawled: number;
|
||||
failed: number;
|
||||
pending: number;
|
||||
currentDepth: number;
|
||||
urlsPerSecond: number;
|
||||
elapsedMs: number;
|
||||
avgResponseTimeMs: number;
|
||||
running: boolean;
|
||||
startUrl: string;
|
||||
}
|
||||
|
||||
export interface CrawlSummary {
|
||||
total: number;
|
||||
byStatus: Record<string, number>;
|
||||
byContentKind: Record<ContentKind, number>;
|
||||
byIndexability: Record<string, number>;
|
||||
avgResponseTimeMs: number;
|
||||
totalBytes: number;
|
||||
}
|
||||
|
||||
export interface DiscoveredLink {
|
||||
fromUrl: string;
|
||||
toUrl: string;
|
||||
anchor: string | null;
|
||||
rel: string | null;
|
||||
isInternal: boolean;
|
||||
}
|
||||
|
||||
export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
|
||||
startUrl: '',
|
||||
maxDepth: 10,
|
||||
maxUrls: 100_000,
|
||||
maxConcurrency: 5,
|
||||
maxRps: 5,
|
||||
requestTimeoutMs: 20_000,
|
||||
userAgent: 'FreeCrawlSEO/0.1 (+https://github.com/kemalai/FreeCrawl-SEO-Tool)',
|
||||
followRedirects: true,
|
||||
respectRobotsTxt: true,
|
||||
crawlExternal: false,
|
||||
includeSubdomains: false,
|
||||
acceptLanguage: 'tr,en;q=0.8',
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from './crawl.js';
|
||||
export * from './ipc.js';
|
||||
export * from './project.js';
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { CrawlConfig, CrawlProgress, CrawlSummary, CrawlUrlRow } from './crawl.js';
|
||||
|
||||
export const IPC = {
|
||||
crawlStart: 'crawl:start',
|
||||
crawlStop: 'crawl:stop',
|
||||
crawlProgress: 'crawl:progress',
|
||||
crawlDone: 'crawl:done',
|
||||
crawlError: 'crawl:error',
|
||||
urlsQuery: 'urls:query',
|
||||
urlsCount: 'urls:count',
|
||||
summaryGet: 'summary:get',
|
||||
exportCsv: 'export:csv',
|
||||
appVersion: 'app:version',
|
||||
} as const;
|
||||
|
||||
export type IpcChannel = (typeof IPC)[keyof typeof IPC];
|
||||
|
||||
export interface UrlsQueryInput {
|
||||
limit: number;
|
||||
offset: number;
|
||||
filter?: 'all' | 'internal' | 'external' | 'errors' | 'redirects';
|
||||
search?: string;
|
||||
sortBy?: keyof CrawlUrlRow;
|
||||
sortDir?: 'asc' | 'desc';
|
||||
}
|
||||
|
||||
export interface UrlsQueryResult {
|
||||
rows: CrawlUrlRow[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface ExportCsvInput {
|
||||
filePath: string;
|
||||
filter?: UrlsQueryInput['filter'];
|
||||
}
|
||||
|
||||
export interface ExportCsvResult {
|
||||
filePath: string;
|
||||
rowsWritten: number;
|
||||
}
|
||||
|
||||
export interface FreeCrawlApi {
|
||||
crawlStart(config: CrawlConfig): Promise<void>;
|
||||
crawlStop(): Promise<void>;
|
||||
urlsQuery(input: UrlsQueryInput): Promise<UrlsQueryResult>;
|
||||
summaryGet(): Promise<CrawlSummary>;
|
||||
exportCsv(input: ExportCsvInput): Promise<ExportCsvResult>;
|
||||
appVersion(): Promise<string>;
|
||||
onProgress(cb: (p: CrawlProgress) => void): () => void;
|
||||
onDone(cb: (summary: CrawlSummary) => void): () => void;
|
||||
onError(cb: (message: string) => void): () => void;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export interface ProjectMeta {
|
||||
version: string;
|
||||
createdAt: string;
|
||||
startUrl: string;
|
||||
notes: string | null;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"composite": true
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"lib": ["ES2022"],
|
||||
"strict": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"skipLibCheck": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"verbatimModuleSyntax": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"files": [],
|
||||
"references": [
|
||||
{ "path": "./packages/shared-types" },
|
||||
{ "path": "./packages/db" },
|
||||
{ "path": "./packages/core" },
|
||||
{ "path": "./apps/desktop" },
|
||||
{ "path": "./apps/cli" }
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user