From f32d32a5a27fe17197dad5106967ecdf07b526f4 Mon Sep 17 00:00:00 2001 From: kemalai Date: Sat, 16 May 2026 15:07:12 +0300 Subject: [PATCH] feat: scheduled crawl + indexability/content-kind reports + linux/mac launcher --- FreeCrawl-SEO-Tool-Start.sh | 190 ++++++++++ apps/cli/package.json | 2 +- apps/desktop/package.json | 2 +- apps/desktop/src/main/db-reader-worker.ts | 2 + apps/desktop/src/main/index.ts | 136 +++++++ apps/desktop/src/main/menu.ts | 6 + apps/desktop/src/main/scheduler.ts | 186 +++++++++ apps/desktop/src/preload/index.ts | 11 + apps/desktop/src/renderer/src/App.tsx | 6 + .../renderer/src/components/ReportsDialog.tsx | 83 +++++ .../src/components/ScheduledCrawlDialog.tsx | 352 ++++++++++++++++++ apps/mcp-server/package.json | 2 +- package.json | 2 +- packages/core/package.json | 2 +- packages/db/package.json | 2 +- packages/db/src/project-db.ts | 36 ++ packages/shared-types/package.json | 2 +- packages/shared-types/src/ipc.ts | 63 ++++ 18 files changed, 1078 insertions(+), 7 deletions(-) create mode 100755 FreeCrawl-SEO-Tool-Start.sh create mode 100644 apps/desktop/src/main/scheduler.ts create mode 100644 apps/desktop/src/renderer/src/components/ScheduledCrawlDialog.tsx diff --git a/FreeCrawl-SEO-Tool-Start.sh b/FreeCrawl-SEO-Tool-Start.sh new file mode 100755 index 0000000..63fb90f --- /dev/null +++ b/FreeCrawl-SEO-Tool-Start.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# FreeCrawl SEO Tool — Launcher (macOS / Linux mirror of FreeCrawl-SEO-Tool-Start.bat) +# +# İlk çalıştırma: +# 1) chmod +x FreeCrawl-SEO-Tool-Start.sh +# 2) ./FreeCrawl-SEO-Tool-Start.sh +# +# .bat tarafındaki Windows-spesifik bölümler (color, title, pause) yerine +# burada ANSI renkleri + sondaki "Enter ile çık" promptu kullanılır. + +set -u + +# Çift-tıklatma ile başlatılan terminallerde script kendi dizininden bağımsız +# olarak fire edilebilir — symlink'i de takip ederek gerçek dizine cd ediyoruz. +SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null || echo "${BASH_SOURCE[0]}")")" && pwd)" +cd "$SCRIPT_DIR" + +# Renk kısayolları — `tput` çıkışa terminal yoksa boş döner, böylece bir log +# dosyasına pipe'lansa bile escape kodları output'u kirletmez. +if [ -t 1 ]; then + C_RESET="$(tput sgr0 2>/dev/null || echo)" + C_OK="$(tput setaf 2 2>/dev/null || echo)" + C_ERR="$(tput setaf 1 2>/dev/null || echo)" + C_INFO="$(tput setaf 6 2>/dev/null || echo)" + C_HEAD="$(tput setaf 14 2>/dev/null || tput setaf 6 2>/dev/null || echo)" +else + C_RESET="" + C_OK="" + C_ERR="" + C_INFO="" + C_HEAD="" +fi + +ok() { printf '%s[OK]%s %s\n' "$C_OK" "$C_RESET" "$*"; } +info() { printf '%s[BILGI]%s %s\n' "$C_INFO" "$C_RESET" "$*"; } +err() { printf '%s[HATA]%s %s\n' "$C_ERR" "$C_RESET" "$*" >&2; } +head_() { printf '%s%s%s\n' "$C_HEAD" "$*" "$C_RESET"; } +hr() { head_ "============================================================"; } +sub() { head_ "------------------------------------------------------------"; } + +# Çift-tıklatma akışlarında pencerenin anında kapanmaması için sondaki +# "Enter ile çık" promptu. Stdin TTY değilse (CI, pipe) atlanır. +final_pause() { + if [ -t 0 ]; then + printf '\n' + read -r -p "Cikmak icin Enter'a basin..." _ || true + fi +} + +trap 'final_pause' EXIT + +# ============================================================================ +hr +head_ " FreeCrawl SEO Tool - Launcher" +hr +echo + +# ---- 1) Node.js kontrolu ---------------------------------------------------- +if ! command -v node >/dev/null 2>&1; then + err "Node.js bulunamadi." + echo + echo "FreeCrawl SEO Tool Node.js 22+ gerektirir." + echo "Lutfen https://nodejs.org/ adresinden LTS surumunu indirip kurun." + echo "Kurulumdan sonra bu betigi tekrar calistirin." + exit 1 +fi +NODE_VERSION="$(node -v)" +ok "Node.js bulundu: ${NODE_VERSION}" + +# ---- 2) npm kontrolu -------------------------------------------------------- +if ! command -v npm >/dev/null 2>&1; then + err "npm bulunamadi. Node.js kurulumunuz bozuk olabilir." + exit 1 +fi +NPM_VERSION="$(npm -v)" +ok "npm bulundu: v${NPM_VERSION}" +echo + +# ---- 3) Bagimlilik kontrolu ------------------------------------------------- +NEED_INSTALL=0 +[ -d "node_modules" ] || NEED_INSTALL=1 +[ -d "apps/desktop/node_modules" ] || NEED_INSTALL=1 +[ -d "node_modules/electron" ] || NEED_INSTALL=1 + +if [ "$NEED_INSTALL" -eq 1 ]; then + info "Gerekli kutuphaneler yuklu degil veya eksik." + echo + echo "Bu islem ilk calistirmada birkac dakika surebilir ve yaklasik" + echo "500-800 MB disk alani kullanir." + echo + CONFIRM="" + read -r -p "Kutuphaneleri simdi yuklemek istiyor musunuz? (E/H): " CONFIRM || true + case "${CONFIRM:-}" in + E|e|Y|y) + echo + sub + head_ " npm install calisiyor..." + sub + if ! npm install; then + err "npm install basarisiz oldu. Internet baglantinizi kontrol edin veya hata mesajini inceleyin." + exit 1 + fi + echo + ok "Kutuphaneler basariyla yuklendi." + echo + ;; + *) + echo + echo "Kurulum iptal edildi. Program baslatilamaz." + exit 1 + ;; + esac +else + ok "Gerekli kutuphaneler yuklu." + echo +fi + +# ---- 4) Paylasilan workspace paketlerini derle ------------------------------ +NEED_BUILD=0 +[ -f "packages/shared-types/dist/index.js" ] || NEED_BUILD=1 +[ -f "packages/db/dist/index.js" ] || NEED_BUILD=1 +[ -f "packages/core/dist/index.js" ] || NEED_BUILD=1 + +if [ "$NEED_BUILD" -eq 1 ]; then + sub + head_ " Paylasilan paketler derleniyor (ilk calisma)..." + sub + if ! npx tsc -b; then + err "Paket derlemesi basarisiz oldu." + exit 1 + fi + ok "Paketler hazir." + echo +else + ok "Paylasilan paketler hazir." + echo +fi + +# ---- 4b) Desktop uygulamasini production build et --------------------------- +# Dev mod (electron-vite dev) ilk acilista 1700+ modulu on-demand transform +# ediyor ve 20-30 sn cold start suruyor. Production build'de tum moduller +# tek bir bundle'a derlenmis oldugu icin acilis 1-2 sn'ye iniyor. +NEED_DESKTOP_BUILD=0 +[ -f "apps/desktop/out/main/index.js" ] || NEED_DESKTOP_BUILD=1 +[ -f "apps/desktop/out/preload/index.js" ] || NEED_DESKTOP_BUILD=1 +[ -f "apps/desktop/out/renderer/index.html" ] || NEED_DESKTOP_BUILD=1 + +if [ "$NEED_DESKTOP_BUILD" -eq 1 ]; then + sub + head_ " Desktop uygulamasi build ediliyor (production)..." + echo " Bu islem yaklasik 10-15 sn surer ve sadece ilk calistirmada" + echo " (veya kod degisikligi sonrasi) yeniden yapilir." + sub + if ! npm --workspace apps/desktop run build; then + err "Desktop build basarisiz oldu." + exit 1 + fi + ok "Desktop bundle hazir." + echo +else + ok "Desktop bundle hazir." + echo +fi + +# ---- 5) Uygulamayi baslat --------------------------------------------------- +hr +head_ " FreeCrawl SEO Tool baslatiliyor (production)..." +echo " Bu pencereyi KAPATMAYIN - uygulamanin yasam dongusu buna bagli." +hr +echo + +# `node:sqlite` Node 24'te kararli ama hala "experimental" bayragi tasiyor +# ve modul yuklenirken ExperimentalWarning basiyor. NODE_NO_WARNINGS=1 +# ile bu kozmetik uyariyi sustururuz; gercek hatalar (TypeError vs.) +# yine konsola yazilir. +export NODE_NO_WARNINGS=1 + +npm --workspace apps/desktop run start +APP_EXIT=$? + +echo +sub +if [ "$APP_EXIT" -eq 0 ]; then + head_ " Uygulama kapatildi." +else + err "Uygulama hatayla sonlandi (exit code: ${APP_EXIT})." +fi +sub + +exit "$APP_EXIT" diff --git a/apps/cli/package.json b/apps/cli/package.json index 2795266..7818926 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/cli", - "version": "0.4.0", + "version": "0.4.1", "private": true, "type": "module", "bin": { diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 0562fce..9f26f76 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/desktop", - "version": "0.4.0", + "version": "0.4.1", "private": true, "description": "FreeCrawl SEO — desktop application", "main": "./out/main/index.js", diff --git a/apps/desktop/src/main/db-reader-worker.ts b/apps/desktop/src/main/db-reader-worker.ts index d350584..410b7dd 100644 --- a/apps/desktop/src/main/db-reader-worker.ts +++ b/apps/desktop/src/main/db-reader-worker.ts @@ -90,6 +90,8 @@ const ALLOWED_METHODS = new Set([ 'sitemapOrphans', 'getPagesPerDirectory', 'getStatusCodeHistogram', + 'getIndexabilityDistribution', + 'getContentKindDistribution', 'getDepthHistogram', 'getResponseTimeHistogram', 'countUrls', diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 6a94644..3f52d77 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -64,6 +64,8 @@ import { type SettingsExportInput, type SettingsExportResult, type SettingsImportResult, + type ScheduleEntry, + type ScheduleSpec, type PagesPerDirectoryInput, type ImagesQueryInput, type ImagesQueryResult, @@ -115,6 +117,13 @@ import { stopMcpBridge, type McpStartCrawlInput, } from './mcp-bridge.js'; +import { + claimIfDue, + getSchedule, + recordFireResult, + setSchedule, + type SchedulerStore, +} from './scheduler.js'; import type { ProjectDb as ProjectDbType } from '@freecrawl/db'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -206,6 +215,93 @@ function fireDataChanged(): void { mainWindow?.webContents.send(IPC.dataChanged); } +/** + * Schedule-store adapter — bridges the scheduler module to the prefs + * cache. `load()` always returns a fresh object reference so the + * module's mutate-then-save flow can't leak references back into the + * prefs cache without an explicit `save()`. + */ +const schedulerStore: SchedulerStore = { + load() { + const raw = prefsCache['schedules']; + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + // Shallow clone — callers mutate this map. + return { ...(raw as Record) }; + } + return {}; + }, + save(map) { + prefsCache['schedules'] = map; + schedulePrefsWrite(); + }, +}; + +let schedulerTickTimer: NodeJS.Timeout | null = null; +/** Set to true at the moment the scheduler tick calls `launchCrawl`, + * cleared on the next crawl-done / crawl-error event. Lets the + * done/error handlers update the schedule's `lastStatus` without + * duplicating the launch path for user-initiated crawls. */ +let crawlerStartedByScheduler = false; + +/** + * Scheduler tick — invoked every 60 s while the app is open. Checks + * whether the currently-loaded project has a due schedule and, if so, + * launches the crawl using the last-used CrawlConfig. + * + * No-ops when: + * - No project is loaded + * - A crawl is already running (we don't preempt a manual crawl) + * - The schedule isn't due yet + * - No CrawlConfig has ever been saved (we'd have nothing to fire) + */ +function schedulerTick(): void { + if (!currentProjectPath) return; + if (activeCrawler) return; + const entry = claimIfDue(schedulerStore, currentProjectPath); + if (!entry) return; + if (!lastCrawlConfig) { + // No saved config — record a failure so the dialog surfaces the + // condition rather than silently retrying on the next tick. + recordFireResult(schedulerStore, currentProjectPath, 'failure'); + logger.log( + 'warn', + 'scheduler', + `Schedule due for ${currentProjectPath} but no CrawlConfig saved yet — run a crawl manually once first.`, + ); + return; + } + if (!crawlController) { + recordFireResult(schedulerStore, currentProjectPath, 'failure'); + return; + } + logger.log( + 'info', + 'scheduler', + `Scheduled crawl firing: ${lastCrawlConfig.startUrl} (cadence=${entry.spec.cadence})`, + ); + crawlerStartedByScheduler = true; + crawlController.launchCrawl(lastCrawlConfig); +} + +function startScheduler(): void { + if (schedulerTickTimer) return; + // 60 s tick is granular enough for hourly/daily/weekly cadences and + // for the 15-min minimum custom interval. Wall-clock drift across an + // hour-long browser session is negligible at this resolution. + schedulerTickTimer = setInterval(schedulerTick, 60_000); + // Fire once shortly after startup so a daily schedule whose target + // time-of-day has already passed today doesn't wait 60 s before + // checking. 3 s lets the renderer mount + initial DB load finish. + setTimeout(schedulerTick, 3_000); +} + +function stopScheduler(): void { + if (schedulerTickTimer) { + clearInterval(schedulerTickTimer); + schedulerTickTimer = null; + } +} + /** * Per-app-session set of diagnostic categories already surfaced in a * popup. Cleared when the user starts a fresh crawl so re-running after @@ -1462,6 +1558,18 @@ function registerIpc(): void { ), ); + ipcMain.handle(IPC.reportsIndexabilityDistribution, () => + callReaderOrFallback('getIndexabilityDistribution', [], () => + getDb().getIndexabilityDistribution(), + ), + ); + + ipcMain.handle(IPC.reportsContentKindDistribution, () => + callReaderOrFallback('getContentKindDistribution', [], () => + getDb().getContentKindDistribution(), + ), + ); + ipcMain.handle(IPC.reportsDepthHistogram, () => callReaderOrFallback('getDepthHistogram', [], () => getDb().getDepthHistogram()), ); @@ -1660,6 +1768,15 @@ function registerIpc(): void { }, ); + // V1 Faz 6 — Scheduled crawl IPC. One schedule per project, stored + // app-level under the `schedules` prefs key keyed by project path. + ipcMain.handle(IPC.scheduleGet, () => { + return getSchedule(schedulerStore, currentProjectPath); + }); + ipcMain.handle(IPC.scheduleSet, (_e, spec: ScheduleSpec | null) => { + return setSchedule(schedulerStore, currentProjectPath, spec); + }); + // Stream new entries to the logs window in coalesced batches. A heavy // crawl emits 100–300 logs/s; sending each as its own IPC message // saturated the renderer's event loop and caused visible UI stutters @@ -1729,6 +1846,15 @@ function registerIpc(): void { mainWindow?.webContents.send(IPC.crawlDone, summary); activeCrawler = null; freezeWatchdog.setMainOp('idle'); + // V1 Faz 6 — if this run was kicked off by the scheduler, record + // the result so the dialog's "Last status" row reflects the + // outcome. The schedule has already been re-armed (nextFiresAt + // advanced) at claim time; we only need to flip the status from + // `running` to a terminal value. + if (crawlerStartedByScheduler) { + crawlerStartedByScheduler = false; + recordFireResult(schedulerStore, currentProjectPath, 'success'); + } if (Notification.isSupported() && !mainWindow?.isFocused()) { try { new Notification({ @@ -1746,6 +1872,10 @@ function registerIpc(): void { logger.log('error', 'crawler', msg); mainWindow?.webContents.send(IPC.crawlError, msg); maybeShowDiagnosticDialog(msg); + if (crawlerStartedByScheduler) { + crawlerStartedByScheduler = false; + recordFireResult(schedulerStore, currentProjectPath, 'failure'); + } }); crawler.on('warn', (msg: string) => { if (activeCrawler !== crawler) return; @@ -3074,6 +3204,11 @@ void app.whenReady().then(async () => { void checkForUpdates({ silent: true }); }, 8_000); + // V1 Faz 6 — start the in-app crawl scheduler. The tick is a no-op + // until the user configures a schedule via the Scheduled Crawl + // dialog, so it's safe to leave running for the lifetime of the app. + startScheduler(); + app.on('activate', () => { if (BrowserWindow.getAllWindows().length === 0) createWindow(); }); @@ -3081,6 +3216,7 @@ void app.whenReady().then(async () => { app.on('window-all-closed', () => { activeCrawler?.stop(); + stopScheduler(); // Tear down the MCP bridge first so any in-flight HTTP calls fail // fast rather than racing with the DB shutdown that follows. try { diff --git a/apps/desktop/src/main/menu.ts b/apps/desktop/src/main/menu.ts index 98de022..1fa83f4 100644 --- a/apps/desktop/src/main/menu.ts +++ b/apps/desktop/src/main/menu.ts @@ -105,6 +105,12 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu { label: 'Compare With Project…', click: () => send('compare-with-project'), }, + { + label: 'Scheduled Crawl…', + toolTip: + 'Set up an in-app recurring crawl for the currently-open project. Fires only while FreeCrawl is open; use the CLI + OS scheduler for triggers that survive restarts.', + click: () => send('open-scheduled-crawl'), + }, { label: 'Save Project As…', accelerator: 'CmdOrCtrl+Shift+S', diff --git a/apps/desktop/src/main/scheduler.ts b/apps/desktop/src/main/scheduler.ts new file mode 100644 index 0000000..dd3eb4a --- /dev/null +++ b/apps/desktop/src/main/scheduler.ts @@ -0,0 +1,186 @@ +/** + * In-app crawl scheduler. One schedule per project, persisted in + * `userData/preferences.json` under the `schedules` key as a map keyed + * by absolute `.seoproject` path: + * + * "schedules": { + * "/abs/path/project.seoproject": { spec: {...}, status: {...} } + * } + * + * The main process ticks every 60 s and fires the schedule for the + * currently-loaded project when due (`nextFiresAt <= now`) — and only + * when no crawl is already in progress. Closed-app firing is out of + * scope; the OS-scheduler-via-CLI path is V2. + */ + +import type { ScheduleEntry, ScheduleSpec, ScheduleStatus } from '@freecrawl/shared-types'; + +/** + * Minimum interval the `custom` cadence will honour. Anything below this + * is clamped — prevents an accidental "every 1 minute" schedule that + * would saturate a target server. + */ +const MIN_CUSTOM_INTERVAL_MIN = 15; + +export interface SchedulerStore { + /** Read the schedules map. Caller mutates and writes back via `save`. */ + load: () => Record; + save: (map: Record) => void; +} + +export function getSchedule( + store: SchedulerStore, + projectPath: string, +): ScheduleEntry | null { + if (!projectPath) return null; + const map = store.load(); + return map[projectPath] ?? null; +} + +export function setSchedule( + store: SchedulerStore, + projectPath: string, + spec: ScheduleSpec | null, + now: number = Date.now(), +): ScheduleEntry | null { + if (!projectPath) return null; + const map = store.load(); + if (spec === null) { + if (map[projectPath]) { + delete map[projectPath]; + store.save(map); + } + return null; + } + const normalized = normalizeSpec(spec); + const previousStatus = map[projectPath]?.status; + const entry: ScheduleEntry = { + spec: normalized, + status: { + lastFiredAt: previousStatus?.lastFiredAt ?? null, + lastStatus: previousStatus?.lastStatus ?? null, + nextFiresAt: normalized.enabled ? computeNextFire(normalized, now, null) : null, + }, + }; + map[projectPath] = entry; + store.save(map); + return entry; +} + +/** + * Determine which schedule (if any) is due to fire for the given + * project. Returns the updated entry with status flipped to `running` + * and `nextFiresAt` re-computed for the *next* tick so the same + * schedule can't double-fire while a crawl is in progress. + */ +export function claimIfDue( + store: SchedulerStore, + projectPath: string, + now: number = Date.now(), +): ScheduleEntry | null { + if (!projectPath) return null; + const map = store.load(); + const entry = map[projectPath]; + if (!entry || !entry.spec.enabled) return null; + const due = entry.status.nextFiresAt; + if (due === null || due > now) return null; + const nextStatus: ScheduleStatus = { + lastFiredAt: now, + lastStatus: 'running', + nextFiresAt: computeNextFire(entry.spec, now, now), + }; + const updated: ScheduleEntry = { spec: entry.spec, status: nextStatus }; + map[projectPath] = updated; + store.save(map); + return updated; +} + +export function recordFireResult( + store: SchedulerStore, + projectPath: string, + result: 'success' | 'failure', +): void { + if (!projectPath) return; + const map = store.load(); + const entry = map[projectPath]; + if (!entry) return; + entry.status.lastStatus = result; + map[projectPath] = entry; + store.save(map); +} + +function normalizeSpec(spec: ScheduleSpec): ScheduleSpec { + const out: ScheduleSpec = { + enabled: !!spec.enabled, + cadence: spec.cadence, + }; + if (spec.cadence === 'custom') { + const raw = Math.floor(spec.intervalMinutes ?? MIN_CUSTOM_INTERVAL_MIN); + out.intervalMinutes = Math.max(MIN_CUSTOM_INTERVAL_MIN, raw); + } + if (spec.cadence === 'daily' || spec.cadence === 'weekly') { + out.hourOfDay = clampInt(spec.hourOfDay ?? 3, 0, 23); + out.minuteOfHour = clampInt(spec.minuteOfHour ?? 0, 0, 59); + } + if (spec.cadence === 'weekly') { + out.dayOfWeek = clampInt(spec.dayOfWeek ?? 1, 0, 6); + } + return out; +} + +/** + * Compute the next fire timestamp (epoch ms) for the given spec. + * + * - `from` anchors when we're computing relative to. For a fresh save, + * pass the user's "now". For a just-fired schedule, pass `lastFiredAt` + * so `daily`/`weekly` still target the same time-of-day on the next + * calendar slot regardless of when the crawl actually completed. + * - `lastFiredAt` only matters for `hourly`/`custom` — those advance + * from when the last fire happened (so the cadence stays steady even + * if the previous crawl was late). + */ +export function computeNextFire( + spec: ScheduleSpec, + now: number, + lastFiredAt: number | null, +): number { + if (spec.cadence === 'hourly') { + const base = lastFiredAt ?? now; + return base + 60 * 60 * 1000; + } + if (spec.cadence === 'custom') { + const interval = + Math.max(MIN_CUSTOM_INTERVAL_MIN, spec.intervalMinutes ?? MIN_CUSTOM_INTERVAL_MIN) * + 60 * + 1000; + const base = lastFiredAt ?? now; + return base + interval; + } + // Daily / weekly — pin to time-of-day in local timezone. We always + // advance by at least 1 minute past `now` to avoid an immediate + // re-fire when the saved spec matches the current clock. + const hour = spec.hourOfDay ?? 3; + const minute = spec.minuteOfHour ?? 0; + const target = new Date(now); + target.setHours(hour, minute, 0, 0); + if (spec.cadence === 'daily') { + if (target.getTime() <= now) { + target.setDate(target.getDate() + 1); + } + return target.getTime(); + } + // weekly + const targetDay = spec.dayOfWeek ?? 1; + let delta = (targetDay - target.getDay() + 7) % 7; + if (delta === 0 && target.getTime() <= now) { + delta = 7; + } + target.setDate(target.getDate() + delta); + return target.getTime(); +} + +function clampInt(v: number, min: number, max: number): number { + const n = Math.floor(v); + if (!Number.isFinite(n)) return min; + return Math.min(max, Math.max(min, n)); +} diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index ca6b2e7..f3f7b80 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -42,6 +42,8 @@ import { type PagesPerDirectoryInput, type PagesPerDirectoryRow, type StatusCodeHistogramRow, + type IndexabilityDistributionRow, + type ContentKindDistributionRow, type DepthHistogramRow, type ResponseTimeHistogramRow, type TopUrlsInput, @@ -60,6 +62,8 @@ import { type SettingsExportInput, type SettingsExportResult, type SettingsImportResult, + type ScheduleEntry, + type ScheduleSpec, type OverviewCounts, type SitemapGenerateInput, type SitemapGenerateResult, @@ -187,6 +191,10 @@ const api: FreeCrawlApi = { ): Promise => ipcRenderer.invoke(IPC.reportsPagesPerDirectory, input), reportsStatusCodeHistogram: (): Promise => ipcRenderer.invoke(IPC.reportsStatusCodeHistogram), + reportsIndexabilityDistribution: (): Promise => + ipcRenderer.invoke(IPC.reportsIndexabilityDistribution), + reportsContentKindDistribution: (): Promise => + ipcRenderer.invoke(IPC.reportsContentKindDistribution), reportsDepthHistogram: (): Promise => ipcRenderer.invoke(IPC.reportsDepthHistogram), reportsResponseTimeHistogram: (): Promise => @@ -228,6 +236,9 @@ const api: FreeCrawlApi = { ipcRenderer.invoke(IPC.prefsExportSettings, input), prefsImportSettings: (): Promise => ipcRenderer.invoke(IPC.prefsImportSettings), + scheduleGet: (): Promise => ipcRenderer.invoke(IPC.scheduleGet), + scheduleSet: (spec: ScheduleSpec | null): Promise => + ipcRenderer.invoke(IPC.scheduleSet, spec), onLogEntry: (cb) => subscribe(IPC.logsEntry, cb), onLogsBatch: (cb) => subscribe(IPC.logsBatch, cb), onLogsBusy: (cb) => subscribe(IPC.logsBusy, cb), diff --git a/apps/desktop/src/renderer/src/App.tsx b/apps/desktop/src/renderer/src/App.tsx index 2b77c07..a3d4fef 100644 --- a/apps/desktop/src/renderer/src/App.tsx +++ b/apps/desktop/src/renderer/src/App.tsx @@ -10,6 +10,7 @@ import { SitemapValidatorDialog } from './components/SitemapValidatorDialog.js'; import { ReportsDialog } from './components/ReportsDialog.js'; import { SettingsDialog } from './components/SettingsDialog.js'; import { CompareDialog } from './components/CompareDialog.js'; +import { ScheduledCrawlDialog } from './components/ScheduledCrawlDialog.js'; import { UrlsTab } from './tabs/UrlsTab.js'; import { ImagesTab } from './tabs/ImagesTab.js'; import { BrokenLinksTab } from './tabs/BrokenLinksTab.js'; @@ -38,6 +39,7 @@ export function App() { const [sitemapValidatorOpen, setSitemapValidatorOpen] = useState(false); const [reportsOpen, setReportsOpen] = useState(false); const [compareOpen, setCompareOpen] = useState(false); + const [scheduleOpen, setScheduleOpen] = useState(false); const [dropFlash, setDropFlash] = useState(null); // Drag & drop URL list — drop a `.txt` / `.csv` of URLs anywhere on the @@ -278,6 +280,9 @@ export function App() { case 'open-visualization': setActiveTab('visualization'); break; + case 'open-scheduled-crawl': + setScheduleOpen(true); + break; } }); return () => { @@ -371,6 +376,7 @@ export function App() { setReportsOpen(false)} /> setSettingsOpen(false)} /> setCompareOpen(false)} /> + setScheduleOpen(false)} /> ); } diff --git a/apps/desktop/src/renderer/src/components/ReportsDialog.tsx b/apps/desktop/src/renderer/src/components/ReportsDialog.tsx index 74a6940..a55c8ca 100644 --- a/apps/desktop/src/renderer/src/components/ReportsDialog.tsx +++ b/apps/desktop/src/renderer/src/components/ReportsDialog.tsx @@ -3,6 +3,8 @@ import { X } from 'lucide-react'; import type { PagesPerDirectoryRow, StatusCodeHistogramRow, + IndexabilityDistributionRow, + ContentKindDistributionRow, DepthHistogramRow, ResponseTimeHistogramRow, TopUrlsRow, @@ -26,6 +28,8 @@ interface Props { type ReportKind = | 'pages-per-dir' | 'status-codes' + | 'indexability-distribution' + | 'content-kind-distribution' | 'depth' | 'response-time' | 'slowest-urls' @@ -63,6 +67,8 @@ interface ReportRow { const REPORT_LABELS: Record = { 'pages-per-dir': 'Pages per Directory', 'status-codes': 'Status Code Histogram', + 'indexability-distribution': 'Indexability Distribution', + 'content-kind-distribution': 'Content-Type Distribution', depth: 'Depth Histogram', 'response-time': 'Response Time Histogram', 'slowest-urls': 'Slowest URLs (Top 25)', @@ -87,6 +93,8 @@ const REPORT_LABELS: Record = { const KEY_LABELS: Record = { 'pages-per-dir': 'Directory', 'status-codes': 'Status', + 'indexability-distribution': 'Indexability', + 'content-kind-distribution': 'Content Type', depth: 'Depth', 'response-time': 'Bucket', 'slowest-urls': 'URL', @@ -111,6 +119,8 @@ const KEY_LABELS: Record = { const TOP_URL_METRIC: Record = { 'pages-per-dir': null, 'status-codes': null, + 'indexability-distribution': null, + 'content-kind-distribution': null, depth: null, 'response-time': null, 'slowest-urls': 'response-time', @@ -135,6 +145,8 @@ const TOP_URL_METRIC: Record = { const VALUE_FORMAT: Record string> = { 'pages-per-dir': (v) => (v ?? 0).toLocaleString(), 'status-codes': (v) => (v ?? 0).toLocaleString(), + 'indexability-distribution': (v) => (v ?? 0).toLocaleString(), + 'content-kind-distribution': (v) => (v ?? 0).toLocaleString(), depth: (v) => (v ?? 0).toLocaleString(), 'response-time': (v) => (v ?? 0).toLocaleString(), 'slowest-urls': (v) => (v == null ? '—' : `${v.toLocaleString()} ms`), @@ -187,6 +199,25 @@ export function ReportsDialog({ open, onClose }: Props) { count: x.count, })), ); + } else if (kind === 'indexability-distribution') { + const r = await window.freecrawl.reportsIndexabilityDistribution(); + if (!cancelled) + setRows( + r.map((x: IndexabilityDistributionRow) => ({ + key: indexabilityLabel(x.indexability), + badge: x.indexability === 'indexable' ? 'OK' : 'NON', + count: x.count, + })), + ); + } else if (kind === 'content-kind-distribution') { + const r = await window.freecrawl.reportsContentKindDistribution(); + if (!cancelled) + setRows( + r.map((x: ContentKindDistributionRow) => ({ + key: contentKindLabel(x.contentKind), + count: x.count, + })), + ); } else if (kind === 'depth') { const r = await window.freecrawl.reportsDepthHistogram(); if (!cancelled) @@ -531,3 +562,55 @@ function rtBadge(label: string): string { if (label === '500ms–1s') return 'WARN'; return 'SLOW'; } + +/** + * Friendly label for the `indexability` enum the crawler writes. Keeps the + * machine-friendly token (`non-indexable:robots-blocked`) out of the UI so + * the dropdown reads naturally — Title Case + human-readable suffix. + */ +function indexabilityLabel(value: string): string { + switch (value) { + case 'indexable': + return 'Indexable'; + case 'non-indexable:noindex': + return 'Non-indexable — noindex'; + case 'non-indexable:canonical': + return 'Non-indexable — canonicalised'; + case 'non-indexable:robots-blocked': + return 'Non-indexable — robots-blocked'; + case 'non-indexable:redirect': + return 'Non-indexable — redirect'; + case 'non-indexable:client-error': + return 'Non-indexable — 4xx'; + case 'non-indexable:server-error': + return 'Non-indexable — 5xx'; + default: + return value; + } +} + +/** + * Friendly label for the `content_kind` enum (html / css / js / image / + * font / pdf / other). Upper-cases acronyms (HTML/CSS/JS/PDF) so the chart + * matches the way users talk about file types. + */ +function contentKindLabel(value: string): string { + switch (value) { + case 'html': + return 'HTML'; + case 'css': + return 'CSS'; + case 'js': + return 'JavaScript'; + case 'image': + return 'Image'; + case 'font': + return 'Font'; + case 'pdf': + return 'PDF'; + case 'other': + return 'Other'; + default: + return value; + } +} diff --git a/apps/desktop/src/renderer/src/components/ScheduledCrawlDialog.tsx b/apps/desktop/src/renderer/src/components/ScheduledCrawlDialog.tsx new file mode 100644 index 0000000..0d6e456 --- /dev/null +++ b/apps/desktop/src/renderer/src/components/ScheduledCrawlDialog.tsx @@ -0,0 +1,352 @@ +import { useEffect, useMemo, useState } from 'react'; +import { X, Clock, AlertTriangle } from 'lucide-react'; +import clsx from 'clsx'; +import type { ScheduleEntry, ScheduleSpec } from '@freecrawl/shared-types'; + +interface Props { + open: boolean; + onClose: () => void; +} + +type Cadence = 'hourly' | 'daily' | 'weekly' | 'custom'; + +const DAY_NAMES = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']; +const MIN_CUSTOM_INTERVAL = 15; + +interface FormState { + enabled: boolean; + cadence: Cadence; + intervalMinutes: string; + hourOfDay: string; + minuteOfHour: string; + dayOfWeek: number; +} + +const DEFAULT_FORM: FormState = { + enabled: false, + cadence: 'daily', + intervalMinutes: '60', + hourOfDay: '03', + minuteOfHour: '00', + dayOfWeek: 1, +}; + +function entryToForm(entry: ScheduleEntry | null): FormState { + if (!entry) return DEFAULT_FORM; + const { spec } = entry; + return { + enabled: spec.enabled, + cadence: spec.cadence, + intervalMinutes: String(spec.intervalMinutes ?? 60), + hourOfDay: String(spec.hourOfDay ?? 3).padStart(2, '0'), + minuteOfHour: String(spec.minuteOfHour ?? 0).padStart(2, '0'), + dayOfWeek: spec.dayOfWeek ?? 1, + }; +} + +function formToSpec(form: FormState): ScheduleSpec { + const out: ScheduleSpec = { + enabled: form.enabled, + cadence: form.cadence, + }; + if (form.cadence === 'custom') { + const n = Math.max(MIN_CUSTOM_INTERVAL, Math.floor(Number(form.intervalMinutes) || 0)); + out.intervalMinutes = n; + } + if (form.cadence === 'daily' || form.cadence === 'weekly') { + out.hourOfDay = clampInt(Number(form.hourOfDay), 0, 23, 3); + out.minuteOfHour = clampInt(Number(form.minuteOfHour), 0, 59, 0); + } + if (form.cadence === 'weekly') { + out.dayOfWeek = clampInt(form.dayOfWeek, 0, 6, 1); + } + return out; +} + +function clampInt(v: number, min: number, max: number, fallback: number): number { + const n = Math.floor(v); + if (!Number.isFinite(n)) return fallback; + return Math.min(max, Math.max(min, n)); +} + +function formatDateTime(ms: number | null): string { + if (ms === null) return '—'; + const d = new Date(ms); + const pad = (n: number) => String(n).padStart(2, '0'); + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`; +} + +export function ScheduledCrawlDialog({ open, onClose }: Props) { + const [entry, setEntry] = useState(null); + const [form, setForm] = useState(DEFAULT_FORM); + const [saving, setSaving] = useState(false); + const [dirty, setDirty] = useState(false); + const [error, setError] = useState(null); + + // Re-fetch the schedule each time the dialog reopens so any external + // mutation (scheduled crawl fired → lastFiredAt updated) shows on the + // status row immediately. Reset dirty flag so the Save button starts + // disabled until the user actually edits something. + useEffect(() => { + if (!open) return; + setError(null); + setDirty(false); + void window.freecrawl.scheduleGet().then((e) => { + setEntry(e); + setForm(entryToForm(e)); + }); + }, [open]); + + useEffect(() => { + if (!open) return; + const onKey = (e: KeyboardEvent) => { + if (e.key === 'Escape') onClose(); + }; + window.addEventListener('keydown', onKey); + return () => window.removeEventListener('keydown', onKey); + }, [open, onClose]); + + function update(key: K, value: FormState[K]) { + setForm((prev) => ({ ...prev, [key]: value })); + setDirty(true); + } + + async function save() { + setSaving(true); + setError(null); + try { + const spec = formToSpec(form); + const result = await window.freecrawl.scheduleSet(spec); + setEntry(result); + setForm(entryToForm(result)); + setDirty(false); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setSaving(false); + } + } + + async function clearSchedule() { + if (!entry) return; + if (!window.confirm('Remove the scheduled crawl for this project?')) return; + setSaving(true); + setError(null); + try { + await window.freecrawl.scheduleSet(null); + setEntry(null); + setForm(DEFAULT_FORM); + setDirty(false); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setSaving(false); + } + } + + const showInterval = form.cadence === 'custom'; + const showTimeOfDay = form.cadence === 'daily' || form.cadence === 'weekly'; + const showDayOfWeek = form.cadence === 'weekly'; + + const nextFiresHint = useMemo(() => { + if (!form.enabled) return 'Disabled — set "Enabled" and Save to arm the schedule.'; + return 'Save to compute the next fire time. The schedule only fires while FreeCrawl is open.'; + }, [form.enabled]); + + if (!open) return null; + + return ( +
+
e.stopPropagation()} + > +
+ +
+ Scheduled Crawl +
+ +
+ +
+
+
+ +
+ The schedule fires only while FreeCrawl is open on this + project. For triggers that survive an app restart, run + freecrawl + (the CLI) from Windows Task Scheduler / macOS launchd / cron. +
+
+
+ + + +
+ + + {showInterval && ( + + )} + + {showTimeOfDay && ( + <> + + + + )} + + {showDayOfWeek && ( + + )} +
+ +
+
+ Next fire + + {entry ? formatDateTime(entry.status.nextFiresAt) : '—'} + + Last fired + + {entry ? formatDateTime(entry.status.lastFiredAt) : '—'} + + Last status + + {entry?.status.lastStatus ?? '—'} + +
+
{nextFiresHint}
+
+ + {error && ( +
+ {error} +
+ )} +
+ +
+ {entry && ( + + )} +
+ + +
+
+
+
+ ); +} diff --git a/apps/mcp-server/package.json b/apps/mcp-server/package.json index 98ae72c..95681a4 100644 --- a/apps/mcp-server/package.json +++ b/apps/mcp-server/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/mcp-server", - "version": "0.4.0", + "version": "0.4.1", "private": true, "type": "module", "bin": { diff --git a/package.json b/package.json index 409e701..d08d2f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "freecrawl-seo", - "version": "0.4.0", + "version": "0.4.1", "private": true, "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog", "license": "MIT", diff --git a/packages/core/package.json b/packages/core/package.json index 9c89521..e3956c2 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/core", - "version": "0.4.0", + "version": "0.4.1", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/packages/db/package.json b/packages/db/package.json index 736809d..f3948f5 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/db", - "version": "0.4.0", + "version": "0.4.1", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/packages/db/src/project-db.ts b/packages/db/src/project-db.ts index b42e400..913b721 100644 --- a/packages/db/src/project-db.ts +++ b/packages/db/src/project-db.ts @@ -2292,6 +2292,42 @@ export class ProjectDb { .all() as { status: number | null; count: number }[]; } + /** + * Indexability distribution across internal URLs. Seven buckets — + * `indexable` + the six `non-indexable:…` reasons — give the user a + * one-glance breakdown of why pages are or aren't reaching Google. + * Ordered by `count DESC` so the dominant bucket sits at the top. + */ + getIndexabilityDistribution(): { indexability: string; count: number }[] { + return this.db + .prepare( + `SELECT indexability, COUNT(*) AS count + FROM urls + WHERE is_external = 0 AND indexability IS NOT NULL + GROUP BY indexability + ORDER BY count DESC, indexability`, + ) + .all() as { indexability: string; count: number }[]; + } + + /** + * Content-kind distribution across internal URLs (HTML / CSS / JS / + * image / font / pdf / other). Mirrors `content_kind` exactly; useful + * for spotting unexpectedly large non-HTML payloads (e.g. an asset CDN + * accidentally crawled in-scope). + */ + getContentKindDistribution(): { contentKind: string; count: number }[] { + return this.db + .prepare( + `SELECT content_kind AS contentKind, COUNT(*) AS count + FROM urls + WHERE is_external = 0 AND content_kind IS NOT NULL + GROUP BY content_kind + ORDER BY count DESC, content_kind`, + ) + .all() as { contentKind: string; count: number }[]; + } + /** * Click-depth distribution for internal HTML pages — the canonical * "site architecture flatness" metric. Shallow sites bias toward depths diff --git a/packages/shared-types/package.json b/packages/shared-types/package.json index 27bfbf6..da369e5 100644 --- a/packages/shared-types/package.json +++ b/packages/shared-types/package.json @@ -1,6 +1,6 @@ { "name": "@freecrawl/shared-types", - "version": "0.4.0", + "version": "0.4.1", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/packages/shared-types/src/ipc.ts b/packages/shared-types/src/ipc.ts index d5a8457..a027d9a 100644 --- a/packages/shared-types/src/ipc.ts +++ b/packages/shared-types/src/ipc.ts @@ -97,6 +97,8 @@ export const IPC = { urlClusterMembers: 'urls:cluster-members', reportsPagesPerDirectory: 'reports:pages-per-directory', reportsStatusCodeHistogram: 'reports:status-code-histogram', + reportsIndexabilityDistribution: 'reports:indexability-distribution', + reportsContentKindDistribution: 'reports:content-kind-distribution', reportsDepthHistogram: 'reports:depth-histogram', reportsResponseTimeHistogram: 'reports:response-time-histogram', reportsTopUrls: 'reports:top-urls', @@ -121,10 +123,48 @@ export const IPC = { rendererLagReport: 'renderer:lag-report', prefsExportSettings: 'prefs:export-settings', prefsImportSettings: 'prefs:import-settings', + scheduleGet: 'schedule:get', + scheduleSet: 'schedule:set', } as const; export type IpcChannel = (typeof IPC)[keyof typeof IPC]; +/** + * In-app scheduled crawl spec. One schedule per project (keyed by the + * absolute `.seoproject` path in app-level prefs). Fires only while the + * desktop app is open and the project is loaded — for triggers that + * survive an app restart, drive the CLI via the OS scheduler (Windows + * Task Scheduler / launchd / cron); that path is V2. + */ +export interface ScheduleSpec { + enabled: boolean; + /** `hourly` = every hour on the minute. `daily` = once at hourOfDay: + * minuteOfHour. `weekly` = once at dayOfWeek+hourOfDay:minuteOfHour. + * `custom` = every `intervalMinutes` (min 15). */ + cadence: 'hourly' | 'daily' | 'weekly' | 'custom'; + /** For `custom`. Minimum enforced at 15 to prevent runaway loops. */ + intervalMinutes?: number; + /** 0–23. Used by daily/weekly. */ + hourOfDay?: number; + /** 0–59. Used by daily/weekly. */ + minuteOfHour?: number; + /** 0 = Sunday … 6 = Saturday. Used by weekly. */ + dayOfWeek?: number; +} + +export interface ScheduleStatus { + /** Epoch ms when last fire completed (success or failure). */ + lastFiredAt: number | null; + /** Epoch ms when the next fire is due. Server-computed on set. */ + nextFiresAt: number | null; + lastStatus: 'success' | 'failure' | 'running' | null; +} + +export interface ScheduleEntry { + spec: ScheduleSpec; + status: ScheduleStatus; +} + export interface UrlsQueryInput { limit: number; offset: number; @@ -192,6 +232,7 @@ export type MenuEvent = | 'open-sitemap-validator' | 'open-reports' | 'open-settings' + | 'open-scheduled-crawl' | 'about'; export interface ExportCsvResult { @@ -544,6 +585,20 @@ export interface StatusCodeHistogramRow { count: number; } +export interface IndexabilityDistributionRow { + /** `indexable` or one of `non-indexable:noindex` / `:canonical` / + * `:robots-blocked` / `:redirect` / `:client-error` / `:server-error`. */ + indexability: string; + count: number; +} + +export interface ContentKindDistributionRow { + /** `html` / `css` / `js` / `image` / `font` / `pdf` / `other` — matches + * the `content_kind` column populated at insert time. */ + contentKind: string; + count: number; +} + export interface DepthHistogramRow { depth: number; count: number; @@ -866,6 +921,8 @@ export interface FreeCrawlApi { urlClusterMembers(urlId: number): Promise; reportsPagesPerDirectory(input: PagesPerDirectoryInput): Promise; reportsStatusCodeHistogram(): Promise; + reportsIndexabilityDistribution(): Promise; + reportsContentKindDistribution(): Promise; reportsDepthHistogram(): Promise; reportsResponseTimeHistogram(): Promise; reportsTopUrls(input: TopUrlsInput): Promise; @@ -886,6 +943,12 @@ export interface FreeCrawlApi { reportRendererLag(lagMs: number): void; prefsExportSettings(input: SettingsExportInput): Promise; prefsImportSettings(): Promise; + /** Read the schedule for the currently-loaded project. `null` when no + * project is open OR no schedule has been configured. */ + scheduleGet(): Promise; + /** Save / update the schedule for the currently-loaded project. Passing + * `null` removes the schedule entirely. */ + scheduleSet(spec: ScheduleSpec | null): Promise; onLogEntry(cb: (entry: LogEntry) => void): () => void; onLogsBatch(cb: (entries: LogEntry[]) => void): () => void; onLogsBusy(cb: (busy: boolean) => void): () => void;