feat: scheduled crawl + indexability/content-kind reports + linux/mac launcher

FreeCrawl-SEO-Tool-Start.sh

@@ -0,0 +1,190 @@
+#!/usr/bin/env bash
+# FreeCrawl SEO Tool — Launcher (macOS / Linux mirror of FreeCrawl-SEO-Tool-Start.bat)
+#
+# İlk çalıştırma:
+#   1) chmod +x FreeCrawl-SEO-Tool-Start.sh
+#   2) ./FreeCrawl-SEO-Tool-Start.sh
+#
+# .bat tarafındaki Windows-spesifik bölümler (color, title, pause) yerine
+# burada ANSI renkleri + sondaki "Enter ile çık" promptu kullanılır.
+
+set -u
+
+# Çift-tıklatma ile başlatılan terminallerde script kendi dizininden bağımsız
+# olarak fire edilebilir — symlink'i de takip ederek gerçek dizine cd ediyoruz.
+SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null || echo "${BASH_SOURCE[0]}")")" && pwd)"
+cd "$SCRIPT_DIR"
+
+# Renk kısayolları — `tput` çıkışa terminal yoksa boş döner, böylece bir log
+# dosyasına pipe'lansa bile escape kodları output'u kirletmez.
+if [ -t 1 ]; then
+  C_RESET="$(tput sgr0 2>/dev/null || echo)"
+  C_OK="$(tput setaf 2 2>/dev/null || echo)"
+  C_ERR="$(tput setaf 1 2>/dev/null || echo)"
+  C_INFO="$(tput setaf 6 2>/dev/null || echo)"
+  C_HEAD="$(tput setaf 14 2>/dev/null || tput setaf 6 2>/dev/null || echo)"
+else
+  C_RESET=""
+  C_OK=""
+  C_ERR=""
+  C_INFO=""
+  C_HEAD=""
+fi
+
+ok()    { printf '%s[OK]%s %s\n'    "$C_OK"   "$C_RESET" "$*"; }
+info()  { printf '%s[BILGI]%s %s\n' "$C_INFO" "$C_RESET" "$*"; }
+err()   { printf '%s[HATA]%s %s\n'  "$C_ERR"  "$C_RESET" "$*" >&2; }
+head_() { printf '%s%s%s\n' "$C_HEAD" "$*" "$C_RESET"; }
+hr()    { head_ "============================================================"; }
+sub()   { head_ "------------------------------------------------------------"; }
+
+# Çift-tıklatma akışlarında pencerenin anında kapanmaması için sondaki
+# "Enter ile çık" promptu. Stdin TTY değilse (CI, pipe) atlanır.
+final_pause() {
+  if [ -t 0 ]; then
+    printf '\n'
+    read -r -p "Cikmak icin Enter'a basin..." _ || true
+  fi
+}
+
+trap 'final_pause' EXIT
+
+# ============================================================================
+hr
+head_ "  FreeCrawl SEO Tool - Launcher"
+hr
+echo
+
+# ---- 1) Node.js kontrolu ----------------------------------------------------
+if ! command -v node >/dev/null 2>&1; then
+  err "Node.js bulunamadi."
+  echo
+  echo "FreeCrawl SEO Tool Node.js 22+ gerektirir."
+  echo "Lutfen https://nodejs.org/ adresinden LTS surumunu indirip kurun."
+  echo "Kurulumdan sonra bu betigi tekrar calistirin."
+  exit 1
+fi
+NODE_VERSION="$(node -v)"
+ok "Node.js bulundu: ${NODE_VERSION}"
+
+# ---- 2) npm kontrolu --------------------------------------------------------
+if ! command -v npm >/dev/null 2>&1; then
+  err "npm bulunamadi. Node.js kurulumunuz bozuk olabilir."
+  exit 1
+fi
+NPM_VERSION="$(npm -v)"
+ok "npm bulundu: v${NPM_VERSION}"
+echo
+
+# ---- 3) Bagimlilik kontrolu -------------------------------------------------
+NEED_INSTALL=0
+[ -d "node_modules" ] || NEED_INSTALL=1
+[ -d "apps/desktop/node_modules" ] || NEED_INSTALL=1
+[ -d "node_modules/electron" ] || NEED_INSTALL=1
+
+if [ "$NEED_INSTALL" -eq 1 ]; then
+  info "Gerekli kutuphaneler yuklu degil veya eksik."
+  echo
+  echo "Bu islem ilk calistirmada birkac dakika surebilir ve yaklasik"
+  echo "500-800 MB disk alani kullanir."
+  echo
+  CONFIRM=""
+  read -r -p "Kutuphaneleri simdi yuklemek istiyor musunuz? (E/H): " CONFIRM || true
+  case "${CONFIRM:-}" in
+    E|e|Y|y)
+      echo
+      sub
+      head_ "  npm install calisiyor..."
+      sub
+      if ! npm install; then
+        err "npm install basarisiz oldu. Internet baglantinizi kontrol edin veya hata mesajini inceleyin."
+        exit 1
+      fi
+      echo
+      ok "Kutuphaneler basariyla yuklendi."
+      echo
+      ;;
+    *)
+      echo
+      echo "Kurulum iptal edildi. Program baslatilamaz."
+      exit 1
+      ;;
+  esac
+else
+  ok "Gerekli kutuphaneler yuklu."
+  echo
+fi
+
+# ---- 4) Paylasilan workspace paketlerini derle ------------------------------
+NEED_BUILD=0
+[ -f "packages/shared-types/dist/index.js" ] || NEED_BUILD=1
+[ -f "packages/db/dist/index.js" ] || NEED_BUILD=1
+[ -f "packages/core/dist/index.js" ] || NEED_BUILD=1
+
+if [ "$NEED_BUILD" -eq 1 ]; then
+  sub
+  head_ "  Paylasilan paketler derleniyor (ilk calisma)..."
+  sub
+  if ! npx tsc -b; then
+    err "Paket derlemesi basarisiz oldu."
+    exit 1
+  fi
+  ok "Paketler hazir."
+  echo
+else
+  ok "Paylasilan paketler hazir."
+  echo
+fi
+
+# ---- 4b) Desktop uygulamasini production build et ---------------------------
+# Dev mod (electron-vite dev) ilk acilista 1700+ modulu on-demand transform
+# ediyor ve 20-30 sn cold start suruyor. Production build'de tum moduller
+# tek bir bundle'a derlenmis oldugu icin acilis 1-2 sn'ye iniyor.
+NEED_DESKTOP_BUILD=0
+[ -f "apps/desktop/out/main/index.js" ] || NEED_DESKTOP_BUILD=1
+[ -f "apps/desktop/out/preload/index.js" ] || NEED_DESKTOP_BUILD=1
+[ -f "apps/desktop/out/renderer/index.html" ] || NEED_DESKTOP_BUILD=1
+
+if [ "$NEED_DESKTOP_BUILD" -eq 1 ]; then
+  sub
+  head_ "  Desktop uygulamasi build ediliyor (production)..."
+  echo "  Bu islem yaklasik 10-15 sn surer ve sadece ilk calistirmada"
+  echo "  (veya kod degisikligi sonrasi) yeniden yapilir."
+  sub
+  if ! npm --workspace apps/desktop run build; then
+    err "Desktop build basarisiz oldu."
+    exit 1
+  fi
+  ok "Desktop bundle hazir."
+  echo
+else
+  ok "Desktop bundle hazir."
+  echo
+fi
+
+# ---- 5) Uygulamayi baslat ---------------------------------------------------
+hr
+head_ "  FreeCrawl SEO Tool baslatiliyor (production)..."
+echo "  Bu pencereyi KAPATMAYIN - uygulamanin yasam dongusu buna bagli."
+hr
+echo
+
+# `node:sqlite` Node 24'te kararli ama hala "experimental" bayragi tasiyor
+# ve modul yuklenirken ExperimentalWarning basiyor. NODE_NO_WARNINGS=1
+# ile bu kozmetik uyariyi sustururuz; gercek hatalar (TypeError vs.)
+# yine konsola yazilir.
+export NODE_NO_WARNINGS=1
+
+npm --workspace apps/desktop run start
+APP_EXIT=$?
+
+echo
+sub
+if [ "$APP_EXIT" -eq 0 ]; then
+  head_ "  Uygulama kapatildi."
+else
+  err "Uygulama hatayla sonlandi (exit code: ${APP_EXIT})."
+fi
+sub
+
+exit "$APP_EXIT"

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": true,
   "type": "module",
   "bin": {

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "main": "./out/main/index.js",

apps/desktop/src/main/db-reader-worker.ts

@@ -90,6 +90,8 @@ const ALLOWED_METHODS = new Set<string>([
   'sitemapOrphans',
   'getPagesPerDirectory',
   'getStatusCodeHistogram',
+  'getIndexabilityDistribution',
+  'getContentKindDistribution',
   'getDepthHistogram',
   'getResponseTimeHistogram',
   'countUrls',

apps/desktop/src/main/index.ts

@@ -64,6 +64,8 @@ import {
   type SettingsExportInput,
   type SettingsExportResult,
   type SettingsImportResult,
+  type ScheduleEntry,
+  type ScheduleSpec,
   type PagesPerDirectoryInput,
   type ImagesQueryInput,
   type ImagesQueryResult,
@@ -115,6 +117,13 @@ import {
   stopMcpBridge,
   type McpStartCrawlInput,
 } from './mcp-bridge.js';
+import {
+  claimIfDue,
+  getSchedule,
+  recordFireResult,
+  setSchedule,
+  type SchedulerStore,
+} from './scheduler.js';
 import type { ProjectDb as ProjectDbType } from '@freecrawl/db';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -206,6 +215,93 @@ function fireDataChanged(): void {
   mainWindow?.webContents.send(IPC.dataChanged);
 }
 
+/**
+ * Schedule-store adapter — bridges the scheduler module to the prefs
+ * cache. `load()` always returns a fresh object reference so the
+ * module's mutate-then-save flow can't leak references back into the
+ * prefs cache without an explicit `save()`.
+ */
+const schedulerStore: SchedulerStore = {
+  load() {
+    const raw = prefsCache['schedules'];
+    if (raw && typeof raw === 'object' && !Array.isArray(raw)) {
+      // Shallow clone — callers mutate this map.
+      return { ...(raw as Record<string, ScheduleEntry>) };
+    }
+    return {};
+  },
+  save(map) {
+    prefsCache['schedules'] = map;
+    schedulePrefsWrite();
+  },
+};
+
+let schedulerTickTimer: NodeJS.Timeout | null = null;
+/** Set to true at the moment the scheduler tick calls `launchCrawl`,
+ *  cleared on the next crawl-done / crawl-error event. Lets the
+ *  done/error handlers update the schedule's `lastStatus` without
+ *  duplicating the launch path for user-initiated crawls. */
+let crawlerStartedByScheduler = false;
+
+/**
+ * Scheduler tick — invoked every 60 s while the app is open. Checks
+ * whether the currently-loaded project has a due schedule and, if so,
+ * launches the crawl using the last-used CrawlConfig.
+ *
+ * No-ops when:
+ *   - No project is loaded
+ *   - A crawl is already running (we don't preempt a manual crawl)
+ *   - The schedule isn't due yet
+ *   - No CrawlConfig has ever been saved (we'd have nothing to fire)
+ */
+function schedulerTick(): void {
+  if (!currentProjectPath) return;
+  if (activeCrawler) return;
+  const entry = claimIfDue(schedulerStore, currentProjectPath);
+  if (!entry) return;
+  if (!lastCrawlConfig) {
+    // No saved config — record a failure so the dialog surfaces the
+    // condition rather than silently retrying on the next tick.
+    recordFireResult(schedulerStore, currentProjectPath, 'failure');
+    logger.log(
+      'warn',
+      'scheduler',
+      `Schedule due for ${currentProjectPath} but no CrawlConfig saved yet — run a crawl manually once first.`,
+    );
+    return;
+  }
+  if (!crawlController) {
+    recordFireResult(schedulerStore, currentProjectPath, 'failure');
+    return;
+  }
+  logger.log(
+    'info',
+    'scheduler',
+    `Scheduled crawl firing: ${lastCrawlConfig.startUrl} (cadence=${entry.spec.cadence})`,
+  );
+  crawlerStartedByScheduler = true;
+  crawlController.launchCrawl(lastCrawlConfig);
+}
+
+function startScheduler(): void {
+  if (schedulerTickTimer) return;
+  // 60 s tick is granular enough for hourly/daily/weekly cadences and
+  // for the 15-min minimum custom interval. Wall-clock drift across an
+  // hour-long browser session is negligible at this resolution.
+  schedulerTickTimer = setInterval(schedulerTick, 60_000);
+  // Fire once shortly after startup so a daily schedule whose target
+  // time-of-day has already passed today doesn't wait 60 s before
+  // checking. 3 s lets the renderer mount + initial DB load finish.
+  setTimeout(schedulerTick, 3_000);
+}
+
+function stopScheduler(): void {
+  if (schedulerTickTimer) {
+    clearInterval(schedulerTickTimer);
+    schedulerTickTimer = null;
+  }
+}
+
 /**
  * Per-app-session set of diagnostic categories already surfaced in a
  * popup. Cleared when the user starts a fresh crawl so re-running after
@@ -1462,6 +1558,18 @@ function registerIpc(): void {
     ),
   );
 
+  ipcMain.handle(IPC.reportsIndexabilityDistribution, () =>
+    callReaderOrFallback('getIndexabilityDistribution', [], () =>
+      getDb().getIndexabilityDistribution(),
+    ),
+  );
+
+  ipcMain.handle(IPC.reportsContentKindDistribution, () =>
+    callReaderOrFallback('getContentKindDistribution', [], () =>
+      getDb().getContentKindDistribution(),
+    ),
+  );
+
   ipcMain.handle(IPC.reportsDepthHistogram, () =>
     callReaderOrFallback('getDepthHistogram', [], () => getDb().getDepthHistogram()),
   );
@@ -1660,6 +1768,15 @@ function registerIpc(): void {
     },
   );
 
+  // V1 Faz 6 — Scheduled crawl IPC. One schedule per project, stored
+  // app-level under the `schedules` prefs key keyed by project path.
+  ipcMain.handle(IPC.scheduleGet, () => {
+    return getSchedule(schedulerStore, currentProjectPath);
+  });
+  ipcMain.handle(IPC.scheduleSet, (_e, spec: ScheduleSpec | null) => {
+    return setSchedule(schedulerStore, currentProjectPath, spec);
+  });
+
   // Stream new entries to the logs window in coalesced batches. A heavy
   // crawl emits 100–300 logs/s; sending each as its own IPC message
   // saturated the renderer's event loop and caused visible UI stutters
@@ -1729,6 +1846,15 @@ function registerIpc(): void {
       mainWindow?.webContents.send(IPC.crawlDone, summary);
       activeCrawler = null;
       freezeWatchdog.setMainOp('idle');
+      // V1 Faz 6 — if this run was kicked off by the scheduler, record
+      // the result so the dialog's "Last status" row reflects the
+      // outcome. The schedule has already been re-armed (nextFiresAt
+      // advanced) at claim time; we only need to flip the status from
+      // `running` to a terminal value.
+      if (crawlerStartedByScheduler) {
+        crawlerStartedByScheduler = false;
+        recordFireResult(schedulerStore, currentProjectPath, 'success');
+      }
       if (Notification.isSupported() && !mainWindow?.isFocused()) {
         try {
           new Notification({
@@ -1746,6 +1872,10 @@ function registerIpc(): void {
       logger.log('error', 'crawler', msg);
       mainWindow?.webContents.send(IPC.crawlError, msg);
       maybeShowDiagnosticDialog(msg);
+      if (crawlerStartedByScheduler) {
+        crawlerStartedByScheduler = false;
+        recordFireResult(schedulerStore, currentProjectPath, 'failure');
+      }
     });
     crawler.on('warn', (msg: string) => {
       if (activeCrawler !== crawler) return;
@@ -3074,6 +3204,11 @@ void app.whenReady().then(async () => {
     void checkForUpdates({ silent: true });
   }, 8_000);
 
+  // V1 Faz 6 — start the in-app crawl scheduler. The tick is a no-op
+  // until the user configures a schedule via the Scheduled Crawl
+  // dialog, so it's safe to leave running for the lifetime of the app.
+  startScheduler();
+
   app.on('activate', () => {
     if (BrowserWindow.getAllWindows().length === 0) createWindow();
   });
@@ -3081,6 +3216,7 @@ void app.whenReady().then(async () => {
 
 app.on('window-all-closed', () => {
   activeCrawler?.stop();
+  stopScheduler();
   // Tear down the MCP bridge first so any in-flight HTTP calls fail
   // fast rather than racing with the DB shutdown that follows.
   try {

apps/desktop/src/main/menu.ts

@@ -105,6 +105,12 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
           label: 'Compare With Project…',
           click: () => send('compare-with-project'),
         },
+        {
+          label: 'Scheduled Crawl…',
+          toolTip:
+            'Set up an in-app recurring crawl for the currently-open project. Fires only while FreeCrawl is open; use the CLI + OS scheduler for triggers that survive restarts.',
+          click: () => send('open-scheduled-crawl'),
+        },
         {
           label: 'Save Project As…',
           accelerator: 'CmdOrCtrl+Shift+S',

apps/desktop/src/main/scheduler.ts

@@ -0,0 +1,186 @@
+/**
+ * In-app crawl scheduler. One schedule per project, persisted in
+ * `userData/preferences.json` under the `schedules` key as a map keyed
+ * by absolute `.seoproject` path:
+ *
+ *   "schedules": {
+ *     "/abs/path/project.seoproject": { spec: {...}, status: {...} }
+ *   }
+ *
+ * The main process ticks every 60 s and fires the schedule for the
+ * currently-loaded project when due (`nextFiresAt <= now`) — and only
+ * when no crawl is already in progress. Closed-app firing is out of
+ * scope; the OS-scheduler-via-CLI path is V2.
+ */
+
+import type { ScheduleEntry, ScheduleSpec, ScheduleStatus } from '@freecrawl/shared-types';
+
+/**
+ * Minimum interval the `custom` cadence will honour. Anything below this
+ * is clamped — prevents an accidental "every 1 minute" schedule that
+ * would saturate a target server.
+ */
+const MIN_CUSTOM_INTERVAL_MIN = 15;
+
+export interface SchedulerStore {
+  /** Read the schedules map. Caller mutates and writes back via `save`. */
+  load: () => Record<string, ScheduleEntry>;
+  save: (map: Record<string, ScheduleEntry>) => void;
+}
+
+export function getSchedule(
+  store: SchedulerStore,
+  projectPath: string,
+): ScheduleEntry | null {
+  if (!projectPath) return null;
+  const map = store.load();
+  return map[projectPath] ?? null;
+}
+
+export function setSchedule(
+  store: SchedulerStore,
+  projectPath: string,
+  spec: ScheduleSpec | null,
+  now: number = Date.now(),
+): ScheduleEntry | null {
+  if (!projectPath) return null;
+  const map = store.load();
+  if (spec === null) {
+    if (map[projectPath]) {
+      delete map[projectPath];
+      store.save(map);
+    }
+    return null;
+  }
+  const normalized = normalizeSpec(spec);
+  const previousStatus = map[projectPath]?.status;
+  const entry: ScheduleEntry = {
+    spec: normalized,
+    status: {
+      lastFiredAt: previousStatus?.lastFiredAt ?? null,
+      lastStatus: previousStatus?.lastStatus ?? null,
+      nextFiresAt: normalized.enabled ? computeNextFire(normalized, now, null) : null,
+    },
+  };
+  map[projectPath] = entry;
+  store.save(map);
+  return entry;
+}
+
+/**
+ * Determine which schedule (if any) is due to fire for the given
+ * project. Returns the updated entry with status flipped to `running`
+ * and `nextFiresAt` re-computed for the *next* tick so the same
+ * schedule can't double-fire while a crawl is in progress.
+ */
+export function claimIfDue(
+  store: SchedulerStore,
+  projectPath: string,
+  now: number = Date.now(),
+): ScheduleEntry | null {
+  if (!projectPath) return null;
+  const map = store.load();
+  const entry = map[projectPath];
+  if (!entry || !entry.spec.enabled) return null;
+  const due = entry.status.nextFiresAt;
+  if (due === null || due > now) return null;
+  const nextStatus: ScheduleStatus = {
+    lastFiredAt: now,
+    lastStatus: 'running',
+    nextFiresAt: computeNextFire(entry.spec, now, now),
+  };
+  const updated: ScheduleEntry = { spec: entry.spec, status: nextStatus };
+  map[projectPath] = updated;
+  store.save(map);
+  return updated;
+}
+
+export function recordFireResult(
+  store: SchedulerStore,
+  projectPath: string,
+  result: 'success' | 'failure',
+): void {
+  if (!projectPath) return;
+  const map = store.load();
+  const entry = map[projectPath];
+  if (!entry) return;
+  entry.status.lastStatus = result;
+  map[projectPath] = entry;
+  store.save(map);
+}
+
+function normalizeSpec(spec: ScheduleSpec): ScheduleSpec {
+  const out: ScheduleSpec = {
+    enabled: !!spec.enabled,
+    cadence: spec.cadence,
+  };
+  if (spec.cadence === 'custom') {
+    const raw = Math.floor(spec.intervalMinutes ?? MIN_CUSTOM_INTERVAL_MIN);
+    out.intervalMinutes = Math.max(MIN_CUSTOM_INTERVAL_MIN, raw);
+  }
+  if (spec.cadence === 'daily' || spec.cadence === 'weekly') {
+    out.hourOfDay = clampInt(spec.hourOfDay ?? 3, 0, 23);
+    out.minuteOfHour = clampInt(spec.minuteOfHour ?? 0, 0, 59);
+  }
+  if (spec.cadence === 'weekly') {
+    out.dayOfWeek = clampInt(spec.dayOfWeek ?? 1, 0, 6);
+  }
+  return out;
+}
+
+/**
+ * Compute the next fire timestamp (epoch ms) for the given spec.
+ *
+ * - `from` anchors when we're computing relative to. For a fresh save,
+ *   pass the user's "now". For a just-fired schedule, pass `lastFiredAt`
+ *   so `daily`/`weekly` still target the same time-of-day on the next
+ *   calendar slot regardless of when the crawl actually completed.
+ * - `lastFiredAt` only matters for `hourly`/`custom` — those advance
+ *   from when the last fire happened (so the cadence stays steady even
+ *   if the previous crawl was late).
+ */
+export function computeNextFire(
+  spec: ScheduleSpec,
+  now: number,
+  lastFiredAt: number | null,
+): number {
+  if (spec.cadence === 'hourly') {
+    const base = lastFiredAt ?? now;
+    return base + 60 * 60 * 1000;
+  }
+  if (spec.cadence === 'custom') {
+    const interval =
+      Math.max(MIN_CUSTOM_INTERVAL_MIN, spec.intervalMinutes ?? MIN_CUSTOM_INTERVAL_MIN) *
+      60 *
+      1000;
+    const base = lastFiredAt ?? now;
+    return base + interval;
+  }
+  // Daily / weekly — pin to time-of-day in local timezone. We always
+  // advance by at least 1 minute past `now` to avoid an immediate
+  // re-fire when the saved spec matches the current clock.
+  const hour = spec.hourOfDay ?? 3;
+  const minute = spec.minuteOfHour ?? 0;
+  const target = new Date(now);
+  target.setHours(hour, minute, 0, 0);
+  if (spec.cadence === 'daily') {
+    if (target.getTime() <= now) {
+      target.setDate(target.getDate() + 1);
+    }
+    return target.getTime();
+  }
+  // weekly
+  const targetDay = spec.dayOfWeek ?? 1;
+  let delta = (targetDay - target.getDay() + 7) % 7;
+  if (delta === 0 && target.getTime() <= now) {
+    delta = 7;
+  }
+  target.setDate(target.getDate() + delta);
+  return target.getTime();
+}
+
+function clampInt(v: number, min: number, max: number): number {
+  const n = Math.floor(v);
+  if (!Number.isFinite(n)) return min;
+  return Math.min(max, Math.max(min, n));
+}

apps/desktop/src/preload/index.ts

@@ -42,6 +42,8 @@ import {
   type PagesPerDirectoryInput,
   type PagesPerDirectoryRow,
   type StatusCodeHistogramRow,
+  type IndexabilityDistributionRow,
+  type ContentKindDistributionRow,
   type DepthHistogramRow,
   type ResponseTimeHistogramRow,
   type TopUrlsInput,
@@ -60,6 +62,8 @@ import {
   type SettingsExportInput,
   type SettingsExportResult,
   type SettingsImportResult,
+  type ScheduleEntry,
+  type ScheduleSpec,
   type OverviewCounts,
   type SitemapGenerateInput,
   type SitemapGenerateResult,
@@ -187,6 +191,10 @@ const api: FreeCrawlApi = {
   ): Promise<PagesPerDirectoryRow[]> => ipcRenderer.invoke(IPC.reportsPagesPerDirectory, input),
   reportsStatusCodeHistogram: (): Promise<StatusCodeHistogramRow[]> =>
     ipcRenderer.invoke(IPC.reportsStatusCodeHistogram),
+  reportsIndexabilityDistribution: (): Promise<IndexabilityDistributionRow[]> =>
+    ipcRenderer.invoke(IPC.reportsIndexabilityDistribution),
+  reportsContentKindDistribution: (): Promise<ContentKindDistributionRow[]> =>
+    ipcRenderer.invoke(IPC.reportsContentKindDistribution),
   reportsDepthHistogram: (): Promise<DepthHistogramRow[]> =>
     ipcRenderer.invoke(IPC.reportsDepthHistogram),
   reportsResponseTimeHistogram: (): Promise<ResponseTimeHistogramRow[]> =>
@@ -228,6 +236,9 @@ const api: FreeCrawlApi = {
     ipcRenderer.invoke(IPC.prefsExportSettings, input),
   prefsImportSettings: (): Promise<SettingsImportResult> =>
     ipcRenderer.invoke(IPC.prefsImportSettings),
+  scheduleGet: (): Promise<ScheduleEntry | null> => ipcRenderer.invoke(IPC.scheduleGet),
+  scheduleSet: (spec: ScheduleSpec | null): Promise<ScheduleEntry | null> =>
+    ipcRenderer.invoke(IPC.scheduleSet, spec),
   onLogEntry: (cb) => subscribe<LogEntry>(IPC.logsEntry, cb),
   onLogsBatch: (cb) => subscribe<LogEntry[]>(IPC.logsBatch, cb),
   onLogsBusy: (cb) => subscribe<boolean>(IPC.logsBusy, cb),

apps/desktop/src/renderer/src/App.tsx

@@ -10,6 +10,7 @@ import { SitemapValidatorDialog } from './components/SitemapValidatorDialog.js';
 import { ReportsDialog } from './components/ReportsDialog.js';
 import { SettingsDialog } from './components/SettingsDialog.js';
 import { CompareDialog } from './components/CompareDialog.js';
+import { ScheduledCrawlDialog } from './components/ScheduledCrawlDialog.js';
 import { UrlsTab } from './tabs/UrlsTab.js';
 import { ImagesTab } from './tabs/ImagesTab.js';
 import { BrokenLinksTab } from './tabs/BrokenLinksTab.js';
@@ -38,6 +39,7 @@ export function App() {
   const [sitemapValidatorOpen, setSitemapValidatorOpen] = useState(false);
   const [reportsOpen, setReportsOpen] = useState(false);
   const [compareOpen, setCompareOpen] = useState(false);
+  const [scheduleOpen, setScheduleOpen] = useState(false);
   const [dropFlash, setDropFlash] = useState<string | null>(null);
 
   // Drag & drop URL list — drop a `.txt` / `.csv` of URLs anywhere on the
@@ -278,6 +280,9 @@ export function App() {
         case 'open-visualization':
           setActiveTab('visualization');
           break;
+        case 'open-scheduled-crawl':
+          setScheduleOpen(true);
+          break;
       }
     });
     return () => {
@@ -371,6 +376,7 @@ export function App() {
       <ReportsDialog open={reportsOpen} onClose={() => setReportsOpen(false)} />
       <SettingsDialog open={settingsOpen} onClose={() => setSettingsOpen(false)} />
       <CompareDialog open={compareOpen} onClose={() => setCompareOpen(false)} />
+      <ScheduledCrawlDialog open={scheduleOpen} onClose={() => setScheduleOpen(false)} />
     </div>
   );
 }

apps/desktop/src/renderer/src/components/ReportsDialog.tsx

@@ -3,6 +3,8 @@ import { X } from 'lucide-react';
 import type {
   PagesPerDirectoryRow,
   StatusCodeHistogramRow,
+  IndexabilityDistributionRow,
+  ContentKindDistributionRow,
   DepthHistogramRow,
   ResponseTimeHistogramRow,
   TopUrlsRow,
@@ -26,6 +28,8 @@ interface Props {
 type ReportKind =
   | 'pages-per-dir'
   | 'status-codes'
+  | 'indexability-distribution'
+  | 'content-kind-distribution'
   | 'depth'
   | 'response-time'
   | 'slowest-urls'
@@ -63,6 +67,8 @@ interface ReportRow {
 const REPORT_LABELS: Record<ReportKind, string> = {
   'pages-per-dir': 'Pages per Directory',
   'status-codes': 'Status Code Histogram',
+  'indexability-distribution': 'Indexability Distribution',
+  'content-kind-distribution': 'Content-Type Distribution',
   depth: 'Depth Histogram',
   'response-time': 'Response Time Histogram',
   'slowest-urls': 'Slowest URLs (Top 25)',
@@ -87,6 +93,8 @@ const REPORT_LABELS: Record<ReportKind, string> = {
 const KEY_LABELS: Record<ReportKind, string> = {
   'pages-per-dir': 'Directory',
   'status-codes': 'Status',
+  'indexability-distribution': 'Indexability',
+  'content-kind-distribution': 'Content Type',
   depth: 'Depth',
   'response-time': 'Bucket',
   'slowest-urls': 'URL',
@@ -111,6 +119,8 @@ const KEY_LABELS: Record<ReportKind, string> = {
 const TOP_URL_METRIC: Record<ReportKind, TopUrlMetric | null> = {
   'pages-per-dir': null,
   'status-codes': null,
+  'indexability-distribution': null,
+  'content-kind-distribution': null,
   depth: null,
   'response-time': null,
   'slowest-urls': 'response-time',
@@ -135,6 +145,8 @@ const TOP_URL_METRIC: Record<ReportKind, TopUrlMetric | null> = {
 const VALUE_FORMAT: Record<ReportKind, (v: number | null) => string> = {
   'pages-per-dir': (v) => (v ?? 0).toLocaleString(),
   'status-codes': (v) => (v ?? 0).toLocaleString(),
+  'indexability-distribution': (v) => (v ?? 0).toLocaleString(),
+  'content-kind-distribution': (v) => (v ?? 0).toLocaleString(),
   depth: (v) => (v ?? 0).toLocaleString(),
   'response-time': (v) => (v ?? 0).toLocaleString(),
   'slowest-urls': (v) => (v == null ? '—' : `${v.toLocaleString()} ms`),
@@ -187,6 +199,25 @@ export function ReportsDialog({ open, onClose }: Props) {
                 count: x.count,
               })),
             );
+        } else if (kind === 'indexability-distribution') {
+          const r = await window.freecrawl.reportsIndexabilityDistribution();
+          if (!cancelled)
+            setRows(
+              r.map((x: IndexabilityDistributionRow) => ({
+                key: indexabilityLabel(x.indexability),
+                badge: x.indexability === 'indexable' ? 'OK' : 'NON',
+                count: x.count,
+              })),
+            );
+        } else if (kind === 'content-kind-distribution') {
+          const r = await window.freecrawl.reportsContentKindDistribution();
+          if (!cancelled)
+            setRows(
+              r.map((x: ContentKindDistributionRow) => ({
+                key: contentKindLabel(x.contentKind),
+                count: x.count,
+              })),
+            );
         } else if (kind === 'depth') {
           const r = await window.freecrawl.reportsDepthHistogram();
           if (!cancelled)
@@ -531,3 +562,55 @@ function rtBadge(label: string): string {
   if (label === '500ms–1s') return 'WARN';
   return 'SLOW';
 }
+
+/**
+ * Friendly label for the `indexability` enum the crawler writes. Keeps the
+ * machine-friendly token (`non-indexable:robots-blocked`) out of the UI so
+ * the dropdown reads naturally — Title Case + human-readable suffix.
+ */
+function indexabilityLabel(value: string): string {
+  switch (value) {
+    case 'indexable':
+      return 'Indexable';
+    case 'non-indexable:noindex':
+      return 'Non-indexable — noindex';
+    case 'non-indexable:canonical':
+      return 'Non-indexable — canonicalised';
+    case 'non-indexable:robots-blocked':
+      return 'Non-indexable — robots-blocked';
+    case 'non-indexable:redirect':
+      return 'Non-indexable — redirect';
+    case 'non-indexable:client-error':
+      return 'Non-indexable — 4xx';
+    case 'non-indexable:server-error':
+      return 'Non-indexable — 5xx';
+    default:
+      return value;
+  }
+}
+
+/**
+ * Friendly label for the `content_kind` enum (html / css / js / image /
+ * font / pdf / other). Upper-cases acronyms (HTML/CSS/JS/PDF) so the chart
+ * matches the way users talk about file types.
+ */
+function contentKindLabel(value: string): string {
+  switch (value) {
+    case 'html':
+      return 'HTML';
+    case 'css':
+      return 'CSS';
+    case 'js':
+      return 'JavaScript';
+    case 'image':
+      return 'Image';
+    case 'font':
+      return 'Font';
+    case 'pdf':
+      return 'PDF';
+    case 'other':
+      return 'Other';
+    default:
+      return value;
+  }
+}

apps/desktop/src/renderer/src/components/ScheduledCrawlDialog.tsx

@@ -0,0 +1,352 @@
+import { useEffect, useMemo, useState } from 'react';
+import { X, Clock, AlertTriangle } from 'lucide-react';
+import clsx from 'clsx';
+import type { ScheduleEntry, ScheduleSpec } from '@freecrawl/shared-types';
+
+interface Props {
+  open: boolean;
+  onClose: () => void;
+}
+
+type Cadence = 'hourly' | 'daily' | 'weekly' | 'custom';
+
+const DAY_NAMES = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
+const MIN_CUSTOM_INTERVAL = 15;
+
+interface FormState {
+  enabled: boolean;
+  cadence: Cadence;
+  intervalMinutes: string;
+  hourOfDay: string;
+  minuteOfHour: string;
+  dayOfWeek: number;
+}
+
+const DEFAULT_FORM: FormState = {
+  enabled: false,
+  cadence: 'daily',
+  intervalMinutes: '60',
+  hourOfDay: '03',
+  minuteOfHour: '00',
+  dayOfWeek: 1,
+};
+
+function entryToForm(entry: ScheduleEntry | null): FormState {
+  if (!entry) return DEFAULT_FORM;
+  const { spec } = entry;
+  return {
+    enabled: spec.enabled,
+    cadence: spec.cadence,
+    intervalMinutes: String(spec.intervalMinutes ?? 60),
+    hourOfDay: String(spec.hourOfDay ?? 3).padStart(2, '0'),
+    minuteOfHour: String(spec.minuteOfHour ?? 0).padStart(2, '0'),
+    dayOfWeek: spec.dayOfWeek ?? 1,
+  };
+}
+
+function formToSpec(form: FormState): ScheduleSpec {
+  const out: ScheduleSpec = {
+    enabled: form.enabled,
+    cadence: form.cadence,
+  };
+  if (form.cadence === 'custom') {
+    const n = Math.max(MIN_CUSTOM_INTERVAL, Math.floor(Number(form.intervalMinutes) || 0));
+    out.intervalMinutes = n;
+  }
+  if (form.cadence === 'daily' || form.cadence === 'weekly') {
+    out.hourOfDay = clampInt(Number(form.hourOfDay), 0, 23, 3);
+    out.minuteOfHour = clampInt(Number(form.minuteOfHour), 0, 59, 0);
+  }
+  if (form.cadence === 'weekly') {
+    out.dayOfWeek = clampInt(form.dayOfWeek, 0, 6, 1);
+  }
+  return out;
+}
+
+function clampInt(v: number, min: number, max: number, fallback: number): number {
+  const n = Math.floor(v);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(max, Math.max(min, n));
+}
+
+function formatDateTime(ms: number | null): string {
+  if (ms === null) return '—';
+  const d = new Date(ms);
+  const pad = (n: number) => String(n).padStart(2, '0');
+  return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
+}
+
+export function ScheduledCrawlDialog({ open, onClose }: Props) {
+  const [entry, setEntry] = useState<ScheduleEntry | null>(null);
+  const [form, setForm] = useState<FormState>(DEFAULT_FORM);
+  const [saving, setSaving] = useState(false);
+  const [dirty, setDirty] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  // Re-fetch the schedule each time the dialog reopens so any external
+  // mutation (scheduled crawl fired → lastFiredAt updated) shows on the
+  // status row immediately. Reset dirty flag so the Save button starts
+  // disabled until the user actually edits something.
+  useEffect(() => {
+    if (!open) return;
+    setError(null);
+    setDirty(false);
+    void window.freecrawl.scheduleGet().then((e) => {
+      setEntry(e);
+      setForm(entryToForm(e));
+    });
+  }, [open]);
+
+  useEffect(() => {
+    if (!open) return;
+    const onKey = (e: KeyboardEvent) => {
+      if (e.key === 'Escape') onClose();
+    };
+    window.addEventListener('keydown', onKey);
+    return () => window.removeEventListener('keydown', onKey);
+  }, [open, onClose]);
+
+  function update<K extends keyof FormState>(key: K, value: FormState[K]) {
+    setForm((prev) => ({ ...prev, [key]: value }));
+    setDirty(true);
+  }
+
+  async function save() {
+    setSaving(true);
+    setError(null);
+    try {
+      const spec = formToSpec(form);
+      const result = await window.freecrawl.scheduleSet(spec);
+      setEntry(result);
+      setForm(entryToForm(result));
+      setDirty(false);
+    } catch (err) {
+      setError(err instanceof Error ? err.message : String(err));
+    } finally {
+      setSaving(false);
+    }
+  }
+
+  async function clearSchedule() {
+    if (!entry) return;
+    if (!window.confirm('Remove the scheduled crawl for this project?')) return;
+    setSaving(true);
+    setError(null);
+    try {
+      await window.freecrawl.scheduleSet(null);
+      setEntry(null);
+      setForm(DEFAULT_FORM);
+      setDirty(false);
+    } catch (err) {
+      setError(err instanceof Error ? err.message : String(err));
+    } finally {
+      setSaving(false);
+    }
+  }
+
+  const showInterval = form.cadence === 'custom';
+  const showTimeOfDay = form.cadence === 'daily' || form.cadence === 'weekly';
+  const showDayOfWeek = form.cadence === 'weekly';
+
+  const nextFiresHint = useMemo(() => {
+    if (!form.enabled) return 'Disabled — set "Enabled" and Save to arm the schedule.';
+    return 'Save to compute the next fire time. The schedule only fires while FreeCrawl is open.';
+  }, [form.enabled]);
+
+  if (!open) return null;
+
+  return (
+    <div
+      className="fixed inset-0 z-30 flex items-center justify-center bg-black/60"
+      onClick={onClose}
+    >
+      <div
+        className="flex max-h-[85vh] w-[520px] flex-col rounded-md border border-surface-700 bg-surface-900 shadow-2xl"
+        onClick={(e) => e.stopPropagation()}
+      >
+        <div className="flex items-center border-b border-surface-800 px-4 py-2.5">
+          <Clock className="h-4 w-4 text-surface-300" />
+          <div className="ml-2 text-sm font-semibold tracking-wide text-surface-100">
+            Scheduled Crawl
+          </div>
+          <button
+            className="ml-auto rounded p-1 text-surface-400 hover:bg-surface-800 hover:text-surface-100"
+            onClick={onClose}
+            title="Close (Esc)"
+          >
+            <X className="h-4 w-4" />
+          </button>
+        </div>
+
+        <div className="flex-1 overflow-y-auto px-4 py-3 text-[12px]">
+          <div className="rounded border border-amber-700/40 bg-amber-900/20 px-3 py-2 text-[11px] text-amber-200">
+            <div className="flex items-start gap-2">
+              <AlertTriangle className="mt-0.5 h-3.5 w-3.5 shrink-0" />
+              <div>
+                The schedule fires only while FreeCrawl is open on this
+                project. For triggers that survive an app restart, run
+                <code className="mx-1 rounded bg-surface-950 px-1 font-mono">freecrawl</code>
+                (the CLI) from Windows Task Scheduler / macOS launchd / cron.
+              </div>
+            </div>
+          </div>
+
+          <label className="mt-3 flex items-center gap-2">
+            <input
+              type="checkbox"
+              checked={form.enabled}
+              onChange={(e) => update('enabled', e.target.checked)}
+              className="h-3.5 w-3.5 accent-blue-500"
+            />
+            <span className="font-medium text-surface-100">Enabled</span>
+          </label>
+
+          <div className="mt-3 grid grid-cols-2 gap-3">
+            <label className="flex flex-col gap-1">
+              <span className="text-[11px] text-surface-400">Cadence</span>
+              <select
+                className="h-7 rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                value={form.cadence}
+                onChange={(e) => update('cadence', e.target.value as Cadence)}
+              >
+                <option value="hourly">Hourly</option>
+                <option value="daily">Daily</option>
+                <option value="weekly">Weekly</option>
+                <option value="custom">Custom interval</option>
+              </select>
+            </label>
+
+            {showInterval && (
+              <label className="flex flex-col gap-1">
+                <span className="text-[11px] text-surface-400">
+                  Interval (minutes, min {MIN_CUSTOM_INTERVAL})
+                </span>
+                <input
+                  type="number"
+                  min={MIN_CUSTOM_INTERVAL}
+                  step={5}
+                  value={form.intervalMinutes}
+                  onChange={(e) => update('intervalMinutes', e.target.value)}
+                  className="h-7 rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                />
+              </label>
+            )}
+
+            {showTimeOfDay && (
+              <>
+                <label className="flex flex-col gap-1">
+                  <span className="text-[11px] text-surface-400">Hour (0–23)</span>
+                  <input
+                    type="number"
+                    min={0}
+                    max={23}
+                    value={form.hourOfDay}
+                    onChange={(e) => update('hourOfDay', e.target.value)}
+                    className="h-7 rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                  />
+                </label>
+                <label className="flex flex-col gap-1">
+                  <span className="text-[11px] text-surface-400">Minute (0–59)</span>
+                  <input
+                    type="number"
+                    min={0}
+                    max={59}
+                    value={form.minuteOfHour}
+                    onChange={(e) => update('minuteOfHour', e.target.value)}
+                    className="h-7 rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
+                  />
+                </label>
+              </>
+            )}
+
+            {showDayOfWeek && (
+              <label className="col-span-2 flex flex-col gap-1">
+                <span className="text-[11px] text-surface-400">Day of week</span>
+                <div className="flex flex-wrap gap-1">
+                  {DAY_NAMES.map((label, idx) => (
+                    <button
+                      key={label}
+                      type="button"
+                      onClick={() => update('dayOfWeek', idx)}
+                      className={clsx(
+                        'rounded border px-2 py-1 text-[11px] transition',
+                        form.dayOfWeek === idx
+                          ? 'border-blue-500 bg-blue-500/15 text-blue-200'
+                          : 'border-surface-700 text-surface-300 hover:bg-surface-800',
+                      )}
+                    >
+                      {label}
+                    </button>
+                  ))}
+                </div>
+              </label>
+            )}
+          </div>
+
+          <div className="mt-3 rounded border border-surface-800 bg-surface-950/60 px-3 py-2">
+            <div className="grid grid-cols-2 gap-y-1 text-[11px]">
+              <span className="text-surface-400">Next fire</span>
+              <span className="font-mono text-surface-200">
+                {entry ? formatDateTime(entry.status.nextFiresAt) : '—'}
+              </span>
+              <span className="text-surface-400">Last fired</span>
+              <span className="font-mono text-surface-200">
+                {entry ? formatDateTime(entry.status.lastFiredAt) : '—'}
+              </span>
+              <span className="text-surface-400">Last status</span>
+              <span
+                className={clsx(
+                  'font-mono',
+                  entry?.status.lastStatus === 'success' && 'text-emerald-400',
+                  entry?.status.lastStatus === 'failure' && 'text-red-400',
+                  entry?.status.lastStatus === 'running' && 'text-blue-300',
+                  !entry?.status.lastStatus && 'text-surface-500',
+                )}
+              >
+                {entry?.status.lastStatus ?? '—'}
+              </span>
+            </div>
+            <div className="mt-2 text-[11px] text-surface-500">{nextFiresHint}</div>
+          </div>
+
+          {error && (
+            <div className="mt-3 rounded border border-red-700/50 bg-red-900/20 px-3 py-2 text-[11px] text-red-200">
+              {error}
+            </div>
+          )}
+        </div>
+
+        <div className="flex items-center gap-2 border-t border-surface-800 px-4 py-2.5">
+          {entry && (
+            <button
+              type="button"
+              onClick={clearSchedule}
+              disabled={saving}
+              className="rounded border border-red-700/60 px-2.5 py-1 text-[11px] text-red-300 hover:bg-red-900/20 disabled:cursor-not-allowed disabled:opacity-50"
+            >
+              Remove
+            </button>
+          )}
+          <div className="ml-auto flex items-center gap-2">
+            <button
+              type="button"
+              onClick={onClose}
+              disabled={saving}
+              className="rounded border border-surface-700 px-2.5 py-1 text-[11px] text-surface-300 hover:bg-surface-800 disabled:cursor-not-allowed disabled:opacity-50"
+            >
+              Cancel
+            </button>
+            <button
+              type="button"
+              onClick={save}
+              disabled={!dirty || saving}
+              className="rounded bg-blue-600 px-3 py-1 text-[11px] font-medium text-white hover:bg-blue-500 disabled:cursor-not-allowed disabled:opacity-50"
+            >
+              {saving ? 'Saving…' : 'Save'}
+            </button>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}

apps/mcp-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/mcp-server",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": true,
   "type": "module",
   "bin": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/project-db.ts

@@ -2292,6 +2292,42 @@ export class ProjectDb {
       .all() as { status: number | null; count: number }[];
   }
 
+  /**
+   * Indexability distribution across internal URLs. Seven buckets —
+   * `indexable` + the six `non-indexable:…` reasons — give the user a
+   * one-glance breakdown of why pages are or aren't reaching Google.
+   * Ordered by `count DESC` so the dominant bucket sits at the top.
+   */
+  getIndexabilityDistribution(): { indexability: string; count: number }[] {
+    return this.db
+      .prepare(
+        `SELECT indexability, COUNT(*) AS count
+         FROM urls
+         WHERE is_external = 0 AND indexability IS NOT NULL
+         GROUP BY indexability
+         ORDER BY count DESC, indexability`,
+      )
+      .all() as { indexability: string; count: number }[];
+  }
+
+  /**
+   * Content-kind distribution across internal URLs (HTML / CSS / JS /
+   * image / font / pdf / other). Mirrors `content_kind` exactly; useful
+   * for spotting unexpectedly large non-HTML payloads (e.g. an asset CDN
+   * accidentally crawled in-scope).
+   */
+  getContentKindDistribution(): { contentKind: string; count: number }[] {
+    return this.db
+      .prepare(
+        `SELECT content_kind AS contentKind, COUNT(*) AS count
+         FROM urls
+         WHERE is_external = 0 AND content_kind IS NOT NULL
+         GROUP BY content_kind
+         ORDER BY count DESC, content_kind`,
+      )
+      .all() as { contentKind: string; count: number }[];
+  }
+
   /**
    * Click-depth distribution for internal HTML pages — the canonical
    * "site architecture flatness" metric. Shallow sites bias toward depths

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/ipc.ts

@@ -97,6 +97,8 @@ export const IPC = {
   urlClusterMembers: 'urls:cluster-members',
   reportsPagesPerDirectory: 'reports:pages-per-directory',
   reportsStatusCodeHistogram: 'reports:status-code-histogram',
+  reportsIndexabilityDistribution: 'reports:indexability-distribution',
+  reportsContentKindDistribution: 'reports:content-kind-distribution',
   reportsDepthHistogram: 'reports:depth-histogram',
   reportsResponseTimeHistogram: 'reports:response-time-histogram',
   reportsTopUrls: 'reports:top-urls',
@@ -121,10 +123,48 @@ export const IPC = {
   rendererLagReport: 'renderer:lag-report',
   prefsExportSettings: 'prefs:export-settings',
   prefsImportSettings: 'prefs:import-settings',
+  scheduleGet: 'schedule:get',
+  scheduleSet: 'schedule:set',
 } as const;
 
 export type IpcChannel = (typeof IPC)[keyof typeof IPC];
 
+/**
+ * In-app scheduled crawl spec. One schedule per project (keyed by the
+ * absolute `.seoproject` path in app-level prefs). Fires only while the
+ * desktop app is open and the project is loaded — for triggers that
+ * survive an app restart, drive the CLI via the OS scheduler (Windows
+ * Task Scheduler / launchd / cron); that path is V2.
+ */
+export interface ScheduleSpec {
+  enabled: boolean;
+  /** `hourly` = every hour on the minute. `daily` = once at hourOfDay:
+   *  minuteOfHour. `weekly` = once at dayOfWeek+hourOfDay:minuteOfHour.
+   *  `custom` = every `intervalMinutes` (min 15). */
+  cadence: 'hourly' | 'daily' | 'weekly' | 'custom';
+  /** For `custom`. Minimum enforced at 15 to prevent runaway loops. */
+  intervalMinutes?: number;
+  /** 0–23. Used by daily/weekly. */
+  hourOfDay?: number;
+  /** 0–59. Used by daily/weekly. */
+  minuteOfHour?: number;
+  /** 0 = Sunday … 6 = Saturday. Used by weekly. */
+  dayOfWeek?: number;
+}
+
+export interface ScheduleStatus {
+  /** Epoch ms when last fire completed (success or failure). */
+  lastFiredAt: number | null;
+  /** Epoch ms when the next fire is due. Server-computed on set. */
+  nextFiresAt: number | null;
+  lastStatus: 'success' | 'failure' | 'running' | null;
+}
+
+export interface ScheduleEntry {
+  spec: ScheduleSpec;
+  status: ScheduleStatus;
+}
+
 export interface UrlsQueryInput {
   limit: number;
   offset: number;
@@ -192,6 +232,7 @@ export type MenuEvent =
   | 'open-sitemap-validator'
   | 'open-reports'
   | 'open-settings'
+  | 'open-scheduled-crawl'
   | 'about';
 
 export interface ExportCsvResult {
@@ -544,6 +585,20 @@ export interface StatusCodeHistogramRow {
   count: number;
 }
 
+export interface IndexabilityDistributionRow {
+  /** `indexable` or one of `non-indexable:noindex` / `:canonical` /
+   *  `:robots-blocked` / `:redirect` / `:client-error` / `:server-error`. */
+  indexability: string;
+  count: number;
+}
+
+export interface ContentKindDistributionRow {
+  /** `html` / `css` / `js` / `image` / `font` / `pdf` / `other` — matches
+   *  the `content_kind` column populated at insert time. */
+  contentKind: string;
+  count: number;
+}
+
 export interface DepthHistogramRow {
   depth: number;
   count: number;
@@ -866,6 +921,8 @@ export interface FreeCrawlApi {
   urlClusterMembers(urlId: number): Promise<UrlClusterMember[]>;
   reportsPagesPerDirectory(input: PagesPerDirectoryInput): Promise<PagesPerDirectoryRow[]>;
   reportsStatusCodeHistogram(): Promise<StatusCodeHistogramRow[]>;
+  reportsIndexabilityDistribution(): Promise<IndexabilityDistributionRow[]>;
+  reportsContentKindDistribution(): Promise<ContentKindDistributionRow[]>;
   reportsDepthHistogram(): Promise<DepthHistogramRow[]>;
   reportsResponseTimeHistogram(): Promise<ResponseTimeHistogramRow[]>;
   reportsTopUrls(input: TopUrlsInput): Promise<TopUrlsRow[]>;
@@ -886,6 +943,12 @@ export interface FreeCrawlApi {
   reportRendererLag(lagMs: number): void;
   prefsExportSettings(input: SettingsExportInput): Promise<SettingsExportResult>;
   prefsImportSettings(): Promise<SettingsImportResult>;
+  /** Read the schedule for the currently-loaded project. `null` when no
+   *  project is open OR no schedule has been configured. */
+  scheduleGet(): Promise<ScheduleEntry | null>;
+  /** Save / update the schedule for the currently-loaded project. Passing
+   *  `null` removes the schedule entirely. */
+  scheduleSet(spec: ScheduleSpec | null): Promise<ScheduleEntry | null>;
   onLogEntry(cb: (entry: LogEntry) => void): () => void;
   onLogsBatch(cb: (entries: LogEntry[]) => void): () => void;
   onLogsBusy(cb: (busy: boolean) => void): () => void;