feat: spider crawl matrix + grid copy + crawl speed fixes

README.md

@@ -33,6 +33,8 @@ FreeCrawl crawls your website and checks it for SEO problems — a free, desktop
 
 It runs **150+ SEO checks across 32 tabs** — page titles, meta descriptions, headings, duplicate content, broken links, redirects, canonicals, hreflang, structured data, security headers, readability, and more. It can also render JavaScript pages with a real headless browser to capture the final DOM, screenshots, and Core Web Vitals.
 
+Every link type — images, CSS, JavaScript, media, canonicals, pagination, hreflang, AMP, iframes — has its own **Crawl** and **Store** switch, so you decide exactly what gets fetched and what gets reported. Right-click any table selection to copy cells, rows, or columns straight to the clipboard.
+
 Pull in real data from **Google Search Console, Analytics 4, PageSpeed Insights, and Chrome UX Report**, check spelling and grammar with **LanguageTool** — with a **bundled offline dictionary** for languages LanguageTool has no rules for, such as Turkish — and run your own **AI prompts** on each page (OpenAI, Claude, or Ollama). Open several projects side by side, each in its own window running its own crawl, and manage them from a Projects dialog with search, tags, and starter templates.
 
 Export to **Excel, CSV, JSON, XML**, an HTML report, sitemaps, **Google Sheets**, or **BigQuery**, save each project as **one compressed file** (or a **password-protected encrypted file**), and export any URL Details sub-table on its own. A built-in **MCP server** (96 tools) lets AI agents like Claude drive crawls live and read every result — every action a human can take in the desktop is callable from an agent. Everything runs **fully local** — no telemetry, no cloud, MIT-licensed.

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/cli",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "private": true,
   "type": "module",
   "bin": {

apps/cli/src/index.ts

@@ -37,6 +37,10 @@ Options:
   --rps <n>           Max requests per second (default: ${DEFAULT_CRAWL_CONFIG.maxRps})
   --user-agent <str>  Custom User-Agent string
   --no-robots         Ignore robots.txt
+  --respect-crawl-delay
+                      Honour a robots.txt Crawl-delay directive as a global rate limit
+                        (one request every N seconds). Off by default — the directive is
+                        not part of RFC 9309 and published values are often crippling.
   --external          Follow external links
   --header <K: V>     Extra request header; repeatable (e.g. --header "Authorization: Bearer X")
   --include <regex>   Only crawl URLs matching this regex; repeatable
@@ -71,6 +75,7 @@ async function main(): Promise<void> {
       rps: { type: 'string' },
       'user-agent': { type: 'string' },
       'no-robots': { type: 'boolean' },
+      'respect-crawl-delay': { type: 'boolean' },
       external: { type: 'boolean' },
       header: { type: 'string', multiple: true },
       include: { type: 'string', multiple: true },
@@ -220,6 +225,7 @@ async function main(): Promise<void> {
     // `--no-robots` / `--external` only flip the flag ON in their direction;
     // absent, the file value (or default) stands.
     ...(values['no-robots'] ? { respectRobotsTxt: false } : {}),
+    ...(values['respect-crawl-delay'] ? { respectCrawlDelay: true } : {}),
     ...(values.external ? { crawlExternal: true } : {}),
     ...(values.header !== undefined ? { customHeaders: parseHeaders(values.header) } : {}),
     ...(values.include !== undefined ? { includePatterns: values.include } : {}),

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/desktop",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "private": true,
   "description": "FreeCrawl SEO — desktop application",
   "homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",

apps/desktop/src/main/index.ts

@@ -109,6 +109,8 @@ import {
   type SitemapGenerateInput,
   type SitemapGenerateResult,
   type UrlBulkContextMenuInput,
+  type UrlGridContextMenuInput,
+  type UrlGridCopyEvent,
   type UrlContextMenuInput,
   type UrlDetail,
   type UrlDetailInput,
@@ -5277,7 +5279,18 @@ function registerIpc(): void {
     // window (multi-window) — these callbacks run outside any IPC handler's
     // AsyncLocalStorage context, so they can't use `currentSession()`.
     crawler.on('progress', (p: CrawlProgress) => {
-      if (sess.activeCrawler !== crawler) return;
+      // A superseded crawler must not clobber the live crawl's UI state
+      // — but its own terminal `running: false` frame is still valid
+      // when NO crawl is active, and dropping it is what left the UI
+      // pinned to "Running" with a dead Stop/Pause button. The 200 ms
+      // progress throttle can push that frame past the 'done' handler,
+      // which has already set `activeCrawler = null` by then.
+      if (
+        sess.activeCrawler !== crawler &&
+        !(sess.activeCrawler === null && !p.running)
+      ) {
+        return;
+      }
       sess.latestProgress = p;
       sess.send(IPC.crawlProgress, p);
       freezeWatchdog.updateCounters({
@@ -5339,11 +5352,24 @@ function registerIpc(): void {
     });
   }
 
-  async function launchCrawl(config: CrawlConfig): Promise<void> {
+  async function launchCrawl(
+    config: CrawlConfig,
+    opts: {
+      /** Re-fetch only these URLs (Re-Spider) instead of walking the site. */
+      respiderSeeds?: string[];
+      /**
+       * Session to run under. Required from Electron menu callbacks: they
+       * fire after the IPC handler has unwound, so AsyncLocalStorage is
+       * gone and `currentSession()` would silently resolve to the primary
+       * window's project instead of the one that was right-clicked.
+       */
+      session?: ProjectSession;
+    } = {},
+  ): Promise<void> {
     // Capture the launching window's session ONCE. The crawler outlives
     // this handler's AsyncLocalStorage context, so its hooks + listeners
     // are bound to `sess` rather than resolved via `currentSession()`.
-    const sess = currentSession();
+    const sess = opts.session ?? currentSession();
     const previousCrawler = sess.activeCrawler;
     if (previousCrawler) {
       previousCrawler.stop();
@@ -5355,7 +5381,9 @@ function registerIpc(): void {
     logger.log(
       'info',
       'crawler',
-      `Crawl starting: ${config.startUrl} (scope=${config.scope}, maxDepth=${config.maxDepth}, maxUrls=${config.maxUrls}, concurrency=${config.maxConcurrency}, rps=${config.maxRps})`,
+      opts.respiderSeeds
+        ? `Re-Spider starting: ${opts.respiderSeeds.length} URL(s)`
+        : `Crawl starting: ${config.startUrl} (scope=${config.scope}, maxDepth=${config.maxDepth}, maxUrls=${config.maxUrls}, concurrency=${config.maxConcurrency}, rps=${config.maxRps})`,
       undefined,
       sess.logTag,
     );
@@ -5625,6 +5653,7 @@ function registerIpc(): void {
       runDbPass: (method) => runDbPassViaPool(sess, method),
       dbCall: (method, args) => dbCallViaPool(sess, method, args),
       renderUrl: renderHook,
+      respiderSeeds: opts.respiderSeeds,
     });
     sess.activeCrawler = crawler;
     attachCrawlerListeners(crawler, sess);
@@ -5639,6 +5668,59 @@ function registerIpc(): void {
     void crawler.start();
   }
 
+  /**
+   * Re-Spider a specific set of URLs, whether or not a crawl is running.
+   *
+   * With a live crawl the URLs are simply pushed back onto its queue. With
+   * none — the normal case once a crawl has finished — a crawler is spun
+   * up for just those URLs. That path is why Re-Spider is no longer gated
+   * on "start a crawl first": re-reading a page the user believes has
+   * changed is the exact moment they do NOT want to re-crawl the site.
+   *
+   * The config is reconstructed from whatever the project last ran with,
+   * so a re-spider honours the same scope, timeouts, rendering mode and
+   * auth as the crawl that produced the rows. `startUrl` is pinned to the
+   * project's recorded seed: a mismatch there is what tells `Crawler` a
+   * different site is being crawled, and the re-spider must never read as
+   * one.
+   */
+  async function launchRespider(sess: ProjectSession, urls: string[]): Promise<void> {
+    if (urls.length === 0) return;
+    const db = sess.getDb();
+    // Clear stale markers on exactly these rows so the green flags that
+    // come back describe this re-spider, not an earlier one.
+    db.clearChangedFlagsForUrls(urls);
+
+    const live = sess.activeCrawler;
+    if (live && live.isRunning) {
+      for (const u of urls) live.requeueUrl(u);
+      fireDataChanged(sess);
+      return;
+    }
+
+    let config = sess.lastCrawlConfig;
+    if (!config) {
+      const stored = db.getMeta('lastCrawlConfig');
+      if (stored) {
+        try {
+          config = JSON.parse(stored) as CrawlConfig;
+        } catch {
+          /* corrupt meta — fall through to defaults */
+        }
+      }
+    }
+    const seed = db.getMeta('startUrl') ?? urls[0]!;
+    // Spider mode explicitly: a project last run in List or Sitemap mode
+    // would otherwise send `start()` down a fixed-URL path that ignores
+    // the re-spider seeds entirely.
+    const respiderConfig: CrawlConfig = {
+      ...(config ?? DEFAULT_CRAWL_CONFIG),
+      mode: 'spider',
+      startUrl: seed,
+    };
+    await launchCrawl(respiderConfig, { respiderSeeds: urls, session: sess });
+  }
+
   /**
    * V2 Faz 1 — Resolve the per-project screenshots sidecar directory.
    * Lives alongside the `.seoproject` file as `<project>.screenshots/`
@@ -5680,8 +5762,23 @@ function registerIpc(): void {
   });
 
   registerHandle(IPC.crawlStop, () => {
-    currentSession().activeCrawler?.stop();
-    void disposeBrowserPool(currentSession());
+    const sess = currentSession();
+    if (sess.activeCrawler) {
+      sess.activeCrawler.stop();
+    } else if (sess.latestProgress?.running) {
+      // No crawler to stop, yet the renderer still thinks one is live —
+      // it missed the terminal progress frame. Re-send it rather than
+      // no-op, so Stop always gets the UI back to an idle state instead
+      // of appearing frozen.
+      const p: CrawlProgress = {
+        ...sess.latestProgress,
+        running: false,
+        paused: false,
+      };
+      sess.latestProgress = p;
+      sess.send(IPC.crawlProgress, p);
+    }
+    void disposeBrowserPool(sess);
   });
 
   registerHandle(IPC.crawlPause, () => {
@@ -5899,27 +5996,20 @@ function registerIpc(): void {
     // ---- Faz 0.5 Increment 3 — URL mutation, exports, config, integrations, project file ----
 
     'respider-urls': async (input) => {
-      // Mirrors the URL-bulk context menu's "Re-Spider" item. Requires
-      // an active crawler — the queued URLs are re-fetched live as part
-      // of the existing crawl. With no active crawl, the DB rows are
-      // still marked dirty so the NEXT crawl will pick them up.
+      // Mirrors the URL-bulk context menu's "Re-Spider" item. A live
+      // crawl absorbs the URLs into its queue; with no crawl running a
+      // crawler is started for exactly these URLs, so an agent can
+      // refresh a page without re-crawling the site.
       const ids = (input as { urlIds?: number[] }).urlIds ?? [];
       if (!Array.isArray(ids) || ids.length === 0) {
         throw new Error('urlIds is required (non-empty array).');
       }
-      const db = getDb();
-      db.markUrlsForRecrawl(ids);
-      let requeued = 0;
-      const requeueCrawler = currentSession().activeCrawler;
-      if (requeueCrawler && requeueCrawler.isRunning) {
-        const urls = db.getUrlsByIds(ids);
-        for (const u of urls) {
-          requeueCrawler.requeueUrl(u);
-          requeued++;
-        }
-      }
+      const sess = currentSession();
+      const urls = sess.getDb().getUrlsByIds(ids);
+      const wasRunning = sess.activeCrawler?.isRunning === true;
+      await launchRespider(sess, urls);
       fireDataChanged();
-      return { marked: ids.length, requeued, hasActiveCrawl: currentSession().activeCrawler?.isRunning === true };
+      return { marked: urls.length, requeued: urls.length, hasActiveCrawl: wasRunning };
     },
 
     'remove-urls': async (input) => {
@@ -7058,8 +7148,6 @@ function registerIpc(): void {
     // getDb() there would silently hit defaultSession (the primary window)
     // and mutate the WRONG project's DB. Bind everything to `sess`.
     const sess = currentSession();
-    const ctxCrawler = sess.activeCrawler;
-    const canRecrawl = ctxCrawler !== null && ctxCrawler.isRunning;
     const L = getMenuLabels(getMenuLang());
 
     const template: MenuItemConstructorOptions[] = [
@@ -7076,16 +7164,11 @@ function registerIpc(): void {
       { type: 'separator' },
       {
         label: L.ctxRespider,
-        enabled: canRecrawl,
-        toolTip: canRecrawl ? undefined : L.ctxStartCrawlFirst,
+        // No longer gated on a live crawl — `launchRespider` starts a
+        // one-URL crawler when none is running, which is when the user
+        // actually reaches for this.
         click: () => {
-          const db = sess.getDb();
-          db.markUrlForRecrawl(input.urlId);
-          const ctxRequeueCrawler = sess.activeCrawler;
-          if (ctxRequeueCrawler) {
-            ctxRequeueCrawler.requeueUrl(input.url);
-          }
-          fireDataChanged(sess);
+          void launchRespider(sess, [input.url]);
         },
       },
       {
@@ -7127,8 +7210,6 @@ function registerIpc(): void {
       const ids = input.urlIds;
       if (ids.length === 0) return;
       const urls = db.getUrlsByIds(ids);
-      const bulkCtxCrawler = sess.activeCrawler;
-      const canRecrawl = bulkCtxCrawler !== null && bulkCtxCrawler.isRunning;
       const n = ids.length.toLocaleString();
       const L = getMenuLabels(getMenuLang());
 
@@ -7149,15 +7230,9 @@ function registerIpc(): void {
         { type: 'separator' },
         {
           label: L.ctxRespiderNUrls.replace('{n}', n),
-          enabled: canRecrawl,
-          toolTip: canRecrawl ? undefined : L.ctxStartCrawlFirst,
+          // Always available — see the single-URL menu above.
           click: () => {
-            db.markUrlsForRecrawl(ids);
-            const bulkRequeueCrawler = sess.activeCrawler;
-            if (bulkRequeueCrawler) {
-              for (const u of urls) bulkRequeueCrawler.requeueUrl(u);
-            }
-            fireDataChanged(sess);
+            void launchRespider(sess, urls);
           },
         },
         {
@@ -7189,6 +7264,129 @@ function registerIpc(): void {
     },
   );
 
+  /**
+   * Selection-aware context menu for the URL grid.
+   *
+   * Supersedes the two menus above for right-clicks inside the table:
+   * those two are row-only, which forced the renderer to collapse a cell
+   * or column selection down to a single row before it could show a
+   * menu. Here the renderer keeps its selection and tells us which layer
+   * the click landed in; the menu names that scope and the row-semantic
+   * actions apply to every row the selection touches.
+   *
+   * The Copy item is pushed back to the renderer over `urlGridCopy`
+   * instead of being executed here — only the renderer knows what a cell
+   * says. Everything else is handled in place.
+   */
+  registerHandle(
+    IPC.urlGridContextMenu,
+    (e, input: UrlGridContextMenuInput): void => {
+      const win = BrowserWindow.fromWebContents(e.sender);
+      // Bind to the calling session while ALS is still live — popup() is
+      // non-blocking, so the click callbacks run after this handler has
+      // unwound (same reasoning as the two menus above).
+      const sess = currentSession();
+      const db = sess.getDb();
+      const ids = input.urlIds;
+      if (ids.length === 0) return;
+      // Display order, not id order — `ids` arrives in the order the rows
+      // appear on screen and every item below should honour that.
+      const urls = db.getUrlsByIdsInOrder(ids);
+      const many = ids.length > 1;
+      const n = ids.length.toLocaleString();
+      const L = getMenuLabels(getMenuLang());
+
+      // Name what the user actually picked, so the menu reads as a
+      // continuation of the selection rather than a reset of it.
+      const copyLabel = ((): string => {
+        if (input.scope === 'columns') {
+          return input.columnCount > 1
+            ? L.ctxCopyNColumns.replace('{n}', input.columnCount.toLocaleString())
+            : L.ctxCopyColumn;
+        }
+        if (input.scope === 'cells') {
+          return input.cellCount > 1
+            ? L.ctxCopyNCells.replace('{n}', input.cellCount.toLocaleString())
+            : L.ctxCopyCell;
+        }
+        return many ? L.ctxCopyNRows.replace('{n}', n) : L.ctxCopyRow;
+      })();
+
+      const template: MenuItemConstructorOptions[] = [
+        {
+          label: copyLabel,
+          click: () => {
+            if (!e.sender.isDestroyed()) {
+              e.sender.send(IPC.urlGridCopy, 'copy-selection' satisfies UrlGridCopyEvent);
+            }
+          },
+        },
+        {
+          label: many ? L.ctxCopyNUrls.replace('{n}', n) : L.ctxCopy,
+          click: () => clipboard.writeText(urls.join('\n')),
+        },
+        {
+          label: many ? L.ctxOpenNUrlsInBrowser.replace('{n}', n) : L.ctxOpenInBrowser,
+          // Opening hundreds of tabs at once is a bad default.
+          enabled: urls.length <= 20,
+          toolTip: urls.length > 20 ? L.ctxOpenLimitTooltip : undefined,
+          click: () => {
+            for (const u of urls) if (isSafeExternalUrl(u)) void shell.openExternal(u);
+          },
+        },
+        { type: 'separator' },
+        {
+          label: many ? L.ctxRespiderNUrls.replace('{n}', n) : L.ctxRespider,
+          click: () => {
+            void launchRespider(sess, urls);
+          },
+        },
+        {
+          label: many ? L.ctxRemoveNUrls.replace('{n}', n) : L.ctxRemove,
+          click: () => {
+            db.deleteUrls(ids);
+            fireDataChanged(sess);
+          },
+        },
+        { type: 'separator' },
+      ];
+      // The tail differs by count, and not only for grammar: bulk export
+      // is meaningless for one row, and robots.txt is per-origin so it
+      // only makes sense when there's a single URL in play. Same split
+      // the two older menus used.
+      if (many) {
+        template.push({
+          label: L.ctxExportNUrlsAsCsv.replace('{n}', n),
+          click: async () => {
+            const w = win ?? mainWindow;
+            if (!w) return;
+            const res = await dialog.showSaveDialog(w, {
+              defaultPath: 'freecrawl-selected.csv',
+              filters: [{ name: 'CSV', extensions: ['csv'] }],
+            });
+            if (res.canceled || !res.filePath) return;
+            await exportUrlsToCsv(db, res.filePath, { selectedIds: ids });
+          },
+        });
+      } else {
+        template.push({
+          label: L.ctxOpenRobotsTxt,
+          click: () => {
+            try {
+              void shell.openExternal(new URL(input.url).origin + '/robots.txt');
+            } catch {
+              /* ignore malformed URL */
+            }
+          },
+        });
+      }
+
+      const menu = Menu.buildFromTemplate(template);
+      if (win) menu.popup({ window: win });
+      else menu.popup();
+    },
+  );
+
   registerHandle(IPC.urlDetailGet, async (_e, input: UrlDetailInput): Promise<UrlDetail | null> =>
     callReaderOrFallback<UrlDetail | null>(
       'getUrlDetail',
@@ -7542,6 +7740,10 @@ function registerIpc(): void {
         recomputeIssues: (defs) => recomputeIssuesViaPool(sess, defs),
         runDbPass: (method) => runDbPassViaPool(sess, method),
         dbCall: (method, args) => dbCallViaPool(sess, method, args),
+        // Continue the interrupted crawl only. Without this the same
+        // start URL would read as "Start pressed again" and re-fetch the
+        // pages that already completed before the crash.
+        resumeOnly: true,
       });
       sess.activeCrawler = crawler;
       attachCrawlerListeners(crawler, sess);

apps/desktop/src/main/menu-i18n.ts

@@ -106,6 +106,13 @@ export interface MenuLabels {
   ctxRespiderNUrls: string;
   ctxRemoveNUrls: string;
   ctxExportNUrlsAsCsv: string;
+  /** Selection-scoped copy items. `{n}` → cell / row / column count. */
+  ctxCopyCell: string;
+  ctxCopyNCells: string;
+  ctxCopyRow: string;
+  ctxCopyNRows: string;
+  ctxCopyColumn: string;
+  ctxCopyNColumns: string;
 
   // ── Recurring dialog buttons ──
   btnOk: string;
@@ -257,6 +264,12 @@ const MENU_EN: MenuLabels = {
   ctxRespiderNUrls: 'Re-Spider {n} URLs',
   ctxRemoveNUrls: 'Remove {n} URLs',
   ctxExportNUrlsAsCsv: 'Export {n} URLs as CSV…',
+  ctxCopyCell: 'Copy Cell',
+  ctxCopyNCells: 'Copy {n} Cells',
+  ctxCopyRow: 'Copy Row',
+  ctxCopyNRows: 'Copy {n} Rows',
+  ctxCopyColumn: 'Copy Column',
+  ctxCopyNColumns: 'Copy {n} Columns',
 
   btnOk: 'OK',
   btnCancel: 'Cancel',
@@ -408,6 +421,12 @@ const MENU_TR: MenuLabels = {
   ctxRespiderNUrls: '{n} URL\'yi Yeniden Tara',
   ctxRemoveNUrls: '{n} URL\'yi Kaldır',
   ctxExportNUrlsAsCsv: '{n} URL\'yi CSV Olarak Dışa Aktar…',
+  ctxCopyCell: 'Hücreyi Kopyala',
+  ctxCopyNCells: '{n} Hücreyi Kopyala',
+  ctxCopyRow: 'Satırı Kopyala',
+  ctxCopyNRows: '{n} Satırı Kopyala',
+  ctxCopyColumn: 'Sütunu Kopyala',
+  ctxCopyNColumns: '{n} Sütunu Kopyala',
 
   btnOk: 'Tamam',
   btnCancel: 'İptal',

apps/desktop/src/preload/index.ts

@@ -87,6 +87,8 @@ import {
   type SitemapGenerateInput,
   type SitemapGenerateResult,
   type UrlBulkContextMenuInput,
+  type UrlGridContextMenuInput,
+  type UrlGridCopyEvent,
   type UrlContextMenuInput,
   type UrlDetail,
   type UrlDetailInput,
@@ -233,6 +235,8 @@ const api: FreeCrawlApi = {
     ipcRenderer.invoke(IPC.urlContextMenu, input),
   urlBulkContextMenu: (input: UrlBulkContextMenuInput): Promise<void> =>
     ipcRenderer.invoke(IPC.urlBulkContextMenu, input),
+  urlGridContextMenu: (input: UrlGridContextMenuInput): Promise<void> =>
+    ipcRenderer.invoke(IPC.urlGridContextMenu, input),
   imagesQuery: (input: ImagesQueryInput): Promise<ImagesQueryResult> =>
     ipcRenderer.invoke(IPC.imagesQuery, input),
   brokenLinksQuery: (input: BrokenLinksQueryInput): Promise<BrokenLinksQueryResult> =>
@@ -487,6 +491,7 @@ const api: FreeCrawlApi = {
   onDone: (cb) => subscribe<CrawlSummary>(IPC.crawlDone, cb),
   onError: (cb) => subscribe<string>(IPC.crawlError, cb),
   onMenuEvent: (cb) => subscribe<MenuEvent>(IPC.menuEvent, cb),
+  onUrlGridCopy: (cb) => subscribe<UrlGridCopyEvent>(IPC.urlGridCopy, cb),
   editCopyNative: (): Promise<void> => ipcRenderer.invoke(IPC.editCopyNative),
   onDataChanged: (cb) => {
     const listener = (): void => cb();

apps/desktop/src/renderer/src/App.tsx

@@ -39,6 +39,7 @@ export function App() {
   const toggleDetailPanel = useAppStore((s) => s.toggleDetailPanel);
   const setProgress = useAppStore((s) => s.setProgress);
   const setSummary = useAppStore((s) => s.setSummary);
+  const markCrawlFinished = useAppStore((s) => s.markCrawlFinished);
   const setError = useAppStore((s) => s.setError);
   const bumpDataVersion = useAppStore((s) => s.bumpDataVersion);
   const reset = useAppStore((s) => s.reset);
@@ -219,7 +220,22 @@ export function App() {
         }
       });
     });
-    const off2 = window.freecrawl.onDone((s) => setSummary(s));
+    const off2 = window.freecrawl.onDone((s) => {
+      setSummary(s);
+      // 'done' is terminal. Flush any frame still buffered for the next
+      // animation frame — dropping it would lose the final counters,
+      // keeping it as-is would re-assert `running: true` one frame after
+      // we clear it — then make sure the toolbar leaves the Running
+      // state even when no frame arrives at all. A crawl that ends
+      // within one throttle window of its last progress emit used to
+      // leave the UI pinned to "Running" with Stop/Pause wired to an
+      // already-detached crawler.
+      if (pendingProgress) {
+        setProgress({ ...pendingProgress, running: false, paused: false });
+        pendingProgress = null;
+      }
+      markCrawlFinished();
+    });
     const off3 = window.freecrawl.onError(setError);
     const offData = window.freecrawl.onDataChanged(() => bumpDataVersion());
     // Per-project settings — when a project opens, swap the UI to its saved
@@ -233,10 +249,12 @@ export function App() {
           // macOS-only round-trip (see menu.ts): the native menu eats
           // Cmd+C before the page sees a keydown, so the Copy item
           // forwards here instead. Re-dispatch a synthetic keydown so
-          // the existing document-level Ctrl/Cmd+C table-copy handlers
-          // (detail panel cells, broken-links cells) run unchanged; if
-          // none of them preventDefault'ed — no table selection — fall
-          // back to the stock native copy for text selections/inputs.
+          // the document-level Ctrl/Cmd+C grid-copy handlers run
+          // unchanged — the URL table, the Broken Links table and the
+          // detail panel's links table all listen for it, and arbitrate
+          // between themselves via `ownsGridCopy`. If none of them
+          // preventDefault'ed — no grid selection — fall back to the
+          // stock native copy for text selections/inputs.
           const ae = document.activeElement as HTMLElement | null;
           const editable =
             ae !== null &&
@@ -449,6 +467,7 @@ export function App() {
     t,
     setProgress,
     setSummary,
+    markCrawlFinished,
     setError,
     toggleSidebar,
     toggleDetailPanel,

apps/desktop/src/renderer/src/components/BottomDetailPanel.tsx

@@ -21,6 +21,20 @@ import type {
 import { GridExportButton } from './GridExportButton.js';
 import { useAppStore } from '../store.js';
 import { diagnoseStatus, type StatusDiagnosis } from '../utils/statusDiagnosis.js';
+import {
+  copySelection,
+  isAdditiveClick,
+  isGridCopyShortcut,
+  isMacSecondaryClick,
+  markGridActive,
+  ownsGridCopy,
+  writeTextToClipboard,
+} from '../utils/clipboard.js';
+
+/** Identifies the detail panel's links grid to the copy-shortcut
+ *  arbitration in `clipboard.ts`. Only one LinksView is mounted at a
+ *  time (the sub-tabs are mutually exclusive), so one id is enough. */
+const GRID_ID = 'detail-links';
 
 type SubTab =
   | 'url-details'
@@ -2078,6 +2092,7 @@ function NameValueView({ row, exportName }: { row: CrawlUrlRow; exportName: stri
     ['Hreflang Count', row.hreflangCount > 0 ? row.hreflangCount : null],
     ['Hreflangs', summarizeHreflangs(row.hreflangs)],
     ['AMP HTML', row.amphtml],
+    ['Mobile Alternate', row.mobileAlternate],
     ['AMP Page', row.ampPage ? 'Yes' : null],
     [
       'AMP Validation Errors',
@@ -2427,17 +2442,10 @@ function LinksView({
   // selected cell in this LinksView is enough.
   useEffect(() => {
     const onKey = (e: KeyboardEvent) => {
-      if (!(e.ctrlKey || e.metaKey)) return;
-      if (e.key !== 'c' && e.key !== 'C') return;
+      if (!isGridCopyShortcut(e)) return;
+      if (!ownsGridCopy(GRID_ID)) return;
       const sel = selectedRef.current;
       if (sel.size === 0) return;
-      // Don't override copy when the user is in an input/textarea — they
-      // probably want the input's selection, not ours.
-      const target = e.target as HTMLElement | null;
-      const tag = target?.tagName;
-      if (tag === 'INPUT' || tag === 'TEXTAREA' || target?.isContentEditable) {
-        return;
-      }
       e.preventDefault();
       void copyCellsToClipboard(sel, rowsRef.current);
     };
@@ -2525,7 +2533,7 @@ function LinksView({
       return;
     }
     // Ctrl/Cmd+Click: toggle the single cell in the current selection.
-    if (e.ctrlKey || e.metaKey) {
+    if (isAdditiveClick(e)) {
       const next = new Set(selected);
       const k = cellKey(r, c);
       if (next.has(k)) next.delete(k);
@@ -2548,7 +2556,7 @@ function LinksView({
 
   const handleHeaderClick = (c: number, e: React.MouseEvent) => {
     const keys = rows.map((_, r) => cellKey(r, c));
-    if (e.ctrlKey || e.metaKey) {
+    if (isAdditiveClick(e)) {
       const next = new Set(selected);
       const allSelected = keys.every((k) => next.has(k));
       if (allSelected) {
@@ -2599,12 +2607,16 @@ function LinksView({
 
   const beginCellDrag = (r: number, c: number, e: React.MouseEvent) => {
     if (e.button !== 0) return;
+    // macOS Ctrl+click is the OS secondary click: it fires mousedown *and*
+    // contextmenu. Starting a drag here would grow the selection at the
+    // exact moment the menu is about to act on it.
+    if (isMacSecondaryClick(e)) return;
     if (e.shiftKey) {
       handleCellClick(r, c, e);
       return;
     }
     e.preventDefault();
-    const additive = e.ctrlKey || e.metaKey;
+    const additive = isAdditiveClick(e);
     dragRef.current = {
       kind: 'cell',
       aR: r,
@@ -2620,12 +2632,16 @@ function LinksView({
 
   const beginColumnDrag = (c: number, e: React.MouseEvent) => {
     if (e.button !== 0) return;
+    // macOS Ctrl+click is the OS secondary click: it fires mousedown *and*
+    // contextmenu. Starting a drag here would grow the selection at the
+    // exact moment the menu is about to act on it.
+    if (isMacSecondaryClick(e)) return;
     if (e.shiftKey) {
       handleHeaderClick(c, e);
       return;
     }
     e.preventDefault();
-    const additive = e.ctrlKey || e.metaKey;
+    const additive = isAdditiveClick(e);
     dragRef.current = {
       kind: 'column',
       aC: c,
@@ -2642,7 +2658,11 @@ function LinksView({
   const menuClickedIsUrl = isUrlLike(menuClickedCell);
 
   return (
-    <div ref={rootRef} className="relative flex h-full select-none flex-col">
+    <div
+      ref={rootRef}
+      className="relative flex h-full select-none flex-col"
+      onMouseDown={() => markGridActive(GRID_ID)}
+    >
       <div className="flex shrink-0 items-center gap-3 px-3 pt-2 text-[11px] text-surface-500">
         <span>
           {t('links.showing', { defaultValue: 'Showing' })}{' '}
@@ -2936,77 +2956,21 @@ function collectUrlsFromSelection(
 }
 
 /**
- * Write `text` to the OS clipboard with the same fallback chain as
- * `copyCellsToClipboard` — async API first, hidden textarea +
- * execCommand if the API is unavailable or permission-denied.
- */
-async function writeTextToClipboard(text: string): Promise<void> {
-  try {
-    await navigator.clipboard.writeText(text);
-  } catch {
-    const ta = document.createElement('textarea');
-    ta.value = text;
-    ta.style.position = 'fixed';
-    ta.style.opacity = '0';
-    document.body.appendChild(ta);
-    ta.select();
-    try {
-      document.execCommand('copy');
-    } finally {
-      document.body.removeChild(ta);
-    }
-  }
-}
-
-/**
- * Build a TSV string from the selected cells of a 2-D table and write it
- * to the OS clipboard. Cells are grouped by row (preserving the row
- * order shown in the UI) and within a row by ascending column index, so
- * paste-ing into a spreadsheet drops cells into matching grid positions.
+ * Copy the selected cells of a 2-D table to the clipboard as TSV. Rows
+ * are a plain array here, so the selection's row key is already the
+ * display index.
  */
 async function copyCellsToClipboard(
   selected: Set<string>,
   rows: string[][],
 ): Promise<void> {
-  if (selected.size === 0) return;
-  // Group selected cells by row index → list of column indexes.
-  const byRow = new Map<number, number[]>();
-  for (const k of selected) {
-    const [rs, cs] = k.split(':');
-    const r = Number(rs);
-    const c = Number(cs);
-    if (!Number.isFinite(r) || !Number.isFinite(c)) continue;
-    const list = byRow.get(r);
-    if (list) list.push(c);
-    else byRow.set(r, [c]);
-  }
-  const sortedRows = [...byRow.keys()].sort((a, b) => a - b);
-  const lines: string[] = [];
-  for (const r of sortedRows) {
-    const row = rows[r];
-    if (!row) continue;
-    const cols = (byRow.get(r) ?? []).sort((a, b) => a - b);
-    lines.push(cols.map((c) => row[c] ?? '').join('\t'));
-  }
-  const text = lines.join('\n');
-  try {
-    await navigator.clipboard.writeText(text);
-  } catch {
-    // Clipboard API can fail (e.g. when window not focused); fall back
-    // to a hidden textarea + execCommand which works in Electron even
-    // when the clipboard permission is unset.
-    const ta = document.createElement('textarea');
-    ta.value = text;
-    ta.style.position = 'fixed';
-    ta.style.opacity = '0';
-    document.body.appendChild(ta);
-    ta.select();
-    try {
-      document.execCommand('copy');
-    } finally {
-      document.body.removeChild(ta);
-    }
-  }
+  await copySelection(selected, {
+    order: (r) => r,
+    text: (r, c) => {
+      const row = rows[r];
+      return row ? (row[c] ?? '') : null;
+    },
+  });
 }
 
 function SerpSnippet({ row }: { row: CrawlUrlRow }) {

apps/desktop/src/renderer/src/components/SettingsDialog.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useMemo, useRef, useState, type RefObject } from 'react';
+import { useEffect, useMemo, useRef, useState, type ReactNode, type RefObject } from 'react';
 import { useTranslation, Trans } from 'react-i18next';
 import {
   SUPPORTED_LANGUAGES,
@@ -41,6 +41,7 @@ import {
   Chrome,
   Target,
   Check,
+  Workflow,
   type LucideIcon,
 } from 'lucide-react';
 import clsx from 'clsx';
@@ -48,6 +49,7 @@ import type {
   BrowserLoginConfig,
   CrawlConfig,
   CrawlMode,
+  CrawlScope,
   CustomExtractionRule,
   FormLoginStep,
   HttpAuth,
@@ -88,12 +90,39 @@ interface FormState {
   retryInitialDelayMs: string;
   followRedirects: boolean;
   respectRobotsTxt: boolean;
+  respectCrawlDelay: boolean;
   crawlExternal: boolean;
   checkImages: boolean;
   checkCss: boolean;
   checkJs: boolean;
   storeNofollowLinks: boolean;
   discoverSitemaps: boolean;
+  // spider → crawl matrix
+  scope: CrawlScope;
+  storeImages: boolean;
+  crawlMedia: boolean;
+  storeMedia: boolean;
+  storeCss: boolean;
+  storeJs: boolean;
+  crawlInternalLinks: boolean;
+  storeInternalLinks: boolean;
+  storeExternalLinks: boolean;
+  storeCanonicals: boolean;
+  storePagination: boolean;
+  crawlHreflang: boolean;
+  storeHreflang: boolean;
+  crawlAmp: boolean;
+  storeAmp: boolean;
+  storeMetaRefresh: boolean;
+  crawlIframes: boolean;
+  storeIframes: boolean;
+  crawlMobileAlternate: boolean;
+  storeMobileAlternate: boolean;
+  storeUncrawlableLinks: boolean;
+  checkLinksOutsideStartFolder: boolean;
+  followExternalNofollow: boolean;
+  crawlInvalidLinks: boolean;
+  crawlLinkedSitemaps: boolean;
   // requests
   userAgent: string;
   acceptLanguage: string;
@@ -199,6 +228,7 @@ export type SettingsSectionKey =
   | 'presets'
   | 'mode'
   | 'crawler'
+  | 'spider-crawl'
   | 'speed'
   | 'requests'
   | 'filters'
@@ -255,6 +285,13 @@ const SECTIONS: SectionDef[] = [
     keywords:
       'depth max urls concurrency rps timeout delay retry follow redirects robots external nofollow sitemap',
   },
+  {
+    key: 'spider-crawl',
+    label: 'Spider Crawl',
+    icon: Workflow,
+    keywords:
+      'spider crawl store link types resource links page links images media css javascript internal hyperlinks external canonicals pagination hreflang amp meta refresh iframes mobile alternate uncrawlable start folder subdomains nofollow invalid xml sitemaps',
+  },
   {
     key: 'speed',
     label: 'Speed',
@@ -424,12 +461,38 @@ function configToForm(c: CrawlConfig): FormState {
     retryInitialDelayMs: String(c.retryInitialDelayMs),
     followRedirects: c.followRedirects,
     respectRobotsTxt: c.respectRobotsTxt,
+    respectCrawlDelay: c.respectCrawlDelay,
     crawlExternal: c.crawlExternal,
     checkImages: c.checkImages,
     checkCss: c.checkCss,
     checkJs: c.checkJs,
     storeNofollowLinks: c.storeNofollowLinks,
     discoverSitemaps: c.discoverSitemaps,
+    scope: c.scope,
+    storeImages: c.storeImages ?? true,
+    crawlMedia: c.crawlMedia ?? false,
+    storeMedia: c.storeMedia ?? false,
+    storeCss: c.storeCss ?? true,
+    storeJs: c.storeJs ?? true,
+    crawlInternalLinks: c.crawlInternalLinks ?? true,
+    storeInternalLinks: c.storeInternalLinks ?? true,
+    storeExternalLinks: c.storeExternalLinks ?? true,
+    storeCanonicals: c.storeCanonicals ?? true,
+    storePagination: c.storePagination ?? true,
+    crawlHreflang: c.crawlHreflang ?? false,
+    storeHreflang: c.storeHreflang ?? true,
+    crawlAmp: c.crawlAmp ?? false,
+    storeAmp: c.storeAmp ?? true,
+    storeMetaRefresh: c.storeMetaRefresh ?? true,
+    crawlIframes: c.crawlIframes ?? false,
+    storeIframes: c.storeIframes ?? false,
+    crawlMobileAlternate: c.crawlMobileAlternate ?? false,
+    storeMobileAlternate: c.storeMobileAlternate ?? true,
+    storeUncrawlableLinks: c.storeUncrawlableLinks ?? true,
+    checkLinksOutsideStartFolder: c.checkLinksOutsideStartFolder ?? true,
+    followExternalNofollow: c.followExternalNofollow ?? false,
+    crawlInvalidLinks: c.crawlInvalidLinks ?? false,
+    crawlLinkedSitemaps: c.crawlLinkedSitemaps ?? false,
     userAgent: c.userAgent,
     acceptLanguage: c.acceptLanguage,
     customHeadersText: Object.entries(c.customHeaders ?? {})
@@ -646,12 +709,38 @@ export function SettingsDialog({ open, onClose }: Props) {
       retryInitialDelayMs: Math.max(0, num(form.retryInitialDelayMs, config.retryInitialDelayMs)),
       followRedirects: form.followRedirects,
       respectRobotsTxt: form.respectRobotsTxt,
+      respectCrawlDelay: form.respectCrawlDelay,
       crawlExternal: form.crawlExternal,
       checkImages: form.checkImages,
       checkCss: form.checkCss,
       checkJs: form.checkJs,
       storeNofollowLinks: form.storeNofollowLinks,
       discoverSitemaps: form.discoverSitemaps,
+      scope: form.scope,
+      storeImages: form.storeImages,
+      crawlMedia: form.crawlMedia,
+      storeMedia: form.storeMedia,
+      storeCss: form.storeCss,
+      storeJs: form.storeJs,
+      crawlInternalLinks: form.crawlInternalLinks,
+      storeInternalLinks: form.storeInternalLinks,
+      storeExternalLinks: form.storeExternalLinks,
+      storeCanonicals: form.storeCanonicals,
+      storePagination: form.storePagination,
+      crawlHreflang: form.crawlHreflang,
+      storeHreflang: form.storeHreflang,
+      crawlAmp: form.crawlAmp,
+      storeAmp: form.storeAmp,
+      storeMetaRefresh: form.storeMetaRefresh,
+      crawlIframes: form.crawlIframes,
+      storeIframes: form.storeIframes,
+      crawlMobileAlternate: form.crawlMobileAlternate,
+      storeMobileAlternate: form.storeMobileAlternate,
+      storeUncrawlableLinks: form.storeUncrawlableLinks,
+      checkLinksOutsideStartFolder: form.checkLinksOutsideStartFolder,
+      followExternalNofollow: form.followExternalNofollow,
+      crawlInvalidLinks: form.crawlInvalidLinks,
+      crawlLinkedSitemaps: form.crawlLinkedSitemaps,
       userAgent: form.userAgent.trim() || config.userAgent,
       acceptLanguage: form.acceptLanguage.trim() || config.acceptLanguage,
       customHeaders: parseHeaders(form.customHeadersText),
@@ -915,6 +1004,9 @@ export function SettingsDialog({ open, onClose }: Props) {
               {active === 'crawler' && (
                 <CrawlerPanel form={form} update={update} />
               )}
+              {active === 'spider-crawl' && (
+                <SpiderCrawlPanel form={form} update={update} />
+              )}
               {active === 'speed' && (
                 <SpeedPanel form={form} update={update} />
               )}
@@ -1397,9 +1489,19 @@ function CrawlerPanel({ form, update }: PanelProps) {
           label={t('settingsPanels.crawler.respectRobotsTxt', { defaultValue: 'Respect robots.txt' })}
           checked={form.respectRobotsTxt}
           onChange={(v) => update('respectRobotsTxt', v)}
-          info="Honor Disallow rules + crawl-delay declared in /robots.txt for the configured User-Agent. Screaming Frog: 'Respect robots.txt' (Configuration → robots.txt)."
+          info="Honor Allow / Disallow rules declared in /robots.txt for the configured User-Agent. Screaming Frog: 'Respect robots.txt' (Configuration → robots.txt)."
           example="On (default). Off only when crawling sites you own and need to bypass."
         />
+        <Bool
+          label={t('settingsPanels.crawler.respectCrawlDelay', {
+            defaultValue: 'Respect robots.txt Crawl-delay',
+          })}
+          checked={form.respectCrawlDelay}
+          onChange={(v) => update('respectCrawlDelay', v)}
+          disabled={!form.respectRobotsTxt}
+          info="Honor a Crawl-delay directive as a global rate limit (one request every N seconds). Crawl-delay is not part of RFC 9309 — Google ignores it and Screaming Frog does not implement it — and published values are often stale: 'Crawl-delay: 30' turns a 500-URL crawl into hours. Ignored by default; the directive is still reported in the log when found."
+          example="Off (default) for normal audits. On when an ops policy requires it — expect the crawl to take Crawl-delay seconds per URL."
+        />
         <Bool
           label={t('settingsPanels.crawler.crawlExternal', { defaultValue: 'Check External Links' })}
           checked={form.crawlExternal}
@@ -1484,6 +1586,408 @@ function CrawlerPanel({ form, update }: PanelProps) {
   );
 }
 
+/* ------------------------------------------------------------------ *
+ * Spider → Crawl — the per-link-type crawl/store matrix, laid out like
+ * Screaming Frog's Configuration → Spider → Crawl tab.
+ *
+ * Two switches per row, and they are genuinely independent: "store but
+ * don't crawl" lists a declaration without spending a request on it,
+ * "crawl but don't store" fetches a resource for its side effects
+ * (proving it loads, mining a stylesheet for its font targets) without
+ * burying the Internal tab under it.
+ *
+ * Screaming Frog's SWF row has no counterpart here — Flash reached
+ * end-of-life in 2020.
+ * ------------------------------------------------------------------ */
+
+/** One bordered group with a legend, mirroring SF's fieldset boxes. */
+function CrawlGroup({
+  title,
+  columns,
+  children,
+}: {
+  title: string;
+  /** Render the Crawl / Store column captions above the first row. */
+  columns?: { crawl: string; store: string };
+  children: ReactNode;
+}) {
+  return (
+    <fieldset className="rounded border border-surface-800 bg-surface-950/40 px-3 pb-3 pt-2">
+      <legend className="px-1 text-[11px] font-medium text-surface-300">{title}</legend>
+      {columns ? (
+        <div className="mb-1 flex items-center gap-2 border-b border-surface-800/70 pb-1">
+          <span className="flex-1" />
+          <span className="w-[58px] text-center text-[10px] uppercase tracking-wide text-surface-500">
+            {columns.crawl}
+          </span>
+          <span className="w-[58px] text-center text-[10px] uppercase tracking-wide text-surface-500">
+            {columns.store}
+          </span>
+        </div>
+      ) : null}
+      <div className="flex flex-col gap-1">{children}</div>
+    </fieldset>
+  );
+}
+
+/**
+ * One matrix row: a label plus up to two checkboxes in fixed columns.
+ * A `null` switch renders an empty cell — Uncrawlable Links has no
+ * crawl side, because an uncrawlable link is by definition not fetched.
+ */
+function CrawlRow({
+  label,
+  info,
+  example,
+  crawl,
+  store,
+}: {
+  label: string;
+  crawl: { checked: boolean; onChange: (v: boolean) => void; disabled?: boolean; title?: string } | null;
+  store: { checked: boolean; onChange: (v: boolean) => void; disabled?: boolean; title?: string } | null;
+} & FieldInfo) {
+  const { i18n } = useTranslation();
+  const cell = (
+    box: { checked: boolean; onChange: (v: boolean) => void; disabled?: boolean; title?: string } | null,
+  ) => (
+    <span className="flex w-[58px] justify-center">
+      {box ? (
+        <input
+          type="checkbox"
+          checked={box.checked}
+          disabled={box.disabled}
+          title={box.title}
+          onChange={(e) => box.onChange(e.target.checked)}
+          className={clsx(box.disabled && 'cursor-not-allowed opacity-40')}
+        />
+      ) : null}
+    </span>
+  );
+  return (
+    <div className="flex items-center gap-2 py-[1px]">
+      <span className="flex flex-1 items-center gap-1">
+        <span className="text-[12px] text-surface-100">
+          {translateLabel(label, i18n.language)}
+        </span>
+        <InfoTip info={info} example={example} />
+      </span>
+      {cell(crawl)}
+      {cell(store)}
+    </div>
+  );
+}
+
+function SpiderCrawlPanel({ form, update }: PanelProps) {
+  const { t } = useTranslation();
+  const cols = {
+    crawl: t('spiderCrawl.colCrawl', { defaultValue: 'Crawl' }),
+    store: t('spiderCrawl.colStore', { defaultValue: 'Store' }),
+  };
+  // Resource rows have nothing to keep unless they were fetched, so their
+  // Store box follows the Crawl box instead of pretending otherwise. The
+  // image row is the exception: `<img>` declarations land in the images
+  // table whether or not the bytes were ever requested.
+  const storeNeedsCrawl = t('spiderCrawl.storeNeedsCrawl', {
+    defaultValue: 'Requires Crawl — an un-fetched resource has no row to store.',
+  });
+
+  // "Crawl Outside of Start Folder" and "Crawl All Subdomains" are two
+  // views of the one scope setting rather than fields of their own, so
+  // they can never contradict it or each other. `exact-url` is a
+  // single-page crawl where neither applies.
+  const scope = form.scope;
+  const scopeLocked = scope === 'exact-url';
+  const outsideStartFolder = scope !== 'subfolder';
+  const allSubdomains = scope === 'all-subdomains';
+
+  return (
+    <>
+      <p className="mb-3 text-[11px] text-surface-400">
+        {t('spiderCrawl.intro', {
+          defaultValue:
+            'Choose which link types to crawl and store. Crawl fetches the target so it gets its own row; Store keeps what the page declared so the matching tab has data. These settings shape how many URLs are discovered, crawled, and reported.',
+        })}
+      </p>
+
+      <div className="grid grid-cols-1 gap-3 xl:grid-cols-2">
+        <div className="flex flex-col gap-3">
+          <CrawlGroup
+            title={t('spiderCrawl.resourceLinks', { defaultValue: 'Resource Links' })}
+            columns={cols}
+          >
+            <CrawlRow
+              label="Images"
+              info="Crawl fetches internal <img> targets (incl. srcset / <picture> sources) so each appears in the Internal tab with status, content type, and size — every one counts toward Max URLs. Store keeps the <img> declarations in the Images tab, which works even with Crawl off: you get the full image inventory with alt text for the cost of zero extra requests."
+              example="Store on, Crawl off is the cheap alt-text audit. Both on for a full image health check."
+              crawl={{ checked: form.checkImages, onChange: (v) => update('checkImages', v) }}
+              store={{ checked: form.storeImages, onChange: (v) => update('storeImages', v) }}
+            />
+            <CrawlRow
+              label="Media"
+              info="<video> / <audio> and the <source> children they own. Off by default — media files are large and rarely what an SEO crawl is looking for."
+              example="On when auditing a video-heavy site for dead media URLs."
+              crawl={{ checked: form.crawlMedia, onChange: (v) => update('crawlMedia', v) }}
+              store={{
+                checked: form.storeMedia,
+                onChange: (v) => update('storeMedia', v),
+                disabled: !form.crawlMedia,
+                title: form.crawlMedia ? undefined : storeNeedsCrawl,
+              }}
+            />
+            <CrawlRow
+              label="CSS"
+              info="<link rel=stylesheet> targets. Crawling a stylesheet is also what discovers the web fonts and background images declared inside it via @font-face / url() — so Crawl on with Store off still populates the Internal tab's Font filter without listing every stylesheet."
+              example="Crawl on, Store off when you want fonts discovered but not hundreds of CSS rows."
+              crawl={{ checked: form.checkCss, onChange: (v) => update('checkCss', v) }}
+              store={{
+                checked: form.storeCss,
+                onChange: (v) => update('storeCss', v),
+                disabled: !form.checkCss,
+                title: form.checkCss ? undefined : storeNeedsCrawl,
+              }}
+            />
+            <CrawlRow
+              label="JavaScript"
+              info="<script src> targets, fetched so each gets its own row with status code, content type, and size. Headers only — the body is discarded, never executed."
+              example="Both on to catch 404ing bundles; both off for HTML-only crawls."
+              crawl={{ checked: form.checkJs, onChange: (v) => update('checkJs', v) }}
+              store={{
+                checked: form.storeJs,
+                onChange: (v) => update('storeJs', v),
+                disabled: !form.checkJs,
+                title: form.checkJs ? undefined : storeNeedsCrawl,
+              }}
+            />
+          </CrawlGroup>
+
+          <CrawlGroup
+            title={t('spiderCrawl.pageLinks', { defaultValue: 'Page Links' })}
+            columns={cols}
+          >
+            <CrawlRow
+              label="Internal Hyperlinks"
+              info="<a href> targets on the same site. Crawl off turns the run into an audit of a fixed set of pages — sitemaps, canonicals, and the other declared alternates below still feed discovery. Store off empties the link graph: inlinks, outlinks, anchor-text reports, and link score all go with it."
+              example="Leave both on. Crawl off only when a sitemap or URL list already defines the exact set you want."
+              crawl={{
+                checked: form.crawlInternalLinks,
+                onChange: (v) => update('crawlInternalLinks', v),
+              }}
+              store={{
+                checked: form.storeInternalLinks,
+                onChange: (v) => update('storeInternalLinks', v),
+              }}
+            />
+            <CrawlRow
+              label="External Links"
+              info="Outbound links to other hosts are always status-checked (one HEAD each) so Broken Links catches dead externals — that does not depend on this row. Crawl here means fully crawling those pages, following their links onward too. Store keeps outbound links in the link graph."
+              example="Crawl off (default) — status-check externals without spidering the whole web."
+              crawl={{ checked: form.crawlExternal, onChange: (v) => update('crawlExternal', v) }}
+              store={{
+                checked: form.storeExternalLinks,
+                onChange: (v) => update('storeExternalLinks', v),
+              }}
+            />
+            <CrawlRow
+              label="Canonicals"
+              info="<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter."
+              example="Crawl off (default) — canonicals are a signal, not a route. Store on."
+              crawl={{ checked: form.followCanonicals, onChange: (v) => update('followCanonicals', v) }}
+              store={{ checked: form.storeCanonicals, onChange: (v) => update('storeCanonicals', v) }}
+            />
+            <CrawlRow
+              label="Pagination (Rel Next/Prev)"
+              info="<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere."
+              example="Both on unless you are debugging an infinite paginated series."
+              crawl={{
+                checked: form.followPaginationLinks,
+                onChange: (v) => update('followPaginationLinks', v),
+              }}
+              store={{ checked: form.storePagination, onChange: (v) => update('storePagination', v) }}
+            />
+            <CrawlRow
+              label="Hreflang"
+              info="<link rel=alternate hreflang>. Crawl enqueues every declared alternate, which is how you reach language versions nothing links to. Store feeds the Hreflang tab and the reciprocity / invalid-code audits."
+              example="Crawl on for a multi-language audit — otherwise unlinked locales stay invisible."
+              crawl={{ checked: form.crawlHreflang, onChange: (v) => update('crawlHreflang', v) }}
+              store={{ checked: form.storeHreflang, onChange: (v) => update('storeHreflang', v) }}
+            />
+            <CrawlRow
+              label="AMP"
+              info="<link rel=amphtml>. Crawl fetches the AMP variant as its own URL; Store keeps the declaration plus the AMP smoke-validator findings."
+              example="Crawl on only if the site still ships AMP pages."
+              crawl={{ checked: form.crawlAmp, onChange: (v) => update('crawlAmp', v) }}
+              store={{ checked: form.storeAmp, onChange: (v) => update('storeAmp', v) }}
+            />
+            <CrawlRow
+              label="Meta Refresh"
+              info='<meta http-equiv="refresh">. Crawl enqueues the parsed target like a redirect; Store keeps the raw directive and its URL for the Meta Refresh tab.'
+              example="Crawl on when auditing a legacy site that still redirects this way."
+              crawl={{ checked: form.followJsRedirects, onChange: (v) => update('followJsRedirects', v) }}
+              store={{ checked: form.storeMetaRefresh, onChange: (v) => update('storeMetaRefresh', v) }}
+            />
+            <CrawlRow
+              label="iframes"
+              info="<iframe src> documents. Crawl fetches each embedded page as its own URL, which can pull in a lot of third-party surface. Store records them in the link graph so a dead embed shows up in Outlinks and Broken Links — without counting toward the page's outlink total, since an embed is not a hyperlink."
+              example="Store on, Crawl off is usually the right pair."
+              crawl={{ checked: form.crawlIframes, onChange: (v) => update('crawlIframes', v) }}
+              store={{ checked: form.storeIframes, onChange: (v) => update('storeIframes', v) }}
+            />
+            <CrawlRow
+              label="Mobile Alternate"
+              info='The separate-URL (m-dot) mobile version: <link rel="alternate" media="only screen and (max-width: …)">. Null on responsive sites, which is most of them — a value here with no reciprocal canonical back is the classic broken m-dot setup.'
+              example="Crawl on only when the site really does serve a separate mobile host."
+              crawl={{
+                checked: form.crawlMobileAlternate,
+                onChange: (v) => update('crawlMobileAlternate', v),
+              }}
+              store={{
+                checked: form.storeMobileAlternate,
+                onChange: (v) => update('storeMobileAlternate', v),
+              }}
+            />
+            <CrawlRow
+              label="Uncrawlable Links"
+              info={'Links a search engine cannot follow: <a> with no href but an onclick, href="javascript:…", and href="#" placeholders wired to a handler. Store-only — an uncrawlable link is by definition never fetched. Drives the JS-Only Navigation issue filter.'}
+              example="On — it is a count, so it costs nothing."
+              crawl={null}
+              store={{
+                checked: form.storeUncrawlableLinks,
+                onChange: (v) => update('storeUncrawlableLinks', v),
+              }}
+            />
+          </CrawlGroup>
+        </div>
+
+        <div className="flex flex-col gap-3">
+          <CrawlGroup title={t('spiderCrawl.behaviour', { defaultValue: 'Crawl Behaviour' })}>
+            <Bool
+              label={t('spiderCrawl.checkOutsideFolder', {
+                defaultValue: 'Check Links Outside of Start Folder',
+              })}
+              checked={form.checkLinksOutsideStartFolder}
+              onChange={(v) => update('checkLinksOutsideStartFolder', v)}
+              disabled={scope !== 'subfolder'}
+              hint={
+                scope === 'subfolder'
+                  ? undefined
+                  : t('spiderCrawl.subfolderOnly', {
+                      defaultValue: 'Only applies while Crawl Scope is Subfolder',
+                    })
+              }
+              info="With a Subfolder-scoped crawl, links pointing outside the start folder are fetched once so their status code is known, then stopped — they are checked, not crawled through. Off leaves them undiscovered entirely."
+              example="On — knowing a link out of /blog/ is a 404 costs one request."
+            />
+            <Bool
+              label={t('spiderCrawl.crawlOutsideFolder', {
+                defaultValue: 'Crawl Outside of Start Folder',
+              })}
+              checked={outsideStartFolder}
+              disabled={scopeLocked}
+              onChange={(v) => update('scope', v ? 'subdomain' : 'subfolder')}
+              hint={
+                scopeLocked
+                  ? t('spiderCrawl.scopeExactUrl', {
+                      defaultValue: 'Disabled — Crawl Scope is Exact URL (single page)',
+                    })
+                  : t('spiderCrawl.scopeBound', {
+                      defaultValue: 'Same setting as Crawl Scope — changing either updates both',
+                    })
+              }
+              info="Off restricts the crawl to URLs under the start URL's path (Crawl Scope = Subfolder). On lets it cover the whole host. This is a view of the Crawl Scope setting, not a separate switch, so the two can never disagree."
+              example="Off to audit just /blog/; on for the whole site."
+            />
+            <Bool
+              label={t('spiderCrawl.crawlAllSubdomains', {
+                defaultValue: 'Crawl All Subdomains',
+              })}
+              checked={allSubdomains}
+              disabled={scopeLocked}
+              onChange={(v) => update('scope', v ? 'all-subdomains' : 'subdomain')}
+              hint={
+                scopeLocked
+                  ? t('spiderCrawl.scopeExactUrl', {
+                      defaultValue: 'Disabled — Crawl Scope is Exact URL (single page)',
+                    })
+                  : t('spiderCrawl.scopeBound', {
+                      defaultValue: 'Same setting as Crawl Scope — changing either updates both',
+                    })
+              }
+              info="Treats every host sharing the registrable domain as internal — shop.example.com and blog.example.com crawl alongside example.com instead of counting as external. Another view of the Crawl Scope setting."
+              example="On when subdomains are part of the same property."
+            />
+            <Bool
+              label={t('spiderCrawl.followInternalNofollow', {
+                defaultValue: 'Follow Internal "nofollow"',
+              })}
+              checked={form.followNofollow}
+              onChange={(v) => update('followNofollow', v)}
+              info='Crawl through rel="nofollow" links pointing at the same site. Off (default) is Screaming Frog "Respect Nofollow" behaviour. Internal and external are separate switches because sites nofollow them for opposite reasons — crawl-budget shaping vs. not vouching for a third party.'
+              example="On when a site nofollows its own faceted navigation and you need behind it."
+            />
+            <Bool
+              label={t('spiderCrawl.followExternalNofollow', {
+                defaultValue: 'Follow External "nofollow"',
+              })}
+              checked={form.followExternalNofollow}
+              onChange={(v) => update('followExternalNofollow', v)}
+              info='Crawl through rel="nofollow" links pointing at other hosts. Only has an effect while External Links → Crawl is on.'
+              example="Off — nofollowed externals are exactly the ones you did not vouch for."
+            />
+            <Bool
+              label={t('spiderCrawl.crawlInvalidLinks', { defaultValue: 'Crawl Invalid Links' })}
+              checked={form.crawlInvalidLinks}
+              onChange={(v) => update('crawlInvalidLinks', v)}
+              info="Record hrefs that cannot be parsed as a URL — unencoded whitespace inside the authority, doubled schemes, stray delimiters. They can never resolve to a crawled page, so every one is reported in Broken Links, which is the point. Deliberate non-navigable schemes (mailto:, tel:, #) are not malformed and never appear."
+              example="On when hunting hand-written markup errors; off keeps Broken Links focused on real 404s."
+            />
+          </CrawlGroup>
+
+          <CrawlGroup title={t('spiderCrawl.xmlSitemaps', { defaultValue: 'XML Sitemaps' })}>
+            <Bool
+              label={t('spiderCrawl.crawlLinkedSitemaps', {
+                defaultValue: 'Crawl Linked XML Sitemaps',
+              })}
+              checked={form.crawlLinkedSitemaps}
+              onChange={(v) => update('crawlLinkedSitemaps', v)}
+              hint={
+                form.discoverSitemaps
+                  ? undefined
+                  : t('spiderCrawl.needsDiscovery', {
+                      defaultValue: 'Needs Auto-Discover or an explicit sitemap below',
+                    })
+              }
+              info="Auto-discovery on its own only records sitemap entries, which is what the sitemap issue filters compare the crawl against. Turning this on crawls them too — and that is what surfaces orphans: pages the sitemap declares but nothing on the site links to."
+              example="On for an orphan-page audit."
+            />
+            <Bool
+              label={t('spiderCrawl.autoDiscover', {
+                defaultValue: 'Auto Discover XML Sitemaps via robots.txt',
+              })}
+              checked={form.discoverSitemaps}
+              onChange={(v) => update('discoverSitemaps', v)}
+              info="Reads Sitemap: directives from /robots.txt plus the conventional /sitemap.xml fallbacks at crawl start. Cheap I/O, and it powers every sitemap issue filter."
+              example="On (default)."
+            />
+            <div className="mt-1">
+              <Area
+                label={t('spiderCrawl.crawlTheseSitemaps', {
+                  defaultValue: 'Crawl These Sitemaps',
+                })}
+                value={form.seedSitemapText}
+                onChange={(v) => update('seedSitemapText', v)}
+                rows={6}
+                placeholder={'https://example.com/sitemap_index.xml\nhttps://example.com/news-sitemap.xml'}
+                info="Explicit sitemap URLs, one per line. Their entries are always both recorded and queued as crawl seeds — use this when the sitemap lives somewhere robots.txt never mentions."
+                example="https://example.com/sitemap_index.xml"
+              />
+            </div>
+          </CrawlGroup>
+        </div>
+      </div>
+    </>
+  );
+}
+
 function SpeedPanel({ form, update }: PanelProps) {
   const { t } = useTranslation();
   const conc = Number.parseInt(form.maxConcurrency, 10);
@@ -3356,19 +3860,25 @@ function Bool({
   hint,
   info,
   example,
+  disabled,
 }: {
   label: string;
   checked: boolean;
   onChange: (v: boolean) => void;
   hint?: string;
+  /** Greys the row out when the setting cannot apply in the current
+   *  configuration, rather than letting a click silently override the
+   *  setting that is actually in charge. */
+  disabled?: boolean;
 } & FieldInfo) {
   return (
-    <label className="flex items-start gap-2">
+    <label className={clsx('flex items-start gap-2', disabled && 'opacity-50')}>
       <input
         type="checkbox"
         checked={checked}
+        disabled={disabled}
         onChange={(e) => onChange(e.target.checked)}
-        className="mt-0.5"
+        className={clsx('mt-0.5', disabled && 'cursor-not-allowed')}
       />
       <BoolLabel label={label} info={info} example={example} hint={hint} />
     </label>

apps/desktop/src/renderer/src/i18n/labels.ts

@@ -32,6 +32,8 @@ const TR_LABELS: Record<string, string> = {
   'OAuth (your own client)': 'OAuth (kendi istemciniz)',
   'Service Account': 'Servis Hesabı',
   Local: 'Yerel',
+  // URL table — re-crawl change marker
+  'Changed since the previous crawl': 'Önceki taramaya göre değişti',
   // Top-level groups
   Summary: 'Özet',
   'Crawl Data': 'Crawl Verisi',
@@ -635,7 +637,18 @@ const TR_LABELS: Record<string, string> = {
   Presets: 'Hazır Ayarlar',
   Mode: 'Mod',
   Crawler: 'Crawler',
+  'Spider Crawl': 'Spider Tarama',
   Speed: 'Hız',
+  // Spider → Crawl matrix row labels (the ones not already registered
+  // above as table columns / sidebar nodes).
+  Media: 'Medya',
+  'Internal Hyperlinks': 'Dahili Bağlantılar',
+  'External Links': 'Harici Bağlantılar',
+  'Pagination (Rel Next/Prev)': 'Sayfalama (Rel Next/Prev)',
+  AMP: 'AMP',
+  iframes: 'iframe’ler',
+  'Mobile Alternate': 'Mobil Alternatif',
+  'Uncrawlable Links': 'Taranamayan Bağlantılar',
   Requests: 'İstekler',
   'Include/Exclude': 'Dahil Et/Hariç Tut',
   'Custom Search': 'Özel Arama',

apps/desktop/src/renderer/src/i18n/locales/en.json

@@ -380,6 +380,29 @@
     "errorCount": "{{n}} error(s)",
     "warningCount": "{{n}} warning(s)"
   },
+  "spiderCrawl": {
+    "intro": "Choose which link types to crawl and store. Crawl fetches the target so it gets its own row; Store keeps what the page declared so the matching tab has data. These settings shape how many URLs are discovered, crawled, and reported.",
+    "colCrawl": "Crawl",
+    "colStore": "Store",
+    "storeNeedsCrawl": "Requires Crawl — an un-fetched resource has no row to store.",
+    "resourceLinks": "Resource Links",
+    "pageLinks": "Page Links",
+    "behaviour": "Crawl Behaviour",
+    "xmlSitemaps": "XML Sitemaps",
+    "checkOutsideFolder": "Check Links Outside of Start Folder",
+    "crawlOutsideFolder": "Crawl Outside of Start Folder",
+    "crawlAllSubdomains": "Crawl All Subdomains",
+    "followInternalNofollow": "Follow Internal \"nofollow\"",
+    "followExternalNofollow": "Follow External \"nofollow\"",
+    "crawlInvalidLinks": "Crawl Invalid Links",
+    "crawlLinkedSitemaps": "Crawl Linked XML Sitemaps",
+    "autoDiscover": "Auto Discover XML Sitemaps via robots.txt",
+    "crawlTheseSitemaps": "Crawl These Sitemaps",
+    "subfolderOnly": "Only applies while Crawl Scope is Subfolder",
+    "scopeBound": "Same setting as Crawl Scope — changing either updates both",
+    "scopeExactUrl": "Disabled — Crawl Scope is Exact URL (single page)",
+    "needsDiscovery": "Needs Auto-Discover or an explicit sitemap below"
+  },
   "settingsGroups": {
     "threshold": "Threshold",
     "scope": "Scope",
@@ -430,6 +453,7 @@
       "requestTimeoutMs": "Response Timeout (ms)",
       "followRedirects": "Always Follow Redirects",
       "respectRobotsTxt": "Respect robots.txt",
+      "respectCrawlDelay": "Respect robots.txt Crawl-delay",
       "crawlExternal": "Check External Links",
       "checkImages": "Check Images",
       "checkCss": "Check CSS",
@@ -1392,6 +1416,9 @@
     "statusSelectedRows": "Selected Rows:",
     "statusSelectedCells": "Selected Cells:",
     "statusSelectedColumns": "Selected Columns:",
+    "copying": "Copying…",
+    "copiedCells": "{{n}} cell(s) copied",
+    "copiedNothing": "Nothing to copy",
     "rowHeader": "Row",
     "rowNumberTitle": "Row number",
     "rowSelectHint": "Click to select row · drag to select multiple rows",

apps/desktop/src/renderer/src/i18n/locales/tr.json

@@ -380,6 +380,29 @@
     "errorCount": "{{n}} hata",
     "warningCount": "{{n}} uyarı"
   },
+  "spiderCrawl": {
+    "intro": "Hangi bağlantı türlerinin taranacağını ve saklanacağını seçin. Tara, hedefi çeker ve kendi satırını oluşturur; Sakla ise sayfanın bildirdiğini kaydeder, böylece ilgili sekme dolu gelir. Bu ayarlar kaç URL'in keşfedileceğini, taranacağını ve raporlanacağını belirler.",
+    "colCrawl": "Tara",
+    "colStore": "Sakla",
+    "storeNeedsCrawl": "Tara gerekir — çekilmemiş bir kaynağın saklanacak satırı olmaz.",
+    "resourceLinks": "Kaynak Bağlantıları",
+    "pageLinks": "Sayfa Bağlantıları",
+    "behaviour": "Tarama Davranışı",
+    "xmlSitemaps": "XML Site Haritaları",
+    "checkOutsideFolder": "Başlangıç Klasörü Dışındaki Bağlantıları Kontrol Et",
+    "crawlOutsideFolder": "Başlangıç Klasörü Dışını Tara",
+    "crawlAllSubdomains": "Tüm Alt Alan Adlarını Tara",
+    "followInternalNofollow": "Dahili \"nofollow\" Bağlantıları Takip Et",
+    "followExternalNofollow": "Harici \"nofollow\" Bağlantıları Takip Et",
+    "crawlInvalidLinks": "Geçersiz Bağlantıları Tara",
+    "crawlLinkedSitemaps": "Bağlantılı XML Site Haritalarını Tara",
+    "autoDiscover": "robots.txt Üzerinden XML Site Haritalarını Otomatik Keşfet",
+    "crawlTheseSitemaps": "Şu Site Haritalarını Tara",
+    "subfolderOnly": "Yalnızca Tarama Kapsamı Alt Klasör iken geçerlidir",
+    "scopeBound": "Tarama Kapsamı ile aynı ayar — birini değiştirmek diğerini de günceller",
+    "scopeExactUrl": "Devre dışı — Tarama Kapsamı Tam URL (tek sayfa)",
+    "needsDiscovery": "Otomatik Keşif ya da aşağıda açık bir site haritası gerekir"
+  },
   "settingsGroups": {
     "threshold": "Eşik",
     "scope": "Kapsam",
@@ -430,6 +453,7 @@
       "requestTimeoutMs": "Yanıt Zaman Aşımı (ms)",
       "followRedirects": "Yönlendirmeleri Daima Takip Et",
       "respectRobotsTxt": "robots.txt'ye Uy",
+      "respectCrawlDelay": "robots.txt Crawl-delay'e Uy",
       "crawlExternal": "Harici Linkleri Kontrol Et",
       "checkImages": "Görselleri Kontrol Et",
       "checkCss": "CSS'i Kontrol Et",
@@ -1392,6 +1416,9 @@
     "statusSelectedRows": "Seçili Satırlar:",
     "statusSelectedCells": "Seçili Hücreler:",
     "statusSelectedColumns": "Seçili Sütunlar:",
+    "copying": "Kopyalanıyor…",
+    "copiedCells": "{{n}} hücre kopyalandı",
+    "copiedNothing": "Kopyalanacak bir şey yok",
     "rowHeader": "Satır",
     "rowNumberTitle": "Satır numarası",
     "rowSelectHint": "Satır seçmek için tıkla · birden çok satır için sürükle",

apps/desktop/src/renderer/src/store.ts

@@ -367,6 +367,11 @@ interface AppState {
   applyProjectConfig: (config: CrawlConfig) => void;
   setProgress: (p: CrawlProgress) => void;
   setSummary: (s: CrawlSummary) => void;
+  /** Force the UI out of the "Running" state. `crawl:done` is terminal,
+   *  so the toolbar must not depend on the crawler's final progress
+   *  frame arriving — if that frame is ever late or lost, Start stays
+   *  disabled and Stop/Pause target a crawler that no longer exists. */
+  markCrawlFinished: () => void;
   setOverview: (o: OverviewCounts) => void;
   setActiveTab: (t: TabKey) => void;
   setActiveCategory: (c: UrlCategory) => void;
@@ -492,6 +497,12 @@ export const useAppStore = create<AppState>((set) => ({
     set({ config: { ...config, startUrl: '' } }),
   setProgress: (p) => set({ progress: p }),
   setSummary: (s) => set({ summary: s }),
+  markCrawlFinished: () =>
+    set((s) =>
+      s.progress === null || (!s.progress.running && !s.progress.paused)
+        ? {}
+        : { progress: { ...s.progress, running: false, paused: false } },
+    ),
   setOverview: (o) => set({ overview: o }),
   setActiveTab: (t) => set({ activeTab: t, activeCategory: categoryForTab(t) }),
   setActiveCategory: (c) => set({ activeCategory: c }),

apps/desktop/src/renderer/src/tabs/BrokenLinksTab.tsx

@@ -7,11 +7,22 @@ import type { BrokenLinkRow } from '@freecrawl/shared-types';
 import { useAppStore } from '../store.js';
 import { InfoTip } from '../components/InfoTip.js';
 import { translateLabel } from '../i18n/labels.js';
+import {
+  copySelection,
+  isAdditiveClick,
+  isGridCopyShortcut,
+  isMacSecondaryClick,
+  markGridActive,
+  ownsGridCopy,
+  writeTextToClipboard,
+} from '../utils/clipboard.js';
 
 const ROW_HEIGHT = 24;
 const HEADER_HEIGHT = 28;
 const ROW_NUM_WIDTH = 56;
 const STATUS_BAR_HEIGHT = 22;
+/** Identifies this grid to the copy-shortcut arbitration in `clipboard.ts`. */
+const GRID_ID = 'broken-links';
 // I-4 — Crawl-aware polling cadence. Live during a crawl, idle when
 // just viewing existing project data. The 30 s idle poll exists only
 // to catch external invalidations (Open Project, Bulk Export); the
@@ -216,15 +227,10 @@ export function BrokenLinksTab() {
   // Ctrl/Cmd+C copies the selected cells as TSV.
   useEffect(() => {
     const onKey = (e: KeyboardEvent) => {
-      if (!(e.ctrlKey || e.metaKey)) return;
-      if (e.key !== 'c' && e.key !== 'C') return;
+      if (!isGridCopyShortcut(e)) return;
+      if (!ownsGridCopy(GRID_ID)) return;
       const sel = selectedRef.current;
       if (sel.size === 0) return;
-      const target = e.target as HTMLElement | null;
-      const tag = target?.tagName;
-      if (tag === 'INPUT' || tag === 'TEXTAREA' || target?.isContentEditable) {
-        return;
-      }
       e.preventDefault();
       void copySelectedCells(sel, rowsRef.current);
     };
@@ -278,13 +284,17 @@ export function BrokenLinksTab() {
   // --- selection handlers --------------------------------------------------
   const cellMouseDown = (r: number, c: number, e: React.MouseEvent) => {
     if (e.button !== 0) return; // left button only
+    // macOS Ctrl+click is the OS secondary click: it fires mousedown *and*
+    // contextmenu. Starting a drag here would grow the selection at the
+    // exact moment the menu is about to act on it.
+    if (isMacSecondaryClick(e)) return;
     if (e.shiftKey && anchorRef.current) {
       // Extend a rectangle from the existing anchor.
       setSelected(rangeKeys(anchorRef.current.r, anchorRef.current.c, r, c));
       dragRef.current = { aR: anchorRef.current.r, aC: anchorRef.current.c, additive: false, base: new Set() };
       return;
     }
-    if (e.ctrlKey || e.metaKey) {
+    if (isAdditiveClick(e)) {
       // Toggle this cell, keep the rest.
       setSelected((prev) => {
         const next = new Set(prev);
@@ -368,7 +378,11 @@ export function BrokenLinksTab() {
         </button>
       </div>
 
-      <div ref={scrollRef} className="relative flex-1 select-none overflow-auto">
+      <div
+        ref={scrollRef}
+        className="relative flex-1 select-none overflow-auto"
+        onMouseDown={() => markGridActive(GRID_ID)}
+      >
         <div style={{ minWidth: totalWidth, width: '100%' }}>
           <div
             className="sticky top-0 z-10 flex bg-surface-900 text-[11px]"
@@ -610,52 +624,20 @@ function rangeKeys(aR: number, aC: number, r: number, c: number): Set<string> {
 }
 
 /**
- * Copy the selected cells to the clipboard as TSV — grouped by row,
- * columns ascending, so a paste into a spreadsheet lands in matching
- * grid positions.
+ * Copy the selected cells to the clipboard as TSV. Rows here live in a
+ * plain array, so the selection's row key is already the display index.
  */
 async function copySelectedCells(
   selected: Set<string>,
   rows: BrokenLinkRow[],
 ): Promise<void> {
-  if (selected.size === 0) return;
-  const byRow = new Map<number, number[]>();
-  for (const k of selected) {
-    const [rs, cs] = k.split(':');
-    const r = Number(rs);
-    const c = Number(cs);
-    if (!Number.isFinite(r) || !Number.isFinite(c)) continue;
-    const list = byRow.get(r);
-    if (list) list.push(c);
-    else byRow.set(r, [c]);
-  }
-  const lines: string[] = [];
-  for (const r of [...byRow.keys()].sort((a, b) => a - b)) {
-    const row = rows[r];
-    if (!row) continue;
-    const cols = (byRow.get(r) ?? []).sort((a, b) => a - b);
-    lines.push(cols.map((c) => cellValue(row, c)).join('\t'));
-  }
-  await writeTextToClipboard(lines.join('\n'));
-}
-
-/** Clipboard write with a hidden-textarea fallback for unfocused windows. */
-async function writeTextToClipboard(text: string): Promise<void> {
-  try {
-    await navigator.clipboard.writeText(text);
-  } catch {
-    const ta = document.createElement('textarea');
-    ta.value = text;
-    ta.style.position = 'fixed';
-    ta.style.opacity = '0';
-    document.body.appendChild(ta);
-    ta.select();
-    try {
-      document.execCommand('copy');
-    } finally {
-      document.body.removeChild(ta);
-    }
-  }
+  await copySelection(selected, {
+    order: (r) => r,
+    text: (r, c) => {
+      const row = rows[r];
+      return row ? cellValue(row, c) : null;
+    },
+  });
 }
 
 function statusClasses(code: number | null): string {

apps/desktop/src/renderer/src/tabs/UrlsTab.tsx

@@ -33,6 +33,7 @@ import type {
   AdvancedFilter,
   CrawlUrlRow,
   UrlCategory,
+  UrlGridMenuScope,
 } from '@freecrawl/shared-types';
 import { useAppStore, TAB_QUICK_FILTERS, type TabKey } from '../store.js';
 import { COLUMN_SPECS, columnId, type ColumnSpec } from './columns.js';
@@ -44,6 +45,17 @@ import { ErrorBoundary } from '../components/ErrorBoundary.js';
 import { ExportDialog } from '../components/ExportDialog.js';
 import { InfoTip } from '../components/InfoTip.js';
 import { translateLabel } from '../i18n/labels.js';
+import {
+  copySelection,
+  isAdditiveClick,
+  isGridCopyShortcut,
+  isMacSecondaryClick,
+  markGridActive,
+  ownsGridCopy,
+  toTsvLine,
+  writeTextToClipboard,
+  type CellSource,
+} from '../utils/clipboard.js';
 
 const ROW_HEIGHT = 24;
 const HEADER_HEIGHT = 28;
@@ -51,6 +63,14 @@ const MIN_COL_WIDTH = 48;
 const ROW_NUM_DEFAULT_WIDTH = 56;
 const ROW_NUM_KEY = '__row_num__';
 const STATUS_BAR_HEIGHT = 22;
+/** Identifies this grid to the copy-shortcut arbitration in `clipboard.ts`. */
+const GRID_ID = 'urls';
+// Rows pulled per query when copying whole columns. Big enough that a
+// 100k-row table is ~50 round-trips, small enough that each one stays a
+// short job on the reader worker instead of a multi-second stall.
+const COPY_FETCH_CHUNK = 2000;
+// How long the "copied N cells" acknowledgement stays up.
+const COPY_ACK_MS = 2500;
 const PREFS_PREFIX = 'col-widths:';
 const PREFS_HIDDEN_PREFIX = 'col-hidden:';
 const PREFS_PINNED_LEFT_PREFIX = 'col-pinned-left:';
@@ -192,6 +212,21 @@ export function UrlsTab() {
       }
     | null
   >(null);
+  /** Row data captured the moment a row is pulled into a selection.
+   *
+   *  The selection sets deliberately store url ids, not row indexes, so
+   *  they survive the re-sorts a running crawl causes. A copy, though,
+   *  needs two things ids alone can't give: the row's values and its
+   *  position in the table. The lazy loader holds at most ~8k rows and
+   *  evicts by LRU, so rows the user selected can be gone from its cache
+   *  by the time Ctrl+C is pressed — capturing them here is what keeps a
+   *  large drag-selection copyable in full instead of silently
+   *  truncated. Fresh data still wins: the copy re-reads the live row
+   *  whenever the loader still has it and only then falls back here. */
+  const rowSnapshots = useRef(new Map<number, { index: number; row: CrawlUrlRow }>());
+  const rememberRow = useCallback((index: number, row: CrawlUrlRow) => {
+    rowSnapshots.current.set(row.id, { index, row });
+  }, []);
   const [columnWidths, setColumnWidths] = useState<Record<string, number>>(() =>
     loadStoredWidths(activeTab),
   );
@@ -366,8 +401,21 @@ export function UrlsTab() {
     selectionAnchor.current = null;
     cellAnchor.current = null;
     dragRef.current = null;
+    rowSnapshots.current.clear();
   }, [activeCategory, activeTab, search, sortBy, sortDir, hiddenColumns]);
 
+  // Cell and column selections are keyed by column *index*, so any
+  // reshuffle of the column layout silently re-points them at a
+  // different column — the highlight jumps and a copy would hand over
+  // the wrong values. Show/hide is covered by the reset above (it keys
+  // on `hiddenColumns`); reorder and pin need their own clear. Row
+  // selection is index-free, so it survives.
+  useEffect(() => {
+    setSelectedCells(new Set());
+    setSelectedColumns(new Set());
+    cellAnchor.current = null;
+  }, [columnOrder, pinnedLeftOrder]);
+
   // Clear the drag flag on any mouseup anywhere — without this, releasing
   // the button outside a cell would leave `dragRef` dangling and the next
   // `mouseenter` would mistakenly extend the "dragged" range.
@@ -544,19 +592,24 @@ export function UrlsTab() {
     // selection because row and cell selections are mutually exclusive
     // semantically (row → "this URL as a record", cell → "this value").
     clearCellAndColumn();
+    const clicked = lazy.rowAt(rowIndex);
+    if (clicked) rememberRow(rowIndex, clicked);
     if (e.shiftKey && selectionAnchor.current !== null) {
       const anchorIdx = selectionAnchor.current;
       const [lo, hi] = anchorIdx < rowIndex ? [anchorIdx, rowIndex] : [rowIndex, anchorIdx];
       const next = new Set(selectedIds);
       for (let i = lo; i <= hi; i++) {
         const r = lazy.rowAt(i);
-        if (r) next.add(r.id);
+        if (r) {
+          next.add(r.id);
+          rememberRow(i, r);
+        }
       }
       setSelectedIds(next);
       setSelectedUrlId(rowId);
       return;
     }
-    if (e.ctrlKey || e.metaKey) {
+    if (isAdditiveClick(e)) {
       const next = new Set(selectedIds);
       if (next.has(rowId)) next.delete(rowId);
       else next.add(rowId);
@@ -589,6 +642,8 @@ export function UrlsTab() {
     setSelectedColumns(new Set());
 
     const k = `${rowId}:${colIdx}`;
+    const clicked = lazy.rowAt(rowIndex);
+    if (clicked) rememberRow(rowIndex, clicked);
 
     // Shift+Click within the same column extends the vertical range from
     // the anchor. Cross-column shift-click falls back to single-pick
@@ -600,13 +655,16 @@ export function UrlsTab() {
       const next = new Set(selectedCells);
       for (let i = lo; i <= hi; i++) {
         const r = lazy.rowAt(i);
-        if (r) next.add(`${r.id}:${colIdx}`);
+        if (r) {
+          next.add(`${r.id}:${colIdx}`);
+          rememberRow(i, r);
+        }
       }
       setSelectedCells(next);
       setSelectedUrlId(rowId);
       return;
     }
-    if (e.ctrlKey || e.metaKey) {
+    if (isAdditiveClick(e)) {
       const next = new Set(selectedCells);
       if (next.has(k)) next.delete(k);
       else next.add(k);
@@ -632,7 +690,7 @@ export function UrlsTab() {
     // Column selection — mutually exclusive with row selection, lives
     // alongside per-cell selection but replaces it on plain clicks.
     clearRow();
-    if (e.ctrlKey || e.metaKey) {
+    if (isAdditiveClick(e)) {
       const next = new Set(selectedColumns);
       if (next.has(colIdx)) next.delete(colIdx);
       else next.add(colIdx);
@@ -666,6 +724,7 @@ export function UrlsTab() {
       for (let i = loR; i <= hiR; i++) {
         const r = lazy.rowAt(i);
         if (!r) continue;
+        rememberRow(i, r);
         for (let c = loC; c <= hiC; c++) {
           next.add(`${r.id}:${c}`);
         }
@@ -674,7 +733,7 @@ export function UrlsTab() {
       const endRow = lazy.rowAt(toIdx);
       if (endRow) setSelectedUrlId(endRow.id);
     },
-    [lazy, setSelectedUrlId],
+    [lazy, rememberRow, setSelectedUrlId],
   );
 
   const applyRowDrag = useCallback(
@@ -687,13 +746,16 @@ export function UrlsTab() {
       const next = new Set(d.baseIds);
       for (let i = lo; i <= hi; i++) {
         const r = lazy.rowAt(i);
-        if (r) next.add(r.id);
+        if (r) {
+          next.add(r.id);
+          rememberRow(i, r);
+        }
       }
       setSelectedIds(next);
       const endRow = lazy.rowAt(toIdx);
       if (endRow) setSelectedUrlId(endRow.id);
     },
-    [lazy, setSelectedUrlId],
+    [lazy, rememberRow, setSelectedUrlId],
   );
 
   const applyColumnDrag = useCallback((toCol: number) => {
@@ -714,13 +776,17 @@ export function UrlsTab() {
     e: React.MouseEvent,
   ) => {
     if (e.button !== 0) return;
+    // macOS Ctrl+click is the OS secondary click: it fires mousedown *and*
+    // contextmenu. Starting a drag here would grow the selection at the
+    // exact moment the menu is about to act on it.
+    if (isMacSecondaryClick(e)) return;
     // Shift extends from the existing anchor using click semantics — no drag.
     if (e.shiftKey) {
       handleCellClick(rowId, colIdx, idx, e);
       return;
     }
     e.preventDefault();
-    const additive = e.ctrlKey || e.metaKey;
+    const additive = isAdditiveClick(e);
     clearRow();
     setSelectedColumns(new Set());
     dragRef.current = {
@@ -739,12 +805,16 @@ export function UrlsTab() {
 
   const beginRowDrag = (rowId: number, idx: number, e: React.MouseEvent) => {
     if (e.button !== 0) return;
+    // macOS Ctrl+click is the OS secondary click: it fires mousedown *and*
+    // contextmenu. Starting a drag here would grow the selection at the
+    // exact moment the menu is about to act on it.
+    if (isMacSecondaryClick(e)) return;
     if (e.shiftKey) {
       handleRowClick(rowId, idx, e);
       return;
     }
     e.preventDefault();
-    const additive = e.ctrlKey || e.metaKey;
+    const additive = isAdditiveClick(e);
     setSelectedCells(new Set());
     setSelectedColumns(new Set());
     dragRef.current = {
@@ -761,12 +831,16 @@ export function UrlsTab() {
 
   const beginColumnDrag = (colIdx: number, e: React.MouseEvent) => {
     if (e.button !== 0) return;
+    // macOS Ctrl+click is the OS secondary click: it fires mousedown *and*
+    // contextmenu. Starting a drag here would grow the selection at the
+    // exact moment the menu is about to act on it.
+    if (isMacSecondaryClick(e)) return;
     if (e.shiftKey) {
       handleHeaderClick(colIdx, e);
       return;
     }
     e.preventDefault();
-    const additive = e.ctrlKey || e.metaKey;
+    const additive = isAdditiveClick(e);
     clearRow();
     setSelectedCells(new Set());
     dragRef.current = {
@@ -779,6 +853,260 @@ export function UrlsTab() {
     applyColumnDrag(colIdx);
   };
 
+  // ──────── Clipboard ────────
+
+  const [copyState, setCopyState] = useState<
+    { phase: 'copying' } | { phase: 'done'; cells: number } | null
+  >(null);
+  const copyInFlight = useRef(false);
+
+  /** Everything the document-level Ctrl+C listener needs, refreshed on
+   *  every render. A ref rather than effect deps so the listener is
+   *  registered once instead of being torn down and re-added on every
+   *  mouseenter of a drag-select. */
+  const copyContext: CopyContext = {
+    columns,
+    lang,
+    rowAt: lazy.rowAt,
+    selectedCells,
+    selectedIds,
+    selectedColumns,
+    snapshots: rowSnapshots.current,
+    total: lazy.total,
+    query: {
+      category: activeCategory,
+      search: search || undefined,
+      sortBy,
+      sortDir,
+      filter: filter ?? undefined,
+    },
+  };
+  const copyCtx = useRef(copyContext);
+  copyCtx.current = copyContext;
+
+  /**
+   * Copy the current selection to the clipboard as TSV.
+   *
+   * The three selection layers are three different asks: a column pick
+   * means "the whole column", a cell pick means "these values", a row
+   * pick means "these records" (every visible column of each). Cells and
+   * rows resolve locally from the snapshot map; a column has to go back
+   * to SQLite, because it extends far past anything the lazy loader is
+   * holding.
+   *
+   * `scope` is passed by the context menu, which already decided which
+   * layer the right-click landed in — the layers can overlap (Ctrl+click
+   * a header while cells are picked), and the menu's label has to match
+   * what actually gets copied. Ctrl+C omits it and takes the default
+   * precedence instead.
+   */
+  const copySelectionToClipboard = useCallback(
+    async (scope?: CopyScope): Promise<void> => {
+      if (copyInFlight.current) return;
+      const ctx = copyCtx.current;
+      const effective =
+        scope ??
+        (ctx.selectedColumns.size > 0
+          ? 'columns'
+          : ctx.selectedCells.size > 0
+            ? 'cells'
+            : ctx.selectedIds.size > 0
+              ? 'rows'
+              : null);
+      if (effective === null) return;
+      copyInFlight.current = true;
+      try {
+        if (effective === 'columns') {
+          setCopyState({ phase: 'copying' });
+          setCopyState({ phase: 'done', cells: await copyWholeColumns(ctx) });
+          return;
+        }
+        backfillSnapshots(
+          ctx,
+          effective === 'cells' ? rowIdsInKeys(ctx.selectedCells) : ctx.selectedIds,
+        );
+        const source = cellSource(ctx);
+        if (effective === 'cells') {
+          setCopyState({
+            phase: 'done',
+            cells: await copySelection(ctx.selectedCells, source),
+          });
+          return;
+        }
+        // Rows: "these records" — every visible column of each.
+        const keys = new Set<string>();
+        for (const id of ctx.selectedIds) {
+          for (let c = 0; c < ctx.columns.length; c++) keys.add(`${id}:${c}`);
+        }
+        setCopyState({ phase: 'done', cells: await copySelection(keys, source) });
+      } catch {
+        // A failed copy leaves the clipboard untouched; drop the pending
+        // "Copying…" so the status bar doesn't hang on it forever.
+        setCopyState(null);
+      } finally {
+        copyInFlight.current = false;
+      }
+    },
+    [],
+  );
+
+  // Ctrl/Cmd+C copies the selection. Document-level, so having a
+  // selection is the only precondition — the user doesn't have to focus
+  // the table first. preventDefault does more than suppress the native
+  // copy here: on macOS the Edit ▸ Copy item re-dispatches a synthetic
+  // keydown and reads `defaultPrevented` to decide whether to fall
+  // through to the native copy (see menu.ts / App.tsx).
+  useEffect(() => {
+    const onKey = (e: KeyboardEvent) => {
+      if (!isGridCopyShortcut(e)) return;
+      // Tree and cluster views render their own bodies — the table's
+      // selection isn't on screen, so Ctrl+C isn't ours to claim.
+      if (viewMode !== 'list') return;
+      // Nor is it when the user's last click was in the detail panel's
+      // links table, which keeps its own selection and its own listener.
+      if (!ownsGridCopy(GRID_ID)) return;
+      const ctx = copyCtx.current;
+      if (
+        ctx.selectedCells.size === 0 &&
+        ctx.selectedIds.size === 0 &&
+        ctx.selectedColumns.size === 0
+      ) {
+        return;
+      }
+      e.preventDefault();
+      void copySelectionToClipboard();
+    };
+    document.addEventListener('keydown', onKey);
+    return () => document.removeEventListener('keydown', onKey);
+  }, [copySelectionToClipboard, viewMode]);
+
+  // The acknowledgement fades on its own — it confirms an action, it
+  // isn't a state the user has to dismiss.
+  useEffect(() => {
+    if (copyState?.phase !== 'done') return;
+    const id = setTimeout(() => setCopyState(null), COPY_ACK_MS);
+    return () => clearTimeout(id);
+  }, [copyState]);
+
+  // Release the snapshots once nothing is selected. Without this, a long
+  // session of drag-selecting across a 100k-row table would keep every
+  // row it ever touched alive on the heap.
+  useEffect(() => {
+    if (selectedCells.size === 0 && selectedIds.size === 0) {
+      rowSnapshots.current.clear();
+    }
+  }, [selectedCells, selectedIds]);
+
+  // ──────── Context menu ────────
+
+  /** Scope of the menu currently open, for the pushed-back Copy item. */
+  const pendingCopyScope = useRef<CopyScope | null>(null);
+
+  // The context menu's Copy item is executed here, not in main — main
+  // has no idea what a cell says. Scope comes from the click that opened
+  // the menu so the copy matches the item's label.
+  useEffect(() => {
+    const off = window.freecrawl.onUrlGridCopy(() => {
+      const scope = pendingCopyScope.current;
+      if (scope === null) return;
+      pendingCopyScope.current = null;
+      void copySelectionToClipboard(scope);
+    });
+    return off;
+  }, [copySelectionToClipboard]);
+
+  /** Url ids in the order their rows appear in the table. */
+  const inDisplayOrder = (ids: Iterable<number>): number[] =>
+    [...new Set(ids)].sort(
+      (a, b) =>
+        (rowSnapshots.current.get(a)?.index ?? Number.MAX_SAFE_INTEGER) -
+        (rowSnapshots.current.get(b)?.index ?? Number.MAX_SAFE_INTEGER),
+    );
+
+  /** Distinct rows touched by the current cell selection, in display order. */
+  const rowIdsFromCells = (): number[] => inDisplayOrder(rowIdsInKeys(selectedCells));
+
+  /** Whether any cell of this row is in the cell selection. Linear, but
+   *  it only runs on right-click and the selection bounds it. */
+  const rowHasSelectedCell = (id: number): boolean => {
+    const prefix = `${id}:`;
+    for (const k of selectedCells) if (k.startsWith(prefix)) return true;
+    return false;
+  };
+
+  /**
+   * Right-click inside the grid.
+   *
+   * This used to collapse everything to the clicked row before showing a
+   * menu, which threw away whatever the user had just selected — pick
+   * eight cells, right-click one of them, and you were left with one row
+   * and a menu that could only act on it. A click *inside* any of the
+   * three selection layers now keeps that selection and scopes the menu
+   * to it; only a click outside every layer replaces the selection,
+   * which is what a grid is expected to do.
+   *
+   * `colIdx` is null for the row-number cell, where a click is
+   * row-semantic by definition.
+   */
+  const handleContextMenu = (
+    row: CrawlUrlRow,
+    rowIndex: number,
+    colIdx: number | null,
+    e: React.MouseEvent,
+  ): void => {
+    e.preventDefault();
+    e.stopPropagation();
+
+    // On a data cell, "inside the selection" means that exact cell. On
+    // the row-number cell there is no column to match, so any selected
+    // cell in the row counts — right-clicking the row number of a row
+    // you just drag-selected shouldn't throw the selection away either.
+    const inCells =
+      selectedCells.size > 0 &&
+      (colIdx !== null
+        ? selectedCells.has(`${row.id}:${colIdx}`)
+        : rowHasSelectedCell(row.id));
+    const inColumns = colIdx !== null && selectedColumns.has(colIdx);
+    const inRows = selectedIds.has(row.id);
+
+    let scope: UrlGridMenuScope;
+    let urlIds: number[];
+    if (inCells) {
+      scope = 'cells';
+      urlIds = rowIdsFromCells();
+    } else if (inColumns) {
+      // A column selection spans the whole table, so Re-Spider / Remove
+      // at that scale would be a footgun — they stay on the clicked row.
+      // Only the copy item honours the full column.
+      scope = 'columns';
+      urlIds = [row.id];
+    } else if (inRows) {
+      scope = selectedIds.size > 1 ? 'rows' : 'row';
+      urlIds = inDisplayOrder(selectedIds);
+    } else {
+      clearCellAndColumn();
+      setSelectedIds(new Set([row.id]));
+      setSelectedUrlId(row.id);
+      selectionAnchor.current = rowIndex;
+      rememberRow(rowIndex, row);
+      scope = 'row';
+      urlIds = [row.id];
+    }
+
+    // Remembered for the Copy item, which main pushes back rather than
+    // returning (see the `urlGridCopy` channel). Only one context menu
+    // can be open at a time, so a single slot is enough.
+    pendingCopyScope.current =
+      scope === 'cells' ? 'cells' : scope === 'columns' ? 'columns' : 'rows';
+    void window.freecrawl.urlGridContextMenu({
+      scope,
+      urlIds,
+      url: row.url,
+      cellCount: selectedCells.size,
+      columnCount: selectedColumns.size,
+    });
+  };
+
   const startResize = (colKey: string, startWidth: number, clientX: number) => {
     const startX = clientX;
     const onMove = (ev: MouseEvent) => {
@@ -1008,7 +1336,11 @@ export function UrlsTab() {
       ) : viewMode === 'cluster' ? (
         <DuplicatesGroupedView />
       ) : (
-      <div ref={scrollRef} className="relative flex-1 select-none overflow-auto">
+      <div
+        ref={scrollRef}
+        className="relative flex-1 select-none overflow-auto"
+        onMouseDown={() => markGridActive(GRID_ID)}
+      >
         <div style={{ minWidth: totalWidth, width: '100%' }}>
           {/* Header row */}
           <div
@@ -1113,35 +1445,11 @@ export function UrlsTab() {
                   key={vi.key}
                   data-index={vi.index}
                   ref={virtualizer.measureElement}
+                  // Row-level fallback: the row-number cell and the
+                  // trailing filler. Data cells handle it themselves so
+                  // the menu knows which column was clicked.
                   onContextMenu={(e) => {
-                    e.preventDefault();
-                    if (!row) return;
-                    // Right-click is always row-semantic. If the clicked
-                    // row isn't in the bulk selection, replace it (and
-                    // drop any cell/column picks so the menu scope is
-                    // unambiguous).
-                    if (!selectedIds.has(row.id)) {
-                      setSelectedIds(new Set([row.id]));
-                      setSelectedUrlId(row.id);
-                      selectionAnchor.current = vi.index;
-                      setSelectedCells(new Set());
-                      setSelectedColumns(new Set());
-                      void window.freecrawl.urlContextMenu({
-                        url: row.url,
-                        urlId: row.id,
-                      });
-                      return;
-                    }
-                    if (selectedIds.size > 1) {
-                      void window.freecrawl.urlBulkContextMenu({
-                        urlIds: [...selectedIds],
-                      });
-                    } else {
-                      void window.freecrawl.urlContextMenu({
-                        url: row.url,
-                        urlId: row.id,
-                      });
-                    }
+                    if (row) handleContextMenu(row, vi.index, null, e);
                   }}
                   className={clsx(
                     'absolute left-0 top-0 flex items-center border-b border-surface-900 text-[11px]',
@@ -1239,6 +1547,9 @@ export function UrlsTab() {
                             applyCellDrag(vi.index, colIdx);
                           }
                         }}
+                        onContextMenu={(e) => {
+                          if (row) handleContextMenu(row, vi.index, colIdx, e);
+                        }}
                       >
                         <Cell row={row} spec={c} lang={lang} />
                       </div>
@@ -1277,9 +1588,20 @@ export function UrlsTab() {
       </div>
       )}
       <div
-        className="flex shrink-0 items-center justify-end gap-4 border-t border-surface-800 bg-surface-900/60 px-3 text-[11px] text-surface-400"
+        className="flex shrink-0 items-center gap-4 border-t border-surface-800 bg-surface-900/60 px-3 text-[11px] text-surface-400"
         style={{ height: STATUS_BAR_HEIGHT }}
       >
+        <span className="flex-1 truncate text-surface-500">
+          {copyState?.phase === 'copying' &&
+            t('urlsTab.copying', { defaultValue: 'Copying…' })}
+          {copyState?.phase === 'done' &&
+            (copyState.cells > 0
+              ? t('urlsTab.copiedCells', {
+                  defaultValue: '{{n}} cell(s) copied',
+                  n: copyState.cells.toLocaleString(),
+                })
+              : t('urlsTab.copiedNothing', { defaultValue: 'Nothing to copy' }))}
+        </span>
         <span>
           {t('urlsTab.statusUrls', { defaultValue: 'URLs:' })}{' '}
           <span className="font-mono tabular-nums text-surface-200">
@@ -1704,6 +2026,11 @@ function ColumnPickerPopover({
   );
 }
 
+/** Hover text for the green `*`. Kept as a constant because `Cell` runs
+ *  for every visible cell and looking the string up through the shared
+ *  label dictionary avoids a `useTranslation` hook on that path. */
+const CHANGED_MARKER_TITLE = 'Changed since the previous crawl';
+
 function Cell({
   row,
   spec,
@@ -1717,15 +2044,6 @@ function Cell({
     return <span className="text-surface-700">…</span>;
   }
   const raw = row[spec.key];
-  // Booleans render as "Y" / "—" rather than the JS-default "true"/"false".
-  // Matches Screaming Frog's compact convention for flag columns
-  // (Loop, Sequence Break, Self-Ref Missing, …).
-  const value =
-    raw === null || raw === undefined
-      ? ''
-      : typeof raw === 'boolean'
-        ? raw ? 'Y' : ''
-        : String(raw);
 
   if (spec.kind === 'status') {
     const code = row.statusCode;
@@ -1787,7 +2105,32 @@ function Cell({
     );
   }
 
+  // Shared with the clipboard path so what you copy is what you read —
+  // booleans as "Y"/blank, nulls as empty (the "—" below is presentation
+  // only). Derived here rather than at the top of the function so the
+  // branches above, which format their own values, don't pay for it.
+  const value = cellText(row, spec, lang);
+
   if (spec.kind === 'mono') {
+    // Re-crawl change marker. Sits outside the truncating span so it
+    // survives a URL too long for the column — the whole point is that
+    // it can be spotted by scanning the list, and a marker that
+    // disappears exactly on the long URLs would be worse than none.
+    if (spec.key === 'url' && row.changed) {
+      return (
+        <span className="flex w-full items-baseline overflow-hidden" title={value}>
+          {/* min-w-0 is what lets a flex child actually truncate — without
+              it the item refuses to shrink below its content width. */}
+          <span className="min-w-0 truncate font-mono text-surface-100">{value}</span>
+          <span
+            className="shrink-0 pl-1 font-mono font-bold text-emerald-400"
+            title={translateLabel(CHANGED_MARKER_TITLE, lang)}
+          >
+            *
+          </span>
+        </span>
+      );
+    }
     return (
       <span className="block truncate font-mono text-surface-100" title={value}>
         {value || <span className="text-surface-700">—</span>}
@@ -1802,6 +2145,163 @@ function Cell({
   );
 }
 
+/** Which selection layer a copy reads from. */
+type CopyScope = 'columns' | 'cells' | 'rows';
+
+/** Everything a clipboard copy needs from the live table. */
+interface CopyContext {
+  columns: ColumnSpec[];
+  lang: string;
+  rowAt: (index: number) => CrawlUrlRow | null;
+  selectedCells: Set<string>;
+  selectedIds: Set<number>;
+  selectedColumns: Set<number>;
+  snapshots: Map<number, { index: number; row: CrawlUrlRow }>;
+  /** Row count of the current view — the backfill scan's upper bound. */
+  total: number;
+  /** The query that produced the current view — replayed verbatim when
+   *  a whole-column copy has to re-read the table from SQLite. */
+  query: {
+    category: UrlCategory;
+    search: string | undefined;
+    sortBy: keyof CrawlUrlRow | undefined;
+    sortDir: 'asc' | 'desc';
+    filter: AdvancedFilter | undefined;
+  };
+}
+
+/** Row ids embedded in `"<urlId>:<colIdx>"` selection keys. */
+function rowIdsInKeys(keys: Iterable<string>): number[] {
+  const out: number[] = [];
+  for (const k of keys) {
+    const id = Number(k.slice(0, k.lastIndexOf(':')));
+    if (Number.isFinite(id)) out.push(id);
+  }
+  return out;
+}
+
+/**
+ * Record display positions for selected rows that were never dragged
+ * over.
+ *
+ * Selections normally build their snapshots as they go, but not every
+ * path goes through the grid — picking a node in tree view sets a row
+ * selection too, and switching back to the list leaves that row with no
+ * snapshot at all, so a copy would drop it silently. Scans only what the
+ * lazy loader is currently holding, and only when something is missing.
+ */
+function backfillSnapshots(ctx: CopyContext, ids: Iterable<number>): void {
+  const missing = new Set<number>();
+  for (const id of ids) if (!ctx.snapshots.has(id)) missing.add(id);
+  if (missing.size === 0) return;
+  for (let i = 0; i < ctx.total && missing.size > 0; i++) {
+    const row = ctx.rowAt(i);
+    if (row && missing.delete(row.id)) {
+      ctx.snapshots.set(row.id, { index: i, row });
+    }
+  }
+}
+
+/**
+ * Resolve `"<urlId>:<colIdx>"` selection keys into ordered plain text.
+ *
+ * Order comes from the snapshot's row index — where the row sat when it
+ * was selected. Values prefer the live row, so a copy taken mid-crawl
+ * carries what the table shows now rather than what it showed at click
+ * time, and fall back to the snapshot once the lazy loader has evicted
+ * the row.
+ */
+function cellSource(ctx: CopyContext): CellSource {
+  const resolve = (id: number): CrawlUrlRow | null => {
+    const snap = ctx.snapshots.get(id);
+    if (!snap) return null;
+    const live = ctx.rowAt(snap.index);
+    return live && live.id === id ? live : snap.row;
+  };
+  return {
+    order: (id) => ctx.snapshots.get(id)?.index ?? Number.MAX_SAFE_INTEGER,
+    text: (id, colIdx) => {
+      const spec = ctx.columns[colIdx];
+      if (!spec) return null;
+      const row = resolve(id);
+      return row ? cellText(row, spec, ctx.lang) : null;
+    },
+  };
+}
+
+/**
+ * Copy whole selected columns.
+ *
+ * Unlike a cell or row copy this can't be served from what's on screen:
+ * selecting a column header means every row of the current view, and the
+ * lazy loader only ever holds a few thousand of them. Page back through
+ * SQLite in the view's own sort order so the clipboard gets the complete
+ * column instead of the visible slice.
+ *
+ * @returns the number of cells written.
+ */
+async function copyWholeColumns(ctx: CopyContext): Promise<number> {
+  const specs = [...ctx.selectedColumns]
+    .sort((a, b) => a - b)
+    .map((c) => ctx.columns[c])
+    .filter((c): c is ColumnSpec => c !== undefined);
+  if (specs.length === 0) return 0;
+
+  const lines: string[] = [];
+  for (let offset = 0; ; offset += COPY_FETCH_CHUNK) {
+    const { rows } = await window.freecrawl.urlsQuery({
+      limit: COPY_FETCH_CHUNK,
+      offset,
+      ...ctx.query,
+    });
+    for (const row of rows) {
+      lines.push(toTsvLine(specs.map((s) => cellText(row, s, ctx.lang))));
+    }
+    if (rows.length < COPY_FETCH_CHUNK) break;
+  }
+  if (lines.length === 0) return 0;
+  await writeTextToClipboard(lines.join('\n'));
+  return lines.length * specs.length;
+}
+
+/**
+ * Plain-text value of one cell — the clipboard counterpart of
+ * {@link Cell}, and the single place the two agree on what a cell "says".
+ *
+ * Two deliberate departures from what's painted on screen, both because
+ * the consumer here is a spreadsheet rather than a reader: numbers are
+ * emitted unformatted (thousands separators would arrive as text, not a
+ * number), and an absent value is emitted as an empty string rather than
+ * the "—" placeholder.
+ */
+function cellText(row: CrawlUrlRow, spec: ColumnSpec, lang: string): string {
+  if (spec.kind === 'status') {
+    const code = row.statusCode;
+    if (code !== null && code !== undefined) return String(code);
+    const label = shortFailureLabel(row.statusText);
+    return label === '—' ? '' : label;
+  }
+
+  if (spec.kind === 'indexability') {
+    return translateLabel(
+      row.indexability === 'indexable' ? 'Indexable' : 'Non-Indexable',
+      lang,
+    );
+  }
+
+  if (spec.kind === 'indexability-status') {
+    const label = indexabilityStatusLabel(row.indexability);
+    return label === '' ? '' : translateLabel(label, lang);
+  }
+
+  const raw = row[spec.key];
+  if (raw === null || raw === undefined) return '';
+  // Booleans render as "Y" / blank rather than the JS-default
+  // "true"/"false" — matches the compact flag-column convention.
+  if (typeof raw === 'boolean') return raw ? 'Y' : '';
+  return String(raw);
+}
+
 function indexabilityStatusLabel(v: CrawlUrlRow['indexability']): string {
   switch (v) {
     case 'indexable':

apps/desktop/src/renderer/src/tabs/columns.ts

@@ -435,6 +435,14 @@ const COL = {
     info: '`<link rel="amphtml" href="…">` value — the AMP version of this page. Empty when the page does not declare an AMP alternate.',
     example: 'https://example.com/amp/post-1',
   } as ColumnSpec,
+  mobileAlternate: {
+    key: 'mobileAlternate',
+    header: 'Mobile Alternate',
+    size: 360,
+    kind: 'mono',
+    info: '`<link rel="alternate" media="only screen and (max-width: …)" href="…">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup.',
+    example: 'https://m.example.com/post-1',
+  } as ColumnSpec,
   schemaTypes: {
     key: 'schemaTypes',
     header: 'Schema Types',
@@ -675,6 +683,7 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
     COL.status,
     COL.indexability,
     COL.amphtml,
+    COL.mobileAlternate,
     COL.title,
     COL.inlinks,
   ],

apps/desktop/src/renderer/src/utils/clipboard.ts

@@ -0,0 +1,215 @@
+/**
+ * Shared clipboard helpers for the selectable data grids.
+ *
+ * Every grid in the app keys its cell selection the same way — a
+ * `Set<string>` of `"<rowKey>:<colIdx>"` — but they disagree on what a
+ * row key means. The detail-panel and Broken Links grids hold their rows
+ * in a plain array, so the key is the array index. The URLs grid is
+ * lazily paged out of SQLite and re-sorts underneath the user mid-crawl,
+ * so it keys on the stable url id instead and resolves display order
+ * separately. {@link CellSource} is the seam: give it an order function
+ * and a text function and the TSV assembly is identical for all of them.
+ */
+
+/** Resolves a grid's selection keys into ordered, plain-text cells. */
+export interface CellSource {
+  /**
+   * Display-order rank of a row key — lower sorts first. For an
+   * array-backed grid this is the identity function.
+   */
+  order: (rowKey: number) => number;
+  /**
+   * Plain text of one cell, or `null` when the row is no longer
+   * available (evicted from a cache, deleted mid-crawl). Null rows are
+   * dropped from the output rather than emitted as blank lines.
+   */
+  text: (rowKey: number, colIdx: number) => string | null;
+}
+
+/**
+ * Flatten a cell value into something a spreadsheet can take.
+ *
+ * Tabs and newlines are the TSV field/record separators, so a value
+ * containing either (a meta description with a hard line break, an
+ * anchor text with a tab) would silently shift every following cell into
+ * the wrong column. Collapse them to spaces — the value stays readable
+ * and the grid stays aligned.
+ */
+function sanitize(value: string): string {
+  if (!/[\t\r\n]/.test(value)) return value;
+  return value.replace(/[\t\r\n]+/g, ' ').trim();
+}
+
+/** Join one row's values into a TSV line, separator-safe. */
+export function toTsvLine(values: readonly string[]): string {
+  return values.map(sanitize).join('\t');
+}
+
+/**
+ * Build a TSV block from a cell selection — grouped by row in display
+ * order, columns ascending — so a paste into a spreadsheet drops the
+ * cells into matching grid positions.
+ *
+ * `cells` counts what actually made it into the text, which is less than
+ * the selection size whenever a row has gone away underneath it.
+ */
+export function selectionToTsv(
+  selected: Iterable<string>,
+  src: CellSource,
+): { text: string; cells: number } {
+  const byRow = new Map<number, number[]>();
+  for (const k of selected) {
+    const sep = k.lastIndexOf(':');
+    if (sep < 0) continue;
+    const r = Number(k.slice(0, sep));
+    const c = Number(k.slice(sep + 1));
+    if (!Number.isFinite(r) || !Number.isFinite(c)) continue;
+    const list = byRow.get(r);
+    if (list) list.push(c);
+    else byRow.set(r, [c]);
+  }
+  const rowKeys = [...byRow.keys()].sort((a, b) => src.order(a) - src.order(b));
+  const lines: string[] = [];
+  let cells = 0;
+  for (const rowKey of rowKeys) {
+    const cols = (byRow.get(rowKey) ?? []).sort((a, b) => a - b);
+    const values: string[] = [];
+    let alive = false;
+    for (const c of cols) {
+      const v = src.text(rowKey, c);
+      if (v !== null) alive = true;
+      values.push(v ?? '');
+    }
+    if (alive) {
+      lines.push(toTsvLine(values));
+      cells += values.length;
+    }
+  }
+  return { text: lines.join('\n'), cells };
+}
+
+/**
+ * Copy a cell selection to the OS clipboard as TSV.
+ *
+ * @returns the number of cells actually written — callers use it to
+ *          confirm the copy in the UI, and to stay silent on a no-op.
+ */
+export async function copySelection(
+  selected: Set<string>,
+  src: CellSource,
+): Promise<number> {
+  if (selected.size === 0) return 0;
+  const { text, cells } = selectionToTsv(selected, src);
+  if (cells === 0) return 0;
+  await writeTextToClipboard(text);
+  return cells;
+}
+
+/**
+ * Write `text` to the OS clipboard.
+ *
+ * The async Clipboard API is tried first and covers the normal case. It
+ * rejects when the renderer isn't focused or the clipboard permission is
+ * unset — situations that do occur in Electron — so a hidden textarea +
+ * `execCommand('copy')` backs it up. The deprecated call is deliberate:
+ * it is the only path that works without focus.
+ */
+export async function writeTextToClipboard(text: string): Promise<void> {
+  try {
+    await navigator.clipboard.writeText(text);
+  } catch {
+    const ta = document.createElement('textarea');
+    ta.value = text;
+    ta.style.position = 'fixed';
+    ta.style.opacity = '0';
+    document.body.appendChild(ta);
+    ta.select();
+    try {
+      document.execCommand('copy');
+    } finally {
+      document.body.removeChild(ta);
+    }
+  }
+}
+
+/**
+ * True when the keyboard event should be treated as "copy the grid
+ * selection" rather than the browser's own copy.
+ *
+ * Both Ctrl and Cmd are accepted on every platform: macOS delivers ⌘C
+ * through the Edit ▸ Copy round-trip in `menu.ts` (which synthesises a
+ * `metaKey` keydown), and a Mac user who reaches for Ctrl+C out of habit
+ * gets the same result rather than nothing.
+ *
+ * Inputs, textareas and contenteditable regions keep the native copy —
+ * the user pressing Ctrl+C inside a filter box means the box's text, not
+ * whatever cells happen to be selected behind it.
+ */
+export function isGridCopyShortcut(e: KeyboardEvent): boolean {
+  if (!(e.ctrlKey || e.metaKey)) return false;
+  if (e.key !== 'c' && e.key !== 'C') return false;
+  const target = e.target as HTMLElement | null;
+  const tag = target?.tagName;
+  if (tag === 'INPUT' || tag === 'TEXTAREA' || target?.isContentEditable) return false;
+  return true;
+}
+
+/** True on macOS. The renderer has no `process`, so sniff the UA. */
+export const IS_MAC = /Mac|iPhone|iPad/i.test(navigator.userAgent);
+
+/**
+ * The modifier that means "add to the selection" on this platform.
+ *
+ * On Windows and Linux that's Ctrl. On macOS it is ⌘ *only*: there,
+ * Ctrl+click is the OS-level secondary click, so treating it as additive
+ * would grow the selection at the exact moment the user is opening a
+ * context menu — and the menu would then act on a cell they never meant
+ * to pick.
+ */
+export function isAdditiveClick(e: { ctrlKey: boolean; metaKey: boolean }): boolean {
+  return IS_MAC ? e.metaKey : e.ctrlKey || e.metaKey;
+}
+
+/**
+ * True when this mouse event is macOS's Ctrl+click secondary click.
+ *
+ * Chromium on macOS fires both `mousedown` (button 0, ctrlKey) and
+ * `contextmenu` for that gesture. Grids use it to skip drag-selection so
+ * the gesture opens a menu and nothing else.
+ */
+export function isMacSecondaryClick(e: {
+  button: number;
+  ctrlKey: boolean;
+}): boolean {
+  return IS_MAC && e.button === 0 && e.ctrlKey;
+}
+
+// ──────── Which grid owns the copy shortcut ────────
+
+/**
+ * Several grids can be on screen at once — the URL table and the detail
+ * panel's Inlinks/Outlinks table, say — and each listens for
+ * Ctrl/Cmd+C at the document level. A user holding a live selection in
+ * both would otherwise fire every handler on one keypress and get
+ * whichever async clipboard write happened to land last.
+ *
+ * The grid the user most recently pressed a mouse button inside wins.
+ * That is also the grid they are looking at, and since a selection can
+ * only be made by clicking inside a grid, whichever grid has something
+ * to copy has necessarily claimed ownership at some point.
+ */
+let activeGrid: string | null = null;
+
+/** Claim the copy shortcut for `id`. Call from the grid root's mousedown. */
+export function markGridActive(id: string): void {
+  activeGrid = id;
+}
+
+/**
+ * Whether `id` should handle a copy shortcut right now. Nothing has been
+ * clicked yet → nobody owns it, and nobody has a selection either, so
+ * letting every grid through is harmless.
+ */
+export function ownsGridCopy(id: string): boolean {
+  return activeGrid === null || activeGrid === id;
+}

apps/mcp-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/mcp-server",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "private": true,
   "type": "module",
   "bin": {

apps/mcp-server/src/tools.ts

@@ -286,6 +286,7 @@ export function buildTools(): Tool[] {
               crawlDelayMs: { type: 'integer', minimum: 0 },
               requestTimeoutMs: { type: 'integer', minimum: 1000 },
               respectRobotsTxt: { type: 'boolean' },
+              respectCrawlDelay: { type: 'boolean' },
               followRedirects: { type: 'boolean' },
               crawlExternal: { type: 'boolean' },
               userAgent: { type: 'string' },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "freecrawl-seo",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "private": true,
   "description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/core",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/crawler.ts

@@ -69,6 +69,30 @@ interface QueueItem {
   depth: number;
   /** How many redirect hops led here (0 for items from link extraction). */
   redirectHopCount?: number;
+  /**
+   * Set only by `enqueueResources`, naming the Spider → Crawl row whose
+   * "Crawl" box put this URL in the queue. The matching "Store" box then
+   * decides whether the fetched resource keeps its URL row.
+   *
+   * Carried on the queue item rather than re-derived from the response's
+   * content type at write time, because the two disagree exactly where it
+   * matters: a `.zip` reached through an ordinary hyperlink and a `.mp4`
+   * reached through `<video src>` both detect as `'other'`, and only the
+   * second one is governed by "Store Media".
+   *
+   * Absent for links, sitemap entries, redirect hops, and the resources
+   * mined out of stylesheets — none of those are gated on storage.
+   */
+  resourceRow?: 'image' | 'css' | 'js' | 'media';
+  /**
+   * Fetch this URL and record it, but do not follow anything it links to.
+   * Set by "Check Links Outside of Start Folder": the point of that
+   * option is to learn a status code, not to leave the start folder.
+   *
+   * A leaf flag rather than an inflated depth, so the URL's reported
+   * crawl depth stays the number of hops it actually took to reach.
+   */
+  checkOnly?: boolean;
 }
 
 const EXT_TO_KIND: Record<string, ContentKind> = {
@@ -332,6 +356,16 @@ export class Crawler extends EventEmitter {
    */
   private readonly dbCall: <T>(method: string, args: unknown[]) => Promise<T>;
 
+  /**
+   * Explicit re-fetch list for a targeted Re-Spider run, or null for a
+   * normal crawl. Non-null also implies "never wipe the project" — a
+   * re-spider updates rows in place, it does not start a new data set.
+   */
+  private readonly respiderSeeds: string[] | null;
+
+  /** True when this run resumes an interrupted crawl from a checkpoint. */
+  private readonly resumeOnly: boolean;
+
   /**
    * V2 — Optional Playwright renderer. Returns the post-JS DOM dump for
    * a URL when `renderingMode === 'js'`. Defaults to undefined so the
@@ -432,9 +466,31 @@ export class Crawler extends EventEmitter {
           tapTargetsSampled: number;
         } | null;
       }>;
+      /**
+       * Re-fetch exactly these URLs instead of walking the site from the
+       * seed — the "Re-Spider" affordance when no crawl is running.
+       *
+       * Everything already in the project is pre-marked as seen, so a
+       * re-spider costs one request per listed URL. Links found on those
+       * pages that the project has never seen ARE followed, which is the
+       * point: a page that gained a link should bring the new target into
+       * the report rather than leaving a gap.
+       */
+      respiderSeeds?: string[];
+      /**
+       * This run continues an interrupted crawl (crash recovery) rather
+       * than starting one. Suppresses the re-fetch-everything behaviour a
+       * repeat Start normally triggers: the caller has already restored
+       * the exact queue that was in flight, and re-reading the pages that
+       * did complete would turn "carry on where you left off" into "crawl
+       * the whole site again".
+       */
+      resumeOnly?: boolean;
     } = {},
   ) {
     super();
+    this.respiderSeeds = opts.respiderSeeds ?? null;
+    this.resumeOnly = opts.resumeOnly === true;
     this.setOp = opts.setOp ?? ((): void => undefined);
     this.parsePage =
       opts.parseHtml ??
@@ -628,7 +684,7 @@ export class Crawler extends EventEmitter {
         `Invalid start URL: ${this.config.startUrl} — neither https:// nor http:// responded within 5s. Check that the host is reachable from this machine (try opening it in a browser).`,
       );
       this.running = false;
-      this.emitProgress();
+      this.emitProgressFinal();
       // Without this the UI hangs in "Running" forever after a bad start URL.
       if (!this.stopped) {
       // Wave 6 — Clean completion clears the checkpoint so the next
@@ -654,14 +710,39 @@ export class Crawler extends EventEmitter {
     this.applyProcessPriority();
     this.startMemoryMonitor();
 
-    // Fresh-start vs. resume decision. If the start URL matches the one
-    // recorded from the previous crawl, we keep existing rows and resume.
-    // If it differs (or there is no previous crawl), we wipe the tables.
+    // Fresh-start vs. refresh decision. A different start URL means a
+    // different site, so the tables are wiped. The same start URL means
+    // the user wants this site looked at again — existing rows are kept
+    // and every one of them is re-fetched (see `hydrateFromDb`), because
+    // a site that was fully crawled has no pending work left and would
+    // otherwise finish instantly without touching a single page.
+    //
+    // A Re-Spider is exempt from both halves: it updates the rows the
+    // user picked and must never wipe the project or re-point it at a
+    // different seed, even if the start URL happens to resolve somewhere
+    // new (a changed redirect would otherwise destroy the whole crawl
+    // the user was looking at).
+    // A crash-recovery resume is exempt from the re-fetch half only — it
+    // is the same site, so the tables must survive.
     const previousStart = this.db.getMeta('startUrl');
-    if (previousStart !== start) {
+    const isRespider = this.respiderSeeds !== null;
+    const isSameSite = previousStart === start;
+    // Wipe ONLY when this is genuinely a different site. Keep this
+    // condition keyed on `isSameSite` rather than on `isRefresh` — a
+    // resume or a re-spider is also "not a refresh", and resetting on
+    // either would destroy the crawl the user is trying to continue.
+    if (!isSameSite && !isRespider) {
       this.db.reset();
+      this.db.setMeta('startUrl', start);
+    }
+    const isRefresh = isSameSite && !isRespider && !this.resumeOnly;
+    // Change markers describe the run that is starting now, so anything
+    // a previous refresh flagged is cleared first. A wiped project has
+    // nothing to clear, and a targeted re-spider clears only its own
+    // seeds (done by the caller) so unrelated markers survive.
+    if (isRefresh) {
+      this.db.clearChangedFlags();
     }
-    this.db.setMeta('startUrl', start);
 
     const origin = new URL(start).origin;
     // robots.txt + sitemap discovery used to block the crawl start
@@ -675,15 +756,32 @@ export class Crawler extends EventEmitter {
       ? loadRobots(origin, this.config.userAgent).then((r) => {
           if (!this.stopped) {
             this.robots = r;
-            // Honour a robots.txt `Crawl-delay` (seconds) by folding it into
-            // the per-worker politeness delay. Cached once — the start
+            // robots.txt `Crawl-delay` (seconds). Cached once — the start
             // origin's directive applies for the crawl, matching the single
             // robots checker used for allow/deny.
+            //
+            // Opt-in (`respectCrawlDelay`, default off). The directive is not
+            // part of RFC 9309; Google ignores it and Screaming Frog does not
+            // implement it. Published values are usually stale copy-paste
+            // (`Crawl-delay: 30` is common) and honouring them literally costs
+            // hours per crawl. Either way we surface the directive so a slow
+            // crawl is never unexplained — silently sleeping for 30 s per URL
+            // with no feedback is the actual bug this replaces.
             const rd = r.getCrawlDelay();
-            this.robotsCrawlDelayMs =
+            const declaredMs =
               typeof rd === 'number' && Number.isFinite(rd) && rd > 0
                 ? Math.round(rd * 1000)
                 : 0;
+            this.robotsCrawlDelayMs = this.config.respectCrawlDelay ? declaredMs : 0;
+            if (declaredMs > 0) {
+              const secs = (declaredMs / 1000).toFixed(declaredMs % 1000 === 0 ? 0 : 1);
+              this.emit(
+                'warn',
+                this.config.respectCrawlDelay
+                  ? `robots.txt declares Crawl-delay: ${secs}s — honouring it caps this crawl at one request every ${secs}s. Turn off "Respect robots.txt Crawl-delay" in Settings to crawl at full speed.`
+                  : `robots.txt declares Crawl-delay: ${secs}s — ignored (matches Google / Screaming Frog; not part of RFC 9309). Enable "Respect robots.txt Crawl-delay" in Settings to honour it.`,
+              );
+            }
           }
         })
       : Promise.resolve();
@@ -691,8 +789,14 @@ export class Crawler extends EventEmitter {
     // seed URLs — both are handled inside `discoverAndIngestSitemaps`, which
     // writes `sitemap_urls` exactly once (merged) to avoid the replace-all
     // race, and additionally enqueues the seed entries as crawl seeds.
+    // Skipped for a targeted Re-Spider: the user asked for specific URLs
+    // to be re-read, and every sitemap entry is already `seen`, so a
+    // rediscovery pass would spend a second or two fetching XML that
+    // cannot add a single fetch. The stored `sitemap_urls` rows survive
+    // (nothing was reset), so sitemap-derived issue counts stay intact.
     const sitemapPromise =
-      this.config.discoverSitemaps || this.config.seedSitemapUrls.length > 0
+      this.respiderSeeds === null &&
+      (this.config.discoverSitemaps || this.config.seedSitemapUrls.length > 0)
         ? this.discoverAndIngestSitemaps(origin)
         : Promise.resolve();
 
@@ -744,7 +848,7 @@ export class Crawler extends EventEmitter {
 
     // Hydrate in-memory state from the DB so resume starts from the right
     // point; then queue whatever work is still pending.
-    this.hydrateFromDb();
+    this.hydrateFromDb({ refresh: isRefresh });
 
     try {
       // Wait for internal crawl first, then drain any external probes still
@@ -758,6 +862,9 @@ export class Crawler extends EventEmitter {
       while (!this.stopped) {
         await this.queue.onIdle();
         await new Promise<void>((r) => setTimeout(r, 50));
+        // Top up a refresh run's backlog as the queue frees up. No-op for
+        // a normal crawl.
+        this.feedRefreshBacklog();
         if (this.queue.size + this.queue.pending === 0) break;
       }
       await this.externalQueue.onIdle();
@@ -824,7 +931,7 @@ export class Crawler extends EventEmitter {
     // heap that's no longer needed once the queue is drained.
     this.seen.clear();
     this.externalSeen.clear();
-    this.emitProgress();
+    this.emitProgressFinal();
     this.setOp('idle');
     // Suppress 'done' if a stop() ran during teardown — otherwise the
     // zombie crawler's done-event clobbers the new crawl's UI state.
@@ -1608,7 +1715,7 @@ export class Crawler extends EventEmitter {
    *  with nothing to fetch): stop, clear the resume checkpoint, emit done. */
   private finishEmptyFixedCrawl(): void {
     this.running = false;
-    this.emitProgress();
+    this.emitProgressFinal();
     if (!this.stopped) {
       try {
         this.db.clearQueueCheckpoint();
@@ -1680,7 +1787,7 @@ export class Crawler extends EventEmitter {
     this.stopMemoryMonitor();
     this.seen.clear();
     this.externalSeen.clear();
-    this.emitProgress();
+    this.emitProgressFinal();
     if (!this.stopped) {
       // Wave 6 — Clean completion clears the checkpoint so the next
       // app launch doesn't offer to "resume" a crawl that already
@@ -1753,6 +1860,22 @@ export class Crawler extends EventEmitter {
               `Sitemap: parsed ${res.sitemapsParsed.length}/${res.sitemapsTried.length}, ${res.entries.length} URLs${res.truncated ? ` (truncated at ${sitemapMaxUrls.toLocaleString()})` : ''}`,
             );
           }
+          // Spider → Crawl "Crawl Linked XML Sitemaps". Auto-discovery on
+          // its own only records entries, which is what the sitemap issue
+          // filters compare against. Crawling them as well is what finds
+          // orphans — pages the sitemap declares but nothing links to.
+          if (this.config.crawlLinkedSitemaps) {
+            let seeded = 0;
+            for (const e of res.entries) {
+              const norm = normalizeUrl(e.url, undefined, this.urlRewrites);
+              if (!norm) continue;
+              this.enqueue({ url: norm, depth: 0 });
+              seeded++;
+            }
+            if (seeded > 0) {
+              this.emit('info', `Sitemap: ${seeded} discovered URL(s) queued as crawl seeds`);
+            }
+          }
         }
 
         // (2) User-supplied seed sitemaps — recorded AND enqueued.
@@ -1792,12 +1915,93 @@ export class Crawler extends EventEmitter {
     }
   }
 
-  private hydrateFromDb(): void {
+  /** Known URLs a refresh run still has to re-fetch, and how far into it
+   *  we are. Fed to the queue in slices rather than all at once — see
+   *  `feedRefreshBacklog`. */
+  private refreshBacklog: { url: string; depth: number }[] = [];
+  private refreshBacklogIndex = 0;
+
+  /** How many backlog URLs to queue per top-up. */
+  private static readonly REFRESH_FEED_CHUNK = 5000;
+
+  /**
+   * Queue the next slice of a refresh run's backlog.
+   *
+   * Dumping a whole site onto the queue in one go would materialise a
+   * queue task per URL up front (a real memory spike past ~100 K pages)
+   * and, worse, silently lose the tail on any project with a
+   * `maxQueueSize`: `enqueue` drops anything arriving above the cap, so
+   * those pages would never be re-crawled and the run would report
+   * success having skipped them. Feeding in slices and topping up as the
+   * queue drains keeps coverage complete at any site size.
+   */
+  private feedRefreshBacklog(): void {
+    if (this.stopped) return;
+    if (this.refreshBacklogIndex >= this.refreshBacklog.length) return;
+    const cap = this.config.maxQueueSize;
+    const room =
+      cap > 0
+        ? Math.max(0, cap - (this.queue.size + this.queue.pending))
+        : Crawler.REFRESH_FEED_CHUNK;
+    const take = Math.min(Crawler.REFRESH_FEED_CHUNK, room);
+    if (take <= 0) return;
+    const end = Math.min(
+      this.refreshBacklog.length,
+      this.refreshBacklogIndex + take,
+    );
+    for (; this.refreshBacklogIndex < end; this.refreshBacklogIndex++) {
+      const row = this.refreshBacklog[this.refreshBacklogIndex]!;
+      // Already known, so `seen` holds it from the hydrate sweep above —
+      // drop it so enqueue accepts the re-fetch. Anything the live
+      // link-follow path reaches first stays deduped: enqueue re-adds it
+      // to `seen`, and the backlog's own enqueue then no-ops.
+      this.seen.delete(row.url);
+      this.enqueue({ url: row.url, depth: row.depth });
+    }
+  }
+
+  private hydrateFromDb(opts: { refresh: boolean } = { refresh: false }): void {
     // Mark every already-known URL as "seen" so enqueue can skip them.
     for (const url of this.db.getAllUrls()) {
       this.seen.add(url);
     }
-    this.crawled = this.db.countCrawledUrls();
+    // A refresh counts its own work from zero — the progress bar should
+    // read "0 of 122 re-crawled", not start at 122 and never move.
+    this.crawled = opts.refresh ? 0 : this.db.countCrawledUrls();
+    // Same reasoning for the subresource quota: a resumed crawl continues
+    // spending the original allowance. A refresh re-crawls the known set, so
+    // it starts over alongside `crawled`.
+    this.resourceAdmitted = opts.refresh ? 0 : this.db.countSubresourceUrls();
+
+    // Targeted Re-Spider: fetch only the listed URLs. They are dropped
+    // from `seen` so enqueue accepts them despite already having rows;
+    // everything else stays seen, so the run costs one request per seed
+    // plus whatever genuinely new links those pages turn up.
+    if (this.respiderSeeds !== null) {
+      for (const url of this.respiderSeeds) {
+        this.seen.delete(url);
+        this.enqueue({ url, depth: this.db.getUrlDepth(url) ?? 0 });
+      }
+      return;
+    }
+
+    // Refresh run — Start pressed again on a site that is already in the
+    // project. Re-fetch every page we hold, so a site the user has since
+    // edited shows its current state and the reports are rebuilt from
+    // fresh responses. Links discovered along the way that the project
+    // has never seen are followed as usual, which is how pages added
+    // since the last crawl get picked up.
+    if (opts.refresh) {
+      this.refreshBacklog = this.db.getCrawledInternalUrls();
+      this.refreshBacklogIndex = 0;
+      if (this.refreshBacklog.length > 0) {
+        this.emit(
+          'info',
+          `Refresh: re-crawling ${this.refreshBacklog.length} known URL(s)…`,
+        );
+      }
+      this.feedRefreshBacklog();
+    }
 
     // If the start URL isn't in the DB yet, kick off a brand-new crawl from it.
     if (!this.db.hasUrl(this.config.startUrl)) {
@@ -2042,7 +2246,9 @@ export class Crawler extends EventEmitter {
       }
     }
     this.setOp('paused');
-    this.emitProgress();
+    // User-initiated state change — never throttle it, the button must
+    // flip on the next frame rather than up to 200 ms later.
+    this.emitProgressFinal();
   }
 
   resume(): void {
@@ -2051,7 +2257,7 @@ export class Crawler extends EventEmitter {
     this.queue.start();
     this.externalQueue.start();
     this.setOp('idle');
-    this.emitProgress();
+    this.emitProgressFinal();
   }
 
   get isRunning(): boolean {
@@ -2281,10 +2487,68 @@ export class Crawler extends EventEmitter {
     return this.seen.size > before;
   }
 
+  /**
+   * Dispatch priority. Documents outrank subresources; p-queue runs the
+   * highest number first and keeps FIFO order within a tier, so pages
+   * still traverse breadth-first and resources still fetch in discovery
+   * order — only the interleaving changes.
+   *
+   * `resourceRow` is exactly the right discriminator: `enqueueResources`
+   * sets it for images / CSS / JS / media and nothing else does. Links,
+   * sitemap entries, redirect hops, manual additions, and iframes (which
+   * are documents that get parsed for their own links) all land in the
+   * document tier.
+   *
+   * Why this matters: resources are enqueued before the link-follow loop
+   * in `fetchAndProcess`, so under a flat FIFO queue a page's entire
+   * asset set lands ahead of every link it found. On an image-heavy home
+   * page that is ~90 fetches of images before the first category page —
+   * the crawl looks stalled to anyone watching the (HTML-filtered) URL
+   * table, and a `maxUrls` budget gets spent on assets instead of pages.
+   * Nothing is skipped either way; assets are fetched as slots free up.
+   */
+  private static priorityFor(item: QueueItem): number {
+    return item.resourceRow ? 0 : 1;
+  }
+
+  /**
+   * Share of `maxUrls` that subresources may claim. `priorityFor` decides the
+   * order pages and assets are *fetched* in; this decides how much of a
+   * limited budget they are *allowed* in the queue for at all — the admission
+   * cap in `enqueue` is first-come-first-served, and resources are enqueued
+   * before the link-follow loop runs, so without a rail one image-heavy page
+   * can spend the whole allowance before a single link is considered. On a
+   * 90-URL budget that produced 83 images and exactly one page.
+   *
+   * A rail, not a redesign: it only binds on small budgets. Measured on a
+   * 500-URL budget the mix moves 323→343 pages and 172→152 images, while a
+   * 90-URL budget goes from 1 page to 55. Deliberately not a setting — the
+   * failure it prevents is one users hit without knowing to look for it, and
+   * an unlimited `maxUrls` (the default) never reaches the cap at all.
+   */
+  private static readonly SUBRESOURCE_BUDGET_SHARE = 0.4;
+
+  /**
+   * Subresources admitted against `maxUrls`. Seeded from the project on
+   * resume so a continued crawl shares one allowance with the run that
+   * created it rather than starting a fresh one.
+   */
+  private resourceAdmitted = 0;
+
   private enqueue(item: QueueItem): void {
     if (this.stopped) return;
     if (this.seen.has(item.url)) return;
     if (this.seen.size >= this.config.maxUrls) return;
+    // Subresource share of the budget (see SUBRESOURCE_BUDGET_SHARE). Checked
+    // after the overall cap so the budget total is unchanged — this only
+    // decides who gets to claim the remaining slots.
+    if (
+      item.resourceRow &&
+      this.resourceAdmitted >=
+        Math.round(this.config.maxUrls * Crawler.SUBRESOURCE_BUDGET_SHARE)
+    ) {
+      return;
+    }
     if (item.depth > this.config.maxDepth) return;
     if (this.robots && !this.robots.isAllowed(item.url)) return;
     if (!this.passesUrlFilter(item.url)) return;
@@ -2301,10 +2565,11 @@ export class Crawler extends EventEmitter {
     }
 
     this.seen.add(item.url);
+    if (item.resourceRow) this.resourceAdmitted++;
     this.pending++;
     this.pendingItems.set(item.url, item.depth);
     this.queue
-      .add(() => this.fetchAndProcess(item))
+      .add(() => this.fetchAndProcess(item), { priority: Crawler.priorityFor(item) })
       .catch((err: unknown) => {
         this.emit(
           'error',
@@ -2331,29 +2596,48 @@ export class Crawler extends EventEmitter {
    * include/exclude filters, dedupe) still apply. Non-HTML responses never
    * reach the link-follow path, so resources stay leaf nodes.
    */
+  /**
+   * The "Store" half of a Spider → Crawl resource row. Split out because
+   * the decision is needed on the write path, far from `enqueueResources`
+   * where the row was chosen.
+   */
+  private storesResourceRow(row: NonNullable<QueueItem['resourceRow']>): boolean {
+    if (row === 'css') return this.config.storeCss;
+    if (row === 'js') return this.config.storeJs;
+    if (row === 'image') return this.config.storeImages;
+    return this.config.storeMedia;
+  }
+
   private enqueueResources(parsed: ParsedPage, depth: number): void {
     if (this.config.checkImages) {
       for (const img of parsed.images) {
-        if (img.isInternal) this.enqueue({ url: img.src, depth });
+        if (img.isInternal) this.enqueue({ url: img.src, depth, resourceRow: 'image' });
         // External images: status/size are filled by the post-crawl image
         // probe (images table), which now covers external images too — so
         // a dead external/CDN image still trips "Broken Image src".
       }
     }
-    // Whether a resource kind should be crawled at all. Fonts ride with the
-    // CSS toggle (they're declared by CSS / font preloads); images obey the
-    // image toggle so `<link rel=preload as=image>` matches `<img>` behaviour.
-    const shouldFollow = (kind: ContentKind): boolean => {
-      if (kind === 'css') return this.config.checkCss;
-      if (kind === 'js') return this.config.checkJs;
-      if (kind === 'image') return this.config.checkImages;
-      if (kind === 'font') return this.config.checkCss;
-      return false;
+    // Which Spider → Crawl row governs each resource kind. Fonts ride with
+    // CSS (they're declared by stylesheets and font preloads); `'other'` is
+    // only ever produced by the media extractor, so it maps to Media.
+    const rowFor = (kind: ContentKind): QueueItem['resourceRow'] | null => {
+      if (kind === 'css' || kind === 'font') return 'css';
+      if (kind === 'js') return 'js';
+      if (kind === 'image') return 'image';
+      if (kind === 'other') return 'media';
+      return null;
+    };
+    const shouldFollow = (row: NonNullable<QueueItem['resourceRow']>): boolean => {
+      if (row === 'css') return this.config.checkCss;
+      if (row === 'js') return this.config.checkJs;
+      if (row === 'image') return this.config.checkImages;
+      return this.config.crawlMedia;
     };
     for (const r of parsed.resources) {
-      if (!shouldFollow(r.kind)) continue;
+      const row = rowFor(r.kind);
+      if (!row || !shouldFollow(row)) continue;
       if (r.isInternal) {
-        this.enqueue({ url: r.url, depth });
+        this.enqueue({ url: r.url, depth, resourceRow: row });
       } else {
         // External CSS/JS/font resources have no dedicated probe, so route
         // them through the external-stub probe to surface their status
@@ -2361,6 +2645,15 @@ export class Crawler extends EventEmitter {
         this.enqueueExternal(r.url);
       }
     }
+    // Iframes are documents, not subresources: crawling one is a full page
+    // fetch that extracts its own links, so it goes through the ordinary
+    // queue at the next depth rather than the leaf-node resource path.
+    if (this.config.crawlIframes) {
+      for (const f of parsed.iframes) {
+        if (f.isInternal) this.enqueue({ url: f.url, depth: depth + 1 });
+        else this.enqueueExternal(f.url);
+      }
+    }
   }
 
   /**
@@ -2414,8 +2707,16 @@ export class Crawler extends EventEmitter {
    * evenly-spaced slots. `maxRps <= 0` disables the gate (unlimited).
    */
   private async acquireRateSlot(): Promise<void> {
-    if (this.config.maxRps <= 0) return;
-    const minInterval = 1000 / this.config.maxRps;
+    // A robots.txt `Crawl-delay` means "one request every N seconds to this
+    // origin" — a *global* spacing, not a per-worker one. Applying it as a
+    // per-worker post-request sleep (as this used to) both violated the
+    // directive (20 workers → 20 requests per window) and multiplied crawl
+    // time by the worker count. Folding it into the dispatch gate honours it
+    // literally when the user opts in. Only reached when `respectCrawlDelay`
+    // is on — `robotsCrawlDelayMs` stays 0 otherwise.
+    const rpsInterval = this.config.maxRps > 0 ? 1000 / this.config.maxRps : 0;
+    const minInterval = Math.max(rpsInterval, this.robotsCrawlDelayMs);
+    if (minInterval <= 0) return;
     const now = Date.now();
     const scheduled = Math.max(now, this.nextDispatchAt);
     this.nextDispatchAt = scheduled + minInterval;
@@ -2781,6 +3082,19 @@ export class Crawler extends EventEmitter {
             : statusCode >= 400
               ? 'non-indexable:client-error'
               : 'indexable';
+        // Spider → Crawl "Store" for the row that queued this resource.
+        // Turning it off keeps the fetch — which is what validates the
+        // resource loads and what lets a stylesheet be mined for its
+        // `@font-face` targets — but drops the URL row, so the Internal
+        // tab isn't buried under a few hundred assets. Anything not
+        // enqueued as a resource has no `resourceRow` and is unaffected.
+        if (item.resourceRow && !this.storesResourceRow(item.resourceRow)) {
+          this.crawled++;
+          if (needsCssParse && nonHtmlBody) {
+            this.enqueueCssResources(nonHtmlBody, item.url, item.depth);
+          }
+          return;
+        }
         const nonHtmlUrlId = await this.dbCall<number>('upsertUrl', [
           {
             url: item.url,
@@ -2985,6 +3299,63 @@ export class Crawler extends EventEmitter {
         ? parsed.links
         : parsed.links.filter((l) => !l.rel?.includes('nofollow'));
 
+      // Spider → Crawl "Store" for the two hyperlink rows. `storableLinks`
+      // itself stays whole: traversal below and the page's outlink count
+      // both describe what the page declared, which doesn't change just
+      // because the user stopped recording one side of the graph.
+      const persistedLinks: DiscoveredLink[] = [];
+      if (this.config.storeInternalLinks || this.config.storeExternalLinks) {
+        for (const l of storableLinks) {
+          const keep = l.isInternal
+            ? this.config.storeInternalLinks
+            : this.config.storeExternalLinks;
+          if (keep) persistedLinks.push(l);
+        }
+      }
+      // Iframes join the link graph as `other`, so they show up in the
+      // page's Outlinks and a dead embed surfaces in Broken Links. They
+      // are excluded from `outlinks` above — an embed is not a hyperlink,
+      // and counting it would move every "Too Many Links" verdict.
+      if (this.config.storeIframes) {
+        for (const f of parsed.iframes) {
+          persistedLinks.push({
+            fromUrl: item.url,
+            toUrl: f.url,
+            type: 'other',
+            anchor: null,
+            altText: null,
+            rel: null,
+            target: null,
+            pathType: 'absolute',
+            linkPath: null,
+            linkPosition: 'content',
+            linkOrigin: 'html',
+            isInternal: f.isInternal,
+          });
+        }
+      }
+      // Malformed hrefs recorded verbatim. They can never resolve to a
+      // crawled URL, so the broken-link join reports every one of them —
+      // which is the whole point of asking for them.
+      if (this.config.crawlInvalidLinks) {
+        for (const raw of parsed.invalidLinks) {
+          persistedLinks.push({
+            fromUrl: item.url,
+            toUrl: raw,
+            type: 'other',
+            anchor: null,
+            altText: null,
+            rel: null,
+            target: null,
+            pathType: 'absolute',
+            linkPath: null,
+            linkPosition: 'content',
+            linkOrigin: 'html',
+            isInternal: true,
+          });
+        }
+      }
+
       const imagesMissingAlt = parsed.images.filter((img) => img.alt === null).length;
       // Phase 1b — Build the entire per-URL write payload up front and
       // ship it across the writer-worker boundary in one shot. The
@@ -3014,9 +3385,13 @@ export class Crawler extends EventEmitter {
           h5Count: parsed.h5Count,
           h6Count: parsed.h6Count,
           wordCount: parsed.wordCount,
-          canonical: parsed.canonical,
-          canonicalCount: parsed.canonicalCount,
-          canonicalHttp,
+          // Spider → Crawl "Store" columns. Indexability was already
+          // decided above from the live values, so switching a row off
+          // empties its tab without silently re-labelling pages as
+          // indexable — the verdict and the evidence stay in agreement.
+          canonical: this.config.storeCanonicals ? parsed.canonical : null,
+          canonicalCount: this.config.storeCanonicals ? parsed.canonicalCount : 0,
+          canonicalHttp: this.config.storeCanonicals ? canonicalHttp : null,
           metaRobots: parsed.metaRobots,
           xRobotsTag,
           contentType,
@@ -3058,15 +3433,22 @@ export class Crawler extends EventEmitter {
           schemaTypes: parsed.schemaTypes.length > 0 ? parsed.schemaTypes.join(', ') : null,
           schemaBlockCount: parsed.schemaBlockCount,
           schemaInvalidCount: parsed.schemaInvalidCount,
-          paginationNext: parsed.paginationNext,
-          paginationPrev: parsed.paginationPrev,
-          hreflangs: parsed.hreflangs.length > 0 ? JSON.stringify(parsed.hreflangs) : null,
-          hreflangCount: parsed.hreflangs.length,
+          paginationNext: this.config.storePagination ? parsed.paginationNext : null,
+          paginationPrev: this.config.storePagination ? parsed.paginationPrev : null,
+          hreflangs:
+            this.config.storeHreflang && parsed.hreflangs.length > 0
+              ? JSON.stringify(parsed.hreflangs)
+              : null,
+          hreflangCount: this.config.storeHreflang ? parsed.hreflangs.length : 0,
           videos: parsed.videos.length > 0 ? JSON.stringify(parsed.videos) : null,
-          amphtml: parsed.amphtml,
-          ampPage: parsed.ampPage,
-          ampValidationErrors: parsed.ampValidationErrors.length > 0
-            ? JSON.stringify(parsed.ampValidationErrors)
+          amphtml: this.config.storeAmp ? parsed.amphtml : null,
+          ampPage: this.config.storeAmp ? parsed.ampPage : false,
+          ampValidationErrors:
+            this.config.storeAmp && parsed.ampValidationErrors.length > 0
+              ? JSON.stringify(parsed.ampValidationErrors)
+              : null,
+          mobileAlternate: this.config.storeMobileAlternate
+            ? parsed.mobileAlternate
             : null,
           favicon: parsed.favicon,
           appleTouchIcon: parsed.appleTouchIcon,
@@ -3076,8 +3458,8 @@ export class Crawler extends EventEmitter {
           mixedContentCount: parsed.mixedContentCount,
           mixedContentActive: parsed.mixedContentActive,
           mixedContentPassive: parsed.mixedContentPassive,
-          metaRefresh: parsed.metaRefresh,
-          metaRefreshUrl: parsed.metaRefreshUrl,
+          metaRefresh: this.config.storeMetaRefresh ? parsed.metaRefresh : null,
+          metaRefreshUrl: this.config.storeMetaRefresh ? parsed.metaRefreshUrl : null,
           charset,
           extractionResults: parsed.extractionResults
             ? JSON.stringify(parsed.extractionResults)
@@ -3122,7 +3504,9 @@ export class Crawler extends EventEmitter {
           headings:
             parsed.headings.length > 0 ? JSON.stringify(parsed.headings) : null,
           serverHeader,
-          jsOnlyLinksCount: parsed.jsOnlyLinksCount,
+          jsOnlyLinksCount: this.config.storeUncrawlableLinks
+            ? parsed.jsOnlyLinksCount
+            : 0,
           textCodeRatio: parsed.textCodeRatio,
           fleschReadingEase: parsed.fleschReadingEase,
           fleschKincaidGrade: parsed.fleschKincaidGrade,
@@ -3140,8 +3524,8 @@ export class Crawler extends EventEmitter {
                   : 1_048_576,
             }
           : null,
-        links: storableLinks,
-        images: parsed.images,
+        links: persistedLinks,
+        images: this.config.storeImages ? parsed.images : [],
         fromDepth: item.depth,
       });
       // V2 Faz 1 — Persist render artefacts inline so they finish before
@@ -3210,20 +3594,43 @@ export class Crawler extends EventEmitter {
         return;
       }
 
-      if (this.config.scope === 'exact-url') {
-        // exact-url / single-page mode: do not follow any links.
+      if (this.config.scope === 'exact-url' || item.checkOnly) {
+        // exact-url / single-page mode: do not follow any links. Same for
+        // a URL pulled in only to check its status (see `checkOnly`).
       } else {
         const nextDepth = item.depth + 1;
-        for (const link of storableLinks) {
-          const inScope = isInScope(this.config.startUrl, link.toUrl, this.config.scope);
-          if (!inScope && !this.config.crawlExternal) continue;
-          // Wave 3 — nofollow follow toggle. By default nofollow links
-          // are stored (when `storeNofollowLinks` is on) but never
-          // recursed into. `followNofollow=true` opts out of the
-          // "respect nofollow" behaviour and treats them like any
-          // other link for the follow decision.
-          if (link.rel?.includes('nofollow') && !this.config.followNofollow) continue;
-          this.enqueue({ url: link.toUrl, depth: nextDepth });
+        // Spider → Crawl "Internal Hyperlinks / Crawl". Off leaves every
+        // other discovery route (sitemaps, canonicals, pagination) intact,
+        // so it audits a fixed set of pages without spidering outward.
+        if (this.config.crawlInternalLinks) {
+          for (const link of storableLinks) {
+            const inScope = isInScope(this.config.startUrl, link.toUrl, this.config.scope);
+            // With `scope: 'subfolder'`, "Check Links Outside of Start
+            // Folder" fetches out-of-folder targets once so their status
+            // is known, then stops — Screaming Frog draws the same line
+            // between checking a link and crawling through it.
+            const checkOnly =
+              !inScope &&
+              this.config.scope === 'subfolder' &&
+              this.config.checkLinksOutsideStartFolder &&
+              isInScope(this.config.startUrl, link.toUrl, 'subdomain');
+            if (!inScope && !checkOnly && !this.config.crawlExternal) continue;
+            // Wave 3 — nofollow follow toggle. By default nofollow links
+            // are stored (when `storeNofollowLinks` is on) but never
+            // recursed into. Turning the matching row on opts out of the
+            // "respect nofollow" behaviour and treats them like any other
+            // link for the follow decision — internal and external are
+            // separate switches because sites nofollow them for opposite
+            // reasons (crawl-budget shaping vs. not vouching for a third
+            // party).
+            if (link.rel?.includes('nofollow')) {
+              const followThis = link.isInternal
+                ? this.config.followNofollow
+                : this.config.followExternalNofollow;
+              if (!followThis) continue;
+            }
+            this.enqueue({ url: link.toUrl, depth: nextDepth, checkOnly });
+          }
         }
         // Wave 3 — Pagination follow toggle. rel=next/prev are part of
         // the standard discovery graph; the toggle exists to debug
@@ -3268,6 +3675,21 @@ export class Crawler extends EventEmitter {
             this.enqueue({ url: parsed.metaRefreshUrl, depth: nextDepth });
           }
         }
+        // Spider → Crawl rows that only add discovery. Each one enqueues a
+        // declared alternate — a translated page, an AMP variant, an m-dot
+        // URL — that an ordinary hyperlink often never points at, which is
+        // exactly why they go uncrawled without an explicit opt-in.
+        const followDeclared = (target: string | null): void => {
+          if (!target || target === item.url) return;
+          const inScope = isInScope(this.config.startUrl, target, this.config.scope);
+          if (!inScope && !this.config.crawlExternal) return;
+          this.enqueue({ url: target, depth: nextDepth });
+        };
+        if (this.config.crawlHreflang) {
+          for (const h of parsed.hreflangs) followDeclared(h.href);
+        }
+        if (this.config.crawlAmp) followDeclared(parsed.amphtml);
+        if (this.config.crawlMobileAlternate) followDeclared(parsed.mobileAlternate);
       }
     } catch (err) {
       // Stop / Clear: abandon silently. No row is written, no failure
@@ -3286,7 +3708,9 @@ export class Crawler extends EventEmitter {
       // and surface them as errors in the URL table. Re-enqueue lands
       // in a paused queue → resume() picks them up cleanly.
       if (this.paused && controller.signal.aborted) {
-        this.queue.add(() => this.fetchAndProcess(item)).catch(() => undefined);
+        this.queue
+          .add(() => this.fetchAndProcess(item), { priority: Crawler.priorityFor(item) })
+          .catch(() => undefined);
         return;
       }
       this.failed++;
@@ -3319,12 +3743,11 @@ export class Crawler extends EventEmitter {
       if (respTimeTimer) clearTimeout(respTimeTimer);
       // Politeness delay — applied per worker *after* each request so a
       // higher concurrency still honours a "one request every N ms per slot"
-      // contract on top of the global RPS cap. Takes the larger of the
-      // configured delay and any robots.txt Crawl-delay for the origin.
-      const politenessMs = Math.max(
-        this.config.crawlDelayMs,
-        this.config.respectRobotsTxt ? this.robotsCrawlDelayMs : 0,
-      );
+      // contract on top of the global RPS cap. This is the user's own
+      // `crawlDelayMs` setting only; a robots.txt Crawl-delay is enforced
+      // globally in `acquireRateSlot` instead, which is what the directive
+      // actually means (see there).
+      const politenessMs = this.config.crawlDelayMs;
       if (politenessMs > 0 && !this.stopped) {
         await sleep(politenessMs);
       }
@@ -3873,6 +4296,27 @@ export class Crawler extends EventEmitter {
     this.emitProgressNow();
   }
 
+  /**
+   * Emit a progress frame immediately, bypassing the 5 Hz throttle.
+   *
+   * Every terminal state transition (crawl finished, paused, resumed)
+   * MUST use this instead of `emitProgress()`. The throttle defers a
+   * too-soon call by up to 200 ms via `progressTrailingTimer`, and the
+   * `'done'` event that follows is emitted synchronously — so the host
+   * processes 'done' first, unbinds this crawler as the session's
+   * active one, and then DROPS the trailing `running: false` frame as
+   * if it came from a superseded crawler. The renderer never learns the
+   * crawl ended: the UI stays pinned to "Running" and Stop/Pause become
+   * no-ops because the crawler they target is already detached.
+   *
+   * The gap only opens when the final frame lands within 200 ms of the
+   * previous one — i.e. when post-crawl passes finish almost instantly,
+   * which is exactly what a re-Start on an already-complete crawl does.
+   */
+  private emitProgressFinal(): void {
+    this.emitProgressNow();
+  }
+
   private emitProgressNow(): void {
     this.lastProgressEmitTs = Date.now();
     if (this.progressTrailingTimer !== null) {

packages/core/src/html-parser.ts

@@ -331,6 +331,27 @@ export interface ParsedPage {
    * separately in `images`.
    */
   resources: DiscoveredResource[];
+  /**
+   * `<iframe src>` targets, normalised and deduped. Embedded documents,
+   * not subresources — crawling one means a full page fetch — so they
+   * are kept out of `resources` and out of the outlink count. Gated by
+   * the Spider → Crawl "iframes" row.
+   */
+  iframes: DiscoveredResource[];
+  /**
+   * Raw `<a href>` values that could not be parsed as an http(s) URL —
+   * malformed markup, not deliberate `mailto:` / `tel:` / `#` links.
+   * Capped at 200 per page. Surfaced only when `crawlInvalidLinks` is on.
+   */
+  invalidLinks: string[];
+  /**
+   * Separate-URL mobile version: `<link rel="alternate" media="only
+   * screen and (max-width: …)" href="…">`. Null on responsive sites.
+   * `media` must actually constrain width — a bare `rel="alternate"`
+   * without it is an RSS feed or an hreflang entry, both handled
+   * elsewhere.
+   */
+  mobileAlternate: string | null;
   hasNoindex: boolean;
   hasNofollow: boolean;
 }
@@ -719,6 +740,46 @@ export function parseHtml(
     const kind = rel === 'modulepreload' ? 'js' : PRELOAD_AS_TO_KIND[as];
     if (kind) addResource($el.attr('href'), kind);
   });
+  // Media — `<video>` / `<audio>` and the `<source>` children they own.
+  // Filed under `'other'`, which no other producer emits, so the crawler
+  // can key the Spider → Crawl "Media" row off the kind alone. `<source>`
+  // is matched through its parent because the same tag inside `<picture>`
+  // is an image and already went through `imageMap`.
+  $('video[src], audio[src]').each((_, el) => addResource($(el).attr('src'), 'other'));
+  $('video > source[src], audio > source[src]').each((_, el) =>
+    addResource($(el).attr('src'), 'other'),
+  );
+
+  // Iframes — embedded documents. Deduped separately from `resources` so
+  // that crawling media never drags in every embed, and so they can be
+  // stored as links rather than as subresources.
+  const iframeMap = new Map<string, DiscoveredResource>();
+  $('iframe[src]').each((_, el) => {
+    const raw = ($(el).attr('src') ?? '').trim();
+    if (!raw || raw.startsWith('data:') || raw.startsWith('about:')) return;
+    const normalized = normalizeUrl(raw, pageUrl, opts.urlRewrites);
+    if (!normalized || !/^https?:/.test(normalized)) return;
+    if (iframeMap.has(normalized)) return;
+    iframeMap.set(normalized, {
+      url: normalized,
+      kind: 'html',
+      isInternal: isSameHost(pageUrl, normalized, opts),
+    });
+  });
+
+  // Mobile alternate (m-dot). `rel="alternate"` alone is ambiguous — it
+  // also marks RSS feeds and hreflang entries — so the `media` attribute
+  // must actually express a width constraint before we claim it.
+  let mobileAlternate: string | null = null;
+  $('link[rel~="alternate"][media][href]').each((_, el) => {
+    if (mobileAlternate) return;
+    const $el = $(el);
+    if (($el.attr('hreflang') ?? '').trim()) return;
+    const media = ($el.attr('media') ?? '').toLowerCase();
+    if (!/max-(device-)?width/.test(media)) return;
+    const href = normalizeUrl(($el.attr('href') ?? '').trim(), pageUrl, opts.urlRewrites);
+    if (href && /^https?:/.test(href)) mobileAlternate = href;
+  });
 
   // Pagination — `<link rel="next">` / `<link rel="prev">`. Resolved to
   // absolute via normalizeUrl so the values are comparable to the URLs
@@ -1060,13 +1121,31 @@ export function parseHtml(
     }
   });
 
+  // Hrefs that survive being an `<a href>` but not being a URL: doubled
+  // schemes, stray whitespace inside the authority, bare `:::`. Kept raw
+  // and deduped so the crawler can surface them under the Spider → Crawl
+  // "Crawl Invalid Links" row instead of dropping them silently. Schemes
+  // that are *meant* to be non-navigable (mailto/tel/javascript/#) are
+  // not malformed and never land here.
+  const NON_NAVIGABLE_SCHEME = /^(mailto|tel|sms|callto|javascript|data|blob|about|file|ftp|news|whatsapp|skype|viber|intent|market|itms-apps|geo|magnet|bitcoin|webcal|feed|irc|ssh|slack|tg|zoommtg|msteams):/i;
+  const invalidLinkSet = new Set<string>();
   const linkMap = new Map<string, DiscoveredLink>();
   $('a[href]').each((_, el) => {
     const href = $(el).attr('href');
     if (!href) return;
     const normalized = normalizeUrl(href, pageUrl, opts.urlRewrites);
-    if (!normalized) return;
-    if (!/^https?:/.test(normalized)) return;
+    if (!normalized || !/^https?:/.test(normalized)) {
+      const raw = href.trim();
+      if (
+        raw &&
+        !raw.startsWith('#') &&
+        !NON_NAVIGABLE_SCHEME.test(raw) &&
+        invalidLinkSet.size < 200
+      ) {
+        invalidLinkSet.add(raw.slice(0, 2000));
+      }
+      return;
+    }
     if (linkMap.has(normalized)) return;
 
     const $el = $(el);
@@ -1471,6 +1550,9 @@ export function parseHtml(
     links: [...linkMap.values()],
     images: [...imageMap.values()],
     resources: [...resourceMap.values()],
+    iframes: [...iframeMap.values()],
+    invalidLinks: [...invalidLinkSet],
+    mobileAlternate,
     hasNoindex,
     hasNofollow,
   };

packages/core/src/xml-export.ts

@@ -96,6 +96,7 @@ const XML_COLUMNS: (keyof CrawlUrlRow)[] = [
   'paginationPrev',
   'paginationSequenceBreak',
   'amphtml',
+  'mobileAlternate',
   'favicon',
   'appleTouchIcon',
   'androidIcon',

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/db",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/db/src/migrations.ts

@@ -1942,6 +1942,48 @@ const MIGRATIONS: Migration[] = [
       );
     },
   },
+  {
+    version: 81,
+    name: 'add_url_changed_flag',
+    // Re-crawling a URL that already has data (Re-Spider, or a second
+    // Start on the same site) now records whether the page actually
+    // moved since the previous fetch, so the URL table can flag it.
+    //
+    // Set by the urls upsert itself — comparing `excluded.*` against the
+    // stored row inside the ON CONFLICT clause costs nothing, whereas a
+    // read-then-compare in JS would add a SELECT per fetched URL.
+    //
+    // Rows written before this column existed have no baseline to be
+    // measured against, so the default of 0 (unchanged) is correct.
+    up: (db) => {
+      const cols = db
+        .prepare('PRAGMA table_info(urls)')
+        .all() as unknown as { name: string }[];
+      if (!cols.some((c) => c.name === 'changed')) {
+        db.exec('ALTER TABLE urls ADD COLUMN changed INTEGER NOT NULL DEFAULT 0');
+      }
+    },
+  },
+  {
+    version: 82,
+    name: 'add_mobile_alternate',
+    // Separate-URL (m-dot) sites declare their mobile version with
+    // `<link rel="alternate" media="only screen and (max-width: …)">`.
+    // The Spider → Crawl matrix can now both follow and store it, so the
+    // target needs somewhere to live.
+    //
+    // Left NULL for rows crawled before this column existed: absent and
+    // "declared nothing" are the same thing for every consumer, and a
+    // backfill would need the page bodies we no longer parse.
+    up: (db) => {
+      const cols = db
+        .prepare('PRAGMA table_info(urls)')
+        .all() as unknown as { name: string }[];
+      if (!cols.some((c) => c.name === 'mobile_alternate')) {
+        db.exec('ALTER TABLE urls ADD COLUMN mobile_alternate TEXT');
+      }
+    },
+  },
 ];
 
 export function runMigrations(db: DatabaseSync): void {

packages/db/src/project-db.ts

@@ -170,6 +170,8 @@ interface UrlRowDb {
   hreflang_count: number;
   videos: string | null;
   amphtml: string | null;
+  /** Separate-URL mobile version from `<link rel="alternate" media=…>`. */
+  mobile_alternate: string | null;
   /** 1 when the page declares `<html ⚡>` / `<html amp>`, else 0. */
   amp_page: number;
   /** JSON string array of AMP smoke-validator error codes. Empty when
@@ -282,6 +284,7 @@ interface UrlRowDb {
   schema_missing_recommended: number;
   heading_order_violations: number;
   subresource_request_count: number;
+  changed: number;
 }
 
 interface ImageRowDb {
@@ -360,6 +363,8 @@ export interface UpsertUrlInput {
   /** JSON-stringified array of `VideoEntry` objects, or null. */
   videos?: string | null;
   amphtml?: string | null;
+  /** Separate-URL mobile version from `<link rel="alternate" media=…>`. */
+  mobileAlternate?: string | null;
   ampPage?: boolean;
   /** JSON-stringified array of AMP validator error codes. */
   ampValidationErrors?: string | null;
@@ -447,6 +452,49 @@ export interface UpsertUrlInput {
   corsAllowHeaders?: string | null;
 }
 
+/**
+ * "Did this page actually move since the last time we fetched it?", as an
+ * expression for the urls upsert's ON CONFLICT clause. `urls.x` is the
+ * stored row, `excluded.x` the incoming one.
+ *
+ * Only fields a user would call a change are compared — response time,
+ * depth and crawl timestamps drift on every fetch and would make the flag
+ * meaningless. `IS NOT` (not `<>`) so NULL→value and value→NULL count as
+ * changes instead of evaluating to NULL.
+ *
+ * A stored row with no `status_code` is a stub — a link target discovered
+ * but never fetched, or the seed row written before the first request goes
+ * out. There is no previous fetch to compare against, so it can never be
+ * "changed"; this is also what keeps a first crawl from flagging every row.
+ *
+ * `content_hash` covers HTML body drift on its own (it hashes the
+ * normalised token stream, so reformatting alone doesn't trip it). Assets
+ * never get one, so they fall back to Content-Length.
+ */
+const CHANGED_FLAG_SQL = `CASE
+      WHEN urls.status_code IS NULL THEN 0
+      WHEN urls.status_code IS NOT excluded.status_code
+        OR urls.content_hash IS NOT excluded.content_hash
+        OR (urls.content_hash IS NULL AND excluded.content_hash IS NULL
+            AND urls.content_length IS NOT excluded.content_length)
+        OR urls.title IS NOT excluded.title
+        OR urls.meta_description IS NOT excluded.meta_description
+        OR urls.h1 IS NOT excluded.h1
+        OR urls.word_count IS NOT excluded.word_count
+        OR urls.canonical IS NOT excluded.canonical
+        OR urls.meta_robots IS NOT excluded.meta_robots
+        OR urls.x_robots_tag IS NOT excluded.x_robots_tag
+        OR urls.indexability IS NOT excluded.indexability
+        OR urls.redirect_target IS NOT excluded.redirect_target
+        OR urls.content_type IS NOT excluded.content_type
+        OR urls.outlinks IS NOT excluded.outlinks
+        OR urls.images_count IS NOT excluded.images_count
+        OR urls.hreflangs IS NOT excluded.hreflangs
+        OR urls.schema_types IS NOT excluded.schema_types
+      THEN 1
+      ELSE 0
+    END`;
+
 const UPSERT_URL_SQL = `
   INSERT INTO urls (
     url, content_kind, status_code, status_text, indexability, indexability_reason,
@@ -488,7 +536,7 @@ const UPSERT_URL_SQL = `
     schema_duplicate_ids, schema_unknown_types, schema_missing_required,
     schema_missing_recommended,
     heading_order_violations, subresource_request_count,
-    amp_page, amp_validation_errors
+    amp_page, amp_validation_errors, mobile_alternate
   ) VALUES (
     :url, :content_kind, :status_code, :status_text, :indexability, :indexability_reason,
     :title, :title_length, :meta_description, :meta_description_length,
@@ -529,7 +577,7 @@ const UPSERT_URL_SQL = `
     :schema_duplicate_ids, :schema_unknown_types, :schema_missing_required,
     :schema_missing_recommended,
     :heading_order_violations, :subresource_request_count,
-    :amp_page, :amp_validation_errors
+    :amp_page, :amp_validation_errors, :mobile_alternate
   )
   ON CONFLICT(url) DO UPDATE SET
     content_kind = excluded.content_kind,
@@ -662,6 +710,8 @@ const UPSERT_URL_SQL = `
     subresource_request_count = excluded.subresource_request_count,
     amp_page = excluded.amp_page,
     amp_validation_errors = excluded.amp_validation_errors,
+    mobile_alternate = excluded.mobile_alternate,
+    changed = ${CHANGED_FLAG_SQL},
     crawled_at = CURRENT_TIMESTAMP
   RETURNING id
 `;
@@ -804,28 +854,47 @@ export class ProjectDb {
   }
 
   reset(): void {
-    // All deletes wrapped in a single transaction so the writer's
-    // WAL only fsyncs once at the end instead of after each
-    // statement. On a 100K-URL project this drops Clear time from
-    // ~2-3 s to ~300-500 ms because the per-statement fsync is the
-    // dominant cost in autocommit mode (10 statements × ~150-300 ms
-    // each on Windows NTFS with WAL).
+    // Foreign-key enforcement is suspended for the wipe. It is the single
+    // dominant cost here — measured on a 6.6K-URL / 820K-link project,
+    // Clear runs in 17.3 s with FKs on and 0.31 s with them off. Every
+    // table in an FK relationship is emptied in the same transaction, so
+    // there is no constraint left to violate when it commits; paying for
+    // per-row parent/child checks on the way to an empty database buys
+    // nothing.
     //
-    // Trade-off: SQLite's "DELETE FROM table" truncate optimisation
-    // (which skips per-row work and just frees the b-tree pages)
-    // only fires in autocommit mode. Inside a transaction it
-    // degrades to a row-by-row delete. For tables in the millions
-    // that would be slower, but for the tables touched here the
-    // fsync dominance still makes the transaction-wrapped variant
-    // faster overall.
+    // A previous revision blamed per-statement fsync and wrapped the
+    // deletes in a transaction to amortise it, accepting the loss of
+    // SQLite's truncate optimisation as the price. That reasoning was
+    // chasing the wrong variable: with FKs off, autocommit (318 ms) and
+    // transaction (311 ms) are indistinguishable. The transaction is kept
+    // anyway — it costs nothing and makes Clear atomic, so an interrupted
+    // wipe can't leave a half-cleared project behind.
+    //
+    // `PRAGMA foreign_keys` is a no-op inside a transaction, so it has to
+    // be toggled outside one; the restore is in a `finally` so a failed
+    // wipe can't leave this connection without FK enforcement. Only the
+    // writer connection is affected (Clear is dispatched to the writer
+    // worker), and the crawler is already stopped by the time it runs.
+    //
+    // This depends on `reset()` being entered with `txDepth === 0`. Every
+    // caller is top-level today. If one is ever wrapped in
+    // `runInTransaction`, the pragma below silently does nothing and Clear
+    // quietly returns to its 17 s behaviour — still correct, just slow
+    // again, which is exactly the kind of regression that is expensive to
+    // rediscover. Dropping FK enforcement stays safe only because all four
+    // tables carrying a foreign key (links, headers, image_usages,
+    // url_sources) are deleted explicitly below, so nothing relies on
+    // ON DELETE CASCADE to be reached.
     //
     // `host_certs` is included so a Clear after a TLS-probe pass
     // doesn't leave stale certificate rows that the next crawl
     // would skip re-probing for (the unprobedHttpsHosts() filter
     // excludes hosts with an existing row).
-    this.runInTransaction(() => {
-      this.db.exec(
-        `DELETE FROM image_usages;
+    this.db.exec('PRAGMA foreign_keys = OFF');
+    try {
+      this.runInTransaction(() => {
+        this.db.exec(
+          `DELETE FROM image_usages;
          DELETE FROM images;
          DELETE FROM links;
          DELETE FROM headers;
@@ -844,8 +913,11 @@ export class ProjectDb {
          DELETE FROM ai_results;
          DELETE FROM seo_results;
          DELETE FROM gsc_inspection_results;`,
-      );
-    });
+        );
+      });
+    } finally {
+      this.db.exec('PRAGMA foreign_keys = ON');
+    }
     // Reclaim freed pages back to the OS so the project file size
     // doesn't keep ballooning across crawl + clear cycles. Cheap
     // (single passive checkpoint) compared to a full VACUUM, but
@@ -1150,6 +1222,27 @@ export class ProjectDb {
     ).map((r) => r.url);
   }
 
+  /**
+   * Internal non-HTML rows — images, CSS, JS, fonts, media. Used to seed the
+   * crawler's subresource budget quota when resuming, so a resumed crawl
+   * doesn't hand out a fresh allowance on top of what it already spent.
+   *
+   * Approximate by design: a `.pdf` reached through an ordinary hyperlink
+   * counts here too even though it was never admitted as a subresource. That
+   * only makes the quota slightly more conservative after a resume, which is
+   * the safe direction for a rail whose job is to stop assets from crowding
+   * out pages.
+   */
+  countSubresourceUrls(): number {
+    return (
+      this.db
+        .prepare(
+          "SELECT COUNT(*) AS c FROM urls WHERE is_external = 0 AND content_kind <> 'html'",
+        )
+        .get() as { c: number }
+    ).c;
+  }
+
   countCrawledUrls(): number {
     return (
       this.db
@@ -1197,16 +1290,24 @@ export class ProjectDb {
     });
   }
 
+  /**
+   * Flag a URL as about to be re-fetched.
+   *
+   * This deliberately leaves the stored response in place. It used to
+   * blank `status_code` / `indexability` so the row read as "fetching",
+   * but that threw away the only baseline the change detector has: the
+   * upsert decides `changed` by diffing the incoming row against the
+   * stored one, and a blanked row looks like a never-fetched stub, so
+   * every Re-Spider came back unflagged. Keeping the old values also
+   * means a re-fetch that fails outright no longer wipes good data off
+   * the row — it stays visible until a real response replaces it.
+   *
+   * The `changed` reset is what makes the flag describe THIS re-spider
+   * rather than whatever an earlier one found.
+   */
   markUrlForRecrawl(id: number): void {
     this.db
-      .prepare(
-        `UPDATE urls SET
-           status_code = NULL,
-           status_text = NULL,
-           indexability = 'indexable',
-           indexability_reason = NULL
-         WHERE id = ? AND is_external = 0`,
-      )
+      .prepare('UPDATE urls SET changed = 0 WHERE id = ? AND is_external = 0')
       .run(id);
   }
 
@@ -1221,17 +1322,72 @@ export class ProjectDb {
       const placeholders = slice.map(() => '?').join(',');
       this.db
         .prepare(
-          `UPDATE urls SET
-             status_code = NULL,
-             status_text = NULL,
-             indexability = 'indexable',
-             indexability_reason = NULL
+          `UPDATE urls SET changed = 0
            WHERE id IN (${placeholders}) AND is_external = 0`,
         )
         .run(...slice);
     }
   }
 
+  /** Crawl depth recorded for a URL, or null if it isn't in the project.
+   *  Lets a Re-Spider re-queue a page at the depth it was found at
+   *  instead of pretending every re-fetched URL is a root-level seed. */
+  getUrlDepth(url: string): number | null {
+    const row = this.db.prepare('SELECT depth FROM urls WHERE url = ?').get(url) as
+      | { depth: number | null }
+      | undefined;
+    return row?.depth ?? null;
+  }
+
+  /** Clear every change flag. Runs at the start of a refresh crawl so the
+   *  green markers describe that run and not a previous one. */
+  clearChangedFlags(): void {
+    this.db.exec('UPDATE urls SET changed = 0 WHERE changed != 0');
+  }
+
+  /** Same, restricted to the URLs a targeted Re-Spider is about to
+   *  re-fetch — markers on pages it won't touch stay as they were. */
+  clearChangedFlagsForUrls(urls: string[]): void {
+    if (urls.length === 0) return;
+    const CHUNK = 500;
+    for (let i = 0; i < urls.length; i += CHUNK) {
+      const slice = urls.slice(i, i + CHUNK);
+      const placeholders = slice.map(() => '?').join(',');
+      this.db
+        .prepare(`UPDATE urls SET changed = 0 WHERE url IN (${placeholders})`)
+        .run(...slice);
+    }
+  }
+
+  /** How many URLs the last refresh crawl found different. Drives the
+   *  "N changed" summary the status bar shows after a re-spider. */
+  countChangedUrls(): number {
+    return (
+      this.db.prepare('SELECT COUNT(*) AS c FROM urls WHERE changed = 1').get() as {
+        c: number;
+      }
+    ).c;
+  }
+
+  /**
+   * Every internal URL that already holds a response, as re-crawl seeds.
+   *
+   * Used when the user presses Start again on a site that is already
+   * crawled: rather than resuming a finished crawl (which has nothing
+   * left to do), every known page is re-fetched so the report reflects
+   * the site as it stands now. Ordered by depth so the re-crawl walks
+   * the site top-down, same as a first crawl.
+   */
+  getCrawledInternalUrls(): { url: string; depth: number }[] {
+    return this.db
+      .prepare(
+        `SELECT url, depth FROM urls
+         WHERE is_external = 0 AND status_code IS NOT NULL
+         ORDER BY depth ASC, id ASC`,
+      )
+      .all() as unknown as { url: string; depth: number }[];
+  }
+
   deleteUrls(ids: number[]): void {
     if (ids.length === 0) return;
     // Chunk to stay under SQLite's bound-parameter limit; wrap in one
@@ -1284,6 +1440,38 @@ export class ProjectDb {
     return out;
   }
 
+  /**
+   * Look up URL strings for a batch of ids, preserving the caller's id
+   * order rather than the table's.
+   *
+   * The grid's context menu passes ids in the order the rows appear on
+   * screen — which, once the user has sorted or filtered, has nothing to
+   * do with ascending id. Copying "8 URLs" in a different order than the
+   * eight rows the user is looking at is a small betrayal, so callers
+   * that care about display order use this instead of
+   * {@link getUrlsByIds}. Ids with no matching row are skipped.
+   */
+  getUrlsByIdsInOrder(ids: number[]): string[] {
+    if (ids.length === 0) return [];
+    const CHUNK = 500;
+    const byId = new Map<number, string>();
+    const unique = [...new Set(ids)];
+    for (let i = 0; i < unique.length; i += CHUNK) {
+      const slice = unique.slice(i, i + CHUNK);
+      const placeholders = slice.map(() => '?').join(',');
+      const rows = this.db
+        .prepare(`SELECT id, url FROM urls WHERE id IN (${placeholders})`)
+        .all(...slice) as unknown as { id: number; url: string }[];
+      for (const r of rows) byId.set(r.id, r.url);
+    }
+    const out: string[] = [];
+    for (const id of ids) {
+      const url = byId.get(id);
+      if (url !== undefined) out.push(url);
+    }
+    return out;
+  }
+
   getUrlRowById(id: number): { url: string; depth: number; isExternal: number } | null {
     const row = this.db
       .prepare('SELECT url, depth, is_external FROM urls WHERE id = ?')
@@ -1388,10 +1576,17 @@ export class ProjectDb {
       : '';
     const discovered = this.db
       .prepare(
+        // Hyperlinks only. The drain exists to catch pages a depth or
+        // robots race skipped, and the non-hyperlink rows the Spider →
+        // Crawl matrix can add — stored iframes, recorded malformed hrefs
+        // — are declarations the user chose to record, not pages awaiting
+        // a crawl. Every row predating that matrix is a hyperlink, so the
+        // filter changes nothing for existing projects.
         `SELECT l.to_url AS url, MIN(u.depth) + 1 AS depth
          FROM links l
          JOIN urls u ON l.from_url_id = u.id
          WHERE l.is_internal = 1
+           AND l.type = 'hyperlink'
            AND l.to_url NOT IN (SELECT url FROM urls)
            ${followFilter}
          GROUP BY l.to_url`,
@@ -1478,6 +1673,7 @@ export class ProjectDb {
       amphtml: input.amphtml ?? null,
       amp_page: input.ampPage ? 1 : 0,
       amp_validation_errors: input.ampValidationErrors ?? null,
+      mobile_alternate: input.mobileAlternate ?? null,
       favicon: input.favicon ?? null,
       mixed_content_count: input.mixedContentCount ?? 0,
       mixed_content_active: input.mixedContentActive ?? 0,
@@ -7776,6 +7972,7 @@ export class ProjectDb {
     hreflangCount: r.hreflang_count,
     videos: r.videos ?? null,
     amphtml: r.amphtml,
+    mobileAlternate: r.mobile_alternate ?? null,
     ampPage: r.amp_page === 1,
     ampValidationErrors: r.amp_validation_errors,
     boilerplateCoverage: r.boilerplate_coverage,
@@ -7875,6 +8072,7 @@ export class ProjectDb {
     schemaMissingRecommended: r.schema_missing_recommended ?? 0,
     headingOrderViolations: r.heading_order_violations ?? 0,
     subresourceRequestCount: r.subresource_request_count ?? 0,
+    changed: r.changed === 1,
     crawledAt: r.crawled_at,
   });
 

packages/shared-types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@freecrawl/shared-types",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/shared-types/src/crawl.ts

@@ -500,6 +500,15 @@ export interface CrawlUrlRow {
    *  objects for the video-sitemap variant, or null. */
   videos: string | null;
   amphtml: string | null;
+  /**
+   * Separate-URL mobile version declared by
+   * `<link rel="alternate" media="only screen and (max-width: …)">`.
+   * The m-dot pattern: desktop page points here, the mobile page points
+   * back with a canonical. Null on responsive sites, which is most of
+   * them — a non-null value without a reciprocal canonical is the
+   * classic broken m-dot setup.
+   */
+  mobileAlternate: string | null;
   /** True when the page declares `<html ⚡>` / `<html amp>`. AMP-for-Ads
    *  / AMP-for-Email variants (`⚡4ads`, `⚡4email`) count too. */
   ampPage: boolean;
@@ -614,6 +623,17 @@ export interface CrawlUrlRow {
   clusterId: number;
   /** Number of pages in this URL's near-duplicate cluster (1 = singleton). */
   clusterSize: number;
+  /**
+   * True when the most recent fetch returned different data than the one
+   * before it — status, body content, title, description, H1, canonical,
+   * robots directives, indexability, word/outlink/image counts, hreflang
+   * or schema types. Set only on a re-crawl of a URL that already had a
+   * response (Re-Spider, or Start pressed again on a crawled site); a
+   * first crawl leaves every row false because there is no baseline.
+   * Reset for the whole table when a refresh crawl begins, so it always
+   * describes the latest run.
+   */
+  changed: boolean;
   crawledAt: string;
 }
 
@@ -689,6 +709,24 @@ export interface CrawlConfig {
   deviceMode: 'desktop' | 'mobile';
   followRedirects: boolean;
   respectRobotsTxt: boolean;
+  /**
+   * Honour a robots.txt `Crawl-delay` directive. Default `false`.
+   *
+   * `Crawl-delay` is not part of the Robots Exclusion Protocol (RFC 9309);
+   * Google ignores it outright and Screaming Frog does not implement it.
+   * Sites routinely publish values tuned for 2005-era bulk scrapers —
+   * `Crawl-delay: 30` is common — which, honoured literally, turns a
+   * 500-URL crawl from ~40 s into ~4 hours. Because the value is almost
+   * never calibrated to the site's actual capacity, obeying it by default
+   * costs the user enormously and buys the server nothing it asked for in
+   * a standardised way.
+   *
+   * Off by default; users who need it (own-site audits under a strict
+   * ops policy, shared hosting) can switch it on. `Allow` / `Disallow`
+   * are unaffected — those are the actual standard and are always
+   * honoured when `respectRobotsTxt` is on.
+   */
+  respectCrawlDelay: boolean;
   crawlExternal: boolean;
   /**
    * Crawl internal page subresources so they appear in the Internal tab as
@@ -1048,6 +1086,110 @@ export interface CrawlConfig {
    * the meta-refresh URL is also enqueued like a redirect target.
    */
   followJsRedirects: boolean;
+
+  /* ------------------------------------------------------------------ *
+   * Spider → Crawl matrix (Screaming Frog "Configuration → Spider →
+   * Crawl"). Every link type the parser can discover gets two
+   * independent switches:
+   *
+   *   • CRAWL — enqueue the target so it is fetched and gets its own row.
+   *   • STORE — persist what the page declared, so the corresponding tab
+   *             / column / table is populated.
+   *
+   * The two are orthogonal: "store but don't crawl" lists a declaration
+   * without spending a request on it (the classic image-inventory case),
+   * while "crawl but don't store" fetches a resource for its side effects
+   * — validating that it loads, mining a stylesheet for its `@font-face`
+   * targets — without cluttering the Internal tab with hundreds of rows.
+   *
+   * The crawl side of Images / CSS / JavaScript / External Links /
+   * Canonicals / Pagination / Meta Refresh is carried by the pre-existing
+   * `checkImages`, `checkCss`, `checkJs`, `crawlExternal`,
+   * `followCanonicals`, `followPaginationLinks` and `followJsRedirects`
+   * fields — the panel binds to those rather than shadowing them, so
+   * there is exactly one source of truth per switch.
+   *
+   * Every default below reproduces the behaviour FreeCrawl had before the
+   * matrix existed, so opening an older project changes nothing.
+   *
+   * Screaming Frog's SWF row is deliberately absent: Flash reached
+   * end-of-life in 2020 and no live site still ships `.swf` objects.
+   * ------------------------------------------------------------------ */
+
+  /** Persist `<img>` declarations into the images table (and the Images
+   *  tab). Independent of `checkImages`, which decides whether the image
+   *  bytes are actually requested. */
+  storeImages: boolean;
+  /** Fetch internal `<video>` / `<audio>` / their `<source>` targets so
+   *  they land in the Internal tab with status + size. Off by default —
+   *  media files are large and rarely the point of an SEO crawl. */
+  crawlMedia: boolean;
+  /** Keep the crawled media resource's URL row. Only has an effect while
+   *  `crawlMedia` is on — an un-fetched media file has no row to keep. */
+  storeMedia: boolean;
+  /** Keep the crawled stylesheet's URL row. Turn off to mine CSS for
+   *  `@font-face` / `url()` targets without listing every stylesheet. */
+  storeCss: boolean;
+  /** Keep the crawled script's URL row. */
+  storeJs: boolean;
+  /** Follow internal `<a href>` hyperlinks. Off turns the crawl into a
+   *  single-page fetch that still records everything the page declares. */
+  crawlInternalLinks: boolean;
+  /** Persist internal hyperlinks in the link graph — inlinks / outlinks,
+   *  anchor-text reports, link score, internal broken links. */
+  storeInternalLinks: boolean;
+  /** Persist outbound hyperlinks in the link graph. */
+  storeExternalLinks: boolean;
+  /** Persist `<link rel="canonical">` (and its HTTP-header form) so the
+   *  Canonicals tab and the canonical issue filters have data. */
+  storeCanonicals: boolean;
+  /** Persist `<link rel="next">` / `<link rel="prev">`. */
+  storePagination: boolean;
+  /** Enqueue every `<link rel="alternate" hreflang>` target. Off by
+   *  default: hreflang alternates are usually in scope anyway and get
+   *  discovered through ordinary links. */
+  crawlHreflang: boolean;
+  /** Persist hreflang declarations (Hreflang tab + reciprocity audit). */
+  storeHreflang: boolean;
+  /** Enqueue the `<link rel="amphtml">` target so the AMP page is crawled
+   *  as its own URL. */
+  crawlAmp: boolean;
+  /** Persist AMP declarations and the smoke-validator findings. */
+  storeAmp: boolean;
+  /** Persist `<meta http-equiv="refresh">` and its parsed target. */
+  storeMetaRefresh: boolean;
+  /** Enqueue internal `<iframe src>` documents as their own URLs. */
+  crawlIframes: boolean;
+  /** Record `<iframe src>` targets in the link graph so they show up in
+   *  the page's outlinks. Excluded from the outlink *count* — an embed is
+   *  not a hyperlink. */
+  storeIframes: boolean;
+  /** Enqueue the mobile-alternate target — the `<link rel="alternate"
+   *  media="only screen and (max-width: …)">` that separate-URL (m-dot)
+   *  sites use to point at their mobile version. */
+  crawlMobileAlternate: boolean;
+  /** Persist the mobile-alternate URL on the page row. */
+  storeMobileAlternate: boolean;
+  /** Persist the count of links a search engine cannot follow —
+   *  `<a>` with no href but an onclick, `href="javascript:…"`, and
+   *  `href="#"` placeholders wired to a handler. */
+  storeUncrawlableLinks: boolean;
+  /** With `scope: 'subfolder'`, still request links that point outside
+   *  the start folder (one hop, no recursion) so their status code is
+   *  known. Off leaves them undiscovered entirely. */
+  checkLinksOutsideStartFolder: boolean;
+  /** Follow `rel="nofollow"` on links pointing to another host. Split
+   *  from `followNofollow`, which now governs internal links only. */
+  followExternalNofollow: boolean;
+  /** Record links whose href cannot be parsed as a URL — unencoded
+   *  spaces, stray angle brackets, doubled schemes. Off drops them
+   *  silently; on surfaces them in the link graph as broken. */
+  crawlInvalidLinks: boolean;
+  /** Crawl the URLs listed inside discovered XML sitemaps, not just
+   *  record them. Turns `discoverSitemaps` from a reporting aid into a
+   *  discovery source, which is what finds orphan pages. */
+  crawlLinkedSitemaps: boolean;
+
   /**
    * Wave 6 — Per-pass crawl-analysis toggles. Each post-crawl pass
    * can be independently disabled when the user knows the data isn't
@@ -1987,6 +2129,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   deviceMode: 'desktop',
   followRedirects: true,
   respectRobotsTxt: true,
+  respectCrawlDelay: false,
   crawlExternal: false,
   checkImages: true,
   checkCss: true,
@@ -2040,6 +2183,32 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
   followPaginationLinks: true,
   followNofollow: false,
   followJsRedirects: false,
+  // Spider → Crawl matrix. Each value reproduces the behaviour that was
+  // hardcoded before the matrix existed, so the defaults are a no-op.
+  storeImages: true,
+  crawlMedia: false,
+  storeMedia: false,
+  storeCss: true,
+  storeJs: true,
+  crawlInternalLinks: true,
+  storeInternalLinks: true,
+  storeExternalLinks: true,
+  storeCanonicals: true,
+  storePagination: true,
+  crawlHreflang: false,
+  storeHreflang: true,
+  crawlAmp: false,
+  storeAmp: true,
+  storeMetaRefresh: true,
+  crawlIframes: false,
+  storeIframes: false,
+  crawlMobileAlternate: false,
+  storeMobileAlternate: true,
+  storeUncrawlableLinks: true,
+  checkLinksOutsideStartFolder: true,
+  followExternalNofollow: false,
+  crawlInvalidLinks: false,
+  crawlLinkedSitemaps: false,
   analyseInlinks: true,
   analyseLinkScore: true,
   analyseRedirectChains: true,

packages/shared-types/src/ipc.ts

@@ -116,6 +116,8 @@ export const IPC = {
   urlCertInfo: 'urls:cert-info',
   urlContextMenu: 'url:context-menu',
   urlBulkContextMenu: 'url:bulk-context-menu',
+  urlGridContextMenu: 'url:grid-context-menu',
+  urlGridCopy: 'url:grid-copy',
   imagesQuery: 'images:query',
   brokenLinksQuery: 'broken-links:query',
   overviewGet: 'overview:get',
@@ -935,6 +937,35 @@ export interface UrlBulkContextMenuInput {
   urlIds: number[];
 }
 
+/**
+ * Which of the URL grid's three selection layers the right-click landed
+ * in. Decides the menu's copy item and what the row-semantic actions
+ * (Re-Spider / Remove / Export) apply to.
+ */
+export type UrlGridMenuScope = 'row' | 'rows' | 'cells' | 'columns';
+
+export interface UrlGridContextMenuInput {
+  scope: UrlGridMenuScope;
+  /** Rows the row-semantic actions apply to, in display order. */
+  urlIds: number[];
+  /** The right-clicked row's URL — used by the single-URL actions. */
+  url: string;
+  /** Selected cell count, for the 'cells' scope menu label. */
+  cellCount: number;
+  /** Selected column count, for the 'columns' scope menu label. */
+  columnCount: number;
+}
+
+/**
+ * Pushed back to the renderer when the grid context menu's Copy item is
+ * clicked. The copy can't run in main — only the renderer knows what a
+ * cell says — and it's pushed rather than returned from the popup call
+ * because Electron documents no ordering between an item's `click` and
+ * `popup()`'s close callback, so a request/response shape would drop the
+ * copy whenever close won the race.
+ */
+export type UrlGridCopyEvent = 'copy-selection';
+
 export interface ConfirmClearResult {
   confirmed: boolean;
   skipNext: boolean;
@@ -1692,6 +1723,8 @@ export interface FreeCrawlApi {
   urlCertInfo(input: UrlCertInfoInput): Promise<UrlCertInfoResult>;
   urlContextMenu(input: UrlContextMenuInput): Promise<void>;
   urlBulkContextMenu(input: UrlBulkContextMenuInput): Promise<void>;
+  urlGridContextMenu(input: UrlGridContextMenuInput): Promise<void>;
+  onUrlGridCopy(cb: (event: UrlGridCopyEvent) => void): () => void;
   imagesQuery(input: ImagesQueryInput): Promise<ImagesQueryResult>;
   brokenLinksQuery(input: BrokenLinksQueryInput): Promise<BrokenLinksQueryResult>;
   overviewGet(): Promise<OverviewCounts>;