Keep link-check-only external rows out of the Overview

External links that were only HEAD-checked (Crawl External off) now get
a link_check_only flag, saved with the crawl. The Overview leaves those
rows out, since those sites were not crawled; the External tab still
lists them. Off-site images and crawled pages are not flagged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/crawl_db.py

@@ -129,6 +129,13 @@ def init_crawl_tables():
         except sqlite3.OperationalError:
             pass  # Column already exists
 
+        # Migration: link_check_only, rows for external links that were only
+        # HEAD-checked (not crawled), which the Overview leaves out
+        try:
+            cursor.execute('ALTER TABLE crawled_urls ADD COLUMN link_check_only BOOLEAN DEFAULT 0')
+        except sqlite3.OperationalError:
+            pass  # Column already exists
+
         # Links table
         cursor.execute('''
             CREATE TABLE IF NOT EXISTS crawl_links (
@@ -306,7 +313,8 @@ def save_url_batch(crawl_id, urls):
                     url_data.get('response_time'),
                     url_data.get('javascript_rendered', False),
                     url_data.get('error_type'),
-                    url_data.get('render_time')
+                    url_data.get('render_time'),
+                    url_data.get('link_check_only', False)
                 )
                 rows.append(row)
 
@@ -318,8 +326,8 @@ def save_url_batch(crawl_id, urls):
                     meta_tags, og_tags, twitter_tags, json_ld, analytics, images,
                     hreflang, schema_org, redirects, linked_from,
                     external_links, internal_links, response_time, javascript_rendered,
-                    error_type, render_time
-                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    error_type, render_time, link_check_only
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ''', rows)
 
             print(f"Saved {len(urls)} URLs to database for crawl {crawl_id}")

src/crawler.py

@@ -1483,9 +1483,9 @@ class WebCrawler:
                               if l.get('placement') != 'image' and not l.get('is_internal')
                               and urlparse(l['target_url']).scheme in ('http', 'https')]
             if external_links:
-                self._head_check_links(external_links, depth)
+                self._head_check_links(external_links, depth, link_check_only=True)
 
-    def _head_check_links(self, links, depth=0):
+    def _head_check_links(self, links, depth=0, link_check_only=False):
         """HEAD-check link targets that won't be fetched as crawl targets.
 
         Uses a per-crawl cache so the same URL is only checked once even if
@@ -1494,7 +1494,9 @@ class WebCrawler:
 
         URLs that won't be fetched as full crawl targets get a result row
         synthesized from the HEAD response so they appear in the Images /
-        External tabs without downloading the body.
+        External tabs without downloading the body. link_check_only marks
+        rows for links to sites that were not crawled, which the Overview
+        leaves out.
         """
         # Statuses set here are journalled at the end: these links were already
         # announced by the link manager, so the UI only learns about a HEAD
@@ -1540,7 +1542,8 @@ class WebCrawler:
 
             # Skip synthesis for URLs that will be (or were) crawled for real
             if not self._should_crawl_url(url):
-                self._record_head_result(url, link['target_status'], content_type, size, depth)
+                self._record_head_result(url, link['target_status'], content_type, size, depth,
+                                         link_check_only)
 
         batch = to_check[:50]
         with ThreadPoolExecutor(max_workers=min(HEAD_CHECK_WORKERS, len(batch))) as pool:
@@ -1554,7 +1557,8 @@ class WebCrawler:
         if links:
             self.event_log.emit_many('link_update', links)
 
-    def _record_head_result(self, url, status_code, content_type, size, depth):
+    def _record_head_result(self, url, status_code, content_type, size, depth,
+                            link_check_only=False):
         """Add a result row for a URL from its HEAD response (no body download)"""
         sources = self.link_manager.get_source_pages(url)
 
@@ -1568,6 +1572,7 @@ class WebCrawler:
             result['size'] = size
             result['is_internal'] = self.link_manager.is_internal(url)
             result['linked_from'] = sources
+            result['link_check_only'] = link_check_only
             self.crawl_results.append(result)
             self.stats['crawled'] += 1
             # a result counts as discovered (issue #94)

tests/fixture_tests.py

@@ -201,6 +201,12 @@ def test_external_images():
         link_statuses = {l['target_url']: l['target_status'] for l in crawler.link_manager.all_links}
         result('external: link status shown in Links tab',
                link_statuses.get(f'{off}/gone') == 404, str(link_statuses))
+        result('external: link rows flagged link_check_only (left out of Overview)',
+               ext is not None and ext.get('link_check_only') is True)
+        result('external: offsite image rows not flagged (stay in Overview)',
+               rows.get(f'{off}/out.png', {}).get('link_check_only') is False)
+        result('external: crawled pages not flagged',
+               not rows[f'http://127.0.0.1:{site}/'].get('link_check_only'))
         result('external: non-http links ignored',
                not any(u.startswith('javascript:') for u in rows))
 

web/static/js/app.js

@@ -523,6 +523,13 @@ function updateCrawlData(data) {
     }
 }
 
+// Rows for external links that were only HEAD-checked (Crawl External off)
+// stay out of the Overview unless the External filter is picked: those sites
+// were not crawled. The External tab always lists them.
+function overviewUrls() {
+    return crawlState.urls.filter(url => !url.link_check_only);
+}
+
 // Rebuild the URL virtual scrollers from crawlState.urls, honoring the
 // active sidebar filter if one is set
 function refreshUrlTables() {
@@ -532,7 +539,7 @@ function refreshUrlTables() {
     }
 
     if (virtualScrollers.overview) {
-        virtualScrollers.overview.setData(crawlState.urls);
+        virtualScrollers.overview.setData(overviewUrls());
     }
     if (virtualScrollers.internal) {
         virtualScrollers.internal.setData(crawlState.urls.filter(url => url.is_internal));
@@ -1041,7 +1048,7 @@ function addUrlToTable(urlData) {
     crawlState.urls.push(urlData);
 
     // Update virtual scrollers with new data
-    if (virtualScrollers.overview) {
+    if (virtualScrollers.overview && !urlData.link_check_only) {
         virtualScrollers.overview.appendData([urlData]);
     }
 
@@ -1221,7 +1228,7 @@ function clearActiveFilters() {
 
     // Reset all virtual scrollers to show full data
     if (virtualScrollers.overview) {
-        virtualScrollers.overview.setData(crawlState.urls);
+        virtualScrollers.overview.setData(overviewUrls());
     }
     if (virtualScrollers.internal) {
         const internalUrls = crawlState.urls.filter(url => url.is_internal);
@@ -1243,7 +1250,9 @@ function filterVirtualScrollerData(scrollerName, filterType) {
     let filteredData = crawlState.urls;
 
     // Apply base filter for internal/external tables
-    if (scrollerName === 'internal') {
+    if (scrollerName === 'overview' && filterType !== 'external') {
+        filteredData = overviewUrls();
+    } else if (scrollerName === 'internal') {
         filteredData = filteredData.filter(url => url.is_internal);
     } else if (scrollerName === 'external') {
         filteredData = filteredData.filter(url => !url.is_internal);
@@ -1338,6 +1347,9 @@ function updateFilterCounts() {
         if (isInternalURL(url.url)) counts.internal++;
         else counts.external++;
 
+        // The other filters match what the Overview shows under them
+        if (url.link_check_only) return;
+
         // Count by status code
         const statusCode = parseInt(url.status_code);
         if (statusCode >= 200 && statusCode < 300) counts['2xx']++;
@@ -1369,46 +1381,46 @@ function updateStatusCodesTable(filterType = null) {
 
     // Count status codes, respecting current filter
     const statusCounts = {};
-    let filteredUrls = crawlState.urls;
+    let filteredUrls = filterType === 'external' ? crawlState.urls : overviewUrls();
 
     // Apply filter if specified
     if (filterType === 'internal') {
-        filteredUrls = crawlState.urls.filter(url => isInternalURL(url.url));
+        filteredUrls = filteredUrls.filter(url => isInternalURL(url.url));
     } else if (filterType === 'external') {
-        filteredUrls = crawlState.urls.filter(url => !isInternalURL(url.url));
+        filteredUrls = filteredUrls.filter(url => !isInternalURL(url.url));
     } else if (filterType === '2xx') {
-        filteredUrls = crawlState.urls.filter(url => {
+        filteredUrls = filteredUrls.filter(url => {
             const status = parseInt(url.status_code);
             return status >= 200 && status < 300;
         });
     } else if (filterType === '3xx') {
-        filteredUrls = crawlState.urls.filter(url => {
+        filteredUrls = filteredUrls.filter(url => {
             const status = parseInt(url.status_code);
             return status >= 300 && status < 400;
         });
     } else if (filterType === '4xx') {
-        filteredUrls = crawlState.urls.filter(url => {
+        filteredUrls = filteredUrls.filter(url => {
             const status = parseInt(url.status_code);
             return status >= 400 && status < 500;
         });
     } else if (filterType === '5xx') {
-        filteredUrls = crawlState.urls.filter(url => {
+        filteredUrls = filteredUrls.filter(url => {
             const status = parseInt(url.status_code);
             return status >= 500;
         });
     } else if (filterType === 'no_response') {
-        filteredUrls = crawlState.urls.filter(url =>
+        filteredUrls = filteredUrls.filter(url =>
             (url.status_code === 0 || url.status_code === null || url.status_code === undefined)
             && url.error_type !== 'file_too_large'
         );
     } else if (filterType === 'html') {
-        filteredUrls = crawlState.urls.filter(url => (url.content_type || '').includes('html'));
+        filteredUrls = filteredUrls.filter(url => (url.content_type || '').includes('html'));
     } else if (filterType === 'css') {
-        filteredUrls = crawlState.urls.filter(url => (url.content_type || '').includes('css'));
+        filteredUrls = filteredUrls.filter(url => (url.content_type || '').includes('css'));
     } else if (filterType === 'js') {
-        filteredUrls = crawlState.urls.filter(url => (url.content_type || '').includes('javascript'));
+        filteredUrls = filteredUrls.filter(url => (url.content_type || '').includes('javascript'));
     } else if (filterType === 'images') {
-        filteredUrls = crawlState.urls.filter(url => (url.content_type || '').includes('image'));
+        filteredUrls = filteredUrls.filter(url => (url.content_type || '').includes('image'));
     }
 
     let totalUrls = filteredUrls.length;