fix: HTTP/2, page size, dead external domain and CORS checks (#35)

packages/core/src/alpn-probe.ts

@@ -0,0 +1,40 @@
+import { isIP } from 'node:net';
+import { connect } from 'node:tls';
+import type { LookupFunction } from 'node:net';
+
+/**
+ * Open one TLS connection offering `h2` and `http/1.1` via ALPN and return
+ * the protocol the server selected. undici's fetch does not expose the
+ * negotiated protocol on a Response, so a direct handshake is the only
+ * reliable HTTP/2 signal.
+ *
+ * Returns `'h2'` or `'http/1.1'`; a server that ignores ALPN negotiates
+ * HTTP/1.1. Returns `null` when the handshake fails or times out.
+ * Certificate errors are ignored: the cert audit is the TLS probe's job.
+ */
+export async function probeAlpn(
+  host: string,
+  port = 443,
+  timeoutMs = 10_000,
+  lookup?: LookupFunction,
+): Promise<string | null> {
+  return await new Promise<string | null>((resolve) => {
+    const socket = connect({
+      host,
+      port,
+      // SNI must be a hostname; Node rejects an IP literal here.
+      ...(isIP(host) ? {} : { servername: host }),
+      rejectUnauthorized: false,
+      ALPNProtocols: ['h2', 'http/1.1'],
+      ...(lookup ? { lookup } : {}),
+    });
+    const finish = (result: string | null): void => {
+      clearTimeout(timer);
+      socket.destroy();
+      resolve(result);
+    };
+    const timer = setTimeout(() => finish(null), timeoutMs);
+    socket.once('secureConnect', () => finish(socket.alpnProtocol === 'h2' ? 'h2' : 'http/1.1'));
+    socket.once('error', () => finish(null));
+  });
+}

packages/core/src/crawler.ts

@@ -38,11 +38,13 @@ import {
   type DigestChallenge,
   formatFetchError,
   initHttpClient,
+  directModeLookup,
   MOBILE_USER_AGENT,
   parseDigestChallenge,
   redactUrlSecrets,
 } from './http-client.js';
 import { setActiveDnsHook } from './dns-resolver.js';
+import { probeAlpn } from './alpn-probe.js';
 import { discoverSitemapUrls, fetchSitemaps, type SitemapEntry } from './sitemap.js';
 import { runJsonExtractionRules } from './extraction.js';
 import { SessionCookieJar } from './cookie-jar.js';
@@ -526,6 +528,8 @@ export class Crawler extends EventEmitter {
   /** Effective proxy resolved in the constructor, reused by the start-URL
    *  probe so it doesn't reset the global dispatcher back to direct. */
   private readonly resolvedProxy: string;
+  /** Negotiated ALPN protocol per `host:port`, probed once per host. */
+  private readonly alpnByHost = new Map<string, Promise<string | null>>();
 
   /**
    * Optional freeze-watchdog hook. The desktop main process injects a
@@ -3600,6 +3604,26 @@ export class Crawler extends EventEmitter {
     );
   }
 
+  private negotiatedProtocol(url: string): Promise<string | null> {
+    // The probe is a direct TLS connect. Under a proxy it would bypass the
+    // proxy and expose the crawler's real address, so skip it. Otherwise it
+    // resolves DNS with the same resolver as the fetch.
+    const lookup = directModeLookup();
+    if (!lookup) return Promise.resolve(null);
+    const { host, hostname, port } = new URL(url);
+    let probe = this.alpnByHost.get(host);
+    if (!probe) {
+      probe = probeAlpn(
+        hostname.replace(/^\[|\]$/g, ''),
+        Number(port) || 443,
+        Math.max(2_000, this.config.requestTimeoutMs / 2),
+        lookup as never,
+      ).catch(() => null);
+      this.alpnByHost.set(host, probe);
+    }
+    return probe;
+  }
+
   private async fetchAndProcess(item: QueueItem): Promise<void> {
     if (this.stopped) return;
     this.setOp(`crawl:fetch:${item.url}`);
@@ -3644,6 +3668,10 @@ export class Crawler extends EventEmitter {
       // Response Codes > 3xx view and `redirect_target` column are
       // populated correctly. When followRedirects is on we enqueue the
       // target, producing a full chain across multiple crawl passes.
+      // Start the per-host ALPN probe beside the fetch. It has its own
+      // timeout, outside this request's AbortController, and is awaited
+      // only at write time, after the body is read.
+      const alpnProbe = item.url.startsWith('https:') ? this.negotiatedProtocol(item.url) : null;
       const res = await this.fetchWithRetry(item.url, controller.signal);
 
       const responseTimeMs = Date.now() - t0;
@@ -3770,9 +3798,29 @@ export class Crawler extends EventEmitter {
         ? normalizeUrl(canonicalHttpRaw, item.url, this.urlRewrites)
         : null;
 
-      // HTTP protocol — heuristic via Alt-Svc (best-effort; undici doesn't
-      // surface the actually-negotiated ALPN protocol on Response).
-      const httpProtocol = detectHttpProtocol(res.headers.get('alt-svc'));
+      // HTTP protocol. undici doesn't surface the negotiated ALPN protocol
+      // on Response, so HTTPS hosts get one ALPN handshake each; an Alt-Svc
+      // h3 advertisement still wins. A failed or skipped probe stores null
+      // (unknown), not 'http/1.1'. Plain-HTTP URLs keep the Alt-Svc reading.
+      const altSvcProtocol = detectHttpProtocol(res.headers.get('alt-svc'));
+      const httpProtocolProbe =
+        altSvcProtocol === 'h3' || !alpnProbe ? Promise.resolve(altSvcProtocol) : alpnProbe;
+      // Awaited at write time. Stop (and Clear, which stops first) rejects
+      // the wait, so the catch branch's stopped-abort guard skips the write
+      // instead of resurrecting a cleared row. A timeout abort does not:
+      // the body is already read.
+      const httpProtocol = (): Promise<string | null> =>
+        new Promise((resolve, reject) => {
+          const onAbort = (): void => {
+            if (this.stopped) reject(new Error('crawler stopped'));
+          };
+          controller.signal.addEventListener('abort', onAbort, { once: true });
+          void httpProtocolProbe.then((value) => {
+            controller.signal.removeEventListener('abort', onAbort);
+            if (this.stopped) reject(new Error('crawler stopped'));
+            else resolve(value);
+          });
+        });
 
       // Query-string length — characters after the first `?` (no `?` → 0).
       const qIdx = item.url.indexOf('?');
@@ -3882,7 +3930,7 @@ export class Crawler extends EventEmitter {
             cookiesInsecure: cookieSummary.insecureCount,
             cookiesNoHttpOnly: cookieSummary.noHttpOnlyCount,
             cookiesNoSameSite: cookieSummary.noSameSiteCount,
-            httpProtocol,
+            httpProtocol: await httpProtocol(),
             queryStringLength,
             keepAlive,
             serverHeader,
@@ -4016,7 +4064,7 @@ export class Crawler extends EventEmitter {
             cookiesInsecure: cookieSummary.insecureCount,
             cookiesNoHttpOnly: cookieSummary.noHttpOnlyCount,
             cookiesNoSameSite: cookieSummary.noSameSiteCount,
-            httpProtocol,
+            httpProtocol: await httpProtocol(),
             queryStringLength,
             keepAlive,
             serverHeader,
@@ -4061,7 +4109,10 @@ export class Crawler extends EventEmitter {
         return;
       }
       const body = decodeBody(capRead.bytes, contentType);
-      const bodyLength = parseIntSafe(contentLengthHeader) ?? capRead.bytes.length;
+      // Decoded size, not Content-Length: under gzip/br the header is the
+      // compressed transfer size, so the page-size checks would depend on
+      // the server's compression and framing.
+      const bodyLength = capRead.bytes.length;
 
       // V2 Faz 1 — JavaScript rendering pass. When `renderingMode === 'js'`
       // and the host injected a Playwright renderer, fetch the post-JS DOM
@@ -4390,7 +4441,7 @@ export class Crawler extends EventEmitter {
           cookiesInsecure: cookieSummary.insecureCount,
           cookiesNoHttpOnly: cookieSummary.noHttpOnlyCount,
           cookiesNoSameSite: cookieSummary.noSameSiteCount,
-          httpProtocol,
+          httpProtocol: await httpProtocol(),
           queryStringLength,
           keepAlive,
           renderBlockingCount: parsed.renderBlockingCount,

packages/core/src/http-client.ts

@@ -15,6 +15,11 @@ let sharedResilientLookup: ReturnType<typeof createResilientLookup> | null = nul
 /** True while the global dispatcher is a direct Agent (no proxy). Lenient TLS
  *  retry is only offered in this mode so we never silently bypass a proxy. */
 let directMode = false;
+/** The crawler's DNS lookup while connections go direct to origins, or
+ *  null under a proxy (direct sockets would bypass it). */
+export function directModeLookup(): ReturnType<typeof createResilientLookup> | null {
+  return directMode ? sharedResilientLookup : null;
+}
 /** Lazily-built dispatcher with certificate verification disabled — used ONLY
  *  as a last-resort retry when an EXTERNAL link fails TLS chain verification
  *  (incomplete chain that browsers complete via AIA fetching but Node does

packages/db/src/project-db.ts

@@ -10048,9 +10048,34 @@ const ISSUE_WRITE_BATCH = 2_000;
  * pass (`issueMaterialisationStale`) instead of showing 0 for those
  * checks until the next crawl.
  */
-export const ISSUE_MATERIALISATION_VERSION = '2';
+export const ISSUE_MATERIALISATION_VERSION = '3';
 const ISSUE_VERSION_META_KEY = 'urlsIssuesVersion';
 
+/**
+ * External URL whose probe failed: an HTTP error status, or a network
+ * failure, which is stored with a NULL status and the `formatFetchError`
+ * chain as status_text. Only real network error codes and the timeout /
+ * abort text count, so local request errors, SSRF-guard skips, robots
+ * blocks and unprobed stubs do not. 401/403/407/429/999 are excluded: an
+ * origin that answers auth walls, bot blocks or rate limits is alive
+ * (999 is LinkedIn's bot block).
+ */
+const EXTERNAL_PROBE_FAILED_SQL = `((status_code >= 400 AND status_code NOT IN (401, 403, 407, 429, 999))
+  OR (status_code IS NULL AND (${[
+    'ENOTFOUND',
+    'EAI_AGAIN',
+    'ECONNREFUSED',
+    'ECONNRESET',
+    'ETIMEDOUT',
+    'EHOSTUNREACH',
+    'ENETUNREACH',
+    'UND_ERR_CONNECT_TIMEOUT',
+    'UND_ERR_HEADERS_TIMEOUT',
+    'aborted',
+  ]
+    .map((code) => `status_text LIKE '%${code}%'`)
+    .join(' OR ')})))`;
+
 export const EXPENSIVE_ISSUE_DEFINITIONS: ReadonlyArray<readonly [string, string]> = [
   [
     'issues:dead-external-domain',
@@ -10059,23 +10084,27 @@ export const EXPENSIVE_ISSUE_DEFINITIONS: ReadonlyArray<readonly [string, string
     // index — one pass over `links` and one over the external rows. The
     // previous correlated `EXISTS (… JOIN …)` re-ran the host grouping
     // per page and scaled with pages × outlinks.
+    // Only links whose own target failed are flagged; a link to a URL
+    // that answered on a mostly-dead host is not.
     `is_external = 0 AND content_kind = 'html'
      AND id IN (
        SELECT l.from_url_id
          FROM links l
         WHERE l.is_internal = 0
           AND l.to_url IN (
-            SELECT eu.url
-              FROM urls eu
-             WHERE eu.is_external = 1
-               AND ${urlHostSql('eu.url')} IN (
+            SELECT url
+              FROM urls
+             WHERE is_external = 1
+               AND ${EXTERNAL_PROBE_FAILED_SQL}
+               AND ${urlHostSql('url')} IN (
                  SELECT host_grouped FROM (
                    SELECT
                      ${urlHostSql('url')} AS host_grouped,
                      COUNT(*) AS total,
-                     SUM(CASE WHEN status_code >= 400 THEN 1 ELSE 0 END) AS errors
+                     SUM(CASE WHEN ${EXTERNAL_PROBE_FAILED_SQL} THEN 1 ELSE 0 END) AS errors
                    FROM urls
-                   WHERE is_external = 1 AND status_code IS NOT NULL
+                   WHERE is_external = 1
+                     AND (status_code IS NOT NULL OR ${EXTERNAL_PROBE_FAILED_SQL})
                    GROUP BY host_grouped
                    HAVING total >= 3 AND CAST(errors AS REAL) / total >= 0.8
                  )
@@ -11020,7 +11049,7 @@ export function categoryWhereClause(
       // hides content from Googlebot's link-graph crawl heuristics.
       return `is_external = 0 AND content_kind = 'html' AND folder_depth > 4`;
     case 'issues:http2-not-supported':
-      // Origin advertises only HTTP/1.1 (no Alt-Svc / no h2 token).
+      // Origin negotiated HTTP/1.1 over TLS ALPN, or is plain HTTP.
       // Informational — modern browsers still work fine, but HTTP/2 is
       // a free CWV win on multi-resource pages.
       return `is_external = 0 AND content_kind = 'html'
@@ -11412,8 +11441,8 @@ export function categoryWhereClause(
     case 'issues:cors-wildcard-with-credentials':
       // `Access-Control-Allow-Origin: *` paired with
       // `Access-Control-Allow-Credentials: true`. Browsers reject this
-      // combination per the CORS spec, but misconfigured servers reach
-      // it and any working XSS could exfiltrate cookies cross-origin.
+      // combination per the Fetch spec, so credentialed cross-origin
+      // requests fail. A broken config, not a credential leak.
       return `is_external = 0 AND content_kind = 'html'
               AND TRIM(cors_allow_origin) = '*'
               AND cors_allow_credentials = 1`;

packages/shared-types/src/issue-severity.ts

@@ -50,7 +50,6 @@ const CRITICAL: UrlCategory[] = [
   'issues:ssl-cert-expired',
   'issues:page-too-large-critical',
   'issues:page-empty',
-  'issues:cors-wildcard-with-credentials',
   'issues:http-not-https',
   'issues:mixed-content-active',
 ];
@@ -81,6 +80,9 @@ const WARNING: UrlCategory[] = [
   'issues:hsts-missing',
   'issues:x-frame-options-missing',
   'issues:x-content-type-options-missing',
+  // Browsers reject a credentialed response with ACAO `*`, so this is a
+  // broken CORS config (credentialed requests fail), not a data leak.
+  'issues:cors-wildcard-with-credentials',
   'issues:structured-data-invalid',
   'issues:pagination-broken',
   'issues:hreflang-x-default-missing',