mirror of
https://github.com/kemalai/FreeCrawl-SEO-Tool.git
synced 2026-10-18 00:39:38 +02:00
feat: crawl images/CSS/JS/font resources into Internal tab
This commit is contained in:
kemalai committed
1 parent
4c7e3b3fdf
commit
473395ca92
7 files changed
+362
-86
No files matched your search
@@ -84,6 +84,9 @@ interface FormState {
|
||||
followRedirects: boolean;
|
||||
respectRobotsTxt: boolean;
|
||||
crawlExternal: boolean;
|
||||
checkImages: boolean;
|
||||
checkCss: boolean;
|
||||
checkJs: boolean;
|
||||
storeNofollowLinks: boolean;
|
||||
discoverSitemaps: boolean;
|
||||
// requests
|
||||
@@ -399,6 +402,9 @@ function configToForm(c: CrawlConfig): FormState {
|
||||
followRedirects: c.followRedirects,
|
||||
respectRobotsTxt: c.respectRobotsTxt,
|
||||
crawlExternal: c.crawlExternal,
|
||||
checkImages: c.checkImages,
|
||||
checkCss: c.checkCss,
|
||||
checkJs: c.checkJs,
|
||||
storeNofollowLinks: c.storeNofollowLinks,
|
||||
discoverSitemaps: c.discoverSitemaps,
|
||||
userAgent: c.userAgent,
|
||||
@@ -598,6 +604,9 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
followRedirects: form.followRedirects,
|
||||
respectRobotsTxt: form.respectRobotsTxt,
|
||||
crawlExternal: form.crawlExternal,
|
||||
checkImages: form.checkImages,
|
||||
checkCss: form.checkCss,
|
||||
checkJs: form.checkJs,
|
||||
storeNofollowLinks: form.storeNofollowLinks,
|
||||
discoverSitemaps: form.discoverSitemaps,
|
||||
userAgent: form.userAgent.trim() || config.userAgent,
|
||||
@@ -1265,6 +1274,27 @@ function CrawlerPanel({ form, update }: PanelProps) {
|
||||
info="Probe outbound links to other hosts (HEAD only) so the Broken Links view catches dead externals. Screaming Frog: 'External Links' (Configuration → Spider → Crawl)."
|
||||
example="On for outbound link audits; off for fast internal-only crawls."
|
||||
/>
|
||||
<Bool
|
||||
label={t('settingsPanels.crawler.checkImages', { defaultValue: 'Check Images' })}
|
||||
checked={form.checkImages}
|
||||
onChange={(v) => update('checkImages', v)}
|
||||
info="Fetch internal <img> resources (incl. srcset / <picture> sources) so they appear in the Internal tab with their own status code, content type, and size. Each counts toward Max URLs. Screaming Frog: 'Check Images' (Configuration → Spider → Crawl)."
|
||||
example="On (default) so the Internal tab shows images, not just HTML; off for HTML-only crawls."
|
||||
/>
|
||||
<Bool
|
||||
label={t('settingsPanels.crawler.checkCss', { defaultValue: 'Check CSS' })}
|
||||
checked={form.checkCss}
|
||||
onChange={(v) => update('checkCss', v)}
|
||||
info="Fetch internal <link rel=stylesheet> resources so they appear in the Internal tab with status code, content type, and size. Each counts toward Max URLs. Screaming Frog: 'Check CSS' (Configuration → Spider → Crawl)."
|
||||
example="On (default); off for HTML-only crawls."
|
||||
/>
|
||||
<Bool
|
||||
label={t('settingsPanels.crawler.checkJs', { defaultValue: 'Check JavaScript' })}
|
||||
checked={form.checkJs}
|
||||
onChange={(v) => update('checkJs', v)}
|
||||
info="Fetch internal <script src> resources so they appear in the Internal tab with status code, content type, and size. Each counts toward Max URLs. Screaming Frog: 'Check JavaScript' (Configuration → Spider → Crawl)."
|
||||
example="On (default); off for HTML-only crawls."
|
||||
/>
|
||||
<Bool
|
||||
label={t('settingsPanels.crawler.storeNofollow', { defaultValue: 'Follow / Store Nofollow Links' })}
|
||||
checked={form.storeNofollowLinks}
|
||||
|
||||
@@ -383,6 +383,9 @@
|
||||
"followRedirects": "Always Follow Redirects",
|
||||
"respectRobotsTxt": "Respect robots.txt",
|
||||
"crawlExternal": "Check External Links",
|
||||
"checkImages": "Check Images",
|
||||
"checkCss": "Check CSS",
|
||||
"checkJs": "Check JavaScript",
|
||||
"storeNofollow": "Follow / Store Nofollow Links",
|
||||
"storeNofollowHint": "Default off — Screaming Frog 'Respect Nofollow' behaviour",
|
||||
"discoverSitemaps": "Auto-Discover XML Sitemaps",
|
||||
|
||||
@@ -382,6 +382,9 @@
|
||||
"followRedirects": "Yönlendirmeleri Daima Takip Et",
|
||||
"respectRobotsTxt": "robots.txt'ye Uy",
|
||||
"crawlExternal": "Harici Linkleri Kontrol Et",
|
||||
"checkImages": "Görselleri Kontrol Et",
|
||||
"checkCss": "CSS'i Kontrol Et",
|
||||
"checkJs": "JavaScript'i Kontrol Et",
|
||||
"storeNofollow": "Nofollow Linkleri Takip Et / Sakla",
|
||||
"storeNofollowHint": "Varsayılan kapalı — Screaming Frog 'Respect Nofollow' davranışı",
|
||||
"discoverSitemaps": "XML Sitemap'leri Otomatik Keşfet",
|
||||
|
||||
+131
-11
@@ -23,7 +23,7 @@ import {
|
||||
compileUrlRegexRewrites,
|
||||
toEscapedFragmentUrl,
|
||||
} from './url-utils.js';
|
||||
import { parseHtml, estimatePixelWidth } from './html-parser.js';
|
||||
import { parseHtml, estimatePixelWidth, type ParsedPage } from './html-parser.js';
|
||||
import { analyseCookies, extractSetCookies } from './cookies.js';
|
||||
import { loadRobots, type RobotsChecker } from './robots.js';
|
||||
import {
|
||||
@@ -903,18 +903,19 @@ export class Crawler extends EventEmitter {
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-crawl HEAD probe over internal images so the DB knows their byte
|
||||
* size for the "Large Image" issue. HEAD-only — no body download — so
|
||||
* cost is one round-trip per image. Concurrency is bounded by the same
|
||||
* setting as the main crawl; failures are silent (probe_status stays
|
||||
* null and the issue check skips them rather than false-positives).
|
||||
* Post-crawl HEAD probe over images (internal + external) so the DB knows
|
||||
* their byte size for the "Large Image" issue and their status for the
|
||||
* "Broken Image src" issue (incl. dead external/CDN images). HEAD-only —
|
||||
* no body download — so cost is one round-trip per image. Concurrency is
|
||||
* bounded by the same setting as the main crawl; failures are silent
|
||||
* (probe_status stays null and the issue check skips them).
|
||||
*/
|
||||
private async runImageSizeProbes(): Promise<void> {
|
||||
if (!this.config.probeImageSizes) return;
|
||||
if (this.stopped) return;
|
||||
let unprobed: { id: number; src: string }[] = [];
|
||||
try {
|
||||
unprobed = this.db.unprobedInternalImages(20_000);
|
||||
unprobed = this.db.unprobedImages(20_000);
|
||||
} catch (err) {
|
||||
this.emit(
|
||||
'info',
|
||||
@@ -2040,6 +2041,92 @@ export class Crawler extends EventEmitter {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue the internal subresources of a freshly-parsed HTML page so they
|
||||
* become their own `urls` rows (Screaming Frog "Check Images / CSS / JS").
|
||||
* Gated per-type by the `checkImages` / `checkCss` / `checkJs` config flags.
|
||||
*
|
||||
* Resources are enqueued at the *referring page's* depth (not depth+1) so a
|
||||
* page within the depth limit always has its assets crawled. Only internal
|
||||
* resources are followed; the standard `enqueue` guards (maxUrls, robots,
|
||||
* include/exclude filters, dedupe) still apply. Non-HTML responses never
|
||||
* reach the link-follow path, so resources stay leaf nodes.
|
||||
*/
|
||||
private enqueueResources(parsed: ParsedPage, depth: number): void {
|
||||
if (this.config.checkImages) {
|
||||
for (const img of parsed.images) {
|
||||
if (img.isInternal) this.enqueue({ url: img.src, depth });
|
||||
// External images: status/size are filled by the post-crawl image
|
||||
// probe (images table), which now covers external images too — so
|
||||
// a dead external/CDN image still trips "Broken Image src".
|
||||
}
|
||||
}
|
||||
// Whether a resource kind should be crawled at all. Fonts ride with the
|
||||
// CSS toggle (they're declared by CSS / font preloads); images obey the
|
||||
// image toggle so `<link rel=preload as=image>` matches `<img>` behaviour.
|
||||
const shouldFollow = (kind: ContentKind): boolean => {
|
||||
if (kind === 'css') return this.config.checkCss;
|
||||
if (kind === 'js') return this.config.checkJs;
|
||||
if (kind === 'image') return this.config.checkImages;
|
||||
if (kind === 'font') return this.config.checkCss;
|
||||
return false;
|
||||
};
|
||||
for (const r of parsed.resources) {
|
||||
if (!shouldFollow(r.kind)) continue;
|
||||
if (r.isInternal) {
|
||||
this.enqueue({ url: r.url, depth });
|
||||
} else {
|
||||
// External CSS/JS/font resources have no dedicated probe, so route
|
||||
// them through the external-stub probe to surface their status
|
||||
// (broken third-party scripts/styles show up in Broken Links).
|
||||
this.enqueueExternal(r.url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mine a fetched stylesheet for the resources it references — `url(...)`
|
||||
* targets (web fonts, background / mask images) and `@import` targets
|
||||
* (nested stylesheets) — and enqueue the INTERNAL ones so they land in the
|
||||
* Internal tab. This is the path that populates the "Font" filter, since
|
||||
* most web fonts are declared in CSS `@font-face` rather than on the page.
|
||||
* External CSS-referenced assets are intentionally skipped to avoid fanning
|
||||
* out into third-party font/CDN hosts. Iteration is capped so a pathological
|
||||
* stylesheet can't stall the crawl.
|
||||
*/
|
||||
private enqueueCssResources(css: string, baseUrl: string, depth: number): void {
|
||||
const opts = {
|
||||
includeSubdomains: this.config.scope === 'all-subdomains',
|
||||
cdnHosts: this.config.cdnHosts,
|
||||
};
|
||||
const seenRaw = new Set<string>();
|
||||
const consider = (raw: string | undefined): void => {
|
||||
if (!raw) return;
|
||||
const ref = raw.trim().replace(/^['"]|['"]$/g, '').trim();
|
||||
if (!ref || ref.startsWith('data:') || ref.startsWith('#')) return;
|
||||
if (seenRaw.has(ref)) return;
|
||||
seenRaw.add(ref);
|
||||
const normalized = normalizeUrl(ref, baseUrl, this.urlRewrites);
|
||||
if (!normalized || !/^https?:/.test(normalized)) return;
|
||||
if (isSameHost(baseUrl, normalized, opts)) {
|
||||
this.enqueue({ url: normalized, depth });
|
||||
}
|
||||
};
|
||||
let m: RegExpExecArray | null;
|
||||
let count = 0;
|
||||
const urlRe = /url\(\s*([^)]+?)\s*\)/gi;
|
||||
while ((m = urlRe.exec(css)) !== null && count < 500) {
|
||||
count++;
|
||||
consider(m[1]);
|
||||
}
|
||||
const importRe = /@import\s+(?:url\(\s*)?['"]?([^'")\s;]+)/gi;
|
||||
count = 0;
|
||||
while ((m = importRe.exec(css)) !== null && count < 200) {
|
||||
count++;
|
||||
consider(m[1]);
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchAndProcess(item: QueueItem): Promise<void> {
|
||||
if (this.stopped) return;
|
||||
this.setOp(`crawl:fetch:${item.url}`);
|
||||
@@ -2344,11 +2431,33 @@ export class Crawler extends EventEmitter {
|
||||
const needsJsonExtraction =
|
||||
statusCode < 400 &&
|
||||
(this.config.customExtractionRules ?? []).some((r) => r.type === 'jsonpath');
|
||||
// We also read the body for a successful CSS resource so we can mine
|
||||
// its `url(...)` / `@import` references (fonts, background images,
|
||||
// nested stylesheets) and crawl the internal ones — this is what
|
||||
// actually populates the Internal tab's "Font" filter.
|
||||
const needsCssParse =
|
||||
kind === 'css' &&
|
||||
statusCode >= 200 &&
|
||||
statusCode < 300 &&
|
||||
this.config.checkCss;
|
||||
// Otherwise only download the body when a JSONPath rule needs it.
|
||||
// `contentLength` is read from the Content-Length header (below), not
|
||||
// the body, so for plain resources (images, fonts, JS, PDFs) we
|
||||
// discard the stream without buffering megabytes into memory — this is
|
||||
// what makes "Check Images / CSS / JS" resource crawling lightweight.
|
||||
let nonHtmlBody = '';
|
||||
try {
|
||||
nonHtmlBody = await res.text();
|
||||
} catch {
|
||||
/* ignore */
|
||||
if (needsJsonExtraction || needsCssParse) {
|
||||
try {
|
||||
nonHtmlBody = await res.text();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await res.body?.cancel();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
let jsonExtraction: Record<string, unknown> | null = null;
|
||||
if (needsJsonExtraction) {
|
||||
@@ -2408,6 +2517,9 @@ export class Crawler extends EventEmitter {
|
||||
]);
|
||||
if (nonHtmlUrlId) await this.dbCall<void>('setUrlHeaders', [nonHtmlUrlId, allHeaders]);
|
||||
this.crawled++;
|
||||
if (needsCssParse && nonHtmlBody) {
|
||||
this.enqueueCssResources(nonHtmlBody, item.url, item.depth);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2749,6 +2861,14 @@ export class Crawler extends EventEmitter {
|
||||
}
|
||||
this.crawled++;
|
||||
|
||||
// Subresource crawling — fetch internal images / CSS / JS referenced by
|
||||
// this page so they appear in the Internal tab with their own status
|
||||
// code, content type, and size (Screaming Frog "Check Images/CSS/JS").
|
||||
// Done before the noindex/nofollow early-return so a page's assets are
|
||||
// crawled regardless of its own indexability. Resources are leaf nodes:
|
||||
// the non-HTML fetch path stores them without extracting further links.
|
||||
this.enqueueResources(parsed, item.depth);
|
||||
|
||||
if (parsed.hasNofollow || indexability === 'non-indexable:noindex') {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import * as cheerio from 'cheerio';
|
||||
import type {
|
||||
ContentKind,
|
||||
CustomExtractionRule,
|
||||
DiscoveredImage,
|
||||
DiscoveredLink,
|
||||
DiscoveredResource,
|
||||
LinkPathType,
|
||||
LinkPosition,
|
||||
} from '@freecrawl/shared-types';
|
||||
@@ -311,6 +313,12 @@ export interface ParsedPage {
|
||||
analyticsTrackers: AnalyticsTracker[];
|
||||
links: DiscoveredLink[];
|
||||
images: DiscoveredImage[];
|
||||
/**
|
||||
* Non-anchor subresources (`<script src>`, `<link rel="stylesheet">`) the
|
||||
* crawler can fetch so they surface in the Internal tab. Images are carried
|
||||
* separately in `images`.
|
||||
*/
|
||||
resources: DiscoveredResource[];
|
||||
hasNoindex: boolean;
|
||||
hasNofollow: boolean;
|
||||
}
|
||||
@@ -638,6 +646,46 @@ export function parseHtml(
|
||||
subresourceRequestCount += $('video[src]').length;
|
||||
subresourceRequestCount += $('audio[src]').length;
|
||||
|
||||
// Subresource URLs the crawler can fetch so they surface in the Internal
|
||||
// tab as their own rows (status code, content type, size). Images are
|
||||
// gathered separately in `imageMap`; here we collect scripts + stylesheets.
|
||||
// Each unique URL is recorded once with its kind pre-classified from the tag.
|
||||
const resourceMap = new Map<string, DiscoveredResource>();
|
||||
const addResource = (raw: string | undefined, kind: ContentKind): void => {
|
||||
const trimmed = (raw ?? '').trim();
|
||||
if (!trimmed || trimmed.startsWith('data:')) return;
|
||||
const normalized = normalizeUrl(trimmed, pageUrl, opts.urlRewrites);
|
||||
if (!normalized) return;
|
||||
if (!/^https?:/.test(normalized)) return;
|
||||
if (resourceMap.has(normalized)) return;
|
||||
resourceMap.set(normalized, {
|
||||
url: normalized,
|
||||
kind,
|
||||
isInternal: isSameHost(pageUrl, normalized, opts),
|
||||
});
|
||||
};
|
||||
$('script[src]').each((_, el) => addResource($(el).attr('src'), 'js'));
|
||||
$('link[rel="stylesheet"][href]').each((_, el) =>
|
||||
addResource($(el).attr('href'), 'css'),
|
||||
);
|
||||
// Preloaded subresources (`<link rel="preload" as="font|style|script|image">`).
|
||||
// The `as` attribute pre-classifies the kind. This is the main way modern
|
||||
// sites declare web fonts on the page itself — without it the Internal tab's
|
||||
// "Font" filter stays empty (most other fonts live in CSS `@font-face`).
|
||||
const PRELOAD_AS_TO_KIND: Record<string, ContentKind> = {
|
||||
font: 'font',
|
||||
style: 'css',
|
||||
script: 'js',
|
||||
image: 'image',
|
||||
};
|
||||
$('link[rel="preload"][as][href], link[rel="modulepreload"][href]').each((_, el) => {
|
||||
const $el = $(el);
|
||||
const rel = ($el.attr('rel') ?? '').trim().toLowerCase();
|
||||
const as = ($el.attr('as') ?? '').trim().toLowerCase();
|
||||
const kind = rel === 'modulepreload' ? 'js' : PRELOAD_AS_TO_KIND[as];
|
||||
if (kind) addResource($el.attr('href'), kind);
|
||||
});
|
||||
|
||||
// Pagination — `<link rel="next">` / `<link rel="prev">`. Resolved to
|
||||
// absolute via normalizeUrl so the values are comparable to the URLs
|
||||
// we crawl and store.
|
||||
@@ -1347,6 +1395,7 @@ export function parseHtml(
|
||||
analyticsTrackers,
|
||||
links: [...linkMap.values()],
|
||||
images: [...imageMap.values()],
|
||||
resources: [...resourceMap.values()],
|
||||
hasNoindex,
|
||||
hasNofollow,
|
||||
};
|
||||
|
||||
+114
-75
@@ -36,6 +36,26 @@ import type {
|
||||
import { issueCategoriesBySeverity } from '@freecrawl/shared-types';
|
||||
import { runMigrations } from './migrations.js';
|
||||
|
||||
/**
|
||||
* Best-effort `Content-Type` → {@link ContentKind} classifier. Used to
|
||||
* re-classify external probe stubs (which are inserted as 'other' before the
|
||||
* HEAD/GET probe resolves) so the `external:html` / `external:other` filters
|
||||
* separate correctly. Returns null when the type is absent/unrecognised so
|
||||
* callers can keep the prior value rather than clobbering it with 'other'.
|
||||
*/
|
||||
function contentKindFromType(ct: string | null | undefined): ContentKind | null {
|
||||
if (!ct) return null;
|
||||
const t = ct.toLowerCase();
|
||||
if (t.includes('text/html') || t.includes('application/xhtml')) return 'html';
|
||||
if (t.includes('text/css')) return 'css';
|
||||
if (t.includes('javascript') || t.includes('ecmascript')) return 'js';
|
||||
if (t.startsWith('image/')) return 'image';
|
||||
if (t.includes('application/pdf')) return 'pdf';
|
||||
if (t.includes('font/') || t.includes('woff') || t.includes('application/vnd.ms-fontobject'))
|
||||
return 'font';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
interface UrlRowDb {
|
||||
id: number;
|
||||
url: string;
|
||||
@@ -1198,12 +1218,19 @@ export class ProjectDb {
|
||||
responseTimeMs?: number | null;
|
||||
},
|
||||
): void {
|
||||
// Re-classify the external stub's content_kind from the probed
|
||||
// Content-Type. Stubs are inserted as 'other' before they're probed;
|
||||
// without this an external HTML page never leaves the 'other' bucket,
|
||||
// so the `external:html` filter stays empty. COALESCE keeps the prior
|
||||
// value when the probe returned no usable Content-Type.
|
||||
const contentKind = contentKindFromType(patch.contentType);
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE urls SET
|
||||
status_code = :status_code,
|
||||
status_text = :status_text,
|
||||
content_type = :content_type,
|
||||
content_kind = COALESCE(:content_kind, content_kind),
|
||||
content_length = :content_length,
|
||||
response_time_ms = :response_time_ms
|
||||
WHERE url = :url AND is_external = 1`,
|
||||
@@ -1213,6 +1240,7 @@ export class ProjectDb {
|
||||
status_code: patch.statusCode,
|
||||
status_text: patch.statusText ?? null,
|
||||
content_type: patch.contentType ?? null,
|
||||
content_kind: contentKind,
|
||||
content_length: patch.contentLength ?? null,
|
||||
response_time_ms: patch.responseTimeMs ?? null,
|
||||
});
|
||||
@@ -4011,8 +4039,15 @@ export class ProjectDb {
|
||||
const totalInternalUrls = countWhere('is_external = 0');
|
||||
const totalExternalUrls = countWhere('is_external = 1');
|
||||
const internalKinds = groupByInternal('content_kind');
|
||||
const totalIndexable = countWhere("is_external = 0 AND indexability = 'indexable' AND status_code IS NOT NULL");
|
||||
const totalNonIndexable = countWhere("is_external = 0 AND indexability LIKE 'non-indexable%'");
|
||||
// Headline indexability reflects PAGES only (content_kind = 'html').
|
||||
// Resources (image/css/js/font) are all `indexable` with a 2xx status,
|
||||
// so without this guard they would inflate the "Indexable URLs" stat.
|
||||
const totalIndexable = countWhere(
|
||||
"is_external = 0 AND content_kind = 'html' AND indexability = 'indexable' AND status_code IS NOT NULL",
|
||||
);
|
||||
const totalNonIndexable = countWhere(
|
||||
"is_external = 0 AND content_kind = 'html' AND indexability LIKE 'non-indexable%'",
|
||||
);
|
||||
|
||||
return {
|
||||
summary: {
|
||||
@@ -4158,16 +4193,16 @@ export class ProjectDb {
|
||||
)`,
|
||||
),
|
||||
contentThin: countWhere(`${html} AND word_count IS NOT NULL AND word_count < 300`),
|
||||
responseSlow: countWhere('is_external = 0 AND response_time_ms > 1000'),
|
||||
responseVerySlow: countWhere('is_external = 0 AND response_time_ms > 3000'),
|
||||
responseSlow: countWhere(`${html} AND response_time_ms > 1000`),
|
||||
responseVerySlow: countWhere(`${html} AND response_time_ms > 3000`),
|
||||
pageLarge: countWhere(`${html} AND content_length > 1048576`),
|
||||
urlTooLong: countWhere('is_external = 0 AND LENGTH(url) > 2048'),
|
||||
urlUppercase: countWhere("is_external = 0 AND url GLOB '*[A-Z]*'"),
|
||||
urlUnderscore: countWhere("is_external = 0 AND INSTR(url, '_') > 0"),
|
||||
urlTooLong: countWhere(`${html} AND LENGTH(url) > 2048`),
|
||||
urlUppercase: countWhere(`${html} AND url GLOB '*[A-Z]*'`),
|
||||
urlUnderscore: countWhere(`${html} AND INSTR(url, '_') > 0`),
|
||||
urlMultipleSlashes: countWhere(
|
||||
"is_external = 0 AND INSTR(SUBSTR(url, INSTR(url, '://') + 3), '//') > 0",
|
||||
`${html} AND INSTR(SUBSTR(url, INSTR(url, '://') + 3), '//') > 0`,
|
||||
),
|
||||
urlNonAscii: countWhere('is_external = 0 AND LENGTH(CAST(url AS BLOB)) != LENGTH(url)'),
|
||||
urlNonAscii: countWhere(`${html} AND LENGTH(CAST(url AS BLOB)) != LENGTH(url)`),
|
||||
langMissing: countWhere(`${html} AND (lang IS NULL OR lang = '')`),
|
||||
viewportMissing: countWhere(`${html} AND (viewport IS NULL OR viewport = '')`),
|
||||
ogMissing: countWhere(
|
||||
@@ -4182,7 +4217,7 @@ export class ProjectDb {
|
||||
AND (twitter_image IS NULL OR twitter_image = '')`,
|
||||
),
|
||||
hstsMissing: countWhere(
|
||||
"is_external = 0 AND url LIKE 'https://%' AND (hsts IS NULL OR hsts = '')",
|
||||
`${html} AND url LIKE 'https://%' AND (hsts IS NULL OR hsts = '')`,
|
||||
),
|
||||
xFrameOptionsMissing: countWhere(
|
||||
`${html} AND (x_frame_options IS NULL OR x_frame_options = '')`,
|
||||
@@ -4221,12 +4256,12 @@ export class ProjectDb {
|
||||
`${html} AND url LIKE 'https://%' AND mixed_content_passive > 0`,
|
||||
),
|
||||
faviconMissing: countWhere(`${html} AND (favicon IS NULL OR favicon = '')`),
|
||||
redirectLoop: countWhere('is_external = 0 AND redirect_loop = 1'),
|
||||
redirectChainLong: countWhere('is_external = 0 AND redirect_chain_length > 3'),
|
||||
redirectLoop: countWhere(`${html} AND redirect_loop = 1`),
|
||||
redirectChainLong: countWhere(`${html} AND redirect_chain_length > 3`),
|
||||
redirectSelf: countWhere(
|
||||
'is_external = 0 AND redirect_target IS NOT NULL AND redirect_target = url',
|
||||
`${html} AND redirect_target IS NOT NULL AND redirect_target = url`,
|
||||
),
|
||||
urlManyParams: countWhere('is_external = 0 AND query_param_count > 5'),
|
||||
urlManyParams: countWhere(`${html} AND query_param_count > 5`),
|
||||
compressionMissing: countWhere(
|
||||
`${html} AND status_code >= 200 AND status_code < 300
|
||||
AND (content_encoding IS NULL OR content_encoding = '')`,
|
||||
@@ -4306,11 +4341,11 @@ export class ProjectDb {
|
||||
h1Empty: countWhere(`${html} AND h1_count > 0 AND (h1 IS NULL OR h1 = '')`),
|
||||
h1TooLong: countWhere(`${html} AND h1_length > 70`),
|
||||
titleMultiple: countWhere(`${html} AND title_count > 1`),
|
||||
urlFragment: countWhere("is_external = 0 AND INSTR(url, '#') > 0"),
|
||||
urlFragment: countWhere(`${html} AND INSTR(url, '#') > 0`),
|
||||
urlSpaces: countWhere(
|
||||
"is_external = 0 AND (INSTR(url, ' ') > 0 OR INSTR(url, '%20') > 0)",
|
||||
`${html} AND (INSTR(url, ' ') > 0 OR INSTR(url, '%20') > 0)`,
|
||||
),
|
||||
urlMalformed: countWhere('is_external = 0 AND url_malformed = 1'),
|
||||
urlMalformed: countWhere(`${html} AND url_malformed = 1`),
|
||||
imageEmptyAlt: countWhere(`${html} AND images_empty_alt > 0`),
|
||||
linkEmptyAnchor: countWhere(`${html} AND empty_anchor_count > 0`),
|
||||
appleTouchIconMissing: countWhere(
|
||||
@@ -4331,17 +4366,17 @@ export class ProjectDb {
|
||||
`${html} AND url LIKE 'https://%' AND insecure_form_action_count > 0`,
|
||||
),
|
||||
missingSri: countWhere(`${html} AND missing_sri_count > 0`),
|
||||
ttfbSlow: countWhere('is_external = 0 AND ttfb_ms IS NOT NULL AND ttfb_ms > 600'),
|
||||
ttfbSlow: countWhere(`${html} AND ttfb_ms IS NOT NULL AND ttfb_ms > 600`),
|
||||
ttfbVerySlow: countWhere(
|
||||
'is_external = 0 AND ttfb_ms IS NOT NULL AND ttfb_ms > 1800',
|
||||
`${html} AND ttfb_ms IS NOT NULL AND ttfb_ms > 1800`,
|
||||
),
|
||||
cookieNoSecure: countWhere(
|
||||
"is_external = 0 AND cookies_insecure > 0 AND url LIKE 'https://%'",
|
||||
`${html} AND cookies_insecure > 0 AND url LIKE 'https://%'`,
|
||||
),
|
||||
cookieNoHttpOnly: countWhere('is_external = 0 AND cookies_no_httponly > 0'),
|
||||
cookieNoSameSite: countWhere('is_external = 0 AND cookies_no_samesite > 0'),
|
||||
queryStringTooLong: countWhere('is_external = 0 AND query_string_length > 100'),
|
||||
folderDepthTooDeep: countWhere('is_external = 0 AND folder_depth > 4'),
|
||||
cookieNoHttpOnly: countWhere(`${html} AND cookies_no_httponly > 0`),
|
||||
cookieNoSameSite: countWhere(`${html} AND cookies_no_samesite > 0`),
|
||||
queryStringTooLong: countWhere(`${html} AND query_string_length > 100`),
|
||||
folderDepthTooDeep: countWhere(`${html} AND folder_depth > 4`),
|
||||
http2NotSupported: countWhere(
|
||||
`${html} AND http_protocol = 'http/1.1'`,
|
||||
),
|
||||
@@ -4349,7 +4384,7 @@ export class ProjectDb {
|
||||
`${html} AND http_protocol IS NOT NULL AND http_protocol != 'h3'`,
|
||||
),
|
||||
renderBlocking: countWhere(`${html} AND render_blocking_count > 5`),
|
||||
keepaliveDisabled: countWhere('is_external = 0 AND keep_alive = 0'),
|
||||
keepaliveDisabled: countWhere(`${html} AND keep_alive = 0`),
|
||||
titlePlaceholder: countWhere(
|
||||
`${html} AND title IS NOT NULL AND title != ''
|
||||
AND (
|
||||
@@ -4366,13 +4401,13 @@ export class ProjectDb {
|
||||
AND (analytics_trackers IS NULL OR analytics_trackers = '[]' OR analytics_trackers = '')`,
|
||||
),
|
||||
analyticsMultipleGa4: countWhere(
|
||||
`is_external = 0 AND analytics_trackers IS NOT NULL
|
||||
`${html} AND analytics_trackers IS NOT NULL
|
||||
AND (
|
||||
LENGTH(analytics_trackers) - LENGTH(REPLACE(analytics_trackers, '"name":"Google Analytics 4"', ''))
|
||||
) / LENGTH('"name":"Google Analytics 4"') > 1`,
|
||||
),
|
||||
analyticsUaLegacy: countWhere(
|
||||
`is_external = 0 AND analytics_trackers IS NOT NULL
|
||||
`${html} AND analytics_trackers IS NOT NULL
|
||||
AND analytics_trackers LIKE '%"name":"Google Analytics (UA)"%'`,
|
||||
),
|
||||
analyticsPixelWithoutPolicy: countWhere(
|
||||
@@ -4394,7 +4429,7 @@ export class ProjectDb {
|
||||
)`,
|
||||
),
|
||||
sslCertExpired: countWhere(
|
||||
`is_external = 0 AND url LIKE 'https://%'
|
||||
`${html} AND url LIKE 'https://%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM host_certs hc
|
||||
WHERE hc.host = LOWER(SUBSTR(urls.url, 9, INSTR(SUBSTR(urls.url, 9), '/') - 1))
|
||||
@@ -4403,7 +4438,7 @@ export class ProjectDb {
|
||||
)`,
|
||||
),
|
||||
sslCertExpiringSoon: countWhere(
|
||||
`is_external = 0 AND url LIKE 'https://%'
|
||||
`${html} AND url LIKE 'https://%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM host_certs hc
|
||||
WHERE hc.host = LOWER(SUBSTR(urls.url, 9, INSTR(SUBSTR(urls.url, 9), '/') - 1))
|
||||
@@ -4413,7 +4448,7 @@ export class ProjectDb {
|
||||
)`,
|
||||
),
|
||||
sslProtocolOld: countWhere(
|
||||
`is_external = 0 AND url LIKE 'https://%'
|
||||
`${html} AND url LIKE 'https://%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM host_certs hc
|
||||
WHERE hc.host = LOWER(SUBSTR(urls.url, 9, INSTR(SUBSTR(urls.url, 9), '/') - 1))
|
||||
@@ -4422,7 +4457,7 @@ export class ProjectDb {
|
||||
)`,
|
||||
),
|
||||
sslSignatureWeak: countWhere(
|
||||
`is_external = 0 AND url LIKE 'https://%'
|
||||
`${html} AND url LIKE 'https://%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM host_certs hc
|
||||
WHERE hc.host = LOWER(SUBSTR(urls.url, 9, INSTR(SUBSTR(urls.url, 9), '/') - 1))
|
||||
@@ -4635,13 +4670,13 @@ export class ProjectDb {
|
||||
`${html} AND gunning_fog_index IS NOT NULL AND gunning_fog_index > 17`,
|
||||
),
|
||||
corsWildcardWithCredentials: countWhere(
|
||||
`is_external = 0 AND TRIM(cors_allow_origin) = '*' AND cors_allow_credentials = 1`,
|
||||
`${html} AND TRIM(cors_allow_origin) = '*' AND cors_allow_credentials = 1`,
|
||||
),
|
||||
corsWildcardOrigin: countWhere(
|
||||
`is_external = 0 AND TRIM(cors_allow_origin) = '*' AND cors_allow_credentials != 1`,
|
||||
`${html} AND TRIM(cors_allow_origin) = '*' AND cors_allow_credentials != 1`,
|
||||
),
|
||||
httpNotHttps: countWhere(
|
||||
`is_external = 0
|
||||
`${html}
|
||||
AND url LIKE 'http://%'
|
||||
AND url NOT LIKE 'http://localhost%'
|
||||
AND url NOT LIKE 'http://127.0.0.1%'
|
||||
@@ -5916,16 +5951,18 @@ export class ProjectDb {
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal images that haven't been HEAD-probed yet — used by the
|
||||
* post-crawl image-size pass to discover oversize PNGs/JPEGs without
|
||||
* re-probing already-sized rows on subsequent crawls.
|
||||
* Images that haven't been HEAD-probed yet — used by the post-crawl
|
||||
* image-size pass to discover oversize PNGs/JPEGs and broken image src
|
||||
* (4xx/5xx) without re-probing already-sized rows on subsequent crawls.
|
||||
* Covers BOTH internal and external images so the "Broken Image src"
|
||||
* issue fires on dead external/CDN images too; the "Large Image" issue
|
||||
* stays internal-only via its own `is_internal = 1` join filter.
|
||||
*/
|
||||
unprobedInternalImages(limit = 20_000): { id: number; src: string }[] {
|
||||
unprobedImages(limit = 20_000): { id: number; src: string }[] {
|
||||
return this.db
|
||||
.prepare(
|
||||
`SELECT id, src FROM images
|
||||
WHERE is_internal = 1
|
||||
AND probe_status IS NULL
|
||||
WHERE probe_status IS NULL
|
||||
AND src LIKE 'http%'
|
||||
ORDER BY id
|
||||
LIMIT ?`,
|
||||
@@ -6738,10 +6775,11 @@ export const EXPENSIVE_ISSUE_DEFINITIONS: ReadonlyArray<readonly [string, string
|
||||
],
|
||||
[
|
||||
'issues:duplicate-url-post-norm',
|
||||
`is_external = 0 AND EXISTS (
|
||||
`is_external = 0 AND content_kind = 'html' AND EXISTS (
|
||||
SELECT 1 FROM urls u2
|
||||
WHERE u2.id <> urls.id
|
||||
AND u2.is_external = 0
|
||||
AND u2.content_kind = 'html'
|
||||
AND RTRIM(
|
||||
LOWER(
|
||||
CASE
|
||||
@@ -7098,26 +7136,26 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
case 'issues:content-thin':
|
||||
return "is_external = 0 AND content_kind = 'html' AND word_count IS NOT NULL AND word_count < 300";
|
||||
case 'issues:response-slow':
|
||||
return 'is_external = 0 AND response_time_ms > 1000';
|
||||
return "is_external = 0 AND content_kind = 'html' AND response_time_ms > 1000";
|
||||
case 'issues:response-very-slow':
|
||||
return 'is_external = 0 AND response_time_ms > 3000';
|
||||
return "is_external = 0 AND content_kind = 'html' AND response_time_ms > 3000";
|
||||
case 'issues:page-large':
|
||||
return "is_external = 0 AND content_kind = 'html' AND content_length > 1048576";
|
||||
case 'issues:url-too-long':
|
||||
return 'is_external = 0 AND LENGTH(url) > 2048';
|
||||
return "is_external = 0 AND content_kind = 'html' AND LENGTH(url) > 2048";
|
||||
case 'issues:url-uppercase':
|
||||
// GLOB with [A-Z] is case-sensitive — unlike LIKE which isn't.
|
||||
return "is_external = 0 AND url GLOB '*[A-Z]*'";
|
||||
return "is_external = 0 AND content_kind = 'html' AND url GLOB '*[A-Z]*'";
|
||||
case 'issues:url-underscore':
|
||||
return "is_external = 0 AND INSTR(url, '_') > 0";
|
||||
return "is_external = 0 AND content_kind = 'html' AND INSTR(url, '_') > 0";
|
||||
case 'issues:url-multiple-slashes':
|
||||
// Strip the `scheme://` prefix, then check for any `//` that remains
|
||||
// (path / query doubled slashes aren't usually intentional).
|
||||
return "is_external = 0 AND INSTR(SUBSTR(url, INSTR(url, '://') + 3), '//') > 0";
|
||||
return "is_external = 0 AND content_kind = 'html' AND INSTR(SUBSTR(url, INSTR(url, '://') + 3), '//') > 0";
|
||||
case 'issues:url-non-ascii':
|
||||
// Byte-length (BLOB cast) > character length only when the string
|
||||
// contains multi-byte UTF-8, i.e. any non-ASCII code point.
|
||||
return 'is_external = 0 AND LENGTH(CAST(url AS BLOB)) != LENGTH(url)';
|
||||
return "is_external = 0 AND content_kind = 'html' AND LENGTH(CAST(url AS BLOB)) != LENGTH(url)";
|
||||
case 'issues:lang-missing':
|
||||
return "is_external = 0 AND content_kind = 'html' AND (lang IS NULL OR lang = '')";
|
||||
case 'issues:viewport-missing':
|
||||
@@ -7137,7 +7175,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// and X-Content-Type-Options matter on any HTML response regardless
|
||||
// of scheme, so they're only scheme-gated on a per-page basis.
|
||||
case 'issues:hsts-missing':
|
||||
return "is_external = 0 AND url LIKE 'https://%' AND (hsts IS NULL OR hsts = '')";
|
||||
return "is_external = 0 AND content_kind = 'html' AND url LIKE 'https://%' AND (hsts IS NULL OR hsts = '')";
|
||||
case 'issues:x-frame-options-missing':
|
||||
return `is_external = 0 AND content_kind = 'html'
|
||||
AND (x_frame_options IS NULL OR x_frame_options = '')`;
|
||||
@@ -7185,18 +7223,18 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
return `is_external = 0 AND content_kind = 'html'
|
||||
AND (favicon IS NULL OR favicon = '')`;
|
||||
case 'issues:redirect-loop':
|
||||
return 'is_external = 0 AND redirect_loop = 1';
|
||||
return "is_external = 0 AND content_kind = 'html' AND redirect_loop = 1";
|
||||
case 'issues:redirect-chain-long':
|
||||
// 3 hops is the conservative SF threshold — every extra redirect
|
||||
// multiplies link-equity loss and crawl-budget waste.
|
||||
return 'is_external = 0 AND redirect_chain_length > 3';
|
||||
return "is_external = 0 AND content_kind = 'html' AND redirect_chain_length > 3";
|
||||
case 'issues:redirect-self':
|
||||
// Redirect target equals the URL itself — a self-loop. Always
|
||||
// broken regardless of `followRedirects`.
|
||||
return 'is_external = 0 AND redirect_target IS NOT NULL AND redirect_target = url';
|
||||
return "is_external = 0 AND content_kind = 'html' AND redirect_target IS NOT NULL AND redirect_target = url";
|
||||
case 'issues:url-many-params':
|
||||
// 5+ query params usually means session IDs / faceted-nav explosion.
|
||||
return 'is_external = 0 AND query_param_count > 5';
|
||||
return "is_external = 0 AND content_kind = 'html' AND query_param_count > 5";
|
||||
case 'issues:compression-missing':
|
||||
// No Content-Encoding on a successful HTML response = ~70% wasted
|
||||
// bandwidth. Skip the scheme-less and non-200 noise.
|
||||
@@ -7323,17 +7361,17 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// A `#` in the URL means the crawler reached a fragment that wasn't
|
||||
// stripped earlier (URL normaliser usually does, but List-mode +
|
||||
// some redirect chains can leave it).
|
||||
return "is_external = 0 AND INSTR(url, '#') > 0";
|
||||
return "is_external = 0 AND content_kind = 'html' AND INSTR(url, '#') > 0";
|
||||
case 'issues:url-spaces':
|
||||
// Literal space or `%20` in URL — encoding bug, often from CMS that
|
||||
// produced filenames with spaces.
|
||||
return "is_external = 0 AND (INSTR(url, ' ') > 0 OR INSTR(url, '%20') > 0)";
|
||||
return "is_external = 0 AND content_kind = 'html' AND (INSTR(url, ' ') > 0 OR INSTR(url, '%20') > 0)";
|
||||
case 'issues:url-malformed':
|
||||
// Computed by `isUrlMalformed` at upsert time. Captures the
|
||||
// ambiguities the URL parser tolerates but middleware diverges on:
|
||||
// multiple `?`/`#`, control chars, unescaped reserved chars,
|
||||
// double-encoding sequences (`%2520` etc).
|
||||
return 'is_external = 0 AND url_malformed = 1';
|
||||
return "is_external = 0 AND content_kind = 'html' AND url_malformed = 1";
|
||||
case 'issues:image-empty-alt':
|
||||
// alt="" specifically. Decorative images use this intentionally, but
|
||||
// many sites apply it to content images by mistake.
|
||||
@@ -7385,30 +7423,30 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// TTFB > 600 ms is the Core Web Vitals "needs improvement" boundary
|
||||
// (Google CrUX). Crawls measure ttfb_ms as request → headers, so
|
||||
// it's not exactly equivalent to a real-user RTT but tracks closely.
|
||||
return `is_external = 0 AND ttfb_ms IS NOT NULL AND ttfb_ms > 600`;
|
||||
return `is_external = 0 AND content_kind = 'html' AND ttfb_ms IS NOT NULL AND ttfb_ms > 600`;
|
||||
case 'issues:ttfb-very-slow':
|
||||
// > 1.8 s is the CrUX "poor" boundary.
|
||||
return `is_external = 0 AND ttfb_ms IS NOT NULL AND ttfb_ms > 1800`;
|
||||
return `is_external = 0 AND content_kind = 'html' AND ttfb_ms IS NOT NULL AND ttfb_ms > 1800`;
|
||||
case 'issues:cookie-no-secure':
|
||||
// At least one Set-Cookie missing the `Secure` flag — over HTTPS
|
||||
// this lets a downgrade-attack snoop the cookie.
|
||||
return `is_external = 0 AND cookies_insecure > 0 AND url LIKE 'https://%'`;
|
||||
return `is_external = 0 AND content_kind = 'html' AND cookies_insecure > 0 AND url LIKE 'https://%'`;
|
||||
case 'issues:cookie-no-httponly':
|
||||
// Missing HttpOnly — JS can read the cookie, expanding the XSS blast
|
||||
// radius. Session cookies should always be HttpOnly.
|
||||
return `is_external = 0 AND cookies_no_httponly > 0`;
|
||||
return `is_external = 0 AND content_kind = 'html' AND cookies_no_httponly > 0`;
|
||||
case 'issues:cookie-no-samesite':
|
||||
// Missing SameSite — Chrome treats absent as `Lax` since 80, but
|
||||
// explicit declaration is recommended for cross-browser parity.
|
||||
return `is_external = 0 AND cookies_no_samesite > 0`;
|
||||
return `is_external = 0 AND content_kind = 'html' AND cookies_no_samesite > 0`;
|
||||
case 'issues:query-string-too-long':
|
||||
// Long query strings indicate session-id explosion or faceted-nav
|
||||
// gone wrong; >100 chars is a practical Screaming-Frog warn.
|
||||
return `is_external = 0 AND query_string_length > 100`;
|
||||
return `is_external = 0 AND content_kind = 'html' AND query_string_length > 100`;
|
||||
case 'issues:folder-depth-too-deep':
|
||||
// Path-segment depth beyond the conservative SF threshold of 4
|
||||
// hides content from Googlebot's link-graph crawl heuristics.
|
||||
return `is_external = 0 AND folder_depth > 4`;
|
||||
return `is_external = 0 AND content_kind = 'html' AND folder_depth > 4`;
|
||||
case 'issues:http2-not-supported':
|
||||
// Origin advertises only HTTP/1.1 (no Alt-Svc / no h2 token).
|
||||
// Informational — modern browsers still work fine, but HTTP/2 is
|
||||
@@ -7451,7 +7489,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// Server explicitly closed the connection. -1 sentinel = no signal,
|
||||
// 0 = Connection: close seen, 1 = keep-alive (or implicit). Only
|
||||
// flag when we have a positive signal of `close`.
|
||||
return `is_external = 0 AND keep_alive = 0`;
|
||||
return `is_external = 0 AND content_kind = 'html' AND keep_alive = 0`;
|
||||
case 'issues:title-placeholder':
|
||||
// Common CMS placeholders: "Untitled", "Default Title", "New Page",
|
||||
// "Home" / "Page N". Catches default theme/template values that
|
||||
@@ -7480,14 +7518,14 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// (Tag Manager + hardcoded gtag, or two property IDs). The pattern
|
||||
// ".*?\"name\":\"Google Analytics 4\".*?\"name\":\"Google Analytics 4\""
|
||||
// is awkward in SQL, so we use a position-based check.
|
||||
return `is_external = 0 AND analytics_trackers IS NOT NULL
|
||||
return `is_external = 0 AND content_kind = 'html' AND analytics_trackers IS NOT NULL
|
||||
AND (
|
||||
LENGTH(analytics_trackers) - LENGTH(REPLACE(analytics_trackers, '"name":"Google Analytics 4"', ''))
|
||||
) / LENGTH('"name":"Google Analytics 4"') > 1`;
|
||||
case 'issues:analytics-ua-legacy':
|
||||
// Universal Analytics (UA-XXXXX-Y) was sunset 2023-07-01. Any page
|
||||
// still loading it is gathering no data — pure dead weight.
|
||||
return `is_external = 0 AND analytics_trackers IS NOT NULL
|
||||
return `is_external = 0 AND content_kind = 'html' AND analytics_trackers IS NOT NULL
|
||||
AND analytics_trackers LIKE '%"name":"Google Analytics (UA)"%'`;
|
||||
case 'issues:analytics-pixel-without-policy':
|
||||
// Tracking pixels (FB / TikTok / Pinterest / LinkedIn) that share
|
||||
@@ -7519,7 +7557,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// HTTPS pages whose host's certificate is past `valid_to`. The
|
||||
// host is parsed inline from the URL because storing it as a column
|
||||
// would balloon the row count for an aggregation we run rarely.
|
||||
return `is_external = 0 AND url LIKE 'https://%'
|
||||
return `is_external = 0 AND content_kind = 'html' AND url LIKE 'https://%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM host_certs hc
|
||||
WHERE hc.host = LOWER(SUBSTR(urls.url, 9, INSTR(SUBSTR(urls.url, 9), '/') - 1))
|
||||
@@ -7529,7 +7567,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
case 'issues:ssl-cert-expiring-soon':
|
||||
// 30-day warning window — matches Let's Encrypt's reminder cadence
|
||||
// and the Mozilla Observatory's "expiring soon" threshold.
|
||||
return `is_external = 0 AND url LIKE 'https://%'
|
||||
return `is_external = 0 AND content_kind = 'html' AND url LIKE 'https://%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM host_certs hc
|
||||
WHERE hc.host = LOWER(SUBSTR(urls.url, 9, INSTR(SUBSTR(urls.url, 9), '/') - 1))
|
||||
@@ -7541,7 +7579,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// TLSv1.0 / TLSv1.1 are deprecated by all major browsers (2020).
|
||||
// Sites still negotiating them fail PCI-DSS and many corporate
|
||||
// proxies block them.
|
||||
return `is_external = 0 AND url LIKE 'https://%'
|
||||
return `is_external = 0 AND content_kind = 'html' AND url LIKE 'https://%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM host_certs hc
|
||||
WHERE hc.host = LOWER(SUBSTR(urls.url, 9, INSTR(SUBSTR(urls.url, 9), '/') - 1))
|
||||
@@ -7552,7 +7590,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// SHA-1 / MD5 signature algorithms are cryptographically broken
|
||||
// for cert chains. Browsers stopped accepting SHA-1 in 2017 but
|
||||
// self-signed / internal certs sometimes still use it.
|
||||
return `is_external = 0 AND url LIKE 'https://%'
|
||||
return `is_external = 0 AND content_kind = 'html' AND url LIKE 'https://%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM host_certs hc
|
||||
WHERE hc.host = LOWER(SUBSTR(urls.url, 9, INSTR(SUBSTR(urls.url, 9), '/') - 1))
|
||||
@@ -7891,11 +7929,12 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// issue and a fragment-only difference is rare in real corpora.
|
||||
// EXISTS pattern matches at least one OTHER row with the same
|
||||
// normalised key; current row excluded by `urls.id <> u2.id`.
|
||||
return `is_external = 0
|
||||
return `is_external = 0 AND content_kind = 'html'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM urls u2
|
||||
WHERE u2.id <> urls.id
|
||||
AND u2.is_external = 0
|
||||
AND u2.content_kind = 'html'
|
||||
AND RTRIM(
|
||||
LOWER(
|
||||
CASE
|
||||
@@ -7994,14 +8033,14 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// `Access-Control-Allow-Credentials: true`. Browsers reject this
|
||||
// combination per the CORS spec, but misconfigured servers reach
|
||||
// it and any working XSS could exfiltrate cookies cross-origin.
|
||||
return `is_external = 0
|
||||
return `is_external = 0 AND content_kind = 'html'
|
||||
AND TRIM(cors_allow_origin) = '*'
|
||||
AND cors_allow_credentials = 1`;
|
||||
case 'issues:cors-wildcard-origin':
|
||||
// `Access-Control-Allow-Origin: *` without credentials. Not always
|
||||
// wrong (public APIs do this) but worth reviewing on production
|
||||
// HTML so an audit can confirm intent.
|
||||
return `is_external = 0
|
||||
return `is_external = 0 AND content_kind = 'html'
|
||||
AND TRIM(cors_allow_origin) = '*'
|
||||
AND cors_allow_credentials != 1`;
|
||||
case 'issues:http-not-https':
|
||||
@@ -8009,7 +8048,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// HTTPS as a positive signal and Chrome marks HTTP pages as "Not
|
||||
// Secure". Excludes the localhost / 127.0.0.1 / .local hosts a
|
||||
// user might legitimately crawl over HTTP for local dev work.
|
||||
return `is_external = 0
|
||||
return `is_external = 0 AND content_kind = 'html'
|
||||
AND url LIKE 'http://%'
|
||||
AND url NOT LIKE 'http://localhost%'
|
||||
AND url NOT LIKE 'http://127.0.0.1%'
|
||||
@@ -8153,7 +8192,7 @@ function categoryWhereClause(cat: UrlCategory): string | null {
|
||||
// pass. `budget_status` is a bitmask (1=response, 2=size, 4=LCP,
|
||||
// 8=CLS); >0 means at least one configured ceiling was exceeded.
|
||||
// NULL/0 (budget off, or within budget) is excluded.
|
||||
return 'is_external = 0 AND budget_status IS NOT NULL AND budget_status > 0';
|
||||
return "is_external = 0 AND content_kind = 'html' AND budget_status IS NOT NULL AND budget_status > 0";
|
||||
case 'tab:directives':
|
||||
// Wave 4 — Directives tab. Pages declaring any indexability
|
||||
// directive: meta robots, X-Robots-Tag header, robots.txt block,
|
||||
|
||||
@@ -644,6 +644,20 @@ export interface CrawlConfig {
|
||||
followRedirects: boolean;
|
||||
respectRobotsTxt: boolean;
|
||||
crawlExternal: boolean;
|
||||
/**
|
||||
* Crawl internal page subresources so they appear in the Internal tab as
|
||||
* their own rows (status code, content type, size) — exactly like
|
||||
* Screaming Frog's "Check Images / CSS / JavaScript". Each fetched resource
|
||||
* counts toward `maxUrls`. Resources are leaf nodes: fetched once (headers
|
||||
* only, body discarded) but never parsed for further links. All default
|
||||
* `true` so the Internal tab shows every internal resource, not just HTML.
|
||||
* - `checkImages` — `<img src>` + `srcset` candidates + `<picture>` sources
|
||||
* - `checkCss` — `<link rel="stylesheet">`
|
||||
* - `checkJs` — `<script src>`
|
||||
*/
|
||||
checkImages: boolean;
|
||||
checkCss: boolean;
|
||||
checkJs: boolean;
|
||||
acceptLanguage: string;
|
||||
/** Per-worker delay inserted *after* each request (ms). 0 = disabled. */
|
||||
crawlDelayMs: number;
|
||||
@@ -1721,6 +1735,21 @@ export interface DiscoveredImage {
|
||||
isInternal: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* A non-anchor subresource referenced by a page — currently `<script src>`
|
||||
* and `<link rel="stylesheet">`. When the matching "Check CSS / JavaScript"
|
||||
* toggle is on, the crawler fetches each internal resource so it appears in
|
||||
* the Internal tab as its own row with status code, content type, and size
|
||||
* (mirrors Screaming Frog's resource crawling). `kind` is pre-classified from
|
||||
* the tag so the crawler can apply the right per-type toggle without
|
||||
* re-sniffing. Images are carried separately in {@link DiscoveredImage}.
|
||||
*/
|
||||
export interface DiscoveredResource {
|
||||
url: string;
|
||||
kind: ContentKind;
|
||||
isInternal: boolean;
|
||||
}
|
||||
|
||||
export interface ImageRow {
|
||||
id: number;
|
||||
src: string;
|
||||
@@ -1861,6 +1890,9 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
|
||||
followRedirects: true,
|
||||
respectRobotsTxt: true,
|
||||
crawlExternal: false,
|
||||
checkImages: true,
|
||||
checkCss: true,
|
||||
checkJs: true,
|
||||
acceptLanguage: 'tr,en;q=0.8',
|
||||
crawlDelayMs: 0,
|
||||
retryAttempts: 2,
|
||||
|
||||
Reference in new issue
Block a user