feat: social image aspect-ratio, CWV INP/TTI, a11y contrast audit

This commit is contained in:
kemalai committed 2026-06-14 00:46:18 +03:00
1 parent 021beb01cb
commit 29f54bff32
27 files changed
+986 -14

No files matched your search

+10
View File
@@ -1,5 +1,15 @@
# Changelog
## [0.6.9] — 2026-06-14
### Added
- Social image aspect-ratio validation — flags OG / Twitter images too small or wrongly shaped to render as a proper share card.
- Core Web Vitals INP, TTI, and Max Potential FID in PageSpeed audits, with INP shown alongside LCP and CLS as a Core Web Vital.
- Accessibility audit (opt-in, JS render) — flags low colour-contrast text (WCAG AA) and stylesheets that suppress the keyboard focus outline.
### Fixed
- macOS: added Edit and Window menus so copy / paste and ⌘M work, and cleared a ⌘R shortcut conflict between Reload and Reports.
## [0.6.8] — 2026-06-04
### Added
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/cli",
"version": "0.6.8",
"version": "0.6.9",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/desktop",
"version": "0.6.8",
"version": "0.6.9",
"private": true,
"description": "FreeCrawl SEO — desktop application",
"homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
@@ -74,6 +74,7 @@ const ALLOWED_METHODS = new Set<string>([
'setUrlScreenshotPaths',
'setUrlLcpCandidate',
'setUrlMobileUsability',
'setUrlA11y',
'setSitemapUrls',
'setHostCert',
'setImageSize',
+10
View File
@@ -739,6 +739,9 @@ function getDb(): ProjectDb {
lcpCandidate:
parsed.jsRender?.lcpCandidate ??
DEFAULT_CRAWL_CONFIG.jsRender.lcpCandidate,
a11yAudit:
parsed.jsRender?.a11yAudit ??
DEFAULT_CRAWL_CONFIG.jsRender.a11yAudit,
},
};
}
@@ -3248,6 +3251,11 @@ function registerIpc(): void {
overflowPx: number;
hasViewportMeta: boolean;
} | null;
a11y?: {
lowContrast: number;
sampled: number;
focusSuppressed: boolean;
} | null;
}>)
| undefined;
if (config.renderingMode === 'js') {
@@ -3365,6 +3373,7 @@ function registerIpc(): void {
waitUntil: config.jsRender.waitUntil,
screenshotMode: config.jsRender.screenshotMode,
detectLcp: config.jsRender.lcpCandidate,
detectA11y: config.jsRender.a11yAudit,
screenshotPaths,
},
signal,
@@ -3442,6 +3451,7 @@ function registerIpc(): void {
screenshots,
lcp: res.lcp ?? null,
mobileUsability,
a11y: res.a11y ?? null,
};
};
}
+2 -1
View File
@@ -153,7 +153,7 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
click: () => send('toggle-detail-panel'),
},
{ type: 'separator' },
{ role: 'reload' },
{ role: 'reload', accelerator: 'CmdOrCtrl+Shift+R' },
{ type: 'separator' },
{ role: 'resetZoom' },
{ role: 'zoomIn' },
@@ -182,6 +182,7 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
},
],
},
{ role: 'windowMenu' as const },
{
label: L.help,
submenu: [
+25
View File
@@ -40,12 +40,32 @@ function errorMetrics(fetchedAt: string, message: string): PagespeedMetrics {
fcp: null,
tbt: null,
speedIndex: null,
tti: null,
maxPotentialFid: null,
inp: null,
status: 'error',
error: message,
fetchedAt,
};
}
/**
* Pull a CrUX field metric's 75th-percentile value (ms) out of the PSI
* `loadingExperience` block. Field metrics only exist when the URL has
* enough real-user data — returns null otherwise.
*/
function fieldMetricMs(
json: Record<string, unknown>,
key: string,
): number | null {
const le = json['loadingExperience'] as
| { metrics?: Record<string, unknown> }
| undefined;
const metric = le?.metrics?.[key] as { percentile?: unknown } | undefined;
const p = metric?.percentile;
return typeof p === 'number' && Number.isFinite(p) ? Math.round(p) : null;
}
/** Pull `numericValue` out of a Lighthouse audit object, rounded. */
function auditValue(
audits: Record<string, unknown>,
@@ -117,6 +137,11 @@ export async function fetchPagespeedAudit(
fcp: auditValue(audits, 'first-contentful-paint'),
tbt: auditValue(audits, 'total-blocking-time'),
speedIndex: auditValue(audits, 'speed-index'),
tti: auditValue(audits, 'interactive'),
maxPotentialFid: auditValue(audits, 'max-potential-fid'),
// INP is a field metric — read from CrUX loadingExperience, not the
// lab audits. Null when the URL lacks real-user data.
inp: fieldMetricMs(json, 'INTERACTION_TO_NEXT_PAINT'),
status: 'ok',
error: null,
fetchedAt,
@@ -1538,6 +1538,12 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
['OG Title', row.ogTitle],
['OG Description', row.ogDescription],
['OG Image', row.ogImage],
[
'OG Image Dimensions',
row.ogImageWidth && row.ogImageWidth > 0 && row.ogImageHeight && row.ogImageHeight > 0
? `${row.ogImageWidth} × ${row.ogImageHeight}`
: null,
],
['OG Type', row.ogType],
['OG URL', row.ogUrl],
['OG Site Name', row.ogSiteName],
@@ -1546,6 +1552,15 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
['Twitter Title', row.twitterTitle],
['Twitter Description', row.twitterDescription],
['Twitter Image', row.twitterImage],
[
'Twitter Image Dimensions',
row.twitterImageWidth &&
row.twitterImageWidth > 0 &&
row.twitterImageHeight &&
row.twitterImageHeight > 0
? `${row.twitterImageWidth} × ${row.twitterImageHeight}`
: null,
],
['Meta Keywords', row.metaKeywords],
['Meta Author', row.metaAuthor],
['Meta Generator', row.metaGenerator],
@@ -1657,6 +1672,20 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
'Form Inputs Without Label',
row.formInputUnlabeled > 0 ? row.formInputUnlabeled : null,
],
[
'Low Contrast Elements',
row.a11yLowContrast !== null && row.a11yLowContrast !== undefined
? row.a11yLowContrast
: null,
],
[
'Focus Outline Suppressed',
row.a11yFocusSuppressed === 1
? 'Yes'
: row.a11yFocusSuppressed === 0
? 'No'
: null,
],
['Empty Anchor Links', row.emptyAnchorCount > 0 ? row.emptyAnchorCount : null],
['Mixed Content (subresources)', row.mixedContentCount > 0 ? row.mixedContentCount : null],
[
@@ -875,6 +875,18 @@ function buildTree(o: OverviewCounts | null): Node[] {
count: o.issues.ariaInvalidRole,
category: 'issues:aria-invalid-role',
},
{
key: 'issues-low-contrast-text',
label: 'Low Contrast Text',
count: o.issues.lowContrastText,
category: 'issues:low-contrast-text',
},
{
key: 'issues-focus-outline-suppressed',
label: 'Focus Outline Suppressed',
count: o.issues.focusOutlineSuppressed,
category: 'issues:focus-outline-suppressed',
},
],
},
{
@@ -929,6 +941,18 @@ function buildTree(o: OverviewCounts | null): Node[] {
count: o.issues.twitterImageTooLarge,
category: 'issues:twitter-image-too-large',
},
{
key: 'issues-og-image-wrong-aspect',
label: 'OG Image Wrong Aspect Ratio',
count: o.issues.ogImageWrongAspect,
category: 'issues:og-image-wrong-aspect',
},
{
key: 'issues-twitter-image-wrong-aspect',
label: 'Twitter Image Wrong Aspect Ratio',
count: o.issues.twitterImageWrongAspect,
category: 'issues:twitter-image-wrong-aspect',
},
],
},
{
@@ -167,6 +167,7 @@ interface FormState {
jsMobileScreenshot: boolean;
jsMobileUsability: boolean;
jsLcpCandidate: boolean;
jsA11yAudit: boolean;
}
type SectionKey =
@@ -451,6 +452,7 @@ function configToForm(c: CrawlConfig): FormState {
jsMobileScreenshot: c.jsRender?.mobileScreenshot ?? false,
jsMobileUsability: c.jsRender?.mobileUsability ?? false,
jsLcpCandidate: c.jsRender?.lcpCandidate ?? false,
jsA11yAudit: c.jsRender?.a11yAudit ?? false,
};
}
@@ -686,6 +688,7 @@ export function SettingsDialog({ open, onClose }: Props) {
mobileScreenshot: form.jsMobileScreenshot,
mobileUsability: form.jsMobileUsability,
lcpCandidate: form.jsLcpCandidate,
a11yAudit: form.jsA11yAudit,
},
});
onClose();
@@ -3194,6 +3197,14 @@ function RenderingPanel({ form, update }: PanelProps) {
info="Identifies the largest element visible in the initial viewport (likely LCP candidate per Google's heuristic) and stores its CSS selector, dimensions, and resource URL. Useful for spotting unoptimised LCP images without a PSI API call."
example="On for performance-focused audits."
/>
<Bool
label={t('settingsPanels.rendering.a11yAudit', { defaultValue: 'Accessibility audit (contrast + focus)' })}
checked={form.jsA11yAudit}
onChange={(v) => update('jsA11yAudit', v)}
hint=""
info="Audits the rendered DOM for WCAG AA colour-contrast failures (4.5:1 normal text, 3:1 large text) and stylesheet rules that suppress the keyboard focus outline without a :focus-visible fallback. Surfaces the Low-Contrast Text and Focus Outline Suppressed issue filters."
example="On for accessibility / WCAG audits."
/>
</div>
</div>
</>
@@ -240,6 +240,13 @@ const TR_LABELS: Record<string, string> = {
'Twitter Card Missing': 'Twitter Card Eksik',
'Twitter Image Not Absolute': 'Twitter Image Mutlak Değil',
'Twitter Image >5MB': 'Twitter Image >5MB',
'OG Image Wrong Aspect Ratio': 'OG Görsel Yanlış En-Boy Oranı',
'Twitter Image Wrong Aspect Ratio': 'Twitter Görsel Yanlış En-Boy Oranı',
'OG Image Dimensions': 'OG Görsel Boyutları',
'Twitter Image Dimensions': 'Twitter Görsel Boyutları',
'Low Contrast Text': 'Düşük Kontrastlı Metin',
'Focus Outline Suppressed': 'Odak Çerçevesi Bastırılmış',
'Low Contrast Elements': 'Düşük Kontrastlı Öğeler',
// PWA / icons
'Viewport Meta Missing': 'Viewport Meta Eksik',
@@ -598,7 +598,8 @@
"mobileScreenshot": "Also capture mobile viewport (375×667)",
"advancedAudits": "Advanced Audits",
"mobileUsability": "Mobile Usability audit",
"lcpCandidate": "LCP candidate detection"
"lcpCandidate": "LCP candidate detection",
"a11yAudit": "Accessibility audit (contrast + focus)"
},
"perHostUa": {
"intro": "Override the User-Agent on a per-host basis. Useful when crawling a mobile subdomain with the mobile-Googlebot UA in the same run as the desktop site, or when a CDN serves a different page based on the requester's UA. The first matching pattern wins; the global User-Agent (Requests tab) is the fallback.",
@@ -917,6 +918,7 @@
"colStatus": "Status",
"colPerf": "Perf",
"colLcp": "LCP",
"colInp": "INP",
"colCls": "CLS",
"colLastRun": "Last run",
"groupMobile": "Mobile",
@@ -597,7 +597,8 @@
"mobileScreenshot": "Mobil viewport (375×667) için de yakala",
"advancedAudits": "Gelişmiş Denetimler",
"mobileUsability": "Mobil Kullanılabilirlik denetimi",
"lcpCandidate": "LCP aday tespiti"
"lcpCandidate": "LCP aday tespiti",
"a11yAudit": "Erişilebilirlik denetimi (kontrast + odak)"
},
"perHostUa": {
"intro": "User-Agent'ı host bazında geçersiz kıl. Mobil bir alt alan adını mobile-Googlebot UA ile masaüstü sitesiyle aynı koşuda crawl ederken veya bir CDN, isteyene göre farklı sayfa servis ettiğinde faydalı. İlk eşleşen kalıp kazanır; global User-Agent (Requests sekmesi) yedektir.",
@@ -916,6 +917,7 @@
"colStatus": "Durum",
"colPerf": "Perf",
"colLcp": "LCP",
"colInp": "INP",
"colCls": "CLS",
"colLastRun": "Son çalıştırma",
"groupMobile": "Mobil",
@@ -371,7 +371,7 @@ export function PageSpeedTab() {
{t('pagespeedTab.colStatus', { defaultValue: 'Status' })}
</div>
{/* Mobile group */}
<div className="flex w-[162px] flex-col border-r border-surface-800">
<div className="flex w-[218px] flex-col border-r border-surface-800">
<div className="flex h-[22px] items-center justify-center gap-1 border-b border-surface-800 bg-surface-900/80 text-surface-300">
<Smartphone size={11} />
{t('pagespeedTab.groupMobile', { defaultValue: 'Mobile' })}
@@ -379,11 +379,12 @@ export function PageSpeedTab() {
<div className="flex flex-1 items-end pb-1 text-center">
<div className="w-[50px]">{t('pagespeedTab.colPerf', { defaultValue: 'Perf' })}</div>
<div className="w-[60px]">{t('pagespeedTab.colLcp', { defaultValue: 'LCP' })}</div>
<div className="w-[56px]">{t('pagespeedTab.colInp', { defaultValue: 'INP' })}</div>
<div className="w-[52px]">{t('pagespeedTab.colCls', { defaultValue: 'CLS' })}</div>
</div>
</div>
{/* Desktop group */}
<div className="flex w-[162px] flex-col border-r border-surface-800">
<div className="flex w-[218px] flex-col border-r border-surface-800">
<div className="flex h-[22px] items-center justify-center gap-1 border-b border-surface-800 bg-surface-900/80 text-surface-300">
<Monitor size={11} />
{t('pagespeedTab.groupDesktop', { defaultValue: 'Desktop' })}
@@ -391,6 +392,7 @@ export function PageSpeedTab() {
<div className="flex flex-1 items-end pb-1 text-center">
<div className="w-[50px]">{t('pagespeedTab.colPerf', { defaultValue: 'Perf' })}</div>
<div className="w-[60px]">{t('pagespeedTab.colLcp', { defaultValue: 'LCP' })}</div>
<div className="w-[56px]">{t('pagespeedTab.colInp', { defaultValue: 'INP' })}</div>
<div className="w-[52px]">{t('pagespeedTab.colCls', { defaultValue: 'CLS' })}</div>
</div>
</div>
@@ -525,6 +527,7 @@ function StrategyCells({ m }: { m: PagespeedMetrics | null }) {
<>
<MetricCell text="—" cls="text-surface-700" width={50} />
<MetricCell text="—" cls="text-surface-700" width={60} />
<MetricCell text="—" cls="text-surface-700" width={56} />
<MetricCell text="—" cls="text-surface-700" width={52} />
</>
);
@@ -533,7 +536,7 @@ function StrategyCells({ m }: { m: PagespeedMetrics | null }) {
const label = classifyError(m.error);
return (
<div
className="flex w-[162px] shrink-0 cursor-help items-center justify-center gap-1 text-red-400"
className="flex w-[218px] shrink-0 cursor-help items-center justify-center gap-1 text-red-400"
title={m.error ?? 'Audit failed'}
>
<span className="font-semibold">{label}</span>
@@ -553,6 +556,12 @@ function StrategyCells({ m }: { m: PagespeedMetrics | null }) {
cls={cwvClass(m.lcp, 2500, 4000)}
width={60}
/>
<MetricCell
text={fmtMs(m.inp)}
cls={cwvClass(m.inp, 200, 500)}
width={56}
title={`INP (field, CrUX) · TTI ${fmtMs(m.tti)} · Max-FID ${fmtMs(m.maxPotentialFid)}`}
/>
<MetricCell
text={fmtCls(m.cls)}
cls={cwvClass(m.cls, 0.1, 0.25)}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/mcp-server",
"version": "0.6.8",
"version": "0.6.9",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "freecrawl-seo",
"version": "0.6.8",
"version": "0.6.9",
"private": true,
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
"license": "MIT",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/core",
"version": "0.6.8",
"version": "0.6.9",
"private": true,
"type": "module",
"main": "./dist/index.js",
+94
View File
@@ -305,6 +305,11 @@ export class Crawler extends EventEmitter {
overflowPx: number;
hasViewportMeta: boolean;
} | null;
a11y?: {
lowContrast: number;
sampled: number;
focusSuppressed: boolean;
} | null;
}>;
constructor(
@@ -360,6 +365,11 @@ export class Crawler extends EventEmitter {
overflowPx: number;
hasViewportMeta: boolean;
} | null;
a11y?: {
lowContrast: number;
sampled: number;
focusSuppressed: boolean;
} | null;
}>;
} = {},
) {
@@ -791,6 +801,9 @@ export class Crawler extends EventEmitter {
await yieldToEventLoop();
this.setOp('post-crawl:manifest-probes');
await this.runManifestProbes();
await yieldToEventLoop();
this.setOp('post-crawl:social-image-probes');
await this.runSocialImageProbes();
this.running = false;
this.stopMemoryMonitor();
// Release per-URL dedup sets — at 1M URLs this is ~80–120 MB of string
@@ -1041,6 +1054,78 @@ export class Crawler extends EventEmitter {
);
}
/**
* V2 Faz 16 #1 — Post-crawl social-image dimension probe. Ranged-GETs
* each distinct `og:image` / `twitter:image` (heavily deduplicated —
* usually a handful per site), reads its pixel width × height from the
* header, and stamps them onto every referencing page. Feeds the
* social-card aspect-ratio issue filters. Undecodable / failed fetches
* record 0×0 so the image isn't re-probed on the next crawl.
*/
private async runSocialImageProbes(): Promise<void> {
if (!this.config.probeSocialImages) return;
if (this.stopped) return;
let urls: string[] = [];
try {
urls = this.db.unprobedSocialImageUrls(5_000);
} catch (err) {
this.emit(
'info',
`social-image probe skipped (DB query failed): ${
err instanceof Error ? err.message : String(err)
}`,
);
return;
}
if (urls.length === 0) return;
this.emit('info', `Probing ${urls.length} social image(s)…`);
const { probeSocialImageDimensions } = await import('./social-image-probe.js');
const concurrency = Math.max(1, Math.min(this.config.maxConcurrency, 8));
let cursor = 0;
let decoded = 0;
const worker = async (): Promise<void> => {
while (!this.stopped) {
const idx = cursor++;
if (idx >= urls.length) return;
const src = urls[idx];
if (!src) return;
let dims: { width: number; height: number } | null = null;
try {
dims = await probeSocialImageDimensions(src, {
userAgent: this.resolveUserAgent(src),
acceptLanguage: this.config.acceptLanguage,
customHeaders: this.config.customHeaders,
auth: this.config.auth,
});
} catch {
dims = null;
}
try {
// 0×0 marks "probed but undecodable" so the row leaves the
// unprobed set and we don't re-fetch a broken image next crawl.
this.db.setSocialImageDimsForReferrers({
src,
width: dims?.width ?? 0,
height: dims?.height ?? 0,
});
} catch {
/* ignore — best-effort */
}
if (dims) decoded++;
}
};
const workers: Promise<void>[] = [];
for (let i = 0; i < concurrency; i++) workers.push(worker());
await Promise.all(workers);
this.emit(
'info',
`Social-image probe complete: ${decoded}/${urls.length} dimensions read`,
);
}
private async runTlsCertProbes(): Promise<void> {
if (!this.config.probeTlsCerts) return;
if (this.stopped) return;
@@ -1249,6 +1334,8 @@ export class Crawler extends EventEmitter {
await this.runImageSizeProbes();
await yieldToEventLoop();
await this.runTlsCertProbes();
await yieldToEventLoop();
await this.runSocialImageProbes();
this.running = false;
this.stopMemoryMonitor();
this.seen.clear();
@@ -2181,6 +2268,9 @@ export class Crawler extends EventEmitter {
let renderMobile:
| { ok: boolean; overflowPx: number; hasViewportMeta: boolean }
| null = null;
let renderA11y:
| { lowContrast: number; sampled: number; focusSuppressed: boolean }
| null = null;
if (
this.config.renderingMode === 'js' &&
this.renderUrlHook &&
@@ -2211,6 +2301,7 @@ export class Crawler extends EventEmitter {
if (renderRes.screenshots) renderScreenshots = renderRes.screenshots;
if (renderRes.lcp) renderLcp = renderRes.lcp;
if (renderRes.mobileUsability) renderMobile = renderRes.mobileUsability;
if (renderRes.a11y) renderA11y = renderRes.a11y;
} catch (err) {
this.emit(
'warn',
@@ -2466,6 +2557,9 @@ export class Crawler extends EventEmitter {
this.dbCall<void>('setUrlMobileUsability', [urlId, renderMobile]),
);
}
if (renderA11y) {
persistOps.push(this.dbCall<void>('setUrlA11y', [urlId, renderA11y]));
}
if (persistOps.length > 0) {
const results = await Promise.allSettled(persistOps);
for (const r of results) {
+183
View File
@@ -0,0 +1,183 @@
/**
* V2 Faz 16 #1 — Pixel-dimension extraction from raw image bytes.
*
* Parses the width × height out of an image's header without decoding the
* pixels — only the first few hundred bytes are needed for every format
* here, which is why the social-image probe fetches just the leading
* ~64 KB of each `og:image` / `twitter:image`.
*
* Supported: PNG, GIF, JPEG, WebP (VP8 / VP8L / VP8X), BMP. SVG has no
* intrinsic pixel size (it scales) so it returns null — the aspect-ratio
* issue filters skip pages whose probe came back without dimensions.
*/
export interface ImageDimensions {
width: number;
height: number;
}
function be16(b: Uint8Array, o: number): number {
return (b[o]! << 8) | b[o + 1]!;
}
function be32(b: Uint8Array, o: number): number {
// `>>> 0` keeps the result an unsigned 32-bit int.
return ((b[o]! << 24) | (b[o + 1]! << 16) | (b[o + 2]! << 8) | b[o + 3]!) >>> 0;
}
function le16(b: Uint8Array, o: number): number {
return b[o]! | (b[o + 1]! << 8);
}
function le32(b: Uint8Array, o: number): number {
return (b[o]! | (b[o + 1]! << 8) | (b[o + 2]! << 16) | (b[o + 3]! << 24)) >>> 0;
}
function ok(width: number, height: number): ImageDimensions | null {
if (!Number.isFinite(width) || !Number.isFinite(height)) return null;
if (width <= 0 || height <= 0) return null;
// Sanity cap — a value larger than this is almost certainly a misparse.
if (width > 100_000 || height > 100_000) return null;
return { width, height };
}
function parsePng(b: Uint8Array): ImageDimensions | null {
// 8-byte signature + IHDR chunk (length + "IHDR" + width + height + …).
// Width is a big-endian uint32 at offset 16, height at offset 20.
if (b.length < 24) return null;
return ok(be32(b, 16), be32(b, 20));
}
function parseGif(b: Uint8Array): ImageDimensions | null {
// "GIF87a" / "GIF89a" (6) then logical-screen width/height as LE uint16.
if (b.length < 10) return null;
return ok(le16(b, 6), le16(b, 8));
}
function parseBmp(b: Uint8Array): ImageDimensions | null {
// "BM" header; BITMAPINFOHEADER width (LE int32 @18) / height (@22).
// Height can be negative (top-down DIB) so take the magnitude.
if (b.length < 26) return null;
const w = le32(b, 18);
let h = le32(b, 22);
if (h > 0x7fffffff) h = 0x100000000 - h; // interpret as signed, abs
return ok(w, h);
}
function parseJpeg(b: Uint8Array): ImageDimensions | null {
// SOI (FFD8) then a sequence of marker segments. Start-of-Frame markers
// (C0–CF excluding the non-SOF C4/C8/CC) carry height then width as
// big-endian uint16. We walk segments by their length field until we
// hit a SOF or run out of bytes.
if (b.length < 4 || b[0] !== 0xff || b[1] !== 0xd8) return null;
let o = 2;
const len = b.length;
while (o + 9 < len) {
// Markers are byte-aligned on 0xFF; skip any fill bytes.
if (b[o] !== 0xff) {
o++;
continue;
}
let marker = b[o + 1]!;
o += 2;
// Skip padding 0xFF run.
while (marker === 0xff && o < len) {
marker = b[o]!;
o++;
}
// Standalone markers (no length payload): RSTn (D0–D7), SOI/EOI, TEM.
if (marker === 0xd8 || marker === 0xd9 || marker === 0x01) continue;
if (marker >= 0xd0 && marker <= 0xd7) continue;
if (o + 1 >= len) break;
const segLen = be16(b, o);
if (segLen < 2) break;
const isSof =
marker >= 0xc0 &&
marker <= 0xcf &&
marker !== 0xc4 &&
marker !== 0xc8 &&
marker !== 0xcc;
if (isSof) {
// segment: [len(2)][precision(1)][height(2)][width(2)] …
if (o + 7 >= len) break;
return ok(be16(b, o + 5), be16(b, o + 3));
}
o += segLen;
}
return null;
}
function parseWebp(b: Uint8Array): ImageDimensions | null {
// RIFF container: "RIFF"(4) size(4) "WEBP"(4) then a chunk fourcc.
if (b.length < 30) return null;
const fourcc = String.fromCharCode(b[12]!, b[13]!, b[14]!, b[15]!);
if (fourcc === 'VP8 ') {
// Lossy: chunk header(8) then 3-byte frame tag + 3-byte start code,
// then 14-bit width / 14-bit height as LE at chunk-data offset 6/8.
const base = 20; // 12 (RIFF/WEBP) + 8 (chunk header) = 20 → frame tag
if (b.length < base + 10) return null;
const w = le16(b, base + 6) & 0x3fff;
const h = le16(b, base + 8) & 0x3fff;
return ok(w, h);
}
if (fourcc === 'VP8L') {
// Lossless: 1-byte signature (0x2F) then 14-bit (width-1), 14-bit
// (height-1) packed little-endian across 4 bytes.
const base = 21; // 12 + 8 + 1 signature byte
if (b.length < base + 4) return null;
const bits = le32(b, base);
const w = (bits & 0x3fff) + 1;
const h = ((bits >> 14) & 0x3fff) + 1;
return ok(w, h);
}
if (fourcc === 'VP8X') {
// Extended: chunk header(8) + flags(4) then 24-bit (canvas width-1)
// and 24-bit (canvas height-1), both little-endian.
const base = 24; // 12 + 8 + 4 flags
if (b.length < base + 6) return null;
const w = (b[base]! | (b[base + 1]! << 8) | (b[base + 2]! << 16)) + 1;
const h = (b[base + 3]! | (b[base + 4]! << 8) | (b[base + 5]! << 16)) + 1;
return ok(w, h);
}
return null;
}
/**
* Best-effort dimension parse. Returns null for unrecognised / truncated
* data so callers can record "probed but undecodable" without guessing.
*/
export function parseImageDimensions(buf: Uint8Array): ImageDimensions | null {
if (buf.length < 10) return null;
// PNG: 89 50 4E 47 0D 0A 1A 0A
if (
buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47
) {
return parsePng(buf);
}
// GIF: "GIF8"
if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x38) {
return parseGif(buf);
}
// JPEG: FF D8 FF
if (buf[0] === 0xff && buf[1] === 0xd8) {
return parseJpeg(buf);
}
// WebP: "RIFF"…"WEBP"
if (
buf[0] === 0x52 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x46 &&
buf[8] === 0x57 &&
buf[9] === 0x45 &&
buf[10] === 0x42 &&
buf[11] === 0x50
) {
return parseWebp(buf);
}
// BMP: "BM"
if (buf[0] === 0x42 && buf[1] === 0x4d) {
return parseBmp(buf);
}
return null;
}
+132
View File
@@ -13,6 +13,8 @@ export interface RenderOptions {
screenshotMode?: 'none' | 'fullpage' | 'fold' | 'both';
/** Detect the LCP candidate element after render. */
detectLcp?: boolean;
/** Run the in-page accessibility audit (contrast + focus outline). */
detectA11y?: boolean;
/** Where to write screenshot PNGs — the caller resolves a per-URL path. */
screenshotPaths?: {
fullpage?: string;
@@ -55,6 +57,18 @@ export interface RenderResult {
screenshots?: { fullpage?: string; fold?: string };
/** LCP candidate from `detectLcp` — null when no eligible element found. */
lcp?: LcpCandidate | null;
/** Accessibility audit from `detectA11y` — null when it could not run. */
a11y?: A11yAudit | null;
}
/** V2 Faz 16 — in-page accessibility audit result. */
export interface A11yAudit {
/** Sampled text elements whose contrast is below WCAG AA. */
lowContrast: number;
/** How many text elements were sampled (context for `lowContrast`). */
sampled: number;
/** A stylesheet rule suppresses the focus outline without a fallback. */
focusSuppressed: boolean;
}
/**
@@ -127,6 +141,112 @@ const LCP_DETECT_FN = `(() => {
};
})()`;
/**
* Browser-side accessibility audit. Stringified + injected via
* `page.evaluate()`. Two checks:
*
* 1. **Colour contrast** — samples up to 500 visible elements that have
* direct text, computes the WCAG contrast ratio between the text
* colour and the effective background (walking ancestors until an
* opaque background-colour is found; default white). Elements over a
* background-image are skipped (contrast is indeterminate). Threshold
* is 3:1 for large text (>=24px, or >=18.66px bold) and 4.5:1
* otherwise — the WCAG 2.1 AA minimums.
* 2. **Focus outline suppression** — scans accessible stylesheets for a
* `:focus` rule that removes the outline (`outline: none/0`) without a
* compensating box-shadow/border, and with no `:focus-visible` rule
* restoring a visible indicator. Catches the common
* `*:focus { outline: none }` keyboard-trap anti-pattern. Cross-origin
* sheets (whose cssRules throw) are skipped.
*/
const A11Y_AUDIT_FN = `(() => {
const parseColor = (s) => {
const m = s && s.match(/rgba?\\(([^)]+)\\)/);
if (!m) return null;
const p = m[1].split(',').map((x) => parseFloat(x.trim()));
if (p.length < 3 || p.some((n) => !isFinite(n))) return null;
return { r: p[0], g: p[1], b: p[2], a: p.length >= 4 ? p[3] : 1 };
};
const relLum = (c) => {
const f = (v) => { v = v / 255; return v <= 0.03928 ? v / 12.92 : Math.pow((v + 0.055) / 1.055, 2.4); };
return 0.2126 * f(c.r) + 0.7152 * f(c.g) + 0.0722 * f(c.b);
};
const ratio = (fg, bg) => {
const l1 = relLum(fg), l2 = relLum(bg);
const hi = Math.max(l1, l2), lo = Math.min(l1, l2);
return (hi + 0.05) / (lo + 0.05);
};
const over = (top, bottom) => {
if (top.a >= 1) return top;
const a = top.a;
return { r: top.r * a + bottom.r * (1 - a), g: top.g * a + bottom.g * (1 - a), b: top.b * a + bottom.b * (1 - a), a: 1 };
};
const effectiveBg = (el) => {
let node = el;
while (node && node.nodeType === 1) {
const cs = getComputedStyle(node);
if (cs.backgroundImage && cs.backgroundImage !== 'none') return null;
const c = parseColor(cs.backgroundColor);
if (c && c.a > 0) return c.a >= 1 ? c : over(c, { r: 255, g: 255, b: 255, a: 1 });
node = node.parentElement;
}
return { r: 255, g: 255, b: 255, a: 1 };
};
let sampled = 0, low = 0;
const all = document.body ? document.body.querySelectorAll('*') : [];
for (const el of all) {
if (sampled >= 500) break;
let hasText = false;
for (const n of el.childNodes) {
if (n.nodeType === 3 && n.textContent && n.textContent.trim().length > 1) { hasText = true; break; }
}
if (!hasText) continue;
const rect = el.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) continue;
const cs = getComputedStyle(el);
if (cs.visibility === 'hidden' || cs.display === 'none' || parseFloat(cs.opacity || '1') === 0) continue;
const fg = parseColor(cs.color);
if (!fg) continue;
const bg = effectiveBg(el);
if (!bg) continue;
const fgc = fg.a < 1 ? over(fg, bg) : fg;
sampled++;
const fontSize = parseFloat(cs.fontSize || '16');
let weight = parseInt(cs.fontWeight, 10);
if (!isFinite(weight)) weight = cs.fontWeight === 'bold' ? 700 : 400;
const large = fontSize >= 24 || (fontSize >= 18.66 && weight >= 700);
const threshold = large ? 3.0 : 4.5;
if (ratio(fgc, bg) < threshold - 0.05) low++;
}
let focusSuppressed = false, focusVisibleRestores = false;
try {
for (const sheet of document.styleSheets) {
let rules = null;
try { rules = sheet.cssRules; } catch (e) { continue; }
if (!rules) continue;
for (const rule of rules) {
if (rule.type !== 1 || !rule.selectorText) continue;
const sel = rule.selectorText;
if (sel.indexOf(':focus') === -1) continue;
const st = rule.style;
const ow = parseFloat(st.outlineWidth || '');
const hasOutline = (st.outlineStyle && st.outlineStyle !== 'none' && (isNaN(ow) || ow !== 0));
const hasShadow = st.boxShadow && st.boxShadow !== 'none';
const hasBorder = st.border && st.border !== 'none' && !/^0/.test(st.border) && st.border.indexOf('none') === -1;
if (sel.indexOf(':focus-visible') !== -1) {
if (hasOutline || hasShadow) focusVisibleRestores = true;
continue;
}
const ct = (rule.cssText || '');
const removesOutline = st.outlineStyle === 'none' || ow === 0 || /outline\\s*:\\s*(none|0)/.test(ct);
if (removesOutline && !hasShadow && !hasBorder && !hasOutline) focusSuppressed = true;
}
}
} catch (e) { /* ignore */ }
if (focusVisibleRestores) focusSuppressed = false;
return { lowContrast: low, sampled: sampled, focusSuppressed: focusSuppressed };
})()`;
/**
* Render a URL via Playwright and return a Response-like result. Cookies are
* cleared from the BrowserContext between calls upstream — the pool keeps
@@ -213,6 +333,17 @@ export async function renderUrl(
}
}
// Accessibility audit — contrast + focus outline. Best-effort: a
// failing evaluate (detached page, hostile CSP) leaves a11y null.
let a11y: A11yAudit | null = null;
if (opts.detectA11y) {
try {
a11y = (await page.evaluate(A11Y_AUDIT_FN)) as A11yAudit | null;
} catch {
a11y = null;
}
}
// Screenshot capture — best-effort. Failures (e.g. detached page)
// do not fail the render result.
const screenshots: { fullpage?: string; fold?: string } = {};
@@ -254,6 +385,7 @@ export async function renderUrl(
ok: true,
screenshots: Object.keys(screenshots).length > 0 ? screenshots : undefined,
lcp: opts.detectLcp ? lcp : undefined,
a11y: opts.detectA11y ? a11y : undefined,
};
} catch (err) {
result = {
+94
View File
@@ -0,0 +1,94 @@
import { fetch as undiciFetch } from 'undici';
import { defaultRequestHeaders } from './http-client.js';
import { parseImageDimensions, type ImageDimensions } from './image-dimensions.js';
import type { CrawlConfig } from '@freecrawl/shared-types';
const PROBE_TIMEOUT_MS = 8_000;
/** 64 KB is enough for every header format we parse, even a JPEG whose
* Start-of-Frame marker sits after a large EXIF/ICC block. */
const MAX_BYTES = 65_536;
/**
* V2 Faz 16 #1 — Fetch just enough of a social image (`og:image` /
* `twitter:image`) to read its pixel dimensions, then stop. A `Range`
* request asks for only the first 64 KB; servers that ignore Range get
* cut off by the streaming read so we never download a multi-MB hero
* image in full. `accept-encoding: identity` keeps the byte offsets in
* the header aligned (a gzipped range would decode to a different prefix).
*
* Best-effort: any failure (network, 4xx/5xx, unknown format, truncated
* header) returns null so the caller records "probed, undecodable" and
* doesn't retry on the next crawl.
*/
export async function probeSocialImageDimensions(
imageUrl: string,
config: Pick<CrawlConfig, 'userAgent' | 'acceptLanguage' | 'customHeaders' | 'auth'>,
): Promise<ImageDimensions | null> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS);
try {
const headers = defaultRequestHeaders(
config.userAgent,
config.acceptLanguage,
config.customHeaders,
config.auth,
);
headers['range'] = `bytes=0-${MAX_BYTES - 1}`;
headers['accept-encoding'] = 'identity';
headers['accept'] = 'image/avif,image/webp,image/png,image/*,*/*;q=0.8';
const res = await undiciFetch(imageUrl, {
method: 'GET',
headers,
redirect: 'follow',
signal: controller.signal,
});
// 200 (Range ignored) and 206 (Partial Content) are both fine.
if (res.status < 200 || res.status >= 300 || !res.body) {
try {
await res.body?.cancel();
} catch {
/* ignore */
}
return null;
}
const reader = res.body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
while (total < MAX_BYTES) {
const { done, value } = await reader.read();
if (done) break;
if (value && value.length > 0) {
chunks.push(value);
total += value.length;
}
}
} finally {
try {
await reader.cancel();
} catch {
/* ignore */
}
}
if (total === 0) return null;
const cap = Math.min(total, MAX_BYTES);
const buf = new Uint8Array(cap);
let off = 0;
for (const chunk of chunks) {
if (off >= cap) break;
const slice = chunk.subarray(0, cap - off);
buf.set(slice, off);
off += slice.length;
}
return parseImageDimensions(buf);
} catch {
return null;
} finally {
clearTimeout(timer);
}
}
export type { ImageDimensions };
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/db",
"version": "0.6.8",
"version": "0.6.9",
"private": true,
"type": "module",
"main": "./dist/index.js",
+81
View File
@@ -1546,6 +1546,87 @@ const MIGRATIONS: Migration[] = [
}
},
},
{
version: 66,
name: 'add_social_image_dimensions',
// V2 Faz 16 #1 — Social image aspect-ratio validation. The post-crawl
// `runSocialImageProbes()` pass does a ranged GET on each distinct
// og:image / twitter:image URL, parses the pixel width × height out
// of the image header (PNG/JPEG/GIF/WebP/BMP), and stamps them onto
// every page that referenced that image. NULL = not yet probed;
// 0 = probed but undecodable / fetch failed (so we don't re-probe a
// broken image every crawl); >0 = real dimensions. The dedicated
// `og-image-wrong-aspect` / `twitter-image-wrong-aspect` issue
// filters read these to flag images that won't render as a proper
// share card (Facebook/LinkedIn 1.91:1, Twitter 2:1 or 1:1).
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(urls)')
.all() as unknown as { name: string }[];
for (const col of [
'og_image_width',
'og_image_height',
'twitter_image_width',
'twitter_image_height',
]) {
if (!cols.some((c) => c.name === col)) {
db.exec(`ALTER TABLE urls ADD COLUMN ${col} INTEGER`);
}
}
},
},
{
version: 67,
name: 'add_pagespeed_interactivity_metrics',
// V2 Faz 15 #1 — extend the PageSpeed audit row with the
// interactivity metrics PSI returns beyond the existing LCP/CLS/FCP/
// TBT/SpeedIndex set:
// tti — Time to Interactive (lab, `interactive`), ms
// max_potential_fid — Max Potential First Input Delay (lab,
// `max-potential-fid`), ms — Lighthouse's
// worst-case input-delay estimate, the closest
// lab proxy for the deprecated field FID.
// inp — Interaction to Next Paint, ms. INP is a field
// (CrUX) metric and the Core Web Vital that
// replaced FID in March 2024; pulled from the
// PSI `loadingExperience` block, NULL when the
// URL has no real-user data.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(pagespeed_results)')
.all() as unknown as { name: string }[];
for (const col of ['tti', 'max_potential_fid', 'inp']) {
if (!cols.some((c) => c.name === col)) {
db.exec(`ALTER TABLE pagespeed_results ADD COLUMN ${col} REAL`);
}
}
},
},
{
version: 68,
name: 'add_a11y_audit_columns',
// V2 Faz 16 — accessibility audit from the JS-render in-page pass.
// a11y_low_contrast — count of sampled text elements whose
// colour contrast is below WCAG AA
// (4.5:1 normal / 3:1 large text). NULL
// when the page wasn't rendered with the
// a11y audit on; >=0 once audited.
// a11y_focus_suppressed — 0/1: a stylesheet rule removes the
// keyboard focus outline (`:focus { outline:
// none }`) without a compensating indicator
// and no `:focus-visible` fallback. NULL =
// not audited.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(urls)')
.all() as unknown as { name: string }[];
for (const col of ['a11y_low_contrast', 'a11y_focus_suppressed']) {
if (!cols.some((c) => c.name === col)) {
db.exec(`ALTER TABLE urls ADD COLUMN ${col} INTEGER`);
}
}
},
},
];
export function runMigrations(db: DatabaseSync): void {
+177 -1
View File
@@ -80,6 +80,12 @@ interface UrlRowDb {
twitter_title: string | null;
twitter_description: string | null;
twitter_image: string | null;
/** V2 Faz 16 #1 — social image pixel dimensions from the post-crawl
* ranged-GET probe. NULL = not probed, 0 = undecodable, >0 = real. */
og_image_width: number | null;
og_image_height: number | null;
twitter_image_width: number | null;
twitter_image_height: number | null;
meta_keywords: string | null;
meta_author: string | null;
meta_generator: string | null;
@@ -105,6 +111,10 @@ interface UrlRowDb {
* from the sampled post-crawl pass. NULL when the page wasn't in
* the sample (body snapshot missing, or sample LIMIT exceeded). */
boilerplate_coverage: number | null;
/** V2 Faz 16 — accessibility audit from the JS-render in-page pass.
* NULL when not audited; a11y_low_contrast >=0, a11y_focus_suppressed 0/1. */
a11y_low_contrast: number | null;
a11y_focus_suppressed: number | null;
favicon: string | null;
mixed_content_count: number;
mixed_content_active: number;
@@ -2991,8 +3001,9 @@ export class ProjectDb {
.prepare(
`INSERT INTO pagespeed_results
(url, strategy, performance, lcp, cls, fcp, tbt, speed_index,
tti, max_potential_fid, inp,
status, error, fetched_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(url, strategy) DO UPDATE SET
performance = excluded.performance,
lcp = excluded.lcp,
@@ -3000,6 +3011,9 @@ export class ProjectDb {
fcp = excluded.fcp,
tbt = excluded.tbt,
speed_index = excluded.speed_index,
tti = excluded.tti,
max_potential_fid = excluded.max_potential_fid,
inp = excluded.inp,
status = excluded.status,
error = excluded.error,
fetched_at = excluded.fetched_at`,
@@ -3013,6 +3027,9 @@ export class ProjectDb {
m.fcp,
m.tbt,
m.speedIndex,
m.tti,
m.maxPotentialFid,
m.inp,
m.status,
m.error,
m.fetchedAt,
@@ -3060,9 +3077,11 @@ export class ProjectDb {
`SELECT u.url AS url, u.status_code AS status_code,
pm.performance AS m_perf, pm.lcp AS m_lcp, pm.cls AS m_cls,
pm.fcp AS m_fcp, pm.tbt AS m_tbt, pm.speed_index AS m_si,
pm.tti AS m_tti, pm.max_potential_fid AS m_mpfid, pm.inp AS m_inp,
pm.status AS m_status, pm.error AS m_error, pm.fetched_at AS m_fetched,
pd.performance AS d_perf, pd.lcp AS d_lcp, pd.cls AS d_cls,
pd.fcp AS d_fcp, pd.tbt AS d_tbt, pd.speed_index AS d_si,
pd.tti AS d_tti, pd.max_potential_fid AS d_mpfid, pd.inp AS d_inp,
pd.status AS d_status, pd.error AS d_error, pd.fetched_at AS d_fetched
${joinSql}
${whereSql}
@@ -3087,6 +3106,9 @@ export class ProjectDb {
fcp: (r[`${p}_fcp`] as number | null) ?? null,
tbt: (r[`${p}_tbt`] as number | null) ?? null,
speedIndex: (r[`${p}_si`] as number | null) ?? null,
tti: (r[`${p}_tti`] as number | null) ?? null,
maxPotentialFid: (r[`${p}_mpfid`] as number | null) ?? null,
inp: (r[`${p}_inp`] as number | null) ?? null,
status: status === 'ok' ? 'ok' : 'error',
error: (r[`${p}_error`] as string | null) ?? null,
fetchedAt: (r[`${p}_fetched`] as string | null) ?? '',
@@ -4533,6 +4555,31 @@ export class ProjectDb {
AND i.byte_size > 5242880
)`,
),
ogImageWrongAspect: countWhere(
`${html} AND og_image IS NOT NULL AND og_image != ''
AND og_image_width > 0 AND og_image_height > 0
AND (
og_image_width < 200 OR og_image_height < 200
OR (CAST(og_image_width AS REAL) / og_image_height) NOT BETWEEN 1.5 AND 2.3
)`,
),
twitterImageWrongAspect: countWhere(
`${html} AND twitter_image IS NOT NULL AND twitter_image != ''
AND twitter_image_width > 0 AND twitter_image_height > 0
AND (
(LOWER(COALESCE(twitter_card, '')) = 'summary_large_image' AND (
twitter_image_width < 300 OR twitter_image_height < 157
OR (CAST(twitter_image_width AS REAL) / twitter_image_height) NOT BETWEEN 1.7 AND 2.3))
OR
(LOWER(COALESCE(twitter_card, '')) != 'summary_large_image' AND (
twitter_image_width < 144 OR twitter_image_height < 144
OR (CAST(twitter_image_width AS REAL) / twitter_image_height) NOT BETWEEN 0.8 AND 1.25))
)`,
),
lowContrastText: countWhere(
`${html} AND a11y_low_contrast IS NOT NULL AND a11y_low_contrast > 0`,
),
focusOutlineSuppressed: countWhere(`${html} AND a11y_focus_suppressed = 1`),
paginationSequenceBreak: countWhere(
`${html} AND pagination_sequence_break = 1`,
),
@@ -5022,6 +5069,30 @@ export class ProjectDb {
);
}
/**
* V2 Faz 16 — persist the in-page accessibility audit (contrast +
* focus outline) for a URL. Called from the JS-render path when the
* a11y audit is enabled. Stores the low-contrast element count and a
* 0/1 focus-suppression flag; NULL stays for pages never audited.
*/
setUrlA11y(
urlId: number,
audit: { lowContrast: number; sampled: number; focusSuppressed: boolean },
): void {
this.db
.prepare(
`UPDATE urls
SET a11y_low_contrast = ?,
a11y_focus_suppressed = ?
WHERE id = ?`,
)
.run(
Math.max(0, Math.round(audit.lowContrast)),
audit.focusSuppressed ? 1 : 0,
urlId,
);
}
/**
* Persist (or overwrite) the post-JS rendered HTML for a URL.
* Called when `renderingMode === 'js'` after Playwright finishes
@@ -5539,6 +5610,61 @@ export class ProjectDb {
});
}
/**
* V2 Faz 16 #1 — Distinct `og:image` / `twitter:image` URLs that
* haven't had their pixel dimensions probed yet. A single image is
* usually shared across many pages (site-wide default OG card), so the
* DISTINCT set is tiny relative to the URL count. "Unprobed" = the
* matching width column is still NULL; a failed probe writes 0 so the
* URL drops out of this set and isn't re-fetched on the next crawl.
*/
unprobedSocialImageUrls(limit = 5_000): string[] {
const rows = this.db
.prepare(
`SELECT DISTINCT src FROM (
SELECT og_image AS src, og_image_width AS w FROM urls
WHERE og_image IS NOT NULL AND og_image != ''
AND content_kind = 'html'
AND og_image_width IS NULL
UNION
SELECT twitter_image AS src, twitter_image_width AS w FROM urls
WHERE twitter_image IS NOT NULL AND twitter_image != ''
AND content_kind = 'html'
AND twitter_image_width IS NULL
)
WHERE src LIKE 'http%'
LIMIT ?`,
)
.all(limit) as { src: string }[];
return rows.map((r) => r.src).filter((s) => typeof s === 'string' && s.length > 0);
}
/**
* Stamp probed pixel dimensions onto every page that references this
* image as its `og:image` and/or `twitter:image`. `width`/`height` of
* 0 marks the image as probed-but-undecodable (fetch failed, unknown
* format, truncated header) so it isn't re-probed next crawl. One image
* → fan-out write across all referrers, mirroring `setManifestForReferrers`.
*/
setSocialImageDimsForReferrers(input: {
src: string;
width: number;
height: number;
}): void {
this.db
.prepare(
`UPDATE urls SET og_image_width = :w, og_image_height = :h
WHERE og_image = :src AND og_image_width IS NULL`,
)
.run({ src: input.src, w: input.width, h: input.height });
this.db
.prepare(
`UPDATE urls SET twitter_image_width = :w, twitter_image_height = :h
WHERE twitter_image = :src AND twitter_image_width IS NULL`,
)
.run({ src: input.src, w: input.width, h: input.height });
}
/**
* Look up the cached TLS cert info for a single URL — joins the URL's
* host (parsed from the stored URL) against `host_certs`. Returns null
@@ -6160,6 +6286,10 @@ export class ProjectDb {
twitterTitle: r.twitter_title,
twitterDescription: r.twitter_description,
twitterImage: r.twitter_image,
ogImageWidth: r.og_image_width ?? null,
ogImageHeight: r.og_image_height ?? null,
twitterImageWidth: r.twitter_image_width ?? null,
twitterImageHeight: r.twitter_image_height ?? null,
metaKeywords: r.meta_keywords,
metaAuthor: r.meta_author,
metaGenerator: r.meta_generator,
@@ -6180,6 +6310,8 @@ export class ProjectDb {
ampPage: r.amp_page === 1,
ampValidationErrors: r.amp_validation_errors,
boilerplateCoverage: r.boilerplate_coverage,
a11yLowContrast: r.a11y_low_contrast ?? null,
a11yFocusSuppressed: r.a11y_focus_suppressed ?? null,
favicon: r.favicon,
mixedContentCount: r.mixed_content_count,
mixedContentActive: r.mixed_content_active ?? 0,
@@ -7711,6 +7843,50 @@ function categoryWhereClause(cat: UrlCategory): string | null {
AND i.byte_size IS NOT NULL
AND i.byte_size > 5242880
)`;
case 'issues:og-image-wrong-aspect':
// og:image dimensions that won't render as a proper Facebook /
// LinkedIn share card. Probed by the post-crawl social-image pass
// (width > 0 = decoded). Flags too-small images (< 200 px either
// side — share renderers drop them) and aspect ratios far from the
// recommended 1.91:1 (1200×630). The 1.5–2.3 band keeps the
// canonical 1.91 ratio comfortably inside while catching square
// (1.0) and portrait images that get awkwardly cropped.
return `is_external = 0 AND content_kind = 'html'
AND og_image IS NOT NULL AND og_image != ''
AND og_image_width > 0 AND og_image_height > 0
AND (
og_image_width < 200 OR og_image_height < 200
OR (CAST(og_image_width AS REAL) / og_image_height) NOT BETWEEN 1.5 AND 2.3
)`;
case 'issues:twitter-image-wrong-aspect':
// twitter:image dimensions that don't fit the declared card type.
// `summary_large_image` wants ~2:1 (min 300×157); `summary` and the
// implicit default want ~1:1 (min 144×144). Probed width > 0 means
// decoded. Card type read from twitter_card (case-insensitive).
return `is_external = 0 AND content_kind = 'html'
AND twitter_image IS NOT NULL AND twitter_image != ''
AND twitter_image_width > 0 AND twitter_image_height > 0
AND (
(LOWER(COALESCE(twitter_card, '')) = 'summary_large_image' AND (
twitter_image_width < 300 OR twitter_image_height < 157
OR (CAST(twitter_image_width AS REAL) / twitter_image_height) NOT BETWEEN 1.7 AND 2.3))
OR
(LOWER(COALESCE(twitter_card, '')) != 'summary_large_image' AND (
twitter_image_width < 144 OR twitter_image_height < 144
OR (CAST(twitter_image_width AS REAL) / twitter_image_height) NOT BETWEEN 0.8 AND 1.25))
)`;
case 'issues:low-contrast-text':
// Pages with at least one text element below the WCAG AA contrast
// ratio, from the JS-render in-page audit. NULL (not audited)
// excluded — only flag pages we actually measured.
return `is_external = 0 AND content_kind = 'html'
AND a11y_low_contrast IS NOT NULL AND a11y_low_contrast > 0`;
case 'issues:focus-outline-suppressed':
// Pages whose stylesheets remove the keyboard focus outline without
// a compensating indicator (and no :focus-visible fallback). NULL
// (not audited) excluded.
return `is_external = 0 AND content_kind = 'html'
AND a11y_focus_suppressed = 1`;
case 'issues:pagination-sequence-break':
// Set post-crawl by `recomputePaginationSequence()` — see the
// implementation comment there for the gap-detection algorithm
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/shared-types",
"version": "0.6.8",
"version": "0.6.9",
"private": true,
"type": "module",
"main": "./dist/index.js",
+67
View File
@@ -194,6 +194,10 @@ export type UrlCategory =
| 'issues:render-blocking-critical'
| 'issues:og-image-too-large'
| 'issues:twitter-image-too-large'
| 'issues:og-image-wrong-aspect'
| 'issues:twitter-image-wrong-aspect'
| 'issues:low-contrast-text'
| 'issues:focus-outline-suppressed'
| 'issues:pagination-sequence-break'
| 'issues:links-per-page-too-many'
| 'tab:redirects'
@@ -389,6 +393,17 @@ export interface CrawlUrlRow {
twitterTitle: string | null;
twitterDescription: string | null;
twitterImage: string | null;
/**
* V2 Faz 16 #1 — Pixel dimensions of the `og:image` / `twitter:image`,
* resolved by the post-crawl social-image probe (ranged GET + header
* parse). `null` = not probed (no social image, or probe disabled);
* `0` = probed but undecodable / fetch failed; `>0` = real dimension.
* Drive the social-card aspect-ratio issue filters.
*/
ogImageWidth: number | null;
ogImageHeight: number | null;
twitterImageWidth: number | null;
twitterImageHeight: number | null;
metaKeywords: string | null;
metaAuthor: string | null;
metaGenerator: string | null;
@@ -462,6 +477,16 @@ export interface CrawlUrlRow {
* are flagged as templated / thin-content. NULL when the page
* wasn't in the sample (no body snapshot, or sample LIMIT exceeded). */
boilerplateCoverage: number | null;
/**
* V2 Faz 16 — accessibility audit from the JS-render in-page pass.
* `a11yLowContrast` = number of sampled text elements below the WCAG AA
* contrast threshold. `a11yFocusSuppressed` = 1 when a stylesheet rule
* removes the keyboard focus outline without a compensating indicator,
* 0 when clean. Both `null` when the page wasn't rendered with the
* a11y audit enabled (text mode, or audit off).
*/
a11yLowContrast: number | null;
a11yFocusSuppressed: number | null;
favicon: string | null;
/** Resolved `<link rel="apple-touch-icon">` URL, else null. */
appleTouchIcon: string | null;
@@ -808,6 +833,16 @@ export interface CrawlConfig {
* fetch per unique manifest URL, typically site-wide singleton.
*/
probeManifestJson: boolean;
/**
* After the HTML crawl finishes, ranged-GET each distinct `og:image` /
* `twitter:image` once to read its pixel dimensions, then stamp them
* onto every page referencing that image. Feeds the social-card
* aspect-ratio issue filters (Facebook/LinkedIn 1.91:1, Twitter 2:1
* or 1:1). Default `true` — social images are heavily deduplicated
* across a site so the probe count is tiny. Only the first ~64 KB of
* each image is read (enough for the header), so cost is minimal.
*/
probeSocialImages: boolean;
/**
* When the post-norm "Duplicate URL" issue filter runs, normalise
* URLs (lowercase, strip query, trim trailing slash) before
@@ -1001,6 +1036,15 @@ export interface JsRenderConfig {
* requiring a PSI API call.
*/
lcpCandidate: boolean;
/**
* Run an in-page accessibility audit on the rendered DOM: count text
* elements whose colour contrast falls below the WCAG AA threshold
* (4.5:1 normal, 3:1 large text) and detect stylesheet rules that
* suppress the keyboard focus outline without a compensating
* indicator. Requires the rendered DOM's computed styles, so it only
* runs in JS mode. Off by default — adds an in-page evaluate pass.
*/
a11yAudit: boolean;
}
export interface HttpAuth {
@@ -1426,6 +1470,27 @@ export interface OverviewCounts {
* conservative threshold that catches both card types' renderer.
*/
twitterImageTooLarge: number;
/**
* Pages whose `og:image` dimensions won't render as a proper share
* card — too small (<200 px either side) or aspect ratio far from
* the Facebook/LinkedIn 1.91:1 recommendation. From the post-crawl
* social-image probe.
*/
ogImageWrongAspect: number;
/**
* V2 Faz 16 — text elements below the WCAG AA contrast threshold,
* and pages whose stylesheets suppress the keyboard focus outline.
* From the JS-render in-page accessibility audit (only populated
* when that audit is enabled).
*/
lowContrastText: number;
focusOutlineSuppressed: number;
/**
* Pages whose `twitter:image` dimensions don't fit the declared
* card type — `summary_large_image` wants ~2:1 (min 300×157),
* `summary` (and the default) wants ~1:1 (min 144×144).
*/
twitterImageWrongAspect: number;
/**
* Pages part of a paginated cluster whose ordinals have a gap
* (e.g. ?page=1, ?page=2, ?page=4 — page 3 missing). Set by the
@@ -1692,6 +1757,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
largeImageBytes: 102_400,
probeTlsCerts: true,
probeManifestJson: true,
probeSocialImages: true,
dedupePreNormalize: true,
cdnHosts: [],
maxLinksPerPage: 100,
@@ -1733,5 +1799,6 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
mobileScreenshot: false,
mobileUsability: false,
lcpCandidate: false,
a11yAudit: false,
},
};
+14
View File
@@ -37,6 +37,20 @@ export interface PagespeedMetrics {
tbt: number | null;
/** Speed Index (ms). */
speedIndex: number | null;
/** Time to Interactive (ms) — lab audit `interactive`. */
tti: number | null;
/**
* Max Potential First Input Delay (ms) — lab audit `max-potential-fid`,
* Lighthouse's worst-case input-delay estimate. The closest lab proxy
* for the deprecated field FID metric.
*/
maxPotentialFid: number | null;
/**
* Interaction to Next Paint (ms) — the Core Web Vital that replaced FID
* in March 2024. Field (CrUX) metric from the PSI `loadingExperience`
* block; `null` when the URL has no real-user data yet.
*/
inp: number | null;
/** `ok` when the audit completed, `error` when PSI/Lighthouse failed. */
status: 'ok' | 'error';
/** Human-readable failure reason when `status` is `error`. */