feat: performance budget + structured-data recommended-props validation

This commit is contained in:
kemalai committed 2026-06-18 02:24:19 +03:00
1 parent 29f54bff32
commit fcbc4b2a5d
21 files changed
+534 -21

No files matched your search

+1 -1
View File
@@ -29,7 +29,7 @@ A high-performance website crawler built for serious SEO audits. Targets <kbd>1M
## ✨ What FreeCrawl Does
FreeCrawl is a desktop SEO crawler that scales to **1M+ URLs on a single machine** at 80–150 URL/s via undici with keep-alive, with optional **JavaScript rendering** through headless Chromium (Playwright) that captures the post-JS DOM, full-page / above-fold / mobile screenshots, LCP candidate elements, and a per-URL mobile usability audit. It runs **150+ on-page SEO checks across 30 top-level tabs** — exact and near-duplicate clustering (SimHash + LSH) with a dedicated Cluster view, full hreflang validation (reciprocity, self-reference, inconsistent lang), OpenGraph / Twitter Card / JSON-LD / Web App Manifest parsing, AMP smoke validation, structured-data validation (duplicate `@id`, malformed `@type`, missing required props), WCAG accessibility checks (`<main>` landmark, skip-link, ARIA roles, heading order), security headers + SSL/TLS chain audit + active/passive mixed content split, readability scores (Flesch, Flesch–Kincaid, Gunning Fog), and a 10-rule custom CSS + regex extraction engine with a **live Preview dialog and JSON import / export** for sharing rule sets.
FreeCrawl is a desktop SEO crawler that scales to **1M+ URLs on a single machine** at 80–150 URL/s via undici with keep-alive, with optional **JavaScript rendering** through headless Chromium (Playwright) that captures the post-JS DOM, full-page / above-fold / mobile screenshots, LCP candidate elements, and a per-URL mobile usability audit. It runs **150+ on-page SEO checks across 30 top-level tabs** — exact and near-duplicate clustering (SimHash + LSH) with a dedicated Cluster view, full hreflang validation (reciprocity, self-reference, inconsistent lang), OpenGraph / Twitter Card / JSON-LD / Web App Manifest parsing, AMP smoke validation, structured-data validation across 17 schema types (duplicate `@id`, malformed `@type`, missing required + Google-recommended props), a configurable performance budget (response-time / page-size / LCP / CLS ceilings), WCAG accessibility checks (`<main>` landmark, skip-link, ARIA roles, heading order), security headers + SSL/TLS chain audit + active/passive mixed content split, readability scores (Flesch, Flesch–Kincaid, Gunning Fog), and a 10-rule custom CSS + regex extraction engine with a **live Preview dialog and JSON import / export** for sharing rule sets.
The **dense dark UI** renders virtualized 1M-row tables that live-stream rows during a crawl, with **List ↔ Tree ↔ Cluster view toggles**, **column pin (sticky-left) + drag-to-reorder + show/hide**, advanced AND/OR filters across 24 fields × 12 operators, per-tab quick-filter dropdowns, a 16-tab Details panel per URL (URL Details, Outline, Inlinks, Outlinks, Images, Resources, Extracted Data, SERP Snippet, HTTP Headers, Cookies, Structured Data, View Source, View Rendered, Screenshot, Duplicates, Analytics), an English + Turkish UI including every Settings panel, **in-app scheduled crawls** (hourly / daily / weekly / custom), a live memory monitor, robots.txt syntax validator, URL rewriting with live preview, project-vs-project compare diff, a Cytoscape link graph in its own native window, and a `.seoproject` file association for OS double-click.
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/cli",
"version": "0.6.9",
"version": "0.7.0",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/desktop",
"version": "0.6.9",
"version": "0.7.0",
"private": true,
"description": "FreeCrawl SEO — desktop application",
"homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
+4
View File
@@ -743,6 +743,10 @@ function getDb(): ProjectDb {
parsed.jsRender?.a11yAudit ??
DEFAULT_CRAWL_CONFIG.jsRender.a11yAudit,
},
performanceBudget: {
...DEFAULT_CRAWL_CONFIG.performanceBudget,
...(parsed.performanceBudget ?? {}),
},
};
}
} catch {
@@ -1686,6 +1686,20 @@ function NameValueView({ row }: { row: CrawlUrlRow }) {
? 'No'
: null,
],
[
'Performance Budget',
(() => {
const s = row.budgetStatus;
if (s === null || s === undefined) return null;
if (s === 0) return 'Within budget';
const parts: string[] = [];
if (s & 1) parts.push('Response');
if (s & 2) parts.push('Size');
if (s & 4) parts.push('LCP');
if (s & 8) parts.push('CLS');
return `Over: ${parts.join(', ')}`;
})(),
],
['Empty Anchor Links', row.emptyAnchorCount > 0 ? row.emptyAnchorCount : null],
['Mixed Content (subresources)', row.mixedContentCount > 0 ? row.mixedContentCount : null],
[
@@ -2921,6 +2935,12 @@ function StructuredDataView({
{row.schemaInvalidCount > 0 && (
<span className="text-amber-400">{t('structured.invalid', { defaultValue: '{{n}} invalid', n: row.schemaInvalidCount })}</span>
)}
{row.schemaMissingRequired > 0 && (
<span className="text-red-400">{t('structured.missingRequired', { defaultValue: '{{n}} missing required', n: row.schemaMissingRequired })}</span>
)}
{row.schemaMissingRecommended > 0 && (
<span className="text-amber-400">{t('structured.missingRecommended', { defaultValue: '{{n}} missing recommended', n: row.schemaMissingRecommended })}</span>
)}
<span>
<span className="font-medium text-surface-200">{row.microdataCount}</span> {t('structured.microdataItems', { defaultValue: 'microdata items' })}
</span>
@@ -695,6 +695,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
count: o.issues.ttfbVerySlow,
category: 'issues:ttfb-very-slow',
},
{
key: 'issues-over-budget',
label: 'Over Performance Budget',
count: o.issues.overBudget,
category: 'issues:over-budget',
},
],
},
{
@@ -1235,6 +1241,12 @@ function buildTree(o: OverviewCounts | null): Node[] {
count: o.issues.schemaMissingRequired,
category: 'issues:schema-missing-required',
},
{
key: 'issues-schema-missing-recommended',
label: 'Missing Recommended Property',
count: o.issues.schemaMissingRecommended,
category: 'issues:schema-missing-recommended',
},
],
},
{
@@ -38,6 +38,7 @@ import {
ExternalLink,
BookOpen,
Chrome,
Target,
type LucideIcon,
} from 'lucide-react';
import clsx from 'clsx';
@@ -168,6 +169,12 @@ interface FormState {
jsMobileUsability: boolean;
jsLcpCandidate: boolean;
jsA11yAudit: boolean;
// V2 Faz 15 — performance budget
budgetEnabled: boolean;
budgetMaxResponseMs: string;
budgetMaxPageKb: string;
budgetMaxLcpMs: string;
budgetMaxCls: string;
}
type SectionKey =
@@ -193,6 +200,7 @@ type SectionKey =
| 'per-host-ua'
| 'integrations'
| 'rendering'
| 'performance-budget'
| 'storage'
| 'privacy'
| 'language';
@@ -346,6 +354,13 @@ const SECTIONS: SectionDef[] = [
keywords:
'rendering javascript js render playwright chromium browser headless headful viewport mobile desktop ajax timeout wait selector resource block image font script media css spa hydration',
},
{
key: 'performance-budget',
label: 'Performance Budget',
icon: Target,
keywords:
'performance budget threshold response time ttfb page size weight lcp cls core web vitals ceiling limit over budget',
},
{
key: 'storage',
label: 'Storage',
@@ -453,6 +468,13 @@ function configToForm(c: CrawlConfig): FormState {
jsMobileUsability: c.jsRender?.mobileUsability ?? false,
jsLcpCandidate: c.jsRender?.lcpCandidate ?? false,
jsA11yAudit: c.jsRender?.a11yAudit ?? false,
budgetEnabled: c.performanceBudget?.enabled ?? false,
budgetMaxResponseMs: String(c.performanceBudget?.maxResponseMs ?? 800),
budgetMaxPageKb: String(
Math.round((c.performanceBudget?.maxPageBytes ?? 1048576) / 1024),
),
budgetMaxLcpMs: String(c.performanceBudget?.maxLcpMs ?? 2500),
budgetMaxCls: String(c.performanceBudget?.maxCls ?? 0.1),
};
}
@@ -690,6 +712,13 @@ export function SettingsDialog({ open, onClose }: Props) {
lcpCandidate: form.jsLcpCandidate,
a11yAudit: form.jsA11yAudit,
},
performanceBudget: {
enabled: form.budgetEnabled,
maxResponseMs: Math.max(0, num(form.budgetMaxResponseMs, 800)),
maxPageBytes: Math.max(0, num(form.budgetMaxPageKb, 1024)) * 1024,
maxLcpMs: Math.max(0, num(form.budgetMaxLcpMs, 2500)),
maxCls: Math.max(0, Number.parseFloat(form.budgetMaxCls) || 0),
},
});
onClose();
}
@@ -855,6 +884,9 @@ export function SettingsDialog({ open, onClose }: Props) {
{active === 'rendering' && (
<RenderingPanel form={form} update={update} />
)}
{active === 'performance-budget' && (
<PerformanceBudgetPanel form={form} update={update} />
)}
{active === 'storage' && <StoragePanel />}
{active === 'privacy' && <PrivacyPanel />}
{active === 'language' && <LanguagePanel />}
@@ -3211,6 +3243,68 @@ function RenderingPanel({ form, update }: PanelProps) {
);
}
function PerformanceBudgetPanel({ form, update }: PanelProps) {
const { t } = useTranslation();
const on = form.budgetEnabled;
return (
<>
<p className="mb-3 text-[11px] text-surface-400">
{t('settingsPanels.performanceBudget.intro', {
defaultValue:
'Set per-page performance ceilings. After each crawl, internal HTML pages that exceed any enabled threshold are flagged under the "Over Performance Budget" issue filter. Response time and page size come from the crawl itself; LCP and CLS use PageSpeed Insights data when present. Set any field to 0 to disable that individual check.',
})}
</p>
<div className="mb-4 rounded border border-surface-800 bg-surface-950/40 p-3">
<Bool
label={t('settingsPanels.performanceBudget.enabled', { defaultValue: 'Enable performance budget' })}
checked={form.budgetEnabled}
onChange={(v) => update('budgetEnabled', v)}
hint=""
info="When off, no budget evaluation runs and the verdict column is cleared. When on, the post-crawl pass scores every internal 200 HTML page against the ceilings below."
example="On for performance-focused audits that should fail pages over a target."
/>
</div>
<div className={clsx('mb-4 rounded border border-surface-800 bg-surface-950/40 p-3', !on && 'opacity-50')}>
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-surface-400">
{t('settingsGroups.budgetThresholds', { defaultValue: 'Budget Thresholds' })}
</div>
<div className="grid grid-cols-2 gap-3">
<Num
label={t('settingsPanels.performanceBudget.maxResponseMs', { defaultValue: 'Max response time (ms)' })}
value={form.budgetMaxResponseMs}
onChange={(v) => update('budgetMaxResponseMs', v)}
info="Server response time (a TTFB proxy) measured during the crawl. Pages slower than this are flagged. Google considers a good server response time under 800 ms."
example="800 default; 200 for CDN-backed static; 0 to disable."
/>
<Num
label={t('settingsPanels.performanceBudget.maxPageKb', { defaultValue: 'Max page size (KB)' })}
value={form.budgetMaxPageKb}
onChange={(v) => update('budgetMaxPageKb', v)}
info="HTML transfer size of the page document. Heavy HTML payloads delay first paint. Stored as bytes internally; entered here in kilobytes."
example="1024 (1 MB) default; 150 for a lean HTML budget; 0 to disable."
/>
<Num
label={t('settingsPanels.performanceBudget.maxLcpMs', { defaultValue: 'Max LCP (ms)' })}
value={form.budgetMaxLcpMs}
onChange={(v) => update('budgetMaxLcpMs', v)}
info="Largest Contentful Paint from PageSpeed Insights lab data, when the URL has been audited. Google's 'good' LCP threshold is 2500 ms. Pages without PSI data are never flagged on this metric."
example="2500 default (Google 'good'); 0 to disable."
/>
<Num
label={t('settingsPanels.performanceBudget.maxCls', { defaultValue: 'Max CLS' })}
value={form.budgetMaxCls}
onChange={(v) => update('budgetMaxCls', v)}
info="Cumulative Layout Shift from PageSpeed Insights, when present. Google's 'good' CLS threshold is 0.1. Unitless; accepts decimals. Pages without PSI data are never flagged."
example="0.1 default (Google 'good'); 0 to disable."
/>
</div>
</div>
</>
);
}
function IntegrationsPanel() {
const { t } = useTranslation();
const [state, setState] = useState<IntegrationsState | null>(null);
@@ -149,6 +149,7 @@ const TR_LABELS: Record<string, string> = {
'Very Slow (>3s)': 'Çok Yavaş (>3sn)',
'TTFB Slow (>600ms)': 'TTFB Yavaş (>600ms)',
'TTFB Very Slow (>1.8s)': 'TTFB Çok Yavaş (>1.8sn)',
'Over Performance Budget': 'Performans Bütçesini Aşan',
'Large (>1MB)': 'Büyük (>1MB)',
'Critical Size (>3MB)': 'Kritik Boyut (>3MB)',
'Render-Blocking Head (>5)': 'Render-Blocking Head (>5)',
@@ -232,6 +233,7 @@ const TR_LABELS: Record<string, string> = {
'Duplicate @id': 'Yinelenen @id',
'Malformed @type': 'Bozuk @type',
'Missing Required Property': 'Gerekli Özellik Eksik',
'Missing Recommended Property': 'Önerilen Özellik Eksik',
// Social / OG
'OpenGraph Tags Missing': 'OpenGraph Etiketleri Eksik',
@@ -247,6 +249,7 @@ const TR_LABELS: Record<string, string> = {
'Low Contrast Text': 'Düşük Kontrastlı Metin',
'Focus Outline Suppressed': 'Odak Çerçevesi Bastırılmış',
'Low Contrast Elements': 'Düşük Kontrastlı Öğeler',
'Performance Budget': 'Performans Bütçesi',
// PWA / icons
'Viewport Meta Missing': 'Viewport Meta Eksik',
@@ -339,6 +339,7 @@
"cpu": "CPU",
"bodySnapshots": "Body Snapshots",
"linkResponseCaps": "Link & Response Caps",
"budgetThresholds": "Budget Thresholds",
"urlStructureThresholds": "URL Structure Thresholds",
"linkFollowBehaviour": "Link Follow Behaviour",
"savedProxyProfiles": "Saved Proxy Profiles",
@@ -601,6 +602,14 @@
"lcpCandidate": "LCP candidate detection",
"a11yAudit": "Accessibility audit (contrast + focus)"
},
"performanceBudget": {
"intro": "Set per-page performance ceilings. After each crawl, internal HTML pages that exceed any enabled threshold are flagged under the \"Over Performance Budget\" issue filter. Response time and page size come from the crawl itself; LCP and CLS use PageSpeed Insights data when present. Set any field to 0 to disable that individual check.",
"enabled": "Enable performance budget",
"maxResponseMs": "Max response time (ms)",
"maxPageKb": "Max page size (KB)",
"maxLcpMs": "Max LCP (ms)",
"maxCls": "Max CLS"
},
"perHostUa": {
"intro": "Override the User-Agent on a per-host basis. Useful when crawling a mobile subdomain with the mobile-Googlebot UA in the same run as the desktop site, or when a CDN serves a different page based on the requester's UA. The first matching pattern wins; the global User-Agent (Requests tab) is the fallback.",
"syntax1": "Pattern syntax: exact host",
@@ -735,6 +744,8 @@
"structured": {
"blocks": "block(s)",
"invalid": "{{n}} invalid",
"missingRequired": "{{n}} missing required",
"missingRecommended": "{{n}} missing recommended",
"microdataItems": "microdata items",
"rdfaAttrs": "RDFa attrs",
"empty": "No structured data declared on this page (no JSON-LD, microdata or RDFa).",
@@ -338,6 +338,7 @@
"cpu": "CPU",
"bodySnapshots": "Gövde Snapshot'ları",
"linkResponseCaps": "Link & Yanıt Limitleri",
"budgetThresholds": "Bütçe Eşikleri",
"urlStructureThresholds": "URL Yapı Eşikleri",
"linkFollowBehaviour": "Link Takip Davranışı",
"savedProxyProfiles": "Kayıtlı Proxy Profilleri",
@@ -600,6 +601,14 @@
"lcpCandidate": "LCP aday tespiti",
"a11yAudit": "Erişilebilirlik denetimi (kontrast + odak)"
},
"performanceBudget": {
"intro": "Sayfa başına performans tavanları belirleyin. Her crawl'dan sonra, etkin eşiklerden herhangi birini aşan dahili HTML sayfaları \"Performans Bütçesini Aşan\" issue filtresinde işaretlenir. Yanıt süresi ve sayfa boyutu crawl'dan; LCP ve CLS mevcutsa PageSpeed Insights verisinden gelir. Bir kontrolü devre dışı bırakmak için ilgili alanı 0 yapın.",
"enabled": "Performans bütçesini etkinleştir",
"maxResponseMs": "Maks yanıt süresi (ms)",
"maxPageKb": "Maks sayfa boyutu (KB)",
"maxLcpMs": "Maks LCP (ms)",
"maxCls": "Maks CLS"
},
"perHostUa": {
"intro": "User-Agent'ı host bazında geçersiz kıl. Mobil bir alt alan adını mobile-Googlebot UA ile masaüstü sitesiyle aynı koşuda crawl ederken veya bir CDN, isteyene göre farklı sayfa servis ettiğinde faydalı. İlk eşleşen kalıp kazanır; global User-Agent (Requests sekmesi) yedektir.",
"syntax1": "Kalıp söz dizimi: tam host",
@@ -734,6 +743,8 @@
"structured": {
"blocks": "blok",
"invalid": "{{n}} geçersiz",
"missingRequired": "{{n}} gerekli eksik",
"missingRecommended": "{{n}} önerilen eksik",
"microdataItems": "microdata öğesi",
"rdfaAttrs": "RDFa özniteliği",
"empty": "Bu sayfada yapısal veri yok (JSON-LD, microdata veya RDFa yok).",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/mcp-server",
"version": "0.6.9",
"version": "0.7.0",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "freecrawl-seo",
"version": "0.6.9",
"version": "0.7.0",
"private": true,
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
"license": "MIT",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/core",
"version": "0.6.9",
"version": "0.7.0",
"private": true,
"type": "module",
"main": "./dist/index.js",
+38
View File
@@ -804,6 +804,9 @@ export class Crawler extends EventEmitter {
await yieldToEventLoop();
this.setOp('post-crawl:social-image-probes');
await this.runSocialImageProbes();
await yieldToEventLoop();
this.setOp('post-crawl:performance-budget');
this.runBudgetPass();
this.running = false;
this.stopMemoryMonitor();
// Release per-URL dedup sets — at 1M URLs this is ~80–120 MB of string
@@ -1126,6 +1129,38 @@ export class Crawler extends EventEmitter {
);
}
/**
* V2 Faz 15 — performance budget pass. Stamps each internal 200 HTML
* page with a bitmask of which configured ceilings it exceeded
* (response time / transfer size / LCP / CLS). Pure SQL on the DB, so
* it runs synchronously; when the budget is disabled the column is
* wiped to NULL. No-ops when unsupported (older DB without the method).
*/
private runBudgetPass(): void {
if (this.stopped) return;
const budget = this.config.performanceBudget;
if (!budget) return;
if (typeof this.db.recomputeBudgetViolations !== 'function') return;
try {
const result = this.db.recomputeBudgetViolations(budget);
if (budget.enabled && result.evaluated > 0) {
this.emit(
'info',
`Performance budget: ${result.overBudget}/${result.evaluated} page(s) over budget`,
);
}
} catch (err) {
this.emit(
'error',
new Error(
`recomputeBudgetViolations failed: ${
err instanceof Error ? err.message : String(err)
}`,
),
);
}
}
private async runTlsCertProbes(): Promise<void> {
if (!this.config.probeTlsCerts) return;
if (this.stopped) return;
@@ -1336,6 +1371,8 @@ export class Crawler extends EventEmitter {
await this.runTlsCertProbes();
await yieldToEventLoop();
await this.runSocialImageProbes();
await yieldToEventLoop();
this.runBudgetPass();
this.running = false;
this.stopMemoryMonitor();
this.seen.clear();
@@ -2496,6 +2533,7 @@ export class Crawler extends EventEmitter {
schemaDuplicateIds: parsed.schemaDuplicateIds,
schemaUnknownTypes: parsed.schemaUnknownTypes,
schemaMissingRequired: parsed.schemaMissingRequired,
schemaMissingRecommended: parsed.schemaMissingRecommended,
headingOrderViolations: parsed.headingOrderViolations,
subresourceRequestCount: parsed.subresourceRequestCount,
headings:
+119 -13
View File
@@ -94,6 +94,12 @@ export interface ParsedPage {
* VideoObject) and which is missing one or more required properties.
*/
schemaMissingRequired: number;
/**
* Count of JSON-LD nodes that satisfy their required properties but
* are missing one or more of the type's Google-recommended properties
* (warning-level enrichment signal). One count per node.
*/
schemaMissingRecommended: number;
/** Number of Microdata `itemscope` elements declared on the page. */
microdataCount: number;
/** Number of RDFa `typeof` / `vocab` / `property` attribute occurrences. */
@@ -515,6 +521,7 @@ export function parseHtml(
const schemaIdCounts = new Map<string, number>();
let schemaUnknownTypes = 0;
let schemaMissingRequired = 0;
let schemaMissingRecommended = 0;
$('script[type="application/ld+json"]').each((_, el) => {
const raw = $(el).text().trim();
if (!raw) return;
@@ -524,6 +531,7 @@ export function parseHtml(
const validation = walkSchemaForValidation(parsed, schemaIdCounts);
schemaUnknownTypes += validation.unknownTypeCount;
schemaMissingRequired += validation.missingRequiredCount;
schemaMissingRecommended += validation.missingRecommendedCount;
schemaBlockCount++;
} catch {
// Malformed JSON-LD — still count presence so Structured Data
@@ -1289,6 +1297,7 @@ export function parseHtml(
schemaDuplicateIds,
schemaUnknownTypes,
schemaMissingRequired,
schemaMissingRecommended,
microdataCount,
rdfaCount,
insecureFormActionCount,
@@ -1606,26 +1615,99 @@ function collectSchemaTypes(node: unknown, out: Set<string>): void {
*/
const SCHEMA_REQUIRED_PROPS: Record<
string,
{ required: string[]; oneOf?: string[][] }
{ required: string[]; oneOf?: string[][]; recommended?: string[] }
> = {
Article: { required: ['headline', 'image', 'datePublished'] },
NewsArticle: { required: ['headline', 'image', 'datePublished'] },
BlogPosting: { required: ['headline', 'image', 'datePublished'] },
Product: { required: ['name'], oneOf: [['image', 'offers']] },
Article: {
required: ['headline', 'image', 'datePublished'],
recommended: ['author', 'dateModified', 'publisher'],
},
NewsArticle: {
required: ['headline', 'image', 'datePublished'],
recommended: ['author', 'dateModified', 'publisher'],
},
BlogPosting: {
required: ['headline', 'image', 'datePublished'],
recommended: ['author', 'dateModified', 'publisher'],
},
Product: {
required: ['name'],
oneOf: [['image', 'offers']],
recommended: ['description', 'brand', 'review', 'aggregateRating', 'sku'],
},
BreadcrumbList: { required: ['itemListElement'] },
Recipe: {
required: ['name', 'image'],
oneOf: [['recipeIngredient', 'recipeInstructions']],
recommended: [
'author',
'datePublished',
'aggregateRating',
'nutrition',
'prepTime',
'cookTime',
'totalTime',
'recipeYield',
],
},
Event: {
required: ['name', 'startDate', 'location'],
recommended: ['endDate', 'offers', 'performer', 'image', 'description'],
},
Event: { required: ['name', 'startDate', 'location'] },
FAQPage: { required: ['mainEntity'] },
HowTo: { required: ['name', 'step'] },
Organization: { required: [], oneOf: [['name', 'url']] },
VideoObject: { required: ['name', 'thumbnailUrl', 'uploadDate'] },
HowTo: {
required: ['name', 'step'],
recommended: ['image', 'totalTime', 'supply', 'tool'],
},
Organization: {
required: [],
oneOf: [['name', 'url']],
recommended: ['logo', 'contactPoint', 'sameAs'],
},
VideoObject: {
required: ['name', 'thumbnailUrl', 'uploadDate'],
recommended: ['description', 'duration', 'contentUrl', 'embedUrl'],
},
LocalBusiness: {
required: ['name'],
recommended: [
'address',
'telephone',
'openingHours',
'geo',
'priceRange',
'image',
],
},
JobPosting: {
required: ['title', 'description', 'datePosted', 'hiringOrganization'],
recommended: ['validThrough', 'employmentType', 'jobLocation', 'baseSalary'],
},
Course: {
required: ['name', 'description'],
recommended: ['provider', 'offers', 'hasCourseInstance'],
},
Review: {
required: ['reviewRating'],
oneOf: [['itemReviewed', 'author']],
recommended: ['datePublished', 'publisher'],
},
Person: {
required: ['name'],
recommended: ['url', 'image', 'jobTitle', 'sameAs'],
},
WebSite: {
required: ['name', 'url'],
recommended: ['potentialAction'],
},
SoftwareApplication: {
required: ['name'],
oneOf: [['offers', 'aggregateRating', 'review']],
recommended: ['applicationCategory', 'operatingSystem'],
},
};
/**
* Walk a parsed JSON-LD payload and accumulate three structured-data
* Walk a parsed JSON-LD payload and accumulate four structured-data
* health signals into the caller's tallies / map:
*
* - **Duplicate `@id`** — populated by recording every `@id` value
@@ -1642,13 +1724,24 @@ const SCHEMA_REQUIRED_PROPS: Record<
* - **Missing required props** — when a node's `@type` matches an
* entry in `SCHEMA_REQUIRED_PROPS`, every entry in `required` and
* at least one entry of each `oneOf[i]` group must be present.
* - **Missing recommended props** — a node that satisfies its
* required props but is missing one or more of the type's
* `recommended` props (Google Rich Results "recommended" fields).
* Counted once per node — a warning-level enrichment signal, not a
* hard validation failure. A node already flagged as missing
* *required* props is not double-counted as recommended.
*/
function walkSchemaForValidation(
node: unknown,
idCounts: Map<string, number>,
): { unknownTypeCount: number; missingRequiredCount: number } {
): {
unknownTypeCount: number;
missingRequiredCount: number;
missingRecommendedCount: number;
} {
let unknownTypeCount = 0;
let missingRequiredCount = 0;
let missingRecommendedCount = 0;
const visit = (n: unknown): void => {
if (!n) return;
@@ -1685,20 +1778,33 @@ function walkSchemaForValidation(
// Required-prop check for the curated high-traffic types.
const rule = SCHEMA_REQUIRED_PROPS[t];
if (!rule) continue;
let nodeMissingRequired = false;
for (const key of rule.required) {
if (!isTruthyProp(obj[key])) {
missingRequiredCount++;
nodeMissingRequired = true;
break; // one failure per node, not per missing prop
}
}
if (rule.oneOf) {
if (!nodeMissingRequired && rule.oneOf) {
for (const group of rule.oneOf) {
if (!group.some((k) => isTruthyProp(obj[k]))) {
missingRequiredCount++;
nodeMissingRequired = true;
break;
}
}
}
// Recommended-prop check — only when the node is otherwise valid,
// so a node already failing a required prop isn't double-flagged.
if (!nodeMissingRequired && rule.recommended) {
for (const key of rule.recommended) {
if (!isTruthyProp(obj[key])) {
missingRecommendedCount++;
break; // one warning per node
}
}
}
}
// Recurse into nested objects/arrays (e.g. `@graph`, `mainEntity`).
@@ -1710,7 +1816,7 @@ function walkSchemaForValidation(
};
visit(node);
return { unknownTypeCount, missingRequiredCount };
return { unknownTypeCount, missingRequiredCount, missingRecommendedCount };
}
function isTruthyProp(v: unknown): boolean {
+1
View File
@@ -124,6 +124,7 @@ const XML_COLUMNS: (keyof CrawlUrlRow)[] = [
'schemaDuplicateIds',
'schemaUnknownTypes',
'schemaMissingRequired',
'schemaMissingRecommended',
'jsOnlyLinksCount',
'textCodeRatio',
'fleschReadingEase',
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/db",
"version": "0.6.9",
"version": "0.7.0",
"private": true,
"type": "module",
"main": "./dist/index.js",
+42
View File
@@ -1627,6 +1627,48 @@ const MIGRATIONS: Migration[] = [
}
},
},
{
version: 69,
name: 'add_budget_status',
// V2 Faz 15 — performance budget verdict. The post-crawl
// `recomputeBudgetViolations()` pass writes a bitmask per internal
// 200 HTML page when the budget is enabled:
// bit 1 — response time (TTFB proxy) over `maxResponseMs`
// bit 2 — HTML transfer size over `maxPageBytes`
// bit 4 — LCP over `maxLcpMs` (from PageSpeed lab data, if any)
// bit 8 — CLS over `maxCls` (from PageSpeed data, if any)
// 0 = evaluated and within budget; NULL = budget disabled or the
// page wasn't an internal 200 HTML page. LCP/CLS bits only set when
// PageSpeed data exists for the URL at crawl-finalize time.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(urls)')
.all() as unknown as { name: string }[];
if (!cols.some((c) => c.name === 'budget_status')) {
db.exec('ALTER TABLE urls ADD COLUMN budget_status INTEGER');
}
},
},
{
version: 70,
name: 'add_schema_missing_recommended',
// V2 Faz 16 — structured-data validation v2. Counts JSON-LD nodes
// that satisfy their required props but omit one or more of the
// type's Google-recommended properties (warning-level enrichment
// signal, e.g. an Article without `author`/`dateModified`, a Product
// without `aggregateRating`/`review`). Populated by the html-parser
// schema-validation pass; 0 = no recommended-prop gaps on the page.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(urls)')
.all() as unknown as { name: string }[];
if (!cols.some((c) => c.name === 'schema_missing_recommended')) {
db.exec(
'ALTER TABLE urls ADD COLUMN schema_missing_recommended INTEGER NOT NULL DEFAULT 0',
);
}
},
},
];
export function runMigrations(db: DatabaseSync): void {
+121
View File
@@ -15,6 +15,7 @@ import type {
InlinkRow,
OutlinkRow,
OverviewCounts,
PerformanceBudgetConfig,
PagespeedMetrics,
PagespeedRow,
PagespeedStrategy,
@@ -115,6 +116,10 @@ interface UrlRowDb {
* NULL when not audited; a11y_low_contrast >=0, a11y_focus_suppressed 0/1. */
a11y_low_contrast: number | null;
a11y_focus_suppressed: number | null;
/** V2 Faz 15 — performance budget verdict bitmask
* (1=response, 2=size, 4=LCP, 8=CLS). 0 = within budget,
* NULL = budget disabled / page not an internal 200 HTML page. */
budget_status: number | null;
favicon: string | null;
mixed_content_count: number;
mixed_content_active: number;
@@ -198,6 +203,7 @@ interface UrlRowDb {
schema_duplicate_ids: number;
schema_unknown_types: number;
schema_missing_required: number;
schema_missing_recommended: number;
heading_order_violations: number;
subresource_request_count: number;
}
@@ -330,6 +336,7 @@ export interface UpsertUrlInput {
schemaDuplicateIds?: number;
schemaUnknownTypes?: number;
schemaMissingRequired?: number;
schemaMissingRecommended?: number;
/** Skipped-level heading events on this page (h1→h3 etc.). */
headingOrderViolations?: number;
/** Total subresource requests declared (img/script/stylesheet/iframe/video/audio). */
@@ -401,6 +408,7 @@ const UPSERT_URL_SQL = `
url_malformed, og_type, og_url, og_site_name, og_locale, android_icon,
landmark_main, skip_link_present, aria_invalid_roles,
schema_duplicate_ids, schema_unknown_types, schema_missing_required,
schema_missing_recommended,
heading_order_violations, subresource_request_count,
amp_page, amp_validation_errors
) VALUES (
@@ -441,6 +449,7 @@ const UPSERT_URL_SQL = `
:url_malformed, :og_type, :og_url, :og_site_name, :og_locale, :android_icon,
:landmark_main, :skip_link_present, :aria_invalid_roles,
:schema_duplicate_ids, :schema_unknown_types, :schema_missing_required,
:schema_missing_recommended,
:heading_order_violations, :subresource_request_count,
:amp_page, :amp_validation_errors
)
@@ -569,6 +578,7 @@ const UPSERT_URL_SQL = `
schema_duplicate_ids = excluded.schema_duplicate_ids,
schema_unknown_types = excluded.schema_unknown_types,
schema_missing_required = excluded.schema_missing_required,
schema_missing_recommended = excluded.schema_missing_recommended,
heading_order_violations = excluded.heading_order_violations,
subresource_request_count = excluded.subresource_request_count,
amp_page = excluded.amp_page,
@@ -1378,6 +1388,7 @@ export class ProjectDb {
schema_duplicate_ids: input.schemaDuplicateIds ?? 0,
schema_unknown_types: input.schemaUnknownTypes ?? 0,
schema_missing_required: input.schemaMissingRequired ?? 0,
schema_missing_recommended: input.schemaMissingRecommended ?? 0,
heading_order_violations: input.headingOrderViolations ?? 0,
subresource_request_count: input.subresourceRequestCount ?? 0,
};
@@ -2178,6 +2189,94 @@ export class ProjectDb {
};
}
/**
* V2 Faz 15 — performance budget pass. Stamps `urls.budget_status`
* for every internal 200 HTML page with a bitmask of which configured
* ceilings it blew through:
* 1 = response time (TTFB proxy) > maxResponseMs
* 2 = HTML transfer size > maxPageBytes
* 4 = LCP > maxLcpMs (from PageSpeed lab data, if present)
* 8 = CLS > maxCls (from PageSpeed data, if present)
* A threshold of 0 disables that individual check. LCP/CLS use the
* worst (MAX) value across whatever PageSpeed strategies were run for
* the URL — a page over budget on mobile OR desktop is flagged. Pages
* with no PageSpeed data simply never light those bits.
*
* Returns the number of pages evaluated and how many exceeded budget.
* When the budget is disabled the column is wiped to NULL so a stale
* verdict from a prior crawl doesn't linger.
*/
recomputeBudgetViolations(
budget: PerformanceBudgetConfig,
): { evaluated: number; overBudget: number } {
const wipe = this.db.prepare(
"UPDATE urls SET budget_status = NULL WHERE is_external = 0 AND content_kind = 'html'",
);
if (!budget.enabled) {
this.db.exec('BEGIN');
try {
wipe.run();
this.db.exec('COMMIT');
} catch (err) {
this.db.exec('ROLLBACK');
throw err;
}
return { evaluated: 0, overBudget: 0 };
}
// A `> 0` threshold means "check enabled". The CASE arms short-
// circuit on a 0 threshold so the bit is never set. NULL metric
// comparisons evaluate to NULL → ELSE 0, so un-measured pages
// (no response time / no PageSpeed) don't false-flag.
const update = this.db.prepare(`
UPDATE urls SET budget_status =
(CASE WHEN :maxResp > 0 AND response_time_ms IS NOT NULL
AND response_time_ms > :maxResp THEN 1 ELSE 0 END)
+ (CASE WHEN :maxBytes > 0 AND content_length IS NOT NULL
AND content_length > :maxBytes THEN 2 ELSE 0 END)
+ (CASE WHEN :maxLcp > 0 AND (
SELECT MAX(p.lcp) FROM pagespeed_results p
WHERE p.url = urls.url AND p.status = 'ok'
) > :maxLcp THEN 4 ELSE 0 END)
+ (CASE WHEN :maxCls > 0 AND (
SELECT MAX(p.cls) FROM pagespeed_results p
WHERE p.url = urls.url AND p.status = 'ok'
) > :maxCls THEN 8 ELSE 0 END)
WHERE is_external = 0 AND content_kind = 'html' AND status_code = 200
`);
this.db.exec('BEGIN');
try {
wipe.run();
update.run({
maxResp: budget.maxResponseMs,
maxBytes: budget.maxPageBytes,
maxLcp: budget.maxLcpMs,
maxCls: budget.maxCls,
});
this.db.exec('COMMIT');
} catch (err) {
this.db.exec('ROLLBACK');
throw err;
}
const evaluated = (
this.db
.prepare(
'SELECT COUNT(*) AS n FROM urls WHERE budget_status IS NOT NULL',
)
.get() as { n: number }
).n;
const overBudget = (
this.db
.prepare(
'SELECT COUNT(*) AS n FROM urls WHERE budget_status IS NOT NULL AND budget_status > 0',
)
.get() as { n: number }
).n;
return { evaluated, overBudget };
}
/**
* Page through near-duplicate clusters for the dedicated Duplicates tab.
* Members are returned grouped: ORDER BY cluster_size DESC, cluster_id,
@@ -4404,6 +4503,9 @@ export class ProjectDb {
schemaMissingRequired: countWhere(
`${html} AND schema_missing_required > 0`,
),
schemaMissingRecommended: countWhere(
`${html} AND schema_missing_recommended > 0`,
),
imageBrokenSrc: countWhere(
`${html} AND EXISTS (
SELECT 1 FROM image_usages iu
@@ -4590,6 +4692,9 @@ export class ProjectDb {
pageManyRequests: countWhere(
`${html} AND subresource_request_count > 100`,
),
overBudget: countWhere(
`${html} AND budget_status IS NOT NULL AND budget_status > 0`,
),
};
}
@@ -6312,6 +6417,7 @@ export class ProjectDb {
boilerplateCoverage: r.boilerplate_coverage,
a11yLowContrast: r.a11y_low_contrast ?? null,
a11yFocusSuppressed: r.a11y_focus_suppressed ?? null,
budgetStatus: r.budget_status ?? null,
favicon: r.favicon,
mixedContentCount: r.mixed_content_count,
mixedContentActive: r.mixed_content_active ?? 0,
@@ -6395,6 +6501,7 @@ export class ProjectDb {
schemaDuplicateIds: r.schema_duplicate_ids ?? 0,
schemaUnknownTypes: r.schema_unknown_types ?? 0,
schemaMissingRequired: r.schema_missing_required ?? 0,
schemaMissingRecommended: r.schema_missing_recommended ?? 0,
headingOrderViolations: r.heading_order_violations ?? 0,
subresourceRequestCount: r.subresource_request_count ?? 0,
crawledAt: r.crawled_at,
@@ -7503,6 +7610,14 @@ function categoryWhereClause(cat: UrlCategory): string | null {
// more required properties.
return `is_external = 0 AND content_kind = 'html'
AND schema_missing_required > 0`;
case 'issues:schema-missing-recommended':
// Page has at least one JSON-LD node that satisfies its required
// props but omits one or more of the type's Google-recommended
// properties (e.g. an Article without author/dateModified, a
// Product without aggregateRating/review). Warning-level — the
// markup is valid but not eligible for richer result features.
return `is_external = 0 AND content_kind = 'html'
AND schema_missing_recommended > 0`;
case 'issues:image-broken-src':
// Pages referencing at least one internal image whose HEAD probe
// returned a 4xx/5xx status. Uses the existing `probe_status`
@@ -7935,6 +8050,12 @@ function categoryWhereClause(cat: UrlCategory): string | null {
// party tag injection.
return `is_external = 0 AND content_kind = 'html'
AND subresource_request_count > 100`;
case 'issues:over-budget':
// V2 Faz 15 — pages flagged by the post-crawl performance-budget
// pass. `budget_status` is a bitmask (1=response, 2=size, 4=LCP,
// 8=CLS); >0 means at least one configured ceiling was exceeded.
// NULL/0 (budget off, or within budget) is excluded.
return 'is_external = 0 AND budget_status IS NOT NULL AND budget_status > 0';
case 'tab:directives':
// Wave 4 — Directives tab. Pages declaring any indexability
// directive: meta robots, X-Robots-Tag header, robots.txt block,
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/shared-types",
"version": "0.6.9",
"version": "0.7.0",
"private": true,
"type": "module",
"main": "./dist/index.js",
+50
View File
@@ -166,6 +166,7 @@ export type UrlCategory =
| 'issues:schema-duplicate-id'
| 'issues:schema-unknown-type'
| 'issues:schema-missing-required'
| 'issues:schema-missing-recommended'
| 'issues:target-blank-no-noopener'
| 'issues:page-empty'
| 'issues:og-image-not-absolute'
@@ -215,6 +216,7 @@ export type UrlCategory =
| 'tab:serp'
| 'issues:hreflang-inconsistent-lang'
| 'issues:page-many-requests'
| 'issues:over-budget'
export type Indexability =
| 'indexable'
@@ -318,6 +320,8 @@ export interface CrawlUrlRow {
schemaUnknownTypes: number;
/** Nodes failing required-property check for Article/Product/Recipe/Event/FAQ/HowTo/etc. */
schemaMissingRequired: number;
/** Nodes that pass required props but omit one or more Google-recommended props (warning). */
schemaMissingRecommended: number;
/** Total user-facing form inputs (input/textarea/select, excluding hidden/submit/button/image/reset). */
formInputCount: number;
/** Form inputs without label / aria-label / title (WCAG 1.3.1, 4.1.2 violation). */
@@ -487,6 +491,13 @@ export interface CrawlUrlRow {
*/
a11yLowContrast: number | null;
a11yFocusSuppressed: number | null;
/**
* V2 Faz 15 — performance budget verdict, a bitmask: 1=response time,
* 2=transfer size, 4=LCP, 8=CLS. 0 = evaluated and within budget,
* >0 = one or more ceilings exceeded. NULL when the budget pass didn't
* run (disabled) or the page wasn't an internal 200 HTML page.
*/
budgetStatus: number | null;
favicon: string | null;
/** Resolved `<link rel="apple-touch-icon">` URL, else null. */
appleTouchIcon: string | null;
@@ -982,6 +993,29 @@ export interface CrawlConfig {
* crash older projects that pre-date this block.
*/
jsRender: JsRenderConfig;
/**
* V2 Faz 15 — performance budget. When enabled, the post-crawl pass
* flags every internal HTML page whose response time, transfer size,
* LCP, or CLS exceeds the configured ceiling. Off by default; the
* per-page verdict lands in `urls.budget_status` and drives the
* "Over Performance Budget" issue filter.
*/
performanceBudget: PerformanceBudgetConfig;
}
/** Per-crawl performance budget — surfaces in Settings → Performance. */
export interface PerformanceBudgetConfig {
/** Master switch. When false the budget pass is skipped entirely. */
enabled: boolean;
/** Max response time (TTFB proxy) in ms; 0 disables this check. */
maxResponseMs: number;
/** Max HTML transfer size in bytes; 0 disables this check. */
maxPageBytes: number;
/** Max LCP in ms from PageSpeed lab data; 0 disables this check. */
maxLcpMs: number;
/** Max CLS (unitless) from PageSpeed; 0 disables this check. */
maxCls: number;
}
/** Per-crawl Playwright settings — surfaces in Settings → Rendering. */
@@ -1341,6 +1375,8 @@ export interface OverviewCounts {
schemaUnknownType: number;
/** Pages where a high-traffic JSON-LD type is missing one or more Google-required properties. */
schemaMissingRequired: number;
/** Pages where a JSON-LD type passes required props but omits Google-recommended props (warning). */
schemaMissingRecommended: number;
/** Pages referencing at least one image whose HEAD probe returned a 4xx/5xx status. */
imageBrokenSrc: number;
/** Pages with at least one `<a target="_blank">` without `rel="noopener"` (reverse-tabnabbing risk). */
@@ -1516,6 +1552,13 @@ export interface OverviewCounts {
* party tag injection.
*/
pageManyRequests: number;
/**
* V2 Faz 15 — internal HTML pages over the configured performance
* budget (response time / transfer size / LCP / CLS). Populated by
* the post-crawl `recomputeBudgetViolations()` pass; 0 when the
* budget is disabled.
*/
overBudget: number;
};
}
@@ -1801,4 +1844,11 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
lcpCandidate: false,
a11yAudit: false,
},
performanceBudget: {
enabled: false,
maxResponseMs: 800,
maxPageBytes: 1048576,
maxLcpMs: 2500,
maxCls: 0.1,
},
};