pretecno/FreeCrawl-SEO-Tool
3
0 Fork 0
mirror of https://github.com/kemalai/FreeCrawl-SEO-Tool.git synced 2026-10-18 00:39:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: video/news sitemaps + mobile a11y checks + tray icon

Browse Source
This commit is contained in:
kemalai committed 2026-07-18 13:27:02 +03:00
1 parent af774d1457
commit e8a7c14e4e
23 files changed
+516 -25
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/cli/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/cli",
"version": "0.8.7",
"version": "0.8.8",
"private": true,
"type": "module",
"bin": {
apps/desktop/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/desktop",
"version": "0.8.7",
"version": "0.8.8",
"private": true,
"description": "FreeCrawl SEO — desktop application",
"homepage": "https://github.com/kemalai/FreeCrawl-SEO-Tool",
apps/desktop/src/main/index.ts
+96 -1
View File
@@ -8,6 +8,8 @@ import {
Notification,
session,
shell,
Tray,
nativeImage,
type DownloadItem,
type MenuItemConstructorOptions,
type IpcMainInvokeEvent,
@@ -1865,6 +1867,82 @@ async function promptOpenProject(): Promise<void> {
}
}
/** Placeholder tray icon — a 32×32 white magnifying glass on a transparent
* background, generated as a PNG (no branded icon asset exists yet; swap
* this base64 for the real icon when one lands). */
const TRAY_ICON_PNG_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAWUlEQVR4nO3OQQoAIAgF0e5/6dqHpZYaxMxavq81IqWu9PR5KsL6PAVhHU9BeEfDESdjAAD8C7AMeu9DEfNdGGA1bg1EOOIGCQIECBA7RBlghSgFSIhygNQAumhY0n/Yk+0AAAAASUVORK5CYII=';
let tray: Tray | null = null;
/** Show the main window if it's hidden/minimized, else hide it to the tray.
* Non-destructive: closing the window still quits (unchanged behaviour) —
* the tray only toggles visibility while the app is running. */
function toggleMainWindow(): void {
const win = mainWindow;
if (!win || win.isDestroyed()) return;
if (win.isVisible() && !win.isMinimized()) {
win.hide();
} else {
if (win.isMinimized()) win.restore();
win.show();
win.focus();
}
}
/** Create the system tray once. Left-click toggles the window; the context
* menu offers show/hide, stop the primary crawl, and quit. Failure is
* non-fatal — the app runs fine without a tray. */
function createTray(): void {
if (tray) return;
try {
const img = nativeImage.createFromBuffer(
Buffer.from(TRAY_ICON_PNG_BASE64, 'base64'),
);
// macOS menu-bar icons are template images so they invert with the theme.
if (process.platform === 'darwin') img.setTemplateImage(true);
const t = new Tray(img);
t.setToolTip('FreeCrawl SEO Tool');
const rebuild = (): void => {
const labels = L();
const win = mainWindow;
const visible =
!!win && !win.isDestroyed() && win.isVisible() && !win.isMinimized();
t.setContextMenu(
Menu.buildFromTemplate([
{
label: visible ? labels.trayHide : labels.trayShow,
click: () => toggleMainWindow(),
},
{ type: 'separator' },
{
label: labels.trayStopCrawl,
click: () => runInPrimarySession(() => crawlController?.stopCrawl()),
},
{ type: 'separator' },
{ label: labels.trayQuit, click: () => app.quit() },
]),
);
};
rebuild();
t.on('click', () => toggleMainWindow());
tray = t;
// Keep the show/hide label in sync with the primary window's state.
const win = mainWindow;
if (win) {
win.on('show', rebuild);
win.on('hide', rebuild);
win.on('minimize', rebuild);
win.on('restore', rebuild);
}
} catch (err) {
logger.log(
'warn',
'main',
`system tray init failed: ${err instanceof Error ? err.message : String(err)}`,
);
}
}
function createWindow(): void {
mainWindow = new BrowserWindow({
width: 1440,
@@ -1971,6 +2049,7 @@ function createWindow(): void {
} else {
void mainWindow.loadFile(join(__dirname, '../renderer/index.html'));
}
createTray();
// The primary window resolves to `defaultSession` via `sessionFor`'s
// fallback, so it needs no registry entry (avoids a stale entry if the
// window is recreated on macOS activate).
@@ -4643,6 +4722,9 @@ function registerIpc(): void {
lowContrast: number;
sampled: number;
focusSuppressed: boolean;
smallFont: number;
tapTargetsSmall: number;
tapTargetsSampled: number;
} | null;
}>)
| undefined;
@@ -6956,7 +7038,11 @@ function registerIpc(): void {
? 'sitemap-images.xml'
: variant === 'hreflang'
? 'sitemap-hreflang.xml'
: 'sitemap.xml';
: variant === 'news'
? 'sitemap-news.xml'
: variant === 'video'
? 'sitemap-video.xml'
: 'sitemap.xml';
const defaultPath = gzip ? `${baseName}.gz` : baseName;
const res = await dialog.showSaveDialog(mainWindow!, {
defaultPath,
@@ -7371,6 +7457,15 @@ let shutdownComplete = false;
function performShutdown(): void {
if (shutdownComplete) return;
shutdownComplete = true;
// Drop the tray so its icon doesn't linger after quit.
if (tray) {
try {
tray.destroy();
} catch {
/* already gone */
}
tray = null;
}
// Close ancillary popup windows first so their renderers can flush
// any in-flight IPC before the worker pools shut down.
if (visualizationWindow && !visualizationWindow.isDestroyed()) {
apps/desktop/src/main/menu-i18n.ts
+29 -2
View File
@@ -27,6 +27,11 @@ export interface MenuLabels {
clearCrawlData: string;
exportAs: string;
generateSitemap: string;
sitemapStandard: string;
sitemapImages: string;
sitemapHreflang: string;
sitemapNews: string;
sitemapVideo: string;
exportHtmlReport: string;
bulkExport: string;
exportSheets: string;
@@ -54,6 +59,10 @@ export interface MenuLabels {
help: string;
documentation: string;
showLogs: string;
trayShow: string;
trayHide: string;
trayStopCrawl: string;
trayQuit: string;
openLogsFolder: string;
openLogsFolderTooltip: string;
robotsTester: string;
@@ -147,7 +156,12 @@ const MENU_EN: MenuLabels = {
emptyRecent: '(empty)',
clearCrawlData: 'Clear Crawl Data',
exportAs: 'Export Crawl Data…',
generateSitemap: 'Generate XML Sitemap…',
generateSitemap: 'Generate XML Sitemap',
sitemapStandard: 'Standard…',
sitemapImages: 'Images…',
sitemapHreflang: 'Hreflang…',
sitemapNews: 'News…',
sitemapVideo: 'Video…',
exportHtmlReport: 'Export HTML Report…',
bulkExport: 'Bulk Export…',
exportSheets: 'Export to Google Sheets…',
@@ -179,6 +193,10 @@ const MENU_EN: MenuLabels = {
help: 'Help',
documentation: 'Documentation',
showLogs: 'Show Logs…',
trayShow: 'Show FreeCrawl',
trayHide: 'Hide to Tray',
trayStopCrawl: 'Stop Crawl',
trayQuit: 'Quit FreeCrawl',
openLogsFolder: 'Open Logs Folder',
openLogsFolderTooltip:
'Open the directory where rotated log files are persisted on disk',
@@ -275,7 +293,12 @@ const MENU_TR: MenuLabels = {
emptyRecent: '(boş)',
clearCrawlData: 'Crawl Verilerini Temizle',
exportAs: 'Crawl Verilerini Dışa Aktar…',
generateSitemap: 'XML Sitemap Oluştur…',
generateSitemap: 'XML Sitemap Oluştur',
sitemapStandard: 'Standart…',
sitemapImages: 'Görseller…',
sitemapHreflang: 'Hreflang…',
sitemapNews: 'Haber…',
sitemapVideo: 'Video…',
exportHtmlReport: 'HTML Rapor Dışa Aktar…',
bulkExport: 'Toplu Dışa Aktarım…',
exportSheets: 'Google Sheets\'e Aktar…',
@@ -307,6 +330,10 @@ const MENU_TR: MenuLabels = {
help: 'Yardım',
documentation: 'Dokümantasyon',
showLogs: 'Logları Göster…',
trayShow: 'FreeCrawl\'i Göster',
trayHide: 'Tepsiye Gizle',
trayStopCrawl: 'Taramayı Durdur',
trayQuit: 'FreeCrawl\'ten Çık',
openLogsFolder: 'Log Klasörünü Aç',
openLogsFolderTooltip:
'Diske kayıtlı, dönen log dosyalarının olduğu dizini aç',
apps/desktop/src/main/menu.ts
+7 -1
View File
@@ -95,7 +95,13 @@ export function buildAppMenu(handlers: AppMenuHandlers): Menu {
},
{
label: L.generateSitemap,
click: () => send('generate-sitemap'),
submenu: [
{ label: L.sitemapStandard, click: () => send('generate-sitemap') },
{ label: L.sitemapImages, click: () => send('generate-sitemap-images') },
{ label: L.sitemapHreflang, click: () => send('generate-sitemap-hreflang') },
{ label: L.sitemapNews, click: () => send('generate-sitemap-news') },
{ label: L.sitemapVideo, click: () => send('generate-sitemap-video') },
],
},
{
label: L.exportHtmlReport,
apps/desktop/src/renderer/src/App.tsx
+12
View File
@@ -366,6 +366,18 @@ export function App() {
case 'generate-sitemap':
void window.freecrawl.sitemapGenerate({ filePath: '' });
break;
case 'generate-sitemap-images':
void window.freecrawl.sitemapGenerate({ filePath: '', variant: 'image' });
break;
case 'generate-sitemap-hreflang':
void window.freecrawl.sitemapGenerate({ filePath: '', variant: 'hreflang' });
break;
case 'generate-sitemap-news':
void window.freecrawl.sitemapGenerate({ filePath: '', variant: 'news' });
break;
case 'generate-sitemap-video':
void window.freecrawl.sitemapGenerate({ filePath: '', variant: 'video' });
break;
case 'export-html-report':
void window.freecrawl.exportHtmlReport({ filePath: '' });
break;
apps/desktop/src/renderer/src/components/OverviewSidebar.tsx
+12
View File
@@ -899,6 +899,18 @@ function buildTree(o: OverviewCounts | null): Node[] {
count: o.issues.focusOutlineSuppressed,
category: 'issues:focus-outline-suppressed',
},
{
key: 'issues-font-too-small',
label: 'Font Size Too Small',
count: o.issues.fontTooSmall,
category: 'issues:font-too-small',
},
{
key: 'issues-tap-targets-too-small',
label: 'Tap Target Too Small',
count: o.issues.tapTargetsTooSmall,
category: 'issues:tap-targets-too-small',
},
],
},
{
apps/desktop/src/renderer/src/i18n/labels.ts
+2
View File
@@ -302,6 +302,8 @@ const TR_LABELS: Record<string, string> = {
'Twitter Image Dimensions': 'Twitter Görsel Boyutları',
'Low Contrast Text': 'Düşük Kontrastlı Metin',
'Focus Outline Suppressed': 'Odak Çerçevesi Bastırılmış',
'Font Size Too Small': 'Yazı Tipi Çok Küçük',
'Tap Target Too Small': 'Dokunma Hedefi Çok Küçük',
'Low Contrast Elements': 'Düşük Kontrastlı Öğeler',
'Performance Budget': 'Performans Bütçesi',
apps/mcp-server/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/mcp-server",
"version": "0.8.7",
"version": "0.8.8",
"private": true,
"type": "module",
"bin": {
apps/mcp-server/src/tools.ts
+2 -2
View File
@@ -1201,14 +1201,14 @@ export function buildTools(): Tool[] {
requiresDb: false,
name: 'sitemap_generate',
description:
'Generate a sitemap.xml from the indexable URLs in the active project. Variant defaults to `standard`; pass `image` for Google Images extension or `hreflang` for international targeting. Gzip and per-file URL caps supported (sharding kicks in over 50,000).',
'Generate a sitemap.xml from the indexable URLs in the active project. Variant defaults to `standard`; pass `image` for Google Images, `hreflang` for international targeting, `news` for a Google News sitemap (publication name/language/date + title per page), or `video` for a Google Video sitemap (only pages with a `<video>` or recognised YouTube/Vimeo embed). Gzip and per-file URL caps supported (sharding kicks in over 50,000).',
inputSchema: {
type: 'object',
properties: {
filePath: { type: 'string', description: 'Absolute output path.' },
variant: {
type: 'string',
enum: ['standard', 'image', 'hreflang'],
enum: ['standard', 'image', 'hreflang', 'news', 'video'],
},
gzip: { type: 'boolean' },
splitAtUrlCount: { type: 'integer', minimum: 1, maximum: 50_000 },
package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "freecrawl-seo",
"version": "0.8.7",
"version": "0.8.8",
"private": true,
"description": "Open-source desktop SEO crawler — a free alternative to Screaming Frog",
"license": "MIT",
packages/core/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/core",
"version": "0.8.7",
"version": "0.8.8",
"private": true,
"type": "module",
"main": "./dist/index.js",
packages/core/src/crawler.ts
+15 -1
View File
@@ -354,6 +354,9 @@ export class Crawler extends EventEmitter {
lowContrast: number;
sampled: number;
focusSuppressed: boolean;
smallFont: number;
tapTargetsSmall: number;
tapTargetsSampled: number;
} | null;
}>;
@@ -414,6 +417,9 @@ export class Crawler extends EventEmitter {
lowContrast: number;
sampled: number;
focusSuppressed: boolean;
smallFont: number;
tapTargetsSmall: number;
tapTargetsSampled: number;
} | null;
}>;
} = {},
@@ -2672,7 +2678,14 @@ export class Crawler extends EventEmitter {
| { ok: boolean; overflowPx: number; hasViewportMeta: boolean }
| null = null;
let renderA11y:
| { lowContrast: number; sampled: number; focusSuppressed: boolean }
| {
lowContrast: number;
sampled: number;
focusSuppressed: boolean;
smallFont: number;
tapTargetsSmall: number;
tapTargetsSampled: number;
}
| null = null;
if (
this.config.renderingMode === 'js' &&
@@ -2846,6 +2859,7 @@ export class Crawler extends EventEmitter {
paginationPrev: parsed.paginationPrev,
hreflangs: parsed.hreflangs.length > 0 ? JSON.stringify(parsed.hreflangs) : null,
hreflangCount: parsed.hreflangs.length,
videos: parsed.videos.length > 0 ? JSON.stringify(parsed.videos) : null,
amphtml: parsed.amphtml,
ampPage: parsed.ampPage,
ampValidationErrors: parsed.ampValidationErrors.length > 0
packages/core/src/html-parser.ts
+90
View File
@@ -19,6 +19,15 @@ export interface HreflangEntry {
href: string;
}
export interface VideoEntry {
/** Direct media URL from `<video src>` / `<source src>` (Google `content_loc`). */
contentLoc: string | null;
/** Embed/player URL from a recognised `<iframe>` (Google `player_loc`). */
playerLoc: string | null;
/** Thumbnail URL (`<video poster>` or a derived YouTube thumbnail). */
thumbnail: string | null;
}
/**
* One detected third-party analytics / marketing tracker on a page. The
* `name` is the canonical product name (e.g. `"Google Analytics 4"`). The
@@ -130,6 +139,9 @@ export interface ParsedPage {
paginationPrev: string | null;
/** All `<link rel="alternate" hreflang>` entries on the page. */
hreflangs: HreflangEntry[];
/** Videos found on the page (`<video>` + recognised embeds) for the
* video-sitemap variant. Empty when the page has no detectable video. */
videos: VideoEntry[];
/** `<link rel="amphtml" href>` if present, else null. */
amphtml: string | null;
/** True when the page declares itself AMP via `<html ⚡>` /
@@ -732,6 +744,40 @@ export function parseHtml(
hreflangs.push({ lang, href });
});
// Video — collected for the video-sitemap variant. Two sources: HTML5
// `<video>` (media = src / first `<source src>`, thumbnail = poster) and
// recognised `<iframe>` embeds (YouTube / Vimeo → player URL, with a
// derived thumbnail for YouTube). Deduped by media/player URL; title and
// description are taken from the page at sitemap-emit time, so only the
// media locations are persisted here.
const videoSet = new Set<string>();
const videos: VideoEntry[] = [];
const pushVideo = (
contentLoc: string | null,
playerLoc: string | null,
thumbnail: string | null,
): void => {
const key = contentLoc ?? playerLoc ?? '';
if (!key || videoSet.has(key)) return;
videoSet.add(key);
videos.push({ contentLoc, playerLoc, thumbnail });
};
$('video').each((_, el) => {
const $v = $(el);
let rawSrc = ($v.attr('src') ?? '').trim();
if (!rawSrc) rawSrc = ($v.find('source[src]').first().attr('src') ?? '').trim();
if (!rawSrc) return;
const contentLoc = normalizeUrl(rawSrc, pageUrl, opts.urlRewrites);
if (!contentLoc) return;
const posterRaw = ($v.attr('poster') ?? '').trim();
const thumbnail = posterRaw ? normalizeUrl(posterRaw, pageUrl, opts.urlRewrites) : null;
pushVideo(contentLoc, null, thumbnail);
});
$('iframe[src]').each((_, el) => {
const embed = parseVideoEmbed(($(el).attr('src') ?? '').trim(), pageUrl, opts.urlRewrites);
if (embed) pushVideo(null, embed.playerLoc, embed.thumbnail);
});
// AMP variant — `<link rel="amphtml">` points to the AMP version of
// the current page, when one exists.
const amphtmlRaw = ($('link[rel="amphtml"]').attr('href') ?? '').trim();
@@ -1376,6 +1422,7 @@ export function parseHtml(
paginationNext,
paginationPrev,
hreflangs,
videos,
amphtml,
ampPage,
ampValidationErrors,
@@ -2253,3 +2300,46 @@ function computeReadability(text: string, wordCount: number): ReadabilityScores
complexWordCount,
};
}
/**
* Recognise a video `<iframe>` embed (YouTube / Vimeo) and return its
* player URL plus a derivable thumbnail. Returns null for iframes that
* aren't known video players (ads, maps, generic embeds) so they never
* pollute the video sitemap.
*/
function parseVideoEmbed(
rawSrc: string,
pageUrl: string,
rewrites?: UrlRewriteOptions,
): { playerLoc: string; thumbnail: string | null } | null {
if (!rawSrc) return null;
const playerLoc = normalizeUrl(rawSrc, pageUrl, rewrites);
if (!playerLoc) return null;
let u: URL;
try {
u = new URL(playerLoc);
} catch {
return null;
}
const host = u.hostname.replace(/^www\./, '').toLowerCase();
// YouTube — /embed/<id> (also youtube-nocookie.com) or youtu.be/<id>.
if (host === 'youtube.com' || host === 'youtube-nocookie.com' || host === 'm.youtube.com') {
const id = u.pathname.match(/^\/embed\/([A-Za-z0-9_-]{6,})/)?.[1];
if (id) return { playerLoc, thumbnail: `https://i.ytimg.com/vi/${id}/hqdefault.jpg` };
return null;
}
if (host === 'youtu.be') {
const id = u.pathname.replace(/^\//, '').split('/')[0];
if (id && /^[A-Za-z0-9_-]{6,}$/.test(id)) {
return { playerLoc, thumbnail: `https://i.ytimg.com/vi/${id}/hqdefault.jpg` };
}
return null;
}
// Vimeo player embeds — /video/<id>. Thumbnail needs the oEmbed API, so
// it falls back to the page's og:image at sitemap-emit time.
if (host === 'player.vimeo.com') {
if (/^\/video\/\d+/.test(u.pathname)) return { playerLoc, thumbnail: null };
return null;
}
return null;
}
packages/core/src/render-fetcher.ts
+33 -2
View File
@@ -69,6 +69,12 @@ export interface A11yAudit {
sampled: number;
/** A stylesheet rule suppresses the focus outline without a fallback. */
focusSuppressed: boolean;
/** Sampled text elements rendered below ~12px (too small on mobile). */
smallFont: number;
/** Interactive elements rendered below the WCAG 2.5.8 24×24px minimum. */
tapTargetsSmall: number;
/** How many interactive elements were sampled (context for the above). */
tapTargetsSampled: number;
}
/**
@@ -192,7 +198,7 @@ const A11Y_AUDIT_FN = `(() => {
}
return { r: 255, g: 255, b: 255, a: 1 };
};
let sampled = 0, low = 0;
let sampled = 0, low = 0, small = 0;
const all = document.body ? document.body.querySelectorAll('*') : [];
for (const el of all) {
if (sampled >= 500) break;
@@ -212,6 +218,8 @@ const A11Y_AUDIT_FN = `(() => {
const fgc = fg.a < 1 ? over(fg, bg) : fg;
sampled++;
const fontSize = parseFloat(cs.fontSize || '16');
// Body copy under ~12px is hard to read on mobile without zooming.
if (fontSize > 0 && fontSize < 12) small++;
let weight = parseInt(cs.fontWeight, 10);
if (!isFinite(weight)) weight = cs.fontWeight === 'bold' ? 700 : 400;
const large = fontSize >= 24 || (fontSize >= 18.66 && weight >= 700);
@@ -244,7 +252,30 @@ const A11Y_AUDIT_FN = `(() => {
}
} catch (e) { /* ignore */ }
if (focusVisibleRestores) focusSuppressed = false;
return { lowContrast: low, sampled: sampled, focusSuppressed: focusSuppressed };
// Tap-target sizing — WCAG 2.5.8 (AA) wants interactive controls to be at
// least 24x24 CSS px. Sample visible interactive elements and count those
// rendered smaller in either dimension. Measured at the render viewport.
let tapSmall = 0, tapSampled = 0;
const interactive = document.querySelectorAll(
'a[href], button, input:not([type="hidden"]), select, textarea, [role="button"], [role="link"], [onclick]',
);
for (const el of interactive) {
if (tapSampled >= 500) break;
const cs2 = getComputedStyle(el);
if (cs2.visibility === 'hidden' || cs2.display === 'none' || parseFloat(cs2.opacity || '1') === 0) continue;
const r = el.getBoundingClientRect();
if (r.width <= 0 || r.height <= 0) continue;
tapSampled++;
if (r.width < 24 || r.height < 24) tapSmall++;
}
return {
lowContrast: low,
sampled: sampled,
focusSuppressed: focusSuppressed,
smallFont: small,
tapTargetsSmall: tapSmall,
tapTargetsSampled: tapSampled,
};
})()`;
/**
packages/core/src/sitemap-export.ts
+117 -4
View File
@@ -5,7 +5,12 @@ import { createGzip } from 'node:zlib';
import path from 'node:path';
import type { ProjectDb } from '@freecrawl/db';
export type SitemapVariant = 'standard' | 'image' | 'hreflang';
export type SitemapVariant =
| 'standard'
| 'image'
| 'hreflang'
| 'news'
| 'video';
export interface SitemapOptions {
/** Default `'weekly'`. Sitemaps.org change-frequency hint. */
@@ -27,6 +32,11 @@ export interface SitemapOptions {
* - `standard` — plain `<url><loc>...</loc></url>` (sitemaps.org core).
* - `image` — adds `<image:image><image:loc>...</image:loc></image:image>` per page (Google Images).
* - `hreflang` — adds `<xhtml:link rel="alternate" hreflang="..." href="..." />` siblings (Google international).
* - `news` — adds a `<news:news>` block (publication name/language, publication
* date, title) per page (Google News). Publication name comes from
* `og:site_name` (host fallback), language from the page `lang`
* attribute, and the date from the crawl time — Google News also
* expects only recent articles, so treat the output as a template.
*/
variant?: SitemapVariant;
/** Gzip the output (`.xml.gz`). When true the file path is auto-suffixed. */
@@ -72,6 +82,31 @@ function priorityForDepth(depth: number, depthBased: boolean): string {
return p.toFixed(1);
}
/** Best-effort publication name for the news variant when `og:site_name`
* is absent — fall back to the URL's hostname (e.g. `www.example.com`). */
function hostnameOf(rawUrl: string): string {
try {
return new URL(rawUrl).hostname || rawUrl;
} catch {
return rawUrl;
}
}
/** ISO 639-1 language code for `<news:language>`: take the primary subtag
* of the page `lang` (e.g. `en-US` → `en`), lowercased; default `en`. */
function newsLanguage(lang: string | null): string {
const primary = (lang ?? '').trim().split(/[-_]/)[0]?.toLowerCase();
return primary && /^[a-z]{2,3}$/.test(primary) ? primary : 'en';
}
/** Full W3C datetime for `<news:publication_date>` — the crawl timestamp is
* the only per-URL date we have; Google accepts complete ISO 8601. */
function newsPublicationDate(crawledAt: string): string {
return crawledAt && crawledAt.length >= 10
? crawledAt
: new Date().toISOString();
}
/**
* Produce the right `<urlset>` opening tag for the chosen variant.
* The XML namespace declarations are extension-aware so consumers
@@ -81,15 +116,35 @@ function urlsetOpen(variant: SitemapVariant): string {
const ns = ['xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"'];
if (variant === 'image') ns.push('xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"');
if (variant === 'hreflang') ns.push('xmlns:xhtml="http://www.w3.org/1999/xhtml"');
if (variant === 'news') ns.push('xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"');
if (variant === 'video') ns.push('xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"');
return `<urlset ${ns.join(' ')}>\n`;
}
interface VideoRow {
contentLoc: string | null;
playerLoc: string | null;
thumbnail: string | null;
}
interface UrlEntry {
id: number;
url: string;
depth: number;
crawledAt: string;
hreflangs: string | null;
title: string | null;
lang: string | null;
ogSiteName: string | null;
videos: string | null;
metaDescription: string | null;
ogImage: string | null;
}
/** Google caps `<video:title>` at 100 chars and `<video:description>` at
* 2048 — clamp so the sitemap validates. */
function clamp(s: string, max: number): string {
return s.length > max ? s.slice(0, max) : s;
}
async function writePart(
@@ -107,9 +162,59 @@ async function writePart(
for (const e of entries) {
yield ' <url>\n';
yield ` <loc>${escapeXml(e.url)}</loc>\n`;
if (e.crawledAt) yield ` <lastmod>${formatLastmod(e.crawledAt)}</lastmod>\n`;
yield ` <changefreq>${changefreq}</changefreq>\n`;
yield ` <priority>${priorityForDepth(e.depth, depthBased)}</priority>\n`;
// News entries carry their own <news:publication_date>; the core
// <lastmod>/<changefreq>/<priority> hints aren't part of the news
// schema, so we omit them for a clean, valid news sitemap.
if (variant !== 'news') {
if (e.crawledAt) yield ` <lastmod>${formatLastmod(e.crawledAt)}</lastmod>\n`;
yield ` <changefreq>${changefreq}</changefreq>\n`;
yield ` <priority>${priorityForDepth(e.depth, depthBased)}</priority>\n`;
}
if (variant === 'news') {
const pubName = e.ogSiteName?.trim() || hostnameOf(e.url);
yield ' <news:news>\n';
yield ' <news:publication>\n';
yield ` <news:name>${escapeXml(pubName)}</news:name>\n`;
yield ` <news:language>${escapeXml(newsLanguage(e.lang))}</news:language>\n`;
yield ' </news:publication>\n';
yield ` <news:publication_date>${escapeXml(newsPublicationDate(e.crawledAt))}</news:publication_date>\n`;
yield ` <news:title>${escapeXml(e.title?.trim() || e.url)}</news:title>\n`;
yield ' </news:news>\n';
}
if (variant === 'video' && e.videos) {
let vids: VideoRow[] = [];
try {
const parsed = JSON.parse(e.videos) as unknown;
if (Array.isArray(parsed)) vids = parsed as VideoRow[];
} catch {
// Malformed JSON — skip this page's videos.
}
const title = clamp((e.title?.trim() || e.url), 100);
const description = clamp(
(e.metaDescription?.trim() || e.title?.trim() || e.url),
2048,
);
for (const v of vids) {
// Google requires a thumbnail; fall back to the page's og:image
// when the video itself doesn't declare one, and skip the entry
// if neither is available (an entry without one is invalid).
const thumb = v.thumbnail?.trim() || e.ogImage?.trim() || '';
const media = v.contentLoc?.trim() || v.playerLoc?.trim() || '';
if (!thumb || !media) continue;
yield ' <video:video>\n';
yield ` <video:thumbnail_loc>${escapeXml(thumb)}</video:thumbnail_loc>\n`;
yield ` <video:title>${escapeXml(title)}</video:title>\n`;
yield ` <video:description>${escapeXml(description)}</video:description>\n`;
if (v.contentLoc?.trim()) {
yield ` <video:content_loc>${escapeXml(v.contentLoc.trim())}</video:content_loc>\n`;
} else if (v.playerLoc?.trim()) {
yield ` <video:player_loc>${escapeXml(v.playerLoc.trim())}</video:player_loc>\n`;
}
yield ' </video:video>\n';
}
}
if (variant === 'image') {
const imgs = imagesFor(e.id);
@@ -224,12 +329,20 @@ export async function exportSitemap(
// outputs we still need the full set up front to compute part counts.
const entries: UrlEntry[] = [];
for (const row of db.iterateIndexableUrls()) {
// A video sitemap only lists pages that actually have a video.
if (variant === 'video' && (!row.videos || row.videos === '[]')) continue;
entries.push({
id: row.id,
url: row.url,
depth: row.depth,
crawledAt: row.crawledAt,
hreflangs: row.hreflangs,
title: row.title,
lang: row.lang,
ogSiteName: row.ogSiteName,
videos: row.videos,
metaDescription: row.metaDescription,
ogImage: row.ogImage,
});
}
packages/db/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/db",
"version": "0.8.7",
"version": "0.8.8",
"private": true,
"type": "module",
"main": "./dist/index.js",
packages/db/src/migrations.ts
+31
View File
@@ -1861,6 +1861,37 @@ const MIGRATIONS: Migration[] = [
`);
},
},
{
version: 77,
name: 'add_urls_videos',
// Video sitemap variant — per-page videos discovered by the parser
// (`<video>` media + recognised YouTube/Vimeo embeds). Stored as a JSON
// array of `{ contentLoc, playerLoc, thumbnail }` on `urls`, mirroring
// the `hreflangs` column; null when the page has no detectable video.
up: (db) => {
db.exec('ALTER TABLE urls ADD COLUMN videos TEXT');
},
},
{
version: 78,
name: 'add_a11y_mobile_columns',
// Mobile-usability checks from the JS-render in-page a11y pass.
// a11y_small_font — count of sampled text elements rendered
// below ~12px (too small on mobile).
// a11y_tap_targets_small — count of interactive elements rendered
// below the WCAG 2.5.8 24x24px minimum.
// Both NULL until the page is rendered with the a11y audit on.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(urls)')
.all() as unknown as { name: string }[];
for (const col of ['a11y_small_font', 'a11y_tap_targets_small']) {
if (!cols.some((c) => c.name === col)) {
db.exec(`ALTER TABLE urls ADD COLUMN ${col} INTEGER`);
}
}
},
},
];
export function runMigrations(db: DatabaseSync): void {
packages/db/src/project-db.ts
+43 -4
View File
@@ -141,6 +141,7 @@ interface UrlRowDb {
pagination_prev: string | null;
hreflangs: string | null;
hreflang_count: number;
videos: string | null;
amphtml: string | null;
/** 1 when the page declares `<html ⚡>` / `<html amp>`, else 0. */
amp_page: number;
@@ -155,6 +156,9 @@ interface UrlRowDb {
* NULL when not audited; a11y_low_contrast >=0, a11y_focus_suppressed 0/1. */
a11y_low_contrast: number | null;
a11y_focus_suppressed: number | null;
/** Mobile-usability a11y counts (JS-render pass); NULL until audited. */
a11y_small_font: number | null;
a11y_tap_targets_small: number | null;
/** V2 Faz 15 — performance budget verdict bitmask
* (1=response, 2=size, 4=LCP, 8=CLS). 0 = within budget,
* NULL = budget disabled / page not an internal 200 HTML page. */
@@ -326,6 +330,8 @@ export interface UpsertUrlInput {
/** JSON-stringified array of `HreflangEntry` objects, or null. */
hreflangs?: string | null;
hreflangCount?: number;
/** JSON-stringified array of `VideoEntry` objects, or null. */
videos?: string | null;
amphtml?: string | null;
ampPage?: boolean;
/** JSON-stringified array of AMP validator error codes. */
@@ -427,7 +433,7 @@ const UPSERT_URL_SQL = `
meta_keywords, meta_author, meta_generator, theme_color,
hsts, x_frame_options, x_content_type_options, content_encoding,
schema_types, schema_block_count, schema_invalid_count,
pagination_next, pagination_prev, hreflangs, hreflang_count,
pagination_next, pagination_prev, hreflangs, hreflang_count, videos,
amphtml, favicon, mixed_content_count, mixed_content_active, mixed_content_passive,
folder_depth, query_param_count,
csp, referrer_policy, permissions_policy,
@@ -468,7 +474,7 @@ const UPSERT_URL_SQL = `
:meta_keywords, :meta_author, :meta_generator, :theme_color,
:hsts, :x_frame_options, :x_content_type_options, :content_encoding,
:schema_types, :schema_block_count, :schema_invalid_count,
:pagination_next, :pagination_prev, :hreflangs, :hreflang_count,
:pagination_next, :pagination_prev, :hreflangs, :hreflang_count, :videos,
:amphtml, :favicon, :mixed_content_count, :mixed_content_active, :mixed_content_passive,
:folder_depth, :query_param_count,
:csp, :referrer_policy, :permissions_policy,
@@ -554,6 +560,7 @@ const UPSERT_URL_SQL = `
pagination_prev = excluded.pagination_prev,
hreflangs = excluded.hreflangs,
hreflang_count = excluded.hreflang_count,
videos = excluded.videos,
amphtml = excluded.amphtml,
favicon = excluded.favicon,
mixed_content_count = excluded.mixed_content_count,
@@ -1427,6 +1434,7 @@ export class ProjectDb {
pagination_next: input.paginationNext ?? null,
pagination_prev: input.paginationPrev ?? null,
hreflangs: input.hreflangs ?? null,
videos: input.videos ?? null,
hreflang_count: input.hreflangCount ?? 0,
amphtml: input.amphtml ?? null,
amp_page: input.ampPage ? 1 : 0,
@@ -5234,6 +5242,12 @@ export class ProjectDb {
`${html} AND a11y_low_contrast IS NOT NULL AND a11y_low_contrast > 0`,
),
focusOutlineSuppressed: countWhere(`${html} AND a11y_focus_suppressed = 1`),
fontTooSmall: countWhere(
`${html} AND a11y_small_font IS NOT NULL AND a11y_small_font > 0`,
),
tapTargetsTooSmall: countWhere(
`${html} AND a11y_tap_targets_small IS NOT NULL AND a11y_tap_targets_small > 0`,
),
paginationSequenceBreak: countWhere(
`${html} AND pagination_sequence_break = 1`,
),
@@ -5734,18 +5748,29 @@ export class ProjectDb {
*/
setUrlA11y(
urlId: number,
audit: { lowContrast: number; sampled: number; focusSuppressed: boolean },
audit: {
lowContrast: number;
sampled: number;
focusSuppressed: boolean;
smallFont?: number;
tapTargetsSmall?: number;
tapTargetsSampled?: number;
},
): void {
this.db
.prepare(
`UPDATE urls
SET a11y_low_contrast = ?,
a11y_focus_suppressed = ?
a11y_focus_suppressed = ?,
a11y_small_font = ?,
a11y_tap_targets_small = ?
WHERE id = ?`,
)
.run(
Math.max(0, Math.round(audit.lowContrast)),
audit.focusSuppressed ? 1 : 0,
Math.max(0, Math.round(audit.smallFont ?? 0)),
Math.max(0, Math.round(audit.tapTargetsSmall ?? 0)),
urlId,
);
}
@@ -7531,12 +7556,15 @@ export class ProjectDb {
paginationSequenceBreak: (r as unknown as { pagination_sequence_break?: number }).pagination_sequence_break === 1,
hreflangs: r.hreflangs,
hreflangCount: r.hreflang_count,
videos: r.videos ?? null,
amphtml: r.amphtml,
ampPage: r.amp_page === 1,
ampValidationErrors: r.amp_validation_errors,
boilerplateCoverage: r.boilerplate_coverage,
a11yLowContrast: r.a11y_low_contrast ?? null,
a11yFocusSuppressed: r.a11y_focus_suppressed ?? null,
a11ySmallFont: r.a11y_small_font ?? null,
a11yTapTargetsSmall: r.a11y_tap_targets_small ?? null,
budgetStatus: r.budget_status ?? null,
favicon: r.favicon,
mixedContentCount: r.mixed_content_count,
@@ -9161,6 +9189,17 @@ function categoryWhereClause(cat: UrlCategory): string | null {
// (not audited) excluded.
return `is_external = 0 AND content_kind = 'html'
AND a11y_focus_suppressed = 1`;
case 'issues:font-too-small':
// Pages with sampled text rendered below ~12px (hard to read on mobile
// without zooming), from the JS-render in-page audit. NULL (not
// audited) excluded — only flag pages we actually measured.
return `is_external = 0 AND content_kind = 'html'
AND a11y_small_font IS NOT NULL AND a11y_small_font > 0`;
case 'issues:tap-targets-too-small':
// Pages with interactive controls rendered below the WCAG 2.5.8
// 24×24px minimum. NULL (not audited) excluded.
return `is_external = 0 AND content_kind = 'html'
AND a11y_tap_targets_small IS NOT NULL AND a11y_tap_targets_small > 0`;
case 'issues:pagination-sequence-break':
// Set post-crawl by `recomputePaginationSequence()` — see the
// implementation comment there for the gap-detection algorithm
packages/shared-types/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@freecrawl/shared-types",
"version": "0.8.7",
"version": "0.8.8",
"private": true,
"type": "module",
"main": "./dist/index.js",
packages/shared-types/src/crawl.ts
+13
View File
@@ -200,6 +200,8 @@ export type UrlCategory =
| 'issues:twitter-image-wrong-aspect'
| 'issues:low-contrast-text'
| 'issues:focus-outline-suppressed'
| 'issues:font-too-small'
| 'issues:tap-targets-too-small'
| 'issues:pagination-sequence-break'
| 'issues:links-per-page-too-many'
| 'tab:redirects'
@@ -488,6 +490,9 @@ export interface CrawlUrlRow {
/** JSON-stringified array of `{ lang, href }` objects, or null. */
hreflangs: string | null;
hreflangCount: number;
/** JSON-stringified array of `{ contentLoc, playerLoc, thumbnail }` video
* objects for the video-sitemap variant, or null. */
videos: string | null;
amphtml: string | null;
/** True when the page declares `<html ⚡>` / `<html amp>`. AMP-for-Ads
* / AMP-for-Email variants (`⚡4ads`, `⚡4email`) count too. */
@@ -512,6 +517,11 @@ export interface CrawlUrlRow {
*/
a11yLowContrast: number | null;
a11yFocusSuppressed: number | null;
/** Mobile-usability a11y counts (JS-render pass), NULL until audited.
* `a11ySmallFont` = sampled text elements rendered below ~12px;
* `a11yTapTargetsSmall` = interactive elements below 24×24 CSS px. */
a11ySmallFont: number | null;
a11yTapTargetsSmall: number | null;
/**
* V2 Faz 15 — performance budget verdict, a bitmask: 1=response time,
* 2=transfer size, 4=LCP, 8=CLS. 0 = evaluated and within budget,
@@ -1677,6 +1687,9 @@ export interface OverviewCounts {
*/
lowContrastText: number;
focusOutlineSuppressed: number;
/** Mobile-usability checks from the same in-page a11y audit. */
fontTooSmall: number;
tapTargetsTooSmall: number;
/**
* Pages whose `twitter:image` dimensions don't fit the declared
* card type — `summary_large_image` wants ~2:1 (min 300×157),
packages/shared-types/src/ipc.ts
+5 -1
View File
@@ -462,6 +462,10 @@ export type MenuEvent =
| 'open-project-encrypted'
| 'open-visualization'
| 'generate-sitemap'
| 'generate-sitemap-images'
| 'generate-sitemap-hreflang'
| 'generate-sitemap-news'
| 'generate-sitemap-video'
| 'open-robots-tester'
| 'open-sitemap-validator'
| 'open-reports'
@@ -743,7 +747,7 @@ export interface SitemapGenerateInput {
* Variant: `standard` (default), `image` (Google Images extension), or
* `hreflang` (international targeting via `<xhtml:link>`).
*/
variant?: 'standard' | 'image' | 'hreflang';
variant?: 'standard' | 'image' | 'hreflang' | 'news' | 'video';
/** Gzip the output (`.xml.gz`). Index file is gzipped too when sharded. */
gzip?: boolean;
/** Per-file URL cap (≤50,000). Sharding kicks in when exceeded. */
packages/shared-types/src/issue-severity.ts
+2
View File
@@ -131,6 +131,8 @@ const WARNING: UrlCategory[] = [
'issues:twitter-image-wrong-aspect',
'issues:low-contrast-text',
'issues:focus-outline-suppressed',
'issues:font-too-small',
'issues:tap-targets-too-small',
'issues:hreflang-inconsistent-lang',
'issues:over-budget',
];