pretecno/FreeCrawl-SEO-Tool
3
0 Fork 0
mirror of https://github.com/kemalai/FreeCrawl-SEO-Tool.git synced 2026-10-18 00:39:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: anchor checks, URL details in grids, MCP and OAuth refresh errors

Browse Source
This commit is contained in:
kemalai committed 2026-10-13 19:33:48 +03:00
1 parent 1a62e1e9af
commit 44bb3d09c6
45 files changed
+480 -94
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/desktop/src/main/google-oauth.ts
+12 -1
View File
@@ -32,6 +32,7 @@ import type { GoogleAccount, GoogleAuthState } from '@freecrawl/shared-types';
import { apiFetch } from './api-fetch.js';
import { resolveCredentials } from './credentials.js';
import * as logger from './logger.js';
import { L } from './menu-i18n.js';
const AUTH_ENDPOINT = 'https://accounts.google.com/o/oauth2/v2/auth';
const TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token';
@@ -568,10 +569,20 @@ export async function getAccessToken(
});
const json = (await res.json().catch(() => null)) as Record<string, unknown> | null;
if (!res.ok || !json || typeof json['access_token'] !== 'string') {
const code = json?.['error'] as string | undefined;
const msg =
(json?.['error_description'] as string | undefined) ??
(json?.['error'] as string | undefined) ??
code ??
`token refresh failed (HTTP ${res.status})`;
// `invalid_grant` (Google's description is just "Bad Request") is the
// refresh token itself being dead. With a BYOC client that is almost
// always the "Testing" publishing status, where Google expires refresh
// tokens after 7 days — tell the user the one-time fix instead of the
// generic "reconnect", which they'd otherwise repeat every week.
if (code === 'invalid_grant') {
logger.log('warn', 'main', `google-oauth: refresh rejected for "${integrationId}": ${msg}`);
throw new Error(L().oauthRefreshExpired.replace('{reason}', msg));
}
throw new Error(`${msg} — you may need to reconnect this integration.`);
}
const token = json['access_token'] as string;
apps/desktop/src/main/menu-i18n.ts
+4
View File
@@ -335,6 +335,10 @@ export interface MenuLabels {
dlgConfirmClearMsg: string;
dlgConfirmClearDetail: string;
dlgDontAskAgain: string;
// ── Google OAuth refresh failure ──
/** `{reason}` = Google's error code / description (e.g. `invalid_grant`). */
oauthRefreshExpired: string;
}
const MENU_LABELS: Record<UiLanguage, MenuLabels> = {
apps/desktop/src/main/menu-locales/az.ts
+4
View File
@@ -309,4 +309,8 @@ export const MENU_AZ: MenuLabels = {
dlgConfirmClearDetail:
'Bu, aktiv layihədəki hər taranmış URL-i, keçidi, şəkli, başlığı və mənbə görüntüsünü həmişəlik silir. Geri qaytarıla bilməz.',
dlgDontAskAgain: 'Bir daha soruşma',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google yadda saxlanılan girişi rədd etdi ({reason}). Bu adətən Google Cloud OAuth tətbiqinizin "Testing" statusunda olduğunu bildirir; bu halda Google bağlantını hər 7 gündə bir bitirir. Bir dəfə düzəldin: Google Cloud Console → OAuth consent screen → "Publish app" (təsdiq yoxlaması tələb olunmur), sonra bu inteqrasiyanı ayırıb yenidən qoşun.',
};
apps/desktop/src/main/menu-locales/en.ts
+4
View File
@@ -302,4 +302,8 @@ export const MENU_EN: MenuLabels = {
dlgConfirmClearDetail:
'This permanently deletes every crawled URL, link, image, header and source snapshot in the active project. This cannot be undone.',
dlgDontAskAgain: "Don't ask me again",
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google rejected the saved sign-in ({reason}). This usually means your Google Cloud OAuth app is in "Testing" status, where Google expires the connection every 7 days. Fix it once: Google Cloud Console → OAuth consent screen → "Publish app" (no verification review needed), then Disconnect and Connect this integration again.',
};
apps/desktop/src/main/menu-locales/es.ts
+4
View File
@@ -304,4 +304,8 @@ export const MENU_ES: MenuLabels = {
dlgConfirmClearDetail:
'Esto elimina de forma permanente cada URL rastreada, enlace, imagen, cabecera e instantánea del código fuente del proyecto activo. No se puede deshacer.',
dlgDontAskAgain: 'No volver a preguntar',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google rechazó el inicio de sesión guardado ({reason}). Normalmente significa que tu app OAuth de Google Cloud está en estado "Testing", donde Google caduca la conexión cada 7 días. Arréglalo una vez: Google Cloud Console → OAuth consent screen → "Publish app" (sin revisión de verificación) y luego Desconecta y Conecta esta integración de nuevo.',
};
apps/desktop/src/main/menu-locales/fr.ts
+4
View File
@@ -304,4 +304,8 @@ export const MENU_FR: MenuLabels = {
dlgConfirmClearDetail:
'Cette action supprime définitivement chaque URL explorée, lien, image, en-tête et instantané du code source du projet actif. Elle est irréversible.',
dlgDontAskAgain: 'Ne plus me demander',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google a rejeté la connexion enregistrée ({reason}). Cela signifie généralement que votre application OAuth Google Cloud est en statut "Testing", où Google fait expirer la connexion tous les 7 jours. Corrigez-le une fois : Google Cloud Console → OAuth consent screen → "Publish app" (aucune vérification requise), puis Déconnectez et Connectez à nouveau cette intégration.',
};
apps/desktop/src/main/menu-locales/hi.ts
+4
View File
@@ -304,4 +304,8 @@ export const MENU_HI: MenuLabels = {
dlgConfirmClearDetail:
'यह सक्रिय प्रोजेक्ट के हर क्रॉल किए गए URL, लिंक, छवि, हेडर और स्रोत स्नैपशॉट को स्थायी रूप से हटा देता है। इसे पूर्ववत नहीं किया जा सकता।',
dlgDontAskAgain: 'मुझसे फिर न पूछें',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google ने सहेजा हुआ साइन-इन अस्वीकार कर दिया ({reason})। इसका आमतौर पर मतलब है कि आपका Google Cloud OAuth ऐप "Testing" स्थिति में है, जहाँ Google हर 7 दिन में कनेक्शन समाप्त कर देता है। एक बार ठीक करें: Google Cloud Console → OAuth consent screen → "Publish app" (सत्यापन समीक्षा की ज़रूरत नहीं), फिर इस इंटीग्रेशन को डिस्कनेक्ट करके दोबारा कनेक्ट करें।',
};
apps/desktop/src/main/menu-locales/it.ts
+4
View File
@@ -304,4 +304,8 @@ export const MENU_IT: MenuLabels = {
dlgConfirmClearDetail:
'Questa operazione elimina in modo permanente ogni URL scansionato, link, immagine, header e snapshot del codice sorgente nel progetto attivo. Non è reversibile.',
dlgDontAskAgain: 'Non chiedermelo più',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google ha rifiutato l\'accesso salvato ({reason}). Di solito significa che la tua app OAuth di Google Cloud è nello stato "Testing", in cui Google fa scadere la connessione ogni 7 giorni. Risolvi una volta per tutte: Google Cloud Console → OAuth consent screen → "Publish app" (nessuna revisione di verifica), poi Disconnetti e Connetti di nuovo questa integrazione.',
};
apps/desktop/src/main/menu-locales/ko.ts
+4
View File
@@ -301,4 +301,8 @@ export const MENU_KO: MenuLabels = {
dlgConfirmClearDetail:
'현재 프로젝트에서 크롤된 모든 URL, 링크, 이미지, 헤더, 소스 스냅샷이 영구적으로 삭제됩니다. 되돌릴 수 없습니다.',
dlgDontAskAgain: '다시 묻지 않기',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google이 저장된 로그인을 거부했습니다({reason}). 보통 Google Cloud OAuth 앱이 "Testing" 상태라는 뜻이며, 이 상태에서는 Google이 7일마다 연결을 만료시킵니다. 한 번만 고치세요: Google Cloud Console → OAuth consent screen → "Publish app"(검증 심사 불필요), 그다음 이 통합을 연결 해제 후 다시 연결하세요.',
};
apps/desktop/src/main/menu-locales/pt-BR.ts
+4
View File
@@ -304,4 +304,8 @@ export const MENU_PT_BR: MenuLabels = {
dlgConfirmClearDetail:
'Isso exclui permanentemente cada URL rastreada, link, imagem, cabeçalho e snapshot do código-fonte do projeto ativo. Não pode ser desfeito.',
dlgDontAskAgain: 'Não perguntar novamente',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'O Google rejeitou o login salvo ({reason}). Isso normalmente significa que seu app OAuth do Google Cloud está no status "Testing", em que o Google expira a conexão a cada 7 dias. Corrija uma vez: Google Cloud Console → OAuth consent screen → "Publish app" (sem revisão de verificação) e depois Desconecte e Conecte esta integração de novo.',
};
apps/desktop/src/main/menu-locales/ru.ts
+4
View File
@@ -304,4 +304,8 @@ export const MENU_RU: MenuLabels = {
dlgConfirmClearDetail:
'Это безвозвратно удалит каждый обойдённый URL, ссылку, изображение, заголовок и снимок исходного кода в активном проекте. Отменить нельзя.',
dlgDontAskAgain: 'Больше не спрашивать',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google отклонил сохранённый вход ({reason}). Обычно это означает, что ваше OAuth-приложение в Google Cloud находится в статусе "Testing", при котором Google сбрасывает подключение каждые 7 дней. Исправьте один раз: Google Cloud Console → OAuth consent screen → "Publish app" (проверка не требуется), затем отключите и снова подключите эту интеграцию.',
};
apps/desktop/src/main/menu-locales/tr.ts
+4
View File
@@ -302,4 +302,8 @@ export const MENU_TR: MenuLabels = {
dlgConfirmClearDetail:
'Bu işlem, aktif projedeki taranmış her URL, link, görsel, başlık ve kaynak snapshot\'ını kalıcı olarak siler. Geri alınamaz.',
dlgDontAskAgain: 'Bir daha sorma',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google kayıtlı oturumu reddetti ({reason}). Bu genellikle Google Cloud OAuth uygulamanızın "Testing" durumunda olduğunu gösterir; Google bu durumda bağlantıyı her 7 günde bir düşürür. Bir kez düzeltin: Google Cloud Console → OAuth consent screen → "Publish app" (verification incelemesi gerekmez), sonra bu entegrasyonu Bağlantıyı Kes ve tekrar Bağla.',
};
apps/desktop/src/main/menu-locales/zh-CN.ts
+4
View File
@@ -299,4 +299,8 @@ export const MENU_ZH_CN: MenuLabels = {
dlgConfirmClearDetail:
'此操作将永久删除当前项目中每个已抓取的 URL、链接、图片、响应头和源码快照，且无法撤销。',
dlgDontAskAgain: '不再询问',
// ── Google OAuth refresh failure ──
oauthRefreshExpired:
'Google 拒绝了已保存的登录（{reason}）。这通常表示你的 Google Cloud OAuth 应用处于 "Testing" 状态，Google 会每 7 天使连接失效。一次性修复：Google Cloud Console → OAuth consent screen → "Publish app"（无需验证审核），然后断开并重新连接此集成。',
};
apps/desktop/src/renderer/src/components/integration-guides/az.ts
+1 -1
View File
@@ -544,7 +544,7 @@ export const GUIDES_AZ: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'Test rejimində OAuth yeniləmə tokenləri hər 7 gündə bitir. Parametrlər → İnteqrasiyalar → Search Console → "Bağlantını kəs" → "Bağlan" ilə yenidən autentifikasiya edin. Bitmədən qurtulmaq üçün tətbiqi Google-un təsdiq prosesindən (1-4 həftə) keçirməlisiniz.',
'Razılıq ekranı "Testing" nəşr statusunda olduğu müddətcə Google OAuth yeniləmə tokenlərini 7 gündən sonra etibarsız edir. Daimi həll: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (status "In production" olur). Təsdiq yoxlaması tələb olunmur — "unverified app" xəbərdarlığı qalır, amma tokenlər artıq bitmir. Sonra Parametrlər → İnteqrasiyalar → Search Console → "Bağlantını kəs" → "Bağlan" ilə son dəfə yenidən daxil olun.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/en.ts
+1 -1
View File
@@ -538,7 +538,7 @@ export const GUIDES_EN: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'OAuth refresh tokens expire every 7 days in testing mode. Settings → Integrations → Search Console → "Disconnect" → "Connect" to re-auth. To get rid of the expiry you need to push the app through Google\'s verification process (1-4 weeks).',
'Google expires OAuth refresh tokens after 7 days while the consent screen is in "Testing" publishing status. Permanent fix: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (status becomes "In production"). No verification review is needed — the "unverified app" warning stays, but tokens stop expiring. Then Settings → Integrations → Search Console → "Disconnect" → "Connect" one last time.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/es.ts
+1 -1
View File
@@ -550,7 +550,7 @@ export const GUIDES_ES: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'Los refresh tokens de OAuth caducan cada 7 días en modo de pruebas. Ajustes → Integraciones → Search Console → "Desconectar" → "Conectar" para volver a autenticarte. Para librarte de la caducidad tienes que pasar la app por el proceso de verificación de Google (1-4 semanas).',
'Google caduca los refresh tokens de OAuth a los 7 días mientras la pantalla de consentimiento está en estado de publicación "Testing". Solución definitiva: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (el estado pasa a "In production"). No hace falta la revisión de verificación: el aviso de "unverified app" se mantiene, pero los tokens dejan de caducar. Después, Ajustes → Integraciones → Search Console → "Desconectar" → "Conectar" por última vez.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/fr.ts
+1 -1
View File
@@ -549,7 +549,7 @@ export const GUIDES_FR: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'Les jetons de rafraîchissement OAuth expirent tous les 7 jours en mode test. Paramètres → Intégrations → Search Console → "Déconnecter" → "Connecter" pour vous réauthentifier. Pour supprimer cette expiration, il faut faire passer l\'application par le processus de validation de Google (1-4 semaines).',
'Google fait expirer les jetons de rafraîchissement OAuth après 7 jours tant que l\'écran de consentement est en statut de publication "Testing". Correctif définitif : Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (le statut passe à "In production"). Aucune vérification n\'est requise : l\'avertissement "unverified app" reste, mais les jetons n\'expirent plus. Ensuite, Paramètres → Intégrations → Search Console → "Déconnecter" → "Connecter" une dernière fois.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/hi.ts
+1 -1
View File
@@ -533,7 +533,7 @@ export const GUIDES_HI: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'टेस्टिंग मोड में OAuth रिफ़्रेश टोकन हर 7 दिन में समाप्त होते हैं। सेटिंग्स → इंटीग्रेशन → Search Console → "डिस्कनेक्ट" → "कनेक्ट" से पुनः प्रमाणित करें। समाप्ति से छुटकारा पाने के लिए ऐप को Google की सत्यापन प्रक्रिया (1-4 सप्ताह) से गुज़ारना होगा।',
'जब तक consent screen "Testing" प्रकाशन स्थिति में है, Google OAuth रिफ़्रेश टोकन को 7 दिन बाद समाप्त कर देता है। स्थायी समाधान: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (स्थिति "In production" हो जाती है)। सत्यापन समीक्षा की ज़रूरत नहीं है — "unverified app" चेतावनी बनी रहती है, पर टोकन समाप्त होना बंद हो जाते हैं। फिर सेटिंग्स → इंटीग्रेशन → Search Console → "डिस्कनेक्ट" → "कनेक्ट" आख़िरी बार करें।',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/it.ts
+1 -1
View File
@@ -546,7 +546,7 @@ export const GUIDES_IT: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'I refresh token OAuth scadono ogni 7 giorni in modalità test. Impostazioni → Integrazioni → Search Console → "Disconnetti" → "Connetti" per riautenticarti. Per eliminare la scadenza devi far passare l\'app per il processo di verifica di Google (1-4 settimane).',
'Google fa scadere i refresh token OAuth dopo 7 giorni finché la schermata di consenso è nello stato di pubblicazione "Testing". Soluzione definitiva: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (lo stato diventa "In production"). Non serve la revisione di verifica: l\'avviso "unverified app" resta, ma i token smettono di scadere. Poi Impostazioni → Integrazioni → Search Console → "Disconnetti" → "Connetti" un\'ultima volta.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/ko.ts
+1 -1
View File
@@ -532,7 +532,7 @@ export const GUIDES_KO: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'테스트 모드에서는 OAuth 리프레시 토큰이 7일마다 만료됩니다. 설정 → 통합 → Search Console → "연결 해제" → "연결"로 다시 인증하세요. 만료를 없애려면 앱을 Google 검증 절차(1-4주)에 통과시켜야 합니다.',
'동의 화면이 "Testing" 게시 상태인 동안 Google은 OAuth 리프레시 토큰을 7일 후 만료시킵니다. 영구 해결: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app"(상태가 "In production"으로 바뀜). 검증 심사는 필요 없습니다. "unverified app" 경고는 남지만 토큰은 더 이상 만료되지 않습니다. 그다음 설정 → 연동 → Search Console → "연결 해제" → "연결"로 마지막 한 번만 다시 인증하세요.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/pt-BR.ts
+1 -1
View File
@@ -548,7 +548,7 @@ export const GUIDES_PT_BR: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'Refresh tokens do OAuth expiram a cada 7 dias no modo de teste. Configurações → Integrações → Search Console → "Desconectar" → "Conectar" para autenticar de novo. Para se livrar da expiração é preciso passar o app pelo processo de verificação do Google (1-4 semanas).',
'O Google expira os refresh tokens do OAuth após 7 dias enquanto a tela de consentimento está no status de publicação "Testing". Correção definitiva: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (o status passa a "In production"). Não é preciso passar pela revisão de verificação: o aviso "unverified app" continua, mas os tokens deixam de expirar. Depois, Configurações → Integrações → Search Console → "Desconectar" → "Conectar" uma última vez.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/ru.ts
+1 -1
View File
@@ -539,7 +539,7 @@ export const GUIDES_RU: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'В режиме тестирования refresh-токены OAuth истекают каждые 7 дней. Настройки → Интеграции → Search Console → «Отключить» → «Подключить» для повторной авторизации. Чтобы избавиться от истечения, нужно провести приложение через проверку Google (1-4 недели).',
'Google аннулирует refresh-токены OAuth через 7 дней, пока экран согласия находится в статусе публикации "Testing". Постоянное решение: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (статус станет "In production"). Проверка приложения не требуется: предупреждение "unverified app" останется, но токены перестанут истекать. Затем Настройки → Интеграции → Search Console → «Отключить» → «Подключить» в последний раз.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/tr.ts
+1 -1
View File
@@ -541,7 +541,7 @@ export const GUIDES_TR: Record<string, Guide> = {
{
problem: 'Bağlantı 7 gün sonra patladı',
solution:
'Testing modunda OAuth refresh token\'lar 7 günde bir expire olur. Settings → Integrations → Search Console → "Disconnect" → "Connect" ile yeniden bağlan. Production\'a geçmek için Google verification süreci gerekir (1-4 hafta).',
'Google, OAuth consent screen "Testing" yayın durumundayken refresh token\'ları 7 gün sonra geçersiz kılar. Kalıcı çözüm: Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app" (durum "In production" olur). Verification incelemesi gerekmez — "unverified app" uyarısı kalır ama token\'lar artık expire olmaz. Ardından Settings → Integrations → Search Console → "Disconnect" → "Connect" ile son bir kez bağlan.',
},
],
notes: [
apps/desktop/src/renderer/src/components/integration-guides/zh-CN.ts
+1 -1
View File
@@ -526,7 +526,7 @@ export const GUIDES_ZH_CN: Record<string, Guide> = {
{
problem: 'Connection broke after 7 days',
solution:
'测试模式下 OAuth 刷新令牌每 7 天过期。设置 → 集成 → Search Console → “断开” → “连接”重新授权。要消除过期限制，需要让应用通过 Google 的验证流程（1-4 周）。',
'同意屏幕处于 "Testing" 发布状态时，Google 会在 7 天后使 OAuth 刷新令牌失效。永久解决：Google Cloud Console → APIs & Services → OAuth consent screen → "Publish app"（状态变为 "In production"）。无需通过验证审核："unverified app" 警告会保留，但令牌不再过期。然后设置 → 集成 → Search Console → “断开” → “连接”，最后再授权一次。',
},
],
notes: [
apps/desktop/src/renderer/src/store.ts
+34 -3
View File
@@ -530,7 +530,7 @@ if (typeof window !== 'undefined') {
window.addEventListener('beforeunload', flushConfigPersist);
}
export const useAppStore = create<AppState>((set) => ({
export const useAppStore = create<AppState>((set, get) => ({
config: loadInitialConfig(),
progress: null,
summary: null,
@@ -584,7 +584,13 @@ export const useAppStore = create<AppState>((set) => ({
navigateToCategory: (c) => {
// Some categories are only meaningful on specific tabs — switch the
// active tab along with the category so the correct view renders.
const tab = tabForCategory(c);
// A category with no dedicated home stays on the current tab — but
// only when that tab is a URL table that honours `activeCategory`.
// Dedicated grids (Broken Links, Images, SERP, GA4, …) ignore it, so
// leaving the user there would show the old table under the new
// filter's label; land on Internal instead.
const tab =
tabForCategory(c) ?? (tabRendersUrlTable(get().activeTab) ? null : 'internal');
set(tab ? { activeCategory: c, activeTab: tab } : { activeCategory: c });
},
setError: (e) => set({ error: e }),
@@ -717,6 +723,28 @@ export const useAppStore = create<AppState>((set) => ({
}),
}));
// Tabs whose grid is the category-filtered URL table (`UrlsTab`). The rest
// mount a dedicated component that never reads `activeCategory` — keep in
// step with the tab switch in App.tsx.
const DEDICATED_GRID_TABS: ReadonlySet<TabKey> = new Set<TabKey>([
'images',
'broken-links',
'serp',
'pagespeed',
'crux',
'spelling',
'search-console',
'analytics',
'metrica',
'ai',
'seo',
'assistant',
]);
function tabRendersUrlTable(tab: TabKey): boolean {
return !DEDICATED_GRID_TABS.has(tab);
}
function tabForCategory(cat: UrlCategory): TabKey | null {
// V2 — Comprehensive sidebar-click → top-level tab routing. Lets the
// user click any OverviewSidebar entry and land on the tab whose
@@ -849,7 +877,10 @@ function tabForCategory(cat: UrlCategory): TabKey | null {
) {
return 'response-codes';
}
if (cat.startsWith('issues:analytics-')) return 'analytics';
// Page-level tag checks (no GA4 snippet, legacy UA, …) filter the URL
// table; the 'analytics' tab is the GA4 traffic grid, which ignores the
// category entirely and would show unrelated data (or a connect screen).
if (cat.startsWith('issues:analytics-')) return 'internal';
if (
cat === 'issues:anchor-text-too-long' ||
cat === 'issues:anchor-text-generic' ||
apps/desktop/src/renderer/src/tabs/AiTab.tsx
+9 -1
View File
@@ -56,6 +56,8 @@ function fmtTokens(a: AiResult): string {
export function AiTab() {
const { t } = useTranslation();
const dataVersion = useAppStore((s) => s.dataVersion);
const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
const setSelectedUrlIds = useAppStore((s) => s.setSelectedUrlIds);
const crawlProgress = useAppStore((s) => s.progress);
const [provider, setProvider] = useState<AiProvider>('openai');
@@ -413,7 +415,13 @@ export function AiTab() {
<AiDataRow
row={row}
selected={selected.has(row.url)}
onToggle={() => toggleRow(row.url)}
onToggle={() => {
toggleRow(row.url);
// The click still picks the row for the next run; it
// also opens the page in the Detail panel below.
setSelectedUrlId(row.id);
setSelectedUrlIds([row.id]);
}}
/>
</div>
);
apps/desktop/src/renderer/src/tabs/AnalyticsTab.tsx
+29 -3
View File
@@ -59,6 +59,11 @@ function relativeTime(iso: string): string {
export function AnalyticsTab() {
const { t } = useTranslation();
const dataVersion = useAppStore((s) => s.dataVersion);
// Clicking a row opens its page in the Detail panel below, as in the URL
// tables — without this the panel kept whatever another tab had picked.
const selectedUrlId = useAppStore((s) => s.selectedUrlId);
const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
const setSelectedUrlIds = useAppStore((s) => s.setSelectedUrlIds);
const crawlProgress = useAppStore((s) => s.progress);
const [stage, setStage] = useState<Stage>('loading');
@@ -522,7 +527,14 @@ export function AnalyticsTab() {
transform: `translateY(${vRow.start}px)`,
}}
>
<Ga4DataRow row={row} />
<Ga4DataRow
row={row}
active={selectedUrlId === row.id}
onSelect={() => {
setSelectedUrlId(row.id);
setSelectedUrlIds([row.id]);
}}
/>
</div>
);
})}
@@ -558,7 +570,16 @@ export function AnalyticsTab() {
);
}
function Ga4DataRow({ row }: { row: Ga4Row }) {
function Ga4DataRow({
row,
active,
onSelect,
}: {
row: Ga4Row;
/** The page the Detail panel is showing. */
active: boolean;
onSelect: () => void;
}) {
const a = row.ga4;
const status = row.statusCode;
const statusCls =
@@ -571,7 +592,12 @@ function Ga4DataRow({ row }: { row: Ga4Row }) {
: 'text-red-400';
return (
<div className="flex h-full items-center text-[11px] odd:bg-surface-900/20 hover:bg-surface-800/40">
<div
onClick={onSelect}
className={`flex h-full items-center text-[11px] cursor-pointer ${
active ? 'bg-accent-500/20' : 'odd:bg-surface-900/20 hover:bg-surface-800/40'
}`}
>
<div className="flex-1 truncate px-2 text-surface-200" title={row.url}>
{row.url}
</div>
apps/desktop/src/renderer/src/tabs/BrokenLinksTab.tsx
+40
View File
@@ -128,6 +128,8 @@ export function BrokenLinksTab() {
const activeCategory = useAppStore((s) => s.activeCategory);
const dataVersion = useAppStore((s) => s.dataVersion);
const progress = useAppStore((s) => s.progress);
const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
const setSelectedUrlIds = useAppStore((s) => s.setSelectedUrlIds);
const [rows, setRows] = useState<BrokenLinkRow[]>([]);
const [total, setTotal] = useState(0);
// Distinct broken target URLs — stable across crawls of different
@@ -227,6 +229,44 @@ export function BrokenLinksTab() {
dragRef.current = null;
}, [search, internal]);
// Mirror the cell selection into the global URL selection so the
// Detail panel below follows the clicked row's *source* page — the
// crawled URL that carries the broken link. Without this the panel
// kept showing whatever was picked on the previous tab. Multi-row
// picks aggregate (distinct pages, table order); the anchor cell's
// row is the primary one. An empty local selection is left alone:
// filter changes reset it, and that must not blank the panel.
useEffect(() => {
if (selected.size === 0) return;
const ids: number[] = [];
const seen = new Set<number>();
const rowIdxs = [...selected]
.map((k) => Number(k.split(':')[0]))
.filter((n) => Number.isFinite(n))
.sort((a, b) => a - b);
for (const ri of rowIdxs) {
const id = rows[ri]?.fromUrlId;
if (id === undefined || seen.has(id)) continue;
seen.add(id);
ids.push(id);
}
if (ids.length === 0) return;
const anchorId = anchorRef.current ? rows[anchorRef.current.r]?.fromUrlId : undefined;
const primary = anchorId !== undefined && seen.has(anchorId) ? anchorId : ids[0]!;
// A poll refresh replaces `rows` with equal content; skip the write
// when nothing changed so the Detail panel isn't re-fetched every tick.
const cur = useAppStore.getState();
if (
cur.selectedUrlId === primary &&
cur.selectedUrlIds.length === ids.length &&
cur.selectedUrlIds.every((id, i) => id === ids[i])
) {
return;
}
setSelectedUrlId(primary);
setSelectedUrlIds(ids);
}, [selected, rows, setSelectedUrlId, setSelectedUrlIds]);
// Any mouseup ends a drag — even one released outside the table.
useEffect(() => {
const onUp = () => {
apps/desktop/src/renderer/src/tabs/CruxTab.tsx
+9 -1
View File
@@ -68,6 +68,8 @@ function relativeTime(iso: string): string {
export function CruxTab() {
const { t } = useTranslation();
const dataVersion = useAppStore((s) => s.dataVersion);
const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
const setSelectedUrlIds = useAppStore((s) => s.setSelectedUrlIds);
const crawlProgress = useAppStore((s) => s.progress);
const setSettingsOpen = useAppStore((s) => s.setSettingsOpen);
@@ -409,7 +411,13 @@ export function CruxTab() {
<CruxTableRow
row={row}
selected={selected.has(row.url)}
onToggle={() => toggleRow(row.url)}
onToggle={() => {
toggleRow(row.url);
// The click still picks the row for the next run; it
// also opens the page in the Detail panel below.
setSelectedUrlId(row.id);
setSelectedUrlIds([row.id]);
}}
/>
</div>
);
apps/desktop/src/renderer/src/tabs/PageSpeedTab.tsx
+9 -1
View File
@@ -76,6 +76,8 @@ function relativeTime(iso: string): string {
export function PageSpeedTab() {
const { t } = useTranslation();
const dataVersion = useAppStore((s) => s.dataVersion);
const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
const setSelectedUrlIds = useAppStore((s) => s.setSelectedUrlIds);
const crawlProgress = useAppStore((s) => s.progress);
const setSettingsOpen = useAppStore((s) => s.setSettingsOpen);
@@ -444,7 +446,13 @@ export function PageSpeedTab() {
<PageSpeedRow
row={row}
selected={selected.has(row.url)}
onToggle={() => toggleRow(row.url)}
onToggle={() => {
toggleRow(row.url);
// The click still picks the row for the next run; it
// also opens the page in the Detail panel below.
setSelectedUrlId(row.id);
setSelectedUrlIds([row.id]);
}}
/>
</div>
);
apps/desktop/src/renderer/src/tabs/SearchConsoleTab.tsx
+31 -3
View File
@@ -72,6 +72,11 @@ function relativeTime(iso: string): string {
export function SearchConsoleTab() {
const { t } = useTranslation();
const dataVersion = useAppStore((s) => s.dataVersion);
// Clicking a row opens its page in the Detail panel below, as in the URL
// tables — without this the panel kept whatever another tab had picked.
const selectedUrlId = useAppStore((s) => s.selectedUrlId);
const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
const setSelectedUrlIds = useAppStore((s) => s.setSelectedUrlIds);
const crawlProgress = useAppStore((s) => s.progress);
const [stage, setStage] = useState<Stage>('loading');
@@ -692,7 +697,16 @@ export function SearchConsoleTab() {
transform: `translateY(${vRow.start}px)`,
}}
>
<GscDataRow row={row} />
<GscDataRow
row={row}
active={row.id !== null && selectedUrlId === row.id}
onSelect={() => {
// A GSC URL the crawl never reached has no page to open.
if (row.id === null) return;
setSelectedUrlId(row.id);
setSelectedUrlIds([row.id]);
}}
/>
</div>
);
})}
@@ -728,7 +742,16 @@ export function SearchConsoleTab() {
);
}
function GscDataRow({ row }: { row: GscRow }) {
function GscDataRow({
row,
active,
onSelect,
}: {
row: GscRow;
/** The page the Detail panel is showing. */
active: boolean;
onSelect: () => void;
}) {
const g = row.gsc;
const status = row.statusCode;
const statusCls =
@@ -741,7 +764,12 @@ function GscDataRow({ row }: { row: GscRow }) {
: 'text-red-400';
return (
<div className="flex h-full items-center text-[11px] odd:bg-surface-900/20 hover:bg-surface-800/40">
<div
onClick={onSelect}
className={`flex h-full items-center text-[11px] ${row.id !== null ? 'cursor-pointer' : ''} ${
active ? 'bg-accent-500/20' : 'odd:bg-surface-900/20 hover:bg-surface-800/40'
}`}
>
<div className="flex-1 truncate px-2 text-surface-200" title={row.url}>
{row.url}
</div>
apps/desktop/src/renderer/src/tabs/SeoTab.tsx
+9 -1
View File
@@ -101,6 +101,8 @@ function metricValue(provider: SeoProvider, metrics: SeoMetrics | null, key: str
export function SeoTab() {
const { t } = useTranslation();
const dataVersion = useAppStore((s) => s.dataVersion);
const setSelectedUrlId = useAppStore((s) => s.setSelectedUrlId);
const setSelectedUrlIds = useAppStore((s) => s.setSelectedUrlIds);
const crawlProgress = useAppStore((s) => s.progress);
const [provider, setProvider] = useState<SeoProvider>('ahrefs');
@@ -388,7 +390,13 @@ export function SeoTab() {
provider={provider}
cols={cols}
selected={selected.has(row.url)}
onToggle={() => toggleRow(row.url)}
onToggle={() => {
toggleRow(row.url);
// The click still picks the row for the next run; it
// also opens the page in the Detail panel below.
setSelectedUrlId(row.id);
setSelectedUrlIds([row.id]);
}}
/>
</div>
);
apps/mcp-server/src/tools.ts
+48 -15
View File
@@ -104,17 +104,38 @@ function resolveUrlId(
throw new Error('Tool requires either `id` (number) or `url` (string).');
}
/** One integration's per-project settings blob, as the desktop saves it
* (`integrationSettings:<id>`), or `{}` when missing or corrupt. */
function integrationSettings(db: ProjectDb, id: string): Record<string, unknown> {
try {
const raw = db.getMeta(`integrationSettings:${id}`);
const parsed = raw ? (JSON.parse(raw) as unknown) : null;
return parsed && typeof parsed === 'object' ? (parsed as Record<string, unknown>) : {};
} catch {
return {};
}
}
/**
* The Google account whose GSC / GA4 rows this project reads — the one
* the desktop saved for it (account picker, `google_account_set`, every
* fetch). Rows are keyed by account since multi-account, so querying
* without it finds nothing. The linked-account list lives in the
* desktop's credential store, out of this process's reach, so a project
* with no saved choice falls back to the account of its latest pull.
*/
function savedGoogleAccount(db: ProjectDb, id: 'gsc' | 'ga4'): string {
const saved = integrationSettings(db, id)['accountId'];
if (typeof saved === 'string' && saved) return saved;
return db.latestGoogleAccountId(id === 'gsc' ? 'gsc_results' : 'ga4_results');
}
/** The Metrica counter last picked or fetched for this project (the desktop
* saves it on a pick in the Metrica tab and after a successful pull), or ''
* when there is none. */
function savedMetricaCounter(db: ProjectDb): string {
try {
const raw = db.getMeta('integrationSettings:metrica');
const saved = raw ? (JSON.parse(raw) as { counterId?: unknown }) : null;
return typeof saved?.counterId === 'string' ? saved.counterId : '';
} catch {
return ''; // corrupt settings — no counter
}
const saved = integrationSettings(db, 'metrica')['counterId'];
return typeof saved === 'string' ? saved : '';
}
/** Sanity-cap a numeric param without falling back to undefined. */
@@ -611,7 +632,12 @@ export function buildTools(): Tool[] {
const id = resolveUrlId(args, db);
const row = db.getUrlRowById(id);
if (!row) throw new Error(`URL id ${id} not found.`);
return db.getUrlAnalytics(row.url, '', '', savedMetricaCounter(db));
return db.getUrlAnalytics(
row.url,
savedGoogleAccount(db, 'gsc'),
savedGoogleAccount(db, 'ga4'),
savedMetricaCounter(db),
);
},
},
@@ -642,7 +668,7 @@ export function buildTools(): Tool[] {
{
name: 'query_gsc',
description:
'Search Console rows: clicks, impressions, CTR, average position per crawled page. Returns null per-URL when the URL has not been pulled from GSC yet (configure the integration in Settings → Integrations → Search Console).',
'Search Console rows: clicks, impressions, CTR, average position per crawled page, from the Google account the project uses (switch with `google_account_set`). Returns null per-URL when the URL has not been pulled from GSC yet (configure the integration in Settings → Integrations → Search Console).',
inputSchema: {
type: 'object',
properties: {
@@ -656,19 +682,25 @@ export function buildTools(): Tool[] {
offset: { type: 'integer', minimum: 0 },
},
},
handler: (args, db) =>
db.queryGsc({
handler: (args, db) => {
// Same URL-matching options the desktop's Search Console tab joins with.
const settings = integrationSettings(db, 'gsc');
return db.queryGsc({
limit: clamp(args.limit, 1, 1000, 50),
offset: clamp(args.offset, 0, 10_000_000, 0),
search: typeof args.search === 'string' ? args.search : undefined,
filter: args.filter as 'all' | 'with-data' | 'without-data' | undefined,
}),
accountId: savedGoogleAccount(db, 'gsc'),
matchSlash: typeof settings['matchSlash'] === 'boolean' ? settings['matchSlash'] : undefined,
matchCase: typeof settings['matchCase'] === 'boolean' ? settings['matchCase'] : undefined,
});
},
},
{
name: 'query_ga4',
description:
'GA4 rows: sessions, users, bounceRate, engagementRate per crawled page. Configure the integration in Settings → Integrations → GA4 to populate.',
'GA4 rows: sessions, users, bounceRate, engagementRate per crawled page, from the Google account the project uses (switch with `google_account_set`). Configure the integration in Settings → Integrations → GA4 to populate.',
inputSchema: {
type: 'object',
properties: {
@@ -687,6 +719,7 @@ export function buildTools(): Tool[] {
offset: clamp(args.offset, 0, 10_000_000, 0),
search: typeof args.search === 'string' ? args.search : undefined,
filter: args.filter as 'all' | 'with-data' | 'without-data' | undefined,
accountId: savedGoogleAccount(db, 'ga4'),
}),
},
@@ -1173,8 +1206,8 @@ export function buildTools(): Tool[] {
handler: (args, db) => ({
rows: db.orphanPagesCrossSource(
clamp(args.limit, 1, 10_000, 1000),
'',
'',
savedGoogleAccount(db, 'gsc'),
savedGoogleAccount(db, 'ga4'),
savedMetricaCounter(db),
),
}),
docs/i18n-glossary.md
+5 -2
View File
@@ -96,8 +96,11 @@ noun and the second the verb.
- **es / pt-BR / fr / it** — use the infinitive for button labels and menu
actions (`Exportar`, `Exporter`, `Esportare` → but prefer the established
UI convention: `Exportar` / `Exporter` / `Esporta`). Address the user with
the formal/impersonal register; avoid `tú` / `tu` imperatives in help text.
UI convention: `Exportar` / `Exporter` / `Esporta`). Address the user in
the register each locale already ships: `es` and `it` use `tú` / `tu`
imperatives (`Abre…`, `Apri…`), `fr` uses `vous` (`Ouvrez…`) and `pt-BR`
uses `você` (`Abra…`). Keep new strings in that register — switching
register inside one language reads as machine translation.
- **ru** — button labels are imperative verbs or verbal nouns
(`Экспорт`, `Остановить`). Respect the three plural forms
(`_one` / `_few` / `_many` / `_other`).
packages/core/src/crawler.ts
+3
View File
@@ -4377,6 +4377,9 @@ export class Crawler extends EventEmitter {
titleCount: parsed.titleCount,
imagesEmptyAlt: parsed.imagesEmptyAlt,
emptyAnchorCount: parsed.emptyAnchorCount,
anchorTooLongCount: parsed.anchorTooLongCount,
genericAnchorCount: parsed.genericAnchorCount,
targetBlankNoNoopenerCount: parsed.targetBlankNoNoopenerCount,
microdataCount: parsed.microdataCount,
rdfaCount: parsed.rdfaCount,
insecureFormActionCount: parsed.insecureFormActionCount,
packages/core/src/html-parser.ts
+48 -23
View File
@@ -9,6 +9,7 @@ import type {
LinkPosition,
SchemaFinding,
} from '@freecrawl/shared-types';
import { GENERIC_ANCHOR_TEXTS } from '@freecrawl/shared-types';
import {
normalizeUrl,
isSameHost,
@@ -18,6 +19,8 @@ import {
import { computeContentFingerprint } from './simhash.js';
import { runExtractionRules } from './extraction.js';
const GENERIC_ANCHORS: ReadonlySet<string> = new Set(GENERIC_ANCHOR_TEXTS);
export interface HreflangEntry {
/** Language tag from `hreflang` attribute (e.g. "tr", "en-US", "x-default"). */
lang: string;
@@ -201,6 +204,12 @@ export interface ParsedPage {
feedUrl: string | null;
/** Number of internal hyperlinks with no usable anchor text or image alt. */
emptyAnchorCount: number;
/** `<a href>` elements whose anchor text is longer than 100 characters. */
anchorTooLongCount: number;
/** `<a href>` elements whose anchor is a generic phrase (`GENERIC_ANCHOR_TEXTS`). */
genericAnchorCount: number;
/** `<a target="_blank">` elements without `rel="noopener"` / `noreferrer`. */
targetBlankNoNoopenerCount: number;
/**
* Number of `<a>` elements that look clickable but are NOT crawlable —
* either no `href` at all (with `onclick`), `href="javascript:…"`, or
@@ -1289,6 +1298,9 @@ export function parseHtml(
}
let emptyAnchorCount = 0;
let anchorTooLongCount = 0;
let genericAnchorCount = 0;
let targetBlankNoNoopenerCount = 0;
// JS-only / non-crawlable anchors. Three patterns count toward this
// number — see the field doc on `parseHtml` return for the rationale:
// 1. `<a onclick="…">` (no href at all)
@@ -1345,6 +1357,38 @@ export function parseHtml(
}
const $el = $(el);
const rel = ($el.attr('rel') ?? '').trim().toLowerCase() || null;
const target = ($el.attr('target') ?? '').trim() || null;
const rawAnchor = $el.text().replace(/\s+/g, ' ').trim().slice(0, 200) || null;
const anchor = rawAnchor ? decodeEntities(rawAnchor) : null;
// Image inside <a>: capture its alt attribute so the detail table can
// show it alongside the plain anchor text.
let altText: string | null = null;
const imgAlt = $el.find('img[alt]').first().attr('alt');
if (imgAlt !== undefined) {
altText = decodeEntities(imgAlt.trim());
if (altText === '') altText = null;
}
const isInternal = isSameHost(pageUrl, normalized, opts);
// Link-level checks count every <a>, before the per-target dedupe
// below: the link graph keeps one record per target, so an icon-only,
// generic or `target=_blank` link to a page the nav already links to
// never reached the `links` table and the page went unflagged.
// Empty anchor: no usable text inside the <a> AND no alt on a nested
// image. Common accessibility/SEO failure on icon-only or image-only
// links. Only count internal links — externals aren't our problem.
if (isInternal && !anchor && !altText) emptyAnchorCount++;
if (anchor && anchor.length > 100) anchorTooLongCount++;
if (anchor && GENERIC_ANCHORS.has(anchor.trim().toLowerCase())) genericAnchorCount++;
if (
target?.toLowerCase() === '_blank' &&
!rel?.includes('noopener') &&
!rel?.includes('noreferrer')
) {
targetBlankNoNoopenerCount++;
}
// Same target linked more than once (a nav/footer link that also
// appears in the body). Keep the MOST permissive occurrence rather
@@ -1360,37 +1404,15 @@ export function parseHtml(
existing.rel = rel;
// A followable content link also carries the more useful anchor
// text than an icon-only nav link, so take it when we had none.
if (!existing.anchor) {
const upgradedAnchor =
$el.text().replace(/\s+/g, ' ').trim().slice(0, 200) || null;
if (upgradedAnchor) existing.anchor = decodeEntities(upgradedAnchor);
}
if (!existing.anchor && anchor) existing.anchor = anchor;
}
return;
}
const target = ($el.attr('target') ?? '').trim() || null;
const rawAnchor = $el.text().replace(/\s+/g, ' ').trim().slice(0, 200) || null;
const anchor = rawAnchor ? decodeEntities(rawAnchor) : null;
// Image inside <a>: capture its alt attribute so the detail table can
// show it alongside the plain anchor text.
let altText: string | null = null;
const imgAlt = $el.find('img[alt]').first().attr('alt');
if (imgAlt !== undefined) {
altText = decodeEntities(imgAlt.trim());
if (altText === '') altText = null;
}
const isInternal = isSameHost(pageUrl, normalized, opts);
const pathType = detectPathType(href);
const linkPath = buildLinkPath(el);
const linkPosition = detectLinkPosition(el);
// Empty anchor: no usable text inside the <a> AND no alt on a nested
// image. Common accessibility/SEO failure on icon-only or image-only
// links. Only count internal links — externals aren't our problem.
if (isInternal && !anchor && !altText) emptyAnchorCount++;
linkMap.set(normalized, {
fromUrl: pageUrl,
toUrl: normalized,
@@ -1739,6 +1761,9 @@ export function parseHtml(
manifestUrl,
feedUrl,
emptyAnchorCount,
anchorTooLongCount,
genericAnchorCount,
targetBlankNoNoopenerCount,
jsOnlyLinksCount,
textCodeRatio,
fleschReadingEase: readability.fleschReadingEase,
packages/db/src/migrations.ts
+23
View File
@@ -2343,6 +2343,29 @@ const MIGRATIONS: Migration[] = [
`);
},
},
{
version: 94,
name: 'add_link_level_anchor_counters',
// Per-page counters the HTML parser takes over every `<a>`. The link
// graph keeps one record per target, so a long, generic or
// `target=_blank` link to a page the nav already links to never
// reached `links` and the anchor checks missed the page.
up: (db) => {
const cols = db
.prepare('PRAGMA table_info(urls)')
.all() as unknown as { name: string }[];
const names = new Set(cols.map((c) => c.name));
for (const col of [
'anchor_too_long_count',
'generic_anchor_count',
'target_blank_no_noopener_count',
]) {
if (!names.has(col)) {
db.exec(`ALTER TABLE urls ADD COLUMN ${col} INTEGER NOT NULL DEFAULT 0`);
}
}
},
},
];
export function runMigrations(db: DatabaseSync): void {
packages/db/src/project-db.ts
+85 -29
View File
@@ -64,6 +64,7 @@ import type {
LogThreatSummary,
} from '@freecrawl/shared-types';
import {
GENERIC_ANCHOR_TEXTS,
ISSUE_COUNT_KEY,
issueCategoriesBySeverity,
normalizeDisabledIssues,
@@ -492,6 +493,12 @@ export interface UpsertUrlInput {
titleCount?: number;
imagesEmptyAlt?: number;
emptyAnchorCount?: number;
/** Per-page link-level counters over every `<a>` (not just the deduped
* link records): anchors > 100 chars, generic anchors, and
* `target=_blank` without noopener / noreferrer. */
anchorTooLongCount?: number;
genericAnchorCount?: number;
targetBlankNoNoopenerCount?: number;
appleTouchIcon?: string | null;
manifestUrl?: string | null;
feedUrl?: string | null;
@@ -632,6 +639,7 @@ const UPSERT_URL_SQL = `
extraction_results,
simhash, content_hash,
title_count, images_empty_alt, empty_anchor_count,
anchor_too_long_count, generic_anchor_count, target_blank_no_noopener_count,
apple_touch_icon, manifest_url, feed_url,
microdata_count, rdfa_count, insecure_form_action_count, missing_sri_count,
title_pixel_width, meta_pixel_width,
@@ -674,6 +682,7 @@ const UPSERT_URL_SQL = `
:extraction_results,
:simhash, :content_hash,
:title_count, :images_empty_alt, :empty_anchor_count,
:anchor_too_long_count, :generic_anchor_count, :target_blank_no_noopener_count,
:apple_touch_icon, :manifest_url, :feed_url,
:microdata_count, :rdfa_count, :insecure_form_action_count, :missing_sri_count,
:title_pixel_width, :meta_pixel_width,
@@ -776,6 +785,9 @@ const UPSERT_URL_SQL = `
title_count = excluded.title_count,
images_empty_alt = excluded.images_empty_alt,
empty_anchor_count = excluded.empty_anchor_count,
anchor_too_long_count = excluded.anchor_too_long_count,
generic_anchor_count = excluded.generic_anchor_count,
target_blank_no_noopener_count = excluded.target_blank_no_noopener_count,
apple_touch_icon = excluded.apple_touch_icon,
manifest_url = excluded.manifest_url,
feed_url = excluded.feed_url,
@@ -2161,6 +2173,9 @@ export class ProjectDb {
title_count: input.titleCount ?? 0,
images_empty_alt: input.imagesEmptyAlt ?? 0,
empty_anchor_count: input.emptyAnchorCount ?? 0,
anchor_too_long_count: input.anchorTooLongCount ?? 0,
generic_anchor_count: input.genericAnchorCount ?? 0,
target_blank_no_noopener_count: input.targetBlankNoNoopenerCount ?? 0,
apple_touch_icon: input.appleTouchIcon ?? null,
manifest_url: input.manifestUrl ?? null,
feed_url: input.feedUrl ?? null,
@@ -4234,7 +4249,7 @@ export class ProjectDb {
.get(...args) as { c: number; u: number };
const rowsDb = this.db
.prepare(
`SELECT f.url AS from_url, f.status_code AS from_status,
`SELECT f.id AS from_url_id, f.url AS from_url, f.status_code AS from_status,
l.to_url AS to_url, t.status_code AS to_status,
l.anchor, l.rel, l.is_internal
FROM links l
@@ -4245,6 +4260,7 @@ export class ProjectDb {
LIMIT ? OFFSET ?`,
)
.all(...args, params.limit, params.offset) as unknown as {
from_url_id: number;
from_url: string;
from_status: number | null;
to_url: string;
@@ -4257,6 +4273,7 @@ export class ProjectDb {
total: totalRow.c,
uniqueTargets: totalRow.u,
rows: rowsDb.map((r) => ({
fromUrlId: r.from_url_id,
fromUrl: r.from_url,
fromStatusCode: r.from_status,
toUrl: r.to_url,
@@ -4354,7 +4371,7 @@ export class ProjectDb {
const rowsDb = this.db
.prepare(
`SELECT u.url AS url, u.status_code AS status_code,
`SELECT u.id AS id, u.url AS url, u.status_code AS status_code,
pm.performance AS m_perf, pm.lcp AS m_lcp, pm.cls AS m_cls,
pm.fcp AS m_fcp, pm.tbt AS m_tbt, pm.speed_index AS m_si,
pm.tti AS m_tti, pm.max_potential_fid AS m_mpfid, pm.inp AS m_inp,
@@ -4398,6 +4415,7 @@ export class ProjectDb {
return {
total: totalRow.c,
rows: rowsDb.map((r) => ({
id: r['id'] as number,
url: r['url'] as string,
statusCode: (r['status_code'] as number | null) ?? null,
mobile: metrics(r, 'm'),
@@ -4484,7 +4502,7 @@ export class ProjectDb {
const rowsDb = this.db
.prepare(
`SELECT u.url AS url, u.status_code AS status_code,
`SELECT u.id AS id, u.url AS url, u.status_code AS status_code,
cp.lcp AS p_lcp, cp.cls AS p_cls, cp.inp AS p_inp,
cp.fcp AS p_fcp, cp.ttfb AS p_ttfb, cp.status AS p_status,
cp.error AS p_error, cp.collection_period AS p_period, cp.fetched_at AS p_fetched,
@@ -4524,6 +4542,7 @@ export class ProjectDb {
return {
total: totalRow.c,
rows: rowsDb.map((r) => ({
id: r['id'] as number,
url: r['url'] as string,
statusCode: (r['status_code'] as number | null) ?? null,
phone: metrics(r, 'p'),
@@ -4785,6 +4804,23 @@ export class ProjectDb {
return moved;
}
/**
* Account id of the most recent pull stored in a GSC / GA4 results
* table, or '' when the table is empty (or holds only pre-multi-account
* rows). For readers that can't see the desktop's linked-account list —
* the MCP server — when the project has no saved account choice: the
* desktop adopts legacy rows into its first linked account without
* saving one.
*/
latestGoogleAccountId(table: 'gsc_results' | 'ga4_results'): string {
const allowed = ['gsc_results', 'ga4_results'];
if (!allowed.includes(table)) return '';
const row = this.db
.prepare(`SELECT account_id FROM ${table} ORDER BY fetched_at DESC LIMIT 1`)
.get() as { account_id: string } | undefined;
return row?.account_id ?? '';
}
/**
* Faz 7 — candidate query for the Search Console tab. Crawled internal
* HTML 2xx pages left-joined with their stored GSC metrics, ordered
@@ -4827,7 +4863,7 @@ export class ProjectDb {
const accountId = params.accountId ?? '';
// The full column projection + row mapper is shared by both the
// crawled-pages branch and the orphan branch below.
const selectCols = `g.clicks AS clicks, g.impressions AS impressions,
const selectCols = `u.id AS id, g.clicks AS clicks, g.impressions AS impressions,
g.ctr AS ctr, g.position AS position,
g.fetched_at AS g_fetched,
i.verdict AS i_verdict,
@@ -4848,6 +4884,7 @@ export class ProjectDb {
const iFetched = r['i_fetched'];
const iStatus = r['i_status'];
return {
id: (r['id'] as number | null) ?? null,
url: r['url'] as string,
statusCode: (r['status_code'] as number | null) ?? null,
gsc:
@@ -5285,7 +5322,7 @@ export class ProjectDb {
const rowsDb = this.db
.prepare(
`SELECT u.url AS url, u.status_code AS status_code,
`SELECT u.id AS id, u.url AS url, u.status_code AS status_code,
s.provider AS provider, s.metrics AS metrics,
s.status AS status, s.error AS error,
s.fetched_at AS fetched_at
@@ -5323,6 +5360,7 @@ export class ProjectDb {
}
: null;
return {
id: r['id'] as number,
url: r['url'] as string,
statusCode: (r['status_code'] as number | null) ?? null,
seo,
@@ -5580,7 +5618,7 @@ export class ProjectDb {
const rowsDb = this.db
.prepare(
`SELECT u.url AS url, u.status_code AS status_code,
`SELECT u.id AS id, u.url AS url, u.status_code AS status_code,
a.sessions AS sessions, a.users AS users,
a.pageviews AS pageviews,
a.engagement_rate AS engagement_rate,
@@ -5601,6 +5639,7 @@ export class ProjectDb {
rows: rowsDb.map((r) => {
const fetchedAt = r['fetched_at'];
return {
id: r['id'] as number,
url: r['url'] as string,
statusCode: (r['status_code'] as number | null) ?? null,
ga4:
@@ -5743,7 +5782,7 @@ export class ProjectDb {
const rowsDb = this.db
.prepare(
`SELECT u.url AS url, u.status_code AS status_code,
`SELECT u.id AS id, u.url AS url, u.status_code AS status_code,
a.provider AS provider, a.model AS model,
a.response AS response,
a.tokens_in AS tokens_in, a.tokens_out AS tokens_out,
@@ -5778,6 +5817,7 @@ export class ProjectDb {
}
: null;
return {
id: r['id'] as number,
url: r['url'] as string,
statusCode: (r['status_code'] as number | null) ?? null,
ai,
@@ -7543,7 +7583,7 @@ export class ProjectDb {
}
const rows = this.db
.prepare(
`SELECT f.url AS from_url, f.status_code AS from_status,
`SELECT f.id AS from_url_id, f.url AS from_url, f.status_code AS from_status,
l.to_url AS to_url, t.status_code AS to_status,
l.anchor, l.rel, l.is_internal
FROM links l
@@ -7558,6 +7598,7 @@ export class ProjectDb {
// footer produces one row per page, so a large site turns one broken
// URL into hundreds of thousands of rows held in memory at once.
.iterate(...args) as unknown as Iterable<{
from_url_id: number;
from_url: string;
from_status: number | null;
to_url: string;
@@ -7568,6 +7609,7 @@ export class ProjectDb {
}>;
for (const r of rows) {
yield {
fromUrlId: r.from_url_id,
fromUrl: r.from_url,
fromStatusCode: r.from_status,
toUrl: r.to_url,
@@ -9990,6 +10032,11 @@ function urlPostNormSql(col: string): string {
)`;
}
/** A JS string as a single-quoted SQL literal. */
function sqlString(value: string): string {
return `'${value.replace(/'/g, "''")}'`;
}
/** Rows per write transaction in the yielding issue recompute. */
const ISSUE_WRITE_BATCH = 2_000;
@@ -10063,39 +10110,48 @@ export const EXPENSIVE_ISSUE_DEFINITIONS: ReadonlyArray<readonly [string, string
// aggregate ran past the 60 s reader budget on every tick.
[
'issues:anchor-text-too-long',
// The column counts every <a> on the page (the link graph keeps one
// record per target); the links sub-select keeps projects crawled
// before the column existed flagged until their next crawl.
`is_external = 0 AND content_kind = 'html'
AND id IN (
SELECT from_url_id FROM links
WHERE anchor IS NOT NULL AND LENGTH(anchor) > 100
AND (
anchor_too_long_count > 0
OR id IN (
SELECT from_url_id FROM links
WHERE anchor IS NOT NULL AND LENGTH(anchor) > 100
)
)`,
],
[
'issues:anchor-text-generic',
// Same column + legacy fallback split as anchor-text-too-long.
`is_external = 0 AND content_kind = 'html'
AND id IN (
SELECT from_url_id FROM links
WHERE anchor IS NOT NULL
AND LOWER(TRIM(anchor)) IN (
'click here', 'click', 'here', 'read more', 'more',
'learn more', 'see more', 'continue reading', 'continue',
'this link', 'link', 'go', 'buraya', 'tıkla', 'devamı',
'devamını oku', 'daha fazla'
)
AND (
generic_anchor_count > 0
OR id IN (
SELECT from_url_id FROM links
WHERE anchor IS NOT NULL
AND LOWER(TRIM(anchor)) IN (${GENERIC_ANCHOR_TEXTS.map(sqlString).join(', ')})
)
)`,
],
[
'issues:target-blank-no-noopener',
// Same column + legacy fallback split as anchor-text-too-long.
`is_external = 0 AND content_kind = 'html'
AND id IN (
SELECT from_url_id FROM links
WHERE LOWER(COALESCE(target, '')) = '_blank'
AND (
rel IS NULL
OR (
LOWER(rel) NOT LIKE '%noopener%'
AND LOWER(rel) NOT LIKE '%noreferrer%'
AND (
target_blank_no_noopener_count > 0
OR id IN (
SELECT from_url_id FROM links
WHERE LOWER(COALESCE(target, '')) = '_blank'
AND (
rel IS NULL
OR (
LOWER(rel) NOT LIKE '%noopener%'
AND LOWER(rel) NOT LIKE '%noreferrer%'
)
)
)
)
)`,
],
[
packages/shared-types/src/ai.ts
+2
View File
@@ -27,6 +27,8 @@ export interface AiResult {
/** One AI tab row — a crawled page joined with its latest AI result. */
export interface AiRow {
/** `urls.id` — what the Detail panel opens when the row is clicked. */
id: number;
url: string;
statusCode: number | null;
/** Null when this page has no result for the queried provider. */
packages/shared-types/src/crawl.ts
+2
View File
@@ -2329,6 +2329,8 @@ export interface AdvancedFilter {
}
export interface BrokenLinkRow {
/** Crawled id of the linking page — lets the Broken Links grid open it in the Detail panel. */
fromUrlId: number;
fromUrl: string;
fromStatusCode: number | null;
toUrl: string;
packages/shared-types/src/crux.ts
+2
View File
@@ -56,6 +56,8 @@ export interface CruxMetrics {
* has been fetched for the URL.
*/
export interface CruxRow {
/** `urls.id` — what the Detail panel opens when the row is clicked. */
id: number;
url: string;
statusCode: number | null;
phone: CruxMetrics | null;
packages/shared-types/src/google.ts
+5
View File
@@ -71,6 +71,9 @@ export interface GscMetrics {
/** One Search Console tab row — a crawled page joined with its GSC data. */
export interface GscRow {
/** `urls.id` the Detail panel opens on a row click; null for a GSC URL
* the crawl never reached (the orphan presets). */
id: number | null;
url: string;
statusCode: number | null;
/** Null when this page has no Search Console impressions in the range. */
@@ -249,6 +252,8 @@ export interface Ga4Metrics {
/** One Analytics tab row — a crawled page joined with its GA4 data. */
export interface Ga4Row {
/** `urls.id` — what the Detail panel opens when the row is clicked. */
id: number;
url: string;
statusCode: number | null;
/** Null when the page had no GA4 traffic in the range. */
packages/shared-types/src/issue-catalog.ts
+12
View File
@@ -385,6 +385,18 @@ export const ISSUE_CATEGORIES: readonly UrlCategory[] = ISSUE_GROUPS.flatMap((g)
g.items.map((i) => i.category),
);
/**
* Anchor texts the "Generic Anchor Text" check flags, lower-cased. One list
* for both sides: the HTML parser counts them over every `<a>` on a page,
* and the issue SQL matches them against stored link anchors.
*/
export const GENERIC_ANCHOR_TEXTS: readonly string[] = [
'click here', 'click', 'here', 'read more', 'more',
'learn more', 'see more', 'continue reading', 'continue',
'this link', 'link', 'go', 'buraya', 'tıkla', 'devamı',
'devamını oku', 'daha fazla',
];
/**
* Normalises a saved `disabledIssues` list: drops unknown ids (a check
* renamed since the project was saved) and duplicates, keeps sidebar order.
packages/shared-types/src/pagespeed.ts
+2
View File
@@ -65,6 +65,8 @@ export interface PagespeedMetrics {
* strategy has been run for the URL.
*/
export interface PagespeedRow {
/** `urls.id` — what the Detail panel opens when the row is clicked. */
id: number;
url: string;
statusCode: number | null;
mobile: PagespeedMetrics | null;
packages/shared-types/src/seo.ts
+2
View File
@@ -61,6 +61,8 @@ export interface SeoResult {
/** One SEO tab row — a crawled page joined with its provider data. */
export interface SeoRow {
/** `urls.id` — what the Detail panel opens when the row is clicked. */
id: number;
url: string;
statusCode: number | null;
/** Null when this page has no result for the currently-queried provider. */