mirror of
https://github.com/kemalai/FreeCrawl-SEO-Tool.git
synced 2026-10-18 00:39:38 +02:00
feat: in-app AI assistant + MCP servers, query-string crawl scope
This commit is contained in:
1 parent
7e16cc0e1e
commit
414e788604
33 files changed
+4151
-84
No files matched your search
File renamed without changes.
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Faz 5 — provider message shapes for the assistant, isolated from IO.
|
||||
*
|
||||
* The conversation is stored once in the provider-neutral
|
||||
* `AssistantMessage` form; each provider then wants something different,
|
||||
* and the differences are exactly where a silent bug lives: OpenAI needs a
|
||||
* `tool_call_id` on every tool turn, Anthropic rejects a `tool_use` turn
|
||||
* that is not answered by a SINGLE user message carrying every
|
||||
* `tool_result` block, and Ollama has no call ids at all so the tool name
|
||||
* is the only thing tying a result to its request.
|
||||
*
|
||||
* Kept free of `electron` and `undici` imports so the conversion can be
|
||||
* unit-tested without a running app.
|
||||
*/
|
||||
import type { AssistantMessage } from '@freecrawl/shared-types';
|
||||
|
||||
export interface OpenAiMessage {
|
||||
role: 'system' | 'user' | 'assistant' | 'tool';
|
||||
content: string;
|
||||
tool_calls?: {
|
||||
id: string;
|
||||
type: 'function';
|
||||
function: { name: string; arguments: string };
|
||||
}[];
|
||||
tool_call_id?: string;
|
||||
}
|
||||
|
||||
export function toOpenAiMessages(
|
||||
system: string,
|
||||
messages: AssistantMessage[],
|
||||
): OpenAiMessage[] {
|
||||
const out: OpenAiMessage[] = [{ role: 'system', content: system }];
|
||||
for (const m of messages) {
|
||||
if (m.role === 'tool') {
|
||||
out.push({
|
||||
role: 'tool',
|
||||
content: m.content,
|
||||
tool_call_id: m.toolCallId ?? '',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (m.role === 'assistant' && m.toolCalls?.length) {
|
||||
out.push({
|
||||
role: 'assistant',
|
||||
content: m.content,
|
||||
tool_calls: m.toolCalls.map((c) => ({
|
||||
id: c.id,
|
||||
type: 'function' as const,
|
||||
function: { name: c.name, arguments: JSON.stringify(c.arguments ?? {}) },
|
||||
})),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
out.push({ role: m.role === 'assistant' ? 'assistant' : 'user', content: m.content });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface AnthropicBlock {
|
||||
type: 'text' | 'tool_use' | 'tool_result';
|
||||
text?: string;
|
||||
id?: string;
|
||||
name?: string;
|
||||
input?: Record<string, unknown>;
|
||||
tool_use_id?: string;
|
||||
content?: string;
|
||||
is_error?: boolean;
|
||||
}
|
||||
|
||||
export interface AnthropicMessage {
|
||||
role: 'user' | 'assistant';
|
||||
content: AnthropicBlock[];
|
||||
}
|
||||
|
||||
export function toAnthropicMessages(messages: AssistantMessage[]): AnthropicMessage[] {
|
||||
const out: AnthropicMessage[] = [];
|
||||
for (const m of messages) {
|
||||
if (m.role === 'tool') {
|
||||
const block: AnthropicBlock = {
|
||||
type: 'tool_result',
|
||||
tool_use_id: m.toolCallId ?? '',
|
||||
content: m.content,
|
||||
...(m.isError ? { is_error: true } : {}),
|
||||
};
|
||||
// Consecutive tool results belong to ONE user turn — Anthropic
|
||||
// rejects a tool_use turn that isn't answered in a single message.
|
||||
const last = out[out.length - 1];
|
||||
if (last && last.role === 'user' && last.content[0]?.type === 'tool_result') {
|
||||
last.content.push(block);
|
||||
} else {
|
||||
out.push({ role: 'user', content: [block] });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (m.role === 'assistant') {
|
||||
const blocks: AnthropicBlock[] = [];
|
||||
if (m.content.trim()) blocks.push({ type: 'text', text: m.content });
|
||||
for (const c of m.toolCalls ?? []) {
|
||||
blocks.push({ type: 'tool_use', id: c.id, name: c.name, input: c.arguments ?? {} });
|
||||
}
|
||||
if (blocks.length) out.push({ role: 'assistant', content: blocks });
|
||||
continue;
|
||||
}
|
||||
out.push({ role: 'user', content: [{ type: 'text', text: m.content }] });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Ollama's `/api/chat` shape — OpenAI-like, but without call ids. */
|
||||
export function toOllamaMessages(
|
||||
system: string,
|
||||
messages: AssistantMessage[],
|
||||
): Record<string, unknown>[] {
|
||||
const out: Record<string, unknown>[] = [{ role: 'system', content: system }];
|
||||
for (const m of messages) {
|
||||
if (m.role === 'tool') {
|
||||
// No tool_call_id in Ollama's protocol — name the tool instead so the
|
||||
// model can match the result to the call it asked for.
|
||||
out.push({ role: 'tool', content: m.content, tool_name: m.toolName ?? '' });
|
||||
continue;
|
||||
}
|
||||
if (m.role === 'assistant' && m.toolCalls?.length) {
|
||||
out.push({
|
||||
role: 'assistant',
|
||||
content: m.content,
|
||||
tool_calls: m.toolCalls.map((c) => ({
|
||||
function: { name: c.name, arguments: c.arguments ?? {} },
|
||||
})),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
out.push({ role: m.role, content: m.content });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Parse an OpenAI-style stringified argument blob into an object. */
|
||||
export function parseArgs(raw: string | undefined): Record<string, unknown> {
|
||||
if (!raw) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
/**
|
||||
* Faz 5 — multi-turn chat with tool calling, for the in-app assistant.
|
||||
*
|
||||
* `ai-providers.ts` runs one prompt and returns text, which is all the AI
|
||||
* tab's per-URL batch needs. The assistant needs the other shape: a running
|
||||
* transcript, a tool catalogue, and an answer that may be "call this tool"
|
||||
* instead of prose. The three providers spell that differently —
|
||||
*
|
||||
* OpenAI `tools: [{type:'function', function:{…}}]`
|
||||
* → `message.tool_calls[].function.arguments` (a JSON string)
|
||||
* Anthropic `tools: [{name, description, input_schema}]`
|
||||
* → `content[]` blocks of `{type:'tool_use', id, name, input}`,
|
||||
* results returned as `tool_result` blocks in a user turn
|
||||
* Ollama OpenAI-shaped tools on `/api/chat`, arguments already parsed
|
||||
*
|
||||
* — so the conversation is stored once in the provider-neutral
|
||||
* `AssistantMessage` shape and converted here, at the edge.
|
||||
*/
|
||||
import type {
|
||||
AiProvider,
|
||||
AssistantMessage,
|
||||
AssistantToolCall,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { resolveCredentials } from './credentials.js';
|
||||
import {
|
||||
parseArgs,
|
||||
toAnthropicMessages,
|
||||
toOllamaMessages,
|
||||
toOpenAiMessages,
|
||||
type AnthropicBlock,
|
||||
} from './ai-chat-format.js';
|
||||
import { apiFetch } from './api-fetch.js';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 180_000;
|
||||
const MAX_TOKENS = 4096;
|
||||
|
||||
/** One tool as handed to the model. */
|
||||
export interface ChatTool {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ChatTurnResult {
|
||||
text: string;
|
||||
toolCalls: AssistantToolCall[];
|
||||
tokensIn: number | null;
|
||||
tokensOut: number | null;
|
||||
}
|
||||
|
||||
export class ChatError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'ChatError';
|
||||
}
|
||||
}
|
||||
|
||||
/** Run one model turn. Returns either prose, tool calls, or both. */
|
||||
export async function runChatTurn(
|
||||
provider: AiProvider,
|
||||
model: string,
|
||||
system: string,
|
||||
messages: AssistantMessage[],
|
||||
tools: ChatTool[],
|
||||
signal: AbortSignal,
|
||||
): Promise<ChatTurnResult> {
|
||||
switch (provider) {
|
||||
case 'openai':
|
||||
return runOpenAiChat(model, system, messages, tools, signal);
|
||||
case 'anthropic':
|
||||
return runAnthropicChat(model, system, messages, tools, signal);
|
||||
case 'ollama':
|
||||
return runOllamaChat(model, system, messages, tools, signal);
|
||||
}
|
||||
}
|
||||
|
||||
// ── OpenAI ───────────────────────────────────────────────────────────
|
||||
|
||||
async function runOpenAiChat(
|
||||
model: string,
|
||||
system: string,
|
||||
messages: AssistantMessage[],
|
||||
tools: ChatTool[],
|
||||
signal: AbortSignal,
|
||||
): Promise<ChatTurnResult> {
|
||||
const apiKey = (resolveCredentials('openai')['apiKey'] ?? '').trim();
|
||||
if (!apiKey) {
|
||||
throw new ChatError('No OpenAI API key — paste yours in Settings → Integrations.');
|
||||
}
|
||||
const body: Record<string, unknown> = {
|
||||
model,
|
||||
messages: toOpenAiMessages(system, messages),
|
||||
max_tokens: MAX_TOKENS,
|
||||
};
|
||||
if (tools.length) {
|
||||
body['tools'] = tools.map((t) => ({
|
||||
type: 'function',
|
||||
function: { name: t.name, description: t.description, parameters: t.parameters },
|
||||
}));
|
||||
body['tool_choice'] = 'auto';
|
||||
}
|
||||
const res = await apiFetch('https://api.openai.com/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
|
||||
body: JSON.stringify(body),
|
||||
signal: mergeSignals(signal),
|
||||
});
|
||||
const json = (await res.json().catch(() => null)) as Record<string, unknown> | null;
|
||||
if (!res.ok || !json) {
|
||||
const err = (json?.['error'] as { message?: string } | undefined)?.message;
|
||||
throw new ChatError(err ?? `OpenAI API error (HTTP ${res.status})`);
|
||||
}
|
||||
const choice = (json['choices'] as
|
||||
| {
|
||||
message?: {
|
||||
content?: string | null;
|
||||
tool_calls?: {
|
||||
id?: string;
|
||||
function?: { name?: string; arguments?: string };
|
||||
}[];
|
||||
};
|
||||
}[]
|
||||
| undefined)?.[0];
|
||||
const calls: AssistantToolCall[] = (choice?.message?.tool_calls ?? []).map((c, i) => ({
|
||||
id: c.id ?? `call_${i}`,
|
||||
name: c.function?.name ?? '',
|
||||
arguments: parseArgs(c.function?.arguments),
|
||||
}));
|
||||
const usage = json['usage'] as
|
||||
| { prompt_tokens?: number; completion_tokens?: number }
|
||||
| undefined;
|
||||
return {
|
||||
text: (choice?.message?.content ?? '').trim(),
|
||||
toolCalls: calls.filter((c) => c.name),
|
||||
tokensIn: usage?.prompt_tokens ?? null,
|
||||
tokensOut: usage?.completion_tokens ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Anthropic ────────────────────────────────────────────────────────
|
||||
|
||||
async function runAnthropicChat(
|
||||
model: string,
|
||||
system: string,
|
||||
messages: AssistantMessage[],
|
||||
tools: ChatTool[],
|
||||
signal: AbortSignal,
|
||||
): Promise<ChatTurnResult> {
|
||||
const apiKey = (resolveCredentials('anthropic')['apiKey'] ?? '').trim();
|
||||
if (!apiKey) {
|
||||
throw new ChatError('No Anthropic API key — paste yours in Settings → Integrations.');
|
||||
}
|
||||
const body: Record<string, unknown> = {
|
||||
model,
|
||||
max_tokens: MAX_TOKENS,
|
||||
system,
|
||||
messages: toAnthropicMessages(messages),
|
||||
};
|
||||
if (tools.length) {
|
||||
body['tools'] = tools.map((t) => ({
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
input_schema: t.parameters,
|
||||
}));
|
||||
}
|
||||
const res = await apiFetch('https://api.anthropic.com/v1/messages', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': apiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: mergeSignals(signal),
|
||||
});
|
||||
const json = (await res.json().catch(() => null)) as Record<string, unknown> | null;
|
||||
if (!res.ok || !json) {
|
||||
const err = (json?.['error'] as { message?: string } | undefined)?.message;
|
||||
throw new ChatError(err ?? `Anthropic API error (HTTP ${res.status})`);
|
||||
}
|
||||
const content = (json['content'] as AnthropicBlock[] | undefined) ?? [];
|
||||
const text = content
|
||||
.filter((b) => b.type === 'text' && typeof b.text === 'string')
|
||||
.map((b) => b.text as string)
|
||||
.join('\n')
|
||||
.trim();
|
||||
const calls: AssistantToolCall[] = content
|
||||
.filter((b) => b.type === 'tool_use' && b.name)
|
||||
.map((b, i) => ({
|
||||
id: b.id ?? `call_${i}`,
|
||||
name: b.name as string,
|
||||
arguments: (b.input ?? {}) as Record<string, unknown>,
|
||||
}));
|
||||
const usage = json['usage'] as
|
||||
| { input_tokens?: number; output_tokens?: number }
|
||||
| undefined;
|
||||
return {
|
||||
text,
|
||||
toolCalls: calls,
|
||||
tokensIn: usage?.input_tokens ?? null,
|
||||
tokensOut: usage?.output_tokens ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Ollama ───────────────────────────────────────────────────────────
|
||||
|
||||
async function runOllamaChat(
|
||||
model: string,
|
||||
system: string,
|
||||
messages: AssistantMessage[],
|
||||
tools: ChatTool[],
|
||||
signal: AbortSignal,
|
||||
): Promise<ChatTurnResult> {
|
||||
const creds = resolveCredentials('ollama');
|
||||
const endpoint =
|
||||
(creds['endpoint'] ?? 'http://localhost:11434').trim() || 'http://localhost:11434';
|
||||
const body: Record<string, unknown> = {
|
||||
model,
|
||||
messages: toOllamaMessages(system, messages),
|
||||
stream: false,
|
||||
};
|
||||
if (tools.length) {
|
||||
body['tools'] = tools.map((t) => ({
|
||||
type: 'function',
|
||||
function: { name: t.name, description: t.description, parameters: t.parameters },
|
||||
}));
|
||||
}
|
||||
let res: Awaited<ReturnType<typeof apiFetch>>;
|
||||
try {
|
||||
res = await apiFetch(`${endpoint.replace(/\/+$/, '')}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
signal: mergeSignals(signal),
|
||||
});
|
||||
} catch (err) {
|
||||
throw new ChatError(
|
||||
`Couldn't reach Ollama at ${endpoint}: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
const json = (await res.json().catch(() => null)) as Record<string, unknown> | null;
|
||||
if (!res.ok || !json) {
|
||||
throw new ChatError(
|
||||
(json?.['error'] as string | undefined) ?? `Ollama returned HTTP ${res.status}`,
|
||||
);
|
||||
}
|
||||
const message = json['message'] as
|
||||
| {
|
||||
content?: string;
|
||||
tool_calls?: { function?: { name?: string; arguments?: unknown } }[];
|
||||
}
|
||||
| undefined;
|
||||
const calls: AssistantToolCall[] = (message?.tool_calls ?? []).map((c, i) => ({
|
||||
id: `call_${Date.now()}_${i}`,
|
||||
name: c.function?.name ?? '',
|
||||
arguments:
|
||||
typeof c.function?.arguments === 'string'
|
||||
? parseArgs(c.function.arguments)
|
||||
: ((c.function?.arguments ?? {}) as Record<string, unknown>),
|
||||
}));
|
||||
return {
|
||||
text: (message?.content ?? '').trim(),
|
||||
toolCalls: calls.filter((c) => c.name),
|
||||
tokensIn:
|
||||
typeof json['prompt_eval_count'] === 'number'
|
||||
? (json['prompt_eval_count'] as number)
|
||||
: null,
|
||||
tokensOut:
|
||||
typeof json['eval_count'] === 'number' ? (json['eval_count'] as number) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
/** Cancel on either the caller's signal or our own request ceiling. */
|
||||
function mergeSignals(signal: AbortSignal): AbortSignal {
|
||||
return AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]);
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
/**
|
||||
* Faz 5 — in-app AI assistant: the agent loop behind the Assistant tab.
|
||||
*
|
||||
* One "send" from the renderer is one turn of:
|
||||
*
|
||||
* model → (tool calls?) → run them → feed results back → model → …
|
||||
*
|
||||
* until the model answers in prose, the user cancels, or the round budget
|
||||
* runs out. Two families of tools are on offer:
|
||||
*
|
||||
* - **built-ins** — read-only queries against the open project (the same
|
||||
* primitives `apps/mcp-server` exposes). They touch nothing outside the
|
||||
* already-open `.seoproject`, so they run without a prompt.
|
||||
* - **MCP tools** — anything the user's registered MCP servers advertise.
|
||||
* These reach the network, the filesystem, or a third-party account, so
|
||||
* every call is gated: allow once, always allow (remembered per tool),
|
||||
* or deny.
|
||||
*
|
||||
* The transcript itself lives in the renderer. The main process owns only
|
||||
* the loop, so a tab switch can't strand a half-finished tool round.
|
||||
*/
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { ProjectDb } from '@freecrawl/db';
|
||||
import type {
|
||||
AiProvider,
|
||||
AssistantEvent,
|
||||
AssistantMessage,
|
||||
AssistantSendInput,
|
||||
AssistantSendResult,
|
||||
AssistantStopReason,
|
||||
AssistantToolCall,
|
||||
McpToolInfo,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { runChatTurn, ChatError, type ChatTool } from './ai-chat.js';
|
||||
import { resolveModel } from './ai-providers.js';
|
||||
import {
|
||||
BUILTIN_SERVER_ID,
|
||||
BUILTIN_TOOL_PREFIX,
|
||||
MCP_TOOL_PREFIX,
|
||||
callTool as callMcpTool,
|
||||
labelFor,
|
||||
policyFor,
|
||||
setPolicy,
|
||||
toolsFor,
|
||||
} from './mcp-client.js';
|
||||
import * as logger from './logger.js';
|
||||
|
||||
/** Tool rounds per send. Enough for "find the issue → drill in → report",
|
||||
* low enough that a looping model can't burn the user's token budget. */
|
||||
const MAX_ROUNDS = 8;
|
||||
/** An unanswered approval prompt expires rather than pinning the turn. */
|
||||
const APPROVAL_TIMEOUT_MS = 10 * 60_000;
|
||||
/** Built-in tool results are JSON — cap them like MCP results are capped. */
|
||||
const MAX_BUILTIN_RESULT_CHARS = 24_000;
|
||||
|
||||
const SYSTEM_PROMPT = [
|
||||
'You are the FreeCrawl SEO assistant, embedded in a desktop SEO crawler.',
|
||||
'The user has an open project containing the results of a site crawl.',
|
||||
'Use the builtin__* tools to read that crawl before answering questions',
|
||||
'about the site — never guess numbers, look them up. Prefer',
|
||||
'builtin__get_summary and builtin__top_issues for orientation, then',
|
||||
'builtin__query_urls with a category filter to list affected pages, then',
|
||||
'builtin__get_url_detail for one page.',
|
||||
'Any mcp__* tools come from servers the user connected themselves; use them',
|
||||
'when the question needs information outside the crawl.',
|
||||
'Answer concisely, in the language the user writes in, and cite concrete',
|
||||
'URLs and counts from the tools rather than generic SEO advice.',
|
||||
].join(' ');
|
||||
|
||||
// ── Approval plumbing ────────────────────────────────────────────────
|
||||
|
||||
type Decision = 'allow-once' | 'allow-always' | 'deny';
|
||||
|
||||
interface PendingApproval {
|
||||
resolve: (decision: Decision) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
const pendingApprovals = new Map<string, PendingApproval>();
|
||||
|
||||
/** Called from the IPC handler when the user answers an approval card. */
|
||||
export function resolveApproval(requestId: string, decision: Decision): boolean {
|
||||
const pending = pendingApprovals.get(requestId);
|
||||
if (!pending) return false;
|
||||
pendingApprovals.delete(requestId);
|
||||
clearTimeout(pending.timer);
|
||||
pending.resolve(decision);
|
||||
return true;
|
||||
}
|
||||
|
||||
// ── Cancellation ─────────────────────────────────────────────────────
|
||||
|
||||
const activeRuns = new Map<number, AbortController>();
|
||||
|
||||
/** Abort the in-flight turn owned by one window. */
|
||||
export function cancelAssistant(runKey: number): void {
|
||||
activeRuns.get(runKey)?.abort();
|
||||
}
|
||||
|
||||
// ── Built-in project tools ───────────────────────────────────────────
|
||||
|
||||
interface BuiltinTool {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: Record<string, unknown>;
|
||||
run: (args: Record<string, unknown>, db: ProjectDb) => unknown;
|
||||
}
|
||||
|
||||
/** Columns returned by `query_urls` when the model doesn't ask for a set —
|
||||
* wide enough to reason about, narrow enough not to flood the context. */
|
||||
const COMMON_URL_FIELDS = [
|
||||
'id',
|
||||
'url',
|
||||
'statusCode',
|
||||
'indexability',
|
||||
'indexabilityStatus',
|
||||
'title',
|
||||
'metaDescription',
|
||||
'h1',
|
||||
'canonical',
|
||||
'wordCount',
|
||||
'depth',
|
||||
'inlinks',
|
||||
'outlinks',
|
||||
];
|
||||
|
||||
const BUILTIN_TOOLS: BuiltinTool[] = [
|
||||
{
|
||||
name: 'get_summary',
|
||||
description:
|
||||
'Headline numbers for the open crawl: URLs crawled, indexable vs non-indexable, status-code distribution, average response time, start/end time. Call this first when asked about the site overall.',
|
||||
parameters: { type: 'object', properties: {} },
|
||||
run: (_args, db) => db.getSummary(),
|
||||
},
|
||||
{
|
||||
name: 'top_issues',
|
||||
description:
|
||||
'Ranked list of non-empty SEO issue categories in the open crawl, biggest first. Use it to answer "what is wrong with this site?" before drilling into a category.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
limit: { type: 'integer', minimum: 1, maximum: 100, description: 'Max categories (default 20).' },
|
||||
},
|
||||
},
|
||||
run: (args, db) => {
|
||||
const limit = clamp(args['limit'], 1, 100, 20);
|
||||
const counts = db.getOverviewCounts() as unknown as Record<string, unknown>;
|
||||
const issues = (counts['issues'] as Record<string, number>) ?? {};
|
||||
const ranked = Object.entries(issues)
|
||||
.filter(([, n]) => n > 0)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, limit)
|
||||
.map(([category, count]) => ({ category: `issues:${category}`, count }));
|
||||
return { issues: ranked };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'get_overview_counts',
|
||||
description:
|
||||
'Every category and issue filter the sidebar shows, with row counts. Useful to discover the exact category string to pass to query_urls.',
|
||||
parameters: { type: 'object', properties: {} },
|
||||
run: (_args, db) => db.getOverviewCounts(),
|
||||
},
|
||||
{
|
||||
name: 'query_urls',
|
||||
description:
|
||||
'Paginated query over the crawled URLs, using the same category filters as the UI (e.g. "all", "internal:html", "status:4xx", "issues:title-missing"). Returns rows plus a total count.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
category: { type: 'string', description: 'Category filter. Default "all".' },
|
||||
search: { type: 'string', description: 'Substring match on URL and title.' },
|
||||
limit: { type: 'integer', minimum: 1, maximum: 200, description: 'Rows to return (default 25).' },
|
||||
offset: { type: 'integer', minimum: 0, description: 'Row offset for paging.' },
|
||||
sortBy: { type: 'string', description: 'Column to sort by, e.g. "depth" or "inlinks".' },
|
||||
sortDir: { type: 'string', enum: ['asc', 'desc'] },
|
||||
},
|
||||
},
|
||||
run: (args, db) => {
|
||||
const limit = clamp(args['limit'], 1, 200, 25);
|
||||
const offset = clamp(args['offset'], 0, 1_000_000, 0);
|
||||
const { rows, total } = db.queryUrls({
|
||||
category: (typeof args['category'] === 'string'
|
||||
? args['category']
|
||||
: 'all') as Parameters<ProjectDb['queryUrls']>[0]['category'],
|
||||
search: typeof args['search'] === 'string' ? args['search'] : undefined,
|
||||
limit,
|
||||
offset,
|
||||
sortBy: typeof args['sortBy'] === 'string' ? args['sortBy'] : undefined,
|
||||
sortDir: args['sortDir'] === 'desc' ? 'desc' : 'asc',
|
||||
});
|
||||
const projected = rows.map((r) => {
|
||||
const row = r as unknown as Record<string, unknown>;
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const key of COMMON_URL_FIELDS) out[key] = row[key];
|
||||
return out;
|
||||
});
|
||||
return { rows: projected, total, limit, offset };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'get_url_detail',
|
||||
description:
|
||||
'Everything stored about one crawled URL — all columns plus inlinks, outlinks, images and HTTP headers. Pass `id` from query_urls, or the exact `url`.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: { type: 'integer', description: 'URL id from query_urls (preferred).' },
|
||||
url: { type: 'string', description: 'Exact URL string.' },
|
||||
linkLimit: { type: 'integer', minimum: 1, maximum: 500, description: 'Cap on links returned (default 50).' },
|
||||
},
|
||||
},
|
||||
run: (args, db) => {
|
||||
let id: number | null = null;
|
||||
if (typeof args['id'] === 'number') id = args['id'];
|
||||
else if (typeof args['url'] === 'string') id = db.getUrlIdByUrl(args['url']);
|
||||
if (id === null) {
|
||||
throw new Error('get_url_detail needs an `id` from query_urls, or a `url` that exists in this crawl.');
|
||||
}
|
||||
const detail = db.getUrlDetail(id, clamp(args['linkLimit'], 1, 500, 50));
|
||||
if (!detail) throw new Error(`URL id ${id} is not in this project.`);
|
||||
return detail;
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
function clamp(raw: unknown, min: number, max: number, fallback: number): number {
|
||||
const v = typeof raw === 'number' && Number.isFinite(raw) ? raw : fallback;
|
||||
return Math.max(min, Math.min(max, Math.floor(v)));
|
||||
}
|
||||
|
||||
/** Tool catalogue the renderer's picker shows, without connecting. */
|
||||
export function builtinToolInfos(): McpToolInfo[] {
|
||||
return BUILTIN_TOOLS.map((t) => ({
|
||||
serverId: BUILTIN_SERVER_ID,
|
||||
name: t.name,
|
||||
qualifiedName: `${BUILTIN_TOOL_PREFIX}${t.name}`,
|
||||
description: t.description,
|
||||
inputSchema: t.parameters,
|
||||
}));
|
||||
}
|
||||
|
||||
// ── The turn ─────────────────────────────────────────────────────────
|
||||
|
||||
export interface AssistantRunContext {
|
||||
/** Resolver for the calling window's project DB. */
|
||||
db: () => ProjectDb;
|
||||
/** Push a live event to the calling window. */
|
||||
emit: (event: AssistantEvent) => void;
|
||||
/** Identifies the window, so Cancel hits the right run. */
|
||||
runKey: number;
|
||||
}
|
||||
|
||||
export async function runAssistantTurn(
|
||||
input: AssistantSendInput,
|
||||
ctx: AssistantRunContext,
|
||||
): Promise<AssistantSendResult> {
|
||||
const provider: AiProvider = input.provider;
|
||||
const model = resolveModel(provider, input.model);
|
||||
const includeBuiltins = input.includeBuiltins !== false;
|
||||
|
||||
// One controller per window: a second send while one is running replaces
|
||||
// the first rather than racing it.
|
||||
activeRuns.get(ctx.runKey)?.abort();
|
||||
const controller = new AbortController();
|
||||
activeRuns.set(ctx.runKey, controller);
|
||||
|
||||
/** Approval ids this turn created — a cancel must only deny its own. */
|
||||
const ownApprovals = new Set<string>();
|
||||
const transcript: AssistantMessage[] = [...input.messages];
|
||||
const produced: AssistantMessage[] = [];
|
||||
let tokensIn = 0;
|
||||
let tokensOut = 0;
|
||||
let stopReason: AssistantStopReason = 'done';
|
||||
let error: string | null = null;
|
||||
|
||||
try {
|
||||
const { tools: mcpTools, errors } = await toolsFor(input.serverIds ?? []);
|
||||
for (const e of errors) {
|
||||
ctx.emit({
|
||||
type: 'error',
|
||||
message: `${labelFor(e.id)}: ${e.message}`,
|
||||
});
|
||||
}
|
||||
const catalogue: ChatTool[] = [
|
||||
...(includeBuiltins
|
||||
? BUILTIN_TOOLS.map((t) => ({
|
||||
name: `${BUILTIN_TOOL_PREFIX}${t.name}`,
|
||||
description: t.description,
|
||||
parameters: t.parameters,
|
||||
}))
|
||||
: []),
|
||||
...mcpTools.map((t) => ({
|
||||
name: t.qualifiedName,
|
||||
description: t.description || `${t.name} (from ${labelFor(t.serverId)})`,
|
||||
parameters: t.inputSchema,
|
||||
})),
|
||||
];
|
||||
|
||||
for (let round = 0; round < MAX_ROUNDS; round++) {
|
||||
if (controller.signal.aborted) {
|
||||
stopReason = 'cancelled';
|
||||
break;
|
||||
}
|
||||
ctx.emit({ type: 'thinking', round: round + 1 });
|
||||
const turn = await runChatTurn(
|
||||
provider,
|
||||
model,
|
||||
SYSTEM_PROMPT,
|
||||
transcript,
|
||||
catalogue,
|
||||
controller.signal,
|
||||
);
|
||||
tokensIn += turn.tokensIn ?? 0;
|
||||
tokensOut += turn.tokensOut ?? 0;
|
||||
|
||||
const assistantMessage: AssistantMessage = {
|
||||
role: 'assistant',
|
||||
content: turn.text,
|
||||
...(turn.toolCalls.length ? { toolCalls: turn.toolCalls } : {}),
|
||||
};
|
||||
transcript.push(assistantMessage);
|
||||
produced.push(assistantMessage);
|
||||
ctx.emit({
|
||||
type: 'assistant-text',
|
||||
text: turn.text,
|
||||
toolCalls: turn.toolCalls,
|
||||
});
|
||||
|
||||
if (!turn.toolCalls.length) {
|
||||
stopReason = 'done';
|
||||
break;
|
||||
}
|
||||
|
||||
for (const call of turn.toolCalls) {
|
||||
if (controller.signal.aborted) {
|
||||
stopReason = 'cancelled';
|
||||
break;
|
||||
}
|
||||
const result = await executeCall(
|
||||
call,
|
||||
ctx,
|
||||
mcpTools,
|
||||
includeBuiltins,
|
||||
ownApprovals,
|
||||
);
|
||||
const toolMessage: AssistantMessage = {
|
||||
role: 'tool',
|
||||
content: result.text,
|
||||
toolCallId: call.id,
|
||||
toolName: call.name,
|
||||
...(result.isError ? { isError: true } : {}),
|
||||
durationMs: result.durationMs,
|
||||
};
|
||||
transcript.push(toolMessage);
|
||||
produced.push(toolMessage);
|
||||
ctx.emit({
|
||||
type: 'tool-result',
|
||||
callId: call.id,
|
||||
name: call.name,
|
||||
result: result.text,
|
||||
isError: result.isError,
|
||||
durationMs: result.durationMs,
|
||||
});
|
||||
}
|
||||
if (stopReason === 'cancelled') break;
|
||||
if (round === MAX_ROUNDS - 1) stopReason = 'max-rounds';
|
||||
}
|
||||
} catch (err) {
|
||||
if (controller.signal.aborted) {
|
||||
stopReason = 'cancelled';
|
||||
} else {
|
||||
stopReason = 'error';
|
||||
error =
|
||||
err instanceof ChatError || err instanceof Error
|
||||
? err.message
|
||||
: String(err);
|
||||
logger.log('warn', 'assistant', `Assistant turn failed: ${error}`);
|
||||
ctx.emit({ type: 'error', message: error });
|
||||
}
|
||||
} finally {
|
||||
if (activeRuns.get(ctx.runKey) === controller) activeRuns.delete(ctx.runKey);
|
||||
// A cancelled turn must not leave its own approval card waiting for
|
||||
// the ten-minute timeout — but another window's turn may be running,
|
||||
// so only this run's requests are answered.
|
||||
if (controller.signal.aborted) {
|
||||
for (const id of ownApprovals) {
|
||||
const pending = pendingApprovals.get(id);
|
||||
if (!pending) continue;
|
||||
clearTimeout(pending.timer);
|
||||
pending.resolve('deny');
|
||||
pendingApprovals.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
messages: produced,
|
||||
stopReason,
|
||||
error,
|
||||
tokensIn: tokensIn || null,
|
||||
tokensOut: tokensOut || null,
|
||||
model,
|
||||
};
|
||||
}
|
||||
|
||||
interface ToolOutcome {
|
||||
text: string;
|
||||
isError: boolean;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
async function executeCall(
|
||||
call: AssistantToolCall,
|
||||
ctx: AssistantRunContext,
|
||||
mcpTools: McpToolInfo[],
|
||||
includeBuiltins: boolean,
|
||||
ownApprovals: Set<string>,
|
||||
): Promise<ToolOutcome> {
|
||||
const started = Date.now();
|
||||
const fail = (message: string): ToolOutcome => ({
|
||||
text: `Error: ${message}`,
|
||||
isError: true,
|
||||
durationMs: Date.now() - started,
|
||||
});
|
||||
|
||||
// Built-in: read-only against the already-open project, so no prompt.
|
||||
if (call.name.startsWith(BUILTIN_TOOL_PREFIX)) {
|
||||
if (!includeBuiltins) return fail('FreeCrawl project tools are disabled for this chat.');
|
||||
const tool = BUILTIN_TOOLS.find(
|
||||
(t) => `${BUILTIN_TOOL_PREFIX}${t.name}` === call.name,
|
||||
);
|
||||
if (!tool) return fail(`Unknown FreeCrawl tool "${call.name}".`);
|
||||
ctx.emit({ type: 'tool-start', call });
|
||||
try {
|
||||
const value = tool.run(call.arguments ?? {}, ctx.db());
|
||||
const json = JSON.stringify(value ?? null);
|
||||
const text =
|
||||
json.length > MAX_BUILTIN_RESULT_CHARS
|
||||
? `${json.slice(0, MAX_BUILTIN_RESULT_CHARS)}\n… [truncated ${json.length - MAX_BUILTIN_RESULT_CHARS} characters — narrow the query with limit/offset]`
|
||||
: json;
|
||||
return { text, isError: false, durationMs: Date.now() - started };
|
||||
} catch (err) {
|
||||
return fail((err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// MCP: find which server owns this qualified name.
|
||||
const owner = mcpTools.find((t) => t.qualifiedName === call.name);
|
||||
if (!owner) {
|
||||
return fail(
|
||||
`Tool "${call.name}" is not available — its server may be disconnected.`,
|
||||
);
|
||||
}
|
||||
|
||||
const decision = await approveIfNeeded(call, owner, ctx, ownApprovals);
|
||||
if (decision === 'deny') {
|
||||
return {
|
||||
text: 'The user denied this tool call. Do not retry it; continue with what you have or ask them what to do instead.',
|
||||
isError: true,
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
}
|
||||
|
||||
ctx.emit({ type: 'tool-start', call });
|
||||
try {
|
||||
const res = await callMcpTool(owner.serverId, owner.name, call.arguments ?? {});
|
||||
return {
|
||||
text: res.text,
|
||||
isError: res.isError,
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
} catch (err) {
|
||||
return fail((err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
async function approveIfNeeded(
|
||||
call: AssistantToolCall,
|
||||
owner: McpToolInfo,
|
||||
ctx: AssistantRunContext,
|
||||
ownApprovals: Set<string>,
|
||||
): Promise<Decision> {
|
||||
const policy = policyFor(owner.serverId, owner.name);
|
||||
if (policy === 'always') return 'allow-once';
|
||||
if (policy === 'never') return 'deny';
|
||||
|
||||
const requestId = randomUUID();
|
||||
ownApprovals.add(requestId);
|
||||
const decision = await new Promise<Decision>((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
pendingApprovals.delete(requestId);
|
||||
resolve('deny');
|
||||
}, APPROVAL_TIMEOUT_MS);
|
||||
pendingApprovals.set(requestId, { resolve, timer });
|
||||
ctx.emit({
|
||||
type: 'tool-approval',
|
||||
requestId,
|
||||
call,
|
||||
serverId: owner.serverId,
|
||||
serverLabel: labelFor(owner.serverId),
|
||||
});
|
||||
});
|
||||
|
||||
ownApprovals.delete(requestId);
|
||||
if (decision === 'allow-always') setPolicy(owner.serverId, owner.name, 'always');
|
||||
return decision;
|
||||
}
|
||||
|
||||
/** Exported for the tool-picker UI: is this a namespaced MCP tool? */
|
||||
export function isMcpToolName(name: string): boolean {
|
||||
return name.startsWith(MCP_TOOL_PREFIX);
|
||||
}
|
||||
+100
-19
@@ -47,6 +47,14 @@ import {
|
||||
DEFAULT_CRAWL_CONFIG,
|
||||
IPC,
|
||||
type MenuEvent,
|
||||
type AssistantApprovalInput,
|
||||
type AssistantEvent,
|
||||
type AssistantSendInput,
|
||||
type AssistantSendResult,
|
||||
type McpServerConfig,
|
||||
type McpServerStatus,
|
||||
type McpToolPolicy,
|
||||
type McpToolsListResult,
|
||||
type ConfirmClearResult,
|
||||
type CrawlConfig,
|
||||
type CrawlProgress,
|
||||
@@ -316,6 +324,13 @@ import { listProperties as ga4ListProperties, runReport as ga4RunReport } from '
|
||||
import { runAiBatch, type AiBatchItem } from './ai-batch.js';
|
||||
import { resolveModel, defaultConcurrency } from './ai-providers.js';
|
||||
import { runSeoBatch } from './seo-batch.js';
|
||||
import * as mcpClient from './mcp-client.js';
|
||||
import {
|
||||
builtinToolInfos,
|
||||
cancelAssistant,
|
||||
resolveApproval,
|
||||
runAssistantTurn,
|
||||
} from './assistant.js';
|
||||
import * as logger from './logger.js';
|
||||
import {
|
||||
dbReaderPool,
|
||||
@@ -6375,6 +6390,86 @@ function registerIpc(): void {
|
||||
async (_e, input: AiRunInput): Promise<AiRunResult> => runAiJob(input),
|
||||
);
|
||||
|
||||
// ── Faz 5 — MCP client + in-app assistant ──────────────────────────
|
||||
//
|
||||
// The registry lives in mcp-client.ts (its own 0600 file under userData,
|
||||
// secrets encrypted with the same safeStorage helpers as the credential
|
||||
// store). These handlers are thin: everything stateful — connections,
|
||||
// approvals, the tool loop — belongs to the two modules, so a second
|
||||
// project window drives the same server processes instead of spawning
|
||||
// its own copy of every MCP server.
|
||||
|
||||
registerHandle(IPC.mcpServersList, (): McpServerConfig[] => mcpClient.listServers());
|
||||
|
||||
registerHandle(
|
||||
IPC.mcpServersSave,
|
||||
(_e, servers: McpServerConfig[]): McpServerConfig[] =>
|
||||
mcpClient.saveServers(Array.isArray(servers) ? servers : []),
|
||||
);
|
||||
|
||||
registerHandle(
|
||||
IPC.mcpServerConnect,
|
||||
async (_e, id: string): Promise<McpServerStatus> => {
|
||||
try {
|
||||
return await mcpClient.connect(id);
|
||||
} catch {
|
||||
// The failure is already recorded on the status row; surfacing it
|
||||
// as a rejected IPC call would make every caller write the same
|
||||
// try/catch to read an error it can just render.
|
||||
return mcpClient.statusOf(id);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
registerHandle(IPC.mcpServerDisconnect, (_e, id: string): McpServerStatus => {
|
||||
mcpClient.disconnect(id);
|
||||
return mcpClient.statusOf(id);
|
||||
});
|
||||
|
||||
registerHandle(IPC.mcpServerStatuses, (): McpServerStatus[] => mcpClient.statuses());
|
||||
|
||||
registerHandle(
|
||||
IPC.mcpToolsList,
|
||||
async (_e, serverIds: string[]): Promise<McpToolsListResult> => {
|
||||
const { tools, errors } = await mcpClient.toolsFor(
|
||||
Array.isArray(serverIds) ? serverIds : [],
|
||||
);
|
||||
return { tools: [...builtinToolInfos(), ...tools], errors };
|
||||
},
|
||||
);
|
||||
|
||||
registerHandle(
|
||||
IPC.mcpPolicySet,
|
||||
(_e, serverId: string, tool: string, policy: McpToolPolicy): void => {
|
||||
mcpClient.setPolicy(serverId, tool, policy);
|
||||
},
|
||||
);
|
||||
|
||||
registerHandle(IPC.mcpPolicyList, (): Record<string, McpToolPolicy> =>
|
||||
mcpClient.listPolicies(),
|
||||
);
|
||||
|
||||
registerHandle(
|
||||
IPC.assistantSend,
|
||||
async (e, input: AssistantSendInput): Promise<AssistantSendResult> => {
|
||||
const runSession = currentSession();
|
||||
const runKey = e.sender.id;
|
||||
return runAssistantTurn(input, {
|
||||
db: () => runSession.getDb(),
|
||||
emit: (event: AssistantEvent) => runSession.send(IPC.assistantEvent, event),
|
||||
runKey,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
registerHandle(IPC.assistantCancel, (e): void => {
|
||||
cancelAssistant(e.sender.id);
|
||||
});
|
||||
|
||||
registerHandle(IPC.assistantApprove, (_e, input: AssistantApprovalInput): void => {
|
||||
resolveApproval(input.requestId, input.decision);
|
||||
});
|
||||
|
||||
registerHandle(IPC.aiCancel, () => {
|
||||
const run = currentSession().fetchRuns.ai;
|
||||
if (run.active) {
|
||||
@@ -7123,6 +7218,7 @@ function registerIpc(): void {
|
||||
screenshotMode: config.jsRender.screenshotMode,
|
||||
detectLcp: config.jsRender.lcpCandidate,
|
||||
detectA11y: config.jsRender.a11yAudit,
|
||||
discoverSpaRoutes: config.jsRender.spaRouting,
|
||||
screenshotPaths,
|
||||
},
|
||||
signal,
|
||||
@@ -7201,6 +7297,7 @@ function registerIpc(): void {
|
||||
lcp: res.lcp ?? null,
|
||||
mobileUsability,
|
||||
a11y: res.a11y ?? null,
|
||||
spaRoutes: res.spaRoutes,
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -8792,25 +8889,6 @@ function registerIpc(): void {
|
||||
};
|
||||
});
|
||||
|
||||
// `setImmediate`-yield wrapper for read-heavy IPC handlers. Yielding
|
||||
// before running the SQL gives the Node event loop one tick to drain
|
||||
// pending IPC and crawler callbacks first — without this, two
|
||||
// simultaneous renderer queries (e.g. table chunk + sidebar refresh)
|
||||
// run back-to-back and freeze input for the duration of both. With
|
||||
// this wrapper they interleave with the crawler's per-URL DB writes
|
||||
// and any other queued IPC, keeping click → response under one frame
|
||||
// even on a saturated main thread. Cost per call: < 1 ms.
|
||||
const yieldThenRun = <T>(fn: () => T): Promise<T> =>
|
||||
new Promise<T>((resolve, reject) => {
|
||||
setImmediate(() => {
|
||||
try {
|
||||
resolve(fn());
|
||||
} catch (err) {
|
||||
reject(err as Error);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
registerHandle(IPC.urlsQuery, async (_e, input: UrlsQueryInput): Promise<UrlsQueryResult> => {
|
||||
const args: Parameters<typeof ProjectDb.prototype.queryUrls> = [
|
||||
{
|
||||
@@ -10248,6 +10326,9 @@ function performShutdown(): void {
|
||||
// Stop an in-flight browser download — otherwise the detached child
|
||||
// keeps writing into the cache after the app window is gone.
|
||||
abortChromiumInstall();
|
||||
// Kill every MCP server child process — they are ours, and an orphaned
|
||||
// stdio server would keep running after the app window is gone.
|
||||
mcpClient.disconnectAll();
|
||||
// Drop the tray so its icon doesn't linger after quit.
|
||||
if (tray) {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,681 @@
|
||||
/**
|
||||
* Faz 5 — MCP client: FreeCrawl connecting *out* to external MCP servers.
|
||||
*
|
||||
* The counterpart to `apps/mcp-server` (FreeCrawl as a server). Here the
|
||||
* desktop app is the client, so the in-app assistant can chain third-party
|
||||
* tools — Brave Search, GitHub, an internal enterprise server — onto the
|
||||
* crawl data it already has.
|
||||
*
|
||||
* Like the server side this is hand-rolled rather than built on
|
||||
* `@modelcontextprotocol/sdk`: the surface we need (`initialize`,
|
||||
* `tools/list`, `tools/call`) is small and stable, and a dependency-free
|
||||
* client keeps the zero-native-module promise intact.
|
||||
*
|
||||
* Two transports:
|
||||
* - **stdio** — spawn the server as a child process and speak
|
||||
* newline-delimited JSON-RPC over its stdin/stdout. Never through a
|
||||
* shell: the argument vector is passed as an array, and on Windows the
|
||||
* executable is resolved against PATH/PATHEXT ourselves so `npx`
|
||||
* (a `.cmd` shim) works without `shell: true` and its quoting hazards.
|
||||
* - **http** — Streamable HTTP: JSON-RPC over POST, accepting either a
|
||||
* plain JSON response or an SSE stream, and echoing back any
|
||||
* `Mcp-Session-Id` the server assigns.
|
||||
*
|
||||
* Registry storage: `<userData>/mcp-servers.json`, mode 0600. Env values
|
||||
* and auth headers are secrets (an MCP server config is where a GitHub PAT
|
||||
* ends up), so they are encrypted at rest with the same `safeStorage`
|
||||
* helpers the integration credential store uses, and the renderer only
|
||||
* ever sees `MCP_SECRET_MASK`.
|
||||
*/
|
||||
import { app } from 'electron';
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import { existsSync, readFileSync, writeFileSync, chmodSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
MCP_MAX_SERVERS,
|
||||
MCP_SECRET_MASK,
|
||||
type McpServerConfig,
|
||||
type McpServerStatus,
|
||||
type McpToolInfo,
|
||||
type McpToolPolicy,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { decryptString, encryptString } from './credentials.js';
|
||||
import {
|
||||
BUILTIN_SERVER_ID,
|
||||
extractJsonRpcMessage,
|
||||
qualifyToolName,
|
||||
resolveExecutable,
|
||||
slugify,
|
||||
truncateToolResult,
|
||||
} from './mcp-protocol.js';
|
||||
import { apiFetch } from './api-fetch.js';
|
||||
import * as logger from './logger.js';
|
||||
|
||||
/** MCP protocol revision we implement. Servers may answer with their own. */
|
||||
const PROTOCOL_VERSION = '2024-11-05';
|
||||
const CLIENT_NAME = 'freecrawl-seo-desktop';
|
||||
|
||||
const CONNECT_TIMEOUT_MS = 30_000;
|
||||
const LIST_TIMEOUT_MS = 30_000;
|
||||
const CALL_TIMEOUT_MS = 120_000;
|
||||
|
||||
/** Tool output handed to an LLM is truncated at this many characters —
|
||||
* an unbounded `tools/call` result would otherwise blow the context
|
||||
* window (and the user's token budget) on one bad call. */
|
||||
const MAX_TOOL_RESULT_CHARS = 24_000;
|
||||
|
||||
// Tool namespacing, JSON-RPC frame extraction and Windows PATH resolution
|
||||
// live in `mcp-protocol.ts` — pure string handling kept Electron-free so it
|
||||
// is directly unit-testable. Re-exported so callers have one import site.
|
||||
export {
|
||||
BUILTIN_SERVER_ID,
|
||||
BUILTIN_TOOL_PREFIX,
|
||||
MCP_TOOL_PREFIX,
|
||||
qualifyToolName,
|
||||
resolveExecutable,
|
||||
slugify,
|
||||
} from './mcp-protocol.js';
|
||||
|
||||
interface StoredServer extends Omit<McpServerConfig, 'env' | 'headers'> {
|
||||
/** Encrypted values, keyed by env var / header name. */
|
||||
env: Record<string, string>;
|
||||
headers: Record<string, string>;
|
||||
}
|
||||
|
||||
interface RegistryFile {
|
||||
servers: StoredServer[];
|
||||
/** `<serverId>:<tool>` → remembered decision. */
|
||||
policies: Record<string, McpToolPolicy>;
|
||||
}
|
||||
|
||||
let registry: RegistryFile | null = null;
|
||||
|
||||
function registryPath(): string {
|
||||
return join(app.getPath('userData'), 'mcp-servers.json');
|
||||
}
|
||||
|
||||
function loadRegistry(): RegistryFile {
|
||||
if (registry) return registry;
|
||||
try {
|
||||
const p = registryPath();
|
||||
if (existsSync(p)) {
|
||||
const parsed = JSON.parse(readFileSync(p, 'utf8')) as Partial<RegistryFile>;
|
||||
registry = {
|
||||
servers: Array.isArray(parsed.servers) ? parsed.servers : [],
|
||||
policies:
|
||||
parsed.policies && typeof parsed.policies === 'object'
|
||||
? (parsed.policies as Record<string, McpToolPolicy>)
|
||||
: {},
|
||||
};
|
||||
return registry;
|
||||
}
|
||||
} catch (err) {
|
||||
logger.log(
|
||||
'warn',
|
||||
'mcp',
|
||||
`MCP server registry unreadable, starting fresh: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
registry = { servers: [], policies: {} };
|
||||
return registry;
|
||||
}
|
||||
|
||||
function persistRegistry(): void {
|
||||
try {
|
||||
const p = registryPath();
|
||||
// 0600 for the same reason credentials.enc.json is: on a Linux box with
|
||||
// no keyring the encrypted values degrade to base64.
|
||||
writeFileSync(p, JSON.stringify(registry ?? { servers: [], policies: {} }, null, 2), {
|
||||
encoding: 'utf8',
|
||||
mode: 0o600,
|
||||
});
|
||||
if (process.platform !== 'win32') {
|
||||
try {
|
||||
chmodSync(p, 0o600);
|
||||
} catch {
|
||||
/* best-effort — a pre-existing file keeps its mode otherwise */
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.log('error', 'mcp', `MCP registry write failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Registry API ─────────────────────────────────────────────────────
|
||||
|
||||
/** Server list for the renderer — secret values replaced by the mask. */
|
||||
export function listServers(): McpServerConfig[] {
|
||||
return loadRegistry().servers.map((s) => ({
|
||||
id: s.id,
|
||||
label: s.label,
|
||||
transport: s.transport,
|
||||
command: s.command,
|
||||
args: [...(s.args ?? [])],
|
||||
env: maskValues(s.env),
|
||||
url: s.url,
|
||||
headers: maskValues(s.headers),
|
||||
enabled: s.enabled,
|
||||
}));
|
||||
}
|
||||
|
||||
function maskValues(rec: Record<string, string>): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const key of Object.keys(rec ?? {})) out[key] = MCP_SECRET_MASK;
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Decrypt one stored secret map for actual use. */
|
||||
function revealValues(rec: Record<string, string>): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(rec ?? {})) {
|
||||
try {
|
||||
out[k] = decryptString(v);
|
||||
} catch {
|
||||
out[k] = '';
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the whole registry with what the renderer sent.
|
||||
*
|
||||
* A value that comes back as `MCP_SECRET_MASK` means "unchanged" — the
|
||||
* renderer never received the plaintext, so it cannot echo it. Anything
|
||||
* else is a new secret and gets encrypted here.
|
||||
*/
|
||||
export function saveServers(incoming: McpServerConfig[]): McpServerConfig[] {
|
||||
const reg = loadRegistry();
|
||||
const previous = new Map(reg.servers.map((s) => [s.id, s]));
|
||||
const seen = new Set<string>();
|
||||
const next: StoredServer[] = [];
|
||||
|
||||
for (const raw of incoming.slice(0, MCP_MAX_SERVERS)) {
|
||||
const id = slugify(raw.id || raw.label);
|
||||
if (!id || seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
const prev = previous.get(id);
|
||||
next.push({
|
||||
id,
|
||||
label: (raw.label || id).slice(0, 60),
|
||||
transport: raw.transport === 'http' ? 'http' : 'stdio',
|
||||
command: (raw.command ?? '').trim().slice(0, 500),
|
||||
args: (raw.args ?? [])
|
||||
.map((a) => String(a))
|
||||
.filter((a) => a.length > 0)
|
||||
.slice(0, 40),
|
||||
env: mergeSecrets(raw.env ?? {}, prev?.env ?? {}),
|
||||
url: (raw.url ?? '').trim().slice(0, 2000),
|
||||
headers: mergeSecrets(raw.headers ?? {}, prev?.headers ?? {}),
|
||||
enabled: raw.enabled !== false,
|
||||
});
|
||||
}
|
||||
|
||||
// A server that disappeared from the list (or was edited) must not keep
|
||||
// a live child process around.
|
||||
for (const id of previous.keys()) {
|
||||
const stillThere = next.find((s) => s.id === id);
|
||||
const before = previous.get(id);
|
||||
if (!stillThere || !sameConnectionShape(before, stillThere)) {
|
||||
void disconnect(id);
|
||||
}
|
||||
}
|
||||
|
||||
reg.servers = next;
|
||||
persistRegistry();
|
||||
return listServers();
|
||||
}
|
||||
|
||||
function sameConnectionShape(a: StoredServer | undefined, b: StoredServer): boolean {
|
||||
if (!a) return false;
|
||||
return (
|
||||
a.transport === b.transport &&
|
||||
a.command === b.command &&
|
||||
a.url === b.url &&
|
||||
JSON.stringify(a.args) === JSON.stringify(b.args) &&
|
||||
JSON.stringify(a.env) === JSON.stringify(b.env) &&
|
||||
JSON.stringify(a.headers) === JSON.stringify(b.headers) &&
|
||||
a.enabled === b.enabled
|
||||
);
|
||||
}
|
||||
|
||||
function mergeSecrets(
|
||||
incoming: Record<string, string>,
|
||||
stored: Record<string, string>,
|
||||
): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(incoming)) {
|
||||
const name = key.trim();
|
||||
if (!name) continue;
|
||||
if (value === MCP_SECRET_MASK) {
|
||||
const kept = stored[name];
|
||||
if (kept !== undefined) out[name] = kept;
|
||||
continue;
|
||||
}
|
||||
out[name] = encryptString(value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Remembered decision for one tool (`ask` when the user never chose). */
|
||||
export function policyFor(serverId: string, tool: string): McpToolPolicy {
|
||||
return loadRegistry().policies[`${serverId}:${tool}`] ?? 'ask';
|
||||
}
|
||||
|
||||
export function setPolicy(serverId: string, tool: string, policy: McpToolPolicy): void {
|
||||
const reg = loadRegistry();
|
||||
if (policy === 'ask') delete reg.policies[`${serverId}:${tool}`];
|
||||
else reg.policies[`${serverId}:${tool}`] = policy;
|
||||
persistRegistry();
|
||||
}
|
||||
|
||||
export function listPolicies(): Record<string, McpToolPolicy> {
|
||||
return { ...loadRegistry().policies };
|
||||
}
|
||||
|
||||
// ── Connections ──────────────────────────────────────────────────────
|
||||
|
||||
interface Pending {
|
||||
resolve: (value: unknown) => void;
|
||||
reject: (err: Error) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
class McpConnection {
|
||||
private child: ChildProcess | null = null;
|
||||
private buffer = '';
|
||||
private nextId = 1;
|
||||
private pending = new Map<number, Pending>();
|
||||
private httpSessionId: string | null = null;
|
||||
private closed = false;
|
||||
|
||||
tools: McpToolInfo[] = [];
|
||||
serverName: string | null = null;
|
||||
serverVersion: string | null = null;
|
||||
protocolVersion: string | null = null;
|
||||
|
||||
constructor(private readonly cfg: StoredServer) {}
|
||||
|
||||
get id(): string {
|
||||
return this.cfg.id;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
if (this.cfg.transport === 'stdio') await this.spawnChild();
|
||||
const init = (await this.request(
|
||||
'initialize',
|
||||
{
|
||||
protocolVersion: PROTOCOL_VERSION,
|
||||
capabilities: {},
|
||||
clientInfo: { name: CLIENT_NAME, version: app.getVersion() },
|
||||
},
|
||||
CONNECT_TIMEOUT_MS,
|
||||
)) as {
|
||||
protocolVersion?: string;
|
||||
serverInfo?: { name?: string; version?: string };
|
||||
};
|
||||
this.protocolVersion = init?.protocolVersion ?? null;
|
||||
this.serverName = init?.serverInfo?.name ?? null;
|
||||
this.serverVersion = init?.serverInfo?.version ?? null;
|
||||
// Notification: no id, no response expected.
|
||||
this.notify('notifications/initialized', {});
|
||||
this.tools = await this.fetchTools();
|
||||
}
|
||||
|
||||
private async fetchTools(): Promise<McpToolInfo[]> {
|
||||
const out: McpToolInfo[] = [];
|
||||
let cursor: string | undefined;
|
||||
// Paginated per the MCP spec; cap the walk so a misbehaving server
|
||||
// can't spin us forever.
|
||||
for (let page = 0; page < 20; page++) {
|
||||
const res = (await this.request(
|
||||
'tools/list',
|
||||
cursor ? { cursor } : {},
|
||||
LIST_TIMEOUT_MS,
|
||||
)) as {
|
||||
tools?: { name?: string; description?: string; inputSchema?: unknown }[];
|
||||
nextCursor?: string;
|
||||
};
|
||||
for (const t of res?.tools ?? []) {
|
||||
if (!t?.name) continue;
|
||||
out.push({
|
||||
serverId: this.cfg.id,
|
||||
name: t.name,
|
||||
qualifiedName: qualifyToolName(this.cfg.id, t.name),
|
||||
description: (t.description ?? '').slice(0, 1000),
|
||||
inputSchema:
|
||||
t.inputSchema && typeof t.inputSchema === 'object'
|
||||
? (t.inputSchema as Record<string, unknown>)
|
||||
: { type: 'object', properties: {} },
|
||||
});
|
||||
}
|
||||
cursor = res?.nextCursor;
|
||||
if (!cursor) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async callTool(
|
||||
tool: string,
|
||||
args: Record<string, unknown>,
|
||||
): Promise<{ text: string; isError: boolean }> {
|
||||
const res = (await this.request(
|
||||
'tools/call',
|
||||
{ name: tool, arguments: args },
|
||||
CALL_TIMEOUT_MS,
|
||||
)) as {
|
||||
content?: { type?: string; text?: string }[];
|
||||
isError?: boolean;
|
||||
structuredContent?: unknown;
|
||||
};
|
||||
const parts = (res?.content ?? [])
|
||||
.map((c) => {
|
||||
if (c?.type === 'text' && typeof c.text === 'string') return c.text;
|
||||
// Images / resources can't be handed to a text-only chat turn —
|
||||
// describe them instead of dropping the block silently.
|
||||
return c?.type ? `[${c.type} content omitted]` : '';
|
||||
})
|
||||
.filter(Boolean);
|
||||
let text = parts.join('\n').trim();
|
||||
if (!text && res?.structuredContent !== undefined) {
|
||||
text = JSON.stringify(res.structuredContent);
|
||||
}
|
||||
if (!text) text = '(tool returned no content)';
|
||||
return {
|
||||
text: truncateToolResult(text, MAX_TOOL_RESULT_CHARS),
|
||||
isError: res?.isError === true,
|
||||
};
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.closed = true;
|
||||
for (const [, p] of this.pending) {
|
||||
clearTimeout(p.timer);
|
||||
p.reject(new Error('Connection closed'));
|
||||
}
|
||||
this.pending.clear();
|
||||
if (this.child) {
|
||||
try {
|
||||
this.child.kill();
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
this.child = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── stdio transport ────────────────────────────────────────────────
|
||||
|
||||
private async spawnChild(): Promise<void> {
|
||||
const command = this.cfg.command.trim();
|
||||
if (!command) throw new Error('No command configured for this stdio server.');
|
||||
const resolved = resolveExecutable(command);
|
||||
const env = { ...process.env, ...revealValues(this.cfg.env) };
|
||||
let child: ChildProcess;
|
||||
try {
|
||||
child = spawn(resolved, this.cfg.args, {
|
||||
env,
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
// Never `shell: true` — the args are a vector, and a shell would
|
||||
// re-split and expand them (and on Windows open a console window).
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch (err) {
|
||||
throw new Error(`Could not start "${command}": ${(err as Error).message}`);
|
||||
}
|
||||
this.child = child;
|
||||
|
||||
child.stdout?.setEncoding('utf8');
|
||||
child.stdout?.on('data', (chunk: string) => this.onStdout(chunk));
|
||||
child.stderr?.setEncoding('utf8');
|
||||
child.stderr?.on('data', (chunk: string) => {
|
||||
const line = String(chunk).trim();
|
||||
if (line) logger.log('debug', 'mcp', `[${this.cfg.id}] ${line.slice(0, 500)}`);
|
||||
});
|
||||
child.on('error', (err) => {
|
||||
this.failAll(new Error(`${command}: ${err.message}`));
|
||||
});
|
||||
child.on('exit', (code, signal) => {
|
||||
if (this.closed) return;
|
||||
this.failAll(
|
||||
new Error(
|
||||
`Server process exited (${signal ? `signal ${signal}` : `code ${code}`}).`,
|
||||
),
|
||||
);
|
||||
connections.delete(this.cfg.id);
|
||||
});
|
||||
|
||||
// A command that does not exist fails asynchronously via 'error' on
|
||||
// some platforms; give it a tick so connect() reports it properly.
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const t = setTimeout(resolve, 60);
|
||||
child.once('error', (err) => {
|
||||
clearTimeout(t);
|
||||
reject(new Error(`Could not start "${command}": ${err.message}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private onStdout(chunk: string): void {
|
||||
this.buffer += chunk;
|
||||
let nl: number;
|
||||
while ((nl = this.buffer.indexOf('\n')) !== -1) {
|
||||
const line = this.buffer.slice(0, nl).trim();
|
||||
this.buffer = this.buffer.slice(nl + 1);
|
||||
if (line) this.handleFrame(line);
|
||||
}
|
||||
// A server that never emits a newline would otherwise grow this
|
||||
// buffer without bound.
|
||||
if (this.buffer.length > 8 * 1024 * 1024) this.buffer = '';
|
||||
}
|
||||
|
||||
private handleFrame(line: string): void {
|
||||
let msg: {
|
||||
id?: number | string | null;
|
||||
result?: unknown;
|
||||
error?: { message?: string; code?: number };
|
||||
};
|
||||
try {
|
||||
msg = JSON.parse(line) as typeof msg;
|
||||
} catch {
|
||||
return; // Non-JSON chatter on stdout — ignore, stderr is the log.
|
||||
}
|
||||
if (msg.id === undefined || msg.id === null) return; // notification
|
||||
const id = typeof msg.id === 'string' ? Number(msg.id) : msg.id;
|
||||
const waiting = this.pending.get(id);
|
||||
if (!waiting) return;
|
||||
this.pending.delete(id);
|
||||
clearTimeout(waiting.timer);
|
||||
if (msg.error) waiting.reject(new Error(msg.error.message ?? 'MCP error'));
|
||||
else waiting.resolve(msg.result);
|
||||
}
|
||||
|
||||
private failAll(err: Error): void {
|
||||
for (const [, p] of this.pending) {
|
||||
clearTimeout(p.timer);
|
||||
p.reject(err);
|
||||
}
|
||||
this.pending.clear();
|
||||
}
|
||||
|
||||
// ── request plumbing ───────────────────────────────────────────────
|
||||
|
||||
private notify(method: string, params: unknown): void {
|
||||
const frame = JSON.stringify({ jsonrpc: '2.0', method, params });
|
||||
if (this.cfg.transport === 'stdio') {
|
||||
this.child?.stdin?.write(frame + '\n');
|
||||
} else {
|
||||
void this.httpPost(frame).catch(() => {
|
||||
/* notifications are best-effort */
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private request(method: string, params: unknown, timeoutMs: number): Promise<unknown> {
|
||||
const id = this.nextId++;
|
||||
const frame = JSON.stringify({ jsonrpc: '2.0', id, method, params });
|
||||
if (this.cfg.transport === 'http') return this.httpRequest(id, frame, timeoutMs);
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = this.child;
|
||||
if (!child?.stdin?.writable) {
|
||||
reject(new Error('Server process is not running.'));
|
||||
return;
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(id);
|
||||
reject(new Error(`${method} timed out after ${Math.round(timeoutMs / 1000)}s.`));
|
||||
}, timeoutMs);
|
||||
this.pending.set(id, { resolve, reject, timer });
|
||||
child.stdin.write(frame + '\n');
|
||||
});
|
||||
}
|
||||
|
||||
// ── Streamable HTTP transport ──────────────────────────────────────
|
||||
|
||||
private async httpPost(frame: string, timeoutMs = CALL_TIMEOUT_MS): Promise<Response> {
|
||||
const url = this.cfg.url.trim();
|
||||
if (!/^https?:\/\//i.test(url)) {
|
||||
throw new Error('HTTP server URL must start with http:// or https://');
|
||||
}
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json, text/event-stream',
|
||||
...revealValues(this.cfg.headers),
|
||||
};
|
||||
if (this.httpSessionId) headers['Mcp-Session-Id'] = this.httpSessionId;
|
||||
const res = await apiFetch(url, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: frame,
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
const session = res.headers.get('mcp-session-id');
|
||||
if (session) this.httpSessionId = session;
|
||||
return res as unknown as Response;
|
||||
}
|
||||
|
||||
private async httpRequest(
|
||||
id: number,
|
||||
frame: string,
|
||||
timeoutMs: number,
|
||||
): Promise<unknown> {
|
||||
const res = await this.httpPost(frame, timeoutMs);
|
||||
const body = await res.text();
|
||||
if (!res.ok) {
|
||||
throw new Error(`HTTP ${res.status}: ${body.slice(0, 300)}`);
|
||||
}
|
||||
const message = extractJsonRpcMessage(body, id);
|
||||
if (!message) throw new Error('Server returned no JSON-RPC response.');
|
||||
if (message.error) throw new Error(message.error.message ?? 'MCP error');
|
||||
return message.result;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Connection registry ──────────────────────────────────────────────
|
||||
|
||||
const connections = new Map<string, McpConnection>();
|
||||
const connecting = new Set<string>();
|
||||
const lastError = new Map<string, string>();
|
||||
|
||||
function storedServer(id: string): StoredServer | null {
|
||||
return loadRegistry().servers.find((s) => s.id === id) ?? null;
|
||||
}
|
||||
|
||||
/** Connect (or return the existing connection for) one server. */
|
||||
export async function connect(id: string): Promise<McpServerStatus> {
|
||||
const existing = connections.get(id);
|
||||
if (existing) return statusOf(id);
|
||||
const cfg = storedServer(id);
|
||||
if (!cfg) throw new Error(`No MCP server registered with id "${id}".`);
|
||||
connecting.add(id);
|
||||
const conn = new McpConnection(cfg);
|
||||
try {
|
||||
await conn.connect();
|
||||
connections.set(id, conn);
|
||||
lastError.delete(id);
|
||||
logger.log(
|
||||
'info',
|
||||
'mcp',
|
||||
`Connected to MCP server "${cfg.label}" (${conn.tools.length} tool(s)).`,
|
||||
);
|
||||
} catch (err) {
|
||||
conn.dispose();
|
||||
const msg = (err as Error).message;
|
||||
lastError.set(id, msg);
|
||||
logger.log('warn', 'mcp', `MCP server "${cfg.label}" failed to connect: ${msg}`);
|
||||
throw err;
|
||||
} finally {
|
||||
connecting.delete(id);
|
||||
}
|
||||
return statusOf(id);
|
||||
}
|
||||
|
||||
export function disconnect(id: string): void {
|
||||
const conn = connections.get(id);
|
||||
if (!conn) return;
|
||||
conn.dispose();
|
||||
connections.delete(id);
|
||||
}
|
||||
|
||||
export function disconnectAll(): void {
|
||||
for (const id of [...connections.keys()]) disconnect(id);
|
||||
}
|
||||
|
||||
export function statusOf(id: string): McpServerStatus {
|
||||
const conn = connections.get(id);
|
||||
return {
|
||||
id,
|
||||
connected: !!conn,
|
||||
connecting: connecting.has(id),
|
||||
error: lastError.get(id) ?? null,
|
||||
toolCount: conn?.tools.length ?? 0,
|
||||
serverName: conn?.serverName ?? null,
|
||||
serverVersion: conn?.serverVersion ?? null,
|
||||
protocolVersion: conn?.protocolVersion ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function statuses(): McpServerStatus[] {
|
||||
return listServers().map((s) => statusOf(s.id));
|
||||
}
|
||||
|
||||
/** Tools of every requested server, connecting on demand. Servers that
|
||||
* fail are reported through `errors` instead of failing the whole turn —
|
||||
* one broken server must not take the assistant down with it. */
|
||||
export async function toolsFor(
|
||||
ids: string[],
|
||||
): Promise<{ tools: McpToolInfo[]; errors: { id: string; message: string }[] }> {
|
||||
const tools: McpToolInfo[] = [];
|
||||
const errors: { id: string; message: string }[] = [];
|
||||
for (const id of ids) {
|
||||
const cfg = storedServer(id);
|
||||
if (!cfg || !cfg.enabled) continue;
|
||||
try {
|
||||
if (!connections.has(id)) await connect(id);
|
||||
tools.push(...(connections.get(id)?.tools ?? []));
|
||||
} catch (err) {
|
||||
errors.push({ id, message: (err as Error).message });
|
||||
}
|
||||
}
|
||||
return { tools, errors };
|
||||
}
|
||||
|
||||
export async function callTool(
|
||||
serverId: string,
|
||||
tool: string,
|
||||
args: Record<string, unknown>,
|
||||
): Promise<{ text: string; isError: boolean }> {
|
||||
let conn = connections.get(serverId);
|
||||
if (!conn) {
|
||||
await connect(serverId);
|
||||
conn = connections.get(serverId);
|
||||
}
|
||||
if (!conn) throw new Error(`MCP server "${serverId}" is not connected.`);
|
||||
return conn.callTool(tool, args);
|
||||
}
|
||||
|
||||
/** Display label for a server id (falls back to the id itself). */
|
||||
export function labelFor(serverId: string): string {
|
||||
if (serverId === BUILTIN_SERVER_ID) return 'FreeCrawl';
|
||||
return storedServer(serverId)?.label ?? serverId;
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Faz 5 — the Electron-free half of the MCP client.
|
||||
*
|
||||
* `mcp-client.ts` owns processes, sockets and the encrypted registry, all
|
||||
* of which need `electron`. The pieces that are pure string handling live
|
||||
* here instead: JSON-RPC frame extraction (plain JSON *and* SSE), tool-name
|
||||
* namespacing, and Windows executable resolution. Keeping them separate
|
||||
* makes them directly testable, which matters because each one is a place
|
||||
* where a wrong answer is silent — a missed SSE frame looks like a server
|
||||
* that never replied, and an unresolved `npx` looks like a missing package.
|
||||
*/
|
||||
import { existsSync } from 'node:fs';
|
||||
import { delimiter, isAbsolute, join } from 'node:path';
|
||||
|
||||
/** Namespace prefix for MCP-provided tools in the LLM's tool list. */
|
||||
export const MCP_TOOL_PREFIX = 'mcp__';
|
||||
/** Namespace prefix for FreeCrawl's own project tools. */
|
||||
export const BUILTIN_TOOL_PREFIX = 'builtin__';
|
||||
/** Pseudo server id used for FreeCrawl's own tools. */
|
||||
export const BUILTIN_SERVER_ID = 'builtin';
|
||||
|
||||
/** OpenAI and Anthropic both cap a tool name at 64 characters. */
|
||||
const MAX_TOOL_NAME = 64;
|
||||
|
||||
export interface RpcEnvelope {
|
||||
id?: number | string | null;
|
||||
result?: unknown;
|
||||
error?: { message?: string; code?: number };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the response for `id` out of a Streamable-HTTP body.
|
||||
*
|
||||
* The transport may answer with a single JSON-RPC object, a batch array,
|
||||
* or an SSE stream carrying one frame per `data:` line. Falls back to the
|
||||
* first parseable frame when no id matches — a server that echoes a
|
||||
* string id where we sent a number should not look like silence.
|
||||
*/
|
||||
export function extractJsonRpcMessage(body: string, id: number): RpcEnvelope | null {
|
||||
const text = body.trim();
|
||||
if (!text) return null;
|
||||
if (text.startsWith('{') || text.startsWith('[')) {
|
||||
const parsed = JSON.parse(text) as RpcEnvelope | RpcEnvelope[];
|
||||
const list = Array.isArray(parsed) ? parsed : [parsed];
|
||||
return list.find((m) => Number(m.id) === id) ?? list[0] ?? null;
|
||||
}
|
||||
let found: RpcEnvelope | null = null;
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
if (!line.startsWith('data:')) continue;
|
||||
const payload = line.slice(5).trim();
|
||||
if (!payload || payload === '[DONE]') continue;
|
||||
try {
|
||||
const parsed = JSON.parse(payload) as RpcEnvelope;
|
||||
if (Number(parsed.id) === id) return parsed;
|
||||
found ??= parsed;
|
||||
} catch {
|
||||
/* keep scanning — a partial frame is not fatal */
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/** `mcp__<server>__<tool>`, clamped to the provider tool-name limit. */
|
||||
export function qualifyToolName(serverId: string, tool: string): string {
|
||||
const safeTool = tool.replace(/[^A-Za-z0-9_-]/g, '_');
|
||||
const name = `${MCP_TOOL_PREFIX}${serverId}__${safeTool}`;
|
||||
return name.length <= MAX_TOOL_NAME ? name : name.slice(0, MAX_TOOL_NAME);
|
||||
}
|
||||
|
||||
/** Lowercase slug usable both as a registry key and inside a tool name.
|
||||
* LLM tool names are restricted to `[A-Za-z0-9_-]`, so the id must be too. */
|
||||
export function slugify(raw: string): string {
|
||||
return raw
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, 32);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a bare command name to a real executable path.
|
||||
*
|
||||
* `spawn` without a shell does not apply Windows' PATHEXT rules, so `npx`
|
||||
* — which exists on disk as `npx.cmd` — fails with ENOENT; and using
|
||||
* `shell: true` instead would re-parse the argument vector and pop a
|
||||
* console window. macOS and Linux resolve PATH themselves, so the name is
|
||||
* returned untouched there.
|
||||
*/
|
||||
export function resolveExecutable(
|
||||
command: string,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
exists: (p: string) => boolean = existsSync,
|
||||
): string {
|
||||
if (platform !== 'win32') return command;
|
||||
if (isAbsolute(command) || command.includes('/') || command.includes('\\')) {
|
||||
return command;
|
||||
}
|
||||
const exts = (env['PATHEXT'] ?? '.COM;.EXE;.BAT;.CMD')
|
||||
.split(';')
|
||||
.map((e) => e.trim())
|
||||
.filter(Boolean);
|
||||
const dirs = (env['PATH'] ?? '').split(delimiter).filter(Boolean);
|
||||
const hasExt = exts.some((e) => command.toLowerCase().endsWith(e.toLowerCase()));
|
||||
for (const dir of dirs) {
|
||||
if (hasExt) {
|
||||
const direct = join(dir, command);
|
||||
if (exists(direct)) return direct;
|
||||
continue;
|
||||
}
|
||||
for (const ext of exts) {
|
||||
const candidate = join(dir, command + ext);
|
||||
if (exists(candidate)) return candidate;
|
||||
}
|
||||
}
|
||||
return command; // Let spawn produce the ENOENT with the original name.
|
||||
}
|
||||
|
||||
/** Truncate a tool result so one huge payload cannot eat the context window. */
|
||||
export function truncateToolResult(text: string, max: number): string {
|
||||
if (text.length <= max) return text;
|
||||
return `${text.slice(0, max)}\n… [truncated ${text.length - max} characters]`;
|
||||
}
|
||||
@@ -43,6 +43,14 @@ import {
|
||||
type ImagesQueryResult,
|
||||
type LogEntry,
|
||||
type MenuEvent,
|
||||
type McpServerConfig,
|
||||
type McpServerStatus,
|
||||
type McpToolPolicy,
|
||||
type McpToolsListResult,
|
||||
type AssistantApprovalInput,
|
||||
type AssistantEvent,
|
||||
type AssistantSendInput,
|
||||
type AssistantSendResult,
|
||||
type RobotsTestInput,
|
||||
type RobotsTestResult,
|
||||
type RobotsValidationIssue,
|
||||
@@ -521,6 +529,31 @@ const api: FreeCrawlApi = {
|
||||
onLogEntry: (cb) => subscribe<LogEntry>(IPC.logsEntry, cb),
|
||||
onLogsBatch: (cb) => subscribe<LogEntry[]>(IPC.logsBatch, cb),
|
||||
onLogsBusy: (cb) => subscribe<boolean>(IPC.logsBusy, cb),
|
||||
mcpServersList: (): Promise<McpServerConfig[]> =>
|
||||
ipcRenderer.invoke(IPC.mcpServersList),
|
||||
mcpServersSave: (servers: McpServerConfig[]): Promise<McpServerConfig[]> =>
|
||||
ipcRenderer.invoke(IPC.mcpServersSave, servers),
|
||||
mcpServerConnect: (id: string): Promise<McpServerStatus> =>
|
||||
ipcRenderer.invoke(IPC.mcpServerConnect, id),
|
||||
mcpServerDisconnect: (id: string): Promise<McpServerStatus> =>
|
||||
ipcRenderer.invoke(IPC.mcpServerDisconnect, id),
|
||||
mcpServerStatuses: (): Promise<McpServerStatus[]> =>
|
||||
ipcRenderer.invoke(IPC.mcpServerStatuses),
|
||||
mcpToolsList: (serverIds: string[]): Promise<McpToolsListResult> =>
|
||||
ipcRenderer.invoke(IPC.mcpToolsList, serverIds),
|
||||
mcpPolicySet: (
|
||||
serverId: string,
|
||||
tool: string,
|
||||
policy: McpToolPolicy,
|
||||
): Promise<void> => ipcRenderer.invoke(IPC.mcpPolicySet, serverId, tool, policy),
|
||||
mcpPolicyList: (): Promise<Record<string, McpToolPolicy>> =>
|
||||
ipcRenderer.invoke(IPC.mcpPolicyList),
|
||||
assistantSend: (input: AssistantSendInput): Promise<AssistantSendResult> =>
|
||||
ipcRenderer.invoke(IPC.assistantSend, input),
|
||||
assistantCancel: (): Promise<void> => ipcRenderer.invoke(IPC.assistantCancel),
|
||||
assistantApprove: (input: AssistantApprovalInput): Promise<void> =>
|
||||
ipcRenderer.invoke(IPC.assistantApprove, input),
|
||||
onAssistantEvent: (cb) => subscribe<AssistantEvent>(IPC.assistantEvent, cb),
|
||||
onProgress: (cb) => subscribe<CrawlProgress>(IPC.crawlProgress, cb),
|
||||
onDone: (cb) => subscribe<CrawlSummary>(IPC.crawlDone, cb),
|
||||
onError: (cb) => subscribe<string>(IPC.crawlError, cb),
|
||||
|
||||
@@ -26,6 +26,7 @@ import { SearchConsoleTab } from './tabs/SearchConsoleTab.js';
|
||||
import { AnalyticsTab } from './tabs/AnalyticsTab.js';
|
||||
import { AiTab } from './tabs/AiTab.js';
|
||||
import { SeoTab } from './tabs/SeoTab.js';
|
||||
import { AssistantTab } from './tabs/AssistantTab.js';
|
||||
import { useAppStore } from './store.js';
|
||||
import type { MenuEvent, CrawlProgress } from '@freecrawl/shared-types';
|
||||
import { clearCrawlWithConfirm } from './utils/clearCrawl.js';
|
||||
@@ -48,6 +49,7 @@ export function App() {
|
||||
const setConfig = useAppStore((s) => s.setConfig);
|
||||
const applyProjectConfig = useAppStore((s) => s.applyProjectConfig);
|
||||
const setActiveTab = useAppStore((s) => s.setActiveTab);
|
||||
const assistantHandleEvent = useAppStore((s) => s.assistantHandleEvent);
|
||||
const [robotsTesterOpen, setRobotsTesterOpen] = useState(false);
|
||||
const [sitemapValidatorOpen, setSitemapValidatorOpen] = useState(false);
|
||||
const [exportAsOpen, setExportAsOpen] = useState(false);
|
||||
@@ -63,6 +65,15 @@ export function App() {
|
||||
| null
|
||||
>(null);
|
||||
|
||||
// Assistant events are subscribed here rather than in the Assistant tab:
|
||||
// a tool round can take minutes, and the user is free to go look at the
|
||||
// URLs table while it runs. A subscription that dies with the tab would
|
||||
// drop the results and leave an approval prompt nobody can answer.
|
||||
useEffect(
|
||||
() => window.freecrawl.onAssistantEvent(assistantHandleEvent),
|
||||
[assistantHandleEvent],
|
||||
);
|
||||
|
||||
// Drag & drop URL list — drop a `.txt` / `.csv` of URLs anywhere on the
|
||||
// window to populate List mode + open Settings so the user can review
|
||||
// before clicking Start. Lines starting with `#` are treated as comments
|
||||
@@ -540,6 +551,8 @@ export function App() {
|
||||
<AiTab />
|
||||
) : activeTab === 'seo' ? (
|
||||
<SeoTab />
|
||||
) : activeTab === 'assistant' ? (
|
||||
<AssistantTab />
|
||||
) : (
|
||||
<UrlsTab />
|
||||
)}
|
||||
|
||||
@@ -17,6 +17,8 @@ import type {
|
||||
LogUrlStatsInput,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { writeTextToClipboard } from './utils/clipboard.js';
|
||||
import { InfoTip } from './components/InfoTip.js';
|
||||
import { translateInfoTip } from './i18n/info-tips.js';
|
||||
|
||||
/**
|
||||
* V2 Faz 2 — standalone Log File Analyzer window (renderer `?loganalyzer=1`).
|
||||
@@ -401,10 +403,29 @@ function TabStrip({ tab, setTab }: { tab: Tab; setTab: (t: Tab) => void }) {
|
||||
);
|
||||
}
|
||||
|
||||
function Th({ children, className = '' }: { children: React.ReactNode; className?: string }) {
|
||||
function Th({
|
||||
children,
|
||||
className = '',
|
||||
info,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
className?: string;
|
||||
/** English tooltip body — rendered as the standard [i] icon, translated
|
||||
* through the shared InfoTip dictionary. */
|
||||
info?: string;
|
||||
}) {
|
||||
return (
|
||||
<th className={`sticky top-0 z-10 bg-surface-900 px-2 py-1 text-left font-medium text-surface-400 ${className}`}>
|
||||
{children}
|
||||
{info ? (
|
||||
<span
|
||||
className={`inline-flex items-center gap-1 ${className.includes('text-right') ? 'flex-row-reverse' : ''}`}
|
||||
>
|
||||
{children}
|
||||
<InfoTip info={info} />
|
||||
</span>
|
||||
) : (
|
||||
children
|
||||
)}
|
||||
</th>
|
||||
);
|
||||
}
|
||||
@@ -428,6 +449,7 @@ function UrlStatsTab({ version, mode }: { version: number; mode: 'all' | 'orphan
|
||||
const [filter, setFilter] = useState<NonNullable<LogUrlStatsInput['filter']>>(mode === 'orphans' ? 'orphans' : 'all');
|
||||
const [bot, setBot] = useState('');
|
||||
const [botList, setBotList] = useState<LogBotRow[]>([]);
|
||||
const [status, setStatus] = useState<NonNullable<LogUrlStatsInput['status']>>('all');
|
||||
const [sortBy, setSortBy] = useState<NonNullable<LogUrlStatsInput['sortBy']>>('totalHits');
|
||||
|
||||
useEffect(() => {
|
||||
@@ -436,7 +458,7 @@ function UrlStatsTab({ version, mode }: { version: number; mode: 'all' | 'orphan
|
||||
|
||||
useEffect(() => {
|
||||
setOffset(0);
|
||||
}, [search, filter, bot, sortBy, version]);
|
||||
}, [search, filter, bot, status, sortBy, version]);
|
||||
|
||||
useEffect(() => {
|
||||
// Cancellation guard: fast typing / filter changes fire overlapping
|
||||
@@ -444,7 +466,15 @@ function UrlStatsTab({ version, mode }: { version: number; mode: 'all' | 'orphan
|
||||
// and overwrite the newer result set, showing rows for a filter the user
|
||||
// no longer has selected.
|
||||
let cancelled = false;
|
||||
const input: LogUrlStatsInput = { limit: PAGE, offset, search, sortBy, filter, bot: bot || undefined };
|
||||
const input: LogUrlStatsInput = {
|
||||
limit: PAGE,
|
||||
offset,
|
||||
search,
|
||||
sortBy,
|
||||
filter,
|
||||
bot: bot || undefined,
|
||||
status,
|
||||
};
|
||||
const call = mode === 'orphans' ? window.freecrawl.logOrphans(input) : window.freecrawl.logUrlStats(input);
|
||||
void call.then((r) => {
|
||||
if (cancelled) return;
|
||||
@@ -454,7 +484,7 @@ function UrlStatsTab({ version, mode }: { version: number; mode: 'all' | 'orphan
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [offset, search, filter, bot, sortBy, version, mode]);
|
||||
}, [offset, search, filter, bot, status, sortBy, version, mode]);
|
||||
|
||||
// The single filter <select> carries either a membership token
|
||||
// (all/bots/orphans/crawled) or a `bot:<name>` per-bot selection.
|
||||
@@ -499,6 +529,20 @@ function UrlStatsTab({ version, mode }: { version: number; mode: 'all' | 'orphan
|
||||
)}
|
||||
</select>
|
||||
)}
|
||||
<span className="inline-flex items-center gap-1">
|
||||
<select
|
||||
value={status}
|
||||
onChange={(e) => setStatus(e.target.value as NonNullable<LogUrlStatsInput['status']>)}
|
||||
className="rounded border border-surface-700 bg-surface-800 px-2 py-1 text-surface-100"
|
||||
>
|
||||
<option value="all">{t('logAnalyzer.urlStatusAll', { defaultValue: 'Any status' })}</option>
|
||||
<option value="2xx">2xx</option>
|
||||
<option value="3xx">3xx</option>
|
||||
<option value="4xx">{t('logAnalyzer.urlStatus4xx', { defaultValue: '4xx (errors)' })}</option>
|
||||
<option value="5xx">{t('logAnalyzer.urlStatus5xx', { defaultValue: '5xx (server errors)' })}</option>
|
||||
</select>
|
||||
<InfoTip info="Filters on the Status column — the most recent response the log recorded for that path. The analyzer keeps one status per URL rather than a full distribution, so this answers 'what is this URL returning now'. Paths whose status could not be parsed are hidden while a class is selected." />
|
||||
</span>
|
||||
<select
|
||||
value={sortBy}
|
||||
onChange={(e) => setSortBy(e.target.value as NonNullable<LogUrlStatsInput['sortBy']>)}
|
||||
@@ -521,7 +565,12 @@ function UrlStatsTab({ version, mode }: { version: number; mode: 'all' | 'orphan
|
||||
<Th className="text-right">Googlebot</Th>
|
||||
<Th className="text-right">Bingbot</Th>
|
||||
<Th className="text-right">Yandex</Th>
|
||||
<Th className="text-right">{t('logAnalyzer.colStatus', { defaultValue: 'Status' })}</Th>
|
||||
<Th
|
||||
className="text-right"
|
||||
info="Most recent HTTP status the log recorded for this path. One value per URL, not a distribution — a path that returned 200 all week and 404 this morning shows 404."
|
||||
>
|
||||
{t('logAnalyzer.colStatus', { defaultValue: 'Status' })}
|
||||
</Th>
|
||||
<Th>{t('logAnalyzer.colLast', { defaultValue: 'Last Hit' })}</Th>
|
||||
<Th>{t('logAnalyzer.colCrawl', { defaultValue: 'In Crawl' })}</Th>
|
||||
</tr>
|
||||
@@ -818,6 +867,26 @@ const THREAT_CATEGORIES: LogThreatCategory[] = [
|
||||
'anomaly',
|
||||
];
|
||||
|
||||
/**
|
||||
* What each category actually means, in the site owner's terms. Shown as
|
||||
* the hover text on every category badge (table rows and the filter
|
||||
* chips) and collected into the Category column's [i] tooltip, because
|
||||
* "Scanner probe" is only obvious to someone who already knows.
|
||||
*/
|
||||
const THREAT_CATEGORY_INFO: Record<LogThreatCategory, string> = {
|
||||
sqli: 'SQL injection — the request tries to smuggle SQL into a parameter (UNION SELECT, sleep(), error-based functions) to read or alter your database.',
|
||||
xss: 'Cross-site scripting — the request carries script markup or a javascript: URL in a parameter, hoping the page echoes it back into the HTML unescaped.',
|
||||
traversal: 'Path traversal — the request walks out of the web root with ../ or encoded variants to reach files like /etc/passwd or win.ini.',
|
||||
cmdi: 'Command injection — the request appends shell syntax (;, |, backticks, $( )) to a parameter to run commands on the server.',
|
||||
scanner: 'Scanner probe — an automated vulnerability scanner walking a wordlist of known admin panels, installers and exploit paths (wp-login, phpmyadmin, /actuator, shell uploads). Not tailored to your site; it hits everyone.',
|
||||
'sensitive-file': 'Sensitive file fetch — a direct request for something that must never be public: .env, .git, backups, SQL dumps, private keys, config files.',
|
||||
anomaly: 'Anomaly — malformed or evasive input (null bytes, CRLF injection, over-encoding, absurd parameter lengths) that matches no single attack class but is not a normal browser request.',
|
||||
};
|
||||
|
||||
/** How a request earns its score — the answer to "what is this number?". */
|
||||
const THREAT_SCORE_INFO =
|
||||
'Sum of the weights of every attack signature the request matched. Each signature carries a weight by how conclusive it is (a UNION SELECT weighs 9, a stray quote 2), and a line is only flagged once the total reaches 5 — so one decisive pattern flags on its own, while weak hints have to add up. Higher score = less room for a false positive; sort by it to triage.';
|
||||
|
||||
const THREAT_COLORS: Record<LogThreatCategory, string> = {
|
||||
sqli: 'text-red-300 bg-red-500/15',
|
||||
xss: 'text-orange-300 bg-orange-500/15',
|
||||
@@ -853,9 +922,25 @@ function useThreatCategoryLabel(): (cat: LogThreatCategory) => string {
|
||||
);
|
||||
}
|
||||
|
||||
function ThreatBadge({ category, label }: { category: LogThreatCategory; label: string }) {
|
||||
function ThreatBadge({
|
||||
category,
|
||||
label,
|
||||
title,
|
||||
}: {
|
||||
category: LogThreatCategory;
|
||||
label: string;
|
||||
/** Extra hover text appended to the category's own explanation. */
|
||||
title?: string;
|
||||
}) {
|
||||
const { i18n } = useTranslation();
|
||||
const info = translateInfoTip(THREAT_CATEGORY_INFO[category], i18n.language);
|
||||
return (
|
||||
<span className={`whitespace-nowrap rounded px-1.5 py-0.5 text-[10px] ${THREAT_COLORS[category]}`}>
|
||||
<span
|
||||
className={`cursor-help whitespace-nowrap rounded px-1.5 py-0.5 text-[10px] ${THREAT_COLORS[category]}`}
|
||||
title={title ? `${info}
|
||||
|
||||
${title}` : info}
|
||||
>
|
||||
{label}
|
||||
</span>
|
||||
);
|
||||
@@ -1072,9 +1157,15 @@ function ThreatsTab({ version, onToast }: { version: number; onToast: (m: string
|
||||
key={c.category}
|
||||
onClick={() => setCategory(category === c.category ? 'all' : c.category)}
|
||||
className={`rounded border px-2 py-0.5 ${category === c.category ? 'border-blue-500' : 'border-surface-800 hover:border-surface-600'}`}
|
||||
title={`${c.ips.toLocaleString()} IP`}
|
||||
>
|
||||
<ThreatBadge category={c.category} label={`${catLabel(c.category)} ${c.hits.toLocaleString()}`} />
|
||||
<ThreatBadge
|
||||
category={c.category}
|
||||
label={`${catLabel(c.category)} ${c.hits.toLocaleString()}`}
|
||||
title={t('logAnalyzer.threatChipIps', {
|
||||
defaultValue: '{{n}} distinct IP(s)',
|
||||
n: c.ips.toLocaleString(),
|
||||
})}
|
||||
/>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
@@ -1160,8 +1251,12 @@ function ThreatsTab({ version, onToast }: { version: number; onToast: (m: string
|
||||
<Th>{t('logAnalyzer.colMethod', { defaultValue: 'Method' })}</Th>
|
||||
<Th>{t('logAnalyzer.colRequest', { defaultValue: 'Request (decoded)' })}</Th>
|
||||
<Th className="text-right">{t('logAnalyzer.colStatus', { defaultValue: 'Status' })}</Th>
|
||||
<Th>{t('logAnalyzer.colCategory', { defaultValue: 'Category' })}</Th>
|
||||
<Th className="text-right">{t('logAnalyzer.colScore', { defaultValue: 'Score' })}</Th>
|
||||
<Th info="Which attack class the strongest matching signature belongs to: SQL injection, XSS, path traversal, command injection, scanner probe, sensitive file, or anomaly. Hover any badge in this column for what that class means in practice.">
|
||||
{t('logAnalyzer.colCategory', { defaultValue: 'Category' })}
|
||||
</Th>
|
||||
<Th className="text-right" info={THREAT_SCORE_INFO}>
|
||||
{t('logAnalyzer.colScore', { defaultValue: 'Score' })}
|
||||
</Th>
|
||||
<Th>{t('logAnalyzer.colEvidence', { defaultValue: 'Evidence' })}</Th>
|
||||
<Th>{t('logAnalyzer.colDeclaredBot', { defaultValue: 'Declared Bot' })}</Th>
|
||||
</tr>
|
||||
|
||||
@@ -1987,7 +1987,7 @@ function fleschBand(score: number): string {
|
||||
}
|
||||
|
||||
function NameValueView({ row, exportName }: { row: CrawlUrlRow; exportName: string }) {
|
||||
const { t, i18n } = useTranslation();
|
||||
const { i18n } = useTranslation();
|
||||
const lang = i18n.language;
|
||||
// Server-side pixel-width is the source of truth (drives the issue
|
||||
// filters); fall back to the renderer estimate only when the column is
|
||||
|
||||
@@ -28,7 +28,6 @@ export class ErrorBoundary extends Component<Props, State> {
|
||||
}
|
||||
|
||||
override componentDidCatch(err: Error, info: { componentStack?: string }): void {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(
|
||||
`[ErrorBoundary${this.props.context ? `:${this.props.context}` : ''}]`,
|
||||
err,
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Loader2, Plus, Plug, Trash2, Unplug } from 'lucide-react';
|
||||
import {
|
||||
MCP_MAX_SERVERS,
|
||||
MCP_SECRET_MASK,
|
||||
type McpServerConfig,
|
||||
type McpServerStatus,
|
||||
type McpToolInfo,
|
||||
type McpToolPolicy,
|
||||
} from '@freecrawl/shared-types';
|
||||
|
||||
/**
|
||||
* Settings → MCP Servers.
|
||||
*
|
||||
* Registers the external MCP servers the in-app assistant may call. Two
|
||||
* transports: a local process (stdio — the common case, e.g.
|
||||
* `npx -y @modelcontextprotocol/server-github`) or a remote Streamable
|
||||
* HTTP endpoint.
|
||||
*
|
||||
* Secret handling: env values and auth headers are encrypted in the main
|
||||
* process and never sent back to the renderer, so a stored value shows as
|
||||
* the mask. Leaving the mask in place keeps the stored secret; typing over
|
||||
* it replaces it.
|
||||
*/
|
||||
export function McpServersSection() {
|
||||
const { t } = useTranslation();
|
||||
const [servers, setServers] = useState<McpServerConfig[]>([]);
|
||||
const [statuses, setStatuses] = useState<Record<string, McpServerStatus>>({});
|
||||
const [tools, setTools] = useState<Record<string, McpToolInfo[]>>({});
|
||||
const [policies, setPolicies] = useState<Record<string, McpToolPolicy>>({});
|
||||
const [busy, setBusy] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loaded, setLoaded] = useState(false);
|
||||
|
||||
const reload = useCallback(async () => {
|
||||
const [list, statusList, policyMap] = await Promise.all([
|
||||
window.freecrawl.mcpServersList(),
|
||||
window.freecrawl.mcpServerStatuses(),
|
||||
window.freecrawl.mcpPolicyList(),
|
||||
]);
|
||||
setServers(list);
|
||||
setStatuses(Object.fromEntries(statusList.map((s) => [s.id, s])));
|
||||
setPolicies(policyMap);
|
||||
setLoaded(true);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void reload().catch((e: unknown) => {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
setLoaded(true);
|
||||
});
|
||||
}, [reload]);
|
||||
|
||||
/** Persist immediately — a half-saved server list is worse than an
|
||||
* extra IPC round-trip, and the dialog's Save button only covers the
|
||||
* crawl config. */
|
||||
async function persist(next: McpServerConfig[]) {
|
||||
setServers(next);
|
||||
const saved = await window.freecrawl.mcpServersSave(next);
|
||||
setServers(saved);
|
||||
const statusList = await window.freecrawl.mcpServerStatuses();
|
||||
setStatuses(Object.fromEntries(statusList.map((s) => [s.id, s])));
|
||||
}
|
||||
|
||||
function update(id: string, patch: Partial<McpServerConfig>) {
|
||||
void persist(servers.map((s) => (s.id === id ? { ...s, ...patch } : s)));
|
||||
}
|
||||
|
||||
function addServer() {
|
||||
if (servers.length >= MCP_MAX_SERVERS) return;
|
||||
let n = servers.length + 1;
|
||||
while (servers.some((s) => s.id === `server-${n}`)) n++;
|
||||
void persist([
|
||||
...servers,
|
||||
{
|
||||
id: `server-${n}`,
|
||||
label: `Server ${n}`,
|
||||
transport: 'stdio',
|
||||
command: '',
|
||||
args: [],
|
||||
env: {},
|
||||
url: '',
|
||||
headers: {},
|
||||
enabled: true,
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
async function connect(id: string) {
|
||||
setBusy(id);
|
||||
setError(null);
|
||||
try {
|
||||
const status = await window.freecrawl.mcpServerConnect(id);
|
||||
setStatuses((prev) => ({ ...prev, [id]: status }));
|
||||
if (status.connected) {
|
||||
const res = await window.freecrawl.mcpToolsList([id]);
|
||||
setTools((prev) => ({
|
||||
...prev,
|
||||
[id]: res.tools.filter((tool) => tool.serverId === id),
|
||||
}));
|
||||
} else if (status.error) {
|
||||
setError(status.error);
|
||||
}
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
}
|
||||
|
||||
async function disconnect(id: string) {
|
||||
const status = await window.freecrawl.mcpServerDisconnect(id);
|
||||
setStatuses((prev) => ({ ...prev, [id]: status }));
|
||||
}
|
||||
|
||||
async function setPolicy(serverId: string, tool: string, policy: McpToolPolicy) {
|
||||
await window.freecrawl.mcpPolicySet(serverId, tool, policy);
|
||||
setPolicies((prev) => {
|
||||
const next = { ...prev };
|
||||
if (policy === 'ask') delete next[`${serverId}:${tool}`];
|
||||
else next[`${serverId}:${tool}`] = policy;
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<p className="mb-3 text-[11px] text-surface-400">
|
||||
{t('settings.mcp.intro', {
|
||||
defaultValue:
|
||||
'Connect external MCP servers so the Assistant tab can use their tools alongside your crawl data. Local servers run as a child process; remote ones are called over HTTP. Every tool call asks for your approval the first time.',
|
||||
})}
|
||||
</p>
|
||||
|
||||
{error && (
|
||||
<div className="mb-3 rounded border border-red-800/60 bg-red-950/20 p-2 text-[11px] text-red-200">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!loaded && (
|
||||
<div className="flex items-center gap-2 text-[11px] text-surface-400">
|
||||
<Loader2 size={12} className="animate-spin" />
|
||||
{t('common.loading', { defaultValue: 'Loading…' })}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loaded && servers.length === 0 && (
|
||||
<p className="mb-3 rounded border border-surface-800 bg-surface-950/40 p-3 text-[11px] text-surface-400">
|
||||
{t('settings.mcp.empty', {
|
||||
defaultValue:
|
||||
'No servers yet. A typical local server looks like: command "npx", arguments "-y" and "@modelcontextprotocol/server-filesystem", plus any API key as an environment variable.',
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{servers.map((server) => {
|
||||
const status = statuses[server.id];
|
||||
const serverTools = tools[server.id] ?? [];
|
||||
return (
|
||||
<div
|
||||
key={server.id}
|
||||
className="mb-3 rounded border border-surface-800 bg-surface-950/40 p-3"
|
||||
>
|
||||
<div className="mb-2 flex items-center gap-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={server.enabled}
|
||||
onChange={(e) => update(server.id, { enabled: e.target.checked })}
|
||||
title={t('settings.mcp.enabled', { defaultValue: 'Enabled' })}
|
||||
/>
|
||||
<input
|
||||
className="h-7 flex-1 rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={server.label}
|
||||
onChange={(e) => update(server.id, { label: e.target.value })}
|
||||
placeholder={t('settings.mcp.labelPlaceholder', {
|
||||
defaultValue: 'Display name',
|
||||
})}
|
||||
/>
|
||||
<span
|
||||
className={
|
||||
status?.connected
|
||||
? 'h-2 w-2 rounded-full bg-emerald-500'
|
||||
: 'h-2 w-2 rounded-full bg-surface-600'
|
||||
}
|
||||
title={
|
||||
status?.connected
|
||||
? t('settings.mcp.connected', { defaultValue: 'Connected' })
|
||||
: t('settings.mcp.disconnected', { defaultValue: 'Not connected' })
|
||||
}
|
||||
/>
|
||||
{status?.connected ? (
|
||||
<button
|
||||
className="flex h-7 items-center gap-1 rounded border border-surface-700 px-2 text-[11px] text-surface-300 hover:bg-surface-800"
|
||||
onClick={() => void disconnect(server.id)}
|
||||
>
|
||||
<Unplug size={12} />
|
||||
{t('settings.mcp.disconnect', { defaultValue: 'Disconnect' })}
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
className="flex h-7 items-center gap-1 rounded border border-surface-700 px-2 text-[11px] text-surface-300 hover:bg-surface-800 disabled:opacity-40"
|
||||
onClick={() => void connect(server.id)}
|
||||
disabled={busy === server.id}
|
||||
>
|
||||
{busy === server.id ? (
|
||||
<Loader2 size={12} className="animate-spin" />
|
||||
) : (
|
||||
<Plug size={12} />
|
||||
)}
|
||||
{t('settings.mcp.connect', { defaultValue: 'Connect' })}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
className="flex h-7 items-center rounded border border-surface-700 px-2 text-[11px] text-red-300 hover:bg-surface-800"
|
||||
onClick={() =>
|
||||
void persist(servers.filter((s) => s.id !== server.id))
|
||||
}
|
||||
title={t('settings.mcp.remove', { defaultValue: 'Remove server' })}
|
||||
>
|
||||
<Trash2 size={12} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="mb-2 flex items-center gap-2">
|
||||
<label className="text-[10px] uppercase tracking-wider text-surface-500">
|
||||
{t('settings.mcp.transport', { defaultValue: 'Transport' })}
|
||||
</label>
|
||||
<select
|
||||
className="h-7 rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={server.transport}
|
||||
onChange={(e) =>
|
||||
update(server.id, {
|
||||
transport: e.target.value === 'http' ? 'http' : 'stdio',
|
||||
})
|
||||
}
|
||||
>
|
||||
<option value="stdio">
|
||||
{t('settings.mcp.transportStdio', {
|
||||
defaultValue: 'Local process (stdio)',
|
||||
})}
|
||||
</option>
|
||||
<option value="http">
|
||||
{t('settings.mcp.transportHttp', { defaultValue: 'Remote (HTTP)' })}
|
||||
</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{server.transport === 'stdio' ? (
|
||||
<>
|
||||
<Field
|
||||
label={t('settings.mcp.command', { defaultValue: 'Command' })}
|
||||
hint={t('settings.mcp.commandHint', {
|
||||
defaultValue:
|
||||
'Executable name or full path. Resolved against PATH on every OS — on Windows, `npx` resolves to npx.cmd automatically.',
|
||||
})}
|
||||
>
|
||||
<input
|
||||
className="h-7 w-full rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={server.command}
|
||||
onChange={(e) => update(server.id, { command: e.target.value })}
|
||||
placeholder="npx"
|
||||
/>
|
||||
</Field>
|
||||
<Field
|
||||
label={t('settings.mcp.args', { defaultValue: 'Arguments' })}
|
||||
hint={t('settings.mcp.argsHint', {
|
||||
defaultValue:
|
||||
'One per line. Passed as a vector — no shell, so quotes and spaces are literal.',
|
||||
})}
|
||||
>
|
||||
<textarea
|
||||
className="h-16 w-full resize-none rounded border border-surface-700 bg-surface-950 px-2 py-1 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={server.args.join('\n')}
|
||||
onChange={(e) =>
|
||||
update(server.id, {
|
||||
args: e.target.value
|
||||
.split('\n')
|
||||
.map((a) => a.trim())
|
||||
.filter(Boolean),
|
||||
})
|
||||
}
|
||||
placeholder={'-y\n@modelcontextprotocol/server-github'}
|
||||
/>
|
||||
</Field>
|
||||
<Field
|
||||
label={t('settings.mcp.env', { defaultValue: 'Environment variables' })}
|
||||
hint={t('settings.mcp.envHint', {
|
||||
defaultValue:
|
||||
'KEY=value per line. Values are stored encrypted with your OS keychain and shown masked.',
|
||||
})}
|
||||
>
|
||||
<textarea
|
||||
className="h-16 w-full resize-none rounded border border-surface-700 bg-surface-950 px-2 py-1 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={pairsToText(server.env, '=')}
|
||||
onChange={(e) =>
|
||||
update(server.id, { env: textToPairs(e.target.value, '=') })
|
||||
}
|
||||
placeholder={`GITHUB_TOKEN=${MCP_SECRET_MASK}`}
|
||||
/>
|
||||
</Field>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Field label={t('settings.mcp.url', { defaultValue: 'Endpoint URL' })}>
|
||||
<input
|
||||
className="h-7 w-full rounded border border-surface-700 bg-surface-950 px-2 text-[12px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={server.url}
|
||||
onChange={(e) => update(server.id, { url: e.target.value })}
|
||||
placeholder="https://example.com/mcp"
|
||||
/>
|
||||
</Field>
|
||||
<Field
|
||||
label={t('settings.mcp.headers', { defaultValue: 'Headers' })}
|
||||
hint={t('settings.mcp.headersHint', {
|
||||
defaultValue:
|
||||
'Name: value per line. Stored encrypted and shown masked.',
|
||||
})}
|
||||
>
|
||||
<textarea
|
||||
className="h-16 w-full resize-none rounded border border-surface-700 bg-surface-950 px-2 py-1 font-mono text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={pairsToText(server.headers, ': ')}
|
||||
onChange={(e) =>
|
||||
update(server.id, { headers: textToPairs(e.target.value, ':') })
|
||||
}
|
||||
placeholder={`Authorization: Bearer ${MCP_SECRET_MASK}`}
|
||||
/>
|
||||
</Field>
|
||||
</>
|
||||
)}
|
||||
|
||||
{status?.error && (
|
||||
<p className="mt-1 text-[10px] text-red-400">{status.error}</p>
|
||||
)}
|
||||
{status?.connected && (
|
||||
<p className="mt-1 text-[10px] text-surface-500">
|
||||
{t('settings.mcp.handshake', {
|
||||
defaultValue: '{{name}} {{version}} · MCP {{protocol}} · {{n}} tool(s)',
|
||||
name: status.serverName ?? server.label,
|
||||
version: status.serverVersion ?? '',
|
||||
protocol: status.protocolVersion ?? '',
|
||||
n: status.toolCount,
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{serverTools.length > 0 && (
|
||||
<div className="mt-2 rounded border border-surface-800">
|
||||
<div className="border-b border-surface-800 px-2 py-1 text-[10px] font-semibold uppercase tracking-wider text-surface-500">
|
||||
{t('settings.mcp.toolPermissions', {
|
||||
defaultValue: 'Tool permissions',
|
||||
})}
|
||||
</div>
|
||||
<div className="max-h-52 overflow-y-auto">
|
||||
{serverTools.map((tool) => (
|
||||
<div
|
||||
key={tool.name}
|
||||
className="flex items-center gap-2 border-b border-surface-800/60 px-2 py-1 last:border-b-0"
|
||||
>
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="block truncate font-mono text-[11px] text-surface-200">
|
||||
{tool.name}
|
||||
</span>
|
||||
<span className="block truncate text-[10px] text-surface-500">
|
||||
{tool.description}
|
||||
</span>
|
||||
</span>
|
||||
<select
|
||||
className="h-6 rounded border border-surface-700 bg-surface-950 px-1 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={policies[`${server.id}:${tool.name}`] ?? 'ask'}
|
||||
onChange={(e) =>
|
||||
void setPolicy(
|
||||
server.id,
|
||||
tool.name,
|
||||
e.target.value as McpToolPolicy,
|
||||
)
|
||||
}
|
||||
>
|
||||
<option value="ask">
|
||||
{t('settings.mcp.policyAsk', { defaultValue: 'Ask' })}
|
||||
</option>
|
||||
<option value="always">
|
||||
{t('settings.mcp.policyAlways', { defaultValue: 'Always allow' })}
|
||||
</option>
|
||||
<option value="never">
|
||||
{t('settings.mcp.policyNever', { defaultValue: 'Never allow' })}
|
||||
</option>
|
||||
</select>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<button
|
||||
className="flex items-center gap-1 rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-300 hover:bg-surface-800 hover:text-surface-100 disabled:opacity-40"
|
||||
onClick={addServer}
|
||||
disabled={servers.length >= MCP_MAX_SERVERS}
|
||||
>
|
||||
<Plus size={12} />
|
||||
{t('settings.mcp.add', { defaultValue: 'Add server' })}
|
||||
</button>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function Field({
|
||||
label,
|
||||
hint,
|
||||
children,
|
||||
}: {
|
||||
label: string;
|
||||
hint?: string;
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<div className="mb-2">
|
||||
<div className="mb-1 text-[10px] font-semibold uppercase tracking-wider text-surface-500">
|
||||
{label}
|
||||
</div>
|
||||
{hint && <p className="mb-1 text-[10px] text-surface-500">{hint}</p>}
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** `{A: 'x'}` → `A=x` lines (values already masked by the main process). */
|
||||
function pairsToText(rec: Record<string, string>, sep: string): string {
|
||||
return Object.entries(rec ?? {})
|
||||
.map(([k, v]) => `${k}${sep}${v}`)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function textToPairs(text: string, sep: string): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const line of text.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
const idx = trimmed.indexOf(sep);
|
||||
if (idx <= 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const value = trimmed.slice(idx + sep.length).trim();
|
||||
if (key) out[key] = value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -95,33 +95,6 @@ const REPORT_LABELS: Record<ReportKind, string> = {
|
||||
'top-words': 'Top Words (Title + Meta + H1, Top 100)',
|
||||
};
|
||||
|
||||
const KEY_LABELS: Record<ReportKind, string> = {
|
||||
'pages-per-dir': 'Directory',
|
||||
'status-codes': 'Status',
|
||||
'indexability-distribution': 'Indexability',
|
||||
'content-kind-distribution': 'Content Type',
|
||||
depth: 'Depth',
|
||||
'response-time': 'Bucket',
|
||||
'slowest-urls': 'URL',
|
||||
'most-inlinks': 'URL',
|
||||
'least-inlinks': 'URL',
|
||||
'most-outlinks': 'URL',
|
||||
'biggest-pages': 'URL',
|
||||
'deepest-urls': 'URL',
|
||||
'external-domain-health': 'Domain',
|
||||
'analytics-coverage': 'Tracker',
|
||||
'link-positions': 'Position',
|
||||
'image-weight': 'URL',
|
||||
'inlinks-histogram': 'Bucket',
|
||||
'word-count-histogram': 'Bucket',
|
||||
'url-length-histogram': 'Bucket',
|
||||
'word-count-per-dir': 'Directory',
|
||||
'sitemap-orphans': 'URL',
|
||||
'orphan-cross-source': 'URL',
|
||||
'server-headers': 'Server',
|
||||
'top-words': 'Word',
|
||||
};
|
||||
|
||||
const TOP_URL_METRIC: Record<ReportKind, TopUrlMetric | null> = {
|
||||
'pages-per-dir': null,
|
||||
'status-codes': null,
|
||||
|
||||
@@ -11,7 +11,6 @@ interface Props {
|
||||
|
||||
type Cadence = 'hourly' | 'daily' | 'weekly' | 'custom';
|
||||
|
||||
const DAY_NAMES = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
|
||||
const MIN_CUSTOM_INTERVAL = 15;
|
||||
|
||||
interface FormState {
|
||||
|
||||
@@ -65,6 +65,7 @@ import { ExtractionPreviewDialog } from './ExtractionPreviewDialog.js';
|
||||
import { IntegrationSetupGuideModal } from './IntegrationSetupGuideModal.js';
|
||||
import { GscSettingsSection } from './GscSettingsSection.js';
|
||||
import { Ga4SettingsSection } from './Ga4SettingsSection.js';
|
||||
import { McpServersSection } from './McpServersSection.js';
|
||||
import { GoogleAccountsSection } from './GoogleAccountsSection.js';
|
||||
|
||||
interface Props {
|
||||
@@ -127,6 +128,8 @@ interface FormState {
|
||||
checkLinksOutsideStartFolder: boolean;
|
||||
followExternalNofollow: boolean;
|
||||
crawlInvalidLinks: boolean;
|
||||
crawlQueryStrings: boolean;
|
||||
crawlQueryStringExceptionsText: string;
|
||||
crawlLinkedSitemaps: boolean;
|
||||
// requests
|
||||
userAgent: string;
|
||||
@@ -221,6 +224,7 @@ interface FormState {
|
||||
jsMobileUsability: boolean;
|
||||
jsLcpCandidate: boolean;
|
||||
jsA11yAudit: boolean;
|
||||
jsSpaRouting: boolean;
|
||||
// V2 Faz 15 — performance budget
|
||||
budgetEnabled: boolean;
|
||||
budgetMaxResponseMs: string;
|
||||
@@ -260,6 +264,7 @@ export type SettingsSectionKey =
|
||||
| 'performance-budget'
|
||||
| 'storage'
|
||||
| 'spelling'
|
||||
| 'mcp-servers'
|
||||
| 'language'
|
||||
/** Per-integration sub-page, e.g. `integration:gsc`. Each integration
|
||||
* gets its own page under the "Integrations" group header, so a
|
||||
@@ -486,6 +491,13 @@ const SECTIONS: SectionDef[] = [
|
||||
icon: SpellCheck,
|
||||
keywords: 'spelling grammar languagetool dictionary ignore words picky rule level yazım dilbilgisi sözlük',
|
||||
},
|
||||
{
|
||||
key: 'mcp-servers',
|
||||
label: 'MCP Servers',
|
||||
icon: Plug,
|
||||
keywords:
|
||||
'mcp model context protocol server assistant ai tools stdio http agent claude tool permission approval',
|
||||
},
|
||||
{
|
||||
key: 'language',
|
||||
label: 'Language',
|
||||
@@ -541,6 +553,8 @@ function configToForm(c: CrawlConfig): FormState {
|
||||
checkLinksOutsideStartFolder: c.checkLinksOutsideStartFolder ?? true,
|
||||
followExternalNofollow: c.followExternalNofollow ?? false,
|
||||
crawlInvalidLinks: c.crawlInvalidLinks ?? false,
|
||||
crawlQueryStrings: c.crawlQueryStrings ?? true,
|
||||
crawlQueryStringExceptionsText: (c.crawlQueryStringExceptions ?? []).join(', '),
|
||||
crawlLinkedSitemaps: c.crawlLinkedSitemaps ?? false,
|
||||
userAgent: c.userAgent,
|
||||
acceptLanguage: c.acceptLanguage,
|
||||
@@ -635,6 +649,7 @@ function configToForm(c: CrawlConfig): FormState {
|
||||
jsMobileUsability: c.jsRender?.mobileUsability ?? false,
|
||||
jsLcpCandidate: c.jsRender?.lcpCandidate ?? false,
|
||||
jsA11yAudit: c.jsRender?.a11yAudit ?? false,
|
||||
jsSpaRouting: c.jsRender?.spaRouting ?? false,
|
||||
budgetEnabled: c.performanceBudget?.enabled ?? false,
|
||||
budgetMaxResponseMs: String(c.performanceBudget?.maxResponseMs ?? 800),
|
||||
budgetMaxPageKb: String(
|
||||
@@ -659,6 +674,22 @@ function parseHeaders(text: string): Record<string, string> {
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Query-parameter names typed as a single line. Accepts the three
|
||||
* separators a user is likely to reach for — comma, whitespace,
|
||||
* newline — because "page, lang" and "page lang" mean the same thing
|
||||
* and neither should silently produce one parameter named "page, lang".
|
||||
* `?` and `&` are stripped so pasting `?page&lang` works too.
|
||||
*/
|
||||
function parseParamNames(text: string): string[] {
|
||||
const seen = new Set<string>();
|
||||
for (const raw of text.split(/[,\s]+/)) {
|
||||
const name = raw.trim().replace(/^[?&]+/, '').replace(/=.*$/, '');
|
||||
if (name) seen.add(name);
|
||||
}
|
||||
return [...seen];
|
||||
}
|
||||
|
||||
function parseLines(text: string): string[] {
|
||||
return text
|
||||
.split(/\r?\n/)
|
||||
@@ -842,6 +873,8 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
checkLinksOutsideStartFolder: form.checkLinksOutsideStartFolder,
|
||||
followExternalNofollow: form.followExternalNofollow,
|
||||
crawlInvalidLinks: form.crawlInvalidLinks,
|
||||
crawlQueryStrings: form.crawlQueryStrings,
|
||||
crawlQueryStringExceptions: parseParamNames(form.crawlQueryStringExceptionsText),
|
||||
crawlLinkedSitemaps: form.crawlLinkedSitemaps,
|
||||
userAgent: form.userAgent.trim() || config.userAgent,
|
||||
acceptLanguage: form.acceptLanguage.trim() || config.acceptLanguage,
|
||||
@@ -991,6 +1024,7 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
mobileUsability: form.jsMobileUsability,
|
||||
lcpCandidate: form.jsLcpCandidate,
|
||||
a11yAudit: form.jsA11yAudit,
|
||||
spaRouting: form.jsSpaRouting,
|
||||
},
|
||||
performanceBudget: {
|
||||
enabled: form.budgetEnabled,
|
||||
@@ -1151,7 +1185,6 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
if (r.unknownFields.length > 0) {
|
||||
// Surface unknown fields as a non-fatal warning by
|
||||
// logging — Settings UI doesn't have a toast system.
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`Import: ignored unknown fields: ${r.unknownFields.join(', ')}`,
|
||||
);
|
||||
@@ -1228,6 +1261,7 @@ export function SettingsDialog({ open, onClose }: Props) {
|
||||
)}
|
||||
{active === 'storage' && <StoragePanel />}
|
||||
{active === 'spelling' && <SpellingPanel />}
|
||||
{active === 'mcp-servers' && <McpServersSection />}
|
||||
{active === 'language' && <LanguagePanel />}
|
||||
</div>
|
||||
</div>
|
||||
@@ -2114,6 +2148,38 @@ function SpiderCrawlPanel({ form, update }: PanelProps) {
|
||||
info="Record hrefs that cannot be parsed as a URL — unencoded whitespace inside the authority, doubled schemes, stray delimiters. They can never resolve to a crawled page, so every one is reported in Broken Links, which is the point. Deliberate non-navigable schemes (mailto:, tel:, #) are not malformed and never appear."
|
||||
example="On when hunting hand-written markup errors; off keeps Broken Links focused on real 404s."
|
||||
/>
|
||||
<Bool
|
||||
label={t('spiderCrawl.crawlQueryStrings', {
|
||||
defaultValue: 'Crawl URLs with Query Strings',
|
||||
})}
|
||||
checked={form.crawlQueryStrings}
|
||||
onChange={(v) => update('crawlQueryStrings', v)}
|
||||
disabled={form.mode !== 'spider'}
|
||||
hint={
|
||||
form.mode === 'spider'
|
||||
? undefined
|
||||
: t('spiderCrawl.spiderModeOnly', {
|
||||
defaultValue:
|
||||
'Only applies in Spider mode — List and Sitemap crawl the URLs you supplied',
|
||||
})
|
||||
}
|
||||
info="Off drops every discovered URL carrying a `?`, before robots and before a request goes out. That is the cheap way to stop a faceted navigation (?color=red&size=xl&sort=price) from spending the whole URL budget on one product listing wearing a thousand URLs. The start URL is always crawled, and subresources are exempt — style.css?v=7 is a cache-buster, not a facet. Skipped URLs are counted and reported in the log, never dropped silently."
|
||||
example="On (default). Off for a first pass over a shop with faceted filters."
|
||||
/>
|
||||
{!form.crawlQueryStrings && form.mode === 'spider' && (
|
||||
<div className="ml-6">
|
||||
<Text
|
||||
label={t('spiderCrawl.queryStringExceptions', {
|
||||
defaultValue: 'Except these parameters',
|
||||
})}
|
||||
value={form.crawlQueryStringExceptionsText}
|
||||
onChange={(v) => update('crawlQueryStringExceptionsText', v)}
|
||||
placeholder="page, lang, p"
|
||||
info="Parameter names that keep a URL in the crawl anyway — pagination, a language switch, a product id. Names only; values are not looked at, and matching ignores case. A URL is admitted only when every parameter it carries is on this list: ?page=2 passes, ?page=2&color=red does not. Any-match would defeat the point, since a facet URL nearly always carries the pagination parameter too."
|
||||
example="page, lang — keeps paginated archives reachable while the facets stay out."
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</CrawlGroup>
|
||||
|
||||
<CrawlGroup title={t('spiderCrawl.xmlSitemaps', { defaultValue: 'XML Sitemaps' })}>
|
||||
@@ -2710,7 +2776,6 @@ function CustomExtractionPanel({ form, update }: PanelProps) {
|
||||
// Surface main-process parse errors inline rather than
|
||||
// silently dropping — invalid JSON is the most common
|
||||
// import failure mode and the user needs to know.
|
||||
// eslint-disable-next-line no-alert
|
||||
alert(
|
||||
t('settingsPanels.customExtraction.importFailedAlert', {
|
||||
defaultValue: 'Import failed: {{error}}',
|
||||
@@ -2721,7 +2786,6 @@ function CustomExtractionPanel({ form, update }: PanelProps) {
|
||||
return;
|
||||
}
|
||||
if (result.rules.length === 0) {
|
||||
// eslint-disable-next-line no-alert
|
||||
alert(
|
||||
result.error ??
|
||||
t('settingsPanels.customExtraction.noValidRules', {
|
||||
@@ -2731,7 +2795,6 @@ function CustomExtractionPanel({ form, update }: PanelProps) {
|
||||
return;
|
||||
}
|
||||
if (rules.length > 0) {
|
||||
// eslint-disable-next-line no-alert, no-restricted-globals
|
||||
const ok = confirm(
|
||||
t('settingsPanels.customExtraction.importConfirm', {
|
||||
defaultValue:
|
||||
@@ -2744,7 +2807,6 @@ function CustomExtractionPanel({ form, update }: PanelProps) {
|
||||
}
|
||||
setRules(result.rules);
|
||||
if (result.skippedCount > 0) {
|
||||
// eslint-disable-next-line no-alert
|
||||
alert(
|
||||
t('settingsPanels.customExtraction.importSkipped', {
|
||||
defaultValue:
|
||||
@@ -4474,6 +4536,14 @@ function RenderingPanel({ form, update }: PanelProps) {
|
||||
info="Audits the rendered DOM for WCAG AA colour-contrast failures (4.5:1 normal text, 3:1 large text) and stylesheet rules that suppress the keyboard focus outline without a :focus-visible fallback. Surfaces the Low-Contrast Text and Focus Outline Suppressed issue filters."
|
||||
example="On for accessibility / WCAG audits."
|
||||
/>
|
||||
<Bool
|
||||
label={t('settingsPanels.rendering.spaRouting', { defaultValue: 'SPA route discovery' })}
|
||||
checked={form.jsSpaRouting}
|
||||
onChange={(v) => update('jsSpaRouting', v)}
|
||||
hint=""
|
||||
info="Hooks the History API before the page's own scripts run, so routes an SPA reaches via pushState / replaceState / popstate are discovered and crawled. Also keeps hash routes (#/about) as distinct URLs instead of collapsing them onto the shell document."
|
||||
example="On for React Router / Vue Router / Angular sites whose pages never produce a document request."
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
|
||||
@@ -12,7 +12,12 @@
|
||||
* degradation), which is also what happens to technical example values
|
||||
* — regexes, header values, URLs — that are deliberately left untranslated.
|
||||
*
|
||||
* GENERATED FILE — do not hand-edit. Regenerate when tooltip copy changes.
|
||||
* Hand-maintained: add the entry in the same commit as the tooltip.
|
||||
* `tests/settings-infotip-i18n.test.ts` enforces that for both places
|
||||
* tooltips are authored — SettingsDialog.tsx and tabs/columns.ts — and a
|
||||
* new prose tooltip with no Turkish entry fails the suite by file and
|
||||
* line. Technical sample values need no entry; the test tells a sentence
|
||||
* apart from a value list on its own.
|
||||
*/
|
||||
|
||||
const TR_INFO_TIPS: Record<string, string> = {
|
||||
@@ -528,6 +533,160 @@ const TR_INFO_TIPS: Record<string, string> = {
|
||||
"Bu URL'den başlayan yönlendirme zinciri, döngüye dayanıklı yürütecin tespit ettiği bir çevrim (A → B → A) içeriyorsa Y olur; aksi halde zincir yürünmemiştir.",
|
||||
["Y when this URL belongs to a paginated cluster whose ordinal sequence has a gap (e.g. ?page=1, 2, 4 — page 3 missing). Set by the post-crawl `recomputePaginationSequence` pass."]:
|
||||
"Bu URL, sıra numaralarında boşluk olan bir sayfalama kümesine aitse Y olur (ör. ?page=1, 2, 4 — 3. sayfa eksik). Crawl sonrası `recomputePaginationSequence` geçişi tarafından ayarlanır.",
|
||||
["SQL injection — the request tries to smuggle SQL into a parameter (UNION SELECT, sleep(), error-based functions) to read or alter your database."]:
|
||||
"SQL enjeksiyonu — istek, veritabanınızı okumak veya değiştirmek için bir parametrenin içine SQL kaçırmaya çalışıyor (UNION SELECT, sleep(), hata tabanlı fonksiyonlar).",
|
||||
["Cross-site scripting — the request carries script markup or a javascript: URL in a parameter, hoping the page echoes it back into the HTML unescaped."]:
|
||||
"Siteler arası betik çalıştırma (XSS) — istek, bir parametrede script işaretlemesi ya da javascript: URL'i taşıyor; sayfanın bunu kaçışsız biçimde HTML'e geri yazmasını umuyor.",
|
||||
["Path traversal — the request walks out of the web root with ../ or encoded variants to reach files like /etc/passwd or win.ini."]:
|
||||
"Dizin aşımı — istek, /etc/passwd veya win.ini gibi dosyalara ulaşmak için ../ ya da kodlanmış varyantlarıyla web kökünün dışına çıkıyor.",
|
||||
["Command injection — the request appends shell syntax (;, |, backticks, $( )) to a parameter to run commands on the server."]:
|
||||
"Komut enjeksiyonu — istek, sunucuda komut çalıştırmak için bir parametreye kabuk sözdizimi ekliyor (;, |, ters tırnak, $( )).",
|
||||
["Scanner probe — an automated vulnerability scanner walking a wordlist of known admin panels, installers and exploit paths (wp-login, phpmyadmin, /actuator, shell uploads). Not tailored to your site; it hits everyone."]:
|
||||
"Tarayıcı sondası — otomatik bir zafiyet tarayıcısı, bilinen yönetim panelleri, kurulum dosyaları ve istismar yollarından oluşan bir listeyi deniyor (wp-login, phpmyadmin, /actuator, shell yüklemeleri). Sitenize özel değildir; herkese aynı listeyi dener.",
|
||||
["Sensitive file fetch — a direct request for something that must never be public: .env, .git, backups, SQL dumps, private keys, config files."]:
|
||||
"Hassas dosya isteği — asla herkese açık olmaması gereken bir şeye doğrudan istek: .env, .git, yedekler, SQL dump'ları, özel anahtarlar, yapılandırma dosyaları.",
|
||||
["Anomaly — malformed or evasive input (null bytes, CRLF injection, over-encoding, absurd parameter lengths) that matches no single attack class but is not a normal browser request."]:
|
||||
"Anormallik — tek bir saldırı sınıfına uymayan ama normal bir tarayıcı isteği de olmayan bozuk ya da atlatma amaçlı girdi (null bayt, CRLF enjeksiyonu, aşırı kodlama, absürt parametre uzunlukları).",
|
||||
["Sum of the weights of every attack signature the request matched. Each signature carries a weight by how conclusive it is (a UNION SELECT weighs 9, a stray quote 2), and a line is only flagged once the total reaches 5 — so one decisive pattern flags on its own, while weak hints have to add up. Higher score = less room for a false positive; sort by it to triage."]:
|
||||
"İsteğin eşleştiği tüm saldırı imzalarının ağırlıklarının toplamı. Her imza ne kadar kesin olduğuna göre bir ağırlık taşır (UNION SELECT 9, başıboş bir tırnak 2 eder) ve bir satır ancak toplam 5'e ulaşınca işaretlenir — yani tek başına belirleyici bir desen yeterken, zayıf ipuçlarının birikmesi gerekir. Skor yükseldikçe yanlış pozitif ihtimali azalır; önceliklendirmek için bu sütuna göre sıralayın.",
|
||||
["Which attack class the strongest matching signature belongs to: SQL injection, XSS, path traversal, command injection, scanner probe, sensitive file, or anomaly. Hover any badge in this column for what that class means in practice."]:
|
||||
"En güçlü eşleşen imzanın hangi saldırı sınıfına ait olduğu: SQL enjeksiyonu, XSS, dizin aşımı, komut enjeksiyonu, tarayıcı sondası, hassas dosya ya da anormallik. Bu sütundaki herhangi bir rozetin üzerine gelerek o sınıfın pratikte ne anlama geldiğini görebilirsiniz.",
|
||||
["Filters on the Status column — the most recent response the log recorded for that path. The analyzer keeps one status per URL rather than a full distribution, so this answers 'what is this URL returning now'. Paths whose status could not be parsed are hidden while a class is selected."]:
|
||||
"Durum sütununa göre filtreler — log'un o yol için kaydettiği en son yanıt. Analiz aracı URL başına tam bir dağılım yerine tek bir durum tutar, dolayısıyla bu filtre “bu URL şu anda ne dönüyor” sorusunu yanıtlar. Bir sınıf seçiliyken durumu ayrıştırılamamış yollar gizlenir.",
|
||||
["Most recent HTTP status the log recorded for this path. One value per URL, not a distribution — a path that returned 200 all week and 404 this morning shows 404."]:
|
||||
"Log'un bu yol için kaydettiği en son HTTP durum kodu. URL başına tek değer tutulur, dağılım değil — bütün hafta 200 dönüp bu sabah 404 dönen bir yol 404 görünür.",
|
||||
["A URL whose path repeats the same segment this many times or more (/shop/shop/shop/…) is treated as a link loop and skipped. This shape comes from a relative-href bug and has no legitimate counterpart. Skipped counts are reported when the crawl finishes."]:
|
||||
"Yolu aynı segmenti bu kadar veya daha fazla kez tekrarlayan bir URL (/shop/shop/shop/…) bağlantı döngüsü sayılır ve atlanır. Bu şekil göreli href hatasından doğar ve meşru bir karşılığı yoktur. Atlanan URL sayısı crawl bittiğinde raporlanır.",
|
||||
["3 is safe for every site; raise to 4–5 only if a real path legitimately repeats a segment; 0 disables the guard."]:
|
||||
"3 her site için güvenlidir; gerçek bir yol meşru olarak bir segmenti tekrarlıyorsa 4–5'e çıkarın; 0 korumayı kapatır.",
|
||||
["URLs with more query parameters than this are flagged as faceted-navigation traps under Issues → URL → Crawl Trap. Detection only — the URLs are still crawled, because legitimate filter pages look the same."]:
|
||||
"Bundan fazla sorgu parametresi taşıyan URL'ler Sorunlar → URL → Crawl Tuzağı altında fasetli navigasyon tuzağı olarak işaretlenir. Yalnızca tespit — URL'ler yine de taranır, çünkü meşru filtre sayfaları da aynı görünür.",
|
||||
["4 surfaces most faceted-nav explosions; 0 disables the check."]:
|
||||
"4 çoğu fasetli navigasyon patlamasını yakalar; 0 kontrolü kapatır.",
|
||||
["Honor Allow / Disallow rules declared in /robots.txt for the configured User-Agent. Screaming Frog: 'Respect robots.txt' (Configuration → robots.txt)."]:
|
||||
"Yapılandırılan User-Agent için /robots.txt'te bildirilen Allow / Disallow kurallarına uyar. Screaming Frog'daki karşılığı: 'Respect robots.txt' (Configuration → robots.txt).",
|
||||
["Honor a Crawl-delay directive as a global rate limit (one request every N seconds). Crawl-delay is not part of RFC 9309 — Google ignores it and Screaming Frog does not implement it — and published values are often stale: 'Crawl-delay: 30' turns a 500-URL crawl into hours. Ignored by default; the directive is still reported in the log when found."]:
|
||||
"Crawl-delay direktifini genel bir hız sınırı olarak uygular (N saniyede bir istek). Crawl-delay RFC 9309'un parçası değildir — Google yok sayar, Screaming Frog uygulamaz — ve yayımlanan değerler çoğu zaman eskimiştir: 'Crawl-delay: 30' 500 URL'lik bir taramayı saatlere çıkarır. Varsayılan olarak yok sayılır; direktif bulunduğunda yine de log'a yazılır.",
|
||||
["Off (default) for normal audits. On when an ops policy requires it — expect the crawl to take Crawl-delay seconds per URL."]:
|
||||
"Normal denetimler için kapalı (varsayılan). Bir operasyon politikası gerektiriyorsa açın — taramanın URL başına Crawl-delay saniye süreceğini hesaba katın.",
|
||||
["Crawl fetches internal <img> targets (incl. srcset / <picture> sources) so each appears in the Internal tab with status, content type, and size — every one counts toward Max URLs. Store keeps the <img> declarations in the Images tab, which works even with Crawl off: you get the full image inventory with alt text for the cost of zero extra requests."]:
|
||||
"Tara, dahili <img> hedeflerini (srcset / <picture> kaynakları dahil) çeker; her biri Internal sekmesinde durum, içerik tipi ve boyutuyla görünür — ve her biri Maks URL kotasından düşer. Sakla, <img> bildirimlerini Images sekmesinde tutar ve Tara kapalıyken de çalışır: sıfır ek istek karşılığında alt metinleriyle birlikte tam görsel envanterini alırsınız.",
|
||||
["Store on, Crawl off is the cheap alt-text audit. Both on for a full image health check."]:
|
||||
"Sakla açık, Tara kapalı — ucuz alt metin denetimi. Tam bir görsel sağlık kontrolü için ikisi de açık.",
|
||||
["<video> / <audio> and the <source> children they own. Off by default — media files are large and rarely what an SEO crawl is looking for."]:
|
||||
"<video> / <audio> ve bunlara ait <source> öğeleri. Varsayılan olarak kapalı — medya dosyaları büyüktür ve bir SEO taramasının aradığı şey nadiren budur.",
|
||||
["On when auditing a video-heavy site for dead media URLs."]:
|
||||
"Video ağırlıklı bir sitede ölü medya URL'lerini denetlerken açın.",
|
||||
["<link rel=stylesheet> targets. Crawling a stylesheet is also what discovers the web fonts and background images declared inside it via @font-face / url() — so Crawl on with Store off still populates the Internal tab's Font filter without listing every stylesheet."]:
|
||||
"<link rel=stylesheet> hedefleri. Bir stylesheet'i taramak, içinde @font-face / url() ile bildirilen web fontlarını ve arka plan görsellerini de keşfeden şeydir — yani Tara açık, Sakla kapalı, her stylesheet'i listelemeden Internal sekmesindeki Font filtresini yine de doldurur.",
|
||||
["Crawl on, Store off when you want fonts discovered but not hundreds of CSS rows."]:
|
||||
"Fontların keşfedilmesini isteyip yüzlerce CSS satırı istemiyorsanız Tara açık, Sakla kapalı.",
|
||||
["<script src> targets, fetched so each gets its own row with status code, content type, and size. Headers only — the body is discarded, never executed."]:
|
||||
"<script src> hedefleri; her biri kendi satırını durum kodu, içerik tipi ve boyutuyla alsın diye çekilir. Yalnızca başlıklar — gövde atılır, asla çalıştırılmaz.",
|
||||
["Both on to catch 404ing bundles; both off for HTML-only crawls."]:
|
||||
"404 dönen bundle'ları yakalamak için ikisi de açık; yalnızca HTML taramaları için ikisi de kapalı.",
|
||||
["<a href> targets on the same site. Crawl off turns the run into an audit of a fixed set of pages — sitemaps, canonicals, and the other declared alternates below still feed discovery. Store off empties the link graph: inlinks, outlinks, anchor-text reports, and link score all go with it."]:
|
||||
"Aynı sitedeki <a href> hedefleri. Tara kapalıyken çalışma, sabit bir sayfa kümesinin denetimine dönüşür — sitemap'ler, canonical'lar ve aşağıdaki diğer bildirilen alternatifler keşfi beslemeye devam eder. Sakla kapalıyken bağlantı grafiği boşalır: inlink'ler, outlink'ler, anchor metni raporları ve link skoru da onunla birlikte gider.",
|
||||
["Leave both on. Crawl off only when a sitemap or URL list already defines the exact set you want."]:
|
||||
"İkisini de açık bırakın. Yalnızca bir sitemap veya URL listesi istediğiniz kümeyi zaten tanımlıyorsa Tara'yı kapatın.",
|
||||
["Outbound links to other hosts are always status-checked (one HEAD each) so Broken Links catches dead externals — that does not depend on this row. Crawl here means fully crawling those pages, following their links onward too. Store keeps outbound links in the link graph."]:
|
||||
"Diğer host'lara giden bağlantıların durumu her zaman kontrol edilir (her biri için bir HEAD), böylece Bozuk Bağlantılar ölü dış bağlantıları yakalar — bu, bu satıra bağlı değildir. Buradaki Tara, o sayfaları tümüyle taramak ve onların bağlantılarını da izlemek demektir. Sakla, giden bağlantıları bağlantı grafiğinde tutar.",
|
||||
["Crawl off (default) — status-check externals without spidering the whole web."]:
|
||||
"Tara kapalı (varsayılan) — tüm web'i taramadan dış bağlantıların durumunu kontrol edin.",
|
||||
["<link rel=canonical> and its HTTP Link: header form. Crawl also enqueues the canonical target, treating it as a navigation hint. Store feeds the Canonicals tab and every canonical issue filter."]:
|
||||
"<link rel=canonical> ve onun HTTP Link: başlığı biçimi. Tara ayrıca canonical hedefini de kuyruğa alır, yani onu bir navigasyon ipucu gibi değerlendirir. Sakla, Canonical'lar sekmesini ve tüm canonical sorun filtrelerini besler.",
|
||||
["Crawl off (default) — canonicals are a signal, not a route. Store on."]:
|
||||
"Tara kapalı (varsayılan) — canonical bir sinyaldir, bir rota değil. Sakla açık.",
|
||||
["<link rel=next> / <link rel=prev>. Part of the standard discovery graph; turn Crawl off to isolate a pagination loop without disabling link-following everywhere."]:
|
||||
"<link rel=next> / <link rel=prev>. Standart keşif grafiğinin parçasıdır; her yerde bağlantı takibini kapatmadan bir sayfalama döngüsünü izole etmek için Tara'yı kapatın.",
|
||||
["Both on unless you are debugging an infinite paginated series."]:
|
||||
"Sonsuz bir sayfalama serisinde hata ayıklamıyorsanız ikisi de açık.",
|
||||
["<link rel=alternate hreflang>. Crawl enqueues every declared alternate, which is how you reach language versions nothing links to. Store feeds the Hreflang tab and the reciprocity / invalid-code audits."]:
|
||||
"<link rel=alternate hreflang>. Tara, bildirilen her alternatifi kuyruğa alır; hiçbir yerden link almayan dil sürümlerine böyle ulaşırsınız. Sakla, Hreflang sekmesini ve karşılıklılık / geçersiz kod denetimlerini besler.",
|
||||
["Crawl on for a multi-language audit — otherwise unlinked locales stay invisible."]:
|
||||
"Çok dilli bir denetim için Tara açık — aksi halde link almayan yerelleştirmeler görünmez kalır.",
|
||||
["<link rel=amphtml>. Crawl fetches the AMP variant as its own URL; Store keeps the declaration plus the AMP smoke-validator findings."]:
|
||||
"<link rel=amphtml>. Tara, AMP sürümünü kendi URL'i olarak çeker; Sakla, bildirimi ve AMP smoke doğrulayıcısının bulgularını tutar.",
|
||||
["Crawl on only if the site still ships AMP pages."]:
|
||||
"Yalnızca site hâlâ AMP sayfası yayınlıyorsa Tara açık.",
|
||||
["<meta http-equiv=\"refresh\">. Crawl enqueues the parsed target like a redirect; Store keeps the raw directive and its URL for the Meta Refresh tab."]:
|
||||
"<meta http-equiv=\"refresh\">. Tara, ayrıştırılan hedefi bir yönlendirme gibi kuyruğa alır; Sakla, ham direktifi ve URL'ini Meta Refresh sekmesi için tutar.",
|
||||
["Crawl on when auditing a legacy site that still redirects this way."]:
|
||||
"Hâlâ bu şekilde yönlendiren eski bir siteyi denetlerken Tara açık.",
|
||||
["<iframe src> documents. Crawl fetches each embedded page as its own URL, which can pull in a lot of third-party surface. Store records them in the link graph so a dead embed shows up in Outlinks and Broken Links — without counting toward the page's outlink total, since an embed is not a hyperlink."]:
|
||||
"<iframe src> dokümanları. Tara, gömülü her sayfayı kendi URL'i olarak çeker; bu da epeyce üçüncü taraf yüzeyini içeri çekebilir. Sakla, bunları bağlantı grafiğine yazar, böylece ölü bir gömme Outlink'lerde ve Bozuk Bağlantılar'da görünür — ama sayfanın outlink toplamına dahil edilmez, çünkü bir gömme hyperlink değildir.",
|
||||
["Store on, Crawl off is usually the right pair."]:
|
||||
"Genellikle doğru ikili şudur: Sakla açık, Tara kapalı.",
|
||||
["The separate-URL (m-dot) mobile version: <link rel=\"alternate\" media=\"only screen and (max-width: …)\">. Null on responsive sites, which is most of them — a value here with no reciprocal canonical back is the classic broken m-dot setup."]:
|
||||
"Ayrı URL'li (m-dot) mobil sürüm: <link rel=\"alternate\" media=\"only screen and (max-width: …)\">. Responsive sitelerde — ki çoğu öyledir — boştur; burada bir değer olup geri işaret eden karşılıklı canonical yoksa bu, klasik bozuk m-dot kurulumudur.",
|
||||
["Crawl on only when the site really does serve a separate mobile host."]:
|
||||
"Yalnızca site gerçekten ayrı bir mobil host sunuyorsa Tara açık.",
|
||||
["Links a search engine cannot follow: <a> with no href but an onclick, href=\"javascript:…\", and href=\"#\" placeholders wired to a handler. Store-only — an uncrawlable link is by definition never fetched. Drives the JS-Only Navigation issue filter."]:
|
||||
"Bir arama motorunun izleyemeyeceği bağlantılar: href'i olmayıp onclick taşıyan <a>'lar, href=\"javascript:…\" ve bir handler'a bağlanmış href=\"#\" yer tutucuları. Yalnızca Sakla — taranamayan bir bağlantı tanımı gereği hiç çekilmez. JS-Only Navigasyon sorun filtresini besler.",
|
||||
["On — it is a count, so it costs nothing."]:
|
||||
"Açık — bu bir sayımdır, hiçbir maliyeti yoktur.",
|
||||
["With a Subfolder-scoped crawl, links pointing outside the start folder are fetched once so their status code is known, then stopped — they are checked, not crawled through. Off leaves them undiscovered entirely."]:
|
||||
"Alt klasör kapsamlı bir taramada, başlangıç klasörünün dışını gösteren bağlantılar durum kodları bilinsin diye bir kez çekilir, sonra durulur — kontrol edilirler, içlerinden geçilerek taranmazlar. Kapalıyken tamamen keşfedilmeden kalırlar.",
|
||||
["On — knowing a link out of /blog/ is a 404 costs one request."]:
|
||||
"Açık — /blog/ dışına çıkan bir bağlantının 404 olduğunu bilmek bir isteğe mal olur.",
|
||||
["Off restricts the crawl to URLs under the start URL's path (Crawl Scope = Subfolder). On lets it cover the whole host. This is a view of the Crawl Scope setting, not a separate switch, so the two can never disagree."]:
|
||||
"Kapalıyken tarama, başlangıç URL'inin yolu altındaki URL'lerle sınırlanır (Tarama Kapsamı = Alt Klasör). Açıkken tüm host'u kapsar. Bu ayrı bir anahtar değil, Tarama Kapsamı ayarının bir görünümüdür; dolayısıyla ikisi asla çelişemez.",
|
||||
["Off to audit just /blog/; on for the whole site."]:
|
||||
"Yalnızca /blog/'u denetlemek için kapalı; tüm site için açık.",
|
||||
["Treats every host sharing the registrable domain as internal — shop.example.com and blog.example.com crawl alongside example.com instead of counting as external. Another view of the Crawl Scope setting."]:
|
||||
"Kayıtlanabilir alan adını paylaşan her host'u dahili sayar — shop.example.com ve blog.example.com harici sayılmak yerine example.com ile birlikte taranır. Tarama Kapsamı ayarının bir başka görünümü.",
|
||||
["On when subdomains are part of the same property."]:
|
||||
"Alt alan adları aynı mülkün parçasıysa açık.",
|
||||
["Crawl through rel=\"nofollow\" links pointing at the same site. Off (default) is Screaming Frog \"Respect Nofollow\" behaviour. Internal and external are separate switches because sites nofollow them for opposite reasons — crawl-budget shaping vs. not vouching for a third party."]:
|
||||
"Aynı siteyi gösteren rel=\"nofollow\" bağlantıların içinden geçerek tarar. Kapalı (varsayılan), Screaming Frog'un \"Respect Nofollow\" davranışıdır. Dahili ve harici ayrı anahtarlardır, çünkü siteler bunlara zıt nedenlerle nofollow koyar — crawl bütçesini şekillendirmek ile üçüncü bir tarafa kefil olmamak.",
|
||||
["On when a site nofollows its own faceted navigation and you need behind it."]:
|
||||
"Bir site kendi fasetli navigasyonuna nofollow koyuyorsa ve arkasına geçmeniz gerekiyorsa açın.",
|
||||
["Crawl through rel=\"nofollow\" links pointing at other hosts. Only has an effect while External Links → Crawl is on."]:
|
||||
"Başka host'ları gösteren rel=\"nofollow\" bağlantıların içinden geçerek tarar. Yalnızca Harici Bağlantılar → Tara açıkken etkilidir.",
|
||||
["Off — nofollowed externals are exactly the ones you did not vouch for."]:
|
||||
"Kapalı — nofollow'lu dış bağlantılar tam da kefil olmadıklarınızdır.",
|
||||
["Record hrefs that cannot be parsed as a URL — unencoded whitespace inside the authority, doubled schemes, stray delimiters. They can never resolve to a crawled page, so every one is reported in Broken Links, which is the point. Deliberate non-navigable schemes (mailto:, tel:, #) are not malformed and never appear."]:
|
||||
"URL olarak ayrıştırılamayan href'leri kaydeder — authority içinde kodlanmamış boşluk, çiftlenmiş şema, başıboş ayraç. Bunlar hiçbir zaman taranan bir sayfaya çözümlenemez, bu yüzden her biri Bozuk Bağlantılar'da raporlanır; amaç da budur. Kasıtlı olarak navigasyon dışı şemalar (mailto:, tel:, #) bozuk sayılmaz ve hiç görünmez.",
|
||||
["On when hunting hand-written markup errors; off keeps Broken Links focused on real 404s."]:
|
||||
"Elle yazılmış işaretleme hatalarını ararken açın; kapalı tutmak Bozuk Bağlantılar'ı gerçek 404'lere odaklı tutar.",
|
||||
["Off drops every discovered URL carrying a `?`, before robots and before a request goes out. That is the cheap way to stop a faceted navigation (?color=red&size=xl&sort=price) from spending the whole URL budget on one product listing wearing a thousand URLs. The start URL is always crawled, and subresources are exempt — style.css?v=7 is a cache-buster, not a facet. Skipped URLs are counted and reported in the log, never dropped silently."]:
|
||||
"Kapalıyken, `?` taşıyan her keşfedilen URL robots kontrolünden ve istek gönderilmeden önce düşürülür. Fasetli bir navigasyonun (?color=red&size=xl&sort=price) tüm URL bütçesini, bin URL kılığına girmiş tek bir ürün listesine harcamasını engellemenin ucuz yolu budur. Başlangıç URL'i her zaman taranır ve alt kaynaklar muaftır — style.css?v=7 bir cache-buster'dır, faset değil. Atlanan URL'ler sayılır ve log'a yazılır, sessizce düşürülmez.",
|
||||
["On (default). Off for a first pass over a shop with faceted filters."]:
|
||||
"Açık (varsayılan). Fasetli filtreleri olan bir mağazada ilk geçiş için kapalı.",
|
||||
["Parameter names that keep a URL in the crawl anyway — pagination, a language switch, a product id. Names only; values are not looked at, and matching ignores case. A URL is admitted only when every parameter it carries is on this list: ?page=2 passes, ?page=2&color=red does not. Any-match would defeat the point, since a facet URL nearly always carries the pagination parameter too."]:
|
||||
"Bir URL'i yine de tarama içinde tutan parametre adları — sayfalama, dil değiştirici, ürün id'si. Yalnızca adlar; değerlere bakılmaz ve eşleşmede büyük/küçük harf önemsizdir. Bir URL ancak taşıdığı her parametre bu listedeyse kabul edilir: ?page=2 geçer, ?page=2&color=red geçmez. Herhangi-biri eşleşmesi amacı boşa çıkarırdı, çünkü bir faset URL'i neredeyse her zaman sayfalama parametresini de taşır.",
|
||||
["page, lang — keeps paginated archives reachable while the facets stay out."]:
|
||||
"page, lang — fasetler dışarıda kalırken sayfalanmış arşivler erişilebilir kalır.",
|
||||
["Auto-discovery on its own only records sitemap entries, which is what the sitemap issue filters compare the crawl against. Turning this on crawls them too — and that is what surfaces orphans: pages the sitemap declares but nothing on the site links to."]:
|
||||
"Otomatik keşif tek başına yalnızca sitemap kayıtlarını yazar; sitemap sorun filtreleri taramayı bununla karşılaştırır. Bunu açmak onları taramaya da başlar — orphan'ları ortaya çıkaran şey budur: sitemap'in bildirdiği ama sitede hiçbir şeyin link vermediği sayfalar.",
|
||||
["On for an orphan-page audit."]:
|
||||
"Orphan sayfa denetimi için açık.",
|
||||
["Reads Sitemap: directives from /robots.txt plus the conventional /sitemap.xml fallbacks at crawl start. Cheap I/O, and it powers every sitemap issue filter."]:
|
||||
"Tarama başlangıcında /robots.txt'teki Sitemap: direktiflerini ve alışılmış /sitemap.xml yedeklerini okur. Ucuz bir I/O'dur ve her sitemap sorun filtresine güç verir.",
|
||||
["On (default)."]:
|
||||
"Açık (varsayılan).",
|
||||
["Explicit sitemap URLs, one per line. Their entries are always both recorded and queued as crawl seeds — use this when the sitemap lives somewhere robots.txt never mentions."]:
|
||||
"Açıkça belirtilen sitemap URL'leri, satır başına bir tane. Kayıtları her zaman hem yazılır hem de tarama tohumu olarak kuyruğa alınır — sitemap, robots.txt'in hiç bahsetmediği bir yerde duruyorsa bunu kullanın.",
|
||||
["Treat the concurrency and RPS above as a ceiling and let the target server set the real pace. On a 429/503 (or a Retry-After header) the crawler pauses for the penalty window and steps the rate + concurrency down; after a sustained run of clean responses it grows them back toward the ceiling. Off = hold the configured rate no matter how the server responds."]:
|
||||
"Yukarıdaki eşzamanlılık ve RPS değerlerini bir tavan olarak alır ve gerçek hızı hedef sunucunun belirlemesine izin verir. Bir 429/503'te (ya da Retry-After başlığında) crawler ceza penceresi boyunca duraklar, hızı ve eşzamanlılığı kademeli olarak düşürür; temiz yanıtlardan oluşan sürekli bir seriden sonra ikisini de tavana doğru geri büyütür. Kapalı = sunucu ne yanıt verirse versin yapılandırılan hızı koru.",
|
||||
["Turn on for sites behind Cloudflare / a WAF that returns 429s; leave off for your own infrastructure where the fixed rate is safe."]:
|
||||
"Cloudflare / WAF arkasında olup 429 dönen siteler için açın; sabit hızın güvenli olduğu kendi altyapınızda kapalı bırakın.",
|
||||
["Sorts query parameters alphabetically at normalisation time. Repeated keys keep their relative order, so ?tag=a&tag=b is preserved. Without this the two orderings occupy separate rows and read as duplicates."]:
|
||||
"Normalleştirme sırasında sorgu parametrelerini alfabetik sıralar. Tekrarlanan anahtarlar göreli sıralarını korur, yani ?tag=a&tag=b bozulmaz. Bu olmadan iki farklı sıralama ayrı satırlar işgal eder ve yinelenen gibi görünür.",
|
||||
["On for most sites; off if your server routes on positional parameter order."]:
|
||||
"Çoğu site için açık; sunucunuz parametre sırasına göre yönlendirme yapıyorsa kapalı.",
|
||||
["Collapses runs of slashes in the path to a single slash. Applied before the trailing-slash policy. Web servers serve these identically, so the duplicate-slash variant is normally a false duplicate."]:
|
||||
"Yoldaki ardışık eğik çizgileri tek bir eğik çizgiye indirger. Sondaki eğik çizgi politikasından önce uygulanır. Web sunucuları bunları aynı şekilde sunar, dolayısıyla çift eğik çizgili varyant normalde sahte bir yinelenendir.",
|
||||
["On if a template bug emits // in links; off if your framework uses empty path segments as data."]:
|
||||
"Bir şablon hatası bağlantılarda // üretiyorsa açık; framework'ünüz boş yol segmentlerini veri olarak kullanıyorsa kapalı.",
|
||||
["Off by default: verify the login page's TLS certificate before typing credentials into it. Enable only for a trusted internal host with a self-signed certificate — an unverifiable certificate on a login page is a man-in-the-middle risk."]:
|
||||
"Varsayılan olarak kapalı: kimlik bilgilerini yazmadan önce giriş sayfasının TLS sertifikası doğrulanır. Yalnızca kendinden imzalı sertifikası olan, güvendiğiniz bir iç host için açın — bir giriş sayfasında doğrulanamayan sertifika, ortadaki adam (MITM) riskidir.",
|
||||
["Hooks the History API before the page's own scripts run, so routes an SPA reaches via pushState / replaceState / popstate are discovered and crawled. Also keeps hash routes (#/about) as distinct URLs instead of collapsing them onto the shell document."]:
|
||||
"Sayfanın kendi script'leri çalışmadan önce History API'ye kanca takar; böylece bir SPA'nın pushState / replaceState / popstate ile ulaştığı rotalar keşfedilir ve taranır. Ayrıca hash rotalarını (#/about) kabuk dokümana indirgemek yerine ayrı URL'ler olarak korur.",
|
||||
["On for React Router / Vue Router / Angular sites whose pages never produce a document request."]:
|
||||
"Sayfaları hiçbir zaman bir doküman isteği üretmeyen React Router / Vue Router / Angular siteleri için açık.",
|
||||
["`<link rel=\"alternate\" media=\"only screen and (max-width: …)\" href=\"…\">` value — the separate-URL (m-dot) mobile version of this page. Empty on responsive sites, which is most of them. A value here with no reciprocal canonical pointing back is the classic broken m-dot setup."]:
|
||||
"`<link rel=\"alternate\" media=\"only screen and (max-width: …)\" href=\"…\">` değeri — bu sayfanın ayrı URL'li (m-dot) mobil sürümü. Responsive sitelerde — ki çoğu öyledir — boştur. Burada bir değer olup geri işaret eden karşılıklı bir canonical yoksa, bu klasik bozuk m-dot kurulumudur.",
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -670,6 +670,8 @@ const TR_LABELS: Record<string, string> = {
|
||||
Integrations: 'Entegrasyonlar',
|
||||
Rendering: 'Render',
|
||||
Storage: 'Depolama',
|
||||
'MCP Servers': 'MCP Sunucuları',
|
||||
Assistant: 'Asistan',
|
||||
Privacy: 'Gizlilik',
|
||||
Language: 'Dil',
|
||||
// Status column — robots-blocked badge + its tooltip.
|
||||
|
||||
@@ -239,7 +239,8 @@
|
||||
"analytics": "GA4",
|
||||
"ai": "AI",
|
||||
"seo": "SEO Authority",
|
||||
"visualization": "Visualization"
|
||||
"visualization": "Visualization",
|
||||
"assistant": "Assistant"
|
||||
},
|
||||
"visualization": {
|
||||
"windowTitle": "Crawl Visualization",
|
||||
@@ -421,6 +422,9 @@
|
||||
"followInternalNofollow": "Follow Internal \"nofollow\"",
|
||||
"followExternalNofollow": "Follow External \"nofollow\"",
|
||||
"crawlInvalidLinks": "Crawl Invalid Links",
|
||||
"crawlQueryStrings": "Crawl URLs with Query Strings",
|
||||
"queryStringExceptions": "Except these parameters",
|
||||
"spiderModeOnly": "Only applies in Spider mode — List and Sitemap crawl the URLs you supplied",
|
||||
"crawlLinkedSitemaps": "Crawl Linked XML Sitemaps",
|
||||
"autoDiscover": "Auto Discover XML Sitemaps via robots.txt",
|
||||
"crawlTheseSitemaps": "Crawl These Sitemaps",
|
||||
@@ -760,7 +764,8 @@
|
||||
"screenshotNone": "None (default)",
|
||||
"screenshotFold": "Above the fold",
|
||||
"screenshotFull": "Full page",
|
||||
"screenshotBoth": "Both (full page + fold)"
|
||||
"screenshotBoth": "Both (full page + fold)",
|
||||
"spaRouting": "SPA route discovery"
|
||||
},
|
||||
"performanceBudget": {
|
||||
"intro": "Set per-page performance ceilings. After each crawl, internal HTML pages that exceed any enabled threshold are flagged under the \"Over Performance Budget\" issue filter. Response time and page size come from the crawl itself; LCP and CLS use PageSpeed Insights data when present. Set any field to 0 to disable that individual check.",
|
||||
@@ -1621,7 +1626,36 @@
|
||||
"telemetryHelp": "Off by default. When on, aggregated counts of feature use (e.g. \"export ran\", \"report opened\") may be sent — never URLs, project contents, or crawl results.",
|
||||
"statusNote": "No telemetry endpoint is currently wired up. Toggling this preference is recorded but nothing is transmitted in the current build. The toggle is exposed now so consent state persists across upgrades when a future build adds an endpoint."
|
||||
},
|
||||
"chooseDefaultProjectFolder": "Choose Default Project Folder"
|
||||
"chooseDefaultProjectFolder": "Choose Default Project Folder",
|
||||
"mcp": {
|
||||
"intro": "Connect external MCP servers so the Assistant tab can use their tools alongside your crawl data. Local servers run as a child process; remote ones are called over HTTP. Every tool call asks for your approval the first time.",
|
||||
"empty": "No servers yet. A typical local server looks like: command \"npx\", arguments \"-y\" and \"@modelcontextprotocol/server-filesystem\", plus any API key as an environment variable.",
|
||||
"enabled": "Enabled",
|
||||
"labelPlaceholder": "Display name",
|
||||
"connected": "Connected",
|
||||
"disconnected": "Not connected",
|
||||
"connect": "Connect",
|
||||
"disconnect": "Disconnect",
|
||||
"remove": "Remove server",
|
||||
"transport": "Transport",
|
||||
"transportStdio": "Local process (stdio)",
|
||||
"transportHttp": "Remote (HTTP)",
|
||||
"command": "Command",
|
||||
"commandHint": "Executable name or full path. Resolved against PATH on every OS — on Windows, `npx` resolves to npx.cmd automatically.",
|
||||
"args": "Arguments",
|
||||
"argsHint": "One per line. Passed as a vector — no shell, so quotes and spaces are literal.",
|
||||
"env": "Environment variables",
|
||||
"envHint": "KEY=value per line. Values are stored encrypted with your OS keychain and shown masked.",
|
||||
"url": "Endpoint URL",
|
||||
"headers": "Headers",
|
||||
"headersHint": "Name: value per line. Stored encrypted and shown masked.",
|
||||
"handshake": "{{name}} {{version}} · MCP {{protocol}} · {{n}} tool(s)",
|
||||
"toolPermissions": "Tool permissions",
|
||||
"policyAsk": "Ask",
|
||||
"policyAlways": "Always allow",
|
||||
"policyNever": "Never allow",
|
||||
"add": "Add server"
|
||||
}
|
||||
},
|
||||
"logAnalyzer": {
|
||||
"title": "Log File Analyzer",
|
||||
@@ -1754,11 +1788,40 @@
|
||||
"threatFilterIp": "Only this IP",
|
||||
"threatCopyIp": "Copy IP",
|
||||
"threatServedBadge": "served",
|
||||
"threatClose": "Close"
|
||||
"threatClose": "Close",
|
||||
"urlStatusAll": "Any status",
|
||||
"urlStatus4xx": "4xx (errors)",
|
||||
"urlStatus5xx": "5xx (server errors)",
|
||||
"threatChipIps": "{{n}} distinct IP(s)"
|
||||
},
|
||||
"detailExport": {
|
||||
"label": "Export",
|
||||
"title": "Export this table to Excel or CSV",
|
||||
"nothing": "Nothing to export"
|
||||
},
|
||||
"assistant": {
|
||||
"modelPlaceholder": "Model (provider default)",
|
||||
"tools": "Tools",
|
||||
"builtinTools": "FreeCrawl project tools",
|
||||
"builtinToolsHint": "Read-only queries against the open crawl. Run without asking.",
|
||||
"mcpServers": "MCP servers",
|
||||
"noServers": "No MCP servers configured yet. Add one in Settings → MCP Servers.",
|
||||
"toolCount": "{{n}} tool(s)",
|
||||
"manageServers": "Manage MCP servers",
|
||||
"tokens": "{{in}} in / {{out}} out tokens",
|
||||
"clear": "Clear",
|
||||
"emptyTitle": "Ask about this crawl in plain language.",
|
||||
"emptyHint": "The assistant reads the open project through read-only tools, and can call any MCP server you connect. Your API key stays on this machine.",
|
||||
"runningTool": "Running {{name}}…",
|
||||
"thinking": "Thinking…",
|
||||
"thinkingRound": "Thinking (round {{n}})…",
|
||||
"approvalTitle": "{{server}} wants to run {{tool}}",
|
||||
"allowOnce": "Allow once",
|
||||
"allowAlways": "Always allow this tool",
|
||||
"deny": "Deny",
|
||||
"resultChars": "{{n}} chars",
|
||||
"placeholder": "e.g. Which pages have the worst title problems, and why? (Ctrl+Enter to send)",
|
||||
"stop": "Stop",
|
||||
"send": "Send"
|
||||
}
|
||||
}
|
||||
@@ -239,7 +239,8 @@
|
||||
"analytics": "GA4",
|
||||
"ai": "AI",
|
||||
"seo": "SEO Authority",
|
||||
"visualization": "Görselleştirme"
|
||||
"visualization": "Görselleştirme",
|
||||
"assistant": "Asistan"
|
||||
},
|
||||
"visualization": {
|
||||
"windowTitle": "Crawl Görselleştirme",
|
||||
@@ -421,6 +422,9 @@
|
||||
"followInternalNofollow": "Dahili \"nofollow\" Bağlantıları Takip Et",
|
||||
"followExternalNofollow": "Harici \"nofollow\" Bağlantıları Takip Et",
|
||||
"crawlInvalidLinks": "Geçersiz Bağlantıları Tara",
|
||||
"crawlQueryStrings": "Parametreli URL’leri Tara",
|
||||
"queryStringExceptions": "Bu parametreler hariç",
|
||||
"spiderModeOnly": "Yalnızca Spider modunda geçerli — Liste ve Site Haritası verdiğiniz URL’leri tarar",
|
||||
"crawlLinkedSitemaps": "Bağlantılı XML Site Haritalarını Tara",
|
||||
"autoDiscover": "robots.txt Üzerinden XML Site Haritalarını Otomatik Keşfet",
|
||||
"crawlTheseSitemaps": "Şu Site Haritalarını Tara",
|
||||
@@ -760,7 +764,8 @@
|
||||
"screenshotNone": "Yok (varsayılan)",
|
||||
"screenshotFold": "Katlama üstü",
|
||||
"screenshotFull": "Tam sayfa",
|
||||
"screenshotBoth": "Her ikisi (tam sayfa + katlama)"
|
||||
"screenshotBoth": "Her ikisi (tam sayfa + katlama)",
|
||||
"spaRouting": "SPA rota keşfi"
|
||||
},
|
||||
"performanceBudget": {
|
||||
"intro": "Sayfa başına performans tavanları belirleyin. Her crawl'dan sonra, etkin eşiklerden herhangi birini aşan dahili HTML sayfaları \"Performans Bütçesini Aşan\" issue filtresinde işaretlenir. Yanıt süresi ve sayfa boyutu crawl'dan; LCP ve CLS mevcutsa PageSpeed Insights verisinden gelir. Bir kontrolü devre dışı bırakmak için ilgili alanı 0 yapın.",
|
||||
@@ -1621,7 +1626,36 @@
|
||||
"telemetryHelp": "Varsayılan olarak kapalı. Açıkken, özellik kullanımının toplulaştırılmış sayıları (örn. \"dışa aktarım çalıştı\", \"rapor açıldı\") gönderilebilir — URL, proje içeriği veya crawl sonuçları asla gönderilmez.",
|
||||
"statusNote": "Şu anda herhangi bir telemetri uç noktası kablolanmamıştır. Bu tercihi açıp kapatmak kaydedilir ancak mevcut sürümde hiçbir şey iletilmez. İlerideki bir sürüm uç nokta eklediğinde rıza durumunun korunması için bu toggle şimdi gösteriliyor."
|
||||
},
|
||||
"chooseDefaultProjectFolder": "Varsayılan Proje Klasörünü Seç"
|
||||
"chooseDefaultProjectFolder": "Varsayılan Proje Klasörünü Seç",
|
||||
"mcp": {
|
||||
"intro": "Asistan sekmesinin crawl verinizin yanında harici araçları da kullanabilmesi için MCP sunucuları bağlayın. Yerel sunucular alt süreç olarak çalışır, uzak sunucular HTTP üzerinden çağrılır. Her araç çağrısı ilk seferinde onayınızı ister.",
|
||||
"empty": "Henüz sunucu yok. Tipik bir yerel sunucu şöyledir: komut \"npx\", argümanlar \"-y\" ve \"@modelcontextprotocol/server-filesystem\", varsa API anahtarı ortam değişkeni olarak.",
|
||||
"enabled": "Etkin",
|
||||
"labelPlaceholder": "Görünen ad",
|
||||
"connected": "Bağlı",
|
||||
"disconnected": "Bağlı değil",
|
||||
"connect": "Bağlan",
|
||||
"disconnect": "Bağlantıyı kes",
|
||||
"remove": "Sunucuyu kaldır",
|
||||
"transport": "Taşıma",
|
||||
"transportStdio": "Yerel süreç (stdio)",
|
||||
"transportHttp": "Uzak (HTTP)",
|
||||
"command": "Komut",
|
||||
"commandHint": "Çalıştırılabilir dosya adı veya tam yol. Her işletim sisteminde PATH üzerinden çözülür — Windows'ta `npx` otomatik olarak npx.cmd'ye çözülür.",
|
||||
"args": "Argümanlar",
|
||||
"argsHint": "Satır başına bir argüman. Dizi olarak geçirilir — kabuk yok, tırnak ve boşluklar birebir kalır.",
|
||||
"env": "Ortam değişkenleri",
|
||||
"envHint": "Satır başına KEY=değer. Değerler işletim sistemi anahtarlığıyla şifrelenir ve maskeli gösterilir.",
|
||||
"url": "Uç nokta URL'si",
|
||||
"headers": "Başlıklar",
|
||||
"headersHint": "Satır başına Ad: değer. Şifreli saklanır, maskeli gösterilir.",
|
||||
"handshake": "{{name}} {{version}} · MCP {{protocol}} · {{n}} araç",
|
||||
"toolPermissions": "Araç izinleri",
|
||||
"policyAsk": "Sor",
|
||||
"policyAlways": "Her zaman izin ver",
|
||||
"policyNever": "Asla izin verme",
|
||||
"add": "Sunucu ekle"
|
||||
}
|
||||
},
|
||||
"logAnalyzer": {
|
||||
"title": "Log Dosyası Analizi",
|
||||
@@ -1754,11 +1788,40 @@
|
||||
"threatServedBadge": "işlendi",
|
||||
"threatClose": "Kapat",
|
||||
"clearFailed": "Log verisi temizlenemedi: {{detail}}",
|
||||
"exportFailed": "Dışa aktarma başarısız: {{detail}}"
|
||||
"exportFailed": "Dışa aktarma başarısız: {{detail}}",
|
||||
"urlStatusAll": "Tüm yanıt kodları",
|
||||
"urlStatus4xx": "4xx (hatalar)",
|
||||
"urlStatus5xx": "5xx (sunucu hataları)",
|
||||
"threatChipIps": "{{n}} farklı IP"
|
||||
},
|
||||
"detailExport": {
|
||||
"label": "Dışa Aktar",
|
||||
"title": "Bu tabloyu Excel veya CSV olarak dışa aktar",
|
||||
"nothing": "Dışa aktarılacak veri yok"
|
||||
},
|
||||
"assistant": {
|
||||
"modelPlaceholder": "Model (sağlayıcı varsayılanı)",
|
||||
"tools": "Araçlar",
|
||||
"builtinTools": "FreeCrawl proje araçları",
|
||||
"builtinToolsHint": "Açık crawl üzerinde salt-okunur sorgular. Onay istemeden çalışır.",
|
||||
"mcpServers": "MCP sunucuları",
|
||||
"noServers": "Henüz MCP sunucusu tanımlı değil. Ayarlar → MCP Sunucuları'ndan ekleyin.",
|
||||
"toolCount": "{{n}} araç",
|
||||
"manageServers": "MCP sunucularını yönet",
|
||||
"tokens": "{{in}} giriş / {{out}} çıkış token",
|
||||
"clear": "Temizle",
|
||||
"emptyTitle": "Bu crawl hakkında gündelik dilde soru sorun.",
|
||||
"emptyHint": "Asistan açık projeyi salt-okunur araçlarla okur ve bağladığınız MCP sunucularını çağırabilir. API anahtarınız bu makinede kalır.",
|
||||
"runningTool": "{{name}} çalışıyor…",
|
||||
"thinking": "Düşünüyor…",
|
||||
"thinkingRound": "Düşünüyor ({{n}}. tur)…",
|
||||
"approvalTitle": "{{server}} sunucusu {{tool}} aracını çalıştırmak istiyor",
|
||||
"allowOnce": "Bir kez izin ver",
|
||||
"allowAlways": "Bu araca her zaman izin ver",
|
||||
"deny": "Reddet",
|
||||
"resultChars": "{{n}} karakter",
|
||||
"placeholder": "ör. Başlık sorunları en kötü olan sayfalar hangileri, neden? (Göndermek için Ctrl+Enter)",
|
||||
"stop": "Durdur",
|
||||
"send": "Gönder"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
import { create } from 'zustand';
|
||||
import {
|
||||
DEFAULT_CRAWL_CONFIG,
|
||||
type AssistantEvent,
|
||||
type AssistantMessage,
|
||||
type AssistantToolCall,
|
||||
type AdvancedFilter,
|
||||
type CrawlConfig,
|
||||
type CrawlProgress,
|
||||
@@ -57,7 +60,8 @@ export type TabKey =
|
||||
| 'search-console'
|
||||
| 'analytics'
|
||||
| 'ai'
|
||||
| 'seo';
|
||||
| 'seo'
|
||||
| 'assistant';
|
||||
|
||||
export const TAB_ORDER: { key: TabKey; label: string }[] = [
|
||||
{ key: 'internal', label: 'Internal' },
|
||||
@@ -92,6 +96,7 @@ export const TAB_ORDER: { key: TabKey; label: string }[] = [
|
||||
{ key: 'analytics', label: 'GA4' },
|
||||
{ key: 'ai', label: 'AI' },
|
||||
{ key: 'seo', label: 'SEO Authority' },
|
||||
{ key: 'assistant', label: 'Assistant' },
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -411,9 +416,38 @@ interface AppState {
|
||||
setSettingsOpen: (open: boolean, target?: SettingsTarget) => void;
|
||||
addRecentUrl: (url: string) => void;
|
||||
bumpDataVersion: () => void;
|
||||
/** Faz 5 — assistant transcript. Lives in the store (not the tab) so a
|
||||
* tab switch mid-turn neither loses the conversation nor strands a
|
||||
* pending tool-approval card. */
|
||||
assistantMessages: AssistantMessage[];
|
||||
/** Tool-approval cards waiting on the user. */
|
||||
assistantApprovals: AssistantApprovalRequest[];
|
||||
/** A tool call currently executing, for the "running" indicator. */
|
||||
assistantActiveCall: AssistantToolCall | null;
|
||||
assistantRunning: boolean;
|
||||
/** Which tool round the model is on (0 when idle). */
|
||||
assistantRound: number;
|
||||
assistantError: string | null;
|
||||
/** Append the user's message and return the index the turn starts at. */
|
||||
assistantStartTurn: (message: AssistantMessage) => number;
|
||||
/** Replace everything produced after `fromIndex` with the authoritative
|
||||
* message list the main process returned. */
|
||||
assistantCommitTurn: (fromIndex: number, messages: AssistantMessage[]) => void;
|
||||
assistantSetRunning: (running: boolean) => void;
|
||||
assistantHandleEvent: (event: AssistantEvent) => void;
|
||||
assistantResolveApproval: (requestId: string) => void;
|
||||
assistantClear: () => void;
|
||||
reset: () => void;
|
||||
}
|
||||
|
||||
/** One pending tool-approval card. */
|
||||
export interface AssistantApprovalRequest {
|
||||
requestId: string;
|
||||
call: AssistantToolCall;
|
||||
serverId: string;
|
||||
serverLabel: string;
|
||||
}
|
||||
|
||||
const RECENT_URLS_MAX = 5;
|
||||
const RECENT_URLS_KEY = 'recent-urls';
|
||||
|
||||
@@ -576,6 +610,100 @@ export const useAppStore = create<AppState>((set) => ({
|
||||
return { recentUrls: next };
|
||||
}),
|
||||
bumpDataVersion: () => set((s) => ({ dataVersion: s.dataVersion + 1 })),
|
||||
assistantMessages: [],
|
||||
assistantApprovals: [],
|
||||
assistantActiveCall: null,
|
||||
assistantRunning: false,
|
||||
assistantRound: 0,
|
||||
assistantError: null,
|
||||
assistantStartTurn: (message) => {
|
||||
let startIndex = 0;
|
||||
set((s) => {
|
||||
startIndex = s.assistantMessages.length + 1;
|
||||
return {
|
||||
assistantMessages: [...s.assistantMessages, message],
|
||||
assistantRunning: true,
|
||||
assistantError: null,
|
||||
assistantRound: 0,
|
||||
};
|
||||
});
|
||||
return startIndex;
|
||||
},
|
||||
assistantCommitTurn: (fromIndex, messages) =>
|
||||
set((s) => ({
|
||||
// The live events already rendered these; replacing the tail keeps
|
||||
// the transcript identical to what the next turn will be sent.
|
||||
assistantMessages: [...s.assistantMessages.slice(0, fromIndex), ...messages],
|
||||
assistantRunning: false,
|
||||
assistantActiveCall: null,
|
||||
assistantRound: 0,
|
||||
})),
|
||||
assistantSetRunning: (running) =>
|
||||
set(() => ({ assistantRunning: running, ...(running ? {} : { assistantActiveCall: null }) })),
|
||||
assistantHandleEvent: (event) =>
|
||||
set((s) => {
|
||||
switch (event.type) {
|
||||
case 'thinking':
|
||||
return { assistantRound: event.round, assistantActiveCall: null };
|
||||
case 'assistant-text':
|
||||
return {
|
||||
assistantMessages: [
|
||||
...s.assistantMessages,
|
||||
{
|
||||
role: 'assistant' as const,
|
||||
content: event.text,
|
||||
...(event.toolCalls.length ? { toolCalls: event.toolCalls } : {}),
|
||||
},
|
||||
],
|
||||
};
|
||||
case 'tool-approval':
|
||||
return {
|
||||
assistantApprovals: [
|
||||
...s.assistantApprovals,
|
||||
{
|
||||
requestId: event.requestId,
|
||||
call: event.call,
|
||||
serverId: event.serverId,
|
||||
serverLabel: event.serverLabel,
|
||||
},
|
||||
],
|
||||
};
|
||||
case 'tool-start':
|
||||
return { assistantActiveCall: event.call };
|
||||
case 'tool-result':
|
||||
return {
|
||||
assistantActiveCall: null,
|
||||
assistantMessages: [
|
||||
...s.assistantMessages,
|
||||
{
|
||||
role: 'tool' as const,
|
||||
content: event.result,
|
||||
toolCallId: event.callId,
|
||||
toolName: event.name,
|
||||
...(event.isError ? { isError: true } : {}),
|
||||
durationMs: event.durationMs,
|
||||
},
|
||||
],
|
||||
};
|
||||
case 'error':
|
||||
return { assistantError: event.message };
|
||||
default:
|
||||
return {};
|
||||
}
|
||||
}),
|
||||
assistantResolveApproval: (requestId) =>
|
||||
set((s) => ({
|
||||
assistantApprovals: s.assistantApprovals.filter((a) => a.requestId !== requestId),
|
||||
})),
|
||||
assistantClear: () =>
|
||||
set(() => ({
|
||||
assistantMessages: [],
|
||||
assistantApprovals: [],
|
||||
assistantActiveCall: null,
|
||||
assistantRunning: false,
|
||||
assistantRound: 0,
|
||||
assistantError: null,
|
||||
})),
|
||||
reset: () =>
|
||||
set({
|
||||
progress: null,
|
||||
|
||||
@@ -0,0 +1,561 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import {
|
||||
AlertTriangle,
|
||||
Check,
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
Loader2,
|
||||
Plug,
|
||||
Send,
|
||||
ShieldQuestion,
|
||||
Square,
|
||||
Trash2,
|
||||
Wrench,
|
||||
} from 'lucide-react';
|
||||
import type {
|
||||
AiProvider,
|
||||
AssistantMessage,
|
||||
AssistantToolCall,
|
||||
McpServerConfig,
|
||||
McpServerStatus,
|
||||
} from '@freecrawl/shared-types';
|
||||
import { useAppStore } from '../store.js';
|
||||
|
||||
const PROVIDER_LABEL: Record<AiProvider, string> = {
|
||||
openai: 'OpenAI',
|
||||
anthropic: 'Anthropic',
|
||||
ollama: 'Ollama (local)',
|
||||
};
|
||||
|
||||
const PROVIDER_PREF = 'assistant:provider';
|
||||
const MODEL_PREF = 'assistant:model';
|
||||
const SERVERS_PREF = 'assistant:servers';
|
||||
const BUILTINS_PREF = 'assistant:builtins';
|
||||
|
||||
/** Pretty-print a tool payload without letting one huge result push the
|
||||
* whole conversation off screen. */
|
||||
function preview(raw: string, max = 4000): string {
|
||||
const text = raw.trim();
|
||||
if (text.length <= max) return text;
|
||||
return `${text.slice(0, max)}\n… (${text.length - max} more characters)`;
|
||||
}
|
||||
|
||||
function prettyJson(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value, null, 2);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
export function AssistantTab() {
|
||||
const { t } = useTranslation();
|
||||
const messages = useAppStore((s) => s.assistantMessages);
|
||||
const approvals = useAppStore((s) => s.assistantApprovals);
|
||||
const activeCall = useAppStore((s) => s.assistantActiveCall);
|
||||
const running = useAppStore((s) => s.assistantRunning);
|
||||
const round = useAppStore((s) => s.assistantRound);
|
||||
const assistantError = useAppStore((s) => s.assistantError);
|
||||
const startTurn = useAppStore((s) => s.assistantStartTurn);
|
||||
const commitTurn = useAppStore((s) => s.assistantCommitTurn);
|
||||
const setRunning = useAppStore((s) => s.assistantSetRunning);
|
||||
const resolveApproval = useAppStore((s) => s.assistantResolveApproval);
|
||||
const clearChat = useAppStore((s) => s.assistantClear);
|
||||
|
||||
const [provider, setProvider] = useState<AiProvider>(() => {
|
||||
const saved = window.freecrawl?.prefsGet(PROVIDER_PREF);
|
||||
return saved === 'anthropic' || saved === 'ollama' ? saved : 'openai';
|
||||
});
|
||||
const [model, setModel] = useState<string>(() => {
|
||||
const saved = window.freecrawl?.prefsGet(MODEL_PREF);
|
||||
return typeof saved === 'string' ? saved : '';
|
||||
});
|
||||
const [servers, setServers] = useState<McpServerConfig[]>([]);
|
||||
const [statuses, setStatuses] = useState<Record<string, McpServerStatus>>({});
|
||||
const [selectedServers, setSelectedServers] = useState<string[]>(() => {
|
||||
const saved = window.freecrawl?.prefsGet(SERVERS_PREF);
|
||||
return Array.isArray(saved) ? (saved as string[]) : [];
|
||||
});
|
||||
const [includeBuiltins, setIncludeBuiltins] = useState<boolean>(() => {
|
||||
const saved = window.freecrawl?.prefsGet(BUILTINS_PREF);
|
||||
return saved !== false;
|
||||
});
|
||||
const [toolsOpen, setToolsOpen] = useState(false);
|
||||
const [draft, setDraft] = useState('');
|
||||
const [lastUsage, setLastUsage] = useState<{ in: number; out: number } | null>(null);
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const setSettingsOpen = useAppStore((s) => s.setSettingsOpen);
|
||||
|
||||
const refreshServers = useCallback(async () => {
|
||||
const [list, statusList] = await Promise.all([
|
||||
window.freecrawl.mcpServersList(),
|
||||
window.freecrawl.mcpServerStatuses(),
|
||||
]);
|
||||
setServers(list);
|
||||
setStatuses(Object.fromEntries(statusList.map((s) => [s.id, s])));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void refreshServers();
|
||||
}, [refreshServers]);
|
||||
|
||||
// Persist the picker choices — re-selecting three servers on every
|
||||
// launch is exactly the kind of friction that kills a chat surface.
|
||||
useEffect(() => {
|
||||
window.freecrawl?.prefsSet(PROVIDER_PREF, provider);
|
||||
}, [provider]);
|
||||
useEffect(() => {
|
||||
window.freecrawl?.prefsSet(MODEL_PREF, model);
|
||||
}, [model]);
|
||||
useEffect(() => {
|
||||
window.freecrawl?.prefsSet(SERVERS_PREF, selectedServers);
|
||||
}, [selectedServers]);
|
||||
useEffect(() => {
|
||||
window.freecrawl?.prefsSet(BUILTINS_PREF, includeBuiltins);
|
||||
}, [includeBuiltins]);
|
||||
|
||||
// Keep the newest message in view while a turn streams in.
|
||||
useEffect(() => {
|
||||
const el = scrollRef.current;
|
||||
if (el) el.scrollTop = el.scrollHeight;
|
||||
}, [messages.length, approvals.length, activeCall, running]);
|
||||
|
||||
const enabledServers = useMemo(() => servers.filter((s) => s.enabled), [servers]);
|
||||
|
||||
const send = useCallback(async () => {
|
||||
const text = draft.trim();
|
||||
if (!text || running) return;
|
||||
setDraft('');
|
||||
const userMessage: AssistantMessage = { role: 'user', content: text };
|
||||
const startIndex = startTurn(userMessage);
|
||||
const history = [...useAppStore.getState().assistantMessages];
|
||||
try {
|
||||
const result = await window.freecrawl.assistantSend({
|
||||
provider,
|
||||
model: model.trim() || undefined,
|
||||
messages: history,
|
||||
serverIds: selectedServers.filter((id) =>
|
||||
enabledServers.some((s) => s.id === id),
|
||||
),
|
||||
includeBuiltins,
|
||||
});
|
||||
commitTurn(startIndex, result.messages);
|
||||
if (result.tokensIn || result.tokensOut) {
|
||||
setLastUsage({ in: result.tokensIn ?? 0, out: result.tokensOut ?? 0 });
|
||||
}
|
||||
} catch (err) {
|
||||
setRunning(false);
|
||||
useAppStore.setState({
|
||||
assistantError: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}, [
|
||||
draft,
|
||||
running,
|
||||
provider,
|
||||
model,
|
||||
selectedServers,
|
||||
enabledServers,
|
||||
includeBuiltins,
|
||||
startTurn,
|
||||
commitTurn,
|
||||
setRunning,
|
||||
]);
|
||||
|
||||
async function decide(
|
||||
requestId: string,
|
||||
decision: 'allow-once' | 'allow-always' | 'deny',
|
||||
) {
|
||||
resolveApproval(requestId);
|
||||
await window.freecrawl.assistantApprove({ requestId, decision });
|
||||
}
|
||||
|
||||
async function connectServer(id: string) {
|
||||
const status = await window.freecrawl.mcpServerConnect(id);
|
||||
setStatuses((prev) => ({ ...prev, [id]: status }));
|
||||
}
|
||||
|
||||
/** Arguments the model passed for one tool result, looked up on the
|
||||
* assistant message that requested it. */
|
||||
const argsByCallId = useMemo(() => {
|
||||
const map = new Map<string, AssistantToolCall>();
|
||||
for (const m of messages) {
|
||||
for (const call of m.toolCalls ?? []) map.set(call.id, call);
|
||||
}
|
||||
return map;
|
||||
}, [messages]);
|
||||
|
||||
return (
|
||||
<div className="flex h-full flex-col bg-surface-950">
|
||||
{/* Toolbar */}
|
||||
<div className="flex h-9 flex-none items-center gap-2 border-b border-surface-800 px-2">
|
||||
<select
|
||||
className="h-6 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 focus:border-blue-500 focus:outline-none"
|
||||
value={provider}
|
||||
onChange={(e) => setProvider(e.target.value as AiProvider)}
|
||||
>
|
||||
{(Object.keys(PROVIDER_LABEL) as AiProvider[]).map((p) => (
|
||||
<option key={p} value={p}>
|
||||
{PROVIDER_LABEL[p]}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<input
|
||||
className="h-6 w-44 rounded border border-surface-700 bg-surface-950 px-1.5 text-[11px] text-surface-100 placeholder:text-surface-600 focus:border-blue-500 focus:outline-none"
|
||||
value={model}
|
||||
onChange={(e) => setModel(e.target.value)}
|
||||
placeholder={t('assistant.modelPlaceholder', {
|
||||
defaultValue: 'Model (provider default)',
|
||||
})}
|
||||
/>
|
||||
<div className="relative">
|
||||
<button
|
||||
className="flex h-6 items-center gap-1 rounded border border-surface-700 px-2 text-[11px] text-surface-300 hover:bg-surface-800 hover:text-surface-100"
|
||||
onClick={() => setToolsOpen((v) => !v)}
|
||||
>
|
||||
<Wrench size={12} />
|
||||
{t('assistant.tools', { defaultValue: 'Tools' })}
|
||||
<span className="text-surface-500">
|
||||
({(includeBuiltins ? 1 : 0) + selectedServers.length})
|
||||
</span>
|
||||
<ChevronDown size={11} />
|
||||
</button>
|
||||
{toolsOpen && (
|
||||
<div className="absolute left-0 top-7 z-50 w-80 rounded border border-surface-700 bg-surface-900 p-2 shadow-xl">
|
||||
<label className="flex cursor-pointer items-start gap-2 rounded p-1.5 hover:bg-surface-800">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="mt-0.5"
|
||||
checked={includeBuiltins}
|
||||
onChange={(e) => setIncludeBuiltins(e.target.checked)}
|
||||
/>
|
||||
<span>
|
||||
<span className="block text-[11px] text-surface-100">
|
||||
{t('assistant.builtinTools', {
|
||||
defaultValue: 'FreeCrawl project tools',
|
||||
})}
|
||||
</span>
|
||||
<span className="block text-[10px] text-surface-500">
|
||||
{t('assistant.builtinToolsHint', {
|
||||
defaultValue:
|
||||
'Read-only queries against the open crawl. Run without asking.',
|
||||
})}
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
<div className="mt-2 mb-1 px-1.5 text-[10px] font-semibold uppercase tracking-wider text-surface-500">
|
||||
{t('assistant.mcpServers', { defaultValue: 'MCP servers' })}
|
||||
</div>
|
||||
{enabledServers.length === 0 && (
|
||||
<p className="px-1.5 pb-1 text-[10px] text-surface-500">
|
||||
{t('assistant.noServers', {
|
||||
defaultValue:
|
||||
'No MCP servers configured yet. Add one in Settings → MCP Servers.',
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
{enabledServers.map((s) => {
|
||||
const status = statuses[s.id];
|
||||
const checked = selectedServers.includes(s.id);
|
||||
return (
|
||||
<label
|
||||
key={s.id}
|
||||
className="flex cursor-pointer items-start gap-2 rounded p-1.5 hover:bg-surface-800"
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
className="mt-0.5"
|
||||
checked={checked}
|
||||
onChange={(e) => {
|
||||
setSelectedServers((prev) =>
|
||||
e.target.checked
|
||||
? [...prev, s.id]
|
||||
: prev.filter((id) => id !== s.id),
|
||||
);
|
||||
if (e.target.checked && !status?.connected) {
|
||||
void connectServer(s.id);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="flex items-center gap-1.5 text-[11px] text-surface-100">
|
||||
{s.label}
|
||||
<span
|
||||
className={
|
||||
status?.connected
|
||||
? 'h-1.5 w-1.5 rounded-full bg-emerald-500'
|
||||
: 'h-1.5 w-1.5 rounded-full bg-surface-600'
|
||||
}
|
||||
/>
|
||||
{status?.connected && (
|
||||
<span className="text-[10px] text-surface-500">
|
||||
{t('assistant.toolCount', {
|
||||
defaultValue: '{{n}} tool(s)',
|
||||
n: status.toolCount,
|
||||
})}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
{status?.error && (
|
||||
<span className="block truncate text-[10px] text-red-400">
|
||||
{status.error}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
<button
|
||||
className="mt-2 flex w-full items-center justify-center gap-1 rounded border border-surface-700 py-1 text-[10px] text-surface-300 hover:bg-surface-800 hover:text-surface-100"
|
||||
onClick={() => {
|
||||
setToolsOpen(false);
|
||||
setSettingsOpen(true, { section: 'mcp-servers' });
|
||||
}}
|
||||
>
|
||||
<Plug size={11} />
|
||||
{t('assistant.manageServers', { defaultValue: 'Manage MCP servers' })}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex-1" />
|
||||
{lastUsage && (
|
||||
<span className="text-[10px] text-surface-500">
|
||||
{t('assistant.tokens', {
|
||||
defaultValue: '{{in}} in / {{out}} out tokens',
|
||||
in: lastUsage.in.toLocaleString(),
|
||||
out: lastUsage.out.toLocaleString(),
|
||||
})}
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
className="flex h-6 items-center gap-1 rounded border border-surface-700 px-2 text-[11px] text-surface-300 hover:bg-surface-800 hover:text-surface-100 disabled:opacity-40"
|
||||
onClick={clearChat}
|
||||
disabled={running || messages.length === 0}
|
||||
>
|
||||
<Trash2 size={12} />
|
||||
{t('assistant.clear', { defaultValue: 'Clear' })}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Transcript */}
|
||||
<div ref={scrollRef} className="flex-1 overflow-y-auto px-3 py-2">
|
||||
{messages.length === 0 && (
|
||||
<div className="mx-auto mt-8 max-w-xl text-center">
|
||||
<p className="text-[12px] text-surface-300">
|
||||
{t('assistant.emptyTitle', {
|
||||
defaultValue: 'Ask about this crawl in plain language.',
|
||||
})}
|
||||
</p>
|
||||
<p className="mt-1 text-[11px] text-surface-500">
|
||||
{t('assistant.emptyHint', {
|
||||
defaultValue:
|
||||
'The assistant reads the open project through read-only tools, and can call any MCP server you connect. Your API key stays on this machine.',
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{messages.map((m, i) => (
|
||||
<MessageBlock
|
||||
key={i}
|
||||
message={m}
|
||||
call={m.toolCallId ? argsByCallId.get(m.toolCallId) : undefined}
|
||||
/>
|
||||
))}
|
||||
|
||||
{activeCall && (
|
||||
<div className="my-1 flex items-center gap-2 rounded border border-surface-800 bg-surface-900/40 px-2 py-1.5 text-[11px] text-surface-400">
|
||||
<Loader2 size={12} className="animate-spin" />
|
||||
{t('assistant.runningTool', {
|
||||
defaultValue: 'Running {{name}}…',
|
||||
name: activeCall.name,
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{running && !activeCall && (
|
||||
<div className="my-1 flex items-center gap-2 px-2 py-1.5 text-[11px] text-surface-500">
|
||||
<Loader2 size={12} className="animate-spin" />
|
||||
{round > 1
|
||||
? t('assistant.thinkingRound', {
|
||||
defaultValue: 'Thinking (round {{n}})…',
|
||||
n: round,
|
||||
})
|
||||
: t('assistant.thinking', { defaultValue: 'Thinking…' })}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{approvals.map((a) => (
|
||||
<div
|
||||
key={a.requestId}
|
||||
className="my-2 rounded border border-amber-700/60 bg-amber-950/20 p-2"
|
||||
>
|
||||
<div className="flex items-center gap-1.5 text-[11px] font-medium text-amber-200">
|
||||
<ShieldQuestion size={13} />
|
||||
{t('assistant.approvalTitle', {
|
||||
defaultValue: '{{server}} wants to run {{tool}}',
|
||||
server: a.serverLabel,
|
||||
tool: a.call.name,
|
||||
})}
|
||||
</div>
|
||||
<pre className="mt-1.5 max-h-40 overflow-auto rounded bg-surface-950/70 p-1.5 text-[10px] text-surface-300">
|
||||
{prettyJson(a.call.arguments)}
|
||||
</pre>
|
||||
<div className="mt-2 flex gap-1.5">
|
||||
<button
|
||||
className="rounded bg-emerald-700 px-2 py-1 text-[11px] text-white hover:bg-emerald-600"
|
||||
onClick={() => void decide(a.requestId, 'allow-once')}
|
||||
>
|
||||
{t('assistant.allowOnce', { defaultValue: 'Allow once' })}
|
||||
</button>
|
||||
<button
|
||||
className="rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-200 hover:bg-surface-800"
|
||||
onClick={() => void decide(a.requestId, 'allow-always')}
|
||||
>
|
||||
{t('assistant.allowAlways', { defaultValue: 'Always allow this tool' })}
|
||||
</button>
|
||||
<button
|
||||
className="rounded border border-surface-700 px-2 py-1 text-[11px] text-surface-200 hover:bg-surface-800"
|
||||
onClick={() => void decide(a.requestId, 'deny')}
|
||||
>
|
||||
{t('assistant.deny', { defaultValue: 'Deny' })}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{assistantError && (
|
||||
<div className="my-2 flex items-start gap-1.5 rounded border border-red-800/60 bg-red-950/20 p-2 text-[11px] text-red-200">
|
||||
<AlertTriangle size={13} className="mt-0.5 flex-none" />
|
||||
<span className="whitespace-pre-wrap">{assistantError}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Composer */}
|
||||
<div className="flex-none border-t border-surface-800 p-2">
|
||||
<div className="flex items-end gap-2">
|
||||
<textarea
|
||||
className="h-16 flex-1 resize-none rounded border border-surface-700 bg-surface-950 px-2 py-1.5 text-[12px] text-surface-100 placeholder:text-surface-600 focus:border-blue-500 focus:outline-none"
|
||||
value={draft}
|
||||
onChange={(e) => setDraft(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) {
|
||||
e.preventDefault();
|
||||
void send();
|
||||
}
|
||||
}}
|
||||
placeholder={t('assistant.placeholder', {
|
||||
defaultValue:
|
||||
'e.g. Which pages have the worst title problems, and why? (Ctrl+Enter to send)',
|
||||
})}
|
||||
/>
|
||||
{running ? (
|
||||
<button
|
||||
className="flex h-8 items-center gap-1 rounded border border-surface-700 px-3 text-[11px] text-surface-200 hover:bg-surface-800"
|
||||
onClick={() => void window.freecrawl.assistantCancel()}
|
||||
>
|
||||
<Square size={12} />
|
||||
{t('assistant.stop', { defaultValue: 'Stop' })}
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
className="flex h-8 items-center gap-1 rounded bg-blue-700 px-3 text-[11px] text-white hover:bg-blue-600 disabled:opacity-40"
|
||||
onClick={() => void send()}
|
||||
disabled={!draft.trim()}
|
||||
>
|
||||
<Send size={12} />
|
||||
{t('assistant.send', { defaultValue: 'Send' })}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MessageBlock({
|
||||
message,
|
||||
call,
|
||||
}: {
|
||||
message: AssistantMessage;
|
||||
call?: AssistantToolCall;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const [open, setOpen] = useState(false);
|
||||
|
||||
if (message.role === 'user') {
|
||||
return (
|
||||
<div className="my-1.5 flex justify-end">
|
||||
<div className="max-w-[80%] whitespace-pre-wrap rounded border border-blue-800/60 bg-blue-950/30 px-2.5 py-1.5 text-[12px] text-surface-100">
|
||||
{message.content}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (message.role === 'tool') {
|
||||
return (
|
||||
<div className="my-1">
|
||||
<button
|
||||
className="flex w-full items-center gap-1.5 rounded border border-surface-800 bg-surface-900/40 px-2 py-1 text-left text-[10px] text-surface-400 hover:bg-surface-900"
|
||||
onClick={() => setOpen((v) => !v)}
|
||||
>
|
||||
{open ? <ChevronDown size={11} /> : <ChevronRight size={11} />}
|
||||
{message.isError ? (
|
||||
<AlertTriangle size={11} className="text-red-400" />
|
||||
) : (
|
||||
<Check size={11} className="text-emerald-500" />
|
||||
)}
|
||||
<span className="font-mono text-surface-300">{message.toolName}</span>
|
||||
{typeof message.durationMs === 'number' && (
|
||||
<span className="text-surface-600">{message.durationMs} ms</span>
|
||||
)}
|
||||
<span className="ml-auto text-surface-600">
|
||||
{t('assistant.resultChars', {
|
||||
defaultValue: '{{n}} chars',
|
||||
n: message.content.length.toLocaleString(),
|
||||
})}
|
||||
</span>
|
||||
</button>
|
||||
{open && (
|
||||
<div className="mt-1 space-y-1">
|
||||
{call && (
|
||||
<pre className="max-h-40 overflow-auto rounded border border-surface-800 bg-surface-950 p-1.5 text-[10px] text-surface-400">
|
||||
{prettyJson(call.arguments)}
|
||||
</pre>
|
||||
)}
|
||||
<pre className="max-h-64 overflow-auto rounded border border-surface-800 bg-surface-950 p-1.5 text-[10px] text-surface-300">
|
||||
{preview(message.content)}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Assistant prose (plus any tool calls it asked for).
|
||||
return (
|
||||
<div className="my-1.5">
|
||||
{message.content.trim() && (
|
||||
<div className="max-w-[85%] whitespace-pre-wrap rounded border border-surface-800 bg-surface-900/50 px-2.5 py-1.5 text-[12px] text-surface-100">
|
||||
{message.content}
|
||||
</div>
|
||||
)}
|
||||
{(message.toolCalls ?? []).length > 0 && (
|
||||
<div className="mt-1 flex flex-wrap gap-1">
|
||||
{(message.toolCalls ?? []).map((c) => (
|
||||
<span
|
||||
key={c.id}
|
||||
className="flex items-center gap-1 rounded border border-surface-800 bg-surface-900/40 px-1.5 py-0.5 font-mono text-[10px] text-surface-400"
|
||||
>
|
||||
<Wrench size={10} />
|
||||
{c.name}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -636,6 +636,7 @@ export const COLUMN_SPECS: Record<TabKey, ColumnSpec[]> = {
|
||||
analytics: [],
|
||||
ai: [],
|
||||
seo: [],
|
||||
assistant: [],
|
||||
canonicals: [
|
||||
COL.url,
|
||||
COL.canonical,
|
||||
|
||||
@@ -7,7 +7,6 @@ import type {
|
||||
ContentKind,
|
||||
CrawlConfig,
|
||||
CrawlProgress,
|
||||
CrawlScope,
|
||||
CrawlSummary,
|
||||
DiscoveredLink,
|
||||
Indexability,
|
||||
@@ -194,6 +193,15 @@ export function xRobotsTagDirectives(
|
||||
if (/\bnoindex\b/.test(directives)) noindex = true;
|
||||
if (/\bnofollow\b/.test(directives)) nofollow = true;
|
||||
};
|
||||
// A bot name scopes every directive that FOLLOWS it in the header, not
|
||||
// just the one it is glued to: `X-Robots-Tag: bingbot: noindex, nofollow`
|
||||
// is Bing's whole instruction list, so a Googlebot-shaped crawl must
|
||||
// ignore the `nofollow` too. Splitting on `,` and judging each segment
|
||||
// in isolation leaked that second directive onto every bot — a page
|
||||
// another engine was told not to follow silently lost its outlinks here.
|
||||
// The scope therefore persists until a different bot name appears; a
|
||||
// directive before any bot name is un-scoped and applies to everyone.
|
||||
let scope: string | null = null;
|
||||
for (const rawSegment of header.split(',')) {
|
||||
const segment = rawSegment.trim().toLowerCase();
|
||||
if (!segment) continue;
|
||||
@@ -203,12 +211,15 @@ export function xRobotsTagDirectives(
|
||||
// `head` is a bot name only if it isn't itself a directive keyword
|
||||
// (which would mean this is a valued directive like `max-snippet: 5`).
|
||||
if (!X_ROBOTS_DIRECTIVES.has(head)) {
|
||||
scope = head;
|
||||
if (head !== 'googlebot' && head !== ua) continue; // other bot
|
||||
apply(segment.slice(colon + 1));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
apply(segment); // un-scoped
|
||||
// Inside another bot's group — not ours to honour.
|
||||
if (scope !== null && scope !== 'googlebot' && scope !== ua) continue;
|
||||
apply(segment);
|
||||
}
|
||||
return { noindex, nofollow };
|
||||
}
|
||||
@@ -498,6 +509,9 @@ export class Crawler extends EventEmitter {
|
||||
private sitemapAbort: AbortController | null = null;
|
||||
private readonly includeRegexes: RegExp[];
|
||||
private readonly excludeRegexes: RegExp[];
|
||||
/** Parameter names exempt from the query-string filter, folded once
|
||||
* because the filter runs on every discovered URL. */
|
||||
private readonly queryStringExceptions: Set<string>;
|
||||
/**
|
||||
* Snapshotted once in the constructor so the URL-rewrite pass costs
|
||||
* nothing per call (no `?:` chains, no per-link `if`s) and so changing
|
||||
@@ -634,6 +648,8 @@ export class Crawler extends EventEmitter {
|
||||
tapTargetsSmall: number;
|
||||
tapTargetsSampled: number;
|
||||
} | null;
|
||||
/** Client-side routes the SPA reached during render (Bölüm 29). */
|
||||
spaRoutes?: string[];
|
||||
}>;
|
||||
|
||||
constructor(
|
||||
@@ -697,6 +713,8 @@ export class Crawler extends EventEmitter {
|
||||
tapTargetsSmall: number;
|
||||
tapTargetsSampled: number;
|
||||
} | null;
|
||||
/** Client-side routes the SPA reached during render (Bölüm 29). */
|
||||
spaRoutes?: string[];
|
||||
}>;
|
||||
/**
|
||||
* Re-fetch exactly these URLs instead of walking the site from the
|
||||
@@ -795,6 +813,12 @@ export class Crawler extends EventEmitter {
|
||||
this.excludeRegexes = compilePatterns(config.excludePatterns, (p, err) => {
|
||||
this.emit('error', `Invalid exclude pattern "${p}": ${err}`);
|
||||
});
|
||||
// Case-folded once: the filter runs on every discovered URL.
|
||||
this.queryStringExceptions = new Set(
|
||||
(config.crawlQueryStringExceptions ?? [])
|
||||
.map((p) => p.trim().toLowerCase())
|
||||
.filter(Boolean),
|
||||
);
|
||||
const compiledRegexRewrites = compileUrlRegexRewrites(config.urlRegexRewrites, (p, err) => {
|
||||
this.emit('error', `Invalid URL regex rewrite "${p}": ${err}`);
|
||||
});
|
||||
@@ -806,6 +830,11 @@ export class Crawler extends EventEmitter {
|
||||
keepQueryParams: config.keepQueryParams,
|
||||
sortQueryParams: config.sortQueryParams,
|
||||
collapseDuplicateSlashes: config.collapseDuplicateSlashes,
|
||||
// A hash-routed SPA identifies its pages only by the fragment, so
|
||||
// the same toggle that hooks the History API also has to stop
|
||||
// normalisation from collapsing `#/a` and `#/b` onto the shell.
|
||||
keepHashRoutes:
|
||||
config.renderingMode === 'js' && config.jsRender?.spaRouting === true,
|
||||
regexRewrites: compiledRegexRewrites,
|
||||
};
|
||||
}
|
||||
@@ -825,6 +854,50 @@ export class Crawler extends EventEmitter {
|
||||
return this.includeRegexes.some((re) => re.test(url));
|
||||
}
|
||||
|
||||
/**
|
||||
* Spider → Crawl Behaviour "Crawl URLs with Query Strings". Off drops
|
||||
* parameterised pages before robots and before a request goes out, which
|
||||
* is the cheap way to stop a faceted navigation from spending the whole
|
||||
* URL budget on one listing wearing a thousand URLs.
|
||||
*
|
||||
* Three deliberate exemptions, each because dropping the URL would be
|
||||
* surprising rather than helpful:
|
||||
* - the start URL, which the user typed themselves;
|
||||
* - fixed-URL modes (List / Sitemap), where the supplied URLs *are*
|
||||
* the crawl and no link-follow can run away;
|
||||
* - subresource rows, because `style.css?v=7` is a cache-buster, not
|
||||
* a facet, and losing it would silently break the CSS/JS audit.
|
||||
*
|
||||
* With exceptions configured, a URL is admitted only when *every*
|
||||
* parameter it carries is on the list — `?page=2` passes, but
|
||||
* `?page=2&color=red` does not. Any-match would defeat the point: a
|
||||
* facet URL nearly always carries the pagination parameter too.
|
||||
*/
|
||||
private passesQueryStringFilter(item: QueueItem): boolean {
|
||||
if (this.config.crawlQueryStrings !== false) return true;
|
||||
if (item.url === this.config.startUrl) return true;
|
||||
if (item.resourceRow) return true;
|
||||
if (this.config.mode === 'list' || this.config.mode === 'sitemap') return true;
|
||||
let params: URLSearchParams;
|
||||
try {
|
||||
const u = new URL(item.url);
|
||||
// `?` with nothing after it is not a parameterised page.
|
||||
if (!u.search || u.search === '?') return true;
|
||||
params = u.searchParams;
|
||||
} catch {
|
||||
// Unparseable URLs are the invalid-link path's business, not ours.
|
||||
return true;
|
||||
}
|
||||
if (this.queryStringExceptions.size === 0) return false;
|
||||
for (const name of params.keys()) {
|
||||
if (!this.queryStringExceptions.has(name.toLowerCase())) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** URLs dropped by `passesQueryStringFilter`, reported at crawl end. */
|
||||
private queryStringsDropped = 0;
|
||||
|
||||
/**
|
||||
* Classify a URL against the crawl-trap shapes, for storage on its row.
|
||||
* Returns the trap kind or null. Detection is independent of whether the
|
||||
@@ -1235,6 +1308,16 @@ export class Crawler extends EventEmitter {
|
||||
`segment ${this.config.maxRepeatedPathSegments}+ times (link loop).`,
|
||||
);
|
||||
}
|
||||
if (this.queryStringsDropped > 0) {
|
||||
const allowed = this.queryStringExceptions.size
|
||||
? ` (kept URLs whose parameters are all in: ${[...this.queryStringExceptions].join(', ')})`
|
||||
: '';
|
||||
this.emit(
|
||||
'info',
|
||||
`Query-string filter: skipped ${this.queryStringsDropped.toLocaleString()} parameterised URL(s)` +
|
||||
`${allowed}. Turn on Spider → Crawl Behaviour → "Crawl URLs with Query Strings" to include them.`,
|
||||
);
|
||||
}
|
||||
this.stopMemoryMonitor();
|
||||
// Release per-URL dedup sets — at 1M URLs this is ~80–120 MB of string
|
||||
// heap that's no longer needed once the queue is drained.
|
||||
@@ -3094,6 +3177,10 @@ export class Crawler extends EventEmitter {
|
||||
}
|
||||
if (!this.passesUrlFilter(item.url)) return;
|
||||
if (!this.passesExtensionFilter(item.url)) return;
|
||||
if (!this.passesQueryStringFilter(item)) {
|
||||
this.queryStringsDropped++;
|
||||
return;
|
||||
}
|
||||
// Hard cap on the in-memory pending queue. Beyond this we drop new
|
||||
// discoveries — the alternative is unbounded heap growth on big
|
||||
// sitemaps / dense link graphs. `seen` still grows, but each entry
|
||||
@@ -3868,6 +3955,7 @@ export class Crawler extends EventEmitter {
|
||||
let renderMobile:
|
||||
| { ok: boolean; overflowPx: number; hasViewportMeta: boolean }
|
||||
| null = null;
|
||||
let renderSpaRoutes: string[] = [];
|
||||
let renderA11y:
|
||||
| {
|
||||
lowContrast: number;
|
||||
@@ -3906,6 +3994,7 @@ export class Crawler extends EventEmitter {
|
||||
);
|
||||
}
|
||||
if (renderRes.screenshots) renderScreenshots = renderRes.screenshots;
|
||||
if (renderRes.spaRoutes?.length) renderSpaRoutes = renderRes.spaRoutes;
|
||||
if (renderRes.lcp) renderLcp = renderRes.lcp;
|
||||
if (renderRes.mobileUsability) renderMobile = renderRes.mobileUsability;
|
||||
if (renderRes.a11y) renderA11y = renderRes.a11y;
|
||||
@@ -4357,6 +4446,24 @@ export class Crawler extends EventEmitter {
|
||||
this.enqueue({ url: target, depth: nextDepth });
|
||||
}
|
||||
}
|
||||
// Bölüm 29 — SPA routes discovered during JS render. The app
|
||||
// reached these itself (pushState / hash change) or links to
|
||||
// them with a hash-router href, so they are as real as any
|
||||
// `<a href>` target — they just never produced a document
|
||||
// request the crawler could see.
|
||||
if (renderSpaRoutes.length > 0) {
|
||||
for (const route of renderSpaRoutes) {
|
||||
const normalized = normalizeUrl(route, item.url, this.urlRewrites);
|
||||
if (!normalized || normalized === item.url) continue;
|
||||
const inScope = isInScope(
|
||||
this.config.startUrl,
|
||||
normalized,
|
||||
this.config.scope,
|
||||
);
|
||||
if (!inScope && !this.config.crawlExternal) continue;
|
||||
this.enqueue({ url: normalized, depth: nextDepth });
|
||||
}
|
||||
}
|
||||
// Wave 3 — Canonical follow toggle. When on, a 200 page that
|
||||
// declares a canonical pointing to a different URL also
|
||||
// enqueues that target. Default off — most crawls treat
|
||||
|
||||
@@ -15,6 +15,11 @@ export interface RenderOptions {
|
||||
detectLcp?: boolean;
|
||||
/** Run the in-page accessibility audit (contrast + focus outline). */
|
||||
detectA11y?: boolean;
|
||||
/**
|
||||
* Record client-side route changes (History API + hash router) so the
|
||||
* crawler can follow routes an SPA reaches without a document request.
|
||||
*/
|
||||
discoverSpaRoutes?: boolean;
|
||||
/** Where to write screenshot PNGs — the caller resolves a per-URL path. */
|
||||
screenshotPaths?: {
|
||||
fullpage?: string;
|
||||
@@ -59,6 +64,13 @@ export interface RenderResult {
|
||||
lcp?: LcpCandidate | null;
|
||||
/** Accessibility audit from `detectA11y` — null when it could not run. */
|
||||
a11y?: A11yAudit | null;
|
||||
/**
|
||||
* Absolute URLs the page navigated to client-side during this render
|
||||
* (`history.pushState` / `replaceState` / `popstate` / `hashchange`),
|
||||
* plus any hash-router links found in the rendered DOM. Empty unless
|
||||
* `discoverSpaRoutes` was set.
|
||||
*/
|
||||
spaRoutes?: string[];
|
||||
}
|
||||
|
||||
/** V2 Faz 16 — in-page accessibility audit result. */
|
||||
@@ -284,6 +296,82 @@ const A11Y_AUDIT_FN = `(() => {
|
||||
* pages warm but does not isolate state per-render. Callers that need state
|
||||
* isolation must drop and recreate the context.
|
||||
*/
|
||||
/**
|
||||
* V2 Bölüm 29 — SPA route discovery.
|
||||
*
|
||||
* A single-page app can serve dozens of "pages" without ever making a
|
||||
* document request: the router calls `history.pushState` (or flips the
|
||||
* hash) and re-renders in place. A crawler that only follows `<a href>`
|
||||
* from the settled DOM sees one URL and reports the rest as missing.
|
||||
*
|
||||
* This init script wraps the History API before any page script runs and
|
||||
* records every URL the app routes to. It is injected once per pooled
|
||||
* page (Playwright has no `removeInitScript`, and pages are reused), so
|
||||
* it must be idempotent — the `__freecrawlSpaHook` guard makes it so.
|
||||
* The recorded list is per-document: Playwright re-runs init scripts on
|
||||
* every navigation, which resets `__freecrawlRoutes` for the next URL.
|
||||
*/
|
||||
const SPA_ROUTE_HOOK = `(() => {
|
||||
if (window.__freecrawlSpaHook) return;
|
||||
window.__freecrawlSpaHook = true;
|
||||
var MAX = 200;
|
||||
var seen = [];
|
||||
window.__freecrawlRoutes = seen;
|
||||
function record() {
|
||||
try {
|
||||
var href = String(location.href);
|
||||
if (!href) return;
|
||||
if (seen.length >= MAX) return;
|
||||
if (seen.indexOf(href) === -1) seen.push(href);
|
||||
} catch (e) { /* cross-origin frame — ignore */ }
|
||||
}
|
||||
['pushState', 'replaceState'].forEach(function (name) {
|
||||
var original = history[name];
|
||||
if (typeof original !== 'function') return;
|
||||
history[name] = function () {
|
||||
var result = original.apply(this, arguments);
|
||||
record();
|
||||
return result;
|
||||
};
|
||||
});
|
||||
window.addEventListener('popstate', record);
|
||||
window.addEventListener('hashchange', record);
|
||||
record();
|
||||
})()`;
|
||||
|
||||
/**
|
||||
* Collect what the hook recorded, plus hash-router links present in the
|
||||
* settled DOM. The DOM half matters because a hash-router site's nav is
|
||||
* `<a href="#/about">` — real links the crawler can follow without
|
||||
* clicking, but only if they are resolved to absolute URLs here.
|
||||
*/
|
||||
const SPA_ROUTE_COLLECT_FN = `(() => {
|
||||
var out = [];
|
||||
function push(href) {
|
||||
if (!href) return;
|
||||
if (out.length >= 300) return;
|
||||
if (out.indexOf(href) === -1) out.push(href);
|
||||
}
|
||||
try {
|
||||
var recorded = window.__freecrawlRoutes || [];
|
||||
for (var i = 0; i < recorded.length; i++) push(recorded[i]);
|
||||
} catch (e) { /* ignore */ }
|
||||
try {
|
||||
var anchors = document.querySelectorAll('a[href*="#/"], a[href*="#!/"]');
|
||||
for (var j = 0; j < anchors.length && j < 500; j++) {
|
||||
var raw = anchors[j].getAttribute('href');
|
||||
if (!raw) continue;
|
||||
try { push(new URL(raw, location.href).href); } catch (e) { /* skip */ }
|
||||
}
|
||||
} catch (e) { /* ignore */ }
|
||||
return out;
|
||||
})()`;
|
||||
|
||||
/** Pages that already carry the SPA hook. Playwright cannot remove an
|
||||
* init script, and the pool hands the same page back for the next URL,
|
||||
* so re-adding it on every render would stack duplicates. */
|
||||
const spaHookedPages = new WeakSet<object>();
|
||||
|
||||
export async function renderUrl(
|
||||
url: string,
|
||||
pool: BrowserPool,
|
||||
@@ -323,6 +411,15 @@ export async function renderUrl(
|
||||
if (opts.viewport) {
|
||||
await page.setViewportSize(opts.viewport);
|
||||
}
|
||||
if (opts.discoverSpaRoutes && !spaHookedPages.has(page)) {
|
||||
try {
|
||||
await page.addInitScript(SPA_ROUTE_HOOK);
|
||||
spaHookedPages.add(page);
|
||||
} catch {
|
||||
// A page that refuses the init script still renders — route
|
||||
// discovery just falls back to whatever the DOM exposes.
|
||||
}
|
||||
}
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: opts.waitUntil,
|
||||
timeout: opts.timeoutMs,
|
||||
@@ -375,6 +472,20 @@ export async function renderUrl(
|
||||
}
|
||||
}
|
||||
|
||||
// SPA routes — read whatever the hook recorded during hydration and
|
||||
// the hash-router links in the settled DOM.
|
||||
let spaRoutes: string[] | undefined;
|
||||
if (opts.discoverSpaRoutes) {
|
||||
try {
|
||||
const collected = (await page.evaluate(SPA_ROUTE_COLLECT_FN)) as unknown;
|
||||
spaRoutes = Array.isArray(collected)
|
||||
? collected.filter((r): r is string => typeof r === 'string')
|
||||
: [];
|
||||
} catch {
|
||||
spaRoutes = [];
|
||||
}
|
||||
}
|
||||
|
||||
// Screenshot capture — best-effort. Failures (e.g. detached page)
|
||||
// do not fail the render result.
|
||||
const screenshots: { fullpage?: string; fold?: string } = {};
|
||||
@@ -417,6 +528,7 @@ export async function renderUrl(
|
||||
screenshots: Object.keys(screenshots).length > 0 ? screenshots : undefined,
|
||||
lcp: opts.detectLcp ? lcp : undefined,
|
||||
a11y: opts.detectA11y ? a11y : undefined,
|
||||
spaRoutes,
|
||||
};
|
||||
} catch (err) {
|
||||
result = {
|
||||
|
||||
@@ -35,6 +35,64 @@ export function robotsUserAgentToken(userAgent: string): string {
|
||||
return leading && leading.length > 0 ? leading : userAgent;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* RFC 9309 §2.5: a crawler MUST parse at least 500 KiB of robots.txt and
|
||||
* MAY ignore the rest. Without a cap the runtime loader called
|
||||
* `res.text()` on whatever the origin sent — a misconfigured server that
|
||||
* answers /robots.txt with a database dump (or a hostile one that streams
|
||||
* forever) would be buffered into memory in full, once per origin, on the
|
||||
* crawl's hot path.
|
||||
*/
|
||||
export const MAX_ROBOTS_BYTES = 512 * 1024;
|
||||
|
||||
/**
|
||||
* Read at most `cap` bytes of a response body, then cancel the stream so
|
||||
* the rest is never transferred. Returns the decoded text plus whether the
|
||||
* cap fired (the caller decides whether that is worth reporting).
|
||||
*/
|
||||
export async function readBodyCapped(
|
||||
body: ReadableStream<Uint8Array> | null,
|
||||
cap = MAX_ROBOTS_BYTES,
|
||||
): Promise<{ text: string; truncated: boolean }> {
|
||||
if (!body) return { text: '', truncated: false };
|
||||
const reader = body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
let truncated = false;
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value || value.byteLength === 0) continue;
|
||||
const remaining = cap - total;
|
||||
if (value.byteLength >= remaining) {
|
||||
chunks.push(value.subarray(0, remaining));
|
||||
total = cap;
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
chunks.push(value);
|
||||
total += value.byteLength;
|
||||
}
|
||||
} finally {
|
||||
// Cancel rather than drain: on a truncated read the origin is still
|
||||
// sending, and we want the socket released now.
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
}
|
||||
const merged = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
merged.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return { text: new TextDecoder('utf-8').decode(merged), truncated };
|
||||
}
|
||||
|
||||
export async function loadRobots(
|
||||
origin: string,
|
||||
userAgent: string,
|
||||
@@ -57,7 +115,12 @@ export async function loadRobots(
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (res.ok) {
|
||||
const body = await res.text();
|
||||
const { text: body, truncated } = await readBodyCapped(res.body);
|
||||
if (truncated && onNotice) {
|
||||
onNotice(
|
||||
`robots.txt for ${origin} is larger than ${Math.round(MAX_ROBOTS_BYTES / 1024)} KB; only the first ${Math.round(MAX_ROBOTS_BYTES / 1024)} KB were parsed (RFC 9309 §2.5).`,
|
||||
);
|
||||
}
|
||||
const parser = robotsParser(robotsUrl, body);
|
||||
return {
|
||||
isAllowed: (url: string) => parser.isAllowed(url, uaToken) ?? true,
|
||||
@@ -174,7 +237,7 @@ export async function testUrlAgainstRobots(
|
||||
});
|
||||
status = res.status;
|
||||
if (res.ok) {
|
||||
const raw = await res.text();
|
||||
const { text: raw } = await readBodyCapped(res.body);
|
||||
body = raw.length > 8192 ? raw.slice(0, 8189) + '...' : raw;
|
||||
} else {
|
||||
try {
|
||||
|
||||
@@ -45,6 +45,19 @@ export interface UrlRewriteOptions {
|
||||
* order, and reordering those would fetch the wrong page.
|
||||
*/
|
||||
sortQueryParams?: boolean;
|
||||
/**
|
||||
* Keep SPA hash routes (`#/about`, `#!/products/1`) as part of the URL
|
||||
* instead of stripping the fragment.
|
||||
*
|
||||
* A fragment is normally not part of a resource's identity — `#pricing`
|
||||
* is a position on a page the server already returned in full. But an
|
||||
* Angular/Vue hash-router app serves its entire site under one document
|
||||
* and distinguishes pages ONLY by that fragment, so stripping it
|
||||
* collapses a 500-page site into a single row. Restricted to the
|
||||
* hash-route forms (`#/…`, `#!/…`): a plain `#section` anchor stays
|
||||
* stripped, because it really is the same page.
|
||||
*/
|
||||
keepHashRoutes?: boolean;
|
||||
/**
|
||||
* Collapse runs of `/` in the path to a single slash (`/a//b` -> `/a/b`).
|
||||
* Web servers serve these identically, so the duplicate-slash variant is
|
||||
@@ -83,6 +96,14 @@ const DEFAULT_TRACKING_PARAMS = [
|
||||
'mc_eid',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Fragments that identify a page rather than a position in one:
|
||||
* `#/path` (Angular / Vue hash mode) and `#!/path` (the legacy
|
||||
* hashbang). Anything else — `#pricing`, `#`, `#L42` — is an in-page
|
||||
* anchor and stays stripped.
|
||||
*/
|
||||
const HASH_ROUTE_RE = /^#!?\/./;
|
||||
|
||||
export function compileUrlRegexRewrites(
|
||||
raw: readonly UrlRegexRewrite[] | undefined,
|
||||
onError?: (pattern: string, err: string) => void,
|
||||
@@ -117,6 +138,9 @@ export function normalizeUrl(
|
||||
// means genuine separators like `?a=1&b=2` are left untouched.
|
||||
const decoded = raw.replace(/&(?:amp|#0*38|#x0*26);/gi, '&');
|
||||
const u = new URL(decoded, base);
|
||||
// Captured before the strip so the hash-route policy below can put a
|
||||
// router path back; everything in between operates on a fragmentless URL.
|
||||
const rawHash = u.hash;
|
||||
u.hash = '';
|
||||
// Drop HTTP-auth userinfo. `https://user:pass@host` is a common way to
|
||||
// crawl a Basic-auth staging site, but the credentials must not survive
|
||||
@@ -182,6 +206,12 @@ export function normalizeUrl(
|
||||
if (u.pathname.includes('%')) u.pathname = upperPercentEscapes(u.pathname);
|
||||
if (u.search.includes('%')) u.search = upperPercentEscapes(u.search);
|
||||
|
||||
// Re-attach a router fragment last, so no earlier rewrite (trailing
|
||||
// slash, lowercase path, percent folding) ever sees or mangles it.
|
||||
if (rewrites.keepHashRoutes && HASH_ROUTE_RE.test(rawHash)) {
|
||||
u.hash = rawHash;
|
||||
}
|
||||
|
||||
let result = u.toString();
|
||||
if (rewrites.regexRewrites && rewrites.regexRewrites.length > 0) {
|
||||
for (const { re, replacement } of rewrites.regexRewrites) {
|
||||
|
||||
@@ -7696,6 +7696,15 @@ export class ProjectDb {
|
||||
args.push(`%${input.search.trim()}%`);
|
||||
}
|
||||
if (filter === 'bots') where.push('bot_hits > 0');
|
||||
const statusClass = input.status ?? 'all';
|
||||
if (statusClass !== 'all') {
|
||||
// `last_status` is the only status the aggregate keeps; a NULL means
|
||||
// the line had no parseable status and is excluded rather than
|
||||
// silently counted as a match.
|
||||
const base = Number(statusClass[0]) * 100;
|
||||
where.push('last_status >= ? AND last_status < ?');
|
||||
args.push(base, base + 100);
|
||||
}
|
||||
if (input.bot && input.bot.trim()) {
|
||||
// Per-named-bot filter — only paths this specific bot hit.
|
||||
where.push(
|
||||
@@ -11040,9 +11049,8 @@ function computeFolderDepth(rawUrl: string): number {
|
||||
function computeQueryParamCount(rawUrl: string): number {
|
||||
try {
|
||||
const u = new URL(rawUrl);
|
||||
let n = 0;
|
||||
for (const _ of u.searchParams) n++;
|
||||
return n;
|
||||
// `size` counts repeated keys the same way the old loop did.
|
||||
return u.searchParams.size;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1250,6 +1250,34 @@ export interface CrawlConfig {
|
||||
* record them. Turns `discoverSitemaps` from a reporting aid into a
|
||||
* discovery source, which is what finds orphan pages. */
|
||||
crawlLinkedSitemaps: boolean;
|
||||
/**
|
||||
* Whether URLs carrying a query string are taken into the crawl at all.
|
||||
* On (default) crawls `?…` URLs like any other. Off drops them at
|
||||
* enqueue, before robots and before a request goes out — the cheap way
|
||||
* to keep a faceted navigation (`?color=red&size=xl&sort=price`) from
|
||||
* eating the whole URL budget on what is one product listing wearing a
|
||||
* thousand URLs.
|
||||
*
|
||||
* Scope of the switch, deliberately narrow:
|
||||
* - Spider mode only. List and Sitemap mode crawl exactly the URLs the
|
||||
* user supplied, and there is no link-follow there to run away with.
|
||||
* - Pages only. Subresources (image / CSS / JS / media rows) are exempt
|
||||
* because `style.css?v=7` is a cache-buster, not a facet.
|
||||
* - The start URL is always crawled, query string and all.
|
||||
*/
|
||||
crawlQueryStrings: boolean;
|
||||
/**
|
||||
* Parameter names that keep a URL crawlable while `crawlQueryStrings`
|
||||
* is off — pagination (`page`), language switches (`lang`), a product
|
||||
* id. Matched case-insensitively against parameter *names*, values are
|
||||
* not looked at.
|
||||
*
|
||||
* A URL is admitted only when **every** parameter it carries is on this
|
||||
* list: `?page=2` passes, `?page=2&color=red` does not. Any-match would
|
||||
* defeat the point, since a facet URL nearly always carries the
|
||||
* pagination parameter too.
|
||||
*/
|
||||
crawlQueryStringExceptions: string[];
|
||||
|
||||
/**
|
||||
* Wave 6 — Per-pass crawl-analysis toggles. Each post-crawl pass
|
||||
@@ -1380,6 +1408,15 @@ export interface JsRenderConfig {
|
||||
* runs in JS mode. Off by default — adds an in-page evaluate pass.
|
||||
*/
|
||||
a11yAudit: boolean;
|
||||
/**
|
||||
* Discover client-side routes an SPA reaches without a document
|
||||
* request: the History API (`pushState` / `replaceState` / `popstate`)
|
||||
* is hooked before the page's own scripts run, and hash-router links
|
||||
* (`#/about`) survive URL normalisation instead of collapsing onto the
|
||||
* shell document. Off by default — a hash-routed site can otherwise
|
||||
* multiply into far more URLs than the operator expected.
|
||||
*/
|
||||
spaRouting: boolean;
|
||||
}
|
||||
|
||||
export interface HttpAuth {
|
||||
@@ -2309,6 +2346,8 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
|
||||
followExternalNofollow: false,
|
||||
crawlInvalidLinks: false,
|
||||
crawlLinkedSitemaps: false,
|
||||
crawlQueryStrings: true,
|
||||
crawlQueryStringExceptions: [],
|
||||
analyseInlinks: true,
|
||||
analyseLinkScore: true,
|
||||
analyseRedirectChains: true,
|
||||
@@ -2343,6 +2382,7 @@ export const DEFAULT_CRAWL_CONFIG: CrawlConfig = {
|
||||
mobileUsability: false,
|
||||
lcpCandidate: false,
|
||||
a11yAudit: false,
|
||||
spaRouting: false,
|
||||
},
|
||||
performanceBudget: {
|
||||
enabled: false,
|
||||
|
||||
@@ -9,6 +9,7 @@ export * from './crux.js';
|
||||
export * from './spelling.js';
|
||||
export * from './google.js';
|
||||
export * from './ai.js';
|
||||
export * from './mcp.js';
|
||||
export * from './seo.js';
|
||||
export * from './loganalyzer.js';
|
||||
export * from './export-datasets.js';
|
||||
@@ -49,6 +49,16 @@ import type {
|
||||
UrlAnalyticsDetail,
|
||||
} from './google.js';
|
||||
import type { AiProvider, AiRow } from './ai.js';
|
||||
import type {
|
||||
AssistantApprovalInput,
|
||||
AssistantEvent,
|
||||
AssistantSendInput,
|
||||
AssistantSendResult,
|
||||
McpServerConfig,
|
||||
McpServerStatus,
|
||||
McpToolPolicy,
|
||||
McpToolsListResult,
|
||||
} from './mcp.js';
|
||||
import type { SeoProvider, SeoRow } from './seo.js';
|
||||
import type {
|
||||
LogAnalyzeInput,
|
||||
@@ -60,7 +70,6 @@ import type {
|
||||
LogStatusRow,
|
||||
LogTrendRow,
|
||||
LogCrawlBudgetRow,
|
||||
LogOrphanRow,
|
||||
LogDiscoveryRow,
|
||||
LogSeedDiscoveryResult,
|
||||
LogExportInput,
|
||||
@@ -341,6 +350,29 @@ export const IPC = {
|
||||
spellingLanguages: 'spelling:languages',
|
||||
/** main → renderer: live progress of an in-flight spelling run. */
|
||||
spellingProgress: 'spelling:progress',
|
||||
/** Faz 5 — MCP client. `mcpServersList` / `mcpServersSave` manage the
|
||||
* registered external MCP servers (secret values masked on the way out,
|
||||
* encrypted on the way in); `mcpServerConnect` / `mcpServerDisconnect`
|
||||
* open and close one session; `mcpServerStatuses` reports them all;
|
||||
* `mcpToolsList` returns the tool catalogue of the chosen servers. */
|
||||
mcpServersList: 'mcp:servers-list',
|
||||
mcpServersSave: 'mcp:servers-save',
|
||||
mcpServerConnect: 'mcp:server-connect',
|
||||
mcpServerDisconnect: 'mcp:server-disconnect',
|
||||
mcpServerStatuses: 'mcp:server-statuses',
|
||||
mcpToolsList: 'mcp:tools-list',
|
||||
/** Remembered per-tool approval decisions (`ask` / `always` / `never`). */
|
||||
mcpPolicySet: 'mcp:policy-set',
|
||||
mcpPolicyList: 'mcp:policy-list',
|
||||
/** Faz 5 — assistant chat. `assistantSend` runs one agentic turn
|
||||
* (tool rounds included) and resolves with the messages to append;
|
||||
* `assistantCancel` aborts it; `assistantApprove` answers a pending
|
||||
* tool-approval card. */
|
||||
assistantSend: 'assistant:send',
|
||||
assistantCancel: 'assistant:cancel',
|
||||
assistantApprove: 'assistant:approve',
|
||||
/** main → renderer: live assistant events (tool calls, results, errors). */
|
||||
assistantEvent: 'assistant:event',
|
||||
/** Faz 7 — Google OAuth keystone (shared by Search Console, GA4,
|
||||
* Sheets). `googleAuthStart` opens the consent screen in the browser
|
||||
* and catches the loopback redirect; `googleAuthStatus` reports the
|
||||
@@ -2097,6 +2129,33 @@ export interface FreeCrawlApi {
|
||||
logThreatSummary(): Promise<LogThreatSummary>;
|
||||
/** Senders of flagged requests, busiest first — backs the blocklist copy. */
|
||||
logThreatIps(limit?: number): Promise<LogThreatIpRow[]>;
|
||||
/** Faz 5 — registered MCP servers, with every secret value masked. */
|
||||
mcpServersList(): Promise<McpServerConfig[]>;
|
||||
/** Replace the server registry. Values that come back as the mask are
|
||||
* left as they were; anything else is re-encrypted at rest. */
|
||||
mcpServersSave(servers: McpServerConfig[]): Promise<McpServerConfig[]>;
|
||||
/** Open a session with one server (spawns it, for stdio). */
|
||||
mcpServerConnect(id: string): Promise<McpServerStatus>;
|
||||
/** Close one server's session. */
|
||||
mcpServerDisconnect(id: string): Promise<McpServerStatus>;
|
||||
/** Connection state of every registered server. */
|
||||
mcpServerStatuses(): Promise<McpServerStatus[]>;
|
||||
/** Tool catalogue for the given servers, plus FreeCrawl's own built-in
|
||||
* project tools. Connects on demand. */
|
||||
mcpToolsList(serverIds: string[]): Promise<McpToolsListResult>;
|
||||
/** Remember (or clear) an approval decision for one tool. */
|
||||
mcpPolicySet(serverId: string, tool: string, policy: McpToolPolicy): Promise<void>;
|
||||
/** Every remembered decision, keyed `<serverId>:<tool>`. */
|
||||
mcpPolicyList(): Promise<Record<string, McpToolPolicy>>;
|
||||
/** Run one assistant turn — the model may call tools several times
|
||||
* before answering. Subscribe to `onAssistantEvent` for live progress. */
|
||||
assistantSend(input: AssistantSendInput): Promise<AssistantSendResult>;
|
||||
/** Abort the in-flight assistant turn for this window. */
|
||||
assistantCancel(): Promise<void>;
|
||||
/** Answer a pending tool-approval card. */
|
||||
assistantApprove(input: AssistantApprovalInput): Promise<void>;
|
||||
/** Live assistant events while a turn runs. */
|
||||
onAssistantEvent(cb: (e: AssistantEvent) => void): () => void;
|
||||
onProgress(cb: (p: CrawlProgress) => void): () => void;
|
||||
onDone(cb: (summary: CrawlSummary) => void): () => void;
|
||||
onError(cb: (message: string) => void): () => void;
|
||||
|
||||
@@ -263,6 +263,16 @@ export interface LogUrlStatsInput {
|
||||
/** Restrict to URLs a specific named bot hit (e.g. "SemrushBot").
|
||||
* Composes with `search` / `filter`. */
|
||||
bot?: string;
|
||||
/**
|
||||
* Response-class filter, matching the Suspicious Requests tab.
|
||||
*
|
||||
* Judged on `lastStatus` — the analyzer stores one status per path
|
||||
* (the most recent one seen), not a per-path distribution, so this
|
||||
* answers "what is this URL returning now", which is the question a
|
||||
* `4xx` / `5xx` filter is actually asked. Paths with no parsed status
|
||||
* are excluded once a class is selected.
|
||||
*/
|
||||
status?: 'all' | '2xx' | '3xx' | '4xx' | '5xx';
|
||||
}
|
||||
|
||||
export interface LogUrlStatsResult {
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
/**
|
||||
* Faz 5 — MCP client (in-app AI assistant).
|
||||
*
|
||||
* FreeCrawl already ships an MCP *server* (`apps/mcp-server`) so external
|
||||
* agents can read a crawl. This is the mirror image: the desktop app acting
|
||||
* as an MCP *client*, so the user can interrogate their own crawl in a chat
|
||||
* pane with their own LLM key, and chain in third-party MCP servers (Brave
|
||||
* Search, GitHub, an internal enterprise tool) for context the crawl itself
|
||||
* does not have.
|
||||
*
|
||||
* Three moving parts:
|
||||
*
|
||||
* 1. **Server registry** — `McpServerConfig` rows the user manages in
|
||||
* Settings → MCP Servers. Secret values (env vars, auth headers) are
|
||||
* encrypted at rest with the same `safeStorage` store the integration
|
||||
* credentials use; the renderer only ever sees `SECRET_MASK`.
|
||||
* 2. **Connections** — one live JSON-RPC session per enabled server,
|
||||
* stdio (child process) or Streamable HTTP.
|
||||
* 3. **Assistant loop** — an LLM turn that may call FreeCrawl's built-in
|
||||
* read-only project tools and any approved MCP tool, feeding results
|
||||
* back until the model answers in prose.
|
||||
*
|
||||
* Every external tool call is gated by an explicit user decision (allow
|
||||
* once / always / deny). Built-in tools are read-only queries against the
|
||||
* open project and run without a prompt, but are still logged in the UI.
|
||||
*/
|
||||
|
||||
/** How FreeCrawl talks to one MCP server. */
|
||||
export type McpTransport = 'stdio' | 'http';
|
||||
|
||||
/** Placeholder the main process substitutes for any stored secret value.
|
||||
* A save that echoes this back means "keep what you already have". */
|
||||
export const MCP_SECRET_MASK = '••••••••';
|
||||
|
||||
/** Hard ceiling on registered servers — keeps the picker and the tool
|
||||
* list a human-sized surface, and bounds the child processes we spawn. */
|
||||
export const MCP_MAX_SERVERS = 12;
|
||||
|
||||
/** One MCP server the user has registered. */
|
||||
export interface McpServerConfig {
|
||||
/** Stable slug, also used to namespace the server's tools. */
|
||||
id: string;
|
||||
/** Display name in the picker. */
|
||||
label: string;
|
||||
transport: McpTransport;
|
||||
/** stdio: executable to spawn (resolved against PATH). */
|
||||
command: string;
|
||||
/** stdio: argument vector — never a shell string, so nothing is split
|
||||
* or expanded on the user's behalf. */
|
||||
args: string[];
|
||||
/** stdio: extra environment variables. Values are secrets at rest. */
|
||||
env: Record<string, string>;
|
||||
/** http: endpoint URL (Streamable HTTP / JSON-RPC over POST). */
|
||||
url: string;
|
||||
/** http: extra request headers (e.g. `Authorization`). Values are
|
||||
* secrets at rest. */
|
||||
headers: Record<string, string>;
|
||||
/** Unchecked servers stay registered but are never connected. */
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
/** Live state of one registered server. */
|
||||
export interface McpServerStatus {
|
||||
id: string;
|
||||
connected: boolean;
|
||||
/** Present while a connect attempt is in flight. */
|
||||
connecting: boolean;
|
||||
/** Last failure, cleared on a successful connect. */
|
||||
error: string | null;
|
||||
/** Tools advertised by the server (0 before the first connect). */
|
||||
toolCount: number;
|
||||
/** `serverInfo` from the MCP handshake, when connected. */
|
||||
serverName: string | null;
|
||||
serverVersion: string | null;
|
||||
protocolVersion: string | null;
|
||||
}
|
||||
|
||||
/** One tool the assistant can call. */
|
||||
export interface McpToolInfo {
|
||||
/** Owning server id, or `builtin` for FreeCrawl's own project tools. */
|
||||
serverId: string;
|
||||
/** Tool name as the server spells it. */
|
||||
name: string;
|
||||
/** Name exposed to the LLM — namespaced so two servers can both ship a
|
||||
* `search` tool without colliding. */
|
||||
qualifiedName: string;
|
||||
description: string;
|
||||
/** JSON Schema for the tool's arguments, as sent by the server. */
|
||||
inputSchema: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* What FreeCrawl does when the assistant wants to call a tool.
|
||||
*
|
||||
* `ask` — prompt the user in the chat pane (default for MCP tools)
|
||||
* `always` — run it without prompting (user chose "Always allow")
|
||||
* `never` — refuse and tell the model the user denied it
|
||||
*/
|
||||
export type McpToolPolicy = 'ask' | 'always' | 'never';
|
||||
|
||||
/** A remembered per-tool decision. */
|
||||
export interface McpToolPolicyEntry {
|
||||
serverId: string;
|
||||
tool: string;
|
||||
policy: McpToolPolicy;
|
||||
}
|
||||
|
||||
// ── Assistant conversation ───────────────────────────────────────────
|
||||
|
||||
export type AssistantRole = 'user' | 'assistant' | 'tool';
|
||||
|
||||
/** One tool invocation requested by the model. */
|
||||
export interface AssistantToolCall {
|
||||
/** Provider-assigned id — echoed back with the result. */
|
||||
id: string;
|
||||
/** Qualified tool name (`builtin__query_urls`, `mcp__github__search`). */
|
||||
name: string;
|
||||
arguments: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** One message in the conversation. Provider-neutral: the main process
|
||||
* converts this shape into whatever OpenAI / Anthropic / Ollama want. */
|
||||
export interface AssistantMessage {
|
||||
role: AssistantRole;
|
||||
content: string;
|
||||
/** Set on assistant messages that requested tools. */
|
||||
toolCalls?: AssistantToolCall[];
|
||||
/** Set on tool messages — which call this answers. */
|
||||
toolCallId?: string;
|
||||
/** Set on tool messages — for the UI, so a result card can be labelled. */
|
||||
toolName?: string;
|
||||
/** Tool result that failed (or was denied). */
|
||||
isError?: boolean;
|
||||
/** Wall-clock duration of a tool call, for the UI. */
|
||||
durationMs?: number;
|
||||
}
|
||||
|
||||
/** Live events emitted while a turn runs. */
|
||||
export type AssistantEvent =
|
||||
| { type: 'thinking'; round: number }
|
||||
| { type: 'assistant-text'; text: string; toolCalls: AssistantToolCall[] }
|
||||
| {
|
||||
type: 'tool-approval';
|
||||
requestId: string;
|
||||
call: AssistantToolCall;
|
||||
serverId: string;
|
||||
serverLabel: string;
|
||||
}
|
||||
| { type: 'tool-start'; call: AssistantToolCall }
|
||||
| {
|
||||
type: 'tool-result';
|
||||
callId: string;
|
||||
name: string;
|
||||
result: string;
|
||||
isError: boolean;
|
||||
durationMs: number;
|
||||
}
|
||||
| { type: 'error'; message: string };
|
||||
|
||||
/** Why a turn stopped. */
|
||||
export type AssistantStopReason =
|
||||
| 'done'
|
||||
| 'cancelled'
|
||||
| 'error'
|
||||
| 'max-rounds';
|
||||
|
||||
export interface AssistantSendInput {
|
||||
provider: 'openai' | 'anthropic' | 'ollama';
|
||||
/** Empty means "use the provider's default model". */
|
||||
model?: string;
|
||||
/** Full conversation so far, oldest first, ending with the new user
|
||||
* message. The renderer owns the transcript; the main process owns the
|
||||
* tool loop. */
|
||||
messages: AssistantMessage[];
|
||||
/** Which registered servers to expose for this turn. */
|
||||
serverIds: string[];
|
||||
/** Expose FreeCrawl's own read-only project tools (default true). */
|
||||
includeBuiltins?: boolean;
|
||||
}
|
||||
|
||||
export interface AssistantSendResult {
|
||||
/** Messages produced by this turn, to append to the transcript. */
|
||||
messages: AssistantMessage[];
|
||||
stopReason: AssistantStopReason;
|
||||
error: string | null;
|
||||
tokensIn: number | null;
|
||||
tokensOut: number | null;
|
||||
/** Model that actually answered — defaults resolve in the main process. */
|
||||
model: string;
|
||||
}
|
||||
|
||||
/** The renderer's answer to a `tool-approval` event. */
|
||||
export interface AssistantApprovalInput {
|
||||
requestId: string;
|
||||
decision: 'allow-once' | 'allow-always' | 'deny';
|
||||
}
|
||||
|
||||
/** Result of asking for the tool catalogue of several servers at once.
|
||||
* A server that fails to connect lands in `errors` instead of taking the
|
||||
* whole listing down with it. */
|
||||
export interface McpToolsListResult {
|
||||
tools: McpToolInfo[];
|
||||
errors: { id: string; message: string }[];
|
||||
}
|
||||
Reference in new issue
Block a user