pretecno/FreeCrawl-SEO-Tool
3
0 Fork 0
mirror of https://github.com/kemalai/FreeCrawl-SEO-Tool.git synced 2026-10-18 00:39:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): Yandex Metrica and desktop-saved projects (#30)

Browse Source
This commit is contained in:
danzerzine authored and GitHub committed 2026-10-17 03:29:35 +03:00
1 parent 0f2c828d2c
commit e31fd1f367
7 files changed
+1304 -525
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/cli/src/index.ts
+719 -25
View File
@@ -1,8 +1,25 @@
#!/usr/bin/env node
import { parseArgs } from 'node:util';
import { basename, resolve } from 'node:path';
import { readFileSync } from 'node:fs';
import { DEFAULT_CRAWL_CONFIG, type LogIngestInput } from '@freecrawl/shared-types';
import { basename, join, resolve } from 'node:path';
import {
closeSync,
existsSync,
mkdtempSync,
openSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { createHash, randomUUID } from 'node:crypto';
import {
DEFAULT_CRAWL_CONFIG,
type CrawlConfig,
type LogIngestInput,
type MetricaFetchMeta,
} from '@freecrawl/shared-types';
import {
Crawler,
compareCrawls,
@@ -12,9 +29,15 @@ import {
fetchSitemaps,
validateSitemap,
analyzeLogFile,
normalizeUrl,
compileUrlRegexRewrites,
listMetricaCounters,
fetchMetricaPageMetrics,
MetricaError,
MetricaCancelledError,
type LogAnalysisResult,
} from '@freecrawl/core';
import { ProjectDb } from '@freecrawl/db';
import { ProjectDb, detectProjectFormat, packProject, unpackProject } from '@freecrawl/db';
function help(): void {
console.log(`freecrawl — headless SEO crawler
@@ -29,6 +52,10 @@ Usage:
freecrawl analyze-logs <access.log> [more.log …] [--project file]
Parse one or more server access logs (Apache / Nginx / IIS / custom), detect bots,
and merge the aggregates into a project for crawl × log analysis
freecrawl metrica-counters [--json] List the Yandex Metrica counters the token can read
freecrawl metrica-fetch --counter <id> [--days 7|28|90] [--project file] [--config file]
Pull per-page Yandex Metrica data (pageviews, visits, users,
search visits) into a project, as the desktop app does
Options:
--depth <n> Max crawl depth (default: ${DEFAULT_CRAWL_CONFIG.maxDepth})
@@ -62,6 +89,25 @@ analyze-logs options:
--regex <pattern> Named-group regex for --format custom (groups: ip ts method path status bytes ua referer)
--verify-bots Reverse-DNS verify a sample of bot IPs (slower; catches spoofed user-agents)
-h, --help Show this help
metrica-counters / metrica-fetch options:
The token comes from the YANDEX_METRICA_TOKEN environment variable
(a personal OAuth token with the metrika:read scope).
--counter <id> Counter to pull (see metrica-counters)
--days <n> 7 | 28 | 90 full days ending yesterday (default: 28)
--project <file> Existing project to store the data in (default: ./crawl.seoproject)
--config <file> Crawl config whose URL rewrite options map Metrica URLs onto crawled
URLs (default: the crawl config the desktop app saved in the project,
else the defaults; a CLI crawl does not save one, so pass the same
--config file it used)
--json Print the result, or the error, as JSON on stdout
Projects:
compare, analyze-logs and metrica-fetch open projects saved by the desktop app as well as
CLI crawl databases. While analyze-logs or metrica-fetch writes to a desktop-saved project,
it holds <project>.lock, and a second such command on the same file is refused. The desktop
app does not check that lock: if it has the same project open, its next Save overwrites what
the CLI wrote (the app works on its own copy of the file).
`);
}
@@ -89,6 +135,8 @@ async function main(): Promise<void> {
format: { type: 'string' },
regex: { type: 'string' },
'verify-bots': { type: 'boolean' },
counter: { type: 'string' },
days: { type: 'string' },
help: { type: 'boolean', short: 'h' },
},
});
@@ -125,7 +173,8 @@ async function main(): Promise<void> {
);
process.exit(2);
}
const exitCode = runCompare(
exitOnInterrupt();
const exitCode = await runCompare(
resolve(positionals[1]),
resolve(positionals[2]),
);
@@ -136,6 +185,7 @@ async function main(): Promise<void> {
console.error('Usage: freecrawl analyze-logs <access.log> [more.log …] [--project file] [--format f] [--regex p] [--verify-bots]');
process.exit(2);
}
exitOnInterrupt();
const exitCode = await runAnalyzeLogs(
positionals.slice(1).map((p) => resolve(p)),
{
@@ -148,6 +198,19 @@ async function main(): Promise<void> {
);
process.exit(exitCode);
}
if (positionals[0] === 'metrica-counters') {
process.exit(await runMetricaCounters(Boolean(values.json)));
}
if (positionals[0] === 'metrica-fetch') {
const exitCode = await runMetricaFetch({
counter: values.counter,
days: values.days,
projectPath: resolve(values.project ?? values.db ?? 'crawl.seoproject'),
configPath: values.config,
json: Boolean(values.json),
});
process.exit(exitCode);
}
// List mode is selected by `--list`; otherwise we need a positional URL.
const listFile = values.list;
@@ -186,13 +249,7 @@ async function main(): Promise<void> {
let fileConfig: Record<string, unknown> = {};
if (values.config) {
try {
const raw = readFileSync(resolve(values.config), 'utf8');
const parsed = JSON.parse(raw) as Record<string, unknown>;
if (parsed && typeof parsed === 'object' && 'config' in parsed) {
fileConfig = (parsed['config'] as Record<string, unknown>) ?? {};
} else {
fileConfig = parsed;
}
fileConfig = readConfigFile(values.config);
} catch (err) {
console.error(
`Cannot read --config file ${values.config}: ${(err as Error).message}`,
@@ -314,6 +371,20 @@ async function main(): Promise<void> {
process.exit(hasErrors ? 1 : 0);
}
/**
* Read a `--config` file: the Settings → Export Settings envelope
* (`{format, version, exportedAt, config}`) or a bare `CrawlConfig`
* fragment. Throws on a missing or malformed file.
*/
function readConfigFile(file: string): Record<string, unknown> {
const raw = readFileSync(resolve(file), 'utf8');
const parsed = JSON.parse(raw) as Record<string, unknown>;
if (parsed && typeof parsed === 'object' && 'config' in parsed) {
return (parsed['config'] as Record<string, unknown>) ?? {};
}
return parsed;
}
function parseNumeric(value: string | undefined, fallback: number): number {
if (value === undefined) return fallback;
const n = Number.parseInt(value, 10);
@@ -429,6 +500,313 @@ async function runAuditRobots(rawUrl: string, userAgent: string): Promise<number
return result.allowed ? 0 : 1;
}
/** An existing project a command reads or updates. */
interface OpenedProject {
db: ProjectDb;
/** Close the database and, for a desktop-saved project, pack the working
* copy back onto the original file. Call only after a successful write. */
save(): Promise<void>;
/** Close the database if still open and delete any working copy. Runs
* on every exit path; never repacks. */
dispose(): void;
}
/** Cleanups still due; the exit hook runs them after Ctrl-C or a crash, so
* no working copy is left in the temp directory. */
const pendingCleanups = new Set<() => void>();
process.on('exit', () => {
for (const cleanup of pendingCleanups) cleanup();
});
/** Ctrl-C for a command with no cancel of its own: exit through
* `process.exit`, so the exit hook still removes working copies. */
function exitOnInterrupt(): void {
// `on`, not `once`: a second Ctrl-C while the exit hook runs must not
// fall back to the default kill and cut the cleanup short.
process.on('SIGINT', () => {
process.stderr.write('\nInterrupted.\n');
process.exit(130);
});
}
/**
* Take `<project>.lock` for a write to a desktop-saved project, so two CLI
* commands can't each repack the archive and drop the other's data. A lock
* whose process is gone (killed, crashed) is taken over. Returns the
* release function, which deletes the lock only while it is still ours;
* throws a user-facing message when the lock is held.
*/
function lockProject(path: string): () => void {
const lockPath = `${path}.lock`;
const inUse = (holder: number | string): Error =>
new Error(
`The project is in use by another FreeCrawl CLI command (pid ${holder}). ` +
`Wait for it to finish, or delete ${lockPath} if no such command is running.`,
);
for (let attempt = 0; attempt < 3; attempt++) {
const mine = createExclusive(lockPath);
if (mine !== null) return () => removeIfUnchanged(lockPath, mine);
const seen = readIfExists(lockPath);
if (seen === null) continue; // released meanwhile
const holder = lockHolder(lockPath, seen);
if (holder !== null) throw inUse(holder);
removeStale(lockPath, seen, inUse);
}
throw inUse('unknown');
}
/**
* Delete the stale `file`, which held `content`, so that only one process
* ever does it. A plain check-then-delete would let a slow process delete
* the fresh lock a faster one had just created. Instead the delete happens
* only under an exclusive guard named after `content`: every lock and guard
* is written with unique content, which never reappears once deleted, so
* the guard holder knows the file it re-reads is the same stale one (a
* half-written file has no id, so for it the age is checked again). A
* guard left by a crashed process is cleared the same way, one level up.
*/
function removeStale(
file: string,
content: string,
inUse: (holder: number | string) => Error,
depth = 0,
): void {
const guardPath = `${file.replace(/\.takeover-[0-9a-f]+$/, '')}.takeover-${createHash('sha256')
.update(content)
.digest('hex')
.slice(0, 16)}`;
let guard = createExclusive(guardPath);
if (guard === null) {
const guardSeen = readIfExists(guardPath);
if (guardSeen !== null) {
const holder = lockHolder(guardPath, guardSeen);
if (holder !== null) throw inUse(holder);
if (depth >= 3) throw inUse('unknown');
removeStale(guardPath, guardSeen, inUse, depth + 1);
}
guard = createExclusive(guardPath);
if (guard === null) throw inUse('unknown');
}
try {
const now = readIfExists(file);
if (now === content && lockHolder(file, now) === null) rmSync(file, { force: true });
} finally {
removeIfUnchanged(guardPath, guard);
}
}
/** Create `file` exclusively with this process's pid, start time and a
* unique id; returns what was written, or null when it already exists. */
function createExclusive(file: string): string | null {
let fd: number;
try {
fd = openSync(file, 'wx');
} catch (err) {
if ((err as NodeJS.ErrnoException).code === 'EEXIST') return null;
throw err;
}
const content = JSON.stringify({
pid: process.pid,
startedAt: new Date().toISOString(),
id: randomUUID(),
});
try {
writeSync(fd, content);
} finally {
closeSync(fd);
}
return content;
}
function readIfExists(file: string): string | null {
try {
return readFileSync(file, 'utf8');
} catch {
return null;
}
}
/** Delete `file` only if it still holds `content`. */
function removeIfUnchanged(file: string, content: string): void {
if (readIfExists(file) === content) rmSync(file, { force: true });
}
/** The pid of a live holder of the lock `file`, which holds `content`, or
* null when the lock is stale. */
function lockHolder(file: string, content: string): number | string | null {
let pid: unknown;
try {
pid = (JSON.parse(content) as { pid?: unknown }).pid;
} catch {
// Unreadable or half-written: its owner may be writing it right now,
// so only an old one counts as stale.
try {
return Date.now() - statSync(file).mtimeMs < 10_000 ? 'unknown' : null;
} catch {
return null;
}
}
if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) return null;
try {
process.kill(pid, 0);
return pid;
} catch (err) {
// EPERM: the process exists but belongs to someone else.
return (err as NodeJS.ErrnoException).code === 'EPERM' ? pid : null;
}
}
const WORKING_COPY_PREFIX = 'freecrawl-cli-';
/** Owner file in each working copy directory, written like a lock. */
const WORKING_COPY_OWNER = 'owner.lock';
/**
* Remove the working copies that commands killed outright (a hard kill,
* a crash, a shutdown) left in the temp directory: no exit hook runs then,
* and each copy is the size of the unpacked project. A copy is orphaned
* when the process in its owner file is gone, or, with no owner file, when
* it is over an hour old. Best effort: anything still in use stays.
*/
function removeOrphanedWorkingCopies(): void {
let entries;
try {
entries = readdirSync(tmpdir(), { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
if (!entry.isDirectory() || !entry.name.startsWith(WORKING_COPY_PREFIX)) continue;
const dir = join(tmpdir(), entry.name);
try {
const ownerPath = join(dir, WORKING_COPY_OWNER);
const owner = readIfExists(ownerPath);
const orphaned =
owner === null
? Date.now() - statSync(dir).mtimeMs > 3_600_000
: lockHolder(ownerPath, owner) === null;
if (orphaned) rmSync(dir, { recursive: true, force: true });
} catch {
/* in use or not ours to remove; tried again on the next run */
}
}
}
/**
* Open an existing project file. The desktop app saves projects as a
* compressed container (`packProject`), which is unpacked into a fresh temp
* directory and opened there; `save()` packs it back. A bare SQLite project
* (a CLI crawl, or a desktop save from before the container) opens in place,
* as it always has. With `createIfMissing`, a missing path becomes a new
* bare project. With `forWrite`, a desktop-saved project is locked (see
* `lockProject`) until `dispose()`; bare SQLite relies on SQLite's own
* locking. Throws with a user-facing message on a missing, unrecognised or
* locked file.
*
* The desktop app works on its own working copy, so if it has the same
* project open, its next Save overwrites whatever the CLI wrote.
*/
async function openProject(
path: string,
opts: { createIfMissing?: boolean; forWrite?: boolean } = {},
): Promise<OpenedProject> {
let dir: string | null = null;
let workingPath: string | null = null;
let unlock: (() => void) | null = null;
if (!existsSync(path)) {
if (!opts.createIfMissing) throw new Error(`Project not found: ${path}`);
} else {
const format = detectProjectFormat(path);
if (format === 'unknown') {
throw new Error(
'This file is not a FreeCrawl project (unrecognised format). It may be corrupt or a different file type.',
);
}
if (format === 'archive') {
removeOrphanedWorkingCopies();
if (opts.forWrite) unlock = lockProject(path);
// Created synchronously and registered below before anything awaits,
// so an exit at any later point still removes it (and the lock).
try {
dir = mkdtempSync(join(tmpdir(), WORKING_COPY_PREFIX));
try {
if (createExclusive(join(dir, WORKING_COPY_OWNER)) === null) {
throw new Error(`Could not create the working copy in ${dir}`);
}
} catch (err) {
rmSync(dir, { recursive: true, force: true });
throw err;
}
} catch (err) {
unlock?.();
throw err;
}
workingPath = join(dir, basename(path));
}
}
let db: ProjectDb | undefined;
let packing = false;
const close = (): void => {
const open = db;
db = undefined;
open?.close();
};
const dispose = (): void => {
pendingCleanups.delete(dispose);
try {
close();
} catch {
/* already gone — the files are removed regardless */
}
// Separate steps, so one busy file (Windows refuses files an
// interrupted pack still holds open) doesn't stop the others.
if (packing) {
// Interrupted mid-pack: drop the half-written file; the original
// project is only replaced by the final rename.
try {
rmSync(`${path}.part`, { force: true });
} catch {
/* left beside the project; the next save overwrites it */
}
}
if (dir) {
try {
// The SQLite sidecars (-wal / -shm) live in the same directory.
rmSync(dir, { recursive: true, force: true });
} catch {
/* left in the temp directory */
}
}
if (unlock) {
try {
unlock();
} catch {
/* a stale lock is taken over by the next command */
}
}
};
pendingCleanups.add(dispose);
try {
if (workingPath) await unpackProject(path, workingPath);
db = new ProjectDb(workingPath ?? path);
} catch (err) {
dispose();
throw err;
}
return {
db,
async save() {
close();
if (!workingPath) return;
packing = true;
await packProject(workingPath, path);
packing = false;
},
dispose,
};
}
/**
* Cross-project diff. Opens two `.seoproject` files (read-only — never
* mutates), runs the same `compareCrawls` engine the desktop Compare
@@ -438,24 +816,24 @@ async function runAuditRobots(rawUrl: string, userAgent: string): Promise<number
* least one diff category had non-zero results (useful for CI gates that
* want to fail on regressions); 2 on fatal load error.
*/
function runCompare(beforePath: string, afterPath: string): number {
let before: ProjectDb;
let after: ProjectDb;
async function runCompare(beforePath: string, afterPath: string): Promise<number> {
let before: OpenedProject;
let after: OpenedProject;
try {
before = new ProjectDb(beforePath);
before = await openProject(beforePath);
} catch (err) {
console.error(`Cannot open before project: ${(err as Error).message}`);
return 2;
}
try {
after = new ProjectDb(afterPath);
after = await openProject(afterPath);
} catch (err) {
before.close();
before.dispose();
console.error(`Cannot open after project: ${(err as Error).message}`);
return 2;
}
try {
const summary = compareCrawls(before, after);
const summary = compareCrawls(before.db, after.db);
console.log(`Compare: ${beforePath} → ${afterPath}`);
console.log(` Total before: ${summary.totalA}`);
console.log(` Total after: ${summary.totalB}`);
@@ -469,8 +847,8 @@ function runCompare(beforePath: string, afterPath: string): number {
);
return totalDiff > 0 ? 1 : 0;
} finally {
before.close();
after.close();
before.dispose();
after.dispose();
}
}
@@ -545,13 +923,20 @@ async function runAnalyzeLogs(
json: boolean;
},
): Promise<number> {
let db: ProjectDb;
let project: OpenedProject;
try {
db = new ProjectDb(opts.projectPath);
// A missing project is created, so logs can be analysed without a crawl.
project = await openProject(opts.projectPath, { createIfMissing: true, forWrite: true });
} catch (err) {
console.error(`Cannot open project ${opts.projectPath}: ${(err as Error).message}`);
const error = `Cannot open project ${opts.projectPath}: ${(err as Error).message}`;
if (opts.json) {
process.stdout.write(JSON.stringify({ ok: false, error, imported: [], errors: [] }, null, 2) + '\n');
} else {
console.error(error);
}
return 2;
}
const { db } = project;
try {
const imported: ReturnType<ProjectDb['ingestLogAnalysis']>[] = [];
const failures: Array<{ file: string; error: string }> = [];
@@ -589,6 +974,7 @@ async function runAnalyzeLogs(
const topUrls = db
.queryLogUrlStats({ limit: 10, offset: 0, sortBy: 'botHits', filter: 'bots' })
.rows;
await project.save();
if (opts.json) {
process.stdout.write(
@@ -624,7 +1010,315 @@ async function runAnalyzeLogs(
console.error(`analyze-logs failed: ${(err as Error).message}`);
return 2;
} finally {
db.close();
project.dispose();
}
}
/**
* Yandex Metrica token from `YANDEX_METRICA_TOKEN` — the CLI has no
* credential store. Never echoed: the messages name the variable only.
*/
function metricaToken(): string {
const token = process.env['YANDEX_METRICA_TOKEN']?.trim() ?? '';
if (!token) {
throw new MetricaError(
'YANDEX_METRICA_TOKEN is not set — export a Yandex Metrica OAuth token with the metrika:read scope.',
'not-connected',
);
}
// Same check as the desktop app: a value with spaces or line breaks would
// otherwise reach undici, whose header error quotes the value.
if (!/^[A-Za-z0-9._~+/=-]+$/.test(token)) {
throw new MetricaError(
'YANDEX_METRICA_TOKEN has an unexpected format — it must be a single OAuth token without spaces or line breaks.',
'token-format',
);
}
return token;
}
/** Print a Metrica failure (JSON on stdout with --json, else stderr) and
* return the exit code. */
function metricaFail(json: boolean, err: unknown, exitCode: number): number {
const error = err instanceof Error ? err.message : String(err);
const errorCode = err instanceof MetricaError ? err.code : null;
const cancelled = err instanceof MetricaCancelledError;
if (json) {
process.stdout.write(
JSON.stringify({ ok: false, error, errorCode, cancelled }, null, 2) + '\n',
);
} else {
console.error(error);
if (errorCode === 'token-rejected') {
console.error('Check the token in YANDEX_METRICA_TOKEN (scope metrika:read).');
}
}
return exitCode;
}
/**
* List the counters the token can read. Exit 0 on success, 1 on an API
* failure, 2 when the token is missing or malformed.
*/
async function runMetricaCounters(json: boolean): Promise<number> {
let token: string;
try {
token = metricaToken();
} catch (err) {
return metricaFail(json, err, 2);
}
try {
const counters = await listMetricaCounters(token);
if (json) {
process.stdout.write(JSON.stringify({ ok: true, counters }, null, 2) + '\n');
return 0;
}
if (counters.length === 0) {
console.log('No Yandex Metrica counters are readable with this token.');
return 0;
}
const idWidth = Math.max(...counters.map((c) => c.counterId.length));
for (const c of counters) {
console.log(
`${c.counterId.padEnd(idWidth)} ${c.name}${c.site && c.site !== c.name ? ` (${c.site})` : ''}`,
);
}
return 0;
} catch (err) {
return metricaFail(json, err, 1);
}
}
/**
* URL normaliser for Metrica rows, mirroring the desktop's
* `crawlUrlNormalizer`: the rewrite options of the crawl config, so the
* stored rows join onto `urls.url`. The config is the `--config` file if
* given, else the project's `lastCrawlConfig` (only the desktop app saves
* one: the `{config, secretsEnc}` envelope or a bare config; secrets are
* irrelevant here), else defaults.
*/
function metricaUrlNormalizer(
db: ProjectDb,
configPath: string | undefined,
): (url: string) => string | null {
let fragment: Record<string, unknown> = {};
if (configPath) {
fragment = readConfigFile(configPath);
} else {
const raw = db.getMeta('lastCrawlConfig');
if (raw) {
try {
const parsed = JSON.parse(raw) as Record<string, unknown>;
if (parsed && typeof parsed === 'object') {
const env = parsed['config'];
fragment = env && typeof env === 'object' ? (env as Record<string, unknown>) : parsed;
}
} catch {
/* corrupt meta — fall back to the default rewrites */
}
}
}
const c = { ...DEFAULT_CRAWL_CONFIG, ...fragment } as CrawlConfig;
const rewrites = {
stripWww: c.stripWww,
forceHttps: c.forceHttps,
lowercasePath: c.lowercasePath,
trailingSlash: c.trailingSlash,
keepQueryParams: c.keepQueryParams,
sortQueryParams: c.sortQueryParams,
collapseDuplicateSlashes: c.collapseDuplicateSlashes,
keepHashRoutes: c.renderingMode === 'js' && c.jsRender?.spaRouting === true,
regexRewrites: compileUrlRegexRewrites(c.urlRegexRewrites),
};
return (url) => normalizeUrl(url, undefined, rewrites);
}
/**
* Pull one counter's per-page data into an existing project, writing what
* the desktop pull writes (rows + fetch meta, and the counter in
* `integrationSettings:metrica`), so the desktop app shows the same data.
* Ctrl-C during the pull cancels and writes nothing; once the write has
* started it exits at once, leaving the project either untouched or fully
* updated. Exit 0 on success, 1 on an API or write failure, 2 on bad
* arguments or a locked project, 130 when cancelled or interrupted.
*/
async function runMetricaFetch(opts: {
counter: string | undefined;
days: string | undefined;
projectPath: string;
configPath: string | undefined;
json: boolean;
}): Promise<number> {
// Bad arguments fail like API errors: JSON on stdout with --json.
const fail = (message: string): number => metricaFail(opts.json, new Error(message), 2);
const counterId = (opts.counter ?? '').trim();
if (!/^\d+$/.test(counterId)) {
if (!opts.json) {
console.error(
'Usage: freecrawl metrica-fetch --counter <id> [--days 7|28|90] [--project file] [--config file] [--json]',
);
}
return fail(
opts.counter === undefined
? '--counter is required.'
: `Invalid --counter: ${opts.counter} (expected a numeric counter id).`,
);
}
const days = opts.days === undefined ? 28 : Number(opts.days);
if (days !== 7 && days !== 28 && days !== 90) {
return fail(`Invalid --days: ${opts.days} (expected 7, 28 or 90).`);
}
// ProjectDb creates a missing file; a pull into an empty project would
// join onto nothing, so a typo in --project must fail instead.
if (!existsSync(opts.projectPath)) {
return fail(`Project not found: ${opts.projectPath} (crawl it first, or pass --project).`);
}
let token: string;
try {
token = metricaToken();
} catch (err) {
return metricaFail(opts.json, err, 2);
}
// Installed before the project opens, so Ctrl-C while a desktop-saved
// project unpacks also cancels. Once the write has started, or on a
// second Ctrl-C, it exits at once instead: the exit hook drops a
// half-written pack, and the original is only replaced by its final
// rename, so the project is either untouched or completely updated.
const abort = new AbortController();
let writing = false;
const onSigint = (): void => {
if (writing || abort.signal.aborted) {
process.stderr.write('\nInterrupted.\n');
process.exit(130);
}
process.stderr.write('\nCancelling the Yandex Metrica fetch…\n');
abort.abort();
};
process.on('SIGINT', onSigint);
let project: OpenedProject;
try {
project = await openProject(opts.projectPath, { forWrite: true });
} catch (err) {
process.off('SIGINT', onSigint);
return fail(`Cannot open project ${opts.projectPath}: ${(err as Error).message}`);
}
const { db } = project;
const tty = process.stderr.isTTY === true;
let progressOpen = false;
const endProgress = (): void => {
if (progressOpen) process.stderr.write('\n');
progressOpen = false;
};
const log = (level: 'info' | 'warn' | 'error', message: string): void => {
if (level === 'info') return;
endProgress();
process.stderr.write(`[${level}] ${message}\n`);
};
try {
let normalize: (url: string) => string | null;
try {
normalize = metricaUrlNormalizer(db, opts.configPath);
} catch (err) {
return fail(`Cannot read --config file ${opts.configPath}: ${(err as Error).message}`);
}
// Best effort, for the name the desktop shows; the pull itself reports
// any real access problem.
let counterName = counterId;
try {
const counters = await listMetricaCounters(token, { signal: abort.signal, log });
const found = counters.find((c) => c.counterId === counterId);
if (found) counterName = found.name;
} catch (err) {
if (err instanceof MetricaCancelledError) throw err;
/* keep the id as the name */
}
const result = await fetchMetricaPageMetrics(token, counterId, days, normalize, {
signal: abort.signal,
onProgress: (p) => {
if (opts.json) return;
const line = `${p.report} report ${p.reportIndex}/${p.reportCount}: ${p.rows} rows`;
if (tty) {
process.stderr.write(`\r${line} `);
progressOpen = true;
} else {
process.stderr.write(`${line}\n`);
}
},
log,
});
endProgress();
const fetchedAt = new Date().toISOString();
const meta: MetricaFetchMeta = {
counterId,
counterName,
startDate: result.startDate,
endDate: result.endDate,
fetchedAt,
rowCount: result.rows.length,
truncated: result.truncated,
sampled: result.sampled,
totals: result.totals,
};
// A cancel acknowledged during the pull writes nothing. The check and
// the flag sit in the same tick as the (synchronous) DB write.
if (abort.signal.aborted) throw new MetricaCancelledError();
writing = true;
db.replaceMetricaResults(result.rows, fetchedAt, counterId, JSON.stringify(meta));
// Merge into the per-project settings blob, as the desktop does, so the
// Metrica tab opens on this counter and range.
let settings: Record<string, unknown> = {};
try {
const parsed = JSON.parse(db.getMeta('integrationSettings:metrica') ?? '{}') as unknown;
if (parsed && typeof parsed === 'object') settings = parsed as Record<string, unknown>;
} catch {
/* corrupt blob — overwrite it */
}
db.setMeta(
'integrationSettings:metrica',
JSON.stringify({ ...settings, counterId, counterName, days }),
);
await project.save();
if (opts.json) {
process.stdout.write(
JSON.stringify(
{ ok: true, projectPath: opts.projectPath, days, dropped: result.dropped, ...meta },
null,
2,
) + '\n',
);
} else {
const t = result.totals;
console.log(
`Metrica: counter ${counterId}${counterName !== counterId ? ` (${counterName})` : ''}`,
);
console.log(` Range: ${result.startDate} → ${result.endDate} (${days} days)`);
console.log(` Pages: ${result.rows.length} stored in ${opts.projectPath}`);
console.log(
` Totals: ${t.pageviews} pageviews · ${t.visits} visits · ${t.users} users · ${t.searchVisits} search visits`,
);
if (result.dropped > 0) {
console.log(` Dropped: ${result.dropped} raw rows whose URL did not normalise`);
}
if (result.truncated) {
console.log(' Warning: a report hit the row cap, so low-traffic URLs are missing.');
}
if (result.sampled) {
console.log(
' Warning: Yandex Metrica returned sampled data; per-page numbers are estimates.',
);
}
}
return 0;
} catch (err) {
endProgress();
return metricaFail(opts.json, err, err instanceof MetricaCancelledError ? 130 : 1);
} finally {
// Cleanup first: without a listener, Ctrl-C kills the process at once
// and would leave the working copy behind.
project.dispose();
process.off('SIGINT', onSigint);
}
}
apps/desktop/src/main/index.ts
+1 -1
View File
@@ -265,7 +265,7 @@ import {
detectProjectFormat,
packProject,
unpackProject,
} from './project-archive.js';
} from '@freecrawl/db';
import {
BrowserPool,
PlaywrightBrowserMissingError,
apps/desktop/src/main/metrica.ts
+30 -499
View File
@@ -1,515 +1,46 @@
/**
* Yandex Metrica API client.
*
* - `listCounters()` — every counter the token can read (Management API).
* - `fetchPageMetrics()` — per-page metrics for a date range from the
* Reporting API, joined onto crawl URLs through the caller's
* `normalize` function (the crawl's own `normalizeUrl` options).
*
* Three reports are pulled, all with `accuracy=full` (unsampled):
*
* 1. hit-level `ym:pv:URL` → pageviews
* 2. session `ym:s:startURL` → visits, users (by landing page)
* 3. session `ym:s:startURL` → visits filtered to
* `ym:s:trafficSource=='organic'` (search-engine visits)
*
* Attribution is left at the API default (cross-device last significant
* source), which is also the Metrica UI default, so the numbers match
* what a user sees in the Metrica web interface.
*
* Dates: the window ends yesterday, because today's numbers keep moving
* while pages are fetched and would shift rows across page boundaries.
* The first request sends a relative range so Metrica resolves it in the
* counter's time zone; every later page and report reuses the resolved
* absolute dates, so all three reports cover the same days.
*
* Paging: `offset` is 1-based and `limit` tops out at 100 000. For big
* reports `total_rows` is capped near `offset + limit` (and flagged
* `total_rows_rounded`), so it can't be used as a stop condition — the
* loop keeps going until a page comes back short.
*
* Auth is a personal OAuth token (`metrika:read`) from the credential
* store; the caller resolves it so this module stays free of Electron.
* Yandex Metrica API client, bound to the desktop app. The client itself
* lives in packages/core `metrica.ts` (shared with the CLI); this binding
* sends its requests through the integration dispatcher (`api-fetch.ts`)
* and its diagnostics to the app log.
*/
import type { MetricaCounter, MetricaErrorCode, MetricaFetchMeta } from '@freecrawl/shared-types';
import type { MetricaCounter } from '@freecrawl/shared-types';
import {
listMetricaCounters,
fetchMetricaPageMetrics,
type MetricaDeps,
type MetricaFetchOptions,
type MetricaPageMetrics,
} from '@freecrawl/core';
import { apiFetch } from './api-fetch.js';
import * as logger from './logger.js';
const API_BASE = 'https://api-metrika.yandex.net';
/** Reporting API maximum rows per request. */
const PAGE_LIMIT = 100_000;
/** Smallest page the "query is too complicated" fallback shrinks to. */
const MIN_PAGE_LIMIT = 6_250;
/** Per-report ceiling on raw rows, so a counter with endless query-string
* variants can't keep the pull running for ever. Rows are folded into
* per-URL totals page by page, so memory follows distinct normalised
* Metrica URLs (crawled or not), not raw rows. */
const MAX_ROWS = 1_000_000;
/** Attempts per request on throttling (420 / 429), "too complicated",
* 5xx and network errors. */
const MAX_ATTEMPTS = 4;
export {
MetricaError,
MetricaCancelledError,
type MetricaPageRow,
type MetricaFetchProgress,
type MetricaFetchOptions,
type MetricaPageMetrics,
} from '@freecrawl/core';
type Totals = MetricaFetchMeta['totals'];
function apiError(json: unknown, status: number): string {
const msg = (json as { message?: unknown } | null)?.message;
if (typeof msg === 'string' && msg) return `Yandex Metrica: ${msg}`;
if (status === 401) {
return 'Yandex Metrica rejected the token — paste a fresh token with the metrika:read scope in Settings → Integrations.';
}
if (status === 403) {
return 'Yandex Metrica denied access — the token’s account cannot read this counter.';
}
return `Yandex Metrica API error (HTTP ${status})`;
}
/** Metrica answers throttling with 429 or 420 (the docs use both) and
* sends no `Retry-After`, so both get exponential backoff. */
function isRetryable(status: number): boolean {
return status === 420 || status === 429 || status >= 500;
}
/** A Metrica failure the UI words in the user's language by `code`; the
* English message stays for logs and MCP clients. */
export class MetricaError extends Error {
constructor(
message: string,
readonly code: MetricaErrorCode,
) {
super(message);
}
}
function codeForStatus(status: number): MetricaErrorCode {
if (status === 401) return 'token-rejected';
if (status === 403) return 'access-denied';
if (status === 0) return 'network';
return 'api';
}
/** API failure with the HTTP status kept for the caller's fallbacks. */
class MetricaApiError extends MetricaError {
constructor(
message: string,
readonly status: number,
code: MetricaErrorCode = codeForStatus(status),
) {
super(message, code);
}
}
/** Metrica's 400 "query is too complicated" for heavy unsampled reports.
* It depends on server load, so the same request is retried with longer
* pauses first; a smaller page is only the last resort, because each
* request costs about the same whatever its page size. */
const TOO_COMPLICATED = /too complicated|слишком сложн/i;
function isTooComplicated(err: unknown): boolean {
return err instanceof MetricaApiError && err.status === 400 && TOO_COMPLICATED.test(err.message);
}
const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
/** Thrown when the caller aborts a pull. */
export class MetricaCancelledError extends Error {
constructor() {
super('Yandex Metrica fetch cancelled.');
this.name = 'MetricaCancelledError';
}
}
function throwIfCancelled(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw new MetricaCancelledError();
}
/** Backoff pause that ends early, with a cancel error, on abort. */
function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(new MetricaCancelledError());
return;
}
const onAbort = (): void => {
clearTimeout(timer);
reject(new MetricaCancelledError());
};
const timer = setTimeout(() => {
signal?.removeEventListener('abort', onAbort);
resolve();
}, ms);
signal?.addEventListener('abort', onAbort, { once: true });
});
}
/** GET a Metrica endpoint as JSON, retrying throttling and server errors. */
async function getJson(
token: string,
path: string,
params: Record<string, string>,
signal?: AbortSignal,
): Promise<unknown> {
const url = `${API_BASE}${path}?${new URLSearchParams(params).toString()}`;
let lastError = '';
let lastStatus = 0;
let backoffMs = 2_000;
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
throwIfCancelled(signal);
const timeout = AbortSignal.timeout(120_000);
try {
const res = await apiFetch(url, {
headers: { Authorization: `OAuth ${token}` },
signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
});
const json = (await res.json().catch(() => null)) as unknown;
// A cancel while the body downloads makes json null; report it as a
// cancel, not as an API error.
throwIfCancelled(signal);
if (res.ok && json) return json;
lastError = apiError(json, res.status);
lastStatus = res.status;
const heavy = res.status === 400 && TOO_COMPLICATED.test(lastError);
if (heavy) backoffMs = 10_000;
else if (!isRetryable(res.status)) break;
} catch (err) {
throwIfCancelled(signal);
// Never echo `err.message`: undici quotes the offending header value
// (the token) in its validation errors.
const name = err instanceof Error ? err.name : 'Error';
lastError = `Yandex Metrica request failed (${name})`;
lastStatus = 0;
}
if (attempt < MAX_ATTEMPTS) {
logger.log('warn', 'metrica', `${lastError} — retry ${attempt}/${MAX_ATTEMPTS - 1}`);
await sleep(backoffMs * 2 ** (attempt - 1), signal);
}
}
throw new MetricaApiError(lastError, lastStatus);
}
const deps: Required<MetricaDeps> = {
fetch: apiFetch,
log: (level, message) => logger.log(level, 'metrica', message),
};
/** Every counter the token can read. */
export async function listCounters(token: string): Promise<MetricaCounter[]> {
const PER_PAGE = 1000;
const out: MetricaCounter[] = [];
for (let offset = 1; ; offset += PER_PAGE) {
const json = (await getJson(token, '/management/v1/counters', {
per_page: String(PER_PAGE),
offset: String(offset),
})) as {
counters?: {
id?: unknown;
name?: unknown;
site?: unknown;
site2?: { site?: unknown } | null;
owner_login?: unknown;
}[];
};
const counters = Array.isArray(json.counters) ? json.counters : [];
for (const c of counters) {
if (typeof c.id !== 'number' && typeof c.id !== 'string') continue;
const site =
typeof c.site2?.site === 'string' ? c.site2.site : typeof c.site === 'string' ? c.site : '';
out.push({
counterId: String(c.id),
name: typeof c.name === 'string' && c.name ? c.name : site || String(c.id),
site,
ownerLogin: typeof c.owner_login === 'string' ? c.owner_login : '',
});
}
if (counters.length < PER_PAGE) break;
}
return out;
export function listCounters(token: string): Promise<MetricaCounter[]> {
return listMetricaCounters(token, deps);
}
interface ReportSpec {
dimension: string;
metrics: string[];
filters?: string;
}
interface ReportResult {
/** Whole-range totals per metric, as the API reports them. */
totals: number[];
/** Raw rows received. */
rowCount: number;
truncated: boolean;
/** True when any page came back sampled despite `accuracy=full`. */
sampled: boolean;
/** Resolved `YYYY-MM-DD` range the API reported on (counter time zone). */
date1: string;
date2: string;
}
/**
* Pull one report in full, paging until a short page. Each row is handed
* to `onRow` as it arrives, so raw rows never pile up in memory.
*/
async function runReport(
token: string,
counterId: string,
date1: string,
date2: string,
spec: ReportSpec,
onRow: (url: string, values: number[]) => void,
signal: AbortSignal | undefined,
onPage: (rowsSoFar: number) => void,
): Promise<ReportResult> {
const result: ReportResult = {
totals: spec.metrics.map(() => 0),
rowCount: 0,
truncated: false,
sampled: false,
date1,
date2,
};
onPage(0);
let limit = PAGE_LIMIT;
for (let offset = 1; ; ) {
const params: Record<string, string> = {
ids: counterId,
// Relative on the very first page, the resolved dates after that.
date1: result.date1,
date2: result.date2,
dimensions: spec.dimension,
metrics: spec.metrics.join(','),
// Metric desc, then URL, so the order is total and pages can't
// overlap or skip rows that tie on the metric.
sort: `-${spec.metrics[0]},${spec.dimension}`,
accuracy: 'full',
limit: String(limit),
offset: String(offset),
};
if (spec.filters) params['filters'] = spec.filters;
let json: {
data?: { dimensions?: { name?: unknown }[]; metrics?: unknown[] }[];
totals?: unknown[];
sampled?: unknown;
query?: { date1?: unknown; date2?: unknown };
};
try {
json = (await getJson(token, '/stat/v1/data', params, signal)) as typeof json;
} catch (err) {
if (!isTooComplicated(err) || limit <= MIN_PAGE_LIMIT) throw err;
// Retries didn't help: same offset, half the page. Rows already
// received stay valid because the sort order is total.
limit = Math.max(MIN_PAGE_LIMIT, Math.floor(limit / 2));
logger.log(
'warn',
'metrica',
`report ${spec.dimension} too complicated at offset ${offset} — retrying with limit ${limit}`,
);
continue;
}
// A response without a row list or resolved dates is a changed or
// broken API, not an empty report: fail so the stored data survives.
const resolved1 = json.query?.date1;
const resolved2 = json.query?.date2;
if (
!Array.isArray(json.data) ||
typeof resolved1 !== 'string' ||
!ISO_DATE.test(resolved1) ||
typeof resolved2 !== 'string' ||
!ISO_DATE.test(resolved2)
) {
throw new MetricaApiError(
'Yandex Metrica returned an unexpected response.',
200,
'unexpected-response',
);
}
if (offset === 1 && !ISO_DATE.test(result.date1)) {
result.date1 = resolved1;
result.date2 = resolved2;
} else if (resolved1 !== result.date1 || resolved2 !== result.date2) {
throw new MetricaApiError(
'Yandex Metrica changed the report date range mid-pull.',
200,
'range-changed',
);
}
if (json.sampled === true) {
// accuracy=full should never sample; record it so "no visits"
// filters refuse to treat missing rows as zero.
result.sampled = true;
logger.log('warn', 'metrica', `report ${spec.dimension} came back sampled`);
}
if (offset === 1 && Array.isArray(json.totals)) {
result.totals = spec.metrics.map((_, i) => Number(json.totals?.[i] ?? 0) || 0);
}
const data = json.data;
for (const r of data) {
const url = r.dimensions?.[0]?.name;
if (typeof url !== 'string' || !url) continue;
const m = Array.isArray(r.metrics) ? r.metrics : [];
onRow(
url,
spec.metrics.map((_, i) => Number(m[i] ?? 0) || 0),
);
}
result.rowCount += data.length;
onPage(result.rowCount);
// `total_rows` is capped near offset + limit on large reports, so a
// short page is the only reliable end-of-data signal.
if (data.length < limit) break;
offset += limit;
if (result.rowCount >= MAX_ROWS) {
result.truncated = true;
logger.log(
'warn',
'metrica',
`report ${spec.dimension} stopped at ${result.rowCount} rows (cap)`,
);
break;
}
}
return result;
}
/** One per-page Metrica row, keyed by the crawl-normalised URL. */
export interface MetricaPageRow {
url: string;
pageviews: number;
visits: number;
users: number;
searchVisits: number;
}
/** Where a running pull is: which of the three reports, and how many raw
* rows that report has returned so far. */
export interface MetricaFetchProgress {
report: 'pageviews' | 'visits' | 'searchVisits';
reportIndex: number;
reportCount: number;
rows: number;
}
export interface MetricaFetchOptions {
/** Aborts the pull, including any backoff pause in progress. */
signal?: AbortSignal;
onProgress?: (progress: MetricaFetchProgress) => void;
}
export interface MetricaPageMetrics {
rows: MetricaPageRow[];
totals: Totals;
/** Inclusive `YYYY-MM-DD` range the API reported on, counter time zone. */
startDate: string;
endDate: string;
/** Raw report rows dropped because their URL failed to normalise. */
dropped: number;
/** True when a report hit the row ceiling. */
truncated: boolean;
/** True when a report came back sampled. */
sampled: boolean;
}
/**
* Pull per-page metrics for the `days` full days ending yesterday and fold
* them onto crawl URLs. The range is first sent as `NdaysAgo`…`yesterday`,
* so Metrica resolves it in the counter's own time zone. Several raw Metrica
* URLs can normalise to the same crawl URL (tracking parameters,
* `#fragments`, host / slash rewrites); their counts are summed. For
* `users` that sum can count a person once per URL variant — exact for
* the common case of one variant per page, an upper bound otherwise.
*/
export async function fetchPageMetrics(
/** Per-page metrics for the `days` full days ending yesterday; see core. */
export function fetchPageMetrics(
token: string,
counterId: string,
days: number,
normalize: (url: string) => string | null,
opts: MetricaFetchOptions = {},
opts: Pick<MetricaFetchOptions, 'signal' | 'onProgress'> = {},
): Promise<MetricaPageMetrics> {
const { signal, onProgress } = opts;
const progress =
(report: MetricaFetchProgress['report'], reportIndex: number) =>
(rows: number): void =>
onProgress?.({ report, reportIndex, reportCount: 3, rows });
const started = Date.now();
const relative1 = `${days}daysAgo`;
const relative2 = 'yesterday';
const byUrl = new Map<string, MetricaPageRow>();
let dropped = 0;
const rowFor = (raw: string): MetricaPageRow | null => {
const url = normalize(raw);
if (!url) {
dropped++;
return null;
}
let row = byUrl.get(url);
if (!row) {
row = { url, pageviews: 0, visits: 0, users: 0, searchVisits: 0 };
byUrl.set(url, row);
}
return row;
};
const pv = await runReport(
token,
counterId,
relative1,
relative2,
{ dimension: 'ym:pv:URL', metrics: ['ym:pv:pageviews'] },
(url, v) => {
const row = rowFor(url);
if (row) row.pageviews += v[0] ?? 0;
},
signal,
progress('pageviews', 1),
);
// Later reports reuse the dates the first one resolved to.
const { date1, date2 } = pv;
const sessions = await runReport(
token,
counterId,
date1,
date2,
{ dimension: 'ym:s:startURL', metrics: ['ym:s:visits', 'ym:s:users'] },
(url, v) => {
const row = rowFor(url);
if (!row) return;
row.visits += v[0] ?? 0;
row.users += v[1] ?? 0;
},
signal,
progress('visits', 2),
);
const search = await runReport(
token,
counterId,
date1,
date2,
{
dimension: 'ym:s:startURL',
metrics: ['ym:s:visits'],
filters: "ym:s:trafficSource=='organic'",
},
(url, v) => {
const row = rowFor(url);
if (row) row.searchVisits += v[0] ?? 0;
},
signal,
progress('searchVisits', 3),
);
const rows = [...byUrl.values()];
logger.log(
'info',
'metrica',
`Metrica report: ${rows.length} page(s) for counter ${counterId} (${pv.date1}…${pv.date2}) from ` +
`${pv.rowCount}/${sessions.rowCount}/${search.rowCount} raw rows, ` +
`${dropped} dropped, ${Math.round((Date.now() - started) / 1000)} s`,
);
return {
rows,
totals: {
pageviews: pv.totals[0] ?? 0,
visits: sessions.totals[0] ?? 0,
users: sessions.totals[1] ?? 0,
searchVisits: search.totals[0] ?? 0,
},
startDate: pv.date1,
endDate: pv.date2,
dropped,
truncated: pv.truncated || sessions.truncated || search.truncated,
sampled: pv.sampled || sessions.sampled || search.sampled,
};
return fetchMetricaPageMetrics(token, counterId, days, normalize, { ...opts, ...deps });
}
packages/core/src/index.ts
+11
View File
@@ -168,3 +168,14 @@ export {
type SheetSpec,
type SheetColumn,
} from './spreadsheet.js';
export {
listCounters as listMetricaCounters,
fetchPageMetrics as fetchMetricaPageMetrics,
MetricaError,
MetricaCancelledError,
type MetricaDeps,
type MetricaPageRow,
type MetricaFetchProgress,
type MetricaFetchOptions,
type MetricaPageMetrics,
} from './metrica.js';
packages/core/src/metrica.ts
+537
View File
@@ -0,0 +1,537 @@
/**
* Yandex Metrica API client.
*
* - `listCounters()` — every counter the token can read (Management API).
* - `fetchPageMetrics()` — per-page metrics for a date range from the
* Reporting API, joined onto crawl URLs through the caller's
* `normalize` function (the crawl's own `normalizeUrl` options).
*
* Three reports are pulled, all with `accuracy=full` (unsampled):
*
* 1. hit-level `ym:pv:URL` → pageviews
* 2. session `ym:s:startURL` → visits, users (by landing page)
* 3. session `ym:s:startURL` → visits filtered to
* `ym:s:trafficSource=='organic'` (search-engine visits)
*
* Attribution is left at the API default (cross-device last significant
* source), which is also the Metrica UI default, so the numbers match
* what a user sees in the Metrica web interface.
*
* Dates: the window ends yesterday, because today's numbers keep moving
* while pages are fetched and would shift rows across page boundaries.
* The first request sends a relative range so Metrica resolves it in the
* counter's time zone; every later page and report reuses the resolved
* absolute dates, so all three reports cover the same days.
*
* Paging: `offset` is 1-based and `limit` tops out at 100 000. For big
* reports `total_rows` is capped near `offset + limit` (and flagged
* `total_rows_rounded`), so it can't be used as a stop condition — the
* loop keeps going until a page comes back short.
*
* Auth is a personal OAuth token (`metrika:read`) from the credential
* store or the environment; the caller resolves it so this module stays
* free of Electron. HTTP and logging come in through `MetricaDeps`.
*/
import { fetch as undiciFetch } from 'undici';
import type { MetricaCounter, MetricaErrorCode, MetricaFetchMeta } from '@freecrawl/shared-types';
const API_BASE = 'https://api-metrika.yandex.net';
/** Reporting API maximum rows per request. */
const PAGE_LIMIT = 100_000;
/** Smallest page the "query is too complicated" fallback shrinks to. */
const MIN_PAGE_LIMIT = 6_250;
/** Per-report ceiling on raw rows, so a counter with endless query-string
* variants can't keep the pull running for ever. Rows are folded into
* per-URL totals page by page, so memory follows distinct normalised
* Metrica URLs (crawled or not), not raw rows. */
const MAX_ROWS = 1_000_000;
/** Attempts per request on throttling (420 / 429), "too complicated",
* 5xx and network errors. */
const MAX_ATTEMPTS = 4;
type Totals = MetricaFetchMeta['totals'];
/** What the host app supplies: its HTTP client (the desktop pins requests
* to its integration dispatcher) and its log sink. */
export interface MetricaDeps {
/** Defaults to undici's `fetch`. */
fetch?: typeof undiciFetch;
/** Defaults to a no-op. */
log?: (level: 'info' | 'warn' | 'error', message: string) => void;
}
type ResolvedDeps = Required<MetricaDeps>;
function resolveDeps(deps: MetricaDeps): ResolvedDeps {
return { fetch: deps.fetch ?? undiciFetch, log: deps.log ?? (() => {}) };
}
function apiError(json: unknown, status: number): string {
const msg = (json as { message?: unknown } | null)?.message;
if (typeof msg === 'string' && msg) return `Yandex Metrica: ${msg}`;
if (status === 401) {
return 'Yandex Metrica rejected the token — paste a fresh token with the metrika:read scope in Settings → Integrations.';
}
if (status === 403) {
return 'Yandex Metrica denied access — the token’s account cannot read this counter.';
}
return `Yandex Metrica API error (HTTP ${status})`;
}
/** Metrica answers throttling with 429 or 420 (the docs use both) and
* sends no `Retry-After`, so both get exponential backoff. */
function isRetryable(status: number): boolean {
return status === 420 || status === 429 || status >= 500;
}
/** A Metrica failure the UI words in the user's language by `code`; the
* English message stays for logs and MCP clients. */
export class MetricaError extends Error {
constructor(
message: string,
readonly code: MetricaErrorCode,
) {
super(message);
}
}
function codeForStatus(status: number): MetricaErrorCode {
if (status === 401) return 'token-rejected';
if (status === 403) return 'access-denied';
if (status === 0) return 'network';
return 'api';
}
/** API failure with the HTTP status kept for the caller's fallbacks. */
class MetricaApiError extends MetricaError {
constructor(
message: string,
readonly status: number,
code: MetricaErrorCode = codeForStatus(status),
) {
super(message, code);
}
}
/** Metrica's 400 "query is too complicated" for heavy unsampled reports.
* It depends on server load, so the same request is retried with longer
* pauses first; a smaller page is only the last resort, because each
* request costs about the same whatever its page size. */
const TOO_COMPLICATED = /too complicated|слишком сложн/i;
function isTooComplicated(err: unknown): boolean {
return err instanceof MetricaApiError && err.status === 400 && TOO_COMPLICATED.test(err.message);
}
const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
/** Thrown when the caller aborts a pull. */
export class MetricaCancelledError extends Error {
constructor() {
super('Yandex Metrica fetch cancelled.');
this.name = 'MetricaCancelledError';
}
}
function throwIfCancelled(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw new MetricaCancelledError();
}
/** Backoff pause that ends early, with a cancel error, on abort. */
function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(new MetricaCancelledError());
return;
}
const onAbort = (): void => {
clearTimeout(timer);
reject(new MetricaCancelledError());
};
const timer = setTimeout(() => {
signal?.removeEventListener('abort', onAbort);
resolve();
}, ms);
signal?.addEventListener('abort', onAbort, { once: true });
});
}
/** GET a Metrica endpoint as JSON, retrying throttling and server errors. */
async function getJson(
deps: ResolvedDeps,
token: string,
path: string,
params: Record<string, string>,
signal?: AbortSignal,
): Promise<unknown> {
const url = `${API_BASE}${path}?${new URLSearchParams(params).toString()}`;
let lastError = '';
let lastStatus = 0;
let backoffMs = 2_000;
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
throwIfCancelled(signal);
const timeout = AbortSignal.timeout(120_000);
try {
const res = await deps.fetch(url, {
headers: { Authorization: `OAuth ${token}` },
signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
});
const json = (await res.json().catch(() => null)) as unknown;
// A cancel while the body downloads makes json null; report it as a
// cancel, not as an API error.
throwIfCancelled(signal);
if (res.ok && json) return json;
lastError = apiError(json, res.status);
lastStatus = res.status;
const heavy = res.status === 400 && TOO_COMPLICATED.test(lastError);
if (heavy) backoffMs = 10_000;
else if (!isRetryable(res.status)) break;
} catch (err) {
throwIfCancelled(signal);
// Never echo `err.message`: undici quotes the offending header value
// (the token) in its validation errors.
const name = err instanceof Error ? err.name : 'Error';
lastError = `Yandex Metrica request failed (${name})`;
lastStatus = 0;
}
if (attempt < MAX_ATTEMPTS) {
deps.log('warn', `${lastError} — retry ${attempt}/${MAX_ATTEMPTS - 1}`);
await sleep(backoffMs * 2 ** (attempt - 1), signal);
}
}
throw new MetricaApiError(lastError, lastStatus);
}
/** Every counter the token can read. */
export async function listCounters(
token: string,
deps: MetricaDeps & { signal?: AbortSignal } = {},
): Promise<MetricaCounter[]> {
const resolved = resolveDeps(deps);
const PER_PAGE = 1000;
const out: MetricaCounter[] = [];
for (let offset = 1; ; offset += PER_PAGE) {
const json = (await getJson(
resolved,
token,
'/management/v1/counters',
{ per_page: String(PER_PAGE), offset: String(offset) },
deps.signal,
)) as {
counters?: {
id?: unknown;
name?: unknown;
site?: unknown;
site2?: { site?: unknown } | null;
owner_login?: unknown;
}[];
};
const counters = Array.isArray(json.counters) ? json.counters : [];
for (const c of counters) {
if (typeof c.id !== 'number' && typeof c.id !== 'string') continue;
const site =
typeof c.site2?.site === 'string' ? c.site2.site : typeof c.site === 'string' ? c.site : '';
out.push({
counterId: String(c.id),
name: typeof c.name === 'string' && c.name ? c.name : site || String(c.id),
site,
ownerLogin: typeof c.owner_login === 'string' ? c.owner_login : '',
});
}
if (counters.length < PER_PAGE) break;
}
return out;
}
interface ReportSpec {
dimension: string;
metrics: string[];
filters?: string;
}
interface ReportResult {
/** Whole-range totals per metric, as the API reports them. */
totals: number[];
/** Raw rows received. */
rowCount: number;
truncated: boolean;
/** True when any page came back sampled despite `accuracy=full`. */
sampled: boolean;
/** Resolved `YYYY-MM-DD` range the API reported on (counter time zone). */
date1: string;
date2: string;
}
/**
* Pull one report in full, paging until a short page. Each row is handed
* to `onRow` as it arrives, so raw rows never pile up in memory.
*/
async function runReport(
deps: ResolvedDeps,
token: string,
counterId: string,
date1: string,
date2: string,
spec: ReportSpec,
onRow: (url: string, values: number[]) => void,
signal: AbortSignal | undefined,
onPage: (rowsSoFar: number) => void,
): Promise<ReportResult> {
const result: ReportResult = {
totals: spec.metrics.map(() => 0),
rowCount: 0,
truncated: false,
sampled: false,
date1,
date2,
};
onPage(0);
let limit = PAGE_LIMIT;
for (let offset = 1; ; ) {
const params: Record<string, string> = {
ids: counterId,
// Relative on the very first page, the resolved dates after that.
date1: result.date1,
date2: result.date2,
dimensions: spec.dimension,
metrics: spec.metrics.join(','),
// Metric desc, then URL, so the order is total and pages can't
// overlap or skip rows that tie on the metric.
sort: `-${spec.metrics[0]},${spec.dimension}`,
accuracy: 'full',
limit: String(limit),
offset: String(offset),
};
if (spec.filters) params['filters'] = spec.filters;
let json: {
data?: { dimensions?: { name?: unknown }[]; metrics?: unknown[] }[];
totals?: unknown[];
sampled?: unknown;
query?: { date1?: unknown; date2?: unknown };
};
try {
json = (await getJson(deps, token, '/stat/v1/data', params, signal)) as typeof json;
} catch (err) {
if (!isTooComplicated(err) || limit <= MIN_PAGE_LIMIT) throw err;
// Retries didn't help: same offset, half the page. Rows already
// received stay valid because the sort order is total.
limit = Math.max(MIN_PAGE_LIMIT, Math.floor(limit / 2));
deps.log(
'warn',
`report ${spec.dimension} too complicated at offset ${offset} — retrying with limit ${limit}`,
);
continue;
}
// A response without a row list or resolved dates is a changed or
// broken API, not an empty report: fail so the stored data survives.
const resolved1 = json.query?.date1;
const resolved2 = json.query?.date2;
if (
!Array.isArray(json.data) ||
typeof resolved1 !== 'string' ||
!ISO_DATE.test(resolved1) ||
typeof resolved2 !== 'string' ||
!ISO_DATE.test(resolved2)
) {
throw new MetricaApiError(
'Yandex Metrica returned an unexpected response.',
200,
'unexpected-response',
);
}
if (offset === 1 && !ISO_DATE.test(result.date1)) {
result.date1 = resolved1;
result.date2 = resolved2;
} else if (resolved1 !== result.date1 || resolved2 !== result.date2) {
throw new MetricaApiError(
'Yandex Metrica changed the report date range mid-pull.',
200,
'range-changed',
);
}
if (json.sampled === true) {
// accuracy=full should never sample; record it so "no visits"
// filters refuse to treat missing rows as zero.
result.sampled = true;
deps.log('warn', `report ${spec.dimension} came back sampled`);
}
if (offset === 1 && Array.isArray(json.totals)) {
result.totals = spec.metrics.map((_, i) => Number(json.totals?.[i] ?? 0) || 0);
}
const data = json.data;
for (const r of data) {
const url = r.dimensions?.[0]?.name;
if (typeof url !== 'string' || !url) continue;
const m = Array.isArray(r.metrics) ? r.metrics : [];
onRow(
url,
spec.metrics.map((_, i) => Number(m[i] ?? 0) || 0),
);
}
result.rowCount += data.length;
onPage(result.rowCount);
// `total_rows` is capped near offset + limit on large reports, so a
// short page is the only reliable end-of-data signal.
if (data.length < limit) break;
offset += limit;
if (result.rowCount >= MAX_ROWS) {
result.truncated = true;
deps.log('warn', `report ${spec.dimension} stopped at ${result.rowCount} rows (cap)`);
break;
}
}
return result;
}
/** One per-page Metrica row, keyed by the crawl-normalised URL. */
export interface MetricaPageRow {
url: string;
pageviews: number;
visits: number;
users: number;
searchVisits: number;
}
/** Where a running pull is: which of the three reports, and how many raw
* rows that report has returned so far. */
export interface MetricaFetchProgress {
report: 'pageviews' | 'visits' | 'searchVisits';
reportIndex: number;
reportCount: number;
rows: number;
}
export interface MetricaFetchOptions extends MetricaDeps {
/** Aborts the pull, including any backoff pause in progress. */
signal?: AbortSignal;
onProgress?: (progress: MetricaFetchProgress) => void;
}
export interface MetricaPageMetrics {
rows: MetricaPageRow[];
totals: Totals;
/** Inclusive `YYYY-MM-DD` range the API reported on, counter time zone. */
startDate: string;
endDate: string;
/** Raw report rows dropped because their URL failed to normalise. */
dropped: number;
/** True when a report hit the row ceiling. */
truncated: boolean;
/** True when a report came back sampled. */
sampled: boolean;
}
/**
* Pull per-page metrics for the `days` full days ending yesterday and fold
* them onto crawl URLs. The range is first sent as `NdaysAgo`…`yesterday`,
* so Metrica resolves it in the counter's own time zone. Several raw Metrica
* URLs can normalise to the same crawl URL (tracking parameters,
* `#fragments`, host / slash rewrites); their counts are summed. For
* `users` that sum can count a person once per URL variant — exact for
* the common case of one variant per page, an upper bound otherwise.
*/
export async function fetchPageMetrics(
token: string,
counterId: string,
days: number,
normalize: (url: string) => string | null,
opts: MetricaFetchOptions = {},
): Promise<MetricaPageMetrics> {
const { signal, onProgress } = opts;
const deps = resolveDeps(opts);
const progress =
(report: MetricaFetchProgress['report'], reportIndex: number) =>
(rows: number): void =>
onProgress?.({ report, reportIndex, reportCount: 3, rows });
const started = Date.now();
const relative1 = `${days}daysAgo`;
const relative2 = 'yesterday';
const byUrl = new Map<string, MetricaPageRow>();
let dropped = 0;
const rowFor = (raw: string): MetricaPageRow | null => {
const url = normalize(raw);
if (!url) {
dropped++;
return null;
}
let row = byUrl.get(url);
if (!row) {
row = { url, pageviews: 0, visits: 0, users: 0, searchVisits: 0 };
byUrl.set(url, row);
}
return row;
};
const pv = await runReport(
deps,
token,
counterId,
relative1,
relative2,
{ dimension: 'ym:pv:URL', metrics: ['ym:pv:pageviews'] },
(url, v) => {
const row = rowFor(url);
if (row) row.pageviews += v[0] ?? 0;
},
signal,
progress('pageviews', 1),
);
// Later reports reuse the dates the first one resolved to.
const { date1, date2 } = pv;
const sessions = await runReport(
deps,
token,
counterId,
date1,
date2,
{ dimension: 'ym:s:startURL', metrics: ['ym:s:visits', 'ym:s:users'] },
(url, v) => {
const row = rowFor(url);
if (!row) return;
row.visits += v[0] ?? 0;
row.users += v[1] ?? 0;
},
signal,
progress('visits', 2),
);
const search = await runReport(
deps,
token,
counterId,
date1,
date2,
{
dimension: 'ym:s:startURL',
metrics: ['ym:s:visits'],
filters: "ym:s:trafficSource=='organic'",
},
(url, v) => {
const row = rowFor(url);
if (row) row.searchVisits += v[0] ?? 0;
},
signal,
progress('searchVisits', 3),
);
const rows = [...byUrl.values()];
deps.log(
'info',
`Metrica report: ${rows.length} page(s) for counter ${counterId} (${pv.date1}…${pv.date2}) from ` +
`${pv.rowCount}/${sessions.rowCount}/${search.rowCount} raw rows, ` +
`${dropped} dropped, ${Math.round((Date.now() - started) / 1000)} s`,
);
return {
rows,
totals: {
pageviews: pv.totals[0] ?? 0,
visits: sessions.totals[0] ?? 0,
users: sessions.totals[1] ?? 0,
searchVisits: search.totals[0] ?? 0,
},
startDate: pv.date1,
endDate: pv.date2,
dropped,
truncated: pv.truncated || sessions.truncated || search.truncated,
sampled: pv.sampled || sessions.sampled || search.sampled,
};
}
packages/db/src/index.ts
+6
View File
@@ -1,3 +1,9 @@
export { ProjectDb, EXPENSIVE_ISSUE_DEFINITIONS } from './project-db.js';
export type { UpsertUrlInput, ProbeWrite } from './project-db.js';
export { runMigrations } from './migrations.js';
export {
detectProjectFormat,
packProject,
unpackProject,
type ProjectFormat,
} from './project-archive.js';
apps/desktop/src/main/project-archive.ts → packages/db/src/project-archive.ts
View File
File renamed without changes.