mirror of
https://github.com/PhialsBasement/LibreCrawl.git
synced 2026-10-18 00:19:37 +02:00
PhialsBasementandClaude Opus 5.5
d451885216
Setting ZOHO_OAUTH_ENABLED with a client ID and secret adds a "Login with Zoho" button to the login page. Zoho users are matched by a new users.zoho_id column. An existing account with the same email is linked to Zoho. Otherwise a new verified account is created, unless ZOHO_ALLOW_SIGNUP is off, which is the default when registration is disabled. Also configurable: redirect URI, data-center accounts server, scopes, allowed email domains and the tier for new accounts. All are documented in .env.example and passed through in docker-compose.yml. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1975 lines
78 KiB
Python
1975 lines
78 KiB
Python
import threading
|
|
import time
|
|
import csv
|
|
import json
|
|
import xml.etree.ElementTree as ET
|
|
import uuid
|
|
import webbrowser
|
|
import argparse
|
|
import secrets
|
|
import string
|
|
import os
|
|
import base64
|
|
from io import StringIO, BytesIO
|
|
from datetime import datetime, timedelta
|
|
from flask import Flask, render_template, request, jsonify, session, redirect, url_for
|
|
from flask_compress import Compress
|
|
from functools import wraps
|
|
from src.crawler import WebCrawler
|
|
from src.settings_manager import SettingsManager
|
|
from src.auth_db import init_db, create_user, authenticate_user, get_user_by_id, log_guest_crawl, get_guest_crawls_last_24h, verify_user, set_user_tier, create_verification_token, verify_token, get_user_by_email, get_or_create_zoho_user
|
|
from src.email_service import send_verification_email, send_welcome_email
|
|
from src.zoho_oauth import ZohoOAuthConfig, ZohoOAuthError
|
|
|
|
# Load environment variables from .env file
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
|
|
# Parse command line arguments
|
|
parser = argparse.ArgumentParser(description='LibreCrawl - SEO Spider Tool')
|
|
parser.add_argument('--local', '-l', action='store_true',
|
|
help='Run in local mode (all users get admin tier, no rate limits)')
|
|
parser.add_argument('--disable-register', '-dr', action='store_true',
|
|
help='Disable new user registrations')
|
|
parser.add_argument('--disable-guest', '-dg', action='store_true',
|
|
help='Disable guest login')
|
|
parser.add_argument('--demo', '-dm', action='store_true',
|
|
help='Demo mode: 1.5GB memory limit per user, crawls auto-stop at limit')
|
|
parser.add_argument('--dangerously-skip-auth', '-dsa', action='store_true',
|
|
help='DANGEROUS: Allow anyone to log in as any username with no password. '
|
|
'The username is only used to separate per-user sessions. '
|
|
'Do NOT use on a public network or in production.')
|
|
args = parser.parse_args()
|
|
|
|
LOCAL_MODE = args.local or os.getenv('LOCAL_MODE', '').lower() in ('true', '1', 'yes')
|
|
DISABLE_REGISTER = args.disable_register or os.getenv('REGISTRATION_DISABLED', '').lower() in ('true', '1', 'yes')
|
|
DISABLE_GUEST = args.disable_guest or os.getenv('DISABLE_GUEST', '').lower() in ('true', '1', 'yes')
|
|
DEMO_MODE = args.demo or os.getenv('DEMO_MODE', '').lower() in ('true', '1', 'yes')
|
|
SKIP_AUTH = args.dangerously_skip_auth or os.getenv('DANGEROUSLY_SKIP_AUTH', '').lower() in ('true', '1', 'yes')
|
|
ZOHO = ZohoOAuthConfig(registration_disabled=DISABLE_REGISTER)
|
|
|
|
app = Flask(__name__, template_folder='web/templates', static_folder='web/static')
|
|
app.secret_key = os.environ.get('SECRET_KEY') or secrets.token_hex(32)
|
|
if not os.environ.get('SECRET_KEY'):
|
|
print('⚠️ WARNING: SECRET_KEY not set — using an ephemeral random key. '
|
|
'Sessions will not persist across restarts. Set SECRET_KEY in production.', flush=True)
|
|
|
|
# Enable compression for all responses
|
|
Compress(app)
|
|
|
|
# Ensure data directory exists before initializing the database
|
|
os.makedirs("data", exist_ok=True)
|
|
|
|
# Initialize database on startup
|
|
init_db()
|
|
|
|
def generate_random_password(length=16):
|
|
"""Generate a random password with letters, digits, and symbols"""
|
|
alphabet = string.ascii_letters + string.digits + string.punctuation
|
|
return ''.join(secrets.choice(alphabet) for _ in range(length))
|
|
|
|
def auto_login_local_mode():
|
|
"""Auto-login for local mode - creates or logs into 'local' admin account"""
|
|
import sqlite3
|
|
try:
|
|
conn = sqlite3.connect(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'users.db'))
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
|
|
# Check if 'local' user exists
|
|
cursor.execute('SELECT id, username, tier FROM users WHERE username = ?', ('local',))
|
|
user = cursor.fetchone()
|
|
|
|
if user:
|
|
# User exists, just log them in
|
|
session['user_id'] = user['id']
|
|
session['username'] = user['username']
|
|
session['tier'] = 'admin'
|
|
session.permanent = True
|
|
print(f"Auto-logged in as existing 'local' user (ID: {user['id']})")
|
|
else:
|
|
# Create new local user with random password
|
|
random_password = generate_random_password()
|
|
from src.auth_db import hash_password
|
|
password_hash = hash_password(random_password)
|
|
|
|
cursor.execute('''
|
|
INSERT INTO users (username, email, password_hash, verified, tier)
|
|
VALUES (?, ?, ?, 1, 'admin')
|
|
''', ('local', 'local@localhost', password_hash))
|
|
conn.commit()
|
|
|
|
user_id = cursor.lastrowid
|
|
|
|
# Log in the new user
|
|
session['user_id'] = user_id
|
|
session['username'] = 'local'
|
|
session['tier'] = 'admin'
|
|
session.permanent = True
|
|
|
|
print(f"Created and auto-logged in as new 'local' admin user (ID: {user_id})")
|
|
print(f"Generated password: {random_password}")
|
|
|
|
conn.close()
|
|
return True
|
|
except Exception as e:
|
|
print(f"Error in auto_login_local_mode: {e}")
|
|
return False
|
|
|
|
def skip_auth_login(username):
|
|
"""Skip-auth login: create user record if missing, log them in.
|
|
|
|
Each username gets its own user_id, which drives per-user crawler
|
|
instance and settings isolation. No password is checked. Always
|
|
grants admin tier (matches local-mode behavior).
|
|
|
|
Returns (success, message).
|
|
"""
|
|
import sqlite3
|
|
try:
|
|
conn = sqlite3.connect(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'users.db'))
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute('SELECT id, username FROM users WHERE username = ?', (username,))
|
|
user = cursor.fetchone()
|
|
|
|
if user:
|
|
user_id = user['id']
|
|
else:
|
|
from src.auth_db import hash_password
|
|
random_password = generate_random_password()
|
|
password_hash = hash_password(random_password)
|
|
cursor.execute('''
|
|
INSERT INTO users (username, email, password_hash, verified, tier)
|
|
VALUES (?, ?, ?, 1, 'admin')
|
|
''', (username, f'{username}@skipauth.local', password_hash))
|
|
conn.commit()
|
|
user_id = cursor.lastrowid
|
|
|
|
conn.close()
|
|
|
|
session['user_id'] = user_id
|
|
session['username'] = username
|
|
session['tier'] = 'admin'
|
|
session.permanent = True
|
|
|
|
return True, 'Logged in (authentication skipped)'
|
|
except sqlite3.IntegrityError as e:
|
|
# Most likely the generated email collides with an existing account
|
|
# whose email happens to match. Fall back to a clearer message.
|
|
return False, f'Username conflict: try a different username ({e})'
|
|
except Exception as e:
|
|
print(f"Error in skip_auth_login: {e}")
|
|
return False, f'Login error: {str(e)}'
|
|
|
|
if LOCAL_MODE:
|
|
print("=" * 60)
|
|
print("LOCAL MODE ENABLED")
|
|
print("All users will have admin tier access")
|
|
print("No rate limits or tier restrictions")
|
|
print("Auto-login enabled with 'local' admin account")
|
|
print("=" * 60)
|
|
|
|
if DISABLE_REGISTER:
|
|
print("=" * 60)
|
|
print("REGISTRATION DISABLED")
|
|
print("New user registrations are not allowed")
|
|
print("=" * 60)
|
|
|
|
if DISABLE_GUEST:
|
|
print("=" * 60)
|
|
print("GUEST MODE DISABLED")
|
|
print("Guest login is not allowed")
|
|
print("=" * 60)
|
|
|
|
if DEMO_MODE:
|
|
print("=" * 60)
|
|
print("DEMO MODE ENABLED")
|
|
print("Memory limit: 1.5GB per user")
|
|
print("Crawls will auto-stop when limit is reached")
|
|
print("=" * 60)
|
|
|
|
if SKIP_AUTH:
|
|
print("=" * 60)
|
|
print("⚠️ DANGEROUSLY SKIP AUTH ENABLED")
|
|
print("Anyone can log in as any username with no password!")
|
|
print("Username is used only to separate per-user sessions.")
|
|
print("DO NOT use on a public network or production server!")
|
|
print("=" * 60)
|
|
|
|
if ZOHO.enabled:
|
|
print("=" * 60)
|
|
print("ZOHO OAUTH LOGIN ENABLED")
|
|
print(f"Accounts server: {ZOHO.accounts_url}")
|
|
if ZOHO.allowed_domains:
|
|
print(f"Allowed email domains: {', '.join(ZOHO.allowed_domains)}")
|
|
print(f"New Zoho users: {'created with tier ' + repr(ZOHO.default_tier) if ZOHO.allow_signup else 'not allowed'}")
|
|
print("=" * 60)
|
|
|
|
def get_client_ip():
|
|
"""Get the real client IP address, checking Cloudflare headers first"""
|
|
# Check Cloudflare header first
|
|
if 'CF-Connecting-IP' in request.headers:
|
|
return request.headers['CF-Connecting-IP']
|
|
# Check other common proxy headers
|
|
if 'X-Forwarded-For' in request.headers:
|
|
# X-Forwarded-For can contain multiple IPs, take the first one
|
|
return request.headers['X-Forwarded-For'].split(',')[0].strip()
|
|
if 'X-Real-IP' in request.headers:
|
|
return request.headers['X-Real-IP']
|
|
# Fall back to direct connection IP
|
|
return request.remote_addr
|
|
|
|
def login_required(f):
|
|
"""Decorator to require login for routes"""
|
|
@wraps(f)
|
|
def decorated_function(*args, **kwargs):
|
|
# In local mode, auto-login if not already logged in
|
|
if LOCAL_MODE and 'user_id' not in session:
|
|
auto_login_local_mode()
|
|
elif 'user_id' not in session:
|
|
# Not in local mode and not logged in
|
|
if request.path.startswith('/api/'):
|
|
return jsonify({'success': False, 'error': 'Authentication required'}), 401
|
|
return redirect(url_for('login_page'))
|
|
return f(*args, **kwargs)
|
|
return decorated_function
|
|
|
|
# Multi-tenant crawler instances
|
|
crawler_instances = {} # session_id -> {'crawler': WebCrawler, 'settings': SettingsManager, 'last_accessed': datetime}
|
|
instances_lock = threading.Lock()
|
|
|
|
def get_or_create_crawler():
|
|
"""Get or create a crawler instance for the current session"""
|
|
# Get or create session ID
|
|
if 'session_id' not in session:
|
|
session['session_id'] = str(uuid.uuid4())
|
|
|
|
session_id = session['session_id']
|
|
user_id = session.get('user_id') # Get user_id from session
|
|
tier = session.get('tier', 'guest') # Get tier from session
|
|
|
|
with instances_lock:
|
|
# Check if crawler exists for this session
|
|
if session_id not in crawler_instances:
|
|
print(f"Creating new crawler instance for session: {session_id}, user: {user_id}, tier: {tier}")
|
|
crawler_instances[session_id] = {
|
|
'crawler': WebCrawler(),
|
|
'settings': SettingsManager(session_id=session_id, user_id=user_id, tier=tier), # Per-user settings
|
|
'last_accessed': datetime.now()
|
|
}
|
|
else:
|
|
# Update last accessed time
|
|
crawler_instances[session_id]['last_accessed'] = datetime.now()
|
|
|
|
return crawler_instances[session_id]['crawler']
|
|
|
|
def get_session_settings():
|
|
"""Get the settings manager for the current session"""
|
|
# Get or create session ID
|
|
if 'session_id' not in session:
|
|
session['session_id'] = str(uuid.uuid4())
|
|
|
|
session_id = session['session_id']
|
|
user_id = session.get('user_id') # Get user_id from session
|
|
tier = session.get('tier', 'guest') # Get tier from session
|
|
|
|
with instances_lock:
|
|
# Create instance if it doesn't exist
|
|
if session_id not in crawler_instances:
|
|
print(f"Creating new settings instance for session: {session_id}, user: {user_id}, tier: {tier}")
|
|
crawler_instances[session_id] = {
|
|
'crawler': WebCrawler(),
|
|
'settings': SettingsManager(session_id=session_id, user_id=user_id, tier=tier),
|
|
'last_accessed': datetime.now()
|
|
}
|
|
else:
|
|
# Update last accessed time
|
|
crawler_instances[session_id]['last_accessed'] = datetime.now()
|
|
|
|
return crawler_instances[session_id]['settings']
|
|
|
|
def cleanup_old_instances():
|
|
"""Remove crawler instances that haven't been accessed in 1 hour"""
|
|
timeout = timedelta(hours=1)
|
|
now = datetime.now()
|
|
|
|
with instances_lock:
|
|
sessions_to_remove = []
|
|
for session_id, instance_data in crawler_instances.items():
|
|
if now - instance_data['last_accessed'] > timeout:
|
|
sessions_to_remove.append(session_id)
|
|
|
|
for session_id in sessions_to_remove:
|
|
print(f"Cleaning up crawler instance for session: {session_id}")
|
|
# Stop any running crawls
|
|
try:
|
|
crawler_instances[session_id]['crawler'].stop_crawl()
|
|
except:
|
|
pass
|
|
del crawler_instances[session_id]
|
|
|
|
if sessions_to_remove:
|
|
print(f"Cleaned up {len(sessions_to_remove)} inactive crawler instances")
|
|
|
|
def start_cleanup_thread():
|
|
"""Start background thread to cleanup old instances"""
|
|
def cleanup_loop():
|
|
while True:
|
|
time.sleep(300) # Check every 5 minutes
|
|
try:
|
|
cleanup_old_instances()
|
|
except Exception as e:
|
|
print(f"Error in cleanup thread: {e}")
|
|
|
|
cleanup_thread = threading.Thread(target=cleanup_loop, daemon=True)
|
|
cleanup_thread.start()
|
|
print("Started crawler instance cleanup thread")
|
|
|
|
def generate_csv_export(urls, fields):
|
|
"""Generate CSV export content"""
|
|
output = StringIO()
|
|
writer = csv.DictWriter(output, fieldnames=fields)
|
|
writer.writeheader()
|
|
|
|
for url_data in urls:
|
|
row = {}
|
|
for field in fields:
|
|
value = url_data.get(field, '')
|
|
|
|
# Handle complex data types for CSV
|
|
if field == 'analytics' and isinstance(value, dict):
|
|
analytics_list = []
|
|
if value.get('gtag') or value.get('ga4_id'): analytics_list.append('GA4')
|
|
if value.get('google_analytics'): analytics_list.append('GA')
|
|
if value.get('gtm_id'): analytics_list.append('GTM')
|
|
if value.get('facebook_pixel'): analytics_list.append('FB')
|
|
if value.get('hotjar'): analytics_list.append('HJ')
|
|
if value.get('mixpanel'): analytics_list.append('MP')
|
|
row[field] = ', '.join(analytics_list)
|
|
elif field == 'og_tags' and isinstance(value, dict):
|
|
row[field] = f"{len(value)} tags" if value else ''
|
|
elif field == 'twitter_tags' and isinstance(value, dict):
|
|
row[field] = f"{len(value)} tags" if value else ''
|
|
elif field == 'json_ld' and isinstance(value, list):
|
|
row[field] = f"{len(value)} scripts" if value else ''
|
|
elif field == 'images' and isinstance(value, list):
|
|
row[field] = f"{len(value)} images" if value else ''
|
|
elif field == 'internal_links' and isinstance(value, (int, float)):
|
|
row[field] = f"{int(value)} internal links" if value else '0 internal links'
|
|
elif field == 'external_links' and isinstance(value, (int, float)):
|
|
row[field] = f"{int(value)} external links" if value else '0 external links'
|
|
elif field == 'h2' and isinstance(value, list):
|
|
row[field] = ', '.join(value[:3]) + ('...' if len(value) > 3 else '')
|
|
elif field == 'h3' and isinstance(value, list):
|
|
row[field] = ', '.join(value[:3]) + ('...' if len(value) > 3 else '')
|
|
elif isinstance(value, (dict, list)):
|
|
row[field] = str(value)
|
|
else:
|
|
row[field] = value
|
|
|
|
writer.writerow(row)
|
|
|
|
return output.getvalue()
|
|
|
|
def generate_json_export(urls, fields):
|
|
"""Generate JSON export content"""
|
|
filtered_urls = []
|
|
for url_data in urls:
|
|
filtered_data = {}
|
|
for field in fields:
|
|
value = url_data.get(field, '')
|
|
# Keep complex data structures intact in JSON
|
|
filtered_data[field] = value
|
|
filtered_urls.append(filtered_data)
|
|
|
|
return json.dumps({
|
|
'export_date': time.strftime('%Y-%m-%d %H:%M:%S'),
|
|
'total_urls': len(filtered_urls),
|
|
'fields': fields,
|
|
'data': filtered_urls
|
|
}, indent=2, default=str)
|
|
|
|
def generate_xml_export(urls, fields):
|
|
"""Generate XML export content"""
|
|
root = ET.Element('librecrawl_export')
|
|
root.set('export_date', time.strftime('%Y-%m-%d %H:%M:%S'))
|
|
root.set('total_urls', str(len(urls)))
|
|
|
|
urls_element = ET.SubElement(root, 'urls')
|
|
|
|
for url_data in urls:
|
|
url_element = ET.SubElement(urls_element, 'url')
|
|
for field in fields:
|
|
field_element = ET.SubElement(url_element, field)
|
|
field_element.text = str(url_data.get(field, ''))
|
|
|
|
return ET.tostring(root, encoding='unicode')
|
|
|
|
def generate_links_csv_export(links):
|
|
"""Generate CSV export for links data"""
|
|
output = StringIO()
|
|
fieldnames = ['source_url', 'target_url', 'anchor_text', 'is_internal', 'target_domain', 'target_status', 'placement']
|
|
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
|
|
for link in links:
|
|
row = {
|
|
'source_url': link.get('source_url', ''),
|
|
'target_url': link.get('target_url', ''),
|
|
'anchor_text': link.get('anchor_text', ''),
|
|
'is_internal': 'Yes' if link.get('is_internal') else 'No',
|
|
'target_domain': link.get('target_domain', ''),
|
|
'target_status': link['target_status'] if link.get('target_status') is not None else 'Not crawled',
|
|
'placement': link.get('placement', 'body')
|
|
}
|
|
writer.writerow(row)
|
|
|
|
return output.getvalue()
|
|
|
|
def generate_links_json_export(links):
|
|
"""Generate JSON export for links data"""
|
|
return json.dumps(links, indent=2)
|
|
|
|
def issue_matches_exclusion(issue, exclusion_patterns):
|
|
"""Check whether a single issue's URL matches any exclusion pattern"""
|
|
from fnmatch import fnmatch
|
|
from urllib.parse import urlparse
|
|
|
|
path = urlparse(issue.get('url', '')).path
|
|
|
|
for pattern in exclusion_patterns:
|
|
if not pattern.strip() or pattern.strip().startswith('#'):
|
|
continue
|
|
|
|
if '*' in pattern:
|
|
if fnmatch(path, pattern):
|
|
return True
|
|
elif path == pattern or path.startswith(pattern.rstrip('*')):
|
|
return True
|
|
|
|
return False
|
|
|
|
def filter_issues_by_exclusion_patterns(issues, exclusion_patterns):
|
|
"""Filter issues based on exclusion patterns (applies current settings to loaded crawls)"""
|
|
if not exclusion_patterns:
|
|
return issues
|
|
return [issue for issue in issues if not issue_matches_exclusion(issue, exclusion_patterns)]
|
|
|
|
def get_exclusion_patterns(settings_manager):
|
|
"""Current issue-exclusion patterns as a list of non-empty lines"""
|
|
current_settings = settings_manager.get_settings()
|
|
exclusion_patterns_text = current_settings.get('issueExclusionPatterns', '')
|
|
return [p.strip() for p in exclusion_patterns_text.split('\n') if p.strip()]
|
|
|
|
def generate_issues_csv_export(issues):
|
|
"""Generate CSV export for issues data"""
|
|
output = StringIO()
|
|
fieldnames = ['url', 'type', 'category', 'issue', 'details']
|
|
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
|
|
for issue in issues:
|
|
row = {
|
|
'url': issue.get('url', ''),
|
|
'type': issue.get('type', ''),
|
|
'category': issue.get('category', ''),
|
|
'issue': issue.get('issue', ''),
|
|
'details': issue.get('details', '')
|
|
}
|
|
writer.writerow(row)
|
|
|
|
return output.getvalue()
|
|
|
|
def generate_issues_json_export(issues):
|
|
"""Generate JSON export for issues data"""
|
|
# Group issues by URL for better organization
|
|
issues_by_url = {}
|
|
for issue in issues:
|
|
url = issue.get('url', '')
|
|
if url not in issues_by_url:
|
|
issues_by_url[url] = []
|
|
issues_by_url[url].append({
|
|
'type': issue.get('type', ''),
|
|
'category': issue.get('category', ''),
|
|
'issue': issue.get('issue', ''),
|
|
'details': issue.get('details', '')
|
|
})
|
|
|
|
return json.dumps({
|
|
'export_date': time.strftime('%Y-%m-%d %H:%M:%S'),
|
|
'total_issues': len(issues),
|
|
'total_urls_with_issues': len(issues_by_url),
|
|
'issues_by_url': issues_by_url,
|
|
'all_issues': issues
|
|
}, indent=2)
|
|
|
|
@app.route('/login')
|
|
def login_page():
|
|
# In local mode, auto-login and redirect to index
|
|
if LOCAL_MODE:
|
|
auto_login_local_mode()
|
|
return redirect(url_for('index'))
|
|
# Redirect to app if already logged in
|
|
if 'user_id' in session:
|
|
return redirect(url_for('index'))
|
|
return render_template('login.html', registration_disabled=DISABLE_REGISTER, guest_disabled=DISABLE_GUEST, skip_auth=SKIP_AUTH,
|
|
zoho_enabled=ZOHO.enabled, login_error=session.pop('login_error', None))
|
|
|
|
def _zoho_redirect_uri():
|
|
return ZOHO.redirect_uri or url_for('zoho_callback', _external=True)
|
|
|
|
@app.route('/auth/zoho/login')
|
|
def zoho_login():
|
|
if not ZOHO.enabled:
|
|
return redirect(url_for('login_page'))
|
|
state = secrets.token_urlsafe(32)
|
|
session['zoho_oauth_state'] = state
|
|
return redirect(ZOHO.authorize_url(_zoho_redirect_uri(), state))
|
|
|
|
@app.route('/auth/zoho/callback')
|
|
def zoho_callback():
|
|
if not ZOHO.enabled:
|
|
return redirect(url_for('login_page'))
|
|
|
|
def fail(message):
|
|
session['login_error'] = message
|
|
return redirect(url_for('login_page'))
|
|
|
|
expected_state = session.pop('zoho_oauth_state', None)
|
|
if request.args.get('error'):
|
|
return fail('Zoho login was cancelled or denied.')
|
|
if not expected_state or not secrets.compare_digest(expected_state, request.args.get('state', '')):
|
|
return fail('Zoho login expired or was invalid. Please try again.')
|
|
code = request.args.get('code')
|
|
if not code:
|
|
return fail('Zoho did not return an authorization code.')
|
|
|
|
try:
|
|
accounts_server = ZOHO.accounts_server_for(request.args.get('accounts-server'))
|
|
identity = ZOHO.fetch_identity(code, _zoho_redirect_uri(), accounts_server)
|
|
except ZohoOAuthError as e:
|
|
print(f"Zoho OAuth error: {e}")
|
|
return fail(str(e))
|
|
|
|
if not ZOHO.domain_allowed(identity['email']):
|
|
return fail('Your Zoho account\'s email domain is not allowed to log in here.')
|
|
|
|
success, message, user_data = get_or_create_zoho_user(
|
|
identity['zoho_id'], identity['email'], identity['name'],
|
|
default_tier=ZOHO.default_tier, allow_create=ZOHO.allow_signup)
|
|
if not success:
|
|
return fail(message)
|
|
|
|
session.clear()
|
|
session['user_id'] = user_data['id']
|
|
session['username'] = user_data['username']
|
|
session['tier'] = user_data['tier']
|
|
session.permanent = True
|
|
return redirect(url_for('index'))
|
|
|
|
@app.route('/register')
|
|
def register_page():
|
|
# Redirect to app if already logged in
|
|
if 'user_id' in session:
|
|
return redirect(url_for('index'))
|
|
return render_template('register.html', registration_disabled=DISABLE_REGISTER)
|
|
|
|
@app.route('/verify')
|
|
def verify_email():
|
|
"""Email verification endpoint"""
|
|
token = request.args.get('token')
|
|
|
|
if not token:
|
|
return render_template('verification_result.html',
|
|
success=False,
|
|
message='Invalid verification link',
|
|
app_source='main')
|
|
|
|
# Verify the token
|
|
success, message, app_source, user_email = verify_token(token)
|
|
|
|
# Send welcome email if successful
|
|
if success and user_email:
|
|
try:
|
|
user = get_user_by_email(user_email)
|
|
if user:
|
|
send_welcome_email(user_email, user['username'], app_source or 'main')
|
|
except Exception as e:
|
|
print(f"Error sending welcome email: {e}")
|
|
|
|
# Determine redirect URL based on app_source
|
|
redirect_url = None
|
|
if success:
|
|
if app_source == 'workshop':
|
|
redirect_url = os.getenv('WORKSHOP_APP_URL', 'https://workshop.librecrawl.com')
|
|
else:
|
|
redirect_url = url_for('login_page')
|
|
|
|
return render_template('verification_result.html',
|
|
success=success,
|
|
message=message,
|
|
app_source=app_source or 'main',
|
|
redirect_url=redirect_url)
|
|
|
|
@app.route('/api/register', methods=['POST'])
|
|
def register():
|
|
# Check if registration is disabled
|
|
if DISABLE_REGISTER:
|
|
return jsonify({'success': False, 'message': 'Registration is currently disabled'})
|
|
|
|
data = request.get_json()
|
|
username = data.get('username')
|
|
email = data.get('email')
|
|
password = data.get('password')
|
|
|
|
success, message, user_id = create_user(username, email, password)
|
|
|
|
# In local mode, auto-verify and set to admin tier
|
|
if success and LOCAL_MODE:
|
|
try:
|
|
from src.auth_db import verify_user, set_user_tier
|
|
# Get the user that was just created
|
|
import sqlite3
|
|
conn = sqlite3.connect(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'users.db'))
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
cursor.execute('SELECT id FROM users WHERE username = ?', (username,))
|
|
user = cursor.fetchone()
|
|
conn.close()
|
|
|
|
if user:
|
|
verify_user(user['id'])
|
|
set_user_tier(user['id'], 'admin')
|
|
message = 'Account created and verified! You have admin access in local mode.'
|
|
except Exception as e:
|
|
print(f"Error during local mode auto-verification: {e}")
|
|
# Don't fail the registration, just log the error
|
|
# The account is still created successfully
|
|
elif success:
|
|
# Not in local mode - send verification email
|
|
is_resend = (message == 'resend')
|
|
try:
|
|
# Create verification token
|
|
token = create_verification_token(user_id, app_source='main')
|
|
if token:
|
|
# Send verification email
|
|
email_success, email_message = send_verification_email(
|
|
email, username, token, app_source='main', is_resend=is_resend
|
|
)
|
|
if email_success:
|
|
if is_resend:
|
|
message = 'A verification email was already sent to this address. We\'ve updated your account details and sent a new verification link.'
|
|
else:
|
|
message = 'Registration successful! Please check your email to verify your account.'
|
|
else:
|
|
message = 'Account created, but we could not send the verification email. Please contact support.'
|
|
print(f"Email error: {email_message}")
|
|
else:
|
|
message = 'Account created, but verification token generation failed. Please contact support.'
|
|
except Exception as e:
|
|
print(f"Error sending verification email: {e}")
|
|
message = 'Account created, but we could not send the verification email. Please contact support.'
|
|
|
|
return jsonify({'success': success, 'message': message})
|
|
|
|
@app.route('/api/login', methods=['POST'])
|
|
def login():
|
|
data = request.get_json()
|
|
username = (data.get('username') or '').strip()
|
|
password = data.get('password') or ''
|
|
|
|
# Dangerously skip auth: accept any username with no password.
|
|
# Username is only used to separate per-user sessions.
|
|
if SKIP_AUTH:
|
|
if not username:
|
|
return jsonify({'success': False, 'message': 'Username required'})
|
|
if len(username) > 50:
|
|
return jsonify({'success': False, 'message': 'Username must be 50 characters or less'})
|
|
success, message = skip_auth_login(username)
|
|
return jsonify({'success': success, 'message': message})
|
|
|
|
success, message, user_data = authenticate_user(username, password)
|
|
|
|
if success:
|
|
session['user_id'] = user_data['id']
|
|
session['username'] = user_data['username']
|
|
# In local mode, always give admin tier
|
|
session['tier'] = 'admin' if LOCAL_MODE else user_data['tier']
|
|
session.permanent = True # Remember login
|
|
|
|
return jsonify({'success': success, 'message': message})
|
|
|
|
@app.route('/api/guest-login', methods=['POST'])
|
|
def guest_login():
|
|
"""Login as a guest user (no account required, limited to 3 crawls/24h)"""
|
|
if DISABLE_GUEST:
|
|
return jsonify({'success': False, 'message': 'Guest login is disabled'})
|
|
|
|
# Create a guest session with no user_id but with tier='guest'
|
|
# In local mode, guests also get admin tier
|
|
session['user_id'] = None
|
|
session['username'] = 'Guest'
|
|
session['tier'] = 'admin' if LOCAL_MODE else 'guest'
|
|
session.permanent = False # Don't persist guest sessions
|
|
|
|
return jsonify({'success': True, 'message': 'Logged in as guest'})
|
|
|
|
@app.route('/api/logout', methods=['POST'])
|
|
@login_required
|
|
def logout():
|
|
session.clear()
|
|
return jsonify({'success': True, 'message': 'Logged out successfully'})
|
|
|
|
@app.route('/api/user/info')
|
|
@login_required
|
|
def user_info():
|
|
"""Get current user info including tier"""
|
|
from src.auth_db import get_crawls_last_24h
|
|
user_id = session.get('user_id')
|
|
tier = session.get('tier', 'guest')
|
|
username = session.get('username')
|
|
|
|
# Get crawl count
|
|
crawls_today = 0
|
|
if tier == 'guest':
|
|
# For guests, count from IP address
|
|
client_ip = get_client_ip()
|
|
crawls_today = get_guest_crawls_last_24h(client_ip)
|
|
else:
|
|
# For registered users, count from database
|
|
crawls_today = get_crawls_last_24h(user_id)
|
|
|
|
return jsonify({
|
|
'success': True,
|
|
'user': {
|
|
'id': user_id,
|
|
'username': username,
|
|
'tier': tier,
|
|
'crawls_today': crawls_today,
|
|
'crawls_remaining': max(0, 3 - crawls_today) if tier == 'guest' else -1
|
|
}
|
|
})
|
|
|
|
@app.route('/')
|
|
def index():
|
|
# In local mode, auto-login if not already logged in
|
|
if LOCAL_MODE and 'user_id' not in session:
|
|
auto_login_local_mode()
|
|
elif 'user_id' not in session:
|
|
# Not in local mode and not logged in, redirect to login
|
|
return redirect(url_for('login_page'))
|
|
return render_template('index.html')
|
|
|
|
@app.route('/dashboard')
|
|
@login_required
|
|
def dashboard():
|
|
"""Crawl history dashboard"""
|
|
return render_template('dashboard.html')
|
|
|
|
@app.route('/debug/memory')
|
|
@login_required
|
|
def debug_memory_page():
|
|
"""Debug page with nice UI for memory monitoring"""
|
|
return render_template('debug_memory.html')
|
|
|
|
@app.route('/api/start_crawl', methods=['POST'])
|
|
@login_required
|
|
def start_crawl():
|
|
from src.auth_db import get_crawls_last_24h, log_crawl_start
|
|
|
|
data = request.get_json()
|
|
url = data.get('url')
|
|
|
|
if not url:
|
|
return jsonify({'success': False, 'error': 'URL is required'})
|
|
|
|
user_id = session.get('user_id')
|
|
tier = session.get('tier', 'guest')
|
|
|
|
# Check guest limits (IP-based) - skip in local mode
|
|
if tier == 'guest' and not LOCAL_MODE:
|
|
client_ip = get_client_ip()
|
|
crawls_from_ip = get_guest_crawls_last_24h(client_ip)
|
|
|
|
if crawls_from_ip >= 3:
|
|
return jsonify({
|
|
'success': False,
|
|
'error': 'Guest limit reached: 3 crawls per 24 hours from your IP address. Please register for unlimited crawls.'
|
|
})
|
|
|
|
# Log this guest crawl
|
|
log_guest_crawl(client_ip)
|
|
|
|
# Get or create crawler for this session (this also ensures the session
|
|
# has a session_id — reading it earlier would return None on a session
|
|
# whose first API call is start_crawl, silently disabling persistence)
|
|
crawler = get_or_create_crawler()
|
|
settings_manager = get_session_settings()
|
|
session_id = session.get('session_id')
|
|
|
|
# Apply current settings to crawler before starting
|
|
try:
|
|
crawler_config = settings_manager.get_crawler_config()
|
|
crawler.update_config(crawler_config)
|
|
except Exception as e:
|
|
print(f"Warning: Could not apply settings: {e}")
|
|
|
|
# Enforce demo mode limits
|
|
if DEMO_MODE:
|
|
crawler.config['demo_mode'] = True
|
|
crawler.config['demo_memory_limit_bytes'] = int(1.5 * 1024 * 1024 * 1024) # 1.5GB
|
|
|
|
# Pass user_id and session_id for database persistence
|
|
success, message = crawler.start_crawl(url, user_id=user_id, session_id=session_id)
|
|
|
|
# Store crawl_id in session
|
|
if success and crawler.crawl_id:
|
|
session['current_crawl_id'] = crawler.crawl_id
|
|
# Also log to old crawl_history for compatibility
|
|
log_crawl_start(user_id, url)
|
|
|
|
return jsonify({'success': success, 'message': message, 'crawl_id': crawler.crawl_id})
|
|
|
|
@app.route('/api/stop_crawl', methods=['POST'])
|
|
@login_required
|
|
def stop_crawl():
|
|
crawler = get_or_create_crawler()
|
|
success, message = crawler.stop_crawl()
|
|
return jsonify({'success': success, 'message': message})
|
|
|
|
@app.route('/api/crawl_status')
|
|
@login_required
|
|
def crawl_status():
|
|
crawler = get_or_create_crawler()
|
|
settings_manager = get_session_settings()
|
|
|
|
# Lightweight probe: status + stats only, no data (export pre-checks etc.)
|
|
if request.args.get('stats_only'):
|
|
status_data = crawler.get_status_light()
|
|
if crawler.base_url:
|
|
status_data['stats']['baseUrl'] = crawler.base_url
|
|
return jsonify(status_data)
|
|
|
|
since_seq = request.args.get('since_seq', type=int)
|
|
|
|
if since_seq is None:
|
|
# Full snapshot (initial page load, save-crawl fetch)
|
|
status_data = crawler.get_status()
|
|
if crawler.base_url:
|
|
status_data['stats']['baseUrl'] = crawler.base_url
|
|
|
|
exclusion_patterns = get_exclusion_patterns(settings_manager)
|
|
status_data['issues'] = filter_issues_by_exclusion_patterns(
|
|
status_data.get('issues', []), exclusion_patterns)
|
|
|
|
return jsonify(status_data)
|
|
|
|
# Event mode: everything that changed after since_seq. If the client's
|
|
# epoch is stale (new crawl / loaded crawl / resume), reset=True and the
|
|
# events replay from zero so the client rebuilds its state.
|
|
#
|
|
# Events are read before the stats snapshot on purpose: a crawl running
|
|
# between the two calls should make the counters look newer than the rows,
|
|
# never older, so the UI never shows fewer crawled URLs than it lists.
|
|
epoch = request.args.get('epoch', '')
|
|
reset, events, latest_seq, current_epoch = crawler.event_log.events_since(since_seq, epoch)
|
|
|
|
status_data = crawler.get_status_light()
|
|
if crawler.base_url:
|
|
status_data['stats']['baseUrl'] = crawler.base_url
|
|
|
|
exclusion_patterns = get_exclusion_patterns(settings_manager)
|
|
if exclusion_patterns:
|
|
events = [ev for ev in events
|
|
if ev['kind'] != 'issue' or not issue_matches_exclusion(ev['data'], exclusion_patterns)]
|
|
|
|
status_data.update({
|
|
'reset': reset,
|
|
'epoch': current_epoch,
|
|
'latest_seq': latest_seq,
|
|
'events': events
|
|
})
|
|
return jsonify(status_data)
|
|
|
|
@app.route('/api/visualization_data')
|
|
@login_required
|
|
def visualization_data():
|
|
"""Get graph data for site structure visualization"""
|
|
try:
|
|
crawler = get_or_create_crawler()
|
|
status_data = crawler.get_status()
|
|
|
|
# Get URLs from the status data
|
|
crawled_pages = status_data.get('urls', [])
|
|
all_links = status_data.get('links', [])
|
|
|
|
# Build nodes and edges for the graph
|
|
nodes = []
|
|
edges = []
|
|
url_to_id = {}
|
|
|
|
# Create nodes from crawled pages (limit to prevent lag)
|
|
max_nodes = 500 # Optimization: limit nodes for performance
|
|
pages_to_visualize = crawled_pages[:max_nodes]
|
|
|
|
for idx, page in enumerate(pages_to_visualize):
|
|
url = page.get('url', '')
|
|
status_code = page.get('status_code', 0)
|
|
|
|
# Assign color based on status code
|
|
if 200 <= status_code < 300:
|
|
color = '#10b981' # Green for 2xx
|
|
elif 300 <= status_code < 400:
|
|
color = '#3b82f6' # Blue for 3xx
|
|
elif 400 <= status_code < 500:
|
|
color = '#f59e0b' # Orange for 4xx
|
|
elif 500 <= status_code < 600:
|
|
color = '#ef4444' # Red for 5xx
|
|
else:
|
|
color = '#6b7280' # Gray for other
|
|
|
|
# Create node
|
|
node = {
|
|
'data': {
|
|
'id': f'node-{idx}',
|
|
'label': url.split('/')[-1] or url.split('//')[-1], # Use last path segment or domain
|
|
'url': url,
|
|
'status_code': status_code,
|
|
'title': page.get('title', ''),
|
|
'color': color,
|
|
'size': 30 if idx == 0 else 20, # Make root node larger
|
|
'depth': page.get('depth', 0)
|
|
}
|
|
}
|
|
nodes.append(node)
|
|
url_to_id[url] = f'node-{idx}'
|
|
|
|
# Create edges from links data
|
|
# Links are stored as: {'source_url': url, 'target_url': url, 'is_internal': bool, ...}
|
|
edges_set = set() # Use set to avoid duplicate edges
|
|
for link in all_links:
|
|
if link.get('is_internal'): # Only use internal links
|
|
source_url = link.get('source_url', '')
|
|
target_url = link.get('target_url', '')
|
|
|
|
source_id = url_to_id.get(source_url)
|
|
target_id = url_to_id.get(target_url)
|
|
|
|
if source_id and target_id and source_id != target_id:
|
|
edge_key = f'{source_id}-{target_id}'
|
|
if edge_key not in edges_set:
|
|
edges_set.add(edge_key)
|
|
edge = {
|
|
'data': {
|
|
'id': f'edge-{edge_key}',
|
|
'source': source_id,
|
|
'target': target_id
|
|
}
|
|
}
|
|
edges.append(edge)
|
|
|
|
return jsonify({
|
|
'success': True,
|
|
'nodes': nodes,
|
|
'edges': edges,
|
|
'total_pages': len(crawled_pages),
|
|
'visualized_pages': len(nodes),
|
|
'truncated': len(crawled_pages) > max_nodes
|
|
})
|
|
|
|
except Exception as e:
|
|
print(f"Error generating visualization data: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
return jsonify({
|
|
'success': False,
|
|
'error': str(e),
|
|
'nodes': [],
|
|
'edges': []
|
|
})
|
|
|
|
@app.route('/api/debug/memory')
|
|
@login_required
|
|
def debug_memory():
|
|
"""Debug endpoint showing memory stats for all active crawler instances"""
|
|
with instances_lock:
|
|
memory_stats = {
|
|
'total_instances': len(crawler_instances),
|
|
'instances': []
|
|
}
|
|
|
|
for session_id, instance_data in crawler_instances.items():
|
|
crawler = instance_data['crawler']
|
|
stats = crawler.memory_monitor.get_stats()
|
|
|
|
memory_stats['instances'].append({
|
|
'session_id': session_id[:8] + '...', # Truncate for privacy
|
|
'last_accessed': instance_data['last_accessed'].isoformat(),
|
|
'urls_crawled': len(crawler.crawl_results),
|
|
'memory': stats,
|
|
'data_sizes': crawler.user_memory.get_stats()
|
|
})
|
|
|
|
return jsonify(memory_stats)
|
|
|
|
@app.route('/api/debug/memory/profile')
|
|
@login_required
|
|
def debug_memory_profile():
|
|
"""Detailed memory profiling - what's actually using the RAM"""
|
|
from src.core.memory_profiler import MemoryProfiler
|
|
|
|
with instances_lock:
|
|
profiles = []
|
|
|
|
for session_id, instance_data in crawler_instances.items():
|
|
crawler = instance_data['crawler']
|
|
|
|
# Get object breakdown
|
|
breakdown = MemoryProfiler.get_object_memory_breakdown()
|
|
|
|
profiles.append({
|
|
'session_id': session_id[:8] + '...',
|
|
'urls_crawled': len(crawler.crawl_results),
|
|
'object_breakdown': breakdown,
|
|
'data_sizes': crawler.user_memory.get_stats()
|
|
})
|
|
|
|
return jsonify({
|
|
'total_instances': len(crawler_instances),
|
|
'profiles': profiles
|
|
})
|
|
|
|
@app.route('/api/filter_issues', methods=['POST'])
|
|
@login_required
|
|
def filter_issues():
|
|
try:
|
|
data = request.get_json()
|
|
issues = data.get('issues', [])
|
|
settings_manager = get_session_settings()
|
|
|
|
# Get current exclusion patterns
|
|
current_settings = settings_manager.get_settings()
|
|
exclusion_patterns_text = current_settings.get('issueExclusionPatterns', '')
|
|
exclusion_patterns = [p.strip() for p in exclusion_patterns_text.split('\n') if p.strip()]
|
|
|
|
# Filter issues
|
|
filtered_issues = filter_issues_by_exclusion_patterns(issues, exclusion_patterns)
|
|
|
|
return jsonify({'success': True, 'issues': filtered_issues})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/get_settings')
|
|
@login_required
|
|
def get_settings():
|
|
try:
|
|
settings_manager = get_session_settings()
|
|
settings = settings_manager.get_settings()
|
|
return jsonify({'success': True, 'settings': settings})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/save_settings', methods=['POST'])
|
|
@login_required
|
|
def save_settings():
|
|
try:
|
|
data = request.get_json()
|
|
settings_manager = get_session_settings()
|
|
success, message = settings_manager.save_settings(data)
|
|
return jsonify({'success': success, 'message': message})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/reset_settings', methods=['POST'])
|
|
@login_required
|
|
def reset_settings():
|
|
try:
|
|
settings_manager = get_session_settings()
|
|
success, message = settings_manager.reset_settings()
|
|
return jsonify({'success': success, 'message': message})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/update_crawler_settings', methods=['POST'])
|
|
@login_required
|
|
def update_crawler_settings():
|
|
try:
|
|
crawler = get_or_create_crawler()
|
|
settings_manager = get_session_settings()
|
|
# Get current settings and update crawler configuration
|
|
crawler_config = settings_manager.get_crawler_config()
|
|
crawler.update_config(crawler_config)
|
|
return jsonify({'success': True, 'message': 'Crawler settings updated'})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/pause_crawl', methods=['POST'])
|
|
@login_required
|
|
def pause_crawl():
|
|
try:
|
|
crawler = get_or_create_crawler()
|
|
success, message = crawler.pause_crawl()
|
|
return jsonify({'success': success, 'message': message})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/resume_crawl', methods=['POST'])
|
|
@login_required
|
|
def resume_crawl():
|
|
try:
|
|
crawler = get_or_create_crawler()
|
|
success, message = crawler.resume_crawl()
|
|
return jsonify({'success': success, 'message': message})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/crawls/list')
|
|
@login_required
|
|
def list_crawls():
|
|
"""Get all crawls for current user"""
|
|
try:
|
|
user_id = session.get('user_id')
|
|
from src.crawl_db import get_user_crawls, get_crawl_count
|
|
|
|
limit = request.args.get('limit', 50, type=int)
|
|
offset = request.args.get('offset', 0, type=int)
|
|
status_filter = request.args.get('status')
|
|
|
|
crawls = get_user_crawls(user_id, limit=limit, offset=offset, status_filter=status_filter)
|
|
total_count = get_crawl_count(user_id)
|
|
|
|
return jsonify({
|
|
'success': True,
|
|
'crawls': crawls,
|
|
'total': total_count
|
|
})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/crawls/<int:crawl_id>')
|
|
@login_required
|
|
def get_crawl(crawl_id):
|
|
"""Get complete crawl data by ID"""
|
|
try:
|
|
user_id = session.get('user_id')
|
|
from src.crawl_db import get_crawl_by_id, load_crawled_urls, load_crawl_links, load_crawl_issues
|
|
|
|
# Get crawl metadata
|
|
crawl = get_crawl_by_id(crawl_id)
|
|
if not crawl:
|
|
return jsonify({'success': False, 'error': 'Crawl not found'}), 404
|
|
|
|
# Check ownership (guests have user_id = None)
|
|
if user_id and crawl.get('user_id') != user_id:
|
|
return jsonify({'success': False, 'error': 'Unauthorized'}), 403
|
|
|
|
# Load all data
|
|
urls = load_crawled_urls(crawl_id)
|
|
links = load_crawl_links(crawl_id)
|
|
issues = load_crawl_issues(crawl_id)
|
|
|
|
return jsonify({
|
|
'success': True,
|
|
'crawl': crawl,
|
|
'urls': urls,
|
|
'links': links,
|
|
'issues': issues
|
|
})
|
|
except Exception as e:
|
|
import traceback
|
|
traceback.print_exc()
|
|
return jsonify({'success': False, 'error': str(e)}), 500
|
|
|
|
@app.route('/api/crawls/<int:crawl_id>/load', methods=['POST'])
|
|
@login_required
|
|
def load_crawl_into_session(crawl_id):
|
|
"""Load a historical crawl into the current session"""
|
|
try:
|
|
user_id = session.get('user_id')
|
|
from src.crawl_db import get_crawl_by_id, load_crawled_urls, load_crawl_links, load_crawl_issues
|
|
|
|
# Get crawl metadata
|
|
crawl = get_crawl_by_id(crawl_id)
|
|
if not crawl:
|
|
return jsonify({'success': False, 'error': 'Crawl not found'}), 404
|
|
|
|
# Check ownership
|
|
if user_id and crawl.get('user_id') != user_id:
|
|
return jsonify({'success': False, 'error': 'Unauthorized'}), 403
|
|
|
|
# Get current crawler instance
|
|
crawler = get_or_create_crawler()
|
|
|
|
# Stop any running crawl
|
|
if crawler.is_running:
|
|
crawler.stop_crawl()
|
|
|
|
# Load all data from database
|
|
urls = load_crawled_urls(crawl_id)
|
|
links = load_crawl_links(crawl_id)
|
|
issues = load_crawl_issues(crawl_id)
|
|
|
|
# Inject into the session's crawler. This starts a new event-log
|
|
# epoch, so any polling client resets and replays the loaded data.
|
|
crawler.load_data(crawl, urls, links, issues)
|
|
|
|
return jsonify({
|
|
'success': True,
|
|
'message': f'Loaded {len(urls)} URLs, {len(links)} links, {len(issues)} issues',
|
|
'urls_count': len(urls),
|
|
'links_count': len(links),
|
|
'issues_count': len(issues),
|
|
'should_refresh_ui': True
|
|
})
|
|
|
|
except Exception as e:
|
|
import traceback
|
|
traceback.print_exc()
|
|
return jsonify({'success': False, 'error': str(e)}), 500
|
|
|
|
@app.route('/api/crawls/<int:crawl_id>/resume', methods=['POST'])
|
|
@login_required
|
|
def resume_crawl_endpoint(crawl_id):
|
|
"""Resume an interrupted crawl"""
|
|
try:
|
|
user_id = session.get('user_id')
|
|
session_id = session.get('session_id')
|
|
|
|
# Get crawler for this session
|
|
crawler = get_or_create_crawler()
|
|
|
|
# Enforce demo mode limits on resumed crawls
|
|
if DEMO_MODE:
|
|
crawler.config['demo_mode'] = True
|
|
crawler.config['demo_memory_limit_bytes'] = int(1.5 * 1024 * 1024 * 1024)
|
|
|
|
# Resume from database
|
|
success, message = crawler.resume_from_database(crawl_id, user_id=user_id, session_id=session_id)
|
|
|
|
if success:
|
|
session['current_crawl_id'] = crawl_id
|
|
|
|
return jsonify({'success': success, 'message': message})
|
|
except Exception as e:
|
|
import traceback
|
|
traceback.print_exc()
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/crawls/<int:crawl_id>/delete', methods=['DELETE'])
|
|
@login_required
|
|
def delete_crawl_endpoint(crawl_id):
|
|
"""Delete a crawl and all associated data"""
|
|
try:
|
|
user_id = session.get('user_id')
|
|
from src.crawl_db import delete_crawl, get_crawl_by_id
|
|
|
|
# Verify ownership
|
|
crawl = get_crawl_by_id(crawl_id)
|
|
if not crawl:
|
|
return jsonify({'success': False, 'error': 'Crawl not found'}), 404
|
|
|
|
if user_id and crawl.get('user_id') != user_id:
|
|
return jsonify({'success': False, 'error': 'Unauthorized'}), 403
|
|
|
|
success = delete_crawl(crawl_id)
|
|
return jsonify({'success': success, 'message': 'Crawl deleted successfully' if success else 'Failed to delete crawl'})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/crawls/<int:crawl_id>/archive', methods=['POST'])
|
|
@login_required
|
|
def archive_crawl(crawl_id):
|
|
"""Archive crawl (mark as archived but keep data)"""
|
|
try:
|
|
user_id = session.get('user_id')
|
|
from src.crawl_db import set_crawl_status, get_crawl_by_id
|
|
|
|
# Verify ownership
|
|
crawl = get_crawl_by_id(crawl_id)
|
|
if not crawl:
|
|
return jsonify({'success': False, 'error': 'Crawl not found'}), 404
|
|
|
|
if user_id and crawl.get('user_id') != user_id:
|
|
return jsonify({'success': False, 'error': 'Unauthorized'}), 403
|
|
|
|
success = set_crawl_status(crawl_id, 'archived')
|
|
return jsonify({'success': success, 'message': 'Crawl archived successfully' if success else 'Failed to archive crawl'})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/crawls/stats')
|
|
@login_required
|
|
def crawl_stats():
|
|
"""Get statistics about user's crawls"""
|
|
try:
|
|
user_id = session.get('user_id')
|
|
from src.crawl_db import get_crawl_count, get_database_size_mb
|
|
import sqlite3
|
|
|
|
# Get counts by status
|
|
conn = sqlite3.connect(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'users.db'))
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute('''
|
|
SELECT status, COUNT(*) as count
|
|
FROM crawls
|
|
WHERE user_id = ?
|
|
GROUP BY status
|
|
''', (user_id,))
|
|
|
|
status_counts = {row[0]: row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
return jsonify({
|
|
'success': True,
|
|
'total_crawls': get_crawl_count(user_id),
|
|
'by_status': status_counts,
|
|
'database_size_mb': get_database_size_mb()
|
|
})
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
@app.route('/api/export_stream')
|
|
@login_required
|
|
def export_stream():
|
|
"""Stream export data directly as a file download — avoids loading full export into memory"""
|
|
from flask import Response
|
|
|
|
try:
|
|
export_format = request.args.get('format', 'csv')
|
|
fields_param = request.args.get('fields', 'url,status_code,title')
|
|
# Restrict field names to identifier-like strings — they become XML tag
|
|
# names and CSV headers, so arbitrary query input must not pass through
|
|
export_fields = [f.strip() for f in fields_param.split(',')
|
|
if f.strip() and f.strip().replace('_', '').isalnum() and not f.strip()[0].isdigit()]
|
|
data_type = request.args.get('type', 'urls') # urls, links, issues
|
|
|
|
# Get data from current crawler (already in memory from active or loaded crawl)
|
|
crawler = get_or_create_crawler()
|
|
timestamp = int(time.time())
|
|
|
|
if data_type == 'links':
|
|
links = crawler.link_manager.all_links if crawler.link_manager else []
|
|
# Update link statuses from crawled URLs
|
|
if links and crawler.crawl_results:
|
|
status_lookup = {u['url']: u.get('status_code') for u in crawler.crawl_results}
|
|
for link in links:
|
|
target = link.get('target_url')
|
|
if target in status_lookup:
|
|
link['target_status'] = status_lookup[target]
|
|
|
|
if export_format == 'json':
|
|
return Response(
|
|
_stream_links_json(links),
|
|
mimetype='application/json',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_links_{timestamp}.json'}
|
|
)
|
|
elif export_format == 'xml':
|
|
return Response(
|
|
_stream_links_xml(links),
|
|
mimetype='application/xml',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_links_{timestamp}.xml'}
|
|
)
|
|
elif export_format == 'xlsx':
|
|
return Response(
|
|
_links_xlsx(links),
|
|
mimetype=XLSX_MIMETYPE,
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_links_{timestamp}.xlsx'}
|
|
)
|
|
else:
|
|
return Response(
|
|
_stream_links_csv(links),
|
|
mimetype='text/csv',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_links_{timestamp}.csv'}
|
|
)
|
|
|
|
elif data_type == 'issues':
|
|
issues = crawler.issue_detector.get_issues() if crawler.issue_detector else []
|
|
# Apply exclusion patterns
|
|
settings_manager = get_session_settings()
|
|
current_settings = settings_manager.get_settings()
|
|
exclusion_text = current_settings.get('issueExclusionPatterns', '')
|
|
exclusion_patterns = [p.strip() for p in exclusion_text.split('\n') if p.strip()]
|
|
issues = filter_issues_by_exclusion_patterns(issues, exclusion_patterns)
|
|
|
|
if export_format == 'json':
|
|
return Response(
|
|
_stream_issues_json(issues),
|
|
mimetype='application/json',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_issues_{timestamp}.json'}
|
|
)
|
|
elif export_format == 'xml':
|
|
return Response(
|
|
_stream_issues_xml(issues),
|
|
mimetype='application/xml',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_issues_{timestamp}.xml'}
|
|
)
|
|
elif export_format == 'xlsx':
|
|
return Response(
|
|
_issues_xlsx(issues),
|
|
mimetype=XLSX_MIMETYPE,
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_issues_{timestamp}.xlsx'}
|
|
)
|
|
else:
|
|
return Response(
|
|
_stream_issues_csv(issues),
|
|
mimetype='text/csv',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_issues_{timestamp}.csv'}
|
|
)
|
|
|
|
else:
|
|
# URLs export
|
|
urls = crawler.crawl_results
|
|
if not urls:
|
|
return jsonify({'success': False, 'error': 'No data to export'}), 404
|
|
|
|
# Remove special fields from regular export
|
|
regular_fields = [f for f in export_fields if f not in ['issues_detected', 'links_detailed']]
|
|
if not regular_fields:
|
|
return jsonify({'success': False, 'error': 'No fields selected'}), 400
|
|
|
|
if export_format == 'json':
|
|
return Response(
|
|
_stream_urls_json(urls, regular_fields),
|
|
mimetype='application/json',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_export_{timestamp}.json'}
|
|
)
|
|
elif export_format == 'xml':
|
|
return Response(
|
|
_stream_urls_xml(urls, regular_fields),
|
|
mimetype='application/xml',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_export_{timestamp}.xml'}
|
|
)
|
|
elif export_format == 'xlsx':
|
|
return Response(
|
|
_urls_xlsx(urls, regular_fields),
|
|
mimetype=XLSX_MIMETYPE,
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_export_{timestamp}.xlsx'}
|
|
)
|
|
else:
|
|
return Response(
|
|
_stream_urls_csv(urls, regular_fields),
|
|
mimetype='text/csv',
|
|
headers={'Content-Disposition': f'attachment; filename=librecrawl_export_{timestamp}.csv'}
|
|
)
|
|
|
|
except Exception as e:
|
|
import traceback
|
|
traceback.print_exc()
|
|
return jsonify({'success': False, 'error': str(e)}), 500
|
|
|
|
|
|
def _format_csv_value(value, field):
|
|
"""Format a value for CSV export (same logic as generate_csv_export)"""
|
|
if field == 'analytics' and isinstance(value, dict):
|
|
parts = []
|
|
if value.get('gtag') or value.get('ga4_id'): parts.append('GA4')
|
|
if value.get('google_analytics'): parts.append('GA')
|
|
if value.get('gtm_id'): parts.append('GTM')
|
|
if value.get('facebook_pixel'): parts.append('FB')
|
|
if value.get('hotjar'): parts.append('HJ')
|
|
if value.get('mixpanel'): parts.append('MP')
|
|
return ', '.join(parts)
|
|
elif field in ('og_tags', 'twitter_tags') and isinstance(value, dict):
|
|
return f"{len(value)} tags" if value else ''
|
|
elif field == 'json_ld' and isinstance(value, list):
|
|
return f"{len(value)} scripts" if value else ''
|
|
elif field == 'images' and isinstance(value, list):
|
|
return f"{len(value)} images" if value else ''
|
|
elif field == 'internal_links' and isinstance(value, (int, float)):
|
|
return f"{int(value)} internal links" if value else '0 internal links'
|
|
elif field == 'external_links' and isinstance(value, (int, float)):
|
|
return f"{int(value)} external links" if value else '0 external links'
|
|
elif field in ('h2', 'h3') and isinstance(value, list):
|
|
return ', '.join(value[:3]) + ('...' if len(value) > 3 else '')
|
|
elif isinstance(value, (dict, list)):
|
|
return str(value)
|
|
return value
|
|
|
|
|
|
XLSX_MIMETYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
_XLSX_CELL_LIMIT = 32767 # Excel's per-cell character limit
|
|
|
|
|
|
def _xlsx_cell(value):
|
|
"""Coerce an export value into something openpyxl will accept."""
|
|
if value is None or isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, (int, float)):
|
|
return value
|
|
from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
|
|
text = ILLEGAL_CHARACTERS_RE.sub('', str(value))
|
|
return text[:_XLSX_CELL_LIMIT]
|
|
|
|
|
|
def _build_xlsx(sheet_title, headers, rows):
|
|
"""Build an .xlsx workbook in memory: one sheet, a header row, then `rows`
|
|
(iterables of cell values). Write-only mode streams rows to a temp file,
|
|
so memory stays flat for large crawls."""
|
|
from openpyxl import Workbook
|
|
workbook = Workbook(write_only=True)
|
|
sheet = workbook.create_sheet(title=sheet_title)
|
|
sheet.append(list(headers))
|
|
for row in rows:
|
|
sheet.append([_xlsx_cell(v) for v in row])
|
|
buffer = BytesIO()
|
|
workbook.save(buffer)
|
|
return buffer.getvalue()
|
|
|
|
|
|
LINK_EXPORT_FIELDS = ['source_url', 'target_url', 'anchor_text', 'is_internal', 'target_domain', 'target_status', 'placement']
|
|
ISSUE_EXPORT_FIELDS = ['url', 'type', 'category', 'issue', 'details']
|
|
|
|
|
|
def _link_export_row(link):
|
|
return {
|
|
'source_url': link.get('source_url', ''),
|
|
'target_url': link.get('target_url', ''),
|
|
'anchor_text': link.get('anchor_text', ''),
|
|
'is_internal': 'Yes' if link.get('is_internal') else 'No',
|
|
'target_domain': link.get('target_domain', ''),
|
|
'target_status': link['target_status'] if link.get('target_status') is not None else 'Not crawled',
|
|
'placement': link.get('placement', 'body')
|
|
}
|
|
|
|
|
|
def _issue_export_row(issue):
|
|
return {
|
|
'url': issue.get('url', ''),
|
|
'type': issue.get('type', ''),
|
|
'category': issue.get('category', ''),
|
|
'issue': issue.get('issue', ''),
|
|
'details': str(issue.get('details', ''))
|
|
}
|
|
|
|
|
|
def _urls_xlsx(urls, fields):
|
|
return _build_xlsx('URLs', fields,
|
|
([_format_csv_value(u.get(f, ''), f) for f in fields] for u in urls))
|
|
|
|
|
|
def _links_xlsx(links):
|
|
return _build_xlsx('Links', LINK_EXPORT_FIELDS,
|
|
([_link_export_row(l)[f] for f in LINK_EXPORT_FIELDS] for l in links))
|
|
|
|
|
|
def _issues_xlsx(issues):
|
|
return _build_xlsx('Issues', ISSUE_EXPORT_FIELDS,
|
|
([_issue_export_row(i)[f] for f in ISSUE_EXPORT_FIELDS] for i in issues))
|
|
|
|
|
|
def _stream_urls_csv(urls, fields):
|
|
"""Generator that yields CSV rows one at a time"""
|
|
output = StringIO()
|
|
writer = csv.DictWriter(output, fieldnames=fields)
|
|
writer.writeheader()
|
|
yield output.getvalue()
|
|
output.truncate(0)
|
|
output.seek(0)
|
|
|
|
for url_data in urls:
|
|
row = {}
|
|
for field in fields:
|
|
value = url_data.get(field, '')
|
|
row[field] = _format_csv_value(value, field)
|
|
writer.writerow(row)
|
|
yield output.getvalue()
|
|
output.truncate(0)
|
|
output.seek(0)
|
|
|
|
|
|
def _stream_urls_json(urls, fields):
|
|
"""Generator that yields JSON array items one at a time"""
|
|
yield '{\n "export_date": "' + time.strftime('%Y-%m-%d %H:%M:%S') + '",\n'
|
|
yield ' "total_urls": ' + str(len(urls)) + ',\n'
|
|
yield ' "fields": ' + json.dumps(fields) + ',\n'
|
|
yield ' "data": [\n'
|
|
for i, url_data in enumerate(urls):
|
|
filtered = {f: url_data.get(f, '') for f in fields}
|
|
prefix = ',\n' if i > 0 else ''
|
|
yield prefix + ' ' + json.dumps(filtered, default=str)
|
|
yield '\n ]\n}\n'
|
|
|
|
|
|
def _stream_urls_xml(urls, fields):
|
|
"""Generator that yields XML elements one at a time"""
|
|
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
yield '<librecrawl_export export_date="' + time.strftime('%Y-%m-%d %H:%M:%S') + '" total_urls="' + str(len(urls)) + '">\n'
|
|
yield ' <urls>\n'
|
|
for url_data in urls:
|
|
yield ' <url>\n'
|
|
for field in fields:
|
|
escaped = str(url_data.get(field, '')).replace('&', '&').replace('<', '<').replace('>', '>')
|
|
yield ' <' + field + '>' + escaped + '</' + field + '>\n'
|
|
yield ' </url>\n'
|
|
yield ' </urls>\n'
|
|
yield '</librecrawl_export>\n'
|
|
|
|
|
|
def _stream_records_xml(root_tag, list_tag, item_tag, total_attr, total, records, fields):
|
|
"""Generator yielding an XML document of flat records, one element per field
|
|
(same shape as _stream_urls_xml)."""
|
|
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
yield f'<{root_tag} export_date="{time.strftime("%Y-%m-%d %H:%M:%S")}" {total_attr}="{total}">\n'
|
|
yield f' <{list_tag}>\n'
|
|
for record in records:
|
|
yield f' <{item_tag}>\n'
|
|
for field in fields:
|
|
escaped = str(record.get(field, '')).replace('&', '&').replace('<', '<').replace('>', '>')
|
|
yield f' <{field}>{escaped}</{field}>\n'
|
|
yield f' </{item_tag}>\n'
|
|
yield f' </{list_tag}>\n'
|
|
yield f'</{root_tag}>\n'
|
|
|
|
|
|
def _stream_links_xml(links):
|
|
return _stream_records_xml('librecrawl_links', 'links', 'link', 'total_links', len(links),
|
|
(_link_export_row(link) for link in links), LINK_EXPORT_FIELDS)
|
|
|
|
|
|
def _stream_issues_xml(issues):
|
|
return _stream_records_xml('librecrawl_issues', 'issues', 'issue', 'total_issues', len(issues),
|
|
(_issue_export_row(issue) for issue in issues), ISSUE_EXPORT_FIELDS)
|
|
|
|
|
|
def _stream_links_csv(links):
|
|
"""Generator that yields link CSV rows one at a time"""
|
|
output = StringIO()
|
|
writer = csv.DictWriter(output, fieldnames=LINK_EXPORT_FIELDS)
|
|
writer.writeheader()
|
|
yield output.getvalue()
|
|
output.truncate(0)
|
|
output.seek(0)
|
|
|
|
for link in links:
|
|
writer.writerow(_link_export_row(link))
|
|
yield output.getvalue()
|
|
output.truncate(0)
|
|
output.seek(0)
|
|
|
|
|
|
def _stream_links_json(links):
|
|
"""Generator that yields link JSON array items one at a time"""
|
|
yield '{\n "export_date": "' + time.strftime('%Y-%m-%d %H:%M:%S') + '",\n'
|
|
yield ' "total_links": ' + str(len(links)) + ',\n'
|
|
yield ' "data": [\n'
|
|
for i, link in enumerate(links):
|
|
entry = {
|
|
'source_url': link.get('source_url', ''),
|
|
'target_url': link.get('target_url', ''),
|
|
'anchor_text': link.get('anchor_text', ''),
|
|
'is_internal': link.get('is_internal', False),
|
|
'target_domain': link.get('target_domain', ''),
|
|
'target_status': link['target_status'] if link.get('target_status') is not None else 'Not crawled',
|
|
'placement': link.get('placement', 'body')
|
|
}
|
|
prefix = ',\n' if i > 0 else ''
|
|
yield prefix + ' ' + json.dumps(entry, default=str)
|
|
yield '\n ]\n}\n'
|
|
|
|
|
|
def _stream_issues_csv(issues):
|
|
"""Generator that yields issue CSV rows one at a time"""
|
|
output = StringIO()
|
|
writer = csv.DictWriter(output, fieldnames=ISSUE_EXPORT_FIELDS)
|
|
writer.writeheader()
|
|
yield output.getvalue()
|
|
output.truncate(0)
|
|
output.seek(0)
|
|
|
|
for issue in issues:
|
|
writer.writerow(_issue_export_row(issue))
|
|
yield output.getvalue()
|
|
output.truncate(0)
|
|
output.seek(0)
|
|
|
|
|
|
def _stream_issues_json(issues):
|
|
"""Generator that yields issue JSON array items one at a time"""
|
|
yield '{\n "export_date": "' + time.strftime('%Y-%m-%d %H:%M:%S') + '",\n'
|
|
yield ' "total_issues": ' + str(len(issues)) + ',\n'
|
|
yield ' "data": [\n'
|
|
for i, issue in enumerate(issues):
|
|
entry = {
|
|
'url': issue.get('url', ''),
|
|
'type': issue.get('type', ''),
|
|
'category': issue.get('category', ''),
|
|
'issue': issue.get('issue', ''),
|
|
'details': issue.get('details', '')
|
|
}
|
|
prefix = ',\n' if i > 0 else ''
|
|
yield prefix + ' ' + json.dumps(entry, default=str)
|
|
yield '\n ]\n}\n'
|
|
|
|
|
|
|
|
@app.route('/api/export_data', methods=['POST'])
|
|
@login_required
|
|
def export_data():
|
|
try:
|
|
data = request.get_json()
|
|
export_format = data.get('format', 'csv')
|
|
export_fields = data.get('fields', ['url', 'status_code', 'title'])
|
|
local_data = data.get('localData', {})
|
|
|
|
# Use local data if provided (from loaded crawl), otherwise get from crawler
|
|
if local_data and local_data.get('urls'):
|
|
urls = local_data.get('urls', [])
|
|
links = local_data.get('links', [])
|
|
issues = local_data.get('issues', [])
|
|
else:
|
|
# Get current crawl results
|
|
crawler = get_or_create_crawler()
|
|
crawl_data = crawler.get_status()
|
|
urls = crawl_data.get('urls', [])
|
|
links = crawl_data.get('links', [])
|
|
issues = crawl_data.get('issues', [])
|
|
|
|
if not urls:
|
|
return jsonify({'success': False, 'error': 'No data to export'})
|
|
|
|
# Update link statuses from crawled URLs (fixes missing status codes in exports)
|
|
if links and urls:
|
|
status_lookup = {url_data['url']: url_data.get('status_code') for url_data in urls}
|
|
for link in links:
|
|
target_url = link.get('target_url')
|
|
if target_url in status_lookup:
|
|
link['target_status'] = status_lookup[target_url]
|
|
|
|
# Apply current issue exclusion patterns (works for loaded crawls too)
|
|
if issues:
|
|
settings_manager = get_session_settings()
|
|
current_settings = settings_manager.get_settings()
|
|
exclusion_patterns_text = current_settings.get('issueExclusionPatterns', '')
|
|
exclusion_patterns = [p.strip() for p in exclusion_patterns_text.split('\n') if p.strip()]
|
|
issues = filter_issues_by_exclusion_patterns(issues, exclusion_patterns)
|
|
print(f"DEBUG: After exclusion filter, {len(issues)} issues remain")
|
|
|
|
# Collect files to export based on special field selections
|
|
files_to_export = []
|
|
|
|
# Check for special export fields and prepare them as separate files
|
|
has_issues_export = 'issues_detected' in export_fields
|
|
has_links_export = 'links_detailed' in export_fields
|
|
|
|
# Remove special fields from regular export fields
|
|
regular_fields = [f for f in export_fields if f not in ['issues_detected', 'links_detailed']]
|
|
|
|
# Debug logging
|
|
print(f"DEBUG: export_fields = {export_fields}")
|
|
print(f"DEBUG: has_issues_export = {has_issues_export}")
|
|
print(f"DEBUG: has_links_export = {has_links_export}")
|
|
print(f"DEBUG: regular_fields = {regular_fields}")
|
|
print(f"DEBUG: len(urls) = {len(urls)}")
|
|
print(f"DEBUG: len(links) = {len(links)}")
|
|
print(f"DEBUG: len(issues) = {len(issues)}")
|
|
|
|
# Generate issues export if requested
|
|
if has_issues_export:
|
|
if export_format == 'csv':
|
|
issues_content = generate_issues_csv_export(issues)
|
|
issues_mimetype = 'text/csv'
|
|
issues_filename = f'librecrawl_issues_{int(time.time())}.csv'
|
|
elif export_format == 'json':
|
|
issues_content = generate_issues_json_export(issues)
|
|
issues_mimetype = 'application/json'
|
|
issues_filename = f'librecrawl_issues_{int(time.time())}.json'
|
|
elif export_format == 'xml':
|
|
issues_content = ''.join(_stream_issues_xml(issues))
|
|
issues_mimetype = 'application/xml'
|
|
issues_filename = f'librecrawl_issues_{int(time.time())}.xml'
|
|
elif export_format == 'xlsx':
|
|
issues_content = base64.b64encode(_issues_xlsx(issues)).decode('ascii')
|
|
issues_mimetype = XLSX_MIMETYPE
|
|
issues_filename = f'librecrawl_issues_{int(time.time())}.xlsx'
|
|
else:
|
|
issues_content = generate_issues_csv_export(issues)
|
|
issues_mimetype = 'text/csv'
|
|
issues_filename = f'librecrawl_issues_{int(time.time())}.csv'
|
|
|
|
files_to_export.append({
|
|
'content': issues_content,
|
|
'mimetype': issues_mimetype,
|
|
'filename': issues_filename,
|
|
**({'encoding': 'base64'} if export_format == 'xlsx' else {})
|
|
})
|
|
|
|
# Generate links export if requested
|
|
if has_links_export:
|
|
if export_format == 'csv':
|
|
links_content = generate_links_csv_export(links)
|
|
links_mimetype = 'text/csv'
|
|
links_filename = f'librecrawl_links_{int(time.time())}.csv'
|
|
elif export_format == 'json':
|
|
links_content = generate_links_json_export(links)
|
|
links_mimetype = 'application/json'
|
|
links_filename = f'librecrawl_links_{int(time.time())}.json'
|
|
elif export_format == 'xml':
|
|
links_content = ''.join(_stream_links_xml(links))
|
|
links_mimetype = 'application/xml'
|
|
links_filename = f'librecrawl_links_{int(time.time())}.xml'
|
|
elif export_format == 'xlsx':
|
|
links_content = base64.b64encode(_links_xlsx(links)).decode('ascii')
|
|
links_mimetype = XLSX_MIMETYPE
|
|
links_filename = f'librecrawl_links_{int(time.time())}.xlsx'
|
|
else:
|
|
links_content = generate_links_csv_export(links)
|
|
links_mimetype = 'text/csv'
|
|
links_filename = f'librecrawl_links_{int(time.time())}.csv'
|
|
|
|
files_to_export.append({
|
|
'content': links_content,
|
|
'mimetype': links_mimetype,
|
|
'filename': links_filename,
|
|
**({'encoding': 'base64'} if export_format == 'xlsx' else {})
|
|
})
|
|
|
|
# Generate regular export if there are regular fields
|
|
if regular_fields:
|
|
if export_format == 'csv':
|
|
regular_content = generate_csv_export(urls, regular_fields)
|
|
regular_mimetype = 'text/csv'
|
|
regular_filename = f'librecrawl_export_{int(time.time())}.csv'
|
|
elif export_format == 'json':
|
|
regular_content = generate_json_export(urls, regular_fields)
|
|
regular_mimetype = 'application/json'
|
|
regular_filename = f'librecrawl_export_{int(time.time())}.json'
|
|
elif export_format == 'xml':
|
|
regular_content = generate_xml_export(urls, regular_fields)
|
|
regular_mimetype = 'application/xml'
|
|
regular_filename = f'librecrawl_export_{int(time.time())}.xml'
|
|
elif export_format == 'xlsx':
|
|
regular_content = base64.b64encode(_urls_xlsx(urls, regular_fields)).decode('ascii')
|
|
regular_mimetype = XLSX_MIMETYPE
|
|
regular_filename = f'librecrawl_export_{int(time.time())}.xlsx'
|
|
else:
|
|
return jsonify({'success': False, 'error': 'Unsupported export format'})
|
|
|
|
files_to_export.append({
|
|
'content': regular_content,
|
|
'mimetype': regular_mimetype,
|
|
'filename': regular_filename,
|
|
**({'encoding': 'base64'} if export_format == 'xlsx' else {})
|
|
})
|
|
|
|
# Handle special case where only special fields are selected but no data
|
|
if not files_to_export:
|
|
if has_issues_export and not issues:
|
|
return jsonify({'success': False, 'error': 'No issues data to export'})
|
|
elif has_links_export and not links:
|
|
return jsonify({'success': False, 'error': 'No links data to export'})
|
|
else:
|
|
return jsonify({'success': False, 'error': 'No data to export'})
|
|
|
|
# Return multiple files if we have more than one, otherwise single file
|
|
if len(files_to_export) > 1:
|
|
return jsonify({
|
|
'success': True,
|
|
'multiple_files': True,
|
|
'files': files_to_export
|
|
})
|
|
else:
|
|
# Single file
|
|
file_data = files_to_export[0]
|
|
return jsonify({
|
|
'success': True,
|
|
'content': file_data['content'],
|
|
'mimetype': file_data['mimetype'],
|
|
'filename': file_data['filename']
|
|
})
|
|
|
|
except Exception as e:
|
|
return jsonify({'success': False, 'error': str(e)})
|
|
|
|
def recover_crashed_crawls():
|
|
"""Check for and recover any crashed crawls on startup"""
|
|
try:
|
|
from src.crawl_db import get_crashed_crawls, set_crawl_status
|
|
|
|
crashed = get_crashed_crawls()
|
|
|
|
if crashed:
|
|
print("\n" + "=" * 60)
|
|
print("CRASH RECOVERY")
|
|
print("=" * 60)
|
|
for crawl in crashed:
|
|
set_crawl_status(crawl['id'], 'failed')
|
|
print(f"Found crashed crawl: {crawl['base_url']} (ID: {crawl['id']})")
|
|
print(f" → Marked as failed. User can resume from dashboard.")
|
|
print("=" * 60 + "\n")
|
|
except Exception as e:
|
|
print(f"Error during crash recovery: {e}")
|
|
|
|
def graceful_shutdown(signum, frame):
|
|
"""Save all active crawls before shutdown"""
|
|
print("\n" + "=" * 60)
|
|
print("GRACEFUL SHUTDOWN")
|
|
print("=" * 60)
|
|
print("Saving all active crawls...")
|
|
|
|
try:
|
|
with instances_lock:
|
|
for session_id, instance_data in list(crawler_instances.items()):
|
|
crawler = instance_data['crawler']
|
|
if crawler.is_running and crawler.crawl_id and crawler.db_save_enabled:
|
|
print(f" → Saving crawl {crawler.crawl_id}...")
|
|
try:
|
|
crawler._save_batch_to_db(force=True)
|
|
crawler._save_queue_checkpoint()
|
|
from src.crawl_db import set_crawl_status
|
|
set_crawl_status(crawler.crawl_id, 'paused')
|
|
except Exception as e:
|
|
print(f" Error saving crawl {crawler.crawl_id}: {e}")
|
|
|
|
print("All crawls saved successfully")
|
|
print("=" * 60)
|
|
except Exception as e:
|
|
print(f"Error during shutdown: {e}")
|
|
|
|
print("Goodbye!")
|
|
import sys
|
|
sys.exit(0)
|
|
|
|
def main():
|
|
import signal
|
|
|
|
# Register signal handlers for graceful shutdown
|
|
signal.signal(signal.SIGINT, graceful_shutdown)
|
|
signal.signal(signal.SIGTERM, graceful_shutdown)
|
|
|
|
# Recover any crashed crawls from previous session
|
|
recover_crashed_crawls()
|
|
|
|
# Start cleanup thread for old crawler instances
|
|
start_cleanup_thread()
|
|
|
|
print("=" * 60)
|
|
print("LibreCrawl - SEO Spider")
|
|
print("=" * 60)
|
|
print(f"\n🚀 Server starting on http://0.0.0.0:5000")
|
|
print(f"🌐 Access from browser: http://localhost:5000")
|
|
print(f"📱 Access from network: http://<your-ip>:5000")
|
|
print(f"\n✨ Multi-tenancy enabled - each browser session is isolated")
|
|
print(f"💾 Settings stored in browser localStorage")
|
|
print(f"\nPress Ctrl+C to stop the server\n")
|
|
print("=" * 60 + "\n")
|
|
|
|
# Open browser in a separate thread after short delay
|
|
def open_browser():
|
|
time.sleep(1.5) # Wait for Flask to start
|
|
webbrowser.open('http://localhost:5000')
|
|
|
|
browser_thread = threading.Thread(target=open_browser, daemon=True)
|
|
browser_thread.start()
|
|
|
|
# Run Flask server with Waitress (production-grade WSGI server)
|
|
from waitress import serve
|
|
print("Starting LibreCrawl on http://localhost:5000")
|
|
print("Using Waitress WSGI server with multi-threading support")
|
|
serve(app, host='0.0.0.0', port=5000, threads=8)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|