commit 473351086aa4c41a5f0be802b91b404359600e7f Author: Steffen Date: Sat Nov 7 16:58:29 2026 +0100 Initial commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..905ec34 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +__pycache__/ +*.pyc +*.pyo +*.pyd +.venv/ +venv/ +.git/ +.gitignore +.vscode/ +.idea/ +*.log diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..50a9014 --- /dev/null +++ b/.env.example @@ -0,0 +1,26 @@ +# Crawler Configuration +CRAWLER_START_URL=https://example.com +CRAWLER_USE_TOR=false +CRAWLER_TOR_CIRCUIT_MODE=iteration +CRAWLER_TOR_CIRCUIT_PAGES=10 +CRAWLER_TOR_MAX_CONSECUTIVE_TIMEOUTS=5 +CRAWLER_TOR_EXIT_NODES=de,nl,fr,ch +CRAWLER_NUM_TABS=5 +CRAWLER_NUM_ITERATIONS=1 +CRAWLER_ITERATION_MODE=all +CRAWLER_MAX_VISITS_PER_ITERATION=10 +CRAWLER_RANDOMIZE_QUEUE=true +CRAWLER_HEADLESS=true +CRAWLER_ACCEPT_COOKIES=true +CRAWLER_SLOW_MO=100 +CRAWLER_TIMEOUT_MS=35000 +CRAWLER_USER_AGENT=random + +# Web Dashboard Port +WEB_PORT=8080 + +# Web Dashboard Authentication (Recommended for Internet-Facing Deployments) +AUTH_ENABLED=true +AUTH_USERNAME=admin +AUTH_PASSWORD=change_this_to_a_secure_password +AUTH_SECRET_KEY=change_this_to_a_random_32_character_secret_key diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..944731b --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Virtual Environments +venv/ +.venv/ +env/ +.env_win/ + +# Python Cache +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Environment Variables & Local Secrets +.env +!.env.example + +# Playwright +test-results/ +playwright-report/ + +# OS / Editor Files +.DS_Store +Thumbs.db +.vscode/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8b72417 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Use official Microsoft Playwright image with Python & browser dependencies +FROM mcr.microsoft.com/playwright/python:v1.49.0-noble + +# Prevent interactive prompts during apt package installation +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 + +# Install Tor daemon and curl +RUN apt-get update && apt-get install -y --no-install-recommends \ + tor \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Configure Tor (SOCKS5 proxy on 9050, ControlPort on 9051 with open localhost authentication) +RUN printf "SocksPort 127.0.0.1:9050\nControlPort 127.0.0.1:9051\nCookieAuthentication 0\nHashedControlPassword \"\"\n" > /etc/tor/torrc + +WORKDIR /app + +# Install Python requirements +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Ensure Chromium browser is ready +RUN playwright install chromium + +# Copy application source code +COPY . . + +# Prepare entrypoint script (strip potential Windows CRLF endings and mark executable) +COPY entrypoint.sh /entrypoint.sh +RUN sed -i 's/\r$//' /entrypoint.sh && chmod +x /entrypoint.sh + +# Expose Web Dashboard port +EXPOSE 8000 + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["python", "web.py", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..d03fcb5 --- /dev/null +++ b/README.md @@ -0,0 +1,179 @@ +# WebSurfer: Parallel Playwright Crawler + +A highly-concurrent, fully automated website crawler built with Python and [Playwright](https://playwright.dev/python/). + +This crawler rapidly crawls an entire domain using multiple visible or headless browser tabs. It uses a Breadth-First Search (BFS) approach powered by an `asyncio.Queue` to spread the crawling workload across a customizable pool of worker tabs. + +## Key Features + +- **Multi-Tab Concurrency**: Spawns multiple asynchronous browser tabs (default: 5) to dramatically speed up crawl times. +- **Automatic Cookie Consent Acceptance**: Automatically detects and accepts cookie consent banners / CMPs (Complianz, Borlabs Cookie, Cookiebot, OneTrust, Usercentrics, Klaro, CookieYes, Real Cookie Banner, CCM19, and generic accept patterns) across pages and iframes so analytics, marketing tags, and tracking scripts fire properly. +- **Intelligent Scope**: + - Recursively crawls all internal links belonging to the starting domain. + - Visits external links (depth 1) to verify they are alive, but intentionally avoids following their sub-links to prevent crawling the entire internet. +- **Crawler Trap Protection**: Automatically strips query parameters (e.g. `?page=2`) and URL fragments (e.g. `#section`) from internal links to prevent infinite loops from endless pagination or dynamically generated URLs. External links retain their original query parameters to prevent breaking things like YouTube videos. +- **Configurable Iteration Scope & Visit Limits**: + - **All Websites (Default)**: Crawls all reachable internal and external level-1 links until the queue is fully exhausted. + - **Limit Visits Mode**: Caps each iteration to a configurable number of website visits before starting a new iteration. Ideal for multi-iteration testing across fresh Tor circuits and rotating User-Agents on large websites without having to crawl every single page. +- **Randomized Crawl Queue**: Uniformly picks pages at random from the queue of discovered URLs instead of strict FIFO breadth-first order. Ensures broad, diverse coverage across various sections and depths of the website on every iteration. +- **Error & Broken Link Tracking**: Tracks any pages that result in HTTP errors (404, 500, etc.) or connection timeouts. At the end of the crawl, it outputs a clean list of broken URLs alongside the exact source page where the broken link was discovered. +- **Tor Proxy, Flexible Circuit Rotation & European Exit Nodes**: Optionally routes all crawler traffic securely through a headless Tor proxy (SOCKS5 on port `9050`). + - **Circuit Rotation Modes**: Rotate per crawl iteration or dynamically after every *N* crawled pages. + - **European Exit Node Selector**: Select 0 to *N* European countries (Germany, Netherlands, France, Switzerland, Sweden, UK, etc.) to strictly/preferentially route traffic through specific European exit relays. If no country is selected, global Tor exit relays are used worldwide. +- **Live Tor Network & IP Geolocation Inspector**: Shows active Tor exit IP, country, city, ISP/organization, exit node filter, and circuit rotation count in real-time. + +## Web Dashboard & Frontend + +WebSurfer includes a modern, real-time Web Dashboard built with FastAPI and WebSockets. + +### Features in the Web Dashboard: +- **Interactive Configuration**: Configure start URL, concurrency, iterations, Tor routing, headless mode, cookie handling, and timeouts directly in the browser. +- **`.env` Synchronization**: One-click saving to `.env` or loading stored parameters. +- **Live Metrics & All-Iterations Cumulative Statistics**: Real-time KPI tiles for Visited Pages, Internal vs. External links, Cookies Accepted, Queue Backlog, and Timer, paired with a dedicated all-iterations summary bar and subtitle badges tracking total websites visited and total errors encountered across all iterations. +- **Dedicated Error & Broken Link Section**: Structured, searchable, and exportable (CSV / JSON) error registry displaying the failed URL, error reason / HTTP status, and referring source page. +- **Live Streaming Terminal**: Real-time color-coded crawl logs with auto-scrolling, search query filtering, and level filtering. + +### Running the Web Dashboard: +```bash +python web.py +``` +*(or `python main.py --web`)* + +Open your browser at [http://127.0.0.1:8000](http://127.0.0.1:8000). + +## Project Structure + +The project is organized into clean, single-responsibility modules: + +``` +websurfer/ +├── crawler/ +│ ├── __init__.py # Public exports (Crawler, CrawlerConfig, TorManager, etc.) +│ ├── config.py # Configuration dataclass & interactive/env/dict loaders +│ ├── tor.py # Tor daemon lifecycle & circuit rotation (Stem/SOCKS5) +│ ├── cookies.py # CMP selectors, text regexes & auto-consent execution +│ ├── url_utils.py # URL normalization, domain matching & static file filters +│ └── engine.py # Playwright browser lifecycle, worker queue, events & crawl loop +├── web/ +│ ├── __init__.py # Web package entry point & server runner +│ ├── app.py # FastAPI REST API & WebSocket endpoints +│ ├── manager.py # Async crawl session manager & event broadcaster +│ └── static/ +│ ├── index.html # Modern responsive dashboard UI +│ ├── style.css # Dark glassmorphism styling +│ └── app.js # Real-time WebSocket controller & data renderer +├── main.py # Primary entry point (supports CLI and --web) +├── web.py # Standalone Web Dashboard launcher +├── browser_crawler.py # Backwards-compatible launcher +├── .env.example # Example configuration settings +└── requirements.txt # Project dependencies +``` + +## Prerequisites + +You need Python 3.8+ installed. If you intend to use the Tor proxy feature, ensure the `tor` command-line package is installed on your system: +- **macOS**: `brew install tor` +- **Linux (Ubuntu/Debian)**: `sudo apt install tor` +- **Windows**: Install Tor Expert Bundle from [torproject.org](https://www.torproject.org/) and add it to your PATH. +*(No Tor Browser GUI needed!)* + +## Installation + +1. Navigate to the project directory: + ```bash + cd websurfer + ``` +2. Create and activate an OS-specific virtual environment: + - **macOS / Linux**: + ```bash + python3 -m venv venv + source venv/bin/activate + ``` + - **Windows (PowerShell)**: + ```powershell + python -m venv .venv + .\.venv\Scripts\Activate.ps1 + ``` + - **Windows (Command Prompt)**: + ```cmd + python -m venv .venv + .\.venv\Scripts\activate.bat + ``` +3. Install the dependencies: + ```bash + pip install -r requirements.txt + ``` +4. Install the required Playwright Chromium binaries: + ```bash + playwright install chromium + ``` + +## Usage + +### 1. Web Dashboard (Recommended) +```bash +python web.py +``` +Or: +```bash +python main.py --web +``` + +### 2. Running with Docker & Tor (Containerized) +WebSurfer can be run in a Docker container with Playwright headless Chromium and an integrated background Tor proxy: + +```bash +# Build and start container (Web Dashboard on port 8080) +docker compose up --build + +# Run in background +docker compose up -d + +# View logs +docker compose logs -f +``` + +Open your browser at [http://127.0.0.1:8080](http://127.0.0.1:8080) (or customize the port with `WEB_PORT=8000` in `.env`). + +To run a headless crawl via CLI inside the container: +```bash +docker compose run --rm websurfer python main.py +``` + +### 3. Running via CLI +Run the main entry point in interactive/CLI mode: +```bash +python main.py +``` +*(or `python browser_crawler.py`)* + +The script will interactively prompt you (or read from `.env`): +1. **Starting URL**: The base website you want to crawl (e.g., `https://example.com`). +2. **Tor Proxy**: Whether you want to route traffic through Tor (y/n). +3. **Concurrent Tabs**: The number of browser tabs you want running simultaneously (defaults to 5). +4. **Iterations**: Number of times to run the crawl. +5. **Headless Mode**: Run browser with or without UI. +6. **Accept Cookies**: Automatically accept cookies on pages (defaults to yes). + +### 4. Programmatic Usage + +You can also use WebSurfer in your own Python scripts: + +```python +import asyncio +from crawler import Crawler, CrawlerConfig + +async def run_my_crawl(): + config = CrawlerConfig( + start_url="https://example.com", + num_tabs=4, + headless=True, + accept_cookies=True, + ) + crawler = Crawler(config) + await crawler.run() + +if __name__ == "__main__": + asyncio.run(run_my_crawl()) +``` + diff --git a/browser_crawler.py b/browser_crawler.py new file mode 100644 index 0000000..cbea904 --- /dev/null +++ b/browser_crawler.py @@ -0,0 +1,54 @@ +""" +WebSurfer Entry Point (browser_crawler.py) + +Maintained for backwards compatibility with existing CLI commands and scripts. +The crawler has been modularized under the `crawler/` package. +""" + +import asyncio +import sys + +from main import main +from crawler import ( + Crawler, + CrawlerConfig, + TorManager, + start_tor_service, + renew_tor_circuit, + stop_tor_service, + auto_accept_cookies, + normalize_url, + COOKIE_BANNER_SELECTORS, + COOKIE_TEXT_PATTERNS, + IGNORED_EXTENSIONS, + IGNORED_PATHS, + DEFAULT_TOR_SOCKS_PORT as TOR_SOCKS_PORT, + DEFAULT_TOR_CONTROL_PORT as TOR_CONTROL_PORT, +) + +__all__ = [ + "main", + "Crawler", + "CrawlerConfig", + "TorManager", + "start_tor_service", + "renew_tor_circuit", + "stop_tor_service", + "auto_accept_cookies", + "normalize_url", + "COOKIE_BANNER_SELECTORS", + "COOKIE_TEXT_PATTERNS", + "IGNORED_EXTENSIONS", + "IGNORED_PATHS", + "TOR_SOCKS_PORT", + "TOR_CONTROL_PORT", +] + +if __name__ == "__main__": + if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + try: + main() + except KeyboardInterrupt: + print("\nCrawling interrupted by user.") + sys.exit(0) diff --git a/crawler/__init__.py b/crawler/__init__.py new file mode 100644 index 0000000..0c65fc3 --- /dev/null +++ b/crawler/__init__.py @@ -0,0 +1,44 @@ +""" +WebSurfer Crawler Package +""" + +from .config import CrawlerConfig +from .cookies import ( + COOKIE_BANNER_SELECTORS, + COOKIE_TEXT_PATTERNS, + auto_accept_cookies, +) +from .engine import Crawler, crawl_level, worker, RandomQueue +from .tor import ( + DEFAULT_TOR_CONTROL_PORT, + DEFAULT_TOR_SOCKS_PORT, + TorManager, + renew_tor_circuit, + start_tor_service, + stop_tor_service, +) +from .url_utils import ( + IGNORED_EXTENSIONS, + IGNORED_PATHS, + normalize_url, +) + +__all__ = [ + "Crawler", + "CrawlerConfig", + "RandomQueue", + "TorManager", + "start_tor_service", + "renew_tor_circuit", + "stop_tor_service", + "auto_accept_cookies", + "COOKIE_BANNER_SELECTORS", + "COOKIE_TEXT_PATTERNS", + "normalize_url", + "IGNORED_EXTENSIONS", + "IGNORED_PATHS", + "crawl_level", + "worker", + "DEFAULT_TOR_SOCKS_PORT", + "DEFAULT_TOR_CONTROL_PORT", +] diff --git a/crawler/config.py b/crawler/config.py new file mode 100644 index 0000000..23defcc --- /dev/null +++ b/crawler/config.py @@ -0,0 +1,432 @@ +import os +import random +import re +from dataclasses import dataclass, field +from typing import Optional, List, Dict, Any +from urllib.parse import urlparse +from dotenv import load_dotenv + +from .url_utils import normalize_url + + +# Modern realistic user agents for rotation to evade bot-detection and WAF blocks +DEFAULT_USER_AGENTS: List[str] = [ + # Chrome on Windows (v133) + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", + # Chrome on macOS (v133) + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", + # Firefox on Windows (v134) + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0", + # Firefox on macOS (v134) + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:134.0) Gecko/20100101 Firefox/134.0", + # Edge on Windows (v133) + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0", + # Safari on macOS (v18.2) + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15", +] + + +# Supported European countries with active Tor exit relays +EUROPEAN_COUNTRIES: Dict[str, Dict[str, str]] = { + "de": {"name": "Germany", "flag": "🇩🇪"}, + "nl": {"name": "Netherlands", "flag": "🇳🇱"}, + "fr": {"name": "France", "flag": "🇫🇷"}, + "ch": {"name": "Switzerland", "flag": "🇨🇭"}, + "at": {"name": "Austria", "flag": "🇦🇹"}, + "se": {"name": "Sweden", "flag": "🇸🇪"}, + "no": {"name": "Norway", "flag": "🇳🇴"}, + "fi": {"name": "Finland", "flag": "🇫🇮"}, + "dk": {"name": "Denmark", "flag": "🇩🇰"}, + "gb": {"name": "United Kingdom", "flag": "🇬🇧"}, + "pl": {"name": "Poland", "flag": "🇵🇱"}, + "it": {"name": "Italy", "flag": "🇮🇹"}, + "es": {"name": "Spain", "flag": "🇪🇸"}, + "be": {"name": "Belgium", "flag": "🇧🇪"}, + "cz": {"name": "Czech Republic", "flag": "🇨🇿"}, + "ro": {"name": "Romania", "flag": "🇷🇴"}, + "ie": {"name": "Ireland", "flag": "🇮🇪"}, + "pt": {"name": "Portugal", "flag": "🇵🇹"}, + "hu": {"name": "Hungary", "flag": "🇭🇺"}, + "is": {"name": "Iceland", "flag": "🇮🇸"}, + "lu": {"name": "Luxembourg", "flag": "🇱🇺"}, + "gr": {"name": "Greece", "flag": "🇬🇷"}, + "bg": {"name": "Bulgaria", "flag": "🇧🇬"}, + "ee": {"name": "Estonia", "flag": "🇪🇪"}, + "lv": {"name": "Latvia", "flag": "🇱🇻"}, + "lt": {"name": "Lithuania", "flag": "🇱🇹"}, + "sk": {"name": "Slovakia", "flag": "🇸🇰"}, + "si": {"name": "Slovenia", "flag": "🇸🇮"}, + "hr": {"name": "Croatia", "flag": "🇭🇷"}, +} + + +@dataclass +class CrawlerConfig: + """Configuration settings for the crawler.""" + start_url: str + use_tor: bool = False + tor_circuit_mode: str = "iteration" # "iteration" (new circuit per run) or "pages" (new circuit per N pages) + tor_circuit_pages: int = 10 # Number of pages to crawl before renewing circuit if mode is "pages" + tor_max_consecutive_timeouts: int = 5 # Number of consecutive ERR_TIMED_OUT failures before fallback circuit renewal + tor_exit_nodes: List[str] = field(default_factory=list) # ISO country codes (empty = any global exit node) + num_tabs: int = 5 + num_iterations: int = 1 + iteration_mode: str = "all" # "all" (crawl all discovered websites) or "visits" (cap visits per iteration) + max_visits_per_iteration: int = 10 # Max website visits before starting a new iteration if mode is "visits" + randomize_queue: bool = True # Randomly pop URLs from queue instead of strict FIFO order + headless: bool = False + accept_cookies: bool = True + slow_mo: int = 100 + timeout_ms: int = 35000 + tor_socks_port: int = 9050 + tor_control_port: int = 9051 + user_agent: str = "random" # "random" (auto-rotate modern desktop UAs), "default" (native browser), or custom string + + @property + def visits_per_iteration(self) -> int: + """Alias for max_visits_per_iteration.""" + return self.max_visits_per_iteration + + @property + def start_domain(self) -> str: + """Extracts the hostname/netloc from the start URL.""" + return urlparse(self.start_url).netloc + + @property + def tor_exit_nodes_formatted(self) -> Optional[str]: + """Formats exit node country codes into Tor syntax (e.g. '{de},{fr}'), or None if empty.""" + if not self.tor_exit_nodes: + return None + valid_codes = [c.strip().lower() for c in self.tor_exit_nodes if c.strip()] + if not valid_codes: + return None + return ",".join(f"{{{c}}}" for c in valid_codes) + + def get_user_agent(self) -> Optional[str]: + """ + Resolves the user agent string based on the configuration setting: + - "random" or empty: randomly selects a modern real-world desktop User-Agent. + - "default": returns None to instruct Playwright to use its native browser User-Agent. + - Custom string: returns the specified string. + """ + if not self.user_agent or self.user_agent.strip().lower() in ["random", "rotate"]: + return random.choice(DEFAULT_USER_AGENTS) + if self.user_agent.strip().lower() == "default": + return None + return self.user_agent.strip() + + @classmethod + def from_env_or_prompt(cls) -> "CrawlerConfig": + """ + Loads configuration values from environment variables / .env file, + falling back to interactive CLI prompts for any unset options. + """ + load_dotenv() + + # 1. Starting URL + start_url = os.getenv("CRAWLER_START_URL") + if not start_url: + start_url = input("Enter the starting URL (e.g., https://example.com): ").strip() + if not start_url.startswith("http://") and not start_url.startswith("https://"): + start_url = "https://" + start_url + + start_domain = urlparse(start_url).netloc + norm_start, _ = normalize_url(start_url, start_domain) + if norm_start: + start_url = norm_start + + # 2. Tor Proxy + use_tor_env = os.getenv("CRAWLER_USE_TOR") + if use_tor_env is not None: + use_tor = use_tor_env.strip().lower() in ["true", "1", "y", "yes"] + else: + use_tor_input = input( + "Use Tor proxy (starts/connects to headless Tor on 127.0.0.1:9050)? (y/n): " + ).strip().lower() + use_tor = (use_tor_input == "y") + + # 2.1 Tor Circuit Rotation Strategy & Exit Nodes (if Tor is enabled) + tor_circuit_mode = "iteration" + tor_circuit_pages = 10 + tor_max_consecutive_timeouts = 5 + tor_exit_nodes = [] + if use_tor: + mode_env = os.getenv("CRAWLER_TOR_CIRCUIT_MODE") + pages_env = os.getenv("CRAWLER_TOR_CIRCUIT_PAGES") + timeouts_env = os.getenv("CRAWLER_TOR_MAX_CONSECUTIVE_TIMEOUTS") + nodes_env = os.getenv("CRAWLER_TOR_EXIT_NODES") + + if mode_env is not None: + tor_circuit_mode = "pages" if mode_env.strip().lower() in ["pages", "page", "per_pages"] else "iteration" + else: + print("\nTor Circuit Renewal Options:") + print(" [1] Renew circuit before every iteration (default)") + print(" [2] Renew circuit after every N crawled pages") + tor_mode_choice = input("Select Tor renewal mode (1/2, default 1): ").strip() + tor_circuit_mode = "pages" if tor_mode_choice == "2" else "iteration" + + if tor_circuit_mode == "pages": + if pages_env and pages_env.isdigit(): + tor_circuit_pages = max(1, int(pages_env)) + else: + pages_input = input("How many crawled pages before renewing Tor circuit? (default 10): ").strip() + tor_circuit_pages = int(pages_input) if pages_input and pages_input.isdigit() else 10 + + if timeouts_env and timeouts_env.isdigit(): + tor_max_consecutive_timeouts = max(1, int(timeouts_env)) + + if nodes_env is not None: + tor_exit_nodes = [c.strip().lower() for c in nodes_env.split(",") if c.strip()] + else: + nodes_input = input( + "Select European Tor Exit Node countries (comma-separated, e.g. de,nl,fr,ch) [Enter for any]: " + ).strip() + if nodes_input: + tor_exit_nodes = [c.strip().lower() for c in nodes_input.split(",") if c.strip()] + + # 3. Concurrent Tabs + num_tabs_str = os.getenv("CRAWLER_NUM_TABS") + if not num_tabs_str: + num_tabs_str = input("How many concurrent tabs would you like to use? (default 5): ").strip() + num_tabs = int(num_tabs_str) if num_tabs_str and num_tabs_str.isdigit() else 5 + + # 4. Iterations + num_iterations_str = os.getenv("CRAWLER_NUM_ITERATIONS") + if not num_iterations_str: + num_iterations_str = input("How many times would you like to run the crawler? (default 1): ").strip() + num_iterations = int(num_iterations_str) if num_iterations_str and num_iterations_str.isdigit() else 1 + + # 4.1 Iteration Scope & Website Visit Cap + iter_mode_env = os.getenv("CRAWLER_ITERATION_MODE") + max_visits_env = os.getenv("CRAWLER_MAX_VISITS_PER_ITERATION") or os.getenv("CRAWLER_VISITS_PER_ITERATION") + + if iter_mode_env is not None: + iteration_mode = "visits" if iter_mode_env.strip().lower() in ["visits", "limit", "pages", "capped"] else "all" + else: + print("\nIteration Scope Options:") + print(" [1] Crawl all reachable websites per iteration (default)") + print(" [2] Limit website visits per iteration before starting a new iteration") + iter_choice = input("Select iteration scope mode (1/2, default 1): ").strip() + iteration_mode = "visits" if iter_choice == "2" else "all" + + max_visits_per_iteration = 10 + if iteration_mode == "visits": + if max_visits_env and max_visits_env.isdigit(): + max_visits_per_iteration = max(1, int(max_visits_env)) + else: + visits_input = input("How many website visits before starting a new iteration? (default 10): ").strip() + max_visits_per_iteration = int(visits_input) if visits_input and visits_input.isdigit() else 10 + + # 5. Headless Mode + use_headless_env = os.getenv("CRAWLER_HEADLESS") + if use_headless_env is not None: + headless = use_headless_env.strip().lower() in ["true", "1", "y", "yes"] + else: + headless = input("Run browser in headless mode? (y/n, default n): ").strip().lower() == "y" + + # 6. Accept Cookies + accept_cookies_env = os.getenv("CRAWLER_ACCEPT_COOKIES") + if accept_cookies_env is not None: + accept_cookies = accept_cookies_env.strip().lower() in ["true", "1", "y", "yes"] + else: + accept_cookies_input = input("Automatically accept cookies on pages? (y/n, default y): ").strip().lower() + accept_cookies = (accept_cookies_input != "n") + + # 7. User Agent + user_agent = os.getenv("CRAWLER_USER_AGENT", "random").strip() + + # 8. Randomize Queue + randomize_queue_env = os.getenv("CRAWLER_RANDOMIZE_QUEUE") + if randomize_queue_env is not None: + randomize_queue = randomize_queue_env.strip().lower() in ["true", "1", "y", "yes"] + else: + randomize_queue = True + + return cls( + start_url=start_url, + use_tor=use_tor, + tor_circuit_mode=tor_circuit_mode, + tor_circuit_pages=tor_circuit_pages, + tor_max_consecutive_timeouts=tor_max_consecutive_timeouts, + tor_exit_nodes=tor_exit_nodes, + num_tabs=num_tabs, + num_iterations=num_iterations, + iteration_mode=iteration_mode, + max_visits_per_iteration=max_visits_per_iteration, + randomize_queue=randomize_queue, + headless=headless, + accept_cookies=accept_cookies, + user_agent=user_agent, + ) + + def to_dict(self) -> dict: + """Serializes the configuration to a dictionary.""" + return { + "start_url": self.start_url, + "use_tor": self.use_tor, + "tor_circuit_mode": self.tor_circuit_mode, + "tor_circuit_pages": self.tor_circuit_pages, + "tor_max_consecutive_timeouts": self.tor_max_consecutive_timeouts, + "tor_exit_nodes": self.tor_exit_nodes, + "num_tabs": self.num_tabs, + "num_iterations": self.num_iterations, + "iteration_mode": self.iteration_mode, + "max_visits_per_iteration": self.max_visits_per_iteration, + "randomize_queue": self.randomize_queue, + "headless": self.headless, + "accept_cookies": self.accept_cookies, + "slow_mo": self.slow_mo, + "timeout_ms": self.timeout_ms, + "tor_socks_port": self.tor_socks_port, + "tor_control_port": self.tor_control_port, + "user_agent": self.user_agent, + } + + @classmethod + def from_dict(cls, data: dict) -> "CrawlerConfig": + """Creates a CrawlerConfig from a dictionary with validation and defaults.""" + start_url = str(data.get("start_url", "")).strip() + if start_url and not start_url.startswith("http://") and not start_url.startswith("https://"): + start_url = "https://" + start_url + + if start_url: + start_domain = urlparse(start_url).netloc + norm_start, _ = normalize_url(start_url, start_domain) + if norm_start: + start_url = norm_start + + circuit_mode = str(data.get("tor_circuit_mode", "iteration")).strip().lower() + if circuit_mode not in ["iteration", "pages"]: + circuit_mode = "iteration" + + try: + circuit_pages = max(1, int(data.get("tor_circuit_pages", 10))) + except (ValueError, TypeError): + circuit_pages = 10 + + try: + max_consecutive_timeouts = max(1, int(data.get("tor_max_consecutive_timeouts", 5))) + except (ValueError, TypeError): + max_consecutive_timeouts = 5 + + raw_exit_nodes = data.get("tor_exit_nodes", []) + if isinstance(raw_exit_nodes, str): + exit_nodes_list = [c.strip().lower() for c in raw_exit_nodes.split(",") if c.strip()] + elif isinstance(raw_exit_nodes, list): + exit_nodes_list = [str(c).strip().lower() for c in raw_exit_nodes if str(c).strip()] + else: + exit_nodes_list = [] + + iter_mode = str(data.get("iteration_mode", "all")).strip().lower() + if iter_mode in ["visits", "limit", "pages", "capped"]: + iteration_mode = "visits" + else: + iteration_mode = "all" + + raw_visits = data.get("max_visits_per_iteration") or data.get("visits_per_iteration") or 10 + try: + val = int(raw_visits) + max_visits_per_iteration = val if val >= 1 else 10 + except (ValueError, TypeError): + max_visits_per_iteration = 10 + + return cls( + start_url=start_url, + use_tor=bool(data.get("use_tor", False)), + tor_circuit_mode=circuit_mode, + tor_circuit_pages=circuit_pages, + tor_max_consecutive_timeouts=max_consecutive_timeouts, + tor_exit_nodes=exit_nodes_list, + num_tabs=int(data.get("num_tabs", 5)), + num_iterations=int(data.get("num_iterations", 1)), + iteration_mode=iteration_mode, + max_visits_per_iteration=max_visits_per_iteration, + randomize_queue=bool(data.get("randomize_queue", True)), + headless=bool(data.get("headless", False)), + accept_cookies=bool(data.get("accept_cookies", True)), + slow_mo=int(data.get("slow_mo", 100)), + timeout_ms=int(data.get("timeout_ms", 35000)), + tor_socks_port=int(data.get("tor_socks_port", 9050)), + tor_control_port=int(data.get("tor_control_port", 9051)), + user_agent=str(data.get("user_agent", "random")).strip(), + ) + + @classmethod + def get_env_dict(cls, env_path: str = ".env") -> dict: + """Reads configuration from .env file or environment variables.""" + load_dotenv(dotenv_path=env_path, override=True) + raw_mode = os.getenv("CRAWLER_TOR_CIRCUIT_MODE", "iteration").strip().lower() + tor_circuit_mode = "pages" if raw_mode in ["pages", "page", "per_pages"] else "iteration" + raw_pages = os.getenv("CRAWLER_TOR_CIRCUIT_PAGES", "10").strip() + tor_circuit_pages = int(raw_pages) if raw_pages.isdigit() else 10 + + raw_timeouts = os.getenv("CRAWLER_TOR_MAX_CONSECUTIVE_TIMEOUTS", "5").strip() + tor_max_consecutive_timeouts = int(raw_timeouts) if raw_timeouts.isdigit() else 5 + + raw_nodes = os.getenv("CRAWLER_TOR_EXIT_NODES", "") + tor_exit_nodes = [c.strip().lower() for c in raw_nodes.split(",") if c.strip()] + + raw_iter_mode = os.getenv("CRAWLER_ITERATION_MODE", "all").strip().lower() + iteration_mode = "visits" if raw_iter_mode in ["visits", "limit", "pages", "capped"] else "all" + + raw_max_visits = ( + os.getenv("CRAWLER_MAX_VISITS_PER_ITERATION") or + os.getenv("CRAWLER_VISITS_PER_ITERATION") or + "10" + ).strip() + max_visits_per_iteration = int(raw_max_visits) if raw_max_visits.isdigit() else 10 + + return { + "start_url": os.getenv("CRAWLER_START_URL", "https://example.com"), + "use_tor": os.getenv("CRAWLER_USE_TOR", "false").strip().lower() in ["true", "1", "y", "yes"], + "tor_circuit_mode": tor_circuit_mode, + "tor_circuit_pages": max(1, tor_circuit_pages), + "tor_max_consecutive_timeouts": max(1, tor_max_consecutive_timeouts), + "tor_exit_nodes": tor_exit_nodes, + "num_tabs": int(os.getenv("CRAWLER_NUM_TABS", "5")) if os.getenv("CRAWLER_NUM_TABS", "").isdigit() else 5, + "num_iterations": int(os.getenv("CRAWLER_NUM_ITERATIONS", "1")) if os.getenv("CRAWLER_NUM_ITERATIONS", "").isdigit() else 1, + "iteration_mode": iteration_mode, + "max_visits_per_iteration": max(1, max_visits_per_iteration), + "randomize_queue": os.getenv("CRAWLER_RANDOMIZE_QUEUE", "true").strip().lower() not in ["false", "0", "n", "no"], + "headless": os.getenv("CRAWLER_HEADLESS", "false").strip().lower() in ["true", "1", "y", "yes"], + "accept_cookies": os.getenv("CRAWLER_ACCEPT_COOKIES", "true").strip().lower() not in ["false", "0", "n", "no"], + "slow_mo": int(os.getenv("CRAWLER_SLOW_MO", "100")) if os.getenv("CRAWLER_SLOW_MO", "").isdigit() else 100, + "timeout_ms": int(os.getenv("CRAWLER_TIMEOUT_MS", "35000")) if os.getenv("CRAWLER_TIMEOUT_MS", "").isdigit() else 35000, + "tor_socks_port": int(os.getenv("CRAWLER_TOR_SOCKS_PORT", "9050")) if os.getenv("CRAWLER_TOR_SOCKS_PORT", "").isdigit() else 9050, + "tor_control_port": int(os.getenv("CRAWLER_TOR_CONTROL_PORT", "9051")) if os.getenv("CRAWLER_TOR_CONTROL_PORT", "").isdigit() else 9051, + "user_agent": os.getenv("CRAWLER_USER_AGENT", "random").strip(), + } + + @classmethod + def save_env_file(cls, data: dict, env_path: str = ".env") -> None: + """Writes configuration key-values to a .env file.""" + raw_exit = data.get("tor_exit_nodes", []) + if isinstance(raw_exit, list): + exit_str = ",".join(raw_exit) + else: + exit_str = str(raw_exit) + + lines = [ + f"CRAWLER_START_URL={data.get('start_url', '')}", + f"CRAWLER_USE_TOR={'true' if data.get('use_tor') else 'false'}", + f"CRAWLER_TOR_CIRCUIT_MODE={data.get('tor_circuit_mode', 'iteration')}", + f"CRAWLER_TOR_CIRCUIT_PAGES={data.get('tor_circuit_pages', 10)}", + f"CRAWLER_TOR_MAX_CONSECUTIVE_TIMEOUTS={data.get('tor_max_consecutive_timeouts', 5)}", + f"CRAWLER_TOR_EXIT_NODES={exit_str}", + f"CRAWLER_NUM_TABS={data.get('num_tabs', 5)}", + f"CRAWLER_NUM_ITERATIONS={data.get('num_iterations', 1)}", + f"CRAWLER_ITERATION_MODE={data.get('iteration_mode', 'all')}", + f"CRAWLER_MAX_VISITS_PER_ITERATION={data.get('max_visits_per_iteration', 10)}", + f"CRAWLER_RANDOMIZE_QUEUE={'true' if data.get('randomize_queue', True) else 'false'}", + f"CRAWLER_HEADLESS={'true' if data.get('headless') else 'false'}", + f"CRAWLER_ACCEPT_COOKIES={'true' if data.get('accept_cookies') else 'false'}", + f"CRAWLER_SLOW_MO={data.get('slow_mo', 100)}", + f"CRAWLER_TIMEOUT_MS={data.get('timeout_ms', 35000)}", + f"CRAWLER_TOR_SOCKS_PORT={data.get('tor_socks_port', 9050)}", + f"CRAWLER_TOR_CONTROL_PORT={data.get('tor_control_port', 9051)}", + f"CRAWLER_USER_AGENT={data.get('user_agent', 'random')}", + "", + ] + with open(env_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + diff --git a/crawler/cookies.py b/crawler/cookies.py new file mode 100644 index 0000000..0a07808 --- /dev/null +++ b/crawler/cookies.py @@ -0,0 +1,177 @@ +import asyncio + +# Selectors for common Consent Management Platforms (CMPs) and cookie banners +COOKIE_BANNER_SELECTORS = [ + # Complianz (WordPress) + "button.cmplz-accept", + "button.cmplz-btn.cmplz-accept", + ".cmplz-btn.cmplz-accept", + "#cmplz-cookiebanner-content .cmplz-accept", + + # Borlabs Cookie (WordPress) + "a._brlbs-accept-all", + "button._brlbs-accept-all", + "._brlbs-btn-accept-all", + "[data-borlabs-cookie-accept-all]", + "#borlabs-cookie-accept-all", + ".borlabs-cookie-preference-accept-all", + + # Cookiebot + "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", + "#CybotCookiebotDialogBodyButtonAccept", + "#CybotCookiebotDialogBodyLevelButtonAccept", + "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowallSelection", + + # OneTrust / CookiePro + "#onetrust-accept-btn-handler", + "button#onetrust-accept-btn-handler", + "#accept-recommended-btn-handler", + + # Usercentrics + "button[data-testid='uc-accept-all-button']", + "#uc-btn-accept-banner", + "button#uc-btn-accept-banner", + "div#usercentrics-root button[data-testid='uc-accept-all-button']", + + # Klaro + ".klaro .cm-btn-accept", + ".klaro .cm-btn-success", + "button.cm-btn-accept-all", + + # CookieYes + "button.cky-btn-accept", + "[data-cky-tag='accept-button']", + + # Real Cookie Banner + "button[data-consent-action='accept-all']", + ".rcb-btn-accept-all", + + # CCM19 + ".ccm--btn-accept-all", + "button.ccm--btn-accept-all", + "button[data-ccm-consent='accept-all']", + + # Google Funding Choices + ".fc-cta-consent", + "button.fc-cta-consent", + + # Quantcast / InMobi + ".qc-cmp2-summary-buttons button[mode='primary']", + + # Iubenda + ".iubenda-cs-accept-btn", + "button.iubenda-cs-btn-primary", + + # Didomi + "#didomi-notice-agree-button", + + # Osano + ".osano-cm-accept-all", + "button.osano-cm-button--type_accept", + + # Axeptio + "#axeptio_btn_acceptAll", + + # Generic IDs, classes, and data attributes + "[data-action='accept-all']", + "[data-consent='accept-all']", + "[data-cookie-accept-all]", + "button[id*='cookie-accept']", + "button[id*='accept-cookies']", + "button[id*='cookieAccept']", + "button[class*='cookie-accept']", + "button[class*='accept-all-cookies']", +] + +COOKIE_TEXT_PATTERNS = [ + # German + r"^(alle\s+)?(cookies\s+)?akzeptieren$", + r"^alles\s+akzeptieren$", + r"^alle\s+annehmen$", + r"^alles\s+annehmen$", + r"^alle\s+ausw[aä]hlen\s+und\s+akzeptieren$", + r"^alle\s+zulassen$", + r"^cookies\s+zulassen$", + r"^einverstanden$", + r"^zustimmen$", + r"^allen\s+zustimmen$", + r"^alle\s+best[aä]tigen$", + r"^auswahl\s+best[aä]tigen$", + r"^einwilligen$", + r"^verstanden$", + r"^ok$", + + # English + r"^(accept\s+all(\s+cookies)?)$", + r"^(allow\s+all(\s+cookies)?)$", + r"^i\s+accept$", + r"^i\s+agree$", + r"^agree(\s+to\s+all)?$", + r"^accept$", + r"^allow$", + r"^agree\s+&\s+proceed$", + r"^got\s+it$", +] + +COOKIE_TEXT_REGEX_STR = "|".join(f"(?:{p})" for p in COOKIE_TEXT_PATTERNS) + + +async def auto_accept_cookies(page) -> bool: + """ + Attempts to detect and click cookie consent / banner acceptance buttons + using a fast, single-pass in-browser JavaScript evaluation. + Only targets actual buttons and known CMP selectors, preventing accidental link navigations. + """ + js_code = """ + () => { + const selectors = """ + str(COOKIE_BANNER_SELECTORS) + """; + + // 1. Try known CMP selectors on main page + for (const sel of selectors) { + try { + const el = document.querySelector(sel); + if (el && (el.offsetWidth > 0 || el.offsetHeight > 0 || el.getClientRects().length > 0)) { + el.click(); + return { clicked: true, method: 'selector', text: sel }; + } + } catch(e) {} + } + + // 2. Check Shadow DOM (e.g. Usercentrics / custom web components) + try { + const ucRoot = document.querySelector('#usercentrics-root, div#usercentrics-cmp-ui'); + if (ucRoot && ucRoot.shadowRoot) { + const ucBtn = ucRoot.shadowRoot.querySelector("button[data-testid='uc-accept-all-button'], button#uc-btn-accept-banner, button.uc-btn-accept"); + if (ucBtn) { + ucBtn.click(); + return { clicked: true, method: 'shadow-dom', text: 'usercentrics-shadow' }; + } + } + } catch(e) {} + + // 3. Text-based matching strictly on button / submit / role=button elements (never plain navigation links) + const textRegex = /""" + COOKIE_TEXT_REGEX_STR + """/i; + const candidates = document.querySelectorAll('button, input[type="button"], input[type="submit"], [role="button"], a.cmplz-accept, a._brlbs-accept-all, a.cky-btn-accept'); + for (const btn of candidates) { + try { + const txt = (btn.innerText || btn.value || '').trim(); + if (txt && (btn.offsetWidth > 0 || btn.offsetHeight > 0 || btn.getClientRects().length > 0) && textRegex.test(txt)) { + btn.click(); + return { clicked: true, method: 'button-text', text: txt }; + } + } catch(e) {} + } + + return { clicked: false }; + } + """ + try: + result = await asyncio.wait_for(page.evaluate(js_code), timeout=2.0) + if result and result.get("clicked"): + print(f" [Cookie] Auto-accepted cookies via {result.get('method')}: '{result.get('text')}'") + await page.wait_for_timeout(600) # Give tracking scripts time to transmit beacons + return True + except Exception: + pass + + return False diff --git a/crawler/engine.py b/crawler/engine.py new file mode 100644 index 0000000..1c70edc --- /dev/null +++ b/crawler/engine.py @@ -0,0 +1,1023 @@ +import asyncio +import inspect +import random +import time +from typing import Dict, List, Set, Tuple, Optional, Callable, Any +from playwright.async_api import async_playwright + +from .config import CrawlerConfig +from .cookies import auto_accept_cookies +from .tor import TorManager +from .url_utils import normalize_url + + +async def _dispatch_event(callback: Optional[Callable[[Dict[str, Any]], Any]], event: Dict[str, Any]): + """Safely dispatches an event dictionary to an async or sync callback.""" + if callback is None: + return + try: + if inspect.iscoroutinefunction(callback): + await callback(event) + else: + callback(event) + except Exception: + pass + + +async def crawl_level( + page, + url: str, + source_url: Optional[str], + visited: Set[str], + errors: List[Tuple[str, Optional[str], str]], + start_domain: str, + accept_cookies_default: bool = True, + stats: Optional[Dict[str, int]] = None, + timeout_ms: int = 35000, + event_callback: Optional[Callable[[Dict[str, Any]], Any]] = None, + worker_name: str = "Worker", + page_result_callback: Optional[Callable[[str, Optional[str], bool, Optional[str]], Any]] = None, +) -> Set[str]: + """Navigates to a URL and returns a set of valid HTTP/HTTPS links found on that page.""" + log_msg = f"[{worker_name}] Browsing: {url}" + print(log_msg) + await _dispatch_event(event_callback, { + "type": "log", + "message": log_msg, + "level": "info", + "timestamp": time.time(), + }) + + start_time = time.time() + try: + # Go to the URL and wait for the page to load + response = await page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms) + visited.add(url) + + if response and response.status >= 400: + retry_after = response.headers.get("retry-after") + status_desc = "" + if response.status == 503: + status_desc = " (Service Temporarily Unavailable / Rate Limited)" + elif response.status == 429: + status_desc = " (Too Many Requests / Rate Limited)" + elif response.status == 403: + status_desc = " (Forbidden / Access Denied)" + + retry_info = f" [Retry-After: {retry_after}s]" if retry_after else "" + err_text = f"HTTP {response.status}{status_desc}{retry_info}" + errors.append((url, source_url, err_text)) + err_log = f" [{worker_name}] [!] HTTP {response.status}{status_desc}{retry_info} returned for {url}" + print(err_log) + await _dispatch_event(event_callback, { + "type": "log", + "message": err_log, + "level": "error", + "timestamp": time.time(), + }) + await _dispatch_event(event_callback, { + "type": "error", + "url": url, + "source_url": source_url, + "error": err_text, + "timestamp": time.time(), + }) + if page_result_callback: + try: + res = page_result_callback(url, source_url, False, err_text) + if inspect.iscoroutine(res): + await res + except Exception: + pass + return set() + + except Exception as e: + duration = time.time() - start_time + visited.add(url) + error_msg = str(e).split("\n")[0] + errors.append((url, source_url, error_msg)) + err_log = f" [{worker_name}] [!] Failed to load {url} after {duration:.2f}s: {error_msg}" + print(err_log) + await _dispatch_event(event_callback, { + "type": "log", + "message": err_log, + "level": "error", + "timestamp": time.time(), + }) + await _dispatch_event(event_callback, { + "type": "error", + "url": url, + "source_url": source_url, + "error": error_msg, + "timestamp": time.time(), + }) + if page_result_callback: + try: + res = page_result_callback(url, source_url, False, error_msg) + if inspect.iscoroutine(res): + await res + except Exception: + pass + return set() + + duration = time.time() - start_time + ok_log = f" [{worker_name}] [OK] Loaded in {duration:.2f} seconds" + print(ok_log) + await _dispatch_event(event_callback, { + "type": "log", + "message": ok_log, + "level": "success", + "timestamp": time.time(), + }) + + # Automatically accept cookies to ensure tracking and scripts are activated + if accept_cookies_default: + accepted = await auto_accept_cookies(page) + if accepted and stats is not None: + stats["cookies_accepted"] += 1 + cookie_log = f" [{worker_name}] [COOKIE] Accepted cookie banner on {url}" + print(cookie_log) + await _dispatch_event(event_callback, { + "type": "log", + "message": cookie_log, + "level": "info", + "timestamp": time.time(), + }) + + if page_result_callback: + try: + res = page_result_callback(url, source_url, True, None) + if inspect.iscoroutine(res): + await res + except Exception: + pass + + # Extract all 'href' attributes from 'a' tags + try: + links = await page.eval_on_selector_all("a[href]", "elements => elements.map(el => el.href)") + except Exception as e: + extract_err = f" [{worker_name}] [!] Failed to extract links from {url}: {e}" + print(extract_err) + await _dispatch_event(event_callback, { + "type": "log", + "message": extract_err, + "level": "warn", + "timestamp": time.time(), + }) + return set() + + # Filter and normalize valid HTTP/HTTPS links + valid_links = set() + for link in links: + norm_url, _ = normalize_url(link, start_domain) + if norm_url: + valid_links.add(norm_url) + + return valid_links + + +class RandomQueue(asyncio.Queue): + """An asyncio Queue that pops items uniformly at random instead of strict FIFO order.""" + + def _init(self, maxsize): + self._queue = [] + + def _put(self, item): + self._queue.append(item) + + def _get(self): + idx = random.randrange(len(self._queue)) + self._queue[idx], self._queue[-1] = self._queue[-1], self._queue[idx] + return self._queue.pop() + + +class IterationVisitTracker: + """Coordinates and limits website visits within a single crawl iteration.""" + + def __init__(self, max_visits: Optional[int]): + self.max_visits = max_visits + self.started_visits = 0 + self.completed_visits = 0 + self._lock = asyncio.Lock() + self.limit_reached_event = asyncio.Event() + + async def acquire_visit_slot(self) -> bool: + """ + Attempts to reserve a visit slot. + Returns True if reservation was successful (or visits unlimited), + False if the max visit limit for this iteration has been reached. + """ + if self.max_visits is None or self.max_visits <= 0: + return True + async with self._lock: + if self.started_visits >= self.max_visits: + return False + self.started_visits += 1 + return True + + async def record_visit_completed(self) -> None: + """Records that a visit attempt has finished, signalling limit_reached_event if target reached.""" + if self.max_visits is None or self.max_visits <= 0: + return + async with self._lock: + self.completed_visits += 1 + if self.completed_visits >= self.max_visits: + self.limit_reached_event.set() + + +async def worker( + name: str, + context, + queue: asyncio.Queue, + visited: Set[str], + enqueued: Set[str], + errors: List[Tuple[str, Optional[str], str]], + start_domain: str, + stats: Dict[str, int], + stop_event: asyncio.Event, + iteration_stop_event: Optional[asyncio.Event] = None, + visit_tracker: Optional[IterationVisitTracker] = None, + accept_cookies_default: bool = True, + timeout_ms: int = 35000, + event_callback: Optional[Callable[[Dict[str, Any]], Any]] = None, + page_done_callback: Optional[Callable[[], Any]] = None, + page_result_callback: Optional[Callable[[str, Optional[str], bool, Optional[str]], Any]] = None, + prior_visited: int = 0, + prior_errors: int = 0, +): + """Worker task that pulls URLs from the queue and crawls them in its own dedicated tab.""" + page = None + try: + if stop_event.is_set() or (iteration_stop_event and iteration_stop_event.is_set()): + return + + page = await context.new_page() + + # Automatically close any popups or new tabs spawned by links or scripts + page.on("popup", lambda popup: asyncio.create_task(popup.close())) + + while not stop_event.is_set() and (iteration_stop_event is None or not iteration_stop_event.is_set()): + # Wait for a URL from the queue with a short timeout to check stop conditions periodically + try: + item = await asyncio.wait_for(queue.get(), timeout=1.0) + except asyncio.TimeoutError: + continue + + current_url, source_url, is_external = item + + try: + if stop_event.is_set() or (iteration_stop_event and iteration_stop_event.is_set()): + break + + if current_url in visited: + continue + + if visit_tracker: + can_visit = await visit_tracker.acquire_visit_slot() + if not can_visit: + if iteration_stop_event: + iteration_stop_event.set() + break + + prefix = f"[EXTERNAL|{name}]" if is_external else f"[INTERNAL|{name}]" + queue_log = f"\n{prefix} (Queue approx: {queue.qsize()})" + print(queue_log) + + try: + found_links = await crawl_level( + page=page, + url=current_url, + source_url=source_url, + visited=visited, + errors=errors, + start_domain=start_domain, + accept_cookies_default=accept_cookies_default, + stats=stats, + timeout_ms=timeout_ms, + event_callback=event_callback, + worker_name=name, + page_result_callback=page_result_callback, + ) + finally: + if visit_tracker: + await visit_tracker.record_visit_completed() + + if is_external: + stats["external"] += 1 + else: + stats["internal"] += 1 + + # Emit updated stats + await _dispatch_event(event_callback, { + "type": "stats", + "stats": stats, + "queue_size": queue.qsize(), + "visited_count": len(visited), + "error_count": len(errors), + "total_visited_all": prior_visited + len(visited), + "total_errors_all": prior_errors + len(errors), + }) + + # Trigger legacy page crawled callback if page_result_callback was not supplied + if page_done_callback and not page_result_callback and not stop_event.is_set(): + try: + res = page_done_callback() + if inspect.iscoroutine(res): + await res + except Exception: + pass + + # If it's an internal link, add its discovered links to the queue + if not is_external and not stop_event.is_set() and (iteration_stop_event is None or not iteration_stop_event.is_set()): + new_links = [] + for link in found_links: + norm_link, link_is_internal = normalize_url(link, start_domain) + if norm_link and norm_link not in enqueued and norm_link not in visited: + new_links.append((norm_link, current_url, not link_is_internal)) + enqueued.add(norm_link) + + # Shuffle new links to maintain a degree of randomness in crawling order + if new_links: + random.shuffle(new_links) + for n_item in new_links: + queue.put_nowait(n_item) + elif is_external: + ext_log = f" [{name}] -> Reached external level 1. Not following further links." + print(ext_log) + await _dispatch_event(event_callback, { + "type": "log", + "message": ext_log, + "level": "info", + "timestamp": time.time(), + }) + + except Exception as e: + err_str = f" [!] Unexpected error in worker {name} processing {current_url}: {e}" + print(err_str) + await _dispatch_event(event_callback, { + "type": "log", + "message": err_str, + "level": "error", + "timestamp": time.time(), + }) + finally: + queue.task_done() + except (asyncio.CancelledError, Exception): + pass + finally: + if page: + try: + await page.close() + except Exception: + pass + + +async def fetch_tor_identity(context, timeout_ms: int = 10000) -> Dict[str, Any]: + """Fetches active exit IP and geolocation details using the browser's Tor proxy context.""" + # 1. Try ipapi.co + try: + res = await context.request.get("https://ipapi.co/json/", timeout=timeout_ms) + if res.ok: + data = await res.json() + if "ip" in data: + return { + "ip": data.get("ip", "Unknown"), + "city": data.get("city", "Unknown"), + "region": data.get("region", "Unknown"), + "country": data.get("country_name", "Unknown"), + "country_code": data.get("country_code", ""), + "org": data.get("org", "Unknown"), + "is_tor": True, + } + except Exception: + pass + + # 2. Try ip-api.com fallback + try: + res = await context.request.get("http://ip-api.com/json/", timeout=timeout_ms) + if res.ok: + data = await res.json() + if data.get("status") == "success": + return { + "ip": data.get("query", "Unknown"), + "city": data.get("city", "Unknown"), + "region": data.get("regionName", "Unknown"), + "country": data.get("country", "Unknown"), + "country_code": data.get("countryCode", ""), + "org": data.get("isp", "Unknown"), + "is_tor": True, + } + except Exception: + pass + + # 3. Try official check.torproject.org fallback + try: + res = await context.request.get("https://check.torproject.org/api/ip", timeout=timeout_ms) + if res.ok: + data = await res.json() + return { + "ip": data.get("IP", "Unknown"), + "city": "Unknown", + "region": "Unknown", + "country": "Unknown", + "country_code": "", + "org": "Tor Exit Relay", + "is_tor": bool(data.get("IsTor", True)), + } + except Exception: + pass + + return { + "ip": "Unknown", + "city": "Unknown", + "region": "Unknown", + "country": "Unknown", + "country_code": "", + "org": "Tor Network", + "is_tor": True, + } + + +class Crawler: + """Core crawler engine orchestrating browser instances, tabs, and queue scheduling.""" + + def __init__( + self, + config: CrawlerConfig, + event_callback: Optional[Callable[[Dict[str, Any]], Any]] = None, + ): + self.config = config + self.event_callback = event_callback + self.tor_manager = TorManager( + socks_port=config.tor_socks_port, + control_port=config.tor_control_port, + exit_nodes=config.tor_exit_nodes_formatted, + ) if config.use_tor else None + self._stop_event = asyncio.Event() + self._is_running = False + self._circuit_lock = asyncio.Lock() + self._pages_in_current_circuit = 0 + self._consecutive_timeouts = 0 + self._total_circuit_rotations = 0 + self._total_visited_all_iterations = 0 + self._total_errors_all_iterations = 0 + + def stop(self) -> None: + """Signals the crawler and active workers to stop immediately.""" + self._stop_event.set() + + @property + def is_running(self) -> bool: + return self._is_running + + @property + def total_visited_all_iterations(self) -> int: + return self._total_visited_all_iterations + + @property + def total_errors_all_iterations(self) -> int: + return self._total_errors_all_iterations + + async def _rotate_tor_circuit(self, context=None, reason: str = "") -> None: + """Safely triggers NEWNYM via TorManager and queries fresh exit node identity.""" + if not self.tor_manager or not self.config.use_tor: + return + + rot_msg = f"[Tor] Renewing Tor circuit (NEWNYM)... (Reason: {reason})" + print(f"\n{rot_msg}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": rot_msg, + "level": "info", + "timestamp": time.time(), + }) + + await asyncio.to_thread(self.tor_manager.renew_circuit) + self._total_circuit_rotations += 1 + self._pages_in_current_circuit = 0 + self._consecutive_timeouts = 0 + + if context: + try: + tor_id = await fetch_tor_identity(context) + tor_loc_str = f"{tor_id['city']}, {tor_id['country']}" if tor_id['city'] != "Unknown" else tor_id['country'] + tor_id_msg = f"[Tor] Circuit #{self._total_circuit_rotations} active! Exit Node IP: {tor_id['ip']} | Location: {tor_loc_str} | ISP: {tor_id['org']}" + print(tor_id_msg) + await _dispatch_event(self.event_callback, { + "type": "log", + "message": tor_id_msg, + "level": "success", + "timestamp": time.time(), + }) + await _dispatch_event(self.event_callback, { + "type": "tor_identity", + "data": { + **tor_id, + "rotation_count": self._total_circuit_rotations, + "exit_nodes": self.config.tor_exit_nodes, + }, + }) + except Exception as e: + print(f" [!] Failed to query Tor identity after rotation: {e}") + + async def _on_page_result( + self, + context, + url: str, + source_url: Optional[str], + success: bool, + error_msg: Optional[str] = None, + ) -> None: + """Handles Tor circuit rotation on N crawled pages as well as consecutive timeout fallback.""" + if not self.config.use_tor: + return + + async with self._circuit_lock: + if success: + self._consecutive_timeouts = 0 + if self.config.tor_circuit_mode == "pages": + self._pages_in_current_circuit += 1 + if self._pages_in_current_circuit >= self.config.tor_circuit_pages: + await self._rotate_tor_circuit( + context=context, + reason=f"Reached {self.config.tor_circuit_pages} crawled pages threshold", + ) + else: + # Check if this error is net::ERR_TIMED_OUT, connection timeout, or rate-limit / WAF block (503, 429, 403) + is_timeout = False + is_rate_limited = False + if error_msg: + err_lower = error_msg.lower() + if ( + "err_timed_out" in err_lower + or "timed out" in err_lower + or "timed_out" in err_lower + or "timeout" in err_lower + ): + is_timeout = True + elif ( + "http 503" in err_lower + or "http 429" in err_lower + or "http 403" in err_lower + ): + is_rate_limited = True + + if is_timeout or is_rate_limited: + self._consecutive_timeouts += 1 + err_type = "Rate limit / WAF block" if is_rate_limited else "Timeout" + timeout_log = ( + f" [Tor] Consecutive {err_type.lower()} ({self._consecutive_timeouts}/" + f"{self.config.tor_max_consecutive_timeouts}) on {url}: {error_msg}" + ) + print(timeout_log) + await _dispatch_event(self.event_callback, { + "type": "log", + "message": timeout_log, + "level": "warn", + "timestamp": time.time(), + }) + + if self._consecutive_timeouts >= self.config.tor_max_consecutive_timeouts: + fallback_reason = ( + f"Fallback: {self.config.tor_max_consecutive_timeouts} consecutive pages failed " + f"({err_type.lower()}: {error_msg})" + ) + fallback_log = ( + f" [Tor] Fallback triggered! {self._consecutive_timeouts} consecutive pages failed " + f"with {err_type.lower()}. Acquiring new Tor circuit..." + ) + print(f"\n{fallback_log}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": fallback_log, + "level": "warn", + "timestamp": time.time(), + }) + self._consecutive_timeouts = 0 + await self._rotate_tor_circuit( + context=context, + reason=fallback_reason, + ) + else: + # Other non-transient errors (e.g. 404) reset consecutive failure counter + self._consecutive_timeouts = 0 + + async def _on_page_crawled(self, context) -> None: + """Checks whether Tor circuit renewal is due in 'pages' mode (legacy compatibility).""" + await self._on_page_result(context, "", None, True, None) + + async def run_iteration(self, playwright_instance, iteration: int) -> Dict[str, any]: + """Runs a single crawl iteration.""" + if self._stop_event.is_set(): + return { + "duration": 0, + "stats": {}, + "errors": [], + "visited_count": 0, + "total_visited_all": self._total_visited_all_iterations, + "total_errors_all": self._total_errors_all_iterations, + } + + # Handle Tor circuit rotation at iteration boundary + if self.config.use_tor and self.tor_manager: + if self.config.tor_circuit_mode == "iteration": + await self._rotate_tor_circuit( + context=None, + reason=f"Starting Iteration {iteration}", + ) + elif self.config.tor_circuit_mode == "pages" and self._total_circuit_rotations == 0: + await self._rotate_tor_circuit( + context=None, + reason="Initial Crawl Setup", + ) + + iteration_start_time = time.time() + + launch_args = { + "headless": self.config.headless, + "slow_mo": self.config.slow_mo, + "args": [ + "--disable-blink-features=AutomationControlled", + "--no-sandbox", + "--disable-dev-shm-usage", + ], + } + + if self.config.use_tor: + tor_msg = f"[INFO] Routing traffic through Tor (socks5://127.0.0.1:{self.config.tor_socks_port})..." + print(f"\n{tor_msg}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": tor_msg, + "level": "info", + "timestamp": time.time(), + }) + launch_args["proxy"] = {"server": f"socks5://127.0.0.1:{self.config.tor_socks_port}"} + launch_args["args"].append("--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE 127.0.0.1") + + chosen_ua = self.config.get_user_agent() + context_kwargs = { + "viewport": {"width": 1920, "height": 1080}, + "locale": "de-DE", + "timezone_id": "Europe/Berlin", + } + if chosen_ua: + context_kwargs["user_agent"] = chosen_ua + + browser = await playwright_instance.chromium.launch(**launch_args) + context = await browser.new_context(**context_kwargs) + + ua_msg = f"[INFO] Active Browser User-Agent: {chosen_ua or 'Native Browser Default'}" + print(ua_msg) + await _dispatch_event(self.event_callback, { + "type": "log", + "message": ua_msg, + "level": "info", + "timestamp": time.time(), + }) + + # If Tor is enabled, discover exit IP and region for the current circuit + if self.config.use_tor: + tor_id = await fetch_tor_identity(context) + tor_loc_str = f"{tor_id['city']}, {tor_id['country']}" if tor_id['city'] != "Unknown" else tor_id['country'] + tor_id_msg = f"[Tor] Exit Node IP: {tor_id['ip']} | Location: {tor_loc_str} | ISP: {tor_id['org']}" + print(tor_id_msg) + await _dispatch_event(self.event_callback, { + "type": "log", + "message": tor_id_msg, + "level": "info", + "timestamp": time.time(), + }) + await _dispatch_event(self.event_callback, { + "type": "tor_identity", + "data": { + **tor_id, + "rotation_count": self._total_circuit_rotations, + "exit_nodes": self.config.tor_exit_nodes, + }, + }) + + visited: Set[str] = set() + enqueued: Set[str] = {self.config.start_url} + errors: List[Tuple[str, Optional[str], str]] = [] + stats: Dict[str, int] = {"internal": 0, "external": 0, "cookies_accepted": 0} + + prior_visited = self._total_visited_all_iterations + prior_errors = self._total_errors_all_iterations + + queue: asyncio.Queue = ( + RandomQueue() if self.config.randomize_queue else asyncio.Queue() + ) + queue.put_nowait((self.config.start_url, None, False)) + + max_visits = ( + self.config.max_visits_per_iteration + if self.config.iteration_mode == "visits" and self.config.max_visits_per_iteration > 0 + else None + ) + visit_tracker = IterationVisitTracker(max_visits=max_visits) if max_visits else None + iteration_stop_event = asyncio.Event() + + start_domain = self.config.start_domain + start_msg = f"=== Starting Crawl on {start_domain} with {self.config.num_tabs} concurrent workers ===" + print(f"\n{start_msg}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": start_msg, + "level": "info", + "timestamp": time.time(), + }) + + if max_visits: + mode_msg = f"[INFO] Iteration visit mode: Limit to {max_visits} website visits before starting next iteration." + else: + mode_msg = "[INFO] Iteration visit mode: Crawling all reachable website links." + print(mode_msg) + await _dispatch_event(self.event_callback, { + "type": "log", + "message": mode_msg, + "level": "info", + "timestamp": time.time(), + }) + + queue_msg = ( + "[INFO] Crawl queue strategy: RANDOM (URLs are picked randomly from discovered links)." + if self.config.randomize_queue + else "[INFO] Crawl queue strategy: FIFO (Standard breadth-first search order)." + ) + print(queue_msg) + await _dispatch_event(self.event_callback, { + "type": "log", + "message": queue_msg, + "level": "info", + "timestamp": time.time(), + }) + + if self.config.accept_cookies: + cookie_enabled_msg = "[INFO] Automatic cookie acceptance is ENABLED." + print(cookie_enabled_msg) + await _dispatch_event(self.event_callback, { + "type": "log", + "message": cookie_enabled_msg, + "level": "info", + "timestamp": time.time(), + }) + + # Emit initial stats + await _dispatch_event(self.event_callback, { + "type": "stats", + "stats": stats, + "queue_size": queue.qsize(), + "visited_count": len(visited), + "error_count": len(errors), + "total_visited_all": prior_visited + len(visited), + "total_errors_all": prior_errors + len(errors), + }) + + # Create worker tasks + workers = [] + for i in range(self.config.num_tabs): + task = asyncio.create_task( + worker( + name=f"Tab-{i+1}", + context=context, + queue=queue, + visited=visited, + enqueued=enqueued, + errors=errors, + start_domain=start_domain, + stats=stats, + stop_event=self._stop_event, + iteration_stop_event=iteration_stop_event, + visit_tracker=visit_tracker, + accept_cookies_default=self.config.accept_cookies, + timeout_ms=self.config.timeout_ms, + event_callback=self.event_callback, + page_result_callback=lambda u, s, ok, err: self._on_page_result(context, u, s, ok, err), + prior_visited=prior_visited, + prior_errors=prior_errors, + ) + ) + workers.append(task) + + # Wait until queue is fully processed or stop signal / limit is reached + join_task = asyncio.create_task(queue.join()) + stop_wait_task = asyncio.create_task(self._stop_event.wait()) + wait_tasks = [join_task, stop_wait_task] + + limit_wait_task = None + if visit_tracker and visit_tracker.max_visits: + limit_wait_task = asyncio.create_task(visit_tracker.limit_reached_event.wait()) + wait_tasks.append(limit_wait_task) + + await asyncio.wait( + wait_tasks, + return_when=asyncio.FIRST_COMPLETED, + ) + + if not join_task.done(): + join_task.cancel() + if not stop_wait_task.done(): + stop_wait_task.cancel() + if limit_wait_task and not limit_wait_task.done(): + limit_wait_task.cancel() + + # Signal workers in this iteration to stop + iteration_stop_event.set() + + # Cancel worker tasks + for task in workers: + task.cancel() + + # Wait for workers to cleanly exit + await asyncio.gather(*workers, return_exceptions=True) + + iteration_duration = time.time() - iteration_start_time + + if visit_tracker and visit_tracker.limit_reached_event.is_set(): + limit_summary = ( + f"[INFO] Reached configured website visit limit " + f"({visit_tracker.completed_visits}/{max_visits} visits) for iteration {iteration}." + ) + print(f"\n{limit_summary}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": limit_summary, + "level": "info", + "timestamp": time.time(), + }) + fin_msg = f"Finished iteration {iteration}! Execution time: {iteration_duration:.2f} seconds." + print(f"\n{fin_msg}") + print(f"Total internal pages visited: {stats['internal']}") + print(f"Total external pages visited: {stats['external']}") + if self.config.accept_cookies: + print(f"Total cookie banners accepted: {stats['cookies_accepted']}") + + await _dispatch_event(self.event_callback, { + "type": "log", + "message": fin_msg, + "level": "success", + "timestamp": time.time(), + }) + + if errors: + err_summary = f"=== Websites with Errors ({len(errors)}) ===" + print(f"\n{err_summary}") + for err_url, src_url, err_msg in errors: + src_display = src_url if src_url else "User Input" + print(f"- {err_url}\n Error: {err_msg}\n Found on: {src_display}") + + self._total_visited_all_iterations = prior_visited + len(visited) + self._total_errors_all_iterations = prior_errors + len(errors) + + await browser.close() + return { + "duration": iteration_duration, + "stats": stats, + "errors": errors, + "visited_count": len(visited), + "total_visited_all": self._total_visited_all_iterations, + "total_errors_all": self._total_errors_all_iterations, + } + + async def run(self) -> None: + """Main execution loop running across all configured iterations.""" + self._is_running = True + self._stop_event.clear() + self._total_visited_all_iterations = 0 + self._total_errors_all_iterations = 0 + + await _dispatch_event(self.event_callback, { + "type": "status", + "status": "running", + "message": "Crawl job started", + "total_visited_all": 0, + "total_errors_all": 0, + }) + + if self.tor_manager: + try: + loop = asyncio.get_running_loop() + def _on_tor_log(msg: str): + if self.event_callback: + asyncio.run_coroutine_threadsafe( + _dispatch_event(self.event_callback, { + "type": "log", + "message": msg, + "level": "info", + "timestamp": time.time(), + }), + loop + ) + self.tor_manager.log_callback = _on_tor_log + + # Launch or connect to Tor in background thread without blocking event loop + await asyncio.to_thread(self.tor_manager.start) + + if self.config.tor_exit_nodes: + countries_display = ", ".join(self.config.tor_exit_nodes).upper() + filter_msg = f"[Tor] Exit node country filter active: {countries_display} ({len(self.config.tor_exit_nodes)} selected)" + else: + filter_msg = "[Tor] Exit node country filter: Global / Any (No restrictions)" + print(f"[INFO] {filter_msg}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": filter_msg, + "level": "info", + "timestamp": time.time(), + }) + except Exception as e: + err_msg = f"[ERROR] Could not initialize Tor: {e}" + print(f"\n{err_msg}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": err_msg, + "level": "error", + "timestamp": time.time(), + }) + await _dispatch_event(self.event_callback, { + "type": "status", + "status": "error", + "message": str(e), + }) + self._is_running = False + return + + try: + global_start_time = time.time() + + for iteration in range(1, self.config.num_iterations + 1): + if self._stop_event.is_set(): + break + + if self.config.num_iterations > 1: + iter_msg = f"=== Starting Iteration {iteration}/{self.config.num_iterations} ===" + print(f"\n{'='*40}\n{iter_msg}\n{'='*40}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": iter_msg, + "level": "info", + "timestamp": time.time(), + }) + + await _dispatch_event(self.event_callback, { + "type": "iteration", + "iteration": iteration, + "total": self.config.num_iterations, + "iteration_mode": self.config.iteration_mode, + "max_visits_per_iteration": self.config.max_visits_per_iteration if self.config.iteration_mode == "visits" else None, + "total_visited_all": self._total_visited_all_iterations, + "total_errors_all": self._total_errors_all_iterations, + }) + + async with async_playwright() as p: + await self.run_iteration(p, iteration) + + total_duration = time.time() - global_start_time + done_msg = ( + f"Finished all {self.config.num_iterations} iterations! " + f"Total execution time: {total_duration:.2f} seconds. " + f"Total websites visited: {self._total_visited_all_iterations}, " + f"total errors: {self._total_errors_all_iterations}." + ) + if self._stop_event.is_set(): + done_msg = ( + f"Crawl stopped by user after {total_duration:.2f} seconds. " + f"Total websites visited: {self._total_visited_all_iterations}, " + f"total errors: {self._total_errors_all_iterations}." + ) + + print(f"\n{done_msg}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": done_msg, + "level": "info", + "timestamp": time.time(), + }) + await _dispatch_event(self.event_callback, { + "type": "status", + "status": "stopped" if self._stop_event.is_set() else "finished", + "message": done_msg, + "total_duration": total_duration, + "total_visited_all": self._total_visited_all_iterations, + "total_errors_all": self._total_errors_all_iterations, + }) + except Exception as e: + err_msg = f"Fatal crawler error: {e}" + print(f"\n[FATAL] {err_msg}") + await _dispatch_event(self.event_callback, { + "type": "log", + "message": err_msg, + "level": "error", + "timestamp": time.time(), + }) + await _dispatch_event(self.event_callback, { + "type": "status", + "status": "error", + "message": str(e), + }) + finally: + self._is_running = False + if self.tor_manager: + self.tor_manager.stop() + diff --git a/crawler/tor.py b/crawler/tor.py new file mode 100644 index 0000000..4beba12 --- /dev/null +++ b/crawler/tor.py @@ -0,0 +1,185 @@ +import atexit +import shutil +import time +from typing import Optional +import stem.process +from stem import Signal +from stem.control import Controller + +DEFAULT_TOR_SOCKS_PORT = 9050 +DEFAULT_TOR_CONTROL_PORT = 9051 + + +class TorManager: + """Manages the standalone headless Tor daemon process and circuit rotation.""" + + def __init__( + self, + socks_port: int = DEFAULT_TOR_SOCKS_PORT, + control_port: int = DEFAULT_TOR_CONTROL_PORT, + exit_nodes: Optional[str] = None, + log_callback: Optional[object] = None, + ): + self.socks_port = socks_port + self.control_port = control_port + self.exit_nodes = exit_nodes + self.log_callback = log_callback + self.process: Optional[object] = None + self._is_external_service = False + + def set_exit_nodes(self, exit_nodes: Optional[str]) -> None: + """Dynamically updates exit node country filter on the active Tor controller.""" + self.exit_nodes = exit_nodes + try: + with Controller.from_port(port=self.control_port) as controller: + controller.authenticate() + if self.exit_nodes: + controller.set_conf("ExitNodes", self.exit_nodes) + print(f"[Tor] Configured ExitNodes on Tor controller: {self.exit_nodes}") + else: + try: + controller.reset_conf("ExitNodes") + except Exception: + pass + print("[Tor] Reset ExitNodes filter (Global / Any)") + except Exception as e: + print(f" [!] Failed to set ExitNodes on ControlPort {self.control_port}: {e}") + + def start(self): + """ + Starts a background Tor process with custom SocksPort and ControlPort using stem, + or attaches to an already running Tor service if available. + """ + # 1. Check if a Tor controller is already running on the control port + try: + with Controller.from_port(port=self.control_port) as controller: + controller.authenticate() + print(f"[Tor] Connected to existing Tor service on ControlPort {self.control_port}.") + if self.exit_nodes: + controller.set_conf("ExitNodes", self.exit_nodes) + print(f"[Tor] Applied ExitNodes: {self.exit_nodes}") + else: + try: + controller.reset_conf("ExitNodes") + except Exception: + pass + self._is_external_service = True + return None + except Exception: + pass + + # 2. Check if 'tor' binary is installed + if not shutil.which("tor"): + raise RuntimeError( + "Tor executable ('tor') was not found on your system PATH.\n" + "Please install Tor:\n" + " - macOS: brew install tor\n" + " - Linux: sudo apt install tor\n" + " - Windows: download Tor expert bundle from torproject.org and add to PATH" + ) + + print( + f"[Tor] Launching headless Tor process " + f"(SocksPort {self.socks_port}, ControlPort {self.control_port})..." + ) + # Always launch Tor with base config so it can bootstrap directory info and GeoIP cleanly without stalling + tor_config = { + "SocksPort": str(self.socks_port), + "ControlPort": str(self.control_port), + } + + def _handle_tor_msg(msg: str): + if "Bootstrapped" in msg: + clean_msg = f"[Tor] {msg.strip()}" + print(f" {clean_msg}") + if self.log_callback and callable(self.log_callback): + try: + self.log_callback(clean_msg) + except Exception: + pass + + self.process = stem.process.launch_tor_with_config( + config=tor_config, + take_ownership=False, # Managed explicitly via atexit/stop to prevent premature shutdowns + init_msg_handler=_handle_tor_msg, + ) + self._is_external_service = False + + # Apply ExitNodes once Tor is successfully bootstrapped + if self.exit_nodes: + try: + with Controller.from_port(port=self.control_port) as controller: + controller.authenticate() + controller.set_conf("ExitNodes", self.exit_nodes) + print(f"[Tor] Applied ExitNodes filter: {self.exit_nodes}") + except Exception as e: + print(f" [!] Failed to apply ExitNodes filter on ControlPort {self.control_port}: {e}") + + # Ensure background process is killed when python script exits + atexit.register(self.stop) + print("[Tor] Standalone Tor process started successfully!") + return self.process + + def renew_circuit(self) -> bool: + """ + Sends NEWNYM signal to Tor ControlPort to switch to a new circuit / IP. + Waits for the necessary cool-off period required by Tor. + Returns True if signal was sent successfully, False otherwise. + """ + try: + with Controller.from_port(port=self.control_port) as controller: + controller.authenticate() + if self.exit_nodes: + controller.set_conf("ExitNodes", self.exit_nodes) + controller.signal(Signal.NEWNYM) + wait_time = controller.get_newnym_wait() + print(f"[Tor] Requested new circuit (NEWNYM). Waiting {wait_time:.1f}s for new circuit to activate...") + if wait_time > 0: + time.sleep(wait_time) + print("[Tor] New circuit is active!") + return True + except Exception as e: + print(f" [!] Failed to renew Tor circuit via ControlPort {self.control_port}: {e}") + return False + + def stop(self) -> None: + """Stops the background Tor process if it was spawned by this manager.""" + if self.process and not self._is_external_service: + try: + if self.process.poll() is None: + print("[Tor] Stopping headless Tor process...") + self.process.kill() + except Exception: + pass + finally: + self.process = None + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + + +# Backward-compatible functional interfaces +def start_tor_service( + socks_port: int = DEFAULT_TOR_SOCKS_PORT, + control_port: int = DEFAULT_TOR_CONTROL_PORT, +): + manager = TorManager(socks_port=socks_port, control_port=control_port) + manager.start() + return manager + + +def renew_tor_circuit(control_port: int = DEFAULT_TOR_CONTROL_PORT) -> None: + TorManager(control_port=control_port).renew_circuit() + + +def stop_tor_service(tor_process) -> None: + if tor_process and hasattr(tor_process, "kill"): + try: + if tor_process.poll() is None: + tor_process.kill() + except Exception: + pass diff --git a/crawler/url_utils.py b/crawler/url_utils.py new file mode 100644 index 0000000..3af992e --- /dev/null +++ b/crawler/url_utils.py @@ -0,0 +1,69 @@ +from typing import Optional, Tuple +from urllib.parse import urlparse + +# Non-HTML static file extensions to ignore +IGNORED_EXTENSIONS = ( + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico", ".bmp", ".tiff", + ".pdf", ".zip", ".tar", ".gz", ".bz2", ".7z", ".rar", ".exe", ".dmg", ".iso", ".bin", + ".mp3", ".mp4", ".avi", ".mov", ".mkv", ".flv", ".wmv", ".webm", ".wav", ".ogg", + ".css", ".js", ".json", ".xml", ".rss", ".atom", ".woff", ".woff2", ".ttf", ".eot", ".otf", + ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".csv" +) + +# WordPress / CMS feed and API paths that are not HTML web pages +IGNORED_PATHS = ( + "/feed", "/comments/feed", "/wp-json", "/xmlrpc.php", "/wp-admin", "/wp-login.php" +) + + +def normalize_url(url: str, start_domain: str) -> Tuple[Optional[str], Optional[bool]]: + """ + Normalizes a URL, strips fragments, filters out non-HTML files / endpoints, + and determines whether the link is internal or external. + + Returns: + (normalized_url, is_internal) or (None, None) if the link should be ignored. + """ + if not isinstance(url, str): + return None, None + + url = url.split("#")[0].strip() + if not url: + return None, None + + parsed = urlparse(url) + if parsed.scheme not in ["http", "https"]: + return None, None + + path = parsed.path or "/" + path_lower = path.lower() + + # Filter out static file extensions + if any(path_lower.endswith(ext) for ext in IGNORED_EXTENSIONS): + return None, None + + # Filter out non-HTML WordPress / CMS endpoints + if any(path_lower == p or path_lower.rstrip("/") == p or path_lower.startswith(p + "/") for p in IGNORED_PATHS): + return None, None + + netloc = parsed.netloc.lower() + if not netloc: + return None, None + + # Check domain matching (treat apex domain and www. as identical) + clean_start_domain = start_domain.lower().replace("www.", "") + clean_netloc = netloc.replace("www.", "") + + is_internal = (clean_netloc == clean_start_domain) + + # Strip query parameters for internal links to avoid infinite loops and duplicate crawls + if is_internal: + norm_path = path.rstrip("/") if path != "/" else "/" + norm_url = f"{parsed.scheme}://{netloc}{norm_path}" + else: + # Keep query parameters for external links (e.g., video IDs) + norm_path = path.rstrip("/") if path != "/" else "/" + query_part = f"?{parsed.query}" if parsed.query else "" + norm_url = f"{parsed.scheme}://{netloc}{norm_path}{query_part}" + + return norm_url, is_internal diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..bc43db3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +services: + websurfer: + build: + context: . + dockerfile: Dockerfile + container_name: websurfer + restart: unless-stopped + ports: + - "127.0.0.1:${WEB_PORT:-8080}:8000" + shm_size: "2gb" + environment: + - CRAWLER_HEADLESS=true + - HOST=0.0.0.0 + - PORT=8000 + volumes: + - ./.env:/app/.env diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..c480f51 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +echo "[Docker] Starting Tor background service..." +service tor start + +# Quick check that Tor SOCKS5 & Control ports are active +for i in {1..10}; do + if python3 -c "import socket; s = socket.socket(); s.connect(('127.0.0.1', 9050)); s.close()" 2>/dev/null; then + echo "[Docker] Tor SOCKS proxy is listening on port 9050!" + break + fi + sleep 1 +done + +# Execute main process (e.g. web server or CLI) +exec "$@" diff --git a/main.py b/main.py new file mode 100644 index 0000000..33edb78 --- /dev/null +++ b/main.py @@ -0,0 +1,35 @@ +import argparse +import asyncio +import os +import sys +from crawler import Crawler, CrawlerConfig + + +async def run_cli(): + config = CrawlerConfig.from_env_or_prompt() + crawler = Crawler(config) + await crawler.run() + + +def main(): + parser = argparse.ArgumentParser(description="Clickduck - Web Crawler & Link Explorer") + parser.add_argument("--web", action="store_true", help="Launch the Web Frontend dashboard") + parser.add_argument("--host", type=str, default=os.getenv("HOST", "127.0.0.1"), help="Host IP for web UI (default: 127.0.0.1 or $HOST)") + parser.add_argument("--port", type=int, default=int(os.getenv("PORT", "8000")), help="Port for web UI (default: 8000 or $PORT)") + args = parser.parse_args() + + if args.web: + from web import run_server + run_server(host=args.host, port=args.port) + else: + try: + asyncio.run(run_cli()) + except KeyboardInterrupt: + print("\nCrawling interrupted by user.") + sys.exit(0) + + +if __name__ == "__main__": + if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6488d31 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +playwright>=1.40.0 +python-dotenv +stem>=1.8.0 +fastapi>=0.110.0 +uvicorn[standard]>=0.28.0 diff --git a/test_iteration_visits.py b/test_iteration_visits.py new file mode 100644 index 0000000..8cef143 --- /dev/null +++ b/test_iteration_visits.py @@ -0,0 +1,372 @@ +import asyncio +import os +import unittest +from unittest.mock import MagicMock, AsyncMock, patch + +from crawler.config import CrawlerConfig +from crawler.engine import Crawler, IterationVisitTracker + + +class TestIterationVisitsConfig(unittest.TestCase): + + def test_default_config(self): + config = CrawlerConfig(start_url="https://example.com") + self.assertEqual(config.iteration_mode, "all") + self.assertEqual(config.max_visits_per_iteration, 10) + self.assertEqual(config.visits_per_iteration, 10) + self.assertTrue(config.randomize_queue) + + def test_dict_serialization(self): + config = CrawlerConfig( + start_url="https://example.com", + num_iterations=3, + iteration_mode="visits", + max_visits_per_iteration=25, + randomize_queue=False, + ) + d = config.to_dict() + self.assertEqual(d["iteration_mode"], "visits") + self.assertEqual(d["max_visits_per_iteration"], 25) + self.assertFalse(d["randomize_queue"]) + + # Restore from dict + restored = CrawlerConfig.from_dict(d) + self.assertEqual(restored.iteration_mode, "visits") + self.assertEqual(restored.max_visits_per_iteration, 25) + self.assertEqual(restored.visits_per_iteration, 25) + self.assertFalse(restored.randomize_queue) + + def test_from_dict_alias_and_normalization(self): + # Using visits_per_iteration alias + cfg1 = CrawlerConfig.from_dict({ + "start_url": "https://example.com", + "iteration_mode": "limit", + "visits_per_iteration": 15, + "randomize_queue": True, + }) + self.assertEqual(cfg1.iteration_mode, "visits") + self.assertEqual(cfg1.max_visits_per_iteration, 15) + self.assertTrue(cfg1.randomize_queue) + + # Non-visits mode defaults to "all" + cfg2 = CrawlerConfig.from_dict({ + "start_url": "https://example.com", + "iteration_mode": "unknown_mode", + "max_visits_per_iteration": -5, + }) + self.assertEqual(cfg2.iteration_mode, "all") + self.assertEqual(cfg2.max_visits_per_iteration, 10) + self.assertTrue(cfg2.randomize_queue) + + def test_env_dict_loading(self): + with patch.dict(os.environ, { + "CRAWLER_START_URL": "https://test.com", + "CRAWLER_ITERATION_MODE": "visits", + "CRAWLER_MAX_VISITS_PER_ITERATION": "42", + "CRAWLER_RANDOMIZE_QUEUE": "false", + }): + env_data = CrawlerConfig.get_env_dict(env_path="nonexistent_test.env") + self.assertEqual(env_data["iteration_mode"], "visits") + self.assertEqual(env_data["max_visits_per_iteration"], 42) + self.assertFalse(env_data["randomize_queue"]) + + +class TestIterationVisitTracker(unittest.IsolatedAsyncioTestCase): + + async def test_unlimited_tracker(self): + tracker = IterationVisitTracker(max_visits=None) + for _ in range(50): + can_visit = await tracker.acquire_visit_slot() + self.assertTrue(can_visit) + await tracker.record_visit_completed() + self.assertFalse(tracker.limit_reached_event.is_set()) + + async def test_limited_tracker_slot_reservation_and_completion(self): + limit = 3 + tracker = IterationVisitTracker(max_visits=limit) + + # Acquire 3 slots + self.assertTrue(await tracker.acquire_visit_slot()) + self.assertTrue(await tracker.acquire_visit_slot()) + self.assertTrue(await tracker.acquire_visit_slot()) + + # 4th acquisition attempt fails + self.assertFalse(await tracker.acquire_visit_slot()) + self.assertEqual(tracker.started_visits, 3) + self.assertFalse(tracker.limit_reached_event.is_set()) + + # Record completions + await tracker.record_visit_completed() + self.assertFalse(tracker.limit_reached_event.is_set()) + await tracker.record_visit_completed() + self.assertFalse(tracker.limit_reached_event.is_set()) + + # 3rd completion triggers limit_reached_event + await tracker.record_visit_completed() + self.assertTrue(tracker.limit_reached_event.is_set()) + self.assertEqual(tracker.completed_visits, 3) + + +class TestCrawlerIterationVisitCap(unittest.IsolatedAsyncioTestCase): + + async def test_iteration_stops_at_configured_visits(self): + config = CrawlerConfig( + start_url="https://example.com", + num_tabs=3, + num_iterations=1, + iteration_mode="visits", + max_visits_per_iteration=3, + ) + + visited_urls = [] + + async def mock_crawl_level( + page, + url, + source_url, + visited, + errors, + start_domain, + accept_cookies_default=True, + stats=None, + timeout_ms=35000, + event_callback=None, + worker_name="Worker", + page_result_callback=None, + ): + visited_urls.append(url) + visited.add(url) + await asyncio.sleep(0.01) + # Return new links to simulate queue populated with many items + return { + f"https://example.com/sub_{i}_{len(visited_urls)}" for i in range(5) + } + + crawler = Crawler(config=config) + + mock_page = MagicMock() + mock_page.close = AsyncMock() + mock_page.on = MagicMock() + + mock_context = MagicMock() + mock_context.new_page = AsyncMock(return_value=mock_page) + + mock_browser = MagicMock() + mock_browser.new_context = AsyncMock(return_value=mock_context) + mock_browser.close = AsyncMock() + + mock_playwright = MagicMock() + mock_playwright.chromium.launch = AsyncMock(return_value=mock_browser) + + with patch("crawler.engine.crawl_level", side_effect=mock_crawl_level): + result = await crawler.run_iteration(mock_playwright, iteration=1) + + # Verified that exactly max_visits_per_iteration (3) were crawled despite many discovered links + self.assertEqual(len(visited_urls), 3) + self.assertEqual(result["stats"]["internal"], 3) + + async def test_multi_iteration_stops_at_configured_visits_each(self): + config = CrawlerConfig( + start_url="https://example.com", + num_tabs=2, + num_iterations=2, + iteration_mode="visits", + max_visits_per_iteration=2, + ) + + visited_urls_per_iteration = [] + + async def mock_crawl_level( + page, + url, + source_url, + visited, + errors, + start_domain, + accept_cookies_default=True, + stats=None, + timeout_ms=35000, + event_callback=None, + worker_name="Worker", + page_result_callback=None, + ): + visited_urls_per_iteration.append(url) + visited.add(url) + await asyncio.sleep(0.01) + return {f"https://example.com/link_{i}" for i in range(5)} + + crawler = Crawler(config=config) + + mock_page = MagicMock() + mock_page.close = AsyncMock() + mock_page.on = MagicMock() + + mock_context = MagicMock() + mock_context.new_page = AsyncMock(return_value=mock_page) + + mock_browser = MagicMock() + mock_browser.new_context = AsyncMock(return_value=mock_context) + mock_browser.close = AsyncMock() + + mock_playwright = MagicMock() + mock_playwright.chromium.launch = AsyncMock(return_value=mock_browser) + + with patch("crawler.engine.crawl_level", side_effect=mock_crawl_level): + # Iteration 1 + res1 = await crawler.run_iteration(mock_playwright, iteration=1) + self.assertEqual(len(visited_urls_per_iteration), 2) + self.assertEqual(res1["stats"]["internal"], 2) + + visited_urls_per_iteration.clear() + + # Iteration 2 + res2 = await crawler.run_iteration(mock_playwright, iteration=2) + self.assertEqual(len(visited_urls_per_iteration), 2) + self.assertEqual(res2["stats"]["internal"], 2) + + +class TestRandomQueue(unittest.IsolatedAsyncioTestCase): + + async def test_random_queue_pops_all_items(self): + from crawler.engine import RandomQueue + + q = RandomQueue() + items = [f"https://example.com/page{i}" for i in range(20)] + for it in items: + q.put_nowait(it) + + self.assertEqual(q.qsize(), 20) + popped = [] + for _ in range(20): + popped.append(await q.get()) + q.task_done() + + await q.join() + self.assertEqual(sorted(popped), sorted(items)) + self.assertEqual(q.qsize(), 0) + + async def test_crawler_uses_random_queue_when_configured(self): + from crawler.engine import RandomQueue + + cfg_random = CrawlerConfig(start_url="https://example.com", randomize_queue=True) + crawler_random = Crawler(config=cfg_random) + + mock_page = MagicMock() + mock_page.close = AsyncMock() + mock_page.on = MagicMock() + mock_context = MagicMock() + mock_context.new_page = AsyncMock(return_value=mock_page) + mock_browser = MagicMock() + mock_browser.new_context = AsyncMock(return_value=mock_context) + mock_browser.close = AsyncMock() + mock_playwright = MagicMock() + mock_playwright.chromium.launch = AsyncMock(return_value=mock_browser) + + # Intercept queue creation by checking instance type + async def dummy_crawl(*args, **kwargs): + return set() + + with patch("crawler.engine.crawl_level", side_effect=dummy_crawl): + with patch("crawler.engine.RandomQueue", wraps=RandomQueue) as mock_rq: + await crawler_random.run_iteration(mock_playwright, iteration=1) + self.assertTrue(mock_rq.called) + + +class TestCumulativeAllIterationsStatistics(unittest.IsolatedAsyncioTestCase): + + async def test_cumulative_visited_and_errors_across_iterations(self): + config = CrawlerConfig( + start_url="https://example.com", + num_tabs=2, + num_iterations=2, + iteration_mode="visits", + max_visits_per_iteration=3, + ) + + events = [] + + async def event_cb(ev): + events.append(ev) + + crawler = Crawler(config=config, event_callback=event_cb) + + visited_per_call = 0 + + async def mock_crawl_level( + page, + url, + source_url, + visited, + errors, + start_domain, + accept_cookies_default=True, + stats=None, + timeout_ms=35000, + event_callback=None, + worker_name="Worker", + page_result_callback=None, + ): + nonlocal visited_per_call + visited_per_call += 1 + visited.add(url) + if visited_per_call == 2: + errors.append((url, source_url, "Simulated 500 error")) + await asyncio.sleep(0.01) + return {f"https://example.com/link_{visited_per_call}_{i}" for i in range(3)} + + mock_page = MagicMock() + mock_page.close = AsyncMock() + mock_page.on = MagicMock() + mock_context = MagicMock() + mock_context.new_page = AsyncMock(return_value=mock_page) + mock_browser = MagicMock() + mock_browser.new_context = AsyncMock(return_value=mock_context) + mock_browser.close = AsyncMock() + mock_playwright = MagicMock() + mock_playwright.chromium.launch = AsyncMock(return_value=mock_browser) + + with patch("crawler.engine.crawl_level", side_effect=mock_crawl_level): + res1 = await crawler.run_iteration(mock_playwright, iteration=1) + self.assertEqual(res1["visited_count"], 3) + self.assertEqual(res1["total_visited_all"], 3) + self.assertEqual(res1["total_errors_all"], 1) + + res2 = await crawler.run_iteration(mock_playwright, iteration=2) + self.assertEqual(res2["visited_count"], 3) + self.assertEqual(res2["total_visited_all"], 6) + self.assertEqual(crawler.total_visited_all_iterations, 6) + + # Check emitted stats events + stats_events = [e for e in events if e.get("type") == "stats"] + self.assertTrue(len(stats_events) > 0) + final_stats_event = stats_events[-1] + self.assertEqual(final_stats_event["total_visited_all"], 6) + + async def test_session_manager_snapshot_and_broadcast(self): + from web.manager import CrawlSessionManager + + manager = CrawlSessionManager() + snapshot = manager.get_snapshot() + self.assertIn("total_visited_all", snapshot) + self.assertIn("total_errors_all", snapshot) + self.assertEqual(snapshot["total_visited_all"], 0) + self.assertEqual(snapshot["total_errors_all"], 0) + + # Simulate receiving crawler stats event + await manager.handle_crawler_event({ + "type": "stats", + "visited_count": 5, + "total_visited_all": 15, + "total_errors_all": 2, + "queue_size": 3, + "stats": {"internal": 5, "external": 0, "cookies_accepted": 0}, + }) + self.assertEqual(manager.total_visited_all, 15) + self.assertEqual(manager.total_errors_all, 2) + snapshot_after = manager.get_snapshot() + self.assertEqual(snapshot_after["total_visited_all"], 15) + self.assertEqual(snapshot_after["total_errors_all"], 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_tor_fallback.py b/test_tor_fallback.py new file mode 100644 index 0000000..cb0c6b8 --- /dev/null +++ b/test_tor_fallback.py @@ -0,0 +1,178 @@ +import asyncio +import unittest +from unittest.mock import MagicMock, AsyncMock, patch + +from crawler.config import CrawlerConfig +from crawler.engine import Crawler + + +class TestTorTimeoutFallback(unittest.IsolatedAsyncioTestCase): + + def test_config_defaults_and_env(self): + config = CrawlerConfig(start_url="https://example.com") + self.assertEqual(config.tor_max_consecutive_timeouts, 5) + + dict_data = config.to_dict() + self.assertEqual(dict_data["tor_max_consecutive_timeouts"], 5) + + dict_data["tor_max_consecutive_timeouts"] = 3 + restored = CrawlerConfig.from_dict(dict_data) + self.assertEqual(restored.tor_max_consecutive_timeouts, 3) + + async def test_consecutive_timeout_fallback_trigger(self): + config = CrawlerConfig( + start_url="https://example.com", + use_tor=True, + tor_max_consecutive_timeouts=5, + ) + events = [] + async def on_event(ev): + events.append(ev) + + crawler = Crawler(config=config, event_callback=on_event) + # Mock tor_manager so we don't need real Tor daemon for unit testing + crawler.tor_manager = MagicMock() + crawler.tor_manager.renew_circuit = MagicMock(return_value=True) + + mock_context = MagicMock() + + # 1. First 4 consecutive ERR_TIMED_OUT errors + for i in range(1, 5): + await crawler._on_page_result( + context=mock_context, + url=f"https://example.com/page{i}", + source_url="https://example.com", + success=False, + error_msg="page.goto: net::ERR_TIMED_OUT at https://example.com", + ) + self.assertEqual(crawler._consecutive_timeouts, i) + self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 0) + + # 2. 5th ERR_TIMED_OUT error triggers fallback circuit rotation! + await crawler._on_page_result( + context=mock_context, + url="https://example.com/page5", + source_url="https://example.com", + success=False, + error_msg="page.goto: net::ERR_TIMED_OUT at https://example.com", + ) + # Counter should be reset to 0 after rotation + self.assertEqual(crawler._consecutive_timeouts, 0) + self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 1) + + async def test_success_resets_counter(self): + config = CrawlerConfig( + start_url="https://example.com", + use_tor=True, + tor_max_consecutive_timeouts=5, + ) + crawler = Crawler(config=config) + crawler.tor_manager = MagicMock() + crawler.tor_manager.renew_circuit = MagicMock(return_value=True) + + mock_context = MagicMock() + + # 3 timeouts + for i in range(1, 4): + await crawler._on_page_result( + context=mock_context, + url=f"https://example.com/page{i}", + source_url=None, + success=False, + error_msg="net::ERR_TIMED_OUT", + ) + self.assertEqual(crawler._consecutive_timeouts, 3) + + # 1 success -> should reset counter to 0 + await crawler._on_page_result( + context=mock_context, + url="https://example.com/success_page", + source_url=None, + success=True, + error_msg=None, + ) + self.assertEqual(crawler._consecutive_timeouts, 0) + self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 0) + + async def test_non_timeout_error_resets_counter(self): + config = CrawlerConfig( + start_url="https://example.com", + use_tor=True, + tor_max_consecutive_timeouts=5, + ) + crawler = Crawler(config=config) + crawler.tor_manager = MagicMock() + + mock_context = MagicMock() + + # 2 timeouts + for i in range(2): + await crawler._on_page_result( + context=mock_context, + url=f"https://example.com/page{i}", + source_url=None, + success=False, + error_msg="net::ERR_TIMED_OUT", + ) + self.assertEqual(crawler._consecutive_timeouts, 2) + + # HTTP 404 error -> non-timeout/non-rate-limit resets counter + await crawler._on_page_result( + context=mock_context, + url="https://example.com/missing", + source_url=None, + success=False, + error_msg="HTTP 404", + ) + self.assertEqual(crawler._consecutive_timeouts, 0) + + def test_user_agent_config(self): + config_random = CrawlerConfig(start_url="https://example.com", user_agent="random") + ua1 = config_random.get_user_agent() + self.assertIsNotNone(ua1) + self.assertIn("Mozilla/5.0", ua1) + + config_custom = CrawlerConfig(start_url="https://example.com", user_agent="MyCustomBot/1.0") + self.assertEqual(config_custom.get_user_agent(), "MyCustomBot/1.0") + + config_default = CrawlerConfig(start_url="https://example.com", user_agent="default") + self.assertIsNone(config_default.get_user_agent()) + + async def test_consecutive_rate_limit_fallback_trigger(self): + config = CrawlerConfig( + start_url="https://example.com", + use_tor=True, + tor_max_consecutive_timeouts=3, + ) + crawler = Crawler(config=config) + crawler.tor_manager = MagicMock() + crawler.tor_manager.renew_circuit = MagicMock(return_value=True) + + mock_context = MagicMock() + + # 2 HTTP 503 errors + for i in range(2): + await crawler._on_page_result( + context=mock_context, + url=f"https://example.com/page{i}", + source_url="https://example.com", + success=False, + error_msg="HTTP 503 (Service Temporarily Unavailable / Rate Limited) [Retry-After: 154s]", + ) + self.assertEqual(crawler._consecutive_timeouts, 2) + self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 0) + + # 3rd HTTP 503 error triggers circuit renewal + await crawler._on_page_result( + context=mock_context, + url="https://example.com/page2", + source_url="https://example.com", + success=False, + error_msg="HTTP 503 (Service Temporarily Unavailable / Rate Limited) [Retry-After: 154s]", + ) + self.assertEqual(crawler._consecutive_timeouts, 0) + self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/web.py b/web.py new file mode 100644 index 0000000..501596f --- /dev/null +++ b/web.py @@ -0,0 +1,18 @@ +import argparse +import asyncio +import os +import sys +from web import run_server + +if __name__ == "__main__": + if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + + parser = argparse.ArgumentParser(description="Websurfer Web Frontend Server") + parser.add_argument("--host", type=str, default=os.getenv("HOST", "127.0.0.1"), help="Host IP address (default: 127.0.0.1 or $HOST)") + parser.add_argument("--port", type=int, default=int(os.getenv("PORT", "8000")), help="Port number (default: 8000 or $PORT)") + parser.add_argument("--reload", action="store_true", help="Enable auto-reload for development") + args = parser.parse_args() + + run_server(host=args.host, port=args.port, reload=args.reload) + diff --git a/web/__init__.py b/web/__init__.py new file mode 100644 index 0000000..a29ccb1 --- /dev/null +++ b/web/__init__.py @@ -0,0 +1,26 @@ +"""Web application package for Websurfer.""" +from web.app import app +from web.manager import session_manager + + +def run_server(host: str = "127.0.0.1", port: int = 8000, reload: bool = False): + """Starts the Websurfer FastAPI Web Dashboard server.""" + import sys + import uvicorn + print("\n=======================================================") + print(f"[*] Websurfer Web UI running at http://{host}:{port}") + print(" Open this URL in your browser to configure & crawl.") + + # On Windows, Uvicorn defaults to SelectorEventLoop when reload=True or workers > 1. + # Playwright requires ProactorEventLoop on Windows for asyncio subprocess support. + loop_arg = "asyncio:ProactorEventLoop" if sys.platform == "win32" else "auto" + uvicorn.run( + "web.app:app" if reload else app, + host=host, + port=port, + reload=reload, + proxy_headers=True, + forwarded_allow_ips="*", + loop=loop_arg, + ) + diff --git a/web/app.py b/web/app.py new file mode 100644 index 0000000..8456cec --- /dev/null +++ b/web/app.py @@ -0,0 +1,218 @@ +import csv +import io +import os +from pathlib import Path +from typing import Dict, Any, Optional + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Depends, Request, Response, status +from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles + +from crawler.config import CrawlerConfig +from .manager import session_manager +from .auth import ( + is_auth_enabled, + verify_credentials, + create_session_token, + verify_session_token, + get_current_user, + verify_websocket, + SESSION_COOKIE_NAME, + SESSION_MAX_AGE_SECONDS, +) + +BASE_DIR = Path(__file__).resolve().parent +STATIC_DIR = BASE_DIR / "static" + +app = FastAPI(title="WebSurfer Control Center", version="1.0.0") + +# Mount static assets directory +app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") + + +@app.get("/", response_class=HTMLResponse) +async def root(request: Request): + """Serves the main dashboard user interface, redirecting to login if unauthenticated.""" + if is_auth_enabled(): + token = request.cookies.get(SESSION_COOKIE_NAME) + if not verify_session_token(token): + return RedirectResponse(url="/login", status_code=status.HTTP_307_TEMPORARY_REDIRECT) + + index_path = STATIC_DIR / "index.html" + if not index_path.exists(): + raise HTTPException(status_code=404, detail="Dashboard UI not found") + with open(index_path, "r", encoding="utf-8") as f: + return f.read() + + +@app.get("/login", response_class=HTMLResponse) +async def login_page(request: Request): + """Serves the login page, redirecting to / if already authenticated.""" + if is_auth_enabled(): + token = request.cookies.get(SESSION_COOKIE_NAME) + if verify_session_token(token): + return RedirectResponse(url="/", status_code=status.HTTP_307_TEMPORARY_REDIRECT) + + login_path = STATIC_DIR / "login.html" + if not login_path.exists(): + raise HTTPException(status_code=404, detail="Login UI not found") + with open(login_path, "r", encoding="utf-8") as f: + return f.read() + + +@app.post("/api/login") +async def api_login(payload: Dict[str, str], response: Response, request: Request): + """Authenticates credentials and establishes a signed session cookie.""" + if not is_auth_enabled(): + return {"success": True, "message": "Authentication not enabled"} + + username = payload.get("username", "") + password = payload.get("password", "") + + if not verify_credentials(username, password): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid username or password", + ) + + token = create_session_token(username) + is_secure = request.headers.get("x-forwarded-proto", "").lower() == "https" or request.url.scheme == "https" + + response.set_cookie( + key=SESSION_COOKIE_NAME, + value=token, + max_age=SESSION_MAX_AGE_SECONDS, + httponly=True, + samesite="lax", + secure=is_secure, + path="/", + ) + return {"success": True, "username": username} + + +@app.post("/api/logout") +async def api_logout(response: Response): + """Clears the session cookie.""" + response.delete_cookie(key=SESSION_COOKIE_NAME, path="/") + return {"success": True, "message": "Logged out successfully"} + + +@app.get("/api/auth/me") +async def auth_me(request: Request): + """Returns current authentication state.""" + enabled = is_auth_enabled() + if not enabled: + return {"authenticated": True, "auth_enabled": False, "username": "anonymous"} + + token = request.cookies.get(SESSION_COOKIE_NAME) + username = verify_session_token(token) + return { + "authenticated": username is not None, + "auth_enabled": True, + "username": username, + } + + +@app.get("/api/config", dependencies=[Depends(get_current_user)]) +async def get_config(): + """Fetches the current crawler configuration stored in .env or defaults.""" + env_config = CrawlerConfig.get_env_dict() + return {"success": True, "config": env_config} + + +@app.get("/api/tor/countries", dependencies=[Depends(get_current_user)]) +async def get_tor_countries(): + """Returns the list of supported European countries for Tor exit node selection.""" + from crawler.config import EUROPEAN_COUNTRIES + countries = [ + {"code": code, "name": info["name"], "flag": info["flag"]} + for code, info in EUROPEAN_COUNTRIES.items() + ] + return {"success": True, "countries": countries} + + +@app.post("/api/config", dependencies=[Depends(get_current_user)]) +async def save_config(payload: Dict[str, Any]): + """Persists updated configuration values to the .env file.""" + try: + CrawlerConfig.save_env_file(payload) + return {"success": True, "message": "Configuration successfully saved to .env"} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to save .env file: {e}") + + +@app.post("/api/crawl/start", dependencies=[Depends(get_current_user)]) +async def start_crawl(payload: Dict[str, Any]): + """Starts a new crawler run with given parameters.""" + result = await session_manager.start_crawl(payload) + if not result.get("success"): + raise HTTPException(status_code=400, detail=result.get("message")) + return result + + +@app.post("/api/crawl/stop", dependencies=[Depends(get_current_user)]) +async def stop_crawl(): + """Stops the currently running crawler job.""" + result = await session_manager.stop_crawl() + if not result.get("success"): + raise HTTPException(status_code=400, detail=result.get("message")) + return result + + +@app.get("/api/crawl/status", dependencies=[Depends(get_current_user)]) +async def get_status(): + """Returns the current state, metrics, errors, and recent logs.""" + return session_manager.get_snapshot() + + +@app.post("/api/crawl/clear-logs", dependencies=[Depends(get_current_user)]) +async def clear_logs(): + """Clears buffered in-memory logs.""" + session_manager.clear_logs() + return {"success": True} + + +@app.get("/api/crawl/export-errors", dependencies=[Depends(get_current_user)]) +async def export_errors(format: str = "csv"): + """Exports all recorded errors from current session as CSV or JSON.""" + errors = session_manager.errors + if format.lower() == "json": + return JSONResponse(content={"total_errors": len(errors), "errors": errors}) + + # Default to CSV export + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(["ID", "Failed URL", "Error Reason", "Found On (Source URL)", "Timestamp"]) + for item in errors: + writer.writerow([ + item.get("id", ""), + item.get("url", ""), + item.get("error", ""), + item.get("source_url", ""), + item.get("timestamp", ""), + ]) + + output.seek(0) + return StreamingResponse( + iter([output.getvalue()]), + media_type="text/csv", + headers={"Content-Disposition": "attachment; filename=websurfer_errors.csv"}, + ) + + +@app.websocket("/ws/stream") +async def websocket_endpoint(websocket: WebSocket): + """WebSocket stream for real-time log, metrics, and error updates.""" + if not verify_websocket(websocket): + await websocket.close(code=status.WS_1008_POLICY_VIOLATION) + return + + await session_manager.connect_ws(websocket) + try: + while True: + # Keep connection open and accept any control pings from client + data = await websocket.receive_text() + except WebSocketDisconnect: + session_manager.disconnect_ws(websocket) + except Exception: + session_manager.disconnect_ws(websocket) diff --git a/web/auth.py b/web/auth.py new file mode 100644 index 0000000..906f173 --- /dev/null +++ b/web/auth.py @@ -0,0 +1,121 @@ +import hashlib +import hmac +import os +import secrets +import time +from typing import Optional + +from fastapi import Depends, HTTPException, Request, WebSocket, status +from fastapi.responses import RedirectResponse + +SESSION_COOKIE_NAME = "websurfer_session" +SESSION_MAX_AGE_SECONDS = 7 * 24 * 3600 # 7 days + + +def is_auth_enabled() -> bool: + """Checks whether authentication is enabled via environment variables.""" + flag = os.getenv("AUTH_ENABLED", "").strip().lower() + has_credentials = bool(os.getenv("AUTH_USERNAME") and os.getenv("AUTH_PASSWORD")) + return flag in ("true", "1", "yes") and has_credentials + + +def get_secret_key() -> str: + """Returns the secret key used to HMAC-sign session cookies.""" + return os.getenv("AUTH_SECRET_KEY") or "websurfer-default-secret-key-please-change-in-prod" + + +def verify_credentials(username: str, password: str) -> bool: + """Verifies user credentials using constant-time string comparison.""" + expected_user = os.getenv("AUTH_USERNAME", "") + expected_pass = os.getenv("AUTH_PASSWORD", "") + + if not expected_user or not expected_pass: + return False + + user_match = secrets.compare_digest(username.strip(), expected_user.strip()) + pass_match = secrets.compare_digest(password, expected_pass) + return user_match and pass_match + + +def create_session_token(username: str) -> str: + """Generates a tamper-proof signed session token (username:timestamp:signature).""" + timestamp = str(int(time.time())) + payload = f"{username}:{timestamp}" + signature = hmac.new( + get_secret_key().encode("utf-8"), + payload.encode("utf-8"), + hashlib.sha256, + ).hexdigest() + return f"{payload}:{signature}" + + +def verify_session_token(token: Optional[str]) -> Optional[str]: + """Validates the signature and timestamp of a session token.""" + if not token or ":" not in token: + return None + + parts = token.split(":") + if len(parts) != 3: + return None + + username, timestamp_str, signature = parts + + try: + token_time = int(timestamp_str) + # Check token expiration + if time.time() - token_time > SESSION_MAX_AGE_SECONDS or token_time > time.time() + 300: + return None + except ValueError: + return None + + payload = f"{username}:{timestamp_str}" + expected_sig = hmac.new( + get_secret_key().encode("utf-8"), + payload.encode("utf-8"), + hashlib.sha256, + ).hexdigest() + + if not secrets.compare_digest(signature, expected_sig): + return None + + return username + + +async def get_current_user(request: Request) -> Optional[str]: + """FastAPI dependency: verifies user session from cookie or Bearer token.""" + if not is_auth_enabled(): + return "anonymous" + + # 1. Check session cookie + token = request.cookies.get(SESSION_COOKIE_NAME) + + # 2. Check Authorization Bearer header as fallback + if not token: + auth_header = request.headers.get("Authorization") + if auth_header and auth_header.startswith("Bearer "): + token = auth_header[7:].strip() + + username = verify_session_token(token) + if not username: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Authentication required", + headers={"WWW-Authenticate": "Bearer"}, + ) + + return username + + +def verify_websocket(websocket: WebSocket) -> bool: + """Validates WebSocket connection session from cookie or query parameter.""" + if not is_auth_enabled(): + return True + + # Check cookie from handshake request + token = websocket.cookies.get(SESSION_COOKIE_NAME) + + # Check query param as alternative (?token=...) + if not token: + token = websocket.query_params.get("token") + + return verify_session_token(token) is not None diff --git a/web/manager.py b/web/manager.py new file mode 100644 index 0000000..ef337d3 --- /dev/null +++ b/web/manager.py @@ -0,0 +1,302 @@ +import asyncio +import sys +import time +from typing import Dict, List, Optional, Set, Any +from fastapi import WebSocket + +from crawler.config import CrawlerConfig +from crawler.engine import Crawler + + +class CrawlSessionManager: + """Manages active crawl sessions, WebSocket broadcasting, metrics, and error aggregation.""" + + def __init__(self): + self.active_connections: Set[WebSocket] = set() + self.crawler: Optional[Crawler] = None + self.task: Optional[asyncio.Task] = None + self.status: str = "idle" # "idle" | "running" | "stopping" | "finished" | "error" + self.status_message: str = "Ready to start" + self.start_time: Optional[float] = None + self.end_time: Optional[float] = None + self.current_iteration: int = 1 + self.total_iterations: int = 1 + self.iteration_mode: str = "all" + self.max_visits_per_iteration: int = 10 + + self.stats: Dict[str, int] = { + "internal": 0, + "external": 0, + "cookies_accepted": 0, + } + self.queue_size: int = 0 + self.visited_count: int = 0 + self.total_visited_all: int = 0 + self.total_errors_all: int = 0 + self.tor_identity: Optional[Dict[str, Any]] = None + + self.errors: List[Dict[str, Any]] = [] + self.logs: List[Dict[str, Any]] = [] + self._log_counter: int = 0 + self._error_counter: int = 0 + + async def connect_ws(self, websocket: WebSocket): + """Registers a new WebSocket connection and transmits initial state snapshot.""" + await websocket.accept() + self.active_connections.add(websocket) + # Send current state snapshot immediately upon connection + snapshot = self.get_snapshot() + await websocket.send_json({ + "type": "init", + "data": snapshot, + }) + + def disconnect_ws(self, websocket: WebSocket): + """Removes a disconnected WebSocket.""" + self.active_connections.discard(websocket) + + async def broadcast(self, message: Dict[str, Any]): + """Dispatches an event payload to all connected WebSockets.""" + disconnected = set() + for ws in self.active_connections: + try: + await ws.send_json(message) + except Exception: + disconnected.add(ws) + + for ws in disconnected: + self.active_connections.discard(ws) + + def get_snapshot(self) -> Dict[str, Any]: + """Generates a complete snapshot of current crawler session.""" + elapsed = 0.0 + if self.start_time: + if self.end_time: + elapsed = self.end_time - self.start_time + else: + elapsed = time.time() - self.start_time + + return { + "status": self.status, + "status_message": self.status_message, + "elapsed_seconds": round(elapsed, 1), + "current_iteration": self.current_iteration, + "total_iterations": self.total_iterations, + "iteration_mode": self.iteration_mode, + "max_visits_per_iteration": self.max_visits_per_iteration, + "stats": self.stats, + "queue_size": self.queue_size, + "visited_count": self.visited_count, + "total_visited_all": self.total_visited_all, + "total_errors_all": max(self.total_errors_all, len(self.errors)), + "tor_identity": self.tor_identity, + "error_count": len(self.errors), + "errors": self.errors[-100:], # last 100 errors for quick load + "logs": self.logs[-500:], # last 500 logs for quick load + } + + async def handle_crawler_event(self, event: Dict[str, Any]): + """Processes events dispatched by the core crawler engine.""" + event_type = event.get("type") + + if event_type == "log": + self._log_counter += 1 + log_entry = { + "id": self._log_counter, + "message": event.get("message", ""), + "level": event.get("level", "info"), + "timestamp": event.get("timestamp", time.time()), + } + self.logs.append(log_entry) + if len(self.logs) > 2000: + self.logs = self.logs[-1500:] + await self.broadcast({"type": "log", "data": log_entry}) + + elif event_type == "stats": + self.stats = event.get("stats", self.stats) + self.queue_size = event.get("queue_size", self.queue_size) + self.visited_count = event.get("visited_count", self.visited_count) + if "total_visited_all" in event: + self.total_visited_all = event["total_visited_all"] + else: + self.total_visited_all = max(self.total_visited_all, self.visited_count) + + if "total_errors_all" in event: + self.total_errors_all = event["total_errors_all"] + else: + self.total_errors_all = len(self.errors) + + await self.broadcast({ + "type": "stats", + "data": { + "stats": self.stats, + "queue_size": self.queue_size, + "visited_count": self.visited_count, + "error_count": event.get("error_count", 0), + "total_visited_all": self.total_visited_all, + "total_errors_all": self.total_errors_all, + }, + }) + + elif event_type == "tor_identity": + self.tor_identity = event.get("data", {}) + await self.broadcast({ + "type": "tor_identity", + "data": self.tor_identity, + }) + + elif event_type == "error": + self._error_counter += 1 + error_entry = { + "id": self._error_counter, + "url": event.get("url", ""), + "source_url": event.get("source_url") or "Root / Direct Input", + "error": event.get("error", "Unknown error"), + "timestamp": event.get("timestamp", time.time()), + } + self.errors.append(error_entry) + self.total_errors_all = len(self.errors) + await self.broadcast({ + "type": "error", + "data": { + **error_entry, + "total_errors_all": self.total_errors_all, + }, + }) + + elif event_type == "iteration": + self.current_iteration = event.get("iteration", 1) + self.total_iterations = event.get("total", 1) + if "iteration_mode" in event: + self.iteration_mode = event["iteration_mode"] + if "max_visits_per_iteration" in event: + self.max_visits_per_iteration = event["max_visits_per_iteration"] + if "total_visited_all" in event: + self.total_visited_all = event["total_visited_all"] + if "total_errors_all" in event: + self.total_errors_all = event["total_errors_all"] + await self.broadcast({ + "type": "iteration", + "data": { + "current": self.current_iteration, + "total": self.total_iterations, + "iteration_mode": self.iteration_mode, + "max_visits_per_iteration": self.max_visits_per_iteration, + "total_visited_all": self.total_visited_all, + "total_errors_all": self.total_errors_all, + }, + }) + + elif event_type == "status": + self.status = event.get("status", self.status) + self.status_message = event.get("message", "") + if "total_visited_all" in event: + self.total_visited_all = event["total_visited_all"] + if "total_errors_all" in event: + self.total_errors_all = event["total_errors_all"] + if self.status in ["finished", "stopped", "error"]: + self.end_time = time.time() + + await self.broadcast({ + "type": "status", + "data": { + "status": self.status, + "message": self.status_message, + "elapsed": round((self.end_time or time.time()) - (self.start_time or time.time()), 1), + "total_visited_all": self.total_visited_all, + "total_errors_all": self.total_errors_all, + }, + }) + + async def start_crawl(self, config_data: Dict[str, Any]) -> Dict[str, Any]: + """Configures and initiates a new crawl in a background task.""" + if self.status == "running": + return {"success": False, "message": "Crawler is already running."} + + # Safeguard: Verify Windows event loop supports subprocesses for Playwright + if sys.platform == "win32": + loop = asyncio.get_running_loop() + if not isinstance(loop, getattr(asyncio, "ProactorEventLoop", ())): + err_msg = ( + "Current asyncio event loop does not support subprocesses on Windows " + "(ProactorEventLoop is required for Playwright). " + "Please launch Websurfer using 'python web.py' or ensure Uvicorn uses '--loop asyncio:ProactorEventLoop'." + ) + return {"success": False, "message": err_msg} + + # Clear session data + self.stats = {"internal": 0, "external": 0, "cookies_accepted": 0} + self.queue_size = 0 + self.visited_count = 0 + self.total_visited_all = 0 + self.total_errors_all = 0 + self.tor_identity = None + + self.errors = [] + self.logs = [] + self._log_counter = 0 + self._error_counter = 0 + self.start_time = time.time() + self.end_time = None + self.status = "running" + self.status_message = "Initializing crawler..." + self.current_iteration = 1 + self.total_iterations = int(config_data.get("num_iterations", 1)) + + # Build configuration object + config = CrawlerConfig.from_dict(config_data) + self.iteration_mode = config.iteration_mode + self.max_visits_per_iteration = config.max_visits_per_iteration + + # Instantiate crawler with event callback + self.crawler = Crawler(config, event_callback=self.handle_crawler_event) + + async def _run_task(): + try: + await self.crawler.run() + except Exception as e: + await self.handle_crawler_event({ + "type": "status", + "status": "error", + "message": f"Execution error: {e}", + }) + finally: + if self.status == "running": + self.status = "finished" + self.end_time = time.time() + await self.broadcast({ + "type": "status", + "data": { + "status": self.status, + "message": "Completed", + "elapsed": round(self.end_time - self.start_time, 1), + }, + }) + + self.task = asyncio.create_task(_run_task()) + await self.broadcast({"type": "init", "data": self.get_snapshot()}) + return {"success": True, "message": "Crawl started"} + + async def stop_crawl(self) -> Dict[str, Any]: + """Stops the active crawler job.""" + if self.status != "running" or not self.crawler: + return {"success": False, "message": "No active crawl to stop."} + + self.status = "stopping" + self.status_message = "Cancelling workers..." + await self.broadcast({ + "type": "status", + "data": {"status": "stopping", "message": "Cancelling workers..."}, + }) + + self.crawler.stop() + return {"success": True, "message": "Stop signal sent."} + + def clear_logs(self): + """Clears in-memory logs.""" + self.logs = [] + self._log_counter = 0 + + +# Global singleton manager instance +session_manager = CrawlSessionManager() diff --git a/web/static/app.js b/web/static/app.js new file mode 100644 index 0000000..54feeac --- /dev/null +++ b/web/static/app.js @@ -0,0 +1,1019 @@ +/** + * Websurfer Web Dashboard - Reactive Client Controller + */ + +class WebsurferDashboard { + constructor() { + this.ws = null; + this.wsReconnectInterval = 2000; + this.status = 'idle'; + this.elapsedSeconds = 0; + this.timerInterval = null; + this.errors = []; + this.logs = []; + + // DOM Elements Cache + this.elements = { + // Status & Header + wsStatus: document.getElementById('wsStatus'), + crawlStatusBadge: document.getElementById('crawlStatusBadge'), + elapsedTimer: document.getElementById('elapsedTimer'), + progressMessage: document.getElementById('progressMessage'), + iterationBadge: document.getElementById('iterationBadge'), + progressBar: document.getElementById('progressBar'), + + // Form Controls + configForm: document.getElementById('configForm'), + startUrl: document.getElementById('startUrl'), + numTabs: document.getElementById('numTabs'), + numTabsVal: document.getElementById('numTabsVal'), + numIterations: document.getElementById('numIterations'), + iterModeAll: document.getElementById('iterModeAll'), + iterModeVisits: document.getElementById('iterModeVisits'), + labelIterModeAll: document.getElementById('labelIterModeAll'), + labelIterModeVisits: document.getElementById('labelIterModeVisits'), + iterVisitsField: document.getElementById('iterVisitsField'), + maxVisitsPerIteration: document.getElementById('maxVisitsPerIteration'), + randomizeQueue: document.getElementById('randomizeQueue'), + headless: document.getElementById('headless'), + acceptCookies: document.getElementById('acceptCookies'), + useTor: document.getElementById('useTor'), + torOptionsGroup: document.getElementById('torOptionsGroup'), + torModeIteration: document.getElementById('torModeIteration'), + torModePages: document.getElementById('torModePages'), + labelTorModeIteration: document.getElementById('labelTorModeIteration'), + labelTorModePages: document.getElementById('labelTorModePages'), + torPagesField: document.getElementById('torPagesField'), + torCircuitPages: document.getElementById('torCircuitPages'), + torMaxConsecutiveTimeouts: document.getElementById('torMaxConsecutiveTimeouts'), + timeoutMs: document.getElementById('timeoutMs'), + slowMo: document.getElementById('slowMo'), + torSocksPort: document.getElementById('torSocksPort'), + torControlPort: document.getElementById('torControlPort'), + userAgentSelect: document.getElementById('userAgentSelect'), + userAgentCustom: document.getElementById('userAgentCustom'), + + // Buttons + btnStart: document.getElementById('btnStart'), + btnStop: document.getElementById('btnStop'), + btnSaveEnv: document.getElementById('btnSaveEnv'), + btnReset: document.getElementById('btnReset'), + + // European Exit Node Selector + countryChipsGrid: document.getElementById('countryChipsGrid'), + exitNodesCountBadge: document.getElementById('exitNodesCountBadge'), + exitNodesSummaryText: document.getElementById('exitNodesSummaryText'), + btnExitMajor: document.getElementById('btnExitMajor'), + btnExitAll: document.getElementById('btnExitAll'), + btnExitClear: document.getElementById('btnExitClear'), + + // Metric Tiles + statVisited: document.getElementById('statVisited'), + statInternal: document.getElementById('statInternal'), + statExternal: document.getElementById('statExternal'), + statCookies: document.getElementById('statCookies'), + statQueue: document.getElementById('statQueue'), + statErrors: document.getElementById('statErrors'), + statTotalVisitedAll: document.getElementById('statTotalVisitedAll'), + statTotalErrorsAll: document.getElementById('statTotalErrorsAll'), + + // Tor Identity Panel + torCard: document.getElementById('torCard'), + torStatusBadge: document.getElementById('torStatusBadge'), + torCircuitHint: document.getElementById('torCircuitHint'), + torIpVal: document.getElementById('torIpVal'), + torCountryVal: document.getElementById('torCountryVal'), + torCityVal: document.getElementById('torCityVal'), + torOrgVal: document.getElementById('torOrgVal'), + torStrategyVal: document.getElementById('torStrategyVal'), + torRotationsVal: document.getElementById('torRotationsVal'), + torFilterVal: document.getElementById('torFilterVal'), + + // Error Table + errorsTableBody: document.getElementById('errorsTableBody'), + errorCountBadge: document.getElementById('errorCountBadge'), + errorSearchInput: document.getElementById('errorSearchInput'), + btnExportCsv: document.getElementById('btnExportCsv'), + btnExportJson: document.getElementById('btnExportJson'), + + // Log Terminal + logOutput: document.getElementById('logOutput'), + terminalContainer: document.getElementById('terminalContainer'), + logCountBadge: document.getElementById('logCountBadge'), + autoScrollCheck: document.getElementById('autoScrollCheck'), + logLevelSelect: document.getElementById('logLevelSelect'), + logSearchInput: document.getElementById('logSearchInput'), + btnClearLogs: document.getElementById('btnClearLogs'), + btnCopyLogs: document.getElementById('btnCopyLogs'), + + // Toasts + toastContainer: document.getElementById('toastContainer'), + }; + + this.selectedExitNodes = new Set(); + this.availableCountries = []; + + this.init(); + } + + async init() { + await this.checkAuth(); + this.bindEvents(); + await this.loadCountries(); + await this.loadConfig(); + this.connectWebSocket(); + } + + async checkAuth() { + try { + const res = await fetch('/api/auth/me'); + if (res.status === 401) { + window.location.href = '/login'; + return; + } + const data = await res.json(); + if (!data.authenticated && data.auth_enabled) { + window.location.href = '/login'; + return; + } + if (data.auth_enabled && data.username) { + const profile = document.getElementById('userProfile'); + const nameSpan = document.getElementById('userName'); + const btnLogout = document.getElementById('btnLogout'); + if (profile && nameSpan) { + nameSpan.textContent = data.username; + profile.style.display = 'flex'; + } + if (btnLogout) { + btnLogout.addEventListener('click', async () => { + await fetch('/api/logout', { method: 'POST' }); + window.location.href = '/login'; + }); + } + } + } catch (e) { + console.warn('Auth check warning:', e); + } + } + + bindEvents() { + // Sync slider value badge + this.elements.numTabs.addEventListener('input', (e) => { + this.elements.numTabsVal.textContent = e.target.value; + }); + + // Form submission (Start Crawling) + this.elements.configForm.addEventListener('submit', (e) => { + e.preventDefault(); + this.startCrawl(); + }); + + // Stop button + this.elements.btnStop.addEventListener('click', () => { + this.stopCrawl(); + }); + + // Save to .env button + this.elements.btnSaveEnv.addEventListener('click', () => { + this.saveConfigToEnv(); + }); + + // Reset button + this.elements.btnReset.addEventListener('click', () => { + this.loadConfig(); + this.showToast('Form reset to saved .env settings', 'info'); + }); + + // Use Tor toggle change + this.elements.useTor.addEventListener('change', () => { + this.updateTorVisibility(); + if (!this.elements.useTor.checked) { + this.updateTorIdentity(null); + } else { + this.elements.torStatusBadge.className = 'tor-pill tor-on'; + this.elements.torStatusBadge.textContent = 'Tor Enabled (Pending Run)'; + } + }); + + // Tor circuit mode radio changes + const onTorModeChange = () => { + this.updateTorModeUI(); + }; + if (this.elements.torModeIteration) { + this.elements.torModeIteration.addEventListener('change', onTorModeChange); + } + if (this.elements.torModePages) { + this.elements.torModePages.addEventListener('change', onTorModeChange); + } + if (this.elements.torCircuitPages) { + this.elements.torCircuitPages.addEventListener('input', () => { + this.updateTorModeUI(); + }); + } + + // Iteration mode radio changes + const onIterModeChange = () => { + this.updateIterationModeUI(); + }; + if (this.elements.iterModeAll) { + this.elements.iterModeAll.addEventListener('change', onIterModeChange); + } + if (this.elements.iterModeVisits) { + this.elements.iterModeVisits.addEventListener('change', onIterModeChange); + } + + // User-Agent Selector Toggle + if (this.elements.userAgentSelect) { + this.elements.userAgentSelect.addEventListener('change', (e) => { + const val = e.target.value; + if (val === 'custom') { + this.elements.userAgentCustom.style.display = 'block'; + this.elements.userAgentCustom.focus(); + } else { + this.elements.userAgentCustom.style.display = 'none'; + this.elements.userAgentCustom.value = val; + } + }); + } + + // Exit Node Action Presets + if (this.elements.btnExitMajor) { + this.elements.btnExitMajor.addEventListener('click', () => { + this.selectedExitNodes = new Set(['de', 'nl', 'fr', 'ch']); + this.updateExitNodesUI(); + }); + } + if (this.elements.btnExitAll) { + this.elements.btnExitAll.addEventListener('click', () => { + this.selectedExitNodes = new Set(this.availableCountries.map(c => c.code)); + this.updateExitNodesUI(); + }); + } + if (this.elements.btnExitClear) { + this.elements.btnExitClear.addEventListener('click', () => { + this.selectedExitNodes.clear(); + this.updateExitNodesUI(); + }); + } + + // Error table search/filter + this.elements.errorSearchInput.addEventListener('input', () => { + this.renderErrorsTable(); + }); + + // Error export buttons + this.elements.btnExportCsv.addEventListener('click', () => { + window.location.href = '/api/crawl/export-errors?format=csv'; + }); + + this.elements.btnExportJson.addEventListener('click', () => { + window.open('/api/crawl/export-errors?format=json', '_blank'); + }); + + // Log terminal controls + this.elements.logLevelSelect.addEventListener('change', () => this.filterLogs()); + this.elements.logSearchInput.addEventListener('input', () => this.filterLogs()); + + this.elements.btnClearLogs.addEventListener('click', async () => { + this.logs = []; + this.elements.logOutput.innerHTML = ''; + this.updateLogCount(); + try { + await fetch('/api/crawl/clear-logs', { method: 'POST' }); + } catch (err) { + console.error('Failed to clear server logs:', err); + } + }); + + this.elements.btnCopyLogs.addEventListener('click', () => { + const allText = this.logs.map(l => `[${this.formatTime(l.timestamp)}] [${l.level.toUpperCase()}] ${l.message}`).join('\n'); + navigator.clipboard.writeText(allText).then(() => { + this.showToast('All logs copied to clipboard!', 'success'); + }); + }); + } + + // --- WebSocket Connection --- + connectWebSocket() { + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const wsUrl = `${protocol}//${window.location.host}/ws/stream`; + + this.updateWsStatus('connecting'); + + try { + this.ws = new WebSocket(wsUrl); + + this.ws.onopen = () => { + this.updateWsStatus('connected'); + }; + + this.ws.onmessage = (event) => { + try { + const payload = JSON.parse(event.data); + this.handleEvent(payload); + } catch (err) { + console.error('Error parsing WebSocket message:', err); + } + }; + + this.ws.onclose = () => { + this.updateWsStatus('disconnected'); + setTimeout(() => this.connectWebSocket(), this.wsReconnectInterval); + }; + + this.ws.onerror = (error) => { + console.error('WebSocket Error:', error); + this.ws.close(); + }; + } catch (e) { + this.updateWsStatus('disconnected'); + setTimeout(() => this.connectWebSocket(), this.wsReconnectInterval); + } + } + + updateWsStatus(status) { + const pill = this.elements.wsStatus; + pill.className = `ws-pill ${status}`; + const textSpan = pill.querySelector('.ws-text'); + if (status === 'connected') { + textSpan.textContent = 'Live Connected'; + } else if (status === 'connecting') { + textSpan.textContent = 'Connecting...'; + } else { + textSpan.textContent = 'Disconnected'; + } + } + + // --- Event Handling --- + handleEvent(event) { + const { type, data } = event; + + if (type === 'init') { + this.hydrateSnapshot(data); + } else if (type === 'status') { + this.updateStatus(data.status, data.message, data.elapsed); + } else if (type === 'stats') { + this.updateStats(data); + } else if (type === 'tor_identity') { + this.updateTorIdentity(data); + } else if (type === 'error') { + this.addError(data); + } else if (type === 'log') { + this.addLog(data); + } else if (type === 'iteration') { + const iterText = `Iteration ${data.current} / ${data.total}`; + const modeText = data.iteration_mode === 'visits' && data.max_visits_per_iteration ? ` (${data.max_visits_per_iteration} visits/iter)` : ''; + this.elements.iterationBadge.textContent = `${iterText}${modeText}`; + } + } + + hydrateSnapshot(data) { + this.updateStatus(data.status, data.status_message, data.elapsed_seconds); + this.updateStats({ + stats: data.stats, + queue_size: data.queue_size, + visited_count: data.visited_count, + error_count: data.error_count, + total_visited_all: data.total_visited_all !== undefined ? data.total_visited_all : (data.visited_count || 0), + total_errors_all: data.total_errors_all !== undefined ? data.total_errors_all : (data.error_count || (data.errors ? data.errors.length : 0)), + }); + const iterText = `Iteration ${data.current_iteration || 1} / ${data.total_iterations || 1}`; + const modeText = data.iteration_mode === 'visits' && data.max_visits_per_iteration ? ` (${data.max_visits_per_iteration} visits/iter)` : ''; + this.elements.iterationBadge.textContent = `${iterText}${modeText}`; + + if (data.tor_identity) { + this.updateTorIdentity(data.tor_identity); + } else { + this.updateTorIdentity(null); + } + + if (data.errors) { + this.errors = data.errors; + this.renderErrorsTable(); + } + + if (data.logs) { + this.logs = data.logs; + this.elements.logOutput.innerHTML = ''; + this.logs.forEach(log => this.renderLogLine(log)); + this.updateLogCount(); + } + } + + updateTorVisibility() { + const isTor = this.elements.useTor.checked; + if (this.elements.torOptionsGroup) { + this.elements.torOptionsGroup.style.display = isTor ? 'block' : 'none'; + } + this.updateTorModeUI(); + this.updateIterationModeUI(); + this.updateExitNodesUI(); + } + + updateIterationModeUI() { + const isVisits = this.elements.iterModeVisits && this.elements.iterModeVisits.checked; + if (this.elements.labelIterModeAll) { + this.elements.labelIterModeAll.classList.toggle('active', !isVisits); + } + if (this.elements.labelIterModeVisits) { + this.elements.labelIterModeVisits.classList.toggle('active', isVisits); + } + if (this.elements.iterVisitsField) { + this.elements.iterVisitsField.style.display = isVisits ? 'block' : 'none'; + } + } + + updateTorModeUI() { + const isPages = this.elements.torModePages && this.elements.torModePages.checked; + if (this.elements.labelTorModeIteration) { + this.elements.labelTorModeIteration.classList.toggle('active', !isPages); + } + if (this.elements.labelTorModePages) { + this.elements.labelTorModePages.classList.toggle('active', isPages); + } + if (this.elements.torPagesField) { + this.elements.torPagesField.style.display = isPages ? 'block' : 'none'; + } + if (this.elements.torStrategyVal) { + if (this.elements.useTor.checked) { + const pages = this.elements.torCircuitPages ? this.elements.torCircuitPages.value || 10 : 10; + this.elements.torStrategyVal.textContent = isPages ? `Every ${pages} Pages` : 'Per Iteration'; + } else { + this.elements.torStrategyVal.textContent = 'Disabled'; + } + } + } + + async loadCountries() { + try { + const res = await fetch('/api/tor/countries'); + const data = await res.json(); + if (data.success && data.countries && data.countries.length > 0) { + this.availableCountries = data.countries; + } else { + this.availableCountries = this.getDefaultCountries(); + } + } catch (e) { + this.availableCountries = this.getDefaultCountries(); + } + this.renderCountryChips(); + } + + getDefaultCountries() { + return [ + { code: 'de', name: 'Germany', flag: '🇩🇪' }, + { code: 'nl', name: 'Netherlands', flag: '🇳🇱' }, + { code: 'fr', name: 'France', flag: '🇫🇷' }, + { code: 'ch', name: 'Switzerland', flag: '🇨🇭' }, + { code: 'at', name: 'Austria', flag: '🇦🇹' }, + { code: 'se', name: 'Sweden', flag: '🇸🇪' }, + { code: 'no', name: 'Norway', flag: '🇳🇴' }, + { code: 'fi', name: 'Finland', flag: '🇫🇮' }, + { code: 'dk', name: 'Denmark', flag: '🇩🇰' }, + { code: 'gb', name: 'United Kingdom', flag: '🇬🇧' }, + { code: 'pl', name: 'Poland', flag: '🇵🇱' }, + { code: 'it', name: 'Italy', flag: '🇮🇹' }, + { code: 'es', name: 'Spain', flag: '🇪🇸' }, + { code: 'be', name: 'Belgium', flag: '🇧🇪' }, + { code: 'cz', name: 'Czech Republic', flag: '🇨🇿' }, + { code: 'ro', name: 'Romania', flag: '🇷🇴' }, + { code: 'ie', name: 'Ireland', flag: '🇮🇪' }, + { code: 'pt', name: 'Portugal', flag: '🇵🇹' }, + { code: 'hu', name: 'Hungary', flag: '🇭🇺' }, + { code: 'is', name: 'Iceland', flag: '🇮🇸' }, + { code: 'lu', name: 'Luxembourg', flag: '🇱🇺' }, + ]; + } + + renderCountryChips() { + const grid = this.elements.countryChipsGrid; + if (!grid) return; + grid.innerHTML = ''; + + this.availableCountries.forEach(c => { + const isSelected = this.selectedExitNodes.has(c.code.toLowerCase()); + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = `country-chip ${isSelected ? 'active' : ''}`; + btn.dataset.code = c.code.toLowerCase(); + btn.innerHTML = ` + ${c.flag} + ${c.name} + ${c.code.toUpperCase()} + `; + + btn.addEventListener('click', () => { + const code = c.code.toLowerCase(); + if (this.selectedExitNodes.has(code)) { + this.selectedExitNodes.delete(code); + } else { + this.selectedExitNodes.add(code); + } + this.updateExitNodesUI(); + }); + + grid.appendChild(btn); + }); + + this.updateExitNodesUI(); + } + + updateExitNodesUI() { + const count = this.selectedExitNodes.size; + const chips = this.elements.countryChipsGrid ? this.elements.countryChipsGrid.querySelectorAll('.country-chip') : []; + + chips.forEach(chip => { + const code = chip.dataset.code; + chip.classList.toggle('active', this.selectedExitNodes.has(code)); + }); + + if (this.elements.exitNodesCountBadge) { + if (count === 0) { + this.elements.exitNodesCountBadge.textContent = 'Any / Global'; + this.elements.exitNodesCountBadge.className = 'count-pill-sm'; + } else { + this.elements.exitNodesCountBadge.textContent = `${count} Selected`; + this.elements.exitNodesCountBadge.className = 'count-pill-sm active'; + } + } + + if (this.elements.exitNodesSummaryText) { + if (count === 0) { + this.elements.exitNodesSummaryText.textContent = 'Active Exit Filter: Worldwide / Any Tor exit node (Global)'; + } else { + const countryObjs = Array.from(this.selectedExitNodes).map(code => { + const found = this.availableCountries.find(c => c.code.toLowerCase() === code); + return found ? `${found.flag} ${found.name}` : code.toUpperCase(); + }); + if (countryObjs.length <= 3) { + this.elements.exitNodesSummaryText.textContent = `Active Exit Filter: ${countryObjs.join(', ')}`; + } else { + this.elements.exitNodesSummaryText.textContent = `Active Exit Filter: ${countryObjs.slice(0, 3).join(', ')} + ${countryObjs.length - 3} more`; + } + } + } + + if (this.elements.torFilterVal) { + if (!this.elements.useTor.checked) { + this.elements.torFilterVal.textContent = 'Disabled'; + } else if (count === 0) { + this.elements.torFilterVal.textContent = 'Global (Any)'; + } else { + const displayCodes = Array.from(this.selectedExitNodes).map(c => c.toUpperCase()); + this.elements.torFilterVal.textContent = displayCodes.length <= 4 ? displayCodes.join(', ') : `${displayCodes.slice(0, 4).join(', ')} (+${displayCodes.length - 4})`; + } + } + } + + updateTorIdentity(torData) { + const isTorEnabled = this.elements.useTor.checked; + this.updateTorModeUI(); + this.updateExitNodesUI(); + if (!torData || !torData.ip || torData.ip === 'Unknown') { + this.elements.torCard.classList.remove('active'); + this.elements.torStatusBadge.className = isTorEnabled ? 'tor-pill tor-on' : 'tor-pill tor-off'; + this.elements.torStatusBadge.textContent = isTorEnabled ? 'Tor Enabled (Awaiting Run)' : 'Tor Inactive'; + this.elements.torIpVal.textContent = '--'; + this.elements.torCountryVal.textContent = '--'; + this.elements.torCityVal.textContent = '--'; + this.elements.torOrgVal.textContent = '--'; + if (this.elements.torRotationsVal) this.elements.torRotationsVal.textContent = '0'; + if (this.elements.torFilterVal) this.elements.torFilterVal.textContent = isTorEnabled ? (this.selectedExitNodes.size > 0 ? Array.from(this.selectedExitNodes).map(c=>c.toUpperCase()).join(', ') : 'Global (Any)') : 'Disabled'; + return; + } + + this.elements.torCard.classList.add('active'); + this.elements.torStatusBadge.className = 'tor-pill tor-on'; + this.elements.torStatusBadge.textContent = 'Tor Routing Active'; + + this.elements.torIpVal.textContent = torData.ip || '--'; + + const countryStr = torData.country_code ? `${torData.country || ''} (${torData.country_code})` : (torData.country || '--'); + this.elements.torCountryVal.textContent = countryStr; + + const cityRegion = [torData.city, torData.region].filter(x => x && x !== 'Unknown').join(', ') || '--'; + this.elements.torCityVal.textContent = cityRegion; + + this.elements.torOrgVal.textContent = torData.org || '--'; + + if (this.elements.torRotationsVal) { + this.elements.torRotationsVal.textContent = torData.rotation_count !== undefined ? torData.rotation_count : '1'; + } + + if (this.elements.torFilterVal) { + if (torData.exit_nodes && torData.exit_nodes.length > 0) { + const codes = torData.exit_nodes.map(c => c.toUpperCase()); + this.elements.torFilterVal.textContent = codes.length <= 4 ? codes.join(', ') : `${codes.slice(0, 4).join(', ')} (+${codes.length - 4})`; + } else if (this.selectedExitNodes.size > 0) { + const codes = Array.from(this.selectedExitNodes).map(c => c.toUpperCase()); + this.elements.torFilterVal.textContent = codes.length <= 4 ? codes.join(', ') : `${codes.slice(0, 4).join(', ')} (+${codes.length - 4})`; + } else { + this.elements.torFilterVal.textContent = 'Global (Any)'; + } + } + } + + + updateStatus(status, message, elapsed) { + this.status = status; + const badge = this.elements.crawlStatusBadge; + const label = badge.querySelector('.status-label'); + + badge.className = `status-badge ${status}`; + label.textContent = status.toUpperCase(); + + if (message) { + this.elements.progressMessage.textContent = message; + } + + if (status === 'running') { + this.elements.btnStart.disabled = true; + this.elements.btnStop.disabled = false; + this.elements.progressBar.className = 'progress-bar-fill running'; + this.startTimer(elapsed || 0); + } else { + this.elements.btnStart.disabled = false; + this.elements.btnStop.disabled = true; + this.elements.progressBar.className = 'progress-bar-fill'; + this.stopTimer(); + if (elapsed !== undefined) { + this.elements.elapsedTimer.textContent = this.formatDuration(elapsed); + } + } + } + + updateStats(data) { + const stats = data.stats || {}; + this.elements.statVisited.textContent = data.visited_count !== undefined ? data.visited_count : 0; + this.elements.statInternal.textContent = stats.internal || 0; + this.elements.statExternal.textContent = stats.external || 0; + this.elements.statCookies.textContent = stats.cookies_accepted || 0; + this.elements.statQueue.textContent = data.queue_size || 0; + + const errorCount = data.error_count !== undefined ? data.error_count : (data.total_errors_all !== undefined ? data.total_errors_all : this.errors.length); + this.elements.statErrors.textContent = errorCount; + + const totalVisitedAll = data.total_visited_all !== undefined ? data.total_visited_all : (data.visited_count || 0); + const totalErrorsAll = data.total_errors_all !== undefined ? data.total_errors_all : this.errors.length; + + this.elements.errorCountBadge.textContent = totalErrorsAll; + + if (this.elements.statTotalVisitedAll) { + this.elements.statTotalVisitedAll.textContent = totalVisitedAll; + } + if (this.elements.statTotalErrorsAll) { + this.elements.statTotalErrorsAll.textContent = totalErrorsAll; + } + } + + // --- Timer Management --- + startTimer(initialElapsed = 0) { + this.stopTimer(); + this.elapsedSeconds = Math.floor(initialElapsed); + this.elements.elapsedTimer.textContent = this.formatDuration(this.elapsedSeconds); + + this.timerInterval = setInterval(() => { + this.elapsedSeconds++; + this.elements.elapsedTimer.textContent = this.formatDuration(this.elapsedSeconds); + }, 1000); + } + + stopTimer() { + if (this.timerInterval) { + clearInterval(this.timerInterval); + this.timerInterval = null; + } + } + + formatDuration(sec) { + const s = Math.floor(sec % 60); + const m = Math.floor((sec / 60) % 60); + const h = Math.floor(sec / 3600); + return `${h.toString().padStart(2, '0')}:${m.toString().padStart(2, '0')}:${s.toString().padStart(2, '0')}`; + } + + formatTime(timestamp) { + const date = timestamp ? new Date(timestamp * 1000) : new Date(); + return date.toTimeString().split(' ')[0]; + } + + // --- Dedicated Error Section --- + addError(errorEntry) { + this.errors.push(errorEntry); + this.renderErrorsTable(); + const totalErr = errorEntry.total_errors_all !== undefined ? errorEntry.total_errors_all : this.errors.length; + this.elements.statErrors.textContent = this.elements.statErrors.textContent || totalErr; + this.elements.errorCountBadge.textContent = totalErr; + if (this.elements.statTotalErrorsAll) { + this.elements.statTotalErrorsAll.textContent = totalErr; + } + } + + renderErrorsTable() { + const filterQuery = (this.elements.errorSearchInput.value || '').toLowerCase().trim(); + const filtered = this.errors.filter(e => { + if (!filterQuery) return true; + return ( + (e.url && e.url.toLowerCase().includes(filterQuery)) || + (e.error && e.error.toLowerCase().includes(filterQuery)) || + (e.source_url && e.source_url.toLowerCase().includes(filterQuery)) + ); + }); + + const tbody = this.elements.errorsTableBody; + if (filtered.length === 0) { + tbody.innerHTML = ` + + +
+ ✨ +

${filterQuery ? 'No matching errors found for search filter.' : 'No website errors recorded yet. Happy crawling!'}

+
+ + + `; + return; + } + + tbody.innerHTML = filtered.map((e, index) => ` + + ${index + 1} + + + ${this.escapeHtml(e.url)} + + + + ${this.escapeHtml(e.error)} + + ${this.escapeHtml(e.source_url || 'Direct Input')} + ${this.formatTime(e.timestamp)} + + + + + `).join(''); + } + + // --- Live Log Console --- + addLog(logEntry) { + this.logs.push(logEntry); + if (this.logs.length > 2000) { + this.logs.shift(); + } + this.renderLogLine(logEntry); + this.updateLogCount(); + } + + renderLogLine(logEntry) { + const activeLevel = this.elements.logLevelSelect.value; + const filterQuery = (this.elements.logSearchInput.value || '').toLowerCase(); + + if (activeLevel !== 'all' && logEntry.level !== activeLevel) { + return; + } + if (filterQuery && !logEntry.message.toLowerCase().includes(filterQuery)) { + return; + } + + const lineDiv = document.createElement('div'); + lineDiv.className = `log-line log-${logEntry.level}`; + + let tagClass = 'tag-sys'; + let tagText = '[INFO]'; + if (logEntry.level === 'error') { tagClass = 'tag-err'; tagText = '[ERROR]'; } + else if (logEntry.level === 'warn') { tagClass = 'tag-warn'; tagText = '[WARN]'; } + else if (logEntry.level === 'success') { tagClass = 'tag-ok'; tagText = '[OK]'; } + + lineDiv.innerHTML = ` + [${this.formatTime(logEntry.timestamp)}] + ${this.escapeHtml(logEntry.message)} + `; + + this.elements.logOutput.appendChild(lineDiv); + + if (this.elements.autoScrollCheck.checked) { + this.elements.terminalContainer.scrollTop = this.elements.terminalContainer.scrollHeight; + } + } + + filterLogs() { + this.elements.logOutput.innerHTML = ''; + this.logs.forEach(log => this.renderLogLine(log)); + } + + updateLogCount() { + this.elements.logCountBadge.textContent = `${this.logs.length} lines`; + } + + // --- API Calls --- + async loadConfig() { + try { + const res = await fetch('/api/config'); + const data = await res.json(); + if (data.success && data.config) { + const c = data.config; + this.elements.startUrl.value = c.start_url || ''; + this.elements.numTabs.value = c.num_tabs || 5; + this.elements.numTabsVal.textContent = c.num_tabs || 5; + this.elements.numIterations.value = c.num_iterations || 1; + this.elements.headless.checked = !!c.headless; + this.elements.acceptCookies.checked = c.accept_cookies !== false; + if (this.elements.randomizeQueue) { + this.elements.randomizeQueue.checked = c.randomize_queue !== false; + } + this.elements.useTor.checked = !!c.use_tor; + + const isVisitsMode = (c.iteration_mode === 'visits'); + if (this.elements.iterModeVisits && this.elements.iterModeAll) { + this.elements.iterModeVisits.checked = isVisitsMode; + this.elements.iterModeAll.checked = !isVisitsMode; + } + if (this.elements.maxVisitsPerIteration) { + this.elements.maxVisitsPerIteration.value = c.max_visits_per_iteration || 10; + } + this.updateIterationModeUI(); + + const isPagesMode = (c.tor_circuit_mode === 'pages'); + if (this.elements.torModePages && this.elements.torModeIteration) { + this.elements.torModePages.checked = isPagesMode; + this.elements.torModeIteration.checked = !isPagesMode; + } + if (this.elements.torCircuitPages) { + this.elements.torCircuitPages.value = c.tor_circuit_pages || 10; + } + if (this.elements.torMaxConsecutiveTimeouts) { + this.elements.torMaxConsecutiveTimeouts.value = c.tor_max_consecutive_timeouts || 5; + } + + const exitNodes = Array.isArray(c.tor_exit_nodes) ? c.tor_exit_nodes : []; + this.selectedExitNodes = new Set(exitNodes.map(x => String(x).toLowerCase().trim())); + + this.elements.timeoutMs.value = c.timeout_ms || 35000; + this.elements.slowMo.value = c.slow_mo || 100; + this.elements.torSocksPort.value = c.tor_socks_port || 9050; + this.elements.torControlPort.value = c.tor_control_port || 9051; + + // User-Agent Population + const ua = (c.user_agent || 'random').trim(); + if (this.elements.userAgentSelect) { + if (ua === 'random' || ua === 'default') { + this.elements.userAgentSelect.value = ua; + if (this.elements.userAgentCustom) { + this.elements.userAgentCustom.style.display = 'none'; + this.elements.userAgentCustom.value = ua; + } + } else { + this.elements.userAgentSelect.value = 'custom'; + if (this.elements.userAgentCustom) { + this.elements.userAgentCustom.style.display = 'block'; + this.elements.userAgentCustom.value = ua; + } + } + } + + this.updateTorVisibility(); + } + } catch (err) { + console.error('Failed to load config:', err); + this.showToast('Could not load .env configuration', 'error'); + } + } + + getFormData() { + const isVisitsMode = this.elements.iterModeVisits ? this.elements.iterModeVisits.checked : false; + const maxVisits = this.elements.maxVisitsPerIteration ? parseInt(this.elements.maxVisitsPerIteration.value, 10) : 10; + const isPagesMode = this.elements.torModePages ? this.elements.torModePages.checked : false; + const circuitPages = this.elements.torCircuitPages ? parseInt(this.elements.torCircuitPages.value, 10) : 10; + const maxTimeouts = this.elements.torMaxConsecutiveTimeouts ? parseInt(this.elements.torMaxConsecutiveTimeouts.value, 10) : 5; + + let userAgent = 'random'; + if (this.elements.userAgentSelect) { + const sel = this.elements.userAgentSelect.value; + if (sel === 'custom') { + userAgent = (this.elements.userAgentCustom ? this.elements.userAgentCustom.value.trim() : '') || 'random'; + } else { + userAgent = sel; + } + } + + return { + start_url: this.elements.startUrl.value.trim(), + num_tabs: parseInt(this.elements.numTabs.value, 10), + num_iterations: parseInt(this.elements.numIterations.value, 10), + iteration_mode: isVisitsMode ? 'visits' : 'all', + max_visits_per_iteration: isNaN(maxVisits) || maxVisits < 1 ? 10 : maxVisits, + randomize_queue: this.elements.randomizeQueue ? this.elements.randomizeQueue.checked : true, + headless: this.elements.headless.checked, + accept_cookies: this.elements.acceptCookies.checked, + use_tor: this.elements.useTor.checked, + tor_circuit_mode: isPagesMode ? 'pages' : 'iteration', + tor_circuit_pages: isNaN(circuitPages) || circuitPages < 1 ? 10 : circuitPages, + tor_max_consecutive_timeouts: isNaN(maxTimeouts) || maxTimeouts < 1 ? 5 : maxTimeouts, + tor_exit_nodes: Array.from(this.selectedExitNodes), + timeout_ms: parseInt(this.elements.timeoutMs.value, 10), + slow_mo: parseInt(this.elements.slowMo.value, 10), + tor_socks_port: parseInt(this.elements.torSocksPort.value, 10), + tor_control_port: parseInt(this.elements.torControlPort.value, 10), + user_agent: userAgent, + }; + } + + async saveConfigToEnv() { + const payload = this.getFormData(); + try { + const res = await fetch('/api/config', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + const data = await res.json(); + if (data.success) { + this.showToast('Configuration saved to .env!', 'success'); + } else { + this.showToast(data.detail || 'Failed to save .env', 'error'); + } + } catch (err) { + this.showToast('Error saving .env file', 'error'); + } + } + + async startCrawl() { + const payload = this.getFormData(); + if (!payload.start_url) { + this.showToast('Please enter a valid starting URL', 'error'); + return; + } + + try { + this.elements.btnStart.disabled = true; + const res = await fetch('/api/crawl/start', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + const data = await res.json(); + if (!data.success) { + this.showToast(data.detail || 'Could not start crawler', 'error'); + this.elements.btnStart.disabled = false; + } else { + this.showToast('Crawl started!', 'success'); + } + } catch (err) { + this.showToast('Network error starting crawl', 'error'); + this.elements.btnStart.disabled = false; + } + } + + async stopCrawl() { + try { + this.elements.btnStop.disabled = true; + const res = await fetch('/api/crawl/stop', { method: 'POST' }); + const data = await res.json(); + if (data.success) { + this.showToast('Stopping crawler workers...', 'info'); + } else { + this.showToast(data.detail || 'Could not stop crawler', 'error'); + } + } catch (err) { + this.showToast('Error sending stop signal', 'error'); + } + } + + showToast(message, type = 'info') { + const container = this.elements.toastContainer; + const toast = document.createElement('div'); + toast.className = `toast toast-${type}`; + + let icon = 'ℹ️'; + if (type === 'success') icon = '✅'; + if (type === 'error') icon = '❌'; + + toast.innerHTML = `${icon}${this.escapeHtml(message)}`; + container.appendChild(toast); + + setTimeout(() => { + toast.style.opacity = '0'; + toast.style.transform = 'translateX(100%)'; + toast.style.transition = 'all 0.3s ease'; + setTimeout(() => toast.remove(), 300); + }, 3500); + } + + escapeHtml(str) { + if (!str) return ''; + return String(str) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + } +} + +// Global initialization +document.addEventListener('DOMContentLoaded', () => { + window.dashboard = new WebsurferDashboard(); +}); diff --git a/web/static/clickduck_logo.png b/web/static/clickduck_logo.png new file mode 100644 index 0000000..fb79204 Binary files /dev/null and b/web/static/clickduck_logo.png differ diff --git a/web/static/clickduck_logo_small.jpg b/web/static/clickduck_logo_small.jpg new file mode 100644 index 0000000..69464d0 Binary files /dev/null and b/web/static/clickduck_logo_small.jpg differ diff --git a/web/static/index.html b/web/static/index.html new file mode 100644 index 0000000..512649e --- /dev/null +++ b/web/static/index.html @@ -0,0 +1,580 @@ + + + + + + + Clickduck Control Center + + + + + + + + +
+ +
+
+
+ Clickduck +
+
+

Clickduck v1.0

+

Autonomous Playwright Web Crawler & Broken Link Detector

+
+
+
+
+ + Connecting... +
+
+ + IDLE +
+ +
+
+ + +
+ + +
+
+
+ + + + + +

Crawler Configuration (.env)

+
+ Synced with .env +
+ +
+ +
+ +
+ 🌐 + +
+ The root URL where the crawler begins crawling and finding internal/external + links. +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ 🎯 Iteration Scope & Website Visits +
+
+ + + +
+ + +
+ + +
+ + + + + + + +
+ + + + + +
+ + ⚙️ Advanced Crawler Options + ▼ + +
+
+ + + + Rotates real modern Chrome, Firefox, Safari & Edge headers to avoid WAF rate-limiting and bot blocks (HTTP 503/429/403). +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+ + + + + + + +
+
+
+ + +
+
+
+ + + + + +

Live Session Metrics

+
+
+ ⏱ + 00:00:00 +
+
+ +
+
+ Ready to start crawl + Iteration 1 / 1 +
+
+
+
+
+ + +
+
+ 📊 All Iterations Cumulative +
+
+
+
🌐
+
+
0
+
Total Websites Visited
+
+
+ +
+
⚠️
+
+
0
+
Total Errors Seen
+
+
+
+
+ + +
+
+ 🔄 Current Iteration +
+
+
+
📄
+
+
0
+
Pages Visited
+
+
+ +
+
🔗
+
+
0
+
Internal Pages
+
+
+ +
+
🌐
+
+
0
+
External Pages
+
+
+ +
+
🍪
+
+
0
+
Cookies Accepted
+
+
+ +
+
📥
+
+
0
+
Queue Backlog
+
+
+ +
+
⚠️
+
+
0
+
Failed Websites
+
+
+
+
+
+ + +
+
+
+ + + +

Tor Network Identity & Geolocation

+ Tor Inactive +
+ Queried once per circuit on start/rotation +
+ +
+
+ Tor Exit IP +
--
+
+
+ Country & Region +
--
+
+
+ City +
--
+
+
+ ISP / Organization +
--
+
+
+ Rotation Strategy +
Per Iteration
+
+
+ Circuits Rotated +
0
+
+
+ Exit Node Filter +
Global (Any)
+
+
+
+ + +
+ +
+
+ + + + + +

Websites with Errors & Broken Links

+ 0 +
+
+ + + +
+
+ +
+ + + + + + + + + + + + + + + + +
#Failed / Broken URLError ReasonFound On (Source URL)TimeAction
+
+ ✨ +

No website errors recorded yet. Broken links or HTTP error codes will appear here.

+
+
+
+
+ + +
+
+
+ + + + +

Live Crawl Logs

+ 0 lines +
+
+ + + + + +
+
+ +
+
+
+ [00:00:00] + [SYSTEM] + Websurfer dashboard initialized. Configure parameters and click Start + Crawling. +
+
+
+
+ +
+ + +
+
+ + + + + \ No newline at end of file diff --git a/web/static/login.html b/web/static/login.html new file mode 100644 index 0000000..5420c25 --- /dev/null +++ b/web/static/login.html @@ -0,0 +1,295 @@ + + + + + + + Login - Clickduck Control Center + + + + + + + + + +
+ +
+ + + + + diff --git a/web/static/style.css b/web/static/style.css new file mode 100644 index 0000000..a6a87b7 --- /dev/null +++ b/web/static/style.css @@ -0,0 +1,1353 @@ +/* ========================================================================== + Websurfer Modern Dark Theme & Glassmorphism Design System + ========================================================================== */ + +:root { + --bg-main: #090d16; + --bg-card: rgba(17, 24, 39, 0.75); + --bg-card-hover: rgba(26, 36, 56, 0.85); + --bg-input: rgba(15, 23, 42, 0.8); + --bg-terminal: #070b12; + + --border-subtle: rgba(255, 255, 255, 0.08); + --border-focus: #38bdf8; + --border-accent: rgba(56, 189, 248, 0.3); + + --text-main: #f8fafc; + --text-muted: #94a3b8; + --text-dim: #64748b; + + --primary: #38bdf8; + --primary-glow: rgba(56, 189, 248, 0.35); + --primary-gradient: linear-gradient(135deg, #38bdf8 0%, #6366f1 100%); + --secondary-gradient: linear-gradient(135deg, #334155 0%, #1e293b 100%); + + --success: #10b981; + --success-glow: rgba(16, 185, 129, 0.25); + --warning: #f59e0b; + --warning-glow: rgba(245, 158, 11, 0.25); + --danger: #f43f5e; + --danger-glow: rgba(244, 63, 94, 0.3); + --info: #818cf8; + + --radius-sm: 6px; + --radius-md: 10px; + --radius-lg: 14px; + --radius-xl: 20px; + + --shadow-card: 0 10px 30px -10px rgba(0, 0, 0, 0.5), 0 0 0 1px var(--border-subtle); + --shadow-glow: 0 0 25px var(--primary-glow); + --font-sans: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + --font-mono: 'JetBrains Mono', 'Fira Code', Menlo, monospace; +} + +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +body { + background-color: var(--bg-main); + background-image: + radial-gradient(circle at 15% 10%, rgba(56, 189, 248, 0.08) 0%, transparent 40%), + radial-gradient(circle at 85% 20%, rgba(99, 102, 241, 0.07) 0%, transparent 45%), + radial-gradient(circle at 50% 80%, rgba(16, 185, 129, 0.04) 0%, transparent 50%); + background-attachment: fixed; + color: var(--text-main); + font-family: var(--font-sans); + font-size: 14px; + line-height: 1.5; + min-height: 100vh; + -webkit-font-smoothing: antialiased; +} + +/* Custom Scrollbars */ +::-webkit-scrollbar { + width: 8px; + height: 8px; +} +::-webkit-scrollbar-track { + background: rgba(0, 0, 0, 0.2); +} +::-webkit-scrollbar-thumb { + background: rgba(255, 255, 255, 0.15); + border-radius: 4px; +} +::-webkit-scrollbar-thumb:hover { + background: rgba(255, 255, 255, 0.25); +} + +.app-container { + max-width: 1440px; + margin: 0 auto; + padding: 24px; + display: flex; + flex-direction: column; + gap: 20px; +} + +/* Header */ +.app-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 16px 24px; + background: var(--bg-card); + backdrop-filter: blur(16px); + -webkit-backdrop-filter: blur(16px); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-card); + border: 1px solid var(--border-subtle); +} + +.brand { + display: flex; + align-items: center; + gap: 14px; +} + +.logo-icon { + width: 44px; + height: 44px; + border-radius: 12px; + background: var(--primary-gradient); + display: flex; + align-items: center; + justify-content: center; + color: #090d16; + box-shadow: 0 4px 15px var(--primary-glow); +} + +.brand-title { + font-size: 20px; + font-weight: 700; + letter-spacing: -0.5px; + display: flex; + align-items: center; + gap: 8px; +} + +.badge-tag { + font-size: 11px; + font-weight: 600; + padding: 2px 8px; + background: rgba(56, 189, 248, 0.15); + color: var(--primary); + border-radius: 20px; + border: 1px solid var(--border-accent); +} + +.brand-subtitle { + font-size: 12px; + color: var(--text-muted); +} + +.header-status { + display: flex; + align-items: center; + gap: 12px; +} + +/* Status Badges */ +.ws-pill { + display: flex; + align-items: center; + gap: 6px; + padding: 6px 12px; + border-radius: 20px; + font-size: 12px; + font-weight: 500; + background: rgba(15, 23, 42, 0.8); + border: 1px solid var(--border-subtle); + transition: all 0.3s ease; +} + +.ws-pill .status-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--dim); +} + +.ws-pill.connected .status-dot { + background: var(--success); + box-shadow: 0 0 8px var(--success); +} +.ws-pill.connecting .status-dot { + background: var(--warning); + animation: blink 1s infinite alternate; +} +.ws-pill.disconnected .status-dot { + background: var(--danger); +} + +@keyframes blink { + from { opacity: 0.3; } + to { opacity: 1; } +} + +.status-badge { + position: relative; + display: flex; + align-items: center; + gap: 8px; + padding: 6px 16px; + border-radius: 20px; + font-size: 12px; + font-weight: 700; + letter-spacing: 0.5px; + text-transform: uppercase; + transition: all 0.3s ease; +} + +.status-badge.idle { + background: rgba(100, 116, 139, 0.15); + color: var(--text-muted); + border: 1px solid rgba(100, 116, 139, 0.3); +} + +.status-badge.running { + background: rgba(16, 185, 129, 0.15); + color: var(--success); + border: 1px solid rgba(16, 185, 129, 0.4); + box-shadow: 0 0 15px var(--success-glow); +} + +.status-badge.stopping { + background: rgba(245, 158, 11, 0.15); + color: var(--warning); + border: 1px solid rgba(245, 158, 11, 0.4); +} + +.status-badge.finished { + background: rgba(56, 189, 248, 0.15); + color: var(--primary); + border: 1px solid var(--border-accent); +} + +.status-badge.error { + background: rgba(244, 63, 94, 0.15); + color: var(--danger); + border: 1px solid rgba(244, 63, 94, 0.4); +} + +.pulse-ring { + width: 8px; + height: 8px; + border-radius: 50%; + background: currentColor; +} +.status-badge.running .pulse-ring { + animation: pulse 1.5s infinite; +} + +@keyframes pulse { + 0% { transform: scale(0.95); opacity: 1; box-shadow: 0 0 0 0 currentColor; } + 70% { transform: scale(1); opacity: 0.7; box-shadow: 0 0 0 8px transparent; } + 100% { transform: scale(0.95); opacity: 1; } +} + +/* Dashboard Grid Layout */ +.dashboard-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 20px; +} + +.full-width-card { + grid-column: 1 / -1; +} + +/* Glass Card */ +.card { + background: var(--bg-card); + backdrop-filter: blur(14px); + -webkit-backdrop-filter: blur(14px); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-card); + padding: 22px; + display: flex; + flex-direction: column; + gap: 18px; +} + +.card-header { + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid var(--border-subtle); + padding-bottom: 14px; +} + +.header-title { + display: flex; + align-items: center; + gap: 10px; +} + +.header-title h2 { + font-size: 16px; + font-weight: 600; + letter-spacing: -0.3px; +} + +.header-hint { + font-size: 11px; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +/* Form Styles */ +.config-form { + display: flex; + flex-direction: column; + gap: 16px; +} + +.form-group { + display: flex; + flex-direction: column; + gap: 6px; + flex: 1; +} + +.form-group label { + font-size: 13px; + font-weight: 500; + color: var(--text-main); + display: flex; + justify-content: space-between; +} + +.required { + color: var(--danger); +} + +.val-badge { + background: rgba(56, 189, 248, 0.2); + color: var(--primary); + padding: 1px 6px; + border-radius: 10px; + font-size: 12px; + font-weight: 600; +} + +.input-with-icon { + position: relative; + display: flex; + align-items: center; +} + +.input-icon { + position: absolute; + left: 12px; + font-size: 14px; + pointer-events: none; +} + +.input-with-icon input { + padding-left: 38px !important; +} + +input[type="url"], +input[type="text"], +input[type="number"], +.form-input { + width: 100%; + padding: 10px 14px; + background: var(--bg-input); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + color: var(--text-main); + font-family: var(--font-sans); + font-size: 13px; + transition: all 0.2s ease; +} + +input:focus { + outline: none; + border-color: var(--border-focus); + box-shadow: 0 0 0 3px rgba(56, 189, 248, 0.15); +} + +.field-help { + font-size: 11px; + color: var(--text-dim); +} + +.form-row { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 14px; +} + +/* Range Slider */ +.range-slider { + accent-color: var(--primary); + height: 6px; + background: rgba(255, 255, 255, 0.1); + border-radius: 4px; + cursor: pointer; +} + +/* Toggle Switches */ +.toggles-grid { + display: flex; + flex-direction: column; + gap: 10px; +} + +.toggle-card { + display: flex; + justify-content: space-between; + align-items: center; + padding: 12px 14px; + background: rgba(15, 23, 42, 0.5); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + cursor: pointer; + transition: all 0.2s ease; +} + +.toggle-card:hover { + background: var(--bg-card-hover); + border-color: rgba(255, 255, 255, 0.12); +} + +.toggle-info { + display: flex; + flex-direction: column; + gap: 2px; +} + +.toggle-title { + font-size: 13px; + font-weight: 500; + color: var(--text-main); +} + +.toggle-desc { + font-size: 11px; + color: var(--text-dim); +} + +.toggle-checkbox { + display: none; +} + +.toggle-switch { + width: 44px; + height: 24px; + background: rgba(255, 255, 255, 0.15); + border-radius: 12px; + position: relative; + transition: all 0.3s ease; + flex-shrink: 0; +} + +.toggle-switch::after { + content: ''; + position: absolute; + top: 3px; + left: 3px; + width: 18px; + height: 18px; + background: #ffffff; + border-radius: 50%; + transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); +} + +.toggle-checkbox:checked + .toggle-switch { + background: var(--primary); + box-shadow: 0 0 10px var(--primary-glow); +} + +.toggle-checkbox:checked + .toggle-switch::after { + transform: translateX(20px); +} + +/* Advanced Collapsible */ +.advanced-section { + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + background: rgba(15, 23, 42, 0.3); + overflow: hidden; +} + +.advanced-summary { + padding: 10px 14px; + font-size: 12px; + font-weight: 600; + color: var(--text-muted); + cursor: pointer; + display: flex; + justify-content: space-between; + align-items: center; + user-select: none; +} + +.advanced-summary:hover { + color: var(--text-main); +} + +.chevron { + font-size: 10px; + transition: transform 0.2s ease; +} + +.advanced-section[open] .chevron { + transform: rotate(180deg); +} + +.advanced-body { + padding: 12px 14px; + border-top: 1px solid var(--border-subtle); + display: grid; + grid-template-columns: 1fr 1fr; + gap: 12px; +} + +/* Form Action Buttons */ +.form-actions { + display: flex; + gap: 10px; + margin-top: 6px; + flex-wrap: wrap; +} + +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 8px; + padding: 11px 18px; + font-size: 13px; + font-weight: 600; + border-radius: var(--radius-md); + border: none; + cursor: pointer; + transition: all 0.2s ease; +} + +.btn:disabled { + opacity: 0.4; + cursor: not-allowed; + pointer-events: none; +} + +.btn-primary { + background: var(--primary-gradient); + color: #090d16; + box-shadow: 0 4px 14px var(--primary-glow); +} +.btn-primary:hover:not(:disabled) { + transform: translateY(-1px); + box-shadow: 0 6px 20px var(--primary-glow); +} + +.btn-danger { + background: rgba(244, 63, 94, 0.18); + color: var(--danger); + border: 1px solid rgba(244, 63, 94, 0.4); +} +.btn-danger:hover:not(:disabled) { + background: var(--danger); + color: #ffffff; + box-shadow: 0 4px 15px var(--danger-glow); +} + +.btn-secondary { + background: rgba(255, 255, 255, 0.08); + color: var(--text-main); + border: 1px solid var(--border-subtle); +} +.btn-secondary:hover { + background: rgba(255, 255, 255, 0.14); +} + +.btn-ghost { + background: transparent; + color: var(--text-muted); +} +.btn-ghost:hover { + color: var(--text-main); + background: rgba(255, 255, 255, 0.05); +} + +.btn-sm { + padding: 6px 12px; + font-size: 12px; +} +.btn-xs { + padding: 4px 8px; + font-size: 11px; +} + +.btn-outline { + background: rgba(15, 23, 42, 0.6); + border: 1px solid var(--border-subtle); + color: var(--text-muted); +} +.btn-outline:hover { + color: var(--text-main); + border-color: rgba(255, 255, 255, 0.2); +} + +/* Stats Card Components */ +.timer-badge { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 12px; + background: rgba(15, 23, 42, 0.8); + border-radius: 20px; + border: 1px solid var(--border-subtle); + font-family: var(--font-mono); + font-size: 13px; + font-weight: 600; + color: var(--primary); +} + +.session-progress-strip { + display: flex; + flex-direction: column; + gap: 8px; + background: rgba(15, 23, 42, 0.5); + padding: 12px 14px; + border-radius: var(--radius-md); + border: 1px solid var(--border-subtle); +} + +.progress-info { + display: flex; + justify-content: space-between; + align-items: center; + font-size: 12px; + color: var(--text-muted); +} + +.iter-badge { + background: rgba(99, 102, 241, 0.2); + color: #a5b4fc; + padding: 2px 8px; + border-radius: 12px; + font-size: 11px; + font-weight: 600; +} + +.progress-bar-container { + height: 6px; + background: rgba(255, 255, 255, 0.08); + border-radius: 3px; + overflow: hidden; + position: relative; +} + +.progress-bar-fill { + height: 100%; + width: 0%; + background: var(--primary-gradient); + transition: width 0.3s ease; +} + +.progress-bar-fill.running { + width: 100%; + animation: shimmer 2s infinite linear; + background: linear-gradient(90deg, #38bdf8 0%, #6366f1 50%, #38bdf8 100%); + background-size: 200% 100%; +} + +@keyframes shimmer { + 0% { background-position: 100% 0; } + 100% { background-position: -100% 0; } +} + +/* Metrics Grids */ +.metrics-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 12px; +} + +.metrics-grid-2col { + grid-template-columns: repeat(2, 1fr); +} + +.metric-tile { + background: rgba(15, 23, 42, 0.6); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + padding: 14px; + display: flex; + align-items: center; + gap: 12px; + transition: all 0.2s ease; +} + +.metric-tile:hover { + transform: translateY(-2px); + border-color: rgba(255, 255, 255, 0.15); +} + +.metric-icon { + font-size: 22px; + width: 40px; + height: 40px; + border-radius: 10px; + display: flex; + align-items: center; + justify-content: center; + background: rgba(255, 255, 255, 0.05); +} + +.metric-val { + font-size: 20px; + font-weight: 700; + font-family: var(--font-mono); + line-height: 1.2; +} + +.metric-label { + font-size: 11px; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.tile-visited .metric-val { color: #f8fafc; } +.tile-internal .metric-val { color: #38bdf8; } +.tile-external .metric-val { color: #f59e0b; } +.tile-cookies .metric-val { color: #c084fc; } +.tile-queue .metric-val { color: #60a5fa; } +.tile-errors .metric-val { color: #f43f5e; } +.tile-total-visited .metric-val { color: #38bdf8; } +.tile-total-errors .metric-val { color: #f43f5e; } + +/* Dedicated Errors Section */ +.count-pill { + background: rgba(244, 63, 94, 0.2); + color: var(--danger); + padding: 2px 10px; + border-radius: 12px; + font-size: 12px; + font-weight: 700; + border: 1px solid rgba(244, 63, 94, 0.3); +} + +.count-pill.gray { + background: rgba(255, 255, 255, 0.1); + color: var(--text-dim); + border-color: var(--border-subtle); +} + +.table-toolbar, .log-toolbar { + display: flex; + align-items: center; + gap: 10px; +} + +.filter-input, .filter-input-sm { + padding: 6px 12px; + background: var(--bg-input); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-sm); + color: var(--text-main); + font-size: 12px; +} + +.filter-input { + width: 220px; +} +.filter-input-sm { + width: 150px; +} + +.table-container { + overflow-x: auto; + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + background: rgba(10, 15, 26, 0.6); + max-height: 360px; + overflow-y: auto; +} + +.data-table { + width: 100%; + border-collapse: collapse; + text-align: left; + font-size: 12px; +} + +.data-table th { + background: rgba(15, 23, 42, 0.95); + color: var(--text-muted); + font-weight: 600; + padding: 10px 14px; + border-bottom: 1px solid var(--border-subtle); + position: sticky; + top: 0; + z-index: 1; +} + +.data-table td { + padding: 10px 14px; + border-bottom: 1px solid rgba(255, 255, 255, 0.04); + color: var(--text-main); + vertical-align: middle; +} + +.data-table tr:hover td { + background: rgba(255, 255, 255, 0.02); +} + +.url-cell { + font-family: var(--font-mono); + font-size: 11px; + color: #38bdf8; + word-break: break-all; + max-width: 320px; +} + +.source-cell { + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-muted); + word-break: break-all; + max-width: 260px; +} + +.error-badge { + display: inline-block; + padding: 3px 8px; + border-radius: 4px; + font-size: 11px; + font-family: var(--font-mono); + font-weight: 600; + background: rgba(244, 63, 94, 0.15); + color: var(--danger); + border: 1px solid rgba(244, 63, 94, 0.3); +} + +.copy-btn { + background: rgba(255, 255, 255, 0.06); + border: 1px solid var(--border-subtle); + color: var(--text-dim); + padding: 4px 8px; + border-radius: 4px; + cursor: pointer; + font-size: 11px; + transition: all 0.2s; +} +.copy-btn:hover { + color: var(--text-main); + background: rgba(255, 255, 255, 0.15); +} + +.empty-state { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 36px 20px; + color: var(--text-dim); + gap: 8px; +} + +.empty-icon { + font-size: 28px; +} + +/* Terminal Log Console */ +.terminal-window { + background: var(--bg-terminal); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + box-shadow: inset 0 2px 8px rgba(0, 0, 0, 0.6); + height: 280px; + overflow-y: auto; + font-family: var(--font-mono); + font-size: 12px; + padding: 12px 16px; +} + +.terminal-body { + display: flex; + flex-direction: column; + gap: 4px; +} + +.log-line { + display: flex; + gap: 8px; + align-items: baseline; + line-height: 1.4; + word-break: break-all; +} + +.log-time { + color: var(--text-dim); + font-size: 11px; + flex-shrink: 0; +} + +.log-tag { + font-weight: 600; + font-size: 11px; + flex-shrink: 0; +} + +.log-msg { + color: #e2e8f0; +} + +.tag-sys { color: #818cf8; } +.log-line.log-info .log-msg { color: #cbd5e1; } +.log-line.log-success .log-msg { color: #34d399; } +.log-line.log-warn .log-msg { color: #fbbf24; } +.log-line.log-error .log-msg { color: #f87171; font-weight: 500; } + +.checkbox-label { + display: flex; + align-items: center; + gap: 6px; + font-size: 12px; + color: var(--text-muted); + cursor: pointer; +} + +.form-select-sm { + background: var(--bg-input); + color: var(--text-main); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-sm); + padding: 4px 8px; + font-size: 12px; +} + +/* Toast Notifications */ +.toast-container { + position: fixed; + bottom: 24px; + right: 24px; + display: flex; + flex-direction: column; + gap: 10px; + z-index: 9999; +} + +.toast { + background: rgba(15, 23, 42, 0.95); + backdrop-filter: blur(10px); + border: 1px solid var(--border-subtle); + color: var(--text-main); + padding: 12px 18px; + border-radius: var(--radius-md); + box-shadow: 0 10px 25px rgba(0,0,0,0.5); + display: flex; + align-items: center; + gap: 10px; + font-size: 13px; + animation: slideIn 0.3s ease forwards; +} + +.toast.toast-success { border-left: 4px solid var(--success); } +.toast.toast-error { border-left: 4px solid var(--danger); } +.toast.toast-info { border-left: 4px solid var(--primary); } + +@keyframes slideIn { + from { transform: translateX(100%); opacity: 0; } + to { transform: translateX(0); opacity: 1; } +} + +/* Tor Identity Card Styling */ +.tor-card { + border-left: 4px solid #a855f7; + background: linear-gradient(180deg, rgba(168, 85, 247, 0.05) 0%, rgba(17, 24, 39, 0.8) 100%); + transition: all 0.3s ease; +} + +.tor-card.active { + border-color: #a855f7; + box-shadow: 0 0 20px rgba(168, 85, 247, 0.15); +} + +.tor-shield-icon { + color: #a855f7; +} + +.tor-pill { + padding: 3px 10px; + border-radius: 12px; + font-size: 11px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.5px; + transition: all 0.3s ease; +} + +.tor-pill.tor-on { + background: rgba(168, 85, 247, 0.2); + color: #d8b4fe; + border: 1px solid rgba(168, 85, 247, 0.4); +} + +.tor-pill.tor-off { + background: rgba(255, 255, 255, 0.06); + color: var(--text-dim); + border: 1px solid var(--border-subtle); +} + +.tor-identity-grid { + display: grid; + grid-template-columns: repeat(6, 1fr); + gap: 12px; +} + +.tor-info-tile { + background: rgba(15, 23, 42, 0.6); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + padding: 14px 16px; + display: flex; + flex-direction: column; + gap: 4px; + transition: all 0.2s ease; +} + +.tor-info-tile:hover { + background: var(--bg-card-hover); + border-color: rgba(168, 85, 247, 0.3); +} + +.tor-label { + font-size: 11px; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.5px; + font-weight: 600; +} + +.tor-val { + font-size: 14px; + font-weight: 600; + color: var(--text-main); + word-break: break-word; +} + +.tor-val.monospace { + font-family: var(--font-mono); + color: #38bdf8; +} + +/* Tor Circuit Strategy Sub-Section */ +.tor-options-subgroup { + margin-top: 12px; + padding: 14px 16px; + background: rgba(168, 85, 247, 0.05); + border: 1px solid rgba(168, 85, 247, 0.2); + border-radius: var(--radius-md); + display: flex; + flex-direction: column; + gap: 12px; + animation: fadeIn 0.3s ease; +} + +@keyframes fadeIn { + from { opacity: 0; transform: translateY(-4px); } + to { opacity: 1; transform: translateY(0); } +} + +.subgroup-header { + display: flex; + align-items: center; + justify-content: space-between; +} + +.subgroup-title { + font-size: 12px; + font-weight: 600; + color: #d8b4fe; + letter-spacing: 0.3px; + text-transform: uppercase; +} + +.circuit-mode-selector { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 10px; +} + +.radio-card { + position: relative; + display: flex; + align-items: flex-start; + padding: 12px; + background: rgba(15, 23, 42, 0.6); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + cursor: pointer; + transition: all 0.2s ease; + user-select: none; +} + +.radio-card input[type="radio"] { + position: absolute; + opacity: 0; + pointer-events: none; +} + +.radio-card:hover { + background: rgba(26, 36, 56, 0.8); + border-color: rgba(168, 85, 247, 0.3); +} + +.radio-card.active { + background: rgba(168, 85, 247, 0.12); + border-color: #a855f7; + box-shadow: 0 0 12px rgba(168, 85, 247, 0.2); +} + +.radio-card-body { + display: flex; + flex-direction: column; + gap: 4px; +} + +.radio-card-header { + display: flex; + align-items: center; + gap: 6px; +} + +.radio-icon { + font-size: 14px; +} + +.radio-card-title { + font-size: 13px; + font-weight: 600; + color: var(--text-main); +} + +.radio-card-desc { + font-size: 11px; + color: var(--text-muted); + line-height: 1.35; +} + +.tor-pages-field { + margin-top: 4px; + animation: fadeIn 0.2s ease; +} + +.input-with-stepper { + position: relative; + display: flex; + align-items: center; +} + +.input-with-stepper .input-icon { + position: absolute; + left: 12px; + font-size: 14px; + pointer-events: none; + z-index: 2; +} + +.input-with-stepper .form-input { + width: 100%; + padding-left: 36px; + padding-right: 60px; +} + +.input-unit { + position: absolute; + right: 12px; + font-size: 12px; + font-weight: 500; + color: var(--text-dim); + pointer-events: none; +} + +/* European Exit Node Country Selector */ +.tor-exit-nodes-section { + display: flex; + flex-direction: column; + gap: 10px; + padding-top: 10px; + border-top: 1px solid rgba(168, 85, 247, 0.15); +} + +.exit-nodes-header { + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: 8px; +} + +.exit-nodes-title-group { + display: flex; + align-items: center; + gap: 8px; +} + +.count-pill-sm { + font-size: 11px; + font-weight: 600; + padding: 2px 8px; + border-radius: 12px; + background: rgba(255, 255, 255, 0.08); + color: var(--text-muted); + border: 1px solid var(--border-subtle); + transition: all 0.2s ease; +} + +.count-pill-sm.active { + background: rgba(168, 85, 247, 0.2); + color: #d8b4fe; + border-color: rgba(168, 85, 247, 0.4); +} + +.country-action-btns { + display: flex; + gap: 6px; +} + +.btn-chip-action { + background: rgba(15, 23, 42, 0.7); + border: 1px solid var(--border-subtle); + color: var(--text-muted); + font-size: 11px; + font-weight: 500; + padding: 3px 8px; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all 0.15s ease; +} + +.btn-chip-action:hover { + background: rgba(168, 85, 247, 0.15); + color: #d8b4fe; + border-color: rgba(168, 85, 247, 0.3); +} + +.btn-chip-action.danger:hover { + background: rgba(244, 63, 94, 0.15); + color: #f43f5e; + border-color: rgba(244, 63, 94, 0.3); +} + +.country-chips-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(118px, 1fr)); + gap: 6px; + max-height: 165px; + overflow-y: auto; + padding: 6px; + background: rgba(7, 11, 18, 0.5); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); +} + +.country-chip { + display: flex; + align-items: center; + gap: 6px; + padding: 6px 8px; + background: rgba(15, 23, 42, 0.8); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-sm); + cursor: pointer; + transition: all 0.2s ease; + user-select: none; + font-family: inherit; + text-align: left; +} + +.country-chip:hover { + background: rgba(26, 36, 56, 0.9); + border-color: rgba(168, 85, 247, 0.3); +} + +.country-chip.active { + background: rgba(168, 85, 247, 0.18); + border-color: #a855f7; + box-shadow: 0 0 10px rgba(168, 85, 247, 0.2); +} + +.chip-flag { + font-size: 14px; + line-height: 1; +} + +.chip-name { + font-size: 11px; + font-weight: 500; + color: var(--text-main); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + flex: 1; +} + +.chip-code { + font-size: 10px; + font-weight: 700; + color: var(--text-dim); + font-family: var(--font-mono); +} + +.country-chip.active .chip-code { + color: #d8b4fe; +} + +.exit-nodes-summary { + display: flex; + align-items: center; + gap: 6px; + font-size: 12px; + color: #cbd5e1; + background: rgba(15, 23, 42, 0.4); + padding: 6px 10px; + border-radius: var(--radius-sm); + border: 1px dashed rgba(168, 85, 247, 0.25); +} + +.exit-nodes-summary .summary-icon { + font-size: 13px; +} + +/* Responsive layout for mobile / smaller screens */ +@media (max-width: 1100px) { + .tor-identity-grid { + grid-template-columns: repeat(3, 1fr); + } +} + +@media (max-width: 960px) { + .dashboard-grid { + grid-template-columns: 1fr; + } + .metrics-grid { + grid-template-columns: repeat(2, 1fr); + } + .tor-identity-grid { + grid-template-columns: repeat(2, 1fr); + } + .circuit-mode-selector { + grid-template-columns: 1fr; + } + .country-chips-grid { + grid-template-columns: repeat(2, 1fr); + } + .app-header { + flex-direction: column; + gap: 14px; + align-items: flex-start; + } +} +