Initial commit
This commit is contained in:
Steffen
commit
473351086a
29 changed files with 6816 additions and 0 deletions
11
.dockerignore
Normal file
11
.dockerignore
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
.venv/
|
||||
venv/
|
||||
.git/
|
||||
.gitignore
|
||||
.vscode/
|
||||
.idea/
|
||||
*.log
|
||||
26
.env.example
Normal file
26
.env.example
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# Crawler Configuration
|
||||
CRAWLER_START_URL=https://example.com
|
||||
CRAWLER_USE_TOR=false
|
||||
CRAWLER_TOR_CIRCUIT_MODE=iteration
|
||||
CRAWLER_TOR_CIRCUIT_PAGES=10
|
||||
CRAWLER_TOR_MAX_CONSECUTIVE_TIMEOUTS=5
|
||||
CRAWLER_TOR_EXIT_NODES=de,nl,fr,ch
|
||||
CRAWLER_NUM_TABS=5
|
||||
CRAWLER_NUM_ITERATIONS=1
|
||||
CRAWLER_ITERATION_MODE=all
|
||||
CRAWLER_MAX_VISITS_PER_ITERATION=10
|
||||
CRAWLER_RANDOMIZE_QUEUE=true
|
||||
CRAWLER_HEADLESS=true
|
||||
CRAWLER_ACCEPT_COOKIES=true
|
||||
CRAWLER_SLOW_MO=100
|
||||
CRAWLER_TIMEOUT_MS=35000
|
||||
CRAWLER_USER_AGENT=random
|
||||
|
||||
# Web Dashboard Port
|
||||
WEB_PORT=8080
|
||||
|
||||
# Web Dashboard Authentication (Recommended for Internet-Facing Deployments)
|
||||
AUTH_ENABLED=true
|
||||
AUTH_USERNAME=admin
|
||||
AUTH_PASSWORD=change_this_to_a_secure_password
|
||||
AUTH_SECRET_KEY=change_this_to_a_random_32_character_secret_key
|
||||
24
.gitignore
vendored
Normal file
24
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Virtual Environments
|
||||
venv/
|
||||
.venv/
|
||||
env/
|
||||
.env_win/
|
||||
|
||||
# Python Cache
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
|
||||
# Environment Variables & Local Secrets
|
||||
.env
|
||||
!.env.example
|
||||
|
||||
# Playwright
|
||||
test-results/
|
||||
playwright-report/
|
||||
|
||||
# OS / Editor Files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.vscode/
|
||||
37
Dockerfile
Normal file
37
Dockerfile
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Use official Microsoft Playwright image with Python & browser dependencies
|
||||
FROM mcr.microsoft.com/playwright/python:v1.49.0-noble
|
||||
|
||||
# Prevent interactive prompts during apt package installation
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# Install Tor daemon and curl
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
tor \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Configure Tor (SOCKS5 proxy on 9050, ControlPort on 9051 with open localhost authentication)
|
||||
RUN printf "SocksPort 127.0.0.1:9050\nControlPort 127.0.0.1:9051\nCookieAuthentication 0\nHashedControlPassword \"\"\n" > /etc/tor/torrc
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install Python requirements
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Ensure Chromium browser is ready
|
||||
RUN playwright install chromium
|
||||
|
||||
# Copy application source code
|
||||
COPY . .
|
||||
|
||||
# Prepare entrypoint script (strip potential Windows CRLF endings and mark executable)
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN sed -i 's/\r$//' /entrypoint.sh && chmod +x /entrypoint.sh
|
||||
|
||||
# Expose Web Dashboard port
|
||||
EXPOSE 8000
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
CMD ["python", "web.py", "--host", "0.0.0.0", "--port", "8000"]
|
||||
179
README.md
Normal file
179
README.md
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
# WebSurfer: Parallel Playwright Crawler
|
||||
|
||||
A highly-concurrent, fully automated website crawler built with Python and [Playwright](https://playwright.dev/python/).
|
||||
|
||||
This crawler rapidly crawls an entire domain using multiple visible or headless browser tabs. It uses a Breadth-First Search (BFS) approach powered by an `asyncio.Queue` to spread the crawling workload across a customizable pool of worker tabs.
|
||||
|
||||
## Key Features
|
||||
|
||||
- **Multi-Tab Concurrency**: Spawns multiple asynchronous browser tabs (default: 5) to dramatically speed up crawl times.
|
||||
- **Automatic Cookie Consent Acceptance**: Automatically detects and accepts cookie consent banners / CMPs (Complianz, Borlabs Cookie, Cookiebot, OneTrust, Usercentrics, Klaro, CookieYes, Real Cookie Banner, CCM19, and generic accept patterns) across pages and iframes so analytics, marketing tags, and tracking scripts fire properly.
|
||||
- **Intelligent Scope**:
|
||||
- Recursively crawls all internal links belonging to the starting domain.
|
||||
- Visits external links (depth 1) to verify they are alive, but intentionally avoids following their sub-links to prevent crawling the entire internet.
|
||||
- **Crawler Trap Protection**: Automatically strips query parameters (e.g. `?page=2`) and URL fragments (e.g. `#section`) from internal links to prevent infinite loops from endless pagination or dynamically generated URLs. External links retain their original query parameters to prevent breaking things like YouTube videos.
|
||||
- **Configurable Iteration Scope & Visit Limits**:
|
||||
- **All Websites (Default)**: Crawls all reachable internal and external level-1 links until the queue is fully exhausted.
|
||||
- **Limit Visits Mode**: Caps each iteration to a configurable number of website visits before starting a new iteration. Ideal for multi-iteration testing across fresh Tor circuits and rotating User-Agents on large websites without having to crawl every single page.
|
||||
- **Randomized Crawl Queue**: Uniformly picks pages at random from the queue of discovered URLs instead of strict FIFO breadth-first order. Ensures broad, diverse coverage across various sections and depths of the website on every iteration.
|
||||
- **Error & Broken Link Tracking**: Tracks any pages that result in HTTP errors (404, 500, etc.) or connection timeouts. At the end of the crawl, it outputs a clean list of broken URLs alongside the exact source page where the broken link was discovered.
|
||||
- **Tor Proxy, Flexible Circuit Rotation & European Exit Nodes**: Optionally routes all crawler traffic securely through a headless Tor proxy (SOCKS5 on port `9050`).
|
||||
- **Circuit Rotation Modes**: Rotate per crawl iteration or dynamically after every *N* crawled pages.
|
||||
- **European Exit Node Selector**: Select 0 to *N* European countries (Germany, Netherlands, France, Switzerland, Sweden, UK, etc.) to strictly/preferentially route traffic through specific European exit relays. If no country is selected, global Tor exit relays are used worldwide.
|
||||
- **Live Tor Network & IP Geolocation Inspector**: Shows active Tor exit IP, country, city, ISP/organization, exit node filter, and circuit rotation count in real-time.
|
||||
|
||||
## Web Dashboard & Frontend
|
||||
|
||||
WebSurfer includes a modern, real-time Web Dashboard built with FastAPI and WebSockets.
|
||||
|
||||
### Features in the Web Dashboard:
|
||||
- **Interactive Configuration**: Configure start URL, concurrency, iterations, Tor routing, headless mode, cookie handling, and timeouts directly in the browser.
|
||||
- **`.env` Synchronization**: One-click saving to `.env` or loading stored parameters.
|
||||
- **Live Metrics & All-Iterations Cumulative Statistics**: Real-time KPI tiles for Visited Pages, Internal vs. External links, Cookies Accepted, Queue Backlog, and Timer, paired with a dedicated all-iterations summary bar and subtitle badges tracking total websites visited and total errors encountered across all iterations.
|
||||
- **Dedicated Error & Broken Link Section**: Structured, searchable, and exportable (CSV / JSON) error registry displaying the failed URL, error reason / HTTP status, and referring source page.
|
||||
- **Live Streaming Terminal**: Real-time color-coded crawl logs with auto-scrolling, search query filtering, and level filtering.
|
||||
|
||||
### Running the Web Dashboard:
|
||||
```bash
|
||||
python web.py
|
||||
```
|
||||
*(or `python main.py --web`)*
|
||||
|
||||
Open your browser at [http://127.0.0.1:8000](http://127.0.0.1:8000).
|
||||
|
||||
## Project Structure
|
||||
|
||||
The project is organized into clean, single-responsibility modules:
|
||||
|
||||
```
|
||||
websurfer/
|
||||
├── crawler/
|
||||
│ ├── __init__.py # Public exports (Crawler, CrawlerConfig, TorManager, etc.)
|
||||
│ ├── config.py # Configuration dataclass & interactive/env/dict loaders
|
||||
│ ├── tor.py # Tor daemon lifecycle & circuit rotation (Stem/SOCKS5)
|
||||
│ ├── cookies.py # CMP selectors, text regexes & auto-consent execution
|
||||
│ ├── url_utils.py # URL normalization, domain matching & static file filters
|
||||
│ └── engine.py # Playwright browser lifecycle, worker queue, events & crawl loop
|
||||
├── web/
|
||||
│ ├── __init__.py # Web package entry point & server runner
|
||||
│ ├── app.py # FastAPI REST API & WebSocket endpoints
|
||||
│ ├── manager.py # Async crawl session manager & event broadcaster
|
||||
│ └── static/
|
||||
│ ├── index.html # Modern responsive dashboard UI
|
||||
│ ├── style.css # Dark glassmorphism styling
|
||||
│ └── app.js # Real-time WebSocket controller & data renderer
|
||||
├── main.py # Primary entry point (supports CLI and --web)
|
||||
├── web.py # Standalone Web Dashboard launcher
|
||||
├── browser_crawler.py # Backwards-compatible launcher
|
||||
├── .env.example # Example configuration settings
|
||||
└── requirements.txt # Project dependencies
|
||||
```
|
||||
|
||||
## Prerequisites
|
||||
|
||||
You need Python 3.8+ installed. If you intend to use the Tor proxy feature, ensure the `tor` command-line package is installed on your system:
|
||||
- **macOS**: `brew install tor`
|
||||
- **Linux (Ubuntu/Debian)**: `sudo apt install tor`
|
||||
- **Windows**: Install Tor Expert Bundle from [torproject.org](https://www.torproject.org/) and add it to your PATH.
|
||||
*(No Tor Browser GUI needed!)*
|
||||
|
||||
## Installation
|
||||
|
||||
1. Navigate to the project directory:
|
||||
```bash
|
||||
cd websurfer
|
||||
```
|
||||
2. Create and activate an OS-specific virtual environment:
|
||||
- **macOS / Linux**:
|
||||
```bash
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
```
|
||||
- **Windows (PowerShell)**:
|
||||
```powershell
|
||||
python -m venv .venv
|
||||
.\.venv\Scripts\Activate.ps1
|
||||
```
|
||||
- **Windows (Command Prompt)**:
|
||||
```cmd
|
||||
python -m venv .venv
|
||||
.\.venv\Scripts\activate.bat
|
||||
```
|
||||
3. Install the dependencies:
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
4. Install the required Playwright Chromium binaries:
|
||||
```bash
|
||||
playwright install chromium
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### 1. Web Dashboard (Recommended)
|
||||
```bash
|
||||
python web.py
|
||||
```
|
||||
Or:
|
||||
```bash
|
||||
python main.py --web
|
||||
```
|
||||
|
||||
### 2. Running with Docker & Tor (Containerized)
|
||||
WebSurfer can be run in a Docker container with Playwright headless Chromium and an integrated background Tor proxy:
|
||||
|
||||
```bash
|
||||
# Build and start container (Web Dashboard on port 8080)
|
||||
docker compose up --build
|
||||
|
||||
# Run in background
|
||||
docker compose up -d
|
||||
|
||||
# View logs
|
||||
docker compose logs -f
|
||||
```
|
||||
|
||||
Open your browser at [http://127.0.0.1:8080](http://127.0.0.1:8080) (or customize the port with `WEB_PORT=8000` in `.env`).
|
||||
|
||||
To run a headless crawl via CLI inside the container:
|
||||
```bash
|
||||
docker compose run --rm websurfer python main.py
|
||||
```
|
||||
|
||||
### 3. Running via CLI
|
||||
Run the main entry point in interactive/CLI mode:
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
*(or `python browser_crawler.py`)*
|
||||
|
||||
The script will interactively prompt you (or read from `.env`):
|
||||
1. **Starting URL**: The base website you want to crawl (e.g., `https://example.com`).
|
||||
2. **Tor Proxy**: Whether you want to route traffic through Tor (y/n).
|
||||
3. **Concurrent Tabs**: The number of browser tabs you want running simultaneously (defaults to 5).
|
||||
4. **Iterations**: Number of times to run the crawl.
|
||||
5. **Headless Mode**: Run browser with or without UI.
|
||||
6. **Accept Cookies**: Automatically accept cookies on pages (defaults to yes).
|
||||
|
||||
### 4. Programmatic Usage
|
||||
|
||||
You can also use WebSurfer in your own Python scripts:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from crawler import Crawler, CrawlerConfig
|
||||
|
||||
async def run_my_crawl():
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
num_tabs=4,
|
||||
headless=True,
|
||||
accept_cookies=True,
|
||||
)
|
||||
crawler = Crawler(config)
|
||||
await crawler.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_my_crawl())
|
||||
```
|
||||
|
||||
54
browser_crawler.py
Normal file
54
browser_crawler.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
WebSurfer Entry Point (browser_crawler.py)
|
||||
|
||||
Maintained for backwards compatibility with existing CLI commands and scripts.
|
||||
The crawler has been modularized under the `crawler/` package.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from main import main
|
||||
from crawler import (
|
||||
Crawler,
|
||||
CrawlerConfig,
|
||||
TorManager,
|
||||
start_tor_service,
|
||||
renew_tor_circuit,
|
||||
stop_tor_service,
|
||||
auto_accept_cookies,
|
||||
normalize_url,
|
||||
COOKIE_BANNER_SELECTORS,
|
||||
COOKIE_TEXT_PATTERNS,
|
||||
IGNORED_EXTENSIONS,
|
||||
IGNORED_PATHS,
|
||||
DEFAULT_TOR_SOCKS_PORT as TOR_SOCKS_PORT,
|
||||
DEFAULT_TOR_CONTROL_PORT as TOR_CONTROL_PORT,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"main",
|
||||
"Crawler",
|
||||
"CrawlerConfig",
|
||||
"TorManager",
|
||||
"start_tor_service",
|
||||
"renew_tor_circuit",
|
||||
"stop_tor_service",
|
||||
"auto_accept_cookies",
|
||||
"normalize_url",
|
||||
"COOKIE_BANNER_SELECTORS",
|
||||
"COOKIE_TEXT_PATTERNS",
|
||||
"IGNORED_EXTENSIONS",
|
||||
"IGNORED_PATHS",
|
||||
"TOR_SOCKS_PORT",
|
||||
"TOR_CONTROL_PORT",
|
||||
]
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\nCrawling interrupted by user.")
|
||||
sys.exit(0)
|
||||
44
crawler/__init__.py
Normal file
44
crawler/__init__.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
"""
|
||||
WebSurfer Crawler Package
|
||||
"""
|
||||
|
||||
from .config import CrawlerConfig
|
||||
from .cookies import (
|
||||
COOKIE_BANNER_SELECTORS,
|
||||
COOKIE_TEXT_PATTERNS,
|
||||
auto_accept_cookies,
|
||||
)
|
||||
from .engine import Crawler, crawl_level, worker, RandomQueue
|
||||
from .tor import (
|
||||
DEFAULT_TOR_CONTROL_PORT,
|
||||
DEFAULT_TOR_SOCKS_PORT,
|
||||
TorManager,
|
||||
renew_tor_circuit,
|
||||
start_tor_service,
|
||||
stop_tor_service,
|
||||
)
|
||||
from .url_utils import (
|
||||
IGNORED_EXTENSIONS,
|
||||
IGNORED_PATHS,
|
||||
normalize_url,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Crawler",
|
||||
"CrawlerConfig",
|
||||
"RandomQueue",
|
||||
"TorManager",
|
||||
"start_tor_service",
|
||||
"renew_tor_circuit",
|
||||
"stop_tor_service",
|
||||
"auto_accept_cookies",
|
||||
"COOKIE_BANNER_SELECTORS",
|
||||
"COOKIE_TEXT_PATTERNS",
|
||||
"normalize_url",
|
||||
"IGNORED_EXTENSIONS",
|
||||
"IGNORED_PATHS",
|
||||
"crawl_level",
|
||||
"worker",
|
||||
"DEFAULT_TOR_SOCKS_PORT",
|
||||
"DEFAULT_TOR_CONTROL_PORT",
|
||||
]
|
||||
432
crawler/config.py
Normal file
432
crawler/config.py
Normal file
|
|
@ -0,0 +1,432 @@
|
|||
import os
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from urllib.parse import urlparse
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from .url_utils import normalize_url
|
||||
|
||||
|
||||
# Modern realistic user agents for rotation to evade bot-detection and WAF blocks
|
||||
DEFAULT_USER_AGENTS: List[str] = [
|
||||
# Chrome on Windows (v133)
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
|
||||
# Chrome on macOS (v133)
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
|
||||
# Firefox on Windows (v134)
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
|
||||
# Firefox on macOS (v134)
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:134.0) Gecko/20100101 Firefox/134.0",
|
||||
# Edge on Windows (v133)
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0",
|
||||
# Safari on macOS (v18.2)
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
|
||||
]
|
||||
|
||||
|
||||
# Supported European countries with active Tor exit relays
|
||||
EUROPEAN_COUNTRIES: Dict[str, Dict[str, str]] = {
|
||||
"de": {"name": "Germany", "flag": "🇩🇪"},
|
||||
"nl": {"name": "Netherlands", "flag": "🇳🇱"},
|
||||
"fr": {"name": "France", "flag": "🇫🇷"},
|
||||
"ch": {"name": "Switzerland", "flag": "🇨🇭"},
|
||||
"at": {"name": "Austria", "flag": "🇦🇹"},
|
||||
"se": {"name": "Sweden", "flag": "🇸🇪"},
|
||||
"no": {"name": "Norway", "flag": "🇳🇴"},
|
||||
"fi": {"name": "Finland", "flag": "🇫🇮"},
|
||||
"dk": {"name": "Denmark", "flag": "🇩🇰"},
|
||||
"gb": {"name": "United Kingdom", "flag": "🇬🇧"},
|
||||
"pl": {"name": "Poland", "flag": "🇵🇱"},
|
||||
"it": {"name": "Italy", "flag": "🇮🇹"},
|
||||
"es": {"name": "Spain", "flag": "🇪🇸"},
|
||||
"be": {"name": "Belgium", "flag": "🇧🇪"},
|
||||
"cz": {"name": "Czech Republic", "flag": "🇨🇿"},
|
||||
"ro": {"name": "Romania", "flag": "🇷🇴"},
|
||||
"ie": {"name": "Ireland", "flag": "🇮🇪"},
|
||||
"pt": {"name": "Portugal", "flag": "🇵🇹"},
|
||||
"hu": {"name": "Hungary", "flag": "🇭🇺"},
|
||||
"is": {"name": "Iceland", "flag": "🇮🇸"},
|
||||
"lu": {"name": "Luxembourg", "flag": "🇱🇺"},
|
||||
"gr": {"name": "Greece", "flag": "🇬🇷"},
|
||||
"bg": {"name": "Bulgaria", "flag": "🇧🇬"},
|
||||
"ee": {"name": "Estonia", "flag": "🇪🇪"},
|
||||
"lv": {"name": "Latvia", "flag": "🇱🇻"},
|
||||
"lt": {"name": "Lithuania", "flag": "🇱🇹"},
|
||||
"sk": {"name": "Slovakia", "flag": "🇸🇰"},
|
||||
"si": {"name": "Slovenia", "flag": "🇸🇮"},
|
||||
"hr": {"name": "Croatia", "flag": "🇭🇷"},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlerConfig:
|
||||
"""Configuration settings for the crawler."""
|
||||
start_url: str
|
||||
use_tor: bool = False
|
||||
tor_circuit_mode: str = "iteration" # "iteration" (new circuit per run) or "pages" (new circuit per N pages)
|
||||
tor_circuit_pages: int = 10 # Number of pages to crawl before renewing circuit if mode is "pages"
|
||||
tor_max_consecutive_timeouts: int = 5 # Number of consecutive ERR_TIMED_OUT failures before fallback circuit renewal
|
||||
tor_exit_nodes: List[str] = field(default_factory=list) # ISO country codes (empty = any global exit node)
|
||||
num_tabs: int = 5
|
||||
num_iterations: int = 1
|
||||
iteration_mode: str = "all" # "all" (crawl all discovered websites) or "visits" (cap visits per iteration)
|
||||
max_visits_per_iteration: int = 10 # Max website visits before starting a new iteration if mode is "visits"
|
||||
randomize_queue: bool = True # Randomly pop URLs from queue instead of strict FIFO order
|
||||
headless: bool = False
|
||||
accept_cookies: bool = True
|
||||
slow_mo: int = 100
|
||||
timeout_ms: int = 35000
|
||||
tor_socks_port: int = 9050
|
||||
tor_control_port: int = 9051
|
||||
user_agent: str = "random" # "random" (auto-rotate modern desktop UAs), "default" (native browser), or custom string
|
||||
|
||||
@property
|
||||
def visits_per_iteration(self) -> int:
|
||||
"""Alias for max_visits_per_iteration."""
|
||||
return self.max_visits_per_iteration
|
||||
|
||||
@property
|
||||
def start_domain(self) -> str:
|
||||
"""Extracts the hostname/netloc from the start URL."""
|
||||
return urlparse(self.start_url).netloc
|
||||
|
||||
@property
|
||||
def tor_exit_nodes_formatted(self) -> Optional[str]:
|
||||
"""Formats exit node country codes into Tor syntax (e.g. '{de},{fr}'), or None if empty."""
|
||||
if not self.tor_exit_nodes:
|
||||
return None
|
||||
valid_codes = [c.strip().lower() for c in self.tor_exit_nodes if c.strip()]
|
||||
if not valid_codes:
|
||||
return None
|
||||
return ",".join(f"{{{c}}}" for c in valid_codes)
|
||||
|
||||
def get_user_agent(self) -> Optional[str]:
|
||||
"""
|
||||
Resolves the user agent string based on the configuration setting:
|
||||
- "random" or empty: randomly selects a modern real-world desktop User-Agent.
|
||||
- "default": returns None to instruct Playwright to use its native browser User-Agent.
|
||||
- Custom string: returns the specified string.
|
||||
"""
|
||||
if not self.user_agent or self.user_agent.strip().lower() in ["random", "rotate"]:
|
||||
return random.choice(DEFAULT_USER_AGENTS)
|
||||
if self.user_agent.strip().lower() == "default":
|
||||
return None
|
||||
return self.user_agent.strip()
|
||||
|
||||
@classmethod
|
||||
def from_env_or_prompt(cls) -> "CrawlerConfig":
|
||||
"""
|
||||
Loads configuration values from environment variables / .env file,
|
||||
falling back to interactive CLI prompts for any unset options.
|
||||
"""
|
||||
load_dotenv()
|
||||
|
||||
# 1. Starting URL
|
||||
start_url = os.getenv("CRAWLER_START_URL")
|
||||
if not start_url:
|
||||
start_url = input("Enter the starting URL (e.g., https://example.com): ").strip()
|
||||
if not start_url.startswith("http://") and not start_url.startswith("https://"):
|
||||
start_url = "https://" + start_url
|
||||
|
||||
start_domain = urlparse(start_url).netloc
|
||||
norm_start, _ = normalize_url(start_url, start_domain)
|
||||
if norm_start:
|
||||
start_url = norm_start
|
||||
|
||||
# 2. Tor Proxy
|
||||
use_tor_env = os.getenv("CRAWLER_USE_TOR")
|
||||
if use_tor_env is not None:
|
||||
use_tor = use_tor_env.strip().lower() in ["true", "1", "y", "yes"]
|
||||
else:
|
||||
use_tor_input = input(
|
||||
"Use Tor proxy (starts/connects to headless Tor on 127.0.0.1:9050)? (y/n): "
|
||||
).strip().lower()
|
||||
use_tor = (use_tor_input == "y")
|
||||
|
||||
# 2.1 Tor Circuit Rotation Strategy & Exit Nodes (if Tor is enabled)
|
||||
tor_circuit_mode = "iteration"
|
||||
tor_circuit_pages = 10
|
||||
tor_max_consecutive_timeouts = 5
|
||||
tor_exit_nodes = []
|
||||
if use_tor:
|
||||
mode_env = os.getenv("CRAWLER_TOR_CIRCUIT_MODE")
|
||||
pages_env = os.getenv("CRAWLER_TOR_CIRCUIT_PAGES")
|
||||
timeouts_env = os.getenv("CRAWLER_TOR_MAX_CONSECUTIVE_TIMEOUTS")
|
||||
nodes_env = os.getenv("CRAWLER_TOR_EXIT_NODES")
|
||||
|
||||
if mode_env is not None:
|
||||
tor_circuit_mode = "pages" if mode_env.strip().lower() in ["pages", "page", "per_pages"] else "iteration"
|
||||
else:
|
||||
print("\nTor Circuit Renewal Options:")
|
||||
print(" [1] Renew circuit before every iteration (default)")
|
||||
print(" [2] Renew circuit after every N crawled pages")
|
||||
tor_mode_choice = input("Select Tor renewal mode (1/2, default 1): ").strip()
|
||||
tor_circuit_mode = "pages" if tor_mode_choice == "2" else "iteration"
|
||||
|
||||
if tor_circuit_mode == "pages":
|
||||
if pages_env and pages_env.isdigit():
|
||||
tor_circuit_pages = max(1, int(pages_env))
|
||||
else:
|
||||
pages_input = input("How many crawled pages before renewing Tor circuit? (default 10): ").strip()
|
||||
tor_circuit_pages = int(pages_input) if pages_input and pages_input.isdigit() else 10
|
||||
|
||||
if timeouts_env and timeouts_env.isdigit():
|
||||
tor_max_consecutive_timeouts = max(1, int(timeouts_env))
|
||||
|
||||
if nodes_env is not None:
|
||||
tor_exit_nodes = [c.strip().lower() for c in nodes_env.split(",") if c.strip()]
|
||||
else:
|
||||
nodes_input = input(
|
||||
"Select European Tor Exit Node countries (comma-separated, e.g. de,nl,fr,ch) [Enter for any]: "
|
||||
).strip()
|
||||
if nodes_input:
|
||||
tor_exit_nodes = [c.strip().lower() for c in nodes_input.split(",") if c.strip()]
|
||||
|
||||
# 3. Concurrent Tabs
|
||||
num_tabs_str = os.getenv("CRAWLER_NUM_TABS")
|
||||
if not num_tabs_str:
|
||||
num_tabs_str = input("How many concurrent tabs would you like to use? (default 5): ").strip()
|
||||
num_tabs = int(num_tabs_str) if num_tabs_str and num_tabs_str.isdigit() else 5
|
||||
|
||||
# 4. Iterations
|
||||
num_iterations_str = os.getenv("CRAWLER_NUM_ITERATIONS")
|
||||
if not num_iterations_str:
|
||||
num_iterations_str = input("How many times would you like to run the crawler? (default 1): ").strip()
|
||||
num_iterations = int(num_iterations_str) if num_iterations_str and num_iterations_str.isdigit() else 1
|
||||
|
||||
# 4.1 Iteration Scope & Website Visit Cap
|
||||
iter_mode_env = os.getenv("CRAWLER_ITERATION_MODE")
|
||||
max_visits_env = os.getenv("CRAWLER_MAX_VISITS_PER_ITERATION") or os.getenv("CRAWLER_VISITS_PER_ITERATION")
|
||||
|
||||
if iter_mode_env is not None:
|
||||
iteration_mode = "visits" if iter_mode_env.strip().lower() in ["visits", "limit", "pages", "capped"] else "all"
|
||||
else:
|
||||
print("\nIteration Scope Options:")
|
||||
print(" [1] Crawl all reachable websites per iteration (default)")
|
||||
print(" [2] Limit website visits per iteration before starting a new iteration")
|
||||
iter_choice = input("Select iteration scope mode (1/2, default 1): ").strip()
|
||||
iteration_mode = "visits" if iter_choice == "2" else "all"
|
||||
|
||||
max_visits_per_iteration = 10
|
||||
if iteration_mode == "visits":
|
||||
if max_visits_env and max_visits_env.isdigit():
|
||||
max_visits_per_iteration = max(1, int(max_visits_env))
|
||||
else:
|
||||
visits_input = input("How many website visits before starting a new iteration? (default 10): ").strip()
|
||||
max_visits_per_iteration = int(visits_input) if visits_input and visits_input.isdigit() else 10
|
||||
|
||||
# 5. Headless Mode
|
||||
use_headless_env = os.getenv("CRAWLER_HEADLESS")
|
||||
if use_headless_env is not None:
|
||||
headless = use_headless_env.strip().lower() in ["true", "1", "y", "yes"]
|
||||
else:
|
||||
headless = input("Run browser in headless mode? (y/n, default n): ").strip().lower() == "y"
|
||||
|
||||
# 6. Accept Cookies
|
||||
accept_cookies_env = os.getenv("CRAWLER_ACCEPT_COOKIES")
|
||||
if accept_cookies_env is not None:
|
||||
accept_cookies = accept_cookies_env.strip().lower() in ["true", "1", "y", "yes"]
|
||||
else:
|
||||
accept_cookies_input = input("Automatically accept cookies on pages? (y/n, default y): ").strip().lower()
|
||||
accept_cookies = (accept_cookies_input != "n")
|
||||
|
||||
# 7. User Agent
|
||||
user_agent = os.getenv("CRAWLER_USER_AGENT", "random").strip()
|
||||
|
||||
# 8. Randomize Queue
|
||||
randomize_queue_env = os.getenv("CRAWLER_RANDOMIZE_QUEUE")
|
||||
if randomize_queue_env is not None:
|
||||
randomize_queue = randomize_queue_env.strip().lower() in ["true", "1", "y", "yes"]
|
||||
else:
|
||||
randomize_queue = True
|
||||
|
||||
return cls(
|
||||
start_url=start_url,
|
||||
use_tor=use_tor,
|
||||
tor_circuit_mode=tor_circuit_mode,
|
||||
tor_circuit_pages=tor_circuit_pages,
|
||||
tor_max_consecutive_timeouts=tor_max_consecutive_timeouts,
|
||||
tor_exit_nodes=tor_exit_nodes,
|
||||
num_tabs=num_tabs,
|
||||
num_iterations=num_iterations,
|
||||
iteration_mode=iteration_mode,
|
||||
max_visits_per_iteration=max_visits_per_iteration,
|
||||
randomize_queue=randomize_queue,
|
||||
headless=headless,
|
||||
accept_cookies=accept_cookies,
|
||||
user_agent=user_agent,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Serializes the configuration to a dictionary."""
|
||||
return {
|
||||
"start_url": self.start_url,
|
||||
"use_tor": self.use_tor,
|
||||
"tor_circuit_mode": self.tor_circuit_mode,
|
||||
"tor_circuit_pages": self.tor_circuit_pages,
|
||||
"tor_max_consecutive_timeouts": self.tor_max_consecutive_timeouts,
|
||||
"tor_exit_nodes": self.tor_exit_nodes,
|
||||
"num_tabs": self.num_tabs,
|
||||
"num_iterations": self.num_iterations,
|
||||
"iteration_mode": self.iteration_mode,
|
||||
"max_visits_per_iteration": self.max_visits_per_iteration,
|
||||
"randomize_queue": self.randomize_queue,
|
||||
"headless": self.headless,
|
||||
"accept_cookies": self.accept_cookies,
|
||||
"slow_mo": self.slow_mo,
|
||||
"timeout_ms": self.timeout_ms,
|
||||
"tor_socks_port": self.tor_socks_port,
|
||||
"tor_control_port": self.tor_control_port,
|
||||
"user_agent": self.user_agent,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "CrawlerConfig":
|
||||
"""Creates a CrawlerConfig from a dictionary with validation and defaults."""
|
||||
start_url = str(data.get("start_url", "")).strip()
|
||||
if start_url and not start_url.startswith("http://") and not start_url.startswith("https://"):
|
||||
start_url = "https://" + start_url
|
||||
|
||||
if start_url:
|
||||
start_domain = urlparse(start_url).netloc
|
||||
norm_start, _ = normalize_url(start_url, start_domain)
|
||||
if norm_start:
|
||||
start_url = norm_start
|
||||
|
||||
circuit_mode = str(data.get("tor_circuit_mode", "iteration")).strip().lower()
|
||||
if circuit_mode not in ["iteration", "pages"]:
|
||||
circuit_mode = "iteration"
|
||||
|
||||
try:
|
||||
circuit_pages = max(1, int(data.get("tor_circuit_pages", 10)))
|
||||
except (ValueError, TypeError):
|
||||
circuit_pages = 10
|
||||
|
||||
try:
|
||||
max_consecutive_timeouts = max(1, int(data.get("tor_max_consecutive_timeouts", 5)))
|
||||
except (ValueError, TypeError):
|
||||
max_consecutive_timeouts = 5
|
||||
|
||||
raw_exit_nodes = data.get("tor_exit_nodes", [])
|
||||
if isinstance(raw_exit_nodes, str):
|
||||
exit_nodes_list = [c.strip().lower() for c in raw_exit_nodes.split(",") if c.strip()]
|
||||
elif isinstance(raw_exit_nodes, list):
|
||||
exit_nodes_list = [str(c).strip().lower() for c in raw_exit_nodes if str(c).strip()]
|
||||
else:
|
||||
exit_nodes_list = []
|
||||
|
||||
iter_mode = str(data.get("iteration_mode", "all")).strip().lower()
|
||||
if iter_mode in ["visits", "limit", "pages", "capped"]:
|
||||
iteration_mode = "visits"
|
||||
else:
|
||||
iteration_mode = "all"
|
||||
|
||||
raw_visits = data.get("max_visits_per_iteration") or data.get("visits_per_iteration") or 10
|
||||
try:
|
||||
val = int(raw_visits)
|
||||
max_visits_per_iteration = val if val >= 1 else 10
|
||||
except (ValueError, TypeError):
|
||||
max_visits_per_iteration = 10
|
||||
|
||||
return cls(
|
||||
start_url=start_url,
|
||||
use_tor=bool(data.get("use_tor", False)),
|
||||
tor_circuit_mode=circuit_mode,
|
||||
tor_circuit_pages=circuit_pages,
|
||||
tor_max_consecutive_timeouts=max_consecutive_timeouts,
|
||||
tor_exit_nodes=exit_nodes_list,
|
||||
num_tabs=int(data.get("num_tabs", 5)),
|
||||
num_iterations=int(data.get("num_iterations", 1)),
|
||||
iteration_mode=iteration_mode,
|
||||
max_visits_per_iteration=max_visits_per_iteration,
|
||||
randomize_queue=bool(data.get("randomize_queue", True)),
|
||||
headless=bool(data.get("headless", False)),
|
||||
accept_cookies=bool(data.get("accept_cookies", True)),
|
||||
slow_mo=int(data.get("slow_mo", 100)),
|
||||
timeout_ms=int(data.get("timeout_ms", 35000)),
|
||||
tor_socks_port=int(data.get("tor_socks_port", 9050)),
|
||||
tor_control_port=int(data.get("tor_control_port", 9051)),
|
||||
user_agent=str(data.get("user_agent", "random")).strip(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_env_dict(cls, env_path: str = ".env") -> dict:
|
||||
"""Reads configuration from .env file or environment variables."""
|
||||
load_dotenv(dotenv_path=env_path, override=True)
|
||||
raw_mode = os.getenv("CRAWLER_TOR_CIRCUIT_MODE", "iteration").strip().lower()
|
||||
tor_circuit_mode = "pages" if raw_mode in ["pages", "page", "per_pages"] else "iteration"
|
||||
raw_pages = os.getenv("CRAWLER_TOR_CIRCUIT_PAGES", "10").strip()
|
||||
tor_circuit_pages = int(raw_pages) if raw_pages.isdigit() else 10
|
||||
|
||||
raw_timeouts = os.getenv("CRAWLER_TOR_MAX_CONSECUTIVE_TIMEOUTS", "5").strip()
|
||||
tor_max_consecutive_timeouts = int(raw_timeouts) if raw_timeouts.isdigit() else 5
|
||||
|
||||
raw_nodes = os.getenv("CRAWLER_TOR_EXIT_NODES", "")
|
||||
tor_exit_nodes = [c.strip().lower() for c in raw_nodes.split(",") if c.strip()]
|
||||
|
||||
raw_iter_mode = os.getenv("CRAWLER_ITERATION_MODE", "all").strip().lower()
|
||||
iteration_mode = "visits" if raw_iter_mode in ["visits", "limit", "pages", "capped"] else "all"
|
||||
|
||||
raw_max_visits = (
|
||||
os.getenv("CRAWLER_MAX_VISITS_PER_ITERATION") or
|
||||
os.getenv("CRAWLER_VISITS_PER_ITERATION") or
|
||||
"10"
|
||||
).strip()
|
||||
max_visits_per_iteration = int(raw_max_visits) if raw_max_visits.isdigit() else 10
|
||||
|
||||
return {
|
||||
"start_url": os.getenv("CRAWLER_START_URL", "https://example.com"),
|
||||
"use_tor": os.getenv("CRAWLER_USE_TOR", "false").strip().lower() in ["true", "1", "y", "yes"],
|
||||
"tor_circuit_mode": tor_circuit_mode,
|
||||
"tor_circuit_pages": max(1, tor_circuit_pages),
|
||||
"tor_max_consecutive_timeouts": max(1, tor_max_consecutive_timeouts),
|
||||
"tor_exit_nodes": tor_exit_nodes,
|
||||
"num_tabs": int(os.getenv("CRAWLER_NUM_TABS", "5")) if os.getenv("CRAWLER_NUM_TABS", "").isdigit() else 5,
|
||||
"num_iterations": int(os.getenv("CRAWLER_NUM_ITERATIONS", "1")) if os.getenv("CRAWLER_NUM_ITERATIONS", "").isdigit() else 1,
|
||||
"iteration_mode": iteration_mode,
|
||||
"max_visits_per_iteration": max(1, max_visits_per_iteration),
|
||||
"randomize_queue": os.getenv("CRAWLER_RANDOMIZE_QUEUE", "true").strip().lower() not in ["false", "0", "n", "no"],
|
||||
"headless": os.getenv("CRAWLER_HEADLESS", "false").strip().lower() in ["true", "1", "y", "yes"],
|
||||
"accept_cookies": os.getenv("CRAWLER_ACCEPT_COOKIES", "true").strip().lower() not in ["false", "0", "n", "no"],
|
||||
"slow_mo": int(os.getenv("CRAWLER_SLOW_MO", "100")) if os.getenv("CRAWLER_SLOW_MO", "").isdigit() else 100,
|
||||
"timeout_ms": int(os.getenv("CRAWLER_TIMEOUT_MS", "35000")) if os.getenv("CRAWLER_TIMEOUT_MS", "").isdigit() else 35000,
|
||||
"tor_socks_port": int(os.getenv("CRAWLER_TOR_SOCKS_PORT", "9050")) if os.getenv("CRAWLER_TOR_SOCKS_PORT", "").isdigit() else 9050,
|
||||
"tor_control_port": int(os.getenv("CRAWLER_TOR_CONTROL_PORT", "9051")) if os.getenv("CRAWLER_TOR_CONTROL_PORT", "").isdigit() else 9051,
|
||||
"user_agent": os.getenv("CRAWLER_USER_AGENT", "random").strip(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def save_env_file(cls, data: dict, env_path: str = ".env") -> None:
|
||||
"""Writes configuration key-values to a .env file."""
|
||||
raw_exit = data.get("tor_exit_nodes", [])
|
||||
if isinstance(raw_exit, list):
|
||||
exit_str = ",".join(raw_exit)
|
||||
else:
|
||||
exit_str = str(raw_exit)
|
||||
|
||||
lines = [
|
||||
f"CRAWLER_START_URL={data.get('start_url', '')}",
|
||||
f"CRAWLER_USE_TOR={'true' if data.get('use_tor') else 'false'}",
|
||||
f"CRAWLER_TOR_CIRCUIT_MODE={data.get('tor_circuit_mode', 'iteration')}",
|
||||
f"CRAWLER_TOR_CIRCUIT_PAGES={data.get('tor_circuit_pages', 10)}",
|
||||
f"CRAWLER_TOR_MAX_CONSECUTIVE_TIMEOUTS={data.get('tor_max_consecutive_timeouts', 5)}",
|
||||
f"CRAWLER_TOR_EXIT_NODES={exit_str}",
|
||||
f"CRAWLER_NUM_TABS={data.get('num_tabs', 5)}",
|
||||
f"CRAWLER_NUM_ITERATIONS={data.get('num_iterations', 1)}",
|
||||
f"CRAWLER_ITERATION_MODE={data.get('iteration_mode', 'all')}",
|
||||
f"CRAWLER_MAX_VISITS_PER_ITERATION={data.get('max_visits_per_iteration', 10)}",
|
||||
f"CRAWLER_RANDOMIZE_QUEUE={'true' if data.get('randomize_queue', True) else 'false'}",
|
||||
f"CRAWLER_HEADLESS={'true' if data.get('headless') else 'false'}",
|
||||
f"CRAWLER_ACCEPT_COOKIES={'true' if data.get('accept_cookies') else 'false'}",
|
||||
f"CRAWLER_SLOW_MO={data.get('slow_mo', 100)}",
|
||||
f"CRAWLER_TIMEOUT_MS={data.get('timeout_ms', 35000)}",
|
||||
f"CRAWLER_TOR_SOCKS_PORT={data.get('tor_socks_port', 9050)}",
|
||||
f"CRAWLER_TOR_CONTROL_PORT={data.get('tor_control_port', 9051)}",
|
||||
f"CRAWLER_USER_AGENT={data.get('user_agent', 'random')}",
|
||||
"",
|
||||
]
|
||||
with open(env_path, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
177
crawler/cookies.py
Normal file
177
crawler/cookies.py
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
import asyncio
|
||||
|
||||
# Selectors for common Consent Management Platforms (CMPs) and cookie banners
|
||||
COOKIE_BANNER_SELECTORS = [
|
||||
# Complianz (WordPress)
|
||||
"button.cmplz-accept",
|
||||
"button.cmplz-btn.cmplz-accept",
|
||||
".cmplz-btn.cmplz-accept",
|
||||
"#cmplz-cookiebanner-content .cmplz-accept",
|
||||
|
||||
# Borlabs Cookie (WordPress)
|
||||
"a._brlbs-accept-all",
|
||||
"button._brlbs-accept-all",
|
||||
"._brlbs-btn-accept-all",
|
||||
"[data-borlabs-cookie-accept-all]",
|
||||
"#borlabs-cookie-accept-all",
|
||||
".borlabs-cookie-preference-accept-all",
|
||||
|
||||
# Cookiebot
|
||||
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
|
||||
"#CybotCookiebotDialogBodyButtonAccept",
|
||||
"#CybotCookiebotDialogBodyLevelButtonAccept",
|
||||
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowallSelection",
|
||||
|
||||
# OneTrust / CookiePro
|
||||
"#onetrust-accept-btn-handler",
|
||||
"button#onetrust-accept-btn-handler",
|
||||
"#accept-recommended-btn-handler",
|
||||
|
||||
# Usercentrics
|
||||
"button[data-testid='uc-accept-all-button']",
|
||||
"#uc-btn-accept-banner",
|
||||
"button#uc-btn-accept-banner",
|
||||
"div#usercentrics-root button[data-testid='uc-accept-all-button']",
|
||||
|
||||
# Klaro
|
||||
".klaro .cm-btn-accept",
|
||||
".klaro .cm-btn-success",
|
||||
"button.cm-btn-accept-all",
|
||||
|
||||
# CookieYes
|
||||
"button.cky-btn-accept",
|
||||
"[data-cky-tag='accept-button']",
|
||||
|
||||
# Real Cookie Banner
|
||||
"button[data-consent-action='accept-all']",
|
||||
".rcb-btn-accept-all",
|
||||
|
||||
# CCM19
|
||||
".ccm--btn-accept-all",
|
||||
"button.ccm--btn-accept-all",
|
||||
"button[data-ccm-consent='accept-all']",
|
||||
|
||||
# Google Funding Choices
|
||||
".fc-cta-consent",
|
||||
"button.fc-cta-consent",
|
||||
|
||||
# Quantcast / InMobi
|
||||
".qc-cmp2-summary-buttons button[mode='primary']",
|
||||
|
||||
# Iubenda
|
||||
".iubenda-cs-accept-btn",
|
||||
"button.iubenda-cs-btn-primary",
|
||||
|
||||
# Didomi
|
||||
"#didomi-notice-agree-button",
|
||||
|
||||
# Osano
|
||||
".osano-cm-accept-all",
|
||||
"button.osano-cm-button--type_accept",
|
||||
|
||||
# Axeptio
|
||||
"#axeptio_btn_acceptAll",
|
||||
|
||||
# Generic IDs, classes, and data attributes
|
||||
"[data-action='accept-all']",
|
||||
"[data-consent='accept-all']",
|
||||
"[data-cookie-accept-all]",
|
||||
"button[id*='cookie-accept']",
|
||||
"button[id*='accept-cookies']",
|
||||
"button[id*='cookieAccept']",
|
||||
"button[class*='cookie-accept']",
|
||||
"button[class*='accept-all-cookies']",
|
||||
]
|
||||
|
||||
COOKIE_TEXT_PATTERNS = [
|
||||
# German
|
||||
r"^(alle\s+)?(cookies\s+)?akzeptieren$",
|
||||
r"^alles\s+akzeptieren$",
|
||||
r"^alle\s+annehmen$",
|
||||
r"^alles\s+annehmen$",
|
||||
r"^alle\s+ausw[aä]hlen\s+und\s+akzeptieren$",
|
||||
r"^alle\s+zulassen$",
|
||||
r"^cookies\s+zulassen$",
|
||||
r"^einverstanden$",
|
||||
r"^zustimmen$",
|
||||
r"^allen\s+zustimmen$",
|
||||
r"^alle\s+best[aä]tigen$",
|
||||
r"^auswahl\s+best[aä]tigen$",
|
||||
r"^einwilligen$",
|
||||
r"^verstanden$",
|
||||
r"^ok$",
|
||||
|
||||
# English
|
||||
r"^(accept\s+all(\s+cookies)?)$",
|
||||
r"^(allow\s+all(\s+cookies)?)$",
|
||||
r"^i\s+accept$",
|
||||
r"^i\s+agree$",
|
||||
r"^agree(\s+to\s+all)?$",
|
||||
r"^accept$",
|
||||
r"^allow$",
|
||||
r"^agree\s+&\s+proceed$",
|
||||
r"^got\s+it$",
|
||||
]
|
||||
|
||||
COOKIE_TEXT_REGEX_STR = "|".join(f"(?:{p})" for p in COOKIE_TEXT_PATTERNS)
|
||||
|
||||
|
||||
async def auto_accept_cookies(page) -> bool:
|
||||
"""
|
||||
Attempts to detect and click cookie consent / banner acceptance buttons
|
||||
using a fast, single-pass in-browser JavaScript evaluation.
|
||||
Only targets actual buttons and known CMP selectors, preventing accidental link navigations.
|
||||
"""
|
||||
js_code = """
|
||||
() => {
|
||||
const selectors = """ + str(COOKIE_BANNER_SELECTORS) + """;
|
||||
|
||||
// 1. Try known CMP selectors on main page
|
||||
for (const sel of selectors) {
|
||||
try {
|
||||
const el = document.querySelector(sel);
|
||||
if (el && (el.offsetWidth > 0 || el.offsetHeight > 0 || el.getClientRects().length > 0)) {
|
||||
el.click();
|
||||
return { clicked: true, method: 'selector', text: sel };
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
// 2. Check Shadow DOM (e.g. Usercentrics / custom web components)
|
||||
try {
|
||||
const ucRoot = document.querySelector('#usercentrics-root, div#usercentrics-cmp-ui');
|
||||
if (ucRoot && ucRoot.shadowRoot) {
|
||||
const ucBtn = ucRoot.shadowRoot.querySelector("button[data-testid='uc-accept-all-button'], button#uc-btn-accept-banner, button.uc-btn-accept");
|
||||
if (ucBtn) {
|
||||
ucBtn.click();
|
||||
return { clicked: true, method: 'shadow-dom', text: 'usercentrics-shadow' };
|
||||
}
|
||||
}
|
||||
} catch(e) {}
|
||||
|
||||
// 3. Text-based matching strictly on button / submit / role=button elements (never plain navigation links)
|
||||
const textRegex = /""" + COOKIE_TEXT_REGEX_STR + """/i;
|
||||
const candidates = document.querySelectorAll('button, input[type="button"], input[type="submit"], [role="button"], a.cmplz-accept, a._brlbs-accept-all, a.cky-btn-accept');
|
||||
for (const btn of candidates) {
|
||||
try {
|
||||
const txt = (btn.innerText || btn.value || '').trim();
|
||||
if (txt && (btn.offsetWidth > 0 || btn.offsetHeight > 0 || btn.getClientRects().length > 0) && textRegex.test(txt)) {
|
||||
btn.click();
|
||||
return { clicked: true, method: 'button-text', text: txt };
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
return { clicked: false };
|
||||
}
|
||||
"""
|
||||
try:
|
||||
result = await asyncio.wait_for(page.evaluate(js_code), timeout=2.0)
|
||||
if result and result.get("clicked"):
|
||||
print(f" [Cookie] Auto-accepted cookies via {result.get('method')}: '{result.get('text')}'")
|
||||
await page.wait_for_timeout(600) # Give tracking scripts time to transmit beacons
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
1023
crawler/engine.py
Normal file
1023
crawler/engine.py
Normal file
File diff suppressed because it is too large
Load diff
185
crawler/tor.py
Normal file
185
crawler/tor.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
import atexit
|
||||
import shutil
|
||||
import time
|
||||
from typing import Optional
|
||||
import stem.process
|
||||
from stem import Signal
|
||||
from stem.control import Controller
|
||||
|
||||
DEFAULT_TOR_SOCKS_PORT = 9050
|
||||
DEFAULT_TOR_CONTROL_PORT = 9051
|
||||
|
||||
|
||||
class TorManager:
|
||||
"""Manages the standalone headless Tor daemon process and circuit rotation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
socks_port: int = DEFAULT_TOR_SOCKS_PORT,
|
||||
control_port: int = DEFAULT_TOR_CONTROL_PORT,
|
||||
exit_nodes: Optional[str] = None,
|
||||
log_callback: Optional[object] = None,
|
||||
):
|
||||
self.socks_port = socks_port
|
||||
self.control_port = control_port
|
||||
self.exit_nodes = exit_nodes
|
||||
self.log_callback = log_callback
|
||||
self.process: Optional[object] = None
|
||||
self._is_external_service = False
|
||||
|
||||
def set_exit_nodes(self, exit_nodes: Optional[str]) -> None:
|
||||
"""Dynamically updates exit node country filter on the active Tor controller."""
|
||||
self.exit_nodes = exit_nodes
|
||||
try:
|
||||
with Controller.from_port(port=self.control_port) as controller:
|
||||
controller.authenticate()
|
||||
if self.exit_nodes:
|
||||
controller.set_conf("ExitNodes", self.exit_nodes)
|
||||
print(f"[Tor] Configured ExitNodes on Tor controller: {self.exit_nodes}")
|
||||
else:
|
||||
try:
|
||||
controller.reset_conf("ExitNodes")
|
||||
except Exception:
|
||||
pass
|
||||
print("[Tor] Reset ExitNodes filter (Global / Any)")
|
||||
except Exception as e:
|
||||
print(f" [!] Failed to set ExitNodes on ControlPort {self.control_port}: {e}")
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
Starts a background Tor process with custom SocksPort and ControlPort using stem,
|
||||
or attaches to an already running Tor service if available.
|
||||
"""
|
||||
# 1. Check if a Tor controller is already running on the control port
|
||||
try:
|
||||
with Controller.from_port(port=self.control_port) as controller:
|
||||
controller.authenticate()
|
||||
print(f"[Tor] Connected to existing Tor service on ControlPort {self.control_port}.")
|
||||
if self.exit_nodes:
|
||||
controller.set_conf("ExitNodes", self.exit_nodes)
|
||||
print(f"[Tor] Applied ExitNodes: {self.exit_nodes}")
|
||||
else:
|
||||
try:
|
||||
controller.reset_conf("ExitNodes")
|
||||
except Exception:
|
||||
pass
|
||||
self._is_external_service = True
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Check if 'tor' binary is installed
|
||||
if not shutil.which("tor"):
|
||||
raise RuntimeError(
|
||||
"Tor executable ('tor') was not found on your system PATH.\n"
|
||||
"Please install Tor:\n"
|
||||
" - macOS: brew install tor\n"
|
||||
" - Linux: sudo apt install tor\n"
|
||||
" - Windows: download Tor expert bundle from torproject.org and add to PATH"
|
||||
)
|
||||
|
||||
print(
|
||||
f"[Tor] Launching headless Tor process "
|
||||
f"(SocksPort {self.socks_port}, ControlPort {self.control_port})..."
|
||||
)
|
||||
# Always launch Tor with base config so it can bootstrap directory info and GeoIP cleanly without stalling
|
||||
tor_config = {
|
||||
"SocksPort": str(self.socks_port),
|
||||
"ControlPort": str(self.control_port),
|
||||
}
|
||||
|
||||
def _handle_tor_msg(msg: str):
|
||||
if "Bootstrapped" in msg:
|
||||
clean_msg = f"[Tor] {msg.strip()}"
|
||||
print(f" {clean_msg}")
|
||||
if self.log_callback and callable(self.log_callback):
|
||||
try:
|
||||
self.log_callback(clean_msg)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.process = stem.process.launch_tor_with_config(
|
||||
config=tor_config,
|
||||
take_ownership=False, # Managed explicitly via atexit/stop to prevent premature shutdowns
|
||||
init_msg_handler=_handle_tor_msg,
|
||||
)
|
||||
self._is_external_service = False
|
||||
|
||||
# Apply ExitNodes once Tor is successfully bootstrapped
|
||||
if self.exit_nodes:
|
||||
try:
|
||||
with Controller.from_port(port=self.control_port) as controller:
|
||||
controller.authenticate()
|
||||
controller.set_conf("ExitNodes", self.exit_nodes)
|
||||
print(f"[Tor] Applied ExitNodes filter: {self.exit_nodes}")
|
||||
except Exception as e:
|
||||
print(f" [!] Failed to apply ExitNodes filter on ControlPort {self.control_port}: {e}")
|
||||
|
||||
# Ensure background process is killed when python script exits
|
||||
atexit.register(self.stop)
|
||||
print("[Tor] Standalone Tor process started successfully!")
|
||||
return self.process
|
||||
|
||||
def renew_circuit(self) -> bool:
|
||||
"""
|
||||
Sends NEWNYM signal to Tor ControlPort to switch to a new circuit / IP.
|
||||
Waits for the necessary cool-off period required by Tor.
|
||||
Returns True if signal was sent successfully, False otherwise.
|
||||
"""
|
||||
try:
|
||||
with Controller.from_port(port=self.control_port) as controller:
|
||||
controller.authenticate()
|
||||
if self.exit_nodes:
|
||||
controller.set_conf("ExitNodes", self.exit_nodes)
|
||||
controller.signal(Signal.NEWNYM)
|
||||
wait_time = controller.get_newnym_wait()
|
||||
print(f"[Tor] Requested new circuit (NEWNYM). Waiting {wait_time:.1f}s for new circuit to activate...")
|
||||
if wait_time > 0:
|
||||
time.sleep(wait_time)
|
||||
print("[Tor] New circuit is active!")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" [!] Failed to renew Tor circuit via ControlPort {self.control_port}: {e}")
|
||||
return False
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stops the background Tor process if it was spawned by this manager."""
|
||||
if self.process and not self._is_external_service:
|
||||
try:
|
||||
if self.process.poll() is None:
|
||||
print("[Tor] Stopping headless Tor process...")
|
||||
self.process.kill()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
self.process = None
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.stop()
|
||||
|
||||
|
||||
# Backward-compatible functional interfaces
|
||||
def start_tor_service(
|
||||
socks_port: int = DEFAULT_TOR_SOCKS_PORT,
|
||||
control_port: int = DEFAULT_TOR_CONTROL_PORT,
|
||||
):
|
||||
manager = TorManager(socks_port=socks_port, control_port=control_port)
|
||||
manager.start()
|
||||
return manager
|
||||
|
||||
|
||||
def renew_tor_circuit(control_port: int = DEFAULT_TOR_CONTROL_PORT) -> None:
|
||||
TorManager(control_port=control_port).renew_circuit()
|
||||
|
||||
|
||||
def stop_tor_service(tor_process) -> None:
|
||||
if tor_process and hasattr(tor_process, "kill"):
|
||||
try:
|
||||
if tor_process.poll() is None:
|
||||
tor_process.kill()
|
||||
except Exception:
|
||||
pass
|
||||
69
crawler/url_utils.py
Normal file
69
crawler/url_utils.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
from typing import Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Non-HTML static file extensions to ignore
|
||||
IGNORED_EXTENSIONS = (
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico", ".bmp", ".tiff",
|
||||
".pdf", ".zip", ".tar", ".gz", ".bz2", ".7z", ".rar", ".exe", ".dmg", ".iso", ".bin",
|
||||
".mp3", ".mp4", ".avi", ".mov", ".mkv", ".flv", ".wmv", ".webm", ".wav", ".ogg",
|
||||
".css", ".js", ".json", ".xml", ".rss", ".atom", ".woff", ".woff2", ".ttf", ".eot", ".otf",
|
||||
".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".csv"
|
||||
)
|
||||
|
||||
# WordPress / CMS feed and API paths that are not HTML web pages
|
||||
IGNORED_PATHS = (
|
||||
"/feed", "/comments/feed", "/wp-json", "/xmlrpc.php", "/wp-admin", "/wp-login.php"
|
||||
)
|
||||
|
||||
|
||||
def normalize_url(url: str, start_domain: str) -> Tuple[Optional[str], Optional[bool]]:
|
||||
"""
|
||||
Normalizes a URL, strips fragments, filters out non-HTML files / endpoints,
|
||||
and determines whether the link is internal or external.
|
||||
|
||||
Returns:
|
||||
(normalized_url, is_internal) or (None, None) if the link should be ignored.
|
||||
"""
|
||||
if not isinstance(url, str):
|
||||
return None, None
|
||||
|
||||
url = url.split("#")[0].strip()
|
||||
if not url:
|
||||
return None, None
|
||||
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ["http", "https"]:
|
||||
return None, None
|
||||
|
||||
path = parsed.path or "/"
|
||||
path_lower = path.lower()
|
||||
|
||||
# Filter out static file extensions
|
||||
if any(path_lower.endswith(ext) for ext in IGNORED_EXTENSIONS):
|
||||
return None, None
|
||||
|
||||
# Filter out non-HTML WordPress / CMS endpoints
|
||||
if any(path_lower == p or path_lower.rstrip("/") == p or path_lower.startswith(p + "/") for p in IGNORED_PATHS):
|
||||
return None, None
|
||||
|
||||
netloc = parsed.netloc.lower()
|
||||
if not netloc:
|
||||
return None, None
|
||||
|
||||
# Check domain matching (treat apex domain and www. as identical)
|
||||
clean_start_domain = start_domain.lower().replace("www.", "")
|
||||
clean_netloc = netloc.replace("www.", "")
|
||||
|
||||
is_internal = (clean_netloc == clean_start_domain)
|
||||
|
||||
# Strip query parameters for internal links to avoid infinite loops and duplicate crawls
|
||||
if is_internal:
|
||||
norm_path = path.rstrip("/") if path != "/" else "/"
|
||||
norm_url = f"{parsed.scheme}://{netloc}{norm_path}"
|
||||
else:
|
||||
# Keep query parameters for external links (e.g., video IDs)
|
||||
norm_path = path.rstrip("/") if path != "/" else "/"
|
||||
query_part = f"?{parsed.query}" if parsed.query else ""
|
||||
norm_url = f"{parsed.scheme}://{netloc}{norm_path}{query_part}"
|
||||
|
||||
return norm_url, is_internal
|
||||
16
docker-compose.yml
Normal file
16
docker-compose.yml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
services:
|
||||
websurfer:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: websurfer
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${WEB_PORT:-8080}:8000"
|
||||
shm_size: "2gb"
|
||||
environment:
|
||||
- CRAWLER_HEADLESS=true
|
||||
- HOST=0.0.0.0
|
||||
- PORT=8000
|
||||
volumes:
|
||||
- ./.env:/app/.env
|
||||
17
entrypoint.sh
Normal file
17
entrypoint.sh
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "[Docker] Starting Tor background service..."
|
||||
service tor start
|
||||
|
||||
# Quick check that Tor SOCKS5 & Control ports are active
|
||||
for i in {1..10}; do
|
||||
if python3 -c "import socket; s = socket.socket(); s.connect(('127.0.0.1', 9050)); s.close()" 2>/dev/null; then
|
||||
echo "[Docker] Tor SOCKS proxy is listening on port 9050!"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Execute main process (e.g. web server or CLI)
|
||||
exec "$@"
|
||||
35
main.py
Normal file
35
main.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from crawler import Crawler, CrawlerConfig
|
||||
|
||||
|
||||
async def run_cli():
|
||||
config = CrawlerConfig.from_env_or_prompt()
|
||||
crawler = Crawler(config)
|
||||
await crawler.run()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Clickduck - Web Crawler & Link Explorer")
|
||||
parser.add_argument("--web", action="store_true", help="Launch the Web Frontend dashboard")
|
||||
parser.add_argument("--host", type=str, default=os.getenv("HOST", "127.0.0.1"), help="Host IP for web UI (default: 127.0.0.1 or $HOST)")
|
||||
parser.add_argument("--port", type=int, default=int(os.getenv("PORT", "8000")), help="Port for web UI (default: 8000 or $PORT)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.web:
|
||||
from web import run_server
|
||||
run_server(host=args.host, port=args.port)
|
||||
else:
|
||||
try:
|
||||
asyncio.run(run_cli())
|
||||
except KeyboardInterrupt:
|
||||
print("\nCrawling interrupted by user.")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
|
||||
main()
|
||||
5
requirements.txt
Normal file
5
requirements.txt
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
playwright>=1.40.0
|
||||
python-dotenv
|
||||
stem>=1.8.0
|
||||
fastapi>=0.110.0
|
||||
uvicorn[standard]>=0.28.0
|
||||
372
test_iteration_visits.py
Normal file
372
test_iteration_visits.py
Normal file
|
|
@ -0,0 +1,372 @@
|
|||
import asyncio
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
from crawler.config import CrawlerConfig
|
||||
from crawler.engine import Crawler, IterationVisitTracker
|
||||
|
||||
|
||||
class TestIterationVisitsConfig(unittest.TestCase):
|
||||
|
||||
def test_default_config(self):
|
||||
config = CrawlerConfig(start_url="https://example.com")
|
||||
self.assertEqual(config.iteration_mode, "all")
|
||||
self.assertEqual(config.max_visits_per_iteration, 10)
|
||||
self.assertEqual(config.visits_per_iteration, 10)
|
||||
self.assertTrue(config.randomize_queue)
|
||||
|
||||
def test_dict_serialization(self):
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
num_iterations=3,
|
||||
iteration_mode="visits",
|
||||
max_visits_per_iteration=25,
|
||||
randomize_queue=False,
|
||||
)
|
||||
d = config.to_dict()
|
||||
self.assertEqual(d["iteration_mode"], "visits")
|
||||
self.assertEqual(d["max_visits_per_iteration"], 25)
|
||||
self.assertFalse(d["randomize_queue"])
|
||||
|
||||
# Restore from dict
|
||||
restored = CrawlerConfig.from_dict(d)
|
||||
self.assertEqual(restored.iteration_mode, "visits")
|
||||
self.assertEqual(restored.max_visits_per_iteration, 25)
|
||||
self.assertEqual(restored.visits_per_iteration, 25)
|
||||
self.assertFalse(restored.randomize_queue)
|
||||
|
||||
def test_from_dict_alias_and_normalization(self):
|
||||
# Using visits_per_iteration alias
|
||||
cfg1 = CrawlerConfig.from_dict({
|
||||
"start_url": "https://example.com",
|
||||
"iteration_mode": "limit",
|
||||
"visits_per_iteration": 15,
|
||||
"randomize_queue": True,
|
||||
})
|
||||
self.assertEqual(cfg1.iteration_mode, "visits")
|
||||
self.assertEqual(cfg1.max_visits_per_iteration, 15)
|
||||
self.assertTrue(cfg1.randomize_queue)
|
||||
|
||||
# Non-visits mode defaults to "all"
|
||||
cfg2 = CrawlerConfig.from_dict({
|
||||
"start_url": "https://example.com",
|
||||
"iteration_mode": "unknown_mode",
|
||||
"max_visits_per_iteration": -5,
|
||||
})
|
||||
self.assertEqual(cfg2.iteration_mode, "all")
|
||||
self.assertEqual(cfg2.max_visits_per_iteration, 10)
|
||||
self.assertTrue(cfg2.randomize_queue)
|
||||
|
||||
def test_env_dict_loading(self):
|
||||
with patch.dict(os.environ, {
|
||||
"CRAWLER_START_URL": "https://test.com",
|
||||
"CRAWLER_ITERATION_MODE": "visits",
|
||||
"CRAWLER_MAX_VISITS_PER_ITERATION": "42",
|
||||
"CRAWLER_RANDOMIZE_QUEUE": "false",
|
||||
}):
|
||||
env_data = CrawlerConfig.get_env_dict(env_path="nonexistent_test.env")
|
||||
self.assertEqual(env_data["iteration_mode"], "visits")
|
||||
self.assertEqual(env_data["max_visits_per_iteration"], 42)
|
||||
self.assertFalse(env_data["randomize_queue"])
|
||||
|
||||
|
||||
class TestIterationVisitTracker(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_unlimited_tracker(self):
|
||||
tracker = IterationVisitTracker(max_visits=None)
|
||||
for _ in range(50):
|
||||
can_visit = await tracker.acquire_visit_slot()
|
||||
self.assertTrue(can_visit)
|
||||
await tracker.record_visit_completed()
|
||||
self.assertFalse(tracker.limit_reached_event.is_set())
|
||||
|
||||
async def test_limited_tracker_slot_reservation_and_completion(self):
|
||||
limit = 3
|
||||
tracker = IterationVisitTracker(max_visits=limit)
|
||||
|
||||
# Acquire 3 slots
|
||||
self.assertTrue(await tracker.acquire_visit_slot())
|
||||
self.assertTrue(await tracker.acquire_visit_slot())
|
||||
self.assertTrue(await tracker.acquire_visit_slot())
|
||||
|
||||
# 4th acquisition attempt fails
|
||||
self.assertFalse(await tracker.acquire_visit_slot())
|
||||
self.assertEqual(tracker.started_visits, 3)
|
||||
self.assertFalse(tracker.limit_reached_event.is_set())
|
||||
|
||||
# Record completions
|
||||
await tracker.record_visit_completed()
|
||||
self.assertFalse(tracker.limit_reached_event.is_set())
|
||||
await tracker.record_visit_completed()
|
||||
self.assertFalse(tracker.limit_reached_event.is_set())
|
||||
|
||||
# 3rd completion triggers limit_reached_event
|
||||
await tracker.record_visit_completed()
|
||||
self.assertTrue(tracker.limit_reached_event.is_set())
|
||||
self.assertEqual(tracker.completed_visits, 3)
|
||||
|
||||
|
||||
class TestCrawlerIterationVisitCap(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_iteration_stops_at_configured_visits(self):
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
num_tabs=3,
|
||||
num_iterations=1,
|
||||
iteration_mode="visits",
|
||||
max_visits_per_iteration=3,
|
||||
)
|
||||
|
||||
visited_urls = []
|
||||
|
||||
async def mock_crawl_level(
|
||||
page,
|
||||
url,
|
||||
source_url,
|
||||
visited,
|
||||
errors,
|
||||
start_domain,
|
||||
accept_cookies_default=True,
|
||||
stats=None,
|
||||
timeout_ms=35000,
|
||||
event_callback=None,
|
||||
worker_name="Worker",
|
||||
page_result_callback=None,
|
||||
):
|
||||
visited_urls.append(url)
|
||||
visited.add(url)
|
||||
await asyncio.sleep(0.01)
|
||||
# Return new links to simulate queue populated with many items
|
||||
return {
|
||||
f"https://example.com/sub_{i}_{len(visited_urls)}" for i in range(5)
|
||||
}
|
||||
|
||||
crawler = Crawler(config=config)
|
||||
|
||||
mock_page = MagicMock()
|
||||
mock_page.close = AsyncMock()
|
||||
mock_page.on = MagicMock()
|
||||
|
||||
mock_context = MagicMock()
|
||||
mock_context.new_page = AsyncMock(return_value=mock_page)
|
||||
|
||||
mock_browser = MagicMock()
|
||||
mock_browser.new_context = AsyncMock(return_value=mock_context)
|
||||
mock_browser.close = AsyncMock()
|
||||
|
||||
mock_playwright = MagicMock()
|
||||
mock_playwright.chromium.launch = AsyncMock(return_value=mock_browser)
|
||||
|
||||
with patch("crawler.engine.crawl_level", side_effect=mock_crawl_level):
|
||||
result = await crawler.run_iteration(mock_playwright, iteration=1)
|
||||
|
||||
# Verified that exactly max_visits_per_iteration (3) were crawled despite many discovered links
|
||||
self.assertEqual(len(visited_urls), 3)
|
||||
self.assertEqual(result["stats"]["internal"], 3)
|
||||
|
||||
async def test_multi_iteration_stops_at_configured_visits_each(self):
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
num_tabs=2,
|
||||
num_iterations=2,
|
||||
iteration_mode="visits",
|
||||
max_visits_per_iteration=2,
|
||||
)
|
||||
|
||||
visited_urls_per_iteration = []
|
||||
|
||||
async def mock_crawl_level(
|
||||
page,
|
||||
url,
|
||||
source_url,
|
||||
visited,
|
||||
errors,
|
||||
start_domain,
|
||||
accept_cookies_default=True,
|
||||
stats=None,
|
||||
timeout_ms=35000,
|
||||
event_callback=None,
|
||||
worker_name="Worker",
|
||||
page_result_callback=None,
|
||||
):
|
||||
visited_urls_per_iteration.append(url)
|
||||
visited.add(url)
|
||||
await asyncio.sleep(0.01)
|
||||
return {f"https://example.com/link_{i}" for i in range(5)}
|
||||
|
||||
crawler = Crawler(config=config)
|
||||
|
||||
mock_page = MagicMock()
|
||||
mock_page.close = AsyncMock()
|
||||
mock_page.on = MagicMock()
|
||||
|
||||
mock_context = MagicMock()
|
||||
mock_context.new_page = AsyncMock(return_value=mock_page)
|
||||
|
||||
mock_browser = MagicMock()
|
||||
mock_browser.new_context = AsyncMock(return_value=mock_context)
|
||||
mock_browser.close = AsyncMock()
|
||||
|
||||
mock_playwright = MagicMock()
|
||||
mock_playwright.chromium.launch = AsyncMock(return_value=mock_browser)
|
||||
|
||||
with patch("crawler.engine.crawl_level", side_effect=mock_crawl_level):
|
||||
# Iteration 1
|
||||
res1 = await crawler.run_iteration(mock_playwright, iteration=1)
|
||||
self.assertEqual(len(visited_urls_per_iteration), 2)
|
||||
self.assertEqual(res1["stats"]["internal"], 2)
|
||||
|
||||
visited_urls_per_iteration.clear()
|
||||
|
||||
# Iteration 2
|
||||
res2 = await crawler.run_iteration(mock_playwright, iteration=2)
|
||||
self.assertEqual(len(visited_urls_per_iteration), 2)
|
||||
self.assertEqual(res2["stats"]["internal"], 2)
|
||||
|
||||
|
||||
class TestRandomQueue(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_random_queue_pops_all_items(self):
|
||||
from crawler.engine import RandomQueue
|
||||
|
||||
q = RandomQueue()
|
||||
items = [f"https://example.com/page{i}" for i in range(20)]
|
||||
for it in items:
|
||||
q.put_nowait(it)
|
||||
|
||||
self.assertEqual(q.qsize(), 20)
|
||||
popped = []
|
||||
for _ in range(20):
|
||||
popped.append(await q.get())
|
||||
q.task_done()
|
||||
|
||||
await q.join()
|
||||
self.assertEqual(sorted(popped), sorted(items))
|
||||
self.assertEqual(q.qsize(), 0)
|
||||
|
||||
async def test_crawler_uses_random_queue_when_configured(self):
|
||||
from crawler.engine import RandomQueue
|
||||
|
||||
cfg_random = CrawlerConfig(start_url="https://example.com", randomize_queue=True)
|
||||
crawler_random = Crawler(config=cfg_random)
|
||||
|
||||
mock_page = MagicMock()
|
||||
mock_page.close = AsyncMock()
|
||||
mock_page.on = MagicMock()
|
||||
mock_context = MagicMock()
|
||||
mock_context.new_page = AsyncMock(return_value=mock_page)
|
||||
mock_browser = MagicMock()
|
||||
mock_browser.new_context = AsyncMock(return_value=mock_context)
|
||||
mock_browser.close = AsyncMock()
|
||||
mock_playwright = MagicMock()
|
||||
mock_playwright.chromium.launch = AsyncMock(return_value=mock_browser)
|
||||
|
||||
# Intercept queue creation by checking instance type
|
||||
async def dummy_crawl(*args, **kwargs):
|
||||
return set()
|
||||
|
||||
with patch("crawler.engine.crawl_level", side_effect=dummy_crawl):
|
||||
with patch("crawler.engine.RandomQueue", wraps=RandomQueue) as mock_rq:
|
||||
await crawler_random.run_iteration(mock_playwright, iteration=1)
|
||||
self.assertTrue(mock_rq.called)
|
||||
|
||||
|
||||
class TestCumulativeAllIterationsStatistics(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
async def test_cumulative_visited_and_errors_across_iterations(self):
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
num_tabs=2,
|
||||
num_iterations=2,
|
||||
iteration_mode="visits",
|
||||
max_visits_per_iteration=3,
|
||||
)
|
||||
|
||||
events = []
|
||||
|
||||
async def event_cb(ev):
|
||||
events.append(ev)
|
||||
|
||||
crawler = Crawler(config=config, event_callback=event_cb)
|
||||
|
||||
visited_per_call = 0
|
||||
|
||||
async def mock_crawl_level(
|
||||
page,
|
||||
url,
|
||||
source_url,
|
||||
visited,
|
||||
errors,
|
||||
start_domain,
|
||||
accept_cookies_default=True,
|
||||
stats=None,
|
||||
timeout_ms=35000,
|
||||
event_callback=None,
|
||||
worker_name="Worker",
|
||||
page_result_callback=None,
|
||||
):
|
||||
nonlocal visited_per_call
|
||||
visited_per_call += 1
|
||||
visited.add(url)
|
||||
if visited_per_call == 2:
|
||||
errors.append((url, source_url, "Simulated 500 error"))
|
||||
await asyncio.sleep(0.01)
|
||||
return {f"https://example.com/link_{visited_per_call}_{i}" for i in range(3)}
|
||||
|
||||
mock_page = MagicMock()
|
||||
mock_page.close = AsyncMock()
|
||||
mock_page.on = MagicMock()
|
||||
mock_context = MagicMock()
|
||||
mock_context.new_page = AsyncMock(return_value=mock_page)
|
||||
mock_browser = MagicMock()
|
||||
mock_browser.new_context = AsyncMock(return_value=mock_context)
|
||||
mock_browser.close = AsyncMock()
|
||||
mock_playwright = MagicMock()
|
||||
mock_playwright.chromium.launch = AsyncMock(return_value=mock_browser)
|
||||
|
||||
with patch("crawler.engine.crawl_level", side_effect=mock_crawl_level):
|
||||
res1 = await crawler.run_iteration(mock_playwright, iteration=1)
|
||||
self.assertEqual(res1["visited_count"], 3)
|
||||
self.assertEqual(res1["total_visited_all"], 3)
|
||||
self.assertEqual(res1["total_errors_all"], 1)
|
||||
|
||||
res2 = await crawler.run_iteration(mock_playwright, iteration=2)
|
||||
self.assertEqual(res2["visited_count"], 3)
|
||||
self.assertEqual(res2["total_visited_all"], 6)
|
||||
self.assertEqual(crawler.total_visited_all_iterations, 6)
|
||||
|
||||
# Check emitted stats events
|
||||
stats_events = [e for e in events if e.get("type") == "stats"]
|
||||
self.assertTrue(len(stats_events) > 0)
|
||||
final_stats_event = stats_events[-1]
|
||||
self.assertEqual(final_stats_event["total_visited_all"], 6)
|
||||
|
||||
async def test_session_manager_snapshot_and_broadcast(self):
|
||||
from web.manager import CrawlSessionManager
|
||||
|
||||
manager = CrawlSessionManager()
|
||||
snapshot = manager.get_snapshot()
|
||||
self.assertIn("total_visited_all", snapshot)
|
||||
self.assertIn("total_errors_all", snapshot)
|
||||
self.assertEqual(snapshot["total_visited_all"], 0)
|
||||
self.assertEqual(snapshot["total_errors_all"], 0)
|
||||
|
||||
# Simulate receiving crawler stats event
|
||||
await manager.handle_crawler_event({
|
||||
"type": "stats",
|
||||
"visited_count": 5,
|
||||
"total_visited_all": 15,
|
||||
"total_errors_all": 2,
|
||||
"queue_size": 3,
|
||||
"stats": {"internal": 5, "external": 0, "cookies_accepted": 0},
|
||||
})
|
||||
self.assertEqual(manager.total_visited_all, 15)
|
||||
self.assertEqual(manager.total_errors_all, 2)
|
||||
snapshot_after = manager.get_snapshot()
|
||||
self.assertEqual(snapshot_after["total_visited_all"], 15)
|
||||
self.assertEqual(snapshot_after["total_errors_all"], 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
178
test_tor_fallback.py
Normal file
178
test_tor_fallback.py
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
import asyncio
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
from crawler.config import CrawlerConfig
|
||||
from crawler.engine import Crawler
|
||||
|
||||
|
||||
class TestTorTimeoutFallback(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
def test_config_defaults_and_env(self):
|
||||
config = CrawlerConfig(start_url="https://example.com")
|
||||
self.assertEqual(config.tor_max_consecutive_timeouts, 5)
|
||||
|
||||
dict_data = config.to_dict()
|
||||
self.assertEqual(dict_data["tor_max_consecutive_timeouts"], 5)
|
||||
|
||||
dict_data["tor_max_consecutive_timeouts"] = 3
|
||||
restored = CrawlerConfig.from_dict(dict_data)
|
||||
self.assertEqual(restored.tor_max_consecutive_timeouts, 3)
|
||||
|
||||
async def test_consecutive_timeout_fallback_trigger(self):
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
use_tor=True,
|
||||
tor_max_consecutive_timeouts=5,
|
||||
)
|
||||
events = []
|
||||
async def on_event(ev):
|
||||
events.append(ev)
|
||||
|
||||
crawler = Crawler(config=config, event_callback=on_event)
|
||||
# Mock tor_manager so we don't need real Tor daemon for unit testing
|
||||
crawler.tor_manager = MagicMock()
|
||||
crawler.tor_manager.renew_circuit = MagicMock(return_value=True)
|
||||
|
||||
mock_context = MagicMock()
|
||||
|
||||
# 1. First 4 consecutive ERR_TIMED_OUT errors
|
||||
for i in range(1, 5):
|
||||
await crawler._on_page_result(
|
||||
context=mock_context,
|
||||
url=f"https://example.com/page{i}",
|
||||
source_url="https://example.com",
|
||||
success=False,
|
||||
error_msg="page.goto: net::ERR_TIMED_OUT at https://example.com",
|
||||
)
|
||||
self.assertEqual(crawler._consecutive_timeouts, i)
|
||||
self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 0)
|
||||
|
||||
# 2. 5th ERR_TIMED_OUT error triggers fallback circuit rotation!
|
||||
await crawler._on_page_result(
|
||||
context=mock_context,
|
||||
url="https://example.com/page5",
|
||||
source_url="https://example.com",
|
||||
success=False,
|
||||
error_msg="page.goto: net::ERR_TIMED_OUT at https://example.com",
|
||||
)
|
||||
# Counter should be reset to 0 after rotation
|
||||
self.assertEqual(crawler._consecutive_timeouts, 0)
|
||||
self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 1)
|
||||
|
||||
async def test_success_resets_counter(self):
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
use_tor=True,
|
||||
tor_max_consecutive_timeouts=5,
|
||||
)
|
||||
crawler = Crawler(config=config)
|
||||
crawler.tor_manager = MagicMock()
|
||||
crawler.tor_manager.renew_circuit = MagicMock(return_value=True)
|
||||
|
||||
mock_context = MagicMock()
|
||||
|
||||
# 3 timeouts
|
||||
for i in range(1, 4):
|
||||
await crawler._on_page_result(
|
||||
context=mock_context,
|
||||
url=f"https://example.com/page{i}",
|
||||
source_url=None,
|
||||
success=False,
|
||||
error_msg="net::ERR_TIMED_OUT",
|
||||
)
|
||||
self.assertEqual(crawler._consecutive_timeouts, 3)
|
||||
|
||||
# 1 success -> should reset counter to 0
|
||||
await crawler._on_page_result(
|
||||
context=mock_context,
|
||||
url="https://example.com/success_page",
|
||||
source_url=None,
|
||||
success=True,
|
||||
error_msg=None,
|
||||
)
|
||||
self.assertEqual(crawler._consecutive_timeouts, 0)
|
||||
self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 0)
|
||||
|
||||
async def test_non_timeout_error_resets_counter(self):
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
use_tor=True,
|
||||
tor_max_consecutive_timeouts=5,
|
||||
)
|
||||
crawler = Crawler(config=config)
|
||||
crawler.tor_manager = MagicMock()
|
||||
|
||||
mock_context = MagicMock()
|
||||
|
||||
# 2 timeouts
|
||||
for i in range(2):
|
||||
await crawler._on_page_result(
|
||||
context=mock_context,
|
||||
url=f"https://example.com/page{i}",
|
||||
source_url=None,
|
||||
success=False,
|
||||
error_msg="net::ERR_TIMED_OUT",
|
||||
)
|
||||
self.assertEqual(crawler._consecutive_timeouts, 2)
|
||||
|
||||
# HTTP 404 error -> non-timeout/non-rate-limit resets counter
|
||||
await crawler._on_page_result(
|
||||
context=mock_context,
|
||||
url="https://example.com/missing",
|
||||
source_url=None,
|
||||
success=False,
|
||||
error_msg="HTTP 404",
|
||||
)
|
||||
self.assertEqual(crawler._consecutive_timeouts, 0)
|
||||
|
||||
def test_user_agent_config(self):
|
||||
config_random = CrawlerConfig(start_url="https://example.com", user_agent="random")
|
||||
ua1 = config_random.get_user_agent()
|
||||
self.assertIsNotNone(ua1)
|
||||
self.assertIn("Mozilla/5.0", ua1)
|
||||
|
||||
config_custom = CrawlerConfig(start_url="https://example.com", user_agent="MyCustomBot/1.0")
|
||||
self.assertEqual(config_custom.get_user_agent(), "MyCustomBot/1.0")
|
||||
|
||||
config_default = CrawlerConfig(start_url="https://example.com", user_agent="default")
|
||||
self.assertIsNone(config_default.get_user_agent())
|
||||
|
||||
async def test_consecutive_rate_limit_fallback_trigger(self):
|
||||
config = CrawlerConfig(
|
||||
start_url="https://example.com",
|
||||
use_tor=True,
|
||||
tor_max_consecutive_timeouts=3,
|
||||
)
|
||||
crawler = Crawler(config=config)
|
||||
crawler.tor_manager = MagicMock()
|
||||
crawler.tor_manager.renew_circuit = MagicMock(return_value=True)
|
||||
|
||||
mock_context = MagicMock()
|
||||
|
||||
# 2 HTTP 503 errors
|
||||
for i in range(2):
|
||||
await crawler._on_page_result(
|
||||
context=mock_context,
|
||||
url=f"https://example.com/page{i}",
|
||||
source_url="https://example.com",
|
||||
success=False,
|
||||
error_msg="HTTP 503 (Service Temporarily Unavailable / Rate Limited) [Retry-After: 154s]",
|
||||
)
|
||||
self.assertEqual(crawler._consecutive_timeouts, 2)
|
||||
self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 0)
|
||||
|
||||
# 3rd HTTP 503 error triggers circuit renewal
|
||||
await crawler._on_page_result(
|
||||
context=mock_context,
|
||||
url="https://example.com/page2",
|
||||
source_url="https://example.com",
|
||||
success=False,
|
||||
error_msg="HTTP 503 (Service Temporarily Unavailable / Rate Limited) [Retry-After: 154s]",
|
||||
)
|
||||
self.assertEqual(crawler._consecutive_timeouts, 0)
|
||||
self.assertEqual(crawler.tor_manager.renew_circuit.call_count, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
18
web.py
Normal file
18
web.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from web import run_server
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
|
||||
|
||||
parser = argparse.ArgumentParser(description="Websurfer Web Frontend Server")
|
||||
parser.add_argument("--host", type=str, default=os.getenv("HOST", "127.0.0.1"), help="Host IP address (default: 127.0.0.1 or $HOST)")
|
||||
parser.add_argument("--port", type=int, default=int(os.getenv("PORT", "8000")), help="Port number (default: 8000 or $PORT)")
|
||||
parser.add_argument("--reload", action="store_true", help="Enable auto-reload for development")
|
||||
args = parser.parse_args()
|
||||
|
||||
run_server(host=args.host, port=args.port, reload=args.reload)
|
||||
|
||||
26
web/__init__.py
Normal file
26
web/__init__.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
"""Web application package for Websurfer."""
|
||||
from web.app import app
|
||||
from web.manager import session_manager
|
||||
|
||||
|
||||
def run_server(host: str = "127.0.0.1", port: int = 8000, reload: bool = False):
|
||||
"""Starts the Websurfer FastAPI Web Dashboard server."""
|
||||
import sys
|
||||
import uvicorn
|
||||
print("\n=======================================================")
|
||||
print(f"[*] Websurfer Web UI running at http://{host}:{port}")
|
||||
print(" Open this URL in your browser to configure & crawl.")
|
||||
|
||||
# On Windows, Uvicorn defaults to SelectorEventLoop when reload=True or workers > 1.
|
||||
# Playwright requires ProactorEventLoop on Windows for asyncio subprocess support.
|
||||
loop_arg = "asyncio:ProactorEventLoop" if sys.platform == "win32" else "auto"
|
||||
uvicorn.run(
|
||||
"web.app:app" if reload else app,
|
||||
host=host,
|
||||
port=port,
|
||||
reload=reload,
|
||||
proxy_headers=True,
|
||||
forwarded_allow_ips="*",
|
||||
loop=loop_arg,
|
||||
)
|
||||
|
||||
218
web/app.py
Normal file
218
web/app.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
import csv
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Depends, Request, Response, status
|
||||
from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, RedirectResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from crawler.config import CrawlerConfig
|
||||
from .manager import session_manager
|
||||
from .auth import (
|
||||
is_auth_enabled,
|
||||
verify_credentials,
|
||||
create_session_token,
|
||||
verify_session_token,
|
||||
get_current_user,
|
||||
verify_websocket,
|
||||
SESSION_COOKIE_NAME,
|
||||
SESSION_MAX_AGE_SECONDS,
|
||||
)
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent
|
||||
STATIC_DIR = BASE_DIR / "static"
|
||||
|
||||
app = FastAPI(title="WebSurfer Control Center", version="1.0.0")
|
||||
|
||||
# Mount static assets directory
|
||||
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def root(request: Request):
|
||||
"""Serves the main dashboard user interface, redirecting to login if unauthenticated."""
|
||||
if is_auth_enabled():
|
||||
token = request.cookies.get(SESSION_COOKIE_NAME)
|
||||
if not verify_session_token(token):
|
||||
return RedirectResponse(url="/login", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
|
||||
|
||||
index_path = STATIC_DIR / "index.html"
|
||||
if not index_path.exists():
|
||||
raise HTTPException(status_code=404, detail="Dashboard UI not found")
|
||||
with open(index_path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
@app.get("/login", response_class=HTMLResponse)
|
||||
async def login_page(request: Request):
|
||||
"""Serves the login page, redirecting to / if already authenticated."""
|
||||
if is_auth_enabled():
|
||||
token = request.cookies.get(SESSION_COOKIE_NAME)
|
||||
if verify_session_token(token):
|
||||
return RedirectResponse(url="/", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
|
||||
|
||||
login_path = STATIC_DIR / "login.html"
|
||||
if not login_path.exists():
|
||||
raise HTTPException(status_code=404, detail="Login UI not found")
|
||||
with open(login_path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
@app.post("/api/login")
|
||||
async def api_login(payload: Dict[str, str], response: Response, request: Request):
|
||||
"""Authenticates credentials and establishes a signed session cookie."""
|
||||
if not is_auth_enabled():
|
||||
return {"success": True, "message": "Authentication not enabled"}
|
||||
|
||||
username = payload.get("username", "")
|
||||
password = payload.get("password", "")
|
||||
|
||||
if not verify_credentials(username, password):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid username or password",
|
||||
)
|
||||
|
||||
token = create_session_token(username)
|
||||
is_secure = request.headers.get("x-forwarded-proto", "").lower() == "https" or request.url.scheme == "https"
|
||||
|
||||
response.set_cookie(
|
||||
key=SESSION_COOKIE_NAME,
|
||||
value=token,
|
||||
max_age=SESSION_MAX_AGE_SECONDS,
|
||||
httponly=True,
|
||||
samesite="lax",
|
||||
secure=is_secure,
|
||||
path="/",
|
||||
)
|
||||
return {"success": True, "username": username}
|
||||
|
||||
|
||||
@app.post("/api/logout")
|
||||
async def api_logout(response: Response):
|
||||
"""Clears the session cookie."""
|
||||
response.delete_cookie(key=SESSION_COOKIE_NAME, path="/")
|
||||
return {"success": True, "message": "Logged out successfully"}
|
||||
|
||||
|
||||
@app.get("/api/auth/me")
|
||||
async def auth_me(request: Request):
|
||||
"""Returns current authentication state."""
|
||||
enabled = is_auth_enabled()
|
||||
if not enabled:
|
||||
return {"authenticated": True, "auth_enabled": False, "username": "anonymous"}
|
||||
|
||||
token = request.cookies.get(SESSION_COOKIE_NAME)
|
||||
username = verify_session_token(token)
|
||||
return {
|
||||
"authenticated": username is not None,
|
||||
"auth_enabled": True,
|
||||
"username": username,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/config", dependencies=[Depends(get_current_user)])
|
||||
async def get_config():
|
||||
"""Fetches the current crawler configuration stored in .env or defaults."""
|
||||
env_config = CrawlerConfig.get_env_dict()
|
||||
return {"success": True, "config": env_config}
|
||||
|
||||
|
||||
@app.get("/api/tor/countries", dependencies=[Depends(get_current_user)])
|
||||
async def get_tor_countries():
|
||||
"""Returns the list of supported European countries for Tor exit node selection."""
|
||||
from crawler.config import EUROPEAN_COUNTRIES
|
||||
countries = [
|
||||
{"code": code, "name": info["name"], "flag": info["flag"]}
|
||||
for code, info in EUROPEAN_COUNTRIES.items()
|
||||
]
|
||||
return {"success": True, "countries": countries}
|
||||
|
||||
|
||||
@app.post("/api/config", dependencies=[Depends(get_current_user)])
|
||||
async def save_config(payload: Dict[str, Any]):
|
||||
"""Persists updated configuration values to the .env file."""
|
||||
try:
|
||||
CrawlerConfig.save_env_file(payload)
|
||||
return {"success": True, "message": "Configuration successfully saved to .env"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save .env file: {e}")
|
||||
|
||||
|
||||
@app.post("/api/crawl/start", dependencies=[Depends(get_current_user)])
|
||||
async def start_crawl(payload: Dict[str, Any]):
|
||||
"""Starts a new crawler run with given parameters."""
|
||||
result = await session_manager.start_crawl(payload)
|
||||
if not result.get("success"):
|
||||
raise HTTPException(status_code=400, detail=result.get("message"))
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/api/crawl/stop", dependencies=[Depends(get_current_user)])
|
||||
async def stop_crawl():
|
||||
"""Stops the currently running crawler job."""
|
||||
result = await session_manager.stop_crawl()
|
||||
if not result.get("success"):
|
||||
raise HTTPException(status_code=400, detail=result.get("message"))
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/api/crawl/status", dependencies=[Depends(get_current_user)])
|
||||
async def get_status():
|
||||
"""Returns the current state, metrics, errors, and recent logs."""
|
||||
return session_manager.get_snapshot()
|
||||
|
||||
|
||||
@app.post("/api/crawl/clear-logs", dependencies=[Depends(get_current_user)])
|
||||
async def clear_logs():
|
||||
"""Clears buffered in-memory logs."""
|
||||
session_manager.clear_logs()
|
||||
return {"success": True}
|
||||
|
||||
|
||||
@app.get("/api/crawl/export-errors", dependencies=[Depends(get_current_user)])
|
||||
async def export_errors(format: str = "csv"):
|
||||
"""Exports all recorded errors from current session as CSV or JSON."""
|
||||
errors = session_manager.errors
|
||||
if format.lower() == "json":
|
||||
return JSONResponse(content={"total_errors": len(errors), "errors": errors})
|
||||
|
||||
# Default to CSV export
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(["ID", "Failed URL", "Error Reason", "Found On (Source URL)", "Timestamp"])
|
||||
for item in errors:
|
||||
writer.writerow([
|
||||
item.get("id", ""),
|
||||
item.get("url", ""),
|
||||
item.get("error", ""),
|
||||
item.get("source_url", ""),
|
||||
item.get("timestamp", ""),
|
||||
])
|
||||
|
||||
output.seek(0)
|
||||
return StreamingResponse(
|
||||
iter([output.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": "attachment; filename=websurfer_errors.csv"},
|
||||
)
|
||||
|
||||
|
||||
@app.websocket("/ws/stream")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
"""WebSocket stream for real-time log, metrics, and error updates."""
|
||||
if not verify_websocket(websocket):
|
||||
await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
|
||||
return
|
||||
|
||||
await session_manager.connect_ws(websocket)
|
||||
try:
|
||||
while True:
|
||||
# Keep connection open and accept any control pings from client
|
||||
data = await websocket.receive_text()
|
||||
except WebSocketDisconnect:
|
||||
session_manager.disconnect_ws(websocket)
|
||||
except Exception:
|
||||
session_manager.disconnect_ws(websocket)
|
||||
121
web/auth.py
Normal file
121
web/auth.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, HTTPException, Request, WebSocket, status
|
||||
from fastapi.responses import RedirectResponse
|
||||
|
||||
SESSION_COOKIE_NAME = "websurfer_session"
|
||||
SESSION_MAX_AGE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||
|
||||
|
||||
def is_auth_enabled() -> bool:
|
||||
"""Checks whether authentication is enabled via environment variables."""
|
||||
flag = os.getenv("AUTH_ENABLED", "").strip().lower()
|
||||
has_credentials = bool(os.getenv("AUTH_USERNAME") and os.getenv("AUTH_PASSWORD"))
|
||||
return flag in ("true", "1", "yes") and has_credentials
|
||||
|
||||
|
||||
def get_secret_key() -> str:
|
||||
"""Returns the secret key used to HMAC-sign session cookies."""
|
||||
return os.getenv("AUTH_SECRET_KEY") or "websurfer-default-secret-key-please-change-in-prod"
|
||||
|
||||
|
||||
def verify_credentials(username: str, password: str) -> bool:
|
||||
"""Verifies user credentials using constant-time string comparison."""
|
||||
expected_user = os.getenv("AUTH_USERNAME", "")
|
||||
expected_pass = os.getenv("AUTH_PASSWORD", "")
|
||||
|
||||
if not expected_user or not expected_pass:
|
||||
return False
|
||||
|
||||
user_match = secrets.compare_digest(username.strip(), expected_user.strip())
|
||||
pass_match = secrets.compare_digest(password, expected_pass)
|
||||
return user_match and pass_match
|
||||
|
||||
|
||||
def create_session_token(username: str) -> str:
|
||||
"""Generates a tamper-proof signed session token (username:timestamp:signature)."""
|
||||
timestamp = str(int(time.time()))
|
||||
payload = f"{username}:{timestamp}"
|
||||
signature = hmac.new(
|
||||
get_secret_key().encode("utf-8"),
|
||||
payload.encode("utf-8"),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
return f"{payload}:{signature}"
|
||||
|
||||
|
||||
def verify_session_token(token: Optional[str]) -> Optional[str]:
|
||||
"""Validates the signature and timestamp of a session token."""
|
||||
if not token or ":" not in token:
|
||||
return None
|
||||
|
||||
parts = token.split(":")
|
||||
if len(parts) != 3:
|
||||
return None
|
||||
|
||||
username, timestamp_str, signature = parts
|
||||
|
||||
try:
|
||||
token_time = int(timestamp_str)
|
||||
# Check token expiration
|
||||
if time.time() - token_time > SESSION_MAX_AGE_SECONDS or token_time > time.time() + 300:
|
||||
return None
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
payload = f"{username}:{timestamp_str}"
|
||||
expected_sig = hmac.new(
|
||||
get_secret_key().encode("utf-8"),
|
||||
payload.encode("utf-8"),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
|
||||
if not secrets.compare_digest(signature, expected_sig):
|
||||
return None
|
||||
|
||||
return username
|
||||
|
||||
|
||||
async def get_current_user(request: Request) -> Optional[str]:
|
||||
"""FastAPI dependency: verifies user session from cookie or Bearer token."""
|
||||
if not is_auth_enabled():
|
||||
return "anonymous"
|
||||
|
||||
# 1. Check session cookie
|
||||
token = request.cookies.get(SESSION_COOKIE_NAME)
|
||||
|
||||
# 2. Check Authorization Bearer header as fallback
|
||||
if not token:
|
||||
auth_header = request.headers.get("Authorization")
|
||||
if auth_header and auth_header.startswith("Bearer "):
|
||||
token = auth_header[7:].strip()
|
||||
|
||||
username = verify_session_token(token)
|
||||
if not username:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Authentication required",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
return username
|
||||
|
||||
|
||||
def verify_websocket(websocket: WebSocket) -> bool:
|
||||
"""Validates WebSocket connection session from cookie or query parameter."""
|
||||
if not is_auth_enabled():
|
||||
return True
|
||||
|
||||
# Check cookie from handshake request
|
||||
token = websocket.cookies.get(SESSION_COOKIE_NAME)
|
||||
|
||||
# Check query param as alternative (?token=...)
|
||||
if not token:
|
||||
token = websocket.query_params.get("token")
|
||||
|
||||
return verify_session_token(token) is not None
|
||||
302
web/manager.py
Normal file
302
web/manager.py
Normal file
|
|
@ -0,0 +1,302 @@
|
|||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
from typing import Dict, List, Optional, Set, Any
|
||||
from fastapi import WebSocket
|
||||
|
||||
from crawler.config import CrawlerConfig
|
||||
from crawler.engine import Crawler
|
||||
|
||||
|
||||
class CrawlSessionManager:
|
||||
"""Manages active crawl sessions, WebSocket broadcasting, metrics, and error aggregation."""
|
||||
|
||||
def __init__(self):
|
||||
self.active_connections: Set[WebSocket] = set()
|
||||
self.crawler: Optional[Crawler] = None
|
||||
self.task: Optional[asyncio.Task] = None
|
||||
self.status: str = "idle" # "idle" | "running" | "stopping" | "finished" | "error"
|
||||
self.status_message: str = "Ready to start"
|
||||
self.start_time: Optional[float] = None
|
||||
self.end_time: Optional[float] = None
|
||||
self.current_iteration: int = 1
|
||||
self.total_iterations: int = 1
|
||||
self.iteration_mode: str = "all"
|
||||
self.max_visits_per_iteration: int = 10
|
||||
|
||||
self.stats: Dict[str, int] = {
|
||||
"internal": 0,
|
||||
"external": 0,
|
||||
"cookies_accepted": 0,
|
||||
}
|
||||
self.queue_size: int = 0
|
||||
self.visited_count: int = 0
|
||||
self.total_visited_all: int = 0
|
||||
self.total_errors_all: int = 0
|
||||
self.tor_identity: Optional[Dict[str, Any]] = None
|
||||
|
||||
self.errors: List[Dict[str, Any]] = []
|
||||
self.logs: List[Dict[str, Any]] = []
|
||||
self._log_counter: int = 0
|
||||
self._error_counter: int = 0
|
||||
|
||||
async def connect_ws(self, websocket: WebSocket):
|
||||
"""Registers a new WebSocket connection and transmits initial state snapshot."""
|
||||
await websocket.accept()
|
||||
self.active_connections.add(websocket)
|
||||
# Send current state snapshot immediately upon connection
|
||||
snapshot = self.get_snapshot()
|
||||
await websocket.send_json({
|
||||
"type": "init",
|
||||
"data": snapshot,
|
||||
})
|
||||
|
||||
def disconnect_ws(self, websocket: WebSocket):
|
||||
"""Removes a disconnected WebSocket."""
|
||||
self.active_connections.discard(websocket)
|
||||
|
||||
async def broadcast(self, message: Dict[str, Any]):
|
||||
"""Dispatches an event payload to all connected WebSockets."""
|
||||
disconnected = set()
|
||||
for ws in self.active_connections:
|
||||
try:
|
||||
await ws.send_json(message)
|
||||
except Exception:
|
||||
disconnected.add(ws)
|
||||
|
||||
for ws in disconnected:
|
||||
self.active_connections.discard(ws)
|
||||
|
||||
def get_snapshot(self) -> Dict[str, Any]:
|
||||
"""Generates a complete snapshot of current crawler session."""
|
||||
elapsed = 0.0
|
||||
if self.start_time:
|
||||
if self.end_time:
|
||||
elapsed = self.end_time - self.start_time
|
||||
else:
|
||||
elapsed = time.time() - self.start_time
|
||||
|
||||
return {
|
||||
"status": self.status,
|
||||
"status_message": self.status_message,
|
||||
"elapsed_seconds": round(elapsed, 1),
|
||||
"current_iteration": self.current_iteration,
|
||||
"total_iterations": self.total_iterations,
|
||||
"iteration_mode": self.iteration_mode,
|
||||
"max_visits_per_iteration": self.max_visits_per_iteration,
|
||||
"stats": self.stats,
|
||||
"queue_size": self.queue_size,
|
||||
"visited_count": self.visited_count,
|
||||
"total_visited_all": self.total_visited_all,
|
||||
"total_errors_all": max(self.total_errors_all, len(self.errors)),
|
||||
"tor_identity": self.tor_identity,
|
||||
"error_count": len(self.errors),
|
||||
"errors": self.errors[-100:], # last 100 errors for quick load
|
||||
"logs": self.logs[-500:], # last 500 logs for quick load
|
||||
}
|
||||
|
||||
async def handle_crawler_event(self, event: Dict[str, Any]):
|
||||
"""Processes events dispatched by the core crawler engine."""
|
||||
event_type = event.get("type")
|
||||
|
||||
if event_type == "log":
|
||||
self._log_counter += 1
|
||||
log_entry = {
|
||||
"id": self._log_counter,
|
||||
"message": event.get("message", ""),
|
||||
"level": event.get("level", "info"),
|
||||
"timestamp": event.get("timestamp", time.time()),
|
||||
}
|
||||
self.logs.append(log_entry)
|
||||
if len(self.logs) > 2000:
|
||||
self.logs = self.logs[-1500:]
|
||||
await self.broadcast({"type": "log", "data": log_entry})
|
||||
|
||||
elif event_type == "stats":
|
||||
self.stats = event.get("stats", self.stats)
|
||||
self.queue_size = event.get("queue_size", self.queue_size)
|
||||
self.visited_count = event.get("visited_count", self.visited_count)
|
||||
if "total_visited_all" in event:
|
||||
self.total_visited_all = event["total_visited_all"]
|
||||
else:
|
||||
self.total_visited_all = max(self.total_visited_all, self.visited_count)
|
||||
|
||||
if "total_errors_all" in event:
|
||||
self.total_errors_all = event["total_errors_all"]
|
||||
else:
|
||||
self.total_errors_all = len(self.errors)
|
||||
|
||||
await self.broadcast({
|
||||
"type": "stats",
|
||||
"data": {
|
||||
"stats": self.stats,
|
||||
"queue_size": self.queue_size,
|
||||
"visited_count": self.visited_count,
|
||||
"error_count": event.get("error_count", 0),
|
||||
"total_visited_all": self.total_visited_all,
|
||||
"total_errors_all": self.total_errors_all,
|
||||
},
|
||||
})
|
||||
|
||||
elif event_type == "tor_identity":
|
||||
self.tor_identity = event.get("data", {})
|
||||
await self.broadcast({
|
||||
"type": "tor_identity",
|
||||
"data": self.tor_identity,
|
||||
})
|
||||
|
||||
elif event_type == "error":
|
||||
self._error_counter += 1
|
||||
error_entry = {
|
||||
"id": self._error_counter,
|
||||
"url": event.get("url", ""),
|
||||
"source_url": event.get("source_url") or "Root / Direct Input",
|
||||
"error": event.get("error", "Unknown error"),
|
||||
"timestamp": event.get("timestamp", time.time()),
|
||||
}
|
||||
self.errors.append(error_entry)
|
||||
self.total_errors_all = len(self.errors)
|
||||
await self.broadcast({
|
||||
"type": "error",
|
||||
"data": {
|
||||
**error_entry,
|
||||
"total_errors_all": self.total_errors_all,
|
||||
},
|
||||
})
|
||||
|
||||
elif event_type == "iteration":
|
||||
self.current_iteration = event.get("iteration", 1)
|
||||
self.total_iterations = event.get("total", 1)
|
||||
if "iteration_mode" in event:
|
||||
self.iteration_mode = event["iteration_mode"]
|
||||
if "max_visits_per_iteration" in event:
|
||||
self.max_visits_per_iteration = event["max_visits_per_iteration"]
|
||||
if "total_visited_all" in event:
|
||||
self.total_visited_all = event["total_visited_all"]
|
||||
if "total_errors_all" in event:
|
||||
self.total_errors_all = event["total_errors_all"]
|
||||
await self.broadcast({
|
||||
"type": "iteration",
|
||||
"data": {
|
||||
"current": self.current_iteration,
|
||||
"total": self.total_iterations,
|
||||
"iteration_mode": self.iteration_mode,
|
||||
"max_visits_per_iteration": self.max_visits_per_iteration,
|
||||
"total_visited_all": self.total_visited_all,
|
||||
"total_errors_all": self.total_errors_all,
|
||||
},
|
||||
})
|
||||
|
||||
elif event_type == "status":
|
||||
self.status = event.get("status", self.status)
|
||||
self.status_message = event.get("message", "")
|
||||
if "total_visited_all" in event:
|
||||
self.total_visited_all = event["total_visited_all"]
|
||||
if "total_errors_all" in event:
|
||||
self.total_errors_all = event["total_errors_all"]
|
||||
if self.status in ["finished", "stopped", "error"]:
|
||||
self.end_time = time.time()
|
||||
|
||||
await self.broadcast({
|
||||
"type": "status",
|
||||
"data": {
|
||||
"status": self.status,
|
||||
"message": self.status_message,
|
||||
"elapsed": round((self.end_time or time.time()) - (self.start_time or time.time()), 1),
|
||||
"total_visited_all": self.total_visited_all,
|
||||
"total_errors_all": self.total_errors_all,
|
||||
},
|
||||
})
|
||||
|
||||
async def start_crawl(self, config_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Configures and initiates a new crawl in a background task."""
|
||||
if self.status == "running":
|
||||
return {"success": False, "message": "Crawler is already running."}
|
||||
|
||||
# Safeguard: Verify Windows event loop supports subprocesses for Playwright
|
||||
if sys.platform == "win32":
|
||||
loop = asyncio.get_running_loop()
|
||||
if not isinstance(loop, getattr(asyncio, "ProactorEventLoop", ())):
|
||||
err_msg = (
|
||||
"Current asyncio event loop does not support subprocesses on Windows "
|
||||
"(ProactorEventLoop is required for Playwright). "
|
||||
"Please launch Websurfer using 'python web.py' or ensure Uvicorn uses '--loop asyncio:ProactorEventLoop'."
|
||||
)
|
||||
return {"success": False, "message": err_msg}
|
||||
|
||||
# Clear session data
|
||||
self.stats = {"internal": 0, "external": 0, "cookies_accepted": 0}
|
||||
self.queue_size = 0
|
||||
self.visited_count = 0
|
||||
self.total_visited_all = 0
|
||||
self.total_errors_all = 0
|
||||
self.tor_identity = None
|
||||
|
||||
self.errors = []
|
||||
self.logs = []
|
||||
self._log_counter = 0
|
||||
self._error_counter = 0
|
||||
self.start_time = time.time()
|
||||
self.end_time = None
|
||||
self.status = "running"
|
||||
self.status_message = "Initializing crawler..."
|
||||
self.current_iteration = 1
|
||||
self.total_iterations = int(config_data.get("num_iterations", 1))
|
||||
|
||||
# Build configuration object
|
||||
config = CrawlerConfig.from_dict(config_data)
|
||||
self.iteration_mode = config.iteration_mode
|
||||
self.max_visits_per_iteration = config.max_visits_per_iteration
|
||||
|
||||
# Instantiate crawler with event callback
|
||||
self.crawler = Crawler(config, event_callback=self.handle_crawler_event)
|
||||
|
||||
async def _run_task():
|
||||
try:
|
||||
await self.crawler.run()
|
||||
except Exception as e:
|
||||
await self.handle_crawler_event({
|
||||
"type": "status",
|
||||
"status": "error",
|
||||
"message": f"Execution error: {e}",
|
||||
})
|
||||
finally:
|
||||
if self.status == "running":
|
||||
self.status = "finished"
|
||||
self.end_time = time.time()
|
||||
await self.broadcast({
|
||||
"type": "status",
|
||||
"data": {
|
||||
"status": self.status,
|
||||
"message": "Completed",
|
||||
"elapsed": round(self.end_time - self.start_time, 1),
|
||||
},
|
||||
})
|
||||
|
||||
self.task = asyncio.create_task(_run_task())
|
||||
await self.broadcast({"type": "init", "data": self.get_snapshot()})
|
||||
return {"success": True, "message": "Crawl started"}
|
||||
|
||||
async def stop_crawl(self) -> Dict[str, Any]:
|
||||
"""Stops the active crawler job."""
|
||||
if self.status != "running" or not self.crawler:
|
||||
return {"success": False, "message": "No active crawl to stop."}
|
||||
|
||||
self.status = "stopping"
|
||||
self.status_message = "Cancelling workers..."
|
||||
await self.broadcast({
|
||||
"type": "status",
|
||||
"data": {"status": "stopping", "message": "Cancelling workers..."},
|
||||
})
|
||||
|
||||
self.crawler.stop()
|
||||
return {"success": True, "message": "Stop signal sent."}
|
||||
|
||||
def clear_logs(self):
|
||||
"""Clears in-memory logs."""
|
||||
self.logs = []
|
||||
self._log_counter = 0
|
||||
|
||||
|
||||
# Global singleton manager instance
|
||||
session_manager = CrawlSessionManager()
|
||||
1019
web/static/app.js
Normal file
1019
web/static/app.js
Normal file
File diff suppressed because it is too large
Load diff
BIN
web/static/clickduck_logo.png
Normal file
BIN
web/static/clickduck_logo.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 546 KiB |
BIN
web/static/clickduck_logo_small.jpg
Normal file
BIN
web/static/clickduck_logo_small.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 104 KiB |
580
web/static/index.html
Normal file
580
web/static/index.html
Normal file
|
|
@ -0,0 +1,580 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Clickduck Control Center</title>
|
||||
<link rel="icon" type="image/png" href="/static/clickduck_logo.png">
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link
|
||||
href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500;600&display=swap"
|
||||
rel="stylesheet">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
</head>
|
||||
|
||||
<body class="dark-theme">
|
||||
<div class="app-container">
|
||||
<!-- Header -->
|
||||
<header class="app-header">
|
||||
<div class="brand">
|
||||
<div class="logo-icon" style="background: none; padding: 0; box-shadow: none;">
|
||||
<img src="/static/clickduck_logo.png" alt="Clickduck" style="width: 38px; height: 38px; border-radius: 50%; object-fit: cover; box-shadow: 0 0 12px var(--primary-glow); border: 1.5px solid rgba(56, 189, 248, 0.4);" onerror="this.src='/static/clickduck_logo_small.jpg'">
|
||||
</div>
|
||||
<div>
|
||||
<h1 class="brand-title">Clickduck <span class="badge-tag">v1.0</span></h1>
|
||||
<p class="brand-subtitle">Autonomous Playwright Web Crawler & Broken Link Detector</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="header-status">
|
||||
<div id="wsStatus" class="ws-pill connecting" title="WebSocket Connection">
|
||||
<span class="status-dot"></span>
|
||||
<span class="ws-text">Connecting...</span>
|
||||
</div>
|
||||
<div id="crawlStatusBadge" class="status-badge idle">
|
||||
<span class="pulse-ring"></span>
|
||||
<span class="status-label">IDLE</span>
|
||||
</div>
|
||||
<div id="userProfile" class="user-profile" style="display: none; align-items: center; gap: 0.5rem; margin-left: 0.5rem;">
|
||||
<span id="userName" style="font-size: 0.8rem; color: var(--text-muted); font-weight: 500;"></span>
|
||||
<button id="btnLogout" class="btn btn-secondary btn-sm" title="Log Out" style="padding: 0.35rem 0.65rem; font-size: 0.8rem; border-radius: var(--radius-sm); display: flex; align-items: center; gap: 0.3rem;">
|
||||
<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M9 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h4"></path>
|
||||
<polyline points="16 17 21 12 16 7"></polyline>
|
||||
<line x1="21" y1="12" x2="9" y2="12"></line>
|
||||
</svg>
|
||||
<span>Logout</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- Main Content Layout -->
|
||||
<main class="dashboard-grid">
|
||||
|
||||
<!-- Top Left: Crawler Configuration -->
|
||||
<section class="card config-card">
|
||||
<div class="card-header">
|
||||
<div class="header-title">
|
||||
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path
|
||||
d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z">
|
||||
</path>
|
||||
<circle cx="12" cy="12" r="3"></circle>
|
||||
</svg>
|
||||
<h2>Crawler Configuration (.env)</h2>
|
||||
</div>
|
||||
<span class="header-hint">Synced with .env</span>
|
||||
</div>
|
||||
|
||||
<form id="configForm" class="config-form">
|
||||
<!-- Start URL Input -->
|
||||
<div class="form-group full-width">
|
||||
<label for="startUrl">Start Target URL <span class="required">*</span></label>
|
||||
<div class="input-with-icon">
|
||||
<span class="input-icon">🌐</span>
|
||||
<input type="url" id="startUrl" name="start_url" placeholder="https://example.com" required>
|
||||
</div>
|
||||
<small class="field-help">The root URL where the crawler begins crawling and finding internal/external
|
||||
links.</small>
|
||||
</div>
|
||||
|
||||
<!-- Numeric controls grid -->
|
||||
<div class="form-row">
|
||||
<div class="form-group">
|
||||
<label for="numTabs">Concurrent Tabs: <span id="numTabsVal" class="val-badge">5</span></label>
|
||||
<input type="range" id="numTabs" name="num_tabs" min="1" max="20" value="5" class="range-slider">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="numIterations">Iterations</label>
|
||||
<input type="number" id="numIterations" name="num_iterations" min="1" max="100" value="1"
|
||||
class="form-input">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Iteration Scope Strategy -->
|
||||
<div class="iteration-mode-section" style="margin-bottom: 16px;">
|
||||
<div class="subgroup-header" style="margin-bottom: 8px;">
|
||||
<span class="subgroup-title">🎯 Iteration Scope & Website Visits</span>
|
||||
</div>
|
||||
<div class="circuit-mode-selector">
|
||||
<label class="radio-card active" id="labelIterModeAll">
|
||||
<input type="radio" name="iteration_mode" value="all" id="iterModeAll" checked>
|
||||
<div class="radio-card-body">
|
||||
<div class="radio-card-header">
|
||||
<span class="radio-icon">🌐</span>
|
||||
<span class="radio-card-title">All Websites</span>
|
||||
</div>
|
||||
<span class="radio-card-desc">Crawl all discovered internal/external links per iteration</span>
|
||||
</div>
|
||||
</label>
|
||||
|
||||
<label class="radio-card" id="labelIterModeVisits">
|
||||
<input type="radio" name="iteration_mode" value="visits" id="iterModeVisits">
|
||||
<div class="radio-card-body">
|
||||
<div class="radio-card-header">
|
||||
<span class="radio-icon">🎯</span>
|
||||
<span class="radio-card-title">Limit Visits</span>
|
||||
</div>
|
||||
<span class="radio-card-desc">Visit up to N pages before starting next iteration</span>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div id="iterVisitsField" class="form-group" style="display: none; margin-top: 10px;">
|
||||
<label for="maxVisitsPerIteration">Website Visits per Iteration:</label>
|
||||
<div class="input-with-stepper">
|
||||
<span class="input-icon">🎯</span>
|
||||
<input type="number" id="maxVisitsPerIteration" name="max_visits_per_iteration" min="1" max="10000" value="10"
|
||||
class="form-input">
|
||||
<span class="input-unit">visits</span>
|
||||
</div>
|
||||
<small class="field-help">Caps page visits in this iteration. Once reached, Websurfer immediately starts the next iteration.</small>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Toggles Grid -->
|
||||
<div class="toggles-grid">
|
||||
<label class="toggle-card">
|
||||
<div class="toggle-info">
|
||||
<span class="toggle-title">Headless Mode</span>
|
||||
<span class="toggle-desc">Run browser in background without visible window</span>
|
||||
</div>
|
||||
<input type="checkbox" id="headless" name="headless" class="toggle-checkbox">
|
||||
<span class="toggle-switch"></span>
|
||||
</label>
|
||||
|
||||
<label class="toggle-card">
|
||||
<div class="toggle-info">
|
||||
<span class="toggle-title">Accept Cookie Banners</span>
|
||||
<span class="toggle-desc">Auto-clicks "Accept All", "Allow", "Agree" dialogs</span>
|
||||
</div>
|
||||
<input type="checkbox" id="acceptCookies" name="accept_cookies" checked class="toggle-checkbox">
|
||||
<span class="toggle-switch"></span>
|
||||
</label>
|
||||
|
||||
<label class="toggle-card">
|
||||
<div class="toggle-info">
|
||||
<span class="toggle-title">Tor Proxy Routing</span>
|
||||
<span class="toggle-desc">Route all Chromium traffic through SOCKS5 127.0.0.1:9050</span>
|
||||
</div>
|
||||
<input type="checkbox" id="useTor" name="use_tor" class="toggle-checkbox">
|
||||
<span class="toggle-switch"></span>
|
||||
</label>
|
||||
|
||||
<label class="toggle-card">
|
||||
<div class="toggle-info">
|
||||
<span class="toggle-title">Randomize Queue</span>
|
||||
<span class="toggle-desc">Pick random URLs from queue instead of FIFO order</span>
|
||||
</div>
|
||||
<input type="checkbox" id="randomizeQueue" name="randomize_queue" checked class="toggle-checkbox">
|
||||
<span class="toggle-switch"></span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<!-- Tor Circuit Strategy Sub-Section (Visible when Tor toggle is active) -->
|
||||
<div id="torOptionsGroup" class="tor-options-subgroup" style="display: none;">
|
||||
<div class="subgroup-header">
|
||||
<span class="subgroup-title">🧅 Tor Circuit Rotation Strategy</span>
|
||||
</div>
|
||||
|
||||
<div class="circuit-mode-selector">
|
||||
<label class="radio-card active" id="labelTorModeIteration">
|
||||
<input type="radio" name="tor_circuit_mode" value="iteration" id="torModeIteration" checked>
|
||||
<div class="radio-card-body">
|
||||
<div class="radio-card-header">
|
||||
<span class="radio-icon">🔄</span>
|
||||
<span class="radio-card-title">Per Iteration</span>
|
||||
</div>
|
||||
<span class="radio-card-desc">Request fresh Tor circuit before each crawl iteration</span>
|
||||
</div>
|
||||
</label>
|
||||
|
||||
<label class="radio-card" id="labelTorModePages">
|
||||
<input type="radio" name="tor_circuit_mode" value="pages" id="torModePages">
|
||||
<div class="radio-card-body">
|
||||
<div class="radio-card-header">
|
||||
<span class="radio-icon">📄</span>
|
||||
<span class="radio-card-title">Per N Pages</span>
|
||||
</div>
|
||||
<span class="radio-card-desc">Rotate Tor circuit dynamically after crawling N pages</span>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div id="torPagesField" class="form-group tor-pages-field" style="display: none;">
|
||||
<label for="torCircuitPages">Rotate Circuit Every (Pages):</label>
|
||||
<div class="input-with-stepper">
|
||||
<span class="input-icon">📄</span>
|
||||
<input type="number" id="torCircuitPages" name="tor_circuit_pages" min="1" max="1000" value="10"
|
||||
class="form-input">
|
||||
<span class="input-unit">pages</span>
|
||||
</div>
|
||||
<small class="field-help">Requests a new Tor exit IP & circuit after every N pages crawled across worker
|
||||
tabs.</small>
|
||||
</div>
|
||||
|
||||
<div class="form-group tor-fallback-field" style="margin-top: 10px;">
|
||||
<label for="torMaxConsecutiveTimeouts">Timeout Fallback (Consecutive Timeouts):</label>
|
||||
<div class="input-with-stepper">
|
||||
<span class="input-icon">⏱</span>
|
||||
<input type="number" id="torMaxConsecutiveTimeouts" name="tor_max_consecutive_timeouts" min="1" max="100" value="5"
|
||||
class="form-input">
|
||||
<span class="input-unit">timeouts</span>
|
||||
</div>
|
||||
<small class="field-help">Automatically requests a fresh Tor circuit if 5 consecutive pages fail with net::ERR_TIMED_OUT.</small>
|
||||
</div>
|
||||
|
||||
<!-- European Exit Node Country Selector -->
|
||||
<div class="tor-exit-nodes-section">
|
||||
<div class="exit-nodes-header">
|
||||
<div class="exit-nodes-title-group">
|
||||
<span class="subgroup-title">🎯 European Exit Node Countries</span>
|
||||
<span id="exitNodesCountBadge" class="count-pill-sm">Any / Global</span>
|
||||
</div>
|
||||
<div class="country-action-btns">
|
||||
<button type="button" id="btnExitMajor" class="btn-chip-action"
|
||||
title="Select Germany, Netherlands, France, Switzerland">Major EU</button>
|
||||
<button type="button" id="btnExitAll" class="btn-chip-action"
|
||||
title="Select all European countries">All EU</button>
|
||||
<button type="button" id="btnExitClear" class="btn-chip-action danger"
|
||||
title="Clear selection to use any Tor node globally">Clear (Any)</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="countryChipsGrid" class="country-chips-grid">
|
||||
<!-- Country chips populated dynamically by app.js -->
|
||||
</div>
|
||||
|
||||
<div class="exit-nodes-summary">
|
||||
<span class="summary-icon">📍</span>
|
||||
<span id="exitNodesSummaryText" class="summary-text">Active Exit Filter: Worldwide / Any Tor exit
|
||||
node</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Collapsible Advanced Settings -->
|
||||
<details class="advanced-section">
|
||||
<summary class="advanced-summary">
|
||||
<span>⚙️ Advanced Crawler Options</span>
|
||||
<span class="chevron">▼</span>
|
||||
</summary>
|
||||
<div class="advanced-body form-row">
|
||||
<div class="form-group" style="grid-column: span 2;">
|
||||
<label for="userAgentSelect">Browser User-Agent</label>
|
||||
<select id="userAgentSelect" class="form-input">
|
||||
<option value="random">🔄 Auto-Rotate Realistic Desktop User-Agents (Recommended)</option>
|
||||
<option value="default">🌐 Native Chromium Browser Default</option>
|
||||
<option value="custom">✏️ Custom User-Agent String...</option>
|
||||
</select>
|
||||
<input type="text" id="userAgentCustom" name="user_agent" value="random" placeholder="Enter custom User-Agent string..." class="form-input" style="display: none; margin-top: 6px;">
|
||||
<small class="field-help">Rotates real modern Chrome, Firefox, Safari & Edge headers to avoid WAF rate-limiting and bot blocks (HTTP 503/429/403).</small>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="timeoutMs">Timeout (ms)</label>
|
||||
<input type="number" id="timeoutMs" name="timeout_ms" value="35000" step="1000" class="form-input">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="slowMo">Slow-Mo Delay (ms)</label>
|
||||
<input type="number" id="slowMo" name="slow_mo" value="100" step="50" class="form-input">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="torSocksPort">Tor SOCKS Port</label>
|
||||
<input type="number" id="torSocksPort" name="tor_socks_port" value="9050" class="form-input">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="torControlPort">Tor Control Port</label>
|
||||
<input type="number" id="torControlPort" name="tor_control_port" value="9051" class="form-input">
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Form Actions -->
|
||||
<div class="form-actions">
|
||||
<button type="submit" id="btnStart" class="btn btn-primary">
|
||||
<svg width="18" height="18" viewBox="0 0 24 24" fill="currentColor">
|
||||
<polygon points="5 3 19 12 5 21 5 3"></polygon>
|
||||
</svg>
|
||||
<span>Start Crawling</span>
|
||||
</button>
|
||||
|
||||
<button type="button" id="btnStop" class="btn btn-danger" disabled>
|
||||
<svg width="18" height="18" viewBox="0 0 24 24" fill="currentColor">
|
||||
<rect x="4" y="4" width="16" height="16" rx="2"></rect>
|
||||
</svg>
|
||||
<span>Stop Crawl</span>
|
||||
</button>
|
||||
|
||||
<button type="button" id="btnSaveEnv" class="btn btn-secondary" title="Save current parameters to .env">
|
||||
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"></path>
|
||||
<polyline points="17 21 17 13 7 13 7 21"></polyline>
|
||||
<polyline points="7 3 7 8 15 8"></polyline>
|
||||
</svg>
|
||||
<span>Save .env</span>
|
||||
</button>
|
||||
|
||||
<button type="button" id="btnReset" class="btn btn-ghost" title="Reset form to saved .env">
|
||||
<span>Reset</span>
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<!-- Top Right: Live Session Metrics -->
|
||||
<section class="card stats-card">
|
||||
<div class="card-header">
|
||||
<div class="header-title">
|
||||
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<line x1="18" y1="20" x2="18" y2="10"></line>
|
||||
<line x1="12" y1="20" x2="12" y2="4"></line>
|
||||
<line x1="6" y1="20" x2="6" y2="14"></line>
|
||||
</svg>
|
||||
<h2>Live Session Metrics</h2>
|
||||
</div>
|
||||
<div class="timer-badge">
|
||||
<span class="timer-icon">⏱</span>
|
||||
<span id="elapsedTimer">00:00:00</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="session-progress-strip">
|
||||
<div class="progress-info">
|
||||
<span id="progressMessage">Ready to start crawl</span>
|
||||
<span id="iterationBadge" class="iter-badge">Iteration 1 / 1</span>
|
||||
</div>
|
||||
<div class="progress-bar-container">
|
||||
<div id="progressBar" class="progress-bar-fill"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- All Iterations Cumulative Stats -->
|
||||
<div class="metrics-subgroup">
|
||||
<div class="subgroup-header" style="margin-top: 14px; margin-bottom: 8px;">
|
||||
<span class="subgroup-title" style="color: #38bdf8;">📊 All Iterations Cumulative</span>
|
||||
</div>
|
||||
<div class="metrics-grid metrics-grid-2col">
|
||||
<div class="metric-tile tile-total-visited">
|
||||
<div class="metric-icon">🌐</div>
|
||||
<div class="metric-content">
|
||||
<div class="metric-val" id="statTotalVisitedAll">0</div>
|
||||
<div class="metric-label">Total Websites Visited</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-tile tile-total-errors">
|
||||
<div class="metric-icon">⚠️</div>
|
||||
<div class="metric-content">
|
||||
<div class="metric-val" id="statTotalErrorsAll">0</div>
|
||||
<div class="metric-label">Total Errors Seen</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Current Iteration Stats -->
|
||||
<div class="metrics-subgroup" style="margin-top: 14px;">
|
||||
<div class="subgroup-header" style="margin-bottom: 8px;">
|
||||
<span class="subgroup-title" style="color: #94a3b8;">🔄 Current Iteration</span>
|
||||
</div>
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-tile tile-visited">
|
||||
<div class="metric-icon">📄</div>
|
||||
<div class="metric-content">
|
||||
<div class="metric-val" id="statVisited">0</div>
|
||||
<div class="metric-label">Pages Visited</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-tile tile-internal">
|
||||
<div class="metric-icon">🔗</div>
|
||||
<div class="metric-content">
|
||||
<div class="metric-val" id="statInternal">0</div>
|
||||
<div class="metric-label">Internal Pages</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-tile tile-external">
|
||||
<div class="metric-icon">🌐</div>
|
||||
<div class="metric-content">
|
||||
<div class="metric-val" id="statExternal">0</div>
|
||||
<div class="metric-label">External Pages</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-tile tile-cookies">
|
||||
<div class="metric-icon">🍪</div>
|
||||
<div class="metric-content">
|
||||
<div class="metric-val" id="statCookies">0</div>
|
||||
<div class="metric-label">Cookies Accepted</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-tile tile-queue">
|
||||
<div class="metric-icon">📥</div>
|
||||
<div class="metric-content">
|
||||
<div class="metric-val" id="statQueue">0</div>
|
||||
<div class="metric-label">Queue Backlog</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-tile tile-errors">
|
||||
<div class="metric-icon">⚠️</div>
|
||||
<div class="metric-content">
|
||||
<div class="metric-val" id="statErrors">0</div>
|
||||
<div class="metric-label">Failed Websites</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Middle 1: Tor Network Identity & Region -->
|
||||
<section class="card tor-card full-width-card" id="torCard">
|
||||
<div class="card-header">
|
||||
<div class="header-title">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
class="tor-shield-icon">
|
||||
<path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10z"></path>
|
||||
</svg>
|
||||
<h2>Tor Network Identity & Geolocation</h2>
|
||||
<span id="torStatusBadge" class="tor-pill tor-off">Tor Inactive</span>
|
||||
</div>
|
||||
<span class="header-hint" id="torCircuitHint">Queried once per circuit on start/rotation</span>
|
||||
</div>
|
||||
|
||||
<div class="tor-identity-grid">
|
||||
<div class="tor-info-tile">
|
||||
<span class="tor-label">Tor Exit IP</span>
|
||||
<div class="tor-val monospace" id="torIpVal">--</div>
|
||||
</div>
|
||||
<div class="tor-info-tile">
|
||||
<span class="tor-label">Country & Region</span>
|
||||
<div class="tor-val" id="torCountryVal">--</div>
|
||||
</div>
|
||||
<div class="tor-info-tile">
|
||||
<span class="tor-label">City</span>
|
||||
<div class="tor-val" id="torCityVal">--</div>
|
||||
</div>
|
||||
<div class="tor-info-tile">
|
||||
<span class="tor-label">ISP / Organization</span>
|
||||
<div class="tor-val" id="torOrgVal">--</div>
|
||||
</div>
|
||||
<div class="tor-info-tile">
|
||||
<span class="tor-label">Rotation Strategy</span>
|
||||
<div class="tor-val" id="torStrategyVal">Per Iteration</div>
|
||||
</div>
|
||||
<div class="tor-info-tile">
|
||||
<span class="tor-label">Circuits Rotated</span>
|
||||
<div class="tor-val monospace" id="torRotationsVal">0</div>
|
||||
</div>
|
||||
<div class="tor-info-tile full-width-sm">
|
||||
<span class="tor-label">Exit Node Filter</span>
|
||||
<div class="tor-val" id="torFilterVal">Global (Any)</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Middle 2: Dedicated Websites with Errors & Broken Links Section -->
|
||||
<section class="card errors-card full-width-card">
|
||||
|
||||
<div class="card-header">
|
||||
<div class="header-title">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="#f43f5e" stroke-width="2">
|
||||
<circle cx="12" cy="12" r="10"></circle>
|
||||
<line x1="12" y1="8" x2="12" y2="12"></line>
|
||||
<line x1="12" y1="16" x2="12.01" y2="16"></line>
|
||||
</svg>
|
||||
<h2>Websites with Errors & Broken Links</h2>
|
||||
<span id="errorCountBadge" class="count-pill">0</span>
|
||||
</div>
|
||||
<div class="table-toolbar">
|
||||
<input type="text" id="errorSearchInput" placeholder="Filter errors or URLs..." class="filter-input">
|
||||
<button id="btnExportCsv" class="btn btn-sm btn-outline">
|
||||
<span>📥 Export CSV</span>
|
||||
</button>
|
||||
<button id="btnExportJson" class="btn btn-sm btn-outline">
|
||||
<span>{ } JSON</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="table-container">
|
||||
<table class="data-table" id="errorsTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th width="50">#</th>
|
||||
<th>Failed / Broken URL</th>
|
||||
<th width="200">Error Reason</th>
|
||||
<th>Found On (Source URL)</th>
|
||||
<th width="110">Time</th>
|
||||
<th width="80">Action</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="errorsTableBody">
|
||||
<tr class="empty-row" id="emptyErrorsRow">
|
||||
<td colspan="6">
|
||||
<div class="empty-state">
|
||||
<span class="empty-icon">✨</span>
|
||||
<p>No website errors recorded yet. Broken links or HTTP error codes will appear here.</p>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Bottom: Live Streaming Crawler Logs -->
|
||||
<section class="card logs-card full-width-card">
|
||||
<div class="card-header">
|
||||
<div class="header-title">
|
||||
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<polyline points="4 17 10 11 4 5"></polyline>
|
||||
<line x1="12" y1="19" x2="20" y2="19"></line>
|
||||
</svg>
|
||||
<h2>Live Crawl Logs</h2>
|
||||
<span id="logCountBadge" class="count-pill gray">0 lines</span>
|
||||
</div>
|
||||
<div class="log-toolbar">
|
||||
<label class="checkbox-label">
|
||||
<input type="checkbox" id="autoScrollCheck" checked>
|
||||
<span>Auto-scroll</span>
|
||||
</label>
|
||||
<select id="logLevelSelect" class="form-select-sm">
|
||||
<option value="all">All Levels</option>
|
||||
<option value="info">Info</option>
|
||||
<option value="success">Success</option>
|
||||
<option value="warn">Warnings</option>
|
||||
<option value="error">Errors</option>
|
||||
</select>
|
||||
<input type="text" id="logSearchInput" placeholder="Search logs..." class="filter-input-sm">
|
||||
<button id="btnClearLogs" class="btn btn-xs btn-ghost" title="Clear log window">Clear</button>
|
||||
<button id="btnCopyLogs" class="btn btn-xs btn-ghost" title="Copy all logs to clipboard">Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="terminal-window" id="terminalContainer">
|
||||
<div class="terminal-body" id="logOutput">
|
||||
<div class="log-line system-line">
|
||||
<span class="log-time">[00:00:00]</span>
|
||||
<span class="log-tag tag-sys">[SYSTEM]</span>
|
||||
<span class="log-msg">Websurfer dashboard initialized. Configure parameters and click Start
|
||||
Crawling.</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
</main>
|
||||
|
||||
<!-- Toast Notification Container -->
|
||||
<div id="toastContainer" class="toast-container"></div>
|
||||
</div>
|
||||
|
||||
<script src="/static/app.js"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
295
web/static/login.html
Normal file
295
web/static/login.html
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Login - Clickduck Control Center</title>
|
||||
<link rel="icon" type="image/png" href="/static/clickduck_logo.png">
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link
|
||||
href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500;600&display=swap"
|
||||
rel="stylesheet">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
<style>
|
||||
.login-wrapper {
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 1.5rem;
|
||||
}
|
||||
|
||||
.login-card {
|
||||
width: 100%;
|
||||
max-width: 440px;
|
||||
background: var(--bg-card);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-xl);
|
||||
padding: 2.75rem 2.25rem;
|
||||
box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.75), 0 0 0 1px var(--border-subtle);
|
||||
backdrop-filter: blur(20px);
|
||||
-webkit-backdrop-filter: blur(20px);
|
||||
animation: fadeIn 0.4s cubic-bezier(0.16, 1, 0.3, 1);
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.login-card::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 3px;
|
||||
background: var(--primary-gradient);
|
||||
}
|
||||
|
||||
@keyframes fadeIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(12px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
.login-brand {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
margin-bottom: 2rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.login-logo-container {
|
||||
position: relative;
|
||||
width: 120px;
|
||||
height: 120px;
|
||||
margin-bottom: 1.25rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.login-logo-glow {
|
||||
position: absolute;
|
||||
inset: -6px;
|
||||
border-radius: 50%;
|
||||
background: radial-gradient(circle, rgba(56, 189, 248, 0.3) 0%, rgba(99, 102, 241, 0.15) 70%, transparent 100%);
|
||||
filter: blur(8px);
|
||||
z-index: 0;
|
||||
animation: pulseGlow 4s ease-in-out infinite alternate;
|
||||
}
|
||||
|
||||
@keyframes pulseGlow {
|
||||
from { opacity: 0.5; transform: scale(0.96); }
|
||||
to { opacity: 0.9; transform: scale(1.04); }
|
||||
}
|
||||
|
||||
.login-logo-img {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border-radius: 50%;
|
||||
object-fit: cover;
|
||||
clip-path: circle(49% at 50% 50%);
|
||||
box-shadow: 0 10px 30px -5px rgba(0, 0, 0, 0.6), 0 0 0 2px rgba(56, 189, 248, 0.3);
|
||||
transition: transform 0.35s cubic-bezier(0.34, 1.56, 0.64, 1), box-shadow 0.35s ease;
|
||||
background: #ffffff;
|
||||
}
|
||||
|
||||
.login-logo-container:hover .login-logo-img {
|
||||
transform: scale(1.05) rotate(2deg);
|
||||
box-shadow: 0 15px 35px -5px rgba(0, 0, 0, 0.7), 0 0 0 3px var(--primary);
|
||||
}
|
||||
|
||||
.login-title {
|
||||
font-size: 1.6rem;
|
||||
font-weight: 700;
|
||||
color: var(--text-main);
|
||||
letter-spacing: -0.02em;
|
||||
margin-bottom: 0.25rem;
|
||||
}
|
||||
|
||||
.login-subtitle {
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted);
|
||||
max-width: 320px;
|
||||
line-height: 1.45;
|
||||
}
|
||||
|
||||
.login-form .form-group {
|
||||
margin-bottom: 1.25rem;
|
||||
}
|
||||
|
||||
.login-form label {
|
||||
display: block;
|
||||
font-size: 0.85rem;
|
||||
font-weight: 500;
|
||||
color: var(--text-muted);
|
||||
margin-bottom: 0.4rem;
|
||||
}
|
||||
|
||||
.login-form input[type="text"],
|
||||
.login-form input[type="password"] {
|
||||
width: 100%;
|
||||
padding: 0.75rem 1rem;
|
||||
background: var(--bg-input);
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--text-main);
|
||||
font-family: inherit;
|
||||
font-size: 0.95rem;
|
||||
outline: none;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.login-form input:focus {
|
||||
border-color: var(--border-focus);
|
||||
box-shadow: 0 0 0 2px rgba(56, 189, 248, 0.2);
|
||||
}
|
||||
|
||||
.login-btn {
|
||||
width: 100%;
|
||||
padding: 0.85rem;
|
||||
margin-top: 0.75rem;
|
||||
background: var(--primary-gradient);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.95rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.5rem;
|
||||
box-shadow: 0 4px 15px var(--primary-glow);
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.login-btn:hover {
|
||||
opacity: 0.95;
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 6px 20px var(--primary-glow);
|
||||
}
|
||||
|
||||
.login-btn:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: not-allowed;
|
||||
transform: none;
|
||||
}
|
||||
|
||||
.login-error {
|
||||
background: rgba(244, 63, 94, 0.15);
|
||||
border: 1px solid var(--danger);
|
||||
color: #fecdd3;
|
||||
padding: 0.75rem 1rem;
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 0.85rem;
|
||||
margin-bottom: 1.25rem;
|
||||
display: none;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body class="dark-theme">
|
||||
<div class="login-wrapper">
|
||||
<div class="login-card">
|
||||
<div class="login-brand">
|
||||
<div class="login-logo-container">
|
||||
<div class="login-logo-glow"></div>
|
||||
<img
|
||||
src="/static/clickduck_logo.png"
|
||||
alt="Clickduck Logo"
|
||||
class="login-logo-img"
|
||||
onerror="this.src='/static/clickduck_logo_small.jpg'"
|
||||
>
|
||||
</div>
|
||||
<h1 class="login-title">Clickduck Portal</h1>
|
||||
<p class="login-subtitle">Autonomous Web Crawler & Broken Link Detector</p>
|
||||
</div>
|
||||
|
||||
<div id="loginError" class="login-error">
|
||||
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<circle cx="12" cy="12" r="10"></circle>
|
||||
<line x1="12" y1="8" x2="12" y2="12"></line>
|
||||
<line x1="12" y1="16" x2="12.01" y2="16"></line>
|
||||
</svg>
|
||||
<span id="errorMessage">Invalid username or password</span>
|
||||
</div>
|
||||
|
||||
<form id="loginForm" class="login-form">
|
||||
<div class="form-group">
|
||||
<label for="username">Username</label>
|
||||
<input type="text" id="username" name="username" required autocomplete="username" placeholder="admin" autofocus>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="password">Password</label>
|
||||
<input type="password" id="password" name="password" required autocomplete="current-password" placeholder="••••••••••••">
|
||||
</div>
|
||||
|
||||
<button type="submit" id="btnSubmit" class="login-btn">
|
||||
<span>Sign In</span>
|
||||
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M5 12h14"></path>
|
||||
<path d="M12 5l7 7-7 7"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const form = document.getElementById('loginForm');
|
||||
const errBox = document.getElementById('loginError');
|
||||
const errMsg = document.getElementById('errorMessage');
|
||||
const btn = document.getElementById('btnSubmit');
|
||||
|
||||
form.addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
errBox.style.display = 'none';
|
||||
btn.disabled = true;
|
||||
btn.querySelector('span').textContent = 'Authenticating...';
|
||||
|
||||
const username = document.getElementById('username').value.trim();
|
||||
const password = document.getElementById('password').value;
|
||||
|
||||
try {
|
||||
const resp = await fetch('/api/login', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ username, password })
|
||||
});
|
||||
|
||||
const data = await resp.json();
|
||||
|
||||
if (resp.ok && data.success) {
|
||||
btn.querySelector('span').textContent = 'Success! Redirecting...';
|
||||
const redirectUrl = new URLSearchParams(window.location.search).get('redirect') || '/';
|
||||
window.location.href = redirectUrl;
|
||||
} else {
|
||||
errMsg.textContent = data.detail || data.message || 'Invalid username or password';
|
||||
errBox.style.display = 'flex';
|
||||
btn.disabled = false;
|
||||
btn.querySelector('span').textContent = 'Sign In';
|
||||
}
|
||||
} catch (err) {
|
||||
errMsg.textContent = 'Connection error. Please try again.';
|
||||
errBox.style.display = 'flex';
|
||||
btn.disabled = false;
|
||||
btn.querySelector('span').textContent = 'Sign In';
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
1353
web/static/style.css
Normal file
1353
web/static/style.css
Normal file
File diff suppressed because it is too large
Load diff
Loading…
Add table
Reference in a new issue