import csv import io import os from pathlib import Path from typing import Dict, Any, Optional from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Depends, Request, Response, status from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from crawler.config import CrawlerConfig from .manager import session_manager from .auth import ( is_auth_enabled, verify_credentials, create_session_token, verify_session_token, get_current_user, verify_websocket, SESSION_COOKIE_NAME, SESSION_MAX_AGE_SECONDS, ) BASE_DIR = Path(__file__).resolve().parent STATIC_DIR = BASE_DIR / "static" app = FastAPI(title="WebSurfer Control Center", version="1.0.0") # Mount static assets directory app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") @app.get("/", response_class=HTMLResponse) async def root(request: Request): """Serves the main dashboard user interface, redirecting to login if unauthenticated.""" if is_auth_enabled(): token = request.cookies.get(SESSION_COOKIE_NAME) if not verify_session_token(token): return RedirectResponse(url="/login", status_code=status.HTTP_307_TEMPORARY_REDIRECT) index_path = STATIC_DIR / "index.html" if not index_path.exists(): raise HTTPException(status_code=404, detail="Dashboard UI not found") with open(index_path, "r", encoding="utf-8") as f: return f.read() @app.get("/login", response_class=HTMLResponse) async def login_page(request: Request): """Serves the login page, redirecting to / if already authenticated.""" if is_auth_enabled(): token = request.cookies.get(SESSION_COOKIE_NAME) if verify_session_token(token): return RedirectResponse(url="/", status_code=status.HTTP_307_TEMPORARY_REDIRECT) login_path = STATIC_DIR / "login.html" if not login_path.exists(): raise HTTPException(status_code=404, detail="Login UI not found") with open(login_path, "r", encoding="utf-8") as f: return f.read() @app.post("/api/login") async def api_login(payload: Dict[str, str], response: Response, request: Request): """Authenticates credentials and establishes a signed session cookie.""" if not is_auth_enabled(): return {"success": True, "message": "Authentication not enabled"} username = payload.get("username", "") password = payload.get("password", "") if not verify_credentials(username, password): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid username or password", ) token = create_session_token(username) is_secure = request.headers.get("x-forwarded-proto", "").lower() == "https" or request.url.scheme == "https" response.set_cookie( key=SESSION_COOKIE_NAME, value=token, max_age=SESSION_MAX_AGE_SECONDS, httponly=True, samesite="lax", secure=is_secure, path="/", ) return {"success": True, "username": username} @app.post("/api/logout") async def api_logout(response: Response): """Clears the session cookie.""" response.delete_cookie(key=SESSION_COOKIE_NAME, path="/") return {"success": True, "message": "Logged out successfully"} @app.get("/api/auth/me") async def auth_me(request: Request): """Returns current authentication state.""" enabled = is_auth_enabled() if not enabled: return {"authenticated": True, "auth_enabled": False, "username": "anonymous"} token = request.cookies.get(SESSION_COOKIE_NAME) username = verify_session_token(token) return { "authenticated": username is not None, "auth_enabled": True, "username": username, } @app.get("/api/config", dependencies=[Depends(get_current_user)]) async def get_config(): """Fetches the current crawler configuration stored in .env or defaults.""" env_config = CrawlerConfig.get_env_dict() return {"success": True, "config": env_config} @app.get("/api/tor/countries", dependencies=[Depends(get_current_user)]) async def get_tor_countries(): """Returns the list of supported European countries for Tor exit node selection.""" from crawler.config import EUROPEAN_COUNTRIES countries = [ {"code": code, "name": info["name"], "flag": info["flag"]} for code, info in EUROPEAN_COUNTRIES.items() ] return {"success": True, "countries": countries} @app.post("/api/config", dependencies=[Depends(get_current_user)]) async def save_config(payload: Dict[str, Any]): """Persists updated configuration values to the .env file.""" try: CrawlerConfig.save_env_file(payload) return {"success": True, "message": "Configuration successfully saved to .env"} except Exception as e: raise HTTPException(status_code=500, detail=f"Failed to save .env file: {e}") @app.post("/api/crawl/start", dependencies=[Depends(get_current_user)]) async def start_crawl(payload: Dict[str, Any]): """Starts a new crawler run with given parameters.""" result = await session_manager.start_crawl(payload) if not result.get("success"): raise HTTPException(status_code=400, detail=result.get("message")) return result @app.post("/api/crawl/stop", dependencies=[Depends(get_current_user)]) async def stop_crawl(): """Stops the currently running crawler job.""" result = await session_manager.stop_crawl() if not result.get("success"): raise HTTPException(status_code=400, detail=result.get("message")) return result @app.get("/api/crawl/status", dependencies=[Depends(get_current_user)]) async def get_status(): """Returns the current state, metrics, errors, and recent logs.""" return session_manager.get_snapshot() @app.post("/api/crawl/clear-logs", dependencies=[Depends(get_current_user)]) async def clear_logs(): """Clears buffered in-memory logs.""" session_manager.clear_logs() return {"success": True} @app.get("/api/crawl/export-errors", dependencies=[Depends(get_current_user)]) async def export_errors(format: str = "csv"): """Exports all recorded errors from current session as CSV or JSON.""" errors = session_manager.errors if format.lower() == "json": return JSONResponse(content={"total_errors": len(errors), "errors": errors}) # Default to CSV export output = io.StringIO() writer = csv.writer(output) writer.writerow(["ID", "Failed URL", "Error Reason", "Found On (Source URL)", "Timestamp"]) for item in errors: writer.writerow([ item.get("id", ""), item.get("url", ""), item.get("error", ""), item.get("source_url", ""), item.get("timestamp", ""), ]) output.seek(0) return StreamingResponse( iter([output.getvalue()]), media_type="text/csv", headers={"Content-Disposition": "attachment; filename=websurfer_errors.csv"}, ) @app.websocket("/ws/stream") async def websocket_endpoint(websocket: WebSocket): """WebSocket stream for real-time log, metrics, and error updates.""" if not verify_websocket(websocket): await websocket.close(code=status.WS_1008_POLICY_VIOLATION) return await session_manager.connect_ws(websocket) try: while True: # Keep connection open and accept any control pings from client data = await websocket.receive_text() except WebSocketDisconnect: session_manager.disconnect_ws(websocket) except Exception: session_manager.disconnect_ws(websocket)