218 lines
7.5 KiB
Python
218 lines
7.5 KiB
Python
import csv
|
|
import io
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Dict, Any, Optional
|
|
|
|
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Depends, Request, Response, status
|
|
from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, RedirectResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
|
|
from crawler.config import CrawlerConfig
|
|
from .manager import session_manager
|
|
from .auth import (
|
|
is_auth_enabled,
|
|
verify_credentials,
|
|
create_session_token,
|
|
verify_session_token,
|
|
get_current_user,
|
|
verify_websocket,
|
|
SESSION_COOKIE_NAME,
|
|
SESSION_MAX_AGE_SECONDS,
|
|
)
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent
|
|
STATIC_DIR = BASE_DIR / "static"
|
|
|
|
app = FastAPI(title="WebSurfer Control Center", version="1.0.0")
|
|
|
|
# Mount static assets directory
|
|
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
|
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def root(request: Request):
|
|
"""Serves the main dashboard user interface, redirecting to login if unauthenticated."""
|
|
if is_auth_enabled():
|
|
token = request.cookies.get(SESSION_COOKIE_NAME)
|
|
if not verify_session_token(token):
|
|
return RedirectResponse(url="/login", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
|
|
|
|
index_path = STATIC_DIR / "index.html"
|
|
if not index_path.exists():
|
|
raise HTTPException(status_code=404, detail="Dashboard UI not found")
|
|
with open(index_path, "r", encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
@app.get("/login", response_class=HTMLResponse)
|
|
async def login_page(request: Request):
|
|
"""Serves the login page, redirecting to / if already authenticated."""
|
|
if is_auth_enabled():
|
|
token = request.cookies.get(SESSION_COOKIE_NAME)
|
|
if verify_session_token(token):
|
|
return RedirectResponse(url="/", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
|
|
|
|
login_path = STATIC_DIR / "login.html"
|
|
if not login_path.exists():
|
|
raise HTTPException(status_code=404, detail="Login UI not found")
|
|
with open(login_path, "r", encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
@app.post("/api/login")
|
|
async def api_login(payload: Dict[str, str], response: Response, request: Request):
|
|
"""Authenticates credentials and establishes a signed session cookie."""
|
|
if not is_auth_enabled():
|
|
return {"success": True, "message": "Authentication not enabled"}
|
|
|
|
username = payload.get("username", "")
|
|
password = payload.get("password", "")
|
|
|
|
if not verify_credentials(username, password):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
detail="Invalid username or password",
|
|
)
|
|
|
|
token = create_session_token(username)
|
|
is_secure = request.headers.get("x-forwarded-proto", "").lower() == "https" or request.url.scheme == "https"
|
|
|
|
response.set_cookie(
|
|
key=SESSION_COOKIE_NAME,
|
|
value=token,
|
|
max_age=SESSION_MAX_AGE_SECONDS,
|
|
httponly=True,
|
|
samesite="lax",
|
|
secure=is_secure,
|
|
path="/",
|
|
)
|
|
return {"success": True, "username": username}
|
|
|
|
|
|
@app.post("/api/logout")
|
|
async def api_logout(response: Response):
|
|
"""Clears the session cookie."""
|
|
response.delete_cookie(key=SESSION_COOKIE_NAME, path="/")
|
|
return {"success": True, "message": "Logged out successfully"}
|
|
|
|
|
|
@app.get("/api/auth/me")
|
|
async def auth_me(request: Request):
|
|
"""Returns current authentication state."""
|
|
enabled = is_auth_enabled()
|
|
if not enabled:
|
|
return {"authenticated": True, "auth_enabled": False, "username": "anonymous"}
|
|
|
|
token = request.cookies.get(SESSION_COOKIE_NAME)
|
|
username = verify_session_token(token)
|
|
return {
|
|
"authenticated": username is not None,
|
|
"auth_enabled": True,
|
|
"username": username,
|
|
}
|
|
|
|
|
|
@app.get("/api/config", dependencies=[Depends(get_current_user)])
|
|
async def get_config():
|
|
"""Fetches the current crawler configuration stored in .env or defaults."""
|
|
env_config = CrawlerConfig.get_env_dict()
|
|
return {"success": True, "config": env_config}
|
|
|
|
|
|
@app.get("/api/tor/countries", dependencies=[Depends(get_current_user)])
|
|
async def get_tor_countries():
|
|
"""Returns the list of supported European countries for Tor exit node selection."""
|
|
from crawler.config import EUROPEAN_COUNTRIES
|
|
countries = [
|
|
{"code": code, "name": info["name"], "flag": info["flag"]}
|
|
for code, info in EUROPEAN_COUNTRIES.items()
|
|
]
|
|
return {"success": True, "countries": countries}
|
|
|
|
|
|
@app.post("/api/config", dependencies=[Depends(get_current_user)])
|
|
async def save_config(payload: Dict[str, Any]):
|
|
"""Persists updated configuration values to the .env file."""
|
|
try:
|
|
CrawlerConfig.save_env_file(payload)
|
|
return {"success": True, "message": "Configuration successfully saved to .env"}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"Failed to save .env file: {e}")
|
|
|
|
|
|
@app.post("/api/crawl/start", dependencies=[Depends(get_current_user)])
|
|
async def start_crawl(payload: Dict[str, Any]):
|
|
"""Starts a new crawler run with given parameters."""
|
|
result = await session_manager.start_crawl(payload)
|
|
if not result.get("success"):
|
|
raise HTTPException(status_code=400, detail=result.get("message"))
|
|
return result
|
|
|
|
|
|
@app.post("/api/crawl/stop", dependencies=[Depends(get_current_user)])
|
|
async def stop_crawl():
|
|
"""Stops the currently running crawler job."""
|
|
result = await session_manager.stop_crawl()
|
|
if not result.get("success"):
|
|
raise HTTPException(status_code=400, detail=result.get("message"))
|
|
return result
|
|
|
|
|
|
@app.get("/api/crawl/status", dependencies=[Depends(get_current_user)])
|
|
async def get_status():
|
|
"""Returns the current state, metrics, errors, and recent logs."""
|
|
return session_manager.get_snapshot()
|
|
|
|
|
|
@app.post("/api/crawl/clear-logs", dependencies=[Depends(get_current_user)])
|
|
async def clear_logs():
|
|
"""Clears buffered in-memory logs."""
|
|
session_manager.clear_logs()
|
|
return {"success": True}
|
|
|
|
|
|
@app.get("/api/crawl/export-errors", dependencies=[Depends(get_current_user)])
|
|
async def export_errors(format: str = "csv"):
|
|
"""Exports all recorded errors from current session as CSV or JSON."""
|
|
errors = session_manager.errors
|
|
if format.lower() == "json":
|
|
return JSONResponse(content={"total_errors": len(errors), "errors": errors})
|
|
|
|
# Default to CSV export
|
|
output = io.StringIO()
|
|
writer = csv.writer(output)
|
|
writer.writerow(["ID", "Failed URL", "Error Reason", "Found On (Source URL)", "Timestamp"])
|
|
for item in errors:
|
|
writer.writerow([
|
|
item.get("id", ""),
|
|
item.get("url", ""),
|
|
item.get("error", ""),
|
|
item.get("source_url", ""),
|
|
item.get("timestamp", ""),
|
|
])
|
|
|
|
output.seek(0)
|
|
return StreamingResponse(
|
|
iter([output.getvalue()]),
|
|
media_type="text/csv",
|
|
headers={"Content-Disposition": "attachment; filename=websurfer_errors.csv"},
|
|
)
|
|
|
|
|
|
@app.websocket("/ws/stream")
|
|
async def websocket_endpoint(websocket: WebSocket):
|
|
"""WebSocket stream for real-time log, metrics, and error updates."""
|
|
if not verify_websocket(websocket):
|
|
await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
|
|
return
|
|
|
|
await session_manager.connect_ws(websocket)
|
|
try:
|
|
while True:
|
|
# Keep connection open and accept any control pings from client
|
|
data = await websocket.receive_text()
|
|
except WebSocketDisconnect:
|
|
session_manager.disconnect_ws(websocket)
|
|
except Exception:
|
|
session_manager.disconnect_ws(websocket)
|