54 lines
1.2 KiB
Python
54 lines
1.2 KiB
Python
"""
|
|
WebSurfer Entry Point (browser_crawler.py)
|
|
|
|
Maintained for backwards compatibility with existing CLI commands and scripts.
|
|
The crawler has been modularized under the `crawler/` package.
|
|
"""
|
|
|
|
import asyncio
|
|
import sys
|
|
|
|
from main import main
|
|
from crawler import (
|
|
Crawler,
|
|
CrawlerConfig,
|
|
TorManager,
|
|
start_tor_service,
|
|
renew_tor_circuit,
|
|
stop_tor_service,
|
|
auto_accept_cookies,
|
|
normalize_url,
|
|
COOKIE_BANNER_SELECTORS,
|
|
COOKIE_TEXT_PATTERNS,
|
|
IGNORED_EXTENSIONS,
|
|
IGNORED_PATHS,
|
|
DEFAULT_TOR_SOCKS_PORT as TOR_SOCKS_PORT,
|
|
DEFAULT_TOR_CONTROL_PORT as TOR_CONTROL_PORT,
|
|
)
|
|
|
|
__all__ = [
|
|
"main",
|
|
"Crawler",
|
|
"CrawlerConfig",
|
|
"TorManager",
|
|
"start_tor_service",
|
|
"renew_tor_circuit",
|
|
"stop_tor_service",
|
|
"auto_accept_cookies",
|
|
"normalize_url",
|
|
"COOKIE_BANNER_SELECTORS",
|
|
"COOKIE_TEXT_PATTERNS",
|
|
"IGNORED_EXTENSIONS",
|
|
"IGNORED_PATHS",
|
|
"TOR_SOCKS_PORT",
|
|
"TOR_CONTROL_PORT",
|
|
]
|
|
|
|
if __name__ == "__main__":
|
|
if sys.platform == "win32":
|
|
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
print("\nCrawling interrupted by user.")
|
|
sys.exit(0)
|