44 lines
899 B
Python
44 lines
899 B
Python
"""
|
|
WebSurfer Crawler Package
|
|
"""
|
|
|
|
from .config import CrawlerConfig
|
|
from .cookies import (
|
|
COOKIE_BANNER_SELECTORS,
|
|
COOKIE_TEXT_PATTERNS,
|
|
auto_accept_cookies,
|
|
)
|
|
from .engine import Crawler, crawl_level, worker, RandomQueue
|
|
from .tor import (
|
|
DEFAULT_TOR_CONTROL_PORT,
|
|
DEFAULT_TOR_SOCKS_PORT,
|
|
TorManager,
|
|
renew_tor_circuit,
|
|
start_tor_service,
|
|
stop_tor_service,
|
|
)
|
|
from .url_utils import (
|
|
IGNORED_EXTENSIONS,
|
|
IGNORED_PATHS,
|
|
normalize_url,
|
|
)
|
|
|
|
__all__ = [
|
|
"Crawler",
|
|
"CrawlerConfig",
|
|
"RandomQueue",
|
|
"TorManager",
|
|
"start_tor_service",
|
|
"renew_tor_circuit",
|
|
"stop_tor_service",
|
|
"auto_accept_cookies",
|
|
"COOKIE_BANNER_SELECTORS",
|
|
"COOKIE_TEXT_PATTERNS",
|
|
"normalize_url",
|
|
"IGNORED_EXTENSIONS",
|
|
"IGNORED_PATHS",
|
|
"crawl_level",
|
|
"worker",
|
|
"DEFAULT_TOR_SOCKS_PORT",
|
|
"DEFAULT_TOR_CONTROL_PORT",
|
|
]
|