69 lines
2.5 KiB
Python
69 lines
2.5 KiB
Python
from typing import Optional, Tuple
|
|
from urllib.parse import urlparse
|
|
|
|
# Non-HTML static file extensions to ignore
|
|
IGNORED_EXTENSIONS = (
|
|
".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico", ".bmp", ".tiff",
|
|
".pdf", ".zip", ".tar", ".gz", ".bz2", ".7z", ".rar", ".exe", ".dmg", ".iso", ".bin",
|
|
".mp3", ".mp4", ".avi", ".mov", ".mkv", ".flv", ".wmv", ".webm", ".wav", ".ogg",
|
|
".css", ".js", ".json", ".xml", ".rss", ".atom", ".woff", ".woff2", ".ttf", ".eot", ".otf",
|
|
".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".csv"
|
|
)
|
|
|
|
# WordPress / CMS feed and API paths that are not HTML web pages
|
|
IGNORED_PATHS = (
|
|
"/feed", "/comments/feed", "/wp-json", "/xmlrpc.php", "/wp-admin", "/wp-login.php"
|
|
)
|
|
|
|
|
|
def normalize_url(url: str, start_domain: str) -> Tuple[Optional[str], Optional[bool]]:
|
|
"""
|
|
Normalizes a URL, strips fragments, filters out non-HTML files / endpoints,
|
|
and determines whether the link is internal or external.
|
|
|
|
Returns:
|
|
(normalized_url, is_internal) or (None, None) if the link should be ignored.
|
|
"""
|
|
if not isinstance(url, str):
|
|
return None, None
|
|
|
|
url = url.split("#")[0].strip()
|
|
if not url:
|
|
return None, None
|
|
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in ["http", "https"]:
|
|
return None, None
|
|
|
|
path = parsed.path or "/"
|
|
path_lower = path.lower()
|
|
|
|
# Filter out static file extensions
|
|
if any(path_lower.endswith(ext) for ext in IGNORED_EXTENSIONS):
|
|
return None, None
|
|
|
|
# Filter out non-HTML WordPress / CMS endpoints
|
|
if any(path_lower == p or path_lower.rstrip("/") == p or path_lower.startswith(p + "/") for p in IGNORED_PATHS):
|
|
return None, None
|
|
|
|
netloc = parsed.netloc.lower()
|
|
if not netloc:
|
|
return None, None
|
|
|
|
# Check domain matching (treat apex domain and www. as identical)
|
|
clean_start_domain = start_domain.lower().replace("www.", "")
|
|
clean_netloc = netloc.replace("www.", "")
|
|
|
|
is_internal = (clean_netloc == clean_start_domain)
|
|
|
|
# Strip query parameters for internal links to avoid infinite loops and duplicate crawls
|
|
if is_internal:
|
|
norm_path = path.rstrip("/") if path != "/" else "/"
|
|
norm_url = f"{parsed.scheme}://{netloc}{norm_path}"
|
|
else:
|
|
# Keep query parameters for external links (e.g., video IDs)
|
|
norm_path = path.rstrip("/") if path != "/" else "/"
|
|
query_part = f"?{parsed.query}" if parsed.query else ""
|
|
norm_url = f"{parsed.scheme}://{netloc}{norm_path}{query_part}"
|
|
|
|
return norm_url, is_internal
|