websurfer/crawler/cookies.py

import asyncio

# Selectors for common Consent Management Platforms (CMPs) and cookie banners
COOKIE_BANNER_SELECTORS = [
    # Complianz (WordPress)
    "button.cmplz-accept",
    "button.cmplz-btn.cmplz-accept",
    ".cmplz-btn.cmplz-accept",
    "#cmplz-cookiebanner-content .cmplz-accept",

    # Borlabs Cookie (WordPress)
    "a._brlbs-accept-all",
    "button._brlbs-accept-all",
    "._brlbs-btn-accept-all",
    "[data-borlabs-cookie-accept-all]",
    "#borlabs-cookie-accept-all",
    ".borlabs-cookie-preference-accept-all",

    # Cookiebot
    "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
    "#CybotCookiebotDialogBodyButtonAccept",
    "#CybotCookiebotDialogBodyLevelButtonAccept",
    "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowallSelection",

    # OneTrust / CookiePro
    "#onetrust-accept-btn-handler",
    "button#onetrust-accept-btn-handler",
    "#accept-recommended-btn-handler",

    # Usercentrics
    "button[data-testid='uc-accept-all-button']",
    "#uc-btn-accept-banner",
    "button#uc-btn-accept-banner",
    "div#usercentrics-root button[data-testid='uc-accept-all-button']",

    # Klaro
    ".klaro .cm-btn-accept",
    ".klaro .cm-btn-success",
    "button.cm-btn-accept-all",

    # CookieYes
    "button.cky-btn-accept",
    "[data-cky-tag='accept-button']",

    # Real Cookie Banner
    "button[data-consent-action='accept-all']",
    ".rcb-btn-accept-all",

    # CCM19
    ".ccm--btn-accept-all",
    "button.ccm--btn-accept-all",
    "button[data-ccm-consent='accept-all']",

    # Google Funding Choices
    ".fc-cta-consent",
    "button.fc-cta-consent",

    # Quantcast / InMobi
    ".qc-cmp2-summary-buttons button[mode='primary']",

    # Iubenda
    ".iubenda-cs-accept-btn",
    "button.iubenda-cs-btn-primary",

    # Didomi
    "#didomi-notice-agree-button",

    # Osano
    ".osano-cm-accept-all",
    "button.osano-cm-button--type_accept",

    # Axeptio
    "#axeptio_btn_acceptAll",

    # Generic IDs, classes, and data attributes
    "[data-action='accept-all']",
    "[data-consent='accept-all']",
    "[data-cookie-accept-all]",
    "button[id*='cookie-accept']",
    "button[id*='accept-cookies']",
    "button[id*='cookieAccept']",
    "button[class*='cookie-accept']",
    "button[class*='accept-all-cookies']",
]

COOKIE_TEXT_PATTERNS = [
    # German
    r"^(alle\s+)?(cookies\s+)?akzeptieren$",
    r"^alles\s+akzeptieren$",
    r"^alle\s+annehmen$",
    r"^alles\s+annehmen$",
    r"^alle\s+ausw[aä]hlen\s+und\s+akzeptieren$",
    r"^alle\s+zulassen$",
    r"^cookies\s+zulassen$",
    r"^einverstanden$",
    r"^zustimmen$",
    r"^allen\s+zustimmen$",
    r"^alle\s+best[aä]tigen$",
    r"^auswahl\s+best[aä]tigen$",
    r"^einwilligen$",
    r"^verstanden$",
    r"^ok$",

    # English
    r"^(accept\s+all(\s+cookies)?)$",
    r"^(allow\s+all(\s+cookies)?)$",
    r"^i\s+accept$",
    r"^i\s+agree$",
    r"^agree(\s+to\s+all)?$",
    r"^accept$",
    r"^allow$",
    r"^agree\s+&\s+proceed$",
    r"^got\s+it$",
]

COOKIE_TEXT_REGEX_STR = "|".join(f"(?:{p})" for p in COOKIE_TEXT_PATTERNS)


async def auto_accept_cookies(page) -> bool:
    """
    Attempts to detect and click cookie consent / banner acceptance buttons
    using a fast, single-pass in-browser JavaScript evaluation.
    Only targets actual buttons and known CMP selectors, preventing accidental link navigations.
    """
    js_code = """
    () => {
        const selectors = """ + str(COOKIE_BANNER_SELECTORS) + """;
        
        // 1. Try known CMP selectors on main page
        for (const sel of selectors) {
            try {
                const el = document.querySelector(sel);
                if (el && (el.offsetWidth > 0 || el.offsetHeight > 0 || el.getClientRects().length > 0)) {
                    el.click();
                    return { clicked: true, method: 'selector', text: sel };
                }
            } catch(e) {}
        }
        
        // 2. Check Shadow DOM (e.g. Usercentrics / custom web components)
        try {
            const ucRoot = document.querySelector('#usercentrics-root, div#usercentrics-cmp-ui');
            if (ucRoot && ucRoot.shadowRoot) {
                const ucBtn = ucRoot.shadowRoot.querySelector("button[data-testid='uc-accept-all-button'], button#uc-btn-accept-banner, button.uc-btn-accept");
                if (ucBtn) {
                    ucBtn.click();
                    return { clicked: true, method: 'shadow-dom', text: 'usercentrics-shadow' };
                }
            }
        } catch(e) {}

        // 3. Text-based matching strictly on button / submit / role=button elements (never plain navigation links)
        const textRegex = /""" + COOKIE_TEXT_REGEX_STR + """/i;
        const candidates = document.querySelectorAll('button, input[type="button"], input[type="submit"], [role="button"], a.cmplz-accept, a._brlbs-accept-all, a.cky-btn-accept');
        for (const btn of candidates) {
            try {
                const txt = (btn.innerText || btn.value || '').trim();
                if (txt && (btn.offsetWidth > 0 || btn.offsetHeight > 0 || btn.getClientRects().length > 0) && textRegex.test(txt)) {
                    btn.click();
                    return { clicked: true, method: 'button-text', text: txt };
                }
            } catch(e) {}
        }

        return { clicked: false };
    }
    """
    try:
        result = await asyncio.wait_for(page.evaluate(js_code), timeout=2.0)
        if result and result.get("clicked"):
            print(f"  [Cookie] Auto-accepted cookies via {result.get('method')}: '{result.get('text')}'")
            await page.wait_for_timeout(600)  # Give tracking scripts time to transmit beacons
            return True
    except Exception:
        pass

    return False