How to Build a GDPR-Compliant Web Scraper With Playwright in 2026

# Document your legal basis in code

SCRAPER_CONFIG = {

    "legal_basis": "legitimate_interest",

    "purpose": "B2B lead enrichment from public company websites",

    "data_categories": ["business_email", "job_title", "company_name"],

    "excludes": ["personal_emails", "home_addresses", "private_profiles"],

    "retention_days": 90

}

from playwright.sync_api import sync_playwright



def scrape_profile(url):

    with sync_playwright() as p:

        browser = p.chromium.launch()

        page = browser.new_page()

        page.goto(url)



        # Scraping entire page DOM

        return page.content()  # GDPR problem: stores everything

from playwright.sync_api import sync_playwright

from datetime import datetime



def scrape_business_profile(url: str, config: dict) -> dict:

    """Only collect fields specified in config.data_categories"""

    with sync_playwright() as p:

        browser = p.chromium.launch(headless=True)

        page = browser.new_page()



        # Set legitimate user agent

        page.set_extra_http_headers({

            "User-Agent": "Mozilla/5.0 (compatible; DataBot/1.0; +https://yourdomain.com/bot)"

        })



        page.goto(url, wait_until="networkidle")



        # Only extract what you need

        result = {}



        if "job_title" in config["data_categories"]:

            result["job_title"] = page.locator(".job-title").first.text_content()



        if "company_name" in config["data_categories"]:

            result["company_name"] = page.locator(".company").first.text_content()



        # Never collect: photos, personal emails, home city unless needed



        result["scraped_at"] = datetime.utcnow().isoformat()

        result["source_url"] = url

        result["legal_basis"] = config["legal_basis"]



        browser.close()

        return result

import urllib.robotparser

from urllib.parse import urlparse



def can_scrape(url: str) -> bool:

    parsed = urlparse(url)

    robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"



    rp = urllib.robotparser.RobotFileParser()

    rp.set_url(robots_url)

    rp.read()



    return rp.can_fetch("*", url)



# Always check before scraping

if not can_scrape(target_url):

    print(f"Skipping {target_url} — robots.txt disallows")

import asyncio

import random

from playwright.async_api import async_playwright



async def scrape_with_rate_limit(urls: list, delay_seconds: float = 2.0):

    results = []



    async with async_playwright() as p:

        browser = await p.chromium.launch(headless=True)



        for url in urls:

            if not can_scrape(url):

                continue



            page = await browser.new_page()



            try:

                await page.goto(url, wait_until="networkidle", timeout=30000)

                data = await extract_minimal_data(page)

                results.append(data)

            except Exception as e:

                print(f"Error scraping {url}: {e}")

            finally:

                await page.close()



            # Human-like delay (GDPR proportionality)

            await asyncio.sleep(delay_seconds + random.uniform(0, 1))



        await browser.close()



    return results

from datetime import datetime, timedelta

import sqlite3



def store_with_retention(data: dict, db_path: str, retention_days: int = 90):

    conn = sqlite3.connect(db_path)

    cursor = conn.cursor()



    # Store with expiry timestamp

    expires_at = (datetime.utcnow() + timedelta(days=retention_days)).isoformat()



    cursor.execute("""

        INSERT INTO scraped_data (url, data_json, scraped_at, expires_at, legal_basis)

        VALUES (?, ?, ?, ?, ?)

    """, (

        data["source_url"],

        str(data),

        data["scraped_at"],

        expires_at,

        data["legal_basis"]

    ))



    conn.commit()

    conn.close()



def purge_expired_data(db_path: str):

    """Run this daily via cron to comply with retention limits"""

    conn = sqlite3.connect(db_path)

    cursor = conn.cursor()



    deleted = cursor.execute("""

        DELETE FROM scraped_data 

        WHERE expires_at < datetime()

    """).rowcount



    conn.commit()

    conn.close()



    print(f"Purged {deleted} expired records")

def handle_erasure_request(email: str, db_path: str):

    """GDPR Art. 17 - Right to erasure"""

    conn = sqlite3.connect(db_path)

    cursor = conn.cursor()



    # Find and delete all data about this person

    deleted = cursor.execute("""

        DELETE FROM scraped_data

        WHERE data_json LIKE ?

    """, (f"%{email}%",)).rowcount



    conn.commit()

    conn.close()



    # Log the erasure (required for audit trail)

    log_gdpr_action("erasure", email, records_deleted=deleted)



    print(f"Erased {deleted} records for {email}")



def log_gdpr_action(action: str, subject: str, **kwargs):

    """Maintain audit log for GDPR compliance"""

    import json

    with open("gdpr_audit.log", "a") as f:

        entry = {

            "timestamp": datetime.utcnow().isoformat(),

            "action": action,

            "subject_identifier": subject,

            **kwargs

        }

        f.write(json.dumps(entry) + "\n")