Sometimes a page loads its data dynamically and a simple HTTP request is not enough. You need a real browser to render JavaScript, wait for API calls and extract the content.
Here is a Playwright function that waits for a specific element and extracts all text:
from playwright.sync_api import sync_playwright
def extract_data(url: str, selector: str):
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page()
page.goto(url)
page.wait_for_selector(selector)
items = page.locator(selector).all_text_contents()
browser.close()
return items
titles = extract_data("https://news.ycombinator.com", ".titleline > a")
for t in titles[:10]:
print(t)
For pages with infinite scroll you need to scroll and wait for new content:
def extract_infinite_scroll(url: str, item_selector: str, max_items=100):
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page()
page.goto(url)
items = set()
prev_count = 0
while len(items) < max_items:
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
page.wait_for_timeout(2000)
new_items = page.locator(item_selector).all_text_contents()
items = set(new_items)
if len(items) == prev_count:
break
prev_count = len(items)
browser.close()
return list(items)[:max_items]
The loop scrolls down, waits for new elements to load, and stops when no new items appear.
You can also capture network requests to get structured data directly from the API:
from playwright.sync_api import sync_playwright
def capture_api_calls(url: str, api_pattern: str):
results = []
def on_response(response):
if api_pattern in response.url:
try:
results.append(response.json())
except:
pass
with sync_playwright() as p:
browser = p.chromium.launch()
page = browser.new_page()
page.on("response", on_response)
page.goto(url)
page.wait_for_timeout(5000)
browser.close()
return results
data = capture_api_calls("https://example.com/products", "/api/products")
print(f"Captured {len(data)} API responses")
This pattern is useful when the page loads data from a JSON API and renders it client side. Instead of parsing the DOM, you get the raw data.
To save everything as CSV:
import csv
def save_as_csv(data: list[dict], filename: str):
if not data:
return
with open(filename, "w", newline="", encoding="utf-8") as f:
w = csv.DictWriter(f, fieldnames=data[0].keys())
w.writeheader()
w.writerows(data)
print(f"Saved {len(data)} rows to {filename}")
Combine these patterns and you can extract data from almost any dynamic website.
That's all for now.
Thanks for reading!
SOCIAL SHARE CARD GENERATOR