Scraping Dynamic Content Without API: JavaScript Sites Guide

from playwright.sync_api import sync_playwright

# Main scraping function, takes the target page as an argument
def run(url):
with sync_playwright() as p:

# Launch the browser in headless mode
browser = p.chromium.launch(headless=True)

# Open a new tab
page = browser.new_page()

# Navigate to the given address
page.goto(url)

# Wait for product cards to appear
page.wait_for_selector(".product-card, .catalog-item")

# Use locators to search for elements
cards = page.locator(".product-card, .catalog-item")

# Count the number of elements found
count = cards.count()

# Print the data to the console
print(f"Elements found: {count}")

# Process the array of elements
for i in range(count):
    title = cards.nth(i).locator(".title, .product-title, h3").inner_text(timeout=1000) if cards.nth(i).locator(".title, .product-title, h3").count() else "-"
    price = cards.nth(i).locator(".price, .product-price").inner_text(timeout=1000) if cards.nth(i).locator(".price, .product-price").count() else "-"
    
    # Output the product title and price
    print(f"{i+1}. {title} — {price}")

# Close the browser
browser.close()

if __name__ == "__main__":
    run("https://example.com")  # replace with your target site

// install and import the fetch module — npm i node-fetch@2
const fetch = require('node-fetch');

// Main function
async function callApi() {
    // Call the internal API
    const url = 'https://example.com/api/products?page=1';

    // Send the request with fetch
    const res = await fetch(url, {
        headers: {
            'Accept': 'application/json',
            'User-Agent': 'Mozilla/5.0 ...'
            // include the required headers: Authorization, X-CSRF-Token, etc.
        }
    });

    // Parse the response data
    const data = await res.json();

    // Output the number of items found
    console.log(data.items.length);
}

callApi();

def run(url, max_scroll=5, pause=2):
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url)

        # how many scrolls to perform
        for i in range(max_scroll):
            # scroll to the bottom
            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            time.sleep(pause)  # wait for new content to load
            print(f"Scroll {i+1}/{max_scroll} done")

        # Continue with your parsing code here

# Define the click limit
def run(url, max_clicks=10):
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url)

        # initialize click counter
        clicks = 0

        # while the counter is below the maximum
        while clicks < max_clicks:
            try:
                # search for the button
                button = page.locator("button:has-text('Show more'), .load-more, .show-more")
                if not button.count():
                    print("The 'Show more' button was not found — stopping.")
                    break

                # click the button
                button.first.click()
                page.wait_for_timeout(2000)  # wait for new data to load

                # increase the counter
                clicks += 1
                print(f"Click on the button {clicks}/{max_clicks}")

            except Exception as e:
                print(f"Error when clicking: {e}")
                break

        # After finishing clicks, collect the data

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=False)  # deliberately disable headless so you can watch the process
        page = browser.new_page()

        # open the site
        page.goto("https://example.com")  # Replace with your target URL.

        # type a query into the search field
        page.fill("input[name='q']", "smartphone")  # Replace the query as needed

        # press Enter (simulate keyboard)
        page.press("input[name='q']", "Enter")

        # wait for results
        page.wait_for_selector(".search-results")  # Can be replaced with a timeout

        # click the first product
        page.locator(".search-results .item").first.click()

        # wait for the product card
        page.wait_for_selector(".product-card")

        print("Product card has been successfully opened!")

        browser.close()

if __name__ == "__main__":
    run()

from playwright.async_api import async_playwright
import asyncio

async def main():
    # Proxy credentials
    proxy_server = "http://proxy.example.com:8080"  # your proxy server
    proxy_username = "your_username"  # username
    proxy_password = "your_password"  # password

    # Launch a headless browser with the proxy
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=False,
            proxy={
                'server': proxy_server,
                'username': proxy_username,
                'password': proxy_password
            }
        )

        # Open a page
        page = await browser.new_page()

        # Check IP via a test site
        await page.goto("https://httpbin.org/ip")

        # Get the result
        content = await page.content()
        print("IP proxy:", content)

        # Close the browser
        await browser.close()

# Run the script
asyncio.run(main())

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=False)  # disable headless mode so you can see the captcha
        page = browser.new_page()

        # Load the target site
        page.goto("https://example.com")  # You can replace with your own URL

        # Check for a captcha block (for example, a reCAPTCHA iframe)
        try:
            page.wait_for_selector("iframe[title='reCAPTCHA']", timeout=5000)
            print("The captcha has been detected! Solve it manually...")

            # Wait until the user solves the captcha and new content appears
            page.wait_for_selector("#content-loaded", timeout=0)  # timeout=0 means wait indefinitely
            print("The captcha is solved, we continue to work.")
        except:
            print("The captcha did not appear, we continue to work.")

        # Further parsing or actions
        browser.close()

if __name__ == "__main__":
    run()

from playwright.async_api import async_playwright
import asyncio

async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            # The list of arguments can be extended
            args=[
                '--no-sandbox',
                '--disable-blink-features=AutomationControlled',
                '--disable-dev-shm-usage',
                '--disable-features=VizDisplayCompositor',
                '--disable-background-timer-throttling',
                '--disable-backgrounding-occluded-windows',
                '--disable-renderer-backgrounding',
                '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
            ]
        )

        page = await browser.new_page()

        # Hide headless-browser attributes using an initialization script
        await page.add_init_script("""
        Object.defineProperty(navigator, 'webdriver', {
            get: () => undefined,
        });
        delete navigator.__proto__.webdriver;
        """)

        # Navigate to the target page
        await page.goto('https://site.com/')

# Run the async function
asyncio.run(main())

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import time

# Setting up Selenium
def setup_driver():
    chrome_options = Options()
    chrome_options.add_argument("--headless")  # Remove this line for debugging
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--disable-dev-shm-usage")
    driver = webdriver.Chrome(options=chrome_options)
    return driver

# Parsing dynamic content
def parse_dynamic_content(url):
    driver = setup_driver()
    try:
        # Load the page
        driver.get(url)

        # Wait for dynamic content to load
        wait = WebDriverWait(driver, 10)
        wait.until(EC.presence_of_element_located((By.CLASS_NAME, "dynamic-content")))

        # Allow some time for JavaScript execution
        time.sleep(2)

        # Get the final HTML after JavaScript execution
        page_source = driver.page_source

        # Pass it to BeautifulSoup for parsing
        soup = BeautifulSoup(page_source, 'html.parser')

        # Extract data
        products = []
        product_elements = soup.find_all('div', class_='product-item')  # Class identifying all product cards

        # Iterate through product cards and extract details
        for product in product_elements:
            name = product.find('h3')
            price = product.find('span', class_='price')
            description = product.find('p', class_='description')
            products.append({
                'name': name.text.strip() if name else 'No name',
                'price': price.text.strip() if price else 'No price',
                'description': description.text.strip() if description else 'No description'
            })
        return products
    finally:
        driver.quit()

# Example usage
if __name__ == "__main__":
    url = "https://example.com/dynamic-products"
    data = parse_dynamic_content(url)
    for item in data:
        print(f"Name: {item['name']}")
        print(f"Price: {item['price']}")
        print(f"Description: {item['description']}")
        print("-" * 50)