How to Bypass Cloudflare’s Anti-Bot Challenges: Puppeteer Web Scraping Version

  npm install puppeteer-extra puppeteer-extra-plugin-stealth puppeteer fs-extra
/**
 * NOTES!
 * Template: Puppeteer + Stealth + 2captcha
 * Configure:
 *  - PROXY (rotating) or an array of proxies
 *  - PATH_TO_CHROME (if you want to use a real Chrome browser instead of headless)
 *  - API_KEY_2CAPTCHA
 *
 * Note: the example uses global fetch (Node v18+). For older Node use node-fetch/axios.
 */
// Import libraries
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
const fs = require('fs-extra');
const path = require('path');

// Use stealth mode
puppeteer.use(StealthPlugin());

// ========== CONFIG ==========
const API_KEY_2CAPTCHA = 'REPLACE_WITH_YOUR_2CAPTCHA_KEY';
const PROFILE_DIR = path.resolve(__dirname, 'profiles'); // path for saving profile and cookies
const COOKIES_FILE = profileId => path.join(PROFILE_DIR, `${profileId}_cookies.json`);
const PATH_TO_CHROME = undefined; // path to a real browser '/usr/bin/google-chrome' or undefined to use the bundled headless browser
const HEADLESS = false; // we recommend false to start
// Proxy: if you have a rotator — just pass the proxy server (socks5/http)
const PROXY = 'http://username:password@proxy-host:port'; // or null

// ========== HELPERS ==========
// Async sleep and promises
async function sleep(ms){ return new Promise(r => setTimeout(r, ms)); }
// Save cookies
async function saveCookies(profileId, page){
  await fs.ensureDir(PROFILE_DIR);
  const cookies = await page.cookies();
  await fs.writeJson(COOKIES_FILE(profileId), cookies, { spaces: 2 });
}
// Load existing cookies
async function loadCookies(profileId, page){
  const f = COOKIES_FILE(profileId);
// If cookies file exists, read it
  if (await fs.pathExists(f)) {
    const cookies = await fs.readJson(f);
    try {
      await page.setCookie(...cookies);
      return true;
    } catch(e){
// If cookies can't be set, log the error to the console
      console.warn(Error when installing cookies:', e.message);
    }
  }
  return false;
}

// 2captcha: submit task and poll result
async function submit2CaptchaTask({ apiKey, sitekey, pageurl, method }) {
  // method: 'userrecaptcha' | 'hcaptcha' | 'turnstile' etc.
  // returns { requestId }
  const params = new URLSearchParams({
    key: apiKey,
    method,
    googlekey: sitekey, // for recaptcha userrecaptcha
    pageurl
  });

  // for turnstile 2captcha requires method "method=turnstile" and "sitekey"
  if (method === 'turnstile') {
    params.delete('googlekey');
    params.set('sitekey', sitekey);
  }

  params.set('json', 1);

  const res = await fetch(`http://2captcha.com/in.php?${params.toString()}`);
  const json = await res.json();
  if (json.status !== 1) throw new Error('2captcha submit error: ' + JSON.stringify(json));
  return { requestId: json.request };
}

// Set the number of CAPTCHA-solving attempts and the interval between requests (in milliseconds)
async function poll2CaptchaResult({ apiKey, requestId, maxAttempts = 10, interval = 5000 }) {
  const params = new URLSearchParams({
    key: apiKey,
    action: 'get',
    id: requestId,
    json: 1
  });

  for (let i = 0; i < maxAttempts; i++){
    await sleep(interval);
    const res = await fetch(`http://2captcha.com/res.php?${params.toString()}`);
    const json = await res.json();
    if (json.status === 1) {
      return json.request; // token
    } else if (json.request === 'CAPCHA_NOT_READY') {
      // keep waiting
      console.log(`2captcha: not ready (${i+1}/${maxAttempts})`);
      continue;
    } else {
      // log error to console
      throw new Error('2captcha error: ' + JSON.stringify(json));
    }
  }
  throw new Error('2captcha: timeout waiting for solution');
}

// CAPTCHA detection in HTML/DOM
async function detectCaptchaOnPage(page){
// Wait for content to load
  const html = await page.content();

  // quick HTML check, look for signs of recaptcha, hcaptcha, and turnstile
  const quick = /challenges\.cloudflare\.com\/turnstile|cdn-cgi\/challenge-platform|g-recaptcha|hcaptcha|www\.google\.com\/recaptcha|Checking your browser|One more step/i.test(html);

  // look for selectors and other indicators
  const hasRecaptcha = !!(await page.$('.g-recaptcha, [data-sitekey][data-widget-id*="recaptcha"], iframe[src*="recaptcha"]'));
  const hasHcaptcha = !!(await page.$('.h-captcha, iframe[src*="hcaptcha.com"]'));
  const hasTurnstile = !!(await page.$('[data-cf-turnstile], .cf-turnstile, iframe[src*="turnstile"]'));

  // try to find an explicit sitekey in HTML
  const sitekeyMatch = html.match(/data-sitekey=["']([^"']+)["']/i);
  const iframeUrls = (await page.frames()).map(f => f.url()).join(' ');
  const iframeSitekeyMatch = iframeUrls.match(/sitekey=([^&]+)/i);

  // check cookies
  const cookies = await page.cookies();
  const cookieNames = cookies.map(c => c.name).join(',');
  const hasCfClearance = /cf_clearance/i.test(cookieNames);

  return {
    quick,
    selectors: { hasRecaptcha, hasHcaptcha, hasTurnstile },
    sitekey: (sitekeyMatch && sitekeyMatch[1]) || (iframeSitekeyMatch && decodeURIComponent(iframeSitekeyMatch[1])) || null,
    cookieNames,
    hasCfClearance,
    iframeUrls
  };
}

// return the solved CAPTCHA into the DOM and submit the form
async function injectCaptchaToken(page, token, type){
  // type: 'recaptcha' | 'hcaptcha' | 'turnstile'
  if (type === 'recaptcha'){
    // classic: g-recaptcha-response textarea
    await page.evaluate((t) => {
      let el = document.querySelector('textarea[name="g-recaptcha-response"]');
      if (!el) {
        el = document.createElement('textarea');
        el.name = 'g-recaptcha-response';
        el.style.display = 'none';
        document.body.appendChild(el);
      }
      el.value = t;
      // some sites expect an explicit callback trigger
      if (window.grecaptcha && grecaptcha && grecaptcha.getResponse) {
      }
    }, token);

    // attempt to inject the solution
    await page.evaluate(() => {
      // triggers that some forms listen for
      document.dispatchEvent(new Event('recaptcha-token-injected'));
    });
  } else if (type === 'turnstile'){
    await page.evaluate((t) => {
      // typical Cloudflare Turnstile input name: cf-turnstile-response
      let el = document.querySelector('textarea[name="cf-turnstile-response"], input[name="cf-turnstile-response"]');
      if (!el) {
        el = document.createElement('textarea');
        el.name = 'cf-turnstile-response';
        el.style.display = 'none';
        document.body.appendChild(el);
      }
      el.value = t;
      document.dispatchEvent(new Event('turnstile-token-injected'));
    }, token);
  } else if (type === 'hcaptcha'){

   // hcaptcha solution injection
    await page.evaluate((t) => {
      let el = document.querySelector('textarea[name="h-captcha-response"], input[name="h-captcha-response"]');
      if (!el) {
        el = document.createElement('textarea');
        el.name = 'h-captcha-response';
        el.style.display = 'none';
        document.body.appendChild(el);
      }
      el.value = t;
      document.dispatchEvent(new Event('hcaptcha-token-injected'));
    }, token);
  }

  // try to auto-submit the form if one exists
  try {
    await page.evaluate(() => {
      const forms = Array.from(document.forms);
      if (forms.length === 1) forms[0].submit();
      else {
        // if a submit button exists
        const btn = document.querySelector('button[type="submit"], input[type="submit"]');
        if (btn) btn.click();
      }
    });
  } catch(e){
    console.warn('Error when auto-submitting the form:', e.message);
  }
}

// Main scraper
async function runScrape({ url, profileId = 'default', proxy = PROXY }) {
// Additional arguments to hide headless browser traces
  const launchArgs = [
    '--no-sandbox',
    '--disable-setuid-sandbox',
    '--disable-dev-shm-usage',
    '--disable-blink-features=AutomationControlled',
    '--lang=en-US,en'
  ];

  if (proxy) launchArgs.push(`--proxy-server=${proxy}`);

  const browser = await puppeteer.launch({
    headless: HEADLESS,
    executablePath: PATH_TO_CHROME || undefined,
    args: launchArgs,

// It's a good idea to set realistic browser window parameters
    defaultViewport: { width: 1366, height: 768 }
  });

  const page = await browser.newPage();

  // HTTP headers: user agent and extra headers
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36');
  await page.setExtraHTTPHeaders({ 'Accept-Language': en-US,en;q=0.9,en-US;q=0.8,en;q=0.7' });

  // Important: early overrides before page load
  await page.evaluateOnNewDocument(() => {
    // navigator.webdriver
    Object.defineProperty(navigator, 'webdriver', { get: () => false });

    // languages
    Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en', 'ru-RU', 'ru'] });

    // plugins - simple emulation
    Object.defineProperty(navigator, 'plugins', {
      get: () => [{ name: 'Chrome PDF Plugin' }, { name: 'Native Client' }]
    });

    // chrome runtime
    window.chrome = window.chrome || { runtime: {} };

    // WebGL patch (injecting a primitive emulation)
    try {
      const getParameter = WebGLRenderingContext.prototype.getParameter;
      WebGLRenderingContext.prototype.getParameter = function(parameter) {
        if (parameter === 37445) return 'Intel Inc.';
        if (parameter === 37446) return 'Intel(R) UHD Graphics 620';
        return getParameter.call(this, parameter);
      };
    } catch(e){}

    try {
      const origQuery = navigator.permissions.query;
      navigator.permissions.query = (params) => (
        params.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : origQuery(params)
      );
    } catch(e){}
  });

  // Try to load cookies from profile
  await loadCookies(profileId, page);

  // Create a CDP session to override timezone, locale, and geolocation if needed
  const client = await page.target().createCDPSession();
  try {
    await client.send('Emulation.setTimezoneOverride', { timezoneId: 'America/New_York' });
    await client.send('Emulation.setLocaleOverride', { locale: 'en_US' });
    // geolocation example (uncomment if needed)
    // await client.send('Emulation.setGeolocationOverride', { latitude: 55.75, longitude: 37.616, accuracy: 20 });
  } catch(e){ console.warn('CDP emulation not available:', e.message); }

  // Navigation + Cloudflare IUAM and CAPTCHA detection
  console.log('Opening the address:', url);
  const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 }).catch(e => null);

  // give the JS challenge some time to execute
  await sleep(8000);

  // additionally, wait for navigation if redirects occur
  try { await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 }).catch(()=>{}); } catch(e){}

  // detection
  const detection = await detectCaptchaOnPage(page);
  console.log('Detection:', detection.selectors, 'quickHtml:', detection.quick);
  console.log('Cookies:', detection.cookieNames);

  if (detection.hasCfClearance) {
    console.log('Cloudflare clearance cookie present — most likely the check was passed.');
  }

  // If a CAPTCHA is detected — solve it via 2captcha
  if (detection.selectors.hasRecaptcha || detection.selectors.hasHcaptcha || detection.selectors.hasTurnstile) {
    const sitekey = detection.sitekey;
    console.log('sitekey detected:', sitekey);
    let captchaType = null;
    if (detection.selectors.hasRecaptcha) captchaType = 'recaptcha';
    else if (detection.selectors.hasHcaptcha) captchaType = 'hcaptcha';
    else if (detection.selectors.hasTurnstile) captchaType = 'turnstile';

    if (!sitekey) {
      // attempt to extract sitekey from iframe URLs
      const iframeUrls = detection.iframeUrls || '';
      const sk = iframeUrls.match(/sitekey=([^&]+)/i);
      if (sk) {
        detection.sitekey = decodeURIComponent(sk[1]);
      }
    }

    if (!detection.sitekey) {
      console.warn('The sitekey could not be found automatically. Try manually specifying a sitekey or adding a detector.');
    } else {
      try {
        console.log('Sending the task to 2captcha (type=${captchaType})...');
        const methodMap = { recaptcha: 'userrecaptcha', hcaptcha: 'hcaptcha', turnstile: 'turnstile' };
        const submit = await submit2CaptchaTask({
          apiKey: API_KEY_2CAPTCHA,
          sitekey: detection.sitekey,
          pageurl: url,
          method: methodMap[captchaType]
        });
        console.log('2captcha requestId=', submit.requestId || submit.request);
        const requestId = submit.requestId || submit.request;
        const token = await poll2CaptchaResult({ apiKey: API_KEY_2CAPTCHA, requestId });
        console.log('2captcha token received, inserted into the page:', token.substring(0, 20) + '...');

        // inject the token and attempt auto-submit
        await injectCaptchaToken(page, token, captchaType);
        // wait for redirect / page update
        await sleep(3000);
        await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 15000 }).catch(()=>{});
        console.log('We check the result after inserting the token...');
        const afterDetection = await detectCaptchaOnPage(page);
        console.log('After:', afterDetection.selectors, 'cookies:', afterDetection.cookieNames);
      } catch (e) {
        console.error('Error solving captcha:', e.message);
      }
    }
  } else {
    console.log('Captcha not found on the page (by selectors).');
  }

  // If everything is OK — save cookies for subsequent sessions
  await saveCookies(profileId, page);
  console.log('Cookies saved.');

  // next — custom page parsing
  const content = await page.content();
  // TODO: parse required data
  console.log('Length HTML:', content.length);

  // Finish
  await browser.close();
}

// ========== Run ==========

(async () => {
  try {
    const targetUrl = process.argv[2] || 'https://target-site.example/';
    await runScrape({ url: targetUrl, profileId: 'default' });
    console.log('Done.');
  } catch (e) {
    console.error('Fatal error:', e);
  }
})();