npm install puppeteer-core

import puppeteer from "puppeteer-core";

// Function to introduce a delay
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));

// Non-headless launch
const startIncogniton = async ({ profileId }) => {
  try {
    const launchUrl = `http://localhost:35000/automation/launch/puppeteer`;
    const requestBody = {
      profileID: profileId,
    };

    // Make a POST request with body data
    const response = await fetch(launchUrl, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
      body: JSON.stringify(requestBody),
    });

    const data = await response.json();
    const { puppeteerUrl } = data;

    // Wait for the browser to launch
    console.log('The Incogniton browser is launching...');
    await delay(30000); // await initial boot process

    // Connect Puppeteer to the launched browser
    const browser = await puppeteer.connect({
      browserURL: puppeteerUrl,
      acceptInsecureCerts: true,
    });

    return browser;
  } catch (error) {
    console.error("Error starting Incogniton session ->", error);
    throw error;
  }
};

// Headless launch
const startIncogniton = async ({ profileId }) => {
  try {
    const launchUrl = `http://localhost:35000/automation/launch/puppeteer`;
    const requestBody = {
      profileID: profileId,
      customArgs: '--headless=new', // use headless mode
    };

    // Rest of the function remains the same...
  }
};

const incognitonProfileId = "YOUR_INCOGNITON_PROFILE_ID";

const ipheyTest = async (browser) => {
  try {
    const page = await browser.newPage();
    // Navigate to the IPHey website and wait till zero network requests
    await page.goto("https://iphey.com/", { waitUntil: "networkidle0" });

    // Check for 'trustworthy status' in the DOM
    const ipResult = await page.$eval(
      ".trustworthy-status:not(.hide)",
      (elt) => (elt ? elt.innerText.trim() : "")
    );

    console.log("IP Result ->", ipResult); // expected output: 'Trustworthy'
    await page.close();
  } catch (error) {
    console.error("Error during IPHEY test ->", error);
  } finally {
    await browser.close();
  }
};

// Execute iphey test
const testIncognitonProfile = async () => {
  const browser = await startIncogniton({ profileId: incognitonProfileId });
  await ipheyTest(browser);
};

testIncognitonProfile();

const scrapeDynamicContent = async (profileId) => {
  try {
    // Start Incogniton browser
    const browser = await startIncogniton({ profileId });
    const page = await browser.newPage();

    // Navigate to the dynamic page with client-side rendering
    await page.goto("https://quotes.toscrape.com/js/", {
      waitUntil: "networkidle0",
    });

    // Extract quotes and authors from dynamically rendered content
    const quotes = await page.$$eval(".quote", (elements) =>
      elements.map((element) => ({
        text: element && element.querySelector(".text").innerText.trim(),
        author: element && element.querySelector(".author").innerText.trim(),
      }))
    );

    console.log("Extracted Quotes ->", quotes);

    // Close the browser after scraping
    await browser.close();
  } catch (error) {
    console.error("Error scraping dynamically loaded content ->", error);
  }
};

Extracted Quotes -> [
  {
    text: '"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking."',
    author: 'Albert Einstein'
  },
  {
    text: '"It is our choices, Harry, that show what we truly are, far more than our abilities."',
    author: 'J.K. Rowling'
  },
  {
    text: '"The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid."',
    author: 'Jane Austen'
  },
  // other quotes...
]

const scrapeRecursively = async ({ browser, givenPage, scrapeUrl, allData }) => {
  try {
    // Use the provided page
    const page = givenPage;
    await page.goto(scrapeUrl, { waitUntil: 'networkidle0' });

    // Extract quotes from current page
    const quotes = await page.$$eval('.quote', elements =>
      elements.map(el => ({
        text: el.querySelector('.text').innerText.trim(),
        author: el.querySelector('.author').innerText.trim(),
      }))
    );

    // Add current page data to the collection
    allData.push(...quotes);

    // Look for next page link
    const nextLink = await page.$('li.next a');

    // If there's a next button, continue scraping
    if (nextLink) {
      const href = await nextLink.evaluate(el => el.href);
      await scrapeRecursively({
        givenPage: page,
        scrapeUrl: href,
        allData,
      });
    }

    return {
      data: allData,
    };
  } catch (error) {
    console.error('Error scraping dynamically loaded content ->', error);
    throw error;
  }
};

// Usage example
const scrapeAllPages = async profileId => {
  try {
    const browser = await startIncogniton({ profileId });
    const page = await browser.newPage();
    const allData = [];

    await scrapeRecursively({
      browser,
      givenPage: page,
      scrapeUrl: 'https://quotes.toscrape.com/js/',
      allData,
    });

    console.log('ALL SCRAPED DATA ->', allData);
    browser.close();
  } catch (err) {
    console.error(err);
    throw err;
  }
};