When I use Pupeteer, Axios and Cheerio, I get different results from those displayed by Chrome's inspection tool. On the contrary, these results are inherent to the site itself, and not to its content (the object of the search).
'use strict';import puppeteer from 'puppeteer';import { scrollPageToBottom} from 'puppeteer-autoscroll-down'; // Import using ES6 syntaxconst productUrl = 'https://www.ouedkniss.com/automobiles/1';async function scrapeProduct() { try { console.log(`Scraping product from: ${productUrl}`); const browser = await puppeteer.launch(); const [page] = await browser.pages(); // Bright data const proxyUsername = 'brd-customer-hl_be1bd57e-zone-carfinder'; const proxyPassword = '0k4kkmsfox1l'; await page.authenticate({ username: proxyUsername, password: proxyPassword }); await page.goto(productUrl, { waitUntil: 'domcontentloaded', timeout: 90000 }); // Increased timeout // Use ES6 import syntax const { scrollPageToBottom } = await import ('puppeteer-autoscroll-down'); const lastPosition = await scrollPageToBottom(page, { size: 210, delay: 350, }); // Wait for a specific element to appear on the page //await page.waitForSelector('span.v-btn__content', { timeout: 90000 }); // Adjust the selector as needed // Retrieve the content after waiting const data = await page.content(); // Example: Extracting the title without Cheerio //const title = await page.$eval('h1.text-h5', element => element.textContent.trim()); console.log(data); // If you need to extract more data, use Puppeteer selectors here await browser.close(); } catch (error) { console.error(`Failed to scrape product from ${productUrl}: ${error.message}`); throw new Error(`Failed to scrape product: ${error.message}`); }}// Call the function to initiate the scrapingscrapeProduct();