How to Use node-fetch for Web Scraping

npm install node-fetch

yarn add node-fetch

const fetch = require('node-fetch');

fetch('https://httpbin.org/get')
  .then(response => response.json())
  .then(data => {
    console.log('Response Body:', data);
  })
  .catch(error => {
    console.error('Error:', error);
  });

const fetch = require('node-fetch');

fetch('https://jsonplaceholder.typicode.com/posts')
  .then(response => response.json())
  .then(posts => {
    posts.forEach(post => {
      console.log(`${post.title} — ${post.body}`);
    });
  })
  .catch(error => {
    console.error('Error:', error);
  });

npm install node-fetch
npm install capsolver

const fetch = require('node-fetch');
const CAPSOLVER_KEY = 'YourKey';
const PAGE_URL = 'https://antcpt.com/score_detector';
const PAGE_KEY = '6LcR_okUAAAAAPYrPe-HK_0RULO1aZM15ENyM-Mf';
const PAGE_ACTION = 'homepage';

async function createTask(url, key, pageAction) {
  try {
    const apiUrl = 'https://api.capsolver.com/createTask';
    const payload = {
      clientKey: CAPSOLVER_KEY,
      task: {
        type: 'ReCaptchaV3TaskProxyLess',
        websiteURL: url,
        websiteKey: key,
        pageAction: pageAction
      }
    };
    
    const response = await fetch(apiUrl, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify(payload)
    });
    
    const data = await response.json();
    return data.taskId;

  } catch (error) {
    console.error('Error creating CAPTCHA task:', error);
    throw error;
  }
}

async function getTaskResult(taskId) {
  try {
    const apiUrl = 'https://api.capsolver.com/getTaskResult';
    const payload = {
      clientKey: CAPSOLVER_KEY,
      taskId: taskId,
    };

    let result;
    do {
      const response = await fetch(apiUrl, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(payload)
      });

      result = await response.json();
      if (result.status === 'ready') {
        return result.solution;
      }

      await new Promise(resolve => setTimeout(resolve, 5000)); // wait 5 seconds
    } while (true);

  } catch (error) {
    console.error('Error fetching CAPTCHA result:', error);
    throw error;
  }
}

async function main() {
  console.log('Creating CAPTCHA task...');
  const taskId = await createTask(PAGE_URL, PAGE_KEY, PAGE_ACTION);
  console.log(`Task ID: ${taskId}`);

  console.log('Retrieving CAPTCHA result...');
  const solution = await getTaskResult(taskId);
  const token = solution.gRecaptchaResponse;
  console.log(`Token Solution: ${token}`);

  const res = await fetch('https://antcpt.com/score_detector/verify.php', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ 'g-recaptcha-response': token })
  });

  const response = await res.json();
  console.log(`Score: ${response.score}`);
}

main().catch(err => {
  console.error(err);
});

npm install https-proxy-agent

const fetch = require('node-fetch');
const HttpsProxyAgent = require('https-proxy-agent');

const proxyAgent = new HttpsProxyAgent('http://username:password@proxyserver:8080');

fetch('https://httpbin.org/ip', { agent: proxyAgent })
  .then(response => response.json())
  .then(data => {
    console.log('Response Body:', data);
  })
  .catch(error => {
    console.error('Error:', error);
  });

npm install fetch-cookie

const fetch = require('node-fetch');
const fetchCookie = require('fetch-cookie');
const cookieFetch = fetchCookie(fetch);

cookieFetch('https://httpbin.org/cookies/set?name=value')
  .then(response => response.json())
  .then(data => {
    console.log('Cookies:', data);
  })
  .catch(error => {
    console.error('Error:', error);
  });

const fetch = require('node-fetch');

const headers = {
  'User-Agent': 'Mozilla/5.0 (compatible)',
  'Accept-Language': 'en-US,en;q=0.5',
};

const data = {
  username: 'testuser',
  password: 'testpass',
};

fetch('https://httpbin.org/post', {
  method: 'POST',
  headers: headers,
  body: JSON.stringify(data),
})
  .then(response => response.json())
  .then(data => {
    console.log('Response JSON:', data);
  })
  .catch(error => {
    console.error('Error:', error);
  });