Axios を用いた Web スクレイピングの方法

npm install axios

yarn add axios

const axios = require('axios');

axios.get('https://httpbin.org/get')
  .then(response => {
    console.log('ステータスコード:', response.status);
    console.log('レスポンスボディ:', response.data);
  })
  .catch(error => {
    console.error('エラー:', error);
  });

const axios = require('axios');

axios.get('https://jsonplaceholder.typicode.com/posts')
  .then(response => {
    const posts = response.data;
    posts.forEach(post => {
      console.log(`${post.title} — ${post.body}`);
    });
  })
  .catch(error => {
    console.error('エラー:', error);
  });

npm install axios
npm install capsolver

const axios = require('axios');
const CAPSOLVER_KEY = 'YourKey';
const PAGE_URL = 'https://antcpt.com/score_detector';
const PAGE_KEY = '6LcR_okUAAAAAPYrPe-HK_0RULO1aZM15ENyM-Mf';
const PAGE_ACTION = 'homepage';

async function createTask(url, key, pageAction) {
  try {
    const apiUrl = 'https://api.capsolver.com/createTask';
    const payload = {
      clientKey: CAPSOLVER_KEY,
      task: {
        type: 'ReCaptchaV3TaskProxyLess',
        websiteURL: url,
        websiteKey: key,
        pageAction: pageAction
      }
    };
    const headers = {
      'Content-Type': 'application/json',
    };
    const response = await axios.post(apiUrl, payload, { headers });
    return response.data.taskId;

  } catch (error) {
    console.error('CAPTCHA タスクの作成エラー:', error);
    throw error;
  }
}

async function getTaskResult(taskId) {
  try {
    const apiUrl = 'https://api.capsolver.com/getTaskResult';
    const payload = {
      clientKey: CAPSOLVER_KEY,
      taskId: taskId,
    };
    const headers = {
      'Content-Type': 'application/json',
    };
    let result;
    do {
      const response = await axios.post(apiUrl, payload, { headers });
      result = response.data;
      if (result.status === 'ready') {
        return result.solution;
      }
      await new Promise(resolve => setTimeout(resolve, 5000)); // 5 秒間待機してから再試行
    } while (true);

  } catch (error) {
    console.error('CAPTCHA 結果の取得エラー:', error);
    throw error;
  }
}

function setSessionHeaders() {
  return {
    'cache-control': 'max-age=0',
    'sec-ch-ua': '"Not/A)Brand";v="99", "Google Chrome";v="107", "Chromium";v="107"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': 'Windows',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'sec-fetch-site': 'same-origin',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-user': '?1',
    'sec-fetch-dest': 'document',
    'accept-encoding': 'gzip, deflate',
    'accept-language': 'en,fr-FR;q=0.9,fr;q=0.8,en-US;q=0.7',
  };
}

async function main() {
  const headers = setSessionHeaders();
  console.log('CAPTCHA タスクを作成しています...');
  const taskId = await createTask(PAGE_URL, PAGE_KEY, PAGE_ACTION);
  console.log(`タスク ID: ${taskId}`);

  console.log('CAPTCHA 結果を取得しています...');
  const solution = await getTaskResult(taskId);
  const token = solution.gRecaptchaResponse;
  console.log(`トークンの解決策: ${token}`);

  const res = await axios.post('https://antcpt.com/score_detector/verify.php', { 'g-recaptcha-response': token }, { headers });
  const response = res.data;
  console.log(`スコア: ${response.score}`);
}

main().catch(err => {
  console.error(err);
});

const axios = require('axios');

axios.get('https://httpbin.org/ip', {
  proxy: {
    host: 'proxyserver',
    port: 8080,
    auth: {
      username: 'username',
      password: 'password'
    }
  }
})
  .then(response => {
    console.log('レスポンスボディ:', response.data);
  })
  .catch(error => {
    console.error('エラー:', error);
  });

const axios = require('axios');

axios.get('https://httpbin.org/cookies/set?name=value', { withCredentials: true })
  .then(response => {
    console.log('Cookie:', response.headers['set-cookie']);
  })
  .catch(error => {
    console.error('エラー:', error);
  });

const axios = require('axios');

const headers = {
  'User-Agent': 'Mozilla/5.0 (compatible)',
  'Accept-Language': 'en-US,en;q=0.5',
};

const data = {
  username: 'testuser',
  password: 'testpass',
};

axios.post('https://httpbin.org/post', data, { headers })
  .then(response => {
    console.log('レスポンス JSON:', response.data);
  })
  .catch(error => {
    console.error('エラー:', error);
  });