Web Scraping in C#: Step-by-Step Tutorial in 2025

Install-Package HtmlAgilityPack        # For HTML parsing
Install-Package Selenium.WebDriver     # For browser automation
Install-Package PuppeteerSharp         # For screenshots & advanced scraping
Install-Package Capsolver.SDK          # For CAPTCHA solving

using HtmlAgilityPack;
using System.Net;

var url = "https://www.scrapethissite.com/pages/simple/";
var client = new WebClient();
client.Headers.Add("User-Agent", "Mozilla/5.0"); // Be polite!
var html = client.DownloadString(url);

var doc = new HtmlDocument();
doc.LoadHtml(html);

var countries = doc.DocumentNode.SelectNodes("//div[@class='country']");
foreach (var country in countries)
{
    var name = country.SelectSingleNode(".//h3").InnerText.Trim();
    var capital = country.SelectSingleNode(".//span[@class='country-capital']").InnerText.Trim();
    Console.WriteLine($"Country: {name}, Capital: {capital}");
}

using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;

var options = new ChromeOptions();
options.AddArgument("--headless"); // Run in background
using var driver = new ChromeDriver(options);

driver.Navigate().GoToUrl("https://www.scrapethissite.com/pages/ajax-javascript/");
var dynamicContent = driver.FindElement(By.CssSelector(".ajax-content")).Text;
Console.WriteLine($"AJAX Content: {dynamicContent}");

using System;
using System.Net.Http;
using System.Threading.Tasks;
using System.Text;
using System.Text.Json;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using System.Threading;

namespace CapSolverSeleniumExample
{
    class Program
    {
        static async Task Main(string[] args)
        {
            string token = await GetCaptchaToken();
            using var driver = new ChromeDriver();
            driver.Navigate().GoToUrl("https://www.example.com");
            Thread.Sleep(5000);
            driver.ExecuteScript("document.getElementById('g-recaptcha-response').innerHTML = arguments[0];", token);
            var submitButton = driver.FindElement(By.Id("submit-button"));
            submitButton.Click();
            Thread.Sleep(5000);
            driver.Quit();
        }

        static async Task<string> GetCaptchaToken()
        {
            string apiKey = "YOUR_API_KEY";
            string siteKey = "6Le-wvkSAAAAAPBMRTvw0Q4Muexq9bi0DJwx_kl-";
            string siteUrl = "https://www.example.com";
            using var client = new HttpClient();
            var payload = new
            {
                clientKey = apiKey,
                task = new
                {
                    type = "ReCaptchaV3TaskProxyLess",
                    websiteKey = siteKey,
                    websiteURL = siteUrl,
                    pageAction = "login"
                }
            };
            var requestContent = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json");
            var createTaskResponse = await client.PostAsync("https://api.capsolver.com/createTask", requestContent);
            var createTaskResponseString = await createTaskResponse.Content.ReadAsStringAsync();
            using var createTaskJsonDoc = JsonDocument.Parse(createTaskResponseString);
            var root = createTaskJsonDoc.RootElement;
            if (!root.TryGetProperty("taskId", out var taskIdElement))
            {
                Console.WriteLine("Failed to create task: " + createTaskResponseString);
                return null;
            }
            int taskId = taskIdElement.GetInt32();
            while (true)
            {
                await Task.Delay(1000);
                var resultPayload = new { clientKey = apiKey, taskId = taskId };
                var resultContent = new StringContent(JsonSerializer.Serialize(resultPayload), Encoding.UTF8, "application/json");
                var getTaskResponse = await client.PostAsync("https://api.capsolver.com/getTaskResult", resultContent);
                var getTaskResponseString = await getTaskResponse.Content.ReadAsStringAsync();
                using var getTaskJsonDoc = JsonDocument.Parse(getTaskResponseString);
                var resultRoot = getTaskJsonDoc.RootElement;
                if (resultRoot.TryGetProperty("status", out var statusElement))
                {
                    string status = statusElement.GetString();
                    if (status == "ready")
                    {
                        if (resultRoot.TryGetProperty("solution", out var solutionElement) && solutionElement.TryGetProperty("gRecaptchaResponse", out var tokenElement))
                        {
                            return tokenElement.GetString();
                        }
                        return null;
                    }
                    if (status == "failed" || resultRoot.TryGetProperty("errorId", out _))
                    {
                        Console.WriteLine("Solve failed! response: " + getTaskResponseString);
                        return null;
                    }
                }
            }
        }
    }
}

using PuppeteerSharp;

await new BrowserFetcher().DownloadAsync();
using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true });
using var page = await browser.NewPageAsync();

await page.GoToAsync("https://www.scrapethissite.com/pages/javascript/");
await page.ScreenshotAsync("screenshot.png");