Pythonウェブスクレイピング徹底解説：BeautifulSoupとScrapyでデータ抽出を学ぶ

pip install beautifulsoup4

pip install requests

import requests
from bs4 import BeautifulSoup

def scrape_example():
    url = "https://example.com"
    response = requests.get(url)
    
    # レスポンスが成功か確認
    if response.status_code != 200:
        print("ページの取得に失敗しました。")
        return
    
    # BeautifulSoupでHTML解析
    soup = BeautifulSoup(response.text, "html.parser")
    
    # タイトルタグの内容を取得
    title = soup.find("title").get_text()
    print("ページタイトル:", title)
    
    # 特定のクラスを持つ段落を取得する例
    paragraphs = soup.find_all("p", class_="content")
    for idx, para in enumerate(paragraphs, start=1):
        print(f"段落{idx}: {para.get_text()}")

if __name__ == "__main__":
    scrape_example()

pip install scrapy

scrapy startproject myproject

# myproject/spiders/example_spider.py
import scrapy

class ExampleSpider(scrapy.Spider):
    name = "example"
    start_urls = ["https://example.com"]

    def parse(self, response):
        # ページタイトルを抽出
        title = response.xpath("//title/text()").get()
        yield {"title": title}

        # もしリンクがあれば、さらにたどる例
        for href in response.xpath("//a/@href").getall():
            yield response.follow(href, self.parse)

# settings.py の一部例
DOWNLOAD_DELAY = 2  # 2秒間隔
USER_AGENT = "Mozilla/5.0 (compatible; MyScraper/1.0)"

headers = {"User-Agent": "Mozilla/5.0 (compatible; MyScraper/1.0)"}
response = requests.get(url, headers=headers)

import requests

def robust_scrape(url):
    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status()
    except requests.exceptions.HTTPError as http_err:
        print("HTTPエラー:", http_err)
    except requests.exceptions.ConnectionError as conn_err:
        print("接続エラー:", conn_err)
    except requests.exceptions.Timeout as timeout_err:
        print("タイムアウトエラー:", timeout_err)
    except Exception as err:
        print("予期せぬエラー:", err)
    else:
        return response.text

if __name__ == "__main__":
    html_content = robust_scrape("https://example.com")
    if html_content:
        print("ページ取得成功")