Python 網頁爬蟲實戰進階技巧

# 建立虛擬環境
python -m venv web_scraping_env

# 在 Windows 上啟用虛擬環境
web_scraping_env\Scripts\activate

# 在 Linux/Mac 上啟用虛擬環境
source web_scraping_env/bin/activate

pip install requests beautifulsoup4

# 若需處理動態網頁
pip install selenium webdriver-manager

# 若需高效能解析
pip install lxml

# 若需開發大型爬蟲專案
pip install scrapy

# 完整爬蟲程式碼

import os
import requests
from bs4 import BeautifulSoup

url = 'https://commons.wikimedia.org/wiki/List_of_dog_breeds'

# 設定請求標頭，模擬瀏覽器行為
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}

# 使用 requests 下載頁面 HTML
response = requests.get(url, headers=headers)

# 確認請求是否成功
if response.status_code == 200:

    # 使用 BeautifulSoup 解析 HTML
    soup = BeautifulSoup(response.text, 'html.parser')

    # 使用 CSS 選擇器找到主要資料表
    table = soup.find('table', {'class': 'wikitable sortable'})

    # 初始化資料儲存列表
    names = []
    groups = []
    local_names = []
    photographs = []

    # 建立圖片儲存資料夾
    os.makedirs('dog_images', exist_ok=True)

    # 逐行處理表格資料，跳過標題行
    for row in table.find_all('tr')[1:]:

        # 使用 CSS 選擇器提取每欄資料
        columns = row.find_all(['td', 'th'])

        name = columns[0].find('a').text.strip()
        group = columns[1].text.strip()

        # 提取本地名稱（如果存在）
        span_tag = columns[2].find('span')
        local_name = span_tag.text.strip() if span_tag else ''

        # 提取圖片 URL（如果存在）
        img_tag = columns[3].find('img')
        photograph = img_tag['src'] if img_tag else ''

        # 下載並儲存圖片（如果存在）
        if photograph:
            response = requests.get(photograph)
            if response.status_code == 200:
                image_filename = os.path.join('dog_images', f'{name}.jpg')
                with open(image_filename, 'wb') as img_file:
                    img_file.write(response.content)

        # 將資料加入對應列表
        names.append(name)
        groups.append(group)
        local_names.append(local_name)
        photographs.append(photograph)

# 輸出擷取到的資料
print(names)
print(groups)
print(local_names)
print(photographs)

import requests
from bs4 import BeautifulSoup
import os

url = 'https://commons.wikimedia.org/wiki/List_of_dog_breeds'

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}

response = requests.get(url, headers=headers)

if response.status_code == 200:
   # 連線成功，繼續處理
   html_content = response.text
else:
   print(f"連線失敗，狀態碼：{response.status_code}")

soup = BeautifulSoup(response.text, 'html.parser')

<div class="book-listing">
  <img align="center" src="/covers/harry-potter.jpg">
  <span class="title">Harry Potter and the Goblet of Fire</span>
  <span class="rating">9.1</span>
</div>

<div class="book-listing">
  <img align="center" src="/covers/lord-of-rings.jpg">
  <span class="title">The Fellowship of the Ring</span>
  <span class="rating">9.3</span>
</div>

titles = soup.select("div.book-listing > span.title")
book_names = [title.text for title in titles]

# 透過 ID 選擇
elements = soup.select("#book-title")

# 透過屬性值完全比對
links = soup.select('a[href="/login"]')

# 透過屬性值部分比對
spans = soup.select('span[class^="title"]')  # 以 "title" 開頭的 class

# 選擇直接子元素
list_items = soup.select("ul > li")

table = soup.find('table', {'class': 'wikitable sortable'})

for row in table.find_all('tr')[1:]:  # 跳過標題行
    columns = row.find_all(['td', 'th'])
    
    if columns:  # 確保行不為空
        name = columns[0].find('a').text.strip()
        group = columns[1].text.strip()
        
        print(f"狗種類別: {name}, 分組: {group}")

# 建立資料夾儲存圖片
if not os.path.exists('dog_images'):
    os.makedirs('dog_images')

for row in table.find_all('tr')[1:]:
    columns = row.find_all(['td', 'th'])
    
    if len(columns) >= 3:  # 確保有足夠的列
        name = columns[0].find('a').text.strip()
        
        # 找出圖片元素
        img_element = columns[2].find('img')
        if img_element and 'src' in img_element.attrs:
            img_url = img_element['src']
            if not img_url.startswith('http'):
                img_url = 'https:' + img_url
            
            # 下載圖片
            img_response = requests.get(img_url)
            if img_response.status_code == 200:
                # 建立安全的檔名
                safe_name = "".join([c if c.isalnum() else "_" for c in name])
                img_path = f'dog_images/{safe_name}.jpg'
                
                with open(img_path, 'wb') as img_file:
                    img_file.write(img_response.content)
                print(f"已下載 {name} 的圖片")

import time
from requests.exceptions import RequestException

def fetch_with_retry(url, max_retries=3):
    for attempt in range(max_retries):
        try:
            response = requests.get(url, headers=headers)
            response.raise_for_status()  # 檢查是否有 HTTP 錯誤
            return response
        except RequestException as e:
            if attempt < max_retries - 1:
                wait_time = 2 ** attempt  # 指數退避策略
                print(f"請求失敗: {e}，{wait_time} 秒後重試...")
                time.sleep(wait_time)
            else:
                print(f"重試 {max_retries} 次後仍然失敗: {e}")
                return None

import random

# 在請求之間增加隨機延遲
time.sleep(random.uniform(1, 3))

base_url = "https://example.com/page/"
all_data = []

for page_num in range(1, 6):  # 處理前 5 頁
    page_url = f"{base_url}{page_num}"
    response = fetch_with_retry(page_url)
    if not response:
        continue
        
    soup = BeautifulSoup(response.text, 'html.parser')
    # 提取目前頁面的資料
    page_data = extract_data_from_page(soup)
    all_data.extend(page_data)
    
    # 避免請求過於頻繁
    time.sleep(random.uniform(2, 5))

import requests
from bs4 import BeautifulSoup
import os
import time
import random
from requests.exceptions import RequestException

# 建立資料夾儲存圖片
if not os.path.exists('dog_breeds'):
    os.makedirs('dog_breeds')

def fetch_with_retry(url, max_retries=3):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }
    
    for attempt in range(max_retries):
        try:
            response = requests.get(url, headers=headers)
            response.raise_for_status()
            return response
        except RequestException as e:
            if attempt < max_retries - 1:
                wait_time = 2 ** attempt
                print(f"請求失敗: {e}，{wait_time} 秒後重試...")
                time.sleep(wait_time)
            else:
                print(f"重試 {max_retries} 次後仍然失敗: {e}")
                return None

def sanitize_filename(name):
    return "".join([c if c.isalnum() or c in "._- " else "_" for c in name]).strip()

def main():
    url = 'https://commons.wikimedia.org/wiki/List_of_dog_breeds'
    
    print("開始爬取狗品種資料...")
    response = fetch_with_retry(url)
    if not response:
        print("無法取得網頁內容，程式結束")
        return
    
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 找到主要的表格
    table = soup.find('table', {'class': 'wikitable sortable'})
    if not table:
        print("找不到包含狗品種資料的表格")
        return
    
    # 準備儲存所有資料的列表
    all_breeds = []
    
    # 處理表格中的每一行
    for row in table.find_all('tr')[1:]:  # 跳過標題行
        columns = row.find_all(['td', 'th'])
        
        if len(columns) >= 3:
            try:
                # 提取名稱
                name_element = columns[0].find('a')
                if name_element:
                    name = name_element.text.strip()
                else:
                    name = columns[0].text.strip()
                
                # 提取分組
                group = columns[1].text.strip()
                
                # 提取圖片
                img_element = columns[2].find('img')
                img_url = None
                if img_element and 'src' in img_element.attrs:
                    img_url = img_element['src']
                    if not img_url.startswith('http'):
                        img_url = 'https:' + img_url
                
                # 儲存資料
                breed_info = {
                    'name': name,
                    'group': group,
                    'image_url': img_url
                }
                all_breeds.append(breed_info)
                
                # 下載圖片
                if img_url:
                    img_response = fetch_with_retry(img_url)
                    if img_response:
                        safe_name = sanitize_filename(name)
                        img_path = f'dog_breeds/{safe_name}.jpg'
                        
                        with open(img_path, 'wb') as img_file:
                            img_file.write(img_response.content)
                        print(f"已下載 {name} 的圖片")
                    
                    # 隨機延遲，避免請求過於頻繁
                    time.sleep(random.uniform(0.5, 1.5))
            
            except Exception as e:
                print(f"處理資料時發生錯誤: {e}")
    
    # 輸出統計資訊
    print(f"\n成功爬取 {

group = columns[1].text.strip()

img_tag = columns[3].find('img')
photograph = img_tag['src'] if img_tag else ''

if photograph:
    response = requests.get(photograph)
    image_filename = os.path.join('dog_images', f'{name}.jpg')
    
    with open(image_filename, 'wb') as img_file:
       img_file.write(response.content)

from selenium import webdriver
from selenium.webdriver.common.by import By

# 初始化 Chrome 瀏覽器驅動
driver = webdriver.Chrome()

# 載入頁面
driver.get("https://example.com")

# 等待動態 JS 執行載入標題
driver.implicitly_wait(10)

# Selenium 可擷取動態載入的元素
print(driver.title)

# Selenium 可點選按鈕觸發 JS 事件
driver.find_element(By.ID, "dynamicBtn").click()

# 也可處理輸入
search = driver.find_element(By.NAME, 'search')
search.send_keys('Automate using Selenium')
search.submit()

# 完成後關閉瀏覽器
driver.quit()

import time
import random
import requests
from urllib.request import ProxyHandler, build_opener

# 免費公共代理伺服器列表
PROXIES = ["104.236.141.243:8080", "104.131.178.157:8085"]

# 每次請求間隔5-15秒
def get_request():
    time.sleep(random.randint(5, 15))
    proxy = random.choice(PROXIES)
    opener = build_opener(ProxyHandler({'https': proxy}))

    resp = opener.open("https://example.com")
    return resp

for i in range(50):
   response = get_request()
   print("Request Success")

import requests
import random

# 桌面瀏覽器User-Agent列表
user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 OPR/43.0.2442.991",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/604.4.7 (KHTML, like Gecko) Version/11.0.2 Safari/604.4.7"
]

# 隨機選擇User-Agent
user_agent = random.choice(user_agents)

# 在請求標頭中設定User-Agent
headers = {"User-Agent": user_agent}

response = requests.get(url, headers=headers)

from selenium import webdriver
import random

# 常見螢幕解析度列表
screen_res = [(1366, 768), (1920, 1080), (1024, 768)]

# 常見字型列表
font_families = ["Arial", "Times New Roman", "Verdana"]

# 選擇隨機解析度
width, height = random.choice(screen_res)

# 建立Chrome選項
opts = webdriver.ChromeOptions()

# 設定隨機螢幕解析度
opts.add_argument(f"--window-size={width},{height}")

# 設定隨機User-Agent
opts.add_argument("--user-agent=Mozilla/5.0...")

# 設定隨機字型列表
random_fonts = random.choices(font_families, k=2)
opts.add_argument(f'--font-list="{random_fonts[0]};{random_fonts[1]}"')

# 初始化瀏覽器驅動
driver = webdriver.Chrome(options=opts)

# 存取目標網站
driver.get(target_url)

import requests
import random

# 裝置設定檔
桌面裝置設定 = {
    'user-agent': 'Mozilla/5.0...',
    'accept-language': ['en-US,en', 'en-GB,en'],
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'accept-encoding': 'gzip, deflate, br',
    'upgrade-insecure-requests': '1',
    'sec-fetch-site': 'none',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-user': '?1',
    'sec-fetch-dest': 'document',
    'cache-control': 'max-age=0'
}

行動裝置設定 = {
    'user-agent': 'Mozilla/5.0... Mobile',
    'accept-language': ['en-US,en', 'en-GB,en'],
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'x-requested-with': 'mark.via.gp',
    'sec-fetch-site': 'same-origin',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-user': '?1',
    'sec-fetch-dest': 'document',
    'referer': 'https://www.example.com/',
    'accept-encoding': 'gzip, deflate, br',
    'cache-control': 'max-age=0'
}

裝置設定檔列表 = [桌面裝置設定, 行動裝置設定]

def 建立請求標頭():
    設定檔 = random.choice(裝置設定檔列表)

    標頭 = {
         'User-Agent': random.choice(設定檔['user-agent']),
         'Accept-Language': random.choice(設定檔['accept-language']),
         # 其他標頭設定...
    }

    return 標頭

<b><font color="#3AC1EF">最新訊息 <p>油價下跌的影響...</font></b></p>

<div>
  <span class="title">Python 網頁爬蟲 <span>
  Lorem ipsum...
</div>

title = soup.find("span", class_="title").text

<album>
  <cisco:song>Believer</cisco:song>
</album>

song = soup.find("cisco:song").text

<p>
  現在趨勢
  <table>...</table>
</p>

paras = soup.select("p > text()")

import time
import random

def 自適應延遲(回應時間):
    基本延遲 = 2  # 最小等待秒數
    # 根據網站回應時間動態調整延遲
    動態延遲 = 回應時間 * 1.5
    # 增加隨機因素
    隨機延遲 = random.uniform(0.5, 1.5)
    
    總延遲 = (基本延遲 + 動態延遲) * 隨機延遲
    time.sleep(總延遲)

import pandas as pd
import sqlite3

class 資料處理器:
    def __init__(self, 資料函式庫='爬蟲資料.db'):
        self.連線 = sqlite3.connect(資料函式庫)
        self.緩衝資料 = []
        self.緩衝大小 = 100
        
    def 處理專案(self, 專案):
        # 進行必要的資料清理和轉換
        清理後專案 = self._清理資料(專案)
        self.緩衝資料.append(清理後專案)
        
        # 達到緩衝大小時批次儲存
        if len(self.緩衝資料) >= self.緩衝大小:
            self._儲存資料()
            
    def _清理資料(self, 專案):
        # 實作資料清理邏輯
        return 專案
        
    def _儲存資料(self):
        df = pd.DataFrame(self.緩衝資料)
        df.to_sql('爬蟲結果', self.連線, if_exists='append', index=False)
        self.緩衝資料 = []
        
    def 關閉(self):
        if self.緩衝資料:  # 確保剩餘資料也被儲存
            self._儲存資料()
        self.連線.close()

import json
import requests

def 分析API呼叫(網址, 引數):
    標頭 = 建立請求標頭()  # 使用前面定義的函式
    
    回應 = requests.get(
        網址,
        params=引數,
        headers=標頭
    )
    
    if 回應.status_code == 200:
        try:
            return json.loads(回應.text)
        except json.JSONDecodeError:
            print("無法解析 JSON 回應")
            return None
    else:
        print(f"API 呼叫失敗: {回應.status_code}")
        return None