Python Requests：BeautifulSoup 進行網頁爬蟲

<div class="grid_9 alpha">
    <h3>Group 1</h3>
    <a name="1"></a>
    <table class="wordlist">
        <tbody>
            <tr>
                <th>Abhor</th>
                <td>hate</td>
            </tr>
            <tr>
                <th>Bigot</th>
                <td>narrow-minded, prejudiced person</td>
            </tr>
            ...
        </tbody>
    </table>
</div>

def make_soup(html_string):
    return BeautifulSoup(html_string)

def get_words_from_soup(soup):
    words = {}
    for count, wordlist_table in enumerate(
            soup.find_all(class_='wordlist')):
        title = "Group %d" % (count + 1)
        new_words = {}
        for word_entry in wordlist_table.find_all('tr'):
            new_words[word_entry.th.text] = word_entry.td.text
        words[title] = new_words
    return words

def save_as_json(data, output_file):
    """ 將給定的資料寫入指定的輸出檔案 """
    with open(output_file, 'w') as outfile:
        json.dump(data, outfile)

import json
import time
import requests
from bs4 import BeautifulSoup

START_PAGE, END_PAGE, OUTPUT_FILE = 1, 10, 'words.json'

# 識別URL模式
URL = "http://www.majortests.com/gre/wordlist_0%d"

def generate_urls(url, start_page, end_page):
    """
    生成URL列表
    
    引數:
      url: URL範本
      start_page: 起始頁碼
      end_page: 結束頁碼
      
    回傳:
      生成的URL列表
    """
    urls = []
    for page in range(start_page, end_page):
        urls.append(url % page)
    return urls

def get_resource(url):
    """
    取得網頁資源
    
    引數:
      url: 目標URL
      
    回傳:
      requests.Response物件
    """
    return requests.get(url)

def make_soup(html_string):
    """
    將HTML字串轉換為BeautifulSoup物件
    
    引數:
      html_string: HTML內容字串
      
    回傳:
      BeautifulSoup物件
    """
    return BeautifulSoup(html_string)

def get_words_from_soup(soup):
    """
    從BeautifulSoup物件中擷取單字組
    
    引數:
      soup: BeautifulSoup物件
      
    回傳:
      擷取的單字組字典
    """
    words = {}
    count = 0
    for wordlist_table in soup.find_all(class_='wordlist'):
        count += 1
        title = "Group %d" % count
        new_words = {}
        for word_entry in wordlist_table.find_all('tr'):
            new_words[word_entry.th.text] = word_entry.td.text
        words[title] = new_words
        print " - - Extracted words from %s" % title
    return words

def save_as_json(data, output_file):
    """ 
    將資料寫入指定的輸出檔案
    """
    json.dump(data, open(output_file, 'w'))

def scrapper_bot(urls):
    """
    爬蟲機器人:
    
    引數:
      urls: URL列表
      
    回傳:
      包含不同單字組的字典
    """
    gre_words = {}
    for url in urls:
        print "Scrapping %s" % url.split('/')[-1]
        # 步驟1：取得URL
        
        # 步驟2：使用requests取得HTML
        html = requests.get(url)
        
        # 步驟3：使用瀏覽器工具識別所需資料
        
        # 步驟4：建立BeautifulSoup物件
        soup = make_soup(html.text)
        
        # 步驟5：從soup中擷取單字
        words = get_words_from_soup(soup)
        gre_words[url.split('/')[-1]] = words
        
        print "sleeping for 5 seconds now"
        time.sleep(5)  # 避免過度頻繁請求
    return gre_words

if __name__ == '__main__':
    urls = generate_urls(URL, START_PAGE, END_PAGE+1)
    gre_words = scrapper_bot(urls)
    save_as_json(gre_words, OUTPUT_FILE)

{
  "wordlist_04": {
    "Group 10": {
      "Devoured": "greedily eaten/consumed",
      "Magnate": "powerful businessman",
      "Cavalcade": "procession of vehicles",
      "Extradite": "deport from one country back to the home..."
      // 更多單字...
    }
    // 更多單字組...
  }
  // 更多單字列表...
}

from flask import Flask
app = Flask(__name__)

@app.route("/")
def home():
    return "Hello Guest!"

if __name__ == "__main__":
    app.run()

survey_project/
├── __init__.py      # 初始化專案並新增到PYTHONPATH
├── server.py        # 啟動應用開發伺服器
└── survey/
    ├── __init__.py  # 初始化應用並整合各元件
    ├── app.db       # SQLite3資料函式庫
    ├── models.py    # 定義應用模型
    ├── templates/   # Jinja2範本目錄
    ├── tests.py     # 應用測試案例
    └── views.py     # 定義應用路由

import os
import sys
current_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
parent_dir = os.path.abspath(os.path.join(current_dir, os.pardir))
sys.path.insert(0, parent_dir)

import os
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy

BASE_DIR = os.path.abspath(os.path.dirname(__file__))
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = \
    'sqlite:///' + os.path.join(BASE_DIR, 'app.db')
db = SQLAlchemy(app)

class Question(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    question_text = db.Column(db.String(200))
    number_of_yes_votes = db.Column(db.Integer, default=0)
    number_of_no_votes = db.Column(db.Integer, default=0)
    number_of_maybe_votes = db.Column(db.Integer, default=0)
    
    def __init__(self,
                question_text,
                number_of_yes_votes=0,
                number_of_no_votes=0,
                number_of_maybe_votes=0):
        self.question_text = question_text
        self.number_of_yes_votes = number_of_yes_votes
        self.number_of_maybe_votes = number_of_maybe_votes
        self.number_of_no_votes = number_of_no_votes
        
    def vote(self, vote_type):
        if vote_type == 'yes':
            self.number_of_yes_votes += 1
        elif vote_type == 'no':
            self.number_of_no_votes += 1
        elif vote_type == 'maybe':
            self.number_of_maybe_votes += 1
        else:
            raise Exception("Invalid vote type")

# 在 runserver.py 中
from survey import db
db.create_all()

question = Question("Are you an American?")
db.session.add(question)
db.session.commit()

# 取得所有問題
Question.query.all()

# 根據 ID 取得特定問題
Question.query.get(1)

question = Question.query.get(1)
question.vote('yes')
db.session.add(question)
db.session.commit()

question = Question.query.get(1)
db.session.delete(question)
db.session.commit()

from flask import render_template
from survey import app
from survey.models import Question

@app.route('/', methods=['GET'])
def home():
    questions = Question.query.all()
    context = {'questions': questions,
              'number_of_questions': len(questions)}
    return render_template('index.html',
                         **context)

@app.route('/questions/new', methods=['GET'])
def new_questions():
    return render_template('new.html')

@app.route('/questions', methods=['POST'])
def create_questions():
    if request.form["question_text"].strip() != "":
        new_question = Question(question_text=request.form["question_text"])
        db.session.add(new_question)
        db.session.commit()
        message = "Succefully added a new poll!"
    else:
        message = "Poll question should not be an empty string!"
    context = {'questions': Question.query.all(),
              'message': message}
    return render_template('index.html',
                         **context)

@app.route('/questions/<int:question_id>', methods=['GET'])
def show_questions(question_id):
    context = {'question': Question.query.get(question_id)}
    return render_template('show.html',
                         **context)

@app.route('/questions/<int:question_id>', methods=['PUT'])
def update_questions(question_id):
    question = Question.query.get(question_id)
    if request.form["question_text"].strip() != "":
        question.question_text = request.form["question_text"]
        db.session.add(question)
        db.session.commit()
        message = "Successfully updated a poll!"
    else:
        message = "Question cannot be empty!"
    context = {'question': question,
              'message': message}
    return render_template('show.html',
                         **context)

@app.route('/questions/<int:question_id>', methods=['DELETE'])
def delete_questions(question_id):
    question = Question.query.get(question_id)
    db.session.delete(question)
    db.session.commit()
    context = {'questions': Question.query.all(),
              'message': 'Successfully deleted'}
    return render_template('index.html',
                         **context)

@app.route('/questions/<int:question_id>/vote', methods=['GET'])
def new_vote_questions(question_id):
    question = Question.query.get(question_id)
    context = {'question': question}
    return render_template('vote.html',
                         **context)

@app.route('/questions/<int:question_id>/vote', methods=['POST'])
def vote_questions(question_id):
    question = Question.query.get(question_id)
    vote_type = request.form["vote_type"]
    question.vote(vote_type)
    db.session.add(question)
    db.session.commit()
    context = {'question': question}
    return render_template('show.html', **context)