命名實體識別技術:深度解析 NLP 核心應用與實戰

import pandas as pd
import re

def load_and_preview_dataset(file_path, num_samples=10):
    """
    載入資料集並預覽前幾筆資料
    協助理解資料結構與內容特性
    """
    try:
        data = pd.read_csv(file_path, encoding='utf-8')
        print(f"資料集總筆數:{len(data)}")
        print(f"\n資料欄位:{data.columns.tolist()}\n")
        
        for i in range(min(num_samples, len(data))):
            print(f"第 {i+1} 筆文章標題:{data.loc[i, 'title']}\n")
            print(f"第 {i+1} 筆文章內容:{data.loc[i, 'description']}\n")
            print("-" * 80)
        
        return data
    except Exception as e:
        print(f"資料載入發生錯誤:{str(e)}")
        return None

data = load_and_preview_dataset('train_data.csv')

def clean_text_data(data, columns):
    """
    執行完整的文字資料清理流程
    處理特殊字元、編碼問題與格式統一
    """
    data_cleaned = data.copy()
    
    for col in columns:
        data_cleaned[col] = data_cleaned[col].str.replace("\\", " ", regex=False)
        data_cleaned[col] = data_cleaned[col].str.replace("#36;", "$", regex=False)
        data_cleaned[col] = data_cleaned[col].str.replace("\xa0", " ", regex=False)
        data_cleaned[col] = data_cleaned[col].str.replace("\u3000", " ", regex=False)
        data_cleaned[col] = data_cleaned[col].str.strip()
        data_cleaned[col] = data_cleaned[col].apply(
            lambda x: re.sub(r'\s+', ' ', x) if isinstance(x, str) else x
        )
    
    return data_cleaned

columns_to_clean = ["title", "description"]
data_cleaned = clean_text_data(data, columns_to_clean)
output_path = 'prepared_data.csv'
data_cleaned.to_csv(output_path, index=False, encoding='utf-8')
print(f"清理完成,資料已儲存至:{output_path}")

from transformers import AutoModelForTokenClassification, AutoTokenizer, pipeline
import torch

class NERSystem:
    """
    基於 Transformer 的命名實體識別系統
    支援多種預訓練模型與繁體中文處理
    """
    
    def __init__(self, model_name="dslim/bert-base-NER"):
        """
        初始化 NER 系統
        載入預訓練模型與對應的分詞器
        """
        try:
            self.model_name = model_name
            self.tokenizer = AutoTokenizer.from_pretrained(model_name)
            self.model = AutoModelForTokenClassification.from_pretrained(model_name)
            self.ner_pipeline = pipeline(
                "ner",
                model=self.model,
                tokenizer=self.tokenizer,
                aggregation_strategy="simple"
            )
            print(f"成功載入模型:{model_name}")
        except Exception as e:
            print(f"模型載入失敗:{str(e)}")
            raise
    
    def extract_entities(self, text):
        """
        從輸入文字中提取命名實體
        返回實體及其類型資訊
        """
        try:
            results = self.ner_pipeline(text)
            entities = []
            for entity in results:
                entities.append({
                    'text': entity['word'],
                    'type': entity['entity_group'],
                    'score': round(entity['score'], 4)
                })
            return entities
        except Exception as e:
            print(f"實體提取發生錯誤:{str(e)}")
            return []
    
    def batch_extract(self, texts):
        """
        批次處理多筆文字的實體識別
        提升處理效率
        """
        all_results = []
        for text in texts:
            entities = self.extract_entities(text)
            all_results.append({'text': text, 'entities': entities})
        return all_results

ner_system = NERSystem(model_name="dslim/bert-base-NER")

sample_text = "Hugging Face is headquartered in New York City, and the company was founded by Clément Delangue."
entities = ner_system.extract_entities(sample_text)

print(f"\n輸入文字:{sample_text}\n")
print("識別出的實體:")
for entity in entities:
    print(f"  - 實體內容:{entity['text']}")
    print(f"    實體類型:{entity['type']}")
    print(f"    信心分數:{entity['score']}\n")

import spacy
from spacy import displacy

class ChineseNERSystem:
    """
    繁體中文命名實體識別系統
    基於 SpaCy 的 transformer 模型
    """
    
    def __init__(self, model_name="zh_core_web_trf"):
        """
        初始化繁體中文 NER 系統
        載入 SpaCy 的繁體中文模型
        """
        try:
            self.nlp = spacy.load(model_name)
            print(f"成功載入 SpaCy 模型:{model_name}")
        except OSError:
            print(f"模型 {model_name} 未安裝,請執行:")
            print(f"python -m spacy download {model_name}")
            raise
    
    def analyze_text(self, text):
        """
        分析繁體中文文字並提取實體
        返回完整的分析結果
        """
        doc = self.nlp(text)
        entities = []
        for ent in doc.ents:
            entities.append({
                'text': ent.text,
                'label': ent.label_,
                'start': ent.start_char,
                'end': ent.end_char
            })
        
        tokens_info = []
        for token in doc:
            tokens_info.append({
                'text': token.text,
                'pos': token.pos_,
                'dep': token.dep_
            })
        
        return {'entities': entities, 'tokens': tokens_info, 'doc': doc}
    
    def batch_process(self, texts, batch_size=32):
        """
        批次處理繁體中文文字
        提升大量資料的處理效率
        """
        results = []
        docs = list(self.nlp.pipe(texts, batch_size=batch_size))
        
        for doc in docs:
            entities = [(ent.text, ent.label_) for ent in doc.ents]
            results.append({'text': doc.text, 'entities': entities})
        return results

zh_ner = ChineseNERSystem(model_name="zh_core_web_trf")

sample_texts = [
    "台積電是台灣最大的半導體製造公司,總部位於新竹科學園區。",
    "聯發科技在智慧型手機晶片市場佔有重要地位。",
    "鴻海精密工業是全球最大的電子製造服務商之一。"
]

result = zh_ner.analyze_text(sample_texts[0])
print(f"\n文字分析結果:{sample_texts[0]}\n")
print("識別的實體:")
for entity in result['entities']:
    print(f"  - 實體:{entity['text']}")
    print(f"    類型:{entity['label']}")
    print(f"    位置:{entity['start']}-{entity['end']}\n")