Python 解析 JSON 並應用 Snorkel 與機器學習技術

import json

# 假設response是包含JSON物件的變數
json_data = response['choices']

# 初始化變數來儲存提取的資訊
name = None
organization = None
location = None
contact_number = None

# 迭代JSON物件中的每個條目
for entry in json_data:
    # 提取text欄位的值
    text = entry.get("text", "")
    
    # 使用字串分割和查詢來提取所需資訊
    lines = text.splitlines()
    for line in lines:
        if line.startswith("Name: "):
            name = line[6:].strip()
        elif line.startswith("Organization: "):
            organization = line[14:].strip()
        elif line.startswith("Geographical location: "):
            location = line[22:].strip()
        elif line.startswith("Contact number: "):
            contact_number = line[16:].strip()

# 驗證是否成功提取了所有所需資訊
if name and organization and location and contact_number:
    print("提取的資訊：")
    print(f"姓名：{name}")
    print(f"組織：{organization}")
    print(f"地理位置：{location}")
    print(f"聯絡號碼：{contact_number}")
else:
    print("未能提取所有所需資訊。")

import json

def extract_information(response):
    json_data = response['choices']
    name = None
    organization = None
    location = None
    contact_number = None

    for entry in json_data:
        text = entry.get("text", "")
        lines = text.splitlines()
        for line in lines:
            if line.startswith("Name: "):
                name = line[6:].strip()
            elif line.startswith("Organization: "):
                organization = line[14:].strip()
            elif line.startswith("Geographical location: "):
                location = line[22:].strip()
            elif line.startswith("Contact number: "):
                contact_number = line[16:].strip()

    if name and organization and location and contact_number:
        print("提取的資訊：")
        print(f"姓名：{name}")
        print(f"組織：{organization}")
        print(f"地理位置：{location}")
        print(f"聯絡號碼：{contact_number}")
    else:
        print("未能提取所有所需資訊。")

# 假設response是包含JSON物件的變數
response = {
    'choices': [
        {
            "text": "Thank you for your time, and I look forward to hearing from you soon. \n\nName: Sarah Johnson\nOrganization: XYZ Tech Solutions\nGeographical location: Austin, Texas\nContact number: (555) 123-4567",
            "index": 0,
            "finish_reason": "stop",
            "logprobs": None,
            "content_filter_results": {
                "hate": {
                    "filtered": False,
                    "severity": "safe"
                },
                "self_harm": {
                    "filtered": False,
                    "severity": "safe"
                },
                "sexual": {
                    "filtered": False,
                    "severity": "safe"
                },
                "violence": {
                    "filtered": False,
                    "severity": "safe"
                }
            }
        }
    ]
}

extract_information(response)

text = "Name: Sarah Johnson\nOrganization: XYZ Tech Solutions\nGeographical location: Austin, Texas\nContact number: (555) 123-4567"

name = text.split("Name:")[1].split("\n")[0].strip()
organization = text.split("Organization:")[1].split("\n")[0].strip()
location = text.split("Geographical location:")[1].split("\n")[0].strip()
contact_number = text.split("Contact number:")[1].split("\n")[0].strip()

print("Name:", name)
print("Organization:", organization)
print("Location:", location)
print("Contact Number:", contact_number)

engine = "gpt3.5"
prompt = "Conduct aspect-based sentiment analysis on the following product reviews:\n Provide an overall sentiment score between 0 and 5 for each review.\n Assign a sentiment polarity score between 0 and 5 for each aspect mentioned. \n Identify the top positive and negative aspects, if any. \n Review 1: \n I recently purchased this smartphone, and it has exceeded my expectations! The camera quality is superb, capturing vivid and detailed photos. The battery life is impressive, easily lasting a full day with regular use. The sleek design adds a premium feel to the device. However, the speaker quality could be improved. Overall sentiment score: 4.8 \nAspects with sentiment polarity score: \n - Camera: 5 \n - Battery Life: 5 \n - Design: 5 \n - Speaker: 3 \n \n Top positive aspect: Camera \n Top negative aspect: Speaker \n \n Review 2: \n This laptop offers powerful performance and a sleek design. The keyboard is comfortable for extended typing sessions, and the display is vibrant with accurate colors. However, the trackpad responsiveness can be inconsistent at times."

response = engine(prompt)
print(response.choices[0].text.strip())

import pandas as pd

# 安裝 Snorkel
!pip install snorkel

# 匯入必要的庫
from snorkel.labeling import labeling_function

# 定義標記函式
@labeling_function()
def label_positive(x):
    if "good" in x:
        return 1
    else:
        return 0

@labeling_function()
def label_negative(x):
    if "bad" in x:
        return -1
    else:
        return 0

# 建立資料框
df = pd.DataFrame({"text": ["This is a good product.", "This is a bad product."]})

# 應用標記函式
df["label"] = df["text"].apply(label_positive)
df["label"] = df["text"].apply(label_negative)

# 顯示結果
print(df)

import pandas as pd
import numpy as np

# 定義常數
ABSTAIN = -1  # 棄權
POS = 0  # 正面
NEG = 1  # 負面

# 建立一個包含更多資料的DataFrame
df = pd.DataFrame({
    'id': [1, 2, 3, 4, 5, 6, 7, 8],
    'review': [
        "這部電影非常精彩!",
        "這部電影很糟糕，很無聊。",
        "我對這部電影有混合的感覺。",
        "我對這部電影沒有意見。",
        "這部電影非常棒，很刺激!",
        "我不喜歡這部電影，它太慢了。",
        "這部電影還可以，不太好也不太壞。",
        "這部電影很混亂，很無趣。"
    ]
})

# 將DataFrame分割成訓練集和測試集
df_train = df.iloc[:6]  # 前6筆記錄用於訓練
df_test = df.iloc[6:]  # 其餘記錄用於測試

# 定義測試集的真實標籤
Y_test = [ABSTAIN, NEG]  # 將其替換為實際標籤

# 將Y_test轉換為NumPy陣列
Y_test = np.array(Y_test)

# 定義根據規則的標籤函式，使用正規表示式
def lf_positive_review(x):
    """判斷評論是否為正面"""
    return POS if '精彩' in x['review'] or '棒' in x['review'] else ABSTAIN

def lf_negative_review(x):
    """判斷評論是否為負面"""
    return NEG if '糟糕' in x['review'] or '無聊' in x['review'] or '慢' in x['review'] or '混亂' in x['review'] else ABSTAIN

def lf_neutral_review(x):
    """判斷評論是否為中立"""
    return POS if '還可以' in x['review'] else ABSTAIN

def lf_neutral_review(x):
    return 0 if 'mixed feelings' in x.review or \
                  'no opinion' in x.review or 'okay' in x.review \
           else 0

lfs = [lf_positive_review, lf_negative_review, 
       lf_neutral_review]

applier = PandasLFApplier(lfs=lfs)

L_train = applier.apply(df=df_train)

L_test = applier.apply(df=df_test)

from snorkel.labeling.model import MajorityLabelVoter, LabelModel

majority_model = MajorityLabelVoter()

# 執行多數投票模型
majority_model.predict(L=L_train)
majority_acc = majority_model.score(L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"]
print(majority_acc)

# 預測訓練集的標籤
preds_train = majority_model.predict(L=L_train)
print(preds_train)

# 建立標籤模型
label_model = LabelModel(cardinality=2, verbose=True)

# 執行標籤模型
label_model.fit(L_train=L_train, n_epochs=500, log_freq=100, seed=123)
label_model_acc = label_model.score(L=L_test, Y=Y_test, tie_break_policy="random")["accuracy"]
print(label_model_acc)

# 預測訓練集的標籤
Y_train_pred = label_model.predict(L=L_train)
print(Y_train_pred)

# 分析標籤函式
lf_analysis = LFAnalysis(L=L, lfs=lfs)
lf_summary = lf_analysis.lf_summary()
print(lf_summary)

from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
import nltk
from nltk.sentiment import SentimentAnalyzer
from nltk.classify import NaiveBayesClassifier

# 下載必要的NLTK資料
nltk.download('movie_reviews')
nltk.download('wordnet')
nltk.download('omw-1.4')
nltk.download('punkt')

# 初始化情感分析器和獲取電影評論ID
sentiment_analyzer = SentimentAnalyzer()
ids = movie_reviews.fileids()

# 預處理設定
lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words('english'))

def preprocess(document):
    words = word_tokenize(document)
    words = [word for word in words if word not in stop_words]
    words = [lemmatizer.lemmatize(word) for word in words]
    return ' '.join(words)

# 載入文字資料
train_data = [preprocess(movie_reviews.raw(fileid)) for fileid in ids]

# 標記文字資料
labels = [1 if fileid.startswith('pos') else 0 for fileid in ids]

# 切分訓練和測試資料
X_train, X_test, y_train, y_test = train_test_split(train_data, labels, test_size=0.2, random_state=42)

# 訓練邏輯回歸模型
vectorizer = TfidfVectorizer()
X_train_vectorized = vectorizer.fit_transform(X_train)
model = LogisticRegression()
model.fit(X_train_vectorized, y_train)

# 評估模型
X_test_vectorized = vectorizer.transform(X_test)
y_pred = model.predict(X_test_vectorized)
print('準確率：', accuracy_score(y_test, y_pred))