微調OpenAI模型實作RAG資料縮減

# 從檔案中檢索API金鑰或手動輸入
from google.colab import drive
drive.mount('/content/drive')
f = open("drive/MyDrive/files/api_key.txt", "r")
API_KEY = f.readline()
f.close()

try:
    import openai
except:
    !pip install openai==1.42.0
import openai

import os
os.environ['OPENAI_API_KEY'] = API_KEY
openai.api_key = os.getenv("OPENAI_API_KEY")

!pip install jsonlines==4.0.0
!pip install datasets==2.20.0

# 下載SciQ資料集
from datasets import load_dataset
dataset = load_dataset("sciq")

# 匯入必要的函式庫
from datasets import load_dataset
import pandas as pd

# 從HuggingFace載入SciQ資料集
dataset_view = load_dataset("sciq", split="train")

# 過濾資料集，只保留包含支援文字的題目
filtered_dataset = dataset_view.filter(lambda x: x["support"] != "")

# 列印過濾後題目的數量
print("包含支援文字的題目數量：", len(filtered_dataset))

# 將過濾後的資料集轉換為pandas DataFrame
df_view = pd.DataFrame(filtered_dataset)

# 需要移除的欄位列表
columns_to_drop = ['distractor3', 'distractor1', 'distractor2']

# 從DataFrame中移除指定的欄位
df_view = df_view.drop(columns=columns_to_drop)

# 顯示DataFrame的前幾行
df_view.head()

# 準備JSONL檔案的資料專案
items = []
for idx, row in df.iterrows():
    detailed_answer = row['correct_answer'] + " 解釋： " + row['support']
    items.append({
        "messages": [
            {"role": "system", "content": "針對科學問題提供詳細答案"},
            {"role": "user", "content": row['question']},
            {"role": "assistant", "content": detailed_answer}
        ]
    })

# 將資料寫入JSONL檔案
import jsonlines
with jsonlines.open('/content/QA_prompts_and_completions.json', mode='w') as writer:
    writer.write_all(items)

from openai import OpenAI
import jsonlines

# 建立OpenAI客戶端
client = OpenAI()

# 上傳訓練檔案
result_file = client.files.create(
    file=open("QA_prompts_and_completions.json", "rb"),
    purpose="fine-tune"
)

# 列印檔案資訊
print(result_file)
param_training_file_name = result_file.id
print(param_training_file_name)

# 建立微調任務
ft_job = client.fine_tuning.jobs.create(
    training_file=param_training_file_name,
    model="gpt-4o-mini-2024-07-18"
)

# 列印微調任務資訊
print(ft_job)

# 取得最近的三個微調任務
response = client.fine_tuning.jobs.list(limit=3)

# 列印任務資訊
for job in response.data:
    print(job.id, job.status, job.created_at)

import pandas as pd
from openai import OpenAI

client = OpenAI()
response = client.fine_tuning.jobs.list(limit=3)

# 初始化列表以儲存提取的資料
job_ids = []
created_ats = []
statuses = []
models = []
training_files = []
error_messages = []
fine_tuned_models = []

# 遍歷任務以檢索所需資訊
for job in response.data:
    job_ids.append(job.id)
    created_ats.append(job.created_at)
    statuses.append(job.status)
    models.append(job.model)
    training_files.append(job.training_file)
    error_message = job.error.message if job.error else None
    error_messages.append(error_message)
    fine_tuned_model = job.fine_tuned_model if hasattr(job, 'fine_tuned_model') else None
    fine_tuned_models.append(fine_tuned_model)

# 建立DataFrame
df = pd.DataFrame({
    'Job ID': job_ids,
    'Created At': created_ats,
    'Status': statuses,
    'Model': models,
    'Training File': training_files,
    'Error Message': error_messages,
    'Fine-Tuned Model': fine_tuned_models
})

# 將時間戳轉換為可讀格式並排序
df['Created At'] = pd.to_datetime(df['Created At'], unit='s')
df = df.sort_values(by='Created At', ascending=False)

# 顯示DataFrame
print(df)

generation = False  # 初始狀態為False，表示未完成訓練

# 尋找第一個非空的Fine-Tuned Model
non_empty_models = df[df['Fine-Tuned Model'].notna()]

if not non_empty_models.empty:
    first_non_empty_model = non_empty_models.iloc[0]['Fine-Tuned Model']
    print("The latest fine-tuned model is:", first_non_empty_model)
    generation = True
else:
    print("No fine-tuned models found.")

if generation:
    prompt = "What phenomenon makes global winds blow northeast to southwest?"
    response = client.chat.completions.create(
        model=first_non_empty_model,
        temperature=0.0,
        messages=[
            {"role": "system", "content": "Given a question, respond with the appropriate answer."},
            {"role": "user", "content": prompt}
        ]
    )
    print(response.choices[0].message.content)
else:
    print("Error: Model is None, cannot proceed with the API request.")

import textwrap

if generation:
    response_text = response.choices[0].message.content
    wrapped_text = textwrap.fill(response_text.strip(), 60)
    print(wrapped_text)