生成式AI評估與根據索引RAG應用

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def calculate_cosine_similarity(text1, text2):
    vectorizer = TfidfVectorizer()
    tfidf = vectorizer.fit_transform([text1, text2])
    similarity = cosine_similarity(tfidf[0:1], tfidf[1:2])
    return similarity[0][0]

# 示例用法
user_prompt = "評估生成式AI輸出品質"
gpt4_response = "在開發和最佳化生成式AI模型的過程中，評估輸出品質是一項至關重要的任務。"
similarity_score = calculate_cosine_similarity(user_prompt, gpt4_response)
print(f"餘弦相似度得分：{similarity_score:.3f}")

augmented_input = "評估生成式AI輸出品質的方法包括餘弦相似度和語義相似性。"
similarity_score = calculate_cosine_similarity(augmented_input, gpt4_response)
print(f"餘弦相似度得分：{similarity_score:.3f}")

from sentence_transformers import SentenceTransformer
import torch

# 載入預訓練模型
model = SentenceTransformer('all-MiniLM-L6-v2')

def calculate_cosine_similarity_with_embeddings(text1, text2):
    embeddings1 = model.encode(text1)
    embeddings2 = model.encode(text2)
    similarity = cosine_similarity([embeddings1], [embeddings2])
    return similarity[0][0]

# 示例用法
similarity_score = calculate_cosine_similarity_with_embeddings(augmented_input, gpt4_response)
print(f"語義相似度得分：{similarity_score:.3f}")

import os
from llama_index import SimpleDirectoryReader, VectorStoreIndex
from llama_index.vector_stores import DeepLakeVectorStore
from llama_index.storage.storage_context import StorageContext

# 設定Deep Lake向量儲存
my_activeloop_org_id = "your_org_id"
my_activeloop_dataset_name = "your_dataset_name"
dataset_path = f"hub://{my_activeloop_org_id}/{my_activeloop_dataset_name}"

# 建立向量儲存索引
vector_store = DeepLakeVectorStore(dataset_path=dataset_path, overwrite=True)
storage_context = StorageContext.from_defaults(vector_store=vector_store)

# 載入檔案並建立索引
documents = SimpleDirectoryReader("./data").load_data()
index = VectorStoreIndex.from_documents(documents, storage_context=storage_context)

# 查詢引擎設定
query_engine = index.as_query_engine()
response = query_engine.query("你的查詢問題")

print(response)

!pip install llama-index-vector-stores-deeplake==0.1.6
!pip install deeplake==3.9.8
!pip install llama-index==0.10.64

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.vector_stores.deeplake import DeepLakeVectorStore

!pip install llama-index-vector-stores-deeplake==0.1.6
!pip install deeplake==3.9.8
!pip install llama-index==0.10.64

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.vector_stores.deeplake import DeepLakeVectorStore

!mkdir data

import requests
from bs4 import BeautifulSoup
import re
import os

# 定義要爬取的網址列表
urls = [
    "https://github.com/VisDrone/VisDrone-Dataset",
    "https://paperswithcode.com/dataset/visdrone",
    "https://openaccess.thecvf.com/content_ECCVW_2018/papers/111",
    "https://github.com/VisDrone/VisDrone2018-MOT-toolkit",
    "https://en.wikipedia.org/wiki/Object_detection",
    "https://en.wikipedia.org/wiki/Computer_vision",
    # ... 其他網址
]

# 定義清理文字的函式
def clean_text(content):
    # 移除參考文獻標籤和不想要的字元
    content = re.sub(r'\[\d+\]', '', content)  # 移除參考文獻標籤
    content = re.sub(r'[^\w\s\.]', '', content)  # 移除標點符號
    return content

# 定義抓取和清理網頁內容的函式
def fetch_and_clean(url):
    try:
        response = requests.get(url)
        response.raise_for_status()  # 若請求失敗則丟擲異常
        soup = BeautifulSoup(response.content, 'html.parser')
        
        # 優先查詢 "mw-parser-output" 類別的 div，若無則查詢 "content" 類別
        content = soup.find('div', {'class': 'mw-parser-output'})
        if content is None:
            content = soup.find('div', {'class': 'content'})
        
        # 移除特定的區塊，如 "References"、"Bibliography" 等
        for section_title in ['References', 'Bibliography', 'External links']:
            section = content.find('span', id=section_title)
            while section:
                for sib in section.parent.find_next_siblings():
                    sib.decompose()
                section.parent.decompose()
                section = content.find('span', id=section_title)
        
        # 提取並清理文字
        text = content.get_text(separator=' ', strip=True)
        text = clean_text(text)
        return text
    except requests.exceptions.RequestException as e:
        print(f"從 {url} 抓取內容時發生錯誤：{e}")
        return None  # 發生錯誤時傳回 None

# 處理每個網址並將內容寫入單獨的檔案
output_dir = './data/'
os.makedirs(output_dir, exist_ok=True)
for url in urls:
    article_name = url.split('/')[-1].replace('.html', '')  # 處理網址以生成檔名
    filename = os.path.join(output_dir, article_name + '.txt')
    clean_article_text = fetch_and_clean(url)
    if clean_article_text is not None:
        with open(filename, 'w', encoding='utf-8') as file:
            file.write(clean_article_text)
print("已將內容寫入 data 目錄下的檔案。")

documents = SimpleDirectoryReader("./data/").load_data()

print(documents[0])