人工智慧高階向量表示方法

from sentence_transformers import SentenceTransformer

# 載入預訓練模型
model = SentenceTransformer('paraphrase-MiniLM-L6-v2')

# 生成句子嵌入
embeddings = model.encode(documents)

# 計算餘弦相似度
from sklearn.metrics.pairwise import cosine_similarity
similarities = cosine_similarity(embeddings)

def diverse_retrieval(query_vector, memory_vectors, top_k=5, diversity_factor=0.3):
    """檢索相關與多樣的記憶"""
    similarities = cosine_similarity([query_vector], memory_vectors)[0]
    selected_indices = []
    
    # 選擇最相似的項
    best_index = np.argmax(similarities)
    selected_indices.append(best_index)
    
    # 迭代選擇其餘項
    while len(selected_indices) < top_k:
        # 計算候選項與已選項的最大相似度
        max_similarities = []
        for i in range(len(similarities)):
            if i in selected_indices:
                max_similarities.append(float('inf'))  # 已選項不再考慮
                continue
            
            # 與已選項的最大相似度
            sim_to_selected = max([cosine_similarity([memory_vectors[i]], [memory_vectors[j]])[0][0] 
                                  for j in selected_indices])
            max_similarities.append(sim_to_selected)
        
        # 結合相關性與多樣性
        scores = similarities - diversity_factor * np.array(max_similarities)
        next_index = np.argmax(scores)
        selected_indices.append(next_index)
    
    return selected_indices

import faiss

def build_faiss_index(vectors, dimension):
    """建立FAISS索參照於高效相似度搜尋"""
    index = faiss.IndexFlatL2(dimension)  # L2距離索引
    vectors = np.array(vectors).astype('float32')
    index.add(vectors)
    return index

def search_memories(query_vector, index, k=5):
    """使用FAISS索引搜尋最相似的記憶"""
    query_vector = np.array([query_vector]).astype('float32')
    distances, indices = index.search(query_vector, k)
    return indices[0]

def check_memory_consistency(new_memory, existing_memories, threshold=0.7):
    """檢查新記憶與現有記憶的一致性"""
    contradictions = []
    
    # 將新記憶向量化
    new_memory_vector = vectorize_memory(new_memory)
    
    # 尋找相似但可能矛盾的記憶
    for memory in existing_memories:
        memory_vector = vectorize_memory(memory)
        similarity = cosine_similarity([new_memory_vector], [memory_vector])[0][0]
        
        # 如果相似度高，進一步檢查是否存在矛盾
        if similarity > threshold:
            if contains_contradiction(new_memory, memory):
                contradictions.append(memory)
    
    if contradictions:
        resolved_memory = resolve_contradictions(new_memory, contradictions)
        return resolved_memory
    else:
        return new_memory

def create_dynamic_prompt(user_query, retrieved_memories, system_prompt):
    """根據檢索到的記憶建立動態提示"""
    memories_text = "\n".join([f"- {memory}" for memory in retrieved_memories])
    
    prompt = f"""{system_prompt}
    
使用者的過往往相關記憶:
{memories_text}

請根據以上記憶和使用者的當前查詢提供回應。確保回應與使用者的歷史互動保持一致。

使用者查詢: {user_query}
"""
    return prompt

def evaluate_memory_importance(query, retrieved_memories, llm_client):
    """使用LLM評估記憶的重要性"""
    prompt = f"""請評估以下記憶對回答使用者查詢的重要性，為每項分配1-10的分數。
    
使用者查詢: {query}

記憶:
"""
    
    for i, memory in enumerate(retrieved_memories):
        prompt += f"{i+1}. {memory}\n"
    
    prompt += "\n請為每項記憶評分(1-10)，並簡要說明原因。格式為 '專案編號: 分數 - 原因'"
    
    response = llm_client.generate(prompt)
    
    # 解析回應提取分數
    importance_scores = parse_importance_scores(response)
    
    # 根據重要性排序記憶
    sorted_memories = [mem for _, mem in sorted(
        zip(importance_scores, retrieved_memories), 
        key=lambda pair: pair[0], 
        reverse=True
    )]
    
    return sorted_memories

# 使用TF-IDF向量化檔案並建立簡易向量資料函式庫
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

# 假設我們有一系列檔案
documents = [
    "The sky is blue and beautiful.",
    "Love this blue and beautiful sky!",
    "The quick brown fox jumps over the lazy dog.",
    # 更多檔案...
]

# 使用TF-IDF向量化檔案
vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(documents)
vector_database = X.toarray()

# 定義相似度搜尋函式
def cosine_similarity_search(query, database, vectorizer, top_n=5):
    query_vec = vectorizer.transform([query]).toarray()
    similarities = cosine_similarity(query_vec, database)[0]
    top_indices = np.argsort(-similarities)[:top_n]  # 取得前n個最相似的索引
    return [(idx, similarities[idx]) for idx in top_indices]

# 主要搜尋迴圈
while True:
    query = input("Enter a search query (or 'exit' to stop): ")
    if query.lower() == 'exit':
        break
    top_n = int(input("How many top matches do you want to see? "))
    search_results = cosine_similarity_search(query, vector_database, vectorizer, top_n)
    print("Top Matched Documents:")
    for idx, score in search_results:
        print(f"- {documents[idx]} (Score: {score:.4f})")
    print("\n")

Enter a search query (or 'exit' to stop): blue
How many top matches do you want to see? 3
Top Matched Documents:
- The sky is blue and beautiful. (Score: 0.4080)
- Love this blue and beautiful sky! (Score: 0.3439)
- The brown fox is quick and the blue dog is lazy! (Score: 0.2560)

import os
from openai import OpenAI
import numpy as np
from sklearn.decomposition import PCA
from dotenv import load_dotenv

# 載入環境變數
load_dotenv()
api_key = os.getenv('OPENAI_API_KEY')
if not api_key:
    raise ValueError("No API key found. Please check your .env file.")

client = OpenAI(api_key=api_key)

def get_embedding(text, model="text-embedding-ada-002"):
    text = text.replace("\n", " ")
    return client.embeddings.create(input=[text],
                                   model=model).data[0].embedding

# 假設檔案集合
documents = [
    "The sky is blue and beautiful.",
    "Love this blue and beautiful sky!",
    "The quick brown fox jumps over the lazy dog.",
    # 更多檔案...
]

# 為每個檔案生成嵌入向量
embeddings = [get_embedding(doc) for doc in documents]
embeddings_array = np.array(embeddings)

# 使用PCA降維以便視覺化
pca = PCA(n_components=3)
reduced_embeddings = pca.fit_transform(embeddings_array)

# 此時可以使用matplotlib等工具將reduced_embeddings視覺化

import chromadb
from openai import OpenAI
import os
from dotenv import load_dotenv

# 載入API金鑰
load_dotenv()
api_key = os.getenv('OPENAI_API_KEY')
if not api_key:
    raise ValueError("No API key found. Please check your .env file.")

client = OpenAI(api_key=api_key)

def get_embedding(text, model="text-embedding-ada-002"):
    text = text.replace("\n", " ")
    return client.embeddings.create(input=[text],
                                   model=model).data[0].embedding

# 範例檔案
documents = [
    "The sky is blue and beautiful.",
    "Love this blue and beautiful sky!",
    "The quick brown fox jumps over the lazy dog.",
    "The dog is lazy but the brown fox is quick!",
    "The brown fox is quick and the blue dog is lazy!",
    # 更多檔案...
]

# 生成嵌入向量並分配ID
embeddings = [get_embedding(doc) for doc in documents]
ids = [f"id{i}" for i in range(len(documents))]

# 建立ChromaDB客戶端和集合
chroma_client = chromadb.Client()
collection = chroma_client.create_collection(name="documents")

# 將檔案嵌入增加到集合中
collection.add(
    embeddings=embeddings,
    documents=documents,
    ids=ids
)

# 定義查詢函式
def query_chromadb(query, top_n=2):
    query_embedding = get_embedding(query)
    results = collection.query(
        query_embeddings=[query_embedding],
        n_results=top_n
    )
    return [(id, score, text) for id, score, text in
            zip(results['ids'][0],
                results['distances'][0],
                results['documents'][0])]

# 主要查詢迴圈
while True:
    query = input("Enter a search query (or 'exit' to stop): ")
    if query.lower() == 'exit':
        break
    top_n = int(input("How many top matches do you want to see? "))
    search_results = query_chromadb(query, top_n)
    print("Top Matched Documents:")
    for id, score, text in search_results:
        print(f"ID:{id} TEXT: {text} SCORE: {round(score, 2)}")
    print("\n")

餘弦距離(A,B) = 1 - 餘弦相似度(A,B)

Enter a search query (or 'exit' to stop): dogs are lazy
How many top matches do you want to see? 3
Top Matched Documents:
ID:id7 TEXT: The dog is lazy but the brown fox is quick! SCORE: 0.24
ID:id5 TEXT: The brown fox is quick and the blue dog is lazy! SCORE: 0.28
ID:id2 TEXT: The quick brown fox jumps over the lazy dog. SCORE: 0.29