知識圖譜索引構建與查詢引擎實作

import time

# 啟動計時器
start_time = time.time()

import llama_index

# 建立知識圖索引
index = llama_index.create_index(documents)

import networkx as nx

# 顯示圖形
G = nx.Graph()
G.add_nodes_from(index.nodes)
G.add_edges_from(index.edges)
nx.draw(G, with_labels=True)

prompt = "What is the meaning of life?"

hyperparams = {"top_k": 5, "threshold": 0.5}

import similarity

def similarity_score(query, document):
    # 計算相似性評分
    return similarity.cosine_similarity(query, document)

# 計算查詢結果的相似性評分
scores = [similarity_score(prompt, doc) for doc in documents]

# 執行樣本相似性比較
sample_query = "What is the meaning of life?"
sample_documents = [doc1, doc2, doc3]
scores = [similarity_score(sample_query, doc) for doc in sample_documents]

# 重新排名輸出向量
ranked_scores = sorted(scores, reverse=True)

# 執行評估樣本和應用metrics和人工反饋評分
evaluation_samples = [sample_query, sample_documents]
metrics = [precision, recall, f1_score]
human_feedback_scores = [0.8, 0.9, 0.7]

# 計算metrics和人工反饋評分
metrics_scores = [metric(evaluation_samples) for metric in metrics]
human_feedback_scores = [score(evaluation_samples) for score in human_feedback_scores]

# 執行metric計算和顯示metrics
metric_calculations = [metric(evaluation_samples) for metric in metrics]
print(metric_calculations)

import time
from knowledge_graph_index import KnowledgeGraphIndex

# 計時開始
start_time = time.time()

# 建立知識圖索引
documents = [...]  # 檔案集合
max_triplets_per_chunk = 2  # 每個 chunk 的三元組數量上限
include_embeddings = True  # 是否包含嵌入式向量

graph_index = KnowledgeGraphIndex.from_documents(
    documents,
    max_triplets_per_chunk=max_triplets_per_chunk,
    include_embeddings=include_embeddings,
)

# 計時結束
end_time = time.time()

# 計算並列印索引建立時間
elapsed_time = end_time - start_time
print(f"索引建立時間：{elapsed_time:.4f} 秒")

print(type(graph_index))

# 相似度前 k 個結果
k = 3

# 回應溫度
temp = 0.1

# 查詢引擎組態
query_engine =...  # 查詢引擎例項化
query_engine.configure(
    similarity_top_k=k,
    temperature=temp,
    output_length=...  # 輸出長度組態
)

graph_query_engine = graph_index.as_query_engine(similarity_top_k=k, temperature=temp, max_tokens=mt)

import networkx as nx
from pyvis.network import Network

# 取得 NetworkX 格式的圖
g = graph_index.get_networkx_graph()

# 建立 PyVis 網路
net = Network(notebook=True, cdn_resources="in_line", directed=True)
net.from_nx(g)

# 設定節點和邊的屬性
for node in net.nodes:
    node['color'] = 'lightgray'
    node['size'] = 10
for edge in net.edges:
    edge['color'] = 'black'
    edge['width'] = 1

# 儲存圖為 HTML 檔案
fgraph = "Knowledge_graph_" + graph_name + ".html"
net.write_html(fgraph)

print(fgraph)

from IPython.display import HTML

# 載入 HTML 內容並顯示
HTML(filename=fgraph)

def execute_query(user_query):
    start_time = time.time()
    response = graph_query_engine.query(user_query)
    end_time = time.time()
    execution_time = end_time - start_time
    print(f"查詢執行時間：{execution_time} 秒")
    return response

user_query = "消費者市場的行銷主要目標是什麼？"
response = execute_query(user_query)

!pip install sentence-transformers==3.0.1

from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer

model = SentenceTransformer('all-MiniLM-L6-v2')

def calculate_cosine_similarity_with_embeddings(text1, text2):
    embeddings1 = model.encode(text1)
    embeddings2 = model.encode(text2)
    similarity = cosine_similarity([embeddings1], [embeddings2])
    return similarity

text1 = "消費者市場的行銷主要目標是什麼？"
text2 = "消費者市場的行銷目標是增加銷售額。"
similarity = calculate_cosine_similarity_with_embeddings(text1, text2)
print(f"相似度：{similarity}")

def calculate_cosine_similarity_with_embeddings(text1, text2):
    embeddings1 = model.encode(text1)
    embeddings2 = model.encode(text2)
    similarity = cosine_similarity([embeddings1], [embeddings2])
    return similarity[0][0]

user_query = "哪些專家經常與市場理論相關聯？"
start_time = time.time()
response = execute_query(user_query)
end_time = time.time()
print(f"查詢執行時間：{end_time - start_time} 秒")

for idx, node_with_score in enumerate(response.source_nodes):
    similarity_score = calculate_cosine_similarity_with_embeddings(user_query, node_with_score.text)
    print(f"相似度：{similarity_score}")

best_score = max(similarity_score)
best_text = response.source_nodes[similarity_score.index(best_score)].text
print(textwrap.fill(str(best_text), 100))

心理學家、文化人類學家和市場研究人員等理論。

最佳排名：2
最佳分數：0.5217772722244263

[...1380 年，德國紡織製造商約翰·富格爾（Johann Fugger）從奧格斯堡（Augsburg）旅行到 Graben，以收集資訊...
倫敦商人發表了有關貿易和經濟資源的資訊...

rscores = []
scores = []

# 執行十個示例
for i in range(10):
    user_query = "..."
    elapsed_time = time.time()
    response = execute_query(user_query)
    similarity_score = calculate_cosine_similarity_with_embeddings(user_query, response.text)
    scores.append(similarity_score)
    rscores.append(get_human_feedback_score(response.text))

import numpy as np

def calculate_cosine_similarity(text1, text2):
    # 文字預處理
    text1 = text1.lower()
    text2 = text2.lower()
    
    # 分割文字為單詞
    words1 = text1.split()
    words2 = text2.split()
    
    # 計算餘弦相似度
    vector1 = np.array([words1.count(word) for word in set(words1 + words2)])
    vector2 = np.array([words2.count(word) for word in set(words1 + words2)])
    cosine_similarity = np.dot(vector1, vector2) / (np.linalg.norm(vector1) * np.linalg.norm(vector2))
    
    return cosine_similarity

# 示例
text1 = "Which experts are often associated with marketing theory?"
text2 = "Psychologists, cultural anthropologists, and other experts in behavioral sciences are often associated with marketing theory."
cosine_similarity = calculate_cosine_similarity(text1, text2)
print("Cosine Similarity:", cosine_similarity)

# 評分
human_feedback = 0.75
print("Human Feedback:", human_feedback)

import numpy as np

# 示例資料
scores = [0.808918, 0.720165, 0.7599532, 0.8513956, 0.5457667, 0.696391]
rscores = [0.75, 0.5, 0.8, 0.9, 0.65, 0.8, 0.9, 0.2, 0.2, 0.9]

# 計算平均相似度和平均評分
mean_score = np.mean(scores)
mean_rscore = np.mean(rscores)
print("Mean Score:", mean_score)
print("Mean RScore:", mean_rscore)