多模態資料集處理與查詢引擎建立

import pandas as pd

# 建立一個空的DataFrame，定義資料集的結構
df = pd.DataFrame(columns=['image', 'boxes', 'labels'])

# 遍歷資料集中的樣本
for i, sample in enumerate(ds):
    # 影像資料（選擇路徑或壓縮表示）
    df.loc[i, 'image'] = sample.images.tobytes()  # 儲存壓縮的影像資料
    
    # 邊界框資料（以列表形式儲存）
    boxes_list = sample.boxes.numpy(aslist=True)
    df.loc[i, 'boxes'] = [box.tolist() for box in boxes_list]
    
    # 標籤資料（以列表形式儲存）
    label_data = sample.labels.data()
    df.loc[i, 'labels'] = label_data['text']

# 顯示資料集中的標籤列表
labels_list = ds.labels.info['class_names']
print(labels_list)

['ignored regions', 'pedestrian', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning-tricycle', 'bus', 'motor', 'others']

# 選取第一張影像
ind = 0
image = ds.images[ind].numpy()

# 顯示影像
import cv2
from PIL import Image
from IPython.display import display

# 將影像從BGR轉換為RGB
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

# 建立PIL影像並顯示
img = Image.fromarray(image_rgb)
display(img)

# 定義一個函式，用於在影像上新增邊界框
def display_image_with_bboxes(image_data, bboxes, labels, label_name):
    # ...
    
    # 新增邊界框
    draw = ImageDraw.Draw(img)
    for idx, box in enumerate(bboxes):
        if idx in relevant_indices:
            x1, y1, w, h = box
            x2, y2 = x1 + w, y1 + h
            draw.rectangle([x1, y1, x2, y2], outline="red", width=2)
            draw.text((x1, y1), label_name, fill="red")
    
    # 儲存影像
    save_path = "boxed_image.jpg"
    img.save(save_path)
    display(img)

# 新增邊界框並顯示
labels = ds.labels[ind].data()['value']
image_data = ds.images[ind].tobytes()
bboxes = ds.boxes[ind].numpy()
label_name = "truck"
display_image_with_bboxes(image_data, bboxes, labels, label_name)

# 為DataFrame中的每一行建立向量索引
# ...

# 建立查詢引擎
# ...

# 為DataFrame新增唯一ID
df['doc_id'] = df.index.astype(str)

documents = []
for _, row in df.iterrows():
    text_labels = row['labels']
    text = " ".join(text_labels)
    document = Document(text=text, doc_id=row['doc_id'])
    documents.append(document)

from llama_index.core import GPTVectorStoreIndex
vector_store_index = GPTVectorStoreIndex.from_documents(documents)

vector_query_engine = vector_store_index.as_query_engine(similarity_top_k=3)

start_time = time.time()
response = vector_query_engine.query(user_input)
end_time = time.time()
elapsed_time = end_time - start_time
print(f"查詢執行時間：{elapsed_time:.4f} 秒")

def get_unique_words(text):
    text = text.lower().strip()
    words = text.split()
    unique_words = [word for word, _ in groupby(sorted(words))]
    return unique_words

for node in response.source_nodes:
    node_text = node.get_text()
    unique_words = get_unique_words(node_text)
    print("節點中的唯一詞彙：", unique_words)

def process_and_display(response, df, ds, unique_words):
    # 處理節點並顯示相關影像
    for node in response.source_nodes:
        # 找到對應的影像索引
        doc_id = node.node_id
        # 顯示影像並加上邊界框
        display_image_with_bboxes(image_data, bboxes, labels, label_names)

process_and_display(response, df, ds, unique_words)

# 1. 使用者輸入
print(user_input)
# 2. LLM回應
print(textwrap.fill(str(llm_response), 100))
# 3. 多模態回應（顯示影像）
image_path = "/content/boxed_image.jpg"
display_source_image(image_path)

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer

model = SentenceTransformer('all-MiniLM-L6-v2')

def calculate_cosine_similarity_with_embeddings(text1, text2):
    embeddings1 = model.encode(text1)
    embeddings2 = model.encode(text2)
    similarity = cosine_similarity([embeddings1], [embeddings2])
    return similarity[0][0]

user_input = "無人機如何識別卡車？"
llm_response = "無人機可以透過視覺檢測和跟蹤來識別卡車。"

llm_similarity_score = calculate_cosine_similarity_with_embeddings(user_input, llm_response)
print(f"Cosine Similarity Score: {llm_similarity_score:.3f}")

Cosine Similarity Score: 0.691

import base64

IMAGE_PATH = "/content/boxed_image.jpg"

def encode_image(image_path):
    with open(image_path, "rb") as image_file:
        return base64.b64encode(image_file.read()).decode("utf-8")

base64_image = encode_image(IMAGE_PATH)

from openai import OpenAI

client = OpenAI(api_key=openai.api_key)
MODEL = "gpt-4o"

response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "你是一個有幫助的助手。"},
        {"role": "user", "content": [
            {"type": "text", "text": "分析以下影像並描述其中包含的物件。"},
            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64_image}"}}
        ]}
    ],
    temperature=0.0,
)

response_image = response.choices[0].message.content
print(response_image)

該影像包含兩個卡車，分別位於兩個邊界框內。
1. **第一個卡車（上邊界框）**：
   - 該卡車似乎是一個平板電腦卡車。
   - 它裝載了各種材料，可能與建築相關。
   - 卡車停放在一個有其他建築材料的區域。
2. **第二個卡車（下邊界框）**：
   - 該卡車也是平板電腦卡車。
   - 它運載了不同型別的材料，與第一個卡車類別似。
   - 卡車所處的環境相似，周圍有建築材料。
兩個卡車都位於建築或工業區域，可能用於建築用途。

u_word = "卡車"
resp = u_word + "s"
multimodal_similarity_score = calculate_cosine_similarity_with_embeddings(resp, response_image)
print(f"Cosine Similarity Score: {multimodal_similarity_score:.3f}")

Cosine Similarity Score: 0.505

score = (llm_similarity_score + multimodal_similarity_score) / 2
print(f"Multimodal, Modular Score: {score:.3f}")

Multimodal, Modular Score: 0.598