多模態資料集載入視覺化與查詢引擎建置

import pandas as pd

# 建立一個空的DataFrame用於儲存影像、邊界框和標籤資料
df = pd.DataFrame(columns=['image', 'boxes', 'labels'])

# 遍歷資料集中的每個樣本
for i, sample in enumerate(ds):
    # 將影像資料以位元組形式儲存
    df.loc[i, 'image'] = sample.images.tobytes()
    
    # 將邊界框資料轉換為列表格式並儲存
    boxes_list = sample.boxes.numpy(aslist=True)
    df.loc[i, 'boxes'] = [box.tolist() for box in boxes_list]
    
    # 將標籤資料儲存
    label_data = sample.labels.data()['text']
    df.loc[i, 'labels'] = label_data

# 顯示資料集中的標籤列表
labels_list = ds.labels.info['class_names']
print(labels_list)

import cv2
from PIL import Image
import io
from PIL import ImageDraw

# 選擇第一張影像
ind = 0
image_data = ds.images[ind].tobytes()

# 顯示原始影像
image = ds.images[ind].numpy()
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
img = Image.fromarray(image_rgb)
display(img)

# 取得標籤和邊界框資料
labels = ds.labels[ind].data()['value']
bboxes = ds.boxes[ind].numpy()

# 定義函式用於在影像上新增邊界框
def display_image_with_bboxes(image_data, bboxes, labels, label_name):
    # ...
    # 篩選特定標籤的邊界框並繪製
    # ...

# 新增邊界框並顯示
label_name = "truck"
display_image_with_bboxes(image_data, bboxes, labels, label_name)

def display_multimodal_response(user_input, llm_response, image_path):
    # 1. 列印使用者輸入
    print(user_input)
    # 2. 列印LLM回應
    print(textwrap.fill(str(llm_response), 100))
    # 3. 顯示多模態回應
    display_source_image(image_path)

# 使用者輸入範例
user_input = "無人機如何識別卡車？"
# 假設 llm_response 是由 LLM 產生的回應
llm_response = "無人機使用視覺檢測和追蹤方法來識別卡車。"
image_path = "/content/boxed_image.jpg"

# 呼叫函式顯示多模態回應
display_multimodal_response(user_input, llm_response, image_path)

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer

model = SentenceTransformer('all-MiniLM-L6-v2')

def calculate_cosine_similarity_with_embeddings(text1, text2):
    embeddings1 = model.encode(text1)
    embeddings2 = model.encode(text2)
    similarity = cosine_similarity([embeddings1], [embeddings2])
    return similarity[0][0]

llm_similarity_score = calculate_cosine_similarity_with_embeddings(user_input, llm_response)
print(f"Cosine Similarity Score: {llm_similarity_score:.3f}")

import base64

def encode_image(image_path):
    with open(image_path, "rb") as image_file:
        return base64.b64encode(image_file.read()).decode("utf-8")

base64_image = encode_image(IMAGE_PATH)

from openai import OpenAI

client = OpenAI(api_key=openai.api_key)
MODEL = "gpt-4o"

response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": f"You are a helpful assistant"},
        {"role": "user", "content": [
            {"type": "text", "text": f"Analyze the following image"},
            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64_image}"}}
        ]}
    ],
    temperature=0.0,
)

response_image = response.choices[0].message.content
print(response_image)

score = (llm_similarity_score + multimodal_similarity_score) / 2
print(f"Multimodal, Modular Score: {score:.3f}")

import numpy as np

def calculate_similarity(query_embedding, document_embeddings):
    # 計算查詢嵌入向量與檔案嵌入向量之間的餘弦相似度
    similarities = np.dot(query_embedding, document_embeddings.T) / (np.linalg.norm(query_embedding) * np.linalg.norm(document_embeddings, axis=1))
    return similarities

def get_top_documents(similarities, top_n=5):
    # 取得最相關的前N個檔案
    top_indices = np.argsort(similarities)[::-1][:top_n]
    return top_indices

# 假設query_embedding和document_embeddings已經計算好
query_embedding = np.random.rand(1, 128)  # 示例查詢嵌入向量
document_embeddings = np.random.rand(100, 128)  # 示例檔案嵌入向量

similarities = calculate_similarity(query_embedding, document_embeddings)
top_documents = get_top_documents(similarities)

#### 內容解密：
1. **`calculate_similarity`函式**：計算查詢向量與檔案向量之間的餘弦相似度，用於評估查詢與檔案的相關性。
2. **`get_top_documents`函式**：根據相似度排序，選取最相關的前N個檔案，以最佳化檢索結果。
3. **向量計算**：使用numpy進行高效的向量運算，以提升效能。

### 圖表說明

```mermaid
graph LR;
    A[查詢輸入] --> B[計算查詢嵌入向量];
    B --> C[計算查詢與檔案的相似度];
    C --> D[選取最相關的檔案];
    D --> E[生成最終回應];