批次處理文字嵌入與Pinecone向量資料函式庫查詢技術

import tqdm

# 假設 batch_size 和 chunks 已經定義好了
batch_size = 32
chunks = [...]  # 列表中的文字資料

for i in tqdm.tqdm(range(0, len(chunks), batch_size)):
    i_end = min(i + batch_size, len(chunks))
    meta_batch = chunks[i:i_end]
    ids_batch = [str(j) for j in range(i, i_end)]
    texts = [item for item in meta_batch]

    try:
        embeds =玄貓(texts)  # 取得嵌入
    except Exception as e:
        # 處理異常，例如速率限制
        while True:
            # 等待幾秒鐘後重試
            time.sleep(5)
            try:
                embeds =玄貓(texts)
                break
            except Exception as e:
                # 如果仍然失敗，可能需要增加重試次數或採取其他策略
                pass

    # 修改批次資料結構
    meta_batch = [{"batch": i, "text": text} for text in meta_batch]

    # 建立待插入列表
    to_upsert = [(id, embed, meta) for id, embed, meta in zip(ids_batch, embeds, meta_batch)]

    # 執行插入操作
    index.upsert(to_upsert)

def pinecone_vector_search(user_query, k):
    # 將使用者查詢轉換為向量表示
    xq = get_vector_embeddings(user_query)
    
    # 執行Pinecone查詢
    res = index.query(vector=xq, top_k=k, include_metadata=True)
    
    return res

user_query = "do we get free unicorn rides?"
res = pinecone_vector_search(user_query, k=1)
print(res)

from langchain_core.documents import Document
from langchain_community.vectorstores.chroma import Chroma
from langchain_openai import OpenAIEmbeddings
import lark
import getpass
import os
import warnings

# 停用警告
warnings.filterwarnings("ignore")

# 建立檔案列表
docs = [
    Document(
        page_content="一個關於年輕巫師在魔法學校的旅程的故事。",
        metadata={
            "title": "哈利波特",
            "author": "J.K.羅琳",
            "genre": "奇幻",
            "ISBN": "978-0-7475-3269-9",
            "publisher": "Bloomsbury",
            "summary": "哈利波特是一個關於年輕巫師在魔法學校的旅程的故事。",
            "rating": 4.5
        }
    )
]

from langchain.chains.query_constructor.base import AttributeInfo
from langchain_openai.chat_models import ChatOpenAI
from langchain.retrievers.self_query.base import SelfQueryRetriever
from langchain.chains.query_constructor.base import OpenAIEmbeddings
from langchain.vectorstores import Chroma

# 定義檔案列表
docs = [
    {
        "title": "Harry Potter and the Philosopher's Stone",
        "author": "J.K. Rowling",
        "year_published": 1997,
        "genre": "Fiction",
        "isbn": "978-0747532699",
        "publisher": "Bloomsbury",
        "language": "English",
        "page_count": 223,
        "summary": "The first book in the Harry Potter series where Harry discovers his magical heritage.",
        "rating": 4.8,
    },
    #... 更多檔案...
]

# 建立嵌入模型
embeddings = OpenAIEmbeddings()

# 建立向量資料函式庫
vectorstore = Chroma.from_documents(docs, embeddings)

# 基本資訊
basic_info = [
    AttributeInfo(name="title", description="書籍標題", type="string"),
    AttributeInfo(name="author", description="書籍作者", type="string"),
    AttributeInfo(name="year_published", description="出版年份", type="integer"),
]

# 詳細資訊
detailed_info = [
    AttributeInfo(name="genre", description="書籍型別", type="string or list[string]"),
    AttributeInfo(name="isbn", description="國際標準書號", type="string"),
]

# 建立自查詢系統
retriever = SelfQueryRetriever(vectorstore, basic_info + detailed_info)

from LangChain import ChatOpenAI, SelfQueryRetriever, AttributeInfo

basic_info = [
    AttributeInfo(
        name="title",
        description="The title of the book",
        type="string",
    ),
    AttributeInfo(
        name="author",
        description="The author of the book",
        type="string",
    ),
    AttributeInfo(
        name="publisher",
        description="The publishing house that published the book",
        type="string",
    ),
    AttributeInfo(
        name="language",
        description="The primary language the book is written in",
        type="string",
    ),
    AttributeInfo(
        name="page_count",
        description="Number of pages in the book",
        type="integer",
    ),
]

detailed_info = [
    # 在這裡新增詳細資訊屬性
]

analysis = [
    AttributeInfo(
        name="summary",
        description="A brief summary or description of the book",
        type="string",
    ),
    AttributeInfo(
        name="rating",
        description="""An average rating for the book (from reviews), ranging from 1-5""",
        type="float",
    ),
]

metadata_field_info = basic_info + detailed_info + analysis