主题
阿里云 MySQL,也称为 ApsaraDB RDS for MySQL。ApsaraDB RDS for MySQL 是基于 MySQL 源代码分支的在线数据库服务,可提供高性能。ApsaraDB RDS for MySQL 是一种经过验证的解决方案,曾在双 11 期间处理大量并发流量。ApsaraDB RDS for MySQL 提供白名单配置、备份与恢复、透明数据加密(TDE)、数据迁移以及实例、账户和数据库管理等基本功能。更多信息,请参见 RDS MySQL 功能概述。
要运行此 notebook,你需要在云中运行一个 ApsaraDB RDS MySQL 实例,创建一个账户并创建所需的数据库。你可以参考此链接。
在本 notebook 中,我们需要在你的 ApsaraDB RDS MySQL 实例中创建名为 llama_index_test 和 llama_index_meta_test 的数据库。
设置
如果你在 colab 上打开此 Notebook,可能需要确保已安装 llama-index:
!pip install llama-index%pip install llama-index-vector-stores-alibabacloud-mysql# choose dashscope as embedding and llm model, your can also use default openai or other model to test
%pip install llama-index-embeddings-dashscope
%pip install llama-index-llms-dashscope配置 dashscope 嵌入和 llm 模型,你也可以使用默认的 openai 或其他模型进行测试。如果你选择使用 dashscope 模型,可以在此处获取 API 密钥,并在以下代码中进行设置:
!export DASHSCOPE_API_KEY="your_api_key"下载示例数据
!mkdir -p 'data/paul_graham/'
!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt'使用阿里云 MySQL 的 RAG 演示
简单查询
为简单查询加载数据
import os
DASHSCOPE_API_KEY = os.environ.get("DASHSCOPE_API_KEY")
from llama_index.core import (
VectorStoreIndex,
SimpleDirectoryReader,
StorageContext,
)
from llama_index.vector_stores.alibabacloud_mysql.base import (
AlibabaCloudMySQLVectorStore,
)
# set Embbeding model
from llama_index.core import Settings
from llama_index.embeddings.dashscope import DashScopeEmbedding
Settings.embed_model = DashScopeEmbedding(api_key=DASHSCOPE_API_KEY)
# config llm model
from llama_index.llms.dashscope import DashScope, DashScopeGenerationModels
dashscope_llm = DashScope(
model_name=DashScopeGenerationModels.QWEN_MAX, api_key=DASHSCOPE_API_KEY
)
documents = SimpleDirectoryReader("data/paul_graham/").load_data()
print(f"Total documents: {len(documents)}")
print(f"First document, id: {documents[0].doc_id}")
print(f"First document, hash: {documents[0].hash}")
print(
"First document, text"
f" ({len(documents[0].text)} characters):\n{'='*20}\n{documents[0].text[:360]} ..."
)
print(
"""
#################
# simple generate vector
#################"""
)
client = AlibabaCloudMySQLVectorStore.from_params(
host="rm-***.mysql.***.rds.aliyuncs.com",
port=3306,
user="user",
password="password",
database="llama_index_test",
distance_method="COSINE",
)
storage_context = StorageContext.from_defaults(vector_store=client)
VectorStoreIndex.from_documents(
documents, storage_context=storage_context, show_progress=True
)使用 AlibabaCloudMySQL 进行搜索测试查询
import os
DASHSCOPE_API_KEY = os.environ.get("DASHSCOPE_API_KEY")
from llama_index.core import VectorStoreIndex
from llama_index.vector_stores.alibabacloud_mysql.base import (
AlibabaCloudMySQLVectorStore,
)
# set Embbeding model
from llama_index.core import Settings
from llama_index.embeddings.dashscope import DashScopeEmbedding
embed_model = DashScopeEmbedding(api_key=DASHSCOPE_API_KEY)
# Global Settings
Settings.embed_model = embed_model
# config llm model
from llama_index.llms.dashscope import DashScope, DashScopeGenerationModels
dashscope_llm = DashScope(
model_name=DashScopeGenerationModels.QWEN_MAX, api_key=DASHSCOPE_API_KEY
)
print(
"""
#################
# Basic Querying including Search Test
#################
"""
)
client = AlibabaCloudMySQLVectorStore.from_params(
host="rm-***.mysql.eu-west-1.rds.***.com",
port=3306,
user="user",
password="password",
database="llama_index_test",
distance_method="COSINE",
)
index = VectorStoreIndex.from_vector_store(
vector_store=client, embed_model=embed_model
)
QUESTION = "What did the author do growing up?"
# Set Retriever
vector_retriever = index.as_retriever()
# search
source_nodes = vector_retriever.retrieve(QUESTION)
# check source_nodes
print(f"Question: {QUESTION}")
for node in source_nodes:
print(f"---------------------------------------------")
print("Search Test")
print(f"---------------------------------------------")
print(f"Score: {node.score:.3f}")
print(node.get_content())
print(f"---------------------------------------------")
# run query
query_engine = index.as_query_engine(llm=dashscope_llm)
res = query_engine.query(QUESTION)
print(f"Answer: {res.response}")
print(f"---------------------------------------------\n\n")元数据过滤
为元数据过滤加载数据
import os
DASHSCOPE_API_KEY = os.environ.get("DASHSCOPE_API_KEY")
from llama_index.core import (
VectorStoreIndex,
SimpleDirectoryReader,
StorageContext,
)
from llama_index.vector_stores.alibabacloud_mysql.base import (
AlibabaCloudMySQLVectorStore,
)
# set Embbeding model
from llama_index.core import Settings
from llama_index.embeddings.dashscope import DashScopeEmbedding
Settings.embed_model = DashScopeEmbedding(api_key=DASHSCOPE_API_KEY)
# config llm model
from llama_index.llms.dashscope import DashScope, DashScopeGenerationModels
dashscope_llm = DashScope(
model_name=DashScopeGenerationModels.QWEN_MAX, api_key=DASHSCOPE_API_KEY
)
documents = SimpleDirectoryReader("data/paul_graham/").load_data()
print(f"Total documents: {len(documents)}")
print(f"First document, id: {documents[0].doc_id}")
print(f"First document, hash: {documents[0].hash}")
print(
"First document, text"
f" ({len(documents[0].text)} characters):\n{'='*20}\n{documents[0].text[:360]} ..."
)
print(
"""
#################
# generate vector with some metadata for Metadata Filtering
#################"""
)
client = AlibabaCloudMySQLVectorStore.from_params(
host="rm-***.mysql.***.rds.aliyuncs.com",
port=3306,
user="user",
password="password",
database="llama_index_meta_test",
distance_method="COSINE",
)
storage_context = StorageContext.from_defaults(vector_store=client)
index = VectorStoreIndex.from_documents(
documents, storage_context=storage_context, show_progress=True
)
from llama_index.core import Document
import regex as re
# Split the text into paragraphs.
text_chunks = documents[0].text.split("\n\n")
# Create a document for each footnote
footnotes = [
Document(
text=chunk,
id=documents[0].doc_id,
metadata={
"is_footnote": bool(re.search(r"^\s*\[\d+\]\s*", chunk)),
"mark_id": i,
},
)
for i, chunk in enumerate(text_chunks)
if bool(re.search(r"^\s*\[\d+\]\s*", chunk))
]
# Insert the footnotes into the index
for f in footnotes:
index.insert(f)使用 AlibabaCloudMySQL 进行搜索测试查询
import os
DASHSCOPE_API_KEY = os.environ.get("DASHSCOPE_API_KEY")
from llama_index.core import VectorStoreIndex
from llama_index.vector_stores.alibabacloud_mysql.base import (
AlibabaCloudMySQLVectorStore,
)
# set Embbeding model
from llama_index.core import Settings
from llama_index.embeddings.dashscope import DashScopeEmbedding
embed_model = DashScopeEmbedding(api_key=DASHSCOPE_API_KEY)
# Global Settings
Settings.embed_model = embed_model
# config llm model
from llama_index.llms.dashscope import DashScope, DashScopeGenerationModels
dashscope_llm = DashScope(
model_name=DashScopeGenerationModels.QWEN_MAX, api_key=DASHSCOPE_API_KEY
)
print(
"""
#################
# Querying with Metadata Filtering including Search Test
#################
"""
)
client = AlibabaCloudMySQLVectorStore.from_params(
host="rm-***.mysql.***.rds.aliyuncs.com",
port=3306,
user="user",
password="password",
database="llama_index_meta_test",
distance_method="COSINE",
)
index = VectorStoreIndex.from_vector_store(
vector_store=client, embed_model=embed_model
)
from llama_index.core.vector_stores import (
MetadataFilters,
MetadataFilter,
FilterOperator,
FilterCondition,
)
QUESTION = "What did the author about space aliens and lisp?"
print(f"---------------------------------------------")
print(f"Question: {QUESTION}")
filters = MetadataFilters(
filters=[
MetadataFilter(
key="is_footnote", value="true", operator=FilterOperator.EQ
),
MetadataFilter(key="mark_id", value=0, operator=FilterOperator.GTE),
],
condition=FilterCondition.AND,
)
print(f"---------------------------------------------")
for i in range(len(filters.filters)):
print(f"Filter[{i}]: {filters.filters[i]}")
print(f"Filter Condition: {filters.condition}")
print(f"---------------------------------------------")
retriever = index.as_retriever(
filters=filters,
)
result = retriever.retrieve(QUESTION)
for node in result:
print("Search Test")
print(f"---------------------------------------------")
print(f"Score: {node.score:.3f}")
print(node.get_content())
print(f"---------------------------------------------")
# Create a query engine that only searches certain footnotes.
footnote_query_engine = index.as_query_engine(
filters=filters,
llm=dashscope_llm,
)
res = footnote_query_engine.query(QUESTION)
print(f"Answer: {res.response}")
print(f"---------------------------------------------\n\n")