How to Evaluate Your LLM Application

1
! pip install -qU datasets ragas langchain langchain-mongodb langchain-openai \
2
pymongo pandas tqdm matplotlib seaborn

1
import getpass
2
MONGODB_URI = getpass.getpass("Enter your MongoDB connection string:")

1
import os
2
from openai import OpenAI
3
os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter your OpenAI API Key:")
4
openai_client = OpenAI()

1
from datasets import load_dataset
2
import pandas as pd
3
4
data = load_dataset("explodinggradients/ragas-wikiqa", split="train")
5
df = pd.DataFrame(data)

1
from langchain.text_splitter import RecursiveCharacterTextSplitter
2
3
# Split text by tokens using the tiktoken tokenizer
4
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
5
    encoding_name="cl100k_base", keep_separator=False, chunk_size=200,   chunk_overlap=30
6
)
7
8
def split_texts(texts):
9
    chunked_texts = []
10
    for text in texts:
11
        chunks = text_splitter.create_documents([text])
12
        chunked_texts.extend([chunk.page_content for chunk in chunks])
13
    return chunked_texts
14
15
# Split the context field into chunks
16
df["chunks"] = df["context"].apply(lambda x: split_texts(x))
17
# Aggregate list of all chunks
18
all_chunks = df["chunks"].tolist()
19
docs = [item for chunk in all_chunks for item in chunk]

1
def get_embeddings(docs: List[str], model: str) -> List[List[float]]:
2
    """
3
    Generate embeddings using the OpenAI API.
4
5
    Args:
6
        docs (List[str]): List of texts to embed
7
        model (str, optional): Model name. Defaults to "text-embedding-3-large".
8
9
    Returns:
10
        List[float]: Array of embeddings
11
    """
12
    # replace newlines, which can negatively affect performance.
13
    docs = [doc.replace("\n", " ") for doc in docs]
14
    response = openai_client.embeddings.create(input=docs, model=model)
15
    response = [r.embedding for r in response.data]
16
    return response

1
{
2
  "data": [
3
    {
4
      "embedding": [
5
        0.018429679796099663,
6
        -0.009457024745643139
7
    .
8
    .
9
    .
10
      ],
11
      "index": 0,
12
      "object": "embedding"
13
    }
14
  ],
15
  "model": "text-embedding-3-small",
16
  "object": "list",
17
  "usage": {
18
    "prompt_tokens": 183,
19
    "total_tokens": 183
20
  }
21
}

1
from pymongo import MongoClient
2
from tqdm.auto import tqdm
3
4
client = MongoClient(MONGODB_URI)
5
DB_NAME = "ragas_evals"
6
db = client[DB_NAME]
7
batch_size = 128
8
9
EVAL_EMBEDDING_MODELS = ["text-embedding-ada-002", "text-embedding-3-small"]
10
11
for model in EVAL_EMBEDDING_MODELS:
12
    embedded_docs = []
13
    print(f"Getting embeddings for the {model} model")
14
    for i in tqdm(range(0, len(docs), batch_size)):
15
        end = min(len(docs), i + batch_size)
16
        batch = docs[i:end]
17
        # Generate embeddings for current batch
18
        batch_embeddings = get_embeddings(batch, model)
19
        # Creating the documents to ingest into MongoDB for current batch
20
        batch_embedded_docs = [
21
            {"text": batch[i], "embedding": batch_embeddings[i]}
22
            for i in range(len(batch))
23
        ]
24
        embedded_docs.extend(batch_embedded_docs)
25
    print(f"Finished getting embeddings for the {model} model")
26
27
    # Bulk insert documents into a MongoDB collection
28
    print(f"Inserting embeddings for the {model} model")
29
    collection = db[model]
30
    collection.delete_many({})
31
    collection.insert_many(embedded_docs)
32
    print(f"Finished inserting embeddings for the {model} model")

1
{
2
  "text": "For the purposes of authentication, most countries require commercial or personal documents which originate from or are signed in another country to be notarized before they can be used or officially recorded or before they can have any legal effect.",
3
  "embedding": [
4
    0.018429679796099663,
5
    -0.009457024745643139,
6
    .
7
    .
8
    .
9
  ]
10
}

1
{
2
  "fields": [
3
    {
4
      "numDimensions": 1536,
5
      "path": "embedding",
6
      "similarity": "cosine",
7
      "type": "vector"
8
    }
9
  ]
10
}

1
from langchain_openai import OpenAIEmbeddings
2
from langchain_mongodb import MongoDBAtlasVectorSearch
3
from langchain_core.vectorstores import VectorStoreRetriever
4
5
def get_retriever(model: str, k: int) -> VectorStoreRetriever:
6
    """
7
    Given an embedding model and top k, get a vector store retriever object
8
9
    Args:
10
        model (str): Embedding model to use
11
        k (int): Number of results to retrieve
12
13
    Returns:
14
        VectorStoreRetriever: A vector store retriever object
15
    """
16
    embeddings = OpenAIEmbeddings(model=model)
17
18
    vector_store = MongoDBAtlasVectorSearch.from_connection_string(
19
        connection_string=MONGODB_URI,
20
        namespace=f"{DB_NAME}.{model}",
21
        embedding=embeddings,
22
        index_name="vector_index",
23
        text_key="text",
24
    )
25
26
    retriever = vector_store.as_retriever(
27
        search_type="similarity", search_kwargs={"k": k}
28
    )
29
    return retriever

1
QUESTIONS = df["question"].to_list()
2
GROUND_TRUTH = df["correct_answer"].tolist()

1
from datasets import Dataset
2
from ragas import evaluate, RunConfig
3
from ragas.metrics import context_precision, context_recall
4
import nest_asyncio
5
6
# Allow nested use of asyncio (used by RAGAS)
7
nest_asyncio.apply()
8
9
for model in EVAL_EMBEDDING_MODELS:
10
    data = {"question": [], "ground_truth": [], "contexts": []}
11
    data["question"] = QUESTIONS
12
    data["ground_truth"] = GROUND_TRUTH
13
14
    retriever = get_retriever(model, 2)
15
    # Getting relevant documents for the evaluation dataset
16
    for i in tqdm(range(0, len(QUESTIONS))):
17
        data["contexts"].append(
18
            [doc.page_content for doc in retriever.get_relevant_documents(QUESTIONS[i])]
19
        )
20
    # RAGAS expects a Dataset object
21
    dataset = Dataset.from_dict(data)
22
    # RAGAS runtime settings to avoid hitting OpenAI rate limits
23
    run_config = RunConfig(max_workers=4, max_wait=180)
24
    result = evaluate(
25
        dataset=dataset,
26
        metrics=[context_precision, context_recall],
27
        run_config=run_config,
28
        raise_exceptions=False,
29
    )
30
    print(f"Result for the {model} model: {result}")

1
from langchain_openai import ChatOpenAI
2
from langchain_core.prompts import ChatPromptTemplate
3
from langchain_core.runnables import RunnablePassthrough
4
from langchain_core.runnables.base import RunnableSequence
5
from langchain_core.output_parsers import StrOutputParser
6
7
def get_rag_chain(retriever: VectorStoreRetriever, model: str) -> RunnableSequence:
8
    """
9
    Create a basic RAG chain
10
11
    Args:
12
        retriever (VectorStoreRetriever): Vector store retriever object
13
        model (str): Chat completion model to use
14
15
    Returns:
16
        RunnableSequence: A RAG chain
17
    """
18
    # Generate context using the retriever, and pass the user question through
19
    retrieve = {
20
        "context": retriever
21
        | (lambda docs: "\n\n".join([d.page_content for d in docs])),
22
        "question": RunnablePassthrough(),
23
    }
24
    template = """Answer the question based only on the following context: \
25
    {context}
26
27
    Question: {question}
28
    """
29
    # Defining the chat prompt
30
    prompt = ChatPromptTemplate.from_template(template)
31
    # Defining the model to be used for chat completion
32
    llm = ChatOpenAI(temperature=0, model=model)
33
    # Parse output as a string
34
    parse_output = StrOutputParser()
35
36
    # Naive RAG chain
37
    rag_chain = retrieve | prompt | llm | parse_output
38
    return rag_chain

1
from ragas.metrics import faithfulness, answer_relevancy
2
3
for model in ["gpt-3.5-turbo-1106", "gpt-3.5-turbo"]:
4
    data = {"question": [], "ground_truth": [], "contexts": [], "answer": []}
5
    data["question"] = QUESTIONS
6
    data["ground_truth"] = GROUND_TRUTH
7
    # Using the best embedding model from the retriever evaluation
8
    retriever = get_retriever("text-embedding-3-small", 2)
9
    rag_chain = get_rag_chain(retriever, model)
10
    for i in tqdm(range(0, len(QUESTIONS))):
11
        question = QUESTIONS[i]
12
        data["answer"].append(rag_chain.invoke(question))
13
        data["contexts"].append(
14
            [doc.page_content for doc in retriever.get_relevant_documents(question)]
15
        )
16
    # RAGAS expects a Dataset object
17
    dataset = Dataset.from_dict(data)
18
    # RAGAS runtime settings to avoid hitting OpenAI rate limits
19
    run_config = RunConfig(max_workers=4, max_wait=180)
20
    result = evaluate(
21
        dataset=dataset,
22
        metrics=[faithfulness, answer_relevancy],
23
        run_config=run_config,
24
        raise_exceptions=False,
25
    )
26
    print(f"Result for the {model} model: {result}")

1
from ragas.metrics import answer_similarity, answer_correctness
2
3
data = {"question": [], "ground_truth": [], "answer": []}
4
data["question"] = QUESTIONS
5
data["ground_truth"] = GROUND_TRUTH
6
# Using the best embedding model from the retriever evaluation
7
retriever = get_retriever("text-embedding-3-small", 2)
8
# Using the best completion model from the generator evaluation
9
rag_chain = get_rag_chain(retriever, "gpt-3.5-turbo")
10
for question in tqdm(QUESTIONS):
11
    data["answer"].append(rag_chain.invoke(question))
12
13
dataset = Dataset.from_dict(data)
14
run_config = RunConfig(max_workers=4, max_wait=180)
15
result = evaluate(
16
    dataset=dataset,
17
    metrics=[answer_similarity, answer_correctness],
18
    run_config=run_config,
19
    raise_exceptions=False,
20
)
21
print(f"Overall metrics: {result}")

1
result_df = result.to_pandas()
2
result_df[result_df["answer_correctness"] < 0.7]

1
import seaborn as sns
2
import matplotlib.pyplot as plt
3
4
plt.figure(figsize=(10, 8))
5
sns.heatmap(
6
    result_df[1:10].set_index("question")[["answer_similarity", "answer_correctness"]],
7
    annot=True,
8
    cmap="flare",
9
)
10
plt.show()

1
from datetime import datetime
2
3
result["timestamp"] = datetime.now()
4
collection = db["metrics"]
5
collection.insert_one(result)

1
{
2
  "answer_similarity": 0.8873,
3
  "answer_correctness": 0.5922,
4
  "timestamp": 2024-04-07T23:27:30.655+00:00
5
}