mkdir -p data/paul_graham/
curl "<https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt>" -o "data/paul_graham/paul_graham_essay.txt"

import asyncio
import os

import nest_asyncio
import pandas as pd
from llama_index.core import (
    SimpleDirectoryReader,
    StorageContext,
    VectorStoreIndex,
    load_index_from_storage,
)
from llama_index.core.evaluation import (
    BatchEvalRunner,
    EmbeddingQAFinetuneDataset,
    FaithfulnessEvaluator,
    RelevancyEvaluator,
    RetrieverEvaluator,
    generate_question_context_pairs,
)
from llama_index.core.node_parser import SentenceSplitter
from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = (
    "sk-proj-xxx"
)
nest_asyncio.apply()

def main():
    PERSIST_DIR = "persist"
    MODEL_NAME = "gpt-4o-mini"
    QA_DATASET_PATH = "qa_dataset.json"

    documents = SimpleDirectoryReader("../data/paul_graham/").load_data()

    llm = OpenAI(model=MODEL_NAME)

    # チャンクサイズ512のドキュメントノードの作成
    node_parser = SentenceSplitter.from_defaults(chunk_size=512)
    nodes = node_parser.get_nodes_from_documents(documents)

    # 実行毎にIDが変わらないよう、手動でnodeのIDを付与します
    for idx, node in enumerate(nodes):
        node.id_ = f"node_{idx}"

    # VectoreStoreIndexを作成済みならそこからロード、なければ作成
    if not os.path.exists(PERSIST_DIR):
        os.makedirs(PERSIST_DIR)
        vector_index = VectorStoreIndex(nodes)
        vector_index.storage_context.persist(PERSIST_DIR)
    else:
        storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
        vector_index = load_index_from_storage(storage_context)

    query_engine = vector_index.as_query_engine()

    response_vector = query_engine.query("What did the author do growing up?")
    print(response_vector.response)

if __name__ == "__main__":
    main()

The author worked on writing short stories and programming, particularly on an IBM 1401 computer in 9th grade using an early version of Fortran.

DEFAULT_EVAL_TEMPLATE = PromptTemplate(
    "Please tell if a given piece of information "
    "is supported by the context.\\n"
    "You need to answer with either YES or NO.\\n"
    "Answer YES if any of the context supports the information, even "
    "if most of the context is unrelated. "
    "Some examples are provided below. \\n\\n"
    "Information: Apple pie is generally double-crusted.\\n"
    "Context: An apple pie is a fruit pie in which the principal filling "
    "ingredient is apples. \\n"
    "Apple pie is often served with whipped cream, ice cream "
    "('apple pie à la mode'), custard or cheddar cheese.\\n"
    "It is generally double-crusted, with pastry both above "
    "and below the filling; the upper crust may be solid or "
    "latticed (woven of crosswise strips).\\n"
    "Answer: YES\\n"
    "Information: Apple pies tastes bad.\\n"
    "Context: An apple pie is a fruit pie in which the principal filling "
    "ingredient is apples. \\n"
    "Apple pie is often served with whipped cream, ice cream "
    "('apple pie à la mode'), custard or cheddar cheese.\\n"
    "It is generally double-crusted, with pastry both above "
    "and below the filling; the upper crust may be solid or "
    "latticed (woven of crosswise strips).\\n"
    "Answer: NO\\n"
    "Information: {query_str}\\n"
    "Context: {context_str}\\n"
    "Answer: "
)

DEFAULT_EVAL_TEMPLATE = PromptTemplate(
    "Your task is to evaluate if the response is relevant to the query.\\n"
    "The evaluation should be performed in a step-by-step manner by answering the following questions:\\n"
    "1. Does the provided response match the subject matter of the user's query?\\n"
    "2. Does the provided response attempt to address the focus or perspective "
    "on the subject matter taken on by the user's query?\\n"
    "Each question above is worth 1 point. Provide detailed feedback on response according to the criteria questions above  "
    "After your feedback provide a final result by strictly following this format: '[RESULT] followed by the integer number representing the total score assigned to the response'\\n\\n"
    "Query: \\n {query}\\n"
    "Response: \\n {response}\\n"
    "Feedback:"
)

import asyncio
import os

import nest_asyncio
import pandas as pd
from llama_index.core import (
    SimpleDirectoryReader,
    StorageContext,
    VectorStoreIndex,
    load_index_from_storage,
)
from llama_index.core.evaluation import (
    BatchEvalRunner,
    EmbeddingQAFinetuneDataset,
    FaithfulnessEvaluator,
    RelevancyEvaluator,
    RetrieverEvaluator,
    generate_question_context_pairs,
)
from llama_index.core.node_parser import SentenceSplitter
from llama_index.llms.openai import OpenAI

os.environ["OPENAI_API_KEY"] = "sk-proj-xxx"
nest_asyncio.apply()

async def get_metrics(retriever_evaluator, dataset) -> dict[str, float]:
    eval_results = await retriever_evaluator.aevaluate_dataset(dataset)

    metric_dicts = []
    for eval_result in eval_results:
        metric_dict = eval_result.metric_vals_dict
        metric_dicts.append(metric_dict)

    full_df = pd.DataFrame(metric_dicts)

    hit_rate = full_df["hit_rate"].mean()
    mrr = full_df["mrr"].mean()

    return {"Hit Rate": float(hit_rate), "MRR": float(mrr)}

def get_eval_results(key: str, eval_results: dict) -> float:
    results = eval_results[key]
    correct = 0
    for result in results:
        if result.passing:
            correct += 1
    score = correct / len(results)
    return score

async def main():
    PERSIST_DIR = "persist"
    MODEL_NAME = "gpt-4o-mini"
    QA_DATASET_PATH = "qa_dataset.json"

    documents = SimpleDirectoryReader("../data/paul_graham/").load_data()

    llm = OpenAI(model=MODEL_NAME)

    # Build index with a chunk_size of 512
    node_parser = SentenceSplitter.from_defaults(chunk_size=512)
    nodes = node_parser.get_nodes_from_documents(documents)

    # Ensure the node ids are same for each run
    for idx, node in enumerate(nodes):
        node.id_ = f"node_{idx}"

    if not os.path.exists(PERSIST_DIR):
        os.makedirs(PERSIST_DIR)
        vector_index = VectorStoreIndex(nodes)
        # persist the index
        vector_index.storage_context.persist(PERSIST_DIR)
    else:
        # rebuild storage context
        storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
        # load index
        vector_index = load_index_from_storage(storage_context)

    if not os.path.exists(QA_DATASET_PATH):
        qa_dataset = generate_question_context_pairs(
            nodes,
            llm=llm,
            num_questions_per_chunk=2,
        )
        qa_dataset.save_json(QA_DATASET_PATH)
    else:
        qa_dataset = EmbeddingQAFinetuneDataset.from_json(QA_DATASET_PATH)

    retriever = vector_index.as_retriever(similarity_top_k=2)

    retriever_evaluator = RetrieverEvaluator.from_metric_names(
        ["mrr", "hit_rate"], retriever=retriever
    )

    metrics = await get_metrics(retriever_evaluator, qa_dataset)
    print(metrics)

    queries = list(qa_dataset.queries.values())
    query_engine = vector_index.as_query_engine()

    # Faithfulness & Relevancy Evaluator
    faithfulness = FaithfulnessEvaluator(llm=llm)
    relevancy = RelevancyEvaluator(llm=llm)

    batch_eval_queries = queries
    runner = BatchEvalRunner(
        {"faithfulness": faithfulness, "relevancy": relevancy},
    )
    eval_results = await runner.aevaluate_queries(
        query_engine, queries=batch_eval_queries
    )

    print("Batch Evaluation Results")
    faithfulness_score = get_eval_results("faithfulness", eval_results)
    relevancy_score = get_eval_results("relevancy", eval_results)
    print(f"Faithfulness Score: {faithfulness_score}")
    print(f"Relevancy Score: {relevancy_score}")

if __name__ == "__main__":
    asyncio.run(main())

{'Hit Rate': 0.7950819672131147, 'MRR': 0.6516393442622951}
Batch Evaluation Results
Faithfulness Score: 0.9098360655737705
Relevancy Score: 0.9590163934426229

DEFAULT_QA_GENERATE_PROMPT_TMPL = """\\
Context information is below.

---------------------
{context_str}
---------------------

Given the context information and not prior knowledge.
generate only questions based on the below query.

You are a Teacher/ Professor. Your task is to setup \\
{num_questions_per_chunk} questions for an upcoming \\
quiz/examination. The questions should be diverse in nature \\
across the document. Restrict the questions to the \\
context information provided."
"""