RAG（2）：三合一评估标准Triad of metrics

• 答案相关性Answer relevance

  # 1.答案相关性
  ## 初始化设置
  import nest_asyncio
  from trulens_eval import OpenAI as fOpenAI
  from trulens_eval import Feedback
  nest_asyncio.apply()
  provider = fOpenAI()
  ## 设置答案相关性
  f_qa_relevance = Feedback(
      provider.relevance_with_cot_reasons,
      name="Answer Relevance"
  ).on_input_output()
  

• 上下文相关性Context relevance

  # 2.上下文相关性
  from trulens_eval import TruLlama
  import numpy as np
  context_selection = TruLlama.select_source_nodes().node.text
  ## 实际设置1
  f_qs_relevance = (
      Feedback(provider.qs_relevance,
               name="Context Relevance")
      .on_input()
      .on(context_selection)
      .aggregate(np.mean)
  )
  # 实际设置2 增加思维链reason在评估时提供分数的理由
  f_qs_relevance = (
      Feedback(provider.qs_relevance_with_cot_reasons,
               name="Context Relevance")
      .on_input()
      .on(context_selection)
      .aggregate(np.mean)
  )
  

• 基础性Groundedness

  from trulens_eval.feedback import Groundedness
  grounded = Groundedness(groundedness_provider=provider)
  
  f_groundedness = (
      Feedback(grounded.groundedness_measure_with_cot_reasons,
               name="Groundedness"
              )
      .on(context_selection)
      .on_output()
      .aggregate(grounded.grounded_statements_aggregator)
  )
  

评估RAG应用

不同评估方法的优劣：

from trulens_eval import TruLlama
from trulens_eval import FeedbackMode
# 1.设计记录执行情况
tru_recorder = TruLlama(
    sentence_window_engine,
    app_id="App_1",
    feedbacks=[
        f_qa_relevance,
        f_qs_relevance,
        f_groundedness
    ]
)
# 2.加载评估问题
eval_questions = []
with open('eval_questions.txt', 'r') as file:
    for line in file:
        # Remove newline character and convert to integer
        item = line.strip()
        eval_questions.append(item)
## 追加问题
eval_questions.append("How can I be successful in AI?")
# 3.对每个问题执行Query
for question in eval_questions:
    with tru_recorder as recording:
        sentence_window_engine.query(question)
## 获取记录和反馈
records, feedback = tru.get_records_and_feedback(app_ids=[])
records.head()
## 查看更口语化的反馈
import pandas as pd
pd.set_option("display.max_colwidth", None)
records[["input", "output"] + feedback]
# 4.评估分数
tru.get_leaderboard(app_ids=[])
tru.run_dashboard()