hallucinations_v1 scores only the sentences it could parse, so one unparsed block hides a contradiction #7963

"""Offline repro: HallucinationsV1Evaluator scores a validator response that
leaves sentences without a usable label as if those sentences did not exist.

A scripted judge (BaseLlm subclass, registered via LLMRegistry) answers the
segmenter with two sentences and the validator with a fixed text. No network.
"""
from __future__ import annotations

import asyncio
import re
from typing import AsyncGenerator, ClassVar

import google.adk
from google.adk.evaluation.eval_case import Invocation
from google.adk.evaluation.eval_metrics import EvalMetric, HallucinationsCriterion, JudgeModelOptions
from google.adk.evaluation.hallucinations_v1 import HallucinationsV1Evaluator
from google.adk.models.base_llm import BaseLlm
from google.adk.models.llm_request import LlmRequest
from google.adk.models.llm_response import LlmResponse
from google.adk.models.registry import LLMRegistry
from google.genai import types

print("google.adk from:", google.adk.__file__)

S1 = "Paris is the capital of France."
S2 = "The Eiffel Tower is 50 meters tall."
SEGMENTS = f"<sentence>{S1}</sentence>\n<sentence>{S2}</sentence>"

VALIDATOR_REPLIES = {
    # Control: both blocks complete -> expected 0.5.
    "complete (control)": (
        f"sentence: {S1}\nlabel: supported\nrationale: in context\n"
        "supporting_excerpt: Paris is the capital\ncontradicting_excerpt: null\n"
        f"sentence: {S2}\nlabel: contradictory\nrationale: context says 330 m\n"
        "supporting_excerpt: null\ncontradicting_excerpt: 330 meters tall"
    ),
    # Judge leaves out the contradicting_excerpt line on a `supported` block.
    "S1 block lacks contradicting_excerpt line": (
        f"sentence: {S1}\nlabel: supported\nrationale: in context\n"
        "supporting_excerpt: Paris is the capital\n"
        f"sentence: {S2}\nlabel: contradictory\nrationale: context says 330 m\n"
        "supporting_excerpt: null\ncontradicting_excerpt: 330 meters tall"
    ),
    # Judge reports only one of the two sentences.
    "S2 omitted by validator": (
        f"sentence: {S1}\nlabel: supported\nrationale: in context\n"
        "supporting_excerpt: Paris is the capital\ncontradicting_excerpt: null"
    ),
    # Judge copies the backticks the prompt itself puts around label names.
    "S2 label written as `contradictory`": (
        f"sentence: {S1}\nlabel: supported\nrationale: in context\n"
        "supporting_excerpt: Paris is the capital\ncontradicting_excerpt: null\n"
        f"sentence: {S2}\nlabel: `contradictory`\nrationale: context says 330 m\n"
        "supporting_excerpt: null\ncontradicting_excerpt: 330 meters tall"
    ),
}


class ScriptedJudge(BaseLlm):
  validator_reply: ClassVar[str] = ""

  @classmethod
  def supported_models(cls) -> list[str]:
    return [r"scripted-judge"]

  async def generate_content_async(
      self, llm_request: LlmRequest, stream: bool = False
  ) -> AsyncGenerator[LlmResponse, None]:
    prompt = llm_request.contents[-1].parts[0].text
    text = SEGMENTS if "segment the response into sentences" in prompt else self.validator_reply
    yield LlmResponse(content=types.Content(role="model", parts=[types.Part(text=text)]))


LLMRegistry.register(ScriptedJudge)

invocation = Invocation(
    user_content=types.Content(role="user", parts=[types.Part(text="Tell me about Paris.")]),
    final_response=types.Content(role="model", parts=[types.Part(text=f"{S1} {S2}")]),
)
metric = EvalMetric(
    metric_name="hallucinations_v1",
    criterion=HallucinationsCriterion(
        threshold=0.8, judge_model_options=JudgeModelOptions(judge_model="scripted-judge")
    ),
)


async def main() -> None:
  for name, reply in VALIDATOR_REPLIES.items():
    ScriptedJudge.validator_reply = reply
    result = await HallucinationsV1Evaluator(metric).evaluate_invocations([invocation])
    print(f"{name:45} score={result.overall_score} status={result.overall_eval_status.name}")


asyncio.run(main())