Mutator evaluation

import asyncio
import json

from core.evaluator.llm.evaluator import LLMEvaluator
from core.evaluator.schema import EvaluationContext
from core.mutator.llm.client.base import BaseLLMClient
from core.mutator.llm.mutator import LLMMutator
from core.mutator.llm.schema import StructuredInput
from core.state.interpretive.schema import ValueConfidence
from core.state.interpretive.state import InterpretiveState


class FixedClient(BaseLLMClient):
    async def generate(self, prompt: str) -> str:
        return json.dumps(
            [
                {
                    "path": "registrant.name",
                    "value": "Ada Lovelace",
                    "confidence": 0.95,
                    "inference": "Extracted the stated name",
                }
            ]
        )


async def main() -> None:
    pre_state = InterpretiveState()
    expected = InterpretiveState()
    expected.add_value(
        "registrant.name",
        ValueConfidence(value="Ada Lovelace", confidence=0.95),
    )

    result = await LLMEvaluator().evaluate(
        EvaluationContext(
            mutator=LLMMutator(FixedClient()),
            pre_state=pre_state,
            expected_post_state=expected,
            mutation_input=StructuredInput(
                prompt="My name is Ada Lovelace",
                message_id="message-1",
            ),
        )
    )

    assert result.comparison.overall_match
    assert result.accuracy_score == 1.0


asyncio.run(main())