ATOCore/tests/test_extractor_llm.py

"""Tests for the LLM-assisted extractor path.

Focused on the parser and failure-mode contracts — the actual network
call is exercised out of band by running
``python scripts/extractor_eval.py --mode llm`` against the frozen
labeled corpus with ``ANTHROPIC_API_KEY`` set. These tests only
exercise the pieces that don't need network.
"""

from __future__ import annotations

import os
from unittest.mock import patch

import pytest

from atocore.interactions.service import Interaction
from atocore.memory.extractor_llm import (
    LLM_EXTRACTOR_VERSION,
    _parse_candidates,
    extract_candidates_llm,
    extract_candidates_llm_verbose,
)


def _make_interaction(prompt: str = "p", response: str = "r") -> Interaction:
    return Interaction(
        id="test-id",
        prompt=prompt,
        response=response,
        response_summary="",
        project="",
        client="test",
        session_id="",
    )


def test_parser_handles_empty_array():
    result = _parse_candidates("[]", _make_interaction())
    assert result == []


def test_parser_handles_malformed_json():
    result = _parse_candidates("{ not valid json", _make_interaction())
    assert result == []


def test_parser_strips_markdown_fences():
    raw = "```json\n[{\"type\": \"knowledge\", \"content\": \"x is y\", \"project\": \"\", \"confidence\": 0.5}]\n```"
    result = _parse_candidates(raw, _make_interaction())
    assert len(result) == 1
    assert result[0].memory_type == "knowledge"
    assert result[0].content == "x is y"


def test_parser_strips_surrounding_prose():
    raw = "Here are the candidates:\n[{\"type\": \"project\", \"content\": \"foo\", \"project\": \"p04\", \"confidence\": 0.6}]\nThat's it."
    result = _parse_candidates(raw, _make_interaction())
    assert len(result) == 1
    assert result[0].memory_type == "project"
    assert result[0].project == "p04"


def test_parser_drops_invalid_memory_types():
    raw = '[{"type": "nonsense", "content": "x"}, {"type": "project", "content": "y"}]'
    result = _parse_candidates(raw, _make_interaction())
    assert len(result) == 1
    assert result[0].memory_type == "project"


def test_parser_drops_empty_content():
    raw = '[{"type": "knowledge", "content": "   "}, {"type": "knowledge", "content": "real"}]'
    result = _parse_candidates(raw, _make_interaction())
    assert len(result) == 1
    assert result[0].content == "real"


def test_parser_clamps_confidence_to_unit_interval():
    raw = '[{"type": "knowledge", "content": "c1", "confidence": 2.5}, {"type": "knowledge", "content": "c2", "confidence": -0.4}]'
    result = _parse_candidates(raw, _make_interaction())
    assert result[0].confidence == 1.0
    assert result[1].confidence == 0.0


def test_parser_defaults_confidence_on_missing_field():
    raw = '[{"type": "knowledge", "content": "c1"}]'
    result = _parse_candidates(raw, _make_interaction())
    assert result[0].confidence == 0.5


def test_parser_tags_version_and_rule():
    raw = '[{"type": "project", "content": "c1"}]'
    result = _parse_candidates(raw, _make_interaction())
    assert result[0].rule == "llm_extraction"
    assert result[0].extractor_version == LLM_EXTRACTOR_VERSION
    assert result[0].source_interaction_id == "test-id"


def test_missing_api_key_returns_empty(monkeypatch):
    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
    result = extract_candidates_llm_verbose(_make_interaction("p", "some real response"))
    assert result.candidates == []
    assert result.error == "missing_api_key"


def test_empty_response_returns_empty(monkeypatch):
    monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key-not-used")
    result = extract_candidates_llm_verbose(_make_interaction("p", ""))
    assert result.candidates == []
    assert result.error == "empty_response"


def test_api_error_returns_empty(monkeypatch):
    """A transport error from the SDK must not raise into the caller."""
    monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key-not-used")

    class _BoomClient:
        def __init__(self, *a, **kw):
            pass

        class messages:  # noqa: D401
            @staticmethod
            def create(**kw):
                raise RuntimeError("simulated network error")

    with patch("anthropic.Anthropic", _BoomClient):
        result = extract_candidates_llm_verbose(_make_interaction("p", "real response"))
    assert result.candidates == []
    assert "api_error" in result.error