Testing LLM Apps Skill

You are an expert in testing AI-powered applications, applying the engineering

discipline used by Anthropic's product teams to build reliable, regression-proof

LLM applications. You write production-ready test suites in British English.

IMI colours: Navy #1A1A2E · Teal #0F3D66 · Gold #E2B95A

Part 1 — Mock LLM Responses for Fast Tests

# pip install pytest pytest-asyncio

import pytest
from unittest.mock import patch, MagicMock
import json


# ── Simple mock factory ────────────────────────────────────────────────────────
def make_anthropic_response(text: str, model: str = "claude-3-5-sonnet-20241022",
                              input_tokens: int = 100, output_tokens: int = 50):
    """Create a mock Anthropic API response."""
    mock = MagicMock()
    mock.content = [MagicMock(text=text)]
    mock.model = model
    mock.usage = MagicMock(
        input_tokens=input_tokens,
        output_tokens=output_tokens,
        cache_read_input_tokens=0,
        cache_creation_input_tokens=0
    )
    mock.stop_reason = "end_turn"
    return mock


def make_openai_response(text: str, model: str = "gpt-4o-mini",
                          input_tokens: int = 100, output_tokens: int = 50):
    """Create a mock OpenAI API response."""
    mock = MagicMock()
    mock.choices = [MagicMock(message=MagicMock(content=text), finish_reason="stop")]
    mock.model = model
    mock.usage = MagicMock(prompt_tokens=input_tokens, completion_tokens=output_tokens)
    return mock


# ── Pytest fixtures ────────────────────────────────────────────────────────────
@pytest.fixture
def mock_claude():
    """Patch Anthropic client for tests."""
    with patch("anthropic.Anthropic") as MockClass:
        instance = MockClass.return_value
        instance.messages.create.return_value = make_anthropic_response(
            "Tribal fans show high identity-based attachment to their club."
        )
        yield instance


@pytest.fixture
def mock_openai():
    """Patch OpenAI client for tests."""
    with patch("openai.OpenAI") as MockClass:
        instance = MockClass.return_value
        instance.chat.completions.create.return_value = make_openai_response(
            '{"segment": "Tribal", "confidence": 0.92}'
        )
        yield instance


# ── Example tests ─────────────────────────────────────────────────────────────
class TestFanSegmentation:
    """Test fan segmentation pipeline without hitting real APIs."""

    def test_classifies_tribal_fan(self, mock_claude):
        """Tribal fan text should be classified correctly."""
        mock_claude.messages.create.return_value = make_anthropic_response(
            '{"segment": "Tribal", "confidence": 0.95}'
        )
        # Import and test your function here
        # result = classify_fan_comment("Been supporting since I was born, it's in my blood")
        # assert result["segment"] == "Tribal"
        assert True  # Replace with real assertion

    def test_handles_api_error_gracefully(self, mock_claude):
        """Should return error dict when API fails, not raise exception."""
        import anthropic
        mock_claude.messages.create.side_effect = anthropic.APIError(
            "Rate limit", request=MagicMock(), body={}
        )
        # result = classify_fan_comment("some text")
        # assert result.get("error") is not None
        assert True

    def test_uses_british_english_in_output(self, mock_claude):
        """Outputs must use British English spelling."""
        response_text = mock_claude.messages.create.return_value.content[0].text
        american_spellings = ["behavior", "analyze", "color", "organize"]
        for word in american_spellings:
            assert word not in response_text.lower(), f"American spelling found: {word}"

Part 2 — VCR Cassette Recording (Record Once, Replay Forever)

# pip install vcrpy pytest-recording

import vcr
import pytest

# ── VCR configuration ─────────────────────────────────────────────────────────
VCR_CONFIG = vcr.VCR(
    cassette_library_dir="tests/cassettes",
    record_mode="none",         # "none"=replay only, "new_episodes"=record new, "all"=always record
    match_on=["method", "scheme", "host", "port", "path", "body"],
    filter_headers=["Authorization", "x-api-key"],  # never record API keys
    filter_post_data_parameters=["api_key"],
    decode_compressed_response=True
)


@vcr.use_cassette("tests/cassettes/fan_segmentation.yaml", record_mode="none")
def test_with_recorded_response():
    """
    Uses recorded API response — runs fast, offline, no API cost.
    To record: change record_mode to "new_episodes" and run once.
    """
    import anthropic
    client = anthropic.Anthropic()  # Will use cassette, not real API
    response = client.messages.create(
        model="claude-3-haiku-20240307",
        max_tokens=100,
        messages=[{"role": "user", "content": "Classify this fan: 'lifelong supporter'"}]
    )
    assert len(response.content[0].text) > 0


# ── pytest-recording integration (simpler) ────────────────────────────────────
# In conftest.py:
CONFTEST_CONTENT = '''
import pytest

@pytest.fixture(scope="module")
def vcr_config():
    return {
        "filter_headers": ["Authorization", "x-api-key"],
        "record_mode": "none",
    }
'''

Part 3 — Golden File Testing

import json, os, difflib
from pathlib import Path


GOLDEN_DIR = Path("tests/golden")
GOLDEN_DIR.mkdir(exist_ok=True)


def save_golden(name: str, output: dict | str):
    """Save output as golden file (run once to establish baseline)."""
    path = GOLDEN_DIR / f"{name}.json"
    content = output if isinstance(output, str) else json.dumps(output, indent=2)
    path.write_text(content)
    print(f"Golden saved: {path}")


def load_golden(name: str) -> str:
    """Load golden file content."""
    path = GOLDEN_DIR / f"{name}.json"
    if not path.exists():
        raise FileNotFoundError(f"Golden file not found: {path}. Run with UPDATE_GOLDEN=1 to create.")
    return path.read_text()


def assert_matches_golden(name: str, actual: dict | str, update: bool = False):
    """
    Assert output matches golden file.
    Set UPDATE_GOLDEN=1 env var to update golden files on intentional changes.
    """
    if update or os.getenv("UPDATE_GOLDEN"):
        save_golden(name, actual)
        return  # Just update, don't assert

    expected = load_golden(name)
    actual_str = actual if isinstance(actual, str) else json.dumps(actual, indent=2)

    if expected != actual_str:
        diff = "\n".join(difflib.unified_diff(
            expected.splitlines(), actual_str.splitlines(),
            fromfile="golden", tofile="actual", lineterm=""
        ))
        raise AssertionError(f"Output differs from golden:\n{diff}")


# ── Golden test examples ──────────────────────────────────────────────────────
class TestGoldenOutputs:
    """Tests that compare against known-good golden outputs."""

    def test_report_structure_matches_golden(self, mock_claude):
        """Verify IMI report structure hasn't changed."""
        # result = generate_imi_report("Premier League fan loyalty 2025")
        # assert_matches_golden("imi_report_structure", result)
        assert True

    def test_segment_distribution_golden(self):
        """Verify segment calculation produces known output."""
        sample_data = {"responses": [{"text": "live and die by this club"}, {"text": "watch occasionally"}]}
        expected = {"Tribal": 1, "Casual": 1}
        # assert_matches_golden("segment_distribution", expected)
        assert True

Part 4 — Deterministic Test Suite (No LLM Needed)

class TestDeterministicComponents:
    """
    Tests for all non-LLM logic — should never call real APIs.
    Fast: <100ms per test.
    """

    def test_token_counter_accuracy(self):
        """Token counting must be within 5% of actual."""
        try:
            import tiktoken
            enc = tiktoken.get_encoding("cl100k_base")
            text = "The quick brown fox jumps over the lazy dog"
            counted = len(enc.encode(text))
            # 9 words → ~9-10 tokens
            assert 8 <= counted <= 12, f"Token count {counted} outside expected range"
        except ImportError:
            pytest.skip("tiktoken not installed")

    def test_pii_detection_catches_email(self):
        """PII detection must catch email addresses."""
        try:
            from presidio_analyzer import AnalyzerEngine
            analyzer = AnalyzerEngine()
            results = analyzer.analyze(text="Contact [email protected] for details", language="en")
            entity_types = [r.entity_type for r in results]
            assert "EMAIL_ADDRESS" in entity_types
        except ImportError:
            pytest.skip("presidio not installed")

    def test_cost_calculation_correct(self):
        """Token cost calculation must be mathematically accurate."""
        # claude-3-5-sonnet: $3.00/1M input, $15.00/1M output
        result = calculate_cost("claude-3-5-sonnet-20241022", 1_000_000, 1_000_000)
        assert abs(result["cost_usd"] - 18.0) < 0.01, f"Expected $18.00, got ${result['cost_usd']}"

    def test_imi_colour_constants(self):
        """IMI brand colours must be exact hex values."""
        assert "#1A1A2E".upper() == "#1A1A2E"  # Navy
        assert "#0F3D66".upper() == "#0F3D66"  # Teal
        assert "#E2B95A".upper() == "#E2B95A"  # Gold

    def test_fan_segments_complete(self):
        """All 5 IMI fan segments must be defined."""
        segments = ["Tribal", "Passionate", "Casual", "Distant", "Corporate"]
        assert len(segments) == 5
        assert "Tribal" in segments
        assert "Corporate" in segments


# ── Test runner config (pytest.ini or pyproject.toml) ─────────────────────────
PYTEST_CONFIG = """
[pytest]
testpaths = tests
asyncio_mode = auto
filterwarnings =
    ignore::DeprecationWarning
markers =
    unit: fast unit tests, no API calls
    integration: tests that hit real APIs (slow, cost money)
    golden: golden file comparison tests
"""

# Run only fast tests: pytest -m "not integration"
# Run integration: pytest -m integration --api-key=real
# Update goldens: UPDATE_GOLDEN=1 pytest -m golden

Part 5 — CI/CD Integration

# .github/workflows/test.yml
CI_WORKFLOW: |
  name: IMI AI Tests
  on: [push, pull_request]

  jobs:
    test:
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v4

        - name: Set up Python
          uses: actions/setup-python@v5
          with:
            python-version: "3.11"

        - name: Install dependencies
          run: pip install pytest pytest-asyncio vcrpy anthropic openai

        - name: Run unit tests (no API calls)
          run: pytest -m "not integration" -v

        - name: Run integration tests (scheduled only)
          if: github.event_name == 'schedule'
          env:
            ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          run: pytest -m integration -v

Output Standards

• Fast tests first: all tests not marked integration must run without API keys
• VCR cassettes: record API interactions once, replay in CI (zero cost)
• Golden files: commit golden outputs to git — review diffs on intentional changes
• Mock precision: mock at the lowest level (client.messages.create), not the module
• Test naming: test_[what]_[when]_[expected_result] convention
• British English in all test descriptions and assertion messages

pip install

pip install pytest pytest-asyncio vcrpy pytest-recording

AXE MCP Server Integration

Every skill in the AXE Skills Hub runs with access to the AXE MCP Server — giving it the full fleet intelligence toolkit automatically. No setup required; tools are available in any AXE-powered session.

Core Tools Available

Quick Start

# In any AXE session, tools are pre-loaded. Example chaining:

# 1. Search for context
results = qdrant_search("user query here", collection="axe_persistent_memory")

# 2. Fetch live data if needed
content = web_fetch("https://docs.example.com/api")

# 3. Write result to memory for next session
write_memory("shared/last_result.md", output)

# 4. Log quality output to Edge training pipeline
hydra_add(prompt=user_query, response=output, score=0.9, source="skill-name")

Edge Training Integration

High-quality skill outputs are automatically eligible for Edge model training via hydra_add. When a response scores ≥0.85 in evals, pipe it to the Hydra pipeline to compound Edge's knowledge. This is how skills make Edge smarter over time.

# After generating a high-quality response:
hydra_add(
    prompt=user_input,
    response=final_output,
    score=0.9,          # eval score
    source="skill-name" # tracks provenance
)