"""Ollama provider: local models via Ollama's HTTP API. Uses JSON mode (`format: "json"`) plus a bounded repair loop, since not every locally-run model supports strict schema-constrained decoding the way Anthropic's tool-use does - the schema is instead embedded in the system prompt as an instruction. """ from __future__ import annotations import json import httpx from pydantic import BaseModel, ValidationError from app.analysis.llm.base import LLMResponseError from app.core.config import Settings class OllamaLLMProvider: provider_name = "ollama" def __init__(self, settings: Settings) -> None: self._settings = settings async def generate_structured[T: BaseModel]( self, system_prompt: str, user_prompt: str, response_model: type[T] ) -> T: schema_instructions = ( f"{system_prompt}\n\nRespond with ONLY a single JSON object matching this JSON " f"schema, no other text, no markdown fences:\n" f"{json.dumps(response_model.model_json_schema())}" ) last_error: Exception | None = None prompt = user_prompt for attempt in range(self._settings.llm_max_retries + 1): if attempt > 0 and last_error is not None: prompt = ( f"{user_prompt}\n\nYour previous response was invalid: {last_error}. " "Try again, returning ONLY valid JSON matching the schema." ) async with httpx.AsyncClient(timeout=120) as client: response = await client.post( f"{self._settings.ollama_base_url}/api/chat", json={ "model": self._settings.ollama_model, "messages": [ {"role": "system", "content": schema_instructions}, {"role": "user", "content": prompt}, ], "format": "json", "stream": False, }, ) response.raise_for_status() content = response.json().get("message", {}).get("content", "") try: data = json.loads(content) return response_model.model_validate(data) except (json.JSONDecodeError, ValidationError) as exc: last_error = exc continue raise LLMResponseError( f"Ollama provider failed to produce a valid {response_model.__name__} after " f"{self._settings.llm_max_retries + 1} attempt(s): {last_error}" ) async def generate_text(self, system_prompt: str, user_prompt: str) -> str: async with httpx.AsyncClient(timeout=120) as client: response = await client.post( f"{self._settings.ollama_base_url}/api/chat", json={ "model": self._settings.ollama_model, "messages": [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], "stream": False, }, ) response.raise_for_status() return response.json().get("message", {}).get("content", "")