Initial commit: CI Agent competitive-intelligence monitoring app
FastAPI + Celery + Next.js + Postgres/Redis app with company monitoring, source collection, LLM-based change analysis, enrichment, and account security (Turnstile, escalating lockout, email verification).
This commit is contained in:
@@ -0,0 +1,85 @@
|
||||
"""Ollama provider: local models via Ollama's HTTP API. Uses JSON mode
|
||||
(`format: "json"`) plus a bounded repair loop, since not every locally-run
|
||||
model supports strict schema-constrained decoding the way Anthropic's
|
||||
tool-use does - the schema is instead embedded in the system prompt as an
|
||||
instruction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from app.analysis.llm.base import LLMResponseError
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
class OllamaLLMProvider:
|
||||
provider_name = "ollama"
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
|
||||
async def generate_structured[T: BaseModel](
|
||||
self, system_prompt: str, user_prompt: str, response_model: type[T]
|
||||
) -> T:
|
||||
schema_instructions = (
|
||||
f"{system_prompt}\n\nRespond with ONLY a single JSON object matching this JSON "
|
||||
f"schema, no other text, no markdown fences:\n"
|
||||
f"{json.dumps(response_model.model_json_schema())}"
|
||||
)
|
||||
|
||||
last_error: Exception | None = None
|
||||
prompt = user_prompt
|
||||
|
||||
for attempt in range(self._settings.llm_max_retries + 1):
|
||||
if attempt > 0 and last_error is not None:
|
||||
prompt = (
|
||||
f"{user_prompt}\n\nYour previous response was invalid: {last_error}. "
|
||||
"Try again, returning ONLY valid JSON matching the schema."
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
response = await client.post(
|
||||
f"{self._settings.ollama_base_url}/api/chat",
|
||||
json={
|
||||
"model": self._settings.ollama_model,
|
||||
"messages": [
|
||||
{"role": "system", "content": schema_instructions},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"format": "json",
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
content = response.json().get("message", {}).get("content", "")
|
||||
try:
|
||||
data = json.loads(content)
|
||||
return response_model.model_validate(data)
|
||||
except (json.JSONDecodeError, ValidationError) as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
|
||||
raise LLMResponseError(
|
||||
f"Ollama provider failed to produce a valid {response_model.__name__} after "
|
||||
f"{self._settings.llm_max_retries + 1} attempt(s): {last_error}"
|
||||
)
|
||||
|
||||
async def generate_text(self, system_prompt: str, user_prompt: str) -> str:
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
response = await client.post(
|
||||
f"{self._settings.ollama_base_url}/api/chat",
|
||||
json={
|
||||
"model": self._settings.ollama_model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json().get("message", {}).get("content", "")
|
||||
Reference in New Issue
Block a user