diff --git a/docs/superpowers/plans/2026-03-24-llm-call-logging.md b/docs/superpowers/plans/2026-03-24-llm-call-logging.md new file mode 100644 index 0000000..c177365 --- /dev/null +++ b/docs/superpowers/plans/2026-03-24-llm-call-logging.md @@ -0,0 +1,469 @@ +# LLM Call Logging — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Log every LLM call during synthesis generation with full prompt/response/timing, viewable per synthesis. + +**Architecture:** New `llm_call_log` table, `log_llm_call` helper wrapping each provider call with timing, API endpoint to retrieve logs by job_id, frontend log viewer page linked from Home. + +**Tech Stack:** Rust (sqlx), SolidJS, PostgreSQL + +**Spec:** `docs/superpowers/specs/2026-03-24-llm-call-logging-design.md` + +--- + +### Task 1: Migration + DB module + +**Files:** +- Create: `backend/migrations/20260324000017_create_llm_call_log.sql` +- Create: `backend/src/db/llm_call_log.rs` +- Modify: `backend/src/db/mod.rs` +- Modify: `CLAUDE.md` + +- [ ] **Step 1: Create migration** + +```sql +CREATE TABLE llm_call_log ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + job_id UUID NOT NULL, + call_type TEXT NOT NULL, + model TEXT NOT NULL, + system_prompt TEXT NOT NULL DEFAULT '', + user_prompt TEXT NOT NULL DEFAULT '', + response_body TEXT NOT NULL DEFAULT '', + duration_ms INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX idx_llm_call_log_job_id ON llm_call_log(job_id); +CREATE INDEX idx_llm_call_log_user_id ON llm_call_log(user_id, created_at); +``` + +- [ ] **Step 2: Create `backend/src/db/llm_call_log.rs`** + +```rust +//! LLM call logging: tracks every LLM interaction during synthesis generation. + +use chrono::{DateTime, Utc}; +use serde::Serialize; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::errors::AppError; + +/// Row returned from llm_call_log queries. +#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +pub struct LlmCallLogRow { + pub id: Uuid, + pub call_type: String, + pub model: String, + pub system_prompt: String, + pub user_prompt: String, + pub response_body: String, + pub duration_ms: i32, + pub created_at: DateTime, +} + +/// Insert a single LLM call log entry. +pub async fn insert( + pool: &PgPool, + user_id: Uuid, + job_id: Uuid, + call_type: &str, + model: &str, + system_prompt: &str, + user_prompt: &str, + response_body: &str, + duration_ms: i32, +) -> Result<(), AppError> { + sqlx::query( + r#" + INSERT INTO llm_call_log (user_id, job_id, call_type, model, system_prompt, user_prompt, response_body, duration_ms) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + "#, + ) + .bind(user_id) + .bind(job_id) + .bind(call_type) + .bind(model) + .bind(system_prompt) + .bind(user_prompt) + .bind(response_body) + .bind(duration_ms) + .execute(pool) + .await?; + Ok(()) +} + +/// List all LLM call log entries for a generation job. +pub async fn list_by_job_id( + pool: &PgPool, + user_id: Uuid, + job_id: Uuid, +) -> Result, AppError> { + let rows = sqlx::query_as::<_, LlmCallLogRow>( + r#" + SELECT id, call_type, model, system_prompt, user_prompt, response_body, duration_ms, created_at + FROM llm_call_log + WHERE user_id = $1 AND job_id = $2 + ORDER BY created_at ASC + "#, + ) + .bind(user_id) + .bind(job_id) + .fetch_all(pool) + .await?; + Ok(rows) +} + +/// Truncate old LLM call log entries: replace prompt/response with first 500 chars. +/// +/// Entries older than `days` get their text fields truncated to save space +/// while preserving metadata (call_type, model, duration). +pub async fn truncate_old( + pool: &PgPool, + user_id: Uuid, + days: i32, +) -> Result { + let result = sqlx::query( + r#" + UPDATE llm_call_log + SET system_prompt = LEFT(system_prompt, 500) || E'\n[truncated]', + user_prompt = LEFT(user_prompt, 500) || E'\n[truncated]', + response_body = LEFT(response_body, 500) || E'\n[truncated]' + WHERE user_id = $1 + AND created_at < now() - make_interval(days => $2) + AND LENGTH(system_prompt) > 510 + "#, + ) + .bind(user_id) + .bind(days) + .execute(pool) + .await?; + Ok(result.rows_affected()) +} +``` + +- [ ] **Step 3: Register module + update CLAUDE.md** + +In `backend/src/db/mod.rs`, add `pub mod llm_call_log;` (alphabetical — after `article_history`). +Update CLAUDE.md migration count to 17. + +- [ ] **Step 4: Verify + commit** + +```bash +cd backend && cargo test --lib && cargo build +git add backend/migrations/20260324000017_create_llm_call_log.sql backend/src/db/llm_call_log.rs backend/src/db/mod.rs CLAUDE.md +git commit -m "feat: create llm_call_log table + DB module" +``` + +--- + +### Task 2: Pipeline instrumentation — log_llm_call helper + wrap all LLM calls + +**Files:** +- Modify: `backend/src/services/synthesis.rs` + +- [ ] **Step 1: Add `log_llm_call` helper** + +Add near the other helper functions (`trace_article`, etc.): + +```rust +/// Log an LLM call with full prompt, response, and timing. +async fn log_llm_call( + pool: &sqlx::PgPool, + user_id: Uuid, + job_id: Uuid, + call_type: &str, + model: &str, + system_prompt: &str, + user_prompt: &str, + response: &serde_json::Value, + duration_ms: u64, +) { + let response_str = serde_json::to_string_pretty(response).unwrap_or_default(); + db::llm_call_log::insert( + pool, user_id, job_id, call_type, model, + system_prompt, user_prompt, &response_str, duration_ms as i32, + ) + .await + .ok(); // Don't fail synthesis if logging fails +} +``` + +- [ ] **Step 2: Wrap each LLM call with timing + logging** + +Find each LLM provider call in `run_generation_inner` and wrap with timing. Pattern: + +```rust +let start = std::time::Instant::now(); +let result = provider.generate_search_pass(...).await?; +let duration = start.elapsed().as_millis() as u64; +log_llm_call(&state.pool, user_id, job_id, "search", &model_research, + &system_prompt, &user_prompt, &result, duration).await; +``` + +Apply to all LLM call sites: +1. **Search pass** (Phase 2) — `call_type: "search"` +2. **Classification Phase 1** — `call_type: "classification_phase1"` +3. **Classification Phase 2** — `call_type: "classification_phase2"` +4. **Rewrite pass** — `call_type: "rewrite"` + +For link extraction and article extraction (inside `source_scraper.rs` and `scrape_single_article_with_llm`), logging requires passing `pool`, `user_id`, and `job_id` through. Since these functions are called from spawned tasks, this is complex. For now, skip logging those calls — we can add them later. Focus on the 4 main calls in `run_generation_inner`. + +- [ ] **Step 3: Add truncation to cleanup** + +In `run_generation_inner`, find the existing `article_history::cleanup_old` call at the start of generation. Add LLM log truncation alongside: + +```rust + if settings.article_history_days > 0 { + // ... existing cleanup_old call ... + + // Truncate old LLM call logs + db::llm_call_log::truncate_old(&state.pool, user_id, settings.article_history_days) + .await + .ok(); + } +``` + +- [ ] **Step 4: Verify + commit** + +```bash +cd backend && cargo test --lib && cargo build +git add backend/src/services/synthesis.rs +git commit -m "feat: log LLM calls with timing at search, classification, and rewrite steps" +``` + +--- + +### Task 3: API endpoint + +**Files:** +- Create: `backend/src/handlers/llm_logs.rs` +- Modify: `backend/src/handlers/mod.rs` +- Modify: `backend/src/router.rs` + +- [ ] **Step 1: Create handler** + +```rust +//! Handler for LLM call log viewing. + +use axum::extract::{Path, State}; +use axum::response::IntoResponse; +use axum::Json; +use uuid::Uuid; + +use crate::app_state::AppState; +use crate::db; +use crate::errors::AppError; +use crate::middleware::auth::AuthUser; + +/// GET /api/v1/llm-logs/:job_id +/// +/// Returns all LLM call log entries for a generation job. +/// Verifies the job belongs to a synthesis owned by the authenticated user. +pub async fn get_logs( + auth_user: AuthUser, + State(state): State, + Path(job_id): Path, +) -> Result { + // Verify the job_id belongs to a synthesis owned by this user + let synthesis = sqlx::query_scalar::<_, Uuid>( + "SELECT id FROM syntheses WHERE job_id = $1 AND user_id = $2 LIMIT 1", + ) + .bind(job_id) + .bind(auth_user.id) + .fetch_optional(&state.pool) + .await?; + + if synthesis.is_none() { + return Err(AppError::NotFound("No synthesis found for this job".into())); + } + + let logs = db::llm_call_log::list_by_job_id(&state.pool, auth_user.id, job_id).await?; + + Ok(Json(logs)) +} +``` + +- [ ] **Step 2: Register + add route** + +In `handlers/mod.rs`, add `pub mod llm_logs;` (alphabetical). + +In `router.rs`, add in the authenticated section: +```rust +.route("/api/v1/llm-logs/:job_id", get(handlers::llm_logs::get_logs)) +``` + +- [ ] **Step 3: Verify + commit** + +```bash +cd backend && cargo test --lib && cargo build +git add backend/src/handlers/llm_logs.rs backend/src/handlers/mod.rs backend/src/router.rs +git commit -m "feat: API endpoint for LLM call logs by job_id" +``` + +--- + +### Task 4: Frontend — types, API client, i18n + +**Files:** +- Modify: `frontend/src/types.ts` +- Create: `frontend/src/api/llmLogs.ts` +- Modify: `frontend/src/i18n/fr.ts` + +- [ ] **Step 1: Add type** + +In `types.ts`: +```typescript +export interface LlmCallLogEntry { + id: string; + call_type: string; + model: string; + system_prompt: string; + user_prompt: string; + response_body: string; + duration_ms: number; + created_at: string; +} +``` + +- [ ] **Step 2: Create API client** + +```typescript +import { api } from './client'; +import type { LlmCallLogEntry } from '~/types'; + +export const llmLogsApi = { + getByJobId: (jobId: string): Promise => + api.get(`/llm-logs/${jobId}`), +}; +``` + +- [ ] **Step 3: Add i18n labels** + +```typescript + // LLM Logs + 'llmLogs.title': 'Logs des appels IA', + 'llmLogs.callType': 'Type', + 'llmLogs.model': 'Modele', + 'llmLogs.duration': 'Duree', + 'llmLogs.systemPrompt': 'Prompt systeme', + 'llmLogs.userPrompt': 'Prompt utilisateur', + 'llmLogs.response': 'Reponse', + 'llmLogs.empty': 'Aucun log disponible pour cette synthese.', + 'llmLogs.viewLogs': 'Voir les logs IA', + 'llmLogs.seconds': 's', +``` + +- [ ] **Step 4: Commit** + +```bash +git add frontend/src/types.ts frontend/src/api/llmLogs.ts frontend/src/i18n/fr.ts +git commit -m "feat: LLM logs types, API client, and i18n labels" +``` + +--- + +### Task 5: Frontend — LlmLogs page + Home button + route + +**Files:** +- Create: `frontend/src/pages/LlmLogs.tsx` +- Modify: `frontend/src/App.tsx` +- Modify: `frontend/src/pages/Home.tsx` + +- [ ] **Step 1: Create LlmLogs page** + +Create `frontend/src/pages/LlmLogs.tsx` — a page that: +- Extracts `job_id` from the URL params +- Calls `llmLogsApi.getByJobId(jobId)` on mount +- Displays each call as a card with: + - Header: call_type badge (colored by type), model name, duration in seconds + - Three collapsible sections: System Prompt, User Prompt, Response + - Text in monospace `
` blocks, scrollable, max-height ~300px
+  - Response: attempt `JSON.parse` + `JSON.stringify(parsed, null, 2)` for pretty-printing; fall back to raw text
+- Back link to home
+
+Use SolidJS patterns consistent with the rest of the app (`createSignal`, `onMount`, `For`, `Show`). Use Tailwind classes.
+
+- [ ] **Step 2: Add route**
+
+In `App.tsx`, add:
+```tsx
+import LlmLogs from './pages/LlmLogs';
+// In routes:
+
+```
+
+Inside the authenticated/protected layout.
+
+- [ ] **Step 3: Add log button on Home page**
+
+In `Home.tsx`, find where each synthesis is rendered (the list/card). The synthesis data should include `job_id` (from the `SynthesisListItem` type — check if it's already there; if not, it needs to be added).
+
+First check `frontend/src/types.ts` for `SynthesisListItem` — if `job_id` is not in it, add `job_id: string | null`.
+
+Then in the synthesis row, next to the delete button, add a small icon/button:
+```tsx
+
+  
+    {/* Use a simple text icon or emoji */}
+    📋
+  
+
+```
+
+Also check the backend — does the `SynthesisListItem` model include `job_id`? If not, add it to the model and the SQL query. Read `backend/src/models/synthesis.rs` for `SynthesisListItem` and `backend/src/handlers/syntheses.rs` for the list handler.
+
+- [ ] **Step 4: Verify + commit**
+
+```bash
+cd frontend && npx tsc --noEmit && npx vitest run
+git add frontend/src/pages/LlmLogs.tsx frontend/src/App.tsx frontend/src/pages/Home.tsx frontend/src/types.ts
+# Also add any backend model changes if needed
+git commit -m "feat: LLM logs viewer page + log button on Home synthesis list"
+```
+
+---
+
+### Task 6: E2E test
+
+**Files:**
+- Modify: `e2e/tests/generation-live.spec.ts`
+
+- [ ] **Step 1: Add LLM logs verification**
+
+After the existing provenance verification, add:
+
+```typescript
+    // ═══════════════════════════════════════════════════════════════
+    // LLM call logs verification
+    // ═══════════════════════════════════════════════════════════════
+    // Get job_id from provenance (used entries have it)
+    const jobId = usedEntries[0]?.job_id;
+    if (jobId) {
+      const logsResp = await apiCall(page, 'GET', `/api/v1/llm-logs/${jobId}`);
+      expect(logsResp.status).toBe(200);
+      const logs = logsResp.data;
+      expect(Array.isArray(logs)).toBe(true);
+      expect(logs.length).toBeGreaterThan(0);
+
+      // Should have at least a rewrite call
+      const callTypes = logs.map((l: any) => l.call_type);
+      expect(callTypes).toContain('rewrite');
+
+      // Each log entry should have prompt and response
+      for (const log of logs) {
+        expect(log.model).toBeTruthy();
+        expect(log.duration_ms).toBeGreaterThanOrEqual(0);
+      }
+    }
+```
+
+- [ ] **Step 2: Commit**
+
+```bash
+git add e2e/tests/generation-live.spec.ts
+git commit -m "test: verify LLM call logs endpoint returns data after generation"
+```