docs: add LLM call logging implementation plan (6 tasks)

3 months ago · 88c16c5d67
parent 314fb7a037
commit 88c16c5d67
1 changed files with 469 additions and 0 deletions
--- a/docs/superpowers/plans/2026-03-24-llm-call-logging.md
+++ b/docs/superpowers/plans/2026-03-24-llm-call-logging.md
@ -0,0 +1,469 @@
 # LLM Call Logging — Implementation Plan
 > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
 **Goal:** Log every LLM call during synthesis generation with full prompt/response/timing, viewable per synthesis.
 **Architecture:** New `llm_call_log` table, `log_llm_call` helper wrapping each provider call with timing, API endpoint to retrieve logs by job_id, frontend log viewer page linked from Home.
 **Tech Stack:** Rust (sqlx), SolidJS, PostgreSQL
 **Spec:** `docs/superpowers/specs/2026-03-24-llm-call-logging-design.md`
 ---
 ### Task 1: Migration + DB module
 **Files:**
 - Create: `backend/migrations/20260324000017_create_llm_call_log.sql`
 - Create: `backend/src/db/llm_call_log.rs`
 - Modify: `backend/src/db/mod.rs`
 - Modify: `CLAUDE.md`
 - [ ] **Step 1: Create migration**
 ```sql
 CREATE TABLE llm_call_log (
    id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    user_id         UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
    job_id          UUID NOT NULL,
    call_type       TEXT NOT NULL,
    model           TEXT NOT NULL,
    system_prompt   TEXT NOT NULL DEFAULT '',
    user_prompt     TEXT NOT NULL DEFAULT '',
    response_body   TEXT NOT NULL DEFAULT '',
    duration_ms     INTEGER NOT NULL DEFAULT 0,
    created_at      TIMESTAMPTZ NOT NULL DEFAULT now()
 );
 CREATE INDEX idx_llm_call_log_job_id ON llm_call_log(job_id);
 CREATE INDEX idx_llm_call_log_user_id ON llm_call_log(user_id, created_at);
 ```
 - [ ] **Step 2: Create `backend/src/db/llm_call_log.rs`**
 ```rust
 //! LLM call logging: tracks every LLM interaction during synthesis generation.
 use chrono::{DateTime, Utc};
 use serde::Serialize;
 use sqlx::PgPool;
 use uuid::Uuid;
 use crate::errors::AppError;
 /// Row returned from llm_call_log queries.
 #[derive(Debug, Clone, Serialize, sqlx::FromRow)]
 pub struct LlmCallLogRow {
    pub id: Uuid,
    pub call_type: String,
    pub model: String,
    pub system_prompt: String,
    pub user_prompt: String,
    pub response_body: String,
    pub duration_ms: i32,
    pub created_at: DateTime<Utc>,
 }
 /// Insert a single LLM call log entry.
 pub async fn insert(
    pool: &PgPool,
    user_id: Uuid,
    job_id: Uuid,
    call_type: &str,
    model: &str,
    system_prompt: &str,
    user_prompt: &str,
    response_body: &str,
    duration_ms: i32,
 ) -> Result<(), AppError> {
    sqlx::query(
        r#"
        INSERT INTO llm_call_log (user_id, job_id, call_type, model, system_prompt, user_prompt, response_body, duration_ms)
        VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
        "#,
    )
    .bind(user_id)
    .bind(job_id)
    .bind(call_type)
    .bind(model)
    .bind(system_prompt)
    .bind(user_prompt)
    .bind(response_body)
    .bind(duration_ms)
    .execute(pool)
    .await?;
    Ok(())
 }
 /// List all LLM call log entries for a generation job.
 pub async fn list_by_job_id(
    pool: &PgPool,
    user_id: Uuid,
    job_id: Uuid,
 ) -> Result<Vec<LlmCallLogRow>, AppError> {
    let rows = sqlx::query_as::<_, LlmCallLogRow>(
        r#"
        SELECT id, call_type, model, system_prompt, user_prompt, response_body, duration_ms, created_at
        FROM llm_call_log
        WHERE user_id = $1 AND job_id = $2
        ORDER BY created_at ASC
        "#,
    )
    .bind(user_id)
    .bind(job_id)
    .fetch_all(pool)
    .await?;
    Ok(rows)
 }
 /// Truncate old LLM call log entries: replace prompt/response with first 500 chars.
 ///
 /// Entries older than `days` get their text fields truncated to save space
 /// while preserving metadata (call_type, model, duration).
 pub async fn truncate_old(
    pool: &PgPool,
    user_id: Uuid,
    days: i32,
 ) -> Result<u64, AppError> {
    let result = sqlx::query(
        r#"
        UPDATE llm_call_log
        SET system_prompt = LEFT(system_prompt, 500) || E'\n[truncated]',
            user_prompt = LEFT(user_prompt, 500) || E'\n[truncated]',
            response_body = LEFT(response_body, 500) || E'\n[truncated]'
        WHERE user_id = $1
            AND created_at < now() - make_interval(days => $2)
            AND LENGTH(system_prompt) > 510
        "#,
    )
    .bind(user_id)
    .bind(days)
    .execute(pool)
    .await?;
    Ok(result.rows_affected())
 }
 ```
 - [ ] **Step 3: Register module + update CLAUDE.md**
 In `backend/src/db/mod.rs`, add `pub mod llm_call_log;` (alphabetical — after `article_history`).
 Update CLAUDE.md migration count to 17.
 - [ ] **Step 4: Verify + commit**
 ```bash
 cd backend && cargo test --lib && cargo build
 git add backend/migrations/20260324000017_create_llm_call_log.sql backend/src/db/llm_call_log.rs backend/src/db/mod.rs CLAUDE.md
 git commit -m "feat: create llm_call_log table + DB module"
 ```
 ---
 ### Task 2: Pipeline instrumentation — log_llm_call helper + wrap all LLM calls
 **Files:**
 - Modify: `backend/src/services/synthesis.rs`
 - [ ] **Step 1: Add `log_llm_call` helper**
 Add near the other helper functions (`trace_article`, etc.):
 ```rust
 /// Log an LLM call with full prompt, response, and timing.
 async fn log_llm_call(
    pool: &sqlx::PgPool,
    user_id: Uuid,
    job_id: Uuid,
    call_type: &str,
    model: &str,
    system_prompt: &str,
    user_prompt: &str,
    response: &serde_json::Value,
    duration_ms: u64,
 ) {
    let response_str = serde_json::to_string_pretty(response).unwrap_or_default();
    db::llm_call_log::insert(
        pool, user_id, job_id, call_type, model,
        system_prompt, user_prompt, &response_str, duration_ms as i32,
    )
    .await
    .ok(); // Don't fail synthesis if logging fails
 }
 ```
 - [ ] **Step 2: Wrap each LLM call with timing + logging**
 Find each LLM provider call in `run_generation_inner` and wrap with timing. Pattern:
 ```rust
 let start = std::time::Instant::now();
 let result = provider.generate_search_pass(...).await?;
 let duration = start.elapsed().as_millis() as u64;
 log_llm_call(&state.pool, user_id, job_id, "search", &model_research,
    &system_prompt, &user_prompt, &result, duration).await;
 ```
 Apply to all LLM call sites:
 1. **Search pass** (Phase 2) — `call_type: "search"`
 2. **Classification Phase 1** — `call_type: "classification_phase1"`
 3. **Classification Phase 2** — `call_type: "classification_phase2"`
 4. **Rewrite pass** — `call_type: "rewrite"`
 For link extraction and article extraction (inside `source_scraper.rs` and `scrape_single_article_with_llm`), logging requires passing `pool`, `user_id`, and `job_id` through. Since these functions are called from spawned tasks, this is complex. For now, skip logging those calls — we can add them later. Focus on the 4 main calls in `run_generation_inner`.
 - [ ] **Step 3: Add truncation to cleanup**
 In `run_generation_inner`, find the existing `article_history::cleanup_old` call at the start of generation. Add LLM log truncation alongside:
 ```rust
    if settings.article_history_days > 0 {
        // ... existing cleanup_old call ...
        // Truncate old LLM call logs
        db::llm_call_log::truncate_old(&state.pool, user_id, settings.article_history_days)
            .await
            .ok();
    }
 ```
 - [ ] **Step 4: Verify + commit**
 ```bash
 cd backend && cargo test --lib && cargo build
 git add backend/src/services/synthesis.rs
 git commit -m "feat: log LLM calls with timing at search, classification, and rewrite steps"
 ```
 ---
 ### Task 3: API endpoint
 **Files:**
 - Create: `backend/src/handlers/llm_logs.rs`
 - Modify: `backend/src/handlers/mod.rs`
 - Modify: `backend/src/router.rs`
 - [ ] **Step 1: Create handler**
 ```rust
 //! Handler for LLM call log viewing.
 use axum::extract::{Path, State};
 use axum::response::IntoResponse;
 use axum::Json;
 use uuid::Uuid;
 use crate::app_state::AppState;
 use crate::db;
 use crate::errors::AppError;
 use crate::middleware::auth::AuthUser;
 /// GET /api/v1/llm-logs/:job_id
 ///
 /// Returns all LLM call log entries for a generation job.
 /// Verifies the job belongs to a synthesis owned by the authenticated user.
 pub async fn get_logs(
    auth_user: AuthUser,
    State(state): State<AppState>,
    Path(job_id): Path<Uuid>,
 ) -> Result<impl IntoResponse, AppError> {
    // Verify the job_id belongs to a synthesis owned by this user
    let synthesis = sqlx::query_scalar::<_, Uuid>(
        "SELECT id FROM syntheses WHERE job_id = $1 AND user_id = $2 LIMIT 1",
    )
    .bind(job_id)
    .bind(auth_user.id)
    .fetch_optional(&state.pool)
    .await?;
    if synthesis.is_none() {
        return Err(AppError::NotFound("No synthesis found for this job".into()));
    }
    let logs = db::llm_call_log::list_by_job_id(&state.pool, auth_user.id, job_id).await?;
    Ok(Json(logs))
 }
 ```
 - [ ] **Step 2: Register + add route**
 In `handlers/mod.rs`, add `pub mod llm_logs;` (alphabetical).
 In `router.rs`, add in the authenticated section:
 ```rust
 .route("/api/v1/llm-logs/:job_id", get(handlers::llm_logs::get_logs))
 ```
 - [ ] **Step 3: Verify + commit**
 ```bash
 cd backend && cargo test --lib && cargo build
 git add backend/src/handlers/llm_logs.rs backend/src/handlers/mod.rs backend/src/router.rs
 git commit -m "feat: API endpoint for LLM call logs by job_id"
 ```
 ---
 ### Task 4: Frontend — types, API client, i18n
 **Files:**
 - Modify: `frontend/src/types.ts`
 - Create: `frontend/src/api/llmLogs.ts`
 - Modify: `frontend/src/i18n/fr.ts`
 - [ ] **Step 1: Add type**
 In `types.ts`:
 ```typescript
 export interface LlmCallLogEntry {
  id: string;
  call_type: string;
  model: string;
  system_prompt: string;
  user_prompt: string;
  response_body: string;
  duration_ms: number;
  created_at: string;
 }
 ```
 - [ ] **Step 2: Create API client**
 ```typescript
 import { api } from './client';
 import type { LlmCallLogEntry } from '~/types';
 export const llmLogsApi = {
  getByJobId: (jobId: string): Promise<LlmCallLogEntry[]> =>
    api.get<LlmCallLogEntry[]>(`/llm-logs/${jobId}`),
 };
 ```
 - [ ] **Step 3: Add i18n labels**
 ```typescript
  // LLM Logs
  'llmLogs.title': 'Logs des appels IA',
  'llmLogs.callType': 'Type',
  'llmLogs.model': 'Modele',
  'llmLogs.duration': 'Duree',
  'llmLogs.systemPrompt': 'Prompt systeme',
  'llmLogs.userPrompt': 'Prompt utilisateur',
  'llmLogs.response': 'Reponse',
  'llmLogs.empty': 'Aucun log disponible pour cette synthese.',
  'llmLogs.viewLogs': 'Voir les logs IA',
  'llmLogs.seconds': 's',
 ```
 - [ ] **Step 4: Commit**
 ```bash
 git add frontend/src/types.ts frontend/src/api/llmLogs.ts frontend/src/i18n/fr.ts
 git commit -m "feat: LLM logs types, API client, and i18n labels"
 ```
 ---
 ### Task 5: Frontend — LlmLogs page + Home button + route
 **Files:**
 - Create: `frontend/src/pages/LlmLogs.tsx`
 - Modify: `frontend/src/App.tsx`
 - Modify: `frontend/src/pages/Home.tsx`
 - [ ] **Step 1: Create LlmLogs page**
 Create `frontend/src/pages/LlmLogs.tsx` — a page that:
 - Extracts `job_id` from the URL params
 - Calls `llmLogsApi.getByJobId(jobId)` on mount
 - Displays each call as a card with:
  - Header: call_type badge (colored by type), model name, duration in seconds
  - Three collapsible sections: System Prompt, User Prompt, Response
  - Text in monospace `<pre>` blocks, scrollable, max-height ~300px
  - Response: attempt `JSON.parse` + `JSON.stringify(parsed, null, 2)` for pretty-printing; fall back to raw text
 - Back link to home
 Use SolidJS patterns consistent with the rest of the app (`createSignal`, `onMount`, `For`, `Show`). Use Tailwind classes.
 - [ ] **Step 2: Add route**
 In `App.tsx`, add:
 ```tsx
 import LlmLogs from './pages/LlmLogs';
 // In routes:
 <Route path="/llm-logs/:jobId" component={LlmLogs} />
 ```
 Inside the authenticated/protected layout.
 - [ ] **Step 3: Add log button on Home page**
 In `Home.tsx`, find where each synthesis is rendered (the list/card). The synthesis data should include `job_id` (from the `SynthesisListItem` type — check if it's already there; if not, it needs to be added).
 First check `frontend/src/types.ts` for `SynthesisListItem` — if `job_id` is not in it, add `job_id: string | null`.
 Then in the synthesis row, next to the delete button, add a small icon/button:
 ```tsx
 <Show when={synthesis.job_id}>
  <A href={`/llm-logs/${synthesis.job_id}`} title={t('llmLogs.viewLogs')}
     class="text-gray-400 hover:text-indigo-600">
    {/* Use a simple text icon or emoji */}
    <span class="text-sm">📋</span>
  </A>
 </Show>
 ```
 Also check the backend — does the `SynthesisListItem` model include `job_id`? If not, add it to the model and the SQL query. Read `backend/src/models/synthesis.rs` for `SynthesisListItem` and `backend/src/handlers/syntheses.rs` for the list handler.
 - [ ] **Step 4: Verify + commit**
 ```bash
 cd frontend && npx tsc --noEmit && npx vitest run
 git add frontend/src/pages/LlmLogs.tsx frontend/src/App.tsx frontend/src/pages/Home.tsx frontend/src/types.ts
 # Also add any backend model changes if needed
 git commit -m "feat: LLM logs viewer page + log button on Home synthesis list"
 ```
 ---
 ### Task 6: E2E test
 **Files:**
 - Modify: `e2e/tests/generation-live.spec.ts`
 - [ ] **Step 1: Add LLM logs verification**
 After the existing provenance verification, add:
 ```typescript
    // ═══════════════════════════════════════════════════════════════
    // LLM call logs verification
    // ═══════════════════════════════════════════════════════════════
    // Get job_id from provenance (used entries have it)
    const jobId = usedEntries[0]?.job_id;
    if (jobId) {
      const logsResp = await apiCall(page, 'GET', `/api/v1/llm-logs/${jobId}`);
      expect(logsResp.status).toBe(200);
      const logs = logsResp.data;
      expect(Array.isArray(logs)).toBe(true);
      expect(logs.length).toBeGreaterThan(0);
      // Should have at least a rewrite call
      const callTypes = logs.map((l: any) => l.call_type);
      expect(callTypes).toContain('rewrite');
      // Each log entry should have prompt and response
      for (const log of logs) {
        expect(log.model).toBeTruthy();
        expect(log.duration_ms).toBeGreaterThanOrEqual(0);
      }
    }
 ```
 - [ ] **Step 2: Commit**
 ```bash
 git add e2e/tests/generation-live.spec.ts
 git commit -m "test: verify LLM call logs endpoint returns data after generation"
 ```