ai_synth/docs/superpowers/plans/2026-03-23-generation-uat.md

Generation UAT with Real OpenAI API — Implementation Plan

For agentic workers: REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (- [ ]) syntax for tracking.

Goal: Add a Playwright E2E test that exercises the full synthesis generation pipeline with a real OpenAI API key.

Architecture: Single Playwright spec that makes API calls via page.evaluate(fetch(...)) in the browser context (sharing the session cookie). Gated by .env.test — skips when no key is available.

Tech Stack: Playwright, dotenv, EventSource (browser API)

Spec: docs/superpowers/specs/2026-03-23-generation-uat-design.md

---

Task 1: Add .env.test to gitignore and create example file

Files:

• Modify: e2e/.gitignore

• Create: e2e/.env.test.example

• Step 1: Add .env.test to e2e/.gitignore

Append to e2e/.gitignore:

.env.test

• Step 2: Create e2e/.env.test.example

# Copy this file to .env.test and fill in your real API key.
# This file is gitignored and MUST NOT be committed.
OPENAI_TEST_API_KEY=sk-your-key-here

• Step 3: Commit

git add e2e/.gitignore e2e/.env.test.example
git commit -m "test: add .env.test template for live generation UAT"

---

Task 2: Install dotenv dependency

Files:

• Modify: e2e/package.json

• Step 1: Install dotenv

cd e2e && npm install --save-dev dotenv

• Step 2: Commit

git add e2e/package.json e2e/package-lock.json
git commit -m "test: add dotenv dependency for E2E test env loading"

---

Task 3: Create the live generation test

Files:

• Create: e2e/tests/generation-live.spec.ts

• Step 1: Create e2e/tests/generation-live.spec.ts

/**
 * E2E UAT: Synthesis generation with a real OpenAI API key.
 *
 * Exercises the full pipeline: settings, API key encryption, model resolution,
 * LLM call (search + rewrite), response parsing, and synthesis storage.
 *
 * Gated by OPENAI_TEST_API_KEY in e2e/.env.test — skips if not set.
 * Uses gpt-4o-mini to keep cost under $0.01 per run.
 */

import { test, expect } from '@playwright/test';
import * as dotenv from 'dotenv';
import * as path from 'path';
import { loginAsUser } from '../helpers/auth';

// Load .env.test from the e2e directory
dotenv.config({ path: path.resolve(__dirname, '..', '.env.test') });

const OPENAI_KEY = process.env.OPENAI_TEST_API_KEY;

/** Helper to make an authenticated API call from the browser context. */
async function apiCall(
  page: any,
  method: string,
  url: string,
  body?: object,
): Promise<{ status: number; data: any }> {
  return page.evaluate(
    async ({
      method,
      url,
      body,
    }: {
      method: string;
      url: string;
      body?: object;
    }) => {
      const options: RequestInit = {
        method,
        headers: {
          'Content-Type': 'application/json',
          'X-Requested-With': 'XMLHttpRequest',
        },
        credentials: 'same-origin',
      };
      if (body) {
        options.body = JSON.stringify(body);
      }
      const resp = await fetch(url, options);
      const data = await resp.json().catch(() => null);
      return { status: resp.status, data };
    },
    { method, url, body },
  );
}

/** Wait for SSE generation to complete, return the synthesis_id. */
async function waitForGenerationComplete(
  page: any,
  jobId: string,
  timeoutMs = 120_000,
): Promise<string> {
  return page.evaluate(
    async ({ jobId, timeoutMs }: { jobId: string; timeoutMs: number }) => {
      return new Promise<string>((resolve, reject) => {
        const timer = setTimeout(
          () => reject(new Error('Generation timed out')),
          timeoutMs,
        );
        const es = new EventSource(
          `/api/v1/syntheses/generate/${jobId}/progress`,
        );
        es.addEventListener('complete', (event: MessageEvent) => {
          clearTimeout(timer);
          es.close();
          const parsed = JSON.parse(event.data);
          resolve(parsed.synthesis_id);
        });
        es.addEventListener('error', (event: MessageEvent) => {
          clearTimeout(timer);
          es.close();
          try {
            const parsed = JSON.parse(event.data);
            reject(new Error(`Generation failed: ${parsed.message}`));
          } catch {
            reject(new Error('Generation failed with unknown error'));
          }
        });
        es.onerror = () => {
          clearTimeout(timer);
          es.close();
          reject(new Error('SSE connection error'));
        };
      });
    },
    { jobId, timeoutMs },
  );
}

test.describe('Live generation with OpenAI', () => {
  test.skip(!OPENAI_KEY, 'OPENAI_TEST_API_KEY not set in e2e/.env.test');

  test('full generation pipeline produces valid synthesis', async ({
    page,
  }) => {
    test.setTimeout(180_000);

    // Navigate to app first (required for cookie domain), then login
    // OpenAI provider is already enabled from the migration seed data
    await page.goto('/');
    await loginAsUser(page);

    // Step 2: Configure settings
    const settingsResp = await apiCall(page, 'PUT', '/api/v1/settings', {
      theme: 'AI Weekly',
      max_age_days: 7,
      categories: ['AI News'],
      max_items_per_category: 5,
      search_agent_behavior: '',
      ai_provider: 'openai',
      ai_model: 'gpt-4o-mini',
      ai_model_writing: 'gpt-4o-mini',
    });
    expect(settingsResp.status).toBe(200);

    // Step 3: Store the real OpenAI API key
    const keyResp = await apiCall(page, 'POST', '/api/v1/user/api-keys', {
      provider_name: 'openai',
      api_key: OPENAI_KEY,
    });
    expect(keyResp.status).toBe(200);

    // Step 4: Add a source
    const sourceResp = await apiCall(page, 'POST', '/api/v1/sources', {
      title: 'OpenAI Blog',
      url: 'https://openai.com/blog',
    });
    expect(sourceResp.status).toBe(200);

    // Step 5: Trigger generation
    const genResp = await apiCall(
      page,
      'POST',
      '/api/v1/syntheses/generate',
    );
    expect(genResp.status).toBe(202);
    const jobId = genResp.data.job_id;
    expect(jobId).toBeTruthy();

    // Step 6: Wait for SSE completion
    const synthesisId = await waitForGenerationComplete(page, jobId);
    expect(synthesisId).toBeTruthy();

    // Step 7: Fetch the full synthesis
    const synthResp = await apiCall(
      page,
      'GET',
      `/api/v1/syntheses/${synthesisId}`,
    );
    expect(synthResp.status).toBe(200);
    const synthesis = synthResp.data;

    // Step 8: Validate structure
    expect(synthesis.status).toBe('completed');
    expect(synthesis.sections).toBeDefined();
    expect(synthesis.sections.length).toBeGreaterThanOrEqual(1);

    for (const section of synthesis.sections) {
      // Section has a title (category name)
      expect(section.title).toBeTruthy();
      expect(typeof section.title).toBe('string');

      // Section has items
      expect(section.items).toBeDefined();
      expect(section.items.length).toBeGreaterThanOrEqual(1);

      for (const item of section.items) {
        // Each item has a non-empty title
        expect(item.title).toBeTruthy();
        expect(typeof item.title).toBe('string');

        // Each item URL starts with http
        expect(item.url).toBeTruthy();
        expect(item.url.startsWith('http')).toBe(true);

        // Each item summary is non-trivial (> 50 chars)
        expect(item.summary).toBeTruthy();
        expect(item.summary.length).toBeGreaterThan(50);
      }
    }
  });
});

• Step 2: Verify the test skips without .env.test

Run (from the e2e directory, with the Docker stack running):

cd e2e && npx playwright test generation-live --reporter=list

Expected: test shows as "skipped" with reason "OPENAI_TEST_API_KEY not set"

• Step 3: Commit

git add e2e/tests/generation-live.spec.ts
git commit -m "test: add live generation UAT with real OpenAI API key"

---

Task 4: Run the full UAT with a real key

This task is manual — requires a real OpenAI API key.

• Step 1: Create e2e/.env.test with your real key

cp e2e/.env.test.example e2e/.env.test
# Edit e2e/.env.test and set OPENAI_TEST_API_KEY=sk-your-real-key

• Step 2: Start the E2E Docker stack

cd e2e && docker compose -f docker-compose.test.yml up --build -d
# Wait for health checks to pass
docker compose -f docker-compose.test.yml ps
# Run seed
npx tsx seed.ts

• Step 3: Run the live generation test

npx playwright test generation-live --reporter=list

Expected: test passes in 60-120s, showing "1 passed".

• Step 4: Verify in Docker logs

docker logs ai-synth-test 2>&1 | grep -E "Generation completed|OpenAI"

Expected: log lines showing successful LLM calls and generation completion.