test: add live generation UAT with real OpenAI API key
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>master
parent
6fe75d77e7
commit
02017db2e0
@ -0,0 +1,3 @@
|
||||
# Copy this file to .env.test and fill in your real API key.
|
||||
# This file is gitignored and MUST NOT be committed.
|
||||
OPENAI_TEST_API_KEY=sk-your-key-here
|
||||
@ -0,0 +1,192 @@
|
||||
/**
|
||||
* E2E UAT: Synthesis generation with a real OpenAI API key.
|
||||
*
|
||||
* Exercises the full pipeline: settings, API key encryption, model resolution,
|
||||
* LLM call (search + rewrite), response parsing, and synthesis storage.
|
||||
*
|
||||
* Gated by OPENAI_TEST_API_KEY in e2e/.env.test — skips if not set.
|
||||
* Uses gpt-4o-mini to keep cost under $0.01 per run.
|
||||
*/
|
||||
|
||||
import { test, expect } from '@playwright/test';
|
||||
import * as dotenv from 'dotenv';
|
||||
import * as path from 'path';
|
||||
import { loginAsUser } from '../helpers/auth';
|
||||
|
||||
// Load .env.test from the e2e directory
|
||||
dotenv.config({ path: path.resolve(__dirname, '..', '.env.test') });
|
||||
|
||||
const OPENAI_KEY = process.env.OPENAI_TEST_API_KEY;
|
||||
|
||||
/** Helper to make an authenticated API call from the browser context. */
|
||||
async function apiCall(
|
||||
page: any,
|
||||
method: string,
|
||||
url: string,
|
||||
body?: object,
|
||||
): Promise<{ status: number; data: any }> {
|
||||
return page.evaluate(
|
||||
async ({
|
||||
method,
|
||||
url,
|
||||
body,
|
||||
}: {
|
||||
method: string;
|
||||
url: string;
|
||||
body?: object;
|
||||
}) => {
|
||||
const options: RequestInit = {
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
credentials: 'same-origin',
|
||||
};
|
||||
if (body) {
|
||||
options.body = JSON.stringify(body);
|
||||
}
|
||||
const resp = await fetch(url, options);
|
||||
const data = await resp.json().catch(() => null);
|
||||
return { status: resp.status, data };
|
||||
},
|
||||
{ method, url, body },
|
||||
);
|
||||
}
|
||||
|
||||
/** Wait for SSE generation to complete, return the synthesis_id. */
|
||||
async function waitForGenerationComplete(
|
||||
page: any,
|
||||
jobId: string,
|
||||
timeoutMs = 120_000,
|
||||
): Promise<string> {
|
||||
return page.evaluate(
|
||||
async ({ jobId, timeoutMs }: { jobId: string; timeoutMs: number }) => {
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => reject(new Error('Generation timed out')),
|
||||
timeoutMs,
|
||||
);
|
||||
const es = new EventSource(
|
||||
`/api/v1/syntheses/generate/${jobId}/progress`,
|
||||
);
|
||||
es.addEventListener('complete', (event: MessageEvent) => {
|
||||
clearTimeout(timer);
|
||||
es.close();
|
||||
const parsed = JSON.parse(event.data);
|
||||
resolve(parsed.synthesis_id);
|
||||
});
|
||||
es.addEventListener('error', (event: MessageEvent) => {
|
||||
clearTimeout(timer);
|
||||
es.close();
|
||||
try {
|
||||
const parsed = JSON.parse(event.data);
|
||||
reject(new Error(`Generation failed: ${parsed.message}`));
|
||||
} catch {
|
||||
reject(new Error('Generation failed with unknown error'));
|
||||
}
|
||||
});
|
||||
es.onerror = () => {
|
||||
clearTimeout(timer);
|
||||
es.close();
|
||||
reject(new Error('SSE connection error'));
|
||||
};
|
||||
});
|
||||
},
|
||||
{ jobId, timeoutMs },
|
||||
);
|
||||
}
|
||||
|
||||
test.describe('Live generation with OpenAI', () => {
|
||||
test.skip(!OPENAI_KEY, 'OPENAI_TEST_API_KEY not set in e2e/.env.test');
|
||||
|
||||
test('full generation pipeline produces valid synthesis', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(180_000);
|
||||
|
||||
// Navigate to app first (required for cookie domain), then login
|
||||
// OpenAI provider is already enabled from the migration seed data
|
||||
await page.goto('/');
|
||||
await loginAsUser(page);
|
||||
|
||||
// Step 2: Configure settings
|
||||
const settingsResp = await apiCall(page, 'PUT', '/api/v1/settings', {
|
||||
theme: 'AI Weekly',
|
||||
max_age_days: 7,
|
||||
categories: ['AI News'],
|
||||
max_items_per_category: 5,
|
||||
search_agent_behavior: '',
|
||||
ai_provider: 'openai',
|
||||
ai_model: 'gpt-4o-mini',
|
||||
ai_model_writing: 'gpt-4o-mini',
|
||||
});
|
||||
expect(settingsResp.status).toBe(200);
|
||||
|
||||
// Step 3: Store the real OpenAI API key
|
||||
const keyResp = await apiCall(page, 'POST', '/api/v1/user/api-keys', {
|
||||
provider_name: 'openai',
|
||||
api_key: OPENAI_KEY,
|
||||
});
|
||||
expect(keyResp.status).toBe(200);
|
||||
|
||||
// Step 4: Add a source
|
||||
const sourceResp = await apiCall(page, 'POST', '/api/v1/sources', {
|
||||
title: 'OpenAI Blog',
|
||||
url: 'https://openai.com/blog',
|
||||
});
|
||||
expect(sourceResp.status).toBe(200);
|
||||
|
||||
// Step 5: Trigger generation
|
||||
const genResp = await apiCall(
|
||||
page,
|
||||
'POST',
|
||||
'/api/v1/syntheses/generate',
|
||||
);
|
||||
expect(genResp.status).toBe(202);
|
||||
const jobId = genResp.data.job_id;
|
||||
expect(jobId).toBeTruthy();
|
||||
|
||||
// Step 6: Wait for SSE completion
|
||||
const synthesisId = await waitForGenerationComplete(page, jobId);
|
||||
expect(synthesisId).toBeTruthy();
|
||||
|
||||
// Step 7: Fetch the full synthesis
|
||||
const synthResp = await apiCall(
|
||||
page,
|
||||
'GET',
|
||||
`/api/v1/syntheses/${synthesisId}`,
|
||||
);
|
||||
expect(synthResp.status).toBe(200);
|
||||
const synthesis = synthResp.data;
|
||||
|
||||
// Step 8: Validate structure
|
||||
expect(synthesis.status).toBe('completed');
|
||||
expect(synthesis.sections).toBeDefined();
|
||||
expect(synthesis.sections.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
for (const section of synthesis.sections) {
|
||||
// Section has a title (category name)
|
||||
expect(section.title).toBeTruthy();
|
||||
expect(typeof section.title).toBe('string');
|
||||
|
||||
// Section has items
|
||||
expect(section.items).toBeDefined();
|
||||
expect(section.items.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
for (const item of section.items) {
|
||||
// Each item has a non-empty title
|
||||
expect(item.title).toBeTruthy();
|
||||
expect(typeof item.title).toBe('string');
|
||||
|
||||
// Each item URL starts with http
|
||||
expect(item.url).toBeTruthy();
|
||||
expect(item.url.startsWith('http')).toBe(true);
|
||||
|
||||
// Each item summary is non-trivial (> 50 chars)
|
||||
expect(item.summary).toBeTruthy();
|
||||
expect(item.summary.length).toBeGreaterThan(50);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue