test: add live generation UAT with real OpenAI API key
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>master
parent
6fe75d77e7
commit
02017db2e0
@ -0,0 +1,3 @@
|
|||||||
|
# Copy this file to .env.test and fill in your real API key.
|
||||||
|
# This file is gitignored and MUST NOT be committed.
|
||||||
|
OPENAI_TEST_API_KEY=sk-your-key-here
|
||||||
@ -0,0 +1,192 @@
|
|||||||
|
/**
|
||||||
|
* E2E UAT: Synthesis generation with a real OpenAI API key.
|
||||||
|
*
|
||||||
|
* Exercises the full pipeline: settings, API key encryption, model resolution,
|
||||||
|
* LLM call (search + rewrite), response parsing, and synthesis storage.
|
||||||
|
*
|
||||||
|
* Gated by OPENAI_TEST_API_KEY in e2e/.env.test — skips if not set.
|
||||||
|
* Uses gpt-4o-mini to keep cost under $0.01 per run.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { test, expect } from '@playwright/test';
|
||||||
|
import * as dotenv from 'dotenv';
|
||||||
|
import * as path from 'path';
|
||||||
|
import { loginAsUser } from '../helpers/auth';
|
||||||
|
|
||||||
|
// Load .env.test from the e2e directory
|
||||||
|
dotenv.config({ path: path.resolve(__dirname, '..', '.env.test') });
|
||||||
|
|
||||||
|
const OPENAI_KEY = process.env.OPENAI_TEST_API_KEY;
|
||||||
|
|
||||||
|
/** Helper to make an authenticated API call from the browser context. */
|
||||||
|
async function apiCall(
|
||||||
|
page: any,
|
||||||
|
method: string,
|
||||||
|
url: string,
|
||||||
|
body?: object,
|
||||||
|
): Promise<{ status: number; data: any }> {
|
||||||
|
return page.evaluate(
|
||||||
|
async ({
|
||||||
|
method,
|
||||||
|
url,
|
||||||
|
body,
|
||||||
|
}: {
|
||||||
|
method: string;
|
||||||
|
url: string;
|
||||||
|
body?: object;
|
||||||
|
}) => {
|
||||||
|
const options: RequestInit = {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'X-Requested-With': 'XMLHttpRequest',
|
||||||
|
},
|
||||||
|
credentials: 'same-origin',
|
||||||
|
};
|
||||||
|
if (body) {
|
||||||
|
options.body = JSON.stringify(body);
|
||||||
|
}
|
||||||
|
const resp = await fetch(url, options);
|
||||||
|
const data = await resp.json().catch(() => null);
|
||||||
|
return { status: resp.status, data };
|
||||||
|
},
|
||||||
|
{ method, url, body },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Wait for SSE generation to complete, return the synthesis_id. */
|
||||||
|
async function waitForGenerationComplete(
|
||||||
|
page: any,
|
||||||
|
jobId: string,
|
||||||
|
timeoutMs = 120_000,
|
||||||
|
): Promise<string> {
|
||||||
|
return page.evaluate(
|
||||||
|
async ({ jobId, timeoutMs }: { jobId: string; timeoutMs: number }) => {
|
||||||
|
return new Promise<string>((resolve, reject) => {
|
||||||
|
const timer = setTimeout(
|
||||||
|
() => reject(new Error('Generation timed out')),
|
||||||
|
timeoutMs,
|
||||||
|
);
|
||||||
|
const es = new EventSource(
|
||||||
|
`/api/v1/syntheses/generate/${jobId}/progress`,
|
||||||
|
);
|
||||||
|
es.addEventListener('complete', (event: MessageEvent) => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
es.close();
|
||||||
|
const parsed = JSON.parse(event.data);
|
||||||
|
resolve(parsed.synthesis_id);
|
||||||
|
});
|
||||||
|
es.addEventListener('error', (event: MessageEvent) => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
es.close();
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(event.data);
|
||||||
|
reject(new Error(`Generation failed: ${parsed.message}`));
|
||||||
|
} catch {
|
||||||
|
reject(new Error('Generation failed with unknown error'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
es.onerror = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
es.close();
|
||||||
|
reject(new Error('SSE connection error'));
|
||||||
|
};
|
||||||
|
});
|
||||||
|
},
|
||||||
|
{ jobId, timeoutMs },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
test.describe('Live generation with OpenAI', () => {
|
||||||
|
test.skip(!OPENAI_KEY, 'OPENAI_TEST_API_KEY not set in e2e/.env.test');
|
||||||
|
|
||||||
|
test('full generation pipeline produces valid synthesis', async ({
|
||||||
|
page,
|
||||||
|
}) => {
|
||||||
|
test.setTimeout(180_000);
|
||||||
|
|
||||||
|
// Navigate to app first (required for cookie domain), then login
|
||||||
|
// OpenAI provider is already enabled from the migration seed data
|
||||||
|
await page.goto('/');
|
||||||
|
await loginAsUser(page);
|
||||||
|
|
||||||
|
// Step 2: Configure settings
|
||||||
|
const settingsResp = await apiCall(page, 'PUT', '/api/v1/settings', {
|
||||||
|
theme: 'AI Weekly',
|
||||||
|
max_age_days: 7,
|
||||||
|
categories: ['AI News'],
|
||||||
|
max_items_per_category: 5,
|
||||||
|
search_agent_behavior: '',
|
||||||
|
ai_provider: 'openai',
|
||||||
|
ai_model: 'gpt-4o-mini',
|
||||||
|
ai_model_writing: 'gpt-4o-mini',
|
||||||
|
});
|
||||||
|
expect(settingsResp.status).toBe(200);
|
||||||
|
|
||||||
|
// Step 3: Store the real OpenAI API key
|
||||||
|
const keyResp = await apiCall(page, 'POST', '/api/v1/user/api-keys', {
|
||||||
|
provider_name: 'openai',
|
||||||
|
api_key: OPENAI_KEY,
|
||||||
|
});
|
||||||
|
expect(keyResp.status).toBe(200);
|
||||||
|
|
||||||
|
// Step 4: Add a source
|
||||||
|
const sourceResp = await apiCall(page, 'POST', '/api/v1/sources', {
|
||||||
|
title: 'OpenAI Blog',
|
||||||
|
url: 'https://openai.com/blog',
|
||||||
|
});
|
||||||
|
expect(sourceResp.status).toBe(200);
|
||||||
|
|
||||||
|
// Step 5: Trigger generation
|
||||||
|
const genResp = await apiCall(
|
||||||
|
page,
|
||||||
|
'POST',
|
||||||
|
'/api/v1/syntheses/generate',
|
||||||
|
);
|
||||||
|
expect(genResp.status).toBe(202);
|
||||||
|
const jobId = genResp.data.job_id;
|
||||||
|
expect(jobId).toBeTruthy();
|
||||||
|
|
||||||
|
// Step 6: Wait for SSE completion
|
||||||
|
const synthesisId = await waitForGenerationComplete(page, jobId);
|
||||||
|
expect(synthesisId).toBeTruthy();
|
||||||
|
|
||||||
|
// Step 7: Fetch the full synthesis
|
||||||
|
const synthResp = await apiCall(
|
||||||
|
page,
|
||||||
|
'GET',
|
||||||
|
`/api/v1/syntheses/${synthesisId}`,
|
||||||
|
);
|
||||||
|
expect(synthResp.status).toBe(200);
|
||||||
|
const synthesis = synthResp.data;
|
||||||
|
|
||||||
|
// Step 8: Validate structure
|
||||||
|
expect(synthesis.status).toBe('completed');
|
||||||
|
expect(synthesis.sections).toBeDefined();
|
||||||
|
expect(synthesis.sections.length).toBeGreaterThanOrEqual(1);
|
||||||
|
|
||||||
|
for (const section of synthesis.sections) {
|
||||||
|
// Section has a title (category name)
|
||||||
|
expect(section.title).toBeTruthy();
|
||||||
|
expect(typeof section.title).toBe('string');
|
||||||
|
|
||||||
|
// Section has items
|
||||||
|
expect(section.items).toBeDefined();
|
||||||
|
expect(section.items.length).toBeGreaterThanOrEqual(1);
|
||||||
|
|
||||||
|
for (const item of section.items) {
|
||||||
|
// Each item has a non-empty title
|
||||||
|
expect(item.title).toBeTruthy();
|
||||||
|
expect(typeof item.title).toBe('string');
|
||||||
|
|
||||||
|
// Each item URL starts with http
|
||||||
|
expect(item.url).toBeTruthy();
|
||||||
|
expect(item.url.startsWith('http')).toBe(true);
|
||||||
|
|
||||||
|
// Each item summary is non-trivial (> 50 chars)
|
||||||
|
expect(item.summary).toBeTruthy();
|
||||||
|
expect(item.summary.length).toBeGreaterThan(50);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Reference in New Issue