oabrivard
/
ai_synth
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix rate limiter bug, simplify v2 code

Browse Source 
Bug fix:
- Per-generation rate limiter was creating a new instance on every check,
  making user rate limit overrides non-functional. Fixed by creating the
  limiter once at pipeline start and reusing for both passes.

Simplifications:
- Extract spawn_task closure in scrape_articles (deduplicate spawn blocks)
- Use idiomatic if let Ok(...) instead of if let Some(..).ok() in scraper
- Replace manual loop with iterator chain in export_keys handler
- Simplify check_rate_limit to single boolean check
- Simplify handleImport settings merge (spread already provides defaults)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
master
oabrivard 3 months ago
parent 0f66c28c38
commit 98528f51bd
6 changed files with 300 additions and 68 deletions
Show Stats Download Patch File Download Diff File

7
.claude/settings.local.json
Unescape Escape View File

@ -16,7 +16,12 @@
"Bash(npx vite:*)", "Bash(npx vite:*)",
"Bash(rm -rf src/ node_modules/)", "Bash(rm -rf src/ node_modules/)",
"Bash(rm -f package.json package-lock.json vite.config.ts tsconfig.json index.html firebase-applet-config.json firebase-blueprint.json firestore.rules metadata.json README.md .DS_Store)", "Bash(rm -f package.json package-lock.json vite.config.ts tsconfig.json index.html firebase-applet-config.json firebase-blueprint.json firestore.rules metadata.json README.md .DS_Store)",
"Bash(git status:*)" "Bash(git status:*)",
"Bash(grep -n \"path\\\\|root\\\\|/$\\\\|empty\\\\|url.*valid\\\\|deep\" /Users/oabrivard/Projects/rust/ai_synth/backend/src/services/synthesis.rs)",
"Bash(git:*)",
"Bash(python3 -c \"import sys,json; scripts=json.load\\(sys.stdin\\).get\\(''''scripts'''',{}\\); [print\\(f''''{k}: {v}''''\\) for k,v in scripts.items\\(\\)]\")",
"Bash(npm run:*)",
"Bash(npm test:*)"
], ],
"defaultMode": "bypassPermissions" "defaultMode": "bypassPermissions"
} }

18
backend/src/handlers/api_keys.rs
Unescape Escape View File

@ -184,14 +184,16 @@ pub async fn export_keys(
let stored_keys = db::api_keys::list_for_user(&state.pool, auth_user.id).await?; let stored_keys = db::api_keys::list_for_user(&state.pool, auth_user.id).await?;
let master_key = encryption::MasterKey::from_hex(&state.config.master_encryption_key)?; let master_key = encryption::MasterKey::from_hex(&state.config.master_encryption_key)?;
let mut exported = Vec::new(); let exported: Vec<serde_json::Value> = stored_keys
for key in &stored_keys { .iter()
let decrypted = encryption::decrypt(&master_key, &key.encrypted_key, &key.nonce)?; .map(|key| {
exported.push(serde_json::json!({ let decrypted = encryption::decrypt(&master_key, &key.encrypted_key, &key.nonce)?;
"provider_name": key.provider_name, Ok(serde_json::json!({
"api_key": decrypted, "provider_name": key.provider_name,
})); "api_key": decrypted,
} }))
})
.collect::<Result<_, AppError>>()?;
tracing::info!(user_id = %auth_user.id, key_count = exported.len(), "API keys exported"); tracing::info!(user_id = %auth_user.id, key_count = exported.len(), "API keys exported");
Ok(Json(exported)) Ok(Json(exported))

8
backend/src/services/scraper.rs
Unescape Escape View File

@ -263,7 +263,7 @@ fn is_private_ip(ip: IpAddr) -> bool {
/// Extract the page title using a priority chain: `<title>` -> `og:title` -> `<h1>` -> None. /// Extract the page title using a priority chain: `<title>` -> `og:title` -> `<h1>` -> None.
fn extract_page_title(doc: &Html) -> Option<String> { fn extract_page_title(doc: &Html) -> Option<String> {
// 1. Try <title> element // 1. Try <title> element
if let Some(sel) = Selector::parse("title").ok() { if let Ok(sel) = Selector::parse("title") {
if let Some(title) = doc if let Some(title) = doc
.select(&sel) .select(&sel)
.next() .next()
@ -275,7 +275,7 @@ fn extract_page_title(doc: &Html) -> Option<String> {
} }
// 2. Try <meta property="og:title"> // 2. Try <meta property="og:title">
if let Some(sel) = Selector::parse(r#"meta[property="og:title"]"#).ok() { if let Ok(sel) = Selector::parse(r#"meta[property="og:title"]"#) {
if let Some(content) = doc if let Some(content) = doc
.select(&sel) .select(&sel)
.next() .next()
@ -288,7 +288,7 @@ fn extract_page_title(doc: &Html) -> Option<String> {
} }
// 3. Try first <h1> // 3. Try first <h1>
if let Some(sel) = Selector::parse("h1").ok() { if let Ok(sel) = Selector::parse("h1") {
if let Some(h1) = doc if let Some(h1) = doc
.select(&sel) .select(&sel)
.next() .next()
@ -439,7 +439,7 @@ fn is_error_path(path: &str) -> bool {
/// 6. `<time datetime="...">` /// 6. `<time datetime="...">`
fn extract_publication_date(doc: &Html) -> Option<DateTime<Utc>> { fn extract_publication_date(doc: &Html) -> Option<DateTime<Utc>> {
// 1. JSON-LD // 1. JSON-LD
if let Some(sel) = Selector::parse(r#"script[type="application/ld+json"]"#).ok() { if let Ok(sel) = Selector::parse(r#"script[type="application/ld+json"]"#) {
for el in doc.select(&sel) { for el in doc.select(&sel) {
let text = el.text().collect::<String>(); let text = el.text().collect::<String>();
if let Ok(json) = serde_json::from_str::<serde_json::Value>(&text) { if let Ok(json) = serde_json::from_str::<serde_json::Value>(&text) {

91
backend/src/services/synthesis.rs
Unescape Escape View File

@ -289,9 +289,12 @@ async fn run_generation_inner(
model_research.clone() model_research.clone()
}; };
// Build per-user rate limiter if the user has overrides configured.
// Created once and reused across both passes so the limit is actually enforced.
let user_rate_limiter = build_user_rate_limiter(&settings);
// Step 5: Rate limit check (pass 1) // Step 5: Rate limit check (pass 1)
// User overrides take priority over global rate limiter check_rate_limit(state, &user_rate_limiter, &provider_name)?;
check_rate_limit(state, &settings, &provider_name)?;
// Step 6: LLM search pass // Step 6: LLM search pass
emit_progress(tx, "search", "Recherche d'actualites en cours...", 30); emit_progress(tx, "search", "Recherche d'actualites en cours...", 30);
@ -335,7 +338,7 @@ async fn run_generation_inner(
let scraped = scrape_articles(state, &parsed, settings.max_age_days as i64, tx).await; let scraped = scrape_articles(state, &parsed, settings.max_age_days as i64, tx).await;
// Rate limit check (pass 2) // Rate limit check (pass 2)
check_rate_limit(state, &settings, &provider_name)?; check_rate_limit(state, &user_rate_limiter, &provider_name)?;
// LLM rewrite pass // LLM rewrite pass
emit_progress(tx, "rewrite", "Redaction des resumes...", 80); emit_progress(tx, "rewrite", "Redaction des resumes...", 80);
@ -376,44 +379,45 @@ fn emit_progress(tx: &watch::Sender<ProgressEvent>, step: &str, message: &str, p
.ok(); .ok();
} }
/// Check rate limits, using user overrides if configured, otherwise the global limiter. /// Build a per-user rate limiter from settings, if both override fields are configured.
/// ///
/// When the user has both `rate_limit_max_requests` and `rate_limit_time_window_seconds` /// Returns `None` if the user has no rate limit overrides, in which case the
/// set, a temporary per-user rate limiter is created with those values. Otherwise the /// global provider rate limiter should be used instead.
/// global provider rate limiter is used. fn build_user_rate_limiter(
fn check_rate_limit(
state: &AppState,
settings: &UserSettings, settings: &UserSettings,
provider_name: &str, ) -> Option<crate::services::rate_limiter::RateLimiter> {
) -> Result<(), AppError> {
match ( match (
settings.rate_limit_max_requests, settings.rate_limit_max_requests,
settings.rate_limit_time_window_seconds, settings.rate_limit_time_window_seconds,
) { ) {
(Some(max_req), Some(window_sec)) => { (Some(max_req), Some(window_sec)) => {
// Create a temporary rate limiter with user's config Some(crate::services::rate_limiter::RateLimiter::new(
let user_limiter = crate::services::rate_limiter::RateLimiter::new(
max_req as usize, max_req as usize,
Duration::from_secs(window_sec as u64), Duration::from_secs(window_sec as u64),
); ))
let key = format!("user_gen_{}", provider_name);
if !user_limiter.check(&key) {
return Err(AppError::RateLimited(
"Limite de requetes personnalisee atteinte. Veuillez reessayer dans quelques instants.".into(),
));
}
Ok(())
}
_ => {
if !state.provider_rate_limiter.check(provider_name) {
return Err(AppError::RateLimited(
"Limite de requetes atteinte. Veuillez reessayer dans quelques instants."
.into(),
));
}
Ok(())
} }
_ => None,
}
}
/// Check rate limits using the user's limiter if provided, otherwise the global limiter.
fn check_rate_limit(
state: &AppState,
user_limiter: &Option<crate::services::rate_limiter::RateLimiter>,
provider_name: &str,
) -> Result<(), AppError> {
let allowed = match user_limiter {
Some(limiter) => limiter.check(&format!("user_gen_{}", provider_name)),
None => state.provider_rate_limiter.check(provider_name),
};
if !allowed {
return Err(AppError::RateLimited(
"Limite de requetes atteinte. Veuillez reessayer dans quelques instants.".into(),
));
} }
Ok(())
} }
/// Filter out articles whose URL is a homepage (path is "/" or empty). /// Filter out articles whose URL is a homepage (path is "/" or empty).
@ -601,10 +605,8 @@ async fn scrape_articles(
let mut pending = tasks.into_iter().peekable(); let mut pending = tasks.into_iter().peekable();
let mut completed = 0usize; let mut completed = 0usize;
// Seed the JoinSet with up to 10 initial tasks let spawn_task =
let max_concurrent = 10; |join_set: &mut tokio::task::JoinSet<_>, cat_key: String, item: NewsItem| {
for _ in 0..max_concurrent {
if let Some((cat_key, item)) = pending.next() {
let client = state.http_client.clone(); let client = state.http_client.clone();
let url = item.url.clone(); let url = item.url.clone();
let mad = max_age_days; let mad = max_age_days;
@ -612,6 +614,13 @@ async fn scrape_articles(
let scraped = scrape_single_article(&client, &url, mad).await; let scraped = scrape_single_article(&client, &url, mad).await;
(cat_key, item, scraped) (cat_key, item, scraped)
}); });
};
// Seed the JoinSet with up to 10 initial tasks
let max_concurrent = 10;
for _ in 0..max_concurrent {
if let Some((cat_key, item)) = pending.next() {
spawn_task(&mut join_set, cat_key, item);
} }
} }
@ -645,13 +654,7 @@ async fn scrape_articles(
// Spawn next task if available // Spawn next task if available
if let Some((cat_key, item)) = pending.next() { if let Some((cat_key, item)) = pending.next() {
let client = state.http_client.clone(); spawn_task(&mut join_set, cat_key, item);
let url = item.url.clone();
let mad = max_age_days;
join_set.spawn(async move {
let scraped = scrape_single_article(&client, &url, mad).await;
(cat_key, item, scraped)
});
} }
} }
@ -780,13 +783,11 @@ fn sanitize_error_message(msg: &str) -> String {
} }
// For other errors, truncate and sanitize // For other errors, truncate and sanitize
let truncated = if msg.len() > 200 { if msg.len() > 200 {
format!("{}...", &msg[..200]) format!("{}...", &msg[..200])
} else { } else {
msg.to_string() msg.to_string()
}; }
truncated
} }
#[cfg(test)] #[cfg(test)]

230
docs/functional-spec-v2.md
Unescape Escape View File

@ -0,0 +1,230 @@
# Functional & UX Specification: ai-weekly-synth v1 → v2
This document describes the v1-to-v2 changes from a user and product perspective. It covers what changes, why, and how the user experiences it — without referencing implementation details.
---
## 1. Data model: dynamic sections replace fixed categories
### What changes
v1 stores each synthesis using 5 hardcoded category slots (major announcements, financial sector, other enterprises, public sector, general public). v2 replaces these with a single ordered list of sections, each with a user-defined title.
### Why
The fixed categories constrained the application to a single topic structure. With dynamic sections, the synthesis adapts to whatever categories the user has configured in Settings — whether that is 2 categories or 15, and regardless of their names.
### User-visible impact
- New syntheses are generated using only the dynamic sections structure.
- Old syntheses that were stored with the fixed category fields will no longer display their content. Instead, a message "Aucune section trouvée dans cette synthèse." appears in the detail view and in emails. This is a **breaking change** for legacy data.
- The home page card preview for legacy syntheses will show "Aucune annonce majeure cette semaine." instead of the first category's items.
---
## 2. Home page
### Title wording
The page heading changes from "Synthèses d'Actualités **IA**" to "Synthèses d'Actualités **par IA**" — a minor phrasing adjustment that clarifies the syntheses are made *by* AI, not about AI (the topic is configurable).
---
## 3. Settings page
The Settings page gains significant new functionality. The existing controls (theme, max age, max items per category, categories list) remain unchanged. All additions are described below in the order they appear on the page.
### 3.1 Header: Export / Import buttons
Two new buttons appear in the page header, to the right of the title:
| Button | Icon | Behavior |
|--------|------|----------|
| **Exporter** | Download arrow | Downloads the current settings as a `settings.json` file |
| **Importer** | Upload arrow | Opens a file picker (`.json` only). Loads the selected file's contents into the form. A success banner reminds the user to click "Enregistrer" to persist the imported settings. Invalid JSON shows an error banner. |
Import merges the uploaded values over the defaults, so missing fields fall back to their default values.
### 3.2 Search agent behavior (new field)
| Property | Value |
|----------|-------|
| **Label** | Comportement de l'agent de recherche |
| **Control** | Multi-line text area (3 rows) |
| **Default** | "Tu peux également utiliser d'autres sources pertinentes trouvées via la recherche Google." |
| **Help text** | Personnalisez les instructions données à l'IA concernant sa méthode de recherche. |
This lets the user customize the instructions injected into the AI research prompt — for example, restricting the AI to only the user's custom sources, or asking it to prioritize certain types of sites.
### 3.3 AI model for research (new field)
| Property | Value |
|----------|-------|
| **Label** | Modèle d'IA (Recherche et Extraction) |
| **Control** | Dropdown |
| **Default** | Gemini 3.1 Pro |
| **Help text** | Choisissez le modèle d'IA utilisé pour rechercher et extraire les informations. Le modèle Pro est conseillé pour des résultats plus pertinents et une meilleure analyse. |
Available options:
| Display label | Model ID |
|---------------|----------|
| Gemini 3.1 Pro (conseillé) | gemini-3.1-pro-preview |
| Gemini 3.0 Flash | gemini-3-flash-preview |
| Gemini 3.1 Flash Lite | gemini-3.1-flash-lite-preview |
| Gemini 2.5 Flash | gemini-2.5-flash |
This model is used for Stage 1 of synthesis generation (searching and collecting news articles).
### 3.4 AI model for writing (new field)
| Property | Value |
|----------|-------|
| **Label** | Modèle d'IA (Rédaction et Synthèse) |
| **Control** | Dropdown |
| **Default** | Gemini 3.1 Pro |
| **Help text** | Choisissez le modèle d'IA utilisé pour le second agent, chargé de rédiger et structurer la synthèse finale. |
Same 4 model options as above. This model is used for Stage 2 (rewriting titles and summaries from scraped content).
Users can mix models — for example, use a faster/cheaper model for research and a higher-quality model for final writing, or vice versa.
### 3.5 Rate limiting (new section)
A new section titled **"Limitation de taux (Rate Limiting)"** appears below the categories list, visually separated by a horizontal rule.
| Field | Label | Control | Constraints | Default |
|-------|-------|---------|-------------|---------|
| Max requests | Requêtes maximum | Number input | min: 1 | 29 |
| Time window | Fenêtre de temps (ms) | Number input | min: 1000, step: 1000 | 60000 |
**Help text:** Configurez le nombre maximum de requêtes autorisées vers l'API Gemini pendant la fenêtre de temps spécifiée (en millisecondes). Par défaut : 29 requêtes par minute (60000 ms).
This gives users control over API throttling — useful for users on free-tier API plans with stricter quotas, or users who want to increase throughput on paid plans.
### 3.6 Danger zone (new section)
A red-themed section titled **"Zone de danger"** (with a warning triangle icon) appears at the bottom of the form, separated by a red horizontal rule.
It contains a single action:
**Nettoyer les données**
- Description: "Cette action supprimera définitivement toutes vos synthèses générées et tous vos articles/sources enregistrés. Vos paramètres de configuration seront conservés."
- A "Supprimer les données" button initiates the flow.
- Clicking it reveals a two-button confirmation state: "Confirmer la suppression" and "Annuler".
- During deletion, the confirm button shows a loading state ("Suppression...") and both buttons are disabled.
- On success, a green banner confirms the deletion. On failure, an error banner appears.
- Only the current user's data is affected. Settings are preserved.
---
## 4. Sources page
### 4.1 CSV Import / Export (new section)
A new card titled **"Import / Export CSV"** is inserted between the existing "Ajouter une source" form and the "Import en masse" text area.
**Description:** "Sauvegardez vos sources ou importez-en de nouvelles depuis un fichier CSV."
Two buttons appear side by side:
| Button | Icon | Behavior |
|--------|------|----------|
| **Exporter en CSV** | Download arrow | Downloads all current sources as a `sources.csv` file with a `Titre,URL` header row. Values are quoted and double-quotes are escaped. |
| **Importer depuis un CSV** | Upload arrow | Opens a file picker (`.csv` only). Parses the file and bulk-creates all valid sources. |
**CSV import behavior:**
- Accepts both comma (`,`) and semicolon (`;`) as delimiters (auto-detected per line).
- Automatically skips the first row if it looks like a header (contains "titre" or "title", case-insensitive).
- Handles quoted fields and escaped double-quotes (`""`).
- Automatically prepends `https://` to URLs missing a protocol.
- If no valid sources are found, an error message is shown below the buttons.
- Importing is additive — existing sources are not deleted.
The existing "Import en masse" text area (semicolon-separated, one per line) remains available below the CSV section as an alternative bulk import method.
---
## 5. Synthesis generation quality improvements
These changes affect the output quality of generated syntheses. They are not directly visible as UI controls but improve the user's experience when reading syntheses.
### 5.1 Deep-link URL enforcement
**Problem in v1:** The AI sometimes returned homepage URLs (e.g., `https://lemonde.fr`) instead of direct article links.
**v2 behavior:**
- The AI prompt now explicitly instructs the model to always provide deep-link URLs pointing to specific articles, and never homepage/root URLs.
- The system instruction reinforces this requirement.
- During validation, any URL whose path is `/` or empty is automatically rejected before content is scraped.
### 5.2 Enhanced broken link detection
**Problem in v1:** Some URLs returned by the AI were broken (soft 404 pages that return HTTP 200). v1 only detected errors by checking the page title and H1 for keywords like "404" or "not found".
**v2 adds three additional detection layers:**
1. **Redirect/canonical URL analysis** — If the page was redirected (via the proxy's final URL), or has a canonical or og:url meta tag that points to a known error path (`/404`, `/404.html`, `/error`), the article is rejected.
2. **Short-page body text analysis** — For pages with less than 1500 characters of body text (typically error pages), the full body text is checked for error phrases in English and French ("page not found", "page introuvable", "could not be found", "the requested page", or combinations of "404" with "not found"/"error").
3. **The existing detection** (title/H1 keyword matching, `noindex` meta tag) remains in place.
The combined effect is significantly fewer broken links in the final synthesis.
### 5.3 Original title preservation
**Problem in v1:** Article titles in the synthesis were generated entirely by the AI based on scraped content, which could result in mistranslated or inaccurate titles.
**v2 behavior:**
- During scraping, the original title is extracted from each article page. The extraction uses the following priority: `og:title` meta tag, then the first `<h1>` element, then the document `<title>`, and finally the AI-provided title as a fallback.
- The original title is passed to the writing AI along with the scraped content.
- The writing AI is instructed to use the original title as a base, with these language rules:
- English titles must be kept in English (not translated).
- French titles must be kept in French.
- Titles in other languages must be translated into French.
### 5.4 Longer summaries
Article summaries increase from 3-4 lines (v1) to 4-5 lines (v2), providing slightly more detailed coverage of each article.
### 5.5 Configurable rate limiting at runtime
The rate limiter (which throttles requests to the Gemini API during synthesis generation) now reads its configuration from the user's settings at the start of each generation. In v1, these values were hardcoded at 29 requests per 60 seconds.
---
## 6. Synthesis detail page
### 6.1 Legacy data handling
When viewing a synthesis that has no `sections` data (i.e., a v1-era synthesis stored with the old fixed-category fields):
| Area | v1 behavior | v2 behavior |
|------|-------------|-------------|
| **Page rendering** | Falls back to rendering the 5 hardcoded category sections | Shows an italic centered message: "Aucune section trouvée dans cette synthèse." |
| **Email body** | Falls back to listing the 5 hardcoded categories | Includes the text: "Aucune section trouvée dans cette synthèse." |
This is an intentional simplification. The old hardcoded field names are no longer referenced anywhere in v2.
---
## 7. Firestore security rules
The settings validation rules are updated to accept the new optional fields (`aiModel`, `secondaryAiModel`, `rateLimitMaxRequests`, `rateLimitTimeWindowMs`). All four are optional — existing settings documents without them remain valid and writable.
- `aiModel` and `secondaryAiModel` must be strings if present.
- `rateLimitMaxRequests` and `rateLimitTimeWindowMs` must be positive numbers if present.
---
## Summary of changes by page
| Page | Changes |
|------|---------|
| **Home** | Title wording ("IA" → "par IA"). Legacy synthesis cards show empty preview instead of old category data. |
| **Settings** | New: Export/Import buttons, search agent behavior textarea, 2 AI model dropdowns, rate limiting controls, danger zone with data clearing. |
| **Sources** | New: CSV export button, CSV import via file picker. Existing bulk text import unchanged. |
| **Synthesis Detail** | Legacy syntheses show "no sections found" instead of the 5 hardcoded categories. |
| **Generate Synthesis** | No UI changes. Backend improvements: better URL quality, original title preservation, longer summaries, configurable models. |
| **Login** | No changes. |

14
frontend/src/pages/Settings.tsx
Unescape Escape View File

@ -191,18 +191,12 @@ const Settings: Component = () => {
const text = await file.text(); const text = await file.text();
const data = JSON.parse(text); const data = JSON.parse(text);
// Merge over DEFAULT_SETTINGS for missing fields // Merge imported data over DEFAULT_SETTINGS so missing fields get defaults.
// The spread provides defaults; explicit overrides handle the `categories`
// field which must be validated as an array.
const merged: UserSettings = { const merged: UserSettings = {
...DEFAULT_SETTINGS, ...DEFAULT_SETTINGS,
theme: data.theme ?? DEFAULT_SETTINGS.theme, ...data,
max_age_days: data.max_age_days ?? DEFAULT_SETTINGS.max_age_days,
max_items_per_category: data.max_items_per_category ?? DEFAULT_SETTINGS.max_items_per_category,
search_agent_behavior: data.search_agent_behavior ?? DEFAULT_SETTINGS.search_agent_behavior,
ai_model: data.ai_model ?? DEFAULT_SETTINGS.ai_model,
ai_model_writing: data.ai_model_writing ?? DEFAULT_SETTINGS.ai_model_writing,
ai_provider: data.ai_provider ?? DEFAULT_SETTINGS.ai_provider,
rate_limit_max_requests: data.rate_limit_max_requests ?? DEFAULT_SETTINGS.rate_limit_max_requests,
rate_limit_time_window_seconds: data.rate_limit_time_window_seconds ?? DEFAULT_SETTINGS.rate_limit_time_window_seconds,
categories: Array.isArray(data.categories) ? data.categories : DEFAULT_SETTINGS.categories, categories: Array.isArray(data.categories) ? data.categories : DEFAULT_SETTINGS.categories,
}; };

Write Preview
Loading…
Cancel
Save