diff --git a/CLAUDE.md b/CLAUDE.md index 01ac14a..2fbd1ad 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,7 +117,7 @@ cd frontend && npx tsc --noEmit - `GET /api/v1/admin/users` — user list - `PUT /api/v1/admin/users/:id/role` — role management -## Database (25 migrations) +## Database (26 migrations) Tables: `users`, `sessions`, `magic_link_tokens`, `user_settings`, `sources`, `syntheses`, `admin_providers`, `admin_rate_limits`, `user_api_keys`, `audit_log` ## Environment Variables diff --git a/backend/migrations/20260326000026_remove_use_llm_for_source_links.sql b/backend/migrations/20260326000026_remove_use_llm_for_source_links.sql new file mode 100644 index 0000000..d166d7c --- /dev/null +++ b/backend/migrations/20260326000026_remove_use_llm_for_source_links.sql @@ -0,0 +1 @@ +ALTER TABLE settings DROP COLUMN use_llm_for_source_links; diff --git a/backend/src/db/settings.rs b/backend/src/db/settings.rs index db3efee..2d84da9 100644 --- a/backend/src/db/settings.rs +++ b/backend/src/db/settings.rs @@ -18,7 +18,6 @@ struct SettingsRow { categories: serde_json::Value, max_items_per_category: i32, max_articles_per_source: i32, - use_llm_for_source_links: bool, use_brave_search: bool, article_history_days: i32, batch_size: i32, @@ -48,7 +47,6 @@ impl TryFrom for UserSettings { categories, max_items_per_category: row.max_items_per_category, max_articles_per_source: row.max_articles_per_source, - use_llm_for_source_links: row.use_llm_for_source_links, use_brave_search: row.use_brave_search, article_history_days: row.article_history_days, batch_size: row.batch_size, @@ -80,10 +78,10 @@ pub async fn get_or_create_default( let row = sqlx::query_as::<_, SettingsRow>( r#" - INSERT INTO settings (user_id, theme, max_age_days, categories, max_items_per_category, search_agent_behavior, ai_provider, ai_model, ai_model_websearch, rate_limit_max_requests, rate_limit_time_window_seconds, max_articles_per_source, use_llm_for_source_links, use_brave_search, article_history_days, batch_size, summary_length, source_extraction_window) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + INSERT INTO settings (user_id, theme, max_age_days, categories, max_items_per_category, search_agent_behavior, ai_provider, ai_model, ai_model_websearch, rate_limit_max_requests, rate_limit_time_window_seconds, max_articles_per_source, use_brave_search, article_history_days, batch_size, summary_length, source_extraction_window) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT (user_id) DO UPDATE SET user_id = settings.user_id - RETURNING user_id, theme, max_age_days, categories, max_items_per_category, search_agent_behavior, ai_provider, ai_model, ai_model_websearch, rate_limit_max_requests, rate_limit_time_window_seconds, max_articles_per_source, use_llm_for_source_links, use_brave_search, article_history_days, batch_size, summary_length, source_extraction_window, updated_at + RETURNING user_id, theme, max_age_days, categories, max_items_per_category, search_agent_behavior, ai_provider, ai_model, ai_model_websearch, rate_limit_max_requests, rate_limit_time_window_seconds, max_articles_per_source, use_brave_search, article_history_days, batch_size, summary_length, source_extraction_window, updated_at "#, ) .bind(user_id) @@ -98,7 +96,6 @@ pub async fn get_or_create_default( .bind(defaults.rate_limit_max_requests) .bind(defaults.rate_limit_time_window_seconds) .bind(defaults.max_articles_per_source) - .bind(defaults.use_llm_for_source_links) .bind(defaults.use_brave_search) .bind(defaults.article_history_days) .bind(defaults.batch_size) @@ -122,8 +119,8 @@ pub async fn upsert( let row = sqlx::query_as::<_, SettingsRow>( r#" - INSERT INTO settings (user_id, theme, max_age_days, categories, max_items_per_category, search_agent_behavior, ai_provider, ai_model, ai_model_websearch, rate_limit_max_requests, rate_limit_time_window_seconds, max_articles_per_source, use_llm_for_source_links, use_brave_search, article_history_days, batch_size, summary_length, source_extraction_window) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + INSERT INTO settings (user_id, theme, max_age_days, categories, max_items_per_category, search_agent_behavior, ai_provider, ai_model, ai_model_websearch, rate_limit_max_requests, rate_limit_time_window_seconds, max_articles_per_source, use_brave_search, article_history_days, batch_size, summary_length, source_extraction_window) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT (user_id) DO UPDATE SET theme = EXCLUDED.theme, max_age_days = EXCLUDED.max_age_days, @@ -136,14 +133,13 @@ pub async fn upsert( rate_limit_max_requests = EXCLUDED.rate_limit_max_requests, rate_limit_time_window_seconds = EXCLUDED.rate_limit_time_window_seconds, max_articles_per_source = EXCLUDED.max_articles_per_source, - use_llm_for_source_links = EXCLUDED.use_llm_for_source_links, use_brave_search = EXCLUDED.use_brave_search, article_history_days = EXCLUDED.article_history_days, batch_size = EXCLUDED.batch_size, summary_length = EXCLUDED.summary_length, source_extraction_window = EXCLUDED.source_extraction_window, updated_at = now() - RETURNING user_id, theme, max_age_days, categories, max_items_per_category, search_agent_behavior, ai_provider, ai_model, ai_model_websearch, rate_limit_max_requests, rate_limit_time_window_seconds, max_articles_per_source, use_llm_for_source_links, use_brave_search, article_history_days, batch_size, summary_length, source_extraction_window, updated_at + RETURNING user_id, theme, max_age_days, categories, max_items_per_category, search_agent_behavior, ai_provider, ai_model, ai_model_websearch, rate_limit_max_requests, rate_limit_time_window_seconds, max_articles_per_source, use_brave_search, article_history_days, batch_size, summary_length, source_extraction_window, updated_at "#, ) .bind(user_id) @@ -158,7 +154,6 @@ pub async fn upsert( .bind(req.rate_limit_max_requests) .bind(req.rate_limit_time_window_seconds) .bind(req.max_articles_per_source) - .bind(req.use_llm_for_source_links) .bind(req.use_brave_search) .bind(req.article_history_days) .bind(req.batch_size) diff --git a/backend/src/models/settings.rs b/backend/src/models/settings.rs index a27d328..7886d6e 100644 --- a/backend/src/models/settings.rs +++ b/backend/src/models/settings.rs @@ -14,7 +14,7 @@ pub struct UserSettings { pub categories: Vec, pub max_items_per_category: i32, pub max_articles_per_source: i32, - pub use_llm_for_source_links: bool, + pub use_brave_search: bool, pub article_history_days: i32, pub batch_size: i32, @@ -38,7 +38,7 @@ pub struct UpdateSettingsRequest { pub categories: Vec, pub max_items_per_category: i32, pub max_articles_per_source: i32, - pub use_llm_for_source_links: bool, + pub use_brave_search: bool, pub article_history_days: i32, pub batch_size: i32, @@ -144,7 +144,7 @@ impl Default for UserSettings { ], max_items_per_category: 4, max_articles_per_source: 3, - use_llm_for_source_links: false, + use_brave_search: false, article_history_days: 90, batch_size: 5, @@ -173,7 +173,7 @@ mod tests { categories: vec!["Category 1".into(), "Category 2".into()], max_items_per_category: 4, max_articles_per_source: 3, - use_llm_for_source_links: false, + use_brave_search: false, article_history_days: 90, batch_size: 5, diff --git a/backend/src/services/llm/mock.rs b/backend/src/services/llm/mock.rs index e02604c..1f03ae6 100644 --- a/backend/src/services/llm/mock.rs +++ b/backend/src/services/llm/mock.rs @@ -10,7 +10,6 @@ use super::LlmProvider; pub struct MockLlmProvider { default_category: String, search_urls: Vec, - link_urls: Vec, } impl MockLlmProvider { @@ -18,7 +17,6 @@ impl MockLlmProvider { Self { default_category: "Autre".to_string(), search_urls: Vec::new(), - link_urls: Vec::new(), } } @@ -32,11 +30,6 @@ impl MockLlmProvider { self } - pub fn with_link_urls(mut self, urls: Vec) -> Self { - self.link_urls = urls; - self - } - pub fn into_arc(self) -> Arc { Arc::new(self) } @@ -69,14 +62,10 @@ impl LlmProvider for MockLlmProvider { "title": title, "summary": format!("Mock summary for: {}", title), "category": self.default_category, + "is_article": true, })); } - // Link extraction call - if sys_lower.contains("liens") { - return Ok(json!({ "urls": self.link_urls })); - } - // Search call if sys_lower.contains("precis") { let items: Vec = self.search_urls.iter().map(|url| { @@ -106,6 +95,7 @@ mod tests { .unwrap(); assert_eq!(result["title"], "GPT-7"); assert_eq!(result["category"], "AI News"); + assert_eq!(result["is_article"], true); } #[tokio::test] @@ -119,16 +109,4 @@ mod tests { let items = result["category_0"].as_array().unwrap(); assert_eq!(items.len(), 1); } - - #[tokio::test] - async fn mock_provider_returns_link_extraction() { - let provider = MockLlmProvider::new() - .with_link_urls(vec!["http://example.com/post-1".into()]); - let result = provider - .call_llm("model", "Tu dois identifier les liens", "Links...", &json!({})) - .await - .unwrap(); - let urls = result["urls"].as_array().unwrap(); - assert_eq!(urls.len(), 1); - } } diff --git a/backend/src/services/llm/schema.rs b/backend/src/services/llm/schema.rs index e47852e..2e3bcde 100644 --- a/backend/src/services/llm/schema.rs +++ b/backend/src/services/llm/schema.rs @@ -90,27 +90,14 @@ pub fn build_article_classify_schema() -> Value { "title": { "type": "string", "description": "Article title" }, "summary": { "type": "string", "description": "Summary of the article" }, "category": { "type": "string", "description": "Category name from the provided list" }, - "date": { "type": "string", "description": "Publication date in YYYY-MM-DD format, or empty string if unknown" } + "date": { "type": "string", "description": "Publication date in YYYY-MM-DD format, or empty string if unknown" }, + "is_article": { "type": "boolean", "description": "true if this is a real news article, false if it is a navigation page, contact page, terms of service, etc." } }, - "required": ["title", "summary", "category", "date"], + "required": ["title", "summary", "category", "date", "is_article"], "additionalProperties": false }) } -/// Build a JSON Schema for LLM link extraction response. -pub fn build_link_extraction_schema() -> Value { - serde_json::json!({ - "type": "object", - "properties": { - "urls": { - "type": "array", - "items": { "type": "string" } - } - }, - "required": ["urls"], - "additionalProperties": false - }) -} #[cfg(test)] mod tests { @@ -302,11 +289,4 @@ mod tests { assert_eq!(schema["additionalProperties"], false); } - #[test] - fn link_extraction_schema_has_urls_array() { - let schema = build_link_extraction_schema(); - assert_eq!(schema["properties"]["urls"]["type"], "array"); - assert_eq!(schema["additionalProperties"], false); - } - } diff --git a/backend/src/services/prompts.rs b/backend/src/services/prompts.rs index b047460..5e8a690 100644 --- a/backend/src/services/prompts.rs +++ b/backend/src/services/prompts.rs @@ -119,30 +119,6 @@ pub fn build_search_prompt( (system_prompt, user_prompt) } -/// Build a prompt for LLM-assisted link extraction from a source page. -/// -/// Receives a pre-formatted list of (href, anchor_text) pairs, not raw HTML. -pub fn build_link_extraction_prompt(links_text: &str) -> (String, String) { - let system_prompt = - "Tu es un assistant qui analyse des listes de liens. \ - Tu dois identifier les liens vers des articles d'actualite. \ - Reponds uniquement au format JSON demande." - .to_string(); - - let user_prompt = format!( - "Voici une liste de liens extraits d'une page de blog ou de site d'actualites.\n\n\ - {links}\n\n\ - Selectionne UNIQUEMENT les URLs qui pointent vers des articles \ - (pas les liens de navigation, tags, categories, login, pages statiques, topics, \ - archive, companies, events, company, event, collections, etc.).\n\ - Retourne les URLs completes, sans les modifier, dans le format JSON demande. \ - Ne change jamais les URLs retournees, et ne les tronque jamais.", - links = links_text, - ); - - (system_prompt, user_prompt) -} - /// Build a prompt for per-article classification and summarization. /// /// The LLM classifies the article into a category and generates a title + summary. @@ -180,7 +156,10 @@ pub fn build_article_classify_prompt( {summary_instruction}\n\ Si le titre fourni est vide, genere un titre a partir du contenu.\n\ Extrais la date de publication de l'article au format YYYY-MM-DD. \ - Si la date n'est pas disponible, retourne une chaine vide.", + Si la date n'est pas disponible, retourne une chaine vide.\n\ + Determine si ce contenu est un veritable article d'actualite. \ + Retourne is_article=true pour un article, false pour une page de contact, \ + mentions legales, page de navigation, FAQ, etc.", title = if title.is_empty() { "(pas de titre)" } else { title }, body = body_snippet, categories = categories_list, @@ -207,7 +186,6 @@ mod tests { ], max_items_per_category: 4, max_articles_per_source: 3, - use_llm_for_source_links: false, use_brave_search: false, article_history_days: 90, batch_size: 5, @@ -365,21 +343,6 @@ mod tests { assert!(user_prompt.contains("exactement")); } - #[test] - fn link_extraction_prompt_includes_links() { - let links = "- https://example.com/post-1 | \"Breaking News\"\n- https://example.com/post-2 | \"Update\""; - let (sys, user) = build_link_extraction_prompt(links); - assert!(user.contains("https://example.com/post-1")); - assert!(user.contains("Breaking News")); - assert!(sys.contains("liens")); - } - - #[test] - fn link_extraction_prompt_empty_links() { - let (_, user) = build_link_extraction_prompt(""); - assert!(user.contains("articles")); - } - #[test] fn article_classify_prompt_includes_content() { let (sys, user) = build_article_classify_prompt( diff --git a/backend/src/services/source_scraper.rs b/backend/src/services/source_scraper.rs index 883e554..71a2530 100644 --- a/backend/src/services/source_scraper.rs +++ b/backend/src/services/source_scraper.rs @@ -3,11 +3,8 @@ //! Used in Phase 1 of the generation pipeline to discover articles //! from user-configured sources before falling back to LLM web search. -use std::sync::{Arc, LazyLock}; +use std::sync::LazyLock; use crate::errors::AppError; -use crate::services::llm::LlmProvider; -use crate::services::llm::schema::build_link_extraction_schema; -use crate::services::prompts::build_link_extraction_prompt; use scraper::{Html, Selector}; use url::Url; @@ -16,7 +13,7 @@ static ANCHOR_SELECTOR: LazyLock = LazyLock::new(|| Selector::parse("a /// Patterns in URL paths that indicate non-article pages. const EXCLUDED_PATH_PATTERNS: &[&str] = &[ "/tag", "/category", "/author", "/page", "/login", "/signup", - "/privacy", "/terms", "/search", "/contact", "/about", "/topics", + "/privacy", "/terms", "/search", "/contact", "/about", "/topics", "/archive", "/companies", "/events", "/company", "/event", "/collections", ]; @@ -125,167 +122,6 @@ pub fn extract_links_from_html( links } -/// Extract all links from HTML as (href, anchor_text) pairs for LLM analysis. -/// -/// Minimal filtering: same-domain, http/https, non-empty path. -/// No article-pattern filtering — the LLM decides which are articles. -pub fn extract_links_as_pairs( - html: &str, - base_url: &Url, -) -> Vec<(String, String)> { - let base_domain = base_url.host_str().unwrap_or("").to_lowercase(); - let document = Html::parse_document(html); - let mut pairs = Vec::new(); - - for element in document.select(&ANCHOR_SELECTOR) { - if let Some(href) = element.value().attr("href") { - let resolved = match base_url.join(href) { - Ok(u) => u, - Err(_) => continue, - }; - - if resolved.scheme() != "http" && resolved.scheme() != "https" { - continue; - } - - let link_domain = resolved.host_str().unwrap_or("").to_lowercase(); - if link_domain != base_domain { - continue; - } - - let path = resolved.path(); - if path.is_empty() || path == "/" { - continue; - } - - let anchor_text: String = element.text().collect::>().join(" "); - let anchor_text = anchor_text.trim().to_string(); - - pairs.push((resolved.to_string(), anchor_text)); - } - } - - pairs -} - -/// Format link pairs as a text list for the LLM prompt. -/// Caps at 200 links to limit token usage. -fn format_links_for_llm(pairs: &[(String, String)]) -> String { - pairs - .iter() - .take(200) - .map(|(href, text)| { - if text.is_empty() { - format!("- {}", href) - } else { - format!("- {} | \"{}\"", href, text) - } - }) - .collect::>() - .join("\n") -} - -/// Extract article links using LLM analysis of the page HTML. -/// -/// Falls back to heuristic extraction if the LLM call fails or returns empty. -#[allow(clippy::too_many_arguments)] -pub async fn extract_article_links_with_llm( - http_client: &reqwest::Client, - source_url: &str, - max_links: usize, - provider: &Arc, - model: &str, - pool: Option<&sqlx::PgPool>, - user_id: Option, - job_id: Option, -) -> Result, AppError> { - let base_url = Url::parse(source_url) - .map_err(|e| AppError::BadRequest(format!("Invalid source URL: {}", e)))?; - - // SSRF check before fetching - if let Err(e) = crate::services::scraper::check_ssrf(&base_url).await { - tracing::warn!(url = source_url, error = %e, "Source URL failed SSRF check"); - return Ok(Vec::new()); - } - - let base_domain = base_url.host_str().unwrap_or("").to_lowercase(); - - let response = http_client.get(source_url).send().await.map_err(|e| { - tracing::warn!(url = source_url, error = %e, "Failed to fetch source page"); - AppError::Internal(anyhow::anyhow!("Failed to fetch source page")) - })?; - - if !response.status().is_success() { - tracing::warn!(url = source_url, status = %response.status(), "Source page returned non-200"); - return Ok(Vec::new()); - } - - let html_text = response.text().await.map_err(|e| { - AppError::Internal(anyhow::anyhow!("Failed to read source page body: {}", e)) - })?; - - let pairs = extract_links_as_pairs(&html_text, &base_url); - let links_text = format_links_for_llm(&pairs); - let (system, user) = build_link_extraction_prompt(&links_text); - let schema = build_link_extraction_schema(); - - let llm_start = std::time::Instant::now(); - let llm_result = provider.call_llm(model, &system, &user, &schema).await; - let llm_duration = llm_start.elapsed().as_millis() as u64; - - // Log the LLM call if pool/user_id/job_id are provided - if let (Some(pool), Some(uid), Some(jid)) = (pool, user_id, job_id) { - let response_str = match &llm_result { - Ok(resp) => serde_json::to_string_pretty(resp).unwrap_or_default(), - Err(e) => format!("Error: {}", e), - }; - crate::db::llm_call_log::insert( - pool, uid, jid, "link_extraction", model, - &system, &user, &response_str, llm_duration as i32, - Some(source_url), - ).await.ok(); - } - - match llm_result { - Ok(llm_response) => { - let urls: Vec = llm_response - .get("urls") - .and_then(|u| u.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str()) - .filter_map(|href| { - let resolved = base_url.join(href).ok()?; - if resolved.scheme() != "http" && resolved.scheme() != "https" { - return None; - } - if resolved.host_str()?.to_lowercase() != base_domain { - return None; - } - Some(resolved.to_string()) - }) - .collect() - }) - .unwrap_or_default(); - - if urls.is_empty() { - tracing::warn!(url = source_url, "LLM returned no links, falling back to heuristic"); - let fallback = extract_links_from_html(&html_text, &base_url, &base_domain); - Ok(fallback.into_iter().take(max_links).collect()) - } else { - let mut seen = std::collections::HashSet::new(); - let deduped: Vec = urls.into_iter().filter(|u| seen.insert(u.clone())).collect(); - Ok(deduped.into_iter().take(max_links).collect()) - } - } - Err(e) => { - tracing::warn!(url = source_url, error = %e, "LLM link extraction failed, falling back to heuristic"); - let fallback = extract_links_from_html(&html_text, &base_url, &base_domain); - Ok(fallback.into_iter().take(max_links).collect()) - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -376,67 +212,4 @@ mod tests { let links = extract_links_from_html("", &base_url("https://example.com"), "example.com"); assert!(links.is_empty()); } - - #[test] - fn extract_pairs_returns_href_and_text() { - let html = r#" - - Breaking AI News - GPT-6 Released - "#; - let base = base_url("https://example.com/blog"); - let pairs = extract_links_as_pairs(html, &base); - assert_eq!(pairs.len(), 2); - assert!(pairs[0].0.contains("/blog/article-1")); - assert_eq!(pairs[0].1, "Breaking AI News"); - assert!(pairs[1].0.contains("/blog/article-2")); - assert_eq!(pairs[1].1, "GPT-6 Released"); - } - - #[test] - fn extract_pairs_filters_external_links() { - let html = r#"External"#; - let base = base_url("https://example.com"); - let pairs = extract_links_as_pairs(html, &base); - assert!(pairs.is_empty()); - } - - #[test] - fn extract_pairs_filters_root_path() { - let html = r#"Home"#; - let base = base_url("https://example.com"); - let pairs = extract_links_as_pairs(html, &base); - assert!(pairs.is_empty()); - } - - #[test] - fn extract_pairs_handles_empty_anchor_text() { - let html = r#""#; - let base = base_url("https://example.com"); - let pairs = extract_links_as_pairs(html, &base); - assert_eq!(pairs.len(), 1); - assert_eq!(pairs[0].1, ""); - } - - #[test] - fn format_links_for_llm_formats_correctly() { - let pairs = vec![ - ("https://example.com/a".to_string(), "Article One".to_string()), - ("https://example.com/b".to_string(), "".to_string()), - ]; - let result = format_links_for_llm(&pairs); - assert!(result.contains("- https://example.com/a | \"Article One\"")); - assert!(result.contains("- https://example.com/b")); - assert!(!result.contains("| \"\"")); - } - - #[test] - fn format_links_for_llm_caps_at_200() { - let pairs: Vec<(String, String)> = (0..300) - .map(|i| (format!("https://example.com/{}", i), format!("Link {}", i))) - .collect(); - let result = format_links_for_llm(&pairs); - let line_count = result.lines().count(); - assert_eq!(line_count, 200); - } } diff --git a/backend/src/services/synthesis.rs b/backend/src/services/synthesis.rs index 7fbcb92..a1d4039 100644 --- a/backend/src/services/synthesis.rs +++ b/backend/src/services/synthesis.rs @@ -314,24 +314,9 @@ pub async fn run_generation_inner( let client = state.http_client.clone(); let source_url = source.url.clone(); let source_title = source.title.clone(); - let use_llm = settings.use_llm_for_source_links; - let provider_clone = std::sync::Arc::clone(&provider); - let model = Arc::clone(&model_research); let max_l = max_links; - let pool = state.pool.clone(); - let uid = user_id; - let jid = job_id; join_set.spawn(async move { - let links = if use_llm { - source_scraper::extract_article_links_with_llm( - &client, &source_url, max_l, &provider_clone, &model, - Some(&pool), Some(uid), Some(jid), - ).await - } else { - source_scraper::extract_article_links( - &client, &source_url, max_l, - ).await - }; + let links = source_scraper::extract_article_links(&client, &source_url, max_l).await; (source_url, source_title, links) }); } @@ -503,6 +488,19 @@ pub async fn run_generation_inner( } }; + // Check if LLM considers this a real article + let is_article = class_response.get("is_article").and_then(|v| v.as_bool()).unwrap_or(true); + if !is_article { + tracing::info!(url = %final_url, "Article filtered by LLM: not a real article"); + pending_traces.push(build_trace_entry(user_id, job_id, &ArticleTrace { + url: &final_url, title: &page_title, source_type: "personalized_source", + source_url: Some(&source_url), category: None, synthesis_id: None, + status: "filtered_not_article", scraped_ok: true, + published_date: None, + })); + continue; + } + // Check LLM-extracted date as fallback for articles without a scraper date if let Some(date_str) = class_response.get("date").and_then(|d| d.as_str()) { if !date_str.is_empty() { @@ -715,6 +713,19 @@ pub async fn run_generation_inner( } }; + // Check if LLM considers this a real article + let is_article = class_response.get("is_article").and_then(|v| v.as_bool()).unwrap_or(true); + if !is_article { + tracing::info!(url = %final_url, "Article filtered by LLM: not a real article"); + pending_traces.push(build_trace_entry(user_id, job_id, &ArticleTrace { + url: &final_url, title: &page_title, source_type: "brave_search", + source_url: None, category: None, synthesis_id: None, + status: "filtered_not_article", scraped_ok: true, + published_date: None, + })); + continue; + } + // Check LLM-extracted date as fallback if let Some(date_str) = class_response.get("date").and_then(|d| d.as_str()) { if !date_str.is_empty() { diff --git a/backend/tests/api_settings_test.rs b/backend/tests/api_settings_test.rs index eb0de5a..b89b235 100644 --- a/backend/tests/api_settings_test.rs +++ b/backend/tests/api_settings_test.rs @@ -47,7 +47,7 @@ async fn put_settings_without_auth_returns_401() { "categories": ["Cat"], "max_items_per_category": 4, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -134,7 +134,7 @@ async fn put_settings_with_valid_data_returns_200() { "categories": ["Vulnerabilites", "Patch Tuesday", "Threat Intel"], "max_items_per_category": 6, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -192,7 +192,7 @@ async fn put_then_get_returns_updated_data() { "categories": ["Macro", "Finance"], "max_items_per_category": 10, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -244,7 +244,7 @@ async fn put_settings_empty_theme_returns_422() { "categories": ["Cat"], "max_items_per_category": 4, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -288,7 +288,7 @@ async fn put_settings_too_many_categories_returns_422() { "categories": categories, "max_items_per_category": 4, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -331,7 +331,7 @@ async fn put_settings_empty_categories_returns_422() { "categories": [], "max_items_per_category": 4, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -375,7 +375,7 @@ async fn put_settings_max_age_days_out_of_range_returns_422() { "categories": ["Cat"], "max_items_per_category": 4, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -404,7 +404,7 @@ async fn put_settings_max_age_days_out_of_range_returns_422() { "categories": ["Cat"], "max_items_per_category": 4, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -445,7 +445,7 @@ async fn put_settings_max_items_out_of_range_returns_422() { "categories": ["Cat"], "max_items_per_category": 51, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -494,7 +494,7 @@ async fn settings_are_per_user_isolated() { "categories": ["A-Category"], "max_items_per_category": 2, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -519,7 +519,7 @@ async fn settings_are_per_user_isolated() { "categories": ["B-Category-1", "B-Category-2"], "max_items_per_category": 8, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -575,7 +575,7 @@ async fn put_settings_boundary_values_succeed() { "categories": ["C"], "max_items_per_category": 1, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -601,7 +601,7 @@ async fn put_settings_boundary_values_succeed() { "categories": categories_max, "max_items_per_category": 50, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, diff --git a/backend/tests/api_syntheses_test.rs b/backend/tests/api_syntheses_test.rs index 6955a2f..0fa58fb 100644 --- a/backend/tests/api_syntheses_test.rs +++ b/backend/tests/api_syntheses_test.rs @@ -631,7 +631,7 @@ async fn generate_pipeline_resolves_model_from_admin_config() { "categories": ["Test Category"], "max_items_per_category": 4, "max_articles_per_source": 3, - "use_llm_for_source_links": false, + "use_brave_search": false, "article_history_days": 90, "batch_size": 5, diff --git a/backend/tests/pipeline_test.rs b/backend/tests/pipeline_test.rs index 4233c80..0548dc7 100644 --- a/backend/tests/pipeline_test.rs +++ b/backend/tests/pipeline_test.rs @@ -45,7 +45,6 @@ async fn setup_user_with_settings( app: &common::TestApp, categories: Vec<&str>, max_items: i32, - use_llm_for_links: bool, ) -> (uuid::Uuid, String) { let email = format!("pipeline-{}@test.com", uuid::Uuid::new_v4()); let (user_id, session) = app.create_authenticated_user(&email).await; @@ -57,7 +56,6 @@ async fn setup_user_with_settings( "categories": categories_json, "max_items_per_category": max_items, "max_articles_per_source": 10, - "use_llm_for_source_links": use_llm_for_links, "use_brave_search": false, "article_history_days": 90, "batch_size": 5, @@ -86,12 +84,11 @@ fn make_progress_channel() -> (Arc>, wat } #[tokio::test] -async fn phase1_with_llm_link_extraction_classifies_articles() { +async fn phase1_heuristic_extraction_classifies_articles() { let app = common::TestApp::new().await; let mock_server = setup_mock_server().await; - // Use LLM link extraction to bypass SSRF on source page - let (user_id, session) = setup_user_with_settings(&app, vec!["AI News"], 4, true).await; + let (user_id, session) = setup_user_with_settings(&app, vec!["AI News"], 4).await; // Add a source pointing to wiremock (same host as article URLs) let source_url = format!("{}/blog", mock_server.uri()); @@ -99,14 +96,8 @@ async fn phase1_with_llm_link_extraction_classifies_articles() { let (status, _) = app.post_with_session("/api/v1/sources", &source, &session).await; assert!(status.is_success()); - // Mock provider: LLM link extraction returns wiremock article URLs (same domain) - let article_urls: Vec = (1..=3) - .map(|i| format!("{}/article-{}", mock_server.uri(), i)) - .collect(); - let mock_provider = MockLlmProvider::new() .with_default_category("AI News") - .with_link_urls(article_urls) .into_arc(); let job_id = uuid::Uuid::new_v4(); @@ -159,7 +150,7 @@ async fn phase2_search_fills_gaps_when_no_sources() { let mock_server = setup_mock_server().await; // No sources — Phase 1 produces nothing - let (user_id, _session) = setup_user_with_settings(&app, vec!["AI News"], 2, false).await; + let (user_id, _session) = setup_user_with_settings(&app, vec!["AI News"], 2).await; let mock_provider = MockLlmProvider::new() .with_default_category("AI News") @@ -202,19 +193,14 @@ async fn category_overflow_spills_to_autre() { let mock_server = setup_mock_server().await; // max_items_per_category=1, but LLM classifies all articles to "AI News" - let (user_id, session) = setup_user_with_settings(&app, vec!["AI News"], 1, true).await; + let (user_id, session) = setup_user_with_settings(&app, vec!["AI News"], 1).await; let source_url = format!("{}/blog", mock_server.uri()); let source = serde_json::json!({"title": "Test Source", "url": source_url}); app.post_with_session("/api/v1/sources", &source, &session).await; - let article_urls: Vec = (1..=3) - .map(|i| format!("{}/article-{}", mock_server.uri(), i)) - .collect(); - let mock_provider = MockLlmProvider::new() .with_default_category("AI News") - .with_link_urls(article_urls) .into_arc(); let job_id = uuid::Uuid::new_v4(); diff --git a/e2e/tests/generation-live.spec.ts b/e2e/tests/generation-live.spec.ts index ff9a72a..fd79fa7 100644 --- a/e2e/tests/generation-live.spec.ts +++ b/e2e/tests/generation-live.spec.ts @@ -141,7 +141,6 @@ test.describe('Live generation with OpenAI', () => { ai_provider: 'openai', ai_model: 'gpt-4o-mini', ai_model_websearch: 'gpt-4o-mini', - use_llm_for_source_links: false, use_brave_search: false, article_history_days: 90, batch_size: 5, diff --git a/frontend/src/components/settings/SettingsAdvanced.tsx b/frontend/src/components/settings/SettingsAdvanced.tsx index 11ecd0a..a521aff 100644 --- a/frontend/src/components/settings/SettingsAdvanced.tsx +++ b/frontend/src/components/settings/SettingsAdvanced.tsx @@ -14,7 +14,6 @@ interface SettingsAdvancedProps { * Groups fields that control extraction and pipeline behaviour: * - `article_history_days` — deduplication window * - `batch_size` — number of sources processed per LLM batch - * - `use_llm_for_source_links` — whether to use LLM to extract links * - `search_agent_behavior` — free-text prompt injection for the search agent */ const SettingsAdvanced: Component = (props) => { @@ -106,32 +105,6 @@ const SettingsAdvanced: Component = (props) => { - {/* Advanced extraction */} -
-

- {t('settings.advancedExtraction')} -

-
-
- - props.setSettings((prev) => ({ - ...prev, - use_llm_for_source_links: e.currentTarget.checked, - })) - } - class="h-4 w-4 text-indigo-600 focus:ring-indigo-500 border-gray-300 rounded" - /> - -
-
-
- {/* Search agent behavior */}