@ -314,25 +314,12 @@ async fn run_generation_inner(
// Step 7b: Filter out homepage URLs (path == "/" or empty)
// Step 7b: Filter out homepage URLs (path == "/" or empty)
let parsed = filter_homepage_urls ( parsed ) ;
let parsed = filter_homepage_urls ( parsed ) ;
// Step 8: Adaptive pipeline — decide whether to scrape+rewrite or use search results directly
// Step 8: Scrape + rewrite pass
//
//
// If the provider supports native web search and the search pass produced high-quality
// Always run the full pipeline: the search pass URLs can be hallucinated
// results (>70% valid URLs starting with http), we can skip the expensive scrape+rewrite
// by the LLM (Wikipedia, corporate sites instead of actual articles).
// pass and use the search results directly.
// The scrape pass fetches each URL and validates the content exists,
let final_sections = if provider . supports_web_search ( ) & & url_quality_sufficient ( & parsed ) {
// then the rewrite pass produces summaries based on actual article content.
tracing ::info ! (
provider = provider . provider_id ( ) ,
"Search pass URL quality sufficient, skipping scrape+rewrite pass"
) ;
emit_progress (
tx ,
"finalizing" ,
"Resultats de recherche de bonne qualite, finalisation directe..." ,
85 ,
) ;
build_final_sections ( & raw_results , & settings . categories ) ?
} else {
// Full pipeline: scrape + rewrite
emit_progress ( tx , "scraping" , "Verification des sources..." , 45 ) ;
emit_progress ( tx , "scraping" , "Verification des sources..." , 45 ) ;
let scraped = scrape_articles ( state , & parsed , settings . max_age_days as i64 , tx ) . await ;
let scraped = scrape_articles ( state , & parsed , settings . max_age_days as i64 , tx ) . await ;
@ -348,8 +335,7 @@ async fn run_generation_inner(
. await ? ;
. await ? ;
emit_progress ( tx , "finalizing" , "Finalisation..." , 90 ) ;
emit_progress ( tx , "finalizing" , "Finalisation..." , 90 ) ;
build_final_sections ( & final_results , & settings . categories ) ?
let final_sections = build_final_sections ( & final_results , & settings . categories ) ? ;
} ;
// Step 12: Save synthesis to DB
// Step 12: Save synthesis to DB
emit_progress ( tx , "saving" , "Sauvegarde de la synthese..." , 95 ) ;
emit_progress ( tx , "saving" , "Sauvegarde de la synthese..." , 95 ) ;