From 8e2d742e59b3923852e1ef6e7a5e2ee1de14ce45 Mon Sep 17 00:00:00 2001
From: dev
Date: Fri, 26 Jun 2026 03:26:07 +0200
Subject: refactor: pipeline SPARQL and wiki data in parallel

- Merge fetchWikiArticles + fetchWikiArticlesData into one pipeline
- SPARQL producer fetches batches, commits each to DB, forwards resolved articles
- Wiki data consumer runs concurrently, fetching at 2s/request
- Each SPARQL batch commits independently (no global transaction)
- Rate limits respected for both Wikidata SPARQL and wiki server
- No parallel requests to either endpoint
---
 src/main.go | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'src/main.go')

diff --git a/src/main.go b/src/main.go
index 9fd53db..272e4b6 100644
--- a/src/main.go
+++ b/src/main.go
@@ -198,8 +198,4 @@ func main() {
 	if err = app.fetchWikiArticles(); err != nil {
 		log.Fatalf("fetchWikiArticles: %v", err)
 	}
-
-	if err = app.fetchWikiArticlesData(); err != nil {
-		log.Fatalf("fetchWikiArticlesData: %v", err)
-	}
 }
-- 
cgit v1.2.3