summaryrefslogtreecommitdiff
path: root/src/main.go
diff options
context:
space:
mode:
authordev2026-06-26 03:26:07 +0200
committerdev2026-06-26 03:26:07 +0200
commit8e2d742e59b3923852e1ef6e7a5e2ee1de14ce45 (patch)
tree9cbcab194f21431255222c7785e15795e494a728 /src/main.go
parentf3ed9396113de4d81ae0fadabc10ae59f7076862 (diff)
downloadhnimdbbot-8e2d742e59b3923852e1ef6e7a5e2ee1de14ce45.tar.gz
refactor: pipeline SPARQL and wiki data in parallel
- Merge fetchWikiArticles + fetchWikiArticlesData into one pipeline - SPARQL producer fetches batches, commits each to DB, forwards resolved articles - Wiki data consumer runs concurrently, fetching at 2s/request - Each SPARQL batch commits independently (no global transaction) - Rate limits respected for both Wikidata SPARQL and wiki server - No parallel requests to either endpoint
Diffstat (limited to 'src/main.go')
-rw-r--r--src/main.go4
1 files changed, 0 insertions, 4 deletions
diff --git a/src/main.go b/src/main.go
index 9fd53db..272e4b6 100644
--- a/src/main.go
+++ b/src/main.go
@@ -198,8 +198,4 @@ func main() {
if err = app.fetchWikiArticles(); err != nil {
log.Fatalf("fetchWikiArticles: %v", err)
}
-
- if err = app.fetchWikiArticlesData(); err != nil {
- log.Fatalf("fetchWikiArticlesData: %v", err)
- }
}