From ddef17e4afc59d614d064c97426e8cedcc6599fc Mon Sep 17 00:00:00 2001
From: Max
Date: Thu, 8 Feb 2018 01:03:36 +0100
Subject: Better parallelization.

---
 crawler/scrape.go | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

(limited to 'crawler/scrape.go')

diff --git a/crawler/scrape.go b/crawler/scrape.go
index 31b3618..6250896 100644
--- a/crawler/scrape.go
+++ b/crawler/scrape.go
@@ -6,32 +6,12 @@ import (
 
 func (app *App) ScrapeHTML(shops []Shop) {
 
-	wait := make(chan int)
+	wait := make(chan bool)
 	count := 0
 
 	for _, shop := range shops {
 
-		go func() {
-			var W []Angebot
-			var err error
-
-			W = ScrapeShop(shop)
-
-			W = sanitize_offer(W)
-
-			err = app.save_offer(W)
-			if err != nil {
-				log.Fatal(err)
-			}
-			err = app.remove_expired(W)
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			wait <- 1
-
-		}()
-
+		go app.Scrape(shop, wait)
 		count++
 
 	}
@@ -44,6 +24,26 @@ func (app *App) ScrapeHTML(shops []Shop) {
 	}
 }
 
+func (app *App) Scrape(shop Shop, wait chan bool) {
+	var W []Angebot
+	var err error
+
+	W = ScrapeShop(shop)
+
+	W = sanitize_offer(W)
+
+	err = app.save_offer(W)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = app.remove_expired(W)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wait <- true
+}
+
 func ScrapeShop(shop Shop) []Angebot {
 
 	switch shop.Name {
-- 
cgit v1.2.3