Parallelized the crawler.

1 files changed, 69 insertions, 0 deletions

diff --git a/crawler/scrape.go b/crawler/scrape.go
new file mode 100644
index 0000000..0595240
--- /dev/null
+++ b/crawler/scrape.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+	"log"
+)
+
+func (app *App) ScrapeHTML(shops []Shop) {
+
+	wait := make(chan int)
+	count := 0
+
+	for _, shop := range shops {
+
+		go func() {
+			var W []Angebot
+			var err error
+
+			W = ScrapeShop(shop)
+
+			err = app.save_offer(W)
+			if err != nil {
+				log.Fatal(err)
+			}
+			err = app.remove_expired(W)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			wait <- 1
+
+		}()
+
+		count++
+
+	}
+
+	/*
+		Wait until all go routines finished
+	*/
+	for i := 0; i < count; i++ {
+		<-wait
+	}
+}
+
+func ScrapeShop(shop Shop) []Angebot {
+
+	switch shop.Name {
+	case "Bottleworld":
+		return ScrapeBottleWord(shop)
+	case "MC Whisky":
+		return ScrapeMCWhisky(shop)
+	case "Rum & Co":
+		return ScrapeRumundCo(shop)
+	case "Whic":
+		return ScrapeWhic(shop)
+	case "Whisky.de":
+		return ScrapeWhiskyde(shop)
+	case "Whiskysite.nl":
+		return ScrapeWhiskysitenl(shop)
+	case "Whisky World":
+		return ScrapeWhiskyworld(shop)
+	case "Whiskyzone":
+		return ScrapeWhiskyzone(shop)
+	default:
+		log.Println(shop.Name + ": No Crawler")
+	}
+
+	return []Angebot{}
+}