package main import ( "log" ) func (app *App) ScrapeHTML(shops []Shop) { wait := make(chan bool) count := 0 for _, shop := range shops { go app.Scrape(shop, wait) count++ } /* Wait until all go routines finished */ for i := 0; i < count; i++ { <-wait } } func (app *App) Scrape(shop Shop, wait chan bool) { var W []Angebot var err error W = ScrapeShop(shop) W = sanitize_offer(W) err = app.save_offer(W) if err != nil { log.Fatal(err) } err = app.remove_expired(W) if err != nil { log.Fatal(err) } wait <- true } func ScrapeShop(shop Shop) []Angebot { switch shop.Name { case "Bottleworld": return ScrapeBottleWord(shop) case "MC Whisky": return ScrapeMCWhisky(shop) case "Rum & Co": return ScrapeRumundCo(shop) case "Whic": return ScrapeWhic(shop) case "Whisky.de": return ScrapeWhiskyde(shop) case "Whiskysite.nl": return ScrapeWhiskysitenl(shop) case "Whisky World": return ScrapeWhiskyworld(shop) case "Whiskyzone": return ScrapeWhiskyzone(shop) default: log.Println(shop.Name + ": No Crawler") } return []Angebot{} }