diff options
| author | Maximilian Möhring | 2019-05-14 17:35:18 +0200 |
|---|---|---|
| committer | Maximilian Möhring | 2019-05-14 17:35:18 +0200 |
| commit | a2d595423e4270d8e644212021ba70c70df075d9 (patch) | |
| tree | a4d7f32400266a47b37d612babf9aa807c93f31d | |
| parent | 64520ffa7b66f6914decf50c43c8698ce47ee1ce (diff) | |
| download | alkobote-a2d595423e4270d8e644212021ba70c70df075d9.tar.gz | |
Refactores logging functions. (crawler)
| -rw-r--r-- | crawler/database.go | 16 | ||||
| -rw-r--r-- | crawler/log.go | 18 | ||||
| -rw-r--r-- | crawler/post_process.go | 2 | ||||
| -rw-r--r-- | crawler/sanitize.go | 20 | ||||
| -rw-r--r-- | crawler/shop_bottleworld.go | 10 | ||||
| -rw-r--r-- | crawler/shop_drankdozijn.go | 35 | ||||
| -rw-r--r-- | crawler/shop_mcwhisky.go | 14 | ||||
| -rw-r--r-- | crawler/shop_rumundco.go | 22 | ||||
| -rw-r--r-- | crawler/shop_whic.go | 12 | ||||
| -rw-r--r-- | crawler/shop_whiskyde.go | 16 | ||||
| -rw-r--r-- | crawler/shop_whiskysitenl.go | 12 | ||||
| -rw-r--r-- | crawler/shop_whiskyworld.go | 14 | ||||
| -rw-r--r-- | crawler/shop_whiskyzone.go | 12 |
13 files changed, 101 insertions, 102 deletions
diff --git a/crawler/database.go b/crawler/database.go index 45f7fda..5de0fb2 100644 --- a/crawler/database.go +++ b/crawler/database.go @@ -122,15 +122,15 @@ func (app *App) save_offer(W []Angebot) error { o.error_msg = err.Error() o.error_ctx = fmt.Sprintf(`INSERT INTO angebot (shop, name, url, abv, volume, age, original_price, discounted_price, base_price, valid_until, image_url, spirit_type, created_at) VALUES (%d, "%s", "%s", %f, %f, %d, %d, %d, %d, %d, "%s", "%s", %d)`, o.Shop, o.Name, o.Url, o.Abv, o.Volume, o.Age, o.Original_price, o.Discounted_price, o.Base_price, o.Valid_until, o.Image_url, o.Spirit_type, app.Now) - WarnOffer(o, fmt.Sprintf("Save Offer: Inserting offer failed (%d)", found)) + o.Warn(fmt.Sprintf("Save Offer: Inserting offer failed (%d)", found)) } - DebugOffer(o, "database.go: Inserting offer.") + o.Debug("database.go: Inserting offer.") } else if err != nil { o.error_msg = err.Error() o.error_ctx = fmt.Sprintf(strings.Replace(detect_duplicate_query, "?", `"%s"`, 1), o.Name) - WarnOffer(o, "database.go: Duplicate query failed") + o.Warn("database.go: Duplicate query failed") } else { /* * If everything went right we update the image url to reflect new changes. @@ -140,13 +140,13 @@ func (app *App) save_offer(W []Angebot) error { if err != nil { o.error_msg = err.Error() o.error_ctx = fmt.Sprintf(`UPDATE _intern_view SET image_url = %s, website_raw = %s WHERE name = %s AND shop_id = %d AND volume = %4.2f AND abv = %4.2f AND original_price = %d AND discounted_price = %d AND valid_until = %d`, o.Image_url, "redacted", o.Name, o.Shop, o.Volume, o.Abv, o.Original_price, o.Discounted_price, o.Valid_until) - WarnOffer(o, "database.go: Preparing update_img_query failed") + o.Warn("database.go: Preparing update_img_query failed") } _, err = update_img_stmt.Exec(o.Image_url, o.Website, o.Name) if err != nil { o.error_msg = err.Error() o.error_ctx = fmt.Sprintf(`UPDATE _intern_view SET image_url = %s WHERE name = %s AND shop_id = %d AND volume = %4.2f AND abv = %4.2f AND original_price = %d AND discounted_price = %d AND valid_until = %d`, o.Image_url, o.Name, o.Shop, o.Volume, o.Abv, o.Original_price, o.Discounted_price, o.Valid_until) - WarnOffer(o, "database.go: Executing update_img_query failed") + o.Warn("database.go: Executing update_img_query failed") } } } @@ -179,17 +179,17 @@ func (app *App) remove_expired(W []Angebot, shop Shop) error { } if !app.offer_contains(W, offer_db) { - DebugOffer(offer_db, "Contains not - Set to expire") + offer_db.Debug("Contains not - Set to expire") expire_query := `UPDATE angebot SET valid_until = ? WHERE id = ?` _, err = app.DB.Exec(expire_query, app.Now, offer_db.Id) if err != nil { offer_db.error_msg = err.Error() offer_db.error_ctx = fmt.Sprintf("UPDATE angebot SET valid_until = %d WHERE id = %d", app.Now, offer_db.Id) - WarnOffer(offer_db, "Remove expired: Update failed") + offer_db.Warn("Remove expired: Update failed") return err } } - DebugOffer(offer_db, "Contains! DOES NOT EXPIRE!") + offer_db.Debug("Contains! DOES NOT EXPIRE!") } return nil diff --git a/crawler/log.go b/crawler/log.go index d284a74..26f4abd 100644 --- a/crawler/log.go +++ b/crawler/log.go @@ -15,8 +15,8 @@ func Fatal(err error, msg string) { log.Fatal(msg) } } -func FatalOffer(offer Angebot, msg string) { - log.WithFields(getFields(offer)).Fatal(msg) +func (offer Angebot) Fatal(msg string) { + log.WithFields(offer.getFields()).Fatal(msg) } func Println(err error, msg string) { @@ -31,9 +31,9 @@ func Println(err error, msg string) { } } -func PrintlnOffer(offer Angebot, msg string) { +func (offer Angebot) Println(msg string) { - log.WithFields(getFields(offer)).Println(msg) + log.WithFields(offer.getFields()).Println(msg) } func Debug(err error, msg string) { @@ -48,17 +48,17 @@ func Debug(err error, msg string) { } } -func DebugOffer(offer Angebot, msg string) { +func (offer Angebot) Debug(msg string) { - log.WithFields(getFields(offer)).Debug(msg) + log.WithFields(offer.getFields()).Debug(msg) } -func WarnOffer(offer Angebot, msg string) { +func (offer Angebot) Warn(msg string) { - log.WithFields(getFields(offer)).Warn(msg) + log.WithFields(offer.getFields()).Warn(msg) } -func getFields(offer Angebot) log.Fields { +func (offer Angebot) getFields() log.Fields { return log.Fields{ "Name": offer.Name, "Shop": offer.Shop, diff --git a/crawler/post_process.go b/crawler/post_process.go index bb1ca46..76fc70f 100644 --- a/crawler/post_process.go +++ b/crawler/post_process.go @@ -118,7 +118,7 @@ func (app *App) fix_db() error { if err != nil { offer_db.error_msg = err.Error() offer_db.error_ctx = fmt.Sprintf(`UPDATE angebot SET name = "%s", age = %d WHERE id = %d`, offer_db.Name, offer_db.Age, offer_db.Id) - WarnOffer(offer_db, "post_process.go: Update query failed.") + offer_db.Warn("post_process.go: Update query failed.") } } diff --git a/crawler/sanitize.go b/crawler/sanitize.go index db0faec..25e8a09 100644 --- a/crawler/sanitize.go +++ b/crawler/sanitize.go @@ -17,7 +17,7 @@ func sanitize_offer(angebote []Angebot, shop Shop, try int) []Angebot { for _, offer := range angebote { if offer.Spirit_type == "Wein" { - DebugOffer(offer, "Sanitizer: Skip offer because it's wine") + offer.Debug("Sanitizer: Skip offer because it's wine") continue } @@ -31,34 +31,34 @@ func sanitize_offer(angebote []Angebot, shop Shop, try int) []Angebot { continue } if offer.Volume == 0 { - WarnOffer(offer, "Sanitizer: Volume is zero") + offer.Warn("Sanitizer: Volume is zero") continue } if offer.Original_price == 0 { - WarnOffer(offer, "Sanitizer: Original price is zero") + offer.Warn("Sanitizer: Original price is zero") continue } if offer.Discounted_price == 0 { - WarnOffer(offer, "Sanitizer: Discounted price is zero") + offer.Warn("Sanitizer: Discounted price is zero") continue } if offer.Base_price == 0 { - WarnOffer(offer, "Sanitizer: Base price is zero") + offer.Warn("Sanitizer: Base price is zero") continue } if offer.Url == "" { - WarnOffer(offer, "Sanitizer: URL is empty") + offer.Warn("Sanitizer: URL is empty") continue } if offer.Image_url == "" { - WarnOffer(offer, "Sanitizer: Image-URL is empty") + offer.Warn("Sanitizer: Image-URL is empty") continue } if err := sanitize_image_url(offer.Image_url); err != nil { offer.error_ctx = offer.Image_url offer.error_msg = err.Error() - WarnOffer(offer, "Sanitizer: Image-URL is not valid") + offer.Warn("Sanitizer: Image-URL is not valid") continue } @@ -66,7 +66,7 @@ func sanitize_offer(angebote []Angebot, shop Shop, try int) []Angebot { offer.Website = "" if offer.Age == 0 { - DebugOffer(offer, "GREP") + offer.Debug("GREP") } W = append(W, offer) @@ -258,7 +258,7 @@ func _check_abv_for_spirit_type(offer Angebot) bool { */ if offer.Abv == 0 { - WarnOffer(offer, "Sanitizer: Abv is zero") + offer.Warn("Sanitizer: Abv is zero") return false } diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go index 8722211..97ec162 100644 --- a/crawler/shop_bottleworld.go +++ b/crawler/shop_bottleworld.go @@ -33,7 +33,7 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price") - PrintlnOffer(W, "Bottleworld: Converting original price failed") + W.Println("Bottleworld: Converting original price failed") return } }) @@ -42,7 +42,7 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price") - PrintlnOffer(W, "Bottleworld: Converting discounted price failed") + W.Println("Bottleworld: Converting discounted price failed") return } }) @@ -53,7 +53,7 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = price_per_litre_noisy - PrintlnOffer(W, "Bottleworld: Sanitizing base price failed") + W.Println("Bottleworld: Sanitizing base price failed") return } W.Base_price = price_per_litre @@ -70,14 +70,14 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot { if W.Volume == 0 { W.error_msg = "Bottleworld: Volume is zero" W.error_ctx = ctx - PrintlnOffer(W, "Bottleworld: Volume is zero") + W.Println("Bottleworld: Volume is zero") return } W.Abv, ctx = get_abv(e) if W.Abv == 0 { W.error_msg = "Bottleworld: Abv is zero" W.error_ctx = ctx - PrintlnOffer(W, "Bottleworld: Abv is zero") + W.Println("Bottleworld: Abv is zero") return } diff --git a/crawler/shop_drankdozijn.go b/crawler/shop_drankdozijn.go index 5cd12d5..20018af 100644 --- a/crawler/shop_drankdozijn.go +++ b/crawler/shop_drankdozijn.go @@ -71,15 +71,14 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { tmp_spirit_type := tmp_desc["description"].(string) if "Bier" == tmp_spirit_type { - DebugOffer(W, "Drankdozijn: skip offer because it's beer") + W.Debug("Drankdozijn: skip offer because it's beer") continue } W.Spirit_type = detect_spirit_type(tmp_desc["description"].(string)) if v, _ := api_data["price"]; v == nil { - //DebugOffer(W, "Drankdozijn: Skip Offer") - DebugOffer(W, "Drankdozijn: price is nil -> skip offer") + W.Debug("Drankdozijn: price is nil -> skip offer") continue } W.Original_price, err = convert_price(api_data["price"].(string)) @@ -94,7 +93,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { } if W.Discounted_price >= W.Original_price { - DebugOffer(W, "Drankdozijn: Discounted price is not cheaper") + W.Debug("Drankdozijn: Discounted price is not cheaper") continue } @@ -103,7 +102,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { // Check for bundle offer if len(tmp_offer_url_img_map) > 1 { - DebugOffer(W, "Drankdozijn: Skip Offer because of bundle") + W.Debug("Drankdozijn: Skip Offer because of bundle") continue } @@ -119,7 +118,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { c.OnHTML(".product_top", func(e *colly.HTMLElement) { if strings.Contains(W.Name, "+ gratis") || strings.Contains(W.Name, "& gratis") { - DebugOffer(W, "Drankdozijn: Skip Offer because it contains gratis ware") + W.Debug("Drankdozijn: Skip Offer because it contains gratis ware") return } @@ -132,7 +131,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price_l") - PrintlnOffer(W, "Drankdozijn: Converting base price failed") + W.Println("Drankdozijn: Converting base price failed") return } }) @@ -150,7 +149,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { if W.Volume == 0 { W.error_msg = e.Text W.error_ctx = err.Error() - PrintlnOffer(W, "Drankdozijn: Volume is zero, returning") + W.Println("Drankdozijn: Volume is zero, returning") return } case "Alkoholgehalt", "Alcoholpercentage": @@ -158,7 +157,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { if W.Abv == 0 { W.error_msg = "Drankdozijn: Abv is zero" W.error_ctx = err.Error() - PrintlnOffer(W, "Drankdozijn: abv is zero") + W.Println("Drankdozijn: abv is zero") return } case "Kategorie", "Categorie": @@ -167,7 +166,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { if "Champagner" == tmp_type { if tmp_type != W.Spirit_type { - DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) + W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type @@ -176,19 +175,19 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { switch W.Spirit_type { case "Cognac": if tmp_type != W.Spirit_type { - DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) + W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type case "Brandy": if tmp_type != W.Spirit_type { - DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) + W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type case "Sherry": if tmp_type != W.Spirit_type { - DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) + W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type @@ -196,28 +195,28 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { retest_type := detect_spirit_type(W.Name) if "Likör" != retest_type && "Verschiedenes" != retest_type { if tmp_type != W.Spirit_type { - DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+retest_type) + W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + retest_type) W.Spirit_type = tmp_type } W.Spirit_type = detect_spirit_type(W.Name) } if "Tequila" == tmp_type { if tmp_type != W.Spirit_type { - DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) + W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } if "Mezcal" == tmp_type { if tmp_type != W.Spirit_type { - DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) + W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type } if "Baijiu" == tmp_type { if tmp_type != W.Spirit_type { - DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type) + W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type) W.Spirit_type = tmp_type } W.Spirit_type = tmp_type @@ -243,7 +242,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot { //log.Println("Visit " + W.Url) } - //DebugOffer(W, "DEBUG OFFER") + //W.Debug("DEBUG OFFER") Offers = append(Offers, W) } diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go index 941f3b9..027aeee 100644 --- a/crawler/shop_mcwhisky.go +++ b/crawler/shop_mcwhisky.go @@ -37,7 +37,7 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price") - PrintlnOffer(W, "MC Whisky: Converting original price failed") + W.Println("MC Whisky: Converting original price failed") return } }) @@ -46,7 +46,7 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price") - PrintlnOffer(W, "MC Whisky: Converting discounted price failed") + W.Println("MC Whisky: Converting discounted price failed") return } }) @@ -57,7 +57,7 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = price_per_litre_noisy - PrintlnOffer(W, "MC Whisky: Sanitizing base price failed") + W.Println("MC Whisky: Sanitizing base price failed") return } @@ -69,14 +69,14 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot { if volume_failed != "" { W.error_msg = "MC Whisky: Volume failed" W.error_ctx = volume_failed - PrintlnOffer(W, "MC Whisky: Volume failed") + W.Println("MC Whisky: Volume failed") return } abv_failed := e.Request.Ctx.Get("abv_failed") if volume_failed != "" { W.error_msg = "MC Whisky: Abv failed" W.error_ctx = abv_failed - PrintlnOffer(W, "MC Whisky: Abv failed") + W.Println("MC Whisky: Abv failed") return } @@ -85,14 +85,14 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot { if W.Volume == 0 { W.error_msg = "MC Whisky: Volume is zero" W.error_ctx = ctx - PrintlnOffer(W, "MC Whisky: Volume is zero") + W.Println("MC Whisky: Volume is zero") return } W.Abv, ctx = get_abv(e) if W.Abv == 0 { W.error_msg = "MC Whisky: Abv is zero" W.error_ctx = ctx - PrintlnOffer(W, "MC Whisky: Abv is zero") + W.Println("MC Whisky: Abv is zero") return } diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go index 86d8b8a..3b7d08a 100644 --- a/crawler/shop_rumundco.go +++ b/crawler/shop_rumundco.go @@ -43,7 +43,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot { if err != nil { W.error_msg = "Rum & Co: Parsing Query from Shop-URL failed" W.error_ctx = e.Request.URL.String() - PrintlnOffer(W, "Rum & Co: Parsing Query from Shop-URL failed") + W.Println("Rum & Co: Parsing Query from Shop-URL failed") } switch param["kf"][0] { case "29": @@ -58,7 +58,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot { W.error_msg = "Rum & Co: Query parameter has unexpected value" W.error_ctx = param["kf"][0] W.Url = e.Request.URL.String() - PrintlnOffer(W, "Rum & Co: Detecting spirit type failed") + W.Println("Rum & Co: Detecting spirit type failed") } log.Debug("Rum & Co: Crawling " + W.Spirit_type + " with param kf=" + param["kf"][0]) @@ -75,7 +75,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot { W.error_ctx = e.ChildText(".delivery-status") W.Url = whisky_url W.Name = whisky_name - PrintlnOffer(W, "Rum & Co: Offer not available") + W.Println("Rum & Co: Offer not available") return } @@ -93,21 +93,21 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot { if "" == regular_price { W.error_msg = "Rum & Co: No regular price found" W.error_ctx = regular_price - PrintlnOffer(W, "Rum & Co: No regular price found") + W.Println("Rum & Co: No regular price found") return } W.Original_price, err = convert_price(regular_price) if err != nil { W.error_msg = err.Error() W.error_ctx = regular_price - PrintlnOffer(W, "Rum & Co: Original price: Convert price failed") + W.Println("Rum & Co: Original price: Convert price failed") return } W.Discounted_price, err = convert_price(e.ChildText(".price-value")) if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price-value") - PrintlnOffer(W, "Rum & Co: Discounted price: Convert price failed") + W.Println("Rum & Co: Discounted price: Convert price failed") return } @@ -117,7 +117,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".value") - PrintlnOffer(W, "Rum & Co: Base price: Sanitizing base price failed") + W.Println("Rum & Co: Base price: Sanitizing base price failed") return } }) @@ -136,7 +136,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot { if len(image_url_noisy_slice) < 2 { W.error_msg = "Rum & Co: (Pagespeed) Image URL not found" W.error_ctx = image_url_noisy - PrintlnOffer(W, "Rum & Co: (Pagespeed) Image URL not found") + W.Println("Rum & Co: (Pagespeed) Image URL not found") return } image_url_noisy = strings.Replace(image_url_noisy, image_url_noisy_slice[1], "", 1) @@ -152,7 +152,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot { if W.Volume == 0 { W.error_msg = "Rum & Co: Volume is zero" W.error_ctx = ctx - PrintlnOffer(W, "Rum & Co: Volume is zero") + W.Println("Rum & Co: Volume is zero") return } @@ -164,14 +164,14 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = abv_noisy - PrintlnOffer(W, "Rum & Co: Base price: Extracting ABV failed") + W.Println("Rum & Co: Base price: Extracting ABV failed") return } } if W.Abv == 0 { W.error_msg = "Rum & Co: Abv is zero" W.error_ctx = abv_noisy - PrintlnOffer(W, "Rum & Co: Abv is zero") + W.Println("Rum & Co: Abv is zero") return } W.Website = e.Request.Ctx.Get("website") diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go index 93bff23..b21c042 100644 --- a/crawler/shop_whic.go +++ b/crawler/shop_whic.go @@ -39,7 +39,7 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price") - PrintlnOffer(W, "Whic: Converting original price failed") + W.Println("Whic: Converting original price failed") return } }) @@ -48,7 +48,7 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".price") - PrintlnOffer(W, "Whic: Converting discounted price failed") + W.Println("Whic: Converting discounted price failed") return } }) @@ -59,7 +59,7 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = base_price_noisy - PrintlnOffer(W, "Whic: Sanitizing base price failed") + W.Println("Whic: Sanitizing base price failed") return } @@ -72,7 +72,7 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = img_link_noisy - WarnOffer(W, "Whic: Parsing document in Goquery failed") + W.Println("Whic: Parsing document in Goquery failed") return } W.Image_url, _ = doc.Find("img").Attr("src") @@ -84,14 +84,14 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot { if W.Volume == 0 { W.error_msg = "Whic: Volume is zero" W.error_ctx = ctx - PrintlnOffer(W, "Whic: Volume is zero") + W.Println("Whic: Volume is zero") return } W.Abv, ctx = get_abv(e) if W.Abv == 0 { W.error_msg = "Whic: Abv is zero" W.error_ctx = ctx - PrintlnOffer(W, "Whic: Abv is zero") + W.Println("Whic: Abv is zero") return } diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go index d3087ca..d06f522 100644 --- a/crawler/shop_whiskyde.go +++ b/crawler/shop_whiskyde.go @@ -37,7 +37,7 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText("del") - PrintlnOffer(W, "Whisky.de: Converting original price failed") + W.Println("Whisky.de: Converting original price failed") return } }) @@ -46,7 +46,7 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".article-price-default") - PrintlnOffer(W, "Whisky.de: Converting discounted price failed") + W.Println("Whisky.de: Converting discounted price failed") return } }) @@ -64,7 +64,7 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot { if !strings.Contains(text_noisy, "Liter") { W.error_ctx = text_noisy W.error_msg = "Whisky.de: String 'Liter' not found." - PrintlnOffer(W, "Whisky.de: String 'Liter' not found.") + W.Println("Whisky.de: String 'Liter' not found.") return } @@ -73,27 +73,27 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = text_noisy - PrintlnOffer(W, "Whisky.de: Extracting volume failed") + W.Println("Whisky.de: Extracting volume failed") return } W.Abv, err = extract_abv(abv_noisy) if err != nil { W.error_msg = err.Error() W.error_ctx = abv_noisy - PrintlnOffer(W, "Whisky.de: Extracting abv failed") + W.Println("Whisky.de: Extracting abv failed") return } if W.Volume == 0 { W.error_msg = "Whisky.de: Volume is zero" W.error_ctx = text_noisy - PrintlnOffer(W, "Whisky.de: Volume is zero") + W.Println("Whisky.de: Volume is zero") return } if W.Abv == 0 { W.error_msg = "Whisky.de: Abv is zero" W.error_ctx = abv_noisy - PrintlnOffer(W, "Whisky.de: Abv is zero") + W.Println("Whisky.de: Abv is zero") return } @@ -101,7 +101,7 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".article-unitprice-default") - PrintlnOffer(W, "Whisky.de: Converting base price failed") + W.Println("Whisky.de: Converting base price failed") return } diff --git a/crawler/shop_whiskysitenl.go b/crawler/shop_whiskysitenl.go index e3ae075..c9a50b0 100644 --- a/crawler/shop_whiskysitenl.go +++ b/crawler/shop_whiskysitenl.go @@ -44,14 +44,14 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = regular_price - PrintlnOffer(W, "Whiskysite.nl: Extracting original price failed") + W.Println("Whiskysite.nl: Extracting original price failed") return } W.Discounted_price, err = convert_price(discounted_price) if err != nil { W.error_msg = err.Error() W.error_ctx = discounted_price - PrintlnOffer(W, "Whiskysite.nl: Extracting discounted price failed") + W.Println("Whiskysite.nl: Extracting discounted price failed") return } @@ -63,13 +63,13 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot { if volume_failed != "" { W.error_msg = "Whiskysite.nl: Extracting volume via Liter-Regex failed" W.error_ctx = volume_failed - PrintlnOffer(W, "Whiskysite.nl: Extracting volume via Liter-Regex failed") + W.Println("Whiskysite.nl: Extracting volume via Liter-Regex failed") return } if e.Request.Ctx.Get("abv_failed") != "" { W.error_msg = "Whiskysite.nl: Extracting abv via Abv-Regex failed" W.error_ctx = e.Request.Ctx.Get("volume_failed") - PrintlnOffer(W, "Whiskysite.nl: Extracting abv via Abv-Regex failed") + W.Println("Whiskysite.nl: Extracting abv via Abv-Regex failed") return } @@ -78,14 +78,14 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot { if W.Volume == 0 { W.error_msg = "Whiskysite.nl: Extracting volume failed" W.error_ctx = ctx - PrintlnOffer(W, "Whiskysite.nl: Extracting volume failed") + W.Println("Whiskysite.nl: Extracting volume failed") return } W.Abv, ctx = get_abv(e) if W.Abv == 0 { W.error_msg = "Whiskysite.nl: Extracting abv failed" W.error_ctx = ctx - PrintlnOffer(W, "Whiskysite.nl: Extracting abv failed") + W.Println("Whiskysite.nl: Extracting abv failed") return } diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go index 3f0874d..78e7244 100644 --- a/crawler/shop_whiskyworld.go +++ b/crawler/shop_whiskyworld.go @@ -43,7 +43,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = regular_price - PrintlnOffer(W, "Whiskyworld: Converting original price failed") + W.Println("Whiskyworld: Converting original price failed") return } @@ -51,7 +51,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.ChildText(".uvp") - PrintlnOffer(W, "Whiskyworld: Converting discounted price failed") + W.Println("Whiskyworld: Converting discounted price failed") return } @@ -61,13 +61,13 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = text_noisy - PrintlnOffer(W, "Whiskyworld: Extracting volume failed") + W.Println("Whiskyworld: Extracting volume failed") return } if W.Volume == 0 { W.error_msg = "Whiskyworld: Volume is zero" W.error_ctx = text_noisy - PrintlnOffer(W, "Whiskyworld: Volume is zero") + W.Println("Whiskyworld: Volume is zero") return } abv_noisy := strings.TrimSpace(strings.SplitAfter(text_noisy, "Liter")[1]) @@ -76,13 +76,13 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = abv_noisy - PrintlnOffer(W, "Whiskyworld: Extracting abv failed") + W.Println("Whiskyworld: Extracting abv failed") return } if W.Abv == 0 { W.error_msg = "Whiskyworld: Abv is zero" W.error_ctx = abv_noisy - PrintlnOffer(W, "Whiskyworld: Abv is zero") + W.Println("Whiskyworld: Abv is zero") return } }) @@ -95,7 +95,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = base_price_noisy - PrintlnOffer(W, "Whiskyworld: Sanitizing base price failed") + W.Println("Whiskyworld: Sanitizing base price failed") return } } diff --git a/crawler/shop_whiskyzone.go b/crawler/shop_whiskyzone.go index dbaf0ba..ffb124d 100644 --- a/crawler/shop_whiskyzone.go +++ b/crawler/shop_whiskyzone.go @@ -37,7 +37,7 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot { e.Request.Visit(W.Url) if "sold_out" == e.Request.Ctx.Get("sold_out") { - PrintlnOffer(W, "Whiskyzone: Sold out") + W.Println("Whiskyzone: Sold out") return } @@ -46,7 +46,7 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.Request.Ctx.Get("discounted_price") - PrintlnOffer(W, "Whiskyzone: Convert discounted price failed") + W.Println("Whiskyzone: Convert discounted price failed") return } @@ -54,7 +54,7 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = e.Request.Ctx.Get("original_price") - PrintlnOffer(W, "Whiskyzone: Convert original price failed") + W.Println("Whiskyzone: Convert original price failed") return } @@ -63,14 +63,14 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot { if W.Volume == 0 { W.error_msg = "Whiskyzone: Volume is zero" W.error_ctx = ctx - PrintlnOffer(W, "Whiskyzone: Volume is zero") + W.Println("Whiskyzone: Volume is zero") return } W.Abv, ctx = get_abv(e) if W.Abv == 0 { W.error_msg = "Whiskyzone: Abv is zero" W.error_ctx = ctx - PrintlnOffer(W, "Whiskyzone: Abv is zero") + W.Println("Whiskyzone: Abv is zero") return } @@ -82,7 +82,7 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot { if err != nil { W.error_msg = err.Error() W.error_ctx = base_price - PrintlnOffer(W, "Whiskyzone: Extracting base price failed") + W.Println("Whiskyzone: Extracting base price failed") return } } |
