summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaximilian Möhring2019-05-14 17:35:18 +0200
committerMaximilian Möhring2019-05-14 17:35:18 +0200
commita2d595423e4270d8e644212021ba70c70df075d9 (patch)
treea4d7f32400266a47b37d612babf9aa807c93f31d
parent64520ffa7b66f6914decf50c43c8698ce47ee1ce (diff)
downloadalkobote-a2d595423e4270d8e644212021ba70c70df075d9.tar.gz
Refactores logging functions. (crawler)
-rw-r--r--crawler/database.go16
-rw-r--r--crawler/log.go18
-rw-r--r--crawler/post_process.go2
-rw-r--r--crawler/sanitize.go20
-rw-r--r--crawler/shop_bottleworld.go10
-rw-r--r--crawler/shop_drankdozijn.go35
-rw-r--r--crawler/shop_mcwhisky.go14
-rw-r--r--crawler/shop_rumundco.go22
-rw-r--r--crawler/shop_whic.go12
-rw-r--r--crawler/shop_whiskyde.go16
-rw-r--r--crawler/shop_whiskysitenl.go12
-rw-r--r--crawler/shop_whiskyworld.go14
-rw-r--r--crawler/shop_whiskyzone.go12
13 files changed, 101 insertions, 102 deletions
diff --git a/crawler/database.go b/crawler/database.go
index 45f7fda..5de0fb2 100644
--- a/crawler/database.go
+++ b/crawler/database.go
@@ -122,15 +122,15 @@ func (app *App) save_offer(W []Angebot) error {
o.error_msg = err.Error()
o.error_ctx = fmt.Sprintf(`INSERT INTO angebot (shop, name, url, abv, volume, age, original_price, discounted_price, base_price, valid_until, image_url, spirit_type, created_at) VALUES (%d, "%s", "%s", %f, %f, %d, %d, %d, %d, %d, "%s", "%s", %d)`, o.Shop, o.Name, o.Url, o.Abv, o.Volume, o.Age, o.Original_price,
o.Discounted_price, o.Base_price, o.Valid_until, o.Image_url, o.Spirit_type, app.Now)
- WarnOffer(o, fmt.Sprintf("Save Offer: Inserting offer failed (%d)", found))
+ o.Warn(fmt.Sprintf("Save Offer: Inserting offer failed (%d)", found))
}
- DebugOffer(o, "database.go: Inserting offer.")
+ o.Debug("database.go: Inserting offer.")
} else if err != nil {
o.error_msg = err.Error()
o.error_ctx = fmt.Sprintf(strings.Replace(detect_duplicate_query, "?", `"%s"`, 1), o.Name)
- WarnOffer(o, "database.go: Duplicate query failed")
+ o.Warn("database.go: Duplicate query failed")
} else {
/*
* If everything went right we update the image url to reflect new changes.
@@ -140,13 +140,13 @@ func (app *App) save_offer(W []Angebot) error {
if err != nil {
o.error_msg = err.Error()
o.error_ctx = fmt.Sprintf(`UPDATE _intern_view SET image_url = %s, website_raw = %s WHERE name = %s AND shop_id = %d AND volume = %4.2f AND abv = %4.2f AND original_price = %d AND discounted_price = %d AND valid_until = %d`, o.Image_url, "redacted", o.Name, o.Shop, o.Volume, o.Abv, o.Original_price, o.Discounted_price, o.Valid_until)
- WarnOffer(o, "database.go: Preparing update_img_query failed")
+ o.Warn("database.go: Preparing update_img_query failed")
}
_, err = update_img_stmt.Exec(o.Image_url, o.Website, o.Name)
if err != nil {
o.error_msg = err.Error()
o.error_ctx = fmt.Sprintf(`UPDATE _intern_view SET image_url = %s WHERE name = %s AND shop_id = %d AND volume = %4.2f AND abv = %4.2f AND original_price = %d AND discounted_price = %d AND valid_until = %d`, o.Image_url, o.Name, o.Shop, o.Volume, o.Abv, o.Original_price, o.Discounted_price, o.Valid_until)
- WarnOffer(o, "database.go: Executing update_img_query failed")
+ o.Warn("database.go: Executing update_img_query failed")
}
}
}
@@ -179,17 +179,17 @@ func (app *App) remove_expired(W []Angebot, shop Shop) error {
}
if !app.offer_contains(W, offer_db) {
- DebugOffer(offer_db, "Contains not - Set to expire")
+ offer_db.Debug("Contains not - Set to expire")
expire_query := `UPDATE angebot SET valid_until = ? WHERE id = ?`
_, err = app.DB.Exec(expire_query, app.Now, offer_db.Id)
if err != nil {
offer_db.error_msg = err.Error()
offer_db.error_ctx = fmt.Sprintf("UPDATE angebot SET valid_until = %d WHERE id = %d", app.Now, offer_db.Id)
- WarnOffer(offer_db, "Remove expired: Update failed")
+ offer_db.Warn("Remove expired: Update failed")
return err
}
}
- DebugOffer(offer_db, "Contains! DOES NOT EXPIRE!")
+ offer_db.Debug("Contains! DOES NOT EXPIRE!")
}
return nil
diff --git a/crawler/log.go b/crawler/log.go
index d284a74..26f4abd 100644
--- a/crawler/log.go
+++ b/crawler/log.go
@@ -15,8 +15,8 @@ func Fatal(err error, msg string) {
log.Fatal(msg)
}
}
-func FatalOffer(offer Angebot, msg string) {
- log.WithFields(getFields(offer)).Fatal(msg)
+func (offer Angebot) Fatal(msg string) {
+ log.WithFields(offer.getFields()).Fatal(msg)
}
func Println(err error, msg string) {
@@ -31,9 +31,9 @@ func Println(err error, msg string) {
}
}
-func PrintlnOffer(offer Angebot, msg string) {
+func (offer Angebot) Println(msg string) {
- log.WithFields(getFields(offer)).Println(msg)
+ log.WithFields(offer.getFields()).Println(msg)
}
func Debug(err error, msg string) {
@@ -48,17 +48,17 @@ func Debug(err error, msg string) {
}
}
-func DebugOffer(offer Angebot, msg string) {
+func (offer Angebot) Debug(msg string) {
- log.WithFields(getFields(offer)).Debug(msg)
+ log.WithFields(offer.getFields()).Debug(msg)
}
-func WarnOffer(offer Angebot, msg string) {
+func (offer Angebot) Warn(msg string) {
- log.WithFields(getFields(offer)).Warn(msg)
+ log.WithFields(offer.getFields()).Warn(msg)
}
-func getFields(offer Angebot) log.Fields {
+func (offer Angebot) getFields() log.Fields {
return log.Fields{
"Name": offer.Name,
"Shop": offer.Shop,
diff --git a/crawler/post_process.go b/crawler/post_process.go
index bb1ca46..76fc70f 100644
--- a/crawler/post_process.go
+++ b/crawler/post_process.go
@@ -118,7 +118,7 @@ func (app *App) fix_db() error {
if err != nil {
offer_db.error_msg = err.Error()
offer_db.error_ctx = fmt.Sprintf(`UPDATE angebot SET name = "%s", age = %d WHERE id = %d`, offer_db.Name, offer_db.Age, offer_db.Id)
- WarnOffer(offer_db, "post_process.go: Update query failed.")
+ offer_db.Warn("post_process.go: Update query failed.")
}
}
diff --git a/crawler/sanitize.go b/crawler/sanitize.go
index db0faec..25e8a09 100644
--- a/crawler/sanitize.go
+++ b/crawler/sanitize.go
@@ -17,7 +17,7 @@ func sanitize_offer(angebote []Angebot, shop Shop, try int) []Angebot {
for _, offer := range angebote {
if offer.Spirit_type == "Wein" {
- DebugOffer(offer, "Sanitizer: Skip offer because it's wine")
+ offer.Debug("Sanitizer: Skip offer because it's wine")
continue
}
@@ -31,34 +31,34 @@ func sanitize_offer(angebote []Angebot, shop Shop, try int) []Angebot {
continue
}
if offer.Volume == 0 {
- WarnOffer(offer, "Sanitizer: Volume is zero")
+ offer.Warn("Sanitizer: Volume is zero")
continue
}
if offer.Original_price == 0 {
- WarnOffer(offer, "Sanitizer: Original price is zero")
+ offer.Warn("Sanitizer: Original price is zero")
continue
}
if offer.Discounted_price == 0 {
- WarnOffer(offer, "Sanitizer: Discounted price is zero")
+ offer.Warn("Sanitizer: Discounted price is zero")
continue
}
if offer.Base_price == 0 {
- WarnOffer(offer, "Sanitizer: Base price is zero")
+ offer.Warn("Sanitizer: Base price is zero")
continue
}
if offer.Url == "" {
- WarnOffer(offer, "Sanitizer: URL is empty")
+ offer.Warn("Sanitizer: URL is empty")
continue
}
if offer.Image_url == "" {
- WarnOffer(offer, "Sanitizer: Image-URL is empty")
+ offer.Warn("Sanitizer: Image-URL is empty")
continue
}
if err := sanitize_image_url(offer.Image_url); err != nil {
offer.error_ctx = offer.Image_url
offer.error_msg = err.Error()
- WarnOffer(offer, "Sanitizer: Image-URL is not valid")
+ offer.Warn("Sanitizer: Image-URL is not valid")
continue
}
@@ -66,7 +66,7 @@ func sanitize_offer(angebote []Angebot, shop Shop, try int) []Angebot {
offer.Website = ""
if offer.Age == 0 {
- DebugOffer(offer, "GREP")
+ offer.Debug("GREP")
}
W = append(W, offer)
@@ -258,7 +258,7 @@ func _check_abv_for_spirit_type(offer Angebot) bool {
*/
if offer.Abv == 0 {
- WarnOffer(offer, "Sanitizer: Abv is zero")
+ offer.Warn("Sanitizer: Abv is zero")
return false
}
diff --git a/crawler/shop_bottleworld.go b/crawler/shop_bottleworld.go
index 8722211..97ec162 100644
--- a/crawler/shop_bottleworld.go
+++ b/crawler/shop_bottleworld.go
@@ -33,7 +33,7 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price")
- PrintlnOffer(W, "Bottleworld: Converting original price failed")
+ W.Println("Bottleworld: Converting original price failed")
return
}
})
@@ -42,7 +42,7 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price")
- PrintlnOffer(W, "Bottleworld: Converting discounted price failed")
+ W.Println("Bottleworld: Converting discounted price failed")
return
}
})
@@ -53,7 +53,7 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = price_per_litre_noisy
- PrintlnOffer(W, "Bottleworld: Sanitizing base price failed")
+ W.Println("Bottleworld: Sanitizing base price failed")
return
}
W.Base_price = price_per_litre
@@ -70,14 +70,14 @@ func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
if W.Volume == 0 {
W.error_msg = "Bottleworld: Volume is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "Bottleworld: Volume is zero")
+ W.Println("Bottleworld: Volume is zero")
return
}
W.Abv, ctx = get_abv(e)
if W.Abv == 0 {
W.error_msg = "Bottleworld: Abv is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "Bottleworld: Abv is zero")
+ W.Println("Bottleworld: Abv is zero")
return
}
diff --git a/crawler/shop_drankdozijn.go b/crawler/shop_drankdozijn.go
index 5cd12d5..20018af 100644
--- a/crawler/shop_drankdozijn.go
+++ b/crawler/shop_drankdozijn.go
@@ -71,15 +71,14 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
tmp_spirit_type := tmp_desc["description"].(string)
if "Bier" == tmp_spirit_type {
- DebugOffer(W, "Drankdozijn: skip offer because it's beer")
+ W.Debug("Drankdozijn: skip offer because it's beer")
continue
}
W.Spirit_type = detect_spirit_type(tmp_desc["description"].(string))
if v, _ := api_data["price"]; v == nil {
- //DebugOffer(W, "Drankdozijn: Skip Offer")
- DebugOffer(W, "Drankdozijn: price is nil -> skip offer")
+ W.Debug("Drankdozijn: price is nil -> skip offer")
continue
}
W.Original_price, err = convert_price(api_data["price"].(string))
@@ -94,7 +93,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
}
if W.Discounted_price >= W.Original_price {
- DebugOffer(W, "Drankdozijn: Discounted price is not cheaper")
+ W.Debug("Drankdozijn: Discounted price is not cheaper")
continue
}
@@ -103,7 +102,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
// Check for bundle offer
if len(tmp_offer_url_img_map) > 1 {
- DebugOffer(W, "Drankdozijn: Skip Offer because of bundle")
+ W.Debug("Drankdozijn: Skip Offer because of bundle")
continue
}
@@ -119,7 +118,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
c.OnHTML(".product_top", func(e *colly.HTMLElement) {
if strings.Contains(W.Name, "+ gratis") || strings.Contains(W.Name, "& gratis") {
- DebugOffer(W, "Drankdozijn: Skip Offer because it contains gratis ware")
+ W.Debug("Drankdozijn: Skip Offer because it contains gratis ware")
return
}
@@ -132,7 +131,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price_l")
- PrintlnOffer(W, "Drankdozijn: Converting base price failed")
+ W.Println("Drankdozijn: Converting base price failed")
return
}
})
@@ -150,7 +149,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
if W.Volume == 0 {
W.error_msg = e.Text
W.error_ctx = err.Error()
- PrintlnOffer(W, "Drankdozijn: Volume is zero, returning")
+ W.Println("Drankdozijn: Volume is zero, returning")
return
}
case "Alkoholgehalt", "Alcoholpercentage":
@@ -158,7 +157,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
if W.Abv == 0 {
W.error_msg = "Drankdozijn: Abv is zero"
W.error_ctx = err.Error()
- PrintlnOffer(W, "Drankdozijn: abv is zero")
+ W.Println("Drankdozijn: abv is zero")
return
}
case "Kategorie", "Categorie":
@@ -167,7 +166,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
if "Champagner" == tmp_type {
if tmp_type != W.Spirit_type {
- DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type)
+ W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type)
W.Spirit_type = tmp_type
}
W.Spirit_type = tmp_type
@@ -176,19 +175,19 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
switch W.Spirit_type {
case "Cognac":
if tmp_type != W.Spirit_type {
- DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type)
+ W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type)
W.Spirit_type = tmp_type
}
W.Spirit_type = tmp_type
case "Brandy":
if tmp_type != W.Spirit_type {
- DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type)
+ W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type)
W.Spirit_type = tmp_type
}
W.Spirit_type = tmp_type
case "Sherry":
if tmp_type != W.Spirit_type {
- DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type)
+ W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type)
W.Spirit_type = tmp_type
}
W.Spirit_type = tmp_type
@@ -196,28 +195,28 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
retest_type := detect_spirit_type(W.Name)
if "Likör" != retest_type && "Verschiedenes" != retest_type {
if tmp_type != W.Spirit_type {
- DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+retest_type)
+ W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + retest_type)
W.Spirit_type = tmp_type
}
W.Spirit_type = detect_spirit_type(W.Name)
}
if "Tequila" == tmp_type {
if tmp_type != W.Spirit_type {
- DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type)
+ W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type)
W.Spirit_type = tmp_type
}
W.Spirit_type = tmp_type
}
if "Mezcal" == tmp_type {
if tmp_type != W.Spirit_type {
- DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type)
+ W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type)
W.Spirit_type = tmp_type
}
W.Spirit_type = tmp_type
}
if "Baijiu" == tmp_type {
if tmp_type != W.Spirit_type {
- DebugOffer(W, "Spirit Type Changed: "+W.Spirit_type+" -> "+tmp_type)
+ W.Debug("Spirit Type Changed: " + W.Spirit_type + " -> " + tmp_type)
W.Spirit_type = tmp_type
}
W.Spirit_type = tmp_type
@@ -243,7 +242,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
//log.Println("Visit " + W.Url)
}
- //DebugOffer(W, "DEBUG OFFER")
+ //W.Debug("DEBUG OFFER")
Offers = append(Offers, W)
}
diff --git a/crawler/shop_mcwhisky.go b/crawler/shop_mcwhisky.go
index 941f3b9..027aeee 100644
--- a/crawler/shop_mcwhisky.go
+++ b/crawler/shop_mcwhisky.go
@@ -37,7 +37,7 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price")
- PrintlnOffer(W, "MC Whisky: Converting original price failed")
+ W.Println("MC Whisky: Converting original price failed")
return
}
})
@@ -46,7 +46,7 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price")
- PrintlnOffer(W, "MC Whisky: Converting discounted price failed")
+ W.Println("MC Whisky: Converting discounted price failed")
return
}
})
@@ -57,7 +57,7 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = price_per_litre_noisy
- PrintlnOffer(W, "MC Whisky: Sanitizing base price failed")
+ W.Println("MC Whisky: Sanitizing base price failed")
return
}
@@ -69,14 +69,14 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
if volume_failed != "" {
W.error_msg = "MC Whisky: Volume failed"
W.error_ctx = volume_failed
- PrintlnOffer(W, "MC Whisky: Volume failed")
+ W.Println("MC Whisky: Volume failed")
return
}
abv_failed := e.Request.Ctx.Get("abv_failed")
if volume_failed != "" {
W.error_msg = "MC Whisky: Abv failed"
W.error_ctx = abv_failed
- PrintlnOffer(W, "MC Whisky: Abv failed")
+ W.Println("MC Whisky: Abv failed")
return
}
@@ -85,14 +85,14 @@ func (app *App) ScrapeMCWhisky(shop Shop) []Angebot {
if W.Volume == 0 {
W.error_msg = "MC Whisky: Volume is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "MC Whisky: Volume is zero")
+ W.Println("MC Whisky: Volume is zero")
return
}
W.Abv, ctx = get_abv(e)
if W.Abv == 0 {
W.error_msg = "MC Whisky: Abv is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "MC Whisky: Abv is zero")
+ W.Println("MC Whisky: Abv is zero")
return
}
diff --git a/crawler/shop_rumundco.go b/crawler/shop_rumundco.go
index 86d8b8a..3b7d08a 100644
--- a/crawler/shop_rumundco.go
+++ b/crawler/shop_rumundco.go
@@ -43,7 +43,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
if err != nil {
W.error_msg = "Rum & Co: Parsing Query from Shop-URL failed"
W.error_ctx = e.Request.URL.String()
- PrintlnOffer(W, "Rum & Co: Parsing Query from Shop-URL failed")
+ W.Println("Rum & Co: Parsing Query from Shop-URL failed")
}
switch param["kf"][0] {
case "29":
@@ -58,7 +58,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
W.error_msg = "Rum & Co: Query parameter has unexpected value"
W.error_ctx = param["kf"][0]
W.Url = e.Request.URL.String()
- PrintlnOffer(W, "Rum & Co: Detecting spirit type failed")
+ W.Println("Rum & Co: Detecting spirit type failed")
}
log.Debug("Rum & Co: Crawling " + W.Spirit_type + " with param kf=" + param["kf"][0])
@@ -75,7 +75,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
W.error_ctx = e.ChildText(".delivery-status")
W.Url = whisky_url
W.Name = whisky_name
- PrintlnOffer(W, "Rum & Co: Offer not available")
+ W.Println("Rum & Co: Offer not available")
return
}
@@ -93,21 +93,21 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
if "" == regular_price {
W.error_msg = "Rum & Co: No regular price found"
W.error_ctx = regular_price
- PrintlnOffer(W, "Rum & Co: No regular price found")
+ W.Println("Rum & Co: No regular price found")
return
}
W.Original_price, err = convert_price(regular_price)
if err != nil {
W.error_msg = err.Error()
W.error_ctx = regular_price
- PrintlnOffer(W, "Rum & Co: Original price: Convert price failed")
+ W.Println("Rum & Co: Original price: Convert price failed")
return
}
W.Discounted_price, err = convert_price(e.ChildText(".price-value"))
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price-value")
- PrintlnOffer(W, "Rum & Co: Discounted price: Convert price failed")
+ W.Println("Rum & Co: Discounted price: Convert price failed")
return
}
@@ -117,7 +117,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".value")
- PrintlnOffer(W, "Rum & Co: Base price: Sanitizing base price failed")
+ W.Println("Rum & Co: Base price: Sanitizing base price failed")
return
}
})
@@ -136,7 +136,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
if len(image_url_noisy_slice) < 2 {
W.error_msg = "Rum & Co: (Pagespeed) Image URL not found"
W.error_ctx = image_url_noisy
- PrintlnOffer(W, "Rum & Co: (Pagespeed) Image URL not found")
+ W.Println("Rum & Co: (Pagespeed) Image URL not found")
return
}
image_url_noisy = strings.Replace(image_url_noisy, image_url_noisy_slice[1], "", 1)
@@ -152,7 +152,7 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
if W.Volume == 0 {
W.error_msg = "Rum & Co: Volume is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "Rum & Co: Volume is zero")
+ W.Println("Rum & Co: Volume is zero")
return
}
@@ -164,14 +164,14 @@ func (app *App) ScrapeRumundCo(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = abv_noisy
- PrintlnOffer(W, "Rum & Co: Base price: Extracting ABV failed")
+ W.Println("Rum & Co: Base price: Extracting ABV failed")
return
}
}
if W.Abv == 0 {
W.error_msg = "Rum & Co: Abv is zero"
W.error_ctx = abv_noisy
- PrintlnOffer(W, "Rum & Co: Abv is zero")
+ W.Println("Rum & Co: Abv is zero")
return
}
W.Website = e.Request.Ctx.Get("website")
diff --git a/crawler/shop_whic.go b/crawler/shop_whic.go
index 93bff23..b21c042 100644
--- a/crawler/shop_whic.go
+++ b/crawler/shop_whic.go
@@ -39,7 +39,7 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price")
- PrintlnOffer(W, "Whic: Converting original price failed")
+ W.Println("Whic: Converting original price failed")
return
}
})
@@ -48,7 +48,7 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price")
- PrintlnOffer(W, "Whic: Converting discounted price failed")
+ W.Println("Whic: Converting discounted price failed")
return
}
})
@@ -59,7 +59,7 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = base_price_noisy
- PrintlnOffer(W, "Whic: Sanitizing base price failed")
+ W.Println("Whic: Sanitizing base price failed")
return
}
@@ -72,7 +72,7 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = img_link_noisy
- WarnOffer(W, "Whic: Parsing document in Goquery failed")
+ W.Println("Whic: Parsing document in Goquery failed")
return
}
W.Image_url, _ = doc.Find("img").Attr("src")
@@ -84,14 +84,14 @@ func (app *App) ScrapeWhic(shop Shop) []Angebot {
if W.Volume == 0 {
W.error_msg = "Whic: Volume is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "Whic: Volume is zero")
+ W.Println("Whic: Volume is zero")
return
}
W.Abv, ctx = get_abv(e)
if W.Abv == 0 {
W.error_msg = "Whic: Abv is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "Whic: Abv is zero")
+ W.Println("Whic: Abv is zero")
return
}
diff --git a/crawler/shop_whiskyde.go b/crawler/shop_whiskyde.go
index d3087ca..d06f522 100644
--- a/crawler/shop_whiskyde.go
+++ b/crawler/shop_whiskyde.go
@@ -37,7 +37,7 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText("del")
- PrintlnOffer(W, "Whisky.de: Converting original price failed")
+ W.Println("Whisky.de: Converting original price failed")
return
}
})
@@ -46,7 +46,7 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".article-price-default")
- PrintlnOffer(W, "Whisky.de: Converting discounted price failed")
+ W.Println("Whisky.de: Converting discounted price failed")
return
}
})
@@ -64,7 +64,7 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
if !strings.Contains(text_noisy, "Liter") {
W.error_ctx = text_noisy
W.error_msg = "Whisky.de: String 'Liter' not found."
- PrintlnOffer(W, "Whisky.de: String 'Liter' not found.")
+ W.Println("Whisky.de: String 'Liter' not found.")
return
}
@@ -73,27 +73,27 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = text_noisy
- PrintlnOffer(W, "Whisky.de: Extracting volume failed")
+ W.Println("Whisky.de: Extracting volume failed")
return
}
W.Abv, err = extract_abv(abv_noisy)
if err != nil {
W.error_msg = err.Error()
W.error_ctx = abv_noisy
- PrintlnOffer(W, "Whisky.de: Extracting abv failed")
+ W.Println("Whisky.de: Extracting abv failed")
return
}
if W.Volume == 0 {
W.error_msg = "Whisky.de: Volume is zero"
W.error_ctx = text_noisy
- PrintlnOffer(W, "Whisky.de: Volume is zero")
+ W.Println("Whisky.de: Volume is zero")
return
}
if W.Abv == 0 {
W.error_msg = "Whisky.de: Abv is zero"
W.error_ctx = abv_noisy
- PrintlnOffer(W, "Whisky.de: Abv is zero")
+ W.Println("Whisky.de: Abv is zero")
return
}
@@ -101,7 +101,7 @@ func (app *App) ScrapeWhiskyde(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".article-unitprice-default")
- PrintlnOffer(W, "Whisky.de: Converting base price failed")
+ W.Println("Whisky.de: Converting base price failed")
return
}
diff --git a/crawler/shop_whiskysitenl.go b/crawler/shop_whiskysitenl.go
index e3ae075..c9a50b0 100644
--- a/crawler/shop_whiskysitenl.go
+++ b/crawler/shop_whiskysitenl.go
@@ -44,14 +44,14 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = regular_price
- PrintlnOffer(W, "Whiskysite.nl: Extracting original price failed")
+ W.Println("Whiskysite.nl: Extracting original price failed")
return
}
W.Discounted_price, err = convert_price(discounted_price)
if err != nil {
W.error_msg = err.Error()
W.error_ctx = discounted_price
- PrintlnOffer(W, "Whiskysite.nl: Extracting discounted price failed")
+ W.Println("Whiskysite.nl: Extracting discounted price failed")
return
}
@@ -63,13 +63,13 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
if volume_failed != "" {
W.error_msg = "Whiskysite.nl: Extracting volume via Liter-Regex failed"
W.error_ctx = volume_failed
- PrintlnOffer(W, "Whiskysite.nl: Extracting volume via Liter-Regex failed")
+ W.Println("Whiskysite.nl: Extracting volume via Liter-Regex failed")
return
}
if e.Request.Ctx.Get("abv_failed") != "" {
W.error_msg = "Whiskysite.nl: Extracting abv via Abv-Regex failed"
W.error_ctx = e.Request.Ctx.Get("volume_failed")
- PrintlnOffer(W, "Whiskysite.nl: Extracting abv via Abv-Regex failed")
+ W.Println("Whiskysite.nl: Extracting abv via Abv-Regex failed")
return
}
@@ -78,14 +78,14 @@ func (app *App) ScrapeWhiskysitenl(shop Shop) []Angebot {
if W.Volume == 0 {
W.error_msg = "Whiskysite.nl: Extracting volume failed"
W.error_ctx = ctx
- PrintlnOffer(W, "Whiskysite.nl: Extracting volume failed")
+ W.Println("Whiskysite.nl: Extracting volume failed")
return
}
W.Abv, ctx = get_abv(e)
if W.Abv == 0 {
W.error_msg = "Whiskysite.nl: Extracting abv failed"
W.error_ctx = ctx
- PrintlnOffer(W, "Whiskysite.nl: Extracting abv failed")
+ W.Println("Whiskysite.nl: Extracting abv failed")
return
}
diff --git a/crawler/shop_whiskyworld.go b/crawler/shop_whiskyworld.go
index 3f0874d..78e7244 100644
--- a/crawler/shop_whiskyworld.go
+++ b/crawler/shop_whiskyworld.go
@@ -43,7 +43,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = regular_price
- PrintlnOffer(W, "Whiskyworld: Converting original price failed")
+ W.Println("Whiskyworld: Converting original price failed")
return
}
@@ -51,7 +51,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".uvp")
- PrintlnOffer(W, "Whiskyworld: Converting discounted price failed")
+ W.Println("Whiskyworld: Converting discounted price failed")
return
}
@@ -61,13 +61,13 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = text_noisy
- PrintlnOffer(W, "Whiskyworld: Extracting volume failed")
+ W.Println("Whiskyworld: Extracting volume failed")
return
}
if W.Volume == 0 {
W.error_msg = "Whiskyworld: Volume is zero"
W.error_ctx = text_noisy
- PrintlnOffer(W, "Whiskyworld: Volume is zero")
+ W.Println("Whiskyworld: Volume is zero")
return
}
abv_noisy := strings.TrimSpace(strings.SplitAfter(text_noisy, "Liter")[1])
@@ -76,13 +76,13 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = abv_noisy
- PrintlnOffer(W, "Whiskyworld: Extracting abv failed")
+ W.Println("Whiskyworld: Extracting abv failed")
return
}
if W.Abv == 0 {
W.error_msg = "Whiskyworld: Abv is zero"
W.error_ctx = abv_noisy
- PrintlnOffer(W, "Whiskyworld: Abv is zero")
+ W.Println("Whiskyworld: Abv is zero")
return
}
})
@@ -95,7 +95,7 @@ func (app *App) ScrapeWhiskyworld(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = base_price_noisy
- PrintlnOffer(W, "Whiskyworld: Sanitizing base price failed")
+ W.Println("Whiskyworld: Sanitizing base price failed")
return
}
}
diff --git a/crawler/shop_whiskyzone.go b/crawler/shop_whiskyzone.go
index dbaf0ba..ffb124d 100644
--- a/crawler/shop_whiskyzone.go
+++ b/crawler/shop_whiskyzone.go
@@ -37,7 +37,7 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot {
e.Request.Visit(W.Url)
if "sold_out" == e.Request.Ctx.Get("sold_out") {
- PrintlnOffer(W, "Whiskyzone: Sold out")
+ W.Println("Whiskyzone: Sold out")
return
}
@@ -46,7 +46,7 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.Request.Ctx.Get("discounted_price")
- PrintlnOffer(W, "Whiskyzone: Convert discounted price failed")
+ W.Println("Whiskyzone: Convert discounted price failed")
return
}
@@ -54,7 +54,7 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.Request.Ctx.Get("original_price")
- PrintlnOffer(W, "Whiskyzone: Convert original price failed")
+ W.Println("Whiskyzone: Convert original price failed")
return
}
@@ -63,14 +63,14 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot {
if W.Volume == 0 {
W.error_msg = "Whiskyzone: Volume is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "Whiskyzone: Volume is zero")
+ W.Println("Whiskyzone: Volume is zero")
return
}
W.Abv, ctx = get_abv(e)
if W.Abv == 0 {
W.error_msg = "Whiskyzone: Abv is zero"
W.error_ctx = ctx
- PrintlnOffer(W, "Whiskyzone: Abv is zero")
+ W.Println("Whiskyzone: Abv is zero")
return
}
@@ -82,7 +82,7 @@ func (app *App) ScrapeWhiskyzone(shop Shop) []Angebot {
if err != nil {
W.error_msg = err.Error()
W.error_ctx = base_price
- PrintlnOffer(W, "Whiskyzone: Extracting base price failed")
+ W.Println("Whiskyzone: Extracting base price failed")
return
}
}