1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
package main
import (
"regexp"
"strings"
// "github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly"
)
func (app *App) ScrapeBottleWord(shop Shop) []Angebot {
Shop_url := "https://www.bottleworld.de/aktuelle-sonderpreise/show/all"
Whiskys := []Angebot{}
c := app.customCollector([]string{"bottleworld.de", "www.bottleworld.de"})
c.OnHTML("li.item", func(e *colly.HTMLElement) {
W := Angebot{}
W.Shop = shop.Id
whisky_name := e.ChildText("h2 > a")
var err error
whisky_url := e.ChildAttr("a", "href")
W.Name = whisky_name
W.Url = whisky_url
e.ForEach(".price-box", func(i int, e *colly.HTMLElement) {
e.ForEach(".old-price", func(i int, e *colly.HTMLElement) {
W.Original_price, err = convert_price(e.ChildText(".price"))
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price")
W.Println("Bottleworld: Converting original price failed")
return
}
})
e.ForEach(".special-price", func(i int, e *colly.HTMLElement) {
W.Discounted_price, err = convert_price(e.ChildText(".price"))
if err != nil {
W.error_msg = err.Error()
W.error_ctx = e.ChildText(".price")
W.Println("Bottleworld: Converting discounted price failed")
return
}
})
})
price_per_litre_noisy := e.ChildText(".price-per-liter")
price_per_litre, err := sanitize_base_price(price_per_litre_noisy)
if err != nil {
W.error_msg = err.Error()
W.error_ctx = price_per_litre_noisy
W.Println("Bottleworld: Sanitizing base price failed")
return
}
W.Base_price = price_per_litre
W.Image_url = e.ChildAttr("img", "src")
erro := e.Request.Visit(W.Url)
if erro != nil {
Warn(nil, W.Url+" "+erro.Error())
}
var ctx string
W.Volume, ctx = get_volume(e)
if W.Volume == 0 {
W.error_msg = "Bottleworld: Volume is zero"
W.error_ctx = ctx
W.Println("Bottleworld: Volume is zero")
return
}
W.Abv, ctx = get_abv(e)
if W.Abv == 0 {
W.error_msg = "Bottleworld: Abv is zero"
W.error_ctx = ctx
W.Println("Bottleworld: Abv is zero")
return
}
W.Spirit_type = e.Request.Ctx.Get("spirit_type")
W.Website = e.Request.Ctx.Get("website")
Whiskys = append(Whiskys, W)
})
c.OnHTML("#product-attribute-specs-table", func(e *colly.HTMLElement) {
// Filter nonalcoholic drinks.
has_abv := false
e.ForEach("tr", func(i int, e *colly.HTMLElement) {
td_str := e.ChildText("td")
matched, err := regexp.MatchString("^[0-9]+([,.][0-9]+)? l$", td_str)
if err != nil {
Fatal(err, "Bottleworld: Volume and ABV Regex failed")
}
if matched {
e.Request.Ctx.Put("volume", td_str)
} else if strings.Contains(td_str, "%") {
has_abv = true
e.Request.Ctx.Put("abv", td_str)
}
})
if !has_abv {
e.Request.Ctx.Put("abv", "")
e.Request.Ctx.Put("debug_info", "No abv found on page thus set to zero. (nonalcoholic?)")
}
e.Request.Ctx.Put("website", string(e.Response.Body))
})
c.OnHTML(".short-description", func(e *colly.HTMLElement) {
if e.Request.URL.String() == Shop_url {
return
}
text_noisy := e.ChildText(".std")
e.Request.Ctx.Put("spirit_type", detect_spirit_type(text_noisy))
})
err := c.Visit(Shop_url)
if err != nil {
shop.error_msg = err.Error()
shop.error_ctx = Shop_url
shop.Warn("Crawling failed")
}
return Whiskys
}
|
