Better handling for nil interfaces. (crawler)

1 files changed, 36 insertions, 26 deletions

diff --git a/crawler/shop_drankdozijn.go b/crawler/shop_drankdozijn.go
index a76148f..e4fd444 100644
--- a/crawler/shop_drankdozijn.go
+++ b/crawler/shop_drankdozijn.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 	//"strconv"
 
-	//log "github.com/Sirupsen/logrus"
+	log "github.com/Sirupsen/logrus"
 	"github.com/gocolly/colly"
 )
 
@@ -20,7 +20,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
 	 */
 	API_URL := "https://api.drankdozijn.nl/sale-products?country=DE&language=de"
 
-	c := http.Client{}
+	http_client := http.Client{}
 
 	req, err := http.NewRequest(http.MethodGet, API_URL, nil)
 	if err != nil {
@@ -31,7 +31,7 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
 	req.Header.Set("accept", "application/json")
 	req.Header.Set("User-Agent", "")
 
-	api_resp, err := c.Do(req)
+	api_resp, err := http_client.Do(req)
 	if err != nil {
 		// TODO
 		panic(err)
@@ -45,9 +45,11 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
 
 	var tmp_api_map map[string]interface{}
 
-	err = json.Unmarshal(api_body, tmp_api_map)
+	err = json.Unmarshal(api_body, &tmp_api_map)
 	if err != nil {
 		// TODO
+
+		log.Println("json unmarshal failed")
 		panic(err)
 	}
 
@@ -64,8 +66,18 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
 
 		W.Name = api_data["saleDescription"].(string)
 
-		W.Spirit_type = detect_spirit_type(api_data["description"].(string))
+		tmp_desc := api_data["group"].(map[string]interface{})
+		W.Spirit_type = detect_spirit_type(tmp_desc["description"].(string))
 
+		//v, ok := api_data["price"]
+		//log.Println(v, ok)
+		if v, _ := api_data["price"]; v == nil {
+			log.Println("price is nil -> skip offer")
+			//DebugOffer(W, "Drankdozijn: Skip Offer")
+			continue
+		} else {
+			//log.Println("price is NOT nil -> NOT SKIPPING!")
+		}
 		W.Original_price, err = convert_price(api_data["price"].(string))
 		if err != nil {
 			// TODO
@@ -86,18 +98,8 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
 
 		c := app.customCollector([]string{"drankdozijn.de", "drankdozijn.nl"})
 
-		err = c.Visit(W.Url)
-		if err != nil {
-			Warn(nil, shop.Name+": Error (Visit): "+err.Error())
-		}
-
 		c.OnHTML(".product_top", func(e *colly.HTMLElement) {
-			/*
-				if e.Request.URL.String() != Shop_url && e.Request.URL.String() != Async_url {
-					//Debug(nil, "Drankdozijn.de: Request url ("+e.Request.URL.String()+") is not shop url ("+Shop_url+").")
-					return
-				}
-			*/
+			//	log.Println(".product_top")
 
 			e.ForEach(".product_image", func(i int, e *colly.HTMLElement) {
 				W.Image_url = e.ChildAttr("img", "src")
@@ -160,20 +162,16 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
 					return
 				}
 			}
-
-			W.Website = e.Request.Ctx.Get("website")
-
-			//DebugOffer(W, "DEBUG")
-
-			Offers = append(Offers, W)
 		})
 
 		c.OnHTML(".main_price", func(e *colly.HTMLElement) {
+			//log.Println(".main_price")
 			//e.Request.Ctx.Put("base_price", strings.TrimPrefix(e.ChildText(".price_l"), "/L"))
 			e.Request.Ctx.Put("base_price", e.ChildText(".price_l"))
 		})
 
 		c.OnHTML(".main_description", func(e *colly.HTMLElement) {
+			//log.Println(".main_price")
 			prev := ""
 			count := 0
 			e.ForEach(".col-xs-6", func(i int, e *colly.HTMLElement) {
@@ -200,13 +198,25 @@ func (app *App) ScrapeDrankdozijn(shop Shop) []Angebot {
 
 		c.OnHTML("body", func(e *colly.HTMLElement) {
 			/*
-				if e.Request.URL.String() == Shop_url {
-					return
-				}
+				log.Println("body")
+				e.Request.Ctx.Put("website", string(e.Response.Body))
+
+				W.Website = e.Request.Ctx.Get("website")
 			*/
-			e.Request.Ctx.Put("website", string(e.Response.Body))
+			W.Website = string(e.Response.Body)
+			//W.Website = string(e.Response.Body)
 		})
 
+		err = c.Visit(W.Url)
+		if err != nil {
+			Warn(nil, shop.Name+": Error (Visit): "+err.Error())
+		} else {
+			log.Println("Visit " + W.Url)
+		}
+
+		//DebugOffer(W, "DEBUG")
+
+		Offers = append(Offers, W)
 	}
 
 	return Offers