From ed6ab4da59f80bf9fa2cbf15da5c9167dff44ea4 Mon Sep 17 00:00:00 2001
From: horus
Date: Fri, 16 Feb 2018 16:57:10 +0100
Subject: Adds structured logging. (crawler)

---
 crawler/database.go | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

(limited to 'crawler/database.go')

diff --git a/crawler/database.go b/crawler/database.go
index 1f38ca1..2608c59 100644
--- a/crawler/database.go
+++ b/crawler/database.go
@@ -2,8 +2,9 @@ package main
 
 import (
 	"database/sql"
-	"log"
 	"strings"
+
+	log "github.com/Sirupsen/logrus"
 )
 
 func (app *App) createTables() error {
@@ -59,8 +60,10 @@ func (app *App) createTables() error {
 
 		view_query := `CREATE OR REPLACE VIEW ` + v + `_view AS
 		SELECT
-			angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url,original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url,
-			shop.name as shop, shop.short_url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping, ROUND(100-((discounted_price/original_price)*100)) AS procent, spirit_type, created_at
+			angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url, spirit_type,
+			original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url,
+			shop.name as shop, shop.id as shop_id, shop.short_url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping,
+			ROUND(100-((discounted_price/original_price)*100)) AS procent, created_at
 		FROM angebot
 		JOIN shop ON angebot.shop = shop.id
 		WHERE 
@@ -80,6 +83,7 @@ func (app *App) save_offer(W []Angebot) error {
 
 	stmt, err := app.DB.Prepare(query)
 	if err != nil {
+		Debug(err, "Save Offer: Preparing query failed")
 		return err
 	}
 	defer stmt.Close()
@@ -92,7 +96,7 @@ func (app *App) save_offer(W []Angebot) error {
 			continue
 		}
 
-		err := app.DB.QueryRow("SELECT 1 FROM angebot WHERE shop = ? AND name = ? AND url = ? AND original_price = ? AND discounted_price = ? AND spirit_type = ?", o.Shop, o.Name, o.Url, o.Original_price, o.Discounted_price, o.Spirit_type).Scan(&found)
+		err := app.DB.QueryRow("SELECT 1 FROM all_view WHERE shop_id = ? AND name = ? AND long_url = ? AND original_price = ? AND discounted_price = ? AND spirit_type = ?", o.Shop, o.Name, o.Url, o.Original_price, o.Discounted_price, o.Spirit_type).Scan(&found)
 
 		/*
 		 */
@@ -105,6 +109,7 @@ func (app *App) save_offer(W []Angebot) error {
 				_, err = stmt.Exec(o.Shop, o.Name, o.Url, o.Abv, o.Volume, o.Original_price, o.Discounted_price, o.Base_price, o.Valid_until, o.Image_url, o.Website, o.Spirit_type, app.Now)
 			}
 			if err != nil {
+				Debug(err, "Save Offer: Inserting offer failed")
 				return err
 			}
 
@@ -115,12 +120,14 @@ func (app *App) save_offer(W []Angebot) error {
 	return nil
 }
 
-func (app *App) remove_expired(W []Angebot, shop_id int) error {
+func (app *App) remove_expired(W []Angebot, shop Shop) error {
 
-	query := `SELECT id, shop, name, url, original_price, discounted_price FROM all_view WHERE shop = ? AND created_at < ?`
+	query := `SELECT id, shop, name, url, original_price, discounted_price FROM angebot WHERE shop = ? AND created_at < ?
+		AND (valid_until IS NULL OR valid_until > ?)`
 
-	rows, err := app.DB.Queryx(query, shop_id, app.Now)
+	rows, err := app.DB.Queryx(query, shop.Id, app.Now, app.Now)
 	if err != nil {
+		Debug(err, "Remove expired: Query failed")
 		return err
 	}
 	defer rows.Close()
@@ -130,16 +137,20 @@ func (app *App) remove_expired(W []Angebot, shop_id int) error {
 		err = rows.StructScan(&offer_db)
 
 		if err != nil {
+			Debug(err, "Remove expired: Struct scan failed")
 			return err
 		}
 
 		if !app.offer_contains(W, offer_db) {
+			DebugOffer(offer_db, "Contains not - Set to expire")
 			expire_query := `UPDATE angebot SET valid_until = ? WHERE id = ?`
 			_, err = app.DB.Exec(expire_query, app.Now, offer_db.Id)
 			if err != nil {
+				Debug(err, "Remove expired: Update failed")
 				return err
 			}
 		}
+		DebugOffer(offer_db, "Contains! DOES NOT EXPIRE!")
 	}
 
 	return nil
@@ -154,8 +165,6 @@ func (app *App) offer_contains(W []Angebot, offer_db Angebot) bool {
 		if (v.Shop == offer_db.Shop) && (v.Name == offer_db.Name) && (v.Original_price == offer_db.Original_price) && (v.Discounted_price == offer_db.Discounted_price) {
 
 			if app.Config.Debug {
-				log.Println("Contains: " + v.Name)
-				log.Println("")
 			}
 
 			return true
@@ -164,8 +173,7 @@ func (app *App) offer_contains(W []Angebot, offer_db Angebot) bool {
 	}
 
 	if app.Config.Debug {
-		log.Println("Contains not: " + offer_db.Name)
-		log.Println("")
+		log.Debug("Contains not: " + offer_db.Name)
 	}
 
 	return false
-- 
cgit v1.2.3