From 71950479fbd6088f249e5fda3b180f294d1d745d Mon Sep 17 00:00:00 2001
From: Max
Date: Tue, 6 Feb 2018 00:35:39 +0100
Subject: Moves crawler to designated directory.

---
 crawler/main.go | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 140 insertions(+)
 create mode 100644 crawler/main.go

(limited to 'crawler/main.go')

diff --git a/crawler/main.go b/crawler/main.go
new file mode 100644
index 0000000..5255e1c
--- /dev/null
+++ b/crawler/main.go
@@ -0,0 +1,140 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"time"
+
+	_ "database/sql"
+	_ "github.com/go-sql-driver/mysql"
+	//_ "github.com/mattn/go-sqlite3"
+
+	"github.com/jmoiron/sqlx"
+)
+
+type App struct {
+	Offers []Angebot
+	Shops  []Shop
+	Config *Config
+	DB     *sqlx.DB
+	Now    int64
+	Debug  bool
+}
+
+type Angebot struct {
+	Id               int
+	Name             string
+	Shop             int
+	Url              string
+	Original_price   int
+	Discounted_price int
+	Image_url        string
+	Spirit_type      string
+	Valid_until      int
+}
+
+type Shop struct {
+	Id             int
+	Name           string
+	Url            string
+	Logo_url       string
+	Shipping_costs int
+	Free_shipping  string
+}
+
+func main() {
+
+	var err error
+
+	app := App{Config: &Config{}}
+	app.Config.parseConfig("")
+
+	app.Now = time.Now().Unix()
+
+	if "sqlite3" == app.Config.DBDriver {
+		//app.DB, err = sqlx.Connect(app.Config.DBDriver, app.Config.DBPath)
+		app.DB, err = sqlx.Connect(app.Config.DBDriver, app.Config.DBPath)
+	} else {
+
+		if app.Config.Debug {
+			log.Println(app.Config.DBUser + ":" + app.Config.DBPassword + "@tcp(" + app.Config.DBHost + ":" + app.Config.DBPort + ")/" + app.Config.DBDBName + app.Config.DBOptions)
+		}
+		app.DB, err = sqlx.Connect(app.Config.DBDriver, app.Config.DBUser+":"+app.Config.DBPassword+"@tcp("+app.Config.DBHost+":"+app.Config.DBPort+")/"+app.Config.DBDBName+app.Config.DBOptions)
+	}
+	defer app.DB.Close()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = app.createTables()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = app.insertShops()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	shops, err := app.getShops()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	W := ScrapeHTML(shops)
+
+	err = app.save_offer(W)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = app.remove_expired(W)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func printName(W []Angebot, name string) {
+	return
+	fmt.Println("-------------------")
+	fmt.Println("Sonderangebote von " + name)
+	fmt.Println("-------------------")
+
+	output, err := json.MarshalIndent(W, "", "	")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(string(output))
+}
+
+func ScrapeHTML(shops []Shop) []Angebot {
+	var W []Angebot
+
+	for _, shop := range shops {
+
+		switch shop.Name {
+		case "Bottleworld":
+			W = append(W, ScrapeBottleWord(shop)...)
+		case "MC Whisky":
+			W = append(W, ScrapeMCWhisky(shop)...)
+		case "Rum & Co":
+			W = append(W, ScrapeRumundCo(shop)...)
+		case "Whic":
+			W = append(W, ScrapeWhic(shop)...)
+		case "Whisky.de":
+			W = append(W, ScrapeWhiskyde(shop)...)
+		case "Whiskysite.nl":
+			W = append(W, ScrapeWhiskysitenl(shop)...)
+		case "Whisky World":
+			W = append(W, ScrapeWhiskyworld(shop)...)
+		case "Whiskyzone":
+			W = append(W, ScrapeWhiskyzone(shop)...)
+		default:
+			log.Println(shop.Name + ": No Crawler")
+		}
+	}
+
+	return W
+}
-- 
cgit v1.2.3