Adds URL shorting with polr. (crawler)

4 files changed, 82 insertions, 2 deletions

diff --git a/crawler/config.go b/crawler/config.go
index af02a6c..552d57a 100644
--- a/crawler/config.go
+++ b/crawler/config.go
@@ -75,4 +75,6 @@ func (c *Config) setsConfig() {
 	c.DBOptions = viper.GetString("DB_Options")
 	c.DBPath = viper.GetString("DB_Path")
 	c.Debug = viper.GetBool("Debug")
+	c.Polr_URL = viper.GetString("Polr_URL")
+	c.Polr_API_Key = viper.GetString("Polr_API_Key")
 }
diff --git a/crawler/database.go b/crawler/database.go
index 278d891..d0bd632 100644
--- a/crawler/database.go
+++ b/crawler/database.go
@@ -58,7 +58,7 @@ func (app *App) createTables() error {
 
 		view_query := `CREATE OR REPLACE VIEW ` + v + `_view AS
 		SELECT
-			angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url,original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url,
+			angebot.id, angebot.name, angebot.abv, angebot.volume, angebot.url as long_url, angebot.short_url as url,original_price/100 as original_price, discounted_price/100 as discounted_price, angebot.base_price/100 as base_price, image_url,
 			shop.name as shop, shop.url as shop_url, shop.shipping_costs/100 as shipping_costs, shop.free_shipping, ROUND(100-((discounted_price/original_price)*100)) AS procent, spirit_type, created_at
 		FROM angebot
 		JOIN shop ON angebot.shop = shop.id
@@ -116,7 +116,7 @@ func (app *App) save_offer(W []Angebot) error {
 
 func (app *App) remove_expired(W []Angebot, shop_id int) error {
 
-	query := `SELECT id, shop, name, url, original_price, discounted_price FROM angebot WHERE shop = ? AND created_at < ? AND valid_until IS NULL`
+	query := `SELECT id, shop, name, url, original_price, discounted_price FROM all_view WHERE shop = ? AND created_at < ?`
 
 	rows, err := app.DB.Queryx(query, shop_id, app.Now)
 	if err != nil {
diff --git a/crawler/main.go b/crawler/main.go
index 779ebba..15e854c 100644
--- a/crawler/main.go
+++ b/crawler/main.go
@@ -29,6 +29,7 @@ type Angebot struct {
 	Volume           float32
 	Shop             int
 	Url              string
+	Short_url        string
 	Original_price   int
 	Discounted_price int
 	Base_price       int
@@ -91,6 +92,12 @@ func main() {
 	}
 
 	app.ScrapeHTML(shops)
+
+	// short url
+	err = app.post_process()
+	if err != nil {
+		log.Fatal(err)
+	}
 }
 
 func printName(W []Angebot, name string) {
diff --git a/crawler/post_process.go b/crawler/post_process.go
new file mode 100644
index 0000000..e527911
--- /dev/null
+++ b/crawler/post_process.go
@@ -0,0 +1,71 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+)
+
+func (app *App) post_process() error {
+
+	return app.short_url()
+}
+
+func (app *App) short_url() error {
+
+	var Angebote []Angebot
+
+	query := `SELECT id, long_url as url FROM all_view WHERE url IS NULL AND long_url IS NOT NULL`
+
+	rows, err := app.DB.Queryx(query)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var offer_db Angebot
+		err = rows.StructScan(&offer_db)
+		if err != nil {
+			return err
+		}
+
+		v := url.Values{}
+		v.Set("key", app.Config.Polr_API_Key)
+		v.Add("url", offer_db.Url)
+		polr_url := app.Config.Polr_URL + "?" + v.Encode()
+
+		if app.Config.Debug {
+			log.Println("DEBUG: polr_url: " + polr_url + " ( " + offer_db.Url + " )")
+		}
+		resp, err := http.Get(polr_url)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+
+		short_url, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+
+		offer_db.Short_url = string(short_url)
+		if app.Config.Debug {
+			log.Println("DEBUG: short_url: " + string(short_url) + " ( " + offer_db.Url + " )")
+		}
+
+		Angebote = append(Angebote, offer_db)
+	}
+
+	for _, offer := range Angebote {
+		_, err := app.DB.Exec(`UPDATE angebot SET short_url = ? WHERE id = ?`, offer.Short_url, offer.Id)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+
+}