Adds URL shorting with polr. (crawler)

1 files changed, 71 insertions, 0 deletions

diff --git a/crawler/post_process.go b/crawler/post_process.go
new file mode 100644
index 0000000..e527911
--- /dev/null
+++ b/crawler/post_process.go
@@ -0,0 +1,71 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+)
+
+func (app *App) post_process() error {
+
+	return app.short_url()
+}
+
+func (app *App) short_url() error {
+
+	var Angebote []Angebot
+
+	query := `SELECT id, long_url as url FROM all_view WHERE url IS NULL AND long_url IS NOT NULL`
+
+	rows, err := app.DB.Queryx(query)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var offer_db Angebot
+		err = rows.StructScan(&offer_db)
+		if err != nil {
+			return err
+		}
+
+		v := url.Values{}
+		v.Set("key", app.Config.Polr_API_Key)
+		v.Add("url", offer_db.Url)
+		polr_url := app.Config.Polr_URL + "?" + v.Encode()
+
+		if app.Config.Debug {
+			log.Println("DEBUG: polr_url: " + polr_url + " ( " + offer_db.Url + " )")
+		}
+		resp, err := http.Get(polr_url)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+
+		short_url, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+
+		offer_db.Short_url = string(short_url)
+		if app.Config.Debug {
+			log.Println("DEBUG: short_url: " + string(short_url) + " ( " + offer_db.Url + " )")
+		}
+
+		Angebote = append(Angebote, offer_db)
+	}
+
+	for _, offer := range Angebote {
+		_, err := app.DB.Exec(`UPDATE angebot SET short_url = ? WHERE id = ?`, offer.Short_url, offer.Id)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+
+}