Adds repair function. (crawler)

2 files changed, 73 insertions, 11 deletions

diff --git a/crawler/config.go b/crawler/config.go
index c9b17a4..f74253e 100644
--- a/crawler/config.go
+++ b/crawler/config.go
@@ -5,19 +5,21 @@ import (
 )
 
 type Config struct {
-	DBDriver          string
-	DBDBName          string
-	DBHost            string
-	DBPort            string
-	DBUser            string
-	DBPassword        string
-	DBOptions         string
-	DBPath            string // for sqlite
+	DBDriver   string
+	DBDBName   string
+	DBHost     string
+	DBPort     string
+	DBUser     string
+	DBPassword string
+	DBOptions  string
+	DBPath     string // for sqlite
+
 	DisableURLShorter bool
 	Polr_URL          string
 	Polr_API_Key      string
 
-	Debug bool
+	Debug       bool
+	FixDatabase bool
 }
 
 // Parses the configuration and sets the configuration struct.
@@ -31,6 +33,7 @@ func (c *Config) parseConfig(configFile string) {
 	viper.SetDefault("DB_Path", "./alkobote.db")
 
 	viper.SetDefault("Debug", false)
+	viper.SetDefault("FixDatabase", false)
 	viper.SetDefault("DisableURLShorter", false)
 
 	// Name of the configuration file
@@ -72,6 +75,7 @@ func (c *Config) setsConfig() {
 	c.DBOptions = viper.GetString("DB_Options")
 	c.DBPath = viper.GetString("DB_Path")
 	c.Debug = viper.GetBool("Debug")
+	c.FixDatabase = viper.GetBool("FixDatabase")
 	c.DisableURLShorter = viper.GetBool("DisableURLShorter")
 	c.Polr_URL = viper.GetString("Polr_URL")
 	c.Polr_API_Key = viper.GetString("Polr_API_Key")
diff --git a/crawler/post_process.go b/crawler/post_process.go
index 40845b9..608592f 100644
--- a/crawler/post_process.go
+++ b/crawler/post_process.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -12,11 +13,23 @@ func (app *App) post_process() error {
 
 	if app.Config.DisableURLShorter {
 		log.Debug("post_process.go: URL Shorter is disabled (via config)")
-		return nil
 	} else {
-		return app.short_url()
+		err := app.short_url()
+		if err != nil {
+			return err
+		}
 	}
 
+	// reruns sanitizing functions over database
+	if app.Config.FixDatabase {
+		err := app.fix_db()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+
 }
 
 func (app *App) short_url() error {
@@ -73,3 +86,48 @@ func (app *App) short_url() error {
 	return nil
 
 }
+func (app *App) fix_db() error {
+
+	query := `SELECT id,name,age FROM all_view`
+
+	rows, err := app.DB.Queryx(query)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var offer_db Angebot
+		err = rows.StructScan(&offer_db)
+		if err != nil {
+			return err
+		}
+
+		update_query := `UPDATE angebot SET name = ?, age = ? WHERE id = ?`
+		update_name := false
+		update_age := false
+		name := sanitize_name(offer_db.Name)
+		age := get_age_from_name(name)
+		if name != offer_db.Name {
+			offer_db.Name = name
+			update_name = true
+		}
+		if age != offer_db.Age {
+			offer_db.Age = age
+			update_age = true
+		}
+
+		if update_name || update_age {
+			log.Debug(`UPDATE angebot SET name = "%s", age = %d WHERE id = %d`, offer_db.Name, offer_db.Age, offer_db.Id)
+			_, err = app.DB.Exec(update_query, offer_db.Name, offer_db.Age, offer_db.Id)
+			if err != nil {
+				offer_db.error_msg = err.Error()
+				offer_db.error_ctx = fmt.Sprintf(`UPDATE angebot SET name = "%s", age = %d WHERE id = %d`, offer_db.Name, offer_db.Age, offer_db.Id)
+				WarnOffer(offer_db, "post_process.go: Update query failed.")
+			}
+		}
+
+	}
+
+	return nil
+}