summaryrefslogtreecommitdiff
path: root/crawler/post_process.go
blob: c0c433dce9d6ce813e0f0ee4f9fd06d35a393c82 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
package main

import (
	"fmt"

	log "github.com/sirupsen/logrus"
)

func (app *App) post_process() error {

	if app.Config.DisableURLShorter {
		log.Debug("post_process.go: URL Shorter is disabled (via config)")
	} else {
		err := app.short_url()
		if err != nil {
			return err
		}
	}

	return nil

}

func (app *App) short_url() error {

	var Angebote []Angebot

	query := `SELECT id, long_url as url FROM all_view WHERE url IS NULL AND long_url IS NOT NULL`

	rows, err := app.DB.Queryx(query)
	if err != nil {
		return err
	}
	defer rows.Close()

	query = `SELECT DISTINCT url FROM all_view ORDER BY URL ASC`
	short_urls := []string{}

	err = app.DB.Select(&short_urls, query)
	if err != nil {
		return err
	}

	for rows.Next() {
		var offer_db Angebot
		err = rows.StructScan(&offer_db)
		if err != nil {
			return err
		}

		for {
			generated_short_url := getRandomString(5)
			if !stringInSlice(generated_short_url, short_urls) {
				offer_db.Short_url = generated_short_url
				log.Debug("short_url: " + string(generated_short_url) + " ( " + offer_db.Url + " )")
				Angebote = append(Angebote, offer_db)
				break
			}
		}

	}

	for _, offer := range Angebote {
		_, err := app.DB.Exec(`UPDATE angebot SET short_url = ? WHERE id = ?`, offer.Short_url, offer.Id)
		if err != nil {
			return err
		}

	}

	return nil

}

func (app *App) fix_db() error {

	query := `SELECT id,name,age FROM all_view`

	rows, err := app.DB.Queryx(query)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var offer_db Angebot
		err = rows.StructScan(&offer_db)
		if err != nil {
			return err
		}

		update_query := `UPDATE angebot SET name = ?, age = ? WHERE id = ?`
		update_name := false
		update_age := false
		name := sanitize_name(offer_db.Name)
		if name != offer_db.Name {
			offer_db.Name = name
			update_name = true
		}
		if offer_db.Age == 0 {
			age := get_age_from_name(name)
			if age != 0 {
				offer_db.Age = age
				update_age = true
			}
		}

		if update_name || update_age {
			log.Debug(fmt.Sprintf(`UPDATE angebot SET name = "%s", age = %d WHERE id = %d`, offer_db.Name, offer_db.Age, offer_db.Id))
			_, err = app.DB.Exec(update_query, offer_db.Name, offer_db.Age, offer_db.Id)
			if err != nil {
				offer_db.error_msg = err.Error()
				offer_db.error_ctx = fmt.Sprintf(`UPDATE angebot SET name = "%s", age = %d WHERE id = %d`, offer_db.Name, offer_db.Age, offer_db.Id)
				offer_db.Warn("post_process.go: Update query failed.")
			}
		}

	}

	return nil
}