package main import ( "fmt" "time" _ "database/sql" _ "github.com/go-sql-driver/mysql" //_ "github.com/mattn/go-sqlite3" log "github.com/Sirupsen/logrus" "github.com/jmoiron/sqlx" flag "github.com/spf13/pflag" ) type App struct { Offers []Angebot Shops []Shop Config *Config DB *sqlx.DB Now int64 Debug bool } func main() { var err error // copy global config to avoid woring with globals _own_config := _conf app := App{Config: &_own_config} // overwrite the global _conf = Config{} app.Now = time.Now().Unix() if "sqlite3" == app.Config.DBDriver { app.DB, err = sqlx.Connect(app.Config.DBDriver, app.Config.DBPath) } else { log.Debug(fmt.Sprintf(`Connecting to "%s" database "%s" as user "%s" on host "%s:%s" with extra options "%s".`, app.Config.DBDriver, app.Config.DBDBName, app.Config.DBUser, app.Config.DBHost, app.Config.DBPort, app.Config.DBOptions)) app.DB, err = sqlx.Connect(app.Config.DBDriver, app.Config.DBUser+":"+app.Config.DBPassword+"@tcp("+app.Config.DBHost+":"+app.Config.DBPort+")/"+app.Config.DBDBName+app.Config.DBOptions) } if err != nil { Fatal(err, "Cannot connect to database") } if err = app.DB.Ping(); err != nil { Fatal(err, "No connection to database") } defer app.DB.Close() err = app.createTables() if err != nil { Fatal(err, "Creating table failed") } err = app.insertShops() if err != nil { Fatal(err, "Inserting shops failed") } shops, err := app.getShops() if err != nil { Fatal(err, "Getting shops failed") } // prints all crawlable shops _flag := flag.Lookup("list-shops") if _flag != nil && "true" == _flag.Value.String() { log.SetLevel(log.InfoLevel) for _, shop := range shops { log.WithFields( log.Fields{ "shop_id": shop.Id, "shop_url": shop.Url, "shop_short_url": shop.Short_url, "shop_logo_url": shop.Logo_url, "shipping costs": shop.Shipping_costs, "free shipping": shop.Free_shipping, }, ).Info(shop.Name) } return } // reruns sanitizing functions over database if app.Config.FixDatabase { err := app.fix_db() if err != nil { Fatal(err, "Fix: Fixing database failed") } return } app.ScrapeHTML(shops) // short url err = app.post_process() if err != nil { Fatal(err, "Post processing failed") } }