Add feature to list all crawlable shops. (crawler)

2 files changed, 20 insertions, 0 deletions

diff --git a/crawler/init.go b/crawler/init.go
index a4fcb21..f23dd9b 100644
--- a/crawler/init.go
+++ b/crawler/init.go
@@ -17,6 +17,7 @@ func init() {
 	verbose := flag.BoolP("verbose", "v", false, "same as --debug")
 	silent := flag.BoolP("silent", "s", false, "suppress outputs except warnings")
 	loglevel_f := flag.StringP("loglevel", "l", "Warn", `sets log level, can be "Warn", "Info" or "Debug"`)
+	flag.Bool("shops", false, `list all crawlable shops`)
 
 	flag.Parse()
 	loglevel := strings.ToLower(*loglevel_f)
diff --git a/crawler/main.go b/crawler/main.go
index 034c588..bfcd647 100644
--- a/crawler/main.go
+++ b/crawler/main.go
@@ -10,6 +10,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/jmoiron/sqlx"
+	flag "github.com/spf13/pflag"
 )
 
 type App struct {
@@ -65,6 +66,24 @@ func main() {
 		Fatal(err, "Getting shops failed")
 	}
 
+	// prints all crawlable shops
+	if "true" == flag.Lookup("shops").Value.String() {
+		log.SetLevel(log.InfoLevel)
+		for _, shop := range shops {
+			log.WithFields(
+				log.Fields{
+					"shop_id":        shop.Id,
+					"shop_url":       shop.Url,
+					"shop_short_url": shop.Short_url,
+					"shop_logo_url":  shop.Logo_url,
+					"shipping costs": shop.Shipping_costs,
+					"free shipping":  shop.Free_shipping,
+				},
+			).Info(shop.Name)
+		}
+		return
+	}
+
 	// reruns sanitizing functions over database
 	if app.Config.FixDatabase {
 		err := app.fix_db()