diff options
Diffstat (limited to 'crawler/config.go')
| -rw-r--r-- | crawler/config.go | 9 |
1 files changed, 6 insertions, 3 deletions
diff --git a/crawler/config.go b/crawler/config.go index a3939c4..ba3c92f 100644 --- a/crawler/config.go +++ b/crawler/config.go @@ -25,9 +25,10 @@ type Config struct { Polr_URL string Polr_API_Key string - Debug bool // sets log level to debug - FixDatabase bool // reruns some sanitizing functions over the db - ShopIDs []string // limits which shops to crawl, wants shop_id + Debug bool // sets log level to debug + FixDatabase bool // reruns some sanitizing functions over the db + ShopIDs []string // limits which shops to crawl, wants shop_id + ExcludeShopIDs []string // excludes shops from being crawled, wants shop_id } // Parses the configuration and sets the configuration struct. @@ -44,6 +45,7 @@ func (c *Config) parseConfig(configFile string) { viper.SetDefault("FixDatabase", false) viper.SetDefault("DisableURLShorter", false) viper.SetDefault("ShopIDs", []string{}) + viper.SetDefault("ExcludeShopIDs", []string{}) viper.SetDefault("Delay", 0) // needs some refactoring to truly respect robots.txt @@ -117,4 +119,5 @@ func (c *Config) setsConfig() { c.Debug = viper.GetBool("Debug") c.FixDatabase = viper.GetBool("FixDatabase") c.ShopIDs = viper.GetStringSlice("ShopIDs") + c.ExcludeShopIDs = viper.GetStringSlice("ExcludeShopIDs") } |