From 0026ba55f03c5378d5773459fcdd7c6931ff42a5 Mon Sep 17 00:00:00 2001
From: Max
Date: Fri, 15 Jun 2018 19:38:04 +0200
Subject: Introduces central crawler config. (crawler)

---
 crawler/utility.go | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'crawler/utility.go')

diff --git a/crawler/utility.go b/crawler/utility.go
index 5fa78c4..e0acf3f 100644
--- a/crawler/utility.go
+++ b/crawler/utility.go
@@ -10,6 +10,12 @@ import (
 	"github.com/gocolly/colly"
 )
 
+func customCollector(allowed_urls []string) *colly.Collector {
+	return colly.NewCollector(
+		colly.AllowedDomains(allowed_urls...),
+	)
+}
+
 func stringInSlice(a string, list []string) bool {
 	for _, b := range list {
 		if b == a {
-- 
cgit v1.2.3