From 8d68ac7c900241eb8499a94c23ab1f60750e7aed Mon Sep 17 00:00:00 2001
From: horus
Date: Fri, 15 Jun 2018 23:28:18 +0200
Subject: Introduces config for user agent, robots.txt and crawler delay.
 (crawler)

---
 crawler/utility.go | 6 ------
 1 file changed, 6 deletions(-)

(limited to 'crawler/utility.go')

diff --git a/crawler/utility.go b/crawler/utility.go
index e0acf3f..5fa78c4 100644
--- a/crawler/utility.go
+++ b/crawler/utility.go
@@ -10,12 +10,6 @@ import (
 	"github.com/gocolly/colly"
 )
 
-func customCollector(allowed_urls []string) *colly.Collector {
-	return colly.NewCollector(
-		colly.AllowedDomains(allowed_urls...),
-	)
-}
-
 func stringInSlice(a string, list []string) bool {
 	for _, b := range list {
 		if b == a {
-- 
cgit v1.2.3