diff options
Diffstat (limited to 'crawler')
| -rw-r--r-- | crawler/card.go | 32 | ||||
| -rw-r--r-- | crawler/kingdom.go | 78 | ||||
| -rw-r--r-- | crawler/main.go | 146 | ||||
| -rw-r--r-- | crawler/set.go | 47 |
4 files changed, 303 insertions, 0 deletions
diff --git a/crawler/card.go b/crawler/card.go new file mode 100644 index 0000000..f7e8979 --- /dev/null +++ b/crawler/card.go @@ -0,0 +1,32 @@ +package main + +type Card struct { + Name string + Name_de string + URL string + Set string + Set_de string + Set_Image string + Type string + text string + //Text_de string + Actions string + Cards string + Buy string + Coin string + Trash string + Junk string + Gain string + Image string + IsEdition1 bool + IsEdition2 bool + IsKingdom bool + Cost Cost + //Cost string +} + +type Cost struct { + Coin int + Potion int + Debt int +} diff --git a/crawler/kingdom.go b/crawler/kingdom.go new file mode 100644 index 0000000..fb62cfb --- /dev/null +++ b/crawler/kingdom.go @@ -0,0 +1,78 @@ +package main + +import ( + "strings" +) + +func IsKingdom(Card Card, SplitPile []Card) bool { + + switch Card.Name { + case "Platinum": + return false + case "Gold": + return false + case "Silver": + return false + case "Copper": + return false + case "Province": + return false + case "Duchy": + return false + case "Estate": + return false + case "Curse": + return false + } + + switch Card.Type { + case "State": + return false + case "Hex": + return false + case "Boon": + return false + case "Landmark": + return false + case "Event": + return false + } + + if strings.Contains(Card.Type, "Prize") { + return false + } + if strings.Contains(Card.Type, "Ruins") { + return false + } + if strings.Contains(Card.Type, "Shelter") { + return false + } + if strings.Contains(Card.Type, "Heirloom") { + return false + } + + if strings.Contains(Card.text, "(This is not in the Supply.)") { + return false + } + + if strings.Contains(Card.Type, "Castle") && Card.Name != "Humble Castle" { + return false + } + + if IsSplitCardOnBottom(Card, SplitPile) { + return false + } + + return true +} + +func IsSplitCardOnBottom(Card Card, SplitPile []Card) bool { + for _, v := range SplitPile { + names := strings.Split(v.Name, "/") + if Card.Name == names[1] { + return true + } + } + + return false +} diff --git a/crawler/main.go b/crawler/main.go new file mode 100644 index 0000000..d91aa2e --- /dev/null +++ b/crawler/main.go @@ -0,0 +1,146 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "regexp" + "strconv" + "strings" + + //"github.com/davecgh/go-spew/spew" + "github.com/gocolly/colly" +) + +func main() { + Cards_URL := "http://wiki.dominionstrategy.com/index.php/All_Cards" + + Cards := []Card{} + SplitPile := []Card{} + + c := colly.NewCollector( + colly.AllowedDomains("wiki.dominionstrategy.com"), + ) + + c.OnHTML("table.sortable tr", func(e *colly.HTMLElement) { + + var err error + + if e.Request.URL.String() != Cards_URL { + return + } + + card := Card{} + + e.ForEach("td", func(i int, e *colly.HTMLElement) { + switch i { + case 0: + card.Name = strings.TrimSpace(e.Text) + card.Image = "http://wiki.dominionstrategy.com" + e.ChildAttr("img", "src") + + card.URL = e.ChildAttr("a", "href") + case 1: + card.Set = strings.TrimSpace(e.Text) + + if strings.Contains(card.Set, "1E") { + card.Set = "Base" + card.IsEdition1 = true + } + if strings.Contains(card.Set, "2E") { + card.Set = "Base" + card.IsEdition2 = true + } + case 2: + card.Type = strings.TrimSpace(e.Text) + case 3: + cost := Cost{} + coin_costs := e.ChildAttrs(".coin-icon img", "alt") + for _, v := range coin_costs { + if strings.Contains(v, "$") { + cost.Coin, err = strconv.Atoi(strings.TrimSuffix(strings.TrimSuffix(strings.TrimPrefix(v, "$"), "star"), "plus")) + if err != nil { + log.Fatal(err) + } + } + if strings.Contains(v, "P") { + cost.Potion = 1 + } + } + debt_costs := e.ChildAttr(".debt-icon img", "alt") + if strings.Contains(debt_costs, "D") { + cost.Debt, err = strconv.Atoi(strings.TrimSuffix(debt_costs, "D")) + if err != nil { + log.Fatal(err) + } + } + card.Cost = cost + case 4: + card.text = strings.TrimSpace(e.Text) + case 5: + card.Actions = strings.TrimSpace(e.Text) + case 6: + card.Cards = strings.TrimSpace(e.Text) + case 7: + card.Buy = strings.TrimSpace(e.Text) + case 8: + card.Coin = strings.TrimSpace(strings.TrimPrefix(e.ChildAttr(".coin-img img", "alt"), "$")) + case 9: + card.Trash = strings.TrimSpace(e.Text) + case 10: + card.Junk = strings.TrimSpace(e.Text) + case 11: + card.Gain = strings.TrimSpace(e.Text) + } + + }) + + if strings.Contains(card.text, "Split pile containing 5 ") { + SplitPile = append(SplitPile, card) + } else { + + e.Request.Visit(card.URL) + card.Name_de = e.Request.Ctx.Get("name") + //card.Text_de = e.Request.Ctx.Get("desc") + Cards = append(Cards, card) + } + }) + + c.OnHTML(".wikitable tr", func(e *colly.HTMLElement) { + if e.Request.URL.String() == Cards_URL { + return + } + if e.ChildText("th") != "German" { + return + } + e.ForEach("td", func(i int, e *colly.HTMLElement) { + switch i { + case 0: + name_de := e.Text + + r := regexp.MustCompile(`(\(.+\))`) + name_de = strings.TrimSpace(r.ReplaceAllString(name_de, "")) + e.Request.Ctx.Put("name", name_de) + case 3: + //e.Request.Ctx.Put("desc", e.Text) + } + }) + }) + + c.Visit(Cards_URL) + + Output := []Card{} + for _, v := range Cards { + if v.Name == "" { + continue + } + v.IsKingdom = IsKingdom(v, SplitPile) + card := SetDetails(v) + Output = append(Output, card) + } + + j, err := json.MarshalIndent(Output, "", " ") + if err != nil { + log.Fatal(err) + } + fmt.Println(string(j)) +} diff --git a/crawler/set.go b/crawler/set.go new file mode 100644 index 0000000..dcd48f1 --- /dev/null +++ b/crawler/set.go @@ -0,0 +1,47 @@ +package main + +func SetDetails(card Card) Card { + switch card.Set { + case "Adventures": + card.Set_de = "Abenteuer" + card.Set_Image = "http://wiki.dominionstrategy.com/images/thumb/d/da/Adventures.jpg/298px-Adventures.jpg" + case "Alchemy": + card.Set_de = "Die Alchemisten" + card.Set_Image = "http://wiki.dominionstrategy.com/images/6/67/Alchemy.jpg" + case "Base": + card.Set_de = "Grundspiel" + card.Set_Image = "http://wiki.dominionstrategy.com/images/c/c4/Dominion.jpg" + case "Cornucopia": + card.Set_de = "Reiche Ernte" + card.Set_Image = "http://wiki.dominionstrategy.com/images/thumb/2/26/Cornucopia.jpg/201px-Cornucopia.jpg" + case "Dark Ages": + card.Set_de = "Dark Ages" + card.Set_Image = "http://wiki.dominionstrategy.com/images/thumb/d/dd/Dark_Ages.jpg/300px-Dark_Ages.jpg" + case "Empires": + card.Set_de = "Empires" + card.Set_Image = "http://wiki.dominionstrategy.com/images/thumb/0/0c/Empires.jpg/300px-Empires.jpg" + case "Guilds": + card.Set_de = "Gilden" + card.Set_Image = "http://wiki.dominionstrategy.com/images/thumb/0/02/Guilds.jpg/202px-Guilds.jpg" + case "Hinterlands": + card.Set_de = "Hinterland" + card.Set_Image = "http://wiki.dominionstrategy.com/images/thumb/a/a1/Hinterlands.jpg/299px-Hinterlands.jpg" + case "Intrigue": + card.Set_de = "Intrige" + card.Set_Image = "http://wiki.dominionstrategy.com/images/f/f2/Intrigue.jpg" + case "Nocturne": + card.Set_de = "Nocturne" + card.Set_Image = "http://wiki.dominionstrategy.com/images/thumb/7/7b/Nocturne.jpg/300px-Nocturne.jpg" + case "Promo": + card.Set_de = "Promo" + card.Set_Image = "" + case "Prosperity": + card.Set_de = "Blütezeit" + card.Set_Image = "http://wiki.dominionstrategy.com/images/9/97/Prosperity.jpg" + case "Seaside": + card.Set_de = "Seaside" + card.Set_Image = "http://wiki.dominionstrategy.com/images/3/37/Seaside.jpg" + } + + return card +} |