1 files changed, 56 insertions, 0 deletions
diff --git a/scanner.go b/scanner.go
new file mode 100644
index 0000000..10be11b
--- /dev/null
+++ b/scanner.go
@@ -0,0 +1,56 @@
+package main
+
+/*
+#include <stdio.h>
+#include <stdlib.h>
+#include "urlscanner.h"
+*/
+import "C"
+import (
+	"log"
+	"strings"
+)
+
+type Link struct {
+	Url   string
+	Text  string
+	IsDir bool
+}
+
+func getToken(input string) []Link {
+	var token C.int
+	var ls []Link
+	var l Link
+
+	C.scan_string(C.CString(input))
+
+	for token = C.yylex(); token != C.MYEOF; token = C.yylex() {
+
+		if token == C.TOKEN_URL {
+			// flex reads the href attr
+
+			l.Url = C.GoString(C.yylval)
+
+			if strings.HasSuffix(l.Url, "/") {
+				l.IsDir = true
+			}
+
+		} else if token == C.TOKEN_TEXT {
+			// flex reads the link description
+
+			l.Text = C.GoString(C.yylval)
+			ls = append(ls, l)
+			l = Link{}
+		} else {
+			// lexical error
+
+			l = Link{}
+			log.Printf("Lexical Error on line %d \n", C.yylineno)
+			continue
+		}
+	}
+
+	return ls
+}
+
+// sort