Added basic tokens and lexing

1 files changed, 65 insertions, 0 deletions

diff --git a/tokens/tokens.go b/tokens/tokens.go
new file mode 100644
index 0000000..b93d9f9
--- /dev/null
+++ b/tokens/tokens.go
@@ -0,0 +1,65 @@
+package tokens
+
+type TokenType string
+
+type Token struct {
+	Type    TokenType
+	Literal string
+}
+
+const (
+	ILLEGAL   = "ILLEGAL"
+	EOF       = "EOF"
+
+	// Identifiers + literals
+	IDENT     = "IDENT"
+	INT       = "INT"
+
+	// Operators
+	ASSIGN    = "="
+	PLUS      = "+"
+	MINUS     = "-"
+	BANG      = "!"
+	ASTERISK  = "*"
+	SLASH     = "/"
+	LT        = "<"
+	GT        = ">"
+	EQ        = "=="
+	NOT_EQ    = "!="
+
+	// Delimiters
+	COMMA     = ","
+	SEMICOLON = ";"
+
+	LPAREN    = "("
+	RPAREN    = ")"
+	LBRACE    = "{"
+	RBRACE    = "}"
+
+	// Keywords
+	FUNCTION  = "FUNCTION"
+	LET       = "LET"
+	IF        = "IF"
+	ELSE      = "ELSE"
+	TRUE      = "TRUE"
+	FALSE     = "FALSE"
+	RETURN    = "RETURN"
+)
+
+var keywords = map[string]TokenType {
+	"fn":  FUNCTION,
+	"let": LET,
+	"if":  IF,
+	"else": ELSE,
+	"true": TRUE,
+	"false": FALSE,
+	"return": RETURN,
+}
+
+// LookupIdent looks up an identifier and returns the TokenType.
+func LookupIdent(ident string) TokenType {
+	if tok, ok := keywords[ident]; ok {
+		return tok
+	}
+	return IDENT
+}