From 3c98969220c0f3c6372aef3207a98e4cbcc9a135 Mon Sep 17 00:00:00 2001 From: Bobby Date: Thu, 2 Nov 2023 22:22:54 -0400 Subject: parser:let|ident --- lexer/lexer.go | 17 ++++--- lexer/lexer_test.go | 9 ++-- main.go | 2 +- parser/parser.go | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++ parser/parser_test.go | 91 ++++++++++++++++++++++++++++++++++ repl/repl.go | 8 +-- 6 files changed, 242 insertions(+), 17 deletions(-) create mode 100644 parser/parser.go create mode 100644 parser/parser_test.go diff --git a/lexer/lexer.go b/lexer/lexer.go index b65d00b..589ed60 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,4 +1,5 @@ package lexer + import "mana/tokens" type Lexer struct { @@ -10,7 +11,7 @@ type Lexer struct { // New returns a new Lexer instance. func New(input string) *Lexer { - l := &Lexer{input: input} + var l *Lexer = &Lexer{input: input} l.readChar() return l } @@ -24,9 +25,9 @@ func (l *Lexer) NextToken() tokens.Token { switch l.ch { case '=': if l.peekChar() == '=' { - ch := l.ch + var ch byte = l.ch l.readChar() - literal := string(ch) + string(l.ch) + var literal string = string(ch) + string(l.ch) tok = tokens.Token{Type: tokens.EQ, Literal: literal} } else { tok = newToken(tokens.ASSIGN, l.ch) @@ -45,9 +46,9 @@ func (l *Lexer) NextToken() tokens.Token { tok = newToken(tokens.GT, l.ch) case '!': if l.peekChar() == '=' { - ch := l.ch + var ch byte = l.ch l.readChar() - literal := string(ch) + string(l.ch) + var literal string = string(ch) + string(l.ch) tok = tokens.Token{Type: tokens.NOT_EQ, Literal: literal} } else { tok = newToken(tokens.BANG, l.ch) @@ -80,7 +81,7 @@ func (l *Lexer) NextToken() tokens.Token { tok = newToken(tokens.ILLEGAL, l.ch) } } - + l.readChar() return tok } @@ -129,7 +130,7 @@ func (l *Lexer) peekChar() byte { // readIdentifier reads an identifier and advances the position in the input string until it encounters a non-letter character. func (l *Lexer) readIdentifier() string { - position := l.position + var position int = l.position for isLetter(l.ch) { l.readChar() } @@ -138,7 +139,7 @@ func (l *Lexer) readIdentifier() string { // readNumber reads a number and advances the position in the input string until it encounters a non-digit character. func (l *Lexer) readNumber() string { - position := l.position + var position int = l.position for isDigit(l.ch) { l.readChar() } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index f8e3eae..5cce79d 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -6,7 +6,7 @@ import ( ) func TestNextToken(t *testing.T) { - input := ` + const input string = ` let five = 5; let ten = 10; @@ -29,7 +29,7 @@ func TestNextToken(t *testing.T) { 10 != 9; ` - tests := []struct { + var tests = []struct { expectedType tokens.TokenType expectedLiteral string }{ @@ -109,10 +109,11 @@ func TestNextToken(t *testing.T) { {tokens.EOF, ""}, } - l := New(input) + + var l *Lexer = New(input) for i, tt := range tests { - tok := l.NextToken() + var tok tokens.Token = l.NextToken() if tok.Type != tt.expectedType { t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q", diff --git a/main.go b/main.go index deba212..92962bf 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,7 @@ import ( ) func main() { - user, err := user.Current() + var user, err = user.Current() if err != nil { panic(err) diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..6e8bba6 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,132 @@ +package parser + +import ( + "fmt" + "mana/ast" + "mana/lexer" + "mana/tokens" +) + +// Parser represents a parser. +type Parser struct { + l *lexer.Lexer + + curToken tokens.Token + peekToken tokens.Token + + errors []string +} + +// New returns a new Parser. +func New(l *lexer.Lexer) *Parser { + var p *Parser = &Parser{ + l: l, + errors: []string{}, + } + + // Read two tokens, so curToken and peekToken are both set. + p.nextToken() + p.nextToken() + + return p +} + +// nextToken advances the tokens. +func (p *Parser) nextToken() { + p.curToken = p.peekToken + p.peekToken = p.l.NextToken() +} + +// ParseProgram parses a program. +func (p *Parser) ParseProgram() *ast.Program { + + // Initialize the program with an empty slice of statements. + var program *ast.Program = &ast.Program{} + program.Statements = []ast.Statement{} + + // Iterate over the tokens until we reach the end of file. + for p.curToken.Type != tokens.EOF { + // Parse the statement and append it to the program. + var stmt ast.Statement = p.parseStatement() + + if stmt != nil { + program.Statements = append(program.Statements, stmt) + } + + // Advance to the next token. + p.nextToken() + } + + // Return the program. + return program +} + +// parseStatement parses a statement. +func (p *Parser) parseStatement() ast.Statement { + switch p.curToken.Type { + case tokens.LET: + return p.parseLetStatement() + default: + return nil + } +} + +// parseLetStatement parses a let statement. +func (p *Parser) parseLetStatement() *ast.LetStatement { + var stmt *ast.LetStatement = &ast.LetStatement{Token: p.curToken} + + if !p.expectPeek(tokens.IDENT) { + return nil + } + + stmt.Name = &ast.Identifier{Token: p.curToken, Value: p.curToken.Literal} + + if !p.expectPeek(tokens.ASSIGN) { + return nil + } + + // TODO: We're skipping the expressions until we + // encounter a semicolon. + + for !p.curTokenIs(tokens.SEMICOLON) { + p.nextToken() + } + + return stmt +} + +// curTokenIs returns true if the current token is of the given type. +func (p *Parser) curTokenIs(t tokens.TokenType) bool { + return p.curToken.Type == t +} + +// peekTokenIs returns true if the next token is of the given type. +func (p *Parser) peekTokenIs(t tokens.TokenType) bool { + return p.peekToken.Type == t +} + +// expectPeek expects the next token to be of the given type. +func (p *Parser) expectPeek(t tokens.TokenType) bool { + if p.peekTokenIs(t) { + p.nextToken() + + return true + } else { + p.peekError(t) + + return false + } +} + +// Errors returns the parser errors. +func (p *Parser) Errors() []string { + return p.errors +} + +// peekError returns an error message. +func (p *Parser) peekError(t tokens.TokenType) { + var msg string = fmt.Sprintf("expected next token to be %s, got %s instead", t, p.peekToken.Type) + + p.errors = append(p.errors, msg) +} + diff --git a/parser/parser_test.go b/parser/parser_test.go new file mode 100644 index 0000000..8f78c8b --- /dev/null +++ b/parser/parser_test.go @@ -0,0 +1,91 @@ +package parser + +import ( + "testing" + "mana/ast" + "mana/lexer" +) + +func TestLetStatements(t *testing.T) { + const input string = ` + let x = 5; + let y = 10; + let foobar = 838383; + ` + + var l *lexer.Lexer = lexer.New(input) + var p *Parser = New(l) + + var program *ast.Program = p.ParseProgram() + checkParserErrors(t, p) + + if program == nil { + t.Fatalf("ParseProgram() returned nil") + } + + if len(program.Statements) != 3 { + t.Fatalf("program.Statements does not contain 3 statements. got=%d", len(program.Statements)) + } + + var tests = []struct { + expectedIdentifier string + }{ + {"x"}, + {"y"}, + {"foobar"}, + } + + for i, tt := range tests { + var stmt ast.Statement = program.Statements[i] + + if !testLetStatement(t, stmt, tt.expectedIdentifier) { + return + } + } +} + +func checkParserErrors(t *testing.T, p *Parser) { + var errors []string = p.Errors() + + if len(errors) == 0 { + return + } + + t.Errorf("parser has %d errors", len(errors)) + + for _, msg := range errors { + t.Errorf("parser error: %q", msg) + } + + t.FailNow() +} + +func testLetStatement(t *testing.T, s ast.Statement, name string) bool { + if s.TokenLiteral() != "let" { + t.Errorf("s.TokenLiteral not 'let'. got=%q", s.TokenLiteral()) + + return false + } + + var letStmt, ok = s.(*ast.LetStatement) + + if !ok { + t.Errorf("s not *ast.LetStatement. got=%T", s) + + return false + } + + if letStmt.Name.Value != name { + t.Errorf("letStmt.Name.Value not '%s'. got=%s", name, letStmt.Name.Value) + + return false + } + + if letStmt.Name.TokenLiteral() != name { + t.Errorf("letStmt.Name.TokenLiteral() not '%s'. got=%s", name, letStmt.Name.TokenLiteral()) + + return false + } + + return true +} diff --git a/repl/repl.go b/repl/repl.go index 37acd49..0002a89 100644 --- a/repl/repl.go +++ b/repl/repl.go @@ -12,18 +12,18 @@ import ( const PROMPT = "=> " func Start(in io.Reader, out io.Writer) { - scanner := bufio.NewScanner(in) + var scanner *bufio.Scanner = bufio.NewScanner(in) for { fmt.Fprint(out, PROMPT) - scanned := scanner.Scan() + var scanned bool = scanner.Scan() if !scanned { return } - line := scanner.Text() - l := lexer.New(line) + var line string = scanner.Text() + var l *lexer.Lexer = lexer.New(line) for tok := l.NextToken(); tok.Type != tokens.EOF; tok = l.NextToken() { fmt.Fprintf(out, "%+v\n", tok) -- cgit v1.2.3