From 77eabae402eec37a04ca7c60d8aca179126c4711 Mon Sep 17 00:00:00 2001 From: Rogier Reerink Date: Mon, 9 Nov 2026 19:44:48 +0100 Subject: [PATCH] Added a bunch of tokens. --- src/lex.rs | 199 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 198 insertions(+), 1 deletion(-) diff --git a/src/lex.rs b/src/lex.rs index d67d2f2..b1cf720 100644 --- a/src/lex.rs +++ b/src/lex.rs @@ -1,15 +1,23 @@ #[derive(Debug, PartialEq, Eq)] pub enum Token { + Add, Comma, + CurlyBracketL, + CurlyBracketR, Equals, Fn, + GreaterThan, + LessThan, Ident, Let, Number, ParenL, ParenR, Semi, + SquareBracketL, + SquareBracketR, String, + Subtract, Unknown, } @@ -159,21 +167,87 @@ impl<'a> Iterator for Lexer<'a> { self.skip_whitespace(); match self.peek_char()? { + ADD_OPERATOR => { + return Some(TokenSpan { + token: Token::Add, + span: self.consume(ADD_OPERATOR.len_utf8()), + }); + } ASSIGN_OPERATOR => { return Some(TokenSpan { token: Token::Equals, span: self.consume(ASSIGN_OPERATOR.len_utf8()), }); } + CURLY_BRACKET_L => { + return Some(TokenSpan { + token: Token::CurlyBracketL, + span: self.consume(CURLY_BRACKET_L.len_utf8()), + }); + } + CURLY_BRACKET_R => { + return Some(TokenSpan { + token: Token::CurlyBracketR, + span: self.consume(CURLY_BRACKET_R.len_utf8()), + }); + } + GREATER_THAN => { + return Some(TokenSpan { + token: Token::GreaterThan, + span: self.consume(GREATER_THAN.len_utf8()), + }); + } + LESS_THAN => { + return Some(TokenSpan { + token: Token::LessThan, + span: self.consume(LESS_THAN.len_utf8()), + }); + } + PAREN_L => { + return Some(TokenSpan { + token: Token::ParenL, + span: self.consume(PAREN_L.len_utf8()), + }); + } + PAREN_R => { + return Some(TokenSpan { + token: Token::ParenR, + span: self.consume(PAREN_R.len_utf8()), + }); + } SEMI => { return Some(TokenSpan { token: Token::Semi, span: self.consume(SEMI.len_utf8()), }); } + SQUARE_BRACKET_L => { + return Some(TokenSpan { + token: Token::SquareBracketL, + span: self.consume(SQUARE_BRACKET_L.len_utf8()), + }); + } + SQUARE_BRACKET_R => { + return Some(TokenSpan { + token: Token::SquareBracketR, + span: self.consume(SQUARE_BRACKET_R.len_utf8()), + }); + } + SUBTRACT_OPERATOR => { + return Some(TokenSpan { + token: Token::Subtract, + span: self.consume(SUBTRACT_OPERATOR.len_utf8()), + }); + } _ => {} } + if self.starts_with(FN) { + return Some(TokenSpan { + token: Token::Fn, + span: self.consume(FN.len()), + }); + } if self.starts_with(LET) { return Some(TokenSpan { token: Token::Let, @@ -205,20 +279,143 @@ impl<'a> Iterator for Lexer<'a> { } } +const ADD_OPERATOR: char = '+'; +const ASSIGN_OPERATOR: char = '='; +const CURLY_BRACKET_L: char = '{'; +const CURLY_BRACKET_R: char = '}'; const DECIMAL_POINT: char = '.'; const DIGIT_SEPARATOR: char = '_'; -const ASSIGN_OPERATOR: char = '='; const ESCAPE: char = '\\'; +const GREATER_THAN: char = '>'; +const LESS_THAN: char = '<'; +const PAREN_L: char = '('; +const PAREN_R: char = ')'; const QUOTE: char = '"'; const SEMI: char = ';'; +const SQUARE_BRACKET_L: char = '['; +const SQUARE_BRACKET_R: char = ']'; +const SUBTRACT_OPERATOR: char = '-'; const IDENT_SEPARATOR: char = '_'; +const FN: &str = "fn"; const LET: &str = "let"; #[cfg(test)] mod tests { use super::*; + #[test] + fn test_operators() { + let input = "+ = - ; ( ) [ ] { } < >"; + let mut lexer = Lexer::new(input); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Add, + span: Span::new(0, "+"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Equals, + span: Span::new(2, "="), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Subtract, + span: Span::new(4, "-"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Semi, + span: Span::new(6, ";"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::ParenL, + span: Span::new(8, "("), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::ParenR, + span: Span::new(10, ")"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::SquareBracketL, + span: Span::new(12, "["), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::SquareBracketR, + span: Span::new(14, "]"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::CurlyBracketL, + span: Span::new(16, "{"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::CurlyBracketR, + span: Span::new(18, "}"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::LessThan, + span: Span::new(20, "<"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::GreaterThan, + span: Span::new(22, ">"), + }) + ); + assert_eq!(lexer.next(), None); + } + + #[test] + fn test_keywords() { + let input = "fn let"; + let mut lexer = Lexer::new(input); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Fn, + span: Span::new(0, "fn"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Let, + span: Span::new(3, "let"), + }) + ); + assert_eq!(lexer.next(), None); + } + #[test] fn test_numbers() { let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000";