diff --git a/src/lex.rs b/src/lex.rs index fa0c799..c679f4c 100644 --- a/src/lex.rs +++ b/src/lex.rs @@ -1,23 +1,24 @@ #[derive(Debug, PartialEq, Eq)] pub enum Token { - Add, + Colon, Comma, CurlyBracketL, CurlyBracketR, Equals, Fn, GreaterThan, + Hyphen, LessThan, Ident, Let, Number, ParenL, ParenR, - Semi, + Plus, + Semicolon, SquareBracketL, SquareBracketR, String, - Subtract, Unknown, } @@ -99,13 +100,13 @@ impl<'a> Lexer<'a> { fn peek_ident(&self) -> Option<&'a str> { let first = self .peek_char() - .filter(|c| c.is_alphabetic() || *c == IDENT_SEPARATOR)?; + .filter(|c| c.is_alphabetic() || *c == UNDERSCORE)?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); let end = remaining + self.input[remaining..] .chars() - .take_while(|c| c.is_alphanumeric() || *c == IDENT_SEPARATOR) + .take_while(|c| c.is_alphanumeric() || *c == UNDERSCORE) .map(|c| c.len_utf8()) .sum::(); Some(&self.input[start..end]) @@ -120,14 +121,14 @@ impl<'a> Lexer<'a> { let end = remaining + self.input[remaining..] .chars() - .take_while(|c| c.is_digit(10) || *c == DIGIT_SEPARATOR || *c == DECIMAL_POINT) + .take_while(|c| c.is_digit(10) || *c == UNDERSCORE || *c == DECIMAL_POINT) .map(|c| c.len_utf8()) .sum::(); Some(&self.input[start..end]) } fn peek_string(&self) -> Option<&'a str> { - let first = self.peek_char().filter(|c| *c == QUOTE)?; + let first = self.peek_char().filter(|c| *c == DOUBLE_QUOTE)?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); let mut end = remaining; @@ -142,7 +143,7 @@ impl<'a> Lexer<'a> { escaped = true; continue; } - if c == QUOTE { + if c == DOUBLE_QUOTE { break; } } @@ -167,16 +168,10 @@ impl<'a> Iterator for Lexer<'a> { self.skip_whitespace(); match self.peek_char()? { - ADD_OPERATOR => { + COLON => { return Some(TokenSpan { - token: Token::Add, - span: self.consume(ADD_OPERATOR.len_utf8()), - }); - } - ASSIGN_OPERATOR => { - return Some(TokenSpan { - token: Token::Equals, - span: self.consume(ASSIGN_OPERATOR.len_utf8()), + token: Token::Colon, + span: self.consume(COLON.len_utf8()), }); } CURLY_BRACKET_L => { @@ -191,12 +186,24 @@ impl<'a> Iterator for Lexer<'a> { span: self.consume(CURLY_BRACKET_R.len_utf8()), }); } + EQUALS => { + return Some(TokenSpan { + token: Token::Equals, + span: self.consume(EQUALS.len_utf8()), + }); + } GREATER_THAN => { return Some(TokenSpan { token: Token::GreaterThan, span: self.consume(GREATER_THAN.len_utf8()), }); } + HYPHEN => { + return Some(TokenSpan { + token: Token::Hyphen, + span: self.consume(HYPHEN.len_utf8()), + }); + } LESS_THAN => { return Some(TokenSpan { token: Token::LessThan, @@ -215,10 +222,16 @@ impl<'a> Iterator for Lexer<'a> { span: self.consume(PAREN_R.len_utf8()), }); } - SEMI => { + PLUS => { return Some(TokenSpan { - token: Token::Semi, - span: self.consume(SEMI.len_utf8()), + token: Token::Plus, + span: self.consume(PLUS.len_utf8()), + }); + } + SEMICOLON => { + return Some(TokenSpan { + token: Token::Semicolon, + span: self.consume(SEMICOLON.len_utf8()), }); } SQUARE_BRACKET_L => { @@ -233,12 +246,6 @@ impl<'a> Iterator for Lexer<'a> { span: self.consume(SQUARE_BRACKET_R.len_utf8()), }); } - SUBTRACT_OPERATOR => { - return Some(TokenSpan { - token: Token::Subtract, - span: self.consume(SUBTRACT_OPERATOR.len_utf8()), - }); - } _ => {} } @@ -279,23 +286,23 @@ impl<'a> Iterator for Lexer<'a> { } } -const ADD_OPERATOR: char = '+'; -const ASSIGN_OPERATOR: char = '='; +const COLON: char = ':'; const CURLY_BRACKET_L: char = '{'; const CURLY_BRACKET_R: char = '}'; const DECIMAL_POINT: char = '.'; -const DIGIT_SEPARATOR: char = '_'; +const DOUBLE_QUOTE: char = '"'; +const EQUALS: char = '='; const ESCAPE: char = '\\'; const GREATER_THAN: char = '>'; +const HYPHEN: char = '-'; const LESS_THAN: char = '<'; const PAREN_L: char = '('; const PAREN_R: char = ')'; -const QUOTE: char = '"'; -const SEMI: char = ';'; +const PLUS: char = '+'; +const SEMICOLON: char = ';'; const SQUARE_BRACKET_L: char = '['; const SQUARE_BRACKET_R: char = ']'; -const SUBTRACT_OPERATOR: char = '-'; -const IDENT_SEPARATOR: char = '_'; +const UNDERSCORE: char = '_'; const FN: &str = "fn"; const LET: &str = "let"; @@ -304,94 +311,33 @@ const LET: &str = "let"; mod tests { use super::*; + fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) { + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: expected, + span: Span::new(offset, raw), + }) + ); + } + #[test] fn operators() { - let input = "+ = - ; ( ) [ ] { } < >"; + let input = "+ = - : ; ( ) [ ] { } < >"; let mut lexer = Lexer::new(input); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Add, - span: Span::new(0, "+"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Equals, - span: Span::new(2, "="), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Subtract, - span: Span::new(4, "-"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Semi, - span: Span::new(6, ";"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::ParenL, - span: Span::new(8, "("), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::ParenR, - span: Span::new(10, ")"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::SquareBracketL, - span: Span::new(12, "["), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::SquareBracketR, - span: Span::new(14, "]"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::CurlyBracketL, - span: Span::new(16, "{"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::CurlyBracketR, - span: Span::new(18, "}"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::LessThan, - span: Span::new(20, "<"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::GreaterThan, - span: Span::new(22, ">"), - }) - ); + assert_next_token(&mut lexer, Token::Plus, "+", 0); + assert_next_token(&mut lexer, Token::Equals, "=", 2); + assert_next_token(&mut lexer, Token::Hyphen, "-", 4); + assert_next_token(&mut lexer, Token::Colon, ":", 6); + assert_next_token(&mut lexer, Token::Semicolon, ";", 8); + assert_next_token(&mut lexer, Token::ParenL, "(", 10); + assert_next_token(&mut lexer, Token::ParenR, ")", 12); + assert_next_token(&mut lexer, Token::SquareBracketL, "[", 14); + assert_next_token(&mut lexer, Token::SquareBracketR, "]", 16); + assert_next_token(&mut lexer, Token::CurlyBracketL, "{", 18); + assert_next_token(&mut lexer, Token::CurlyBracketR, "}", 20); + assert_next_token(&mut lexer, Token::LessThan, "<", 22); + assert_next_token(&mut lexer, Token::GreaterThan, ">", 24); assert_eq!(lexer.next(), None); } @@ -399,20 +345,8 @@ mod tests { fn keywords() { let input = "fn let"; let mut lexer = Lexer::new(input); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Fn, - span: Span::new(0, "fn"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Let, - span: Span::new(3, "let"), - }) - ); + assert_next_token(&mut lexer, Token::Fn, "fn", 0); + assert_next_token(&mut lexer, Token::Let, "let", 3); assert_eq!(lexer.next(), None); } @@ -420,48 +354,12 @@ mod tests { fn numbers() { let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000"; let mut lexer = Lexer::new(input); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Number, - span: Span::new(0, ".5"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Number, - span: Span::new(3, "5"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Number, - span: Span::new(5, "50.0"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Number, - span: Span::new(10, "5_000"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Number, - span: Span::new(16, "50.000_0"), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Number, - span: Span::new(25, "50_000.000_000"), - }) - ); + assert_next_token(&mut lexer, Token::Number, ".5", 0); + assert_next_token(&mut lexer, Token::Number, "5", 3); + assert_next_token(&mut lexer, Token::Number, "50.0", 5); + assert_next_token(&mut lexer, Token::Number, "5_000", 10); + assert_next_token(&mut lexer, Token::Number, "50.000_0", 16); + assert_next_token(&mut lexer, Token::Number, "50_000.000_000", 25); assert_eq!(lexer.next(), None); } @@ -469,27 +367,9 @@ mod tests { fn strings() { let input = "\"hello, world\" \"hello \\\", world\" \"hello"; let mut lexer = Lexer::new(input); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::String, - span: Span::new(0, "\"hello, world\""), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::String, - span: Span::new(15, "\"hello \\\", world\""), - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::String, - span: Span::new(33, "\"hello"), - }) - ); + assert_next_token(&mut lexer, Token::String, "\"hello, world\"", 0); + assert_next_token(&mut lexer, Token::String, "\"hello \\\", world\"", 15); + assert_next_token(&mut lexer, Token::String, "\"hello", 33); assert_eq!(lexer.next(), None); } }