diff --git a/src/lex.rs b/src/lex.rs index 75ef8d8..d67d2f2 100644 --- a/src/lex.rs +++ b/src/lex.rs @@ -35,6 +35,10 @@ impl<'a> Span<'a> { pub fn raw(&self) -> &'a str { self.raw } + + pub fn len(&self) -> usize { + self.raw.len() + } } #[derive(Debug, PartialEq, Eq)] @@ -87,26 +91,28 @@ impl<'a> Lexer<'a> { fn peek_ident(&self) -> Option<&'a str> { let first = self .peek_char() - .filter(|c| c.is_alphabetic() || *c == '_')?; + .filter(|c| c.is_alphabetic() || *c == IDENT_SEPARATOR)?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); let end = remaining + self.input[remaining..] .chars() - .take_while(|c| c.is_alphanumeric() || *c == '_') + .take_while(|c| c.is_alphanumeric() || *c == IDENT_SEPARATOR) .map(|c| c.len_utf8()) .sum::(); Some(&self.input[start..end]) } fn peek_number(&self) -> Option<&'a str> { - let first = self.peek_char().filter(|c| c.is_digit(10))?; + let first = self + .peek_char() + .filter(|c| c.is_digit(10) || *c == DECIMAL_POINT)?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); let end = remaining + self.input[remaining..] .chars() - .take_while(|c| c.is_digit(10) || *c == '_' || *c == '.') + .take_while(|c| c.is_digit(10) || *c == DIGIT_SEPARATOR || *c == DECIMAL_POINT) .map(|c| c.len_utf8()) .sum::(); Some(&self.input[start..end]) @@ -124,7 +130,7 @@ impl<'a> Lexer<'a> { escaped = false; continue; } - if c == '\\' { + if c == ESCAPE { escaped = true; continue; } @@ -153,10 +159,10 @@ impl<'a> Iterator for Lexer<'a> { self.skip_whitespace(); match self.peek_char()? { - EQUALS => { + ASSIGN_OPERATOR => { return Some(TokenSpan { token: Token::Equals, - span: self.consume(EQUALS.len_utf8()), + span: self.consume(ASSIGN_OPERATOR.len_utf8()), }); } SEMI => { @@ -199,9 +205,13 @@ impl<'a> Iterator for Lexer<'a> { } } -const EQUALS: char = '='; +const DECIMAL_POINT: char = '.'; +const DIGIT_SEPARATOR: char = '_'; +const ASSIGN_OPERATOR: char = '='; +const ESCAPE: char = '\\'; const QUOTE: char = '"'; const SEMI: char = ';'; +const IDENT_SEPARATOR: char = '_'; const LET: &str = "let"; @@ -210,84 +220,79 @@ mod tests { use super::*; #[test] - fn test_let_number() { - let input = "let x = 5_000.0;"; + fn test_numbers() { + let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000"; let mut lexer = Lexer::new(input); assert_eq!( lexer.next(), Some(TokenSpan { - token: Token::Let, - span: Span::new(0, "let") - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Ident, - span: Span::new(4, "x") - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Equals, - span: Span::new(6, "=") + token: Token::Number, + span: Span::new(0, ".5"), }) ); assert_eq!( lexer.next(), Some(TokenSpan { token: Token::Number, - span: Span::new(8, "5_000.0") + span: Span::new(3, "5"), }) ); assert_eq!( lexer.next(), Some(TokenSpan { - token: Token::Semi, - span: Span::new(15, ";") + token: Token::Number, + span: Span::new(5, "50.0"), }) ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Number, + span: Span::new(10, "5_000"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Number, + span: Span::new(16, "50.000_0"), + }) + ); + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: Token::Number, + span: Span::new(25, "50_000.000_000"), + }) + ); + assert_eq!(lexer.next(), None); } #[test] - fn test_let_string() { - let input = "let x = \"hello, world\";"; + fn test_strings() { + let input = "\"hello, world\" \"hello \\\", world\" \"hello"; let mut lexer = Lexer::new(input); assert_eq!( lexer.next(), Some(TokenSpan { - token: Token::Let, - span: Span::new(0, "let") - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Ident, - span: Span::new(4, "x") - }) - ); - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: Token::Equals, - span: Span::new(6, "=") + token: Token::String, + span: Span::new(0, "\"hello, world\""), }) ); assert_eq!( lexer.next(), Some(TokenSpan { token: Token::String, - span: Span::new(8, "\"hello, world\"") + span: Span::new(15, "\"hello \\\", world\""), }) ); assert_eq!( lexer.next(), Some(TokenSpan { - token: Token::Semi, - span: Span::new(22, ";") + token: Token::String, + span: Span::new(33, "\"hello"), }) ); + assert_eq!(lexer.next(), None); } }