From 01f86a4363255448ad2b82170aa0d279e86fdf79 Mon Sep 17 00:00:00 2001 From: Rogier Reerink Date: Mon, 9 Nov 2026 21:23:22 +0100 Subject: [PATCH] Stream-lined token matching. --- src/lex.rs | 243 +++++++++++++++++------------------------------------ 1 file changed, 76 insertions(+), 167 deletions(-) diff --git a/src/lex.rs b/src/lex.rs index c679f4c..226ad3c 100644 --- a/src/lex.rs +++ b/src/lex.rs @@ -93,6 +93,13 @@ impl<'a> Lexer<'a> { span } + fn emit(&mut self, token: Token, len: usize) -> TokenSpan<'a> { + TokenSpan { + token, + span: self.consume(len), + } + } + fn peek_char(&self) -> Option { self.input[self.cursor.pos..].chars().next() } @@ -100,53 +107,57 @@ impl<'a> Lexer<'a> { fn peek_ident(&self) -> Option<&'a str> { let first = self .peek_char() - .filter(|c| c.is_alphabetic() || *c == UNDERSCORE)?; + .filter(|c| c.is_alphabetic() || *c == '_')?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); - let end = remaining - + self.input[remaining..] - .chars() - .take_while(|c| c.is_alphanumeric() || *c == UNDERSCORE) - .map(|c| c.len_utf8()) - .sum::(); - Some(&self.input[start..end]) + + let offset = self.input[remaining..] + .char_indices() + .find(|(_, c)| !c.is_alphanumeric() && *c != '_') + .map(|(idx, _)| idx) + .unwrap_or(self.input[remaining..].len()); + + Some(&self.input[start..remaining + offset]) } fn peek_number(&self) -> Option<&'a str> { - let first = self - .peek_char() - .filter(|c| c.is_digit(10) || *c == DECIMAL_POINT)?; + let first = self.peek_char().filter(|c| c.is_digit(10))?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); - let end = remaining - + self.input[remaining..] - .chars() - .take_while(|c| c.is_digit(10) || *c == UNDERSCORE || *c == DECIMAL_POINT) - .map(|c| c.len_utf8()) - .sum::(); - Some(&self.input[start..end]) + + let offset = self.input[remaining..] + .char_indices() + .find(|(_, c)| !c.is_digit(10) && *c != '_' && *c != '.') + .map(|(idx, _)| idx) + .unwrap_or(self.input[remaining..].len()); + + Some(&self.input[start..remaining + offset]) } fn peek_string(&self) -> Option<&'a str> { - let first = self.peek_char().filter(|c| *c == DOUBLE_QUOTE)?; + let first = self.peek_char().filter(|c| *c == '"')?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); + let mut end = remaining; let mut escaped = false; - while let Some(c) = self.input[end..].chars().next() { + + let mut chars = self.input[end..].chars(); + while let Some(c) = chars.next() { end += c.len_utf8(); if escaped { escaped = false; continue; } - if c == ESCAPE { + if c == '\\' { escaped = true; continue; } - if c == DOUBLE_QUOTE { + if c == '"' { break; } } + Some(&self.input[start..end]) } @@ -155,10 +166,6 @@ impl<'a> Lexer<'a> { self.cursor.advance_by(c.len_utf8()); } } - - fn starts_with(&self, s: &str) -> bool { - self.input[self.cursor.pos..].starts_with(s) - } } impl<'a> Iterator for Lexer<'a> { @@ -167,146 +174,48 @@ impl<'a> Iterator for Lexer<'a> { fn next(&mut self) -> Option { self.skip_whitespace(); - match self.peek_char()? { - COLON => { - return Some(TokenSpan { - token: Token::Colon, - span: self.consume(COLON.len_utf8()), - }); - } - CURLY_BRACKET_L => { - return Some(TokenSpan { - token: Token::CurlyBracketL, - span: self.consume(CURLY_BRACKET_L.len_utf8()), - }); - } - CURLY_BRACKET_R => { - return Some(TokenSpan { - token: Token::CurlyBracketR, - span: self.consume(CURLY_BRACKET_R.len_utf8()), - }); - } - EQUALS => { - return Some(TokenSpan { - token: Token::Equals, - span: self.consume(EQUALS.len_utf8()), - }); - } - GREATER_THAN => { - return Some(TokenSpan { - token: Token::GreaterThan, - span: self.consume(GREATER_THAN.len_utf8()), - }); - } - HYPHEN => { - return Some(TokenSpan { - token: Token::Hyphen, - span: self.consume(HYPHEN.len_utf8()), - }); - } - LESS_THAN => { - return Some(TokenSpan { - token: Token::LessThan, - span: self.consume(LESS_THAN.len_utf8()), - }); - } - PAREN_L => { - return Some(TokenSpan { - token: Token::ParenL, - span: self.consume(PAREN_L.len_utf8()), - }); - } - PAREN_R => { - return Some(TokenSpan { - token: Token::ParenR, - span: self.consume(PAREN_R.len_utf8()), - }); - } - PLUS => { - return Some(TokenSpan { - token: Token::Plus, - span: self.consume(PLUS.len_utf8()), - }); - } - SEMICOLON => { - return Some(TokenSpan { - token: Token::Semicolon, - span: self.consume(SEMICOLON.len_utf8()), - }); - } - SQUARE_BRACKET_L => { - return Some(TokenSpan { - token: Token::SquareBracketL, - span: self.consume(SQUARE_BRACKET_L.len_utf8()), - }); - } - SQUARE_BRACKET_R => { - return Some(TokenSpan { - token: Token::SquareBracketR, - span: self.consume(SQUARE_BRACKET_R.len_utf8()), - }); - } - _ => {} - } - - if self.starts_with(FN) { - return Some(TokenSpan { - token: Token::Fn, - span: self.consume(FN.len()), - }); - } - if self.starts_with(LET) { - return Some(TokenSpan { - token: Token::Let, - span: self.consume(LET.len()), - }); - } + let c = self.peek_char()?; + if let Some(token) = match c { + ':' => Some(Token::Colon), + ',' => Some(Token::Comma), + '{' => Some(Token::CurlyBracketL), + '}' => Some(Token::CurlyBracketR), + '=' => Some(Token::Equals), + '>' => Some(Token::GreaterThan), + '-' => Some(Token::Hyphen), + '<' => Some(Token::LessThan), + '(' => Some(Token::ParenL), + ')' => Some(Token::ParenR), + '+' => Some(Token::Plus), + ';' => Some(Token::Semicolon), + '[' => Some(Token::SquareBracketL), + ']' => Some(Token::SquareBracketR), + _ => None, + } { + return Some(self.emit(token, c.len_utf8())); + }; if let Some(ident) = self.peek_ident() { - Some(TokenSpan { - token: Token::Ident, - span: self.consume(ident.len()), - }) - } else if let Some(number) = self.peek_number() { - Some(TokenSpan { - token: Token::Number, - span: self.consume(number.len()), - }) - } else if let Some(string) = self.peek_string() { - Some(TokenSpan { - token: Token::String, - span: self.consume(string.len()), - }) - } else { - Some(TokenSpan { - token: Token::Unknown, - span: self.consume(1), - }) + let token = match ident { + "fn" => Token::Fn, + "let" => Token::Let, + _ => Token::Ident, + }; + return Some(self.emit(token, ident.len())); } + + if let Some(num) = self.peek_number() { + return Some(self.emit(Token::Number, num.len())); + } + + if let Some(str) = self.peek_string() { + return Some(self.emit(Token::String, str.len())); + } + + Some(self.emit(Token::Unknown, c.len_utf8())) } } -const COLON: char = ':'; -const CURLY_BRACKET_L: char = '{'; -const CURLY_BRACKET_R: char = '}'; -const DECIMAL_POINT: char = '.'; -const DOUBLE_QUOTE: char = '"'; -const EQUALS: char = '='; -const ESCAPE: char = '\\'; -const GREATER_THAN: char = '>'; -const HYPHEN: char = '-'; -const LESS_THAN: char = '<'; -const PAREN_L: char = '('; -const PAREN_R: char = ')'; -const PLUS: char = '+'; -const SEMICOLON: char = ';'; -const SQUARE_BRACKET_L: char = '['; -const SQUARE_BRACKET_R: char = ']'; -const UNDERSCORE: char = '_'; - -const FN: &str = "fn"; -const LET: &str = "let"; - #[cfg(test)] mod tests { use super::*; @@ -352,14 +261,14 @@ mod tests { #[test] fn numbers() { - let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000"; + let input = "0.5 5 50.0 5_000 50.000_0 50_000.000_000"; let mut lexer = Lexer::new(input); - assert_next_token(&mut lexer, Token::Number, ".5", 0); - assert_next_token(&mut lexer, Token::Number, "5", 3); - assert_next_token(&mut lexer, Token::Number, "50.0", 5); - assert_next_token(&mut lexer, Token::Number, "5_000", 10); - assert_next_token(&mut lexer, Token::Number, "50.000_0", 16); - assert_next_token(&mut lexer, Token::Number, "50_000.000_000", 25); + assert_next_token(&mut lexer, Token::Number, "0.5", 0); + assert_next_token(&mut lexer, Token::Number, "5", 4); + assert_next_token(&mut lexer, Token::Number, "50.0", 6); + assert_next_token(&mut lexer, Token::Number, "5_000", 11); + assert_next_token(&mut lexer, Token::Number, "50.000_0", 17); + assert_next_token(&mut lexer, Token::Number, "50_000.000_000", 26); assert_eq!(lexer.next(), None); }