Stream-lined token matching.

This commit is contained in:
2026-11-09 21:23:22 +01:00
parent 235fba0c90
commit 01f86a4363
+76 -167
View File
@@ -93,6 +93,13 @@ impl<'a> Lexer<'a> {
span span
} }
fn emit(&mut self, token: Token, len: usize) -> TokenSpan<'a> {
TokenSpan {
token,
span: self.consume(len),
}
}
fn peek_char(&self) -> Option<char> { fn peek_char(&self) -> Option<char> {
self.input[self.cursor.pos..].chars().next() self.input[self.cursor.pos..].chars().next()
} }
@@ -100,53 +107,57 @@ impl<'a> Lexer<'a> {
fn peek_ident(&self) -> Option<&'a str> { fn peek_ident(&self) -> Option<&'a str> {
let first = self let first = self
.peek_char() .peek_char()
.filter(|c| c.is_alphabetic() || *c == UNDERSCORE)?; .filter(|c| c.is_alphabetic() || *c == '_')?;
let start = self.cursor.pos; let start = self.cursor.pos;
let remaining = start + first.len_utf8(); let remaining = start + first.len_utf8();
let end = remaining
+ self.input[remaining..] let offset = self.input[remaining..]
.chars() .char_indices()
.take_while(|c| c.is_alphanumeric() || *c == UNDERSCORE) .find(|(_, c)| !c.is_alphanumeric() && *c != '_')
.map(|c| c.len_utf8()) .map(|(idx, _)| idx)
.sum::<usize>(); .unwrap_or(self.input[remaining..].len());
Some(&self.input[start..end])
Some(&self.input[start..remaining + offset])
} }
fn peek_number(&self) -> Option<&'a str> { fn peek_number(&self) -> Option<&'a str> {
let first = self let first = self.peek_char().filter(|c| c.is_digit(10))?;
.peek_char()
.filter(|c| c.is_digit(10) || *c == DECIMAL_POINT)?;
let start = self.cursor.pos; let start = self.cursor.pos;
let remaining = start + first.len_utf8(); let remaining = start + first.len_utf8();
let end = remaining
+ self.input[remaining..] let offset = self.input[remaining..]
.chars() .char_indices()
.take_while(|c| c.is_digit(10) || *c == UNDERSCORE || *c == DECIMAL_POINT) .find(|(_, c)| !c.is_digit(10) && *c != '_' && *c != '.')
.map(|c| c.len_utf8()) .map(|(idx, _)| idx)
.sum::<usize>(); .unwrap_or(self.input[remaining..].len());
Some(&self.input[start..end])
Some(&self.input[start..remaining + offset])
} }
fn peek_string(&self) -> Option<&'a str> { fn peek_string(&self) -> Option<&'a str> {
let first = self.peek_char().filter(|c| *c == DOUBLE_QUOTE)?; let first = self.peek_char().filter(|c| *c == '"')?;
let start = self.cursor.pos; let start = self.cursor.pos;
let remaining = start + first.len_utf8(); let remaining = start + first.len_utf8();
let mut end = remaining; let mut end = remaining;
let mut escaped = false; let mut escaped = false;
while let Some(c) = self.input[end..].chars().next() {
let mut chars = self.input[end..].chars();
while let Some(c) = chars.next() {
end += c.len_utf8(); end += c.len_utf8();
if escaped { if escaped {
escaped = false; escaped = false;
continue; continue;
} }
if c == ESCAPE { if c == '\\' {
escaped = true; escaped = true;
continue; continue;
} }
if c == DOUBLE_QUOTE { if c == '"' {
break; break;
} }
} }
Some(&self.input[start..end]) Some(&self.input[start..end])
} }
@@ -155,10 +166,6 @@ impl<'a> Lexer<'a> {
self.cursor.advance_by(c.len_utf8()); self.cursor.advance_by(c.len_utf8());
} }
} }
fn starts_with(&self, s: &str) -> bool {
self.input[self.cursor.pos..].starts_with(s)
}
} }
impl<'a> Iterator for Lexer<'a> { impl<'a> Iterator for Lexer<'a> {
@@ -167,145 +174,47 @@ impl<'a> Iterator for Lexer<'a> {
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.skip_whitespace(); self.skip_whitespace();
match self.peek_char()? { let c = self.peek_char()?;
COLON => { if let Some(token) = match c {
return Some(TokenSpan { ':' => Some(Token::Colon),
token: Token::Colon, ',' => Some(Token::Comma),
span: self.consume(COLON.len_utf8()), '{' => Some(Token::CurlyBracketL),
}); '}' => Some(Token::CurlyBracketR),
} '=' => Some(Token::Equals),
CURLY_BRACKET_L => { '>' => Some(Token::GreaterThan),
return Some(TokenSpan { '-' => Some(Token::Hyphen),
token: Token::CurlyBracketL, '<' => Some(Token::LessThan),
span: self.consume(CURLY_BRACKET_L.len_utf8()), '(' => Some(Token::ParenL),
}); ')' => Some(Token::ParenR),
} '+' => Some(Token::Plus),
CURLY_BRACKET_R => { ';' => Some(Token::Semicolon),
return Some(TokenSpan { '[' => Some(Token::SquareBracketL),
token: Token::CurlyBracketR, ']' => Some(Token::SquareBracketR),
span: self.consume(CURLY_BRACKET_R.len_utf8()), _ => None,
}); } {
} return Some(self.emit(token, c.len_utf8()));
EQUALS => { };
return Some(TokenSpan {
token: Token::Equals,
span: self.consume(EQUALS.len_utf8()),
});
}
GREATER_THAN => {
return Some(TokenSpan {
token: Token::GreaterThan,
span: self.consume(GREATER_THAN.len_utf8()),
});
}
HYPHEN => {
return Some(TokenSpan {
token: Token::Hyphen,
span: self.consume(HYPHEN.len_utf8()),
});
}
LESS_THAN => {
return Some(TokenSpan {
token: Token::LessThan,
span: self.consume(LESS_THAN.len_utf8()),
});
}
PAREN_L => {
return Some(TokenSpan {
token: Token::ParenL,
span: self.consume(PAREN_L.len_utf8()),
});
}
PAREN_R => {
return Some(TokenSpan {
token: Token::ParenR,
span: self.consume(PAREN_R.len_utf8()),
});
}
PLUS => {
return Some(TokenSpan {
token: Token::Plus,
span: self.consume(PLUS.len_utf8()),
});
}
SEMICOLON => {
return Some(TokenSpan {
token: Token::Semicolon,
span: self.consume(SEMICOLON.len_utf8()),
});
}
SQUARE_BRACKET_L => {
return Some(TokenSpan {
token: Token::SquareBracketL,
span: self.consume(SQUARE_BRACKET_L.len_utf8()),
});
}
SQUARE_BRACKET_R => {
return Some(TokenSpan {
token: Token::SquareBracketR,
span: self.consume(SQUARE_BRACKET_R.len_utf8()),
});
}
_ => {}
}
if self.starts_with(FN) {
return Some(TokenSpan {
token: Token::Fn,
span: self.consume(FN.len()),
});
}
if self.starts_with(LET) {
return Some(TokenSpan {
token: Token::Let,
span: self.consume(LET.len()),
});
}
if let Some(ident) = self.peek_ident() { if let Some(ident) = self.peek_ident() {
Some(TokenSpan { let token = match ident {
token: Token::Ident, "fn" => Token::Fn,
span: self.consume(ident.len()), "let" => Token::Let,
}) _ => Token::Ident,
} else if let Some(number) = self.peek_number() { };
Some(TokenSpan { return Some(self.emit(token, ident.len()));
token: Token::Number,
span: self.consume(number.len()),
})
} else if let Some(string) = self.peek_string() {
Some(TokenSpan {
token: Token::String,
span: self.consume(string.len()),
})
} else {
Some(TokenSpan {
token: Token::Unknown,
span: self.consume(1),
})
}
}
} }
const COLON: char = ':'; if let Some(num) = self.peek_number() {
const CURLY_BRACKET_L: char = '{'; return Some(self.emit(Token::Number, num.len()));
const CURLY_BRACKET_R: char = '}'; }
const DECIMAL_POINT: char = '.';
const DOUBLE_QUOTE: char = '"';
const EQUALS: char = '=';
const ESCAPE: char = '\\';
const GREATER_THAN: char = '>';
const HYPHEN: char = '-';
const LESS_THAN: char = '<';
const PAREN_L: char = '(';
const PAREN_R: char = ')';
const PLUS: char = '+';
const SEMICOLON: char = ';';
const SQUARE_BRACKET_L: char = '[';
const SQUARE_BRACKET_R: char = ']';
const UNDERSCORE: char = '_';
const FN: &str = "fn"; if let Some(str) = self.peek_string() {
const LET: &str = "let"; return Some(self.emit(Token::String, str.len()));
}
Some(self.emit(Token::Unknown, c.len_utf8()))
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
@@ -352,14 +261,14 @@ mod tests {
#[test] #[test]
fn numbers() { fn numbers() {
let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000"; let input = "0.5 5 50.0 5_000 50.000_0 50_000.000_000";
let mut lexer = Lexer::new(input); let mut lexer = Lexer::new(input);
assert_next_token(&mut lexer, Token::Number, ".5", 0); assert_next_token(&mut lexer, Token::Number, "0.5", 0);
assert_next_token(&mut lexer, Token::Number, "5", 3); assert_next_token(&mut lexer, Token::Number, "5", 4);
assert_next_token(&mut lexer, Token::Number, "50.0", 5); assert_next_token(&mut lexer, Token::Number, "50.0", 6);
assert_next_token(&mut lexer, Token::Number, "5_000", 10); assert_next_token(&mut lexer, Token::Number, "5_000", 11);
assert_next_token(&mut lexer, Token::Number, "50.000_0", 16); assert_next_token(&mut lexer, Token::Number, "50.000_0", 17);
assert_next_token(&mut lexer, Token::Number, "50_000.000_000", 25); assert_next_token(&mut lexer, Token::Number, "50_000.000_000", 26);
assert_eq!(lexer.next(), None); assert_eq!(lexer.next(), None);
} }