Moved some stuff around.

This commit is contained in:
2026-11-10 08:53:54 +01:00
parent 6269aaf981
commit 713ffcc8ff
2 changed files with 59 additions and 59 deletions
+58 -58
View File
@@ -77,6 +77,54 @@ pub struct Lexer<'a> {
cursor: Cursor,
}
impl<'a> Iterator for Lexer<'a> {
type Item = TokenSpan<'a>;
fn next(&mut self) -> Option<Self::Item> {
self.skip_whitespace();
let c = self.peek_char()?;
if let Some(token) = match c {
':' => Some(Token::Colon),
',' => Some(Token::Comma),
'{' => Some(Token::CurlyBracketL),
'}' => Some(Token::CurlyBracketR),
'=' => Some(Token::Equals),
'>' => Some(Token::GreaterThan),
'-' => Some(Token::Hyphen),
'<' => Some(Token::LessThan),
'(' => Some(Token::ParenL),
')' => Some(Token::ParenR),
'+' => Some(Token::Plus),
';' => Some(Token::Semicolon),
'[' => Some(Token::SquareBracketL),
']' => Some(Token::SquareBracketR),
_ => None,
} {
return Some(self.emit(token, c.len_utf8()));
};
if let Some(ident) = self.peek_ident() {
let token = match ident {
"fn" => Token::Fn,
"let" => Token::Let,
_ => Token::Ident,
};
return Some(self.emit(token, ident.len()));
}
if let Some(num) = self.peek_number() {
return Some(self.emit(Token::Number, num.len()));
}
if let Some(str) = self.peek_string() {
return Some(self.emit(Token::String, str.len()));
}
Some(self.emit(Token::Unknown, c.len_utf8()))
}
}
impl<'a> Lexer<'a> {
pub fn new(input: &'a str) -> Self {
Lexer {
@@ -167,68 +215,10 @@ impl<'a> Lexer<'a> {
}
}
impl<'a> Iterator for Lexer<'a> {
type Item = TokenSpan<'a>;
fn next(&mut self) -> Option<Self::Item> {
self.skip_whitespace();
let c = self.peek_char()?;
if let Some(token) = match c {
':' => Some(Token::Colon),
',' => Some(Token::Comma),
'{' => Some(Token::CurlyBracketL),
'}' => Some(Token::CurlyBracketR),
'=' => Some(Token::Equals),
'>' => Some(Token::GreaterThan),
'-' => Some(Token::Hyphen),
'<' => Some(Token::LessThan),
'(' => Some(Token::ParenL),
')' => Some(Token::ParenR),
'+' => Some(Token::Plus),
';' => Some(Token::Semicolon),
'[' => Some(Token::SquareBracketL),
']' => Some(Token::SquareBracketR),
_ => None,
} {
return Some(self.emit(token, c.len_utf8()));
};
if let Some(ident) = self.peek_ident() {
let token = match ident {
"fn" => Token::Fn,
"let" => Token::Let,
_ => Token::Ident,
};
return Some(self.emit(token, ident.len()));
}
if let Some(num) = self.peek_number() {
return Some(self.emit(Token::Number, num.len()));
}
if let Some(str) = self.peek_string() {
return Some(self.emit(Token::String, str.len()));
}
Some(self.emit(Token::Unknown, c.len_utf8()))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) {
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: expected,
span: Span::new(offset, raw),
})
);
}
#[test]
fn operators() {
let input = "+ = - : ; ( ) [ ] { } < >";
@@ -280,4 +270,14 @@ mod tests {
assert_next_token(&mut lexer, Token::String, "\"hello", 33);
assert_eq!(lexer.next(), None);
}
fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) {
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: expected,
span: Span::new(offset, raw),
})
);
}
}
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod ast;
pub mod lex;
pub mod lexer;