From 713ffcc8ff6e7b3b846b29199a8234bdd1033004 Mon Sep 17 00:00:00 2001 From: Rogier Reerink Date: Tue, 10 Nov 2026 08:53:54 +0100 Subject: [PATCH] Moved some stuff around. --- src/{lex.rs => lexer.rs} | 116 +++++++++++++++++++-------------------- src/lib.rs | 2 +- 2 files changed, 59 insertions(+), 59 deletions(-) rename src/{lex.rs => lexer.rs} (100%) diff --git a/src/lex.rs b/src/lexer.rs similarity index 100% rename from src/lex.rs rename to src/lexer.rs index e3dd445..0b09dfd 100644 --- a/src/lex.rs +++ b/src/lexer.rs @@ -77,6 +77,54 @@ pub struct Lexer<'a> { cursor: Cursor, } +impl<'a> Iterator for Lexer<'a> { + type Item = TokenSpan<'a>; + + fn next(&mut self) -> Option { + self.skip_whitespace(); + + let c = self.peek_char()?; + if let Some(token) = match c { + ':' => Some(Token::Colon), + ',' => Some(Token::Comma), + '{' => Some(Token::CurlyBracketL), + '}' => Some(Token::CurlyBracketR), + '=' => Some(Token::Equals), + '>' => Some(Token::GreaterThan), + '-' => Some(Token::Hyphen), + '<' => Some(Token::LessThan), + '(' => Some(Token::ParenL), + ')' => Some(Token::ParenR), + '+' => Some(Token::Plus), + ';' => Some(Token::Semicolon), + '[' => Some(Token::SquareBracketL), + ']' => Some(Token::SquareBracketR), + _ => None, + } { + return Some(self.emit(token, c.len_utf8())); + }; + + if let Some(ident) = self.peek_ident() { + let token = match ident { + "fn" => Token::Fn, + "let" => Token::Let, + _ => Token::Ident, + }; + return Some(self.emit(token, ident.len())); + } + + if let Some(num) = self.peek_number() { + return Some(self.emit(Token::Number, num.len())); + } + + if let Some(str) = self.peek_string() { + return Some(self.emit(Token::String, str.len())); + } + + Some(self.emit(Token::Unknown, c.len_utf8())) + } +} + impl<'a> Lexer<'a> { pub fn new(input: &'a str) -> Self { Lexer { @@ -167,68 +215,10 @@ impl<'a> Lexer<'a> { } } -impl<'a> Iterator for Lexer<'a> { - type Item = TokenSpan<'a>; - - fn next(&mut self) -> Option { - self.skip_whitespace(); - - let c = self.peek_char()?; - if let Some(token) = match c { - ':' => Some(Token::Colon), - ',' => Some(Token::Comma), - '{' => Some(Token::CurlyBracketL), - '}' => Some(Token::CurlyBracketR), - '=' => Some(Token::Equals), - '>' => Some(Token::GreaterThan), - '-' => Some(Token::Hyphen), - '<' => Some(Token::LessThan), - '(' => Some(Token::ParenL), - ')' => Some(Token::ParenR), - '+' => Some(Token::Plus), - ';' => Some(Token::Semicolon), - '[' => Some(Token::SquareBracketL), - ']' => Some(Token::SquareBracketR), - _ => None, - } { - return Some(self.emit(token, c.len_utf8())); - }; - - if let Some(ident) = self.peek_ident() { - let token = match ident { - "fn" => Token::Fn, - "let" => Token::Let, - _ => Token::Ident, - }; - return Some(self.emit(token, ident.len())); - } - - if let Some(num) = self.peek_number() { - return Some(self.emit(Token::Number, num.len())); - } - - if let Some(str) = self.peek_string() { - return Some(self.emit(Token::String, str.len())); - } - - Some(self.emit(Token::Unknown, c.len_utf8())) - } -} - #[cfg(test)] mod tests { use super::*; - fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) { - assert_eq!( - lexer.next(), - Some(TokenSpan { - token: expected, - span: Span::new(offset, raw), - }) - ); - } - #[test] fn operators() { let input = "+ = - : ; ( ) [ ] { } < >"; @@ -280,4 +270,14 @@ mod tests { assert_next_token(&mut lexer, Token::String, "\"hello", 33); assert_eq!(lexer.next(), None); } + + fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) { + assert_eq!( + lexer.next(), + Some(TokenSpan { + token: expected, + span: Span::new(offset, raw), + }) + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 851fe46..c3f86b1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,2 @@ pub mod ast; -pub mod lex; +pub mod lexer;