#[derive(Debug, PartialEq, Eq)] pub enum Token { Colon, Comma, CurlyBracketL, CurlyBracketR, Equals, Fn, GreaterThan, Hyphen, LessThan, Ident, Let, Number, ParenL, ParenR, Plus, Semicolon, SquareBracketL, SquareBracketR, String, Unknown, } #[derive(Debug, PartialEq, Eq)] pub struct Span<'a> { start: usize, raw: &'a str, } impl<'a> Span<'a> { fn new(start: usize, raw: &'a str) -> Self { Span { start, raw } } pub fn start(&self) -> usize { self.start } pub fn end(&self) -> usize { self.start + self.raw.len() } pub fn raw(&self) -> &'a str { self.raw } pub fn len(&self) -> usize { self.raw.len() } } #[derive(Debug, PartialEq, Eq)] pub struct TokenSpan<'a> { pub token: Token, pub span: Span<'a>, } struct Cursor { pos: usize, } impl Cursor { fn advance_by(&mut self, n: usize) { self.pos += n; } } impl Default for Cursor { fn default() -> Self { Cursor { pos: 0 } } } pub struct Lexer<'a> { input: &'a str, cursor: Cursor, } impl<'a> Lexer<'a> { pub fn new(input: &'a str) -> Self { Lexer { cursor: Cursor::default(), input, } } fn consume(&mut self, n: usize) -> Span<'a> { let start = self.cursor.pos; let end = start + n; let span = Span::new(start, &self.input[start..end]); self.cursor.advance_by(n); span } fn emit(&mut self, token: Token, len: usize) -> TokenSpan<'a> { TokenSpan { token, span: self.consume(len), } } fn peek_char(&self) -> Option { self.input[self.cursor.pos..].chars().next() } fn peek_ident(&self) -> Option<&'a str> { let first = self .peek_char() .filter(|c| c.is_alphabetic() || *c == '_')?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); let offset = self.input[remaining..] .char_indices() .find(|(_, c)| !c.is_alphanumeric() && *c != '_') .map(|(idx, _)| idx) .unwrap_or(self.input[remaining..].len()); Some(&self.input[start..remaining + offset]) } fn peek_number(&self) -> Option<&'a str> { let first = self.peek_char().filter(|c| c.is_digit(10))?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); let offset = self.input[remaining..] .char_indices() .find(|(_, c)| !c.is_digit(10) && *c != '_' && *c != '.') .map(|(idx, _)| idx) .unwrap_or(self.input[remaining..].len()); Some(&self.input[start..remaining + offset]) } fn peek_string(&self) -> Option<&'a str> { let first = self.peek_char().filter(|c| *c == '"')?; let start = self.cursor.pos; let remaining = start + first.len_utf8(); let mut escaped = false; let end = self.input[remaining..] .char_indices() .find(|&(_, c)| match (escaped, c) { (true, _) => { escaped = false; false } (false, '\\') => { escaped = true; false } (false, '"') => true, (false, _) => false, }) .map(|(idx, c)| remaining + idx + c.len_utf8()) .unwrap_or(self.input.len()); Some(&self.input[start..end]) } fn skip_whitespace(&mut self) { while let Some(c) = self.peek_char().filter(|c| c.is_whitespace()) { self.cursor.advance_by(c.len_utf8()); } } } impl<'a> Iterator for Lexer<'a> { type Item = TokenSpan<'a>; fn next(&mut self) -> Option { self.skip_whitespace(); let c = self.peek_char()?; if let Some(token) = match c { ':' => Some(Token::Colon), ',' => Some(Token::Comma), '{' => Some(Token::CurlyBracketL), '}' => Some(Token::CurlyBracketR), '=' => Some(Token::Equals), '>' => Some(Token::GreaterThan), '-' => Some(Token::Hyphen), '<' => Some(Token::LessThan), '(' => Some(Token::ParenL), ')' => Some(Token::ParenR), '+' => Some(Token::Plus), ';' => Some(Token::Semicolon), '[' => Some(Token::SquareBracketL), ']' => Some(Token::SquareBracketR), _ => None, } { return Some(self.emit(token, c.len_utf8())); }; if let Some(ident) = self.peek_ident() { let token = match ident { "fn" => Token::Fn, "let" => Token::Let, _ => Token::Ident, }; return Some(self.emit(token, ident.len())); } if let Some(num) = self.peek_number() { return Some(self.emit(Token::Number, num.len())); } if let Some(str) = self.peek_string() { return Some(self.emit(Token::String, str.len())); } Some(self.emit(Token::Unknown, c.len_utf8())) } } #[cfg(test)] mod tests { use super::*; fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) { assert_eq!( lexer.next(), Some(TokenSpan { token: expected, span: Span::new(offset, raw), }) ); } #[test] fn operators() { let input = "+ = - : ; ( ) [ ] { } < >"; let mut lexer = Lexer::new(input); assert_next_token(&mut lexer, Token::Plus, "+", 0); assert_next_token(&mut lexer, Token::Equals, "=", 2); assert_next_token(&mut lexer, Token::Hyphen, "-", 4); assert_next_token(&mut lexer, Token::Colon, ":", 6); assert_next_token(&mut lexer, Token::Semicolon, ";", 8); assert_next_token(&mut lexer, Token::ParenL, "(", 10); assert_next_token(&mut lexer, Token::ParenR, ")", 12); assert_next_token(&mut lexer, Token::SquareBracketL, "[", 14); assert_next_token(&mut lexer, Token::SquareBracketR, "]", 16); assert_next_token(&mut lexer, Token::CurlyBracketL, "{", 18); assert_next_token(&mut lexer, Token::CurlyBracketR, "}", 20); assert_next_token(&mut lexer, Token::LessThan, "<", 22); assert_next_token(&mut lexer, Token::GreaterThan, ">", 24); assert_eq!(lexer.next(), None); } #[test] fn keywords() { let input = "fn let"; let mut lexer = Lexer::new(input); assert_next_token(&mut lexer, Token::Fn, "fn", 0); assert_next_token(&mut lexer, Token::Let, "let", 3); assert_eq!(lexer.next(), None); } #[test] fn numbers() { let input = "0. 5 50.0 5_000 50.000_0 50_000.000_000"; let mut lexer = Lexer::new(input); assert_next_token(&mut lexer, Token::Number, "0.", 0); assert_next_token(&mut lexer, Token::Number, "5", 3); assert_next_token(&mut lexer, Token::Number, "50.0", 5); assert_next_token(&mut lexer, Token::Number, "5_000", 10); assert_next_token(&mut lexer, Token::Number, "50.000_0", 16); assert_next_token(&mut lexer, Token::Number, "50_000.000_000", 25); assert_eq!(lexer.next(), None); } #[test] fn strings() { let input = "\"hello, world\" \"hello \\\", world\" \"hello"; let mut lexer = Lexer::new(input); assert_next_token(&mut lexer, Token::String, "\"hello, world\"", 0); assert_next_token(&mut lexer, Token::String, "\"hello \\\", world\"", 15); assert_next_token(&mut lexer, Token::String, "\"hello", 33); assert_eq!(lexer.next(), None); } }