Cleaner tests.

This commit is contained in:
2026-11-09 20:23:02 +01:00
parent 190a5f99e5
commit 235fba0c90
+75 -195
View File
@@ -1,23 +1,24 @@
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub enum Token { pub enum Token {
Add, Colon,
Comma, Comma,
CurlyBracketL, CurlyBracketL,
CurlyBracketR, CurlyBracketR,
Equals, Equals,
Fn, Fn,
GreaterThan, GreaterThan,
Hyphen,
LessThan, LessThan,
Ident, Ident,
Let, Let,
Number, Number,
ParenL, ParenL,
ParenR, ParenR,
Semi, Plus,
Semicolon,
SquareBracketL, SquareBracketL,
SquareBracketR, SquareBracketR,
String, String,
Subtract,
Unknown, Unknown,
} }
@@ -99,13 +100,13 @@ impl<'a> Lexer<'a> {
fn peek_ident(&self) -> Option<&'a str> { fn peek_ident(&self) -> Option<&'a str> {
let first = self let first = self
.peek_char() .peek_char()
.filter(|c| c.is_alphabetic() || *c == IDENT_SEPARATOR)?; .filter(|c| c.is_alphabetic() || *c == UNDERSCORE)?;
let start = self.cursor.pos; let start = self.cursor.pos;
let remaining = start + first.len_utf8(); let remaining = start + first.len_utf8();
let end = remaining let end = remaining
+ self.input[remaining..] + self.input[remaining..]
.chars() .chars()
.take_while(|c| c.is_alphanumeric() || *c == IDENT_SEPARATOR) .take_while(|c| c.is_alphanumeric() || *c == UNDERSCORE)
.map(|c| c.len_utf8()) .map(|c| c.len_utf8())
.sum::<usize>(); .sum::<usize>();
Some(&self.input[start..end]) Some(&self.input[start..end])
@@ -120,14 +121,14 @@ impl<'a> Lexer<'a> {
let end = remaining let end = remaining
+ self.input[remaining..] + self.input[remaining..]
.chars() .chars()
.take_while(|c| c.is_digit(10) || *c == DIGIT_SEPARATOR || *c == DECIMAL_POINT) .take_while(|c| c.is_digit(10) || *c == UNDERSCORE || *c == DECIMAL_POINT)
.map(|c| c.len_utf8()) .map(|c| c.len_utf8())
.sum::<usize>(); .sum::<usize>();
Some(&self.input[start..end]) Some(&self.input[start..end])
} }
fn peek_string(&self) -> Option<&'a str> { fn peek_string(&self) -> Option<&'a str> {
let first = self.peek_char().filter(|c| *c == QUOTE)?; let first = self.peek_char().filter(|c| *c == DOUBLE_QUOTE)?;
let start = self.cursor.pos; let start = self.cursor.pos;
let remaining = start + first.len_utf8(); let remaining = start + first.len_utf8();
let mut end = remaining; let mut end = remaining;
@@ -142,7 +143,7 @@ impl<'a> Lexer<'a> {
escaped = true; escaped = true;
continue; continue;
} }
if c == QUOTE { if c == DOUBLE_QUOTE {
break; break;
} }
} }
@@ -167,16 +168,10 @@ impl<'a> Iterator for Lexer<'a> {
self.skip_whitespace(); self.skip_whitespace();
match self.peek_char()? { match self.peek_char()? {
ADD_OPERATOR => { COLON => {
return Some(TokenSpan { return Some(TokenSpan {
token: Token::Add, token: Token::Colon,
span: self.consume(ADD_OPERATOR.len_utf8()), span: self.consume(COLON.len_utf8()),
});
}
ASSIGN_OPERATOR => {
return Some(TokenSpan {
token: Token::Equals,
span: self.consume(ASSIGN_OPERATOR.len_utf8()),
}); });
} }
CURLY_BRACKET_L => { CURLY_BRACKET_L => {
@@ -191,12 +186,24 @@ impl<'a> Iterator for Lexer<'a> {
span: self.consume(CURLY_BRACKET_R.len_utf8()), span: self.consume(CURLY_BRACKET_R.len_utf8()),
}); });
} }
EQUALS => {
return Some(TokenSpan {
token: Token::Equals,
span: self.consume(EQUALS.len_utf8()),
});
}
GREATER_THAN => { GREATER_THAN => {
return Some(TokenSpan { return Some(TokenSpan {
token: Token::GreaterThan, token: Token::GreaterThan,
span: self.consume(GREATER_THAN.len_utf8()), span: self.consume(GREATER_THAN.len_utf8()),
}); });
} }
HYPHEN => {
return Some(TokenSpan {
token: Token::Hyphen,
span: self.consume(HYPHEN.len_utf8()),
});
}
LESS_THAN => { LESS_THAN => {
return Some(TokenSpan { return Some(TokenSpan {
token: Token::LessThan, token: Token::LessThan,
@@ -215,10 +222,16 @@ impl<'a> Iterator for Lexer<'a> {
span: self.consume(PAREN_R.len_utf8()), span: self.consume(PAREN_R.len_utf8()),
}); });
} }
SEMI => { PLUS => {
return Some(TokenSpan { return Some(TokenSpan {
token: Token::Semi, token: Token::Plus,
span: self.consume(SEMI.len_utf8()), span: self.consume(PLUS.len_utf8()),
});
}
SEMICOLON => {
return Some(TokenSpan {
token: Token::Semicolon,
span: self.consume(SEMICOLON.len_utf8()),
}); });
} }
SQUARE_BRACKET_L => { SQUARE_BRACKET_L => {
@@ -233,12 +246,6 @@ impl<'a> Iterator for Lexer<'a> {
span: self.consume(SQUARE_BRACKET_R.len_utf8()), span: self.consume(SQUARE_BRACKET_R.len_utf8()),
}); });
} }
SUBTRACT_OPERATOR => {
return Some(TokenSpan {
token: Token::Subtract,
span: self.consume(SUBTRACT_OPERATOR.len_utf8()),
});
}
_ => {} _ => {}
} }
@@ -279,23 +286,23 @@ impl<'a> Iterator for Lexer<'a> {
} }
} }
const ADD_OPERATOR: char = '+'; const COLON: char = ':';
const ASSIGN_OPERATOR: char = '=';
const CURLY_BRACKET_L: char = '{'; const CURLY_BRACKET_L: char = '{';
const CURLY_BRACKET_R: char = '}'; const CURLY_BRACKET_R: char = '}';
const DECIMAL_POINT: char = '.'; const DECIMAL_POINT: char = '.';
const DIGIT_SEPARATOR: char = '_'; const DOUBLE_QUOTE: char = '"';
const EQUALS: char = '=';
const ESCAPE: char = '\\'; const ESCAPE: char = '\\';
const GREATER_THAN: char = '>'; const GREATER_THAN: char = '>';
const HYPHEN: char = '-';
const LESS_THAN: char = '<'; const LESS_THAN: char = '<';
const PAREN_L: char = '('; const PAREN_L: char = '(';
const PAREN_R: char = ')'; const PAREN_R: char = ')';
const QUOTE: char = '"'; const PLUS: char = '+';
const SEMI: char = ';'; const SEMICOLON: char = ';';
const SQUARE_BRACKET_L: char = '['; const SQUARE_BRACKET_L: char = '[';
const SQUARE_BRACKET_R: char = ']'; const SQUARE_BRACKET_R: char = ']';
const SUBTRACT_OPERATOR: char = '-'; const UNDERSCORE: char = '_';
const IDENT_SEPARATOR: char = '_';
const FN: &str = "fn"; const FN: &str = "fn";
const LET: &str = "let"; const LET: &str = "let";
@@ -304,94 +311,33 @@ const LET: &str = "let";
mod tests { mod tests {
use super::*; use super::*;
fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) {
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: expected,
span: Span::new(offset, raw),
})
);
}
#[test] #[test]
fn operators() { fn operators() {
let input = "+ = - ; ( ) [ ] { } < >"; let input = "+ = - : ; ( ) [ ] { } < >";
let mut lexer = Lexer::new(input); let mut lexer = Lexer::new(input);
assert_eq!( assert_next_token(&mut lexer, Token::Plus, "+", 0);
lexer.next(), assert_next_token(&mut lexer, Token::Equals, "=", 2);
Some(TokenSpan { assert_next_token(&mut lexer, Token::Hyphen, "-", 4);
token: Token::Add, assert_next_token(&mut lexer, Token::Colon, ":", 6);
span: Span::new(0, "+"), assert_next_token(&mut lexer, Token::Semicolon, ";", 8);
}) assert_next_token(&mut lexer, Token::ParenL, "(", 10);
); assert_next_token(&mut lexer, Token::ParenR, ")", 12);
assert_eq!( assert_next_token(&mut lexer, Token::SquareBracketL, "[", 14);
lexer.next(), assert_next_token(&mut lexer, Token::SquareBracketR, "]", 16);
Some(TokenSpan { assert_next_token(&mut lexer, Token::CurlyBracketL, "{", 18);
token: Token::Equals, assert_next_token(&mut lexer, Token::CurlyBracketR, "}", 20);
span: Span::new(2, "="), assert_next_token(&mut lexer, Token::LessThan, "<", 22);
}) assert_next_token(&mut lexer, Token::GreaterThan, ">", 24);
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Subtract,
span: Span::new(4, "-"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Semi,
span: Span::new(6, ";"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::ParenL,
span: Span::new(8, "("),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::ParenR,
span: Span::new(10, ")"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::SquareBracketL,
span: Span::new(12, "["),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::SquareBracketR,
span: Span::new(14, "]"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::CurlyBracketL,
span: Span::new(16, "{"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::CurlyBracketR,
span: Span::new(18, "}"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::LessThan,
span: Span::new(20, "<"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::GreaterThan,
span: Span::new(22, ">"),
})
);
assert_eq!(lexer.next(), None); assert_eq!(lexer.next(), None);
} }
@@ -399,20 +345,8 @@ mod tests {
fn keywords() { fn keywords() {
let input = "fn let"; let input = "fn let";
let mut lexer = Lexer::new(input); let mut lexer = Lexer::new(input);
assert_eq!( assert_next_token(&mut lexer, Token::Fn, "fn", 0);
lexer.next(), assert_next_token(&mut lexer, Token::Let, "let", 3);
Some(TokenSpan {
token: Token::Fn,
span: Span::new(0, "fn"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Let,
span: Span::new(3, "let"),
})
);
assert_eq!(lexer.next(), None); assert_eq!(lexer.next(), None);
} }
@@ -420,48 +354,12 @@ mod tests {
fn numbers() { fn numbers() {
let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000"; let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000";
let mut lexer = Lexer::new(input); let mut lexer = Lexer::new(input);
assert_eq!( assert_next_token(&mut lexer, Token::Number, ".5", 0);
lexer.next(), assert_next_token(&mut lexer, Token::Number, "5", 3);
Some(TokenSpan { assert_next_token(&mut lexer, Token::Number, "50.0", 5);
token: Token::Number, assert_next_token(&mut lexer, Token::Number, "5_000", 10);
span: Span::new(0, ".5"), assert_next_token(&mut lexer, Token::Number, "50.000_0", 16);
}) assert_next_token(&mut lexer, Token::Number, "50_000.000_000", 25);
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(3, "5"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(5, "50.0"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(10, "5_000"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(16, "50.000_0"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(25, "50_000.000_000"),
})
);
assert_eq!(lexer.next(), None); assert_eq!(lexer.next(), None);
} }
@@ -469,27 +367,9 @@ mod tests {
fn strings() { fn strings() {
let input = "\"hello, world\" \"hello \\\", world\" \"hello"; let input = "\"hello, world\" \"hello \\\", world\" \"hello";
let mut lexer = Lexer::new(input); let mut lexer = Lexer::new(input);
assert_eq!( assert_next_token(&mut lexer, Token::String, "\"hello, world\"", 0);
lexer.next(), assert_next_token(&mut lexer, Token::String, "\"hello \\\", world\"", 15);
Some(TokenSpan { assert_next_token(&mut lexer, Token::String, "\"hello", 33);
token: Token::String,
span: Span::new(0, "\"hello, world\""),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::String,
span: Span::new(15, "\"hello \\\", world\""),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::String,
span: Span::new(33, "\"hello"),
})
);
assert_eq!(lexer.next(), None); assert_eq!(lexer.next(), None);
} }
} }