Constants.

This commit is contained in:
2026-11-09 18:42:03 +01:00
parent 1b1f5dbae3
commit 5bb80a0468
+55 -50
View File
@@ -35,6 +35,10 @@ impl<'a> Span<'a> {
pub fn raw(&self) -> &'a str {
self.raw
}
pub fn len(&self) -> usize {
self.raw.len()
}
}
#[derive(Debug, PartialEq, Eq)]
@@ -87,26 +91,28 @@ impl<'a> Lexer<'a> {
fn peek_ident(&self) -> Option<&'a str> {
let first = self
.peek_char()
.filter(|c| c.is_alphabetic() || *c == '_')?;
.filter(|c| c.is_alphabetic() || *c == IDENT_SEPARATOR)?;
let start = self.cursor.pos;
let remaining = start + first.len_utf8();
let end = remaining
+ self.input[remaining..]
.chars()
.take_while(|c| c.is_alphanumeric() || *c == '_')
.take_while(|c| c.is_alphanumeric() || *c == IDENT_SEPARATOR)
.map(|c| c.len_utf8())
.sum::<usize>();
Some(&self.input[start..end])
}
fn peek_number(&self) -> Option<&'a str> {
let first = self.peek_char().filter(|c| c.is_digit(10))?;
let first = self
.peek_char()
.filter(|c| c.is_digit(10) || *c == DECIMAL_POINT)?;
let start = self.cursor.pos;
let remaining = start + first.len_utf8();
let end = remaining
+ self.input[remaining..]
.chars()
.take_while(|c| c.is_digit(10) || *c == '_' || *c == '.')
.take_while(|c| c.is_digit(10) || *c == DIGIT_SEPARATOR || *c == DECIMAL_POINT)
.map(|c| c.len_utf8())
.sum::<usize>();
Some(&self.input[start..end])
@@ -124,7 +130,7 @@ impl<'a> Lexer<'a> {
escaped = false;
continue;
}
if c == '\\' {
if c == ESCAPE {
escaped = true;
continue;
}
@@ -153,10 +159,10 @@ impl<'a> Iterator for Lexer<'a> {
self.skip_whitespace();
match self.peek_char()? {
EQUALS => {
ASSIGN_OPERATOR => {
return Some(TokenSpan {
token: Token::Equals,
span: self.consume(EQUALS.len_utf8()),
span: self.consume(ASSIGN_OPERATOR.len_utf8()),
});
}
SEMI => {
@@ -199,9 +205,13 @@ impl<'a> Iterator for Lexer<'a> {
}
}
const EQUALS: char = '=';
const DECIMAL_POINT: char = '.';
const DIGIT_SEPARATOR: char = '_';
const ASSIGN_OPERATOR: char = '=';
const ESCAPE: char = '\\';
const QUOTE: char = '"';
const SEMI: char = ';';
const IDENT_SEPARATOR: char = '_';
const LET: &str = "let";
@@ -210,84 +220,79 @@ mod tests {
use super::*;
#[test]
fn test_let_number() {
let input = "let x = 5_000.0;";
fn test_numbers() {
let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000";
let mut lexer = Lexer::new(input);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Let,
span: Span::new(0, "let")
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Ident,
span: Span::new(4, "x")
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Equals,
span: Span::new(6, "=")
token: Token::Number,
span: Span::new(0, ".5"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(8, "5_000.0")
span: Span::new(3, "5"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Semi,
span: Span::new(15, ";")
token: Token::Number,
span: Span::new(5, "50.0"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(10, "5_000"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(16, "50.000_0"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Number,
span: Span::new(25, "50_000.000_000"),
})
);
assert_eq!(lexer.next(), None);
}
#[test]
fn test_let_string() {
let input = "let x = \"hello, world\";";
fn test_strings() {
let input = "\"hello, world\" \"hello \\\", world\" \"hello";
let mut lexer = Lexer::new(input);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Let,
span: Span::new(0, "let")
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Ident,
span: Span::new(4, "x")
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Equals,
span: Span::new(6, "=")
token: Token::String,
span: Span::new(0, "\"hello, world\""),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::String,
span: Span::new(8, "\"hello, world\"")
span: Span::new(15, "\"hello \\\", world\""),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Semi,
span: Span::new(22, ";")
token: Token::String,
span: Span::new(33, "\"hello"),
})
);
assert_eq!(lexer.next(), None);
}
}