Added a bunch of tokens.

This commit is contained in:
2026-11-09 19:44:48 +01:00
parent 5bb80a0468
commit 77eabae402
+198 -1
View File
@@ -1,15 +1,23 @@
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub enum Token { pub enum Token {
Add,
Comma, Comma,
CurlyBracketL,
CurlyBracketR,
Equals, Equals,
Fn, Fn,
GreaterThan,
LessThan,
Ident, Ident,
Let, Let,
Number, Number,
ParenL, ParenL,
ParenR, ParenR,
Semi, Semi,
SquareBracketL,
SquareBracketR,
String, String,
Subtract,
Unknown, Unknown,
} }
@@ -159,21 +167,87 @@ impl<'a> Iterator for Lexer<'a> {
self.skip_whitespace(); self.skip_whitespace();
match self.peek_char()? { match self.peek_char()? {
ADD_OPERATOR => {
return Some(TokenSpan {
token: Token::Add,
span: self.consume(ADD_OPERATOR.len_utf8()),
});
}
ASSIGN_OPERATOR => { ASSIGN_OPERATOR => {
return Some(TokenSpan { return Some(TokenSpan {
token: Token::Equals, token: Token::Equals,
span: self.consume(ASSIGN_OPERATOR.len_utf8()), span: self.consume(ASSIGN_OPERATOR.len_utf8()),
}); });
} }
CURLY_BRACKET_L => {
return Some(TokenSpan {
token: Token::CurlyBracketL,
span: self.consume(CURLY_BRACKET_L.len_utf8()),
});
}
CURLY_BRACKET_R => {
return Some(TokenSpan {
token: Token::CurlyBracketR,
span: self.consume(CURLY_BRACKET_R.len_utf8()),
});
}
GREATER_THAN => {
return Some(TokenSpan {
token: Token::GreaterThan,
span: self.consume(GREATER_THAN.len_utf8()),
});
}
LESS_THAN => {
return Some(TokenSpan {
token: Token::LessThan,
span: self.consume(LESS_THAN.len_utf8()),
});
}
PAREN_L => {
return Some(TokenSpan {
token: Token::ParenL,
span: self.consume(PAREN_L.len_utf8()),
});
}
PAREN_R => {
return Some(TokenSpan {
token: Token::ParenR,
span: self.consume(PAREN_R.len_utf8()),
});
}
SEMI => { SEMI => {
return Some(TokenSpan { return Some(TokenSpan {
token: Token::Semi, token: Token::Semi,
span: self.consume(SEMI.len_utf8()), span: self.consume(SEMI.len_utf8()),
}); });
} }
SQUARE_BRACKET_L => {
return Some(TokenSpan {
token: Token::SquareBracketL,
span: self.consume(SQUARE_BRACKET_L.len_utf8()),
});
}
SQUARE_BRACKET_R => {
return Some(TokenSpan {
token: Token::SquareBracketR,
span: self.consume(SQUARE_BRACKET_R.len_utf8()),
});
}
SUBTRACT_OPERATOR => {
return Some(TokenSpan {
token: Token::Subtract,
span: self.consume(SUBTRACT_OPERATOR.len_utf8()),
});
}
_ => {} _ => {}
} }
if self.starts_with(FN) {
return Some(TokenSpan {
token: Token::Fn,
span: self.consume(FN.len()),
});
}
if self.starts_with(LET) { if self.starts_with(LET) {
return Some(TokenSpan { return Some(TokenSpan {
token: Token::Let, token: Token::Let,
@@ -205,20 +279,143 @@ impl<'a> Iterator for Lexer<'a> {
} }
} }
const ADD_OPERATOR: char = '+';
const ASSIGN_OPERATOR: char = '=';
const CURLY_BRACKET_L: char = '{';
const CURLY_BRACKET_R: char = '}';
const DECIMAL_POINT: char = '.'; const DECIMAL_POINT: char = '.';
const DIGIT_SEPARATOR: char = '_'; const DIGIT_SEPARATOR: char = '_';
const ASSIGN_OPERATOR: char = '=';
const ESCAPE: char = '\\'; const ESCAPE: char = '\\';
const GREATER_THAN: char = '>';
const LESS_THAN: char = '<';
const PAREN_L: char = '(';
const PAREN_R: char = ')';
const QUOTE: char = '"'; const QUOTE: char = '"';
const SEMI: char = ';'; const SEMI: char = ';';
const SQUARE_BRACKET_L: char = '[';
const SQUARE_BRACKET_R: char = ']';
const SUBTRACT_OPERATOR: char = '-';
const IDENT_SEPARATOR: char = '_'; const IDENT_SEPARATOR: char = '_';
const FN: &str = "fn";
const LET: &str = "let"; const LET: &str = "let";
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn test_operators() {
let input = "+ = - ; ( ) [ ] { } < >";
let mut lexer = Lexer::new(input);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Add,
span: Span::new(0, "+"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Equals,
span: Span::new(2, "="),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Subtract,
span: Span::new(4, "-"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Semi,
span: Span::new(6, ";"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::ParenL,
span: Span::new(8, "("),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::ParenR,
span: Span::new(10, ")"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::SquareBracketL,
span: Span::new(12, "["),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::SquareBracketR,
span: Span::new(14, "]"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::CurlyBracketL,
span: Span::new(16, "{"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::CurlyBracketR,
span: Span::new(18, "}"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::LessThan,
span: Span::new(20, "<"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::GreaterThan,
span: Span::new(22, ">"),
})
);
assert_eq!(lexer.next(), None);
}
#[test]
fn test_keywords() {
let input = "fn let";
let mut lexer = Lexer::new(input);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Fn,
span: Span::new(0, "fn"),
})
);
assert_eq!(
lexer.next(),
Some(TokenSpan {
token: Token::Let,
span: Span::new(3, "let"),
})
);
assert_eq!(lexer.next(), None);
}
#[test] #[test]
fn test_numbers() { fn test_numbers() {
let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000"; let input = ".5 5 50.0 5_000 50.000_0 50_000.000_000";