284 lines
7.8 KiB
Rust
284 lines
7.8 KiB
Rust
#[derive(Debug, PartialEq, Eq)]
|
|
pub enum Token {
|
|
Colon,
|
|
Comma,
|
|
CurlyBracketL,
|
|
CurlyBracketR,
|
|
Equals,
|
|
Fn,
|
|
GreaterThan,
|
|
Hyphen,
|
|
LessThan,
|
|
Ident,
|
|
Let,
|
|
Number,
|
|
ParenL,
|
|
ParenR,
|
|
Plus,
|
|
Semicolon,
|
|
SquareBracketL,
|
|
SquareBracketR,
|
|
String,
|
|
Unknown,
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
pub struct Span<'a> {
|
|
start: usize,
|
|
raw: &'a str,
|
|
}
|
|
|
|
impl<'a> Span<'a> {
|
|
fn new(start: usize, raw: &'a str) -> Self {
|
|
Span { start, raw }
|
|
}
|
|
|
|
pub fn start(&self) -> usize {
|
|
self.start
|
|
}
|
|
|
|
pub fn end(&self) -> usize {
|
|
self.start + self.raw.len()
|
|
}
|
|
|
|
pub fn raw(&self) -> &'a str {
|
|
self.raw
|
|
}
|
|
|
|
pub fn len(&self) -> usize {
|
|
self.raw.len()
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
pub struct TokenSpan<'a> {
|
|
pub token: Token,
|
|
pub span: Span<'a>,
|
|
}
|
|
|
|
struct Cursor {
|
|
pos: usize,
|
|
}
|
|
|
|
impl Cursor {
|
|
fn advance_by(&mut self, n: usize) {
|
|
self.pos += n;
|
|
}
|
|
}
|
|
|
|
impl Default for Cursor {
|
|
fn default() -> Self {
|
|
Cursor { pos: 0 }
|
|
}
|
|
}
|
|
|
|
pub struct Lexer<'a> {
|
|
input: &'a str,
|
|
cursor: Cursor,
|
|
}
|
|
|
|
impl<'a> Lexer<'a> {
|
|
pub fn new(input: &'a str) -> Self {
|
|
Lexer {
|
|
cursor: Cursor::default(),
|
|
input,
|
|
}
|
|
}
|
|
|
|
fn consume(&mut self, n: usize) -> Span<'a> {
|
|
let start = self.cursor.pos;
|
|
let end = start + n;
|
|
let span = Span::new(start, &self.input[start..end]);
|
|
self.cursor.advance_by(n);
|
|
span
|
|
}
|
|
|
|
fn emit(&mut self, token: Token, len: usize) -> TokenSpan<'a> {
|
|
TokenSpan {
|
|
token,
|
|
span: self.consume(len),
|
|
}
|
|
}
|
|
|
|
fn peek_char(&self) -> Option<char> {
|
|
self.input[self.cursor.pos..].chars().next()
|
|
}
|
|
|
|
fn peek_ident(&self) -> Option<&'a str> {
|
|
let first = self
|
|
.peek_char()
|
|
.filter(|c| c.is_alphabetic() || *c == '_')?;
|
|
let start = self.cursor.pos;
|
|
let remaining = start + first.len_utf8();
|
|
|
|
let offset = self.input[remaining..]
|
|
.char_indices()
|
|
.find(|(_, c)| !c.is_alphanumeric() && *c != '_')
|
|
.map(|(idx, _)| idx)
|
|
.unwrap_or(self.input[remaining..].len());
|
|
|
|
Some(&self.input[start..remaining + offset])
|
|
}
|
|
|
|
fn peek_number(&self) -> Option<&'a str> {
|
|
let first = self.peek_char().filter(|c| c.is_digit(10))?;
|
|
let start = self.cursor.pos;
|
|
let remaining = start + first.len_utf8();
|
|
|
|
let offset = self.input[remaining..]
|
|
.char_indices()
|
|
.find(|(_, c)| !c.is_digit(10) && *c != '_' && *c != '.')
|
|
.map(|(idx, _)| idx)
|
|
.unwrap_or(self.input[remaining..].len());
|
|
|
|
Some(&self.input[start..remaining + offset])
|
|
}
|
|
|
|
fn peek_string(&self) -> Option<&'a str> {
|
|
let first = self.peek_char().filter(|c| *c == '"')?;
|
|
let start = self.cursor.pos;
|
|
let remaining = start + first.len_utf8();
|
|
|
|
let mut escaped = false;
|
|
let end = self.input[remaining..]
|
|
.char_indices()
|
|
.find(|&(_, c)| match (escaped, c) {
|
|
(true, _) => {
|
|
escaped = false;
|
|
false
|
|
}
|
|
(false, '\\') => {
|
|
escaped = true;
|
|
false
|
|
}
|
|
(false, '"') => true,
|
|
(false, _) => false,
|
|
})
|
|
.map(|(idx, c)| remaining + idx + c.len_utf8())
|
|
.unwrap_or(self.input.len());
|
|
|
|
Some(&self.input[start..end])
|
|
}
|
|
|
|
fn skip_whitespace(&mut self) {
|
|
while let Some(c) = self.peek_char().filter(|c| c.is_whitespace()) {
|
|
self.cursor.advance_by(c.len_utf8());
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<'a> Iterator for Lexer<'a> {
|
|
type Item = TokenSpan<'a>;
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
self.skip_whitespace();
|
|
|
|
let c = self.peek_char()?;
|
|
if let Some(token) = match c {
|
|
':' => Some(Token::Colon),
|
|
',' => Some(Token::Comma),
|
|
'{' => Some(Token::CurlyBracketL),
|
|
'}' => Some(Token::CurlyBracketR),
|
|
'=' => Some(Token::Equals),
|
|
'>' => Some(Token::GreaterThan),
|
|
'-' => Some(Token::Hyphen),
|
|
'<' => Some(Token::LessThan),
|
|
'(' => Some(Token::ParenL),
|
|
')' => Some(Token::ParenR),
|
|
'+' => Some(Token::Plus),
|
|
';' => Some(Token::Semicolon),
|
|
'[' => Some(Token::SquareBracketL),
|
|
']' => Some(Token::SquareBracketR),
|
|
_ => None,
|
|
} {
|
|
return Some(self.emit(token, c.len_utf8()));
|
|
};
|
|
|
|
if let Some(ident) = self.peek_ident() {
|
|
let token = match ident {
|
|
"fn" => Token::Fn,
|
|
"let" => Token::Let,
|
|
_ => Token::Ident,
|
|
};
|
|
return Some(self.emit(token, ident.len()));
|
|
}
|
|
|
|
if let Some(num) = self.peek_number() {
|
|
return Some(self.emit(Token::Number, num.len()));
|
|
}
|
|
|
|
if let Some(str) = self.peek_string() {
|
|
return Some(self.emit(Token::String, str.len()));
|
|
}
|
|
|
|
Some(self.emit(Token::Unknown, c.len_utf8()))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) {
|
|
assert_eq!(
|
|
lexer.next(),
|
|
Some(TokenSpan {
|
|
token: expected,
|
|
span: Span::new(offset, raw),
|
|
})
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn operators() {
|
|
let input = "+ = - : ; ( ) [ ] { } < >";
|
|
let mut lexer = Lexer::new(input);
|
|
assert_next_token(&mut lexer, Token::Plus, "+", 0);
|
|
assert_next_token(&mut lexer, Token::Equals, "=", 2);
|
|
assert_next_token(&mut lexer, Token::Hyphen, "-", 4);
|
|
assert_next_token(&mut lexer, Token::Colon, ":", 6);
|
|
assert_next_token(&mut lexer, Token::Semicolon, ";", 8);
|
|
assert_next_token(&mut lexer, Token::ParenL, "(", 10);
|
|
assert_next_token(&mut lexer, Token::ParenR, ")", 12);
|
|
assert_next_token(&mut lexer, Token::SquareBracketL, "[", 14);
|
|
assert_next_token(&mut lexer, Token::SquareBracketR, "]", 16);
|
|
assert_next_token(&mut lexer, Token::CurlyBracketL, "{", 18);
|
|
assert_next_token(&mut lexer, Token::CurlyBracketR, "}", 20);
|
|
assert_next_token(&mut lexer, Token::LessThan, "<", 22);
|
|
assert_next_token(&mut lexer, Token::GreaterThan, ">", 24);
|
|
assert_eq!(lexer.next(), None);
|
|
}
|
|
|
|
#[test]
|
|
fn keywords() {
|
|
let input = "fn let";
|
|
let mut lexer = Lexer::new(input);
|
|
assert_next_token(&mut lexer, Token::Fn, "fn", 0);
|
|
assert_next_token(&mut lexer, Token::Let, "let", 3);
|
|
assert_eq!(lexer.next(), None);
|
|
}
|
|
|
|
#[test]
|
|
fn numbers() {
|
|
let input = "0. 5 50.0 5_000 50.000_0 50_000.000_000";
|
|
let mut lexer = Lexer::new(input);
|
|
assert_next_token(&mut lexer, Token::Number, "0.", 0);
|
|
assert_next_token(&mut lexer, Token::Number, "5", 3);
|
|
assert_next_token(&mut lexer, Token::Number, "50.0", 5);
|
|
assert_next_token(&mut lexer, Token::Number, "5_000", 10);
|
|
assert_next_token(&mut lexer, Token::Number, "50.000_0", 16);
|
|
assert_next_token(&mut lexer, Token::Number, "50_000.000_000", 25);
|
|
assert_eq!(lexer.next(), None);
|
|
}
|
|
|
|
#[test]
|
|
fn strings() {
|
|
let input = "\"hello, world\" \"hello \\\", world\" \"hello";
|
|
let mut lexer = Lexer::new(input);
|
|
assert_next_token(&mut lexer, Token::String, "\"hello, world\"", 0);
|
|
assert_next_token(&mut lexer, Token::String, "\"hello \\\", world\"", 15);
|
|
assert_next_token(&mut lexer, Token::String, "\"hello", 33);
|
|
assert_eq!(lexer.next(), None);
|
|
}
|
|
}
|